Coverage Report

Created: 2026-03-26 07:41

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/rust/registry/src/index.crates.io-1949cf8c6b5b557f/regalloc2-0.13.5/src/ion/process.rs
Line
Count
Source
1
/*
2
 * This file was initially derived from the files
3
 * `js/src/jit/BacktrackingAllocator.h` and
4
 * `js/src/jit/BacktrackingAllocator.cpp` in Mozilla Firefox, and was
5
 * originally licensed under the Mozilla Public License 2.0. We
6
 * subsequently relicensed it to Apache-2.0 WITH LLVM-exception (see
7
 * https://github.com/bytecodealliance/regalloc2/issues/7).
8
 *
9
 * Since the initial port, the design has been substantially evolved
10
 * and optimized.
11
 */
12
13
//! Main allocation loop that processes bundles.
14
15
use super::{
16
    spill_weight_from_constraint, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag,
17
    LiveRangeIndex, LiveRangeKey, LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter,
18
    Requirement, SpillWeight, UseList, VRegIndex,
19
};
20
use crate::{
21
    ion::data_structures::{
22
        CodeRange, Use, BUNDLE_MAX_NORMAL_SPILL_WEIGHT, MAX_SPLITS_PER_SPILLSET,
23
        MINIMAL_BUNDLE_SPILL_WEIGHT, MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT,
24
        MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT,
25
    },
26
    Allocation, Function, Inst, InstPosition, OperandConstraint, OperandKind, PReg, ProgPoint,
27
    RegAllocError,
28
};
29
use core::fmt::Debug;
30
use smallvec::{smallvec, SmallVec};
31
32
#[derive(Clone, Debug, PartialEq, Eq)]
33
pub enum AllocRegResult<'a> {
34
    Allocated(Allocation),
35
    Conflict(&'a [LiveBundleIndex], ProgPoint),
36
    ConflictWithFixed(u32, ProgPoint),
37
    ConflictHighCost,
38
}
39
40
impl<'a, F: Function> Env<'a, F> {
41
198k
    pub fn process_bundles(&mut self) -> Result<(), RegAllocError> {
42
10.0M
        while let Some((bundle, hint)) = self.ctx.allocation_queue.pop() {
43
9.89M
            self.ctx.output.stats.process_bundle_count += 1;
44
9.89M
            self.process_bundle(bundle, hint)?;
45
        }
46
198k
        self.ctx.output.stats.final_liverange_count = self.ranges.len();
47
198k
        self.ctx.output.stats.final_bundle_count = self.bundles.len();
48
198k
        self.ctx.output.stats.spill_bundle_count = self.spilled_bundles.len();
49
50
198k
        Ok(())
51
198k
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundles
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundles
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundles
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::process_bundles
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundles
Line
Count
Source
41
198k
    pub fn process_bundles(&mut self) -> Result<(), RegAllocError> {
42
10.0M
        while let Some((bundle, hint)) = self.ctx.allocation_queue.pop() {
43
9.89M
            self.ctx.output.stats.process_bundle_count += 1;
44
9.89M
            self.process_bundle(bundle, hint)?;
45
        }
46
198k
        self.ctx.output.stats.final_liverange_count = self.ranges.len();
47
198k
        self.ctx.output.stats.final_bundle_count = self.bundles.len();
48
198k
        self.ctx.output.stats.spill_bundle_count = self.spilled_bundles.len();
49
50
198k
        Ok(())
51
198k
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundles
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundles
52
53
59.3M
    pub fn try_to_allocate_bundle_to_reg<'b>(
54
59.3M
        &mut self,
55
59.3M
        bundle: LiveBundleIndex,
56
59.3M
        reg: PRegIndex,
57
59.3M
        // if the max bundle weight in the conflict set exceeds this
58
59.3M
        // cost (if provided), just return
59
59.3M
        // `AllocRegResult::ConflictHighCost`.
60
59.3M
        max_allowable_cost: Option<u32>,
61
59.3M
        conflicts: &'b mut LiveBundleVec,
62
59.3M
    ) -> AllocRegResult<'b> {
63
59.3M
        trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg);
64
59.3M
        conflicts.clear();
65
59.3M
        self.ctx.conflict_set.clear();
66
59.3M
        let mut max_conflict_weight = 0;
67
        // Traverse the BTreeMap in order by requesting the whole
68
        // range spanned by the bundle and iterating over that
69
        // concurrently with our ranges. Because our ranges are in
70
        // order, and the BTreeMap is as well, this allows us to have
71
        // an overall O(n log n) + O(b) complexity, where the PReg has
72
        // n current ranges and the bundle has b ranges, rather than
73
        // O(b * n log n) with the simple probe-for-each-bundle-range
74
        // approach.
75
        //
76
        // Note that the comparator function on a CodeRange tests for
77
        // *overlap*, so we are checking whether the BTree contains
78
        // any preg range that *overlaps* with range `range`, not
79
        // literally the range `range`.
80
59.3M
        let bundle_ranges = &self.ctx.bundles[bundle].ranges;
81
59.3M
        let from_key = LiveRangeKey::from_range(&CodeRange {
82
59.3M
            from: bundle_ranges.first().unwrap().range.from,
83
59.3M
            to: bundle_ranges.first().unwrap().range.from,
84
59.3M
        });
85
59.3M
        let mut preg_range_iter = self.ctx.pregs[reg.index()]
86
59.3M
            .allocations
87
59.3M
            .btree
88
59.3M
            .range(from_key..)
89
59.3M
            .peekable();
90
59.3M
        trace!(
91
            "alloc map for {:?} in range {:?}..: {:?}",
92
            reg,
93
            from_key,
94
0
            self.ctx.pregs[reg.index()].allocations.btree
95
        );
96
59.3M
        let mut first_conflict: Option<ProgPoint> = None;
97
98
65.9M
        'ranges: for entry in bundle_ranges {
99
65.9M
            trace!(" -> range LR {:?}: {:?}", entry.index, entry.range);
100
65.9M
            let key = LiveRangeKey::from_range(&entry.range);
101
102
65.9M
            let mut skips = 0;
103
            'alloc: loop {
104
101M
                trace!("  -> PReg range {:?}", preg_range_iter.peek());
105
106
                // Advance our BTree traversal until it is >= this bundle
107
                // range (i.e., skip PReg allocations in the BTree that
108
                // are completely before this bundle range).
109
110
101M
                if preg_range_iter.peek().is_some() && *preg_range_iter.peek().unwrap().0 < key {
111
298k
                    trace!(
112
                        "Skipping PReg range {:?}",
113
0
                        preg_range_iter.peek().unwrap().0
114
                    );
115
298k
                    preg_range_iter.next();
116
298k
                    skips += 1;
117
298k
                    if skips >= 16 {
118
1.56k
                        let from_pos = entry.range.from;
119
1.56k
                        let from_key = LiveRangeKey::from_range(&CodeRange {
120
1.56k
                            from: from_pos,
121
1.56k
                            to: from_pos,
122
1.56k
                        });
123
1.56k
                        preg_range_iter = self.ctx.pregs[reg.index()]
124
1.56k
                            .allocations
125
1.56k
                            .btree
126
1.56k
                            .range(from_key..)
127
1.56k
                            .peekable();
128
1.56k
                        skips = 0;
129
296k
                    }
130
298k
                    continue 'alloc;
131
101M
                }
132
101M
                skips = 0;
133
134
                // If there are no more PReg allocations, we're done!
135
101M
                if preg_range_iter.peek().is_none() {
136
13.5M
                    trace!(" -> no more PReg allocations; so no conflict possible!");
137
13.5M
                    break 'ranges;
138
87.6M
                }
139
140
                // If the current PReg range is beyond this range, there is no conflict; continue.
141
87.6M
                if *preg_range_iter.peek().unwrap().0 > key {
142
25.9M
                    trace!(
143
                        " -> next PReg allocation is at {:?}; moving to next VReg range",
144
0
                        preg_range_iter.peek().unwrap().0
145
                    );
146
25.9M
                    break 'alloc;
147
61.6M
                }
148
149
                // Otherwise, there is a conflict.
150
61.6M
                let preg_key = *preg_range_iter.peek().unwrap().0;
151
61.6M
                debug_assert_eq!(preg_key, key); // Assert that this range overlaps.
152
61.6M
                let preg_range = preg_range_iter.next().unwrap().1;
153
154
61.6M
                trace!(" -> btree contains range {:?} that overlaps", preg_range);
155
61.6M
                if preg_range.is_valid() {
156
35.2M
                    trace!("   -> from vreg {:?}", self.ctx.ranges[*preg_range].vreg);
157
                    // range from an allocated bundle: find the bundle and add to
158
                    // conflicts list.
159
35.2M
                    let conflict_bundle = self.ctx.ranges[*preg_range].bundle;
160
35.2M
                    trace!("   -> conflict bundle {:?}", conflict_bundle);
161
35.2M
                    if self.ctx.conflict_set.insert(conflict_bundle) {
162
33.0M
                        conflicts.push(conflict_bundle);
163
33.0M
                        max_conflict_weight = core::cmp::max(
164
33.0M
                            max_conflict_weight,
165
33.0M
                            self.ctx.bundles[conflict_bundle].cached_spill_weight(),
166
33.0M
                        );
167
33.0M
                        if max_allowable_cost.is_some()
168
14.1M
                            && max_conflict_weight > max_allowable_cost.unwrap()
169
                        {
170
59.2k
                            trace!("   -> reached high cost, retrying early");
171
59.2k
                            return AllocRegResult::ConflictHighCost;
172
32.9M
                        }
173
2.22M
                    }
174
175
35.2M
                    if first_conflict.is_none() {
176
25.2M
                        first_conflict = Some(ProgPoint::from_index(core::cmp::max(
177
25.2M
                            preg_key.from,
178
25.2M
                            key.from,
179
25.2M
                        )));
180
25.2M
                    }
181
                } else {
182
26.3M
                    trace!("   -> conflict with fixed reservation");
183
                    // range from a direct use of the PReg (due to clobber).
184
26.3M
                    return AllocRegResult::ConflictWithFixed(
185
26.3M
                        max_conflict_weight,
186
26.3M
                        ProgPoint::from_index(preg_key.from),
187
26.3M
                    );
188
                }
189
            }
190
        }
191
192
32.9M
        if conflicts.len() > 0 {
193
24.8M
            return AllocRegResult::Conflict(conflicts, first_conflict.unwrap());
194
8.09M
        }
195
196
        // We can allocate! Add our ranges to the preg's BTree.
197
8.09M
        let preg = PReg::from_index(reg.index());
198
8.09M
        trace!("  -> bundle {:?} assigned to preg {:?}", bundle, preg);
199
8.09M
        self.ctx.bundles[bundle].allocation = Allocation::reg(preg);
200
9.41M
        for entry in &self.ctx.bundles[bundle].ranges {
201
9.41M
            let key = LiveRangeKey::from_range(&entry.range);
202
9.41M
            let res = self.ctx.pregs[reg.index()]
203
9.41M
                .allocations
204
9.41M
                .btree
205
9.41M
                .insert(key, entry.index);
206
207
            // We disallow LR overlap within bundles, so this should never be possible.
208
9.41M
            debug_assert!(res.is_none());
209
        }
210
211
8.09M
        AllocRegResult::Allocated(Allocation::reg(preg))
212
59.3M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::try_to_allocate_bundle_to_reg
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg
Line
Count
Source
53
59.3M
    pub fn try_to_allocate_bundle_to_reg<'b>(
54
59.3M
        &mut self,
55
59.3M
        bundle: LiveBundleIndex,
56
59.3M
        reg: PRegIndex,
57
59.3M
        // if the max bundle weight in the conflict set exceeds this
58
59.3M
        // cost (if provided), just return
59
59.3M
        // `AllocRegResult::ConflictHighCost`.
60
59.3M
        max_allowable_cost: Option<u32>,
61
59.3M
        conflicts: &'b mut LiveBundleVec,
62
59.3M
    ) -> AllocRegResult<'b> {
63
59.3M
        trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg);
64
59.3M
        conflicts.clear();
65
59.3M
        self.ctx.conflict_set.clear();
66
59.3M
        let mut max_conflict_weight = 0;
67
        // Traverse the BTreeMap in order by requesting the whole
68
        // range spanned by the bundle and iterating over that
69
        // concurrently with our ranges. Because our ranges are in
70
        // order, and the BTreeMap is as well, this allows us to have
71
        // an overall O(n log n) + O(b) complexity, where the PReg has
72
        // n current ranges and the bundle has b ranges, rather than
73
        // O(b * n log n) with the simple probe-for-each-bundle-range
74
        // approach.
75
        //
76
        // Note that the comparator function on a CodeRange tests for
77
        // *overlap*, so we are checking whether the BTree contains
78
        // any preg range that *overlaps* with range `range`, not
79
        // literally the range `range`.
80
59.3M
        let bundle_ranges = &self.ctx.bundles[bundle].ranges;
81
59.3M
        let from_key = LiveRangeKey::from_range(&CodeRange {
82
59.3M
            from: bundle_ranges.first().unwrap().range.from,
83
59.3M
            to: bundle_ranges.first().unwrap().range.from,
84
59.3M
        });
85
59.3M
        let mut preg_range_iter = self.ctx.pregs[reg.index()]
86
59.3M
            .allocations
87
59.3M
            .btree
88
59.3M
            .range(from_key..)
89
59.3M
            .peekable();
90
59.3M
        trace!(
91
            "alloc map for {:?} in range {:?}..: {:?}",
92
            reg,
93
            from_key,
94
0
            self.ctx.pregs[reg.index()].allocations.btree
95
        );
96
59.3M
        let mut first_conflict: Option<ProgPoint> = None;
97
98
65.9M
        'ranges: for entry in bundle_ranges {
99
65.9M
            trace!(" -> range LR {:?}: {:?}", entry.index, entry.range);
100
65.9M
            let key = LiveRangeKey::from_range(&entry.range);
101
102
65.9M
            let mut skips = 0;
103
            'alloc: loop {
104
101M
                trace!("  -> PReg range {:?}", preg_range_iter.peek());
105
106
                // Advance our BTree traversal until it is >= this bundle
107
                // range (i.e., skip PReg allocations in the BTree that
108
                // are completely before this bundle range).
109
110
101M
                if preg_range_iter.peek().is_some() && *preg_range_iter.peek().unwrap().0 < key {
111
298k
                    trace!(
112
                        "Skipping PReg range {:?}",
113
0
                        preg_range_iter.peek().unwrap().0
114
                    );
115
298k
                    preg_range_iter.next();
116
298k
                    skips += 1;
117
298k
                    if skips >= 16 {
118
1.56k
                        let from_pos = entry.range.from;
119
1.56k
                        let from_key = LiveRangeKey::from_range(&CodeRange {
120
1.56k
                            from: from_pos,
121
1.56k
                            to: from_pos,
122
1.56k
                        });
123
1.56k
                        preg_range_iter = self.ctx.pregs[reg.index()]
124
1.56k
                            .allocations
125
1.56k
                            .btree
126
1.56k
                            .range(from_key..)
127
1.56k
                            .peekable();
128
1.56k
                        skips = 0;
129
296k
                    }
130
298k
                    continue 'alloc;
131
101M
                }
132
101M
                skips = 0;
133
134
                // If there are no more PReg allocations, we're done!
135
101M
                if preg_range_iter.peek().is_none() {
136
13.5M
                    trace!(" -> no more PReg allocations; so no conflict possible!");
137
13.5M
                    break 'ranges;
138
87.6M
                }
139
140
                // If the current PReg range is beyond this range, there is no conflict; continue.
141
87.6M
                if *preg_range_iter.peek().unwrap().0 > key {
142
25.9M
                    trace!(
143
                        " -> next PReg allocation is at {:?}; moving to next VReg range",
144
0
                        preg_range_iter.peek().unwrap().0
145
                    );
146
25.9M
                    break 'alloc;
147
61.6M
                }
148
149
                // Otherwise, there is a conflict.
150
61.6M
                let preg_key = *preg_range_iter.peek().unwrap().0;
151
61.6M
                debug_assert_eq!(preg_key, key); // Assert that this range overlaps.
152
61.6M
                let preg_range = preg_range_iter.next().unwrap().1;
153
154
61.6M
                trace!(" -> btree contains range {:?} that overlaps", preg_range);
155
61.6M
                if preg_range.is_valid() {
156
35.2M
                    trace!("   -> from vreg {:?}", self.ctx.ranges[*preg_range].vreg);
157
                    // range from an allocated bundle: find the bundle and add to
158
                    // conflicts list.
159
35.2M
                    let conflict_bundle = self.ctx.ranges[*preg_range].bundle;
160
35.2M
                    trace!("   -> conflict bundle {:?}", conflict_bundle);
161
35.2M
                    if self.ctx.conflict_set.insert(conflict_bundle) {
162
33.0M
                        conflicts.push(conflict_bundle);
163
33.0M
                        max_conflict_weight = core::cmp::max(
164
33.0M
                            max_conflict_weight,
165
33.0M
                            self.ctx.bundles[conflict_bundle].cached_spill_weight(),
166
33.0M
                        );
167
33.0M
                        if max_allowable_cost.is_some()
168
14.1M
                            && max_conflict_weight > max_allowable_cost.unwrap()
169
                        {
170
59.2k
                            trace!("   -> reached high cost, retrying early");
171
59.2k
                            return AllocRegResult::ConflictHighCost;
172
32.9M
                        }
173
2.22M
                    }
174
175
35.2M
                    if first_conflict.is_none() {
176
25.2M
                        first_conflict = Some(ProgPoint::from_index(core::cmp::max(
177
25.2M
                            preg_key.from,
178
25.2M
                            key.from,
179
25.2M
                        )));
180
25.2M
                    }
181
                } else {
182
26.3M
                    trace!("   -> conflict with fixed reservation");
183
                    // range from a direct use of the PReg (due to clobber).
184
26.3M
                    return AllocRegResult::ConflictWithFixed(
185
26.3M
                        max_conflict_weight,
186
26.3M
                        ProgPoint::from_index(preg_key.from),
187
26.3M
                    );
188
                }
189
            }
190
        }
191
192
32.9M
        if conflicts.len() > 0 {
193
24.8M
            return AllocRegResult::Conflict(conflicts, first_conflict.unwrap());
194
8.09M
        }
195
196
        // We can allocate! Add our ranges to the preg's BTree.
197
8.09M
        let preg = PReg::from_index(reg.index());
198
8.09M
        trace!("  -> bundle {:?} assigned to preg {:?}", bundle, preg);
199
8.09M
        self.ctx.bundles[bundle].allocation = Allocation::reg(preg);
200
9.41M
        for entry in &self.ctx.bundles[bundle].ranges {
201
9.41M
            let key = LiveRangeKey::from_range(&entry.range);
202
9.41M
            let res = self.ctx.pregs[reg.index()]
203
9.41M
                .allocations
204
9.41M
                .btree
205
9.41M
                .insert(key, entry.index);
206
207
            // We disallow LR overlap within bundles, so this should never be possible.
208
9.41M
            debug_assert!(res.is_none());
209
        }
210
211
8.09M
        AllocRegResult::Allocated(Allocation::reg(preg))
212
59.3M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg
213
214
1.03M
    pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) {
215
1.03M
        trace!(
216
            "evicting bundle {:?}: alloc {:?}",
217
            bundle,
218
0
            self.ctx.bundles[bundle].allocation
219
        );
220
1.03M
        let preg = match self.ctx.bundles[bundle].allocation.as_reg() {
221
1.03M
            Some(preg) => preg,
222
            None => {
223
0
                trace!(
224
                    "  -> has no allocation! {:?}",
225
0
                    self.ctx.bundles[bundle].allocation
226
                );
227
0
                return;
228
            }
229
        };
230
1.03M
        let preg_idx = PRegIndex::new(preg.index());
231
1.03M
        self.ctx.bundles[bundle].allocation = Allocation::none();
232
1.24M
        for entry in &self.ctx.bundles[bundle].ranges {
233
1.24M
            trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx);
234
1.24M
            self.ctx.pregs[preg_idx.index()]
235
1.24M
                .allocations
236
1.24M
                .btree
237
1.24M
                .remove(&LiveRangeKey::from_range(&entry.range));
238
        }
239
1.03M
        let prio = self.ctx.bundles[bundle].prio;
240
1.03M
        trace!(" -> prio {}; back into queue", prio);
241
1.03M
        self.ctx
242
1.03M
            .allocation_queue
243
1.03M
            .insert(bundle, prio as usize, PReg::invalid());
244
1.03M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::evict_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::evict_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::evict_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::evict_bundle
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::evict_bundle
Line
Count
Source
214
1.03M
    pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) {
215
1.03M
        trace!(
216
            "evicting bundle {:?}: alloc {:?}",
217
            bundle,
218
0
            self.ctx.bundles[bundle].allocation
219
        );
220
1.03M
        let preg = match self.ctx.bundles[bundle].allocation.as_reg() {
221
1.03M
            Some(preg) => preg,
222
            None => {
223
0
                trace!(
224
                    "  -> has no allocation! {:?}",
225
0
                    self.ctx.bundles[bundle].allocation
226
                );
227
0
                return;
228
            }
229
        };
230
1.03M
        let preg_idx = PRegIndex::new(preg.index());
231
1.03M
        self.ctx.bundles[bundle].allocation = Allocation::none();
232
1.24M
        for entry in &self.ctx.bundles[bundle].ranges {
233
1.24M
            trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx);
234
1.24M
            self.ctx.pregs[preg_idx.index()]
235
1.24M
                .allocations
236
1.24M
                .btree
237
1.24M
                .remove(&LiveRangeKey::from_range(&entry.range));
238
        }
239
1.03M
        let prio = self.ctx.bundles[bundle].prio;
240
1.03M
        trace!(" -> prio {}; back into queue", prio);
241
1.03M
        self.ctx
242
1.03M
            .allocation_queue
243
1.03M
            .insert(bundle, prio as usize, PReg::invalid());
244
1.03M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::evict_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::evict_bundle
245
246
2.32M
    pub fn bundle_spill_weight(&self, bundle: LiveBundleIndex) -> u32 {
247
2.32M
        self.ctx.bundles[bundle].cached_spill_weight()
248
2.32M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::bundle_spill_weight
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::bundle_spill_weight
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::bundle_spill_weight
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::bundle_spill_weight
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::bundle_spill_weight
Line
Count
Source
246
2.32M
    pub fn bundle_spill_weight(&self, bundle: LiveBundleIndex) -> u32 {
247
2.32M
        self.ctx.bundles[bundle].cached_spill_weight()
248
2.32M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::bundle_spill_weight
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::bundle_spill_weight
249
250
18.5M
    pub fn maximum_spill_weight_in_bundle_set(&self, bundles: &[LiveBundleIndex]) -> u32 {
251
18.5M
        trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles);
252
18.5M
        let m = bundles
253
18.5M
            .iter()
254
18.5M
            .map(|&b| {
255
18.5M
                let w = self.ctx.bundles[b].cached_spill_weight();
256
18.5M
                trace!("bundle{}: {}", b.index(), w);
257
18.5M
                w
258
18.5M
            })
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::maximum_spill_weight_in_bundle_set::{closure#0}
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}
Line
Count
Source
254
18.5M
            .map(|&b| {
255
18.5M
                let w = self.ctx.bundles[b].cached_spill_weight();
256
18.5M
                trace!("bundle{}: {}", b.index(), w);
257
18.5M
                w
258
18.5M
            })
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}
259
18.5M
            .max()
260
18.5M
            .unwrap_or(0);
261
18.5M
        trace!(" -> max: {}", m);
262
18.5M
        m
263
18.5M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::maximum_spill_weight_in_bundle_set
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set
Line
Count
Source
250
18.5M
    pub fn maximum_spill_weight_in_bundle_set(&self, bundles: &[LiveBundleIndex]) -> u32 {
251
18.5M
        trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles);
252
18.5M
        let m = bundles
253
18.5M
            .iter()
254
18.5M
            .map(|&b| {
255
                let w = self.ctx.bundles[b].cached_spill_weight();
256
                trace!("bundle{}: {}", b.index(), w);
257
                w
258
            })
259
18.5M
            .max()
260
18.5M
            .unwrap_or(0);
261
18.5M
        trace!(" -> max: {}", m);
262
18.5M
        m
263
18.5M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set
264
265
8.86M
    pub fn recompute_bundle_properties(&mut self, bundle: LiveBundleIndex) {
266
8.86M
        trace!("recompute bundle properties: bundle {:?}", bundle);
267
268
        let minimal;
269
8.86M
        let mut fixed = false;
270
8.86M
        let mut fixed_def = false;
271
8.86M
        let mut stack = false;
272
8.86M
        let bundledata = &self.ctx.bundles[bundle];
273
8.86M
        let num_ranges = bundledata.ranges.len();
274
8.86M
        let first_range = bundledata.ranges[0].index;
275
8.86M
        let first_range_data = &self.ctx.ranges[first_range];
276
277
8.86M
        self.ctx.bundles[bundle].prio = self.compute_bundle_prio(bundle);
278
8.86M
        self.ctx.bundles[bundle].limit = self.compute_bundle_limit(bundle);
279
280
8.86M
        if first_range_data.vreg.is_invalid() {
281
0
            trace!("  -> no vreg; minimal and fixed");
282
0
            minimal = true;
283
0
            fixed = true;
284
8.86M
        } else if num_ranges == 1 {
285
21.4M
            for u in &first_range_data.uses {
286
21.4M
                trace!("  -> use: {:?}", u);
287
21.4M
                if let OperandConstraint::FixedReg(_) = u.operand.constraint() {
288
1.80M
                    trace!("  -> fixed operand at {:?}: {:?}", u.pos, u.operand);
289
1.80M
                    fixed = true;
290
1.80M
                    if u.operand.kind() == OperandKind::Def {
291
919k
                        trace!("  -> is fixed def");
292
919k
                        fixed_def = true;
293
882k
                    }
294
19.6M
                }
295
21.4M
                if let OperandConstraint::Stack = u.operand.constraint() {
296
0
                    trace!("  -> stack operand at {:?}: {:?}", u.pos, u.operand);
297
0
                    stack = true;
298
21.4M
                }
299
21.4M
                if stack && fixed {
300
0
                    break;
301
21.4M
                }
302
            }
303
            // Minimal if there is only one LR and only up to one use
304
            // in that LR, and extent of range is consistent with the
305
            // "minimal range for use" definition.
306
8.25M
            minimal = match &first_range_data.uses[..] {
307
8.25M
                [] => true,
308
2.77M
                [only_use] => {
309
                    // We use a "contains" rather than "exact" range
310
                    // check because a smaller-than-min range is also
311
                    // OK, if some logic produced it for a valid
312
                    // reason -- for example, a dead def. We don't
313
                    // want to livelock on "smaller than minimal"
314
                    // ranges.
315
2.77M
                    let min_range = minimal_range_for_use(only_use);
316
2.77M
                    min_range.contains(&first_range_data.range)
317
                }
318
5.47M
                _ => false,
319
            };
320
8.25M
            trace!("  -> minimal: {}", minimal);
321
608k
        } else {
322
608k
            minimal = false;
323
608k
        }
324
325
8.86M
        let spill_weight = if minimal {
326
2.53M
            if fixed {
327
449k
                trace!("  -> fixed and minimal");
328
449k
                MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT
329
2.08M
            } else if let Some(limit) = self.ctx.bundles[bundle].limit {
330
0
                trace!("  -> limited({limit}) and minimal");
331
0
                MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT - u32::from(limit)
332
            } else {
333
2.08M
                trace!("  -> non-fixed and minimal");
334
2.08M
                MINIMAL_BUNDLE_SPILL_WEIGHT
335
            }
336
        } else {
337
6.32M
            let mut total = SpillWeight::zero();
338
7.75M
            for entry in &self.ctx.bundles[bundle].ranges {
339
7.75M
                let range_data = &self.ctx.ranges[entry.index];
340
7.75M
                trace!(
341
                    "  -> uses spill weight: +{:?}",
342
0
                    range_data.uses_spill_weight()
343
                );
344
7.75M
                total = total + range_data.uses_spill_weight();
345
            }
346
347
6.32M
            if self.ctx.bundles[bundle].prio > 0 {
348
6.32M
                let final_weight = (total.to_f32() as u32) / self.ctx.bundles[bundle].prio;
349
6.32M
                trace!(
350
                    " -> dividing by prio {}; final weight {}",
351
0
                    self.ctx.bundles[bundle].prio,
352
                    final_weight
353
                );
354
6.32M
                core::cmp::min(BUNDLE_MAX_NORMAL_SPILL_WEIGHT, final_weight)
355
            } else {
356
1.46k
                0
357
            }
358
        };
359
360
8.86M
        self.ctx.bundles[bundle].set_cached_spill_weight_and_props(
361
8.86M
            spill_weight,
362
8.86M
            minimal,
363
8.86M
            fixed,
364
8.86M
            fixed_def,
365
8.86M
            stack,
366
        );
367
8.86M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::recompute_bundle_properties
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties
Line
Count
Source
265
8.86M
    pub fn recompute_bundle_properties(&mut self, bundle: LiveBundleIndex) {
266
8.86M
        trace!("recompute bundle properties: bundle {:?}", bundle);
267
268
        let minimal;
269
8.86M
        let mut fixed = false;
270
8.86M
        let mut fixed_def = false;
271
8.86M
        let mut stack = false;
272
8.86M
        let bundledata = &self.ctx.bundles[bundle];
273
8.86M
        let num_ranges = bundledata.ranges.len();
274
8.86M
        let first_range = bundledata.ranges[0].index;
275
8.86M
        let first_range_data = &self.ctx.ranges[first_range];
276
277
8.86M
        self.ctx.bundles[bundle].prio = self.compute_bundle_prio(bundle);
278
8.86M
        self.ctx.bundles[bundle].limit = self.compute_bundle_limit(bundle);
279
280
8.86M
        if first_range_data.vreg.is_invalid() {
281
0
            trace!("  -> no vreg; minimal and fixed");
282
0
            minimal = true;
283
0
            fixed = true;
284
8.86M
        } else if num_ranges == 1 {
285
21.4M
            for u in &first_range_data.uses {
286
21.4M
                trace!("  -> use: {:?}", u);
287
21.4M
                if let OperandConstraint::FixedReg(_) = u.operand.constraint() {
288
1.80M
                    trace!("  -> fixed operand at {:?}: {:?}", u.pos, u.operand);
289
1.80M
                    fixed = true;
290
1.80M
                    if u.operand.kind() == OperandKind::Def {
291
919k
                        trace!("  -> is fixed def");
292
919k
                        fixed_def = true;
293
882k
                    }
294
19.6M
                }
295
21.4M
                if let OperandConstraint::Stack = u.operand.constraint() {
296
0
                    trace!("  -> stack operand at {:?}: {:?}", u.pos, u.operand);
297
0
                    stack = true;
298
21.4M
                }
299
21.4M
                if stack && fixed {
300
0
                    break;
301
21.4M
                }
302
            }
303
            // Minimal if there is only one LR and only up to one use
304
            // in that LR, and extent of range is consistent with the
305
            // "minimal range for use" definition.
306
8.25M
            minimal = match &first_range_data.uses[..] {
307
8.25M
                [] => true,
308
2.77M
                [only_use] => {
309
                    // We use a "contains" rather than "exact" range
310
                    // check because a smaller-than-min range is also
311
                    // OK, if some logic produced it for a valid
312
                    // reason -- for example, a dead def. We don't
313
                    // want to livelock on "smaller than minimal"
314
                    // ranges.
315
2.77M
                    let min_range = minimal_range_for_use(only_use);
316
2.77M
                    min_range.contains(&first_range_data.range)
317
                }
318
5.47M
                _ => false,
319
            };
320
8.25M
            trace!("  -> minimal: {}", minimal);
321
608k
        } else {
322
608k
            minimal = false;
323
608k
        }
324
325
8.86M
        let spill_weight = if minimal {
326
2.53M
            if fixed {
327
449k
                trace!("  -> fixed and minimal");
328
449k
                MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT
329
2.08M
            } else if let Some(limit) = self.ctx.bundles[bundle].limit {
330
0
                trace!("  -> limited({limit}) and minimal");
331
0
                MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT - u32::from(limit)
332
            } else {
333
2.08M
                trace!("  -> non-fixed and minimal");
334
2.08M
                MINIMAL_BUNDLE_SPILL_WEIGHT
335
            }
336
        } else {
337
6.32M
            let mut total = SpillWeight::zero();
338
7.75M
            for entry in &self.ctx.bundles[bundle].ranges {
339
7.75M
                let range_data = &self.ctx.ranges[entry.index];
340
7.75M
                trace!(
341
                    "  -> uses spill weight: +{:?}",
342
0
                    range_data.uses_spill_weight()
343
                );
344
7.75M
                total = total + range_data.uses_spill_weight();
345
            }
346
347
6.32M
            if self.ctx.bundles[bundle].prio > 0 {
348
6.32M
                let final_weight = (total.to_f32() as u32) / self.ctx.bundles[bundle].prio;
349
6.32M
                trace!(
350
                    " -> dividing by prio {}; final weight {}",
351
0
                    self.ctx.bundles[bundle].prio,
352
                    final_weight
353
                );
354
6.32M
                core::cmp::min(BUNDLE_MAX_NORMAL_SPILL_WEIGHT, final_weight)
355
            } else {
356
1.46k
                0
357
            }
358
        };
359
360
8.86M
        self.ctx.bundles[bundle].set_cached_spill_weight_and_props(
361
8.86M
            spill_weight,
362
8.86M
            minimal,
363
8.86M
            fixed,
364
8.86M
            fixed_def,
365
8.86M
            stack,
366
        );
367
8.86M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties
368
369
4.64M
    pub fn minimal_bundle(&self, bundle: LiveBundleIndex) -> bool {
370
4.64M
        self.ctx.bundles[bundle].cached_minimal()
371
4.64M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::minimal_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::minimal_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::minimal_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::minimal_bundle
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::minimal_bundle
Line
Count
Source
369
4.64M
    pub fn minimal_bundle(&self, bundle: LiveBundleIndex) -> bool {
370
4.64M
        self.ctx.bundles[bundle].cached_minimal()
371
4.64M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::minimal_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::minimal_bundle
372
373
2.70M
    pub fn recompute_range_properties(&mut self, range: LiveRangeIndex) {
374
2.70M
        let rangedata = &mut self.ctx.ranges[range];
375
2.70M
        let mut w = SpillWeight::zero();
376
5.43M
        for u in &rangedata.uses {
377
5.43M
            w = w + SpillWeight::from_bits(u.weight);
378
5.43M
            trace!("range{}: use {:?}", range.index(), u);
379
        }
380
2.70M
        rangedata.set_uses_spill_weight(w);
381
2.70M
        if rangedata.uses.len() > 0 && rangedata.uses[0].operand.kind() == OperandKind::Def {
382
1.33M
            // Note that we *set* the flag here, but we never *clear*
383
1.33M
            // it: it may be set by a progmove as well (which does not
384
1.33M
            // create an explicit use or def), and we want to preserve
385
1.33M
            // that. We will never split or trim ranges in a way that
386
1.33M
            // removes a def at the front and requires the flag to be
387
1.33M
            // cleared.
388
1.33M
            rangedata.set_flag(LiveRangeFlag::StartsAtDef);
389
1.37M
        }
390
2.70M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::recompute_range_properties
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::recompute_range_properties
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::recompute_range_properties
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::recompute_range_properties
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::recompute_range_properties
Line
Count
Source
373
2.70M
    pub fn recompute_range_properties(&mut self, range: LiveRangeIndex) {
374
2.70M
        let rangedata = &mut self.ctx.ranges[range];
375
2.70M
        let mut w = SpillWeight::zero();
376
5.43M
        for u in &rangedata.uses {
377
5.43M
            w = w + SpillWeight::from_bits(u.weight);
378
5.43M
            trace!("range{}: use {:?}", range.index(), u);
379
        }
380
2.70M
        rangedata.set_uses_spill_weight(w);
381
2.70M
        if rangedata.uses.len() > 0 && rangedata.uses[0].operand.kind() == OperandKind::Def {
382
1.33M
            // Note that we *set* the flag here, but we never *clear*
383
1.33M
            // it: it may be set by a progmove as well (which does not
384
1.33M
            // create an explicit use or def), and we want to preserve
385
1.33M
            // that. We will never split or trim ranges in a way that
386
1.33M
            // removes a def at the front and requires the flag to be
387
1.33M
            // cleared.
388
1.33M
            rangedata.set_flag(LiveRangeFlag::StartsAtDef);
389
1.37M
        }
390
2.70M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::recompute_range_properties
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::recompute_range_properties
391
392
2.12M
    pub fn get_or_create_spill_bundle(
393
2.12M
        &mut self,
394
2.12M
        bundle: LiveBundleIndex,
395
2.12M
        create_if_absent: bool,
396
2.12M
    ) -> Option<LiveBundleIndex> {
397
2.12M
        let ssidx = self.ctx.bundles[bundle].spillset;
398
2.12M
        let idx = self.ctx.spillsets[ssidx].spill_bundle;
399
2.12M
        if idx.is_valid() {
400
1.01M
            Some(idx)
401
1.10M
        } else if create_if_absent {
402
1.08M
            let idx = self.ctx.bundles.add(self.ctx.bump());
403
1.08M
            self.ctx.spillsets[ssidx].spill_bundle = idx;
404
1.08M
            self.ctx.bundles[idx].spillset = ssidx;
405
1.08M
            self.ctx.spilled_bundles.push(idx);
406
1.08M
            Some(idx)
407
        } else {
408
20.8k
            None
409
        }
410
2.12M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::get_or_create_spill_bundle
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle
Line
Count
Source
392
2.12M
    pub fn get_or_create_spill_bundle(
393
2.12M
        &mut self,
394
2.12M
        bundle: LiveBundleIndex,
395
2.12M
        create_if_absent: bool,
396
2.12M
    ) -> Option<LiveBundleIndex> {
397
2.12M
        let ssidx = self.ctx.bundles[bundle].spillset;
398
2.12M
        let idx = self.ctx.spillsets[ssidx].spill_bundle;
399
2.12M
        if idx.is_valid() {
400
1.01M
            Some(idx)
401
1.10M
        } else if create_if_absent {
402
1.08M
            let idx = self.ctx.bundles.add(self.ctx.bump());
403
1.08M
            self.ctx.spillsets[ssidx].spill_bundle = idx;
404
1.08M
            self.ctx.bundles[idx].spillset = ssidx;
405
1.08M
            self.ctx.spilled_bundles.push(idx);
406
1.08M
            Some(idx)
407
        } else {
408
20.8k
            None
409
        }
410
2.12M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle
411
412
1.35M
    pub fn split_and_requeue_bundle(
413
1.35M
        &mut self,
414
1.35M
        bundle: LiveBundleIndex,
415
1.35M
        mut split_at: ProgPoint,
416
1.35M
        hint: PReg,
417
1.35M
        // Do we trim the parts around the split and put them in the
418
1.35M
        // spill bundle?
419
1.35M
        mut trim_ends_into_spill_bundle: bool,
420
1.35M
    ) {
421
1.35M
        self.ctx.output.stats.splits += 1;
422
1.35M
        trace!(
423
            "split bundle {bundle:?} at {split_at:?} and requeue with reg hint (for first part) {hint:?}"
424
        );
425
426
        // Split `bundle` at `split_at`, creating new LiveRanges and
427
        // bundles (and updating vregs' linked lists appropriately),
428
        // and enqueue the new bundles.
429
430
1.35M
        let spillset = self.ctx.bundles[bundle].spillset;
431
432
        // Have we reached the maximum split count? If so, fall back
433
        // to a "minimal bundles and spill bundle" setup for this
434
        // bundle. See the doc-comment on
435
        // `split_into_minimal_bundles()` above for more.
436
1.35M
        if self.ctx.spillsets[spillset].splits >= MAX_SPLITS_PER_SPILLSET {
437
2.16k
            self.split_into_minimal_bundles(bundle, hint);
438
2.16k
            return;
439
1.35M
        }
440
1.35M
        self.ctx.spillsets[spillset].splits += 1;
441
442
1.35M
        debug_assert!(!self.ctx.bundles[bundle].ranges.is_empty());
443
        // Split point *at* start is OK; this means we peel off
444
        // exactly one use to create a minimal bundle.
445
1.35M
        let bundle_start = self.ctx.bundles[bundle].ranges.first().unwrap().range.from;
446
1.35M
        debug_assert!(split_at >= bundle_start);
447
1.35M
        let bundle_end = self.ctx.bundles[bundle].ranges.last().unwrap().range.to;
448
1.35M
        debug_assert!(split_at < bundle_end);
449
450
1.35M
        trace!(
451
            "bundle_start = {:?} bundle_end = {:?}",
452
            bundle_start,
453
            bundle_end
454
        );
455
456
        // Is the bundle at most spanning one instruction? Then
457
        // there's nothing we can do with this logic (we will not
458
        // split in the middle of the instruction). Fall back to the
459
        // minimal-bundle splitting in this case as well.
460
1.35M
        if bundle_end.prev().inst() == bundle_start.inst() {
461
1
            trace!(" -> spans only one inst; splitting into minimal bundles");
462
1
            self.split_into_minimal_bundles(bundle, hint);
463
1
            return;
464
1.35M
        }
465
466
        // Is the split point *at* the start? If so, peel off the
467
        // program point with the first use: set the split point just
468
        // after it, or just before it if it comes after the start of
469
        // the bundle.
470
1.35M
        if split_at == bundle_start {
471
            // Find any uses; if none, just chop off one instruction.
472
739k
            let mut first_use = None;
473
739k
            'outer: for entry in &self.ctx.bundles[bundle].ranges {
474
739k
                for u in &self.ctx.ranges[entry.index].uses {
475
739k
                    first_use = Some(u.pos);
476
739k
                    break 'outer;
477
                }
478
            }
479
739k
            trace!(" -> first use loc is {:?}", first_use);
480
739k
            split_at = match first_use {
481
739k
                Some(pos) => {
482
739k
                    if pos.inst() == bundle_start.inst() {
483
739k
                        ProgPoint::before(pos.inst().next())
484
                    } else {
485
102
                        ProgPoint::before(pos.inst())
486
                    }
487
                }
488
0
                None => ProgPoint::before(
489
0
                    self.ctx.bundles[bundle]
490
0
                        .ranges
491
0
                        .first()
492
0
                        .unwrap()
493
0
                        .range
494
0
                        .from
495
0
                        .inst()
496
0
                        .next(),
497
                ),
498
            };
499
739k
            trace!(
500
                "split point is at bundle start; advancing to {:?}",
501
                split_at
502
            );
503
        } else {
504
            // Don't split in the middle of an instruction -- this could
505
            // create impossible moves (we cannot insert a move between an
506
            // instruction's uses and defs).
507
614k
            if split_at.pos() == InstPosition::After {
508
485k
                split_at = split_at.next();
509
485k
            }
510
614k
            if split_at >= bundle_end {
511
38.4k
                split_at = split_at.prev().prev();
512
575k
            }
513
        }
514
515
1.35M
        debug_assert!(split_at > bundle_start && split_at < bundle_end);
516
517
        // We need to find which LRs fall on each side of the split,
518
        // which LR we need to split down the middle, then update the
519
        // current bundle, create a new one, and (re)-queue both.
520
521
1.35M
        trace!(" -> LRs: {:?}", self.ctx.bundles[bundle].ranges);
522
523
1.35M
        let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle
524
1.35M
        let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle
525
1.38M
        for (i, entry) in self.ctx.bundles[bundle].ranges.iter().enumerate() {
526
1.38M
            if split_at > entry.range.from {
527
1.38M
                last_lr_in_old_bundle_idx = i;
528
1.38M
                first_lr_in_new_bundle_idx = i;
529
1.38M
            }
530
1.38M
            if split_at < entry.range.to {
531
1.35M
                first_lr_in_new_bundle_idx = i;
532
533
                // When the bundle contains a fixed constraint, we advance the split point to right
534
                // before the first instruction with a fixed use present.
535
1.35M
                if self.ctx.bundles[bundle].cached_fixed() {
536
3.24M
                    for u in &self.ctx.ranges[entry.index].uses {
537
3.24M
                        if u.pos < split_at {
538
1.48M
                            continue;
539
1.76M
                        }
540
541
1.76M
                        if matches!(u.operand.constraint(), OperandConstraint::FixedReg { .. }) {
542
242k
                            split_at = ProgPoint::before(u.pos.inst());
543
544
242k
                            if split_at > entry.range.from {
545
242k
                                last_lr_in_old_bundle_idx = i;
546
242k
                            }
547
548
242k
                            trace!(" -> advancing split point to {split_at:?}");
549
550
242k
                            trim_ends_into_spill_bundle = false;
551
552
242k
                            break;
553
1.52M
                        }
554
                    }
555
886k
                }
556
557
1.35M
                break;
558
29.3k
            }
559
        }
560
561
1.35M
        trace!(
562
            " -> last LR in old bundle: LR {:?}",
563
0
            self.ctx.bundles[bundle].ranges[last_lr_in_old_bundle_idx]
564
        );
565
1.35M
        trace!(
566
            " -> first LR in new bundle: LR {:?}",
567
0
            self.ctx.bundles[bundle].ranges[first_lr_in_new_bundle_idx]
568
        );
569
570
        // Take the sublist of LRs that will go in the new bundle.
571
1.35M
        let mut new_lr_list: LiveRangeList = LiveRangeList::new_in(self.ctx.bump());
572
1.35M
        new_lr_list.extend(
573
1.35M
            self.ctx.bundles[bundle]
574
1.35M
                .ranges
575
1.35M
                .iter()
576
1.35M
                .cloned()
577
1.35M
                .skip(first_lr_in_new_bundle_idx),
578
        );
579
1.35M
        self.ctx.bundles[bundle]
580
1.35M
            .ranges
581
1.35M
            .truncate(last_lr_in_old_bundle_idx + 1);
582
1.35M
        self.ctx.bundles[bundle].ranges.shrink_to_fit();
583
584
        // If the first entry in `new_lr_list` is a LR that is split
585
        // down the middle, replace it with a new LR and chop off the
586
        // end of the same LR in the original list.
587
1.35M
        if split_at > new_lr_list[0].range.from {
588
1.35M
            debug_assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx);
589
1.35M
            let orig_lr = new_lr_list[0].index;
590
1.35M
            let new_lr = self.ctx.ranges.add(
591
1.35M
                CodeRange {
592
1.35M
                    from: split_at,
593
1.35M
                    to: new_lr_list[0].range.to,
594
1.35M
                },
595
1.35M
                self.ctx.bump(),
596
            );
597
1.35M
            self.ctx.ranges[new_lr].vreg = self.ranges[orig_lr].vreg;
598
1.35M
            trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr);
599
1.35M
            let first_use = self.ctx.ranges[orig_lr]
600
1.35M
                .uses
601
1.35M
                .iter()
602
3.82M
                .position(|u| u.pos >= split_at)
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::split_and_requeue_bundle::{closure#0}
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}
Line
Count
Source
602
3.82M
                .position(|u| u.pos >= split_at)
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}
603
1.35M
                .unwrap_or(self.ctx.ranges[orig_lr].uses.len());
604
1.35M
            let mut rest_uses = UseList::new_in(self.ctx.bump());
605
1.35M
            rest_uses.extend(
606
1.35M
                self.ctx.ranges[orig_lr]
607
1.35M
                    .uses
608
1.35M
                    .iter()
609
1.35M
                    .cloned()
610
1.35M
                    .skip(first_use),
611
            );
612
1.35M
            self.ctx.ranges[new_lr].uses = rest_uses;
613
1.35M
            self.ctx.ranges[orig_lr].uses.truncate(first_use);
614
1.35M
            self.ctx.ranges[orig_lr].uses.shrink_to_fit();
615
1.35M
            self.recompute_range_properties(orig_lr);
616
1.35M
            self.recompute_range_properties(new_lr);
617
1.35M
            new_lr_list[0].index = new_lr;
618
1.35M
            new_lr_list[0].range = self.ctx.ranges[new_lr].range;
619
1.35M
            self.ctx.ranges[orig_lr].range.to = split_at;
620
1.35M
            self.ctx.bundles[bundle].ranges[last_lr_in_old_bundle_idx].range =
621
1.35M
                self.ctx.ranges[orig_lr].range;
622
623
            // Perform a lazy split in the VReg data. We just
624
            // append the new LR and its range; we will sort by
625
            // start of range, and fix up range ends, once when we
626
            // iterate over the VReg's ranges after allocation
627
            // completes (this is the only time when order
628
            // matters).
629
1.35M
            self.ctx.vregs[self.ctx.ranges[new_lr].vreg]
630
1.35M
                .ranges
631
1.35M
                .push(LiveRangeListEntry {
632
1.35M
                    range: self.ctx.ranges[new_lr].range,
633
1.35M
                    index: new_lr,
634
1.35M
                });
635
303
        }
636
637
1.35M
        let new_bundle = self.ctx.bundles.add(self.ctx.bump());
638
1.35M
        trace!(" -> creating new bundle {:?}", new_bundle);
639
1.35M
        self.ctx.bundles[new_bundle].spillset = spillset;
640
1.70M
        for entry in &new_lr_list {
641
1.70M
            self.ctx.ranges[entry.index].bundle = new_bundle;
642
1.70M
        }
643
1.35M
        self.ctx.bundles[new_bundle].ranges = new_lr_list;
644
645
1.35M
        if trim_ends_into_spill_bundle {
646
            // Finally, handle moving LRs to the spill bundle when
647
            // appropriate: If the first range in `new_bundle` or last
648
            // range in `bundle` has "empty space" beyond the first or
649
            // last use (respectively), trim it and put an empty LR into
650
            // the spill bundle.  (We are careful to treat the "starts at
651
            // def" flag as an implicit first def even if no def-type Use
652
            // is present.)
653
1.12M
            while let Some(entry) = self.ctx.bundles[bundle].ranges.last().cloned() {
654
1.12M
                let end = entry.range.to;
655
1.12M
                let vreg = self.ctx.ranges[entry.index].vreg;
656
1.12M
                let last_use = self.ctx.ranges[entry.index].uses.last().map(|u| u.pos);
657
1.12M
                if last_use.is_none() {
658
9.05k
                    let spill = self
659
9.05k
                        .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true)
660
9.05k
                        .unwrap();
661
9.05k
                    trace!(
662
                        " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}",
663
                        bundle,
664
                        entry.index,
665
                        spill
666
                    );
667
9.05k
                    self.ctx.bundles[spill].ranges.push(entry);
668
9.05k
                    self.ctx.bundles[bundle].ranges.pop();
669
9.05k
                    self.ctx.ranges[entry.index].bundle = spill;
670
9.05k
                    continue;
671
1.11M
                }
672
1.11M
                let last_use = last_use.unwrap();
673
1.11M
                let split = ProgPoint::before(last_use.inst().next());
674
1.11M
                if split < end {
675
389k
                    let spill = self
676
389k
                        .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true)
677
389k
                        .unwrap();
678
389k
                    self.ctx.bundles[bundle].ranges.last_mut().unwrap().range.to = split;
679
389k
                    self.ctx.ranges[self.ctx.bundles[bundle].ranges.last().unwrap().index]
680
389k
                        .range
681
389k
                        .to = split;
682
389k
                    let range = CodeRange {
683
389k
                        from: split,
684
389k
                        to: end,
685
389k
                    };
686
389k
                    let empty_lr = self.ctx.ranges.add(range, self.ctx.bump());
687
389k
                    self.ctx.bundles[spill].ranges.push(LiveRangeListEntry {
688
389k
                        range,
689
389k
                        index: empty_lr,
690
389k
                    });
691
389k
                    self.ctx.ranges[empty_lr].bundle = spill;
692
389k
                    self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry {
693
389k
                        range,
694
389k
                        index: empty_lr,
695
389k
                    });
696
389k
                    trace!(
697
                        " -> bundle {:?} range {:?}: last use implies split point {:?}",
698
                        bundle,
699
                        entry.index,
700
                        split
701
                    );
702
389k
                    trace!(
703
                    " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}",
704
                    spill,
705
                    empty_lr
706
                );
707
721k
                }
708
1.11M
                break;
709
            }
710
1.43M
            while let Some(entry) = self.ctx.bundles[new_bundle].ranges.first().cloned() {
711
1.23M
                if self.ctx.ranges[entry.index].has_flag(LiveRangeFlag::StartsAtDef) {
712
364
                    break;
713
1.23M
                }
714
1.23M
                let start = entry.range.from;
715
1.23M
                let vreg = self.ctx.ranges[entry.index].vreg;
716
1.23M
                let first_use = self.ctx.ranges[entry.index].uses.first().map(|u| u.pos);
717
1.23M
                if first_use.is_none() {
718
319k
                    let spill = self
719
319k
                        .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true)
720
319k
                        .unwrap();
721
319k
                    trace!(
722
                        " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}",
723
                        new_bundle,
724
                        entry.index,
725
                        spill
726
                    );
727
319k
                    self.ctx.bundles[spill].ranges.push(entry);
728
319k
                    self.ctx.bundles[new_bundle].ranges.drain(..1);
729
319k
                    self.ctx.ranges[entry.index].bundle = spill;
730
319k
                    continue;
731
913k
                }
732
913k
                let first_use = first_use.unwrap();
733
913k
                let split = ProgPoint::before(first_use.inst());
734
913k
                if split > start {
735
772k
                    let spill = self
736
772k
                        .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true)
737
772k
                        .unwrap();
738
772k
                    self.ctx.bundles[new_bundle]
739
772k
                        .ranges
740
772k
                        .first_mut()
741
772k
                        .unwrap()
742
772k
                        .range
743
772k
                        .from = split;
744
772k
                    self.ctx.ranges[self.ctx.bundles[new_bundle].ranges.first().unwrap().index]
745
772k
                        .range
746
772k
                        .from = split;
747
772k
                    let range = CodeRange {
748
772k
                        from: start,
749
772k
                        to: split,
750
772k
                    };
751
772k
                    let empty_lr = self.ctx.ranges.add(range, self.ctx.bump());
752
772k
                    self.ctx.bundles[spill].ranges.push(LiveRangeListEntry {
753
772k
                        range,
754
772k
                        index: empty_lr,
755
772k
                    });
756
772k
                    self.ctx.ranges[empty_lr].bundle = spill;
757
772k
                    self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry {
758
772k
                        range,
759
772k
                        index: empty_lr,
760
772k
                    });
761
772k
                    trace!(
762
                        " -> bundle {:?} range {:?}: first use implies split point {:?}",
763
                        bundle,
764
                        entry.index,
765
                        first_use,
766
                    );
767
772k
                    trace!(
768
                        " -> moving leading empty region to new spill bundle {:?} with new LR {:?}",
769
                        spill,
770
                        empty_lr
771
                    );
772
141k
                }
773
913k
                break;
774
            }
775
242k
        }
776
777
1.35M
        if self.ctx.bundles[bundle].ranges.len() > 0 {
778
1.35M
            self.recompute_bundle_properties(bundle);
779
1.35M
            let prio = self.ctx.bundles[bundle].prio;
780
1.35M
            self.ctx
781
1.35M
                .allocation_queue
782
1.35M
                .insert(bundle, prio as usize, hint);
783
1.35M
        }
784
1.35M
        if self.ctx.bundles[new_bundle].ranges.len() > 0 {
785
1.15M
            self.recompute_bundle_properties(new_bundle);
786
1.15M
            let prio = self.ctx.bundles[new_bundle].prio;
787
1.15M
            self.ctx
788
1.15M
                .allocation_queue
789
1.15M
                .insert(new_bundle, prio as usize, hint);
790
1.15M
        }
791
1.35M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::split_and_requeue_bundle
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle
Line
Count
Source
412
1.35M
    pub fn split_and_requeue_bundle(
413
1.35M
        &mut self,
414
1.35M
        bundle: LiveBundleIndex,
415
1.35M
        mut split_at: ProgPoint,
416
1.35M
        hint: PReg,
417
1.35M
        // Do we trim the parts around the split and put them in the
418
1.35M
        // spill bundle?
419
1.35M
        mut trim_ends_into_spill_bundle: bool,
420
1.35M
    ) {
421
1.35M
        self.ctx.output.stats.splits += 1;
422
1.35M
        trace!(
423
            "split bundle {bundle:?} at {split_at:?} and requeue with reg hint (for first part) {hint:?}"
424
        );
425
426
        // Split `bundle` at `split_at`, creating new LiveRanges and
427
        // bundles (and updating vregs' linked lists appropriately),
428
        // and enqueue the new bundles.
429
430
1.35M
        let spillset = self.ctx.bundles[bundle].spillset;
431
432
        // Have we reached the maximum split count? If so, fall back
433
        // to a "minimal bundles and spill bundle" setup for this
434
        // bundle. See the doc-comment on
435
        // `split_into_minimal_bundles()` above for more.
436
1.35M
        if self.ctx.spillsets[spillset].splits >= MAX_SPLITS_PER_SPILLSET {
437
2.16k
            self.split_into_minimal_bundles(bundle, hint);
438
2.16k
            return;
439
1.35M
        }
440
1.35M
        self.ctx.spillsets[spillset].splits += 1;
441
442
1.35M
        debug_assert!(!self.ctx.bundles[bundle].ranges.is_empty());
443
        // Split point *at* start is OK; this means we peel off
444
        // exactly one use to create a minimal bundle.
445
1.35M
        let bundle_start = self.ctx.bundles[bundle].ranges.first().unwrap().range.from;
446
1.35M
        debug_assert!(split_at >= bundle_start);
447
1.35M
        let bundle_end = self.ctx.bundles[bundle].ranges.last().unwrap().range.to;
448
1.35M
        debug_assert!(split_at < bundle_end);
449
450
1.35M
        trace!(
451
            "bundle_start = {:?} bundle_end = {:?}",
452
            bundle_start,
453
            bundle_end
454
        );
455
456
        // Is the bundle at most spanning one instruction? Then
457
        // there's nothing we can do with this logic (we will not
458
        // split in the middle of the instruction). Fall back to the
459
        // minimal-bundle splitting in this case as well.
460
1.35M
        if bundle_end.prev().inst() == bundle_start.inst() {
461
1
            trace!(" -> spans only one inst; splitting into minimal bundles");
462
1
            self.split_into_minimal_bundles(bundle, hint);
463
1
            return;
464
1.35M
        }
465
466
        // Is the split point *at* the start? If so, peel off the
467
        // program point with the first use: set the split point just
468
        // after it, or just before it if it comes after the start of
469
        // the bundle.
470
1.35M
        if split_at == bundle_start {
471
            // Find any uses; if none, just chop off one instruction.
472
739k
            let mut first_use = None;
473
739k
            'outer: for entry in &self.ctx.bundles[bundle].ranges {
474
739k
                for u in &self.ctx.ranges[entry.index].uses {
475
739k
                    first_use = Some(u.pos);
476
739k
                    break 'outer;
477
                }
478
            }
479
739k
            trace!(" -> first use loc is {:?}", first_use);
480
739k
            split_at = match first_use {
481
739k
                Some(pos) => {
482
739k
                    if pos.inst() == bundle_start.inst() {
483
739k
                        ProgPoint::before(pos.inst().next())
484
                    } else {
485
102
                        ProgPoint::before(pos.inst())
486
                    }
487
                }
488
0
                None => ProgPoint::before(
489
0
                    self.ctx.bundles[bundle]
490
0
                        .ranges
491
0
                        .first()
492
0
                        .unwrap()
493
0
                        .range
494
0
                        .from
495
0
                        .inst()
496
0
                        .next(),
497
                ),
498
            };
499
739k
            trace!(
500
                "split point is at bundle start; advancing to {:?}",
501
                split_at
502
            );
503
        } else {
504
            // Don't split in the middle of an instruction -- this could
505
            // create impossible moves (we cannot insert a move between an
506
            // instruction's uses and defs).
507
614k
            if split_at.pos() == InstPosition::After {
508
485k
                split_at = split_at.next();
509
485k
            }
510
614k
            if split_at >= bundle_end {
511
38.4k
                split_at = split_at.prev().prev();
512
575k
            }
513
        }
514
515
1.35M
        debug_assert!(split_at > bundle_start && split_at < bundle_end);
516
517
        // We need to find which LRs fall on each side of the split,
518
        // which LR we need to split down the middle, then update the
519
        // current bundle, create a new one, and (re)-queue both.
520
521
1.35M
        trace!(" -> LRs: {:?}", self.ctx.bundles[bundle].ranges);
522
523
1.35M
        let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle
524
1.35M
        let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle
525
1.38M
        for (i, entry) in self.ctx.bundles[bundle].ranges.iter().enumerate() {
526
1.38M
            if split_at > entry.range.from {
527
1.38M
                last_lr_in_old_bundle_idx = i;
528
1.38M
                first_lr_in_new_bundle_idx = i;
529
1.38M
            }
530
1.38M
            if split_at < entry.range.to {
531
1.35M
                first_lr_in_new_bundle_idx = i;
532
533
                // When the bundle contains a fixed constraint, we advance the split point to right
534
                // before the first instruction with a fixed use present.
535
1.35M
                if self.ctx.bundles[bundle].cached_fixed() {
536
3.24M
                    for u in &self.ctx.ranges[entry.index].uses {
537
3.24M
                        if u.pos < split_at {
538
1.48M
                            continue;
539
1.76M
                        }
540
541
1.76M
                        if matches!(u.operand.constraint(), OperandConstraint::FixedReg { .. }) {
542
242k
                            split_at = ProgPoint::before(u.pos.inst());
543
544
242k
                            if split_at > entry.range.from {
545
242k
                                last_lr_in_old_bundle_idx = i;
546
242k
                            }
547
548
242k
                            trace!(" -> advancing split point to {split_at:?}");
549
550
242k
                            trim_ends_into_spill_bundle = false;
551
552
242k
                            break;
553
1.52M
                        }
554
                    }
555
886k
                }
556
557
1.35M
                break;
558
29.3k
            }
559
        }
560
561
1.35M
        trace!(
562
            " -> last LR in old bundle: LR {:?}",
563
0
            self.ctx.bundles[bundle].ranges[last_lr_in_old_bundle_idx]
564
        );
565
1.35M
        trace!(
566
            " -> first LR in new bundle: LR {:?}",
567
0
            self.ctx.bundles[bundle].ranges[first_lr_in_new_bundle_idx]
568
        );
569
570
        // Take the sublist of LRs that will go in the new bundle.
571
1.35M
        let mut new_lr_list: LiveRangeList = LiveRangeList::new_in(self.ctx.bump());
572
1.35M
        new_lr_list.extend(
573
1.35M
            self.ctx.bundles[bundle]
574
1.35M
                .ranges
575
1.35M
                .iter()
576
1.35M
                .cloned()
577
1.35M
                .skip(first_lr_in_new_bundle_idx),
578
        );
579
1.35M
        self.ctx.bundles[bundle]
580
1.35M
            .ranges
581
1.35M
            .truncate(last_lr_in_old_bundle_idx + 1);
582
1.35M
        self.ctx.bundles[bundle].ranges.shrink_to_fit();
583
584
        // If the first entry in `new_lr_list` is a LR that is split
585
        // down the middle, replace it with a new LR and chop off the
586
        // end of the same LR in the original list.
587
1.35M
        if split_at > new_lr_list[0].range.from {
588
1.35M
            debug_assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx);
589
1.35M
            let orig_lr = new_lr_list[0].index;
590
1.35M
            let new_lr = self.ctx.ranges.add(
591
1.35M
                CodeRange {
592
1.35M
                    from: split_at,
593
1.35M
                    to: new_lr_list[0].range.to,
594
1.35M
                },
595
1.35M
                self.ctx.bump(),
596
            );
597
1.35M
            self.ctx.ranges[new_lr].vreg = self.ranges[orig_lr].vreg;
598
1.35M
            trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr);
599
1.35M
            let first_use = self.ctx.ranges[orig_lr]
600
1.35M
                .uses
601
1.35M
                .iter()
602
1.35M
                .position(|u| u.pos >= split_at)
603
1.35M
                .unwrap_or(self.ctx.ranges[orig_lr].uses.len());
604
1.35M
            let mut rest_uses = UseList::new_in(self.ctx.bump());
605
1.35M
            rest_uses.extend(
606
1.35M
                self.ctx.ranges[orig_lr]
607
1.35M
                    .uses
608
1.35M
                    .iter()
609
1.35M
                    .cloned()
610
1.35M
                    .skip(first_use),
611
            );
612
1.35M
            self.ctx.ranges[new_lr].uses = rest_uses;
613
1.35M
            self.ctx.ranges[orig_lr].uses.truncate(first_use);
614
1.35M
            self.ctx.ranges[orig_lr].uses.shrink_to_fit();
615
1.35M
            self.recompute_range_properties(orig_lr);
616
1.35M
            self.recompute_range_properties(new_lr);
617
1.35M
            new_lr_list[0].index = new_lr;
618
1.35M
            new_lr_list[0].range = self.ctx.ranges[new_lr].range;
619
1.35M
            self.ctx.ranges[orig_lr].range.to = split_at;
620
1.35M
            self.ctx.bundles[bundle].ranges[last_lr_in_old_bundle_idx].range =
621
1.35M
                self.ctx.ranges[orig_lr].range;
622
623
            // Perform a lazy split in the VReg data. We just
624
            // append the new LR and its range; we will sort by
625
            // start of range, and fix up range ends, once when we
626
            // iterate over the VReg's ranges after allocation
627
            // completes (this is the only time when order
628
            // matters).
629
1.35M
            self.ctx.vregs[self.ctx.ranges[new_lr].vreg]
630
1.35M
                .ranges
631
1.35M
                .push(LiveRangeListEntry {
632
1.35M
                    range: self.ctx.ranges[new_lr].range,
633
1.35M
                    index: new_lr,
634
1.35M
                });
635
303
        }
636
637
1.35M
        let new_bundle = self.ctx.bundles.add(self.ctx.bump());
638
1.35M
        trace!(" -> creating new bundle {:?}", new_bundle);
639
1.35M
        self.ctx.bundles[new_bundle].spillset = spillset;
640
1.70M
        for entry in &new_lr_list {
641
1.70M
            self.ctx.ranges[entry.index].bundle = new_bundle;
642
1.70M
        }
643
1.35M
        self.ctx.bundles[new_bundle].ranges = new_lr_list;
644
645
1.35M
        if trim_ends_into_spill_bundle {
646
            // Finally, handle moving LRs to the spill bundle when
647
            // appropriate: If the first range in `new_bundle` or last
648
            // range in `bundle` has "empty space" beyond the first or
649
            // last use (respectively), trim it and put an empty LR into
650
            // the spill bundle.  (We are careful to treat the "starts at
651
            // def" flag as an implicit first def even if no def-type Use
652
            // is present.)
653
1.12M
            while let Some(entry) = self.ctx.bundles[bundle].ranges.last().cloned() {
654
1.12M
                let end = entry.range.to;
655
1.12M
                let vreg = self.ctx.ranges[entry.index].vreg;
656
1.12M
                let last_use = self.ctx.ranges[entry.index].uses.last().map(|u| u.pos);
657
1.12M
                if last_use.is_none() {
658
9.05k
                    let spill = self
659
9.05k
                        .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true)
660
9.05k
                        .unwrap();
661
9.05k
                    trace!(
662
                        " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}",
663
                        bundle,
664
                        entry.index,
665
                        spill
666
                    );
667
9.05k
                    self.ctx.bundles[spill].ranges.push(entry);
668
9.05k
                    self.ctx.bundles[bundle].ranges.pop();
669
9.05k
                    self.ctx.ranges[entry.index].bundle = spill;
670
9.05k
                    continue;
671
1.11M
                }
672
1.11M
                let last_use = last_use.unwrap();
673
1.11M
                let split = ProgPoint::before(last_use.inst().next());
674
1.11M
                if split < end {
675
389k
                    let spill = self
676
389k
                        .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true)
677
389k
                        .unwrap();
678
389k
                    self.ctx.bundles[bundle].ranges.last_mut().unwrap().range.to = split;
679
389k
                    self.ctx.ranges[self.ctx.bundles[bundle].ranges.last().unwrap().index]
680
389k
                        .range
681
389k
                        .to = split;
682
389k
                    let range = CodeRange {
683
389k
                        from: split,
684
389k
                        to: end,
685
389k
                    };
686
389k
                    let empty_lr = self.ctx.ranges.add(range, self.ctx.bump());
687
389k
                    self.ctx.bundles[spill].ranges.push(LiveRangeListEntry {
688
389k
                        range,
689
389k
                        index: empty_lr,
690
389k
                    });
691
389k
                    self.ctx.ranges[empty_lr].bundle = spill;
692
389k
                    self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry {
693
389k
                        range,
694
389k
                        index: empty_lr,
695
389k
                    });
696
389k
                    trace!(
697
                        " -> bundle {:?} range {:?}: last use implies split point {:?}",
698
                        bundle,
699
                        entry.index,
700
                        split
701
                    );
702
389k
                    trace!(
703
                    " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}",
704
                    spill,
705
                    empty_lr
706
                );
707
721k
                }
708
1.11M
                break;
709
            }
710
1.43M
            while let Some(entry) = self.ctx.bundles[new_bundle].ranges.first().cloned() {
711
1.23M
                if self.ctx.ranges[entry.index].has_flag(LiveRangeFlag::StartsAtDef) {
712
364
                    break;
713
1.23M
                }
714
1.23M
                let start = entry.range.from;
715
1.23M
                let vreg = self.ctx.ranges[entry.index].vreg;
716
1.23M
                let first_use = self.ctx.ranges[entry.index].uses.first().map(|u| u.pos);
717
1.23M
                if first_use.is_none() {
718
319k
                    let spill = self
719
319k
                        .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true)
720
319k
                        .unwrap();
721
319k
                    trace!(
722
                        " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}",
723
                        new_bundle,
724
                        entry.index,
725
                        spill
726
                    );
727
319k
                    self.ctx.bundles[spill].ranges.push(entry);
728
319k
                    self.ctx.bundles[new_bundle].ranges.drain(..1);
729
319k
                    self.ctx.ranges[entry.index].bundle = spill;
730
319k
                    continue;
731
913k
                }
732
913k
                let first_use = first_use.unwrap();
733
913k
                let split = ProgPoint::before(first_use.inst());
734
913k
                if split > start {
735
772k
                    let spill = self
736
772k
                        .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true)
737
772k
                        .unwrap();
738
772k
                    self.ctx.bundles[new_bundle]
739
772k
                        .ranges
740
772k
                        .first_mut()
741
772k
                        .unwrap()
742
772k
                        .range
743
772k
                        .from = split;
744
772k
                    self.ctx.ranges[self.ctx.bundles[new_bundle].ranges.first().unwrap().index]
745
772k
                        .range
746
772k
                        .from = split;
747
772k
                    let range = CodeRange {
748
772k
                        from: start,
749
772k
                        to: split,
750
772k
                    };
751
772k
                    let empty_lr = self.ctx.ranges.add(range, self.ctx.bump());
752
772k
                    self.ctx.bundles[spill].ranges.push(LiveRangeListEntry {
753
772k
                        range,
754
772k
                        index: empty_lr,
755
772k
                    });
756
772k
                    self.ctx.ranges[empty_lr].bundle = spill;
757
772k
                    self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry {
758
772k
                        range,
759
772k
                        index: empty_lr,
760
772k
                    });
761
772k
                    trace!(
762
                        " -> bundle {:?} range {:?}: first use implies split point {:?}",
763
                        bundle,
764
                        entry.index,
765
                        first_use,
766
                    );
767
772k
                    trace!(
768
                        " -> moving leading empty region to new spill bundle {:?} with new LR {:?}",
769
                        spill,
770
                        empty_lr
771
                    );
772
141k
                }
773
913k
                break;
774
            }
775
242k
        }
776
777
1.35M
        if self.ctx.bundles[bundle].ranges.len() > 0 {
778
1.35M
            self.recompute_bundle_properties(bundle);
779
1.35M
            let prio = self.ctx.bundles[bundle].prio;
780
1.35M
            self.ctx
781
1.35M
                .allocation_queue
782
1.35M
                .insert(bundle, prio as usize, hint);
783
1.35M
        }
784
1.35M
        if self.ctx.bundles[new_bundle].ranges.len() > 0 {
785
1.15M
            self.recompute_bundle_properties(new_bundle);
786
1.15M
            let prio = self.ctx.bundles[new_bundle].prio;
787
1.15M
            self.ctx
788
1.15M
                .allocation_queue
789
1.15M
                .insert(new_bundle, prio as usize, hint);
790
1.15M
        }
791
1.35M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle
792
793
    /// Splits the given bundle into minimal bundles per Use, falling
794
    /// back onto the spill bundle. This must work for any bundle no
795
    /// matter how many conflicts.
796
    ///
797
    /// This is meant to solve a quadratic-cost problem that exists
798
    /// with "normal" splitting as implemented above. With that
799
    /// procedure, splitting a bundle produces two
800
    /// halves. Furthermore, it has cost linear in the length of the
801
    /// bundle, because the resulting half-bundles have their
802
    /// requirements recomputed with a new scan, and because we copy
803
    /// half the use-list over to the tail end sub-bundle.
804
    ///
805
    /// This works fine when a bundle has a handful of splits overall,
806
    /// but not when an input has a systematic pattern of conflicts
807
    /// that will require O(|bundle|) splits (e.g., every Use is
808
    /// constrained to a different fixed register than the last
809
    /// one). In such a case, we get quadratic behavior.
810
    ///
811
    /// This method implements a direct split into minimal bundles
812
    /// along the whole length of the bundle, putting the regions
813
    /// without uses in the spill bundle. We do this once the number
814
    /// of splits in an original bundle (tracked by spillset) reaches
815
    /// a pre-determined limit.
816
    ///
817
    /// This basically approximates what a non-splitting allocator
818
    /// would do: it "spills" the whole bundle to possibly a
819
    /// stackslot, or a second-chance register allocation at best, via
820
    /// the spill bundle; and then does minimal reservations of
821
    /// registers just at uses/defs and moves the "spilled" value
822
    /// into/out of them immediately.
823
2.16k
    pub fn split_into_minimal_bundles(&mut self, bundle: LiveBundleIndex, hint: PReg) {
824
2.16k
        assert_eq!(self.ctx.scratch_removed_lrs_vregs.len(), 0);
825
2.16k
        self.ctx.scratch_removed_lrs.clear();
826
827
2.16k
        let mut new_lrs: SmallVec<[(VRegIndex, LiveRangeIndex); 16]> = smallvec![];
828
2.16k
        let mut new_bundles: SmallVec<[LiveBundleIndex; 16]> = smallvec![];
829
830
2.16k
        let spillset = self.ctx.bundles[bundle].spillset;
831
2.16k
        let spill = self
832
2.16k
            .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true)
833
2.16k
            .unwrap();
834
835
2.16k
        trace!("Splitting bundle {bundle:?} into minimal bundles with reg hint {hint:?}");
836
837
2.16k
        let mut spill_uses = UseList::new_in(self.ctx.bump());
838
839
2.16k
        let empty_vec = LiveRangeList::new_in(self.ctx.bump());
840
18.1k
        for entry in core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty_vec) {
841
18.1k
            let vreg = self.ctx.ranges[entry.index].vreg;
842
843
18.1k
            self.ctx.scratch_removed_lrs.insert(entry.index);
844
18.1k
            self.ctx.scratch_removed_lrs_vregs.insert(vreg);
845
846
18.1k
            trace!(" -> removing old LR {:?} for vreg {:?}", entry.index, vreg);
847
848
18.1k
            let mut spill_range = entry.range;
849
18.1k
            let mut spill_starts_def = false;
850
851
18.1k
            let empty_vec = UseList::new_in(self.ctx.bump());
852
68.1k
            for u in core::mem::replace(&mut self.ctx.ranges[entry.index].uses, empty_vec) {
853
68.1k
                trace!("   -> use {:?}", u);
854
855
68.1k
                let is_def = u.operand.kind() == OperandKind::Def;
856
857
                // If this use has an `any` constraint, eagerly migrate it to the spill range. The
858
                // reasoning here is that in the second-chance allocation for the spill bundle,
859
                // any-constrained uses will be easy to satisfy. Solving those constraints earlier
860
                // could create unnecessary conflicts with existing bundles that need to fit in a
861
                // register, more strict requirements, so we delay them eagerly.
862
68.1k
                if u.operand.constraint() == OperandConstraint::Any {
863
17.1k
                    trace!("    -> migrating this any-constrained use to the spill range");
864
17.1k
                    spill_uses.push(u);
865
866
                    // Remember if we're moving the def of this vreg into the spill range, so that
867
                    // we can set the appropriate flags on it later.
868
17.1k
                    spill_starts_def = spill_starts_def || is_def;
869
870
17.1k
                    continue;
871
51.0k
                }
872
873
                // If this is a def, make sure that the spill range
874
                // starts before the next instruction rather than at
875
                // this position: the value is available only *after*
876
                // this position.
877
51.0k
                if is_def {
878
668
                    trace!("    -> moving the spill range forward by one");
879
668
                    spill_range.from = ProgPoint::before(u.pos.inst().next());
880
50.3k
                }
881
882
                // Create a new LR.
883
51.0k
                let cr = minimal_range_for_use(&u);
884
51.0k
                let lr = self.ctx.ranges.add(cr, self.ctx.bump());
885
51.0k
                new_lrs.push((vreg, lr));
886
51.0k
                self.ctx.ranges[lr].uses.push(u);
887
51.0k
                self.ctx.ranges[lr].vreg = vreg;
888
889
                // Create a new bundle that contains only this LR.
890
51.0k
                let new_bundle = self.ctx.bundles.add(self.ctx.bump());
891
51.0k
                self.ctx.ranges[lr].bundle = new_bundle;
892
51.0k
                self.ctx.bundles[new_bundle].spillset = spillset;
893
51.0k
                self.ctx.bundles[new_bundle]
894
51.0k
                    .ranges
895
51.0k
                    .push(LiveRangeListEntry {
896
51.0k
                        range: cr,
897
51.0k
                        index: lr,
898
51.0k
                    });
899
51.0k
                new_bundles.push(new_bundle);
900
901
                // If this use was a Def, set the StartsAtDef flag for the new LR.
902
51.0k
                if is_def {
903
668
                    self.ctx.ranges[lr].set_flag(LiveRangeFlag::StartsAtDef);
904
50.3k
                }
905
906
51.0k
                trace!(
907
                    "    -> created new LR {:?} range {:?} with new bundle {:?} for this use",
908
                    lr,
909
                    cr,
910
                    new_bundle
911
                );
912
            }
913
914
18.1k
            if !spill_range.is_empty() {
915
                // Make one entry in the spill bundle that covers the whole range.
916
                // TODO: it might be worth tracking enough state to only create this LR when there is
917
                // open space in the original LR.
918
18.1k
                let spill_lr = self.ctx.ranges.add(spill_range, self.ctx.bump());
919
18.1k
                self.ctx.ranges[spill_lr].vreg = vreg;
920
18.1k
                self.ctx.ranges[spill_lr].bundle = spill;
921
18.1k
                self.ctx.ranges[spill_lr].uses.extend(spill_uses.drain(..));
922
18.1k
                new_lrs.push((vreg, spill_lr));
923
924
18.1k
                if spill_starts_def {
925
2
                    self.ctx.ranges[spill_lr].set_flag(LiveRangeFlag::StartsAtDef);
926
18.1k
                }
927
928
18.1k
                self.ctx.bundles[spill].ranges.push(LiveRangeListEntry {
929
18.1k
                    range: spill_range,
930
18.1k
                    index: spill_lr,
931
18.1k
                });
932
18.1k
                self.ctx.ranges[spill_lr].bundle = spill;
933
18.1k
                trace!(
934
                    "  -> added spill range {:?} in new LR {:?} in spill bundle {:?}",
935
                    spill_range,
936
                    spill_lr,
937
                    spill
938
                );
939
            } else {
940
0
                assert!(spill_uses.is_empty());
941
            }
942
        }
943
944
        // Remove all of the removed LRs from respective vregs' lists.
945
2.74k
        for vreg in self.ctx.scratch_removed_lrs_vregs.drain() {
946
2.74k
            let lrs = &mut self.ctx.scratch_removed_lrs;
947
2.74k
            self.ctx.vregs[vreg]
948
2.74k
                .ranges
949
45.1k
                .retain(|entry| !lrs.contains(&entry.index));
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::split_into_minimal_bundles::{closure#0}
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}
Line
Count
Source
949
45.1k
                .retain(|entry| !lrs.contains(&entry.index));
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}
950
        }
951
952
        // Add the new LRs to their respective vreg lists.
953
69.2k
        for (vreg, lr) in new_lrs {
954
69.2k
            let range = self.ctx.ranges[lr].range;
955
69.2k
            let entry = LiveRangeListEntry { range, index: lr };
956
69.2k
            self.ctx.vregs[vreg].ranges.push(entry);
957
69.2k
        }
958
959
        // Recompute bundle properties for all new bundles and enqueue
960
        // them.
961
51.0k
        for bundle in new_bundles {
962
51.0k
            if self.ctx.bundles[bundle].ranges.len() > 0 {
963
51.0k
                self.recompute_bundle_properties(bundle);
964
51.0k
                let prio = self.ctx.bundles[bundle].prio;
965
51.0k
                self.ctx
966
51.0k
                    .allocation_queue
967
51.0k
                    .insert(bundle, prio as usize, hint);
968
51.0k
            }
969
        }
970
2.16k
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::split_into_minimal_bundles
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles
Line
Count
Source
823
2.16k
    pub fn split_into_minimal_bundles(&mut self, bundle: LiveBundleIndex, hint: PReg) {
824
2.16k
        assert_eq!(self.ctx.scratch_removed_lrs_vregs.len(), 0);
825
2.16k
        self.ctx.scratch_removed_lrs.clear();
826
827
2.16k
        let mut new_lrs: SmallVec<[(VRegIndex, LiveRangeIndex); 16]> = smallvec![];
828
2.16k
        let mut new_bundles: SmallVec<[LiveBundleIndex; 16]> = smallvec![];
829
830
2.16k
        let spillset = self.ctx.bundles[bundle].spillset;
831
2.16k
        let spill = self
832
2.16k
            .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true)
833
2.16k
            .unwrap();
834
835
2.16k
        trace!("Splitting bundle {bundle:?} into minimal bundles with reg hint {hint:?}");
836
837
2.16k
        let mut spill_uses = UseList::new_in(self.ctx.bump());
838
839
2.16k
        let empty_vec = LiveRangeList::new_in(self.ctx.bump());
840
18.1k
        for entry in core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty_vec) {
841
18.1k
            let vreg = self.ctx.ranges[entry.index].vreg;
842
843
18.1k
            self.ctx.scratch_removed_lrs.insert(entry.index);
844
18.1k
            self.ctx.scratch_removed_lrs_vregs.insert(vreg);
845
846
18.1k
            trace!(" -> removing old LR {:?} for vreg {:?}", entry.index, vreg);
847
848
18.1k
            let mut spill_range = entry.range;
849
18.1k
            let mut spill_starts_def = false;
850
851
18.1k
            let empty_vec = UseList::new_in(self.ctx.bump());
852
68.1k
            for u in core::mem::replace(&mut self.ctx.ranges[entry.index].uses, empty_vec) {
853
68.1k
                trace!("   -> use {:?}", u);
854
855
68.1k
                let is_def = u.operand.kind() == OperandKind::Def;
856
857
                // If this use has an `any` constraint, eagerly migrate it to the spill range. The
858
                // reasoning here is that in the second-chance allocation for the spill bundle,
859
                // any-constrained uses will be easy to satisfy. Solving those constraints earlier
860
                // could create unnecessary conflicts with existing bundles that need to fit in a
861
                // register, more strict requirements, so we delay them eagerly.
862
68.1k
                if u.operand.constraint() == OperandConstraint::Any {
863
17.1k
                    trace!("    -> migrating this any-constrained use to the spill range");
864
17.1k
                    spill_uses.push(u);
865
866
                    // Remember if we're moving the def of this vreg into the spill range, so that
867
                    // we can set the appropriate flags on it later.
868
17.1k
                    spill_starts_def = spill_starts_def || is_def;
869
870
17.1k
                    continue;
871
51.0k
                }
872
873
                // If this is a def, make sure that the spill range
874
                // starts before the next instruction rather than at
875
                // this position: the value is available only *after*
876
                // this position.
877
51.0k
                if is_def {
878
668
                    trace!("    -> moving the spill range forward by one");
879
668
                    spill_range.from = ProgPoint::before(u.pos.inst().next());
880
50.3k
                }
881
882
                // Create a new LR.
883
51.0k
                let cr = minimal_range_for_use(&u);
884
51.0k
                let lr = self.ctx.ranges.add(cr, self.ctx.bump());
885
51.0k
                new_lrs.push((vreg, lr));
886
51.0k
                self.ctx.ranges[lr].uses.push(u);
887
51.0k
                self.ctx.ranges[lr].vreg = vreg;
888
889
                // Create a new bundle that contains only this LR.
890
51.0k
                let new_bundle = self.ctx.bundles.add(self.ctx.bump());
891
51.0k
                self.ctx.ranges[lr].bundle = new_bundle;
892
51.0k
                self.ctx.bundles[new_bundle].spillset = spillset;
893
51.0k
                self.ctx.bundles[new_bundle]
894
51.0k
                    .ranges
895
51.0k
                    .push(LiveRangeListEntry {
896
51.0k
                        range: cr,
897
51.0k
                        index: lr,
898
51.0k
                    });
899
51.0k
                new_bundles.push(new_bundle);
900
901
                // If this use was a Def, set the StartsAtDef flag for the new LR.
902
51.0k
                if is_def {
903
668
                    self.ctx.ranges[lr].set_flag(LiveRangeFlag::StartsAtDef);
904
50.3k
                }
905
906
51.0k
                trace!(
907
                    "    -> created new LR {:?} range {:?} with new bundle {:?} for this use",
908
                    lr,
909
                    cr,
910
                    new_bundle
911
                );
912
            }
913
914
18.1k
            if !spill_range.is_empty() {
915
                // Make one entry in the spill bundle that covers the whole range.
916
                // TODO: it might be worth tracking enough state to only create this LR when there is
917
                // open space in the original LR.
918
18.1k
                let spill_lr = self.ctx.ranges.add(spill_range, self.ctx.bump());
919
18.1k
                self.ctx.ranges[spill_lr].vreg = vreg;
920
18.1k
                self.ctx.ranges[spill_lr].bundle = spill;
921
18.1k
                self.ctx.ranges[spill_lr].uses.extend(spill_uses.drain(..));
922
18.1k
                new_lrs.push((vreg, spill_lr));
923
924
18.1k
                if spill_starts_def {
925
2
                    self.ctx.ranges[spill_lr].set_flag(LiveRangeFlag::StartsAtDef);
926
18.1k
                }
927
928
18.1k
                self.ctx.bundles[spill].ranges.push(LiveRangeListEntry {
929
18.1k
                    range: spill_range,
930
18.1k
                    index: spill_lr,
931
18.1k
                });
932
18.1k
                self.ctx.ranges[spill_lr].bundle = spill;
933
18.1k
                trace!(
934
                    "  -> added spill range {:?} in new LR {:?} in spill bundle {:?}",
935
                    spill_range,
936
                    spill_lr,
937
                    spill
938
                );
939
            } else {
940
0
                assert!(spill_uses.is_empty());
941
            }
942
        }
943
944
        // Remove all of the removed LRs from respective vregs' lists.
945
2.74k
        for vreg in self.ctx.scratch_removed_lrs_vregs.drain() {
946
2.74k
            let lrs = &mut self.ctx.scratch_removed_lrs;
947
2.74k
            self.ctx.vregs[vreg]
948
2.74k
                .ranges
949
2.74k
                .retain(|entry| !lrs.contains(&entry.index));
950
        }
951
952
        // Add the new LRs to their respective vreg lists.
953
69.2k
        for (vreg, lr) in new_lrs {
954
69.2k
            let range = self.ctx.ranges[lr].range;
955
69.2k
            let entry = LiveRangeListEntry { range, index: lr };
956
69.2k
            self.ctx.vregs[vreg].ranges.push(entry);
957
69.2k
        }
958
959
        // Recompute bundle properties for all new bundles and enqueue
960
        // them.
961
51.0k
        for bundle in new_bundles {
962
51.0k
            if self.ctx.bundles[bundle].ranges.len() > 0 {
963
51.0k
                self.recompute_bundle_properties(bundle);
964
51.0k
                let prio = self.ctx.bundles[bundle].prio;
965
51.0k
                self.ctx
966
51.0k
                    .allocation_queue
967
51.0k
                    .insert(bundle, prio as usize, hint);
968
51.0k
            }
969
        }
970
2.16k
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles
971
972
9.89M
    pub fn process_bundle(
973
9.89M
        &mut self,
974
9.89M
        bundle: LiveBundleIndex,
975
9.89M
        hint: PReg,
976
9.89M
    ) -> Result<(), RegAllocError> {
977
9.89M
        let class = self.ctx.spillsets[self.bundles[bundle].spillset].class;
978
979
        // Grab a hint from either the queue or our spillset, if any.
980
9.89M
        let mut hint = if hint != PReg::invalid() {
981
2.43M
            hint
982
        } else {
983
7.46M
            self.ctx.spillsets[self.bundles[bundle].spillset].hint
984
        };
985
9.89M
        if self.ctx.pregs[hint.index()].is_stack {
986
0
            hint = PReg::invalid();
987
9.89M
        }
988
9.89M
        trace!("process_bundle: bundle {bundle:?} hint {hint:?}");
989
990
9.89M
        let req = match self.compute_requirement(bundle) {
991
9.82M
            Ok(req) => req,
992
64.2k
            Err(conflict) => {
993
64.2k
                trace!("conflict!: {:?}", conflict);
994
                // We have to split right away. We'll find a point to
995
                // split that would allow at least the first half of the
996
                // split to be conflict-free.
997
64.2k
                debug_assert!(
998
0
                    !self.minimal_bundle(bundle),
999
                    "Minimal bundle with conflict!"
1000
                );
1001
64.2k
                self.split_and_requeue_bundle(
1002
64.2k
                    bundle,
1003
64.2k
                    /* split_at_point = */ conflict.suggested_split_point(),
1004
64.2k
                    hint,
1005
                    /* trim_ends_into_spill_bundle = */
1006
64.2k
                    conflict.should_trim_edges_around_split(),
1007
                );
1008
64.2k
                return Ok(());
1009
            }
1010
        };
1011
1012
        // If no requirement at all (because no uses), and *if* a
1013
        // spill bundle is already present, then move the LRs over to
1014
        // the spill bundle right away.
1015
9.82M
        match req {
1016
            Requirement::Any => {
1017
614k
                if let Some(spill) =
1018
635k
                    self.get_or_create_spill_bundle(bundle, /* create_if_absent = */ false)
1019
                {
1020
614k
                    let empty_vec = LiveRangeList::new_in(self.ctx.bump());
1021
614k
                    let mut list =
1022
614k
                        core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty_vec);
1023
614k
                    for entry in &list {
1024
614k
                        self.ctx.ranges[entry.index].bundle = spill;
1025
614k
                    }
1026
614k
                    self.ctx.bundles[spill].ranges.extend(list.drain(..));
1027
614k
                    return Ok(());
1028
20.8k
                }
1029
            }
1030
9.19M
            _ => {}
1031
        }
1032
1033
        // Try to allocate!
1034
9.21M
        let mut attempts = 0;
1035
9.21M
        let mut scratch = core::mem::take(&mut self.ctx.scratch_conflicts);
1036
9.21M
        let mut lowest_cost_evict_conflict_set = core::mem::take(&mut self.ctx.scratch_bundle);
1037
        'outer: loop {
1038
10.2M
            attempts += 1;
1039
10.2M
            trace!("attempt {}, req {:?}", attempts, req);
1040
10.2M
            debug_assert!(attempts < 100 * self.func.num_insts());
1041
1042
10.2M
            let fixed_preg = match req {
1043
2.38M
                Requirement::FixedReg(preg) | Requirement::FixedStack(preg) => Some(preg),
1044
7.83M
                Requirement::Register | Requirement::Limit(..) => None,
1045
                Requirement::Stack => {
1046
                    // If we must be on the stack, mark our spillset
1047
                    // as required immediately.
1048
0
                    let spillset = self.bundles[bundle].spillset;
1049
0
                    self.spillsets[spillset].required = true;
1050
0
                    return Ok(());
1051
                }
1052
1053
                Requirement::Any => {
1054
20.8k
                    self.ctx.spilled_bundles.push(bundle);
1055
20.8k
                    break;
1056
                }
1057
            };
1058
            // Scan all pregs, or the one fixed preg, and attempt to allocate.
1059
1060
10.2M
            let mut lowest_cost_evict_conflict_cost: Option<u32> = None;
1061
10.2M
            lowest_cost_evict_conflict_set.clear();
1062
1063
10.2M
            let mut lowest_cost_split_conflict_cost: Option<u32> = None;
1064
10.2M
            let mut lowest_cost_split_conflict_point = ProgPoint::before(Inst::new(0));
1065
10.2M
            let mut lowest_cost_split_conflict_reg = PReg::invalid();
1066
1067
            // Heuristic: start the scan for an available
1068
            // register at an offset influenced both by our
1069
            // location in the code and by the bundle we're
1070
            // considering. This has the effect of spreading
1071
            // demand more evenly across registers.
1072
10.2M
            let scan_offset = self.ctx.ranges[self.bundles[bundle].ranges[0].index]
1073
10.2M
                .range
1074
10.2M
                .from
1075
10.2M
                .inst()
1076
10.2M
                .index()
1077
10.2M
                + bundle.index();
1078
1079
10.2M
            self.ctx.output.stats.process_bundle_reg_probe_start_any += 1;
1080
10.2M
            let limit = self.bundles[bundle].limit.map(|l| l as usize);
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::process_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}
1081
44.4M
            for preg in RegTraversalIter::new(
1082
10.2M
                self.env,
1083
10.2M
                class,
1084
10.2M
                fixed_preg,
1085
10.2M
                hint.as_valid(),
1086
10.2M
                scan_offset,
1087
10.2M
                limit,
1088
            ) {
1089
44.4M
                self.ctx.output.stats.process_bundle_reg_probes_any += 1;
1090
44.4M
                let preg_idx = PRegIndex::new(preg.index());
1091
44.4M
                trace!("trying preg {:?}", preg_idx);
1092
1093
44.4M
                let scan_limit_cost = match (
1094
44.4M
                    lowest_cost_evict_conflict_cost,
1095
44.4M
                    lowest_cost_split_conflict_cost,
1096
                ) {
1097
30.4M
                    (Some(a), Some(b)) => Some(core::cmp::max(a, b)),
1098
14.0M
                    _ => None,
1099
                };
1100
44.4M
                match self.try_to_allocate_bundle_to_reg(
1101
44.4M
                    bundle,
1102
44.4M
                    preg_idx,
1103
44.4M
                    scan_limit_cost,
1104
44.4M
                    &mut scratch,
1105
44.4M
                ) {
1106
7.90M
                    AllocRegResult::Allocated(alloc) => {
1107
7.90M
                        self.ctx.output.stats.process_bundle_reg_success_any += 1;
1108
7.90M
                        trace!(" -> allocated to any {:?}", preg_idx);
1109
7.90M
                        self.ctx.spillsets[self.ctx.bundles[bundle].spillset].hint =
1110
7.90M
                            alloc.as_reg().unwrap();
1111
                        // Success, return scratch memory to context and finish
1112
7.90M
                        break 'outer;
1113
                    }
1114
18.5M
                    AllocRegResult::Conflict(bundles, first_conflict_point) => {
1115
18.5M
                        trace!(
1116
                            " -> conflict with bundles {:?}, first conflict at {:?}",
1117
                            bundles,
1118
                            first_conflict_point
1119
                        );
1120
1121
18.5M
                        let conflict_cost = self.maximum_spill_weight_in_bundle_set(bundles);
1122
1123
18.5M
                        if lowest_cost_evict_conflict_cost.is_none()
1124
14.0M
                            || conflict_cost < lowest_cost_evict_conflict_cost.unwrap()
1125
7.28M
                        {
1126
7.28M
                            lowest_cost_evict_conflict_cost = Some(conflict_cost);
1127
7.28M
                            lowest_cost_evict_conflict_set.clear();
1128
7.28M
                            lowest_cost_evict_conflict_set.extend(bundles);
1129
11.2M
                        }
1130
1131
18.5M
                        let loop_depth =
1132
18.5M
                            self.ctx.cfginfo.approx_loop_depth[self.ctx.cfginfo.insn_block
1133
18.5M
                                [first_conflict_point.inst().index()]
1134
18.5M
                            .index()];
1135
18.5M
                        let move_cost = spill_weight_from_constraint(
1136
18.5M
                            OperandConstraint::Reg,
1137
18.5M
                            loop_depth as usize,
1138
                            /* is_def = */ true,
1139
                        )
1140
18.5M
                        .to_int();
1141
18.5M
                        if lowest_cost_split_conflict_cost.is_none()
1142
14.8M
                            || (conflict_cost + move_cost)
1143
14.8M
                                < lowest_cost_split_conflict_cost.unwrap()
1144
4.75M
                        {
1145
4.75M
                            lowest_cost_split_conflict_cost = Some(conflict_cost + move_cost);
1146
4.75M
                            lowest_cost_split_conflict_point = first_conflict_point;
1147
4.75M
                            lowest_cost_split_conflict_reg = preg;
1148
13.7M
                        }
1149
                    }
1150
17.9M
                    AllocRegResult::ConflictWithFixed(max_cost, point) => {
1151
17.9M
                        trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point);
1152
1153
17.9M
                        let loop_depth = self.ctx.cfginfo.approx_loop_depth
1154
17.9M
                            [self.ctx.cfginfo.insn_block[point.inst().index()].index()];
1155
17.9M
                        let move_cost = spill_weight_from_constraint(
1156
17.9M
                            OperandConstraint::Reg,
1157
17.9M
                            loop_depth as usize,
1158
                            /* is_def = */ true,
1159
                        )
1160
17.9M
                        .to_int();
1161
1162
17.9M
                        if lowest_cost_split_conflict_cost.is_none()
1163
16.7M
                            || (max_cost + move_cost) < lowest_cost_split_conflict_cost.unwrap()
1164
2.12M
                        {
1165
2.12M
                            lowest_cost_split_conflict_cost = Some(max_cost + move_cost);
1166
2.12M
                            lowest_cost_split_conflict_point = point;
1167
2.12M
                            lowest_cost_split_conflict_reg = preg;
1168
15.8M
                        }
1169
                    }
1170
                    AllocRegResult::ConflictHighCost => {
1171
                        // Simply don't consider -- we already have
1172
                        // a lower-cost conflict bundle option
1173
                        // to evict.
1174
59.2k
                        continue;
1175
                    }
1176
                }
1177
            }
1178
1179
            // Otherwise, we *require* a register, but didn't fit into
1180
            // any with current bundle assignments. Hence, we will need
1181
            // to either split or attempt to evict some bundles.
1182
1183
2.32M
            trace!(
1184
                " -> lowest cost evict: set {:?}, cost {:?}",
1185
                lowest_cost_evict_conflict_set,
1186
                lowest_cost_evict_conflict_cost,
1187
            );
1188
2.32M
            trace!(
1189
                " -> lowest cost split: cost {:?}, point {:?}, reg {:?}",
1190
                lowest_cost_split_conflict_cost,
1191
                lowest_cost_split_conflict_point,
1192
                lowest_cost_split_conflict_reg
1193
            );
1194
1195
            // If we reach here, we *must* have an option either to split or evict.
1196
2.32M
            debug_assert!(
1197
0
                lowest_cost_split_conflict_cost.is_some()
1198
0
                    || lowest_cost_evict_conflict_cost.is_some()
1199
            );
1200
1201
2.32M
            let our_spill_weight = self.bundle_spill_weight(bundle);
1202
2.32M
            trace!(" -> our spill weight: {}", our_spill_weight);
1203
1204
            // We detect the "too-many-live-registers" case here and
1205
            // return an error cleanly, rather than panicking, because
1206
            // the regalloc.rs fuzzer depends on the register
1207
            // allocator to correctly reject impossible-to-allocate
1208
            // programs in order to discard invalid test cases.
1209
2.32M
            if self.minimal_bundle(bundle)
1210
78.6k
                && (attempts >= 2
1211
78.6k
                    || lowest_cost_evict_conflict_cost.is_none()
1212
78.6k
                    || lowest_cost_evict_conflict_cost.unwrap() >= our_spill_weight)
1213
            {
1214
0
                if matches!(req, Requirement::Register | Requirement::Limit(_)) {
1215
                    // Check if this is a too-many-live-registers situation.
1216
0
                    let range = self.ctx.bundles[bundle].ranges[0].range;
1217
0
                    trace!("checking for too many live regs");
1218
0
                    let mut min_bundles_assigned = 0;
1219
0
                    let mut fixed_assigned = 0;
1220
0
                    let mut total_regs = 0;
1221
0
                    for preg in self.env.preferred_regs_by_class[class as u8 as usize]
1222
0
                        .iter()
1223
0
                        .chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter())
1224
                    {
1225
0
                        trace!(" -> PR {:?}", preg);
1226
0
                        let start = LiveRangeKey::from_range(&CodeRange {
1227
0
                            from: range.from.prev(),
1228
0
                            to: range.from.prev(),
1229
0
                        });
1230
0
                        for (key, lr) in self.ctx.pregs[preg.index()]
1231
0
                            .allocations
1232
0
                            .btree
1233
0
                            .range(start..)
1234
                        {
1235
0
                            let preg_range = key.to_range();
1236
0
                            if preg_range.to <= range.from {
1237
0
                                continue;
1238
0
                            }
1239
0
                            if preg_range.from >= range.to {
1240
0
                                break;
1241
0
                            }
1242
0
                            if lr.is_valid() {
1243
0
                                if self.minimal_bundle(self.ranges[*lr].bundle) {
1244
0
                                    trace!("  -> min bundle {:?}", lr);
1245
0
                                    min_bundles_assigned += 1;
1246
                                } else {
1247
0
                                    trace!("  -> non-min bundle {:?}", lr);
1248
                                }
1249
                            } else {
1250
0
                                trace!("  -> fixed bundle");
1251
0
                                fixed_assigned += 1;
1252
                            }
1253
                        }
1254
1255
                        // We also need to discard any registers that do not fit
1256
                        // under the limit--we cannot allocate to them.
1257
0
                        if let Requirement::Limit(limit) = req {
1258
0
                            if preg.hw_enc() >= limit as usize {
1259
0
                                continue;
1260
0
                            }
1261
0
                        }
1262
1263
0
                        total_regs += 1;
1264
                    }
1265
0
                    trace!(
1266
                        " -> total {}, fixed {}, min {}",
1267
                        total_regs,
1268
                        fixed_assigned,
1269
                        min_bundles_assigned
1270
                    );
1271
0
                    if min_bundles_assigned + fixed_assigned >= total_regs {
1272
0
                        return Err(RegAllocError::TooManyLiveRegs);
1273
0
                    }
1274
0
                }
1275
1276
0
                panic!("Could not allocate minimal bundle, but the allocation problem should be possible to solve");
1277
2.32M
            }
1278
1279
            // If our bundle's weight is less than or equal to(*) the
1280
            // evict cost, choose to split.  Also pick splitting if
1281
            // we're on our second or more attempt and we didn't
1282
            // allocate.  Also pick splitting if the conflict set is
1283
            // empty, meaning a fixed conflict that can't be evicted.
1284
            //
1285
            // (*) the "equal to" part is very important: it prevents
1286
            // an infinite loop where two bundles with equal spill
1287
            // cost continually evict each other in an infinite
1288
            // allocation loop. In such a case, the first bundle in
1289
            // wins, and the other splits.
1290
            //
1291
            // Note that we don't split if the bundle is minimal.
1292
2.32M
            if !self.minimal_bundle(bundle)
1293
2.24M
                && (attempts >= 2
1294
2.24M
                    || lowest_cost_evict_conflict_cost.is_none()
1295
2.13M
                    || our_spill_weight <= lowest_cost_evict_conflict_cost.unwrap())
1296
            {
1297
1.29M
                trace!(
1298
                    " -> deciding to split: our spill weight is {}",
1299
0
                    self.bundle_spill_weight(bundle)
1300
                );
1301
1.29M
                let bundle_start = self.ctx.bundles[bundle].ranges[0].range.from;
1302
1.29M
                let mut split_at_point =
1303
1.29M
                    core::cmp::max(lowest_cost_split_conflict_point, bundle_start);
1304
1.29M
                let requeue_with_reg = lowest_cost_split_conflict_reg;
1305
1306
                // Adjust `split_at_point` if it is within a deeper loop
1307
                // than the bundle start -- hoist it to just before the
1308
                // first loop header it encounters.
1309
1.29M
                let bundle_start_depth = self.ctx.cfginfo.approx_loop_depth
1310
1.29M
                    [self.ctx.cfginfo.insn_block[bundle_start.inst().index()].index()];
1311
1.29M
                let split_at_depth = self.ctx.cfginfo.approx_loop_depth
1312
1.29M
                    [self.ctx.cfginfo.insn_block[split_at_point.inst().index()].index()];
1313
1.29M
                if split_at_depth > bundle_start_depth {
1314
7.00k
                    for block in (self.ctx.cfginfo.insn_block[bundle_start.inst().index()].index()
1315
927
                        + 1)
1316
927
                        ..=self.ctx.cfginfo.insn_block[split_at_point.inst().index()].index()
1317
                    {
1318
7.00k
                        if self.ctx.cfginfo.approx_loop_depth[block] > bundle_start_depth {
1319
927
                            split_at_point = self.ctx.cfginfo.block_entry[block];
1320
927
                            break;
1321
6.07k
                        }
1322
                    }
1323
1.29M
                }
1324
1325
1.29M
                self.split_and_requeue_bundle(
1326
1.29M
                    bundle,
1327
1.29M
                    split_at_point,
1328
1.29M
                    requeue_with_reg,
1329
                    /* should_trim = */ true,
1330
                );
1331
1332
                // Success, return scratch memory to context and finish
1333
1.29M
                break 'outer;
1334
            } else {
1335
                // Evict all bundles in `conflicting bundles` and try again.
1336
1.03M
                self.ctx.output.stats.evict_bundle_event += 1;
1337
1.03M
                for &bundle in &lowest_cost_evict_conflict_set {
1338
1.03M
                    trace!(" -> evicting {:?}", bundle);
1339
1.03M
                    self.evict_bundle(bundle);
1340
1.03M
                    self.ctx.output.stats.evict_bundle_count += 1;
1341
                }
1342
            }
1343
        }
1344
1345
9.21M
        self.ctx.scratch_conflicts = scratch;
1346
9.21M
        self.ctx.scratch_bundle = lowest_cost_evict_conflict_set;
1347
9.21M
        return Ok(());
1348
9.89M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::process_bundle
<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundle
Line
Count
Source
972
9.89M
    pub fn process_bundle(
973
9.89M
        &mut self,
974
9.89M
        bundle: LiveBundleIndex,
975
9.89M
        hint: PReg,
976
9.89M
    ) -> Result<(), RegAllocError> {
977
9.89M
        let class = self.ctx.spillsets[self.bundles[bundle].spillset].class;
978
979
        // Grab a hint from either the queue or our spillset, if any.
980
9.89M
        let mut hint = if hint != PReg::invalid() {
981
2.43M
            hint
982
        } else {
983
7.46M
            self.ctx.spillsets[self.bundles[bundle].spillset].hint
984
        };
985
9.89M
        if self.ctx.pregs[hint.index()].is_stack {
986
0
            hint = PReg::invalid();
987
9.89M
        }
988
9.89M
        trace!("process_bundle: bundle {bundle:?} hint {hint:?}");
989
990
9.89M
        let req = match self.compute_requirement(bundle) {
991
9.82M
            Ok(req) => req,
992
64.2k
            Err(conflict) => {
993
64.2k
                trace!("conflict!: {:?}", conflict);
994
                // We have to split right away. We'll find a point to
995
                // split that would allow at least the first half of the
996
                // split to be conflict-free.
997
64.2k
                debug_assert!(
998
0
                    !self.minimal_bundle(bundle),
999
                    "Minimal bundle with conflict!"
1000
                );
1001
64.2k
                self.split_and_requeue_bundle(
1002
64.2k
                    bundle,
1003
64.2k
                    /* split_at_point = */ conflict.suggested_split_point(),
1004
64.2k
                    hint,
1005
                    /* trim_ends_into_spill_bundle = */
1006
64.2k
                    conflict.should_trim_edges_around_split(),
1007
                );
1008
64.2k
                return Ok(());
1009
            }
1010
        };
1011
1012
        // If no requirement at all (because no uses), and *if* a
1013
        // spill bundle is already present, then move the LRs over to
1014
        // the spill bundle right away.
1015
9.82M
        match req {
1016
            Requirement::Any => {
1017
614k
                if let Some(spill) =
1018
635k
                    self.get_or_create_spill_bundle(bundle, /* create_if_absent = */ false)
1019
                {
1020
614k
                    let empty_vec = LiveRangeList::new_in(self.ctx.bump());
1021
614k
                    let mut list =
1022
614k
                        core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty_vec);
1023
614k
                    for entry in &list {
1024
614k
                        self.ctx.ranges[entry.index].bundle = spill;
1025
614k
                    }
1026
614k
                    self.ctx.bundles[spill].ranges.extend(list.drain(..));
1027
614k
                    return Ok(());
1028
20.8k
                }
1029
            }
1030
9.19M
            _ => {}
1031
        }
1032
1033
        // Try to allocate!
1034
9.21M
        let mut attempts = 0;
1035
9.21M
        let mut scratch = core::mem::take(&mut self.ctx.scratch_conflicts);
1036
9.21M
        let mut lowest_cost_evict_conflict_set = core::mem::take(&mut self.ctx.scratch_bundle);
1037
        'outer: loop {
1038
10.2M
            attempts += 1;
1039
10.2M
            trace!("attempt {}, req {:?}", attempts, req);
1040
10.2M
            debug_assert!(attempts < 100 * self.func.num_insts());
1041
1042
10.2M
            let fixed_preg = match req {
1043
2.38M
                Requirement::FixedReg(preg) | Requirement::FixedStack(preg) => Some(preg),
1044
7.83M
                Requirement::Register | Requirement::Limit(..) => None,
1045
                Requirement::Stack => {
1046
                    // If we must be on the stack, mark our spillset
1047
                    // as required immediately.
1048
0
                    let spillset = self.bundles[bundle].spillset;
1049
0
                    self.spillsets[spillset].required = true;
1050
0
                    return Ok(());
1051
                }
1052
1053
                Requirement::Any => {
1054
20.8k
                    self.ctx.spilled_bundles.push(bundle);
1055
20.8k
                    break;
1056
                }
1057
            };
1058
            // Scan all pregs, or the one fixed preg, and attempt to allocate.
1059
1060
10.2M
            let mut lowest_cost_evict_conflict_cost: Option<u32> = None;
1061
10.2M
            lowest_cost_evict_conflict_set.clear();
1062
1063
10.2M
            let mut lowest_cost_split_conflict_cost: Option<u32> = None;
1064
10.2M
            let mut lowest_cost_split_conflict_point = ProgPoint::before(Inst::new(0));
1065
10.2M
            let mut lowest_cost_split_conflict_reg = PReg::invalid();
1066
1067
            // Heuristic: start the scan for an available
1068
            // register at an offset influenced both by our
1069
            // location in the code and by the bundle we're
1070
            // considering. This has the effect of spreading
1071
            // demand more evenly across registers.
1072
10.2M
            let scan_offset = self.ctx.ranges[self.bundles[bundle].ranges[0].index]
1073
10.2M
                .range
1074
10.2M
                .from
1075
10.2M
                .inst()
1076
10.2M
                .index()
1077
10.2M
                + bundle.index();
1078
1079
10.2M
            self.ctx.output.stats.process_bundle_reg_probe_start_any += 1;
1080
10.2M
            let limit = self.bundles[bundle].limit.map(|l| l as usize);
1081
44.4M
            for preg in RegTraversalIter::new(
1082
10.2M
                self.env,
1083
10.2M
                class,
1084
10.2M
                fixed_preg,
1085
10.2M
                hint.as_valid(),
1086
10.2M
                scan_offset,
1087
10.2M
                limit,
1088
            ) {
1089
44.4M
                self.ctx.output.stats.process_bundle_reg_probes_any += 1;
1090
44.4M
                let preg_idx = PRegIndex::new(preg.index());
1091
44.4M
                trace!("trying preg {:?}", preg_idx);
1092
1093
44.4M
                let scan_limit_cost = match (
1094
44.4M
                    lowest_cost_evict_conflict_cost,
1095
44.4M
                    lowest_cost_split_conflict_cost,
1096
                ) {
1097
30.4M
                    (Some(a), Some(b)) => Some(core::cmp::max(a, b)),
1098
14.0M
                    _ => None,
1099
                };
1100
44.4M
                match self.try_to_allocate_bundle_to_reg(
1101
44.4M
                    bundle,
1102
44.4M
                    preg_idx,
1103
44.4M
                    scan_limit_cost,
1104
44.4M
                    &mut scratch,
1105
44.4M
                ) {
1106
7.90M
                    AllocRegResult::Allocated(alloc) => {
1107
7.90M
                        self.ctx.output.stats.process_bundle_reg_success_any += 1;
1108
7.90M
                        trace!(" -> allocated to any {:?}", preg_idx);
1109
7.90M
                        self.ctx.spillsets[self.ctx.bundles[bundle].spillset].hint =
1110
7.90M
                            alloc.as_reg().unwrap();
1111
                        // Success, return scratch memory to context and finish
1112
7.90M
                        break 'outer;
1113
                    }
1114
18.5M
                    AllocRegResult::Conflict(bundles, first_conflict_point) => {
1115
18.5M
                        trace!(
1116
                            " -> conflict with bundles {:?}, first conflict at {:?}",
1117
                            bundles,
1118
                            first_conflict_point
1119
                        );
1120
1121
18.5M
                        let conflict_cost = self.maximum_spill_weight_in_bundle_set(bundles);
1122
1123
18.5M
                        if lowest_cost_evict_conflict_cost.is_none()
1124
14.0M
                            || conflict_cost < lowest_cost_evict_conflict_cost.unwrap()
1125
7.28M
                        {
1126
7.28M
                            lowest_cost_evict_conflict_cost = Some(conflict_cost);
1127
7.28M
                            lowest_cost_evict_conflict_set.clear();
1128
7.28M
                            lowest_cost_evict_conflict_set.extend(bundles);
1129
11.2M
                        }
1130
1131
18.5M
                        let loop_depth =
1132
18.5M
                            self.ctx.cfginfo.approx_loop_depth[self.ctx.cfginfo.insn_block
1133
18.5M
                                [first_conflict_point.inst().index()]
1134
18.5M
                            .index()];
1135
18.5M
                        let move_cost = spill_weight_from_constraint(
1136
18.5M
                            OperandConstraint::Reg,
1137
18.5M
                            loop_depth as usize,
1138
                            /* is_def = */ true,
1139
                        )
1140
18.5M
                        .to_int();
1141
18.5M
                        if lowest_cost_split_conflict_cost.is_none()
1142
14.8M
                            || (conflict_cost + move_cost)
1143
14.8M
                                < lowest_cost_split_conflict_cost.unwrap()
1144
4.75M
                        {
1145
4.75M
                            lowest_cost_split_conflict_cost = Some(conflict_cost + move_cost);
1146
4.75M
                            lowest_cost_split_conflict_point = first_conflict_point;
1147
4.75M
                            lowest_cost_split_conflict_reg = preg;
1148
13.7M
                        }
1149
                    }
1150
17.9M
                    AllocRegResult::ConflictWithFixed(max_cost, point) => {
1151
17.9M
                        trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point);
1152
1153
17.9M
                        let loop_depth = self.ctx.cfginfo.approx_loop_depth
1154
17.9M
                            [self.ctx.cfginfo.insn_block[point.inst().index()].index()];
1155
17.9M
                        let move_cost = spill_weight_from_constraint(
1156
17.9M
                            OperandConstraint::Reg,
1157
17.9M
                            loop_depth as usize,
1158
                            /* is_def = */ true,
1159
                        )
1160
17.9M
                        .to_int();
1161
1162
17.9M
                        if lowest_cost_split_conflict_cost.is_none()
1163
16.7M
                            || (max_cost + move_cost) < lowest_cost_split_conflict_cost.unwrap()
1164
2.12M
                        {
1165
2.12M
                            lowest_cost_split_conflict_cost = Some(max_cost + move_cost);
1166
2.12M
                            lowest_cost_split_conflict_point = point;
1167
2.12M
                            lowest_cost_split_conflict_reg = preg;
1168
15.8M
                        }
1169
                    }
1170
                    AllocRegResult::ConflictHighCost => {
1171
                        // Simply don't consider -- we already have
1172
                        // a lower-cost conflict bundle option
1173
                        // to evict.
1174
59.2k
                        continue;
1175
                    }
1176
                }
1177
            }
1178
1179
            // Otherwise, we *require* a register, but didn't fit into
1180
            // any with current bundle assignments. Hence, we will need
1181
            // to either split or attempt to evict some bundles.
1182
1183
2.32M
            trace!(
1184
                " -> lowest cost evict: set {:?}, cost {:?}",
1185
                lowest_cost_evict_conflict_set,
1186
                lowest_cost_evict_conflict_cost,
1187
            );
1188
2.32M
            trace!(
1189
                " -> lowest cost split: cost {:?}, point {:?}, reg {:?}",
1190
                lowest_cost_split_conflict_cost,
1191
                lowest_cost_split_conflict_point,
1192
                lowest_cost_split_conflict_reg
1193
            );
1194
1195
            // If we reach here, we *must* have an option either to split or evict.
1196
2.32M
            debug_assert!(
1197
0
                lowest_cost_split_conflict_cost.is_some()
1198
0
                    || lowest_cost_evict_conflict_cost.is_some()
1199
            );
1200
1201
2.32M
            let our_spill_weight = self.bundle_spill_weight(bundle);
1202
2.32M
            trace!(" -> our spill weight: {}", our_spill_weight);
1203
1204
            // We detect the "too-many-live-registers" case here and
1205
            // return an error cleanly, rather than panicking, because
1206
            // the regalloc.rs fuzzer depends on the register
1207
            // allocator to correctly reject impossible-to-allocate
1208
            // programs in order to discard invalid test cases.
1209
2.32M
            if self.minimal_bundle(bundle)
1210
78.6k
                && (attempts >= 2
1211
78.6k
                    || lowest_cost_evict_conflict_cost.is_none()
1212
78.6k
                    || lowest_cost_evict_conflict_cost.unwrap() >= our_spill_weight)
1213
            {
1214
0
                if matches!(req, Requirement::Register | Requirement::Limit(_)) {
1215
                    // Check if this is a too-many-live-registers situation.
1216
0
                    let range = self.ctx.bundles[bundle].ranges[0].range;
1217
0
                    trace!("checking for too many live regs");
1218
0
                    let mut min_bundles_assigned = 0;
1219
0
                    let mut fixed_assigned = 0;
1220
0
                    let mut total_regs = 0;
1221
0
                    for preg in self.env.preferred_regs_by_class[class as u8 as usize]
1222
0
                        .iter()
1223
0
                        .chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter())
1224
                    {
1225
0
                        trace!(" -> PR {:?}", preg);
1226
0
                        let start = LiveRangeKey::from_range(&CodeRange {
1227
0
                            from: range.from.prev(),
1228
0
                            to: range.from.prev(),
1229
0
                        });
1230
0
                        for (key, lr) in self.ctx.pregs[preg.index()]
1231
0
                            .allocations
1232
0
                            .btree
1233
0
                            .range(start..)
1234
                        {
1235
0
                            let preg_range = key.to_range();
1236
0
                            if preg_range.to <= range.from {
1237
0
                                continue;
1238
0
                            }
1239
0
                            if preg_range.from >= range.to {
1240
0
                                break;
1241
0
                            }
1242
0
                            if lr.is_valid() {
1243
0
                                if self.minimal_bundle(self.ranges[*lr].bundle) {
1244
0
                                    trace!("  -> min bundle {:?}", lr);
1245
0
                                    min_bundles_assigned += 1;
1246
                                } else {
1247
0
                                    trace!("  -> non-min bundle {:?}", lr);
1248
                                }
1249
                            } else {
1250
0
                                trace!("  -> fixed bundle");
1251
0
                                fixed_assigned += 1;
1252
                            }
1253
                        }
1254
1255
                        // We also need to discard any registers that do not fit
1256
                        // under the limit--we cannot allocate to them.
1257
0
                        if let Requirement::Limit(limit) = req {
1258
0
                            if preg.hw_enc() >= limit as usize {
1259
0
                                continue;
1260
0
                            }
1261
0
                        }
1262
1263
0
                        total_regs += 1;
1264
                    }
1265
0
                    trace!(
1266
                        " -> total {}, fixed {}, min {}",
1267
                        total_regs,
1268
                        fixed_assigned,
1269
                        min_bundles_assigned
1270
                    );
1271
0
                    if min_bundles_assigned + fixed_assigned >= total_regs {
1272
0
                        return Err(RegAllocError::TooManyLiveRegs);
1273
0
                    }
1274
0
                }
1275
1276
0
                panic!("Could not allocate minimal bundle, but the allocation problem should be possible to solve");
1277
2.32M
            }
1278
1279
            // If our bundle's weight is less than or equal to(*) the
1280
            // evict cost, choose to split.  Also pick splitting if
1281
            // we're on our second or more attempt and we didn't
1282
            // allocate.  Also pick splitting if the conflict set is
1283
            // empty, meaning a fixed conflict that can't be evicted.
1284
            //
1285
            // (*) the "equal to" part is very important: it prevents
1286
            // an infinite loop where two bundles with equal spill
1287
            // cost continually evict each other in an infinite
1288
            // allocation loop. In such a case, the first bundle in
1289
            // wins, and the other splits.
1290
            //
1291
            // Note that we don't split if the bundle is minimal.
1292
2.32M
            if !self.minimal_bundle(bundle)
1293
2.24M
                && (attempts >= 2
1294
2.24M
                    || lowest_cost_evict_conflict_cost.is_none()
1295
2.13M
                    || our_spill_weight <= lowest_cost_evict_conflict_cost.unwrap())
1296
            {
1297
1.29M
                trace!(
1298
                    " -> deciding to split: our spill weight is {}",
1299
0
                    self.bundle_spill_weight(bundle)
1300
                );
1301
1.29M
                let bundle_start = self.ctx.bundles[bundle].ranges[0].range.from;
1302
1.29M
                let mut split_at_point =
1303
1.29M
                    core::cmp::max(lowest_cost_split_conflict_point, bundle_start);
1304
1.29M
                let requeue_with_reg = lowest_cost_split_conflict_reg;
1305
1306
                // Adjust `split_at_point` if it is within a deeper loop
1307
                // than the bundle start -- hoist it to just before the
1308
                // first loop header it encounters.
1309
1.29M
                let bundle_start_depth = self.ctx.cfginfo.approx_loop_depth
1310
1.29M
                    [self.ctx.cfginfo.insn_block[bundle_start.inst().index()].index()];
1311
1.29M
                let split_at_depth = self.ctx.cfginfo.approx_loop_depth
1312
1.29M
                    [self.ctx.cfginfo.insn_block[split_at_point.inst().index()].index()];
1313
1.29M
                if split_at_depth > bundle_start_depth {
1314
7.00k
                    for block in (self.ctx.cfginfo.insn_block[bundle_start.inst().index()].index()
1315
927
                        + 1)
1316
927
                        ..=self.ctx.cfginfo.insn_block[split_at_point.inst().index()].index()
1317
                    {
1318
7.00k
                        if self.ctx.cfginfo.approx_loop_depth[block] > bundle_start_depth {
1319
927
                            split_at_point = self.ctx.cfginfo.block_entry[block];
1320
927
                            break;
1321
6.07k
                        }
1322
                    }
1323
1.29M
                }
1324
1325
1.29M
                self.split_and_requeue_bundle(
1326
1.29M
                    bundle,
1327
1.29M
                    split_at_point,
1328
1.29M
                    requeue_with_reg,
1329
                    /* should_trim = */ true,
1330
                );
1331
1332
                // Success, return scratch memory to context and finish
1333
1.29M
                break 'outer;
1334
            } else {
1335
                // Evict all bundles in `conflicting bundles` and try again.
1336
1.03M
                self.ctx.output.stats.evict_bundle_event += 1;
1337
1.03M
                for &bundle in &lowest_cost_evict_conflict_set {
1338
1.03M
                    trace!(" -> evicting {:?}", bundle);
1339
1.03M
                    self.evict_bundle(bundle);
1340
1.03M
                    self.ctx.output.stats.evict_bundle_count += 1;
1341
                }
1342
            }
1343
        }
1344
1345
9.21M
        self.ctx.scratch_conflicts = scratch;
1346
9.21M
        self.ctx.scratch_bundle = lowest_cost_evict_conflict_set;
1347
9.21M
        return Ok(());
1348
9.89M
    }
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundle
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundle
1349
}
1350
1351
/// Compute the minimal range that covers a given use in a minimal
1352
/// bundle. This definition needs to be consistent between bundle
1353
/// property computation and minimal-range splitting (fallback
1354
/// splitting).
1355
///
1356
/// The cases are:
1357
/// - early def: whole instruction
1358
/// - late def: Late only
1359
/// - early use: Early only
1360
/// - late use: whole instruction
1361
2.82M
fn minimal_range_for_use(u: &Use) -> CodeRange {
1362
2.82M
    let early = ProgPoint::before(u.pos.inst());
1363
2.82M
    let late = ProgPoint::after(u.pos.inst());
1364
2.82M
    let next_early = ProgPoint::before(u.pos.inst().next());
1365
2.82M
    match (u.pos.pos(), u.operand.kind()) {
1366
401k
        (InstPosition::Before, OperandKind::Def) => CodeRange {
1367
401k
            from: early,
1368
401k
            to: next_early,
1369
401k
        },
1370
1.14M
        (InstPosition::Before, OperandKind::Use) => CodeRange {
1371
1.14M
            from: early,
1372
1.14M
            to: late,
1373
1.14M
        },
1374
1.28M
        (InstPosition::After, OperandKind::Def) => CodeRange {
1375
1.28M
            from: late,
1376
1.28M
            to: next_early,
1377
1.28M
        },
1378
3.20k
        (InstPosition::After, OperandKind::Use) => CodeRange {
1379
3.20k
            from: early,
1380
3.20k
            to: next_early,
1381
3.20k
        },
1382
    }
1383
2.82M
}