/rust/registry/src/index.crates.io-1949cf8c6b5b557f/regalloc2-0.13.5/src/ion/process.rs
Line | Count | Source |
1 | | /* |
2 | | * This file was initially derived from the files |
3 | | * `js/src/jit/BacktrackingAllocator.h` and |
4 | | * `js/src/jit/BacktrackingAllocator.cpp` in Mozilla Firefox, and was |
5 | | * originally licensed under the Mozilla Public License 2.0. We |
6 | | * subsequently relicensed it to Apache-2.0 WITH LLVM-exception (see |
7 | | * https://github.com/bytecodealliance/regalloc2/issues/7). |
8 | | * |
9 | | * Since the initial port, the design has been substantially evolved |
10 | | * and optimized. |
11 | | */ |
12 | | |
13 | | //! Main allocation loop that processes bundles. |
14 | | |
15 | | use super::{ |
16 | | spill_weight_from_constraint, Env, LiveBundleIndex, LiveBundleVec, LiveRangeFlag, |
17 | | LiveRangeIndex, LiveRangeKey, LiveRangeList, LiveRangeListEntry, PRegIndex, RegTraversalIter, |
18 | | Requirement, SpillWeight, UseList, VRegIndex, |
19 | | }; |
20 | | use crate::{ |
21 | | ion::data_structures::{ |
22 | | CodeRange, Use, BUNDLE_MAX_NORMAL_SPILL_WEIGHT, MAX_SPLITS_PER_SPILLSET, |
23 | | MINIMAL_BUNDLE_SPILL_WEIGHT, MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT, |
24 | | MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT, |
25 | | }, |
26 | | Allocation, Function, Inst, InstPosition, OperandConstraint, OperandKind, PReg, ProgPoint, |
27 | | RegAllocError, |
28 | | }; |
29 | | use core::fmt::Debug; |
30 | | use smallvec::{smallvec, SmallVec}; |
31 | | |
32 | | #[derive(Clone, Debug, PartialEq, Eq)] |
33 | | pub enum AllocRegResult<'a> { |
34 | | Allocated(Allocation), |
35 | | Conflict(&'a [LiveBundleIndex], ProgPoint), |
36 | | ConflictWithFixed(u32, ProgPoint), |
37 | | ConflictHighCost, |
38 | | } |
39 | | |
40 | | impl<'a, F: Function> Env<'a, F> { |
41 | 198k | pub fn process_bundles(&mut self) -> Result<(), RegAllocError> { |
42 | 10.0M | while let Some((bundle, hint)) = self.ctx.allocation_queue.pop() { |
43 | 9.89M | self.ctx.output.stats.process_bundle_count += 1; |
44 | 9.89M | self.process_bundle(bundle, hint)?; |
45 | | } |
46 | 198k | self.ctx.output.stats.final_liverange_count = self.ranges.len(); |
47 | 198k | self.ctx.output.stats.final_bundle_count = self.bundles.len(); |
48 | 198k | self.ctx.output.stats.spill_bundle_count = self.spilled_bundles.len(); |
49 | | |
50 | 198k | Ok(()) |
51 | 198k | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundles Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundles Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundles Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::process_bundles <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundles Line | Count | Source | 41 | 198k | pub fn process_bundles(&mut self) -> Result<(), RegAllocError> { | 42 | 10.0M | while let Some((bundle, hint)) = self.ctx.allocation_queue.pop() { | 43 | 9.89M | self.ctx.output.stats.process_bundle_count += 1; | 44 | 9.89M | self.process_bundle(bundle, hint)?; | 45 | | } | 46 | 198k | self.ctx.output.stats.final_liverange_count = self.ranges.len(); | 47 | 198k | self.ctx.output.stats.final_bundle_count = self.bundles.len(); | 48 | 198k | self.ctx.output.stats.spill_bundle_count = self.spilled_bundles.len(); | 49 | | | 50 | 198k | Ok(()) | 51 | 198k | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundles Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundles |
52 | | |
53 | 59.3M | pub fn try_to_allocate_bundle_to_reg<'b>( |
54 | 59.3M | &mut self, |
55 | 59.3M | bundle: LiveBundleIndex, |
56 | 59.3M | reg: PRegIndex, |
57 | 59.3M | // if the max bundle weight in the conflict set exceeds this |
58 | 59.3M | // cost (if provided), just return |
59 | 59.3M | // `AllocRegResult::ConflictHighCost`. |
60 | 59.3M | max_allowable_cost: Option<u32>, |
61 | 59.3M | conflicts: &'b mut LiveBundleVec, |
62 | 59.3M | ) -> AllocRegResult<'b> { |
63 | 59.3M | trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg); |
64 | 59.3M | conflicts.clear(); |
65 | 59.3M | self.ctx.conflict_set.clear(); |
66 | 59.3M | let mut max_conflict_weight = 0; |
67 | | // Traverse the BTreeMap in order by requesting the whole |
68 | | // range spanned by the bundle and iterating over that |
69 | | // concurrently with our ranges. Because our ranges are in |
70 | | // order, and the BTreeMap is as well, this allows us to have |
71 | | // an overall O(n log n) + O(b) complexity, where the PReg has |
72 | | // n current ranges and the bundle has b ranges, rather than |
73 | | // O(b * n log n) with the simple probe-for-each-bundle-range |
74 | | // approach. |
75 | | // |
76 | | // Note that the comparator function on a CodeRange tests for |
77 | | // *overlap*, so we are checking whether the BTree contains |
78 | | // any preg range that *overlaps* with range `range`, not |
79 | | // literally the range `range`. |
80 | 59.3M | let bundle_ranges = &self.ctx.bundles[bundle].ranges; |
81 | 59.3M | let from_key = LiveRangeKey::from_range(&CodeRange { |
82 | 59.3M | from: bundle_ranges.first().unwrap().range.from, |
83 | 59.3M | to: bundle_ranges.first().unwrap().range.from, |
84 | 59.3M | }); |
85 | 59.3M | let mut preg_range_iter = self.ctx.pregs[reg.index()] |
86 | 59.3M | .allocations |
87 | 59.3M | .btree |
88 | 59.3M | .range(from_key..) |
89 | 59.3M | .peekable(); |
90 | 59.3M | trace!( |
91 | | "alloc map for {:?} in range {:?}..: {:?}", |
92 | | reg, |
93 | | from_key, |
94 | 0 | self.ctx.pregs[reg.index()].allocations.btree |
95 | | ); |
96 | 59.3M | let mut first_conflict: Option<ProgPoint> = None; |
97 | | |
98 | 65.9M | 'ranges: for entry in bundle_ranges { |
99 | 65.9M | trace!(" -> range LR {:?}: {:?}", entry.index, entry.range); |
100 | 65.9M | let key = LiveRangeKey::from_range(&entry.range); |
101 | | |
102 | 65.9M | let mut skips = 0; |
103 | | 'alloc: loop { |
104 | 101M | trace!(" -> PReg range {:?}", preg_range_iter.peek()); |
105 | | |
106 | | // Advance our BTree traversal until it is >= this bundle |
107 | | // range (i.e., skip PReg allocations in the BTree that |
108 | | // are completely before this bundle range). |
109 | | |
110 | 101M | if preg_range_iter.peek().is_some() && *preg_range_iter.peek().unwrap().0 < key { |
111 | 298k | trace!( |
112 | | "Skipping PReg range {:?}", |
113 | 0 | preg_range_iter.peek().unwrap().0 |
114 | | ); |
115 | 298k | preg_range_iter.next(); |
116 | 298k | skips += 1; |
117 | 298k | if skips >= 16 { |
118 | 1.56k | let from_pos = entry.range.from; |
119 | 1.56k | let from_key = LiveRangeKey::from_range(&CodeRange { |
120 | 1.56k | from: from_pos, |
121 | 1.56k | to: from_pos, |
122 | 1.56k | }); |
123 | 1.56k | preg_range_iter = self.ctx.pregs[reg.index()] |
124 | 1.56k | .allocations |
125 | 1.56k | .btree |
126 | 1.56k | .range(from_key..) |
127 | 1.56k | .peekable(); |
128 | 1.56k | skips = 0; |
129 | 296k | } |
130 | 298k | continue 'alloc; |
131 | 101M | } |
132 | 101M | skips = 0; |
133 | | |
134 | | // If there are no more PReg allocations, we're done! |
135 | 101M | if preg_range_iter.peek().is_none() { |
136 | 13.5M | trace!(" -> no more PReg allocations; so no conflict possible!"); |
137 | 13.5M | break 'ranges; |
138 | 87.6M | } |
139 | | |
140 | | // If the current PReg range is beyond this range, there is no conflict; continue. |
141 | 87.6M | if *preg_range_iter.peek().unwrap().0 > key { |
142 | 25.9M | trace!( |
143 | | " -> next PReg allocation is at {:?}; moving to next VReg range", |
144 | 0 | preg_range_iter.peek().unwrap().0 |
145 | | ); |
146 | 25.9M | break 'alloc; |
147 | 61.6M | } |
148 | | |
149 | | // Otherwise, there is a conflict. |
150 | 61.6M | let preg_key = *preg_range_iter.peek().unwrap().0; |
151 | 61.6M | debug_assert_eq!(preg_key, key); // Assert that this range overlaps. |
152 | 61.6M | let preg_range = preg_range_iter.next().unwrap().1; |
153 | | |
154 | 61.6M | trace!(" -> btree contains range {:?} that overlaps", preg_range); |
155 | 61.6M | if preg_range.is_valid() { |
156 | 35.2M | trace!(" -> from vreg {:?}", self.ctx.ranges[*preg_range].vreg); |
157 | | // range from an allocated bundle: find the bundle and add to |
158 | | // conflicts list. |
159 | 35.2M | let conflict_bundle = self.ctx.ranges[*preg_range].bundle; |
160 | 35.2M | trace!(" -> conflict bundle {:?}", conflict_bundle); |
161 | 35.2M | if self.ctx.conflict_set.insert(conflict_bundle) { |
162 | 33.0M | conflicts.push(conflict_bundle); |
163 | 33.0M | max_conflict_weight = core::cmp::max( |
164 | 33.0M | max_conflict_weight, |
165 | 33.0M | self.ctx.bundles[conflict_bundle].cached_spill_weight(), |
166 | 33.0M | ); |
167 | 33.0M | if max_allowable_cost.is_some() |
168 | 14.1M | && max_conflict_weight > max_allowable_cost.unwrap() |
169 | | { |
170 | 59.2k | trace!(" -> reached high cost, retrying early"); |
171 | 59.2k | return AllocRegResult::ConflictHighCost; |
172 | 32.9M | } |
173 | 2.22M | } |
174 | | |
175 | 35.2M | if first_conflict.is_none() { |
176 | 25.2M | first_conflict = Some(ProgPoint::from_index(core::cmp::max( |
177 | 25.2M | preg_key.from, |
178 | 25.2M | key.from, |
179 | 25.2M | ))); |
180 | 25.2M | } |
181 | | } else { |
182 | 26.3M | trace!(" -> conflict with fixed reservation"); |
183 | | // range from a direct use of the PReg (due to clobber). |
184 | 26.3M | return AllocRegResult::ConflictWithFixed( |
185 | 26.3M | max_conflict_weight, |
186 | 26.3M | ProgPoint::from_index(preg_key.from), |
187 | 26.3M | ); |
188 | | } |
189 | | } |
190 | | } |
191 | | |
192 | 32.9M | if conflicts.len() > 0 { |
193 | 24.8M | return AllocRegResult::Conflict(conflicts, first_conflict.unwrap()); |
194 | 8.09M | } |
195 | | |
196 | | // We can allocate! Add our ranges to the preg's BTree. |
197 | 8.09M | let preg = PReg::from_index(reg.index()); |
198 | 8.09M | trace!(" -> bundle {:?} assigned to preg {:?}", bundle, preg); |
199 | 8.09M | self.ctx.bundles[bundle].allocation = Allocation::reg(preg); |
200 | 9.41M | for entry in &self.ctx.bundles[bundle].ranges { |
201 | 9.41M | let key = LiveRangeKey::from_range(&entry.range); |
202 | 9.41M | let res = self.ctx.pregs[reg.index()] |
203 | 9.41M | .allocations |
204 | 9.41M | .btree |
205 | 9.41M | .insert(key, entry.index); |
206 | | |
207 | | // We disallow LR overlap within bundles, so this should never be possible. |
208 | 9.41M | debug_assert!(res.is_none()); |
209 | | } |
210 | | |
211 | 8.09M | AllocRegResult::Allocated(Allocation::reg(preg)) |
212 | 59.3M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::try_to_allocate_bundle_to_reg <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg Line | Count | Source | 53 | 59.3M | pub fn try_to_allocate_bundle_to_reg<'b>( | 54 | 59.3M | &mut self, | 55 | 59.3M | bundle: LiveBundleIndex, | 56 | 59.3M | reg: PRegIndex, | 57 | 59.3M | // if the max bundle weight in the conflict set exceeds this | 58 | 59.3M | // cost (if provided), just return | 59 | 59.3M | // `AllocRegResult::ConflictHighCost`. | 60 | 59.3M | max_allowable_cost: Option<u32>, | 61 | 59.3M | conflicts: &'b mut LiveBundleVec, | 62 | 59.3M | ) -> AllocRegResult<'b> { | 63 | 59.3M | trace!("try_to_allocate_bundle_to_reg: {:?} -> {:?}", bundle, reg); | 64 | 59.3M | conflicts.clear(); | 65 | 59.3M | self.ctx.conflict_set.clear(); | 66 | 59.3M | let mut max_conflict_weight = 0; | 67 | | // Traverse the BTreeMap in order by requesting the whole | 68 | | // range spanned by the bundle and iterating over that | 69 | | // concurrently with our ranges. Because our ranges are in | 70 | | // order, and the BTreeMap is as well, this allows us to have | 71 | | // an overall O(n log n) + O(b) complexity, where the PReg has | 72 | | // n current ranges and the bundle has b ranges, rather than | 73 | | // O(b * n log n) with the simple probe-for-each-bundle-range | 74 | | // approach. | 75 | | // | 76 | | // Note that the comparator function on a CodeRange tests for | 77 | | // *overlap*, so we are checking whether the BTree contains | 78 | | // any preg range that *overlaps* with range `range`, not | 79 | | // literally the range `range`. | 80 | 59.3M | let bundle_ranges = &self.ctx.bundles[bundle].ranges; | 81 | 59.3M | let from_key = LiveRangeKey::from_range(&CodeRange { | 82 | 59.3M | from: bundle_ranges.first().unwrap().range.from, | 83 | 59.3M | to: bundle_ranges.first().unwrap().range.from, | 84 | 59.3M | }); | 85 | 59.3M | let mut preg_range_iter = self.ctx.pregs[reg.index()] | 86 | 59.3M | .allocations | 87 | 59.3M | .btree | 88 | 59.3M | .range(from_key..) | 89 | 59.3M | .peekable(); | 90 | 59.3M | trace!( | 91 | | "alloc map for {:?} in range {:?}..: {:?}", | 92 | | reg, | 93 | | from_key, | 94 | 0 | self.ctx.pregs[reg.index()].allocations.btree | 95 | | ); | 96 | 59.3M | let mut first_conflict: Option<ProgPoint> = None; | 97 | | | 98 | 65.9M | 'ranges: for entry in bundle_ranges { | 99 | 65.9M | trace!(" -> range LR {:?}: {:?}", entry.index, entry.range); | 100 | 65.9M | let key = LiveRangeKey::from_range(&entry.range); | 101 | | | 102 | 65.9M | let mut skips = 0; | 103 | | 'alloc: loop { | 104 | 101M | trace!(" -> PReg range {:?}", preg_range_iter.peek()); | 105 | | | 106 | | // Advance our BTree traversal until it is >= this bundle | 107 | | // range (i.e., skip PReg allocations in the BTree that | 108 | | // are completely before this bundle range). | 109 | | | 110 | 101M | if preg_range_iter.peek().is_some() && *preg_range_iter.peek().unwrap().0 < key { | 111 | 298k | trace!( | 112 | | "Skipping PReg range {:?}", | 113 | 0 | preg_range_iter.peek().unwrap().0 | 114 | | ); | 115 | 298k | preg_range_iter.next(); | 116 | 298k | skips += 1; | 117 | 298k | if skips >= 16 { | 118 | 1.56k | let from_pos = entry.range.from; | 119 | 1.56k | let from_key = LiveRangeKey::from_range(&CodeRange { | 120 | 1.56k | from: from_pos, | 121 | 1.56k | to: from_pos, | 122 | 1.56k | }); | 123 | 1.56k | preg_range_iter = self.ctx.pregs[reg.index()] | 124 | 1.56k | .allocations | 125 | 1.56k | .btree | 126 | 1.56k | .range(from_key..) | 127 | 1.56k | .peekable(); | 128 | 1.56k | skips = 0; | 129 | 296k | } | 130 | 298k | continue 'alloc; | 131 | 101M | } | 132 | 101M | skips = 0; | 133 | | | 134 | | // If there are no more PReg allocations, we're done! | 135 | 101M | if preg_range_iter.peek().is_none() { | 136 | 13.5M | trace!(" -> no more PReg allocations; so no conflict possible!"); | 137 | 13.5M | break 'ranges; | 138 | 87.6M | } | 139 | | | 140 | | // If the current PReg range is beyond this range, there is no conflict; continue. | 141 | 87.6M | if *preg_range_iter.peek().unwrap().0 > key { | 142 | 25.9M | trace!( | 143 | | " -> next PReg allocation is at {:?}; moving to next VReg range", | 144 | 0 | preg_range_iter.peek().unwrap().0 | 145 | | ); | 146 | 25.9M | break 'alloc; | 147 | 61.6M | } | 148 | | | 149 | | // Otherwise, there is a conflict. | 150 | 61.6M | let preg_key = *preg_range_iter.peek().unwrap().0; | 151 | 61.6M | debug_assert_eq!(preg_key, key); // Assert that this range overlaps. | 152 | 61.6M | let preg_range = preg_range_iter.next().unwrap().1; | 153 | | | 154 | 61.6M | trace!(" -> btree contains range {:?} that overlaps", preg_range); | 155 | 61.6M | if preg_range.is_valid() { | 156 | 35.2M | trace!(" -> from vreg {:?}", self.ctx.ranges[*preg_range].vreg); | 157 | | // range from an allocated bundle: find the bundle and add to | 158 | | // conflicts list. | 159 | 35.2M | let conflict_bundle = self.ctx.ranges[*preg_range].bundle; | 160 | 35.2M | trace!(" -> conflict bundle {:?}", conflict_bundle); | 161 | 35.2M | if self.ctx.conflict_set.insert(conflict_bundle) { | 162 | 33.0M | conflicts.push(conflict_bundle); | 163 | 33.0M | max_conflict_weight = core::cmp::max( | 164 | 33.0M | max_conflict_weight, | 165 | 33.0M | self.ctx.bundles[conflict_bundle].cached_spill_weight(), | 166 | 33.0M | ); | 167 | 33.0M | if max_allowable_cost.is_some() | 168 | 14.1M | && max_conflict_weight > max_allowable_cost.unwrap() | 169 | | { | 170 | 59.2k | trace!(" -> reached high cost, retrying early"); | 171 | 59.2k | return AllocRegResult::ConflictHighCost; | 172 | 32.9M | } | 173 | 2.22M | } | 174 | | | 175 | 35.2M | if first_conflict.is_none() { | 176 | 25.2M | first_conflict = Some(ProgPoint::from_index(core::cmp::max( | 177 | 25.2M | preg_key.from, | 178 | 25.2M | key.from, | 179 | 25.2M | ))); | 180 | 25.2M | } | 181 | | } else { | 182 | 26.3M | trace!(" -> conflict with fixed reservation"); | 183 | | // range from a direct use of the PReg (due to clobber). | 184 | 26.3M | return AllocRegResult::ConflictWithFixed( | 185 | 26.3M | max_conflict_weight, | 186 | 26.3M | ProgPoint::from_index(preg_key.from), | 187 | 26.3M | ); | 188 | | } | 189 | | } | 190 | | } | 191 | | | 192 | 32.9M | if conflicts.len() > 0 { | 193 | 24.8M | return AllocRegResult::Conflict(conflicts, first_conflict.unwrap()); | 194 | 8.09M | } | 195 | | | 196 | | // We can allocate! Add our ranges to the preg's BTree. | 197 | 8.09M | let preg = PReg::from_index(reg.index()); | 198 | 8.09M | trace!(" -> bundle {:?} assigned to preg {:?}", bundle, preg); | 199 | 8.09M | self.ctx.bundles[bundle].allocation = Allocation::reg(preg); | 200 | 9.41M | for entry in &self.ctx.bundles[bundle].ranges { | 201 | 9.41M | let key = LiveRangeKey::from_range(&entry.range); | 202 | 9.41M | let res = self.ctx.pregs[reg.index()] | 203 | 9.41M | .allocations | 204 | 9.41M | .btree | 205 | 9.41M | .insert(key, entry.index); | 206 | | | 207 | | // We disallow LR overlap within bundles, so this should never be possible. | 208 | 9.41M | debug_assert!(res.is_none()); | 209 | | } | 210 | | | 211 | 8.09M | AllocRegResult::Allocated(Allocation::reg(preg)) | 212 | 59.3M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::try_to_allocate_bundle_to_reg |
213 | | |
214 | 1.03M | pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) { |
215 | 1.03M | trace!( |
216 | | "evicting bundle {:?}: alloc {:?}", |
217 | | bundle, |
218 | 0 | self.ctx.bundles[bundle].allocation |
219 | | ); |
220 | 1.03M | let preg = match self.ctx.bundles[bundle].allocation.as_reg() { |
221 | 1.03M | Some(preg) => preg, |
222 | | None => { |
223 | 0 | trace!( |
224 | | " -> has no allocation! {:?}", |
225 | 0 | self.ctx.bundles[bundle].allocation |
226 | | ); |
227 | 0 | return; |
228 | | } |
229 | | }; |
230 | 1.03M | let preg_idx = PRegIndex::new(preg.index()); |
231 | 1.03M | self.ctx.bundles[bundle].allocation = Allocation::none(); |
232 | 1.24M | for entry in &self.ctx.bundles[bundle].ranges { |
233 | 1.24M | trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx); |
234 | 1.24M | self.ctx.pregs[preg_idx.index()] |
235 | 1.24M | .allocations |
236 | 1.24M | .btree |
237 | 1.24M | .remove(&LiveRangeKey::from_range(&entry.range)); |
238 | | } |
239 | 1.03M | let prio = self.ctx.bundles[bundle].prio; |
240 | 1.03M | trace!(" -> prio {}; back into queue", prio); |
241 | 1.03M | self.ctx |
242 | 1.03M | .allocation_queue |
243 | 1.03M | .insert(bundle, prio as usize, PReg::invalid()); |
244 | 1.03M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::evict_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::evict_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::evict_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::evict_bundle <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::evict_bundle Line | Count | Source | 214 | 1.03M | pub fn evict_bundle(&mut self, bundle: LiveBundleIndex) { | 215 | 1.03M | trace!( | 216 | | "evicting bundle {:?}: alloc {:?}", | 217 | | bundle, | 218 | 0 | self.ctx.bundles[bundle].allocation | 219 | | ); | 220 | 1.03M | let preg = match self.ctx.bundles[bundle].allocation.as_reg() { | 221 | 1.03M | Some(preg) => preg, | 222 | | None => { | 223 | 0 | trace!( | 224 | | " -> has no allocation! {:?}", | 225 | 0 | self.ctx.bundles[bundle].allocation | 226 | | ); | 227 | 0 | return; | 228 | | } | 229 | | }; | 230 | 1.03M | let preg_idx = PRegIndex::new(preg.index()); | 231 | 1.03M | self.ctx.bundles[bundle].allocation = Allocation::none(); | 232 | 1.24M | for entry in &self.ctx.bundles[bundle].ranges { | 233 | 1.24M | trace!(" -> removing LR {:?} from reg {:?}", entry.index, preg_idx); | 234 | 1.24M | self.ctx.pregs[preg_idx.index()] | 235 | 1.24M | .allocations | 236 | 1.24M | .btree | 237 | 1.24M | .remove(&LiveRangeKey::from_range(&entry.range)); | 238 | | } | 239 | 1.03M | let prio = self.ctx.bundles[bundle].prio; | 240 | 1.03M | trace!(" -> prio {}; back into queue", prio); | 241 | 1.03M | self.ctx | 242 | 1.03M | .allocation_queue | 243 | 1.03M | .insert(bundle, prio as usize, PReg::invalid()); | 244 | 1.03M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::evict_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::evict_bundle |
245 | | |
246 | 2.32M | pub fn bundle_spill_weight(&self, bundle: LiveBundleIndex) -> u32 { |
247 | 2.32M | self.ctx.bundles[bundle].cached_spill_weight() |
248 | 2.32M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::bundle_spill_weight Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::bundle_spill_weight Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::bundle_spill_weight Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::bundle_spill_weight <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::bundle_spill_weight Line | Count | Source | 246 | 2.32M | pub fn bundle_spill_weight(&self, bundle: LiveBundleIndex) -> u32 { | 247 | 2.32M | self.ctx.bundles[bundle].cached_spill_weight() | 248 | 2.32M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::bundle_spill_weight Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::bundle_spill_weight |
249 | | |
250 | 18.5M | pub fn maximum_spill_weight_in_bundle_set(&self, bundles: &[LiveBundleIndex]) -> u32 { |
251 | 18.5M | trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles); |
252 | 18.5M | let m = bundles |
253 | 18.5M | .iter() |
254 | 18.5M | .map(|&b| { |
255 | 18.5M | let w = self.ctx.bundles[b].cached_spill_weight(); |
256 | 18.5M | trace!("bundle{}: {}", b.index(), w); |
257 | 18.5M | w |
258 | 18.5M | }) Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::maximum_spill_weight_in_bundle_set::{closure#0}<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}Line | Count | Source | 254 | 18.5M | .map(|&b| { | 255 | 18.5M | let w = self.ctx.bundles[b].cached_spill_weight(); | 256 | 18.5M | trace!("bundle{}: {}", b.index(), w); | 257 | 18.5M | w | 258 | 18.5M | }) |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set::{closure#0} |
259 | 18.5M | .max() |
260 | 18.5M | .unwrap_or(0); |
261 | 18.5M | trace!(" -> max: {}", m); |
262 | 18.5M | m |
263 | 18.5M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::maximum_spill_weight_in_bundle_set <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set Line | Count | Source | 250 | 18.5M | pub fn maximum_spill_weight_in_bundle_set(&self, bundles: &[LiveBundleIndex]) -> u32 { | 251 | 18.5M | trace!("maximum_spill_weight_in_bundle_set: {:?}", bundles); | 252 | 18.5M | let m = bundles | 253 | 18.5M | .iter() | 254 | 18.5M | .map(|&b| { | 255 | | let w = self.ctx.bundles[b].cached_spill_weight(); | 256 | | trace!("bundle{}: {}", b.index(), w); | 257 | | w | 258 | | }) | 259 | 18.5M | .max() | 260 | 18.5M | .unwrap_or(0); | 261 | 18.5M | trace!(" -> max: {}", m); | 262 | 18.5M | m | 263 | 18.5M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::maximum_spill_weight_in_bundle_set |
264 | | |
265 | 8.86M | pub fn recompute_bundle_properties(&mut self, bundle: LiveBundleIndex) { |
266 | 8.86M | trace!("recompute bundle properties: bundle {:?}", bundle); |
267 | | |
268 | | let minimal; |
269 | 8.86M | let mut fixed = false; |
270 | 8.86M | let mut fixed_def = false; |
271 | 8.86M | let mut stack = false; |
272 | 8.86M | let bundledata = &self.ctx.bundles[bundle]; |
273 | 8.86M | let num_ranges = bundledata.ranges.len(); |
274 | 8.86M | let first_range = bundledata.ranges[0].index; |
275 | 8.86M | let first_range_data = &self.ctx.ranges[first_range]; |
276 | | |
277 | 8.86M | self.ctx.bundles[bundle].prio = self.compute_bundle_prio(bundle); |
278 | 8.86M | self.ctx.bundles[bundle].limit = self.compute_bundle_limit(bundle); |
279 | | |
280 | 8.86M | if first_range_data.vreg.is_invalid() { |
281 | 0 | trace!(" -> no vreg; minimal and fixed"); |
282 | 0 | minimal = true; |
283 | 0 | fixed = true; |
284 | 8.86M | } else if num_ranges == 1 { |
285 | 21.4M | for u in &first_range_data.uses { |
286 | 21.4M | trace!(" -> use: {:?}", u); |
287 | 21.4M | if let OperandConstraint::FixedReg(_) = u.operand.constraint() { |
288 | 1.80M | trace!(" -> fixed operand at {:?}: {:?}", u.pos, u.operand); |
289 | 1.80M | fixed = true; |
290 | 1.80M | if u.operand.kind() == OperandKind::Def { |
291 | 919k | trace!(" -> is fixed def"); |
292 | 919k | fixed_def = true; |
293 | 882k | } |
294 | 19.6M | } |
295 | 21.4M | if let OperandConstraint::Stack = u.operand.constraint() { |
296 | 0 | trace!(" -> stack operand at {:?}: {:?}", u.pos, u.operand); |
297 | 0 | stack = true; |
298 | 21.4M | } |
299 | 21.4M | if stack && fixed { |
300 | 0 | break; |
301 | 21.4M | } |
302 | | } |
303 | | // Minimal if there is only one LR and only up to one use |
304 | | // in that LR, and extent of range is consistent with the |
305 | | // "minimal range for use" definition. |
306 | 8.25M | minimal = match &first_range_data.uses[..] { |
307 | 8.25M | [] => true, |
308 | 2.77M | [only_use] => { |
309 | | // We use a "contains" rather than "exact" range |
310 | | // check because a smaller-than-min range is also |
311 | | // OK, if some logic produced it for a valid |
312 | | // reason -- for example, a dead def. We don't |
313 | | // want to livelock on "smaller than minimal" |
314 | | // ranges. |
315 | 2.77M | let min_range = minimal_range_for_use(only_use); |
316 | 2.77M | min_range.contains(&first_range_data.range) |
317 | | } |
318 | 5.47M | _ => false, |
319 | | }; |
320 | 8.25M | trace!(" -> minimal: {}", minimal); |
321 | 608k | } else { |
322 | 608k | minimal = false; |
323 | 608k | } |
324 | | |
325 | 8.86M | let spill_weight = if minimal { |
326 | 2.53M | if fixed { |
327 | 449k | trace!(" -> fixed and minimal"); |
328 | 449k | MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT |
329 | 2.08M | } else if let Some(limit) = self.ctx.bundles[bundle].limit { |
330 | 0 | trace!(" -> limited({limit}) and minimal"); |
331 | 0 | MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT - u32::from(limit) |
332 | | } else { |
333 | 2.08M | trace!(" -> non-fixed and minimal"); |
334 | 2.08M | MINIMAL_BUNDLE_SPILL_WEIGHT |
335 | | } |
336 | | } else { |
337 | 6.32M | let mut total = SpillWeight::zero(); |
338 | 7.75M | for entry in &self.ctx.bundles[bundle].ranges { |
339 | 7.75M | let range_data = &self.ctx.ranges[entry.index]; |
340 | 7.75M | trace!( |
341 | | " -> uses spill weight: +{:?}", |
342 | 0 | range_data.uses_spill_weight() |
343 | | ); |
344 | 7.75M | total = total + range_data.uses_spill_weight(); |
345 | | } |
346 | | |
347 | 6.32M | if self.ctx.bundles[bundle].prio > 0 { |
348 | 6.32M | let final_weight = (total.to_f32() as u32) / self.ctx.bundles[bundle].prio; |
349 | 6.32M | trace!( |
350 | | " -> dividing by prio {}; final weight {}", |
351 | 0 | self.ctx.bundles[bundle].prio, |
352 | | final_weight |
353 | | ); |
354 | 6.32M | core::cmp::min(BUNDLE_MAX_NORMAL_SPILL_WEIGHT, final_weight) |
355 | | } else { |
356 | 1.46k | 0 |
357 | | } |
358 | | }; |
359 | | |
360 | 8.86M | self.ctx.bundles[bundle].set_cached_spill_weight_and_props( |
361 | 8.86M | spill_weight, |
362 | 8.86M | minimal, |
363 | 8.86M | fixed, |
364 | 8.86M | fixed_def, |
365 | 8.86M | stack, |
366 | | ); |
367 | 8.86M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::recompute_bundle_properties <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties Line | Count | Source | 265 | 8.86M | pub fn recompute_bundle_properties(&mut self, bundle: LiveBundleIndex) { | 266 | 8.86M | trace!("recompute bundle properties: bundle {:?}", bundle); | 267 | | | 268 | | let minimal; | 269 | 8.86M | let mut fixed = false; | 270 | 8.86M | let mut fixed_def = false; | 271 | 8.86M | let mut stack = false; | 272 | 8.86M | let bundledata = &self.ctx.bundles[bundle]; | 273 | 8.86M | let num_ranges = bundledata.ranges.len(); | 274 | 8.86M | let first_range = bundledata.ranges[0].index; | 275 | 8.86M | let first_range_data = &self.ctx.ranges[first_range]; | 276 | | | 277 | 8.86M | self.ctx.bundles[bundle].prio = self.compute_bundle_prio(bundle); | 278 | 8.86M | self.ctx.bundles[bundle].limit = self.compute_bundle_limit(bundle); | 279 | | | 280 | 8.86M | if first_range_data.vreg.is_invalid() { | 281 | 0 | trace!(" -> no vreg; minimal and fixed"); | 282 | 0 | minimal = true; | 283 | 0 | fixed = true; | 284 | 8.86M | } else if num_ranges == 1 { | 285 | 21.4M | for u in &first_range_data.uses { | 286 | 21.4M | trace!(" -> use: {:?}", u); | 287 | 21.4M | if let OperandConstraint::FixedReg(_) = u.operand.constraint() { | 288 | 1.80M | trace!(" -> fixed operand at {:?}: {:?}", u.pos, u.operand); | 289 | 1.80M | fixed = true; | 290 | 1.80M | if u.operand.kind() == OperandKind::Def { | 291 | 919k | trace!(" -> is fixed def"); | 292 | 919k | fixed_def = true; | 293 | 882k | } | 294 | 19.6M | } | 295 | 21.4M | if let OperandConstraint::Stack = u.operand.constraint() { | 296 | 0 | trace!(" -> stack operand at {:?}: {:?}", u.pos, u.operand); | 297 | 0 | stack = true; | 298 | 21.4M | } | 299 | 21.4M | if stack && fixed { | 300 | 0 | break; | 301 | 21.4M | } | 302 | | } | 303 | | // Minimal if there is only one LR and only up to one use | 304 | | // in that LR, and extent of range is consistent with the | 305 | | // "minimal range for use" definition. | 306 | 8.25M | minimal = match &first_range_data.uses[..] { | 307 | 8.25M | [] => true, | 308 | 2.77M | [only_use] => { | 309 | | // We use a "contains" rather than "exact" range | 310 | | // check because a smaller-than-min range is also | 311 | | // OK, if some logic produced it for a valid | 312 | | // reason -- for example, a dead def. We don't | 313 | | // want to livelock on "smaller than minimal" | 314 | | // ranges. | 315 | 2.77M | let min_range = minimal_range_for_use(only_use); | 316 | 2.77M | min_range.contains(&first_range_data.range) | 317 | | } | 318 | 5.47M | _ => false, | 319 | | }; | 320 | 8.25M | trace!(" -> minimal: {}", minimal); | 321 | 608k | } else { | 322 | 608k | minimal = false; | 323 | 608k | } | 324 | | | 325 | 8.86M | let spill_weight = if minimal { | 326 | 2.53M | if fixed { | 327 | 449k | trace!(" -> fixed and minimal"); | 328 | 449k | MINIMAL_FIXED_BUNDLE_SPILL_WEIGHT | 329 | 2.08M | } else if let Some(limit) = self.ctx.bundles[bundle].limit { | 330 | 0 | trace!(" -> limited({limit}) and minimal"); | 331 | 0 | MINIMAL_LIMITED_BUNDLE_SPILL_WEIGHT - u32::from(limit) | 332 | | } else { | 333 | 2.08M | trace!(" -> non-fixed and minimal"); | 334 | 2.08M | MINIMAL_BUNDLE_SPILL_WEIGHT | 335 | | } | 336 | | } else { | 337 | 6.32M | let mut total = SpillWeight::zero(); | 338 | 7.75M | for entry in &self.ctx.bundles[bundle].ranges { | 339 | 7.75M | let range_data = &self.ctx.ranges[entry.index]; | 340 | 7.75M | trace!( | 341 | | " -> uses spill weight: +{:?}", | 342 | 0 | range_data.uses_spill_weight() | 343 | | ); | 344 | 7.75M | total = total + range_data.uses_spill_weight(); | 345 | | } | 346 | | | 347 | 6.32M | if self.ctx.bundles[bundle].prio > 0 { | 348 | 6.32M | let final_weight = (total.to_f32() as u32) / self.ctx.bundles[bundle].prio; | 349 | 6.32M | trace!( | 350 | | " -> dividing by prio {}; final weight {}", | 351 | 0 | self.ctx.bundles[bundle].prio, | 352 | | final_weight | 353 | | ); | 354 | 6.32M | core::cmp::min(BUNDLE_MAX_NORMAL_SPILL_WEIGHT, final_weight) | 355 | | } else { | 356 | 1.46k | 0 | 357 | | } | 358 | | }; | 359 | | | 360 | 8.86M | self.ctx.bundles[bundle].set_cached_spill_weight_and_props( | 361 | 8.86M | spill_weight, | 362 | 8.86M | minimal, | 363 | 8.86M | fixed, | 364 | 8.86M | fixed_def, | 365 | 8.86M | stack, | 366 | | ); | 367 | 8.86M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::recompute_bundle_properties |
368 | | |
369 | 4.64M | pub fn minimal_bundle(&self, bundle: LiveBundleIndex) -> bool { |
370 | 4.64M | self.ctx.bundles[bundle].cached_minimal() |
371 | 4.64M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::minimal_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::minimal_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::minimal_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::minimal_bundle <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::minimal_bundle Line | Count | Source | 369 | 4.64M | pub fn minimal_bundle(&self, bundle: LiveBundleIndex) -> bool { | 370 | 4.64M | self.ctx.bundles[bundle].cached_minimal() | 371 | 4.64M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::minimal_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::minimal_bundle |
372 | | |
373 | 2.70M | pub fn recompute_range_properties(&mut self, range: LiveRangeIndex) { |
374 | 2.70M | let rangedata = &mut self.ctx.ranges[range]; |
375 | 2.70M | let mut w = SpillWeight::zero(); |
376 | 5.43M | for u in &rangedata.uses { |
377 | 5.43M | w = w + SpillWeight::from_bits(u.weight); |
378 | 5.43M | trace!("range{}: use {:?}", range.index(), u); |
379 | | } |
380 | 2.70M | rangedata.set_uses_spill_weight(w); |
381 | 2.70M | if rangedata.uses.len() > 0 && rangedata.uses[0].operand.kind() == OperandKind::Def { |
382 | 1.33M | // Note that we *set* the flag here, but we never *clear* |
383 | 1.33M | // it: it may be set by a progmove as well (which does not |
384 | 1.33M | // create an explicit use or def), and we want to preserve |
385 | 1.33M | // that. We will never split or trim ranges in a way that |
386 | 1.33M | // removes a def at the front and requires the flag to be |
387 | 1.33M | // cleared. |
388 | 1.33M | rangedata.set_flag(LiveRangeFlag::StartsAtDef); |
389 | 1.37M | } |
390 | 2.70M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::recompute_range_properties Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::recompute_range_properties Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::recompute_range_properties Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::recompute_range_properties <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::recompute_range_properties Line | Count | Source | 373 | 2.70M | pub fn recompute_range_properties(&mut self, range: LiveRangeIndex) { | 374 | 2.70M | let rangedata = &mut self.ctx.ranges[range]; | 375 | 2.70M | let mut w = SpillWeight::zero(); | 376 | 5.43M | for u in &rangedata.uses { | 377 | 5.43M | w = w + SpillWeight::from_bits(u.weight); | 378 | 5.43M | trace!("range{}: use {:?}", range.index(), u); | 379 | | } | 380 | 2.70M | rangedata.set_uses_spill_weight(w); | 381 | 2.70M | if rangedata.uses.len() > 0 && rangedata.uses[0].operand.kind() == OperandKind::Def { | 382 | 1.33M | // Note that we *set* the flag here, but we never *clear* | 383 | 1.33M | // it: it may be set by a progmove as well (which does not | 384 | 1.33M | // create an explicit use or def), and we want to preserve | 385 | 1.33M | // that. We will never split or trim ranges in a way that | 386 | 1.33M | // removes a def at the front and requires the flag to be | 387 | 1.33M | // cleared. | 388 | 1.33M | rangedata.set_flag(LiveRangeFlag::StartsAtDef); | 389 | 1.37M | } | 390 | 2.70M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::recompute_range_properties Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::recompute_range_properties |
391 | | |
392 | 2.12M | pub fn get_or_create_spill_bundle( |
393 | 2.12M | &mut self, |
394 | 2.12M | bundle: LiveBundleIndex, |
395 | 2.12M | create_if_absent: bool, |
396 | 2.12M | ) -> Option<LiveBundleIndex> { |
397 | 2.12M | let ssidx = self.ctx.bundles[bundle].spillset; |
398 | 2.12M | let idx = self.ctx.spillsets[ssidx].spill_bundle; |
399 | 2.12M | if idx.is_valid() { |
400 | 1.01M | Some(idx) |
401 | 1.10M | } else if create_if_absent { |
402 | 1.08M | let idx = self.ctx.bundles.add(self.ctx.bump()); |
403 | 1.08M | self.ctx.spillsets[ssidx].spill_bundle = idx; |
404 | 1.08M | self.ctx.bundles[idx].spillset = ssidx; |
405 | 1.08M | self.ctx.spilled_bundles.push(idx); |
406 | 1.08M | Some(idx) |
407 | | } else { |
408 | 20.8k | None |
409 | | } |
410 | 2.12M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::get_or_create_spill_bundle <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle Line | Count | Source | 392 | 2.12M | pub fn get_or_create_spill_bundle( | 393 | 2.12M | &mut self, | 394 | 2.12M | bundle: LiveBundleIndex, | 395 | 2.12M | create_if_absent: bool, | 396 | 2.12M | ) -> Option<LiveBundleIndex> { | 397 | 2.12M | let ssidx = self.ctx.bundles[bundle].spillset; | 398 | 2.12M | let idx = self.ctx.spillsets[ssidx].spill_bundle; | 399 | 2.12M | if idx.is_valid() { | 400 | 1.01M | Some(idx) | 401 | 1.10M | } else if create_if_absent { | 402 | 1.08M | let idx = self.ctx.bundles.add(self.ctx.bump()); | 403 | 1.08M | self.ctx.spillsets[ssidx].spill_bundle = idx; | 404 | 1.08M | self.ctx.bundles[idx].spillset = ssidx; | 405 | 1.08M | self.ctx.spilled_bundles.push(idx); | 406 | 1.08M | Some(idx) | 407 | | } else { | 408 | 20.8k | None | 409 | | } | 410 | 2.12M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::get_or_create_spill_bundle |
411 | | |
412 | 1.35M | pub fn split_and_requeue_bundle( |
413 | 1.35M | &mut self, |
414 | 1.35M | bundle: LiveBundleIndex, |
415 | 1.35M | mut split_at: ProgPoint, |
416 | 1.35M | hint: PReg, |
417 | 1.35M | // Do we trim the parts around the split and put them in the |
418 | 1.35M | // spill bundle? |
419 | 1.35M | mut trim_ends_into_spill_bundle: bool, |
420 | 1.35M | ) { |
421 | 1.35M | self.ctx.output.stats.splits += 1; |
422 | 1.35M | trace!( |
423 | | "split bundle {bundle:?} at {split_at:?} and requeue with reg hint (for first part) {hint:?}" |
424 | | ); |
425 | | |
426 | | // Split `bundle` at `split_at`, creating new LiveRanges and |
427 | | // bundles (and updating vregs' linked lists appropriately), |
428 | | // and enqueue the new bundles. |
429 | | |
430 | 1.35M | let spillset = self.ctx.bundles[bundle].spillset; |
431 | | |
432 | | // Have we reached the maximum split count? If so, fall back |
433 | | // to a "minimal bundles and spill bundle" setup for this |
434 | | // bundle. See the doc-comment on |
435 | | // `split_into_minimal_bundles()` above for more. |
436 | 1.35M | if self.ctx.spillsets[spillset].splits >= MAX_SPLITS_PER_SPILLSET { |
437 | 2.16k | self.split_into_minimal_bundles(bundle, hint); |
438 | 2.16k | return; |
439 | 1.35M | } |
440 | 1.35M | self.ctx.spillsets[spillset].splits += 1; |
441 | | |
442 | 1.35M | debug_assert!(!self.ctx.bundles[bundle].ranges.is_empty()); |
443 | | // Split point *at* start is OK; this means we peel off |
444 | | // exactly one use to create a minimal bundle. |
445 | 1.35M | let bundle_start = self.ctx.bundles[bundle].ranges.first().unwrap().range.from; |
446 | 1.35M | debug_assert!(split_at >= bundle_start); |
447 | 1.35M | let bundle_end = self.ctx.bundles[bundle].ranges.last().unwrap().range.to; |
448 | 1.35M | debug_assert!(split_at < bundle_end); |
449 | | |
450 | 1.35M | trace!( |
451 | | "bundle_start = {:?} bundle_end = {:?}", |
452 | | bundle_start, |
453 | | bundle_end |
454 | | ); |
455 | | |
456 | | // Is the bundle at most spanning one instruction? Then |
457 | | // there's nothing we can do with this logic (we will not |
458 | | // split in the middle of the instruction). Fall back to the |
459 | | // minimal-bundle splitting in this case as well. |
460 | 1.35M | if bundle_end.prev().inst() == bundle_start.inst() { |
461 | 1 | trace!(" -> spans only one inst; splitting into minimal bundles"); |
462 | 1 | self.split_into_minimal_bundles(bundle, hint); |
463 | 1 | return; |
464 | 1.35M | } |
465 | | |
466 | | // Is the split point *at* the start? If so, peel off the |
467 | | // program point with the first use: set the split point just |
468 | | // after it, or just before it if it comes after the start of |
469 | | // the bundle. |
470 | 1.35M | if split_at == bundle_start { |
471 | | // Find any uses; if none, just chop off one instruction. |
472 | 739k | let mut first_use = None; |
473 | 739k | 'outer: for entry in &self.ctx.bundles[bundle].ranges { |
474 | 739k | for u in &self.ctx.ranges[entry.index].uses { |
475 | 739k | first_use = Some(u.pos); |
476 | 739k | break 'outer; |
477 | | } |
478 | | } |
479 | 739k | trace!(" -> first use loc is {:?}", first_use); |
480 | 739k | split_at = match first_use { |
481 | 739k | Some(pos) => { |
482 | 739k | if pos.inst() == bundle_start.inst() { |
483 | 739k | ProgPoint::before(pos.inst().next()) |
484 | | } else { |
485 | 102 | ProgPoint::before(pos.inst()) |
486 | | } |
487 | | } |
488 | 0 | None => ProgPoint::before( |
489 | 0 | self.ctx.bundles[bundle] |
490 | 0 | .ranges |
491 | 0 | .first() |
492 | 0 | .unwrap() |
493 | 0 | .range |
494 | 0 | .from |
495 | 0 | .inst() |
496 | 0 | .next(), |
497 | | ), |
498 | | }; |
499 | 739k | trace!( |
500 | | "split point is at bundle start; advancing to {:?}", |
501 | | split_at |
502 | | ); |
503 | | } else { |
504 | | // Don't split in the middle of an instruction -- this could |
505 | | // create impossible moves (we cannot insert a move between an |
506 | | // instruction's uses and defs). |
507 | 614k | if split_at.pos() == InstPosition::After { |
508 | 485k | split_at = split_at.next(); |
509 | 485k | } |
510 | 614k | if split_at >= bundle_end { |
511 | 38.4k | split_at = split_at.prev().prev(); |
512 | 575k | } |
513 | | } |
514 | | |
515 | 1.35M | debug_assert!(split_at > bundle_start && split_at < bundle_end); |
516 | | |
517 | | // We need to find which LRs fall on each side of the split, |
518 | | // which LR we need to split down the middle, then update the |
519 | | // current bundle, create a new one, and (re)-queue both. |
520 | | |
521 | 1.35M | trace!(" -> LRs: {:?}", self.ctx.bundles[bundle].ranges); |
522 | | |
523 | 1.35M | let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle |
524 | 1.35M | let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle |
525 | 1.38M | for (i, entry) in self.ctx.bundles[bundle].ranges.iter().enumerate() { |
526 | 1.38M | if split_at > entry.range.from { |
527 | 1.38M | last_lr_in_old_bundle_idx = i; |
528 | 1.38M | first_lr_in_new_bundle_idx = i; |
529 | 1.38M | } |
530 | 1.38M | if split_at < entry.range.to { |
531 | 1.35M | first_lr_in_new_bundle_idx = i; |
532 | | |
533 | | // When the bundle contains a fixed constraint, we advance the split point to right |
534 | | // before the first instruction with a fixed use present. |
535 | 1.35M | if self.ctx.bundles[bundle].cached_fixed() { |
536 | 3.24M | for u in &self.ctx.ranges[entry.index].uses { |
537 | 3.24M | if u.pos < split_at { |
538 | 1.48M | continue; |
539 | 1.76M | } |
540 | | |
541 | 1.76M | if matches!(u.operand.constraint(), OperandConstraint::FixedReg { .. }) { |
542 | 242k | split_at = ProgPoint::before(u.pos.inst()); |
543 | | |
544 | 242k | if split_at > entry.range.from { |
545 | 242k | last_lr_in_old_bundle_idx = i; |
546 | 242k | } |
547 | | |
548 | 242k | trace!(" -> advancing split point to {split_at:?}"); |
549 | | |
550 | 242k | trim_ends_into_spill_bundle = false; |
551 | | |
552 | 242k | break; |
553 | 1.52M | } |
554 | | } |
555 | 886k | } |
556 | | |
557 | 1.35M | break; |
558 | 29.3k | } |
559 | | } |
560 | | |
561 | 1.35M | trace!( |
562 | | " -> last LR in old bundle: LR {:?}", |
563 | 0 | self.ctx.bundles[bundle].ranges[last_lr_in_old_bundle_idx] |
564 | | ); |
565 | 1.35M | trace!( |
566 | | " -> first LR in new bundle: LR {:?}", |
567 | 0 | self.ctx.bundles[bundle].ranges[first_lr_in_new_bundle_idx] |
568 | | ); |
569 | | |
570 | | // Take the sublist of LRs that will go in the new bundle. |
571 | 1.35M | let mut new_lr_list: LiveRangeList = LiveRangeList::new_in(self.ctx.bump()); |
572 | 1.35M | new_lr_list.extend( |
573 | 1.35M | self.ctx.bundles[bundle] |
574 | 1.35M | .ranges |
575 | 1.35M | .iter() |
576 | 1.35M | .cloned() |
577 | 1.35M | .skip(first_lr_in_new_bundle_idx), |
578 | | ); |
579 | 1.35M | self.ctx.bundles[bundle] |
580 | 1.35M | .ranges |
581 | 1.35M | .truncate(last_lr_in_old_bundle_idx + 1); |
582 | 1.35M | self.ctx.bundles[bundle].ranges.shrink_to_fit(); |
583 | | |
584 | | // If the first entry in `new_lr_list` is a LR that is split |
585 | | // down the middle, replace it with a new LR and chop off the |
586 | | // end of the same LR in the original list. |
587 | 1.35M | if split_at > new_lr_list[0].range.from { |
588 | 1.35M | debug_assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx); |
589 | 1.35M | let orig_lr = new_lr_list[0].index; |
590 | 1.35M | let new_lr = self.ctx.ranges.add( |
591 | 1.35M | CodeRange { |
592 | 1.35M | from: split_at, |
593 | 1.35M | to: new_lr_list[0].range.to, |
594 | 1.35M | }, |
595 | 1.35M | self.ctx.bump(), |
596 | | ); |
597 | 1.35M | self.ctx.ranges[new_lr].vreg = self.ranges[orig_lr].vreg; |
598 | 1.35M | trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr); |
599 | 1.35M | let first_use = self.ctx.ranges[orig_lr] |
600 | 1.35M | .uses |
601 | 1.35M | .iter() |
602 | 3.82M | .position(|u| u.pos >= split_at) Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::split_and_requeue_bundle::{closure#0}<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}Line | Count | Source | 602 | 3.82M | .position(|u| u.pos >= split_at) |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle::{closure#0} |
603 | 1.35M | .unwrap_or(self.ctx.ranges[orig_lr].uses.len()); |
604 | 1.35M | let mut rest_uses = UseList::new_in(self.ctx.bump()); |
605 | 1.35M | rest_uses.extend( |
606 | 1.35M | self.ctx.ranges[orig_lr] |
607 | 1.35M | .uses |
608 | 1.35M | .iter() |
609 | 1.35M | .cloned() |
610 | 1.35M | .skip(first_use), |
611 | | ); |
612 | 1.35M | self.ctx.ranges[new_lr].uses = rest_uses; |
613 | 1.35M | self.ctx.ranges[orig_lr].uses.truncate(first_use); |
614 | 1.35M | self.ctx.ranges[orig_lr].uses.shrink_to_fit(); |
615 | 1.35M | self.recompute_range_properties(orig_lr); |
616 | 1.35M | self.recompute_range_properties(new_lr); |
617 | 1.35M | new_lr_list[0].index = new_lr; |
618 | 1.35M | new_lr_list[0].range = self.ctx.ranges[new_lr].range; |
619 | 1.35M | self.ctx.ranges[orig_lr].range.to = split_at; |
620 | 1.35M | self.ctx.bundles[bundle].ranges[last_lr_in_old_bundle_idx].range = |
621 | 1.35M | self.ctx.ranges[orig_lr].range; |
622 | | |
623 | | // Perform a lazy split in the VReg data. We just |
624 | | // append the new LR and its range; we will sort by |
625 | | // start of range, and fix up range ends, once when we |
626 | | // iterate over the VReg's ranges after allocation |
627 | | // completes (this is the only time when order |
628 | | // matters). |
629 | 1.35M | self.ctx.vregs[self.ctx.ranges[new_lr].vreg] |
630 | 1.35M | .ranges |
631 | 1.35M | .push(LiveRangeListEntry { |
632 | 1.35M | range: self.ctx.ranges[new_lr].range, |
633 | 1.35M | index: new_lr, |
634 | 1.35M | }); |
635 | 303 | } |
636 | | |
637 | 1.35M | let new_bundle = self.ctx.bundles.add(self.ctx.bump()); |
638 | 1.35M | trace!(" -> creating new bundle {:?}", new_bundle); |
639 | 1.35M | self.ctx.bundles[new_bundle].spillset = spillset; |
640 | 1.70M | for entry in &new_lr_list { |
641 | 1.70M | self.ctx.ranges[entry.index].bundle = new_bundle; |
642 | 1.70M | } |
643 | 1.35M | self.ctx.bundles[new_bundle].ranges = new_lr_list; |
644 | | |
645 | 1.35M | if trim_ends_into_spill_bundle { |
646 | | // Finally, handle moving LRs to the spill bundle when |
647 | | // appropriate: If the first range in `new_bundle` or last |
648 | | // range in `bundle` has "empty space" beyond the first or |
649 | | // last use (respectively), trim it and put an empty LR into |
650 | | // the spill bundle. (We are careful to treat the "starts at |
651 | | // def" flag as an implicit first def even if no def-type Use |
652 | | // is present.) |
653 | 1.12M | while let Some(entry) = self.ctx.bundles[bundle].ranges.last().cloned() { |
654 | 1.12M | let end = entry.range.to; |
655 | 1.12M | let vreg = self.ctx.ranges[entry.index].vreg; |
656 | 1.12M | let last_use = self.ctx.ranges[entry.index].uses.last().map(|u| u.pos); |
657 | 1.12M | if last_use.is_none() { |
658 | 9.05k | let spill = self |
659 | 9.05k | .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) |
660 | 9.05k | .unwrap(); |
661 | 9.05k | trace!( |
662 | | " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", |
663 | | bundle, |
664 | | entry.index, |
665 | | spill |
666 | | ); |
667 | 9.05k | self.ctx.bundles[spill].ranges.push(entry); |
668 | 9.05k | self.ctx.bundles[bundle].ranges.pop(); |
669 | 9.05k | self.ctx.ranges[entry.index].bundle = spill; |
670 | 9.05k | continue; |
671 | 1.11M | } |
672 | 1.11M | let last_use = last_use.unwrap(); |
673 | 1.11M | let split = ProgPoint::before(last_use.inst().next()); |
674 | 1.11M | if split < end { |
675 | 389k | let spill = self |
676 | 389k | .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) |
677 | 389k | .unwrap(); |
678 | 389k | self.ctx.bundles[bundle].ranges.last_mut().unwrap().range.to = split; |
679 | 389k | self.ctx.ranges[self.ctx.bundles[bundle].ranges.last().unwrap().index] |
680 | 389k | .range |
681 | 389k | .to = split; |
682 | 389k | let range = CodeRange { |
683 | 389k | from: split, |
684 | 389k | to: end, |
685 | 389k | }; |
686 | 389k | let empty_lr = self.ctx.ranges.add(range, self.ctx.bump()); |
687 | 389k | self.ctx.bundles[spill].ranges.push(LiveRangeListEntry { |
688 | 389k | range, |
689 | 389k | index: empty_lr, |
690 | 389k | }); |
691 | 389k | self.ctx.ranges[empty_lr].bundle = spill; |
692 | 389k | self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { |
693 | 389k | range, |
694 | 389k | index: empty_lr, |
695 | 389k | }); |
696 | 389k | trace!( |
697 | | " -> bundle {:?} range {:?}: last use implies split point {:?}", |
698 | | bundle, |
699 | | entry.index, |
700 | | split |
701 | | ); |
702 | 389k | trace!( |
703 | | " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", |
704 | | spill, |
705 | | empty_lr |
706 | | ); |
707 | 721k | } |
708 | 1.11M | break; |
709 | | } |
710 | 1.43M | while let Some(entry) = self.ctx.bundles[new_bundle].ranges.first().cloned() { |
711 | 1.23M | if self.ctx.ranges[entry.index].has_flag(LiveRangeFlag::StartsAtDef) { |
712 | 364 | break; |
713 | 1.23M | } |
714 | 1.23M | let start = entry.range.from; |
715 | 1.23M | let vreg = self.ctx.ranges[entry.index].vreg; |
716 | 1.23M | let first_use = self.ctx.ranges[entry.index].uses.first().map(|u| u.pos); |
717 | 1.23M | if first_use.is_none() { |
718 | 319k | let spill = self |
719 | 319k | .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true) |
720 | 319k | .unwrap(); |
721 | 319k | trace!( |
722 | | " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", |
723 | | new_bundle, |
724 | | entry.index, |
725 | | spill |
726 | | ); |
727 | 319k | self.ctx.bundles[spill].ranges.push(entry); |
728 | 319k | self.ctx.bundles[new_bundle].ranges.drain(..1); |
729 | 319k | self.ctx.ranges[entry.index].bundle = spill; |
730 | 319k | continue; |
731 | 913k | } |
732 | 913k | let first_use = first_use.unwrap(); |
733 | 913k | let split = ProgPoint::before(first_use.inst()); |
734 | 913k | if split > start { |
735 | 772k | let spill = self |
736 | 772k | .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true) |
737 | 772k | .unwrap(); |
738 | 772k | self.ctx.bundles[new_bundle] |
739 | 772k | .ranges |
740 | 772k | .first_mut() |
741 | 772k | .unwrap() |
742 | 772k | .range |
743 | 772k | .from = split; |
744 | 772k | self.ctx.ranges[self.ctx.bundles[new_bundle].ranges.first().unwrap().index] |
745 | 772k | .range |
746 | 772k | .from = split; |
747 | 772k | let range = CodeRange { |
748 | 772k | from: start, |
749 | 772k | to: split, |
750 | 772k | }; |
751 | 772k | let empty_lr = self.ctx.ranges.add(range, self.ctx.bump()); |
752 | 772k | self.ctx.bundles[spill].ranges.push(LiveRangeListEntry { |
753 | 772k | range, |
754 | 772k | index: empty_lr, |
755 | 772k | }); |
756 | 772k | self.ctx.ranges[empty_lr].bundle = spill; |
757 | 772k | self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { |
758 | 772k | range, |
759 | 772k | index: empty_lr, |
760 | 772k | }); |
761 | 772k | trace!( |
762 | | " -> bundle {:?} range {:?}: first use implies split point {:?}", |
763 | | bundle, |
764 | | entry.index, |
765 | | first_use, |
766 | | ); |
767 | 772k | trace!( |
768 | | " -> moving leading empty region to new spill bundle {:?} with new LR {:?}", |
769 | | spill, |
770 | | empty_lr |
771 | | ); |
772 | 141k | } |
773 | 913k | break; |
774 | | } |
775 | 242k | } |
776 | | |
777 | 1.35M | if self.ctx.bundles[bundle].ranges.len() > 0 { |
778 | 1.35M | self.recompute_bundle_properties(bundle); |
779 | 1.35M | let prio = self.ctx.bundles[bundle].prio; |
780 | 1.35M | self.ctx |
781 | 1.35M | .allocation_queue |
782 | 1.35M | .insert(bundle, prio as usize, hint); |
783 | 1.35M | } |
784 | 1.35M | if self.ctx.bundles[new_bundle].ranges.len() > 0 { |
785 | 1.15M | self.recompute_bundle_properties(new_bundle); |
786 | 1.15M | let prio = self.ctx.bundles[new_bundle].prio; |
787 | 1.15M | self.ctx |
788 | 1.15M | .allocation_queue |
789 | 1.15M | .insert(new_bundle, prio as usize, hint); |
790 | 1.15M | } |
791 | 1.35M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::split_and_requeue_bundle <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle Line | Count | Source | 412 | 1.35M | pub fn split_and_requeue_bundle( | 413 | 1.35M | &mut self, | 414 | 1.35M | bundle: LiveBundleIndex, | 415 | 1.35M | mut split_at: ProgPoint, | 416 | 1.35M | hint: PReg, | 417 | 1.35M | // Do we trim the parts around the split and put them in the | 418 | 1.35M | // spill bundle? | 419 | 1.35M | mut trim_ends_into_spill_bundle: bool, | 420 | 1.35M | ) { | 421 | 1.35M | self.ctx.output.stats.splits += 1; | 422 | 1.35M | trace!( | 423 | | "split bundle {bundle:?} at {split_at:?} and requeue with reg hint (for first part) {hint:?}" | 424 | | ); | 425 | | | 426 | | // Split `bundle` at `split_at`, creating new LiveRanges and | 427 | | // bundles (and updating vregs' linked lists appropriately), | 428 | | // and enqueue the new bundles. | 429 | | | 430 | 1.35M | let spillset = self.ctx.bundles[bundle].spillset; | 431 | | | 432 | | // Have we reached the maximum split count? If so, fall back | 433 | | // to a "minimal bundles and spill bundle" setup for this | 434 | | // bundle. See the doc-comment on | 435 | | // `split_into_minimal_bundles()` above for more. | 436 | 1.35M | if self.ctx.spillsets[spillset].splits >= MAX_SPLITS_PER_SPILLSET { | 437 | 2.16k | self.split_into_minimal_bundles(bundle, hint); | 438 | 2.16k | return; | 439 | 1.35M | } | 440 | 1.35M | self.ctx.spillsets[spillset].splits += 1; | 441 | | | 442 | 1.35M | debug_assert!(!self.ctx.bundles[bundle].ranges.is_empty()); | 443 | | // Split point *at* start is OK; this means we peel off | 444 | | // exactly one use to create a minimal bundle. | 445 | 1.35M | let bundle_start = self.ctx.bundles[bundle].ranges.first().unwrap().range.from; | 446 | 1.35M | debug_assert!(split_at >= bundle_start); | 447 | 1.35M | let bundle_end = self.ctx.bundles[bundle].ranges.last().unwrap().range.to; | 448 | 1.35M | debug_assert!(split_at < bundle_end); | 449 | | | 450 | 1.35M | trace!( | 451 | | "bundle_start = {:?} bundle_end = {:?}", | 452 | | bundle_start, | 453 | | bundle_end | 454 | | ); | 455 | | | 456 | | // Is the bundle at most spanning one instruction? Then | 457 | | // there's nothing we can do with this logic (we will not | 458 | | // split in the middle of the instruction). Fall back to the | 459 | | // minimal-bundle splitting in this case as well. | 460 | 1.35M | if bundle_end.prev().inst() == bundle_start.inst() { | 461 | 1 | trace!(" -> spans only one inst; splitting into minimal bundles"); | 462 | 1 | self.split_into_minimal_bundles(bundle, hint); | 463 | 1 | return; | 464 | 1.35M | } | 465 | | | 466 | | // Is the split point *at* the start? If so, peel off the | 467 | | // program point with the first use: set the split point just | 468 | | // after it, or just before it if it comes after the start of | 469 | | // the bundle. | 470 | 1.35M | if split_at == bundle_start { | 471 | | // Find any uses; if none, just chop off one instruction. | 472 | 739k | let mut first_use = None; | 473 | 739k | 'outer: for entry in &self.ctx.bundles[bundle].ranges { | 474 | 739k | for u in &self.ctx.ranges[entry.index].uses { | 475 | 739k | first_use = Some(u.pos); | 476 | 739k | break 'outer; | 477 | | } | 478 | | } | 479 | 739k | trace!(" -> first use loc is {:?}", first_use); | 480 | 739k | split_at = match first_use { | 481 | 739k | Some(pos) => { | 482 | 739k | if pos.inst() == bundle_start.inst() { | 483 | 739k | ProgPoint::before(pos.inst().next()) | 484 | | } else { | 485 | 102 | ProgPoint::before(pos.inst()) | 486 | | } | 487 | | } | 488 | 0 | None => ProgPoint::before( | 489 | 0 | self.ctx.bundles[bundle] | 490 | 0 | .ranges | 491 | 0 | .first() | 492 | 0 | .unwrap() | 493 | 0 | .range | 494 | 0 | .from | 495 | 0 | .inst() | 496 | 0 | .next(), | 497 | | ), | 498 | | }; | 499 | 739k | trace!( | 500 | | "split point is at bundle start; advancing to {:?}", | 501 | | split_at | 502 | | ); | 503 | | } else { | 504 | | // Don't split in the middle of an instruction -- this could | 505 | | // create impossible moves (we cannot insert a move between an | 506 | | // instruction's uses and defs). | 507 | 614k | if split_at.pos() == InstPosition::After { | 508 | 485k | split_at = split_at.next(); | 509 | 485k | } | 510 | 614k | if split_at >= bundle_end { | 511 | 38.4k | split_at = split_at.prev().prev(); | 512 | 575k | } | 513 | | } | 514 | | | 515 | 1.35M | debug_assert!(split_at > bundle_start && split_at < bundle_end); | 516 | | | 517 | | // We need to find which LRs fall on each side of the split, | 518 | | // which LR we need to split down the middle, then update the | 519 | | // current bundle, create a new one, and (re)-queue both. | 520 | | | 521 | 1.35M | trace!(" -> LRs: {:?}", self.ctx.bundles[bundle].ranges); | 522 | | | 523 | 1.35M | let mut last_lr_in_old_bundle_idx = 0; // last LR-list index in old bundle | 524 | 1.35M | let mut first_lr_in_new_bundle_idx = 0; // first LR-list index in new bundle | 525 | 1.38M | for (i, entry) in self.ctx.bundles[bundle].ranges.iter().enumerate() { | 526 | 1.38M | if split_at > entry.range.from { | 527 | 1.38M | last_lr_in_old_bundle_idx = i; | 528 | 1.38M | first_lr_in_new_bundle_idx = i; | 529 | 1.38M | } | 530 | 1.38M | if split_at < entry.range.to { | 531 | 1.35M | first_lr_in_new_bundle_idx = i; | 532 | | | 533 | | // When the bundle contains a fixed constraint, we advance the split point to right | 534 | | // before the first instruction with a fixed use present. | 535 | 1.35M | if self.ctx.bundles[bundle].cached_fixed() { | 536 | 3.24M | for u in &self.ctx.ranges[entry.index].uses { | 537 | 3.24M | if u.pos < split_at { | 538 | 1.48M | continue; | 539 | 1.76M | } | 540 | | | 541 | 1.76M | if matches!(u.operand.constraint(), OperandConstraint::FixedReg { .. }) { | 542 | 242k | split_at = ProgPoint::before(u.pos.inst()); | 543 | | | 544 | 242k | if split_at > entry.range.from { | 545 | 242k | last_lr_in_old_bundle_idx = i; | 546 | 242k | } | 547 | | | 548 | 242k | trace!(" -> advancing split point to {split_at:?}"); | 549 | | | 550 | 242k | trim_ends_into_spill_bundle = false; | 551 | | | 552 | 242k | break; | 553 | 1.52M | } | 554 | | } | 555 | 886k | } | 556 | | | 557 | 1.35M | break; | 558 | 29.3k | } | 559 | | } | 560 | | | 561 | 1.35M | trace!( | 562 | | " -> last LR in old bundle: LR {:?}", | 563 | 0 | self.ctx.bundles[bundle].ranges[last_lr_in_old_bundle_idx] | 564 | | ); | 565 | 1.35M | trace!( | 566 | | " -> first LR in new bundle: LR {:?}", | 567 | 0 | self.ctx.bundles[bundle].ranges[first_lr_in_new_bundle_idx] | 568 | | ); | 569 | | | 570 | | // Take the sublist of LRs that will go in the new bundle. | 571 | 1.35M | let mut new_lr_list: LiveRangeList = LiveRangeList::new_in(self.ctx.bump()); | 572 | 1.35M | new_lr_list.extend( | 573 | 1.35M | self.ctx.bundles[bundle] | 574 | 1.35M | .ranges | 575 | 1.35M | .iter() | 576 | 1.35M | .cloned() | 577 | 1.35M | .skip(first_lr_in_new_bundle_idx), | 578 | | ); | 579 | 1.35M | self.ctx.bundles[bundle] | 580 | 1.35M | .ranges | 581 | 1.35M | .truncate(last_lr_in_old_bundle_idx + 1); | 582 | 1.35M | self.ctx.bundles[bundle].ranges.shrink_to_fit(); | 583 | | | 584 | | // If the first entry in `new_lr_list` is a LR that is split | 585 | | // down the middle, replace it with a new LR and chop off the | 586 | | // end of the same LR in the original list. | 587 | 1.35M | if split_at > new_lr_list[0].range.from { | 588 | 1.35M | debug_assert_eq!(last_lr_in_old_bundle_idx, first_lr_in_new_bundle_idx); | 589 | 1.35M | let orig_lr = new_lr_list[0].index; | 590 | 1.35M | let new_lr = self.ctx.ranges.add( | 591 | 1.35M | CodeRange { | 592 | 1.35M | from: split_at, | 593 | 1.35M | to: new_lr_list[0].range.to, | 594 | 1.35M | }, | 595 | 1.35M | self.ctx.bump(), | 596 | | ); | 597 | 1.35M | self.ctx.ranges[new_lr].vreg = self.ranges[orig_lr].vreg; | 598 | 1.35M | trace!(" -> splitting LR {:?} into {:?}", orig_lr, new_lr); | 599 | 1.35M | let first_use = self.ctx.ranges[orig_lr] | 600 | 1.35M | .uses | 601 | 1.35M | .iter() | 602 | 1.35M | .position(|u| u.pos >= split_at) | 603 | 1.35M | .unwrap_or(self.ctx.ranges[orig_lr].uses.len()); | 604 | 1.35M | let mut rest_uses = UseList::new_in(self.ctx.bump()); | 605 | 1.35M | rest_uses.extend( | 606 | 1.35M | self.ctx.ranges[orig_lr] | 607 | 1.35M | .uses | 608 | 1.35M | .iter() | 609 | 1.35M | .cloned() | 610 | 1.35M | .skip(first_use), | 611 | | ); | 612 | 1.35M | self.ctx.ranges[new_lr].uses = rest_uses; | 613 | 1.35M | self.ctx.ranges[orig_lr].uses.truncate(first_use); | 614 | 1.35M | self.ctx.ranges[orig_lr].uses.shrink_to_fit(); | 615 | 1.35M | self.recompute_range_properties(orig_lr); | 616 | 1.35M | self.recompute_range_properties(new_lr); | 617 | 1.35M | new_lr_list[0].index = new_lr; | 618 | 1.35M | new_lr_list[0].range = self.ctx.ranges[new_lr].range; | 619 | 1.35M | self.ctx.ranges[orig_lr].range.to = split_at; | 620 | 1.35M | self.ctx.bundles[bundle].ranges[last_lr_in_old_bundle_idx].range = | 621 | 1.35M | self.ctx.ranges[orig_lr].range; | 622 | | | 623 | | // Perform a lazy split in the VReg data. We just | 624 | | // append the new LR and its range; we will sort by | 625 | | // start of range, and fix up range ends, once when we | 626 | | // iterate over the VReg's ranges after allocation | 627 | | // completes (this is the only time when order | 628 | | // matters). | 629 | 1.35M | self.ctx.vregs[self.ctx.ranges[new_lr].vreg] | 630 | 1.35M | .ranges | 631 | 1.35M | .push(LiveRangeListEntry { | 632 | 1.35M | range: self.ctx.ranges[new_lr].range, | 633 | 1.35M | index: new_lr, | 634 | 1.35M | }); | 635 | 303 | } | 636 | | | 637 | 1.35M | let new_bundle = self.ctx.bundles.add(self.ctx.bump()); | 638 | 1.35M | trace!(" -> creating new bundle {:?}", new_bundle); | 639 | 1.35M | self.ctx.bundles[new_bundle].spillset = spillset; | 640 | 1.70M | for entry in &new_lr_list { | 641 | 1.70M | self.ctx.ranges[entry.index].bundle = new_bundle; | 642 | 1.70M | } | 643 | 1.35M | self.ctx.bundles[new_bundle].ranges = new_lr_list; | 644 | | | 645 | 1.35M | if trim_ends_into_spill_bundle { | 646 | | // Finally, handle moving LRs to the spill bundle when | 647 | | // appropriate: If the first range in `new_bundle` or last | 648 | | // range in `bundle` has "empty space" beyond the first or | 649 | | // last use (respectively), trim it and put an empty LR into | 650 | | // the spill bundle. (We are careful to treat the "starts at | 651 | | // def" flag as an implicit first def even if no def-type Use | 652 | | // is present.) | 653 | 1.12M | while let Some(entry) = self.ctx.bundles[bundle].ranges.last().cloned() { | 654 | 1.12M | let end = entry.range.to; | 655 | 1.12M | let vreg = self.ctx.ranges[entry.index].vreg; | 656 | 1.12M | let last_use = self.ctx.ranges[entry.index].uses.last().map(|u| u.pos); | 657 | 1.12M | if last_use.is_none() { | 658 | 9.05k | let spill = self | 659 | 9.05k | .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) | 660 | 9.05k | .unwrap(); | 661 | 9.05k | trace!( | 662 | | " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", | 663 | | bundle, | 664 | | entry.index, | 665 | | spill | 666 | | ); | 667 | 9.05k | self.ctx.bundles[spill].ranges.push(entry); | 668 | 9.05k | self.ctx.bundles[bundle].ranges.pop(); | 669 | 9.05k | self.ctx.ranges[entry.index].bundle = spill; | 670 | 9.05k | continue; | 671 | 1.11M | } | 672 | 1.11M | let last_use = last_use.unwrap(); | 673 | 1.11M | let split = ProgPoint::before(last_use.inst().next()); | 674 | 1.11M | if split < end { | 675 | 389k | let spill = self | 676 | 389k | .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) | 677 | 389k | .unwrap(); | 678 | 389k | self.ctx.bundles[bundle].ranges.last_mut().unwrap().range.to = split; | 679 | 389k | self.ctx.ranges[self.ctx.bundles[bundle].ranges.last().unwrap().index] | 680 | 389k | .range | 681 | 389k | .to = split; | 682 | 389k | let range = CodeRange { | 683 | 389k | from: split, | 684 | 389k | to: end, | 685 | 389k | }; | 686 | 389k | let empty_lr = self.ctx.ranges.add(range, self.ctx.bump()); | 687 | 389k | self.ctx.bundles[spill].ranges.push(LiveRangeListEntry { | 688 | 389k | range, | 689 | 389k | index: empty_lr, | 690 | 389k | }); | 691 | 389k | self.ctx.ranges[empty_lr].bundle = spill; | 692 | 389k | self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { | 693 | 389k | range, | 694 | 389k | index: empty_lr, | 695 | 389k | }); | 696 | 389k | trace!( | 697 | | " -> bundle {:?} range {:?}: last use implies split point {:?}", | 698 | | bundle, | 699 | | entry.index, | 700 | | split | 701 | | ); | 702 | 389k | trace!( | 703 | | " -> moving trailing empty region to new spill bundle {:?} with new LR {:?}", | 704 | | spill, | 705 | | empty_lr | 706 | | ); | 707 | 721k | } | 708 | 1.11M | break; | 709 | | } | 710 | 1.43M | while let Some(entry) = self.ctx.bundles[new_bundle].ranges.first().cloned() { | 711 | 1.23M | if self.ctx.ranges[entry.index].has_flag(LiveRangeFlag::StartsAtDef) { | 712 | 364 | break; | 713 | 1.23M | } | 714 | 1.23M | let start = entry.range.from; | 715 | 1.23M | let vreg = self.ctx.ranges[entry.index].vreg; | 716 | 1.23M | let first_use = self.ctx.ranges[entry.index].uses.first().map(|u| u.pos); | 717 | 1.23M | if first_use.is_none() { | 718 | 319k | let spill = self | 719 | 319k | .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true) | 720 | 319k | .unwrap(); | 721 | 319k | trace!( | 722 | | " -> bundle {:?} range {:?}: no uses; moving to spill bundle {:?}", | 723 | | new_bundle, | 724 | | entry.index, | 725 | | spill | 726 | | ); | 727 | 319k | self.ctx.bundles[spill].ranges.push(entry); | 728 | 319k | self.ctx.bundles[new_bundle].ranges.drain(..1); | 729 | 319k | self.ctx.ranges[entry.index].bundle = spill; | 730 | 319k | continue; | 731 | 913k | } | 732 | 913k | let first_use = first_use.unwrap(); | 733 | 913k | let split = ProgPoint::before(first_use.inst()); | 734 | 913k | if split > start { | 735 | 772k | let spill = self | 736 | 772k | .get_or_create_spill_bundle(new_bundle, /* create_if_absent = */ true) | 737 | 772k | .unwrap(); | 738 | 772k | self.ctx.bundles[new_bundle] | 739 | 772k | .ranges | 740 | 772k | .first_mut() | 741 | 772k | .unwrap() | 742 | 772k | .range | 743 | 772k | .from = split; | 744 | 772k | self.ctx.ranges[self.ctx.bundles[new_bundle].ranges.first().unwrap().index] | 745 | 772k | .range | 746 | 772k | .from = split; | 747 | 772k | let range = CodeRange { | 748 | 772k | from: start, | 749 | 772k | to: split, | 750 | 772k | }; | 751 | 772k | let empty_lr = self.ctx.ranges.add(range, self.ctx.bump()); | 752 | 772k | self.ctx.bundles[spill].ranges.push(LiveRangeListEntry { | 753 | 772k | range, | 754 | 772k | index: empty_lr, | 755 | 772k | }); | 756 | 772k | self.ctx.ranges[empty_lr].bundle = spill; | 757 | 772k | self.ctx.vregs[vreg].ranges.push(LiveRangeListEntry { | 758 | 772k | range, | 759 | 772k | index: empty_lr, | 760 | 772k | }); | 761 | 772k | trace!( | 762 | | " -> bundle {:?} range {:?}: first use implies split point {:?}", | 763 | | bundle, | 764 | | entry.index, | 765 | | first_use, | 766 | | ); | 767 | 772k | trace!( | 768 | | " -> moving leading empty region to new spill bundle {:?} with new LR {:?}", | 769 | | spill, | 770 | | empty_lr | 771 | | ); | 772 | 141k | } | 773 | 913k | break; | 774 | | } | 775 | 242k | } | 776 | | | 777 | 1.35M | if self.ctx.bundles[bundle].ranges.len() > 0 { | 778 | 1.35M | self.recompute_bundle_properties(bundle); | 779 | 1.35M | let prio = self.ctx.bundles[bundle].prio; | 780 | 1.35M | self.ctx | 781 | 1.35M | .allocation_queue | 782 | 1.35M | .insert(bundle, prio as usize, hint); | 783 | 1.35M | } | 784 | 1.35M | if self.ctx.bundles[new_bundle].ranges.len() > 0 { | 785 | 1.15M | self.recompute_bundle_properties(new_bundle); | 786 | 1.15M | let prio = self.ctx.bundles[new_bundle].prio; | 787 | 1.15M | self.ctx | 788 | 1.15M | .allocation_queue | 789 | 1.15M | .insert(new_bundle, prio as usize, hint); | 790 | 1.15M | } | 791 | 1.35M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_and_requeue_bundle |
792 | | |
793 | | /// Splits the given bundle into minimal bundles per Use, falling |
794 | | /// back onto the spill bundle. This must work for any bundle no |
795 | | /// matter how many conflicts. |
796 | | /// |
797 | | /// This is meant to solve a quadratic-cost problem that exists |
798 | | /// with "normal" splitting as implemented above. With that |
799 | | /// procedure, splitting a bundle produces two |
800 | | /// halves. Furthermore, it has cost linear in the length of the |
801 | | /// bundle, because the resulting half-bundles have their |
802 | | /// requirements recomputed with a new scan, and because we copy |
803 | | /// half the use-list over to the tail end sub-bundle. |
804 | | /// |
805 | | /// This works fine when a bundle has a handful of splits overall, |
806 | | /// but not when an input has a systematic pattern of conflicts |
807 | | /// that will require O(|bundle|) splits (e.g., every Use is |
808 | | /// constrained to a different fixed register than the last |
809 | | /// one). In such a case, we get quadratic behavior. |
810 | | /// |
811 | | /// This method implements a direct split into minimal bundles |
812 | | /// along the whole length of the bundle, putting the regions |
813 | | /// without uses in the spill bundle. We do this once the number |
814 | | /// of splits in an original bundle (tracked by spillset) reaches |
815 | | /// a pre-determined limit. |
816 | | /// |
817 | | /// This basically approximates what a non-splitting allocator |
818 | | /// would do: it "spills" the whole bundle to possibly a |
819 | | /// stackslot, or a second-chance register allocation at best, via |
820 | | /// the spill bundle; and then does minimal reservations of |
821 | | /// registers just at uses/defs and moves the "spilled" value |
822 | | /// into/out of them immediately. |
823 | 2.16k | pub fn split_into_minimal_bundles(&mut self, bundle: LiveBundleIndex, hint: PReg) { |
824 | 2.16k | assert_eq!(self.ctx.scratch_removed_lrs_vregs.len(), 0); |
825 | 2.16k | self.ctx.scratch_removed_lrs.clear(); |
826 | | |
827 | 2.16k | let mut new_lrs: SmallVec<[(VRegIndex, LiveRangeIndex); 16]> = smallvec![]; |
828 | 2.16k | let mut new_bundles: SmallVec<[LiveBundleIndex; 16]> = smallvec![]; |
829 | | |
830 | 2.16k | let spillset = self.ctx.bundles[bundle].spillset; |
831 | 2.16k | let spill = self |
832 | 2.16k | .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) |
833 | 2.16k | .unwrap(); |
834 | | |
835 | 2.16k | trace!("Splitting bundle {bundle:?} into minimal bundles with reg hint {hint:?}"); |
836 | | |
837 | 2.16k | let mut spill_uses = UseList::new_in(self.ctx.bump()); |
838 | | |
839 | 2.16k | let empty_vec = LiveRangeList::new_in(self.ctx.bump()); |
840 | 18.1k | for entry in core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty_vec) { |
841 | 18.1k | let vreg = self.ctx.ranges[entry.index].vreg; |
842 | | |
843 | 18.1k | self.ctx.scratch_removed_lrs.insert(entry.index); |
844 | 18.1k | self.ctx.scratch_removed_lrs_vregs.insert(vreg); |
845 | | |
846 | 18.1k | trace!(" -> removing old LR {:?} for vreg {:?}", entry.index, vreg); |
847 | | |
848 | 18.1k | let mut spill_range = entry.range; |
849 | 18.1k | let mut spill_starts_def = false; |
850 | | |
851 | 18.1k | let empty_vec = UseList::new_in(self.ctx.bump()); |
852 | 68.1k | for u in core::mem::replace(&mut self.ctx.ranges[entry.index].uses, empty_vec) { |
853 | 68.1k | trace!(" -> use {:?}", u); |
854 | | |
855 | 68.1k | let is_def = u.operand.kind() == OperandKind::Def; |
856 | | |
857 | | // If this use has an `any` constraint, eagerly migrate it to the spill range. The |
858 | | // reasoning here is that in the second-chance allocation for the spill bundle, |
859 | | // any-constrained uses will be easy to satisfy. Solving those constraints earlier |
860 | | // could create unnecessary conflicts with existing bundles that need to fit in a |
861 | | // register, more strict requirements, so we delay them eagerly. |
862 | 68.1k | if u.operand.constraint() == OperandConstraint::Any { |
863 | 17.1k | trace!(" -> migrating this any-constrained use to the spill range"); |
864 | 17.1k | spill_uses.push(u); |
865 | | |
866 | | // Remember if we're moving the def of this vreg into the spill range, so that |
867 | | // we can set the appropriate flags on it later. |
868 | 17.1k | spill_starts_def = spill_starts_def || is_def; |
869 | | |
870 | 17.1k | continue; |
871 | 51.0k | } |
872 | | |
873 | | // If this is a def, make sure that the spill range |
874 | | // starts before the next instruction rather than at |
875 | | // this position: the value is available only *after* |
876 | | // this position. |
877 | 51.0k | if is_def { |
878 | 668 | trace!(" -> moving the spill range forward by one"); |
879 | 668 | spill_range.from = ProgPoint::before(u.pos.inst().next()); |
880 | 50.3k | } |
881 | | |
882 | | // Create a new LR. |
883 | 51.0k | let cr = minimal_range_for_use(&u); |
884 | 51.0k | let lr = self.ctx.ranges.add(cr, self.ctx.bump()); |
885 | 51.0k | new_lrs.push((vreg, lr)); |
886 | 51.0k | self.ctx.ranges[lr].uses.push(u); |
887 | 51.0k | self.ctx.ranges[lr].vreg = vreg; |
888 | | |
889 | | // Create a new bundle that contains only this LR. |
890 | 51.0k | let new_bundle = self.ctx.bundles.add(self.ctx.bump()); |
891 | 51.0k | self.ctx.ranges[lr].bundle = new_bundle; |
892 | 51.0k | self.ctx.bundles[new_bundle].spillset = spillset; |
893 | 51.0k | self.ctx.bundles[new_bundle] |
894 | 51.0k | .ranges |
895 | 51.0k | .push(LiveRangeListEntry { |
896 | 51.0k | range: cr, |
897 | 51.0k | index: lr, |
898 | 51.0k | }); |
899 | 51.0k | new_bundles.push(new_bundle); |
900 | | |
901 | | // If this use was a Def, set the StartsAtDef flag for the new LR. |
902 | 51.0k | if is_def { |
903 | 668 | self.ctx.ranges[lr].set_flag(LiveRangeFlag::StartsAtDef); |
904 | 50.3k | } |
905 | | |
906 | 51.0k | trace!( |
907 | | " -> created new LR {:?} range {:?} with new bundle {:?} for this use", |
908 | | lr, |
909 | | cr, |
910 | | new_bundle |
911 | | ); |
912 | | } |
913 | | |
914 | 18.1k | if !spill_range.is_empty() { |
915 | | // Make one entry in the spill bundle that covers the whole range. |
916 | | // TODO: it might be worth tracking enough state to only create this LR when there is |
917 | | // open space in the original LR. |
918 | 18.1k | let spill_lr = self.ctx.ranges.add(spill_range, self.ctx.bump()); |
919 | 18.1k | self.ctx.ranges[spill_lr].vreg = vreg; |
920 | 18.1k | self.ctx.ranges[spill_lr].bundle = spill; |
921 | 18.1k | self.ctx.ranges[spill_lr].uses.extend(spill_uses.drain(..)); |
922 | 18.1k | new_lrs.push((vreg, spill_lr)); |
923 | | |
924 | 18.1k | if spill_starts_def { |
925 | 2 | self.ctx.ranges[spill_lr].set_flag(LiveRangeFlag::StartsAtDef); |
926 | 18.1k | } |
927 | | |
928 | 18.1k | self.ctx.bundles[spill].ranges.push(LiveRangeListEntry { |
929 | 18.1k | range: spill_range, |
930 | 18.1k | index: spill_lr, |
931 | 18.1k | }); |
932 | 18.1k | self.ctx.ranges[spill_lr].bundle = spill; |
933 | 18.1k | trace!( |
934 | | " -> added spill range {:?} in new LR {:?} in spill bundle {:?}", |
935 | | spill_range, |
936 | | spill_lr, |
937 | | spill |
938 | | ); |
939 | | } else { |
940 | 0 | assert!(spill_uses.is_empty()); |
941 | | } |
942 | | } |
943 | | |
944 | | // Remove all of the removed LRs from respective vregs' lists. |
945 | 2.74k | for vreg in self.ctx.scratch_removed_lrs_vregs.drain() { |
946 | 2.74k | let lrs = &mut self.ctx.scratch_removed_lrs; |
947 | 2.74k | self.ctx.vregs[vreg] |
948 | 2.74k | .ranges |
949 | 45.1k | .retain(|entry| !lrs.contains(&entry.index)); Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::split_into_minimal_bundles::{closure#0}<regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}Line | Count | Source | 949 | 45.1k | .retain(|entry| !lrs.contains(&entry.index)); |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles::{closure#0} |
950 | | } |
951 | | |
952 | | // Add the new LRs to their respective vreg lists. |
953 | 69.2k | for (vreg, lr) in new_lrs { |
954 | 69.2k | let range = self.ctx.ranges[lr].range; |
955 | 69.2k | let entry = LiveRangeListEntry { range, index: lr }; |
956 | 69.2k | self.ctx.vregs[vreg].ranges.push(entry); |
957 | 69.2k | } |
958 | | |
959 | | // Recompute bundle properties for all new bundles and enqueue |
960 | | // them. |
961 | 51.0k | for bundle in new_bundles { |
962 | 51.0k | if self.ctx.bundles[bundle].ranges.len() > 0 { |
963 | 51.0k | self.recompute_bundle_properties(bundle); |
964 | 51.0k | let prio = self.ctx.bundles[bundle].prio; |
965 | 51.0k | self.ctx |
966 | 51.0k | .allocation_queue |
967 | 51.0k | .insert(bundle, prio as usize, hint); |
968 | 51.0k | } |
969 | | } |
970 | 2.16k | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::split_into_minimal_bundles <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles Line | Count | Source | 823 | 2.16k | pub fn split_into_minimal_bundles(&mut self, bundle: LiveBundleIndex, hint: PReg) { | 824 | 2.16k | assert_eq!(self.ctx.scratch_removed_lrs_vregs.len(), 0); | 825 | 2.16k | self.ctx.scratch_removed_lrs.clear(); | 826 | | | 827 | 2.16k | let mut new_lrs: SmallVec<[(VRegIndex, LiveRangeIndex); 16]> = smallvec![]; | 828 | 2.16k | let mut new_bundles: SmallVec<[LiveBundleIndex; 16]> = smallvec![]; | 829 | | | 830 | 2.16k | let spillset = self.ctx.bundles[bundle].spillset; | 831 | 2.16k | let spill = self | 832 | 2.16k | .get_or_create_spill_bundle(bundle, /* create_if_absent = */ true) | 833 | 2.16k | .unwrap(); | 834 | | | 835 | 2.16k | trace!("Splitting bundle {bundle:?} into minimal bundles with reg hint {hint:?}"); | 836 | | | 837 | 2.16k | let mut spill_uses = UseList::new_in(self.ctx.bump()); | 838 | | | 839 | 2.16k | let empty_vec = LiveRangeList::new_in(self.ctx.bump()); | 840 | 18.1k | for entry in core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty_vec) { | 841 | 18.1k | let vreg = self.ctx.ranges[entry.index].vreg; | 842 | | | 843 | 18.1k | self.ctx.scratch_removed_lrs.insert(entry.index); | 844 | 18.1k | self.ctx.scratch_removed_lrs_vregs.insert(vreg); | 845 | | | 846 | 18.1k | trace!(" -> removing old LR {:?} for vreg {:?}", entry.index, vreg); | 847 | | | 848 | 18.1k | let mut spill_range = entry.range; | 849 | 18.1k | let mut spill_starts_def = false; | 850 | | | 851 | 18.1k | let empty_vec = UseList::new_in(self.ctx.bump()); | 852 | 68.1k | for u in core::mem::replace(&mut self.ctx.ranges[entry.index].uses, empty_vec) { | 853 | 68.1k | trace!(" -> use {:?}", u); | 854 | | | 855 | 68.1k | let is_def = u.operand.kind() == OperandKind::Def; | 856 | | | 857 | | // If this use has an `any` constraint, eagerly migrate it to the spill range. The | 858 | | // reasoning here is that in the second-chance allocation for the spill bundle, | 859 | | // any-constrained uses will be easy to satisfy. Solving those constraints earlier | 860 | | // could create unnecessary conflicts with existing bundles that need to fit in a | 861 | | // register, more strict requirements, so we delay them eagerly. | 862 | 68.1k | if u.operand.constraint() == OperandConstraint::Any { | 863 | 17.1k | trace!(" -> migrating this any-constrained use to the spill range"); | 864 | 17.1k | spill_uses.push(u); | 865 | | | 866 | | // Remember if we're moving the def of this vreg into the spill range, so that | 867 | | // we can set the appropriate flags on it later. | 868 | 17.1k | spill_starts_def = spill_starts_def || is_def; | 869 | | | 870 | 17.1k | continue; | 871 | 51.0k | } | 872 | | | 873 | | // If this is a def, make sure that the spill range | 874 | | // starts before the next instruction rather than at | 875 | | // this position: the value is available only *after* | 876 | | // this position. | 877 | 51.0k | if is_def { | 878 | 668 | trace!(" -> moving the spill range forward by one"); | 879 | 668 | spill_range.from = ProgPoint::before(u.pos.inst().next()); | 880 | 50.3k | } | 881 | | | 882 | | // Create a new LR. | 883 | 51.0k | let cr = minimal_range_for_use(&u); | 884 | 51.0k | let lr = self.ctx.ranges.add(cr, self.ctx.bump()); | 885 | 51.0k | new_lrs.push((vreg, lr)); | 886 | 51.0k | self.ctx.ranges[lr].uses.push(u); | 887 | 51.0k | self.ctx.ranges[lr].vreg = vreg; | 888 | | | 889 | | // Create a new bundle that contains only this LR. | 890 | 51.0k | let new_bundle = self.ctx.bundles.add(self.ctx.bump()); | 891 | 51.0k | self.ctx.ranges[lr].bundle = new_bundle; | 892 | 51.0k | self.ctx.bundles[new_bundle].spillset = spillset; | 893 | 51.0k | self.ctx.bundles[new_bundle] | 894 | 51.0k | .ranges | 895 | 51.0k | .push(LiveRangeListEntry { | 896 | 51.0k | range: cr, | 897 | 51.0k | index: lr, | 898 | 51.0k | }); | 899 | 51.0k | new_bundles.push(new_bundle); | 900 | | | 901 | | // If this use was a Def, set the StartsAtDef flag for the new LR. | 902 | 51.0k | if is_def { | 903 | 668 | self.ctx.ranges[lr].set_flag(LiveRangeFlag::StartsAtDef); | 904 | 50.3k | } | 905 | | | 906 | 51.0k | trace!( | 907 | | " -> created new LR {:?} range {:?} with new bundle {:?} for this use", | 908 | | lr, | 909 | | cr, | 910 | | new_bundle | 911 | | ); | 912 | | } | 913 | | | 914 | 18.1k | if !spill_range.is_empty() { | 915 | | // Make one entry in the spill bundle that covers the whole range. | 916 | | // TODO: it might be worth tracking enough state to only create this LR when there is | 917 | | // open space in the original LR. | 918 | 18.1k | let spill_lr = self.ctx.ranges.add(spill_range, self.ctx.bump()); | 919 | 18.1k | self.ctx.ranges[spill_lr].vreg = vreg; | 920 | 18.1k | self.ctx.ranges[spill_lr].bundle = spill; | 921 | 18.1k | self.ctx.ranges[spill_lr].uses.extend(spill_uses.drain(..)); | 922 | 18.1k | new_lrs.push((vreg, spill_lr)); | 923 | | | 924 | 18.1k | if spill_starts_def { | 925 | 2 | self.ctx.ranges[spill_lr].set_flag(LiveRangeFlag::StartsAtDef); | 926 | 18.1k | } | 927 | | | 928 | 18.1k | self.ctx.bundles[spill].ranges.push(LiveRangeListEntry { | 929 | 18.1k | range: spill_range, | 930 | 18.1k | index: spill_lr, | 931 | 18.1k | }); | 932 | 18.1k | self.ctx.ranges[spill_lr].bundle = spill; | 933 | 18.1k | trace!( | 934 | | " -> added spill range {:?} in new LR {:?} in spill bundle {:?}", | 935 | | spill_range, | 936 | | spill_lr, | 937 | | spill | 938 | | ); | 939 | | } else { | 940 | 0 | assert!(spill_uses.is_empty()); | 941 | | } | 942 | | } | 943 | | | 944 | | // Remove all of the removed LRs from respective vregs' lists. | 945 | 2.74k | for vreg in self.ctx.scratch_removed_lrs_vregs.drain() { | 946 | 2.74k | let lrs = &mut self.ctx.scratch_removed_lrs; | 947 | 2.74k | self.ctx.vregs[vreg] | 948 | 2.74k | .ranges | 949 | 2.74k | .retain(|entry| !lrs.contains(&entry.index)); | 950 | | } | 951 | | | 952 | | // Add the new LRs to their respective vreg lists. | 953 | 69.2k | for (vreg, lr) in new_lrs { | 954 | 69.2k | let range = self.ctx.ranges[lr].range; | 955 | 69.2k | let entry = LiveRangeListEntry { range, index: lr }; | 956 | 69.2k | self.ctx.vregs[vreg].ranges.push(entry); | 957 | 69.2k | } | 958 | | | 959 | | // Recompute bundle properties for all new bundles and enqueue | 960 | | // them. | 961 | 51.0k | for bundle in new_bundles { | 962 | 51.0k | if self.ctx.bundles[bundle].ranges.len() > 0 { | 963 | 51.0k | self.recompute_bundle_properties(bundle); | 964 | 51.0k | let prio = self.ctx.bundles[bundle].prio; | 965 | 51.0k | self.ctx | 966 | 51.0k | .allocation_queue | 967 | 51.0k | .insert(bundle, prio as usize, hint); | 968 | 51.0k | } | 969 | | } | 970 | 2.16k | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::split_into_minimal_bundles |
971 | | |
972 | 9.89M | pub fn process_bundle( |
973 | 9.89M | &mut self, |
974 | 9.89M | bundle: LiveBundleIndex, |
975 | 9.89M | hint: PReg, |
976 | 9.89M | ) -> Result<(), RegAllocError> { |
977 | 9.89M | let class = self.ctx.spillsets[self.bundles[bundle].spillset].class; |
978 | | |
979 | | // Grab a hint from either the queue or our spillset, if any. |
980 | 9.89M | let mut hint = if hint != PReg::invalid() { |
981 | 2.43M | hint |
982 | | } else { |
983 | 7.46M | self.ctx.spillsets[self.bundles[bundle].spillset].hint |
984 | | }; |
985 | 9.89M | if self.ctx.pregs[hint.index()].is_stack { |
986 | 0 | hint = PReg::invalid(); |
987 | 9.89M | } |
988 | 9.89M | trace!("process_bundle: bundle {bundle:?} hint {hint:?}"); |
989 | | |
990 | 9.89M | let req = match self.compute_requirement(bundle) { |
991 | 9.82M | Ok(req) => req, |
992 | 64.2k | Err(conflict) => { |
993 | 64.2k | trace!("conflict!: {:?}", conflict); |
994 | | // We have to split right away. We'll find a point to |
995 | | // split that would allow at least the first half of the |
996 | | // split to be conflict-free. |
997 | 64.2k | debug_assert!( |
998 | 0 | !self.minimal_bundle(bundle), |
999 | | "Minimal bundle with conflict!" |
1000 | | ); |
1001 | 64.2k | self.split_and_requeue_bundle( |
1002 | 64.2k | bundle, |
1003 | 64.2k | /* split_at_point = */ conflict.suggested_split_point(), |
1004 | 64.2k | hint, |
1005 | | /* trim_ends_into_spill_bundle = */ |
1006 | 64.2k | conflict.should_trim_edges_around_split(), |
1007 | | ); |
1008 | 64.2k | return Ok(()); |
1009 | | } |
1010 | | }; |
1011 | | |
1012 | | // If no requirement at all (because no uses), and *if* a |
1013 | | // spill bundle is already present, then move the LRs over to |
1014 | | // the spill bundle right away. |
1015 | 9.82M | match req { |
1016 | | Requirement::Any => { |
1017 | 614k | if let Some(spill) = |
1018 | 635k | self.get_or_create_spill_bundle(bundle, /* create_if_absent = */ false) |
1019 | | { |
1020 | 614k | let empty_vec = LiveRangeList::new_in(self.ctx.bump()); |
1021 | 614k | let mut list = |
1022 | 614k | core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty_vec); |
1023 | 614k | for entry in &list { |
1024 | 614k | self.ctx.ranges[entry.index].bundle = spill; |
1025 | 614k | } |
1026 | 614k | self.ctx.bundles[spill].ranges.extend(list.drain(..)); |
1027 | 614k | return Ok(()); |
1028 | 20.8k | } |
1029 | | } |
1030 | 9.19M | _ => {} |
1031 | | } |
1032 | | |
1033 | | // Try to allocate! |
1034 | 9.21M | let mut attempts = 0; |
1035 | 9.21M | let mut scratch = core::mem::take(&mut self.ctx.scratch_conflicts); |
1036 | 9.21M | let mut lowest_cost_evict_conflict_set = core::mem::take(&mut self.ctx.scratch_bundle); |
1037 | | 'outer: loop { |
1038 | 10.2M | attempts += 1; |
1039 | 10.2M | trace!("attempt {}, req {:?}", attempts, req); |
1040 | 10.2M | debug_assert!(attempts < 100 * self.func.num_insts()); |
1041 | | |
1042 | 10.2M | let fixed_preg = match req { |
1043 | 2.38M | Requirement::FixedReg(preg) | Requirement::FixedStack(preg) => Some(preg), |
1044 | 7.83M | Requirement::Register | Requirement::Limit(..) => None, |
1045 | | Requirement::Stack => { |
1046 | | // If we must be on the stack, mark our spillset |
1047 | | // as required immediately. |
1048 | 0 | let spillset = self.bundles[bundle].spillset; |
1049 | 0 | self.spillsets[spillset].required = true; |
1050 | 0 | return Ok(()); |
1051 | | } |
1052 | | |
1053 | | Requirement::Any => { |
1054 | 20.8k | self.ctx.spilled_bundles.push(bundle); |
1055 | 20.8k | break; |
1056 | | } |
1057 | | }; |
1058 | | // Scan all pregs, or the one fixed preg, and attempt to allocate. |
1059 | | |
1060 | 10.2M | let mut lowest_cost_evict_conflict_cost: Option<u32> = None; |
1061 | 10.2M | lowest_cost_evict_conflict_set.clear(); |
1062 | | |
1063 | 10.2M | let mut lowest_cost_split_conflict_cost: Option<u32> = None; |
1064 | 10.2M | let mut lowest_cost_split_conflict_point = ProgPoint::before(Inst::new(0)); |
1065 | 10.2M | let mut lowest_cost_split_conflict_reg = PReg::invalid(); |
1066 | | |
1067 | | // Heuristic: start the scan for an available |
1068 | | // register at an offset influenced both by our |
1069 | | // location in the code and by the bundle we're |
1070 | | // considering. This has the effect of spreading |
1071 | | // demand more evenly across registers. |
1072 | 10.2M | let scan_offset = self.ctx.ranges[self.bundles[bundle].ranges[0].index] |
1073 | 10.2M | .range |
1074 | 10.2M | .from |
1075 | 10.2M | .inst() |
1076 | 10.2M | .index() |
1077 | 10.2M | + bundle.index(); |
1078 | | |
1079 | 10.2M | self.ctx.output.stats.process_bundle_reg_probe_start_any += 1; |
1080 | 10.2M | let limit = self.bundles[bundle].limit.map(|l| l as usize); Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::process_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0}Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundle::{closure#0} |
1081 | 44.4M | for preg in RegTraversalIter::new( |
1082 | 10.2M | self.env, |
1083 | 10.2M | class, |
1084 | 10.2M | fixed_preg, |
1085 | 10.2M | hint.as_valid(), |
1086 | 10.2M | scan_offset, |
1087 | 10.2M | limit, |
1088 | | ) { |
1089 | 44.4M | self.ctx.output.stats.process_bundle_reg_probes_any += 1; |
1090 | 44.4M | let preg_idx = PRegIndex::new(preg.index()); |
1091 | 44.4M | trace!("trying preg {:?}", preg_idx); |
1092 | | |
1093 | 44.4M | let scan_limit_cost = match ( |
1094 | 44.4M | lowest_cost_evict_conflict_cost, |
1095 | 44.4M | lowest_cost_split_conflict_cost, |
1096 | | ) { |
1097 | 30.4M | (Some(a), Some(b)) => Some(core::cmp::max(a, b)), |
1098 | 14.0M | _ => None, |
1099 | | }; |
1100 | 44.4M | match self.try_to_allocate_bundle_to_reg( |
1101 | 44.4M | bundle, |
1102 | 44.4M | preg_idx, |
1103 | 44.4M | scan_limit_cost, |
1104 | 44.4M | &mut scratch, |
1105 | 44.4M | ) { |
1106 | 7.90M | AllocRegResult::Allocated(alloc) => { |
1107 | 7.90M | self.ctx.output.stats.process_bundle_reg_success_any += 1; |
1108 | 7.90M | trace!(" -> allocated to any {:?}", preg_idx); |
1109 | 7.90M | self.ctx.spillsets[self.ctx.bundles[bundle].spillset].hint = |
1110 | 7.90M | alloc.as_reg().unwrap(); |
1111 | | // Success, return scratch memory to context and finish |
1112 | 7.90M | break 'outer; |
1113 | | } |
1114 | 18.5M | AllocRegResult::Conflict(bundles, first_conflict_point) => { |
1115 | 18.5M | trace!( |
1116 | | " -> conflict with bundles {:?}, first conflict at {:?}", |
1117 | | bundles, |
1118 | | first_conflict_point |
1119 | | ); |
1120 | | |
1121 | 18.5M | let conflict_cost = self.maximum_spill_weight_in_bundle_set(bundles); |
1122 | | |
1123 | 18.5M | if lowest_cost_evict_conflict_cost.is_none() |
1124 | 14.0M | || conflict_cost < lowest_cost_evict_conflict_cost.unwrap() |
1125 | 7.28M | { |
1126 | 7.28M | lowest_cost_evict_conflict_cost = Some(conflict_cost); |
1127 | 7.28M | lowest_cost_evict_conflict_set.clear(); |
1128 | 7.28M | lowest_cost_evict_conflict_set.extend(bundles); |
1129 | 11.2M | } |
1130 | | |
1131 | 18.5M | let loop_depth = |
1132 | 18.5M | self.ctx.cfginfo.approx_loop_depth[self.ctx.cfginfo.insn_block |
1133 | 18.5M | [first_conflict_point.inst().index()] |
1134 | 18.5M | .index()]; |
1135 | 18.5M | let move_cost = spill_weight_from_constraint( |
1136 | 18.5M | OperandConstraint::Reg, |
1137 | 18.5M | loop_depth as usize, |
1138 | | /* is_def = */ true, |
1139 | | ) |
1140 | 18.5M | .to_int(); |
1141 | 18.5M | if lowest_cost_split_conflict_cost.is_none() |
1142 | 14.8M | || (conflict_cost + move_cost) |
1143 | 14.8M | < lowest_cost_split_conflict_cost.unwrap() |
1144 | 4.75M | { |
1145 | 4.75M | lowest_cost_split_conflict_cost = Some(conflict_cost + move_cost); |
1146 | 4.75M | lowest_cost_split_conflict_point = first_conflict_point; |
1147 | 4.75M | lowest_cost_split_conflict_reg = preg; |
1148 | 13.7M | } |
1149 | | } |
1150 | 17.9M | AllocRegResult::ConflictWithFixed(max_cost, point) => { |
1151 | 17.9M | trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point); |
1152 | | |
1153 | 17.9M | let loop_depth = self.ctx.cfginfo.approx_loop_depth |
1154 | 17.9M | [self.ctx.cfginfo.insn_block[point.inst().index()].index()]; |
1155 | 17.9M | let move_cost = spill_weight_from_constraint( |
1156 | 17.9M | OperandConstraint::Reg, |
1157 | 17.9M | loop_depth as usize, |
1158 | | /* is_def = */ true, |
1159 | | ) |
1160 | 17.9M | .to_int(); |
1161 | | |
1162 | 17.9M | if lowest_cost_split_conflict_cost.is_none() |
1163 | 16.7M | || (max_cost + move_cost) < lowest_cost_split_conflict_cost.unwrap() |
1164 | 2.12M | { |
1165 | 2.12M | lowest_cost_split_conflict_cost = Some(max_cost + move_cost); |
1166 | 2.12M | lowest_cost_split_conflict_point = point; |
1167 | 2.12M | lowest_cost_split_conflict_reg = preg; |
1168 | 15.8M | } |
1169 | | } |
1170 | | AllocRegResult::ConflictHighCost => { |
1171 | | // Simply don't consider -- we already have |
1172 | | // a lower-cost conflict bundle option |
1173 | | // to evict. |
1174 | 59.2k | continue; |
1175 | | } |
1176 | | } |
1177 | | } |
1178 | | |
1179 | | // Otherwise, we *require* a register, but didn't fit into |
1180 | | // any with current bundle assignments. Hence, we will need |
1181 | | // to either split or attempt to evict some bundles. |
1182 | | |
1183 | 2.32M | trace!( |
1184 | | " -> lowest cost evict: set {:?}, cost {:?}", |
1185 | | lowest_cost_evict_conflict_set, |
1186 | | lowest_cost_evict_conflict_cost, |
1187 | | ); |
1188 | 2.32M | trace!( |
1189 | | " -> lowest cost split: cost {:?}, point {:?}, reg {:?}", |
1190 | | lowest_cost_split_conflict_cost, |
1191 | | lowest_cost_split_conflict_point, |
1192 | | lowest_cost_split_conflict_reg |
1193 | | ); |
1194 | | |
1195 | | // If we reach here, we *must* have an option either to split or evict. |
1196 | 2.32M | debug_assert!( |
1197 | 0 | lowest_cost_split_conflict_cost.is_some() |
1198 | 0 | || lowest_cost_evict_conflict_cost.is_some() |
1199 | | ); |
1200 | | |
1201 | 2.32M | let our_spill_weight = self.bundle_spill_weight(bundle); |
1202 | 2.32M | trace!(" -> our spill weight: {}", our_spill_weight); |
1203 | | |
1204 | | // We detect the "too-many-live-registers" case here and |
1205 | | // return an error cleanly, rather than panicking, because |
1206 | | // the regalloc.rs fuzzer depends on the register |
1207 | | // allocator to correctly reject impossible-to-allocate |
1208 | | // programs in order to discard invalid test cases. |
1209 | 2.32M | if self.minimal_bundle(bundle) |
1210 | 78.6k | && (attempts >= 2 |
1211 | 78.6k | || lowest_cost_evict_conflict_cost.is_none() |
1212 | 78.6k | || lowest_cost_evict_conflict_cost.unwrap() >= our_spill_weight) |
1213 | | { |
1214 | 0 | if matches!(req, Requirement::Register | Requirement::Limit(_)) { |
1215 | | // Check if this is a too-many-live-registers situation. |
1216 | 0 | let range = self.ctx.bundles[bundle].ranges[0].range; |
1217 | 0 | trace!("checking for too many live regs"); |
1218 | 0 | let mut min_bundles_assigned = 0; |
1219 | 0 | let mut fixed_assigned = 0; |
1220 | 0 | let mut total_regs = 0; |
1221 | 0 | for preg in self.env.preferred_regs_by_class[class as u8 as usize] |
1222 | 0 | .iter() |
1223 | 0 | .chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter()) |
1224 | | { |
1225 | 0 | trace!(" -> PR {:?}", preg); |
1226 | 0 | let start = LiveRangeKey::from_range(&CodeRange { |
1227 | 0 | from: range.from.prev(), |
1228 | 0 | to: range.from.prev(), |
1229 | 0 | }); |
1230 | 0 | for (key, lr) in self.ctx.pregs[preg.index()] |
1231 | 0 | .allocations |
1232 | 0 | .btree |
1233 | 0 | .range(start..) |
1234 | | { |
1235 | 0 | let preg_range = key.to_range(); |
1236 | 0 | if preg_range.to <= range.from { |
1237 | 0 | continue; |
1238 | 0 | } |
1239 | 0 | if preg_range.from >= range.to { |
1240 | 0 | break; |
1241 | 0 | } |
1242 | 0 | if lr.is_valid() { |
1243 | 0 | if self.minimal_bundle(self.ranges[*lr].bundle) { |
1244 | 0 | trace!(" -> min bundle {:?}", lr); |
1245 | 0 | min_bundles_assigned += 1; |
1246 | | } else { |
1247 | 0 | trace!(" -> non-min bundle {:?}", lr); |
1248 | | } |
1249 | | } else { |
1250 | 0 | trace!(" -> fixed bundle"); |
1251 | 0 | fixed_assigned += 1; |
1252 | | } |
1253 | | } |
1254 | | |
1255 | | // We also need to discard any registers that do not fit |
1256 | | // under the limit--we cannot allocate to them. |
1257 | 0 | if let Requirement::Limit(limit) = req { |
1258 | 0 | if preg.hw_enc() >= limit as usize { |
1259 | 0 | continue; |
1260 | 0 | } |
1261 | 0 | } |
1262 | | |
1263 | 0 | total_regs += 1; |
1264 | | } |
1265 | 0 | trace!( |
1266 | | " -> total {}, fixed {}, min {}", |
1267 | | total_regs, |
1268 | | fixed_assigned, |
1269 | | min_bundles_assigned |
1270 | | ); |
1271 | 0 | if min_bundles_assigned + fixed_assigned >= total_regs { |
1272 | 0 | return Err(RegAllocError::TooManyLiveRegs); |
1273 | 0 | } |
1274 | 0 | } |
1275 | | |
1276 | 0 | panic!("Could not allocate minimal bundle, but the allocation problem should be possible to solve"); |
1277 | 2.32M | } |
1278 | | |
1279 | | // If our bundle's weight is less than or equal to(*) the |
1280 | | // evict cost, choose to split. Also pick splitting if |
1281 | | // we're on our second or more attempt and we didn't |
1282 | | // allocate. Also pick splitting if the conflict set is |
1283 | | // empty, meaning a fixed conflict that can't be evicted. |
1284 | | // |
1285 | | // (*) the "equal to" part is very important: it prevents |
1286 | | // an infinite loop where two bundles with equal spill |
1287 | | // cost continually evict each other in an infinite |
1288 | | // allocation loop. In such a case, the first bundle in |
1289 | | // wins, and the other splits. |
1290 | | // |
1291 | | // Note that we don't split if the bundle is minimal. |
1292 | 2.32M | if !self.minimal_bundle(bundle) |
1293 | 2.24M | && (attempts >= 2 |
1294 | 2.24M | || lowest_cost_evict_conflict_cost.is_none() |
1295 | 2.13M | || our_spill_weight <= lowest_cost_evict_conflict_cost.unwrap()) |
1296 | | { |
1297 | 1.29M | trace!( |
1298 | | " -> deciding to split: our spill weight is {}", |
1299 | 0 | self.bundle_spill_weight(bundle) |
1300 | | ); |
1301 | 1.29M | let bundle_start = self.ctx.bundles[bundle].ranges[0].range.from; |
1302 | 1.29M | let mut split_at_point = |
1303 | 1.29M | core::cmp::max(lowest_cost_split_conflict_point, bundle_start); |
1304 | 1.29M | let requeue_with_reg = lowest_cost_split_conflict_reg; |
1305 | | |
1306 | | // Adjust `split_at_point` if it is within a deeper loop |
1307 | | // than the bundle start -- hoist it to just before the |
1308 | | // first loop header it encounters. |
1309 | 1.29M | let bundle_start_depth = self.ctx.cfginfo.approx_loop_depth |
1310 | 1.29M | [self.ctx.cfginfo.insn_block[bundle_start.inst().index()].index()]; |
1311 | 1.29M | let split_at_depth = self.ctx.cfginfo.approx_loop_depth |
1312 | 1.29M | [self.ctx.cfginfo.insn_block[split_at_point.inst().index()].index()]; |
1313 | 1.29M | if split_at_depth > bundle_start_depth { |
1314 | 7.00k | for block in (self.ctx.cfginfo.insn_block[bundle_start.inst().index()].index() |
1315 | 927 | + 1) |
1316 | 927 | ..=self.ctx.cfginfo.insn_block[split_at_point.inst().index()].index() |
1317 | | { |
1318 | 7.00k | if self.ctx.cfginfo.approx_loop_depth[block] > bundle_start_depth { |
1319 | 927 | split_at_point = self.ctx.cfginfo.block_entry[block]; |
1320 | 927 | break; |
1321 | 6.07k | } |
1322 | | } |
1323 | 1.29M | } |
1324 | | |
1325 | 1.29M | self.split_and_requeue_bundle( |
1326 | 1.29M | bundle, |
1327 | 1.29M | split_at_point, |
1328 | 1.29M | requeue_with_reg, |
1329 | | /* should_trim = */ true, |
1330 | | ); |
1331 | | |
1332 | | // Success, return scratch memory to context and finish |
1333 | 1.29M | break 'outer; |
1334 | | } else { |
1335 | | // Evict all bundles in `conflicting bundles` and try again. |
1336 | 1.03M | self.ctx.output.stats.evict_bundle_event += 1; |
1337 | 1.03M | for &bundle in &lowest_cost_evict_conflict_set { |
1338 | 1.03M | trace!(" -> evicting {:?}", bundle); |
1339 | 1.03M | self.evict_bundle(bundle); |
1340 | 1.03M | self.ctx.output.stats.evict_bundle_count += 1; |
1341 | | } |
1342 | | } |
1343 | | } |
1344 | | |
1345 | 9.21M | self.ctx.scratch_conflicts = scratch; |
1346 | 9.21M | self.ctx.scratch_bundle = lowest_cost_evict_conflict_set; |
1347 | 9.21M | return Ok(()); |
1348 | 9.89M | } Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<_>>::process_bundle <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::x64::lower::isle::generated_code::MInst>>>::process_bundle Line | Count | Source | 972 | 9.89M | pub fn process_bundle( | 973 | 9.89M | &mut self, | 974 | 9.89M | bundle: LiveBundleIndex, | 975 | 9.89M | hint: PReg, | 976 | 9.89M | ) -> Result<(), RegAllocError> { | 977 | 9.89M | let class = self.ctx.spillsets[self.bundles[bundle].spillset].class; | 978 | | | 979 | | // Grab a hint from either the queue or our spillset, if any. | 980 | 9.89M | let mut hint = if hint != PReg::invalid() { | 981 | 2.43M | hint | 982 | | } else { | 983 | 7.46M | self.ctx.spillsets[self.bundles[bundle].spillset].hint | 984 | | }; | 985 | 9.89M | if self.ctx.pregs[hint.index()].is_stack { | 986 | 0 | hint = PReg::invalid(); | 987 | 9.89M | } | 988 | 9.89M | trace!("process_bundle: bundle {bundle:?} hint {hint:?}"); | 989 | | | 990 | 9.89M | let req = match self.compute_requirement(bundle) { | 991 | 9.82M | Ok(req) => req, | 992 | 64.2k | Err(conflict) => { | 993 | 64.2k | trace!("conflict!: {:?}", conflict); | 994 | | // We have to split right away. We'll find a point to | 995 | | // split that would allow at least the first half of the | 996 | | // split to be conflict-free. | 997 | 64.2k | debug_assert!( | 998 | 0 | !self.minimal_bundle(bundle), | 999 | | "Minimal bundle with conflict!" | 1000 | | ); | 1001 | 64.2k | self.split_and_requeue_bundle( | 1002 | 64.2k | bundle, | 1003 | 64.2k | /* split_at_point = */ conflict.suggested_split_point(), | 1004 | 64.2k | hint, | 1005 | | /* trim_ends_into_spill_bundle = */ | 1006 | 64.2k | conflict.should_trim_edges_around_split(), | 1007 | | ); | 1008 | 64.2k | return Ok(()); | 1009 | | } | 1010 | | }; | 1011 | | | 1012 | | // If no requirement at all (because no uses), and *if* a | 1013 | | // spill bundle is already present, then move the LRs over to | 1014 | | // the spill bundle right away. | 1015 | 9.82M | match req { | 1016 | | Requirement::Any => { | 1017 | 614k | if let Some(spill) = | 1018 | 635k | self.get_or_create_spill_bundle(bundle, /* create_if_absent = */ false) | 1019 | | { | 1020 | 614k | let empty_vec = LiveRangeList::new_in(self.ctx.bump()); | 1021 | 614k | let mut list = | 1022 | 614k | core::mem::replace(&mut self.ctx.bundles[bundle].ranges, empty_vec); | 1023 | 614k | for entry in &list { | 1024 | 614k | self.ctx.ranges[entry.index].bundle = spill; | 1025 | 614k | } | 1026 | 614k | self.ctx.bundles[spill].ranges.extend(list.drain(..)); | 1027 | 614k | return Ok(()); | 1028 | 20.8k | } | 1029 | | } | 1030 | 9.19M | _ => {} | 1031 | | } | 1032 | | | 1033 | | // Try to allocate! | 1034 | 9.21M | let mut attempts = 0; | 1035 | 9.21M | let mut scratch = core::mem::take(&mut self.ctx.scratch_conflicts); | 1036 | 9.21M | let mut lowest_cost_evict_conflict_set = core::mem::take(&mut self.ctx.scratch_bundle); | 1037 | | 'outer: loop { | 1038 | 10.2M | attempts += 1; | 1039 | 10.2M | trace!("attempt {}, req {:?}", attempts, req); | 1040 | 10.2M | debug_assert!(attempts < 100 * self.func.num_insts()); | 1041 | | | 1042 | 10.2M | let fixed_preg = match req { | 1043 | 2.38M | Requirement::FixedReg(preg) | Requirement::FixedStack(preg) => Some(preg), | 1044 | 7.83M | Requirement::Register | Requirement::Limit(..) => None, | 1045 | | Requirement::Stack => { | 1046 | | // If we must be on the stack, mark our spillset | 1047 | | // as required immediately. | 1048 | 0 | let spillset = self.bundles[bundle].spillset; | 1049 | 0 | self.spillsets[spillset].required = true; | 1050 | 0 | return Ok(()); | 1051 | | } | 1052 | | | 1053 | | Requirement::Any => { | 1054 | 20.8k | self.ctx.spilled_bundles.push(bundle); | 1055 | 20.8k | break; | 1056 | | } | 1057 | | }; | 1058 | | // Scan all pregs, or the one fixed preg, and attempt to allocate. | 1059 | | | 1060 | 10.2M | let mut lowest_cost_evict_conflict_cost: Option<u32> = None; | 1061 | 10.2M | lowest_cost_evict_conflict_set.clear(); | 1062 | | | 1063 | 10.2M | let mut lowest_cost_split_conflict_cost: Option<u32> = None; | 1064 | 10.2M | let mut lowest_cost_split_conflict_point = ProgPoint::before(Inst::new(0)); | 1065 | 10.2M | let mut lowest_cost_split_conflict_reg = PReg::invalid(); | 1066 | | | 1067 | | // Heuristic: start the scan for an available | 1068 | | // register at an offset influenced both by our | 1069 | | // location in the code and by the bundle we're | 1070 | | // considering. This has the effect of spreading | 1071 | | // demand more evenly across registers. | 1072 | 10.2M | let scan_offset = self.ctx.ranges[self.bundles[bundle].ranges[0].index] | 1073 | 10.2M | .range | 1074 | 10.2M | .from | 1075 | 10.2M | .inst() | 1076 | 10.2M | .index() | 1077 | 10.2M | + bundle.index(); | 1078 | | | 1079 | 10.2M | self.ctx.output.stats.process_bundle_reg_probe_start_any += 1; | 1080 | 10.2M | let limit = self.bundles[bundle].limit.map(|l| l as usize); | 1081 | 44.4M | for preg in RegTraversalIter::new( | 1082 | 10.2M | self.env, | 1083 | 10.2M | class, | 1084 | 10.2M | fixed_preg, | 1085 | 10.2M | hint.as_valid(), | 1086 | 10.2M | scan_offset, | 1087 | 10.2M | limit, | 1088 | | ) { | 1089 | 44.4M | self.ctx.output.stats.process_bundle_reg_probes_any += 1; | 1090 | 44.4M | let preg_idx = PRegIndex::new(preg.index()); | 1091 | 44.4M | trace!("trying preg {:?}", preg_idx); | 1092 | | | 1093 | 44.4M | let scan_limit_cost = match ( | 1094 | 44.4M | lowest_cost_evict_conflict_cost, | 1095 | 44.4M | lowest_cost_split_conflict_cost, | 1096 | | ) { | 1097 | 30.4M | (Some(a), Some(b)) => Some(core::cmp::max(a, b)), | 1098 | 14.0M | _ => None, | 1099 | | }; | 1100 | 44.4M | match self.try_to_allocate_bundle_to_reg( | 1101 | 44.4M | bundle, | 1102 | 44.4M | preg_idx, | 1103 | 44.4M | scan_limit_cost, | 1104 | 44.4M | &mut scratch, | 1105 | 44.4M | ) { | 1106 | 7.90M | AllocRegResult::Allocated(alloc) => { | 1107 | 7.90M | self.ctx.output.stats.process_bundle_reg_success_any += 1; | 1108 | 7.90M | trace!(" -> allocated to any {:?}", preg_idx); | 1109 | 7.90M | self.ctx.spillsets[self.ctx.bundles[bundle].spillset].hint = | 1110 | 7.90M | alloc.as_reg().unwrap(); | 1111 | | // Success, return scratch memory to context and finish | 1112 | 7.90M | break 'outer; | 1113 | | } | 1114 | 18.5M | AllocRegResult::Conflict(bundles, first_conflict_point) => { | 1115 | 18.5M | trace!( | 1116 | | " -> conflict with bundles {:?}, first conflict at {:?}", | 1117 | | bundles, | 1118 | | first_conflict_point | 1119 | | ); | 1120 | | | 1121 | 18.5M | let conflict_cost = self.maximum_spill_weight_in_bundle_set(bundles); | 1122 | | | 1123 | 18.5M | if lowest_cost_evict_conflict_cost.is_none() | 1124 | 14.0M | || conflict_cost < lowest_cost_evict_conflict_cost.unwrap() | 1125 | 7.28M | { | 1126 | 7.28M | lowest_cost_evict_conflict_cost = Some(conflict_cost); | 1127 | 7.28M | lowest_cost_evict_conflict_set.clear(); | 1128 | 7.28M | lowest_cost_evict_conflict_set.extend(bundles); | 1129 | 11.2M | } | 1130 | | | 1131 | 18.5M | let loop_depth = | 1132 | 18.5M | self.ctx.cfginfo.approx_loop_depth[self.ctx.cfginfo.insn_block | 1133 | 18.5M | [first_conflict_point.inst().index()] | 1134 | 18.5M | .index()]; | 1135 | 18.5M | let move_cost = spill_weight_from_constraint( | 1136 | 18.5M | OperandConstraint::Reg, | 1137 | 18.5M | loop_depth as usize, | 1138 | | /* is_def = */ true, | 1139 | | ) | 1140 | 18.5M | .to_int(); | 1141 | 18.5M | if lowest_cost_split_conflict_cost.is_none() | 1142 | 14.8M | || (conflict_cost + move_cost) | 1143 | 14.8M | < lowest_cost_split_conflict_cost.unwrap() | 1144 | 4.75M | { | 1145 | 4.75M | lowest_cost_split_conflict_cost = Some(conflict_cost + move_cost); | 1146 | 4.75M | lowest_cost_split_conflict_point = first_conflict_point; | 1147 | 4.75M | lowest_cost_split_conflict_reg = preg; | 1148 | 13.7M | } | 1149 | | } | 1150 | 17.9M | AllocRegResult::ConflictWithFixed(max_cost, point) => { | 1151 | 17.9M | trace!(" -> conflict with fixed alloc; cost of other bundles up to point is {}, conflict at {:?}", max_cost, point); | 1152 | | | 1153 | 17.9M | let loop_depth = self.ctx.cfginfo.approx_loop_depth | 1154 | 17.9M | [self.ctx.cfginfo.insn_block[point.inst().index()].index()]; | 1155 | 17.9M | let move_cost = spill_weight_from_constraint( | 1156 | 17.9M | OperandConstraint::Reg, | 1157 | 17.9M | loop_depth as usize, | 1158 | | /* is_def = */ true, | 1159 | | ) | 1160 | 17.9M | .to_int(); | 1161 | | | 1162 | 17.9M | if lowest_cost_split_conflict_cost.is_none() | 1163 | 16.7M | || (max_cost + move_cost) < lowest_cost_split_conflict_cost.unwrap() | 1164 | 2.12M | { | 1165 | 2.12M | lowest_cost_split_conflict_cost = Some(max_cost + move_cost); | 1166 | 2.12M | lowest_cost_split_conflict_point = point; | 1167 | 2.12M | lowest_cost_split_conflict_reg = preg; | 1168 | 15.8M | } | 1169 | | } | 1170 | | AllocRegResult::ConflictHighCost => { | 1171 | | // Simply don't consider -- we already have | 1172 | | // a lower-cost conflict bundle option | 1173 | | // to evict. | 1174 | 59.2k | continue; | 1175 | | } | 1176 | | } | 1177 | | } | 1178 | | | 1179 | | // Otherwise, we *require* a register, but didn't fit into | 1180 | | // any with current bundle assignments. Hence, we will need | 1181 | | // to either split or attempt to evict some bundles. | 1182 | | | 1183 | 2.32M | trace!( | 1184 | | " -> lowest cost evict: set {:?}, cost {:?}", | 1185 | | lowest_cost_evict_conflict_set, | 1186 | | lowest_cost_evict_conflict_cost, | 1187 | | ); | 1188 | 2.32M | trace!( | 1189 | | " -> lowest cost split: cost {:?}, point {:?}, reg {:?}", | 1190 | | lowest_cost_split_conflict_cost, | 1191 | | lowest_cost_split_conflict_point, | 1192 | | lowest_cost_split_conflict_reg | 1193 | | ); | 1194 | | | 1195 | | // If we reach here, we *must* have an option either to split or evict. | 1196 | 2.32M | debug_assert!( | 1197 | 0 | lowest_cost_split_conflict_cost.is_some() | 1198 | 0 | || lowest_cost_evict_conflict_cost.is_some() | 1199 | | ); | 1200 | | | 1201 | 2.32M | let our_spill_weight = self.bundle_spill_weight(bundle); | 1202 | 2.32M | trace!(" -> our spill weight: {}", our_spill_weight); | 1203 | | | 1204 | | // We detect the "too-many-live-registers" case here and | 1205 | | // return an error cleanly, rather than panicking, because | 1206 | | // the regalloc.rs fuzzer depends on the register | 1207 | | // allocator to correctly reject impossible-to-allocate | 1208 | | // programs in order to discard invalid test cases. | 1209 | 2.32M | if self.minimal_bundle(bundle) | 1210 | 78.6k | && (attempts >= 2 | 1211 | 78.6k | || lowest_cost_evict_conflict_cost.is_none() | 1212 | 78.6k | || lowest_cost_evict_conflict_cost.unwrap() >= our_spill_weight) | 1213 | | { | 1214 | 0 | if matches!(req, Requirement::Register | Requirement::Limit(_)) { | 1215 | | // Check if this is a too-many-live-registers situation. | 1216 | 0 | let range = self.ctx.bundles[bundle].ranges[0].range; | 1217 | 0 | trace!("checking for too many live regs"); | 1218 | 0 | let mut min_bundles_assigned = 0; | 1219 | 0 | let mut fixed_assigned = 0; | 1220 | 0 | let mut total_regs = 0; | 1221 | 0 | for preg in self.env.preferred_regs_by_class[class as u8 as usize] | 1222 | 0 | .iter() | 1223 | 0 | .chain(self.env.non_preferred_regs_by_class[class as u8 as usize].iter()) | 1224 | | { | 1225 | 0 | trace!(" -> PR {:?}", preg); | 1226 | 0 | let start = LiveRangeKey::from_range(&CodeRange { | 1227 | 0 | from: range.from.prev(), | 1228 | 0 | to: range.from.prev(), | 1229 | 0 | }); | 1230 | 0 | for (key, lr) in self.ctx.pregs[preg.index()] | 1231 | 0 | .allocations | 1232 | 0 | .btree | 1233 | 0 | .range(start..) | 1234 | | { | 1235 | 0 | let preg_range = key.to_range(); | 1236 | 0 | if preg_range.to <= range.from { | 1237 | 0 | continue; | 1238 | 0 | } | 1239 | 0 | if preg_range.from >= range.to { | 1240 | 0 | break; | 1241 | 0 | } | 1242 | 0 | if lr.is_valid() { | 1243 | 0 | if self.minimal_bundle(self.ranges[*lr].bundle) { | 1244 | 0 | trace!(" -> min bundle {:?}", lr); | 1245 | 0 | min_bundles_assigned += 1; | 1246 | | } else { | 1247 | 0 | trace!(" -> non-min bundle {:?}", lr); | 1248 | | } | 1249 | | } else { | 1250 | 0 | trace!(" -> fixed bundle"); | 1251 | 0 | fixed_assigned += 1; | 1252 | | } | 1253 | | } | 1254 | | | 1255 | | // We also need to discard any registers that do not fit | 1256 | | // under the limit--we cannot allocate to them. | 1257 | 0 | if let Requirement::Limit(limit) = req { | 1258 | 0 | if preg.hw_enc() >= limit as usize { | 1259 | 0 | continue; | 1260 | 0 | } | 1261 | 0 | } | 1262 | | | 1263 | 0 | total_regs += 1; | 1264 | | } | 1265 | 0 | trace!( | 1266 | | " -> total {}, fixed {}, min {}", | 1267 | | total_regs, | 1268 | | fixed_assigned, | 1269 | | min_bundles_assigned | 1270 | | ); | 1271 | 0 | if min_bundles_assigned + fixed_assigned >= total_regs { | 1272 | 0 | return Err(RegAllocError::TooManyLiveRegs); | 1273 | 0 | } | 1274 | 0 | } | 1275 | | | 1276 | 0 | panic!("Could not allocate minimal bundle, but the allocation problem should be possible to solve"); | 1277 | 2.32M | } | 1278 | | | 1279 | | // If our bundle's weight is less than or equal to(*) the | 1280 | | // evict cost, choose to split. Also pick splitting if | 1281 | | // we're on our second or more attempt and we didn't | 1282 | | // allocate. Also pick splitting if the conflict set is | 1283 | | // empty, meaning a fixed conflict that can't be evicted. | 1284 | | // | 1285 | | // (*) the "equal to" part is very important: it prevents | 1286 | | // an infinite loop where two bundles with equal spill | 1287 | | // cost continually evict each other in an infinite | 1288 | | // allocation loop. In such a case, the first bundle in | 1289 | | // wins, and the other splits. | 1290 | | // | 1291 | | // Note that we don't split if the bundle is minimal. | 1292 | 2.32M | if !self.minimal_bundle(bundle) | 1293 | 2.24M | && (attempts >= 2 | 1294 | 2.24M | || lowest_cost_evict_conflict_cost.is_none() | 1295 | 2.13M | || our_spill_weight <= lowest_cost_evict_conflict_cost.unwrap()) | 1296 | | { | 1297 | 1.29M | trace!( | 1298 | | " -> deciding to split: our spill weight is {}", | 1299 | 0 | self.bundle_spill_weight(bundle) | 1300 | | ); | 1301 | 1.29M | let bundle_start = self.ctx.bundles[bundle].ranges[0].range.from; | 1302 | 1.29M | let mut split_at_point = | 1303 | 1.29M | core::cmp::max(lowest_cost_split_conflict_point, bundle_start); | 1304 | 1.29M | let requeue_with_reg = lowest_cost_split_conflict_reg; | 1305 | | | 1306 | | // Adjust `split_at_point` if it is within a deeper loop | 1307 | | // than the bundle start -- hoist it to just before the | 1308 | | // first loop header it encounters. | 1309 | 1.29M | let bundle_start_depth = self.ctx.cfginfo.approx_loop_depth | 1310 | 1.29M | [self.ctx.cfginfo.insn_block[bundle_start.inst().index()].index()]; | 1311 | 1.29M | let split_at_depth = self.ctx.cfginfo.approx_loop_depth | 1312 | 1.29M | [self.ctx.cfginfo.insn_block[split_at_point.inst().index()].index()]; | 1313 | 1.29M | if split_at_depth > bundle_start_depth { | 1314 | 7.00k | for block in (self.ctx.cfginfo.insn_block[bundle_start.inst().index()].index() | 1315 | 927 | + 1) | 1316 | 927 | ..=self.ctx.cfginfo.insn_block[split_at_point.inst().index()].index() | 1317 | | { | 1318 | 7.00k | if self.ctx.cfginfo.approx_loop_depth[block] > bundle_start_depth { | 1319 | 927 | split_at_point = self.ctx.cfginfo.block_entry[block]; | 1320 | 927 | break; | 1321 | 6.07k | } | 1322 | | } | 1323 | 1.29M | } | 1324 | | | 1325 | 1.29M | self.split_and_requeue_bundle( | 1326 | 1.29M | bundle, | 1327 | 1.29M | split_at_point, | 1328 | 1.29M | requeue_with_reg, | 1329 | | /* should_trim = */ true, | 1330 | | ); | 1331 | | | 1332 | | // Success, return scratch memory to context and finish | 1333 | 1.29M | break 'outer; | 1334 | | } else { | 1335 | | // Evict all bundles in `conflicting bundles` and try again. | 1336 | 1.03M | self.ctx.output.stats.evict_bundle_event += 1; | 1337 | 1.03M | for &bundle in &lowest_cost_evict_conflict_set { | 1338 | 1.03M | trace!(" -> evicting {:?}", bundle); | 1339 | 1.03M | self.evict_bundle(bundle); | 1340 | 1.03M | self.ctx.output.stats.evict_bundle_count += 1; | 1341 | | } | 1342 | | } | 1343 | | } | 1344 | | | 1345 | 9.21M | self.ctx.scratch_conflicts = scratch; | 1346 | 9.21M | self.ctx.scratch_bundle = lowest_cost_evict_conflict_set; | 1347 | 9.21M | return Ok(()); | 1348 | 9.89M | } |
Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::aarch64::lower::isle::generated_code::MInst>>>::process_bundle Unexecuted instantiation: <regalloc2::ion::data_structures::Env<cranelift_codegen::machinst::vcode::VCode<cranelift_codegen::isa::riscv64::lower::isle::generated_code::MInst>>>::process_bundle |
1349 | | } |
1350 | | |
1351 | | /// Compute the minimal range that covers a given use in a minimal |
1352 | | /// bundle. This definition needs to be consistent between bundle |
1353 | | /// property computation and minimal-range splitting (fallback |
1354 | | /// splitting). |
1355 | | /// |
1356 | | /// The cases are: |
1357 | | /// - early def: whole instruction |
1358 | | /// - late def: Late only |
1359 | | /// - early use: Early only |
1360 | | /// - late use: whole instruction |
1361 | 2.82M | fn minimal_range_for_use(u: &Use) -> CodeRange { |
1362 | 2.82M | let early = ProgPoint::before(u.pos.inst()); |
1363 | 2.82M | let late = ProgPoint::after(u.pos.inst()); |
1364 | 2.82M | let next_early = ProgPoint::before(u.pos.inst().next()); |
1365 | 2.82M | match (u.pos.pos(), u.operand.kind()) { |
1366 | 401k | (InstPosition::Before, OperandKind::Def) => CodeRange { |
1367 | 401k | from: early, |
1368 | 401k | to: next_early, |
1369 | 401k | }, |
1370 | 1.14M | (InstPosition::Before, OperandKind::Use) => CodeRange { |
1371 | 1.14M | from: early, |
1372 | 1.14M | to: late, |
1373 | 1.14M | }, |
1374 | 1.28M | (InstPosition::After, OperandKind::Def) => CodeRange { |
1375 | 1.28M | from: late, |
1376 | 1.28M | to: next_early, |
1377 | 1.28M | }, |
1378 | 3.20k | (InstPosition::After, OperandKind::Use) => CodeRange { |
1379 | 3.20k | from: early, |
1380 | 3.20k | to: next_early, |
1381 | 3.20k | }, |
1382 | | } |
1383 | 2.82M | } |