/src/regalloc2/src/ion/spill.rs
Line | Count | Source |
1 | | /* |
2 | | * This file was initially derived from the files |
3 | | * `js/src/jit/BacktrackingAllocator.h` and |
4 | | * `js/src/jit/BacktrackingAllocator.cpp` in Mozilla Firefox, and was |
5 | | * originally licensed under the Mozilla Public License 2.0. We |
6 | | * subsequently relicensed it to Apache-2.0 WITH LLVM-exception (see |
7 | | * https://github.com/bytecodealliance/regalloc2/issues/7). |
8 | | * |
9 | | * Since the initial port, the design has been substantially evolved |
10 | | * and optimized. |
11 | | */ |
12 | | |
13 | | //! Spillslot allocation. |
14 | | |
15 | | use super::{ |
16 | | AllocRegResult, Env, LiveRangeKey, PRegIndex, RegTraversalIter, SpillSetIndex, SpillSlotData, |
17 | | SpillSlotIndex, |
18 | | }; |
19 | | use crate::{Allocation, Function, SpillSlot}; |
20 | | |
21 | | impl<'a, F: Function> Env<'a, F> { |
22 | 10.5k | pub fn try_allocating_regs_for_spilled_bundles(&mut self) { |
23 | 10.5k | trace!("allocating regs for spilled bundles"); |
24 | 10.5k | let mut scratch = core::mem::take(&mut self.ctx.scratch_conflicts); |
25 | 5.90M | for i in 0..self.ctx.spilled_bundles.len() { |
26 | 5.90M | let bundle = self.ctx.spilled_bundles[i]; // don't borrow self |
27 | | |
28 | 5.90M | if self.ctx.bundles[bundle].ranges.is_empty() { |
29 | 0 | continue; |
30 | 5.90M | } |
31 | | |
32 | 5.90M | let class = self.ctx.spillsets[self.ctx.bundles[bundle].spillset].class; |
33 | 5.90M | let hint = self.ctx.spillsets[self.ctx.bundles[bundle].spillset] |
34 | 5.90M | .hint |
35 | 5.90M | .as_valid(); |
36 | | |
37 | | // This may be an empty-range bundle whose ranges are not |
38 | | // sorted; sort all range-lists again here. |
39 | 5.90M | self.ctx.bundles[bundle] |
40 | 5.90M | .ranges |
41 | 5.90M | .sort_unstable_by_key(|entry| entry.range.from); |
42 | | |
43 | 5.90M | let mut success = false; |
44 | 5.90M | self.ctx.output.stats.spill_bundle_reg_probes += 1; |
45 | 5.90M | let limit = self.bundles[bundle].limit.map(|l| l as usize); |
46 | 24.4M | for preg in RegTraversalIter::new(self.env, class, None, hint, bundle.index(), limit) { |
47 | 24.4M | trace!("trying bundle {:?} to preg {:?}", bundle, preg); |
48 | 24.4M | let preg_idx = PRegIndex::new(preg.index()); |
49 | | if let AllocRegResult::Allocated(_) = |
50 | 24.4M | self.try_to_allocate_bundle_to_reg(bundle, preg_idx, None, &mut scratch) |
51 | | { |
52 | 5.61M | self.ctx.output.stats.spill_bundle_reg_success += 1; |
53 | 5.61M | success = true; |
54 | 5.61M | break; |
55 | 18.8M | } |
56 | | } |
57 | | |
58 | 5.90M | if !success { |
59 | 285k | trace!( |
60 | 0 | "spilling bundle {:?}: marking spillset {:?} as required", |
61 | | bundle, |
62 | 0 | self.ctx.bundles[bundle].spillset |
63 | | ); |
64 | 285k | self.ctx.spillsets[self.ctx.bundles[bundle].spillset].required = true; |
65 | 5.61M | } |
66 | | } |
67 | 10.5k | self.ctx.scratch_conflicts = scratch; |
68 | 10.5k | } |
69 | | |
70 | 1.87M | pub fn spillslot_can_fit_spillset( |
71 | 1.87M | &mut self, |
72 | 1.87M | spillslot: SpillSlotIndex, |
73 | 1.87M | spillset: SpillSetIndex, |
74 | 1.87M | ) -> bool { |
75 | 1.87M | !self.ctx.spillslots[spillslot.index()] |
76 | 1.87M | .ranges |
77 | 1.87M | .btree |
78 | 1.87M | .contains_key(&LiveRangeKey::from_range( |
79 | 1.87M | &self.ctx.spillsets[spillset].range, |
80 | 1.87M | )) |
81 | 1.87M | } |
82 | | |
83 | 285k | pub fn allocate_spillset_to_spillslot( |
84 | 285k | &mut self, |
85 | 285k | spillset: SpillSetIndex, |
86 | 285k | spillslot: SpillSlotIndex, |
87 | 285k | ) { |
88 | 285k | self.ctx.spillsets[spillset].slot = spillslot; |
89 | | |
90 | 285k | let res = self.ctx.spillslots[spillslot.index()].ranges.btree.insert( |
91 | 285k | LiveRangeKey::from_range(&self.ctx.spillsets[spillset].range), |
92 | 285k | spillset, |
93 | | ); |
94 | | |
95 | 285k | debug_assert!(res.is_none()); |
96 | 285k | } |
97 | | |
98 | 10.5k | pub fn allocate_spillslots(&mut self) { |
99 | | const MAX_ATTEMPTS: usize = 10; |
100 | | |
101 | 7.46M | for spillset in 0..self.ctx.spillsets.len() { |
102 | 7.46M | trace!("allocate spillslot: {}", spillset); |
103 | 7.46M | let spillset = SpillSetIndex::new(spillset); |
104 | 7.46M | if !self.ctx.spillsets[spillset].required { |
105 | 7.18M | continue; |
106 | 285k | } |
107 | 285k | let class = self.ctx.spillsets[spillset].class as usize; |
108 | | // Try a few existing spillslots. |
109 | 285k | let mut i = self.ctx.slots_by_class[class].probe_start; |
110 | 285k | let mut success = false; |
111 | | // Never probe the same element more than once: limit the |
112 | | // attempt count to the number of slots in existence. |
113 | 1.87M | for _attempt in |
114 | 285k | 0..core::cmp::min(self.ctx.slots_by_class[class].slots.len(), MAX_ATTEMPTS) |
115 | | { |
116 | | // Note: this indexing of `slots` is always valid |
117 | | // because either the `slots` list is empty and the |
118 | | // iteration limit above consequently means we don't |
119 | | // run this loop at all, or else `probe_start` is |
120 | | // in-bounds (because it is made so below when we add |
121 | | // a slot, and it always takes on the last index `i` |
122 | | // after this loop). |
123 | 1.87M | let spillslot = self.ctx.slots_by_class[class].slots[i]; |
124 | | |
125 | 1.87M | if self.spillslot_can_fit_spillset(spillslot, spillset) { |
126 | 94.6k | self.allocate_spillset_to_spillslot(spillset, spillslot); |
127 | 94.6k | success = true; |
128 | 94.6k | self.ctx.slots_by_class[class].probe_start = i; |
129 | 94.6k | break; |
130 | 1.78M | } |
131 | | |
132 | 1.78M | i = self.ctx.slots_by_class[class].next_index(i); |
133 | | } |
134 | | |
135 | 285k | if !success { |
136 | 190k | // Allocate a new spillslot. |
137 | 190k | let spillslot = SpillSlotIndex::new(self.ctx.spillslots.len()); |
138 | 190k | self.ctx.spillslots.push(SpillSlotData { |
139 | 190k | ranges: self.ctx.scratch_spillset_pool.pop().unwrap_or_default(), |
140 | 190k | alloc: Allocation::none(), |
141 | 190k | slots: self.func.spillslot_size(self.ctx.spillsets[spillset].class) as u32, |
142 | 190k | }); |
143 | 190k | self.ctx.slots_by_class[class].slots.push(spillslot); |
144 | 190k | self.ctx.slots_by_class[class].probe_start = |
145 | 190k | self.ctx.slots_by_class[class].slots.len() - 1; |
146 | 190k | |
147 | 190k | self.allocate_spillset_to_spillslot(spillset, spillslot); |
148 | 190k | } |
149 | | } |
150 | | |
151 | | // Assign actual slot indices to spillslots. |
152 | 190k | for i in 0..self.ctx.spillslots.len() { |
153 | 190k | self.ctx.spillslots[i].alloc = self.allocate_spillslot(self.ctx.spillslots[i].slots); |
154 | 190k | } |
155 | | |
156 | 10.5k | trace!("spillslot allocator done"); |
157 | 10.5k | } |
158 | | |
159 | 190k | pub fn allocate_spillslot(&mut self, size: u32) -> Allocation { |
160 | 190k | let mut offset = self.ctx.output.num_spillslots as u32; |
161 | | // Align up to `size`. |
162 | 190k | debug_assert!(size.is_power_of_two()); |
163 | 190k | offset = (offset + size - 1) & !(size - 1); |
164 | 190k | let slot = if self.func.multi_spillslot_named_by_last_slot() { |
165 | 0 | offset + size - 1 |
166 | | } else { |
167 | 190k | offset |
168 | | }; |
169 | 190k | offset += size; |
170 | 190k | self.ctx.output.num_spillslots = offset as _; |
171 | 190k | Allocation::stack(SpillSlot::new(slot as usize)) |
172 | 190k | } |
173 | | } |