Coverage Report

Created: 2025-12-04 07:01

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/regalloc2/src/ion/spill.rs
Line
Count
Source
1
/*
2
 * This file was initially derived from the files
3
 * `js/src/jit/BacktrackingAllocator.h` and
4
 * `js/src/jit/BacktrackingAllocator.cpp` in Mozilla Firefox, and was
5
 * originally licensed under the Mozilla Public License 2.0. We
6
 * subsequently relicensed it to Apache-2.0 WITH LLVM-exception (see
7
 * https://github.com/bytecodealliance/regalloc2/issues/7).
8
 *
9
 * Since the initial port, the design has been substantially evolved
10
 * and optimized.
11
 */
12
13
//! Spillslot allocation.
14
15
use super::{
16
    AllocRegResult, Env, LiveRangeKey, PRegIndex, RegTraversalIter, SpillSetIndex, SpillSlotData,
17
    SpillSlotIndex,
18
};
19
use crate::{Allocation, Function, SpillSlot};
20
21
impl<'a, F: Function> Env<'a, F> {
22
10.5k
    pub fn try_allocating_regs_for_spilled_bundles(&mut self) {
23
10.5k
        trace!("allocating regs for spilled bundles");
24
10.5k
        let mut scratch = core::mem::take(&mut self.ctx.scratch_conflicts);
25
5.90M
        for i in 0..self.ctx.spilled_bundles.len() {
26
5.90M
            let bundle = self.ctx.spilled_bundles[i]; // don't borrow self
27
28
5.90M
            if self.ctx.bundles[bundle].ranges.is_empty() {
29
0
                continue;
30
5.90M
            }
31
32
5.90M
            let class = self.ctx.spillsets[self.ctx.bundles[bundle].spillset].class;
33
5.90M
            let hint = self.ctx.spillsets[self.ctx.bundles[bundle].spillset]
34
5.90M
                .hint
35
5.90M
                .as_valid();
36
37
            // This may be an empty-range bundle whose ranges are not
38
            // sorted; sort all range-lists again here.
39
5.90M
            self.ctx.bundles[bundle]
40
5.90M
                .ranges
41
5.90M
                .sort_unstable_by_key(|entry| entry.range.from);
42
43
5.90M
            let mut success = false;
44
5.90M
            self.ctx.output.stats.spill_bundle_reg_probes += 1;
45
5.90M
            let limit = self.bundles[bundle].limit.map(|l| l as usize);
46
24.4M
            for preg in RegTraversalIter::new(self.env, class, None, hint, bundle.index(), limit) {
47
24.4M
                trace!("trying bundle {:?} to preg {:?}", bundle, preg);
48
24.4M
                let preg_idx = PRegIndex::new(preg.index());
49
                if let AllocRegResult::Allocated(_) =
50
24.4M
                    self.try_to_allocate_bundle_to_reg(bundle, preg_idx, None, &mut scratch)
51
                {
52
5.61M
                    self.ctx.output.stats.spill_bundle_reg_success += 1;
53
5.61M
                    success = true;
54
5.61M
                    break;
55
18.8M
                }
56
            }
57
58
5.90M
            if !success {
59
285k
                trace!(
60
0
                    "spilling bundle {:?}: marking spillset {:?} as required",
61
                    bundle,
62
0
                    self.ctx.bundles[bundle].spillset
63
                );
64
285k
                self.ctx.spillsets[self.ctx.bundles[bundle].spillset].required = true;
65
5.61M
            }
66
        }
67
10.5k
        self.ctx.scratch_conflicts = scratch;
68
10.5k
    }
69
70
1.87M
    pub fn spillslot_can_fit_spillset(
71
1.87M
        &mut self,
72
1.87M
        spillslot: SpillSlotIndex,
73
1.87M
        spillset: SpillSetIndex,
74
1.87M
    ) -> bool {
75
1.87M
        !self.ctx.spillslots[spillslot.index()]
76
1.87M
            .ranges
77
1.87M
            .btree
78
1.87M
            .contains_key(&LiveRangeKey::from_range(
79
1.87M
                &self.ctx.spillsets[spillset].range,
80
1.87M
            ))
81
1.87M
    }
82
83
285k
    pub fn allocate_spillset_to_spillslot(
84
285k
        &mut self,
85
285k
        spillset: SpillSetIndex,
86
285k
        spillslot: SpillSlotIndex,
87
285k
    ) {
88
285k
        self.ctx.spillsets[spillset].slot = spillslot;
89
90
285k
        let res = self.ctx.spillslots[spillslot.index()].ranges.btree.insert(
91
285k
            LiveRangeKey::from_range(&self.ctx.spillsets[spillset].range),
92
285k
            spillset,
93
        );
94
95
285k
        debug_assert!(res.is_none());
96
285k
    }
97
98
10.5k
    pub fn allocate_spillslots(&mut self) {
99
        const MAX_ATTEMPTS: usize = 10;
100
101
7.46M
        for spillset in 0..self.ctx.spillsets.len() {
102
7.46M
            trace!("allocate spillslot: {}", spillset);
103
7.46M
            let spillset = SpillSetIndex::new(spillset);
104
7.46M
            if !self.ctx.spillsets[spillset].required {
105
7.18M
                continue;
106
285k
            }
107
285k
            let class = self.ctx.spillsets[spillset].class as usize;
108
            // Try a few existing spillslots.
109
285k
            let mut i = self.ctx.slots_by_class[class].probe_start;
110
285k
            let mut success = false;
111
            // Never probe the same element more than once: limit the
112
            // attempt count to the number of slots in existence.
113
1.87M
            for _attempt in
114
285k
                0..core::cmp::min(self.ctx.slots_by_class[class].slots.len(), MAX_ATTEMPTS)
115
            {
116
                // Note: this indexing of `slots` is always valid
117
                // because either the `slots` list is empty and the
118
                // iteration limit above consequently means we don't
119
                // run this loop at all, or else `probe_start` is
120
                // in-bounds (because it is made so below when we add
121
                // a slot, and it always takes on the last index `i`
122
                // after this loop).
123
1.87M
                let spillslot = self.ctx.slots_by_class[class].slots[i];
124
125
1.87M
                if self.spillslot_can_fit_spillset(spillslot, spillset) {
126
94.6k
                    self.allocate_spillset_to_spillslot(spillset, spillslot);
127
94.6k
                    success = true;
128
94.6k
                    self.ctx.slots_by_class[class].probe_start = i;
129
94.6k
                    break;
130
1.78M
                }
131
132
1.78M
                i = self.ctx.slots_by_class[class].next_index(i);
133
            }
134
135
285k
            if !success {
136
190k
                // Allocate a new spillslot.
137
190k
                let spillslot = SpillSlotIndex::new(self.ctx.spillslots.len());
138
190k
                self.ctx.spillslots.push(SpillSlotData {
139
190k
                    ranges: self.ctx.scratch_spillset_pool.pop().unwrap_or_default(),
140
190k
                    alloc: Allocation::none(),
141
190k
                    slots: self.func.spillslot_size(self.ctx.spillsets[spillset].class) as u32,
142
190k
                });
143
190k
                self.ctx.slots_by_class[class].slots.push(spillslot);
144
190k
                self.ctx.slots_by_class[class].probe_start =
145
190k
                    self.ctx.slots_by_class[class].slots.len() - 1;
146
190k
147
190k
                self.allocate_spillset_to_spillslot(spillset, spillslot);
148
190k
            }
149
        }
150
151
        // Assign actual slot indices to spillslots.
152
190k
        for i in 0..self.ctx.spillslots.len() {
153
190k
            self.ctx.spillslots[i].alloc = self.allocate_spillslot(self.ctx.spillslots[i].slots);
154
190k
        }
155
156
10.5k
        trace!("spillslot allocator done");
157
10.5k
    }
158
159
190k
    pub fn allocate_spillslot(&mut self, size: u32) -> Allocation {
160
190k
        let mut offset = self.ctx.output.num_spillslots as u32;
161
        // Align up to `size`.
162
190k
        debug_assert!(size.is_power_of_two());
163
190k
        offset = (offset + size - 1) & !(size - 1);
164
190k
        let slot = if self.func.multi_spillslot_named_by_last_slot() {
165
0
            offset + size - 1
166
        } else {
167
190k
            offset
168
        };
169
190k
        offset += size;
170
190k
        self.ctx.output.num_spillslots = offset as _;
171
190k
        Allocation::stack(SpillSlot::new(slot as usize))
172
190k
    }
173
}