Coverage Report

Created: 2026-06-07 07:42

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/regalloc2/src/indexset.rs
Line
Count
Source
1
/*
2
 * Released under the terms of the Apache 2.0 license with LLVM
3
 * exception. See `LICENSE` for details.
4
 */
5
6
//! Index sets: sets of integers that represent indices into a space.
7
8
use alloc::vec::Vec;
9
use core::cell::Cell;
10
11
use crate::FxHashMap;
12
13
const SMALL_ELEMS: usize = 12;
14
15
/// A hybrid large/small-mode sparse mapping from integer indices to
16
/// elements.
17
///
18
/// The trailing `(u32, u64)` elements in each variant is a one-item
19
/// cache to allow fast access when streaming through.
20
#[derive(Clone, Debug)]
21
enum AdaptiveMap {
22
    Small {
23
        len: u32,
24
        keys: [u32; SMALL_ELEMS],
25
        values: [u64; SMALL_ELEMS],
26
    },
27
    Large(FxHashMap<u32, u64>),
28
}
29
30
const INVALID: u32 = 0xffff_ffff;
31
32
impl AdaptiveMap {
33
1.00M
    fn new() -> Self {
34
1.00M
        Self::Small {
35
1.00M
            len: 0,
36
1.00M
            keys: [INVALID; SMALL_ELEMS],
37
1.00M
            values: [0; SMALL_ELEMS],
38
1.00M
        }
39
1.00M
    }
40
41
    #[inline(always)]
42
57.0M
    fn get_or_insert<'a>(&'a mut self, key: u32) -> &'a mut u64 {
43
        // Check whether the key is present and we are in small mode;
44
        // if no to both, we need to expand first.
45
57.0M
        let small_mode_idx = match self {
46
            &mut Self::Small {
47
53.9M
                len,
48
53.9M
                ref mut keys,
49
53.9M
                ref values,
50
            } => {
51
                // Perform this scan but do not return right away;
52
                // doing so runs into overlapping-borrow issues
53
                // because the current non-lexical lifetimes
54
                // implementation is not able to see that the `self`
55
                // mutable borrow on return is only on the
56
                // early-return path.
57
151M
                if let Some(i) = keys[..len as usize].iter().position(|&k| k == key) {
58
51.4M
                    Some(i)
59
2.47M
                } else if len != SMALL_ELEMS as u32 {
60
2.43M
                    debug_assert!(len < SMALL_ELEMS as u32);
61
2.43M
                    None
62
451k
                } else if let Some(i) = values.iter().position(|&v| v == 0) {
63
                    // If an existing value is zero, reuse that slot.
64
16.7k
                    keys[i] = key;
65
16.7k
                    Some(i)
66
                } else {
67
24.6k
                    *self = Self::Large(keys.iter().copied().zip(values.iter().copied()).collect());
68
24.6k
                    None
69
                }
70
            }
71
3.17M
            _ => None,
72
        };
73
74
57.0M
        match self {
75
53.8M
            Self::Small { len, keys, values } => {
76
                // If we found the key already while checking whether
77
                // we need to expand above, use that index to return
78
                // early.
79
53.8M
                if let Some(i) = small_mode_idx {
80
51.4M
                    return &mut values[i];
81
2.43M
                }
82
                // Otherwise, the key must not be present; add a new
83
                // entry.
84
2.43M
                debug_assert!(*len < SMALL_ELEMS as u32);
85
2.43M
                let idx = *len as usize;
86
2.43M
                *len += 1;
87
2.43M
                keys[idx] = key;
88
2.43M
                values[idx] = 0;
89
2.43M
                &mut values[idx]
90
            }
91
3.20M
            Self::Large(map) => map.entry(key).or_insert(0),
92
        }
93
57.0M
    }
94
95
    #[inline(always)]
96
36.5M
    fn get_mut(&mut self, key: u32) -> Option<&mut u64> {
97
36.5M
        match self {
98
            &mut Self::Small {
99
35.0M
                len,
100
35.0M
                ref keys,
101
35.0M
                ref mut values,
102
            } => {
103
141M
                for i in 0..len {
104
141M
                    if keys[i as usize] == key {
105
17.9M
                        return Some(&mut values[i as usize]);
106
123M
                    }
107
                }
108
17.1M
                None
109
            }
110
1.51M
            &mut Self::Large(ref mut map) => map.get_mut(&key),
111
        }
112
36.5M
    }
113
    #[inline(always)]
114
95.7M
    fn get(&self, key: u32) -> Option<u64> {
115
95.7M
        match self {
116
            &Self::Small {
117
90.5M
                len,
118
90.5M
                ref keys,
119
90.5M
                ref values,
120
            } => {
121
294M
                for i in 0..len {
122
294M
                    if keys[i as usize] == key {
123
72.3M
                        let value = values[i as usize];
124
72.3M
                        return Some(value);
125
222M
                    }
126
                }
127
18.2M
                None
128
            }
129
5.17M
            &Self::Large(ref map) => {
130
5.17M
                let value = map.get(&key).cloned();
131
5.17M
                value
132
            }
133
        }
134
95.7M
    }
135
2.16M
    fn iter<'a>(&'a self) -> AdaptiveMapIter<'a> {
136
2.16M
        match self {
137
            &Self::Small {
138
2.10M
                len,
139
2.10M
                ref keys,
140
2.10M
                ref values,
141
2.10M
            } => AdaptiveMapIter::Small(&keys[0..len as usize], &values[0..len as usize]),
142
56.1k
            &Self::Large(ref map) => AdaptiveMapIter::Large(map.iter()),
143
        }
144
2.16M
    }
145
146
11.4k
    fn is_empty(&self) -> bool {
147
11.4k
        match self {
148
137k
            AdaptiveMap::Small { values, .. } => values.iter().all(|&value| value == 0),
149
0
            AdaptiveMap::Large(m) => m.values().all(|&value| value == 0),
150
        }
151
11.4k
    }
152
}
153
154
enum AdaptiveMapIter<'a> {
155
    Small(&'a [u32], &'a [u64]),
156
    Large(hashbrown::hash_map::Iter<'a, u32, u64>),
157
}
158
159
impl<'a> core::iter::Iterator for AdaptiveMapIter<'a> {
160
    type Item = (u32, u64);
161
162
    #[inline]
163
11.2M
    fn next(&mut self) -> Option<Self::Item> {
164
11.2M
        match self {
165
10.4M
            &mut Self::Small(ref mut keys, ref mut values) => {
166
10.4M
                if keys.is_empty() {
167
2.10M
                    None
168
                } else {
169
8.33M
                    let (k, v) = ((*keys)[0], (*values)[0]);
170
8.33M
                    *keys = &(*keys)[1..];
171
8.33M
                    *values = &(*values)[1..];
172
8.33M
                    Some((k, v))
173
                }
174
            }
175
819k
            &mut Self::Large(ref mut it) => it.next().map(|(&k, &v)| (k, v)),
176
        }
177
11.2M
    }
178
}
179
180
/// A conceptually infinite-length set of indices that allows union
181
/// and efficient iteration over elements.
182
#[derive(Clone)]
183
pub struct IndexSet {
184
    elems: AdaptiveMap,
185
    cache: Cell<(u32, u64)>,
186
}
187
188
const BITS_PER_WORD: usize = 64;
189
190
impl IndexSet {
191
1.00M
    pub fn new() -> Self {
192
1.00M
        Self {
193
1.00M
            elems: AdaptiveMap::new(),
194
1.00M
            cache: Cell::new((INVALID, 0)),
195
1.00M
        }
196
1.00M
    }
197
198
    #[inline(always)]
199
57.0M
    fn elem(&mut self, bit_index: usize) -> &mut u64 {
200
57.0M
        let word_index = (bit_index / BITS_PER_WORD) as u32;
201
57.0M
        if self.cache.get().0 == word_index {
202
0
            self.cache.set((INVALID, 0));
203
57.0M
        }
204
57.0M
        self.elems.get_or_insert(word_index)
205
57.0M
    }
206
207
    #[inline(always)]
208
36.5M
    fn maybe_elem_mut(&mut self, bit_index: usize) -> Option<&mut u64> {
209
36.5M
        let word_index = (bit_index / BITS_PER_WORD) as u32;
210
36.5M
        if self.cache.get().0 == word_index {
211
0
            self.cache.set((INVALID, 0));
212
36.5M
        }
213
36.5M
        self.elems.get_mut(word_index)
214
36.5M
    }
215
216
    #[inline(always)]
217
95.7M
    fn maybe_elem(&self, bit_index: usize) -> Option<u64> {
218
95.7M
        let word_index = (bit_index / BITS_PER_WORD) as u32;
219
95.7M
        if self.cache.get().0 == word_index {
220
0
            Some(self.cache.get().1)
221
        } else {
222
95.7M
            self.elems.get(word_index)
223
        }
224
95.7M
    }
225
226
    #[inline(always)]
227
86.4M
    pub fn set(&mut self, idx: usize, val: bool) {
228
86.4M
        let bit = idx % BITS_PER_WORD;
229
86.4M
        if val {
230
49.8M
            *self.elem(idx) |= 1 << bit;
231
49.8M
        } else if let Some(word) = self.maybe_elem_mut(idx) {
232
18.8M
            *word &= !(1 << bit);
233
18.8M
        }
234
86.4M
    }
235
236
0
    pub fn assign(&mut self, other: &Self) {
237
0
        self.elems = other.elems.clone();
238
0
        self.cache = other.cache.clone();
239
0
    }
240
241
    #[inline(always)]
242
95.7M
    pub fn get(&self, idx: usize) -> bool {
243
95.7M
        let bit = idx % BITS_PER_WORD;
244
95.7M
        if let Some(word) = self.maybe_elem(idx) {
245
76.8M
            (word & (1 << bit)) != 0
246
        } else {
247
18.8M
            false
248
        }
249
95.7M
    }
250
251
1.66M
    pub fn union_with(&mut self, other: &Self) -> bool {
252
1.66M
        let mut changed = 0;
253
7.65M
        for (word_idx, bits) in other.elems.iter() {
254
7.65M
            if bits == 0 {
255
427k
                continue;
256
7.22M
            }
257
7.22M
            let word_idx = word_idx as usize;
258
7.22M
            let self_word = self.elem(word_idx * BITS_PER_WORD);
259
7.22M
            changed |= bits & !*self_word;
260
7.22M
            *self_word |= bits;
261
        }
262
1.66M
        changed != 0
263
1.66M
    }
264
265
500k
    pub fn iter<'a>(&'a self) -> impl Iterator<Item = usize> + 'a {
266
1.44M
        self.elems.iter().flat_map(|(word_idx, bits)| {
267
1.44M
            let word_idx = word_idx as usize;
268
16.2M
            SetBitsIter(bits).map(move |i| BITS_PER_WORD * word_idx + i)
269
1.44M
        })
270
500k
    }
271
272
    /// Is the adaptive data structure in "small" mode? This is meant
273
    /// for testing assertions only.
274
0
    pub(crate) fn is_small(&self) -> bool {
275
0
        match &self.elems {
276
0
            &AdaptiveMap::Small { .. } => true,
277
0
            _ => false,
278
        }
279
0
    }
280
281
    /// Is the set empty?
282
11.4k
    pub(crate) fn is_empty(&self) -> bool {
283
11.4k
        self.elems.is_empty()
284
11.4k
    }
285
}
286
287
pub struct SetBitsIter(u64);
288
289
impl Iterator for SetBitsIter {
290
    type Item = usize;
291
292
    #[inline]
293
17.6M
    fn next(&mut self) -> Option<usize> {
294
        // Build an `Option<NonZeroU64>` so that on the nonzero path,
295
        // the compiler can optimize the trailing-zeroes operator
296
        // using that knowledge.
297
17.6M
        core::num::NonZeroU64::new(self.0).map(|nz| {
298
16.2M
            let bitidx = nz.trailing_zeros();
299
16.2M
            self.0 &= self.0 - 1; // clear highest set bit
300
16.2M
            bitidx as usize
301
16.2M
        })
302
17.6M
    }
303
}
304
305
impl core::fmt::Debug for IndexSet {
306
0
    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
307
0
        let vals = self.iter().collect::<Vec<_>>();
308
0
        write!(f, "{:?}", vals)
309
0
    }
310
}
311
312
#[cfg(test)]
313
mod test {
314
    use super::IndexSet;
315
316
    #[test]
317
    fn test_set_bits_iter() {
318
        let mut vec = IndexSet::new();
319
        let mut sum = 0;
320
        for i in 0..1024 {
321
            if i % 17 == 0 {
322
                vec.set(i, true);
323
                sum += i;
324
            }
325
        }
326
327
        let mut checksum = 0;
328
        for bit in vec.iter() {
329
            debug_assert!(bit % 17 == 0);
330
            checksum += bit;
331
        }
332
333
        debug_assert_eq!(sum, checksum);
334
    }
335
336
    #[test]
337
    fn test_expand_remove_zero_elems() {
338
        let mut vec = IndexSet::new();
339
        // Set 12 different words (this is the max small-mode size).
340
        for i in 0..12 {
341
            vec.set(64 * i, true);
342
        }
343
        // Now clear a bit, and set a bit in a different word. We
344
        // should still be in small mode.
345
        vec.set(64 * 5, false);
346
        vec.set(64 * 100, true);
347
        debug_assert!(vec.is_small());
348
    }
349
}