Coverage Report

Created: 2024-10-16 07:58

/rust/registry/src/index.crates.io-6f17d22bba15001f/cranelift-codegen-0.91.1/src/ir/dfg.rs
Line
Count
Source (jump to first uncovered line)
1
//! Data flow graph tracking Instructions, Values, and blocks.
2
3
use crate::entity::{self, PrimaryMap, SecondaryMap};
4
use crate::ir;
5
use crate::ir::builder::ReplaceBuilder;
6
use crate::ir::dynamic_type::{DynamicTypeData, DynamicTypes};
7
use crate::ir::immediates::HeapImmData;
8
use crate::ir::instructions::{BranchInfo, CallInfo, InstructionData};
9
use crate::ir::{types, ConstantData, ConstantPool, Immediate};
10
use crate::ir::{
11
    Block, DynamicType, FuncRef, HeapImm, Inst, SigRef, Signature, Type, Value,
12
    ValueLabelAssignments, ValueList, ValueListPool,
13
};
14
use crate::ir::{ExtFuncData, RelSourceLoc};
15
use crate::packed_option::ReservedValue;
16
use crate::write::write_operands;
17
use core::fmt;
18
use core::iter;
19
use core::mem;
20
use core::ops::{Index, IndexMut};
21
use core::u16;
22
23
use alloc::collections::BTreeMap;
24
#[cfg(feature = "enable-serde")]
25
use serde::{Deserialize, Serialize};
26
27
/// A data flow graph defines all instructions and basic blocks in a function as well as
28
/// the data flow dependencies between them. The DFG also tracks values which can be either
29
/// instruction results or block parameters.
30
///
31
/// The layout of blocks in the function and of instructions in each block is recorded by the
32
/// `Layout` data structure which forms the other half of the function representation.
33
///
34
#[derive(Clone, PartialEq, Hash)]
35
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
36
pub struct DataFlowGraph {
37
    /// Data about all of the instructions in the function, including opcodes and operands.
38
    /// The instructions in this map are not in program order. That is tracked by `Layout`, along
39
    /// with the block containing each instruction.
40
    insts: PrimaryMap<Inst, InstructionData>,
41
42
    /// List of result values for each instruction.
43
    ///
44
    /// This map gets resized automatically by `make_inst()` so it is always in sync with the
45
    /// primary `insts` map.
46
    results: SecondaryMap<Inst, ValueList>,
47
48
    /// basic blocks in the function and their parameters.
49
    ///
50
    /// This map is not in program order. That is handled by `Layout`, and so is the sequence of
51
    /// instructions contained in each block.
52
    blocks: PrimaryMap<Block, BlockData>,
53
54
    /// Dynamic types created.
55
    pub dynamic_types: DynamicTypes,
56
57
    /// Memory pool of value lists.
58
    ///
59
    /// The `ValueList` references into this pool appear in many places:
60
    ///
61
    /// - Instructions in `insts` that don't have room for their entire argument list inline.
62
    /// - Instruction result values in `results`.
63
    /// - block parameters in `blocks`.
64
    pub value_lists: ValueListPool,
65
66
    /// Primary value table with entries for all values.
67
    values: PrimaryMap<Value, ValueDataPacked>,
68
69
    /// Function signature table. These signatures are referenced by indirect call instructions as
70
    /// well as the external function references.
71
    pub signatures: PrimaryMap<SigRef, Signature>,
72
73
    /// The pre-legalization signature for each entry in `signatures`, if any.
74
    pub old_signatures: SecondaryMap<SigRef, Option<Signature>>,
75
76
    /// External function references. These are functions that can be called directly.
77
    pub ext_funcs: PrimaryMap<FuncRef, ExtFuncData>,
78
79
    /// Saves Value labels.
80
    pub values_labels: Option<BTreeMap<Value, ValueLabelAssignments>>,
81
82
    /// Constants used within the function
83
    pub constants: ConstantPool,
84
85
    /// Stores large immediates that otherwise will not fit on InstructionData
86
    pub immediates: PrimaryMap<Immediate, ConstantData>,
87
88
    /// Out-of-line heap access immediates that don't fit in `InstructionData`.
89
    pub heap_imms: PrimaryMap<HeapImm, HeapImmData>,
90
}
91
92
impl DataFlowGraph {
93
    /// Create a new empty `DataFlowGraph`.
94
191k
    pub fn new() -> Self {
95
191k
        Self {
96
191k
            insts: PrimaryMap::new(),
97
191k
            results: SecondaryMap::new(),
98
191k
            blocks: PrimaryMap::new(),
99
191k
            dynamic_types: DynamicTypes::new(),
100
191k
            value_lists: ValueListPool::new(),
101
191k
            values: PrimaryMap::new(),
102
191k
            signatures: PrimaryMap::new(),
103
191k
            old_signatures: SecondaryMap::new(),
104
191k
            ext_funcs: PrimaryMap::new(),
105
191k
            values_labels: None,
106
191k
            constants: ConstantPool::new(),
107
191k
            immediates: PrimaryMap::new(),
108
191k
            heap_imms: PrimaryMap::new(),
109
191k
        }
110
191k
    }
111
112
    /// Clear everything.
113
0
    pub fn clear(&mut self) {
114
0
        self.insts.clear();
115
0
        self.results.clear();
116
0
        self.blocks.clear();
117
0
        self.dynamic_types.clear();
118
0
        self.value_lists.clear();
119
0
        self.values.clear();
120
0
        self.signatures.clear();
121
0
        self.old_signatures.clear();
122
0
        self.ext_funcs.clear();
123
0
        self.values_labels = None;
124
0
        self.constants.clear();
125
0
        self.immediates.clear();
126
0
    }
127
128
    /// Clear all instructions, but keep blocks and other metadata
129
    /// (signatures, constants, immediates). Everything to do with
130
    /// `Value`s is cleared, including block params and debug info.
131
    ///
132
    /// Used during egraph-based optimization to clear out the pre-opt
133
    /// body so that we can regenerate it from the egraph.
134
0
    pub(crate) fn clear_insts(&mut self) {
135
0
        self.insts.clear();
136
0
        self.results.clear();
137
0
        self.value_lists.clear();
138
0
        self.values.clear();
139
0
        self.values_labels = None;
140
0
        for block in self.blocks.values_mut() {
141
0
            block.params = ValueList::new();
142
0
        }
143
0
    }
144
145
    /// Get the total number of instructions created in this function, whether they are currently
146
    /// inserted in the layout or not.
147
    ///
148
    /// This is intended for use with `SecondaryMap::with_capacity`.
149
3.03M
    pub fn num_insts(&self) -> usize {
150
3.03M
        self.insts.len()
151
3.03M
    }
152
153
    /// Returns `true` if the given instruction reference is valid.
154
13.9M
    pub fn inst_is_valid(&self, inst: Inst) -> bool {
155
13.9M
        self.insts.is_valid(inst)
156
13.9M
    }
157
158
    /// Get the total number of basic blocks created in this function, whether they are
159
    /// currently inserted in the layout or not.
160
    ///
161
    /// This is intended for use with `SecondaryMap::with_capacity`.
162
3.62M
    pub fn num_blocks(&self) -> usize {
163
3.62M
        self.blocks.len()
164
3.62M
    }
165
166
    /// Returns `true` if the given block reference is valid.
167
4.81M
    pub fn block_is_valid(&self, block: Block) -> bool {
168
4.81M
        self.blocks.is_valid(block)
169
4.81M
    }
170
171
    /// Get the total number of values.
172
139k
    pub fn num_values(&self) -> usize {
173
139k
        self.values.len()
174
139k
    }
175
176
    /// Starts collection of debug information.
177
0
    pub fn collect_debug_info(&mut self) {
178
0
        if self.values_labels.is_none() {
179
0
            self.values_labels = Some(Default::default());
180
0
        }
181
0
    }
182
183
    /// Inserts a `ValueLabelAssignments::Alias` for `to_alias` if debug info
184
    /// collection is enabled.
185
50.4k
    pub fn add_value_label_alias(&mut self, to_alias: Value, from: RelSourceLoc, value: Value) {
186
50.4k
        if let Some(values_labels) = self.values_labels.as_mut() {
187
0
            values_labels.insert(to_alias, ir::ValueLabelAssignments::Alias { from, value });
188
50.4k
        }
189
50.4k
    }
190
}
191
192
/// Resolve value aliases.
193
///
194
/// Find the original SSA value that `value` aliases, or None if an
195
/// alias cycle is detected.
196
24.5M
fn maybe_resolve_aliases(
197
24.5M
    values: &PrimaryMap<Value, ValueDataPacked>,
198
24.5M
    value: Value,
199
24.5M
) -> Option<Value> {
200
24.5M
    let mut v = value;
201
24.5M
202
24.5M
    // Note that values may be empty here.
203
24.5M
    for _ in 0..=values.len() {
204
25.4M
        if let ValueData::Alias { original, .. } = ValueData::from(values[v]) {
205
901k
            v = original;
206
901k
        } else {
207
24.5M
            return Some(v);
208
        }
209
    }
210
211
0
    None
212
24.5M
}
213
214
/// Resolve value aliases.
215
///
216
/// Find the original SSA value that `value` aliases.
217
24.5M
fn resolve_aliases(values: &PrimaryMap<Value, ValueDataPacked>, value: Value) -> Value {
218
24.5M
    if let Some(v) = maybe_resolve_aliases(values, value) {
219
24.5M
        v
220
    } else {
221
0
        panic!("Value alias loop detected for {}", value);
222
    }
223
24.5M
}
224
225
/// Iterator over all Values in a DFG
226
pub struct Values<'a> {
227
    inner: entity::Iter<'a, Value, ValueDataPacked>,
228
}
229
230
/// Check for non-values
231
0
fn valid_valuedata(data: ValueDataPacked) -> bool {
232
0
    let data = ValueData::from(data);
233
    if let ValueData::Alias {
234
        ty: types::INVALID,
235
0
        original,
236
0
    } = ValueData::from(data)
237
    {
238
0
        if original == Value::reserved_value() {
239
0
            return false;
240
0
        }
241
0
    }
242
0
    true
243
0
}
244
245
impl<'a> Iterator for Values<'a> {
246
    type Item = Value;
247
248
0
    fn next(&mut self) -> Option<Self::Item> {
249
0
        self.inner
250
0
            .by_ref()
251
0
            .find(|kv| valid_valuedata(*kv.1))
252
0
            .map(|kv| kv.0)
253
0
    }
254
}
255
256
/// Handling values.
257
///
258
/// Values are either block parameters or instruction results.
259
impl DataFlowGraph {
260
    /// Allocate an extended value entry.
261
3.26M
    fn make_value(&mut self, data: ValueData) -> Value {
262
3.26M
        self.values.push(data.into())
263
3.26M
    }
264
265
    /// Get an iterator over all values.
266
0
    pub fn values<'a>(&'a self) -> Values {
267
0
        Values {
268
0
            inner: self.values.iter(),
269
0
        }
270
0
    }
271
272
    /// Check if a value reference is valid.
273
32.2M
    pub fn value_is_valid(&self, v: Value) -> bool {
274
32.2M
        self.values.is_valid(v)
275
32.2M
    }
276
277
    /// Get the type of a value.
278
86.4M
    pub fn value_type(&self, v: Value) -> Type {
279
86.4M
        self.values[v].ty()
280
86.4M
    }
281
282
    /// Fill in the type of a value, only if currently invalid (as a placeholder).
283
0
    pub(crate) fn fill_in_value_type(&mut self, v: Value, ty: Type) {
284
0
        debug_assert!(self.values[v].ty().is_invalid() || self.values[v].ty() == ty);
285
0
        self.values[v].set_type(ty);
286
0
    }
287
288
    /// Get the definition of a value.
289
    ///
290
    /// This is either the instruction that defined it or the Block that has the value as an
291
    /// parameter.
292
79.4M
    pub fn value_def(&self, v: Value) -> ValueDef {
293
79.4M
        match ValueData::from(self.values[v]) {
294
31.9M
            ValueData::Inst { inst, num, .. } => ValueDef::Result(inst, num as usize),
295
47.1M
            ValueData::Param { block, num, .. } => ValueDef::Param(block, num as usize),
296
275k
            ValueData::Alias { original, .. } => {
297
275k
                // Make sure we only recurse one level. `resolve_aliases` has safeguards to
298
275k
                // detect alias loops without overrunning the stack.
299
275k
                self.value_def(self.resolve_aliases(original))
300
            }
301
        }
302
79.4M
    }
303
304
    /// Determine if `v` is an attached instruction result / block parameter.
305
    ///
306
    /// An attached value can't be attached to something else without first being detached.
307
    ///
308
    /// Value aliases are not considered to be attached to anything. Use `resolve_aliases()` to
309
    /// determine if the original aliased value is attached.
310
16.8M
    pub fn value_is_attached(&self, v: Value) -> bool {
311
16.8M
        use self::ValueData::*;
312
16.8M
        match ValueData::from(self.values[v]) {
313
13.9M
            Inst { inst, num, .. } => Some(&v) == self.inst_results(inst).get(num as usize),
314
2.89M
            Param { block, num, .. } => Some(&v) == self.block_params(block).get(num as usize),
315
0
            Alias { .. } => false,
316
        }
317
16.8M
    }
318
319
    /// Resolve value aliases.
320
    ///
321
    /// Find the original SSA value that `value` aliases.
322
21.1M
    pub fn resolve_aliases(&self, value: Value) -> Value {
323
21.1M
        resolve_aliases(&self.values, value)
324
21.1M
    }
325
326
    /// Resolve all aliases among inst's arguments.
327
    ///
328
    /// For each argument of inst which is defined by an alias, replace the
329
    /// alias with the aliased value.
330
3.98M
    pub fn resolve_aliases_in_arguments(&mut self, inst: Inst) {
331
3.98M
        for arg in self.insts[inst].arguments_mut(&mut self.value_lists) {
332
3.42M
            let resolved = resolve_aliases(&self.values, *arg);
333
3.42M
            if resolved != *arg {
334
476k
                *arg = resolved;
335
2.94M
            }
336
        }
337
3.98M
    }
338
339
    /// Turn a value into an alias of another.
340
    ///
341
    /// Change the `dest` value to behave as an alias of `src`. This means that all uses of `dest`
342
    /// will behave as if they used that value `src`.
343
    ///
344
    /// The `dest` value can't be attached to an instruction or block.
345
88.2k
    pub fn change_to_alias(&mut self, dest: Value, src: Value) {
346
88.2k
        debug_assert!(!self.value_is_attached(dest));
347
        // Try to create short alias chains by finding the original source value.
348
        // This also avoids the creation of loops.
349
88.2k
        let original = self.resolve_aliases(src);
350
88.2k
        debug_assert_ne!(
351
            dest, original,
352
0
            "Aliasing {} to {} would create a loop",
353
            dest, src
354
        );
355
88.2k
        let ty = self.value_type(original);
356
88.2k
        debug_assert_eq!(
357
0
            self.value_type(dest),
358
            ty,
359
0
            "Aliasing {} to {} would change its type {} to {}",
360
0
            dest,
361
0
            src,
362
0
            self.value_type(dest),
363
            ty
364
        );
365
88.2k
        debug_assert_ne!(ty, types::INVALID);
366
367
88.2k
        self.values[dest] = ValueData::Alias { ty, original }.into();
368
88.2k
    }
369
370
    /// Replace the results of one instruction with aliases to the results of another.
371
    ///
372
    /// Change all the results of `dest_inst` to behave as aliases of
373
    /// corresponding results of `src_inst`, as if calling change_to_alias for
374
    /// each.
375
    ///
376
    /// After calling this instruction, `dest_inst` will have had its results
377
    /// cleared, so it likely needs to be removed from the graph.
378
    ///
379
1.25M
    pub fn replace_with_aliases(&mut self, dest_inst: Inst, src_inst: Inst) {
380
1.25M
        debug_assert_ne!(
381
            dest_inst, src_inst,
382
0
            "Replacing {} with itself would create a loop",
383
            dest_inst
384
        );
385
1.25M
        debug_assert_eq!(
386
0
            self.results[dest_inst].len(&self.value_lists),
387
0
            self.results[src_inst].len(&self.value_lists),
388
0
            "Replacing {} with {} would produce a different number of results.",
389
            dest_inst,
390
            src_inst
391
        );
392
393
1.25M
        for (&dest, &src) in self.results[dest_inst]
394
1.25M
            .as_slice(&self.value_lists)
395
1.25M
            .iter()
396
1.25M
            .zip(self.results[src_inst].as_slice(&self.value_lists))
397
        {
398
1.21M
            let original = src;
399
1.21M
            let ty = self.value_type(original);
400
1.21M
            debug_assert_eq!(
401
0
                self.value_type(dest),
402
                ty,
403
0
                "Aliasing {} to {} would change its type {} to {}",
404
0
                dest,
405
0
                src,
406
0
                self.value_type(dest),
407
                ty
408
            );
409
1.21M
            debug_assert_ne!(ty, types::INVALID);
410
411
1.21M
            self.values[dest] = ValueData::Alias { ty, original }.into();
412
        }
413
414
1.25M
        self.clear_results(dest_inst);
415
1.25M
    }
416
}
417
418
/// Where did a value come from?
419
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
420
pub enum ValueDef {
421
    /// Value is the n'th result of an instruction.
422
    Result(Inst, usize),
423
    /// Value is the n'th parameter to a block.
424
    Param(Block, usize),
425
}
426
427
impl ValueDef {
428
    /// Unwrap the instruction where the value was defined, or panic.
429
50.4k
    pub fn unwrap_inst(&self) -> Inst {
430
50.4k
        self.inst().expect("Value is not an instruction result")
431
50.4k
    }
432
433
    /// Get the instruction where the value was defined, if any.
434
2.76M
    pub fn inst(&self) -> Option<Inst> {
435
2.76M
        match *self {
436
1.95M
            Self::Result(inst, _) => Some(inst),
437
812k
            _ => None,
438
        }
439
2.76M
    }
440
441
    /// Unwrap the block there the parameter is defined, or panic.
442
0
    pub fn unwrap_block(&self) -> Block {
443
0
        match *self {
444
0
            Self::Param(block, _) => block,
445
0
            _ => panic!("Value is not a block parameter"),
446
        }
447
0
    }
448
449
    /// Get the program point where the value was defined.
450
0
    pub fn pp(self) -> ir::ExpandedProgramPoint {
451
0
        self.into()
452
0
    }
453
454
    /// Get the number component of this definition.
455
    ///
456
    /// When multiple values are defined at the same program point, this indicates the index of
457
    /// this value.
458
0
    pub fn num(self) -> usize {
459
0
        match self {
460
0
            Self::Result(_, n) | Self::Param(_, n) => n,
461
0
        }
462
0
    }
463
}
464
465
/// Internal table storage for extended values.
466
#[derive(Clone, Debug, PartialEq, Hash)]
467
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
468
enum ValueData {
469
    /// Value is defined by an instruction.
470
    Inst { ty: Type, num: u16, inst: Inst },
471
472
    /// Value is a block parameter.
473
    Param { ty: Type, num: u16, block: Block },
474
475
    /// Value is an alias of another value.
476
    /// An alias value can't be linked as an instruction result or block parameter. It is used as a
477
    /// placeholder when the original instruction or block has been rewritten or modified.
478
    Alias { ty: Type, original: Value },
479
}
480
481
/// Bit-packed version of ValueData, for efficiency.
482
///
483
/// Layout:
484
///
485
/// ```plain
486
///        | tag:2 |  type:14        |    num:16       | index:32          |
487
/// ```
488
#[derive(Clone, Copy, Debug, PartialEq, Hash)]
489
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
490
struct ValueDataPacked(u64);
491
492
impl ValueDataPacked {
493
    const INDEX_SHIFT: u64 = 0;
494
    const INDEX_BITS: u64 = 32;
495
    const NUM_SHIFT: u64 = Self::INDEX_SHIFT + Self::INDEX_BITS;
496
    const NUM_BITS: u64 = 16;
497
    const TYPE_SHIFT: u64 = Self::NUM_SHIFT + Self::NUM_BITS;
498
    const TYPE_BITS: u64 = 14;
499
    const TAG_SHIFT: u64 = Self::TYPE_SHIFT + Self::TYPE_BITS;
500
    const TAG_BITS: u64 = 2;
501
502
    const TAG_INST: u64 = 1;
503
    const TAG_PARAM: u64 = 2;
504
    const TAG_ALIAS: u64 = 3;
505
506
4.72M
    fn make(tag: u64, ty: Type, num: u16, index: u32) -> ValueDataPacked {
507
4.72M
        debug_assert!(tag < (1 << Self::TAG_BITS));
508
4.72M
        debug_assert!(ty.repr() < (1 << Self::TYPE_BITS));
509
510
4.72M
        ValueDataPacked(
511
4.72M
            (tag << Self::TAG_SHIFT)
512
4.72M
                | ((ty.repr() as u64) << Self::TYPE_SHIFT)
513
4.72M
                | ((num as u64) << Self::NUM_SHIFT)
514
4.72M
                | ((index as u64) << Self::INDEX_SHIFT),
515
4.72M
        )
516
4.72M
    }
517
518
    #[inline(always)]
519
574M
    fn field(self, shift: u64, bits: u64) -> u64 {
520
574M
        (self.0 >> shift) & ((1 << bits) - 1)
521
574M
    }
522
523
    #[inline(always)]
524
86.4M
    fn ty(self) -> Type {
525
86.4M
        let ty = self.field(ValueDataPacked::TYPE_SHIFT, ValueDataPacked::TYPE_BITS) as u16;
526
86.4M
        Type::from_repr(ty)
527
86.4M
    }
528
529
    #[inline(always)]
530
0
    fn set_type(&mut self, ty: Type) {
531
0
        self.0 &= !(((1 << Self::TYPE_BITS) - 1) << Self::TYPE_SHIFT);
532
0
        self.0 |= (ty.repr() as u64) << Self::TYPE_SHIFT;
533
0
    }
534
}
535
536
impl From<ValueData> for ValueDataPacked {
537
4.72M
    fn from(data: ValueData) -> Self {
538
4.72M
        match data {
539
2.76M
            ValueData::Inst { ty, num, inst } => {
540
2.76M
                Self::make(Self::TAG_INST, ty, num, inst.as_bits())
541
            }
542
666k
            ValueData::Param { ty, num, block } => {
543
666k
                Self::make(Self::TAG_PARAM, ty, num, block.as_bits())
544
            }
545
1.29M
            ValueData::Alias { ty, original } => {
546
1.29M
                Self::make(Self::TAG_ALIAS, ty, 0, original.as_bits())
547
            }
548
        }
549
4.72M
    }
550
}
551
552
impl From<ValueDataPacked> for ValueData {
553
121M
    fn from(data: ValueDataPacked) -> Self {
554
121M
        let tag = data.field(ValueDataPacked::TAG_SHIFT, ValueDataPacked::TAG_BITS);
555
121M
        let ty = data.field(ValueDataPacked::TYPE_SHIFT, ValueDataPacked::TYPE_BITS) as u16;
556
121M
        let num = data.field(ValueDataPacked::NUM_SHIFT, ValueDataPacked::NUM_BITS) as u16;
557
121M
        let index = data.field(ValueDataPacked::INDEX_SHIFT, ValueDataPacked::INDEX_BITS) as u32;
558
121M
559
121M
        let ty = Type::from_repr(ty);
560
121M
        match tag {
561
65.9M
            ValueDataPacked::TAG_INST => ValueData::Inst {
562
65.9M
                ty,
563
65.9M
                num,
564
65.9M
                inst: Inst::from_bits(index),
565
65.9M
            },
566
54.7M
            ValueDataPacked::TAG_PARAM => ValueData::Param {
567
54.7M
                ty,
568
54.7M
                num,
569
54.7M
                block: Block::from_bits(index),
570
54.7M
            },
571
1.17M
            ValueDataPacked::TAG_ALIAS => ValueData::Alias {
572
1.17M
                ty,
573
1.17M
                original: Value::from_bits(index),
574
1.17M
            },
575
0
            _ => panic!("Invalid tag {} in ValueDataPacked 0x{:x}", tag, data.0),
576
        }
577
121M
    }
578
}
579
580
/// Instructions.
581
///
582
impl DataFlowGraph {
583
    /// Create a new instruction.
584
    ///
585
    /// The type of the first result is indicated by `data.ty`. If the instruction produces
586
    /// multiple results, also call `make_inst_results` to allocate value table entries.
587
3.03M
    pub fn make_inst(&mut self, data: InstructionData) -> Inst {
588
3.03M
        let n = self.num_insts() + 1;
589
3.03M
        self.results.resize(n);
590
3.03M
        self.insts.push(data)
591
3.03M
    }
592
593
    /// Declares a dynamic vector type
594
0
    pub fn make_dynamic_ty(&mut self, data: DynamicTypeData) -> DynamicType {
595
0
        self.dynamic_types.push(data)
596
0
    }
597
598
    /// Returns an object that displays `inst`.
599
0
    pub fn display_inst<'a>(&'a self, inst: Inst) -> DisplayInst<'a> {
600
0
        DisplayInst(self, inst)
601
0
    }
602
603
    /// Returns an object that displays the given `value`'s defining instruction.
604
    ///
605
    /// Panics if the value is not defined by an instruction (i.e. it is a basic
606
    /// block argument).
607
0
    pub fn display_value_inst(&self, value: Value) -> DisplayInst<'_> {
608
0
        match self.value_def(value) {
609
0
            ir::ValueDef::Result(inst, _) => self.display_inst(inst),
610
0
            ir::ValueDef::Param(_, _) => panic!("value is not defined by an instruction"),
611
        }
612
0
    }
613
614
    /// Get all value arguments on `inst` as a slice.
615
66.4M
    pub fn inst_args(&self, inst: Inst) -> &[Value] {
616
66.4M
        self.insts[inst].arguments(&self.value_lists)
617
66.4M
    }
618
619
    /// Get all value arguments on `inst` as a mutable slice.
620
0
    pub fn inst_args_mut(&mut self, inst: Inst) -> &mut [Value] {
621
0
        self.insts[inst].arguments_mut(&mut self.value_lists)
622
0
    }
623
624
    /// Get the fixed value arguments on `inst` as a slice.
625
19.9M
    pub fn inst_fixed_args(&self, inst: Inst) -> &[Value] {
626
19.9M
        let num_fixed_args = self[inst]
627
19.9M
            .opcode()
628
19.9M
            .constraints()
629
19.9M
            .num_fixed_value_arguments();
630
19.9M
        &self.inst_args(inst)[..num_fixed_args]
631
19.9M
    }
632
633
    /// Get the fixed value arguments on `inst` as a mutable slice.
634
0
    pub fn inst_fixed_args_mut(&mut self, inst: Inst) -> &mut [Value] {
635
0
        let num_fixed_args = self[inst]
636
0
            .opcode()
637
0
            .constraints()
638
0
            .num_fixed_value_arguments();
639
0
        &mut self.inst_args_mut(inst)[..num_fixed_args]
640
0
    }
641
642
    /// Get the variable value arguments on `inst` as a slice.
643
4.24M
    pub fn inst_variable_args(&self, inst: Inst) -> &[Value] {
644
4.24M
        let num_fixed_args = self[inst]
645
4.24M
            .opcode()
646
4.24M
            .constraints()
647
4.24M
            .num_fixed_value_arguments();
648
4.24M
        &self.inst_args(inst)[num_fixed_args..]
649
4.24M
    }
650
651
    /// Get the variable value arguments on `inst` as a mutable slice.
652
0
    pub fn inst_variable_args_mut(&mut self, inst: Inst) -> &mut [Value] {
653
0
        let num_fixed_args = self[inst]
654
0
            .opcode()
655
0
            .constraints()
656
0
            .num_fixed_value_arguments();
657
0
        &mut self.inst_args_mut(inst)[num_fixed_args..]
658
0
    }
659
660
    /// Create result values for an instruction that produces multiple results.
661
    ///
662
    /// Instructions that produce no result values only need to be created with `make_inst`,
663
    /// otherwise call `make_inst_results` to allocate value table entries for the results.
664
    ///
665
    /// The result value types are determined from the instruction's value type constraints and the
666
    /// provided `ctrl_typevar` type for polymorphic instructions. For non-polymorphic
667
    /// instructions, `ctrl_typevar` is ignored, and `INVALID` can be used.
668
    ///
669
    /// The type of the first result value is also set, even if it was already set in the
670
    /// `InstructionData` passed to `make_inst`. If this function is called with a single-result
671
    /// instruction, that is the only effect.
672
2.93M
    pub fn make_inst_results(&mut self, inst: Inst, ctrl_typevar: Type) -> usize {
673
2.93M
        self.make_inst_results_reusing(inst, ctrl_typevar, iter::empty())
674
2.93M
    }
675
676
    /// Create result values for `inst`, reusing the provided detached values.
677
    ///
678
    /// Create a new set of result values for `inst` using `ctrl_typevar` to determine the result
679
    /// types. Any values provided by `reuse` will be reused. When `reuse` is exhausted or when it
680
    /// produces `None`, a new value is created.
681
3.09M
    pub fn make_inst_results_reusing<I>(
682
3.09M
        &mut self,
683
3.09M
        inst: Inst,
684
3.09M
        ctrl_typevar: Type,
685
3.09M
        reuse: I,
686
3.09M
    ) -> usize
687
3.09M
    where
688
3.09M
        I: Iterator<Item = Option<Value>>,
689
3.09M
    {
690
3.09M
        let mut reuse = reuse.fuse();
691
3.09M
692
3.09M
        self.results[inst].clear(&mut self.value_lists);
693
694
        // Get the call signature if this is a function call.
695
3.09M
        if let Some(sig) = self.call_signature(inst) {
696
            // Create result values corresponding to the call return types.
697
104k
            debug_assert_eq!(
698
0
                self.insts[inst].opcode().constraints().num_fixed_results(),
699
                0
700
            );
701
104k
            let num_results = self.signatures[sig].returns.len();
702
104k
            for res_idx in 0..num_results {
703
49.6k
                let ty = self.signatures[sig].returns[res_idx].value_type;
704
49.6k
                if let Some(Some(v)) = reuse.next() {
705
0
                    debug_assert_eq!(self.value_type(v), ty, "Reused {} is wrong type", ty);
706
0
                    self.attach_result(inst, v);
707
49.6k
                } else {
708
49.6k
                    self.append_result(inst, ty);
709
49.6k
                }
710
            }
711
104k
            num_results
712
        } else {
713
            // Create result values corresponding to the opcode's constraints.
714
2.99M
            let constraints = self.insts[inst].opcode().constraints();
715
2.99M
            let num_results = constraints.num_fixed_results();
716
2.99M
            for res_idx in 0..num_results {
717
2.54M
                let ty = constraints.result_type(res_idx, ctrl_typevar);
718
2.54M
                if let Some(Some(v)) = reuse.next() {
719
165k
                    debug_assert_eq!(self.value_type(v), ty, "Reused {} is wrong type", ty);
720
165k
                    self.attach_result(inst, v);
721
2.38M
                } else {
722
2.38M
                    self.append_result(inst, ty);
723
2.38M
                }
724
            }
725
2.99M
            num_results
726
        }
727
3.09M
    }
<cranelift_codegen::ir::dfg::DataFlowGraph>::make_inst_results_reusing::<core::iter::sources::empty::Empty<core::option::Option<cranelift_codegen::ir::entities::Value>>>
Line
Count
Source
681
2.93M
    pub fn make_inst_results_reusing<I>(
682
2.93M
        &mut self,
683
2.93M
        inst: Inst,
684
2.93M
        ctrl_typevar: Type,
685
2.93M
        reuse: I,
686
2.93M
    ) -> usize
687
2.93M
    where
688
2.93M
        I: Iterator<Item = Option<Value>>,
689
2.93M
    {
690
2.93M
        let mut reuse = reuse.fuse();
691
2.93M
692
2.93M
        self.results[inst].clear(&mut self.value_lists);
693
694
        // Get the call signature if this is a function call.
695
2.93M
        if let Some(sig) = self.call_signature(inst) {
696
            // Create result values corresponding to the call return types.
697
104k
            debug_assert_eq!(
698
0
                self.insts[inst].opcode().constraints().num_fixed_results(),
699
                0
700
            );
701
104k
            let num_results = self.signatures[sig].returns.len();
702
104k
            for res_idx in 0..num_results {
703
49.6k
                let ty = self.signatures[sig].returns[res_idx].value_type;
704
49.6k
                if let Some(Some(v)) = reuse.next() {
705
0
                    debug_assert_eq!(self.value_type(v), ty, "Reused {} is wrong type", ty);
706
0
                    self.attach_result(inst, v);
707
49.6k
                } else {
708
49.6k
                    self.append_result(inst, ty);
709
49.6k
                }
710
            }
711
104k
            num_results
712
        } else {
713
            // Create result values corresponding to the opcode's constraints.
714
2.82M
            let constraints = self.insts[inst].opcode().constraints();
715
2.82M
            let num_results = constraints.num_fixed_results();
716
2.82M
            for res_idx in 0..num_results {
717
2.38M
                let ty = constraints.result_type(res_idx, ctrl_typevar);
718
2.38M
                if let Some(Some(v)) = reuse.next() {
719
0
                    debug_assert_eq!(self.value_type(v), ty, "Reused {} is wrong type", ty);
720
0
                    self.attach_result(inst, v);
721
2.38M
                } else {
722
2.38M
                    self.append_result(inst, ty);
723
2.38M
                }
724
            }
725
2.82M
            num_results
726
        }
727
2.93M
    }
Unexecuted instantiation: <cranelift_codegen::ir::dfg::DataFlowGraph>::make_inst_results_reusing::<core::iter::adapters::map::Map<core::slice::iter::Iter<cranelift_codegen::ir::entities::Value>, <cranelift_codegen::ir::dfg::DataFlowGraph>::make_inst_results_for_parser::{closure#1}>>
<cranelift_codegen::ir::dfg::DataFlowGraph>::make_inst_results_reusing::<core::iter::adapters::cloned::Cloned<core::slice::iter::Iter<core::option::Option<cranelift_codegen::ir::entities::Value>>>>
Line
Count
Source
681
165k
    pub fn make_inst_results_reusing<I>(
682
165k
        &mut self,
683
165k
        inst: Inst,
684
165k
        ctrl_typevar: Type,
685
165k
        reuse: I,
686
165k
    ) -> usize
687
165k
    where
688
165k
        I: Iterator<Item = Option<Value>>,
689
165k
    {
690
165k
        let mut reuse = reuse.fuse();
691
165k
692
165k
        self.results[inst].clear(&mut self.value_lists);
693
694
        // Get the call signature if this is a function call.
695
165k
        if let Some(sig) = self.call_signature(inst) {
696
            // Create result values corresponding to the call return types.
697
0
            debug_assert_eq!(
698
0
                self.insts[inst].opcode().constraints().num_fixed_results(),
699
                0
700
            );
701
0
            let num_results = self.signatures[sig].returns.len();
702
0
            for res_idx in 0..num_results {
703
0
                let ty = self.signatures[sig].returns[res_idx].value_type;
704
0
                if let Some(Some(v)) = reuse.next() {
705
0
                    debug_assert_eq!(self.value_type(v), ty, "Reused {} is wrong type", ty);
706
0
                    self.attach_result(inst, v);
707
0
                } else {
708
0
                    self.append_result(inst, ty);
709
0
                }
710
            }
711
0
            num_results
712
        } else {
713
            // Create result values corresponding to the opcode's constraints.
714
165k
            let constraints = self.insts[inst].opcode().constraints();
715
165k
            let num_results = constraints.num_fixed_results();
716
165k
            for res_idx in 0..num_results {
717
165k
                let ty = constraints.result_type(res_idx, ctrl_typevar);
718
165k
                if let Some(Some(v)) = reuse.next() {
719
165k
                    debug_assert_eq!(self.value_type(v), ty, "Reused {} is wrong type", ty);
720
165k
                    self.attach_result(inst, v);
721
0
                } else {
722
0
                    self.append_result(inst, ty);
723
0
                }
724
            }
725
165k
            num_results
726
        }
727
165k
    }
728
729
    /// Create a `ReplaceBuilder` that will replace `inst` with a new instruction in place.
730
232k
    pub fn replace(&mut self, inst: Inst) -> ReplaceBuilder {
731
232k
        ReplaceBuilder::new(self, inst)
732
232k
    }
733
734
    /// Detach the list of result values from `inst` and return it.
735
    ///
736
    /// This leaves `inst` without any result values. New result values can be created by calling
737
    /// `make_inst_results` or by using a `replace(inst)` builder.
738
51.1k
    pub fn detach_results(&mut self, inst: Inst) -> ValueList {
739
51.1k
        self.results[inst].take()
740
51.1k
    }
741
742
    /// Clear the list of result values from `inst`.
743
    ///
744
    /// This leaves `inst` without any result values. New result values can be created by calling
745
    /// `make_inst_results` or by using a `replace(inst)` builder.
746
1.28M
    pub fn clear_results(&mut self, inst: Inst) {
747
1.28M
        self.results[inst].clear(&mut self.value_lists)
748
1.28M
    }
749
750
    /// Attach an existing value to the result value list for `inst`.
751
    ///
752
    /// The `res` value is appended to the end of the result list.
753
    ///
754
    /// This is a very low-level operation. Usually, instruction results with the correct types are
755
    /// created automatically. The `res` value must not be attached to anything else.
756
165k
    pub fn attach_result(&mut self, inst: Inst, res: Value) {
757
165k
        debug_assert!(!self.value_is_attached(res));
758
165k
        let num = self.results[inst].push(res, &mut self.value_lists);
759
165k
        debug_assert!(num <= u16::MAX as usize, "Too many result values");
760
165k
        let ty = self.value_type(res);
761
165k
        self.values[res] = ValueData::Inst {
762
165k
            ty,
763
165k
            num: num as u16,
764
165k
            inst,
765
165k
        }
766
165k
        .into();
767
165k
    }
768
769
    /// Replace an instruction result with a new value of type `new_type`.
770
    ///
771
    /// The `old_value` must be an attached instruction result.
772
    ///
773
    /// The old value is left detached, so it should probably be changed into something else.
774
    ///
775
    /// Returns the new value.
776
165k
    pub fn replace_result(&mut self, old_value: Value, new_type: Type) -> Value {
777
165k
        let (num, inst) = match ValueData::from(self.values[old_value]) {
778
165k
            ValueData::Inst { num, inst, .. } => (num, inst),
779
0
            _ => panic!("{} is not an instruction result value", old_value),
780
        };
781
165k
        let new_value = self.make_value(ValueData::Inst {
782
165k
            ty: new_type,
783
165k
            num,
784
165k
            inst,
785
165k
        });
786
165k
        let num = num as usize;
787
165k
        let attached = mem::replace(
788
165k
            self.results[inst]
789
165k
                .get_mut(num, &mut self.value_lists)
790
165k
                .expect("Replacing detached result"),
791
165k
            new_value,
792
165k
        );
793
165k
        debug_assert_eq!(
794
            attached,
795
            old_value,
796
0
            "{} wasn't detached from {}",
797
0
            old_value,
798
0
            self.display_inst(inst)
799
        );
800
165k
        new_value
801
165k
    }
802
803
    /// Append a new instruction result value to `inst`.
804
2.43M
    pub fn append_result(&mut self, inst: Inst, ty: Type) -> Value {
805
2.43M
        let res = self.values.next_key();
806
2.43M
        let num = self.results[inst].push(res, &mut self.value_lists);
807
2.43M
        debug_assert!(num <= u16::MAX as usize, "Too many result values");
808
2.43M
        self.make_value(ValueData::Inst {
809
2.43M
            ty,
810
2.43M
            inst,
811
2.43M
            num: num as u16,
812
2.43M
        })
813
2.43M
    }
814
815
    /// Append a new value argument to an instruction.
816
    ///
817
    /// Panics if the instruction doesn't support arguments.
818
194
    pub fn append_inst_arg(&mut self, inst: Inst, new_arg: Value) {
819
194
        let mut branch_values = self.insts[inst]
820
194
            .take_value_list()
821
194
            .expect("the instruction doesn't have value arguments");
822
194
        branch_values.push(new_arg, &mut self.value_lists);
823
194
        self.insts[inst].put_value_list(branch_values)
824
194
    }
825
826
    /// Get the first result of an instruction.
827
    ///
828
    /// This function panics if the instruction doesn't have any result.
829
15.3M
    pub fn first_result(&self, inst: Inst) -> Value {
830
15.3M
        self.results[inst]
831
15.3M
            .first(&self.value_lists)
832
15.3M
            .expect("Instruction has no results")
833
15.3M
    }
834
835
    /// Test if `inst` has any result values currently.
836
232k
    pub fn has_results(&self, inst: Inst) -> bool {
837
232k
        !self.results[inst].is_empty()
838
232k
    }
839
840
    /// Return all the results of an instruction.
841
85.9M
    pub fn inst_results(&self, inst: Inst) -> &[Value] {
842
85.9M
        self.results[inst].as_slice(&self.value_lists)
843
85.9M
    }
844
845
    /// Return all the results of an instruction as ValueList.
846
0
    pub fn inst_results_list(&self, inst: Inst) -> ValueList {
847
0
        self.results[inst]
848
0
    }
849
850
    /// Get the call signature of a direct or indirect call instruction.
851
    /// Returns `None` if `inst` is not a call instruction.
852
43.2M
    pub fn call_signature(&self, inst: Inst) -> Option<SigRef> {
853
43.2M
        match self.insts[inst].analyze_call(&self.value_lists) {
854
40.5M
            CallInfo::NotACall => None,
855
602k
            CallInfo::Direct(f, _) => Some(self.ext_funcs[f].signature),
856
2.09M
            CallInfo::Indirect(s, _) => Some(s),
857
        }
858
43.2M
    }
859
860
    /// Check if `inst` is a branch.
861
50.6M
    pub fn analyze_branch(&self, inst: Inst) -> BranchInfo {
862
50.6M
        self.insts[inst].analyze_branch(&self.value_lists)
863
50.6M
    }
864
865
    /// Compute the type of an instruction result from opcode constraints and call signatures.
866
    ///
867
    /// This computes the same sequence of result types that `make_inst_results()` above would
868
    /// assign to the created result values, but it does not depend on `make_inst_results()` being
869
    /// called first.
870
    ///
871
    /// Returns `None` if asked about a result index that is too large.
872
35.1M
    pub fn compute_result_type(
873
35.1M
        &self,
874
35.1M
        inst: Inst,
875
35.1M
        result_idx: usize,
876
35.1M
        ctrl_typevar: Type,
877
35.1M
    ) -> Option<Type> {
878
35.1M
        let constraints = self.insts[inst].opcode().constraints();
879
35.1M
        let num_fixed_results = constraints.num_fixed_results();
880
35.1M
881
35.1M
        if result_idx < num_fixed_results {
882
14.8M
            return Some(constraints.result_type(result_idx, ctrl_typevar));
883
20.3M
        }
884
20.3M
885
20.3M
        // Not a fixed result, try to extract a return type from the call signature.
886
20.3M
        self.call_signature(inst).and_then(|sigref| {
887
1.54M
            self.signatures[sigref]
888
1.54M
                .returns
889
1.54M
                .get(result_idx - num_fixed_results)
890
1.54M
                .map(|&arg| arg.value_type)
891
20.3M
        })
892
35.1M
    }
893
894
    /// Get the controlling type variable, or `INVALID` if `inst` isn't polymorphic.
895
15.8M
    pub fn ctrl_typevar(&self, inst: Inst) -> Type {
896
15.8M
        let constraints = self[inst].opcode().constraints();
897
15.8M
898
15.8M
        if !constraints.is_polymorphic() {
899
730k
            types::INVALID
900
15.1M
        } else if constraints.requires_typevar_operand() {
901
            // Not all instruction formats have a designated operand, but in that case
902
            // `requires_typevar_operand()` should never be true.
903
2.75M
            self.value_type(
904
2.75M
                self[inst]
905
2.75M
                    .typevar_operand(&self.value_lists)
906
2.75M
                    .unwrap_or_else(|| {
907
0
                        panic!(
908
0
                            "Instruction format for {:?} doesn't have a designated operand",
909
0
                            self[inst]
910
0
                        )
911
2.75M
                    }),
912
2.75M
            )
913
        } else {
914
12.3M
            self.value_type(self.first_result(inst))
915
        }
916
15.8M
    }
917
}
918
919
/// Allow immutable access to instructions via indexing.
920
impl Index<Inst> for DataFlowGraph {
921
    type Output = InstructionData;
922
923
294M
    fn index(&self, inst: Inst) -> &InstructionData {
924
294M
        &self.insts[inst]
925
294M
    }
926
}
927
928
/// Allow mutable access to instructions via indexing.
929
impl IndexMut<Inst> for DataFlowGraph {
930
241k
    fn index_mut(&mut self, inst: Inst) -> &mut InstructionData {
931
241k
        &mut self.insts[inst]
932
241k
    }
933
}
934
935
/// basic blocks.
936
impl DataFlowGraph {
937
    /// Create a new basic block.
938
505k
    pub fn make_block(&mut self) -> Block {
939
505k
        self.blocks.push(BlockData::new())
940
505k
    }
941
942
    /// Get the number of parameters on `block`.
943
1.43M
    pub fn num_block_params(&self, block: Block) -> usize {
944
1.43M
        self.blocks[block].params.len(&self.value_lists)
945
1.43M
    }
946
947
    /// Get the parameters on `block`.
948
28.1M
    pub fn block_params(&self, block: Block) -> &[Value] {
949
28.1M
        self.blocks[block].params.as_slice(&self.value_lists)
950
28.1M
    }
951
952
    /// Get the types of the parameters on `block`.
953
8.60k
    pub fn block_param_types(&self, block: Block) -> impl Iterator<Item = Type> + '_ {
954
8.60k
        self.block_params(block).iter().map(|&v| self.value_type(v))
955
8.60k
    }
956
957
    /// Append a parameter with type `ty` to `block`.
958
665k
    pub fn append_block_param(&mut self, block: Block, ty: Type) -> Value {
959
665k
        let param = self.values.next_key();
960
665k
        let num = self.blocks[block].params.push(param, &mut self.value_lists);
961
665k
        debug_assert!(num <= u16::MAX as usize, "Too many parameters on block");
962
665k
        self.make_value(ValueData::Param {
963
665k
            ty,
964
665k
            num: num as u16,
965
665k
            block,
966
665k
        })
967
665k
    }
968
969
    /// Removes `val` from `block`'s parameters by swapping it with the last parameter on `block`.
970
    /// Returns the position of `val` before removal.
971
    ///
972
    /// *Important*: to ensure O(1) deletion, this method swaps the removed parameter with the
973
    /// last `block` parameter. This can disrupt all the branch instructions jumping to this
974
    /// `block` for which you have to change the branch argument order if necessary.
975
    ///
976
    /// Panics if `val` is not a block parameter.
977
0
    pub fn swap_remove_block_param(&mut self, val: Value) -> usize {
978
0
        let (block, num) =
979
0
            if let ValueData::Param { num, block, .. } = ValueData::from(self.values[val]) {
980
0
                (block, num)
981
            } else {
982
0
                panic!("{} must be a block parameter", val);
983
            };
984
0
        self.blocks[block]
985
0
            .params
986
0
            .swap_remove(num as usize, &mut self.value_lists);
987
0
        if let Some(last_arg_val) = self.blocks[block]
988
0
            .params
989
0
            .get(num as usize, &self.value_lists)
990
        {
991
            // We update the position of the old last arg.
992
0
            let mut last_arg_data = ValueData::from(self.values[last_arg_val]);
993
            if let ValueData::Param {
994
0
                num: ref mut old_num,
995
                ..
996
0
            } = &mut last_arg_data
997
0
            {
998
0
                *old_num = num;
999
0
                self.values[last_arg_val] = last_arg_data.into();
1000
0
            } else {
1001
0
                panic!("{} should be a Block parameter", last_arg_val);
1002
            }
1003
0
        }
1004
0
        num as usize
1005
0
    }
1006
1007
    /// Removes `val` from `block`'s parameters by a standard linear time list removal which
1008
    /// preserves ordering. Also updates the values' data.
1009
10.7k
    pub fn remove_block_param(&mut self, val: Value) {
1010
10.7k
        let (block, num) =
1011
10.7k
            if let ValueData::Param { num, block, .. } = ValueData::from(self.values[val]) {
1012
10.7k
                (block, num)
1013
            } else {
1014
0
                panic!("{} must be a block parameter", val);
1015
            };
1016
10.7k
        self.blocks[block]
1017
10.7k
            .params
1018
10.7k
            .remove(num as usize, &mut self.value_lists);
1019
10.7k
        for index in num..(self.num_block_params(block) as u16) {
1020
828
            let packed = &mut self.values[self.blocks[block]
1021
828
                .params
1022
828
                .get(index as usize, &self.value_lists)
1023
828
                .unwrap()];
1024
828
            let mut data = ValueData::from(*packed);
1025
828
            match &mut data {
1026
828
                ValueData::Param { ref mut num, .. } => {
1027
828
                    *num -= 1;
1028
828
                    *packed = data.into();
1029
828
                }
1030
0
                _ => panic!(
1031
0
                    "{} must be a block parameter",
1032
0
                    self.blocks[block]
1033
0
                        .params
1034
0
                        .get(index as usize, &self.value_lists)
1035
0
                        .unwrap()
1036
0
                ),
1037
            }
1038
        }
1039
10.7k
    }
1040
1041
    /// Append an existing value to `block`'s parameters.
1042
    ///
1043
    /// The appended value can't already be attached to something else.
1044
    ///
1045
    /// In almost all cases, you should be using `append_block_param()` instead of this method.
1046
0
    pub fn attach_block_param(&mut self, block: Block, param: Value) {
1047
0
        debug_assert!(!self.value_is_attached(param));
1048
0
        let num = self.blocks[block].params.push(param, &mut self.value_lists);
1049
0
        debug_assert!(num <= u16::MAX as usize, "Too many parameters on block");
1050
0
        let ty = self.value_type(param);
1051
0
        self.values[param] = ValueData::Param {
1052
0
            ty,
1053
0
            num: num as u16,
1054
0
            block,
1055
0
        }
1056
0
        .into();
1057
0
    }
1058
1059
    /// Replace a block parameter with a new value of type `ty`.
1060
    ///
1061
    /// The `old_value` must be an attached block parameter. It is removed from its place in the list
1062
    /// of parameters and replaced by a new value of type `new_type`. The new value gets the same
1063
    /// position in the list, and other parameters are not disturbed.
1064
    ///
1065
    /// The old value is left detached, so it should probably be changed into something else.
1066
    ///
1067
    /// Returns the new value.
1068
0
    pub fn replace_block_param(&mut self, old_value: Value, new_type: Type) -> Value {
1069
        // Create new value identical to the old one except for the type.
1070
0
        let (block, num) =
1071
0
            if let ValueData::Param { num, block, .. } = ValueData::from(self.values[old_value]) {
1072
0
                (block, num)
1073
            } else {
1074
0
                panic!("{} must be a block parameter", old_value);
1075
            };
1076
0
        let new_arg = self.make_value(ValueData::Param {
1077
0
            ty: new_type,
1078
0
            num,
1079
0
            block,
1080
0
        });
1081
0
1082
0
        self.blocks[block]
1083
0
            .params
1084
0
            .as_mut_slice(&mut self.value_lists)[num as usize] = new_arg;
1085
0
        new_arg
1086
0
    }
1087
1088
    /// Detach all the parameters from `block` and return them as a `ValueList`.
1089
    ///
1090
    /// This is a quite low-level operation. Sensible things to do with the detached block parameters
1091
    /// is to put them back on the same block with `attach_block_param()` or change them into aliases
1092
    /// with `change_to_alias()`.
1093
0
    pub fn detach_block_params(&mut self, block: Block) -> ValueList {
1094
0
        self.blocks[block].params.take()
1095
0
    }
1096
}
1097
1098
/// Contents of a basic block.
1099
///
1100
/// Parameters on a basic block are values that dominate everything in the block. All
1101
/// branches to this block must provide matching arguments, and the arguments to the entry block must
1102
/// match the function arguments.
1103
#[derive(Clone, PartialEq, Hash)]
1104
#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
1105
struct BlockData {
1106
    /// List of parameters to this block.
1107
    params: ValueList,
1108
}
1109
1110
impl BlockData {
1111
505k
    fn new() -> Self {
1112
505k
        Self {
1113
505k
            params: ValueList::new(),
1114
505k
        }
1115
505k
    }
1116
}
1117
1118
/// Object that can display an instruction.
1119
pub struct DisplayInst<'a>(&'a DataFlowGraph, Inst);
1120
1121
impl<'a> fmt::Display for DisplayInst<'a> {
1122
0
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1123
0
        let dfg = self.0;
1124
0
        let inst = self.1;
1125
1126
0
        if let Some((first, rest)) = dfg.inst_results(inst).split_first() {
1127
0
            write!(f, "{}", first)?;
1128
0
            for v in rest {
1129
0
                write!(f, ", {}", v)?;
1130
            }
1131
0
            write!(f, " = ")?;
1132
0
        }
1133
1134
0
        let typevar = dfg.ctrl_typevar(inst);
1135
0
        if typevar.is_invalid() {
1136
0
            write!(f, "{}", dfg[inst].opcode())?;
1137
        } else {
1138
0
            write!(f, "{}.{}", dfg[inst].opcode(), typevar)?;
1139
        }
1140
0
        write_operands(f, dfg, inst)
1141
0
    }
1142
}
1143
1144
/// Parser routines. These routines should not be used outside the parser.
1145
impl DataFlowGraph {
1146
    /// Set the type of a value. This is only for use in the parser, which needs
1147
    /// to create invalid values for index padding which may be reassigned later.
1148
    #[cold]
1149
0
    fn set_value_type_for_parser(&mut self, v: Value, t: Type) {
1150
0
        assert_eq!(
1151
0
            self.value_type(v),
1152
            types::INVALID,
1153
0
            "this function is only for assigning types to previously invalid values"
1154
        );
1155
0
        self.values[v].set_type(t);
1156
0
    }
1157
1158
    /// Check that the given concrete `Type` has been defined in the function.
1159
0
    pub fn check_dynamic_type(&mut self, ty: Type) -> Option<Type> {
1160
0
        debug_assert!(ty.is_dynamic_vector());
1161
0
        if self
1162
0
            .dynamic_types
1163
0
            .values()
1164
0
            .any(|dyn_ty_data| dyn_ty_data.concrete().unwrap() == ty)
1165
        {
1166
0
            Some(ty)
1167
        } else {
1168
0
            None
1169
        }
1170
0
    }
1171
1172
    /// Create result values for `inst`, reusing the provided detached values.
1173
    /// This is similar to `make_inst_results_reusing` except it's only for use
1174
    /// in the parser, which needs to reuse previously invalid values.
1175
    #[cold]
1176
0
    pub fn make_inst_results_for_parser(
1177
0
        &mut self,
1178
0
        inst: Inst,
1179
0
        ctrl_typevar: Type,
1180
0
        reuse: &[Value],
1181
0
    ) -> usize {
1182
        // Get the call signature if this is a function call.
1183
0
        if let Some(sig) = self.call_signature(inst) {
1184
0
            assert_eq!(
1185
0
                self.insts[inst].opcode().constraints().num_fixed_results(),
1186
0
                0
1187
0
            );
1188
0
            for res_idx in 0..self.signatures[sig].returns.len() {
1189
0
                let ty = self.signatures[sig].returns[res_idx].value_type;
1190
0
                if let Some(v) = reuse.get(res_idx) {
1191
0
                    self.set_value_type_for_parser(*v, ty);
1192
0
                }
1193
            }
1194
        } else {
1195
0
            let constraints = self.insts[inst].opcode().constraints();
1196
0
            for res_idx in 0..constraints.num_fixed_results() {
1197
0
                let ty = constraints.result_type(res_idx, ctrl_typevar);
1198
0
                if ty.is_dynamic_vector() {
1199
0
                    self.check_dynamic_type(ty)
1200
0
                        .unwrap_or_else(|| panic!("Use of undeclared dynamic type: {}", ty));
1201
0
                }
1202
0
                if let Some(v) = reuse.get(res_idx) {
1203
0
                    self.set_value_type_for_parser(*v, ty);
1204
0
                }
1205
            }
1206
        }
1207
1208
0
        self.make_inst_results_reusing(inst, ctrl_typevar, reuse.iter().map(|x| Some(*x)))
1209
0
    }
1210
1211
    /// Similar to `append_block_param`, append a parameter with type `ty` to
1212
    /// `block`, but using value `val`. This is only for use by the parser to
1213
    /// create parameters with specific values.
1214
    #[cold]
1215
0
    pub fn append_block_param_for_parser(&mut self, block: Block, ty: Type, val: Value) {
1216
0
        let num = self.blocks[block].params.push(val, &mut self.value_lists);
1217
0
        assert!(num <= u16::MAX as usize, "Too many parameters on block");
1218
0
        self.values[val] = ValueData::Param {
1219
0
            ty,
1220
0
            num: num as u16,
1221
0
            block,
1222
0
        }
1223
0
        .into();
1224
0
    }
1225
1226
    /// Create a new value alias. This is only for use by the parser to create
1227
    /// aliases with specific values, and the printer for testing.
1228
    #[cold]
1229
0
    pub fn make_value_alias_for_serialization(&mut self, src: Value, dest: Value) {
1230
0
        assert_ne!(src, Value::reserved_value());
1231
0
        assert_ne!(dest, Value::reserved_value());
1232
1233
0
        let ty = if self.values.is_valid(src) {
1234
0
            self.value_type(src)
1235
        } else {
1236
            // As a special case, if we can't resolve the aliasee yet, use INVALID
1237
            // temporarily. It will be resolved later in parsing.
1238
0
            types::INVALID
1239
        };
1240
0
        let data = ValueData::Alias { ty, original: src };
1241
0
        self.values[dest] = data.into();
1242
0
    }
1243
1244
    /// If `v` is already defined as an alias, return its destination value.
1245
    /// Otherwise return None. This allows the parser to coalesce identical
1246
    /// alias definitions, and the printer to identify an alias's immediate target.
1247
    #[cold]
1248
0
    pub fn value_alias_dest_for_serialization(&self, v: Value) -> Option<Value> {
1249
0
        if let ValueData::Alias { original, .. } = ValueData::from(self.values[v]) {
1250
0
            Some(original)
1251
        } else {
1252
0
            None
1253
        }
1254
0
    }
1255
1256
    /// Compute the type of an alias. This is only for use in the parser.
1257
    /// Returns false if an alias cycle was encountered.
1258
    #[cold]
1259
0
    pub fn set_alias_type_for_parser(&mut self, v: Value) -> bool {
1260
0
        if let Some(resolved) = maybe_resolve_aliases(&self.values, v) {
1261
0
            let old_ty = self.value_type(v);
1262
0
            let new_ty = self.value_type(resolved);
1263
0
            if old_ty == types::INVALID {
1264
0
                self.set_value_type_for_parser(v, new_ty);
1265
0
            } else {
1266
0
                assert_eq!(old_ty, new_ty);
1267
            }
1268
0
            true
1269
        } else {
1270
0
            false
1271
        }
1272
0
    }
1273
1274
    /// Create an invalid value, to pad the index space. This is only for use by
1275
    /// the parser to pad out the value index space.
1276
    #[cold]
1277
0
    pub fn make_invalid_value_for_parser(&mut self) {
1278
0
        let data = ValueData::Alias {
1279
0
            ty: types::INVALID,
1280
0
            original: Value::reserved_value(),
1281
0
        };
1282
0
        self.make_value(data);
1283
0
    }
1284
1285
    /// Check if a value reference is valid, while being aware of aliases which
1286
    /// may be unresolved while parsing.
1287
    #[cold]
1288
0
    pub fn value_is_valid_for_parser(&self, v: Value) -> bool {
1289
0
        if !self.value_is_valid(v) {
1290
0
            return false;
1291
0
        }
1292
0
        if let ValueData::Alias { ty, .. } = ValueData::from(self.values[v]) {
1293
0
            ty != types::INVALID
1294
        } else {
1295
0
            true
1296
        }
1297
0
    }
1298
}
1299
1300
#[cfg(test)]
1301
mod tests {
1302
    use super::*;
1303
    use crate::cursor::{Cursor, FuncCursor};
1304
    use crate::ir::types;
1305
    use crate::ir::{Function, InstructionData, Opcode, TrapCode};
1306
    use alloc::string::ToString;
1307
1308
    #[test]
1309
    fn make_inst() {
1310
        let mut dfg = DataFlowGraph::new();
1311
1312
        let idata = InstructionData::UnaryImm {
1313
            opcode: Opcode::Iconst,
1314
            imm: 0.into(),
1315
        };
1316
        let inst = dfg.make_inst(idata);
1317
1318
        dfg.make_inst_results(inst, types::I32);
1319
        assert_eq!(inst.to_string(), "inst0");
1320
        assert_eq!(dfg.display_inst(inst).to_string(), "v0 = iconst.i32 0");
1321
1322
        // Immutable reference resolution.
1323
        {
1324
            let immdfg = &dfg;
1325
            let ins = &immdfg[inst];
1326
            assert_eq!(ins.opcode(), Opcode::Iconst);
1327
        }
1328
1329
        // Results.
1330
        let val = dfg.first_result(inst);
1331
        assert_eq!(dfg.inst_results(inst), &[val]);
1332
1333
        assert_eq!(dfg.value_def(val), ValueDef::Result(inst, 0));
1334
        assert_eq!(dfg.value_type(val), types::I32);
1335
1336
        // Replacing results.
1337
        assert!(dfg.value_is_attached(val));
1338
        let v2 = dfg.replace_result(val, types::F64);
1339
        assert!(!dfg.value_is_attached(val));
1340
        assert!(dfg.value_is_attached(v2));
1341
        assert_eq!(dfg.inst_results(inst), &[v2]);
1342
        assert_eq!(dfg.value_def(v2), ValueDef::Result(inst, 0));
1343
        assert_eq!(dfg.value_type(v2), types::F64);
1344
    }
1345
1346
    #[test]
1347
    fn no_results() {
1348
        let mut dfg = DataFlowGraph::new();
1349
1350
        let idata = InstructionData::Trap {
1351
            opcode: Opcode::Trap,
1352
            code: TrapCode::User(0),
1353
        };
1354
        let inst = dfg.make_inst(idata);
1355
        assert_eq!(dfg.display_inst(inst).to_string(), "trap user0");
1356
1357
        // Result slice should be empty.
1358
        assert_eq!(dfg.inst_results(inst), &[]);
1359
    }
1360
1361
    #[test]
1362
    fn block() {
1363
        let mut dfg = DataFlowGraph::new();
1364
1365
        let block = dfg.make_block();
1366
        assert_eq!(block.to_string(), "block0");
1367
        assert_eq!(dfg.num_block_params(block), 0);
1368
        assert_eq!(dfg.block_params(block), &[]);
1369
        assert!(dfg.detach_block_params(block).is_empty());
1370
        assert_eq!(dfg.num_block_params(block), 0);
1371
        assert_eq!(dfg.block_params(block), &[]);
1372
1373
        let arg1 = dfg.append_block_param(block, types::F32);
1374
        assert_eq!(arg1.to_string(), "v0");
1375
        assert_eq!(dfg.num_block_params(block), 1);
1376
        assert_eq!(dfg.block_params(block), &[arg1]);
1377
1378
        let arg2 = dfg.append_block_param(block, types::I16);
1379
        assert_eq!(arg2.to_string(), "v1");
1380
        assert_eq!(dfg.num_block_params(block), 2);
1381
        assert_eq!(dfg.block_params(block), &[arg1, arg2]);
1382
1383
        assert_eq!(dfg.value_def(arg1), ValueDef::Param(block, 0));
1384
        assert_eq!(dfg.value_def(arg2), ValueDef::Param(block, 1));
1385
        assert_eq!(dfg.value_type(arg1), types::F32);
1386
        assert_eq!(dfg.value_type(arg2), types::I16);
1387
1388
        // Swap the two block parameters.
1389
        let vlist = dfg.detach_block_params(block);
1390
        assert_eq!(dfg.num_block_params(block), 0);
1391
        assert_eq!(dfg.block_params(block), &[]);
1392
        assert_eq!(vlist.as_slice(&dfg.value_lists), &[arg1, arg2]);
1393
        dfg.attach_block_param(block, arg2);
1394
        let arg3 = dfg.append_block_param(block, types::I32);
1395
        dfg.attach_block_param(block, arg1);
1396
        assert_eq!(dfg.block_params(block), &[arg2, arg3, arg1]);
1397
    }
1398
1399
    #[test]
1400
    fn replace_block_params() {
1401
        let mut dfg = DataFlowGraph::new();
1402
1403
        let block = dfg.make_block();
1404
        let arg1 = dfg.append_block_param(block, types::F32);
1405
1406
        let new1 = dfg.replace_block_param(arg1, types::I64);
1407
        assert_eq!(dfg.value_type(arg1), types::F32);
1408
        assert_eq!(dfg.value_type(new1), types::I64);
1409
        assert_eq!(dfg.block_params(block), &[new1]);
1410
1411
        dfg.attach_block_param(block, arg1);
1412
        assert_eq!(dfg.block_params(block), &[new1, arg1]);
1413
1414
        let new2 = dfg.replace_block_param(arg1, types::I8);
1415
        assert_eq!(dfg.value_type(arg1), types::F32);
1416
        assert_eq!(dfg.value_type(new2), types::I8);
1417
        assert_eq!(dfg.block_params(block), &[new1, new2]);
1418
1419
        dfg.attach_block_param(block, arg1);
1420
        assert_eq!(dfg.block_params(block), &[new1, new2, arg1]);
1421
1422
        let new3 = dfg.replace_block_param(new2, types::I16);
1423
        assert_eq!(dfg.value_type(new1), types::I64);
1424
        assert_eq!(dfg.value_type(new2), types::I8);
1425
        assert_eq!(dfg.value_type(new3), types::I16);
1426
        assert_eq!(dfg.block_params(block), &[new1, new3, arg1]);
1427
    }
1428
1429
    #[test]
1430
    fn swap_remove_block_params() {
1431
        let mut dfg = DataFlowGraph::new();
1432
1433
        let block = dfg.make_block();
1434
        let arg1 = dfg.append_block_param(block, types::F32);
1435
        let arg2 = dfg.append_block_param(block, types::F32);
1436
        let arg3 = dfg.append_block_param(block, types::F32);
1437
        assert_eq!(dfg.block_params(block), &[arg1, arg2, arg3]);
1438
1439
        dfg.swap_remove_block_param(arg1);
1440
        assert_eq!(dfg.value_is_attached(arg1), false);
1441
        assert_eq!(dfg.value_is_attached(arg2), true);
1442
        assert_eq!(dfg.value_is_attached(arg3), true);
1443
        assert_eq!(dfg.block_params(block), &[arg3, arg2]);
1444
        dfg.swap_remove_block_param(arg2);
1445
        assert_eq!(dfg.value_is_attached(arg2), false);
1446
        assert_eq!(dfg.value_is_attached(arg3), true);
1447
        assert_eq!(dfg.block_params(block), &[arg3]);
1448
        dfg.swap_remove_block_param(arg3);
1449
        assert_eq!(dfg.value_is_attached(arg3), false);
1450
        assert_eq!(dfg.block_params(block), &[]);
1451
    }
1452
1453
    #[test]
1454
    fn aliases() {
1455
        use crate::ir::InstBuilder;
1456
1457
        let mut func = Function::new();
1458
        let block0 = func.dfg.make_block();
1459
        let mut pos = FuncCursor::new(&mut func);
1460
        pos.insert_block(block0);
1461
1462
        // Build a little test program.
1463
        let v1 = pos.ins().iconst(types::I32, 42);
1464
1465
        // Make sure we can resolve value aliases even when values is empty.
1466
        assert_eq!(pos.func.dfg.resolve_aliases(v1), v1);
1467
1468
        let arg0 = pos.func.dfg.append_block_param(block0, types::I32);
1469
        let (s, c) = pos.ins().iadd_ifcout(v1, arg0);
1470
        let iadd = match pos.func.dfg.value_def(s) {
1471
            ValueDef::Result(i, 0) => i,
1472
            _ => panic!(),
1473
        };
1474
1475
        // Remove `c` from the result list.
1476
        pos.func.dfg.clear_results(iadd);
1477
        pos.func.dfg.attach_result(iadd, s);
1478
1479
        // Replace `iadd_ifcout` with a normal `iadd` and an `ifcmp`.
1480
        pos.func.dfg.replace(iadd).iadd(v1, arg0);
1481
        let c2 = pos.ins().ifcmp(s, v1);
1482
        pos.func.dfg.change_to_alias(c, c2);
1483
1484
        assert_eq!(pos.func.dfg.resolve_aliases(c2), c2);
1485
        assert_eq!(pos.func.dfg.resolve_aliases(c), c2);
1486
    }
1487
}