/src/wasmtime/crates/cranelift/src/trap.rs
Line | Count | Source |
1 | | use crate::TRAP_INTERNAL_ASSERT; |
2 | | use crate::alias_region::AliasRegions; |
3 | | use crate::compiler::Compiler; |
4 | | use cranelift_codegen::cursor::FuncCursor; |
5 | | use cranelift_codegen::ir::condcodes::IntCC; |
6 | | use cranelift_codegen::ir::types::I8; |
7 | | use cranelift_codegen::ir::{self, InstBuilder}; |
8 | | use cranelift_frontend::FunctionBuilder; |
9 | | use wasmtime_environ::{BuiltinFunctionIndex, GetPtrSize, TripleExt}; |
10 | | |
11 | | /// Helper trait to share translation of traps between core functions and |
12 | | /// component trampolines. |
13 | | /// |
14 | | /// Traps are conditionally performed as libcalls when signals-based-traps are |
15 | | /// disabled, for example, but otherwise use the native CLIF `trap` instruction. |
16 | | pub trait TranslateTrap<Offsets> |
17 | | where |
18 | | Offsets: GetPtrSize, |
19 | | { |
20 | | fn compiler(&self) -> &Compiler; |
21 | | |
22 | | fn vmctx_val(&mut self, cursor: &mut FuncCursor<'_>) -> ir::Value; |
23 | | |
24 | | fn alias_regions(&mut self) -> &mut AliasRegions<Offsets>; |
25 | | |
26 | | fn builtin_funcref( |
27 | | &mut self, |
28 | | builder: &mut FunctionBuilder<'_>, |
29 | | index: BuiltinFunctionIndex, |
30 | | ) -> ir::FuncRef; |
31 | | |
32 | 23.0k | fn debug_tags(&self, _srcloc: ir::SourceLoc) -> Vec<ir::DebugTag> { |
33 | 23.0k | vec![] |
34 | 23.0k | } |
35 | | |
36 | 2.75M | fn trap(&mut self, builder: &mut FunctionBuilder, trap: ir::TrapCode) { |
37 | | match ( |
38 | 2.75M | self.clif_instruction_traps_enabled(), |
39 | 2.75M | crate::clif_trap_to_env_trap(trap, self.compiler().tunables()), |
40 | | ) { |
41 | | // If libcall traps are disabled or there's no wasmtime-defined trap |
42 | | // code for this, then emit a native trap instruction. |
43 | 835k | (true, _) | (_, None) => { |
44 | 835k | builder.ins().trap(trap); |
45 | 835k | } |
46 | | // ... otherwise with libcall traps explicitly enabled and a |
47 | | // wasmtime-based trap code invoke the libcall to raise a trap and |
48 | | // pass in our trap code. Leave a debug `unreachable` in place |
49 | | // afterwards as a defense-in-depth measure. |
50 | 1.91M | (false, Some(trap)) => { |
51 | 1.91M | let debug_tags = self.debug_tags(builder.srcloc()); |
52 | 1.91M | let trap_libcall = self.builtin_funcref(builder, BuiltinFunctionIndex::trap()); |
53 | 1.91M | let vmctx = self.vmctx_val(&mut builder.cursor()); |
54 | 1.91M | let trap_code = builder.ins().iconst(I8, i64::from(trap.as_u8())); |
55 | 1.91M | builder.ins().call(trap_libcall, &[vmctx, trap_code]); |
56 | 1.91M | let raise_libcall = self.builtin_funcref(builder, BuiltinFunctionIndex::raise()); |
57 | 1.91M | let inst = builder.ins().call(raise_libcall, &[vmctx]); |
58 | 1.91M | builder.func.debug_tags.set(inst, debug_tags); |
59 | 1.91M | builder.ins().trap(TRAP_INTERNAL_ASSERT); |
60 | 1.91M | } |
61 | | } |
62 | 2.75M | } <wasmtime_internal_cranelift::func_environ::FuncEnvironment as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::vmoffsets::VMOffsets<u8>>>::trap Line | Count | Source | 36 | 2.72M | fn trap(&mut self, builder: &mut FunctionBuilder, trap: ir::TrapCode) { | 37 | | match ( | 38 | 2.72M | self.clif_instruction_traps_enabled(), | 39 | 2.72M | crate::clif_trap_to_env_trap(trap, self.compiler().tunables()), | 40 | | ) { | 41 | | // If libcall traps are disabled or there's no wasmtime-defined trap | 42 | | // code for this, then emit a native trap instruction. | 43 | 828k | (true, _) | (_, None) => { | 44 | 828k | builder.ins().trap(trap); | 45 | 828k | } | 46 | | // ... otherwise with libcall traps explicitly enabled and a | 47 | | // wasmtime-based trap code invoke the libcall to raise a trap and | 48 | | // pass in our trap code. Leave a debug `unreachable` in place | 49 | | // afterwards as a defense-in-depth measure. | 50 | 1.89M | (false, Some(trap)) => { | 51 | 1.89M | let debug_tags = self.debug_tags(builder.srcloc()); | 52 | 1.89M | let trap_libcall = self.builtin_funcref(builder, BuiltinFunctionIndex::trap()); | 53 | 1.89M | let vmctx = self.vmctx_val(&mut builder.cursor()); | 54 | 1.89M | let trap_code = builder.ins().iconst(I8, i64::from(trap.as_u8())); | 55 | 1.89M | builder.ins().call(trap_libcall, &[vmctx, trap_code]); | 56 | 1.89M | let raise_libcall = self.builtin_funcref(builder, BuiltinFunctionIndex::raise()); | 57 | 1.89M | let inst = builder.ins().call(raise_libcall, &[vmctx]); | 58 | 1.89M | builder.func.debug_tags.set(inst, debug_tags); | 59 | 1.89M | builder.ins().trap(TRAP_INTERNAL_ASSERT); | 60 | 1.89M | } | 61 | | } | 62 | 2.72M | } |
<wasmtime_internal_cranelift::compiler::component::TrapTranslator<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>> as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>>>::trap Line | Count | Source | 36 | 29.8k | fn trap(&mut self, builder: &mut FunctionBuilder, trap: ir::TrapCode) { | 37 | | match ( | 38 | 29.8k | self.clif_instruction_traps_enabled(), | 39 | 29.8k | crate::clif_trap_to_env_trap(trap, self.compiler().tunables()), | 40 | | ) { | 41 | | // If libcall traps are disabled or there's no wasmtime-defined trap | 42 | | // code for this, then emit a native trap instruction. | 43 | 6.77k | (true, _) | (_, None) => { | 44 | 6.77k | builder.ins().trap(trap); | 45 | 6.77k | } | 46 | | // ... otherwise with libcall traps explicitly enabled and a | 47 | | // wasmtime-based trap code invoke the libcall to raise a trap and | 48 | | // pass in our trap code. Leave a debug `unreachable` in place | 49 | | // afterwards as a defense-in-depth measure. | 50 | 23.0k | (false, Some(trap)) => { | 51 | 23.0k | let debug_tags = self.debug_tags(builder.srcloc()); | 52 | 23.0k | let trap_libcall = self.builtin_funcref(builder, BuiltinFunctionIndex::trap()); | 53 | 23.0k | let vmctx = self.vmctx_val(&mut builder.cursor()); | 54 | 23.0k | let trap_code = builder.ins().iconst(I8, i64::from(trap.as_u8())); | 55 | 23.0k | builder.ins().call(trap_libcall, &[vmctx, trap_code]); | 56 | 23.0k | let raise_libcall = self.builtin_funcref(builder, BuiltinFunctionIndex::raise()); | 57 | 23.0k | let inst = builder.ins().call(raise_libcall, &[vmctx]); | 58 | 23.0k | builder.func.debug_tags.set(inst, debug_tags); | 59 | 23.0k | builder.ins().trap(TRAP_INTERNAL_ASSERT); | 60 | 23.0k | } | 61 | | } | 62 | 29.8k | } |
|
63 | | |
64 | 408k | fn trapz(&mut self, builder: &mut FunctionBuilder, value: ir::Value, trap: ir::TrapCode) { |
65 | 408k | if self.clif_instruction_traps_enabled() { |
66 | 237k | builder.ins().trapz(value, trap); |
67 | 237k | } else { |
68 | 170k | let ty = builder.func.dfg.value_type(value); |
69 | 170k | let zero = builder.ins().iconst(ty, 0); |
70 | 170k | let cmp = builder.ins().icmp(IntCC::Equal, value, zero); |
71 | 170k | self.conditionally_trap(builder, cmp, trap); |
72 | 170k | } |
73 | 408k | } <wasmtime_internal_cranelift::func_environ::FuncEnvironment as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::vmoffsets::VMOffsets<u8>>>::trapz Line | Count | Source | 64 | 381k | fn trapz(&mut self, builder: &mut FunctionBuilder, value: ir::Value, trap: ir::TrapCode) { | 65 | 381k | if self.clif_instruction_traps_enabled() { | 66 | 226k | builder.ins().trapz(value, trap); | 67 | 226k | } else { | 68 | 154k | let ty = builder.func.dfg.value_type(value); | 69 | 154k | let zero = builder.ins().iconst(ty, 0); | 70 | 154k | let cmp = builder.ins().icmp(IntCC::Equal, value, zero); | 71 | 154k | self.conditionally_trap(builder, cmp, trap); | 72 | 154k | } | 73 | 381k | } |
<wasmtime_internal_cranelift::compiler::component::TrapTranslator<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>> as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>>>::trapz Line | Count | Source | 64 | 26.7k | fn trapz(&mut self, builder: &mut FunctionBuilder, value: ir::Value, trap: ir::TrapCode) { | 65 | 26.7k | if self.clif_instruction_traps_enabled() { | 66 | 11.1k | builder.ins().trapz(value, trap); | 67 | 15.5k | } else { | 68 | 15.5k | let ty = builder.func.dfg.value_type(value); | 69 | 15.5k | let zero = builder.ins().iconst(ty, 0); | 70 | 15.5k | let cmp = builder.ins().icmp(IntCC::Equal, value, zero); | 71 | 15.5k | self.conditionally_trap(builder, cmp, trap); | 72 | 15.5k | } | 73 | 26.7k | } |
|
74 | | |
75 | 1.68M | fn trapnz(&mut self, builder: &mut FunctionBuilder, value: ir::Value, trap: ir::TrapCode) { |
76 | 1.68M | if self.clif_instruction_traps_enabled() { |
77 | 624k | builder.ins().trapnz(value, trap); |
78 | 1.06M | } else { |
79 | 1.06M | let ty = builder.func.dfg.value_type(value); |
80 | 1.06M | let zero = builder.ins().iconst(ty, 0); |
81 | 1.06M | let cmp = builder.ins().icmp(IntCC::NotEqual, value, zero); |
82 | 1.06M | self.conditionally_trap(builder, cmp, trap); |
83 | 1.06M | } |
84 | 1.68M | } <wasmtime_internal_cranelift::func_environ::FuncEnvironment as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::vmoffsets::VMOffsets<u8>>>::trapnz Line | Count | Source | 75 | 1.68M | fn trapnz(&mut self, builder: &mut FunctionBuilder, value: ir::Value, trap: ir::TrapCode) { | 76 | 1.68M | if self.clif_instruction_traps_enabled() { | 77 | 624k | builder.ins().trapnz(value, trap); | 78 | 1.06M | } else { | 79 | 1.06M | let ty = builder.func.dfg.value_type(value); | 80 | 1.06M | let zero = builder.ins().iconst(ty, 0); | 81 | 1.06M | let cmp = builder.ins().icmp(IntCC::NotEqual, value, zero); | 82 | 1.06M | self.conditionally_trap(builder, cmp, trap); | 83 | 1.06M | } | 84 | 1.68M | } |
Unexecuted instantiation: <wasmtime_internal_cranelift::compiler::component::TrapTranslator<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>> as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>>>::trapnz |
85 | | |
86 | 1.05M | fn uadd_overflow_trap( |
87 | 1.05M | &mut self, |
88 | 1.05M | builder: &mut FunctionBuilder, |
89 | 1.05M | lhs: ir::Value, |
90 | 1.05M | rhs: ir::Value, |
91 | 1.05M | trap: ir::TrapCode, |
92 | 1.05M | ) -> ir::Value { |
93 | 1.05M | if self.clif_instruction_traps_enabled() { |
94 | 480k | builder.ins().uadd_overflow_trap(lhs, rhs, trap) |
95 | | } else { |
96 | 575k | let (ret, overflow) = builder.ins().uadd_overflow(lhs, rhs); |
97 | 575k | self.conditionally_trap(builder, overflow, trap); |
98 | 575k | ret |
99 | | } |
100 | 1.05M | } |
101 | | |
102 | | /// Helper to emit a conditional trap based on `trap_cond`. |
103 | | /// |
104 | | /// This should only be used if `self.clif_instruction_traps_enabled()` is |
105 | | /// false, otherwise native CLIF instructions should be used instead. |
106 | 2.11M | fn conditionally_trap( |
107 | 2.11M | &mut self, |
108 | 2.11M | builder: &mut FunctionBuilder, |
109 | 2.11M | trap_cond: ir::Value, |
110 | 2.11M | trap: ir::TrapCode, |
111 | 2.11M | ) { |
112 | 2.11M | assert!(!self.clif_instruction_traps_enabled()); |
113 | | |
114 | 2.11M | let trap_block = builder.create_block(); |
115 | 2.11M | builder.set_cold_block(trap_block); |
116 | 2.11M | let continuation_block = builder.create_block(); |
117 | | |
118 | 2.11M | builder |
119 | 2.11M | .ins() |
120 | 2.11M | .brif(trap_cond, trap_block, &[], continuation_block, &[]); |
121 | | |
122 | 2.11M | builder.seal_block(trap_block); |
123 | 2.11M | builder.seal_block(continuation_block); |
124 | | |
125 | 2.11M | builder.switch_to_block(trap_block); |
126 | 2.11M | self.trap(builder, trap); |
127 | 2.11M | builder.switch_to_block(continuation_block); |
128 | 2.11M | } <wasmtime_internal_cranelift::func_environ::FuncEnvironment as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::vmoffsets::VMOffsets<u8>>>::conditionally_trap Line | Count | Source | 106 | 2.10M | fn conditionally_trap( | 107 | 2.10M | &mut self, | 108 | 2.10M | builder: &mut FunctionBuilder, | 109 | 2.10M | trap_cond: ir::Value, | 110 | 2.10M | trap: ir::TrapCode, | 111 | 2.10M | ) { | 112 | 2.10M | assert!(!self.clif_instruction_traps_enabled()); | 113 | | | 114 | 2.10M | let trap_block = builder.create_block(); | 115 | 2.10M | builder.set_cold_block(trap_block); | 116 | 2.10M | let continuation_block = builder.create_block(); | 117 | | | 118 | 2.10M | builder | 119 | 2.10M | .ins() | 120 | 2.10M | .brif(trap_cond, trap_block, &[], continuation_block, &[]); | 121 | | | 122 | 2.10M | builder.seal_block(trap_block); | 123 | 2.10M | builder.seal_block(continuation_block); | 124 | | | 125 | 2.10M | builder.switch_to_block(trap_block); | 126 | 2.10M | self.trap(builder, trap); | 127 | 2.10M | builder.switch_to_block(continuation_block); | 128 | 2.10M | } |
<wasmtime_internal_cranelift::compiler::component::TrapTranslator<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>> as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>>>::conditionally_trap Line | Count | Source | 106 | 15.5k | fn conditionally_trap( | 107 | 15.5k | &mut self, | 108 | 15.5k | builder: &mut FunctionBuilder, | 109 | 15.5k | trap_cond: ir::Value, | 110 | 15.5k | trap: ir::TrapCode, | 111 | 15.5k | ) { | 112 | 15.5k | assert!(!self.clif_instruction_traps_enabled()); | 113 | | | 114 | 15.5k | let trap_block = builder.create_block(); | 115 | 15.5k | builder.set_cold_block(trap_block); | 116 | 15.5k | let continuation_block = builder.create_block(); | 117 | | | 118 | 15.5k | builder | 119 | 15.5k | .ins() | 120 | 15.5k | .brif(trap_cond, trap_block, &[], continuation_block, &[]); | 121 | | | 122 | 15.5k | builder.seal_block(trap_block); | 123 | 15.5k | builder.seal_block(continuation_block); | 124 | | | 125 | 15.5k | builder.switch_to_block(trap_block); | 126 | 15.5k | self.trap(builder, trap); | 127 | 15.5k | builder.switch_to_block(continuation_block); | 128 | 15.5k | } |
|
129 | | |
130 | | /// Returns whether it's acceptable to have CLIF instructions natively trap, |
131 | | /// such as division-by-zero. |
132 | | /// |
133 | | /// This is enabled if `signals_based_traps` is `true` or on |
134 | | /// Pulley unconditionally since Pulley doesn't use hardware-based |
135 | | /// traps in its runtime. However, if guest debugging is enabled, |
136 | | /// then we cannot rely on Pulley traps and still need a libcall |
137 | | /// to gain proper ownership of the store in the runtime's |
138 | | /// debugger hooks. |
139 | 8.34M | fn clif_instruction_traps_enabled(&self) -> bool { |
140 | 8.34M | let tunables = self.compiler().tunables(); |
141 | 8.34M | tunables.signals_based_traps || (self.is_pulley() && !tunables.debug_guest) |
142 | 8.34M | } <wasmtime_internal_cranelift::func_environ::FuncEnvironment as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::vmoffsets::VMOffsets<u8>>>::clif_instruction_traps_enabled Line | Count | Source | 139 | 8.27M | fn clif_instruction_traps_enabled(&self) -> bool { | 140 | 8.27M | let tunables = self.compiler().tunables(); | 141 | 8.27M | tunables.signals_based_traps || (self.is_pulley() && !tunables.debug_guest) | 142 | 8.27M | } |
<wasmtime_internal_cranelift::compiler::component::TrapTranslator<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>> as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>>>::clif_instruction_traps_enabled Line | Count | Source | 139 | 72.1k | fn clif_instruction_traps_enabled(&self) -> bool { | 140 | 72.1k | let tunables = self.compiler().tunables(); | 141 | 72.1k | tunables.signals_based_traps || (self.is_pulley() && !tunables.debug_guest) | 142 | 72.1k | } |
|
143 | | |
144 | | /// Returns whether translation is happening for Pulley bytecode. |
145 | 10.3M | fn is_pulley(&self) -> bool { |
146 | 10.3M | self.compiler().isa().triple().is_pulley() |
147 | 10.3M | } <wasmtime_internal_cranelift::func_environ::FuncEnvironment as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::vmoffsets::VMOffsets<u8>>>::is_pulley Line | Count | Source | 145 | 10.2M | fn is_pulley(&self) -> bool { | 146 | 10.2M | self.compiler().isa().triple().is_pulley() | 147 | 10.2M | } |
<wasmtime_internal_cranelift::compiler::component::TrapTranslator<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>> as wasmtime_internal_cranelift::trap::TranslateTrap<wasmtime_environ::component::vmcomponent_offsets::VMComponentOffsets<u8>>>::is_pulley Line | Count | Source | 145 | 57.7k | fn is_pulley(&self) -> bool { | 146 | 57.7k | self.compiler().isa().triple().is_pulley() | 147 | 57.7k | } |
|
148 | | } |