/src/wasmtime/fuzz/fuzz_targets/differential.rs
Line | Count | Source |
1 | | #![no_main] |
2 | | |
3 | | use libfuzzer_sys::arbitrary::{self, Result, Unstructured}; |
4 | | use libfuzzer_sys::fuzz_target; |
5 | | use std::sync::atomic::AtomicUsize; |
6 | | use std::sync::atomic::Ordering::SeqCst; |
7 | | use std::sync::{Mutex, Once}; |
8 | | use wasmtime_fuzzing::generators::{Config, DiffValue, DiffValueType, SingleInstModule}; |
9 | | use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance; |
10 | | use wasmtime_fuzzing::oracles::engine::{build_allowed_env_list, parse_env_list}; |
11 | | use wasmtime_fuzzing::oracles::{DiffEqResult, differential, engine, log_wasm}; |
12 | | |
13 | | // Upper limit on the number of invocations for each WebAssembly function |
14 | | // executed by this fuzz target. |
15 | | const NUM_INVOCATIONS: usize = 5; |
16 | | |
17 | | // Only run once when the fuzz target loads. |
18 | | static SETUP: Once = Once::new(); |
19 | | |
20 | | // Environment-specified configuration for controlling the kinds of engines and |
21 | | // modules used by this fuzz target. E.g.: |
22 | | // - ALLOWED_ENGINES=wasmi,spec cargo +nightly fuzz run ... |
23 | | // - ALLOWED_ENGINES=-v8 cargo +nightly fuzz run ... |
24 | | // - ALLOWED_MODULES=single-inst cargo +nightly fuzz run ... |
25 | | static ALLOWED_ENGINES: Mutex<Vec<Option<&str>>> = Mutex::new(vec![]); |
26 | | static ALLOWED_MODULES: Mutex<Vec<Option<&str>>> = Mutex::new(vec![]); |
27 | | |
28 | | // Statistics about what's actually getting executed during fuzzing |
29 | | static STATS: RuntimeStats = RuntimeStats::new(); |
30 | | |
31 | | fuzz_target!(|data: &[u8]| { |
32 | 1 | SETUP.call_once(|| { |
33 | | // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on |
34 | | // `setup_ocaml_runtime`. |
35 | 1 | engine::setup_engine_runtimes(); |
36 | | |
37 | | // Retrieve the configuration for this fuzz target from `ALLOWED_*` |
38 | | // environment variables. |
39 | 1 | let allowed_engines = build_allowed_env_list( |
40 | 1 | parse_env_list("ALLOWED_ENGINES"), |
41 | 1 | &["wasmtime", "wasmi", "spec", "v8", "winch", "pulley"], |
42 | | ); |
43 | 1 | let allowed_modules = build_allowed_env_list( |
44 | 1 | parse_env_list("ALLOWED_MODULES"), |
45 | 1 | &["wasm-smith", "single-inst"], |
46 | | ); |
47 | | |
48 | 1 | *ALLOWED_ENGINES.lock().unwrap() = allowed_engines; |
49 | 1 | *ALLOWED_MODULES.lock().unwrap() = allowed_modules; |
50 | 1 | }); |
51 | | |
52 | | // Errors in `run` have to do with not enough input in `data`, which we |
53 | | // ignore here since it doesn't affect how we'd like to fuzz. |
54 | | let _ = execute_one(&data); |
55 | | }); |
56 | | |
57 | 12.4k | fn execute_one(data: &[u8]) -> Result<()> { |
58 | 12.4k | wasmtime_fuzzing::init_fuzzing(); |
59 | 12.4k | STATS.bump_attempts(); |
60 | | |
61 | 12.4k | let mut u = Unstructured::new(data); |
62 | | |
63 | | // Generate a Wasmtime and module configuration and update its settings |
64 | | // initially to be suitable for differential execution where the generated |
65 | | // wasm will behave the same in two different engines. This will get further |
66 | | // refined below. |
67 | 12.4k | let mut config: Config = u.arbitrary()?; |
68 | 12.2k | config.set_differential_config(); |
69 | | |
70 | 12.2k | let allowed_engines = ALLOWED_ENGINES.lock().unwrap(); |
71 | 12.2k | let allowed_modules = ALLOWED_MODULES.lock().unwrap(); |
72 | | |
73 | | // Choose an engine that Wasmtime will be differentially executed against. |
74 | | // The chosen engine is then created, which might update `config`, and |
75 | | // returned as a trait object. |
76 | 12.2k | let lhs = match *u.choose(&allowed_engines)? { |
77 | 12.2k | Some(engine) => engine, |
78 | | None => { |
79 | 0 | log::debug!("test case uses a runtime-disabled engine"); |
80 | 0 | return Ok(()); |
81 | | } |
82 | | }; |
83 | | |
84 | 12.2k | log::trace!("Building LHS engine"); |
85 | 12.2k | let mut lhs = match engine::build(&mut u, lhs, &mut config)? { |
86 | 12.0k | Some(engine) => engine, |
87 | | // The chosen engine does not have support compiled into the fuzzer, |
88 | | // discard this test case. |
89 | 0 | None => return Ok(()), |
90 | | }; |
91 | 12.0k | log::debug!("lhs engine: {}", lhs.name()); |
92 | | |
93 | | // Using the now-legalized module configuration generate the Wasm module; |
94 | | // this is specified by either the ALLOWED_MODULES environment variable or a |
95 | | // random selection between wasm-smith and single-inst. |
96 | 12.0k | let build_wasm_smith_module = |u: &mut Unstructured, config: &Config| -> Result<_> { |
97 | 9.59k | log::debug!("build wasm-smith with {config:?}"); |
98 | 9.59k | STATS.wasm_smith_modules.fetch_add(1, SeqCst); |
99 | 9.59k | let module = config.generate(u, Some(1000))?; |
100 | 9.55k | Ok(module.to_bytes()) |
101 | 9.59k | }; |
102 | 12.0k | let build_single_inst_module = |u: &mut Unstructured, config: &Config| -> Result<_> { |
103 | 2.48k | log::debug!("build single-inst with {config:?}"); |
104 | 2.48k | STATS.single_instruction_modules.fetch_add(1, SeqCst); |
105 | 2.48k | let module = SingleInstModule::new(u, &config.module_config)?; |
106 | 2.48k | Ok(module.to_bytes()) |
107 | 2.48k | }; |
108 | 12.0k | if allowed_modules.is_empty() { |
109 | 0 | panic!("unable to generate a module to fuzz against; check `ALLOWED_MODULES`") |
110 | 12.0k | } |
111 | 12.0k | let wasm = match *u.choose(&allowed_modules)? { |
112 | 12.0k | Some("wasm-smith") => build_wasm_smith_module(&mut u, &config)?, |
113 | 2.48k | Some("single-inst") => build_single_inst_module(&mut u, &config)?, |
114 | | None => { |
115 | 0 | log::debug!("test case uses a runtime-disabled module strategy"); |
116 | 0 | return Ok(()); |
117 | | } |
118 | 0 | _ => unreachable!(), |
119 | | }; |
120 | | |
121 | 12.0k | log_wasm(&wasm); |
122 | | |
123 | | // Instantiate the generated wasm file in the chosen differential engine. |
124 | 12.0k | let lhs_instance = lhs.instantiate(&wasm); |
125 | 12.0k | STATS.bump_engine(lhs.name()); |
126 | | |
127 | | // Always use Wasmtime as the second engine to instantiate within. |
128 | 12.0k | log::debug!("Building RHS Wasmtime"); |
129 | 12.0k | let rhs_store = config.to_store(); |
130 | 12.0k | let rhs_module = wasmtime::Module::new(rhs_store.engine(), &wasm).unwrap(); |
131 | 12.0k | let rhs_instance = WasmtimeInstance::new(rhs_store, rhs_module); |
132 | | |
133 | 11.4k | let (mut lhs_instance, mut rhs_instance) = |
134 | 12.0k | match DiffEqResult::new(&*lhs, lhs_instance, rhs_instance) { |
135 | | // Both sides successful, continue below to invoking exports. |
136 | 11.4k | DiffEqResult::Success(l, r) => (l, r), |
137 | | |
138 | | // Both sides failed, or computation has diverged. In both cases this |
139 | | // test case is done. |
140 | 547 | DiffEqResult::Poisoned | DiffEqResult::Failed => return Ok(()), |
141 | | }; |
142 | | |
143 | | // Call each exported function with different sets of arguments. |
144 | 35.1k | 'outer: for (name, signature) in rhs_instance.exported_functions() { |
145 | 35.1k | let mut invocations = 0; |
146 | | loop { |
147 | 91.1k | let arguments = match signature |
148 | 91.1k | .params() |
149 | 112k | .map(|ty| { |
150 | 112k | let ty = ty |
151 | 112k | .try_into() |
152 | 112k | .map_err(|_| arbitrary::Error::IncorrectFormat)?; |
153 | 112k | DiffValue::arbitrary_of_type(&mut u, ty) |
154 | 112k | }) |
155 | 91.1k | .collect::<Result<Vec<_>>>() |
156 | | { |
157 | 90.7k | Ok(args) => args, |
158 | | // This function signature isn't compatible with differential |
159 | | // fuzzing yet, try the next exported function in the meantime. |
160 | 428 | Err(_) => continue 'outer, |
161 | | }; |
162 | | |
163 | 90.7k | let result_tys = match signature |
164 | 90.7k | .results() |
165 | 142k | .map(|ty| { |
166 | 142k | DiffValueType::try_from(ty).map_err(|_| arbitrary::Error::IncorrectFormat) |
167 | 142k | }) |
168 | 90.7k | .collect::<Result<Vec<_>>>() |
169 | | { |
170 | 90.4k | Ok(tys) => tys, |
171 | | // This function signature isn't compatible with differential |
172 | | // fuzzing yet, try the next exported function in the meantime. |
173 | 303 | Err(_) => continue 'outer, |
174 | | }; |
175 | | |
176 | 90.4k | let ok = differential( |
177 | 90.4k | lhs_instance.as_mut(), |
178 | 90.4k | lhs.as_ref(), |
179 | 90.4k | &mut rhs_instance, |
180 | 90.4k | &name, |
181 | 90.4k | &arguments, |
182 | 90.4k | &result_tys, |
183 | | ) |
184 | 90.4k | .expect("failed to run differential evaluation"); |
185 | | |
186 | 90.4k | invocations += 1; |
187 | 90.4k | STATS.total_invocations.fetch_add(1, SeqCst); |
188 | | |
189 | | // If this differential execution has resulted in the two instances |
190 | | // diverging in state we can't keep executing so don't execute any |
191 | | // more functions. |
192 | 90.4k | if !ok { |
193 | 388 | break 'outer; |
194 | 90.0k | } |
195 | | |
196 | | // We evaluate the same function with different arguments until we |
197 | | // Hit a predetermined limit or we run out of unstructured data--it |
198 | | // does not make sense to re-evaluate the same arguments over and |
199 | | // over. |
200 | 90.0k | if invocations > NUM_INVOCATIONS || u.is_empty() { |
201 | 34.0k | break; |
202 | 55.9k | } |
203 | | } |
204 | | } |
205 | | |
206 | 11.4k | STATS.successes.fetch_add(1, SeqCst); |
207 | 11.4k | Ok(()) |
208 | 12.4k | } |
209 | | |
210 | | #[derive(Default)] |
211 | | struct RuntimeStats { |
212 | | /// Total number of fuzz inputs processed |
213 | | attempts: AtomicUsize, |
214 | | |
215 | | /// Number of times we've invoked engines |
216 | | total_invocations: AtomicUsize, |
217 | | |
218 | | /// Number of times a fuzz input finished all the way to the end without any |
219 | | /// sort of error (including `Arbitrary` errors) |
220 | | successes: AtomicUsize, |
221 | | |
222 | | // Counters for which engine was chosen |
223 | | wasmi: AtomicUsize, |
224 | | v8: AtomicUsize, |
225 | | spec: AtomicUsize, |
226 | | wasmtime: AtomicUsize, |
227 | | winch: AtomicUsize, |
228 | | pulley: AtomicUsize, |
229 | | |
230 | | // Counters for which style of module is chosen |
231 | | wasm_smith_modules: AtomicUsize, |
232 | | single_instruction_modules: AtomicUsize, |
233 | | } |
234 | | |
235 | | impl RuntimeStats { |
236 | 0 | const fn new() -> RuntimeStats { |
237 | 0 | RuntimeStats { |
238 | 0 | attempts: AtomicUsize::new(0), |
239 | 0 | total_invocations: AtomicUsize::new(0), |
240 | 0 | successes: AtomicUsize::new(0), |
241 | 0 | wasmi: AtomicUsize::new(0), |
242 | 0 | v8: AtomicUsize::new(0), |
243 | 0 | spec: AtomicUsize::new(0), |
244 | 0 | wasmtime: AtomicUsize::new(0), |
245 | 0 | winch: AtomicUsize::new(0), |
246 | 0 | pulley: AtomicUsize::new(0), |
247 | 0 | wasm_smith_modules: AtomicUsize::new(0), |
248 | 0 | single_instruction_modules: AtomicUsize::new(0), |
249 | 0 | } |
250 | 0 | } |
251 | | |
252 | 12.4k | fn bump_attempts(&self) { |
253 | 12.4k | let attempts = self.attempts.fetch_add(1, SeqCst); |
254 | 12.4k | if attempts == 0 || attempts % 1_000 != 0 { |
255 | 12.4k | return; |
256 | 12 | } |
257 | 12 | let successes = self.successes.load(SeqCst); |
258 | 12 | println!( |
259 | 12 | "=== Execution rate ({} successes / {} attempted modules): {:.02}% ===", |
260 | | successes, |
261 | | attempts, |
262 | 12 | successes as f64 / attempts as f64 * 100f64, |
263 | | ); |
264 | | |
265 | 12 | let v8 = self.v8.load(SeqCst); |
266 | 12 | let spec = self.spec.load(SeqCst); |
267 | 12 | let wasmi = self.wasmi.load(SeqCst); |
268 | 12 | let wasmtime = self.wasmtime.load(SeqCst); |
269 | 12 | let winch = self.winch.load(SeqCst); |
270 | 12 | let pulley = self.pulley.load(SeqCst); |
271 | 12 | let total = v8 + spec + wasmi + wasmtime + winch + pulley; |
272 | 12 | println!( |
273 | 12 | "\twasmi: {:.02}%, spec: {:.02}%, wasmtime: {:.02}%, v8: {:.02}%, \ |
274 | 12 | winch: {:.02}, \ |
275 | 12 | pulley: {:.02}%", |
276 | 12 | wasmi as f64 / total as f64 * 100f64, |
277 | 12 | spec as f64 / total as f64 * 100f64, |
278 | 12 | wasmtime as f64 / total as f64 * 100f64, |
279 | 12 | v8 as f64 / total as f64 * 100f64, |
280 | 12 | winch as f64 / total as f64 * 100f64, |
281 | 12 | pulley as f64 / total as f64 * 100f64, |
282 | | ); |
283 | | |
284 | 12 | let wasm_smith = self.wasm_smith_modules.load(SeqCst); |
285 | 12 | let single_inst = self.single_instruction_modules.load(SeqCst); |
286 | 12 | let total = wasm_smith + single_inst; |
287 | 12 | println!( |
288 | 12 | "\twasm-smith: {:.02}%, single-inst: {:.02}%", |
289 | 12 | wasm_smith as f64 / total as f64 * 100f64, |
290 | 12 | single_inst as f64 / total as f64 * 100f64, |
291 | | ); |
292 | 12.4k | } |
293 | | |
294 | 12.0k | fn bump_engine(&self, name: &str) { |
295 | 12.0k | match name { |
296 | 12.0k | "wasmi" => self.wasmi.fetch_add(1, SeqCst), |
297 | 9.55k | "wasmtime" => self.wasmtime.fetch_add(1, SeqCst), |
298 | 6.96k | "spec" => self.spec.fetch_add(1, SeqCst), |
299 | 5.52k | "v8" => self.v8.fetch_add(1, SeqCst), |
300 | 4.18k | "winch" => self.winch.fetch_add(1, SeqCst), |
301 | 2.40k | "pulley" => self.pulley.fetch_add(1, SeqCst), |
302 | 0 | _ => return, |
303 | | }; |
304 | 12.0k | } |
305 | | } |