Coverage Report

Created: 2026-04-24 07:45

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/wasmtime/cranelift/jit/src/memory/system.rs
Line
Count
Source
1
use cranelift_module::{ModuleError, ModuleResult};
2
3
#[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]
4
use memmap2::MmapMut;
5
6
#[cfg(not(any(feature = "selinux-fix", windows)))]
7
use std::alloc;
8
use std::io;
9
use std::mem;
10
use std::ptr;
11
12
use super::{BranchProtection, JITMemoryKind, JITMemoryProvider};
13
14
/// A simple struct consisting of a pointer and length.
15
struct PtrLen {
16
    #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]
17
    map: Option<MmapMut>,
18
19
    ptr: *mut u8,
20
    len: usize,
21
}
22
23
impl PtrLen {
24
    /// Create a new empty `PtrLen`.
25
67.4k
    fn new() -> Self {
26
67.4k
        Self {
27
67.4k
            #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]
28
67.4k
            map: None,
29
67.4k
30
67.4k
            ptr: ptr::null_mut(),
31
67.4k
            len: 0,
32
67.4k
        }
33
67.4k
    }
34
35
    /// Create a new `PtrLen` pointing to at least `size` bytes of memory,
36
    /// suitably sized and aligned for memory protection.
37
    #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]
38
13.8k
    fn with_size(size: usize) -> io::Result<Self> {
39
13.8k
        let alloc_size = region::page::ceil(size as *const ()) as usize;
40
13.8k
        MmapMut::map_anon(alloc_size).map(|mut mmap| {
41
            // The order here is important; we assign the pointer first to get
42
            // around compile time borrow errors.
43
13.8k
            Self {
44
13.8k
                ptr: mmap.as_mut_ptr(),
45
13.8k
                map: Some(mmap),
46
13.8k
                len: alloc_size,
47
13.8k
            }
48
13.8k
        })
49
13.8k
    }
50
51
    #[cfg(all(not(target_os = "windows"), not(feature = "selinux-fix")))]
52
    fn with_size(size: usize) -> io::Result<Self> {
53
        assert_ne!(size, 0);
54
        let page_size = region::page::size();
55
        let alloc_size = region::page::ceil(size as *const ()) as usize;
56
        let layout = alloc::Layout::from_size_align(alloc_size, page_size).unwrap();
57
        // Safety: We assert that the size is non-zero above.
58
        let ptr = unsafe { alloc::alloc(layout) };
59
60
        if !ptr.is_null() {
61
            Ok(Self {
62
                ptr,
63
                len: alloc_size,
64
            })
65
        } else {
66
            Err(io::Error::from(io::ErrorKind::OutOfMemory))
67
        }
68
    }
69
70
    #[cfg(target_os = "windows")]
71
    fn with_size(size: usize) -> io::Result<Self> {
72
        use windows_sys::Win32::System::Memory::{
73
            MEM_COMMIT, MEM_RESERVE, PAGE_READWRITE, VirtualAlloc,
74
        };
75
76
        // VirtualAlloc always rounds up to the next multiple of the page size
77
        let ptr = unsafe {
78
            VirtualAlloc(
79
                ptr::null_mut(),
80
                size,
81
                MEM_COMMIT | MEM_RESERVE,
82
                PAGE_READWRITE,
83
            )
84
        };
85
        if !ptr.is_null() {
86
            Ok(Self {
87
                ptr: ptr as *mut u8,
88
                len: region::page::ceil(size as *const ()) as usize,
89
            })
90
        } else {
91
            Err(io::Error::last_os_error())
92
        }
93
    }
94
}
95
96
// `MMapMut` from `cfg(feature = "selinux-fix")` already deallocates properly.
97
#[cfg(all(not(target_os = "windows"), not(feature = "selinux-fix")))]
98
impl Drop for PtrLen {
99
    fn drop(&mut self) {
100
        if !self.ptr.is_null() {
101
            let page_size = region::page::size();
102
            let layout = alloc::Layout::from_size_align(self.len, page_size).unwrap();
103
            unsafe {
104
                region::protect(self.ptr, self.len, region::Protection::READ_WRITE)
105
                    .expect("unable to unprotect memory");
106
                alloc::dealloc(self.ptr, layout)
107
            }
108
        }
109
    }
110
}
111
112
// TODO: add a `Drop` impl for `cfg(target_os = "windows")`
113
114
/// JIT memory manager. This manages pages of suitably aligned and
115
/// accessible memory. Memory will be leaked by default to have
116
/// function pointers remain valid for the remainder of the
117
/// program's life.
118
pub(crate) struct Memory {
119
    allocations: Vec<PtrLen>,
120
    already_protected: usize,
121
    current: PtrLen,
122
    position: usize,
123
}
124
125
unsafe impl Send for Memory {}
126
127
impl Memory {
128
32.1k
    pub(crate) fn new() -> Self {
129
32.1k
        Self {
130
32.1k
            allocations: Vec::new(),
131
32.1k
            already_protected: 0,
132
32.1k
            current: PtrLen::new(),
133
32.1k
            position: 0,
134
32.1k
        }
135
32.1k
    }
136
137
35.3k
    fn finish_current(&mut self) {
138
35.3k
        self.allocations
139
35.3k
            .push(mem::replace(&mut self.current, PtrLen::new()));
140
35.3k
        self.position = 0;
141
35.3k
    }
142
143
37.7k
    pub(crate) fn allocate(&mut self, size: usize, align: u64) -> io::Result<*mut u8> {
144
37.7k
        let align = usize::try_from(align).expect("alignment too big");
145
37.7k
        if self.position % align != 0 {
146
1.10k
            self.position += align - self.position % align;
147
1.10k
            debug_assert!(self.position % align == 0);
148
36.6k
        }
149
150
37.7k
        if size <= self.current.len - self.position {
151
            // TODO: Ensure overflow is not possible.
152
23.8k
            let ptr = unsafe { self.current.ptr.add(self.position) };
153
23.8k
            self.position += size;
154
23.8k
            return Ok(ptr);
155
13.8k
        }
156
157
13.8k
        self.finish_current();
158
159
        // TODO: Allocate more at a time.
160
13.8k
        self.current = PtrLen::with_size(size)?;
161
13.8k
        self.position = size;
162
163
13.8k
        Ok(self.current.ptr)
164
37.7k
    }
165
166
    /// Set all memory allocated in this `Memory` up to now as readable and executable.
167
10.7k
    pub(crate) fn set_readable_and_executable(
168
10.7k
        &mut self,
169
10.7k
        branch_protection: BranchProtection,
170
10.7k
    ) -> ModuleResult<()> {
171
10.7k
        self.finish_current();
172
173
13.8k
        for &PtrLen { ptr, len, .. } in self.non_protected_allocations_iter() {
174
13.8k
            super::set_readable_and_executable(ptr, len, branch_protection)?;
175
        }
176
177
        // Flush any in-flight instructions from the pipeline
178
10.7k
        wasmtime_jit_icache_coherence::pipeline_flush_mt().expect("Failed pipeline flush");
179
180
10.7k
        self.already_protected = self.allocations.len();
181
10.7k
        Ok(())
182
10.7k
    }
183
184
    /// Set all memory allocated in this `Memory` up to now as readonly.
185
10.7k
    pub(crate) fn set_readonly(&mut self) -> ModuleResult<()> {
186
10.7k
        self.finish_current();
187
188
10.7k
        for &PtrLen { ptr, len, .. } in self.non_protected_allocations_iter() {
189
            unsafe {
190
0
                region::protect(ptr, len, region::Protection::READ).map_err(|e| {
191
0
                    ModuleError::Backend(
192
0
                        anyhow::Error::new(e).context("unable to make memory readonly"),
193
0
                    )
194
0
                })?;
195
            }
196
        }
197
198
10.7k
        self.already_protected = self.allocations.len();
199
10.7k
        Ok(())
200
10.7k
    }
201
202
    /// Iterates non protected memory allocations that are of not zero bytes in size.
203
21.4k
    fn non_protected_allocations_iter(&self) -> impl Iterator<Item = &PtrLen> {
204
21.4k
        let iter = self.allocations[self.already_protected..].iter();
205
206
        #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]
207
35.3k
        return iter.filter(|&PtrLen { map, len, .. }| *len != 0 && map.is_some());
208
209
        #[cfg(any(target_os = "windows", not(feature = "selinux-fix")))]
210
        return iter.filter(|&PtrLen { len, .. }| *len != 0);
211
21.4k
    }
212
213
    /// Frees all allocated memory regions that would be leaked otherwise.
214
    /// Likely to invalidate existing function pointers, causing unsafety.
215
32.1k
    pub(crate) unsafe fn free_memory(&mut self) {
216
32.1k
        self.allocations.clear();
217
32.1k
        self.already_protected = 0;
218
32.1k
    }
219
}
220
221
impl Drop for Memory {
222
32.1k
    fn drop(&mut self) {
223
        // leak memory to guarantee validity of function pointers
224
32.1k
        mem::replace(&mut self.allocations, Vec::new())
225
32.1k
            .into_iter()
226
32.1k
            .for_each(mem::forget);
227
32.1k
    }
228
}
229
230
/// A memory provider that allocates memory on-demand using the system
231
/// allocator.
232
///
233
/// Note: Memory will be leaked by default unless
234
/// [`JITMemoryProvider::free_memory`] is called to ensure function pointers
235
/// remain valid for the remainder of the program's life.
236
pub struct SystemMemoryProvider {
237
    code: Memory,
238
    readonly: Memory,
239
    writable: Memory,
240
}
241
242
impl SystemMemoryProvider {
243
    /// Create a new memory handle with the given branch protection.
244
10.7k
    pub fn new() -> Self {
245
10.7k
        Self {
246
10.7k
            code: Memory::new(),
247
10.7k
            readonly: Memory::new(),
248
10.7k
            writable: Memory::new(),
249
10.7k
        }
250
10.7k
    }
251
}
252
253
impl JITMemoryProvider for SystemMemoryProvider {
254
10.7k
    unsafe fn free_memory(&mut self) {
255
10.7k
        self.code.free_memory();
256
10.7k
        self.readonly.free_memory();
257
10.7k
        self.writable.free_memory();
258
10.7k
    }
259
260
10.7k
    fn finalize(&mut self, branch_protection: BranchProtection) -> ModuleResult<()> {
261
10.7k
        self.readonly.set_readonly()?;
262
10.7k
        self.code.set_readable_and_executable(branch_protection)
263
10.7k
    }
264
265
37.7k
    fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8> {
266
37.7k
        match kind {
267
37.7k
            JITMemoryKind::Executable => self.code.allocate(size, align),
268
0
            JITMemoryKind::Writable => self.writable.allocate(size, align),
269
0
            JITMemoryKind::ReadOnly => self.readonly.allocate(size, align),
270
        }
271
37.7k
    }
272
}