/src/wasmtime/cranelift/jit/src/memory/system.rs
Line | Count | Source |
1 | | use cranelift_module::{ModuleError, ModuleResult}; |
2 | | |
3 | | #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))] |
4 | | use memmap2::MmapMut; |
5 | | |
6 | | #[cfg(not(any(feature = "selinux-fix", windows)))] |
7 | | use std::alloc; |
8 | | use std::io; |
9 | | use std::mem; |
10 | | use std::ptr; |
11 | | |
12 | | use super::{BranchProtection, JITMemoryKind, JITMemoryProvider}; |
13 | | |
14 | | /// A simple struct consisting of a pointer and length. |
15 | | struct PtrLen { |
16 | | #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))] |
17 | | map: Option<MmapMut>, |
18 | | |
19 | | ptr: *mut u8, |
20 | | len: usize, |
21 | | } |
22 | | |
23 | | impl PtrLen { |
24 | | /// Create a new empty `PtrLen`. |
25 | 67.4k | fn new() -> Self { |
26 | 67.4k | Self { |
27 | 67.4k | #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))] |
28 | 67.4k | map: None, |
29 | 67.4k | |
30 | 67.4k | ptr: ptr::null_mut(), |
31 | 67.4k | len: 0, |
32 | 67.4k | } |
33 | 67.4k | } |
34 | | |
35 | | /// Create a new `PtrLen` pointing to at least `size` bytes of memory, |
36 | | /// suitably sized and aligned for memory protection. |
37 | | #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))] |
38 | 13.8k | fn with_size(size: usize) -> io::Result<Self> { |
39 | 13.8k | let alloc_size = region::page::ceil(size as *const ()) as usize; |
40 | 13.8k | MmapMut::map_anon(alloc_size).map(|mut mmap| { |
41 | | // The order here is important; we assign the pointer first to get |
42 | | // around compile time borrow errors. |
43 | 13.8k | Self { |
44 | 13.8k | ptr: mmap.as_mut_ptr(), |
45 | 13.8k | map: Some(mmap), |
46 | 13.8k | len: alloc_size, |
47 | 13.8k | } |
48 | 13.8k | }) |
49 | 13.8k | } |
50 | | |
51 | | #[cfg(all(not(target_os = "windows"), not(feature = "selinux-fix")))] |
52 | | fn with_size(size: usize) -> io::Result<Self> { |
53 | | assert_ne!(size, 0); |
54 | | let page_size = region::page::size(); |
55 | | let alloc_size = region::page::ceil(size as *const ()) as usize; |
56 | | let layout = alloc::Layout::from_size_align(alloc_size, page_size).unwrap(); |
57 | | // Safety: We assert that the size is non-zero above. |
58 | | let ptr = unsafe { alloc::alloc(layout) }; |
59 | | |
60 | | if !ptr.is_null() { |
61 | | Ok(Self { |
62 | | ptr, |
63 | | len: alloc_size, |
64 | | }) |
65 | | } else { |
66 | | Err(io::Error::from(io::ErrorKind::OutOfMemory)) |
67 | | } |
68 | | } |
69 | | |
70 | | #[cfg(target_os = "windows")] |
71 | | fn with_size(size: usize) -> io::Result<Self> { |
72 | | use windows_sys::Win32::System::Memory::{ |
73 | | MEM_COMMIT, MEM_RESERVE, PAGE_READWRITE, VirtualAlloc, |
74 | | }; |
75 | | |
76 | | // VirtualAlloc always rounds up to the next multiple of the page size |
77 | | let ptr = unsafe { |
78 | | VirtualAlloc( |
79 | | ptr::null_mut(), |
80 | | size, |
81 | | MEM_COMMIT | MEM_RESERVE, |
82 | | PAGE_READWRITE, |
83 | | ) |
84 | | }; |
85 | | if !ptr.is_null() { |
86 | | Ok(Self { |
87 | | ptr: ptr as *mut u8, |
88 | | len: region::page::ceil(size as *const ()) as usize, |
89 | | }) |
90 | | } else { |
91 | | Err(io::Error::last_os_error()) |
92 | | } |
93 | | } |
94 | | } |
95 | | |
96 | | // `MMapMut` from `cfg(feature = "selinux-fix")` already deallocates properly. |
97 | | #[cfg(all(not(target_os = "windows"), not(feature = "selinux-fix")))] |
98 | | impl Drop for PtrLen { |
99 | | fn drop(&mut self) { |
100 | | if !self.ptr.is_null() { |
101 | | let page_size = region::page::size(); |
102 | | let layout = alloc::Layout::from_size_align(self.len, page_size).unwrap(); |
103 | | unsafe { |
104 | | region::protect(self.ptr, self.len, region::Protection::READ_WRITE) |
105 | | .expect("unable to unprotect memory"); |
106 | | alloc::dealloc(self.ptr, layout) |
107 | | } |
108 | | } |
109 | | } |
110 | | } |
111 | | |
112 | | // TODO: add a `Drop` impl for `cfg(target_os = "windows")` |
113 | | |
114 | | /// JIT memory manager. This manages pages of suitably aligned and |
115 | | /// accessible memory. Memory will be leaked by default to have |
116 | | /// function pointers remain valid for the remainder of the |
117 | | /// program's life. |
118 | | pub(crate) struct Memory { |
119 | | allocations: Vec<PtrLen>, |
120 | | already_protected: usize, |
121 | | current: PtrLen, |
122 | | position: usize, |
123 | | } |
124 | | |
125 | | unsafe impl Send for Memory {} |
126 | | |
127 | | impl Memory { |
128 | 32.1k | pub(crate) fn new() -> Self { |
129 | 32.1k | Self { |
130 | 32.1k | allocations: Vec::new(), |
131 | 32.1k | already_protected: 0, |
132 | 32.1k | current: PtrLen::new(), |
133 | 32.1k | position: 0, |
134 | 32.1k | } |
135 | 32.1k | } |
136 | | |
137 | 35.3k | fn finish_current(&mut self) { |
138 | 35.3k | self.allocations |
139 | 35.3k | .push(mem::replace(&mut self.current, PtrLen::new())); |
140 | 35.3k | self.position = 0; |
141 | 35.3k | } |
142 | | |
143 | 37.7k | pub(crate) fn allocate(&mut self, size: usize, align: u64) -> io::Result<*mut u8> { |
144 | 37.7k | let align = usize::try_from(align).expect("alignment too big"); |
145 | 37.7k | if self.position % align != 0 { |
146 | 1.10k | self.position += align - self.position % align; |
147 | 1.10k | debug_assert!(self.position % align == 0); |
148 | 36.6k | } |
149 | | |
150 | 37.7k | if size <= self.current.len - self.position { |
151 | | // TODO: Ensure overflow is not possible. |
152 | 23.8k | let ptr = unsafe { self.current.ptr.add(self.position) }; |
153 | 23.8k | self.position += size; |
154 | 23.8k | return Ok(ptr); |
155 | 13.8k | } |
156 | | |
157 | 13.8k | self.finish_current(); |
158 | | |
159 | | // TODO: Allocate more at a time. |
160 | 13.8k | self.current = PtrLen::with_size(size)?; |
161 | 13.8k | self.position = size; |
162 | | |
163 | 13.8k | Ok(self.current.ptr) |
164 | 37.7k | } |
165 | | |
166 | | /// Set all memory allocated in this `Memory` up to now as readable and executable. |
167 | 10.7k | pub(crate) fn set_readable_and_executable( |
168 | 10.7k | &mut self, |
169 | 10.7k | branch_protection: BranchProtection, |
170 | 10.7k | ) -> ModuleResult<()> { |
171 | 10.7k | self.finish_current(); |
172 | | |
173 | 13.8k | for &PtrLen { ptr, len, .. } in self.non_protected_allocations_iter() { |
174 | 13.8k | super::set_readable_and_executable(ptr, len, branch_protection)?; |
175 | | } |
176 | | |
177 | | // Flush any in-flight instructions from the pipeline |
178 | 10.7k | wasmtime_jit_icache_coherence::pipeline_flush_mt().expect("Failed pipeline flush"); |
179 | | |
180 | 10.7k | self.already_protected = self.allocations.len(); |
181 | 10.7k | Ok(()) |
182 | 10.7k | } |
183 | | |
184 | | /// Set all memory allocated in this `Memory` up to now as readonly. |
185 | 10.7k | pub(crate) fn set_readonly(&mut self) -> ModuleResult<()> { |
186 | 10.7k | self.finish_current(); |
187 | | |
188 | 10.7k | for &PtrLen { ptr, len, .. } in self.non_protected_allocations_iter() { |
189 | | unsafe { |
190 | 0 | region::protect(ptr, len, region::Protection::READ).map_err(|e| { |
191 | 0 | ModuleError::Backend( |
192 | 0 | anyhow::Error::new(e).context("unable to make memory readonly"), |
193 | 0 | ) |
194 | 0 | })?; |
195 | | } |
196 | | } |
197 | | |
198 | 10.7k | self.already_protected = self.allocations.len(); |
199 | 10.7k | Ok(()) |
200 | 10.7k | } |
201 | | |
202 | | /// Iterates non protected memory allocations that are of not zero bytes in size. |
203 | 21.4k | fn non_protected_allocations_iter(&self) -> impl Iterator<Item = &PtrLen> { |
204 | 21.4k | let iter = self.allocations[self.already_protected..].iter(); |
205 | | |
206 | | #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))] |
207 | 35.3k | return iter.filter(|&PtrLen { map, len, .. }| *len != 0 && map.is_some()); |
208 | | |
209 | | #[cfg(any(target_os = "windows", not(feature = "selinux-fix")))] |
210 | | return iter.filter(|&PtrLen { len, .. }| *len != 0); |
211 | 21.4k | } |
212 | | |
213 | | /// Frees all allocated memory regions that would be leaked otherwise. |
214 | | /// Likely to invalidate existing function pointers, causing unsafety. |
215 | 32.1k | pub(crate) unsafe fn free_memory(&mut self) { |
216 | 32.1k | self.allocations.clear(); |
217 | 32.1k | self.already_protected = 0; |
218 | 32.1k | } |
219 | | } |
220 | | |
221 | | impl Drop for Memory { |
222 | 32.1k | fn drop(&mut self) { |
223 | | // leak memory to guarantee validity of function pointers |
224 | 32.1k | mem::replace(&mut self.allocations, Vec::new()) |
225 | 32.1k | .into_iter() |
226 | 32.1k | .for_each(mem::forget); |
227 | 32.1k | } |
228 | | } |
229 | | |
230 | | /// A memory provider that allocates memory on-demand using the system |
231 | | /// allocator. |
232 | | /// |
233 | | /// Note: Memory will be leaked by default unless |
234 | | /// [`JITMemoryProvider::free_memory`] is called to ensure function pointers |
235 | | /// remain valid for the remainder of the program's life. |
236 | | pub struct SystemMemoryProvider { |
237 | | code: Memory, |
238 | | readonly: Memory, |
239 | | writable: Memory, |
240 | | } |
241 | | |
242 | | impl SystemMemoryProvider { |
243 | | /// Create a new memory handle with the given branch protection. |
244 | 10.7k | pub fn new() -> Self { |
245 | 10.7k | Self { |
246 | 10.7k | code: Memory::new(), |
247 | 10.7k | readonly: Memory::new(), |
248 | 10.7k | writable: Memory::new(), |
249 | 10.7k | } |
250 | 10.7k | } |
251 | | } |
252 | | |
253 | | impl JITMemoryProvider for SystemMemoryProvider { |
254 | 10.7k | unsafe fn free_memory(&mut self) { |
255 | 10.7k | self.code.free_memory(); |
256 | 10.7k | self.readonly.free_memory(); |
257 | 10.7k | self.writable.free_memory(); |
258 | 10.7k | } |
259 | | |
260 | 10.7k | fn finalize(&mut self, branch_protection: BranchProtection) -> ModuleResult<()> { |
261 | 10.7k | self.readonly.set_readonly()?; |
262 | 10.7k | self.code.set_readable_and_executable(branch_protection) |
263 | 10.7k | } |
264 | | |
265 | 37.7k | fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8> { |
266 | 37.7k | match kind { |
267 | 37.7k | JITMemoryKind::Executable => self.code.allocate(size, align), |
268 | 0 | JITMemoryKind::Writable => self.writable.allocate(size, align), |
269 | 0 | JITMemoryKind::ReadOnly => self.readonly.allocate(size, align), |
270 | | } |
271 | 37.7k | } |
272 | | } |