1extern crate libc;
2
3use std::fs::File;
4use std::mem::ManuallyDrop;
5use std::os::unix::io::{FromRawFd, RawFd};
6use std::sync::atomic::{AtomicUsize, Ordering};
7use std::{io, ptr};
8
9#[cfg(any(
10 all(target_os = "linux", not(target_arch = "mips")),
11 target_os = "freebsd",
12 target_os = "android"
13))]
14const MAP_STACK: libc::c_int = libc::MAP_STACK;
15
16#[cfg(not(any(
17 all(target_os = "linux", not(target_arch = "mips")),
18 target_os = "freebsd",
19 target_os = "android"
20)))]
21const MAP_STACK: libc::c_int = 0;
22
23#[cfg(any(target_os = "linux", target_os = "android"))]
24const MAP_POPULATE: libc::c_int = libc::MAP_POPULATE;
25
26#[cfg(not(any(target_os = "linux", target_os = "android")))]
27const MAP_POPULATE: libc::c_int = 0;
28
29#[cfg(any(target_os = "linux", target_os = "android"))]
30const MAP_HUGETLB: libc::c_int = libc::MAP_HUGETLB;
31
32#[cfg(target_os = "linux")]
33const MAP_HUGE_MASK: libc::c_int = libc::MAP_HUGE_MASK;
34
35#[cfg(any(target_os = "linux", target_os = "android"))]
36const MAP_HUGE_SHIFT: libc::c_int = libc::MAP_HUGE_SHIFT;
37
38#[cfg(not(any(target_os = "linux", target_os = "android")))]
39const MAP_HUGETLB: libc::c_int = 0;
40
41#[cfg(not(target_os = "linux"))]
42const MAP_HUGE_MASK: libc::c_int = 0;
43
44#[cfg(not(any(target_os = "linux", target_os = "android")))]
45const MAP_HUGE_SHIFT: libc::c_int = 0;
46
47#[cfg(any(
48 target_os = "android",
49 all(target_os = "linux", not(target_env = "musl"))
50))]
51use libc::{mmap64 as mmap, off64_t as off_t};
52
53#[cfg(not(any(
54 target_os = "android",
55 all(target_os = "linux", not(target_env = "musl"))
56)))]
57use libc::{mmap, off_t};
58
59pub struct MmapInner {
60 ptr: *mut libc::c_void,
61 len: usize,
62}
63
64impl MmapInner {
65 /// Creates a new `MmapInner`.
66 ///
67 /// This is a thin wrapper around the `mmap` system call.
68 fn new(
69 len: usize,
70 prot: libc::c_int,
71 flags: libc::c_int,
72 file: RawFd,
73 offset: u64,
74 ) -> io::Result<MmapInner> {
75 let alignment = offset % page_size() as u64;
76 let aligned_offset = offset - alignment;
77
78 let (map_len, map_offset) = Self::adjust_mmap_params(len, alignment as usize)?;
79
80 unsafe {
81 let ptr = mmap(
82 ptr::null_mut(),
83 map_len as libc::size_t,
84 prot,
85 flags,
86 file,
87 aligned_offset as off_t,
88 );
89
90 if ptr == libc::MAP_FAILED {
91 Err(io::Error::last_os_error())
92 } else {
93 Ok(Self::from_raw_parts(ptr, len, map_offset))
94 }
95 }
96 }
97
98 fn adjust_mmap_params(len: usize, alignment: usize) -> io::Result<(usize, usize)> {
99 use std::isize;
100
101 // Rust's slice cannot be larger than isize::MAX.
102 // See https://doc.rust-lang.org/std/slice/fn.from_raw_parts.html
103 //
104 // This is not a problem on 64-bit targets, but on 32-bit one
105 // having a file or an anonymous mapping larger than 2GB is quite normal
106 // and we have to prevent it.
107 //
108 // The code below is essentially the same as in Rust's std:
109 // https://github.com/rust-lang/rust/blob/db78ab70a88a0a5e89031d7ee4eccec835dcdbde/library/alloc/src/raw_vec.rs#L495
110 if std::mem::size_of::<usize>() < 8 && len > isize::MAX as usize {
111 return Err(io::Error::new(
112 io::ErrorKind::InvalidData,
113 "memory map length overflows isize",
114 ));
115 }
116
117 let map_len = len + alignment;
118 let map_offset = alignment;
119
120 // `libc::mmap` does not support zero-size mappings. POSIX defines:
121 //
122 // https://pubs.opengroup.org/onlinepubs/9699919799/functions/mmap.html
123 // > If `len` is zero, `mmap()` shall fail and no mapping shall be established.
124 //
125 // So if we would create such a mapping, crate a one-byte mapping instead:
126 let map_len = map_len.max(1);
127
128 // Note that in that case `MmapInner::len` is still set to zero,
129 // and `Mmap` will still dereferences to an empty slice.
130 //
131 // If this mapping is backed by an empty file, we create a mapping larger than the file.
132 // This is unusual but well-defined. On the same man page, POSIX further defines:
133 //
134 // > The `mmap()` function can be used to map a region of memory that is larger
135 // > than the current size of the object.
136 //
137 // (The object here is the file.)
138 //
139 // > Memory access within the mapping but beyond the current end of the underlying
140 // > objects may result in SIGBUS signals being sent to the process. The reason for this
141 // > is that the size of the object can be manipulated by other processes and can change
142 // > at any moment. The implementation should tell the application that a memory reference
143 // > is outside the object where this can be detected; otherwise, written data may be lost
144 // > and read data may not reflect actual data in the object.
145 //
146 // Because `MmapInner::len` is not incremented, this increment of `aligned_len`
147 // will not allow accesses past the end of the file and will not cause SIGBUS.
148 //
149 // (SIGBUS is still possible by mapping a non-empty file and then truncating it
150 // to a shorter size, but that is unrelated to this handling of empty files.)
151 Ok((map_len, map_offset))
152 }
153
154 /// Get the current memory mapping as a `(ptr, map_len, offset)` tuple.
155 ///
156 /// Note that `map_len` is the length of the memory mapping itself and
157 /// _not_ the one that would be passed to `from_raw_parts`.
158 fn as_mmap_params(&self) -> (*mut libc::c_void, usize, usize) {
159 let offset = self.ptr as usize % page_size();
160 let len = self.len + offset;
161
162 // There are two possible memory layouts we could have, depending on
163 // the length and offset passed when constructing this instance:
164 //
165 // 1. The "normal" memory layout looks like this:
166 //
167 // |<------------------>|<---------------------->|
168 // mmap ptr offset ptr public slice
169 //
170 // That is, we have
171 // - The start of the page-aligned memory mapping returned by mmap,
172 // followed by,
173 // - Some number of bytes that are memory mapped but ignored since
174 // they are before the byte offset requested by the user, followed
175 // by,
176 // - The actual memory mapped slice requested by the user.
177 //
178 // This maps cleanly to a (ptr, len, offset) tuple.
179 //
180 // 2. Then, we have the case where the user requested a zero-length
181 // memory mapping. mmap(2) does not support zero-length mappings so
182 // this crate works around that by actually making a mapping of
183 // length one. This means that we have
184 // - A length zero slice, followed by,
185 // - A single memory mapped byte
186 //
187 // Note that this only happens if the offset within the page is also
188 // zero. Otherwise, we have a memory map of offset bytes and not a
189 // zero-length memory map.
190 //
191 // This doesn't fit cleanly into a (ptr, len, offset) tuple. Instead,
192 // we fudge it slightly: a zero-length memory map turns into a
193 // mapping of length one and can't be told apart outside of this
194 // method without knowing the original length.
195 if len == 0 {
196 (self.ptr, 1, 0)
197 } else {
198 (unsafe { self.ptr.offset(-(offset as isize)) }, len, offset)
199 }
200 }
201
202 /// Construct this `MmapInner` from its raw components
203 ///
204 /// # Safety
205 ///
206 /// - `ptr` must point to the start of memory mapping that can be freed
207 /// using `munmap(2)` (i.e. returned by `mmap(2)` or `mremap(2)`)
208 /// - The memory mapping at `ptr` must have a length of `len + offset`.
209 /// - If `len + offset == 0` then the memory mapping must be of length 1.
210 /// - `offset` must be less than the current page size.
211 unsafe fn from_raw_parts(ptr: *mut libc::c_void, len: usize, offset: usize) -> Self {
212 debug_assert_eq!(ptr as usize % page_size(), 0, "ptr not page-aligned");
213 debug_assert!(offset < page_size(), "offset larger than page size");
214
215 Self {
216 ptr: ptr.add(offset),
217 len,
218 }
219 }
220
221 pub fn map(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result<MmapInner> {
222 let populate = if populate { MAP_POPULATE } else { 0 };
223 MmapInner::new(
224 len,
225 libc::PROT_READ,
226 libc::MAP_SHARED | populate,
227 file,
228 offset,
229 )
230 }
231
232 pub fn map_exec(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result<MmapInner> {
233 let populate = if populate { MAP_POPULATE } else { 0 };
234 MmapInner::new(
235 len,
236 libc::PROT_READ | libc::PROT_EXEC,
237 libc::MAP_SHARED | populate,
238 file,
239 offset,
240 )
241 }
242
243 pub fn map_mut(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result<MmapInner> {
244 let populate = if populate { MAP_POPULATE } else { 0 };
245 MmapInner::new(
246 len,
247 libc::PROT_READ | libc::PROT_WRITE,
248 libc::MAP_SHARED | populate,
249 file,
250 offset,
251 )
252 }
253
254 pub fn map_copy(len: usize, file: RawFd, offset: u64, populate: bool) -> io::Result<MmapInner> {
255 let populate = if populate { MAP_POPULATE } else { 0 };
256 MmapInner::new(
257 len,
258 libc::PROT_READ | libc::PROT_WRITE,
259 libc::MAP_PRIVATE | populate,
260 file,
261 offset,
262 )
263 }
264
265 pub fn map_copy_read_only(
266 len: usize,
267 file: RawFd,
268 offset: u64,
269 populate: bool,
270 ) -> io::Result<MmapInner> {
271 let populate = if populate { MAP_POPULATE } else { 0 };
272 MmapInner::new(
273 len,
274 libc::PROT_READ,
275 libc::MAP_PRIVATE | populate,
276 file,
277 offset,
278 )
279 }
280
281 /// Open an anonymous memory map.
282 pub fn map_anon(
283 len: usize,
284 stack: bool,
285 populate: bool,
286 huge: Option<u8>,
287 ) -> io::Result<MmapInner> {
288 let stack = if stack { MAP_STACK } else { 0 };
289 let populate = if populate { MAP_POPULATE } else { 0 };
290 let hugetlb = if huge.is_some() { MAP_HUGETLB } else { 0 };
291 let offset = huge
292 .map(|mask| ((mask as u64) & (MAP_HUGE_MASK as u64)) << MAP_HUGE_SHIFT)
293 .unwrap_or(0);
294 MmapInner::new(
295 len,
296 libc::PROT_READ | libc::PROT_WRITE,
297 libc::MAP_PRIVATE | libc::MAP_ANON | stack | populate | hugetlb,
298 -1,
299 offset,
300 )
301 }
302
303 pub fn flush(&self, offset: usize, len: usize) -> io::Result<()> {
304 let alignment = (self.ptr as usize + offset) % page_size();
305 let offset = offset as isize - alignment as isize;
306 let len = len + alignment;
307 let result =
308 unsafe { libc::msync(self.ptr.offset(offset), len as libc::size_t, libc::MS_SYNC) };
309 if result == 0 {
310 Ok(())
311 } else {
312 Err(io::Error::last_os_error())
313 }
314 }
315
316 pub fn flush_async(&self, offset: usize, len: usize) -> io::Result<()> {
317 let alignment = (self.ptr as usize + offset) % page_size();
318 let offset = offset as isize - alignment as isize;
319 let len = len + alignment;
320 let result =
321 unsafe { libc::msync(self.ptr.offset(offset), len as libc::size_t, libc::MS_ASYNC) };
322 if result == 0 {
323 Ok(())
324 } else {
325 Err(io::Error::last_os_error())
326 }
327 }
328
329 fn mprotect(&mut self, prot: libc::c_int) -> io::Result<()> {
330 unsafe {
331 let alignment = self.ptr as usize % page_size();
332 let ptr = self.ptr.offset(-(alignment as isize));
333 let len = self.len + alignment;
334 let len = len.max(1);
335 if libc::mprotect(ptr, len, prot) == 0 {
336 Ok(())
337 } else {
338 Err(io::Error::last_os_error())
339 }
340 }
341 }
342
343 pub fn make_read_only(&mut self) -> io::Result<()> {
344 self.mprotect(libc::PROT_READ)
345 }
346
347 pub fn make_exec(&mut self) -> io::Result<()> {
348 self.mprotect(libc::PROT_READ | libc::PROT_EXEC)
349 }
350
351 pub fn make_mut(&mut self) -> io::Result<()> {
352 self.mprotect(libc::PROT_READ | libc::PROT_WRITE)
353 }
354
355 #[inline]
356 pub fn ptr(&self) -> *const u8 {
357 self.ptr as *const u8
358 }
359
360 #[inline]
361 pub fn mut_ptr(&mut self) -> *mut u8 {
362 self.ptr as *mut u8
363 }
364
365 #[inline]
366 pub fn len(&self) -> usize {
367 self.len
368 }
369
370 pub fn advise(&self, advice: libc::c_int, offset: usize, len: usize) -> io::Result<()> {
371 let alignment = (self.ptr as usize + offset) % page_size();
372 let offset = offset as isize - alignment as isize;
373 let len = len + alignment;
374 unsafe {
375 if libc::madvise(self.ptr.offset(offset), len, advice) != 0 {
376 Err(io::Error::last_os_error())
377 } else {
378 Ok(())
379 }
380 }
381 }
382
383 #[cfg(target_os = "linux")]
384 pub fn remap(&mut self, new_len: usize, options: crate::RemapOptions) -> io::Result<()> {
385 let (old_ptr, old_len, offset) = self.as_mmap_params();
386 let (map_len, offset) = Self::adjust_mmap_params(new_len, offset)?;
387
388 unsafe {
389 let new_ptr = libc::mremap(old_ptr, old_len, map_len, options.into_flags());
390
391 if new_ptr == libc::MAP_FAILED {
392 Err(io::Error::last_os_error())
393 } else {
394 // We explicitly don't drop self since the pointer within is no longer valid.
395 ptr::write(self, Self::from_raw_parts(new_ptr, new_len, offset));
396 Ok(())
397 }
398 }
399 }
400
401 pub fn lock(&self) -> io::Result<()> {
402 unsafe {
403 if libc::mlock(self.ptr, self.len) != 0 {
404 Err(io::Error::last_os_error())
405 } else {
406 Ok(())
407 }
408 }
409 }
410
411 pub fn unlock(&self) -> io::Result<()> {
412 unsafe {
413 if libc::munlock(self.ptr, self.len) != 0 {
414 Err(io::Error::last_os_error())
415 } else {
416 Ok(())
417 }
418 }
419 }
420}
421
422impl Drop for MmapInner {
423 fn drop(&mut self) {
424 let (ptr: *mut c_void, len: usize, _) = self.as_mmap_params();
425
426 // Any errors during unmapping/closing are ignored as the only way
427 // to report them would be through panicking which is highly discouraged
428 // in Drop impls, c.f. https://github.com/rust-lang/lang-team/issues/97
429 unsafe { libc::munmap(addr:ptr, len as libc::size_t) };
430 }
431}
432
433unsafe impl Sync for MmapInner {}
434unsafe impl Send for MmapInner {}
435
436fn page_size() -> usize {
437 static PAGE_SIZE: AtomicUsize = AtomicUsize::new(0);
438
439 match PAGE_SIZE.load(order:Ordering::Relaxed) {
440 0 => {
441 let page_size: usize = unsafe { libc::sysconf(name:libc::_SC_PAGESIZE) as usize };
442
443 PAGE_SIZE.store(val:page_size, order:Ordering::Relaxed);
444
445 page_size
446 }
447 page_size: usize => page_size,
448 }
449}
450
451pub fn file_len(file: RawFd) -> io::Result<u64> {
452 // SAFETY: We must not close the passed-in fd by dropping the File we create,
453 // we ensure this by immediately wrapping it in a ManuallyDrop.
454 unsafe {
455 let file: ManuallyDrop = ManuallyDrop::new(File::from_raw_fd(file));
456 Ok(file.metadata()?.len())
457 }
458}
459