1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | //! Kernel page allocation and management. |
4 | |
5 | use crate::{ |
6 | alloc::{AllocError, Flags}, |
7 | bindings, |
8 | error::code::*, |
9 | error::Result, |
10 | uaccess::UserSliceReader, |
11 | }; |
12 | use core::ptr::{self, NonNull}; |
13 | |
14 | /// A bitwise shift for the page size. |
15 | pub const PAGE_SHIFT: usize = bindings::PAGE_SHIFT as usize; |
16 | |
17 | /// The number of bytes in a page. |
18 | pub const PAGE_SIZE: usize = bindings::PAGE_SIZE; |
19 | |
20 | /// A bitmask that gives the page containing a given address. |
21 | pub const PAGE_MASK: usize = !(PAGE_SIZE - 1); |
22 | |
23 | /// Round up the given number to the next multiple of [`PAGE_SIZE`]. |
24 | /// |
25 | /// It is incorrect to pass an address where the next multiple of [`PAGE_SIZE`] doesn't fit in a |
26 | /// [`usize`]. |
27 | pub const fn page_align(addr: usize) -> usize { |
28 | // Parentheses around `PAGE_SIZE - 1` to avoid triggering overflow sanitizers in the wrong |
29 | // cases. |
30 | (addr + (PAGE_SIZE - 1)) & PAGE_MASK |
31 | } |
32 | |
33 | /// A pointer to a page that owns the page allocation. |
34 | /// |
35 | /// # Invariants |
36 | /// |
37 | /// The pointer is valid, and has ownership over the page. |
38 | pub struct Page { |
39 | page: NonNull<bindings::page>, |
40 | } |
41 | |
42 | // SAFETY: Pages have no logic that relies on them staying on a given thread, so moving them across |
43 | // threads is safe. |
44 | unsafe impl Send for Page {} |
45 | |
46 | // SAFETY: Pages have no logic that relies on them not being accessed concurrently, so accessing |
47 | // them concurrently is safe. |
48 | unsafe impl Sync for Page {} |
49 | |
50 | impl Page { |
51 | /// Allocates a new page. |
52 | /// |
53 | /// # Examples |
54 | /// |
55 | /// Allocate memory for a page. |
56 | /// |
57 | /// ``` |
58 | /// use kernel::page::Page; |
59 | /// |
60 | /// let page = Page::alloc_page(GFP_KERNEL)?; |
61 | /// # Ok::<(), kernel::alloc::AllocError>(()) |
62 | /// ``` |
63 | /// |
64 | /// Allocate memory for a page and zero its contents. |
65 | /// |
66 | /// ``` |
67 | /// use kernel::page::Page; |
68 | /// |
69 | /// let page = Page::alloc_page(GFP_KERNEL | __GFP_ZERO)?; |
70 | /// # Ok::<(), kernel::alloc::AllocError>(()) |
71 | /// ``` |
72 | #[inline] |
73 | pub fn alloc_page(flags: Flags) -> Result<Self, AllocError> { |
74 | // SAFETY: Depending on the value of `gfp_flags`, this call may sleep. Other than that, it |
75 | // is always safe to call this method. |
76 | let page = unsafe { bindings::alloc_pages(flags.as_raw(), 0) }; |
77 | let page = NonNull::new(page).ok_or(AllocError)?; |
78 | // INVARIANT: We just successfully allocated a page, so we now have ownership of the newly |
79 | // allocated page. We transfer that ownership to the new `Page` object. |
80 | Ok(Self { page }) |
81 | } |
82 | |
83 | /// Returns a raw pointer to the page. |
84 | pub fn as_ptr(&self) -> *mut bindings::page { |
85 | self.page.as_ptr() |
86 | } |
87 | |
88 | /// Runs a piece of code with this page mapped to an address. |
89 | /// |
90 | /// The page is unmapped when this call returns. |
91 | /// |
92 | /// # Using the raw pointer |
93 | /// |
94 | /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for |
95 | /// `PAGE_SIZE` bytes and for the duration in which the closure is called. The pointer might |
96 | /// only be mapped on the current thread, and when that is the case, dereferencing it on other |
97 | /// threads is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't |
98 | /// cause data races, the memory may be uninitialized, and so on. |
99 | /// |
100 | /// If multiple threads map the same page at the same time, then they may reference with |
101 | /// different addresses. However, even if the addresses are different, the underlying memory is |
102 | /// still the same for these purposes (e.g., it's still a data race if they both write to the |
103 | /// same underlying byte at the same time). |
104 | fn with_page_mapped<T>(&self, f: impl FnOnce(*mut u8) -> T) -> T { |
105 | // SAFETY: `page` is valid due to the type invariants on `Page`. |
106 | let mapped_addr = unsafe { bindings::kmap_local_page(self.as_ptr()) }; |
107 | |
108 | let res = f(mapped_addr.cast()); |
109 | |
110 | // This unmaps the page mapped above. |
111 | // |
112 | // SAFETY: Since this API takes the user code as a closure, it can only be used in a manner |
113 | // where the pages are unmapped in reverse order. This is as required by `kunmap_local`. |
114 | // |
115 | // In other words, if this call to `kunmap_local` happens when a different page should be |
116 | // unmapped first, then there must necessarily be a call to `kmap_local_page` other than the |
117 | // call just above in `with_page_mapped` that made that possible. In this case, it is the |
118 | // unsafe block that wraps that other call that is incorrect. |
119 | unsafe { bindings::kunmap_local(mapped_addr) }; |
120 | |
121 | res |
122 | } |
123 | |
124 | /// Runs a piece of code with a raw pointer to a slice of this page, with bounds checking. |
125 | /// |
126 | /// If `f` is called, then it will be called with a pointer that points at `off` bytes into the |
127 | /// page, and the pointer will be valid for at least `len` bytes. The pointer is only valid on |
128 | /// this task, as this method uses a local mapping. |
129 | /// |
130 | /// If `off` and `len` refers to a region outside of this page, then this method returns |
131 | /// [`EINVAL`] and does not call `f`. |
132 | /// |
133 | /// # Using the raw pointer |
134 | /// |
135 | /// It is up to the caller to use the provided raw pointer correctly. The pointer is valid for |
136 | /// `len` bytes and for the duration in which the closure is called. The pointer might only be |
137 | /// mapped on the current thread, and when that is the case, dereferencing it on other threads |
138 | /// is UB. Other than that, the usual rules for dereferencing a raw pointer apply: don't cause |
139 | /// data races, the memory may be uninitialized, and so on. |
140 | /// |
141 | /// If multiple threads map the same page at the same time, then they may reference with |
142 | /// different addresses. However, even if the addresses are different, the underlying memory is |
143 | /// still the same for these purposes (e.g., it's still a data race if they both write to the |
144 | /// same underlying byte at the same time). |
145 | fn with_pointer_into_page<T>( |
146 | &self, |
147 | off: usize, |
148 | len: usize, |
149 | f: impl FnOnce(*mut u8) -> Result<T>, |
150 | ) -> Result<T> { |
151 | let bounds_ok = off <= PAGE_SIZE && len <= PAGE_SIZE && (off + len) <= PAGE_SIZE; |
152 | |
153 | if bounds_ok { |
154 | self.with_page_mapped(move |page_addr| { |
155 | // SAFETY: The `off` integer is at most `PAGE_SIZE`, so this pointer offset will |
156 | // result in a pointer that is in bounds or one off the end of the page. |
157 | f(unsafe { page_addr.add(off) }) |
158 | }) |
159 | } else { |
160 | Err(EINVAL) |
161 | } |
162 | } |
163 | |
164 | /// Maps the page and reads from it into the given buffer. |
165 | /// |
166 | /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes |
167 | /// outside of the page, then this call returns [`EINVAL`]. |
168 | /// |
169 | /// # Safety |
170 | /// |
171 | /// * Callers must ensure that `dst` is valid for writing `len` bytes. |
172 | /// * Callers must ensure that this call does not race with a write to the same page that |
173 | /// overlaps with this read. |
174 | pub unsafe fn read_raw(&self, dst: *mut u8, offset: usize, len: usize) -> Result { |
175 | self.with_pointer_into_page(offset, len, move |src| { |
176 | // SAFETY: If `with_pointer_into_page` calls into this closure, then |
177 | // it has performed a bounds check and guarantees that `src` is |
178 | // valid for `len` bytes. |
179 | // |
180 | // There caller guarantees that there is no data race. |
181 | unsafe { ptr::copy_nonoverlapping(src, dst, len) }; |
182 | Ok(()) |
183 | }) |
184 | } |
185 | |
186 | /// Maps the page and writes into it from the given buffer. |
187 | /// |
188 | /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes |
189 | /// outside of the page, then this call returns [`EINVAL`]. |
190 | /// |
191 | /// # Safety |
192 | /// |
193 | /// * Callers must ensure that `src` is valid for reading `len` bytes. |
194 | /// * Callers must ensure that this call does not race with a read or write to the same page |
195 | /// that overlaps with this write. |
196 | pub unsafe fn write_raw(&self, src: *const u8, offset: usize, len: usize) -> Result { |
197 | self.with_pointer_into_page(offset, len, move |dst| { |
198 | // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a |
199 | // bounds check and guarantees that `dst` is valid for `len` bytes. |
200 | // |
201 | // There caller guarantees that there is no data race. |
202 | unsafe { ptr::copy_nonoverlapping(src, dst, len) }; |
203 | Ok(()) |
204 | }) |
205 | } |
206 | |
207 | /// Maps the page and zeroes the given slice. |
208 | /// |
209 | /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes |
210 | /// outside of the page, then this call returns [`EINVAL`]. |
211 | /// |
212 | /// # Safety |
213 | /// |
214 | /// Callers must ensure that this call does not race with a read or write to the same page that |
215 | /// overlaps with this write. |
216 | pub unsafe fn fill_zero_raw(&self, offset: usize, len: usize) -> Result { |
217 | self.with_pointer_into_page(offset, len, move |dst| { |
218 | // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a |
219 | // bounds check and guarantees that `dst` is valid for `len` bytes. |
220 | // |
221 | // There caller guarantees that there is no data race. |
222 | unsafe { ptr::write_bytes(dst, 0u8, len) }; |
223 | Ok(()) |
224 | }) |
225 | } |
226 | |
227 | /// Copies data from userspace into this page. |
228 | /// |
229 | /// This method will perform bounds checks on the page offset. If `offset .. offset+len` goes |
230 | /// outside of the page, then this call returns [`EINVAL`]. |
231 | /// |
232 | /// Like the other `UserSliceReader` methods, data races are allowed on the userspace address. |
233 | /// However, they are not allowed on the page you are copying into. |
234 | /// |
235 | /// # Safety |
236 | /// |
237 | /// Callers must ensure that this call does not race with a read or write to the same page that |
238 | /// overlaps with this write. |
239 | pub unsafe fn copy_from_user_slice_raw( |
240 | &self, |
241 | reader: &mut UserSliceReader, |
242 | offset: usize, |
243 | len: usize, |
244 | ) -> Result { |
245 | self.with_pointer_into_page(offset, len, move |dst| { |
246 | // SAFETY: If `with_pointer_into_page` calls into this closure, then it has performed a |
247 | // bounds check and guarantees that `dst` is valid for `len` bytes. Furthermore, we have |
248 | // exclusive access to the slice since the caller guarantees that there are no races. |
249 | reader.read_raw(unsafe { core::slice::from_raw_parts_mut(dst.cast(), len) }) |
250 | }) |
251 | } |
252 | } |
253 | |
254 | impl Drop for Page { |
255 | #[inline] |
256 | fn drop(&mut self) { |
257 | // SAFETY: By the type invariants, we have ownership of the page and can free it. |
258 | unsafe { bindings::__free_pages(self.page.as_ptr(), 0) }; |
259 | } |
260 | } |
261 | |