1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (C) 2002 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) |
4 | */ |
5 | |
6 | #include <linux/err.h> |
7 | #include <linux/highmem.h> |
8 | #include <linux/mm.h> |
9 | #include <linux/module.h> |
10 | #include <linux/sched.h> |
11 | #include <asm/current.h> |
12 | #include <asm/page.h> |
13 | #include <kern_util.h> |
14 | #include <asm/futex.h> |
15 | #include <os.h> |
16 | |
17 | pte_t *virt_to_pte(struct mm_struct *mm, unsigned long addr) |
18 | { |
19 | pgd_t *pgd; |
20 | p4d_t *p4d; |
21 | pud_t *pud; |
22 | pmd_t *pmd; |
23 | |
24 | if (mm == NULL) |
25 | return NULL; |
26 | |
27 | pgd = pgd_offset(mm, addr); |
28 | if (!pgd_present(pgd: *pgd)) |
29 | return NULL; |
30 | |
31 | p4d = p4d_offset(pgd, address: addr); |
32 | if (!p4d_present(p4d: *p4d)) |
33 | return NULL; |
34 | |
35 | pud = pud_offset(p4d, address: addr); |
36 | if (!pud_present(pud: *pud)) |
37 | return NULL; |
38 | |
39 | pmd = pmd_offset(pud, address: addr); |
40 | if (!pmd_present(pmd: *pmd)) |
41 | return NULL; |
42 | |
43 | return pte_offset_kernel(pmd, address: addr); |
44 | } |
45 | |
46 | static pte_t *maybe_map(unsigned long virt, int is_write) |
47 | { |
48 | pte_t *pte = virt_to_pte(current->mm, addr: virt); |
49 | int err, dummy_code; |
50 | |
51 | if ((pte == NULL) || !pte_present(a: *pte) || |
52 | (is_write && !pte_write(pte: *pte))) { |
53 | err = handle_page_fault(virt, 0, is_write, 1, &dummy_code); |
54 | if (err) |
55 | return NULL; |
56 | pte = virt_to_pte(current->mm, addr: virt); |
57 | } |
58 | if (!pte_present(a: *pte)) |
59 | pte = NULL; |
60 | |
61 | return pte; |
62 | } |
63 | |
64 | static int do_op_one_page(unsigned long addr, int len, int is_write, |
65 | int (*op)(unsigned long addr, int len, void *arg), void *arg) |
66 | { |
67 | struct page *page; |
68 | pte_t *pte; |
69 | int n; |
70 | |
71 | pte = maybe_map(virt: addr, is_write); |
72 | if (pte == NULL) |
73 | return -1; |
74 | |
75 | page = pte_page(*pte); |
76 | #ifdef CONFIG_64BIT |
77 | pagefault_disable(); |
78 | addr = (unsigned long) page_address(page) + |
79 | (addr & ~PAGE_MASK); |
80 | #else |
81 | addr = (unsigned long) kmap_atomic(page) + |
82 | (addr & ~PAGE_MASK); |
83 | #endif |
84 | n = (*op)(addr, len, arg); |
85 | |
86 | #ifdef CONFIG_64BIT |
87 | pagefault_enable(); |
88 | #else |
89 | kunmap_atomic((void *)addr); |
90 | #endif |
91 | |
92 | return n; |
93 | } |
94 | |
95 | static long buffer_op(unsigned long addr, int len, int is_write, |
96 | int (*op)(unsigned long, int, void *), void *arg) |
97 | { |
98 | long size, remain, n; |
99 | |
100 | size = min(PAGE_ALIGN(addr) - addr, (unsigned long) len); |
101 | remain = len; |
102 | |
103 | n = do_op_one_page(addr, len: size, is_write, op, arg); |
104 | if (n != 0) { |
105 | remain = (n < 0 ? remain : 0); |
106 | goto out; |
107 | } |
108 | |
109 | addr += size; |
110 | remain -= size; |
111 | if (remain == 0) |
112 | goto out; |
113 | |
114 | while (addr < ((addr + remain) & PAGE_MASK)) { |
115 | n = do_op_one_page(addr, PAGE_SIZE, is_write, op, arg); |
116 | if (n != 0) { |
117 | remain = (n < 0 ? remain : 0); |
118 | goto out; |
119 | } |
120 | |
121 | addr += PAGE_SIZE; |
122 | remain -= PAGE_SIZE; |
123 | } |
124 | if (remain == 0) |
125 | goto out; |
126 | |
127 | n = do_op_one_page(addr, len: remain, is_write, op, arg); |
128 | if (n != 0) { |
129 | remain = (n < 0 ? remain : 0); |
130 | goto out; |
131 | } |
132 | |
133 | return 0; |
134 | out: |
135 | return remain; |
136 | } |
137 | |
138 | static int copy_chunk_from_user(unsigned long from, int len, void *arg) |
139 | { |
140 | unsigned long *to_ptr = arg, to = *to_ptr; |
141 | |
142 | memcpy((void *) to, (void *) from, len); |
143 | *to_ptr += len; |
144 | return 0; |
145 | } |
146 | |
147 | unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) |
148 | { |
149 | return buffer_op(addr: (unsigned long) from, len: n, is_write: 0, op: copy_chunk_from_user, arg: &to); |
150 | } |
151 | EXPORT_SYMBOL(raw_copy_from_user); |
152 | |
153 | static int copy_chunk_to_user(unsigned long to, int len, void *arg) |
154 | { |
155 | unsigned long *from_ptr = arg, from = *from_ptr; |
156 | |
157 | memcpy((void *) to, (void *) from, len); |
158 | *from_ptr += len; |
159 | return 0; |
160 | } |
161 | |
162 | unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) |
163 | { |
164 | return buffer_op(addr: (unsigned long) to, len: n, is_write: 1, op: copy_chunk_to_user, arg: &from); |
165 | } |
166 | EXPORT_SYMBOL(raw_copy_to_user); |
167 | |
168 | static int strncpy_chunk_from_user(unsigned long from, int len, void *arg) |
169 | { |
170 | char **to_ptr = arg, *to = *to_ptr; |
171 | int n; |
172 | |
173 | strncpy(p: to, q: (void *) from, size: len); |
174 | n = strnlen(p: to, maxlen: len); |
175 | *to_ptr += n; |
176 | |
177 | if (n < len) |
178 | return 1; |
179 | return 0; |
180 | } |
181 | |
182 | long strncpy_from_user(char *dst, const char __user *src, long count) |
183 | { |
184 | long n; |
185 | char *ptr = dst; |
186 | |
187 | if (!access_ok(src, 1)) |
188 | return -EFAULT; |
189 | n = buffer_op(addr: (unsigned long) src, len: count, is_write: 0, op: strncpy_chunk_from_user, |
190 | arg: &ptr); |
191 | if (n != 0) |
192 | return -EFAULT; |
193 | return strnlen(p: dst, maxlen: count); |
194 | } |
195 | EXPORT_SYMBOL(strncpy_from_user); |
196 | |
197 | static int clear_chunk(unsigned long addr, int len, void *unused) |
198 | { |
199 | memset((void *) addr, 0, len); |
200 | return 0; |
201 | } |
202 | |
203 | unsigned long __clear_user(void __user *mem, unsigned long len) |
204 | { |
205 | return buffer_op(addr: (unsigned long) mem, len, is_write: 1, op: clear_chunk, NULL); |
206 | } |
207 | EXPORT_SYMBOL(__clear_user); |
208 | |
209 | static int strnlen_chunk(unsigned long str, int len, void *arg) |
210 | { |
211 | int *len_ptr = arg, n; |
212 | |
213 | n = strnlen(p: (void *) str, maxlen: len); |
214 | *len_ptr += n; |
215 | |
216 | if (n < len) |
217 | return 1; |
218 | return 0; |
219 | } |
220 | |
221 | long strnlen_user(const char __user *str, long len) |
222 | { |
223 | int count = 0, n; |
224 | |
225 | if (!access_ok(str, 1)) |
226 | return -EFAULT; |
227 | n = buffer_op(addr: (unsigned long) str, len, is_write: 0, op: strnlen_chunk, arg: &count); |
228 | if (n == 0) |
229 | return count + 1; |
230 | return 0; |
231 | } |
232 | EXPORT_SYMBOL(strnlen_user); |
233 | |
234 | /** |
235 | * arch_futex_atomic_op_inuser() - Atomic arithmetic operation with constant |
236 | * argument and comparison of the previous |
237 | * futex value with another constant. |
238 | * |
239 | * @encoded_op: encoded operation to execute |
240 | * @uaddr: pointer to user space address |
241 | * |
242 | * Return: |
243 | * 0 - On success |
244 | * -EFAULT - User access resulted in a page fault |
245 | * -EAGAIN - Atomic operation was unable to complete due to contention |
246 | * -ENOSYS - Operation not supported |
247 | */ |
248 | |
249 | int arch_futex_atomic_op_inuser(int op, u32 oparg, int *oval, u32 __user *uaddr) |
250 | { |
251 | int oldval, ret; |
252 | struct page *page; |
253 | unsigned long addr = (unsigned long) uaddr; |
254 | pte_t *pte; |
255 | |
256 | ret = -EFAULT; |
257 | if (!access_ok(uaddr, sizeof(*uaddr))) |
258 | return -EFAULT; |
259 | preempt_disable(); |
260 | pte = maybe_map(virt: addr, is_write: 1); |
261 | if (pte == NULL) |
262 | goto out_inuser; |
263 | |
264 | page = pte_page(*pte); |
265 | #ifdef CONFIG_64BIT |
266 | pagefault_disable(); |
267 | addr = (unsigned long) page_address(page) + |
268 | (((unsigned long) addr) & ~PAGE_MASK); |
269 | #else |
270 | addr = (unsigned long) kmap_atomic(page) + |
271 | ((unsigned long) addr & ~PAGE_MASK); |
272 | #endif |
273 | uaddr = (u32 *) addr; |
274 | oldval = *uaddr; |
275 | |
276 | ret = 0; |
277 | |
278 | switch (op) { |
279 | case FUTEX_OP_SET: |
280 | *uaddr = oparg; |
281 | break; |
282 | case FUTEX_OP_ADD: |
283 | *uaddr += oparg; |
284 | break; |
285 | case FUTEX_OP_OR: |
286 | *uaddr |= oparg; |
287 | break; |
288 | case FUTEX_OP_ANDN: |
289 | *uaddr &= ~oparg; |
290 | break; |
291 | case FUTEX_OP_XOR: |
292 | *uaddr ^= oparg; |
293 | break; |
294 | default: |
295 | ret = -ENOSYS; |
296 | } |
297 | #ifdef CONFIG_64BIT |
298 | pagefault_enable(); |
299 | #else |
300 | kunmap_atomic((void *)addr); |
301 | #endif |
302 | |
303 | out_inuser: |
304 | preempt_enable(); |
305 | |
306 | if (ret == 0) |
307 | *oval = oldval; |
308 | |
309 | return ret; |
310 | } |
311 | EXPORT_SYMBOL(arch_futex_atomic_op_inuser); |
312 | |
313 | /** |
314 | * futex_atomic_cmpxchg_inatomic() - Compare and exchange the content of the |
315 | * uaddr with newval if the current value is |
316 | * oldval. |
317 | * @uval: pointer to store content of @uaddr |
318 | * @uaddr: pointer to user space address |
319 | * @oldval: old value |
320 | * @newval: new value to store to @uaddr |
321 | * |
322 | * Return: |
323 | * 0 - On success |
324 | * -EFAULT - User access resulted in a page fault |
325 | * -EAGAIN - Atomic operation was unable to complete due to contention |
326 | */ |
327 | |
328 | int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, |
329 | u32 oldval, u32 newval) |
330 | { |
331 | struct page *page; |
332 | pte_t *pte; |
333 | int ret = -EFAULT; |
334 | |
335 | if (!access_ok(uaddr, sizeof(*uaddr))) |
336 | return -EFAULT; |
337 | |
338 | preempt_disable(); |
339 | pte = maybe_map(virt: (unsigned long) uaddr, is_write: 1); |
340 | if (pte == NULL) |
341 | goto out_inatomic; |
342 | |
343 | page = pte_page(*pte); |
344 | #ifdef CONFIG_64BIT |
345 | pagefault_disable(); |
346 | uaddr = page_address(page) + (((unsigned long) uaddr) & ~PAGE_MASK); |
347 | #else |
348 | uaddr = kmap_atomic(page) + ((unsigned long) uaddr & ~PAGE_MASK); |
349 | #endif |
350 | |
351 | *uval = *uaddr; |
352 | |
353 | ret = cmpxchg(uaddr, oldval, newval); |
354 | |
355 | #ifdef CONFIG_64BIT |
356 | pagefault_enable(); |
357 | #else |
358 | kunmap_atomic(uaddr); |
359 | #endif |
360 | ret = 0; |
361 | |
362 | out_inatomic: |
363 | preempt_enable(); |
364 | return ret; |
365 | } |
366 | EXPORT_SYMBOL(futex_atomic_cmpxchg_inatomic); |
367 | |