1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Userfaultfd unit tests. |
4 | * |
5 | * Copyright (C) 2015-2023 Red Hat, Inc. |
6 | */ |
7 | |
8 | #include "uffd-common.h" |
9 | |
10 | #include "../../../../mm/gup_test.h" |
11 | |
12 | #ifdef __NR_userfaultfd |
13 | |
14 | /* The unit test doesn't need a large or random size, make it 32MB for now */ |
15 | #define UFFD_TEST_MEM_SIZE (32UL << 20) |
16 | |
17 | #define MEM_ANON BIT_ULL(0) |
18 | #define MEM_SHMEM BIT_ULL(1) |
19 | #define MEM_SHMEM_PRIVATE BIT_ULL(2) |
20 | #define MEM_HUGETLB BIT_ULL(3) |
21 | #define MEM_HUGETLB_PRIVATE BIT_ULL(4) |
22 | |
23 | #define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \ |
24 | MEM_HUGETLB | MEM_HUGETLB_PRIVATE) |
25 | |
26 | #define ALIGN_UP(x, align_to) \ |
27 | ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1))) |
28 | |
29 | struct mem_type { |
30 | const char *name; |
31 | unsigned int mem_flag; |
32 | uffd_test_ops_t *mem_ops; |
33 | bool shared; |
34 | }; |
35 | typedef struct mem_type mem_type_t; |
36 | |
37 | mem_type_t mem_types[] = { |
38 | { |
39 | .name = "anon" , |
40 | .mem_flag = MEM_ANON, |
41 | .mem_ops = &anon_uffd_test_ops, |
42 | .shared = false, |
43 | }, |
44 | { |
45 | .name = "shmem" , |
46 | .mem_flag = MEM_SHMEM, |
47 | .mem_ops = &shmem_uffd_test_ops, |
48 | .shared = true, |
49 | }, |
50 | { |
51 | .name = "shmem-private" , |
52 | .mem_flag = MEM_SHMEM_PRIVATE, |
53 | .mem_ops = &shmem_uffd_test_ops, |
54 | .shared = false, |
55 | }, |
56 | { |
57 | .name = "hugetlb" , |
58 | .mem_flag = MEM_HUGETLB, |
59 | .mem_ops = &hugetlb_uffd_test_ops, |
60 | .shared = true, |
61 | }, |
62 | { |
63 | .name = "hugetlb-private" , |
64 | .mem_flag = MEM_HUGETLB_PRIVATE, |
65 | .mem_ops = &hugetlb_uffd_test_ops, |
66 | .shared = false, |
67 | }, |
68 | }; |
69 | |
70 | /* Arguments to be passed over to each uffd unit test */ |
71 | struct uffd_test_args { |
72 | mem_type_t *mem_type; |
73 | }; |
74 | typedef struct uffd_test_args uffd_test_args_t; |
75 | |
76 | /* Returns: UFFD_TEST_* */ |
77 | typedef void (*uffd_test_fn)(uffd_test_args_t *); |
78 | |
79 | typedef struct { |
80 | const char *name; |
81 | uffd_test_fn uffd_fn; |
82 | unsigned int mem_targets; |
83 | uint64_t uffd_feature_required; |
84 | uffd_test_case_ops_t *test_case_ops; |
85 | } uffd_test_case_t; |
86 | |
87 | static void uffd_test_report(void) |
88 | { |
89 | printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n" , |
90 | ksft_get_pass_cnt(), |
91 | ksft_get_xskip_cnt(), |
92 | ksft_get_fail_cnt(), |
93 | ksft_test_num()); |
94 | } |
95 | |
96 | static void uffd_test_pass(void) |
97 | { |
98 | printf("done\n" ); |
99 | ksft_inc_pass_cnt(); |
100 | } |
101 | |
102 | #define uffd_test_start(...) do { \ |
103 | printf("Testing "); \ |
104 | printf(__VA_ARGS__); \ |
105 | printf("... "); \ |
106 | fflush(stdout); \ |
107 | } while (0) |
108 | |
109 | #define uffd_test_fail(...) do { \ |
110 | printf("failed [reason: "); \ |
111 | printf(__VA_ARGS__); \ |
112 | printf("]\n"); \ |
113 | ksft_inc_fail_cnt(); \ |
114 | } while (0) |
115 | |
116 | static void uffd_test_skip(const char *message) |
117 | { |
118 | printf("skipped [reason: %s]\n" , message); |
119 | ksft_inc_xskip_cnt(); |
120 | } |
121 | |
122 | /* |
123 | * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll |
124 | * return 1 even if some test failed as long as uffd supported, because in |
125 | * that case we still want to proceed with the rest uffd unit tests. |
126 | */ |
127 | static int test_uffd_api(bool use_dev) |
128 | { |
129 | struct uffdio_api uffdio_api; |
130 | int uffd; |
131 | |
132 | uffd_test_start("UFFDIO_API (with %s)" , |
133 | use_dev ? "/dev/userfaultfd" : "syscall" ); |
134 | |
135 | if (use_dev) |
136 | uffd = uffd_open_dev(UFFD_FLAGS); |
137 | else |
138 | uffd = uffd_open_sys(UFFD_FLAGS); |
139 | if (uffd < 0) { |
140 | uffd_test_skip("cannot open userfaultfd handle" ); |
141 | return 0; |
142 | } |
143 | |
144 | /* Test wrong UFFD_API */ |
145 | uffdio_api.api = 0xab; |
146 | uffdio_api.features = 0; |
147 | if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { |
148 | uffd_test_fail("UFFDIO_API should fail with wrong api but didn't" ); |
149 | goto out; |
150 | } |
151 | |
152 | /* Test wrong feature bit */ |
153 | uffdio_api.api = UFFD_API; |
154 | uffdio_api.features = BIT_ULL(63); |
155 | if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { |
156 | uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't" ); |
157 | goto out; |
158 | } |
159 | |
160 | /* Test normal UFFDIO_API */ |
161 | uffdio_api.api = UFFD_API; |
162 | uffdio_api.features = 0; |
163 | if (ioctl(uffd, UFFDIO_API, &uffdio_api)) { |
164 | uffd_test_fail("UFFDIO_API should succeed but failed" ); |
165 | goto out; |
166 | } |
167 | |
168 | /* Test double requests of UFFDIO_API with a random feature set */ |
169 | uffdio_api.features = BIT_ULL(0); |
170 | if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { |
171 | uffd_test_fail("UFFDIO_API should reject initialized uffd" ); |
172 | goto out; |
173 | } |
174 | |
175 | uffd_test_pass(); |
176 | out: |
177 | close(uffd); |
178 | /* We have a valid uffd handle */ |
179 | return 1; |
180 | } |
181 | |
182 | /* |
183 | * This function initializes the global variables. TODO: remove global |
184 | * vars and then remove this. |
185 | */ |
186 | static int |
187 | uffd_setup_environment(uffd_test_args_t *args, uffd_test_case_t *test, |
188 | mem_type_t *mem_type, const char **errmsg) |
189 | { |
190 | map_shared = mem_type->shared; |
191 | uffd_test_ops = mem_type->mem_ops; |
192 | uffd_test_case_ops = test->test_case_ops; |
193 | |
194 | if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) |
195 | page_size = default_huge_page_size(); |
196 | else |
197 | page_size = psize(); |
198 | |
199 | nr_pages = UFFD_TEST_MEM_SIZE / page_size; |
200 | /* TODO: remove this global var.. it's so ugly */ |
201 | nr_cpus = 1; |
202 | |
203 | /* Initialize test arguments */ |
204 | args->mem_type = mem_type; |
205 | |
206 | return uffd_test_ctx_init(test->uffd_feature_required, errmsg); |
207 | } |
208 | |
209 | static bool uffd_feature_supported(uffd_test_case_t *test) |
210 | { |
211 | uint64_t features; |
212 | |
213 | if (uffd_get_features(&features)) |
214 | return false; |
215 | |
216 | return (features & test->uffd_feature_required) == |
217 | test->uffd_feature_required; |
218 | } |
219 | |
220 | static int pagemap_open(void) |
221 | { |
222 | int fd = open("/proc/self/pagemap" , O_RDONLY); |
223 | |
224 | if (fd < 0) |
225 | err("open pagemap" ); |
226 | |
227 | return fd; |
228 | } |
229 | |
230 | /* This macro let __LINE__ works in err() */ |
231 | #define pagemap_check_wp(value, wp) do { \ |
232 | if (!!(value & PM_UFFD_WP) != wp) \ |
233 | err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \ |
234 | } while (0) |
235 | |
236 | typedef struct { |
237 | int parent_uffd, child_uffd; |
238 | } fork_event_args; |
239 | |
240 | static void *fork_event_consumer(void *data) |
241 | { |
242 | fork_event_args *args = data; |
243 | struct uffd_msg msg = { 0 }; |
244 | |
245 | /* Read until a full msg received */ |
246 | while (uffd_read_msg(args->parent_uffd, &msg)); |
247 | |
248 | if (msg.event != UFFD_EVENT_FORK) |
249 | err("wrong message: %u\n" , msg.event); |
250 | |
251 | /* Just to be properly freed later */ |
252 | args->child_uffd = msg.arg.fork.ufd; |
253 | return NULL; |
254 | } |
255 | |
256 | typedef struct { |
257 | int gup_fd; |
258 | bool pinned; |
259 | } pin_args; |
260 | |
261 | /* |
262 | * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired |
263 | * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy |
264 | * all needs of the test cases (e.g., trigger unshare, trigger fork() early |
265 | * CoW, etc.). |
266 | */ |
267 | static int pin_pages(pin_args *args, void *buffer, size_t size) |
268 | { |
269 | struct pin_longterm_test test = { |
270 | .addr = (uintptr_t)buffer, |
271 | .size = size, |
272 | /* Read-only pins */ |
273 | .flags = 0, |
274 | }; |
275 | |
276 | if (args->pinned) |
277 | err("already pinned" ); |
278 | |
279 | args->gup_fd = open("/sys/kernel/debug/gup_test" , O_RDWR); |
280 | if (args->gup_fd < 0) |
281 | return -errno; |
282 | |
283 | if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) { |
284 | /* Even if gup_test existed, can be an old gup_test / kernel */ |
285 | close(args->gup_fd); |
286 | return -errno; |
287 | } |
288 | args->pinned = true; |
289 | return 0; |
290 | } |
291 | |
292 | static void unpin_pages(pin_args *args) |
293 | { |
294 | if (!args->pinned) |
295 | err("unpin without pin first" ); |
296 | if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP)) |
297 | err("PIN_LONGTERM_TEST_STOP" ); |
298 | close(args->gup_fd); |
299 | args->pinned = false; |
300 | } |
301 | |
302 | static int pagemap_test_fork(int uffd, bool with_event, bool test_pin) |
303 | { |
304 | fork_event_args args = { .parent_uffd = uffd, .child_uffd = -1 }; |
305 | pthread_t thread; |
306 | pid_t child; |
307 | uint64_t value; |
308 | int fd, result; |
309 | |
310 | /* Prepare a thread to resolve EVENT_FORK */ |
311 | if (with_event) { |
312 | if (pthread_create(&thread, NULL, fork_event_consumer, &args)) |
313 | err("pthread_create()" ); |
314 | } |
315 | |
316 | child = fork(); |
317 | if (!child) { |
318 | /* Open the pagemap fd of the child itself */ |
319 | pin_args args = {}; |
320 | |
321 | fd = pagemap_open(); |
322 | |
323 | if (test_pin && pin_pages(&args, area_dst, page_size)) |
324 | /* |
325 | * Normally when reach here we have pinned in |
326 | * previous tests, so shouldn't fail anymore |
327 | */ |
328 | err("pin page failed in child" ); |
329 | |
330 | value = pagemap_get_entry(fd, area_dst); |
331 | /* |
332 | * After fork(), we should handle uffd-wp bit differently: |
333 | * |
334 | * (1) when with EVENT_FORK, it should persist |
335 | * (2) when without EVENT_FORK, it should be dropped |
336 | */ |
337 | pagemap_check_wp(value, with_event); |
338 | if (test_pin) |
339 | unpin_pages(&args); |
340 | /* Succeed */ |
341 | exit(0); |
342 | } |
343 | waitpid(child, &result, 0); |
344 | |
345 | if (with_event) { |
346 | if (pthread_join(thread, NULL)) |
347 | err("pthread_join()" ); |
348 | if (args.child_uffd < 0) |
349 | err("Didn't receive child uffd" ); |
350 | close(args.child_uffd); |
351 | } |
352 | |
353 | return result; |
354 | } |
355 | |
356 | static void uffd_wp_unpopulated_test(uffd_test_args_t *args) |
357 | { |
358 | uint64_t value; |
359 | int pagemap_fd; |
360 | |
361 | if (uffd_register(uffd, area_dst, nr_pages * page_size, |
362 | false, true, false)) |
363 | err("register failed" ); |
364 | |
365 | pagemap_fd = pagemap_open(); |
366 | |
367 | /* Test applying pte marker to anon unpopulated */ |
368 | wp_range(uffd, (uint64_t)area_dst, page_size, true); |
369 | value = pagemap_get_entry(pagemap_fd, area_dst); |
370 | pagemap_check_wp(value, true); |
371 | |
372 | /* Test unprotect on anon pte marker */ |
373 | wp_range(uffd, (uint64_t)area_dst, page_size, false); |
374 | value = pagemap_get_entry(pagemap_fd, area_dst); |
375 | pagemap_check_wp(value, false); |
376 | |
377 | /* Test zap on anon marker */ |
378 | wp_range(uffd, (uint64_t)area_dst, page_size, true); |
379 | if (madvise(area_dst, page_size, MADV_DONTNEED)) |
380 | err("madvise(MADV_DONTNEED) failed" ); |
381 | value = pagemap_get_entry(pagemap_fd, area_dst); |
382 | pagemap_check_wp(value, false); |
383 | |
384 | /* Test fault in after marker removed */ |
385 | *area_dst = 1; |
386 | value = pagemap_get_entry(pagemap_fd, area_dst); |
387 | pagemap_check_wp(value, false); |
388 | /* Drop it to make pte none again */ |
389 | if (madvise(area_dst, page_size, MADV_DONTNEED)) |
390 | err("madvise(MADV_DONTNEED) failed" ); |
391 | |
392 | /* Test read-zero-page upon pte marker */ |
393 | wp_range(uffd, (uint64_t)area_dst, page_size, true); |
394 | *(volatile char *)area_dst; |
395 | /* Drop it to make pte none again */ |
396 | if (madvise(area_dst, page_size, MADV_DONTNEED)) |
397 | err("madvise(MADV_DONTNEED) failed" ); |
398 | |
399 | uffd_test_pass(); |
400 | } |
401 | |
402 | static void uffd_wp_fork_test_common(uffd_test_args_t *args, |
403 | bool with_event) |
404 | { |
405 | int pagemap_fd; |
406 | uint64_t value; |
407 | |
408 | if (uffd_register(uffd, area_dst, nr_pages * page_size, |
409 | false, true, false)) |
410 | err("register failed" ); |
411 | |
412 | pagemap_fd = pagemap_open(); |
413 | |
414 | /* Touch the page */ |
415 | *area_dst = 1; |
416 | wp_range(uffd, (uint64_t)area_dst, page_size, true); |
417 | value = pagemap_get_entry(pagemap_fd, area_dst); |
418 | pagemap_check_wp(value, true); |
419 | if (pagemap_test_fork(uffd, with_event, false)) { |
420 | uffd_test_fail("Detected %s uffd-wp bit in child in present pte" , |
421 | with_event ? "missing" : "stall" ); |
422 | goto out; |
423 | } |
424 | |
425 | /* |
426 | * This is an attempt for zapping the pgtable so as to test the |
427 | * markers. |
428 | * |
429 | * For private mappings, PAGEOUT will only work on exclusive ptes |
430 | * (PM_MMAP_EXCLUSIVE) which we should satisfy. |
431 | * |
432 | * For shared, PAGEOUT may not work. Use DONTNEED instead which |
433 | * plays a similar role of zapping (rather than freeing the page) |
434 | * to expose pte markers. |
435 | */ |
436 | if (args->mem_type->shared) { |
437 | if (madvise(area_dst, page_size, MADV_DONTNEED)) |
438 | err("MADV_DONTNEED" ); |
439 | } else { |
440 | /* |
441 | * NOTE: ignore retval because private-hugetlb doesn't yet |
442 | * support swapping, so it could fail. |
443 | */ |
444 | madvise(area_dst, page_size, MADV_PAGEOUT); |
445 | } |
446 | |
447 | /* Uffd-wp should persist even swapped out */ |
448 | value = pagemap_get_entry(pagemap_fd, area_dst); |
449 | pagemap_check_wp(value, true); |
450 | if (pagemap_test_fork(uffd, with_event, false)) { |
451 | uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte" , |
452 | with_event ? "missing" : "stall" ); |
453 | goto out; |
454 | } |
455 | |
456 | /* Unprotect; this tests swap pte modifications */ |
457 | wp_range(uffd, (uint64_t)area_dst, page_size, false); |
458 | value = pagemap_get_entry(pagemap_fd, area_dst); |
459 | pagemap_check_wp(value, false); |
460 | |
461 | /* Fault in the page from disk */ |
462 | *area_dst = 2; |
463 | value = pagemap_get_entry(pagemap_fd, area_dst); |
464 | pagemap_check_wp(value, false); |
465 | uffd_test_pass(); |
466 | out: |
467 | if (uffd_unregister(uffd, area_dst, nr_pages * page_size)) |
468 | err("unregister failed" ); |
469 | close(pagemap_fd); |
470 | } |
471 | |
472 | static void uffd_wp_fork_test(uffd_test_args_t *args) |
473 | { |
474 | uffd_wp_fork_test_common(args, false); |
475 | } |
476 | |
477 | static void uffd_wp_fork_with_event_test(uffd_test_args_t *args) |
478 | { |
479 | uffd_wp_fork_test_common(args, true); |
480 | } |
481 | |
482 | static void uffd_wp_fork_pin_test_common(uffd_test_args_t *args, |
483 | bool with_event) |
484 | { |
485 | int pagemap_fd; |
486 | pin_args pin_args = {}; |
487 | |
488 | if (uffd_register(uffd, area_dst, page_size, false, true, false)) |
489 | err("register failed" ); |
490 | |
491 | pagemap_fd = pagemap_open(); |
492 | |
493 | /* Touch the page */ |
494 | *area_dst = 1; |
495 | wp_range(uffd, (uint64_t)area_dst, page_size, true); |
496 | |
497 | /* |
498 | * 1. First pin, then fork(). This tests fork() special path when |
499 | * doing early CoW if the page is private. |
500 | */ |
501 | if (pin_pages(&pin_args, area_dst, page_size)) { |
502 | uffd_test_skip("Possibly CONFIG_GUP_TEST missing " |
503 | "or unprivileged" ); |
504 | close(pagemap_fd); |
505 | uffd_unregister(uffd, area_dst, page_size); |
506 | return; |
507 | } |
508 | |
509 | if (pagemap_test_fork(uffd, with_event, false)) { |
510 | uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()" , |
511 | with_event ? "missing" : "stall" ); |
512 | unpin_pages(&pin_args); |
513 | goto out; |
514 | } |
515 | |
516 | unpin_pages(&pin_args); |
517 | |
518 | /* |
519 | * 2. First fork(), then pin (in the child, where test_pin==true). |
520 | * This tests COR, aka, page unsharing on private memories. |
521 | */ |
522 | if (pagemap_test_fork(uffd, with_event, true)) { |
523 | uffd_test_fail("Detected %s uffd-wp bit when RO pin" , |
524 | with_event ? "missing" : "stall" ); |
525 | goto out; |
526 | } |
527 | uffd_test_pass(); |
528 | out: |
529 | if (uffd_unregister(uffd, area_dst, page_size)) |
530 | err("register failed" ); |
531 | close(pagemap_fd); |
532 | } |
533 | |
534 | static void uffd_wp_fork_pin_test(uffd_test_args_t *args) |
535 | { |
536 | uffd_wp_fork_pin_test_common(args, false); |
537 | } |
538 | |
539 | static void uffd_wp_fork_pin_with_event_test(uffd_test_args_t *args) |
540 | { |
541 | uffd_wp_fork_pin_test_common(args, true); |
542 | } |
543 | |
544 | static void check_memory_contents(char *p) |
545 | { |
546 | unsigned long i, j; |
547 | uint8_t expected_byte; |
548 | |
549 | for (i = 0; i < nr_pages; ++i) { |
550 | expected_byte = ~((uint8_t)(i % ((uint8_t)-1))); |
551 | for (j = 0; j < page_size; j++) { |
552 | uint8_t v = *(uint8_t *)(p + (i * page_size) + j); |
553 | if (v != expected_byte) |
554 | err("unexpected page contents" ); |
555 | } |
556 | } |
557 | } |
558 | |
559 | static void uffd_minor_test_common(bool test_collapse, bool test_wp) |
560 | { |
561 | unsigned long p; |
562 | pthread_t uffd_mon; |
563 | char c; |
564 | struct uffd_args args = { 0 }; |
565 | |
566 | /* |
567 | * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing |
568 | * both do not make much sense. |
569 | */ |
570 | assert(!(test_collapse && test_wp)); |
571 | |
572 | if (uffd_register(uffd, area_dst_alias, nr_pages * page_size, |
573 | /* NOTE! MADV_COLLAPSE may not work with uffd-wp */ |
574 | false, test_wp, true)) |
575 | err("register failure" ); |
576 | |
577 | /* |
578 | * After registering with UFFD, populate the non-UFFD-registered side of |
579 | * the shared mapping. This should *not* trigger any UFFD minor faults. |
580 | */ |
581 | for (p = 0; p < nr_pages; ++p) |
582 | memset(area_dst + (p * page_size), p % ((uint8_t)-1), |
583 | page_size); |
584 | |
585 | args.apply_wp = test_wp; |
586 | if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) |
587 | err("uffd_poll_thread create" ); |
588 | |
589 | /* |
590 | * Read each of the pages back using the UFFD-registered mapping. We |
591 | * expect that the first time we touch a page, it will result in a minor |
592 | * fault. uffd_poll_thread will resolve the fault by bit-flipping the |
593 | * page's contents, and then issuing a CONTINUE ioctl. |
594 | */ |
595 | check_memory_contents(area_dst_alias); |
596 | |
597 | if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) |
598 | err("pipe write" ); |
599 | if (pthread_join(uffd_mon, NULL)) |
600 | err("join() failed" ); |
601 | |
602 | if (test_collapse) { |
603 | if (madvise(area_dst_alias, nr_pages * page_size, |
604 | MADV_COLLAPSE)) { |
605 | /* It's fine to fail for this one... */ |
606 | uffd_test_skip("MADV_COLLAPSE failed" ); |
607 | return; |
608 | } |
609 | |
610 | uffd_test_ops->check_pmd_mapping(area_dst, |
611 | nr_pages * page_size / |
612 | read_pmd_pagesize()); |
613 | /* |
614 | * This won't cause uffd-fault - it purely just makes sure there |
615 | * was no corruption. |
616 | */ |
617 | check_memory_contents(area_dst_alias); |
618 | } |
619 | |
620 | if (args.missing_faults != 0 || args.minor_faults != nr_pages) |
621 | uffd_test_fail("stats check error" ); |
622 | else |
623 | uffd_test_pass(); |
624 | } |
625 | |
626 | void uffd_minor_test(uffd_test_args_t *args) |
627 | { |
628 | uffd_minor_test_common(false, false); |
629 | } |
630 | |
631 | void uffd_minor_wp_test(uffd_test_args_t *args) |
632 | { |
633 | uffd_minor_test_common(false, true); |
634 | } |
635 | |
636 | void uffd_minor_collapse_test(uffd_test_args_t *args) |
637 | { |
638 | uffd_minor_test_common(true, false); |
639 | } |
640 | |
641 | static sigjmp_buf jbuf, *sigbuf; |
642 | |
643 | static void sighndl(int sig, siginfo_t *siginfo, void *ptr) |
644 | { |
645 | if (sig == SIGBUS) { |
646 | if (sigbuf) |
647 | siglongjmp(*sigbuf, 1); |
648 | abort(); |
649 | } |
650 | } |
651 | |
652 | /* |
653 | * For non-cooperative userfaultfd test we fork() a process that will |
654 | * generate pagefaults, will mremap the area monitored by the |
655 | * userfaultfd and at last this process will release the monitored |
656 | * area. |
657 | * For the anonymous and shared memory the area is divided into two |
658 | * parts, the first part is accessed before mremap, and the second |
659 | * part is accessed after mremap. Since hugetlbfs does not support |
660 | * mremap, the entire monitored area is accessed in a single pass for |
661 | * HUGETLB_TEST. |
662 | * The release of the pages currently generates event for shmem and |
663 | * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked |
664 | * for hugetlb. |
665 | * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register |
666 | * monitored area, generate pagefaults and test that signal is delivered. |
667 | * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2 |
668 | * test robustness use case - we release monitored area, fork a process |
669 | * that will generate pagefaults and verify signal is generated. |
670 | * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal |
671 | * feature. Using monitor thread, verify no userfault events are generated. |
672 | */ |
673 | static int faulting_process(int signal_test, bool wp) |
674 | { |
675 | unsigned long nr, i; |
676 | unsigned long long count; |
677 | unsigned long split_nr_pages; |
678 | unsigned long lastnr; |
679 | struct sigaction act; |
680 | volatile unsigned long signalled = 0; |
681 | |
682 | split_nr_pages = (nr_pages + 1) / 2; |
683 | |
684 | if (signal_test) { |
685 | sigbuf = &jbuf; |
686 | memset(&act, 0, sizeof(act)); |
687 | act.sa_sigaction = sighndl; |
688 | act.sa_flags = SA_SIGINFO; |
689 | if (sigaction(SIGBUS, &act, 0)) |
690 | err("sigaction" ); |
691 | lastnr = (unsigned long)-1; |
692 | } |
693 | |
694 | for (nr = 0; nr < split_nr_pages; nr++) { |
695 | volatile int steps = 1; |
696 | unsigned long offset = nr * page_size; |
697 | |
698 | if (signal_test) { |
699 | if (sigsetjmp(*sigbuf, 1) != 0) { |
700 | if (steps == 1 && nr == lastnr) |
701 | err("Signal repeated" ); |
702 | |
703 | lastnr = nr; |
704 | if (signal_test == 1) { |
705 | if (steps == 1) { |
706 | /* This is a MISSING request */ |
707 | steps++; |
708 | if (copy_page(uffd, offset, wp)) |
709 | signalled++; |
710 | } else { |
711 | /* This is a WP request */ |
712 | assert(steps == 2); |
713 | wp_range(uffd, |
714 | (__u64)area_dst + |
715 | offset, |
716 | page_size, false); |
717 | } |
718 | } else { |
719 | signalled++; |
720 | continue; |
721 | } |
722 | } |
723 | } |
724 | |
725 | count = *area_count(area_dst, nr); |
726 | if (count != count_verify[nr]) |
727 | err("nr %lu memory corruption %llu %llu\n" , |
728 | nr, count, count_verify[nr]); |
729 | /* |
730 | * Trigger write protection if there is by writing |
731 | * the same value back. |
732 | */ |
733 | *area_count(area_dst, nr) = count; |
734 | } |
735 | |
736 | if (signal_test) |
737 | return signalled != split_nr_pages; |
738 | |
739 | area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size, |
740 | MREMAP_MAYMOVE | MREMAP_FIXED, area_src); |
741 | if (area_dst == MAP_FAILED) |
742 | err("mremap" ); |
743 | /* Reset area_src since we just clobbered it */ |
744 | area_src = NULL; |
745 | |
746 | for (; nr < nr_pages; nr++) { |
747 | count = *area_count(area_dst, nr); |
748 | if (count != count_verify[nr]) { |
749 | err("nr %lu memory corruption %llu %llu\n" , |
750 | nr, count, count_verify[nr]); |
751 | } |
752 | /* |
753 | * Trigger write protection if there is by writing |
754 | * the same value back. |
755 | */ |
756 | *area_count(area_dst, nr) = count; |
757 | } |
758 | |
759 | uffd_test_ops->release_pages(area_dst); |
760 | |
761 | for (nr = 0; nr < nr_pages; nr++) |
762 | for (i = 0; i < page_size; i++) |
763 | if (*(area_dst + nr * page_size + i) != 0) |
764 | err("page %lu offset %lu is not zero" , nr, i); |
765 | |
766 | return 0; |
767 | } |
768 | |
769 | static void uffd_sigbus_test_common(bool wp) |
770 | { |
771 | unsigned long userfaults; |
772 | pthread_t uffd_mon; |
773 | pid_t pid; |
774 | int err; |
775 | char c; |
776 | struct uffd_args args = { 0 }; |
777 | |
778 | ready_for_fork = false; |
779 | |
780 | fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); |
781 | |
782 | if (uffd_register(uffd, area_dst, nr_pages * page_size, |
783 | true, wp, false)) |
784 | err("register failure" ); |
785 | |
786 | if (faulting_process(1, wp)) |
787 | err("faulting process failed" ); |
788 | |
789 | uffd_test_ops->release_pages(area_dst); |
790 | |
791 | args.apply_wp = wp; |
792 | if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) |
793 | err("uffd_poll_thread create" ); |
794 | |
795 | while (!ready_for_fork) |
796 | ; /* Wait for the poll_thread to start executing before forking */ |
797 | |
798 | pid = fork(); |
799 | if (pid < 0) |
800 | err("fork" ); |
801 | |
802 | if (!pid) |
803 | exit(faulting_process(2, wp)); |
804 | |
805 | waitpid(pid, &err, 0); |
806 | if (err) |
807 | err("faulting process failed" ); |
808 | if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) |
809 | err("pipe write" ); |
810 | if (pthread_join(uffd_mon, (void **)&userfaults)) |
811 | err("pthread_join()" ); |
812 | |
813 | if (userfaults) |
814 | uffd_test_fail("Signal test failed, userfaults: %ld" , userfaults); |
815 | else |
816 | uffd_test_pass(); |
817 | } |
818 | |
819 | static void uffd_sigbus_test(uffd_test_args_t *args) |
820 | { |
821 | uffd_sigbus_test_common(false); |
822 | } |
823 | |
824 | static void uffd_sigbus_wp_test(uffd_test_args_t *args) |
825 | { |
826 | uffd_sigbus_test_common(true); |
827 | } |
828 | |
829 | static void uffd_events_test_common(bool wp) |
830 | { |
831 | pthread_t uffd_mon; |
832 | pid_t pid; |
833 | int err; |
834 | char c; |
835 | struct uffd_args args = { 0 }; |
836 | |
837 | ready_for_fork = false; |
838 | |
839 | fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); |
840 | if (uffd_register(uffd, area_dst, nr_pages * page_size, |
841 | true, wp, false)) |
842 | err("register failure" ); |
843 | |
844 | args.apply_wp = wp; |
845 | if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) |
846 | err("uffd_poll_thread create" ); |
847 | |
848 | while (!ready_for_fork) |
849 | ; /* Wait for the poll_thread to start executing before forking */ |
850 | |
851 | pid = fork(); |
852 | if (pid < 0) |
853 | err("fork" ); |
854 | |
855 | if (!pid) |
856 | exit(faulting_process(0, wp)); |
857 | |
858 | waitpid(pid, &err, 0); |
859 | if (err) |
860 | err("faulting process failed" ); |
861 | if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) |
862 | err("pipe write" ); |
863 | if (pthread_join(uffd_mon, NULL)) |
864 | err("pthread_join()" ); |
865 | |
866 | if (args.missing_faults != nr_pages) |
867 | uffd_test_fail("Fault counts wrong" ); |
868 | else |
869 | uffd_test_pass(); |
870 | } |
871 | |
872 | static void uffd_events_test(uffd_test_args_t *args) |
873 | { |
874 | uffd_events_test_common(false); |
875 | } |
876 | |
877 | static void uffd_events_wp_test(uffd_test_args_t *args) |
878 | { |
879 | uffd_events_test_common(true); |
880 | } |
881 | |
882 | static void retry_uffdio_zeropage(int ufd, |
883 | struct uffdio_zeropage *uffdio_zeropage) |
884 | { |
885 | uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start, |
886 | uffdio_zeropage->range.len, |
887 | 0); |
888 | if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) { |
889 | if (uffdio_zeropage->zeropage != -EEXIST) |
890 | err("UFFDIO_ZEROPAGE error: %" PRId64, |
891 | (int64_t)uffdio_zeropage->zeropage); |
892 | } else { |
893 | err("UFFDIO_ZEROPAGE error: %" PRId64, |
894 | (int64_t)uffdio_zeropage->zeropage); |
895 | } |
896 | } |
897 | |
898 | static bool do_uffdio_zeropage(int ufd, bool has_zeropage) |
899 | { |
900 | struct uffdio_zeropage uffdio_zeropage = { 0 }; |
901 | int ret; |
902 | __s64 res; |
903 | |
904 | uffdio_zeropage.range.start = (unsigned long) area_dst; |
905 | uffdio_zeropage.range.len = page_size; |
906 | uffdio_zeropage.mode = 0; |
907 | ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage); |
908 | res = uffdio_zeropage.zeropage; |
909 | if (ret) { |
910 | /* real retval in ufdio_zeropage.zeropage */ |
911 | if (has_zeropage) |
912 | err("UFFDIO_ZEROPAGE error: %" PRId64, (int64_t)res); |
913 | else if (res != -EINVAL) |
914 | err("UFFDIO_ZEROPAGE not -EINVAL" ); |
915 | } else if (has_zeropage) { |
916 | if (res != page_size) |
917 | err("UFFDIO_ZEROPAGE unexpected size" ); |
918 | else |
919 | retry_uffdio_zeropage(ufd, &uffdio_zeropage); |
920 | return true; |
921 | } else |
922 | err("UFFDIO_ZEROPAGE succeeded" ); |
923 | |
924 | return false; |
925 | } |
926 | |
927 | /* |
928 | * Registers a range with MISSING mode only for zeropage test. Return true |
929 | * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register() |
930 | * because we want to detect .ioctls along the way. |
931 | */ |
932 | static bool |
933 | uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len) |
934 | { |
935 | uint64_t ioctls = 0; |
936 | |
937 | if (uffd_register_with_ioctls(uffd, addr, len, true, |
938 | false, false, &ioctls)) |
939 | err("zeropage register fail" ); |
940 | |
941 | return ioctls & (1 << _UFFDIO_ZEROPAGE); |
942 | } |
943 | |
944 | /* exercise UFFDIO_ZEROPAGE */ |
945 | static void uffd_zeropage_test(uffd_test_args_t *args) |
946 | { |
947 | bool has_zeropage; |
948 | int i; |
949 | |
950 | has_zeropage = uffd_register_detect_zeropage(uffd, area_dst, page_size); |
951 | if (area_dst_alias) |
952 | /* Ignore the retval; we already have it */ |
953 | uffd_register_detect_zeropage(uffd, area_dst_alias, page_size); |
954 | |
955 | if (do_uffdio_zeropage(uffd, has_zeropage)) |
956 | for (i = 0; i < page_size; i++) |
957 | if (area_dst[i] != 0) |
958 | err("data non-zero at offset %d\n" , i); |
959 | |
960 | if (uffd_unregister(uffd, area_dst, page_size)) |
961 | err("unregister" ); |
962 | |
963 | if (area_dst_alias && uffd_unregister(uffd, area_dst_alias, page_size)) |
964 | err("unregister" ); |
965 | |
966 | uffd_test_pass(); |
967 | } |
968 | |
969 | static void uffd_register_poison(int uffd, void *addr, uint64_t len) |
970 | { |
971 | uint64_t ioctls = 0; |
972 | uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON); |
973 | |
974 | if (uffd_register_with_ioctls(uffd, addr, len, true, |
975 | false, false, &ioctls)) |
976 | err("poison register fail" ); |
977 | |
978 | if ((ioctls & expected) != expected) |
979 | err("registered area doesn't support COPY and POISON ioctls" ); |
980 | } |
981 | |
982 | static void do_uffdio_poison(int uffd, unsigned long offset) |
983 | { |
984 | struct uffdio_poison uffdio_poison = { 0 }; |
985 | int ret; |
986 | __s64 res; |
987 | |
988 | uffdio_poison.range.start = (unsigned long) area_dst + offset; |
989 | uffdio_poison.range.len = page_size; |
990 | uffdio_poison.mode = 0; |
991 | ret = ioctl(uffd, UFFDIO_POISON, &uffdio_poison); |
992 | res = uffdio_poison.updated; |
993 | |
994 | if (ret) |
995 | err("UFFDIO_POISON error: %" PRId64, (int64_t)res); |
996 | else if (res != page_size) |
997 | err("UFFDIO_POISON unexpected size: %" PRId64, (int64_t)res); |
998 | } |
999 | |
1000 | static void uffd_poison_handle_fault( |
1001 | struct uffd_msg *msg, struct uffd_args *args) |
1002 | { |
1003 | unsigned long offset; |
1004 | |
1005 | if (msg->event != UFFD_EVENT_PAGEFAULT) |
1006 | err("unexpected msg event %u" , msg->event); |
1007 | |
1008 | if (msg->arg.pagefault.flags & |
1009 | (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR)) |
1010 | err("unexpected fault type %llu" , msg->arg.pagefault.flags); |
1011 | |
1012 | offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; |
1013 | offset &= ~(page_size-1); |
1014 | |
1015 | /* Odd pages -> copy zeroed page; even pages -> poison. */ |
1016 | if (offset & page_size) |
1017 | copy_page(uffd, offset, false); |
1018 | else |
1019 | do_uffdio_poison(uffd, offset); |
1020 | } |
1021 | |
1022 | static void uffd_poison_test(uffd_test_args_t *targs) |
1023 | { |
1024 | pthread_t uffd_mon; |
1025 | char c; |
1026 | struct uffd_args args = { 0 }; |
1027 | struct sigaction act = { 0 }; |
1028 | unsigned long nr_sigbus = 0; |
1029 | unsigned long nr; |
1030 | |
1031 | fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK); |
1032 | |
1033 | uffd_register_poison(uffd, area_dst, nr_pages * page_size); |
1034 | memset(area_src, 0, nr_pages * page_size); |
1035 | |
1036 | args.handle_fault = uffd_poison_handle_fault; |
1037 | if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) |
1038 | err("uffd_poll_thread create" ); |
1039 | |
1040 | sigbuf = &jbuf; |
1041 | act.sa_sigaction = sighndl; |
1042 | act.sa_flags = SA_SIGINFO; |
1043 | if (sigaction(SIGBUS, &act, 0)) |
1044 | err("sigaction" ); |
1045 | |
1046 | for (nr = 0; nr < nr_pages; ++nr) { |
1047 | unsigned long offset = nr * page_size; |
1048 | const char *bytes = (const char *) area_dst + offset; |
1049 | const char *i; |
1050 | |
1051 | if (sigsetjmp(*sigbuf, 1)) { |
1052 | /* |
1053 | * Access below triggered a SIGBUS, which was caught by |
1054 | * sighndl, which then jumped here. Count this SIGBUS, |
1055 | * and move on to next page. |
1056 | */ |
1057 | ++nr_sigbus; |
1058 | continue; |
1059 | } |
1060 | |
1061 | for (i = bytes; i < bytes + page_size; ++i) { |
1062 | if (*i) |
1063 | err("nonzero byte in area_dst (%p) at %p: %u" , |
1064 | area_dst, i, *i); |
1065 | } |
1066 | } |
1067 | |
1068 | if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) |
1069 | err("pipe write" ); |
1070 | if (pthread_join(uffd_mon, NULL)) |
1071 | err("pthread_join()" ); |
1072 | |
1073 | if (nr_sigbus != nr_pages / 2) |
1074 | err("expected to receive %lu SIGBUS, actually received %lu" , |
1075 | nr_pages / 2, nr_sigbus); |
1076 | |
1077 | uffd_test_pass(); |
1078 | } |
1079 | |
1080 | static void |
1081 | uffd_move_handle_fault_common(struct uffd_msg *msg, struct uffd_args *args, |
1082 | unsigned long len) |
1083 | { |
1084 | unsigned long offset; |
1085 | |
1086 | if (msg->event != UFFD_EVENT_PAGEFAULT) |
1087 | err("unexpected msg event %u" , msg->event); |
1088 | |
1089 | if (msg->arg.pagefault.flags & |
1090 | (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE)) |
1091 | err("unexpected fault type %llu" , msg->arg.pagefault.flags); |
1092 | |
1093 | offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst; |
1094 | offset &= ~(len-1); |
1095 | |
1096 | if (move_page(uffd, offset, len)) |
1097 | args->missing_faults++; |
1098 | } |
1099 | |
1100 | static void uffd_move_handle_fault(struct uffd_msg *msg, |
1101 | struct uffd_args *args) |
1102 | { |
1103 | uffd_move_handle_fault_common(msg, args, page_size); |
1104 | } |
1105 | |
1106 | static void uffd_move_pmd_handle_fault(struct uffd_msg *msg, |
1107 | struct uffd_args *args) |
1108 | { |
1109 | uffd_move_handle_fault_common(msg, args, read_pmd_pagesize()); |
1110 | } |
1111 | |
1112 | static void |
1113 | uffd_move_test_common(uffd_test_args_t *targs, unsigned long chunk_size, |
1114 | void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args)) |
1115 | { |
1116 | unsigned long nr; |
1117 | pthread_t uffd_mon; |
1118 | char c; |
1119 | unsigned long long count; |
1120 | struct uffd_args args = { 0 }; |
1121 | char *orig_area_src, *orig_area_dst; |
1122 | unsigned long step_size, step_count; |
1123 | unsigned long src_offs = 0; |
1124 | unsigned long dst_offs = 0; |
1125 | |
1126 | /* Prevent source pages from being mapped more than once */ |
1127 | if (madvise(area_src, nr_pages * page_size, MADV_DONTFORK)) |
1128 | err("madvise(MADV_DONTFORK) failure" ); |
1129 | |
1130 | if (uffd_register(uffd, area_dst, nr_pages * page_size, |
1131 | true, false, false)) |
1132 | err("register failure" ); |
1133 | |
1134 | args.handle_fault = handle_fault; |
1135 | if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) |
1136 | err("uffd_poll_thread create" ); |
1137 | |
1138 | step_size = chunk_size / page_size; |
1139 | step_count = nr_pages / step_size; |
1140 | |
1141 | if (chunk_size > page_size) { |
1142 | char *aligned_src = ALIGN_UP(area_src, chunk_size); |
1143 | char *aligned_dst = ALIGN_UP(area_dst, chunk_size); |
1144 | |
1145 | if (aligned_src != area_src || aligned_dst != area_dst) { |
1146 | src_offs = (aligned_src - area_src) / page_size; |
1147 | dst_offs = (aligned_dst - area_dst) / page_size; |
1148 | step_count--; |
1149 | } |
1150 | orig_area_src = area_src; |
1151 | orig_area_dst = area_dst; |
1152 | area_src = aligned_src; |
1153 | area_dst = aligned_dst; |
1154 | } |
1155 | |
1156 | /* |
1157 | * Read each of the pages back using the UFFD-registered mapping. We |
1158 | * expect that the first time we touch a page, it will result in a missing |
1159 | * fault. uffd_poll_thread will resolve the fault by moving source |
1160 | * page to destination. |
1161 | */ |
1162 | for (nr = 0; nr < step_count * step_size; nr += step_size) { |
1163 | unsigned long i; |
1164 | |
1165 | /* Check area_src content */ |
1166 | for (i = 0; i < step_size; i++) { |
1167 | count = *area_count(area_src, nr + i); |
1168 | if (count != count_verify[src_offs + nr + i]) |
1169 | err("nr %lu source memory invalid %llu %llu\n" , |
1170 | nr + i, count, count_verify[src_offs + nr + i]); |
1171 | } |
1172 | |
1173 | /* Faulting into area_dst should move the page or the huge page */ |
1174 | for (i = 0; i < step_size; i++) { |
1175 | count = *area_count(area_dst, nr + i); |
1176 | if (count != count_verify[dst_offs + nr + i]) |
1177 | err("nr %lu memory corruption %llu %llu\n" , |
1178 | nr, count, count_verify[dst_offs + nr + i]); |
1179 | } |
1180 | |
1181 | /* Re-check area_src content which should be empty */ |
1182 | for (i = 0; i < step_size; i++) { |
1183 | count = *area_count(area_src, nr + i); |
1184 | if (count != 0) |
1185 | err("nr %lu move failed %llu %llu\n" , |
1186 | nr, count, count_verify[src_offs + nr + i]); |
1187 | } |
1188 | } |
1189 | if (step_size > page_size) { |
1190 | area_src = orig_area_src; |
1191 | area_dst = orig_area_dst; |
1192 | } |
1193 | |
1194 | if (write(pipefd[1], &c, sizeof(c)) != sizeof(c)) |
1195 | err("pipe write" ); |
1196 | if (pthread_join(uffd_mon, NULL)) |
1197 | err("join() failed" ); |
1198 | |
1199 | if (args.missing_faults != step_count || args.minor_faults != 0) |
1200 | uffd_test_fail("stats check error" ); |
1201 | else |
1202 | uffd_test_pass(); |
1203 | } |
1204 | |
1205 | static void uffd_move_test(uffd_test_args_t *targs) |
1206 | { |
1207 | uffd_move_test_common(targs, page_size, uffd_move_handle_fault); |
1208 | } |
1209 | |
1210 | static void uffd_move_pmd_test(uffd_test_args_t *targs) |
1211 | { |
1212 | if (madvise(area_dst, nr_pages * page_size, MADV_HUGEPAGE)) |
1213 | err("madvise(MADV_HUGEPAGE) failure" ); |
1214 | uffd_move_test_common(targs, read_pmd_pagesize(), |
1215 | uffd_move_pmd_handle_fault); |
1216 | } |
1217 | |
1218 | static void uffd_move_pmd_split_test(uffd_test_args_t *targs) |
1219 | { |
1220 | if (madvise(area_dst, nr_pages * page_size, MADV_NOHUGEPAGE)) |
1221 | err("madvise(MADV_NOHUGEPAGE) failure" ); |
1222 | uffd_move_test_common(targs, read_pmd_pagesize(), |
1223 | uffd_move_pmd_handle_fault); |
1224 | } |
1225 | |
1226 | static int prevent_hugepages(const char **errmsg) |
1227 | { |
1228 | /* This should be done before source area is populated */ |
1229 | if (madvise(area_src, nr_pages * page_size, MADV_NOHUGEPAGE)) { |
1230 | /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */ |
1231 | if (errno != EINVAL) { |
1232 | if (errmsg) |
1233 | *errmsg = "madvise(MADV_NOHUGEPAGE) failed" ; |
1234 | return -errno; |
1235 | } |
1236 | } |
1237 | return 0; |
1238 | } |
1239 | |
1240 | static int request_hugepages(const char **errmsg) |
1241 | { |
1242 | /* This should be done before source area is populated */ |
1243 | if (madvise(area_src, nr_pages * page_size, MADV_HUGEPAGE)) { |
1244 | if (errmsg) { |
1245 | *errmsg = (errno == EINVAL) ? |
1246 | "CONFIG_TRANSPARENT_HUGEPAGE is not set" : |
1247 | "madvise(MADV_HUGEPAGE) failed" ; |
1248 | } |
1249 | return -errno; |
1250 | } |
1251 | return 0; |
1252 | } |
1253 | |
1254 | struct uffd_test_case_ops uffd_move_test_case_ops = { |
1255 | .post_alloc = prevent_hugepages, |
1256 | }; |
1257 | |
1258 | struct uffd_test_case_ops uffd_move_test_pmd_case_ops = { |
1259 | .post_alloc = request_hugepages, |
1260 | }; |
1261 | |
1262 | /* |
1263 | * Test the returned uffdio_register.ioctls with different register modes. |
1264 | * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test. |
1265 | */ |
1266 | static void |
1267 | do_register_ioctls_test(uffd_test_args_t *args, bool miss, bool wp, bool minor) |
1268 | { |
1269 | uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE); |
1270 | mem_type_t *mem_type = args->mem_type; |
1271 | int ret; |
1272 | |
1273 | ret = uffd_register_with_ioctls(uffd, area_dst, page_size, |
1274 | miss, wp, minor, &ioctls); |
1275 | |
1276 | /* |
1277 | * Handle special cases of UFFDIO_REGISTER here where it should |
1278 | * just fail with -EINVAL first.. |
1279 | * |
1280 | * Case 1: register MINOR on anon |
1281 | * Case 2: register with no mode selected |
1282 | */ |
1283 | if ((minor && (mem_type->mem_flag == MEM_ANON)) || |
1284 | (!miss && !wp && !minor)) { |
1285 | if (ret != -EINVAL) |
1286 | err("register (miss=%d, wp=%d, minor=%d) failed " |
1287 | "with wrong errno=%d" , miss, wp, minor, ret); |
1288 | return; |
1289 | } |
1290 | |
1291 | /* UFFDIO_REGISTER should succeed, then check ioctls returned */ |
1292 | if (miss) |
1293 | expected |= BIT_ULL(_UFFDIO_COPY); |
1294 | if (wp) |
1295 | expected |= BIT_ULL(_UFFDIO_WRITEPROTECT); |
1296 | if (minor) |
1297 | expected |= BIT_ULL(_UFFDIO_CONTINUE); |
1298 | |
1299 | if ((ioctls & expected) != expected) |
1300 | err("unexpected uffdio_register.ioctls " |
1301 | "(miss=%d, wp=%d, minor=%d): expected=0x%" PRIx64", " |
1302 | "returned=0x%" PRIx64, miss, wp, minor, expected, ioctls); |
1303 | |
1304 | if (uffd_unregister(uffd, area_dst, page_size)) |
1305 | err("unregister" ); |
1306 | } |
1307 | |
1308 | static void uffd_register_ioctls_test(uffd_test_args_t *args) |
1309 | { |
1310 | int miss, wp, minor; |
1311 | |
1312 | for (miss = 0; miss <= 1; miss++) |
1313 | for (wp = 0; wp <= 1; wp++) |
1314 | for (minor = 0; minor <= 1; minor++) |
1315 | do_register_ioctls_test(args, miss, wp, minor); |
1316 | |
1317 | uffd_test_pass(); |
1318 | } |
1319 | |
1320 | uffd_test_case_t uffd_tests[] = { |
1321 | { |
1322 | /* Test returned uffdio_register.ioctls. */ |
1323 | .name = "register-ioctls" , |
1324 | .uffd_fn = uffd_register_ioctls_test, |
1325 | .mem_targets = MEM_ALL, |
1326 | .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS | |
1327 | UFFD_FEATURE_MISSING_SHMEM | |
1328 | UFFD_FEATURE_PAGEFAULT_FLAG_WP | |
1329 | UFFD_FEATURE_WP_HUGETLBFS_SHMEM | |
1330 | UFFD_FEATURE_MINOR_HUGETLBFS | |
1331 | UFFD_FEATURE_MINOR_SHMEM, |
1332 | }, |
1333 | { |
1334 | .name = "zeropage" , |
1335 | .uffd_fn = uffd_zeropage_test, |
1336 | .mem_targets = MEM_ALL, |
1337 | .uffd_feature_required = 0, |
1338 | }, |
1339 | { |
1340 | .name = "move" , |
1341 | .uffd_fn = uffd_move_test, |
1342 | .mem_targets = MEM_ANON, |
1343 | .uffd_feature_required = UFFD_FEATURE_MOVE, |
1344 | .test_case_ops = &uffd_move_test_case_ops, |
1345 | }, |
1346 | { |
1347 | .name = "move-pmd" , |
1348 | .uffd_fn = uffd_move_pmd_test, |
1349 | .mem_targets = MEM_ANON, |
1350 | .uffd_feature_required = UFFD_FEATURE_MOVE, |
1351 | .test_case_ops = &uffd_move_test_pmd_case_ops, |
1352 | }, |
1353 | { |
1354 | .name = "move-pmd-split" , |
1355 | .uffd_fn = uffd_move_pmd_split_test, |
1356 | .mem_targets = MEM_ANON, |
1357 | .uffd_feature_required = UFFD_FEATURE_MOVE, |
1358 | .test_case_ops = &uffd_move_test_pmd_case_ops, |
1359 | }, |
1360 | { |
1361 | .name = "wp-fork" , |
1362 | .uffd_fn = uffd_wp_fork_test, |
1363 | .mem_targets = MEM_ALL, |
1364 | .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | |
1365 | UFFD_FEATURE_WP_HUGETLBFS_SHMEM, |
1366 | }, |
1367 | { |
1368 | .name = "wp-fork-with-event" , |
1369 | .uffd_fn = uffd_wp_fork_with_event_test, |
1370 | .mem_targets = MEM_ALL, |
1371 | .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | |
1372 | UFFD_FEATURE_WP_HUGETLBFS_SHMEM | |
1373 | /* when set, child process should inherit uffd-wp bits */ |
1374 | UFFD_FEATURE_EVENT_FORK, |
1375 | }, |
1376 | { |
1377 | .name = "wp-fork-pin" , |
1378 | .uffd_fn = uffd_wp_fork_pin_test, |
1379 | .mem_targets = MEM_ALL, |
1380 | .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | |
1381 | UFFD_FEATURE_WP_HUGETLBFS_SHMEM, |
1382 | }, |
1383 | { |
1384 | .name = "wp-fork-pin-with-event" , |
1385 | .uffd_fn = uffd_wp_fork_pin_with_event_test, |
1386 | .mem_targets = MEM_ALL, |
1387 | .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | |
1388 | UFFD_FEATURE_WP_HUGETLBFS_SHMEM | |
1389 | /* when set, child process should inherit uffd-wp bits */ |
1390 | UFFD_FEATURE_EVENT_FORK, |
1391 | }, |
1392 | { |
1393 | .name = "wp-unpopulated" , |
1394 | .uffd_fn = uffd_wp_unpopulated_test, |
1395 | .mem_targets = MEM_ANON, |
1396 | .uffd_feature_required = |
1397 | UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED, |
1398 | }, |
1399 | { |
1400 | .name = "minor" , |
1401 | .uffd_fn = uffd_minor_test, |
1402 | .mem_targets = MEM_SHMEM | MEM_HUGETLB, |
1403 | .uffd_feature_required = |
1404 | UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM, |
1405 | }, |
1406 | { |
1407 | .name = "minor-wp" , |
1408 | .uffd_fn = uffd_minor_wp_test, |
1409 | .mem_targets = MEM_SHMEM | MEM_HUGETLB, |
1410 | .uffd_feature_required = |
1411 | UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM | |
1412 | UFFD_FEATURE_PAGEFAULT_FLAG_WP | |
1413 | /* |
1414 | * HACK: here we leveraged WP_UNPOPULATED to detect whether |
1415 | * minor mode supports wr-protect. There's no feature flag |
1416 | * for it so this is the best we can test against. |
1417 | */ |
1418 | UFFD_FEATURE_WP_UNPOPULATED, |
1419 | }, |
1420 | { |
1421 | .name = "minor-collapse" , |
1422 | .uffd_fn = uffd_minor_collapse_test, |
1423 | /* MADV_COLLAPSE only works with shmem */ |
1424 | .mem_targets = MEM_SHMEM, |
1425 | /* We can't test MADV_COLLAPSE, so try our luck */ |
1426 | .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM, |
1427 | }, |
1428 | { |
1429 | .name = "sigbus" , |
1430 | .uffd_fn = uffd_sigbus_test, |
1431 | .mem_targets = MEM_ALL, |
1432 | .uffd_feature_required = UFFD_FEATURE_SIGBUS | |
1433 | UFFD_FEATURE_EVENT_FORK, |
1434 | }, |
1435 | { |
1436 | .name = "sigbus-wp" , |
1437 | .uffd_fn = uffd_sigbus_wp_test, |
1438 | .mem_targets = MEM_ALL, |
1439 | .uffd_feature_required = UFFD_FEATURE_SIGBUS | |
1440 | UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP | |
1441 | UFFD_FEATURE_WP_HUGETLBFS_SHMEM, |
1442 | }, |
1443 | { |
1444 | .name = "events" , |
1445 | .uffd_fn = uffd_events_test, |
1446 | .mem_targets = MEM_ALL, |
1447 | .uffd_feature_required = UFFD_FEATURE_EVENT_FORK | |
1448 | UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE, |
1449 | }, |
1450 | { |
1451 | .name = "events-wp" , |
1452 | .uffd_fn = uffd_events_wp_test, |
1453 | .mem_targets = MEM_ALL, |
1454 | .uffd_feature_required = UFFD_FEATURE_EVENT_FORK | |
1455 | UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE | |
1456 | UFFD_FEATURE_PAGEFAULT_FLAG_WP | |
1457 | UFFD_FEATURE_WP_HUGETLBFS_SHMEM, |
1458 | }, |
1459 | { |
1460 | .name = "poison" , |
1461 | .uffd_fn = uffd_poison_test, |
1462 | .mem_targets = MEM_ALL, |
1463 | .uffd_feature_required = UFFD_FEATURE_POISON, |
1464 | }, |
1465 | }; |
1466 | |
1467 | static void usage(const char *prog) |
1468 | { |
1469 | printf("usage: %s [-f TESTNAME]\n" , prog); |
1470 | puts("" ); |
1471 | puts(" -f: test name to filter (e.g., event)" ); |
1472 | puts(" -h: show the help msg" ); |
1473 | puts(" -l: list tests only" ); |
1474 | puts("" ); |
1475 | exit(KSFT_FAIL); |
1476 | } |
1477 | |
1478 | int main(int argc, char *argv[]) |
1479 | { |
1480 | int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t); |
1481 | int n_mems = sizeof(mem_types) / sizeof(mem_type_t); |
1482 | const char *test_filter = NULL; |
1483 | bool list_only = false; |
1484 | uffd_test_case_t *test; |
1485 | mem_type_t *mem_type; |
1486 | uffd_test_args_t args; |
1487 | const char *errmsg; |
1488 | int has_uffd, opt; |
1489 | int i, j; |
1490 | |
1491 | while ((opt = getopt(argc, argv, "f:hl" )) != -1) { |
1492 | switch (opt) { |
1493 | case 'f': |
1494 | test_filter = optarg; |
1495 | break; |
1496 | case 'l': |
1497 | list_only = true; |
1498 | break; |
1499 | case 'h': |
1500 | default: |
1501 | /* Unknown */ |
1502 | usage(argv[0]); |
1503 | break; |
1504 | } |
1505 | } |
1506 | |
1507 | if (!test_filter && !list_only) { |
1508 | has_uffd = test_uffd_api(false); |
1509 | has_uffd |= test_uffd_api(true); |
1510 | |
1511 | if (!has_uffd) { |
1512 | printf("Userfaultfd not supported or unprivileged, skip all tests\n" ); |
1513 | exit(KSFT_SKIP); |
1514 | } |
1515 | } |
1516 | |
1517 | for (i = 0; i < n_tests; i++) { |
1518 | test = &uffd_tests[i]; |
1519 | if (test_filter && !strstr(test->name, test_filter)) |
1520 | continue; |
1521 | if (list_only) { |
1522 | printf("%s\n" , test->name); |
1523 | continue; |
1524 | } |
1525 | for (j = 0; j < n_mems; j++) { |
1526 | mem_type = &mem_types[j]; |
1527 | if (!(test->mem_targets & mem_type->mem_flag)) |
1528 | continue; |
1529 | |
1530 | uffd_test_start("%s on %s" , test->name, mem_type->name); |
1531 | if ((mem_type->mem_flag == MEM_HUGETLB || |
1532 | mem_type->mem_flag == MEM_HUGETLB_PRIVATE) && |
1533 | (default_huge_page_size() == 0)) { |
1534 | uffd_test_skip("huge page size is 0, feature missing?" ); |
1535 | continue; |
1536 | } |
1537 | if (!uffd_feature_supported(test)) { |
1538 | uffd_test_skip("feature missing" ); |
1539 | continue; |
1540 | } |
1541 | if (uffd_setup_environment(&args, test, mem_type, |
1542 | &errmsg)) { |
1543 | uffd_test_skip(errmsg); |
1544 | continue; |
1545 | } |
1546 | test->uffd_fn(&args); |
1547 | uffd_test_ctx_clear(); |
1548 | } |
1549 | } |
1550 | |
1551 | if (!list_only) |
1552 | uffd_test_report(); |
1553 | |
1554 | return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; |
1555 | } |
1556 | |
1557 | #else /* __NR_userfaultfd */ |
1558 | |
1559 | #warning "missing __NR_userfaultfd definition" |
1560 | |
1561 | int main(void) |
1562 | { |
1563 | printf("Skipping %s (missing __NR_userfaultfd)\n" , __file__); |
1564 | return KSFT_SKIP; |
1565 | } |
1566 | |
1567 | #endif /* __NR_userfaultfd */ |
1568 | |