1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Userfaultfd tests common header |
4 | * |
5 | * Copyright (C) 2015-2023 Red Hat, Inc. |
6 | */ |
7 | #ifndef __UFFD_COMMON_H__ |
8 | #define __UFFD_COMMON_H__ |
9 | |
10 | #define _GNU_SOURCE |
11 | #include <stdio.h> |
12 | #include <errno.h> |
13 | #include <unistd.h> |
14 | #include <stdlib.h> |
15 | #include <sys/types.h> |
16 | #include <sys/stat.h> |
17 | #include <fcntl.h> |
18 | #include <time.h> |
19 | #include <signal.h> |
20 | #include <poll.h> |
21 | #include <string.h> |
22 | #include <linux/mman.h> |
23 | #include <sys/mman.h> |
24 | #include <sys/syscall.h> |
25 | #include <sys/ioctl.h> |
26 | #include <sys/wait.h> |
27 | #include <pthread.h> |
28 | #include <linux/userfaultfd.h> |
29 | #include <setjmp.h> |
30 | #include <stdbool.h> |
31 | #include <assert.h> |
32 | #include <inttypes.h> |
33 | #include <stdint.h> |
34 | #include <sys/random.h> |
35 | #include <stdatomic.h> |
36 | |
37 | #include "../kselftest.h" |
38 | #include "vm_util.h" |
39 | |
40 | #define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY) |
41 | |
42 | #define _err(fmt, ...) \ |
43 | do { \ |
44 | int ret = errno; \ |
45 | fprintf(stderr, "ERROR: " fmt, ##__VA_ARGS__); \ |
46 | fprintf(stderr, " (errno=%d, @%s:%d)\n", \ |
47 | ret, __FILE__, __LINE__); \ |
48 | } while (0) |
49 | |
50 | #define errexit(exitcode, fmt, ...) \ |
51 | do { \ |
52 | _err(fmt, ##__VA_ARGS__); \ |
53 | exit(exitcode); \ |
54 | } while (0) |
55 | |
56 | #define err(fmt, ...) errexit(1, fmt, ##__VA_ARGS__) |
57 | |
58 | /* pthread_mutex_t starts at page offset 0 */ |
59 | #define area_mutex(___area, ___nr) \ |
60 | ((pthread_mutex_t *) ((___area) + (___nr)*page_size)) |
61 | /* |
62 | * count is placed in the page after pthread_mutex_t naturally aligned |
63 | * to avoid non alignment faults on non-x86 archs. |
64 | */ |
65 | #define area_count(___area, ___nr) \ |
66 | ((volatile unsigned long long *) ((unsigned long) \ |
67 | ((___area) + (___nr)*page_size + \ |
68 | sizeof(pthread_mutex_t) + \ |
69 | sizeof(unsigned long long) - 1) & \ |
70 | ~(unsigned long)(sizeof(unsigned long long) \ |
71 | - 1))) |
72 | |
73 | /* Userfaultfd test statistics */ |
74 | struct uffd_args { |
75 | int cpu; |
76 | /* Whether apply wr-protects when installing pages */ |
77 | bool apply_wp; |
78 | unsigned long missing_faults; |
79 | unsigned long wp_faults; |
80 | unsigned long minor_faults; |
81 | |
82 | /* A custom fault handler; defaults to uffd_handle_page_fault. */ |
83 | void (*handle_fault)(struct uffd_msg *msg, struct uffd_args *args); |
84 | }; |
85 | |
86 | struct uffd_test_ops { |
87 | int (*allocate_area)(void **alloc_area, bool is_src); |
88 | void (*release_pages)(char *rel_area); |
89 | void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset); |
90 | void (*check_pmd_mapping)(void *p, int expect_nr_hpages); |
91 | }; |
92 | typedef struct uffd_test_ops uffd_test_ops_t; |
93 | |
94 | struct uffd_test_case_ops { |
95 | int (*pre_alloc)(const char **errmsg); |
96 | int (*post_alloc)(const char **errmsg); |
97 | }; |
98 | typedef struct uffd_test_case_ops uffd_test_case_ops_t; |
99 | |
100 | extern unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size; |
101 | extern char *area_src, *area_src_alias, *area_dst, *area_dst_alias, *area_remap; |
102 | extern int uffd, uffd_flags, finished, *pipefd, test_type; |
103 | extern bool map_shared; |
104 | extern bool test_uffdio_wp; |
105 | extern unsigned long long *count_verify; |
106 | extern volatile bool test_uffdio_copy_eexist; |
107 | extern atomic_bool ready_for_fork; |
108 | |
109 | extern uffd_test_ops_t anon_uffd_test_ops; |
110 | extern uffd_test_ops_t shmem_uffd_test_ops; |
111 | extern uffd_test_ops_t hugetlb_uffd_test_ops; |
112 | extern uffd_test_ops_t *uffd_test_ops; |
113 | extern uffd_test_case_ops_t *uffd_test_case_ops; |
114 | |
115 | void uffd_stats_report(struct uffd_args *args, int n_cpus); |
116 | int uffd_test_ctx_init(uint64_t features, const char **errmsg); |
117 | void uffd_test_ctx_clear(void); |
118 | int userfaultfd_open(uint64_t *features); |
119 | int uffd_read_msg(int ufd, struct uffd_msg *msg); |
120 | void wp_range(int ufd, __u64 start, __u64 len, bool wp); |
121 | void uffd_handle_page_fault(struct uffd_msg *msg, struct uffd_args *args); |
122 | int __copy_page(int ufd, unsigned long offset, bool retry, bool wp); |
123 | int copy_page(int ufd, unsigned long offset, bool wp); |
124 | int move_page(int ufd, unsigned long offset, unsigned long len); |
125 | void *uffd_poll_thread(void *arg); |
126 | |
127 | int uffd_open_dev(unsigned int flags); |
128 | int uffd_open_sys(unsigned int flags); |
129 | int uffd_open(unsigned int flags); |
130 | int uffd_get_features(uint64_t *features); |
131 | |
132 | #define TEST_ANON 1 |
133 | #define TEST_HUGETLB 2 |
134 | #define TEST_SHMEM 3 |
135 | |
136 | #endif |
137 | |