1 | #include <fcntl.h> |
2 | #include <errno.h> |
3 | #include <stdio.h> |
4 | #include <stdlib.h> |
5 | #include <unistd.h> |
6 | #include <dirent.h> |
7 | #include <sys/ioctl.h> |
8 | #include <sys/mman.h> |
9 | #include <sys/stat.h> |
10 | #include <sys/types.h> |
11 | #include <pthread.h> |
12 | #include <assert.h> |
13 | #include <mm/gup_test.h> |
14 | #include "../kselftest.h" |
15 | #include "vm_util.h" |
16 | |
17 | #define MB (1UL << 20) |
18 | |
19 | /* Just the flags we need, copied from mm.h: */ |
20 | #define FOLL_WRITE 0x01 /* check pte is writable */ |
21 | #define FOLL_TOUCH 0x02 /* mark page accessed */ |
22 | |
23 | #define GUP_TEST_FILE "/sys/kernel/debug/gup_test" |
24 | |
25 | static unsigned long cmd = GUP_FAST_BENCHMARK; |
26 | static int gup_fd, repeats = 1; |
27 | static unsigned long size = 128 * MB; |
28 | /* Serialize prints */ |
29 | static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER; |
30 | |
31 | static char *cmd_to_str(unsigned long cmd) |
32 | { |
33 | switch (cmd) { |
34 | case GUP_FAST_BENCHMARK: |
35 | return "GUP_FAST_BENCHMARK" ; |
36 | case PIN_FAST_BENCHMARK: |
37 | return "PIN_FAST_BENCHMARK" ; |
38 | case PIN_LONGTERM_BENCHMARK: |
39 | return "PIN_LONGTERM_BENCHMARK" ; |
40 | case GUP_BASIC_TEST: |
41 | return "GUP_BASIC_TEST" ; |
42 | case PIN_BASIC_TEST: |
43 | return "PIN_BASIC_TEST" ; |
44 | case DUMP_USER_PAGES_TEST: |
45 | return "DUMP_USER_PAGES_TEST" ; |
46 | } |
47 | return "Unknown command" ; |
48 | } |
49 | |
50 | void *gup_thread(void *data) |
51 | { |
52 | struct gup_test gup = *(struct gup_test *)data; |
53 | int i, status; |
54 | |
55 | /* Only report timing information on the *_BENCHMARK commands: */ |
56 | if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) || |
57 | (cmd == PIN_LONGTERM_BENCHMARK)) { |
58 | for (i = 0; i < repeats; i++) { |
59 | gup.size = size; |
60 | status = ioctl(gup_fd, cmd, &gup); |
61 | if (status) |
62 | break; |
63 | |
64 | pthread_mutex_lock(&print_mutex); |
65 | ksft_print_msg(msg: "%s: Time: get:%lld put:%lld us" , |
66 | cmd_to_str(cmd), gup.get_delta_usec, |
67 | gup.put_delta_usec); |
68 | if (gup.size != size) |
69 | ksft_print_msg(msg: ", truncated (size: %lld)" , gup.size); |
70 | ksft_print_msg(msg: "\n" ); |
71 | pthread_mutex_unlock(&print_mutex); |
72 | } |
73 | } else { |
74 | gup.size = size; |
75 | status = ioctl(gup_fd, cmd, &gup); |
76 | if (status) |
77 | goto return_; |
78 | |
79 | pthread_mutex_lock(&print_mutex); |
80 | ksft_print_msg(msg: "%s: done\n" , cmd_to_str(cmd)); |
81 | if (gup.size != size) |
82 | ksft_print_msg(msg: "Truncated (size: %lld)\n" , gup.size); |
83 | pthread_mutex_unlock(&print_mutex); |
84 | } |
85 | |
86 | return_: |
87 | ksft_test_result(!status, "ioctl status %d\n" , status); |
88 | return NULL; |
89 | } |
90 | |
91 | int main(int argc, char **argv) |
92 | { |
93 | struct gup_test gup = { 0 }; |
94 | int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret; |
95 | int flags = MAP_PRIVATE, touch = 0; |
96 | char *file = "/dev/zero" ; |
97 | pthread_t *tid; |
98 | char *p; |
99 | |
100 | while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz" )) != -1) { |
101 | switch (opt) { |
102 | case 'a': |
103 | cmd = PIN_FAST_BENCHMARK; |
104 | break; |
105 | case 'b': |
106 | cmd = PIN_BASIC_TEST; |
107 | break; |
108 | case 'L': |
109 | cmd = PIN_LONGTERM_BENCHMARK; |
110 | break; |
111 | case 'c': |
112 | cmd = DUMP_USER_PAGES_TEST; |
113 | /* |
114 | * Dump page 0 (index 1). May be overridden later, by |
115 | * user's non-option arguments. |
116 | * |
117 | * .which_pages is zero-based, so that zero can mean "do |
118 | * nothing". |
119 | */ |
120 | gup.which_pages[0] = 1; |
121 | break; |
122 | case 'p': |
123 | /* works only with DUMP_USER_PAGES_TEST */ |
124 | gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN; |
125 | break; |
126 | case 'F': |
127 | /* strtol, so you can pass flags in hex form */ |
128 | gup.gup_flags = strtol(optarg, 0, 0); |
129 | break; |
130 | case 'j': |
131 | nthreads = atoi(optarg); |
132 | break; |
133 | case 'm': |
134 | size = atoi(optarg) * MB; |
135 | break; |
136 | case 'r': |
137 | repeats = atoi(optarg); |
138 | break; |
139 | case 'n': |
140 | nr_pages = atoi(optarg); |
141 | break; |
142 | case 't': |
143 | thp = 1; |
144 | break; |
145 | case 'T': |
146 | thp = 0; |
147 | break; |
148 | case 'U': |
149 | cmd = GUP_BASIC_TEST; |
150 | break; |
151 | case 'u': |
152 | cmd = GUP_FAST_BENCHMARK; |
153 | break; |
154 | case 'w': |
155 | write = 1; |
156 | break; |
157 | case 'W': |
158 | write = 0; |
159 | break; |
160 | case 'f': |
161 | file = optarg; |
162 | break; |
163 | case 'S': |
164 | flags &= ~MAP_PRIVATE; |
165 | flags |= MAP_SHARED; |
166 | break; |
167 | case 'H': |
168 | flags |= (MAP_HUGETLB | MAP_ANONYMOUS); |
169 | break; |
170 | case 'z': |
171 | /* fault pages in gup, do not fault in userland */ |
172 | touch = 1; |
173 | break; |
174 | default: |
175 | ksft_exit_fail_msg(msg: "Wrong argument\n" ); |
176 | } |
177 | } |
178 | |
179 | if (optind < argc) { |
180 | int = 0; |
181 | /* |
182 | * For example: |
183 | * |
184 | * ./gup_test -c 0 1 0x1001 |
185 | * |
186 | * ...to dump pages 0, 1, and 4097 |
187 | */ |
188 | |
189 | while ((optind < argc) && |
190 | (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) { |
191 | /* |
192 | * Do the 1-based indexing here, so that the user can |
193 | * use normal 0-based indexing on the command line. |
194 | */ |
195 | long page_index = strtol(argv[optind], 0, 0) + 1; |
196 | |
197 | gup.which_pages[extra_arg_count] = page_index; |
198 | extra_arg_count++; |
199 | optind++; |
200 | } |
201 | } |
202 | |
203 | ksft_print_header(); |
204 | ksft_set_plan(plan: nthreads); |
205 | |
206 | filed = open(file, O_RDWR|O_CREAT, 0664); |
207 | if (filed < 0) |
208 | ksft_exit_fail_msg("Unable to open %s: %s\n" , file, strerror(errno)); |
209 | |
210 | gup.nr_pages_per_call = nr_pages; |
211 | if (write) |
212 | gup.gup_flags |= FOLL_WRITE; |
213 | |
214 | gup_fd = open(GUP_TEST_FILE, O_RDWR); |
215 | if (gup_fd == -1) { |
216 | switch (errno) { |
217 | case EACCES: |
218 | if (getuid()) |
219 | ksft_print_msg(msg: "Please run this test as root\n" ); |
220 | break; |
221 | case ENOENT: |
222 | if (opendir("/sys/kernel/debug" ) == NULL) |
223 | ksft_print_msg(msg: "mount debugfs at /sys/kernel/debug\n" ); |
224 | ksft_print_msg(msg: "check if CONFIG_GUP_TEST is enabled in kernel config\n" ); |
225 | break; |
226 | default: |
227 | ksft_print_msg("failed to open %s: %s\n" , GUP_TEST_FILE, strerror(errno)); |
228 | break; |
229 | } |
230 | ksft_test_result_skip(msg: "Please run this test as root\n" ); |
231 | return ksft_exit_pass(); |
232 | } |
233 | |
234 | p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0); |
235 | if (p == MAP_FAILED) |
236 | ksft_exit_fail_msg("mmap: %s\n" , strerror(errno)); |
237 | gup.addr = (unsigned long)p; |
238 | |
239 | if (thp == 1) |
240 | madvise(p, size, MADV_HUGEPAGE); |
241 | else if (thp == 0) |
242 | madvise(p, size, MADV_NOHUGEPAGE); |
243 | |
244 | /* |
245 | * FOLL_TOUCH, in gup_test, is used as an either/or case: either |
246 | * fault pages in from the kernel via FOLL_TOUCH, or fault them |
247 | * in here, from user space. This allows comparison of performance |
248 | * between those two cases. |
249 | */ |
250 | if (touch) { |
251 | gup.gup_flags |= FOLL_TOUCH; |
252 | } else { |
253 | for (; (unsigned long)p < gup.addr + size; p += psize()) |
254 | p[0] = 0; |
255 | } |
256 | |
257 | tid = malloc(sizeof(pthread_t) * nthreads); |
258 | assert(tid); |
259 | for (i = 0; i < nthreads; i++) { |
260 | ret = pthread_create(&tid[i], NULL, gup_thread, &gup); |
261 | assert(ret == 0); |
262 | } |
263 | for (i = 0; i < nthreads; i++) { |
264 | ret = pthread_join(tid[i], NULL); |
265 | assert(ret == 0); |
266 | } |
267 | |
268 | free(tid); |
269 | |
270 | return ksft_exit_pass(); |
271 | } |
272 | |