1#include <fcntl.h>
2#include <errno.h>
3#include <stdio.h>
4#include <stdlib.h>
5#include <unistd.h>
6#include <dirent.h>
7#include <sys/ioctl.h>
8#include <sys/mman.h>
9#include <sys/stat.h>
10#include <sys/types.h>
11#include <pthread.h>
12#include <assert.h>
13#include <mm/gup_test.h>
14#include "../kselftest.h"
15#include "vm_util.h"
16
17#define MB (1UL << 20)
18
19/* Just the flags we need, copied from mm.h: */
20#define FOLL_WRITE 0x01 /* check pte is writable */
21#define FOLL_TOUCH 0x02 /* mark page accessed */
22
23#define GUP_TEST_FILE "/sys/kernel/debug/gup_test"
24
25static unsigned long cmd = GUP_FAST_BENCHMARK;
26static int gup_fd, repeats = 1;
27static unsigned long size = 128 * MB;
28/* Serialize prints */
29static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER;
30
31static char *cmd_to_str(unsigned long cmd)
32{
33 switch (cmd) {
34 case GUP_FAST_BENCHMARK:
35 return "GUP_FAST_BENCHMARK";
36 case PIN_FAST_BENCHMARK:
37 return "PIN_FAST_BENCHMARK";
38 case PIN_LONGTERM_BENCHMARK:
39 return "PIN_LONGTERM_BENCHMARK";
40 case GUP_BASIC_TEST:
41 return "GUP_BASIC_TEST";
42 case PIN_BASIC_TEST:
43 return "PIN_BASIC_TEST";
44 case DUMP_USER_PAGES_TEST:
45 return "DUMP_USER_PAGES_TEST";
46 }
47 return "Unknown command";
48}
49
50void *gup_thread(void *data)
51{
52 struct gup_test gup = *(struct gup_test *)data;
53 int i, status;
54
55 /* Only report timing information on the *_BENCHMARK commands: */
56 if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) ||
57 (cmd == PIN_LONGTERM_BENCHMARK)) {
58 for (i = 0; i < repeats; i++) {
59 gup.size = size;
60 status = ioctl(gup_fd, cmd, &gup);
61 if (status)
62 break;
63
64 pthread_mutex_lock(&print_mutex);
65 ksft_print_msg(msg: "%s: Time: get:%lld put:%lld us",
66 cmd_to_str(cmd), gup.get_delta_usec,
67 gup.put_delta_usec);
68 if (gup.size != size)
69 ksft_print_msg(msg: ", truncated (size: %lld)", gup.size);
70 ksft_print_msg(msg: "\n");
71 pthread_mutex_unlock(&print_mutex);
72 }
73 } else {
74 gup.size = size;
75 status = ioctl(gup_fd, cmd, &gup);
76 if (status)
77 goto return_;
78
79 pthread_mutex_lock(&print_mutex);
80 ksft_print_msg(msg: "%s: done\n", cmd_to_str(cmd));
81 if (gup.size != size)
82 ksft_print_msg(msg: "Truncated (size: %lld)\n", gup.size);
83 pthread_mutex_unlock(&print_mutex);
84 }
85
86return_:
87 ksft_test_result(!status, "ioctl status %d\n", status);
88 return NULL;
89}
90
91int main(int argc, char **argv)
92{
93 struct gup_test gup = { 0 };
94 int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret;
95 int flags = MAP_PRIVATE, touch = 0;
96 char *file = "/dev/zero";
97 pthread_t *tid;
98 char *p;
99
100 while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) {
101 switch (opt) {
102 case 'a':
103 cmd = PIN_FAST_BENCHMARK;
104 break;
105 case 'b':
106 cmd = PIN_BASIC_TEST;
107 break;
108 case 'L':
109 cmd = PIN_LONGTERM_BENCHMARK;
110 break;
111 case 'c':
112 cmd = DUMP_USER_PAGES_TEST;
113 /*
114 * Dump page 0 (index 1). May be overridden later, by
115 * user's non-option arguments.
116 *
117 * .which_pages is zero-based, so that zero can mean "do
118 * nothing".
119 */
120 gup.which_pages[0] = 1;
121 break;
122 case 'p':
123 /* works only with DUMP_USER_PAGES_TEST */
124 gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN;
125 break;
126 case 'F':
127 /* strtol, so you can pass flags in hex form */
128 gup.gup_flags = strtol(optarg, 0, 0);
129 break;
130 case 'j':
131 nthreads = atoi(optarg);
132 break;
133 case 'm':
134 size = atoi(optarg) * MB;
135 break;
136 case 'r':
137 repeats = atoi(optarg);
138 break;
139 case 'n':
140 nr_pages = atoi(optarg);
141 break;
142 case 't':
143 thp = 1;
144 break;
145 case 'T':
146 thp = 0;
147 break;
148 case 'U':
149 cmd = GUP_BASIC_TEST;
150 break;
151 case 'u':
152 cmd = GUP_FAST_BENCHMARK;
153 break;
154 case 'w':
155 write = 1;
156 break;
157 case 'W':
158 write = 0;
159 break;
160 case 'f':
161 file = optarg;
162 break;
163 case 'S':
164 flags &= ~MAP_PRIVATE;
165 flags |= MAP_SHARED;
166 break;
167 case 'H':
168 flags |= (MAP_HUGETLB | MAP_ANONYMOUS);
169 break;
170 case 'z':
171 /* fault pages in gup, do not fault in userland */
172 touch = 1;
173 break;
174 default:
175 ksft_exit_fail_msg(msg: "Wrong argument\n");
176 }
177 }
178
179 if (optind < argc) {
180 int extra_arg_count = 0;
181 /*
182 * For example:
183 *
184 * ./gup_test -c 0 1 0x1001
185 *
186 * ...to dump pages 0, 1, and 4097
187 */
188
189 while ((optind < argc) &&
190 (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) {
191 /*
192 * Do the 1-based indexing here, so that the user can
193 * use normal 0-based indexing on the command line.
194 */
195 long page_index = strtol(argv[optind], 0, 0) + 1;
196
197 gup.which_pages[extra_arg_count] = page_index;
198 extra_arg_count++;
199 optind++;
200 }
201 }
202
203 ksft_print_header();
204 ksft_set_plan(plan: nthreads);
205
206 filed = open(file, O_RDWR|O_CREAT, 0664);
207 if (filed < 0)
208 ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno));
209
210 gup.nr_pages_per_call = nr_pages;
211 if (write)
212 gup.gup_flags |= FOLL_WRITE;
213
214 gup_fd = open(GUP_TEST_FILE, O_RDWR);
215 if (gup_fd == -1) {
216 switch (errno) {
217 case EACCES:
218 if (getuid())
219 ksft_print_msg(msg: "Please run this test as root\n");
220 break;
221 case ENOENT:
222 if (opendir("/sys/kernel/debug") == NULL)
223 ksft_print_msg(msg: "mount debugfs at /sys/kernel/debug\n");
224 ksft_print_msg(msg: "check if CONFIG_GUP_TEST is enabled in kernel config\n");
225 break;
226 default:
227 ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno));
228 break;
229 }
230 ksft_test_result_skip(msg: "Please run this test as root\n");
231 return ksft_exit_pass();
232 }
233
234 p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0);
235 if (p == MAP_FAILED)
236 ksft_exit_fail_msg("mmap: %s\n", strerror(errno));
237 gup.addr = (unsigned long)p;
238
239 if (thp == 1)
240 madvise(p, size, MADV_HUGEPAGE);
241 else if (thp == 0)
242 madvise(p, size, MADV_NOHUGEPAGE);
243
244 /*
245 * FOLL_TOUCH, in gup_test, is used as an either/or case: either
246 * fault pages in from the kernel via FOLL_TOUCH, or fault them
247 * in here, from user space. This allows comparison of performance
248 * between those two cases.
249 */
250 if (touch) {
251 gup.gup_flags |= FOLL_TOUCH;
252 } else {
253 for (; (unsigned long)p < gup.addr + size; p += psize())
254 p[0] = 0;
255 }
256
257 tid = malloc(sizeof(pthread_t) * nthreads);
258 assert(tid);
259 for (i = 0; i < nthreads; i++) {
260 ret = pthread_create(&tid[i], NULL, gup_thread, &gup);
261 assert(ret == 0);
262 }
263 for (i = 0; i < nthreads; i++) {
264 ret = pthread_join(tid[i], NULL);
265 assert(ret == 0);
266 }
267
268 free(tid);
269
270 return ksft_exit_pass();
271}
272

source code of linux/tools/testing/selftests/mm/gup_test.c