1 | // SPDX-License-Identifier: GPL-2.0 |
2 | |
3 | #include <sys/mman.h> |
4 | #include <sys/prctl.h> |
5 | #include <sys/wait.h> |
6 | #include <stdbool.h> |
7 | #include <time.h> |
8 | #include <string.h> |
9 | #include <numa.h> |
10 | #include <unistd.h> |
11 | #include <fcntl.h> |
12 | #include <stdint.h> |
13 | #include <err.h> |
14 | |
15 | #include "../kselftest.h" |
16 | #include <include/vdso/time64.h> |
17 | #include "vm_util.h" |
18 | |
19 | #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" |
20 | #define KSM_FP(s) (KSM_SYSFS_PATH s) |
21 | #define KSM_SCAN_LIMIT_SEC_DEFAULT 120 |
22 | #define KSM_PAGE_COUNT_DEFAULT 10l |
23 | #define KSM_PROT_STR_DEFAULT "rw" |
24 | #define KSM_USE_ZERO_PAGES_DEFAULT false |
25 | #define KSM_MERGE_ACROSS_NODES_DEFAULT true |
26 | #define KSM_MERGE_TYPE_DEFAULT 0 |
27 | #define MB (1ul << 20) |
28 | |
29 | struct ksm_sysfs { |
30 | unsigned long max_page_sharing; |
31 | unsigned long merge_across_nodes; |
32 | unsigned long pages_to_scan; |
33 | unsigned long run; |
34 | unsigned long sleep_millisecs; |
35 | unsigned long stable_node_chains_prune_millisecs; |
36 | unsigned long use_zero_pages; |
37 | }; |
38 | |
39 | enum ksm_merge_type { |
40 | KSM_MERGE_MADVISE, |
41 | KSM_MERGE_PRCTL, |
42 | KSM_MERGE_LAST = KSM_MERGE_PRCTL |
43 | }; |
44 | |
45 | enum ksm_test_name { |
46 | CHECK_KSM_MERGE, |
47 | CHECK_KSM_UNMERGE, |
48 | CHECK_KSM_GET_MERGE_TYPE, |
49 | CHECK_KSM_ZERO_PAGE_MERGE, |
50 | CHECK_KSM_NUMA_MERGE, |
51 | KSM_MERGE_TIME, |
52 | KSM_MERGE_TIME_HUGE_PAGES, |
53 | KSM_UNMERGE_TIME, |
54 | KSM_COW_TIME |
55 | }; |
56 | |
57 | int debug; |
58 | |
59 | static int ksm_write_sysfs(const char *file_path, unsigned long val) |
60 | { |
61 | FILE *f = fopen(file_path, "w" ); |
62 | |
63 | if (!f) { |
64 | fprintf(stderr, "f %s\n" , file_path); |
65 | perror("fopen" ); |
66 | return 1; |
67 | } |
68 | if (fprintf(f, "%lu" , val) < 0) { |
69 | perror("fprintf" ); |
70 | fclose(f); |
71 | return 1; |
72 | } |
73 | fclose(f); |
74 | |
75 | return 0; |
76 | } |
77 | |
78 | static int ksm_read_sysfs(const char *file_path, unsigned long *val) |
79 | { |
80 | FILE *f = fopen(file_path, "r" ); |
81 | |
82 | if (!f) { |
83 | fprintf(stderr, "f %s\n" , file_path); |
84 | perror("fopen" ); |
85 | return 1; |
86 | } |
87 | if (fscanf(f, "%lu" , val) != 1) { |
88 | perror("fscanf" ); |
89 | fclose(f); |
90 | return 1; |
91 | } |
92 | fclose(f); |
93 | |
94 | return 0; |
95 | } |
96 | |
97 | static void ksm_print_sysfs(void) |
98 | { |
99 | unsigned long max_page_sharing, pages_sharing, pages_shared; |
100 | unsigned long full_scans, pages_unshared, pages_volatile; |
101 | unsigned long stable_node_chains, stable_node_dups; |
102 | long general_profit; |
103 | |
104 | if (ksm_read_sysfs(KSM_FP("pages_shared" ), val: &pages_shared) || |
105 | ksm_read_sysfs(KSM_FP("pages_sharing" ), val: &pages_sharing) || |
106 | ksm_read_sysfs(KSM_FP("max_page_sharing" ), val: &max_page_sharing) || |
107 | ksm_read_sysfs(KSM_FP("full_scans" ), val: &full_scans) || |
108 | ksm_read_sysfs(KSM_FP("pages_unshared" ), val: &pages_unshared) || |
109 | ksm_read_sysfs(KSM_FP("pages_volatile" ), val: &pages_volatile) || |
110 | ksm_read_sysfs(KSM_FP("stable_node_chains" ), val: &stable_node_chains) || |
111 | ksm_read_sysfs(KSM_FP("stable_node_dups" ), val: &stable_node_dups) || |
112 | ksm_read_sysfs(KSM_FP("general_profit" ), val: (unsigned long *)&general_profit)) |
113 | return; |
114 | |
115 | printf("pages_shared : %lu\n" , pages_shared); |
116 | printf("pages_sharing : %lu\n" , pages_sharing); |
117 | printf("max_page_sharing : %lu\n" , max_page_sharing); |
118 | printf("full_scans : %lu\n" , full_scans); |
119 | printf("pages_unshared : %lu\n" , pages_unshared); |
120 | printf("pages_volatile : %lu\n" , pages_volatile); |
121 | printf("stable_node_chains: %lu\n" , stable_node_chains); |
122 | printf("stable_node_dups : %lu\n" , stable_node_dups); |
123 | printf("general_profit : %ld\n" , general_profit); |
124 | } |
125 | |
126 | static void ksm_print_procfs(void) |
127 | { |
128 | const char *file_name = "/proc/self/ksm_stat" ; |
129 | char buffer[512]; |
130 | FILE *f = fopen(file_name, "r" ); |
131 | |
132 | if (!f) { |
133 | fprintf(stderr, "f %s\n" , file_name); |
134 | perror("fopen" ); |
135 | return; |
136 | } |
137 | |
138 | while (fgets(buffer, sizeof(buffer), f)) |
139 | printf("%s" , buffer); |
140 | |
141 | fclose(f); |
142 | } |
143 | |
144 | static int str_to_prot(char *prot_str) |
145 | { |
146 | int prot = 0; |
147 | |
148 | if ((strchr(prot_str, 'r')) != NULL) |
149 | prot |= PROT_READ; |
150 | if ((strchr(prot_str, 'w')) != NULL) |
151 | prot |= PROT_WRITE; |
152 | if ((strchr(prot_str, 'x')) != NULL) |
153 | prot |= PROT_EXEC; |
154 | |
155 | return prot; |
156 | } |
157 | |
158 | static void print_help(void) |
159 | { |
160 | printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n" |
161 | "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n" ); |
162 | |
163 | printf("Supported <test type>:\n" |
164 | " -M (page merging)\n" |
165 | " -Z (zero pages merging)\n" |
166 | " -N (merging of pages in different NUMA nodes)\n" |
167 | " -U (page unmerging)\n" |
168 | " -P evaluate merging time and speed.\n" |
169 | " For this test, the size of duplicated memory area (in MiB)\n" |
170 | " must be provided using -s option\n" |
171 | " -H evaluate merging time and speed of area allocated mostly with huge pages\n" |
172 | " For this test, the size of duplicated memory area (in MiB)\n" |
173 | " must be provided using -s option\n" |
174 | " -D evaluate unmerging time and speed when disabling KSM.\n" |
175 | " For this test, the size of duplicated memory area (in MiB)\n" |
176 | " must be provided using -s option\n" |
177 | " -C evaluate the time required to break COW of merged pages.\n\n" ); |
178 | |
179 | printf(" -a: specify the access protections of pages.\n" |
180 | " <prot> must be of the form [rwx].\n" |
181 | " Default: %s\n" , KSM_PROT_STR_DEFAULT); |
182 | printf(" -p: specify the number of pages to test.\n" |
183 | " Default: %ld\n" , KSM_PAGE_COUNT_DEFAULT); |
184 | printf(" -l: limit the maximum running time (in seconds) for a test.\n" |
185 | " Default: %d seconds\n" , KSM_SCAN_LIMIT_SEC_DEFAULT); |
186 | printf(" -z: change use_zero_pages tunable\n" |
187 | " Default: %d\n" , KSM_USE_ZERO_PAGES_DEFAULT); |
188 | printf(" -m: change merge_across_nodes tunable\n" |
189 | " Default: %d\n" , KSM_MERGE_ACROSS_NODES_DEFAULT); |
190 | printf(" -d: turn debugging output on\n" ); |
191 | printf(" -s: the size of duplicated memory area (in MiB)\n" ); |
192 | printf(" -t: KSM merge type\n" |
193 | " Default: 0\n" |
194 | " 0: madvise merging\n" |
195 | " 1: prctl merging\n" ); |
196 | |
197 | exit(0); |
198 | } |
199 | |
200 | static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size) |
201 | { |
202 | void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); |
203 | |
204 | if (!map_ptr) { |
205 | perror("mmap" ); |
206 | return NULL; |
207 | } |
208 | memset(map_ptr, data, map_size); |
209 | if (mprotect(map_ptr, map_size, prot)) { |
210 | perror("mprotect" ); |
211 | munmap(map_ptr, map_size); |
212 | return NULL; |
213 | } |
214 | |
215 | return map_ptr; |
216 | } |
217 | |
218 | static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) |
219 | { |
220 | struct timespec cur_time; |
221 | unsigned long cur_scan, init_scan; |
222 | |
223 | if (ksm_read_sysfs(KSM_FP("full_scans" ), val: &init_scan)) |
224 | return 1; |
225 | cur_scan = init_scan; |
226 | |
227 | while (cur_scan < init_scan + scan_count) { |
228 | if (ksm_read_sysfs(KSM_FP("full_scans" ), val: &cur_scan)) |
229 | return 1; |
230 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { |
231 | perror("clock_gettime" ); |
232 | return 1; |
233 | } |
234 | if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { |
235 | printf("Scan time limit exceeded\n" ); |
236 | return 1; |
237 | } |
238 | } |
239 | |
240 | return 0; |
241 | } |
242 | |
243 | static int ksm_merge_pages(int merge_type, void *addr, size_t size, |
244 | struct timespec start_time, int timeout) |
245 | { |
246 | if (merge_type == KSM_MERGE_MADVISE) { |
247 | if (madvise(addr, size, MADV_MERGEABLE)) { |
248 | perror("madvise" ); |
249 | return 1; |
250 | } |
251 | } else if (merge_type == KSM_MERGE_PRCTL) { |
252 | if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { |
253 | perror("prctl" ); |
254 | return 1; |
255 | } |
256 | } |
257 | |
258 | if (ksm_write_sysfs(KSM_FP("run" ), val: 1)) |
259 | return 1; |
260 | |
261 | /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */ |
262 | if (ksm_do_scan(scan_count: 2, start_time: start_time, timeout)) |
263 | return 1; |
264 | |
265 | return 0; |
266 | } |
267 | |
268 | static int ksm_unmerge_pages(void *addr, size_t size, |
269 | struct timespec start_time, int timeout) |
270 | { |
271 | if (madvise(addr, size, MADV_UNMERGEABLE)) { |
272 | perror("madvise" ); |
273 | return 1; |
274 | } |
275 | return 0; |
276 | } |
277 | |
278 | static bool assert_ksm_pages_count(long dupl_page_count) |
279 | { |
280 | unsigned long max_page_sharing, pages_sharing, pages_shared; |
281 | |
282 | if (ksm_read_sysfs(KSM_FP("pages_shared" ), &pages_shared) || |
283 | ksm_read_sysfs(KSM_FP("pages_sharing" ), &pages_sharing) || |
284 | ksm_read_sysfs(KSM_FP("max_page_sharing" ), &max_page_sharing)) |
285 | return false; |
286 | |
287 | if (debug) { |
288 | ksm_print_sysfs(); |
289 | ksm_print_procfs(); |
290 | } |
291 | |
292 | /* |
293 | * Since there must be at least 2 pages for merging and 1 page can be |
294 | * shared with the limited number of pages (max_page_sharing), sometimes |
295 | * there are 'leftover' pages that cannot be merged. For example, if there |
296 | * are 11 pages and max_page_sharing = 10, then only 10 pages will be |
297 | * merged and the 11th page won't be affected. As a result, when the number |
298 | * of duplicate pages is divided by max_page_sharing and the remainder is 1, |
299 | * pages_shared and pages_sharing values will be equal between dupl_page_count |
300 | * and dupl_page_count - 1. |
301 | */ |
302 | if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) { |
303 | if (pages_shared == dupl_page_count / max_page_sharing && |
304 | pages_sharing == pages_shared * (max_page_sharing - 1)) |
305 | return true; |
306 | } else { |
307 | if (pages_shared == (dupl_page_count / max_page_sharing + 1) && |
308 | pages_sharing == dupl_page_count - pages_shared) |
309 | return true; |
310 | } |
311 | |
312 | return false; |
313 | } |
314 | |
315 | static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) |
316 | { |
317 | if (ksm_read_sysfs(KSM_FP("max_page_sharing" ), val: &ksm_sysfs->max_page_sharing) || |
318 | numa_available() ? 0 : |
319 | ksm_read_sysfs(KSM_FP("merge_across_nodes" ), val: &ksm_sysfs->merge_across_nodes) || |
320 | ksm_read_sysfs(KSM_FP("sleep_millisecs" ), val: &ksm_sysfs->sleep_millisecs) || |
321 | ksm_read_sysfs(KSM_FP("pages_to_scan" ), val: &ksm_sysfs->pages_to_scan) || |
322 | ksm_read_sysfs(KSM_FP("run" ), val: &ksm_sysfs->run) || |
323 | ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs" ), |
324 | val: &ksm_sysfs->stable_node_chains_prune_millisecs) || |
325 | ksm_read_sysfs(KSM_FP("use_zero_pages" ), val: &ksm_sysfs->use_zero_pages)) |
326 | return 1; |
327 | |
328 | return 0; |
329 | } |
330 | |
331 | static int ksm_restore(struct ksm_sysfs *ksm_sysfs) |
332 | { |
333 | if (ksm_write_sysfs(KSM_FP("max_page_sharing" ), val: ksm_sysfs->max_page_sharing) || |
334 | numa_available() ? 0 : |
335 | ksm_write_sysfs(KSM_FP("merge_across_nodes" ), val: ksm_sysfs->merge_across_nodes) || |
336 | ksm_write_sysfs(KSM_FP("pages_to_scan" ), val: ksm_sysfs->pages_to_scan) || |
337 | ksm_write_sysfs(KSM_FP("run" ), val: ksm_sysfs->run) || |
338 | ksm_write_sysfs(KSM_FP("sleep_millisecs" ), val: ksm_sysfs->sleep_millisecs) || |
339 | ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs" ), |
340 | val: ksm_sysfs->stable_node_chains_prune_millisecs) || |
341 | ksm_write_sysfs(KSM_FP("use_zero_pages" ), val: ksm_sysfs->use_zero_pages)) |
342 | return 1; |
343 | |
344 | return 0; |
345 | } |
346 | |
347 | static int check_ksm_merge(int merge_type, int mapping, int prot, |
348 | long page_count, int timeout, size_t page_size) |
349 | { |
350 | void *map_ptr; |
351 | struct timespec start_time; |
352 | |
353 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
354 | perror("clock_gettime" ); |
355 | return KSFT_FAIL; |
356 | } |
357 | |
358 | /* fill pages with the same data and merge them */ |
359 | map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); |
360 | if (!map_ptr) |
361 | return KSFT_FAIL; |
362 | |
363 | if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) |
364 | goto err_out; |
365 | |
366 | /* verify that the right number of pages are merged */ |
367 | if (assert_ksm_pages_count(page_count)) { |
368 | printf("OK\n" ); |
369 | munmap(map_ptr, page_size * page_count); |
370 | if (merge_type == KSM_MERGE_PRCTL) |
371 | prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); |
372 | return KSFT_PASS; |
373 | } |
374 | |
375 | err_out: |
376 | printf("Not OK\n" ); |
377 | munmap(map_ptr, page_size * page_count); |
378 | return KSFT_FAIL; |
379 | } |
380 | |
381 | static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size) |
382 | { |
383 | void *map_ptr; |
384 | struct timespec start_time; |
385 | int page_count = 2; |
386 | |
387 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
388 | perror("clock_gettime" ); |
389 | return KSFT_FAIL; |
390 | } |
391 | |
392 | /* fill pages with the same data and merge them */ |
393 | map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); |
394 | if (!map_ptr) |
395 | return KSFT_FAIL; |
396 | |
397 | if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) |
398 | goto err_out; |
399 | |
400 | /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */ |
401 | memset(map_ptr, '-', 1); |
402 | memset(map_ptr + page_size, '+', 1); |
403 | |
404 | /* get at least 1 scan, so KSM can detect that the pages were modified */ |
405 | if (ksm_do_scan(scan_count: 1, start_time: start_time, timeout)) |
406 | goto err_out; |
407 | |
408 | /* check that unmerging was successful and 0 pages are currently merged */ |
409 | if (assert_ksm_pages_count(0)) { |
410 | printf("OK\n" ); |
411 | munmap(map_ptr, page_size * page_count); |
412 | return KSFT_PASS; |
413 | } |
414 | |
415 | err_out: |
416 | printf("Not OK\n" ); |
417 | munmap(map_ptr, page_size * page_count); |
418 | return KSFT_FAIL; |
419 | } |
420 | |
421 | static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count, |
422 | int timeout, bool use_zero_pages, size_t page_size) |
423 | { |
424 | void *map_ptr; |
425 | struct timespec start_time; |
426 | |
427 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
428 | perror("clock_gettime" ); |
429 | return KSFT_FAIL; |
430 | } |
431 | |
432 | if (ksm_write_sysfs(KSM_FP("use_zero_pages" ), val: use_zero_pages)) |
433 | return KSFT_FAIL; |
434 | |
435 | /* fill pages with zero and try to merge them */ |
436 | map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count); |
437 | if (!map_ptr) |
438 | return KSFT_FAIL; |
439 | |
440 | if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) |
441 | goto err_out; |
442 | |
443 | /* |
444 | * verify that the right number of pages are merged: |
445 | * 1) if use_zero_pages is set to 1, empty pages are merged |
446 | * with the kernel zero page instead of with each other; |
447 | * 2) if use_zero_pages is set to 0, empty pages are not treated specially |
448 | * and merged as usual. |
449 | */ |
450 | if (use_zero_pages && !assert_ksm_pages_count(0)) |
451 | goto err_out; |
452 | else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) |
453 | goto err_out; |
454 | |
455 | printf("OK\n" ); |
456 | munmap(map_ptr, page_size * page_count); |
457 | return KSFT_PASS; |
458 | |
459 | err_out: |
460 | printf("Not OK\n" ); |
461 | munmap(map_ptr, page_size * page_count); |
462 | return KSFT_FAIL; |
463 | } |
464 | |
465 | static int get_next_mem_node(int node) |
466 | { |
467 | |
468 | long node_size; |
469 | int mem_node = 0; |
470 | int i, max_node = numa_max_node(); |
471 | |
472 | for (i = node + 1; i <= max_node + node; i++) { |
473 | mem_node = i % (max_node + 1); |
474 | node_size = numa_node_size(mem_node, NULL); |
475 | if (node_size > 0) |
476 | break; |
477 | } |
478 | return mem_node; |
479 | } |
480 | |
481 | static int get_first_mem_node(void) |
482 | { |
483 | return get_next_mem_node(node: numa_max_node()); |
484 | } |
485 | |
486 | static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout, |
487 | bool merge_across_nodes, size_t page_size) |
488 | { |
489 | void *numa1_map_ptr, *numa2_map_ptr; |
490 | struct timespec start_time; |
491 | int page_count = 2; |
492 | int first_node; |
493 | |
494 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
495 | perror("clock_gettime" ); |
496 | return KSFT_FAIL; |
497 | } |
498 | |
499 | if (numa_available() < 0) { |
500 | perror("NUMA support not enabled" ); |
501 | return KSFT_SKIP; |
502 | } |
503 | if (numa_num_configured_nodes() <= 1) { |
504 | printf("At least 2 NUMA nodes must be available\n" ); |
505 | return KSFT_SKIP; |
506 | } |
507 | if (ksm_write_sysfs(KSM_FP("merge_across_nodes" ), val: merge_across_nodes)) |
508 | return KSFT_FAIL; |
509 | |
510 | /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ |
511 | first_node = get_first_mem_node(); |
512 | numa1_map_ptr = numa_alloc_onnode(page_size, first_node); |
513 | numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(node: first_node)); |
514 | if (!numa1_map_ptr || !numa2_map_ptr) { |
515 | perror("numa_alloc_onnode" ); |
516 | return KSFT_FAIL; |
517 | } |
518 | |
519 | memset(numa1_map_ptr, '*', page_size); |
520 | memset(numa2_map_ptr, '*', page_size); |
521 | |
522 | /* try to merge the pages */ |
523 | if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) || |
524 | ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout)) |
525 | goto err_out; |
526 | |
527 | /* |
528 | * verify that the right number of pages are merged: |
529 | * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged; |
530 | * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is |
531 | * only 1 unique page in each node and they can't be shared. |
532 | */ |
533 | if (merge_across_nodes && !assert_ksm_pages_count(page_count)) |
534 | goto err_out; |
535 | else if (!merge_across_nodes && !assert_ksm_pages_count(0)) |
536 | goto err_out; |
537 | |
538 | numa_free(numa1_map_ptr, page_size); |
539 | numa_free(numa2_map_ptr, page_size); |
540 | printf("OK\n" ); |
541 | return KSFT_PASS; |
542 | |
543 | err_out: |
544 | numa_free(numa1_map_ptr, page_size); |
545 | numa_free(numa2_map_ptr, page_size); |
546 | printf("Not OK\n" ); |
547 | return KSFT_FAIL; |
548 | } |
549 | |
550 | static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, |
551 | int timeout, size_t map_size) |
552 | { |
553 | void *map_ptr, *map_ptr_orig; |
554 | struct timespec start_time, end_time; |
555 | unsigned long scan_time_ns; |
556 | int pagemap_fd, n_normal_pages, n_huge_pages; |
557 | |
558 | map_size *= MB; |
559 | size_t len = map_size; |
560 | |
561 | len -= len % HPAGE_SIZE; |
562 | map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE, |
563 | MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); |
564 | map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE; |
565 | |
566 | if (map_ptr_orig == MAP_FAILED) |
567 | err(2, "initial mmap" ); |
568 | |
569 | if (madvise(map_ptr, len, MADV_HUGEPAGE)) |
570 | err(2, "MADV_HUGEPAGE" ); |
571 | |
572 | pagemap_fd = open("/proc/self/pagemap" , O_RDONLY); |
573 | if (pagemap_fd < 0) |
574 | err(2, "open pagemap" ); |
575 | |
576 | n_normal_pages = 0; |
577 | n_huge_pages = 0; |
578 | for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) { |
579 | if (allocate_transhuge(p, pagemap_fd) < 0) |
580 | n_normal_pages++; |
581 | else |
582 | n_huge_pages++; |
583 | } |
584 | printf("Number of normal pages: %d\n" , n_normal_pages); |
585 | printf("Number of huge pages: %d\n" , n_huge_pages); |
586 | |
587 | memset(map_ptr, '*', len); |
588 | |
589 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
590 | perror("clock_gettime" ); |
591 | goto err_out; |
592 | } |
593 | if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) |
594 | goto err_out; |
595 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { |
596 | perror("clock_gettime" ); |
597 | goto err_out; |
598 | } |
599 | |
600 | scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + |
601 | (end_time.tv_nsec - start_time.tv_nsec); |
602 | |
603 | printf("Total size: %lu MiB\n" , map_size / MB); |
604 | printf("Total time: %ld.%09ld s\n" , scan_time_ns / NSEC_PER_SEC, |
605 | scan_time_ns % NSEC_PER_SEC); |
606 | printf("Average speed: %.3f MiB/s\n" , (map_size / MB) / |
607 | ((double)scan_time_ns / NSEC_PER_SEC)); |
608 | |
609 | munmap(map_ptr_orig, len + HPAGE_SIZE); |
610 | return KSFT_PASS; |
611 | |
612 | err_out: |
613 | printf("Not OK\n" ); |
614 | munmap(map_ptr_orig, len + HPAGE_SIZE); |
615 | return KSFT_FAIL; |
616 | } |
617 | |
618 | static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) |
619 | { |
620 | void *map_ptr; |
621 | struct timespec start_time, end_time; |
622 | unsigned long scan_time_ns; |
623 | |
624 | map_size *= MB; |
625 | |
626 | map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); |
627 | if (!map_ptr) |
628 | return KSFT_FAIL; |
629 | |
630 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
631 | perror("clock_gettime" ); |
632 | goto err_out; |
633 | } |
634 | if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) |
635 | goto err_out; |
636 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { |
637 | perror("clock_gettime" ); |
638 | goto err_out; |
639 | } |
640 | |
641 | scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + |
642 | (end_time.tv_nsec - start_time.tv_nsec); |
643 | |
644 | printf("Total size: %lu MiB\n" , map_size / MB); |
645 | printf("Total time: %ld.%09ld s\n" , scan_time_ns / NSEC_PER_SEC, |
646 | scan_time_ns % NSEC_PER_SEC); |
647 | printf("Average speed: %.3f MiB/s\n" , (map_size / MB) / |
648 | ((double)scan_time_ns / NSEC_PER_SEC)); |
649 | |
650 | munmap(map_ptr, map_size); |
651 | return KSFT_PASS; |
652 | |
653 | err_out: |
654 | printf("Not OK\n" ); |
655 | munmap(map_ptr, map_size); |
656 | return KSFT_FAIL; |
657 | } |
658 | |
659 | static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) |
660 | { |
661 | void *map_ptr; |
662 | struct timespec start_time, end_time; |
663 | unsigned long scan_time_ns; |
664 | |
665 | map_size *= MB; |
666 | |
667 | map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); |
668 | if (!map_ptr) |
669 | return KSFT_FAIL; |
670 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
671 | perror("clock_gettime" ); |
672 | goto err_out; |
673 | } |
674 | if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) |
675 | goto err_out; |
676 | |
677 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
678 | perror("clock_gettime" ); |
679 | goto err_out; |
680 | } |
681 | if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) |
682 | goto err_out; |
683 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { |
684 | perror("clock_gettime" ); |
685 | goto err_out; |
686 | } |
687 | |
688 | scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + |
689 | (end_time.tv_nsec - start_time.tv_nsec); |
690 | |
691 | printf("Total size: %lu MiB\n" , map_size / MB); |
692 | printf("Total time: %ld.%09ld s\n" , scan_time_ns / NSEC_PER_SEC, |
693 | scan_time_ns % NSEC_PER_SEC); |
694 | printf("Average speed: %.3f MiB/s\n" , (map_size / MB) / |
695 | ((double)scan_time_ns / NSEC_PER_SEC)); |
696 | |
697 | munmap(map_ptr, map_size); |
698 | return KSFT_PASS; |
699 | |
700 | err_out: |
701 | printf("Not OK\n" ); |
702 | munmap(map_ptr, map_size); |
703 | return KSFT_FAIL; |
704 | } |
705 | |
706 | static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size) |
707 | { |
708 | void *map_ptr; |
709 | struct timespec start_time, end_time; |
710 | unsigned long cow_time_ns; |
711 | |
712 | /* page_count must be less than 2*page_size */ |
713 | size_t page_count = 4000; |
714 | |
715 | map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); |
716 | if (!map_ptr) |
717 | return KSFT_FAIL; |
718 | |
719 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
720 | perror("clock_gettime" ); |
721 | return KSFT_FAIL; |
722 | } |
723 | for (size_t i = 0; i < page_count - 1; i = i + 2) |
724 | memset(map_ptr + page_size * i, '-', 1); |
725 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { |
726 | perror("clock_gettime" ); |
727 | return KSFT_FAIL; |
728 | } |
729 | |
730 | cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + |
731 | (end_time.tv_nsec - start_time.tv_nsec); |
732 | |
733 | printf("Total size: %lu MiB\n\n" , (page_size * page_count) / MB); |
734 | printf("Not merged pages:\n" ); |
735 | printf("Total time: %ld.%09ld s\n" , cow_time_ns / NSEC_PER_SEC, |
736 | cow_time_ns % NSEC_PER_SEC); |
737 | printf("Average speed: %.3f MiB/s\n\n" , ((page_size * (page_count / 2)) / MB) / |
738 | ((double)cow_time_ns / NSEC_PER_SEC)); |
739 | |
740 | /* Create 2000 pairs of duplicate pages */ |
741 | for (size_t i = 0; i < page_count - 1; i = i + 2) { |
742 | memset(map_ptr + page_size * i, '+', i / 2 + 1); |
743 | memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1); |
744 | } |
745 | if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) |
746 | goto err_out; |
747 | |
748 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { |
749 | perror("clock_gettime" ); |
750 | goto err_out; |
751 | } |
752 | for (size_t i = 0; i < page_count - 1; i = i + 2) |
753 | memset(map_ptr + page_size * i, '-', 1); |
754 | if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { |
755 | perror("clock_gettime" ); |
756 | goto err_out; |
757 | } |
758 | |
759 | cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + |
760 | (end_time.tv_nsec - start_time.tv_nsec); |
761 | |
762 | printf("Merged pages:\n" ); |
763 | printf("Total time: %ld.%09ld s\n" , cow_time_ns / NSEC_PER_SEC, |
764 | cow_time_ns % NSEC_PER_SEC); |
765 | printf("Average speed: %.3f MiB/s\n" , ((page_size * (page_count / 2)) / MB) / |
766 | ((double)cow_time_ns / NSEC_PER_SEC)); |
767 | |
768 | munmap(map_ptr, page_size * page_count); |
769 | return KSFT_PASS; |
770 | |
771 | err_out: |
772 | printf("Not OK\n" ); |
773 | munmap(map_ptr, page_size * page_count); |
774 | return KSFT_FAIL; |
775 | } |
776 | |
777 | int main(int argc, char *argv[]) |
778 | { |
779 | int ret, opt; |
780 | int prot = 0; |
781 | int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT; |
782 | int merge_type = KSM_MERGE_TYPE_DEFAULT; |
783 | long page_count = KSM_PAGE_COUNT_DEFAULT; |
784 | size_t page_size = sysconf(_SC_PAGESIZE); |
785 | struct ksm_sysfs ksm_sysfs_old; |
786 | int test_name = CHECK_KSM_MERGE; |
787 | bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; |
788 | bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; |
789 | long size_MB = 0; |
790 | |
791 | while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD" )) != -1) { |
792 | switch (opt) { |
793 | case 'a': |
794 | prot = str_to_prot(optarg); |
795 | break; |
796 | case 'p': |
797 | page_count = atol(optarg); |
798 | if (page_count <= 0) { |
799 | printf("The number of pages must be greater than 0\n" ); |
800 | return KSFT_FAIL; |
801 | } |
802 | break; |
803 | case 'l': |
804 | ksm_scan_limit_sec = atoi(optarg); |
805 | if (ksm_scan_limit_sec <= 0) { |
806 | printf("Timeout value must be greater than 0\n" ); |
807 | return KSFT_FAIL; |
808 | } |
809 | break; |
810 | case 'h': |
811 | print_help(); |
812 | break; |
813 | case 'z': |
814 | if (strcmp(optarg, "0" ) == 0) |
815 | use_zero_pages = 0; |
816 | else |
817 | use_zero_pages = 1; |
818 | break; |
819 | case 'm': |
820 | if (strcmp(optarg, "0" ) == 0) |
821 | merge_across_nodes = 0; |
822 | else |
823 | merge_across_nodes = 1; |
824 | break; |
825 | case 'd': |
826 | debug = 1; |
827 | break; |
828 | case 's': |
829 | size_MB = atoi(optarg); |
830 | if (size_MB <= 0) { |
831 | printf("Size must be greater than 0\n" ); |
832 | return KSFT_FAIL; |
833 | } |
834 | break; |
835 | case 't': |
836 | { |
837 | int tmp = atoi(optarg); |
838 | |
839 | if (tmp < 0 || tmp > KSM_MERGE_LAST) { |
840 | printf("Invalid merge type\n" ); |
841 | return KSFT_FAIL; |
842 | } |
843 | merge_type = tmp; |
844 | } |
845 | break; |
846 | case 'M': |
847 | break; |
848 | case 'U': |
849 | test_name = CHECK_KSM_UNMERGE; |
850 | break; |
851 | case 'Z': |
852 | test_name = CHECK_KSM_ZERO_PAGE_MERGE; |
853 | break; |
854 | case 'N': |
855 | test_name = CHECK_KSM_NUMA_MERGE; |
856 | break; |
857 | case 'P': |
858 | test_name = KSM_MERGE_TIME; |
859 | break; |
860 | case 'H': |
861 | test_name = KSM_MERGE_TIME_HUGE_PAGES; |
862 | break; |
863 | case 'D': |
864 | test_name = KSM_UNMERGE_TIME; |
865 | break; |
866 | case 'C': |
867 | test_name = KSM_COW_TIME; |
868 | break; |
869 | default: |
870 | return KSFT_FAIL; |
871 | } |
872 | } |
873 | |
874 | if (prot == 0) |
875 | prot = str_to_prot(KSM_PROT_STR_DEFAULT); |
876 | |
877 | if (access(KSM_SYSFS_PATH, F_OK)) { |
878 | printf("Config KSM not enabled\n" ); |
879 | return KSFT_SKIP; |
880 | } |
881 | |
882 | if (ksm_save_def(ksm_sysfs: &ksm_sysfs_old)) { |
883 | printf("Cannot save default tunables\n" ); |
884 | return KSFT_FAIL; |
885 | } |
886 | |
887 | if (ksm_write_sysfs(KSM_FP("run" ), val: 2) || |
888 | ksm_write_sysfs(KSM_FP("sleep_millisecs" ), val: 0) || |
889 | numa_available() ? 0 : |
890 | ksm_write_sysfs(KSM_FP("merge_across_nodes" ), val: 1) || |
891 | ksm_write_sysfs(KSM_FP("pages_to_scan" ), val: page_count)) |
892 | return KSFT_FAIL; |
893 | |
894 | switch (test_name) { |
895 | case CHECK_KSM_MERGE: |
896 | ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, |
897 | ksm_scan_limit_sec, page_size); |
898 | break; |
899 | case CHECK_KSM_UNMERGE: |
900 | ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, |
901 | ksm_scan_limit_sec, page_size); |
902 | break; |
903 | case CHECK_KSM_ZERO_PAGE_MERGE: |
904 | ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, |
905 | page_count, ksm_scan_limit_sec, use_zero_pages, |
906 | page_size); |
907 | break; |
908 | case CHECK_KSM_NUMA_MERGE: |
909 | ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, |
910 | ksm_scan_limit_sec, merge_across_nodes, page_size); |
911 | break; |
912 | case KSM_MERGE_TIME: |
913 | if (size_MB == 0) { |
914 | printf("Option '-s' is required.\n" ); |
915 | return KSFT_FAIL; |
916 | } |
917 | ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, |
918 | ksm_scan_limit_sec, size_MB); |
919 | break; |
920 | case KSM_MERGE_TIME_HUGE_PAGES: |
921 | if (size_MB == 0) { |
922 | printf("Option '-s' is required.\n" ); |
923 | return KSFT_FAIL; |
924 | } |
925 | ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, |
926 | ksm_scan_limit_sec, size_MB); |
927 | break; |
928 | case KSM_UNMERGE_TIME: |
929 | if (size_MB == 0) { |
930 | printf("Option '-s' is required.\n" ); |
931 | return KSFT_FAIL; |
932 | } |
933 | ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, |
934 | ksm_scan_limit_sec, size_MB); |
935 | break; |
936 | case KSM_COW_TIME: |
937 | ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, |
938 | ksm_scan_limit_sec, page_size); |
939 | break; |
940 | } |
941 | |
942 | if (ksm_restore(ksm_sysfs: &ksm_sysfs_old)) { |
943 | printf("Cannot restore default tunables\n" ); |
944 | return KSFT_FAIL; |
945 | } |
946 | |
947 | return ret; |
948 | } |
949 | |