1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * KSM functional tests
4 *
5 * Copyright 2022, Red Hat, Inc.
6 *
7 * Author(s): David Hildenbrand <david@redhat.com>
8 */
9#define _GNU_SOURCE
10#include <stdlib.h>
11#include <string.h>
12#include <stdbool.h>
13#include <stdint.h>
14#include <unistd.h>
15#include <errno.h>
16#include <fcntl.h>
17#include <sys/mman.h>
18#include <sys/prctl.h>
19#include <sys/syscall.h>
20#include <sys/ioctl.h>
21#include <sys/wait.h>
22#include <linux/userfaultfd.h>
23
24#include "../kselftest.h"
25#include "vm_util.h"
26
27#define KiB 1024u
28#define MiB (1024 * KiB)
29#define FORK_EXEC_CHILD_PRG_NAME "ksm_fork_exec_child"
30
31static int mem_fd;
32static int ksm_fd;
33static int ksm_full_scans_fd;
34static int proc_self_ksm_stat_fd;
35static int proc_self_ksm_merging_pages_fd;
36static int ksm_use_zero_pages_fd;
37static int pagemap_fd;
38static size_t pagesize;
39
40static bool range_maps_duplicates(char *addr, unsigned long size)
41{
42 unsigned long offs_a, offs_b, pfn_a, pfn_b;
43
44 /*
45 * There is no easy way to check if there are KSM pages mapped into
46 * this range. We only check that the range does not map the same PFN
47 * twice by comparing each pair of mapped pages.
48 */
49 for (offs_a = 0; offs_a < size; offs_a += pagesize) {
50 pfn_a = pagemap_get_pfn(fd: pagemap_fd, start: addr + offs_a);
51 /* Page not present or PFN not exposed by the kernel. */
52 if (pfn_a == -1ul || !pfn_a)
53 continue;
54
55 for (offs_b = offs_a + pagesize; offs_b < size;
56 offs_b += pagesize) {
57 pfn_b = pagemap_get_pfn(fd: pagemap_fd, start: addr + offs_b);
58 if (pfn_b == -1ul || !pfn_b)
59 continue;
60 if (pfn_a == pfn_b)
61 return true;
62 }
63 }
64 return false;
65}
66
67static long get_my_ksm_zero_pages(void)
68{
69 char buf[200];
70 char *substr_ksm_zero;
71 size_t value_pos;
72 ssize_t read_size;
73 unsigned long my_ksm_zero_pages;
74
75 if (!proc_self_ksm_stat_fd)
76 return 0;
77
78 read_size = pread(proc_self_ksm_stat_fd, buf, sizeof(buf) - 1, 0);
79 if (read_size < 0)
80 return -errno;
81
82 buf[read_size] = 0;
83
84 substr_ksm_zero = strstr(buf, "ksm_zero_pages");
85 if (!substr_ksm_zero)
86 return 0;
87
88 value_pos = strcspn(substr_ksm_zero, "0123456789");
89 my_ksm_zero_pages = strtol(substr_ksm_zero + value_pos, NULL, 10);
90
91 return my_ksm_zero_pages;
92}
93
94static long get_my_merging_pages(void)
95{
96 char buf[10];
97 ssize_t ret;
98
99 if (proc_self_ksm_merging_pages_fd < 0)
100 return proc_self_ksm_merging_pages_fd;
101
102 ret = pread(proc_self_ksm_merging_pages_fd, buf, sizeof(buf) - 1, 0);
103 if (ret <= 0)
104 return -errno;
105 buf[ret] = 0;
106
107 return strtol(buf, NULL, 10);
108}
109
110static long ksm_get_full_scans(void)
111{
112 char buf[10];
113 ssize_t ret;
114
115 ret = pread(ksm_full_scans_fd, buf, sizeof(buf) - 1, 0);
116 if (ret <= 0)
117 return -errno;
118 buf[ret] = 0;
119
120 return strtol(buf, NULL, 10);
121}
122
123static int ksm_merge(void)
124{
125 long start_scans, end_scans;
126
127 /* Wait for two full scans such that any possible merging happened. */
128 start_scans = ksm_get_full_scans();
129 if (start_scans < 0)
130 return start_scans;
131 if (write(ksm_fd, "1", 1) != 1)
132 return -errno;
133 do {
134 end_scans = ksm_get_full_scans();
135 if (end_scans < 0)
136 return end_scans;
137 } while (end_scans < start_scans + 2);
138
139 return 0;
140}
141
142static int ksm_unmerge(void)
143{
144 if (write(ksm_fd, "2", 1) != 1)
145 return -errno;
146 return 0;
147}
148
149static char *mmap_and_merge_range(char val, unsigned long size, int prot,
150 bool use_prctl)
151{
152 char *map;
153 int ret;
154
155 /* Stabilize accounting by disabling KSM completely. */
156 if (ksm_unmerge()) {
157 ksft_test_result_fail(msg: "Disabling (unmerging) KSM failed\n");
158 return MAP_FAILED;
159 }
160
161 if (get_my_merging_pages() > 0) {
162 ksft_test_result_fail(msg: "Still pages merged\n");
163 return MAP_FAILED;
164 }
165
166 map = mmap(NULL, size, PROT_READ|PROT_WRITE,
167 MAP_PRIVATE|MAP_ANON, -1, 0);
168 if (map == MAP_FAILED) {
169 ksft_test_result_fail(msg: "mmap() failed\n");
170 return MAP_FAILED;
171 }
172
173 /* Don't use THP. Ignore if THP are not around on a kernel. */
174 if (madvise(map, size, MADV_NOHUGEPAGE) && errno != EINVAL) {
175 ksft_test_result_fail(msg: "MADV_NOHUGEPAGE failed\n");
176 goto unmap;
177 }
178
179 /* Make sure each page contains the same values to merge them. */
180 memset(map, val, size);
181
182 if (mprotect(map, size, prot)) {
183 ksft_test_result_skip(msg: "mprotect() failed\n");
184 goto unmap;
185 }
186
187 if (use_prctl) {
188 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);
189 if (ret < 0 && errno == EINVAL) {
190 ksft_test_result_skip(msg: "PR_SET_MEMORY_MERGE not supported\n");
191 goto unmap;
192 } else if (ret) {
193 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=1 failed\n");
194 goto unmap;
195 }
196 } else if (madvise(map, size, MADV_MERGEABLE)) {
197 ksft_test_result_fail(msg: "MADV_MERGEABLE failed\n");
198 goto unmap;
199 }
200
201 /* Run KSM to trigger merging and wait. */
202 if (ksm_merge()) {
203 ksft_test_result_fail(msg: "Running KSM failed\n");
204 goto unmap;
205 }
206
207 /*
208 * Check if anything was merged at all. Ignore the zero page that is
209 * accounted differently (depending on kernel support).
210 */
211 if (val && !get_my_merging_pages()) {
212 ksft_test_result_fail(msg: "No pages got merged\n");
213 goto unmap;
214 }
215
216 return map;
217unmap:
218 munmap(map, size);
219 return MAP_FAILED;
220}
221
222static void test_unmerge(void)
223{
224 const unsigned int size = 2 * MiB;
225 char *map;
226
227 ksft_print_msg(msg: "[RUN] %s\n", __func__);
228
229 map = mmap_and_merge_range(val: 0xcf, size, prot: PROT_READ | PROT_WRITE, use_prctl: false);
230 if (map == MAP_FAILED)
231 return;
232
233 if (madvise(map, size, MADV_UNMERGEABLE)) {
234 ksft_test_result_fail(msg: "MADV_UNMERGEABLE failed\n");
235 goto unmap;
236 }
237
238 ksft_test_result(!range_maps_duplicates(map, size),
239 "Pages were unmerged\n");
240unmap:
241 munmap(map, size);
242}
243
244static void test_unmerge_zero_pages(void)
245{
246 const unsigned int size = 2 * MiB;
247 char *map;
248 unsigned int offs;
249 unsigned long pages_expected;
250
251 ksft_print_msg(msg: "[RUN] %s\n", __func__);
252
253 if (proc_self_ksm_stat_fd < 0) {
254 ksft_test_result_skip(msg: "open(\"/proc/self/ksm_stat\") failed\n");
255 return;
256 }
257 if (ksm_use_zero_pages_fd < 0) {
258 ksft_test_result_skip(msg: "open \"/sys/kernel/mm/ksm/use_zero_pages\" failed\n");
259 return;
260 }
261 if (write(ksm_use_zero_pages_fd, "1", 1) != 1) {
262 ksft_test_result_skip(msg: "write \"/sys/kernel/mm/ksm/use_zero_pages\" failed\n");
263 return;
264 }
265
266 /* Let KSM deduplicate zero pages. */
267 map = mmap_and_merge_range(val: 0x00, size, prot: PROT_READ | PROT_WRITE, use_prctl: false);
268 if (map == MAP_FAILED)
269 return;
270
271 /* Check if ksm_zero_pages is updated correctly after KSM merging */
272 pages_expected = size / pagesize;
273 if (pages_expected != get_my_ksm_zero_pages()) {
274 ksft_test_result_fail(msg: "'ksm_zero_pages' updated after merging\n");
275 goto unmap;
276 }
277
278 /* Try to unmerge half of the region */
279 if (madvise(map, size / 2, MADV_UNMERGEABLE)) {
280 ksft_test_result_fail(msg: "MADV_UNMERGEABLE failed\n");
281 goto unmap;
282 }
283
284 /* Check if ksm_zero_pages is updated correctly after unmerging */
285 pages_expected /= 2;
286 if (pages_expected != get_my_ksm_zero_pages()) {
287 ksft_test_result_fail(msg: "'ksm_zero_pages' updated after unmerging\n");
288 goto unmap;
289 }
290
291 /* Trigger unmerging of the other half by writing to the pages. */
292 for (offs = size / 2; offs < size; offs += pagesize)
293 *((unsigned int *)&map[offs]) = offs;
294
295 /* Now we should have no zeropages remaining. */
296 if (get_my_ksm_zero_pages()) {
297 ksft_test_result_fail(msg: "'ksm_zero_pages' updated after write fault\n");
298 goto unmap;
299 }
300
301 /* Check if ksm zero pages are really unmerged */
302 ksft_test_result(!range_maps_duplicates(map, size),
303 "KSM zero pages were unmerged\n");
304unmap:
305 munmap(map, size);
306}
307
308static void test_unmerge_discarded(void)
309{
310 const unsigned int size = 2 * MiB;
311 char *map;
312
313 ksft_print_msg(msg: "[RUN] %s\n", __func__);
314
315 map = mmap_and_merge_range(val: 0xcf, size, prot: PROT_READ | PROT_WRITE, use_prctl: false);
316 if (map == MAP_FAILED)
317 return;
318
319 /* Discard half of all mapped pages so we have pte_none() entries. */
320 if (madvise(map, size / 2, MADV_DONTNEED)) {
321 ksft_test_result_fail(msg: "MADV_DONTNEED failed\n");
322 goto unmap;
323 }
324
325 if (madvise(map, size, MADV_UNMERGEABLE)) {
326 ksft_test_result_fail(msg: "MADV_UNMERGEABLE failed\n");
327 goto unmap;
328 }
329
330 ksft_test_result(!range_maps_duplicates(map, size),
331 "Pages were unmerged\n");
332unmap:
333 munmap(map, size);
334}
335
336#ifdef __NR_userfaultfd
337static void test_unmerge_uffd_wp(void)
338{
339 struct uffdio_writeprotect uffd_writeprotect;
340 const unsigned int size = 2 * MiB;
341 struct uffdio_api uffdio_api;
342 char *map;
343 int uffd;
344
345 ksft_print_msg("[RUN] %s\n", __func__);
346
347 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, false);
348 if (map == MAP_FAILED)
349 return;
350
351 /* See if UFFD is around. */
352 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
353 if (uffd < 0) {
354 ksft_test_result_skip("__NR_userfaultfd failed\n");
355 goto unmap;
356 }
357
358 /* See if UFFD-WP is around. */
359 uffdio_api.api = UFFD_API;
360 uffdio_api.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP;
361 if (ioctl(uffd, UFFDIO_API, &uffdio_api) < 0) {
362 ksft_test_result_fail("UFFDIO_API failed\n");
363 goto close_uffd;
364 }
365 if (!(uffdio_api.features & UFFD_FEATURE_PAGEFAULT_FLAG_WP)) {
366 ksft_test_result_skip("UFFD_FEATURE_PAGEFAULT_FLAG_WP not available\n");
367 goto close_uffd;
368 }
369
370 /* Register UFFD-WP, no need for an actual handler. */
371 if (uffd_register(uffd, map, size, false, true, false)) {
372 ksft_test_result_fail("UFFDIO_REGISTER_MODE_WP failed\n");
373 goto close_uffd;
374 }
375
376 /* Write-protect the range using UFFD-WP. */
377 uffd_writeprotect.range.start = (unsigned long) map;
378 uffd_writeprotect.range.len = size;
379 uffd_writeprotect.mode = UFFDIO_WRITEPROTECT_MODE_WP;
380 if (ioctl(uffd, UFFDIO_WRITEPROTECT, &uffd_writeprotect)) {
381 ksft_test_result_fail("UFFDIO_WRITEPROTECT failed\n");
382 goto close_uffd;
383 }
384
385 if (madvise(map, size, MADV_UNMERGEABLE)) {
386 ksft_test_result_fail("MADV_UNMERGEABLE failed\n");
387 goto close_uffd;
388 }
389
390 ksft_test_result(!range_maps_duplicates(map, size),
391 "Pages were unmerged\n");
392close_uffd:
393 close(uffd);
394unmap:
395 munmap(map, size);
396}
397#endif
398
399/* Verify that KSM can be enabled / queried with prctl. */
400static void test_prctl(void)
401{
402 int ret;
403
404 ksft_print_msg(msg: "[RUN] %s\n", __func__);
405
406 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);
407 if (ret < 0 && errno == EINVAL) {
408 ksft_test_result_skip(msg: "PR_SET_MEMORY_MERGE not supported\n");
409 return;
410 } else if (ret) {
411 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=1 failed\n");
412 return;
413 }
414
415 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0);
416 if (ret < 0) {
417 ksft_test_result_fail(msg: "PR_GET_MEMORY_MERGE failed\n");
418 return;
419 } else if (ret != 1) {
420 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=1 not effective\n");
421 return;
422 }
423
424 ret = prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0);
425 if (ret) {
426 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=0 failed\n");
427 return;
428 }
429
430 ret = prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0);
431 if (ret < 0) {
432 ksft_test_result_fail(msg: "PR_GET_MEMORY_MERGE failed\n");
433 return;
434 } else if (ret != 0) {
435 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=0 not effective\n");
436 return;
437 }
438
439 ksft_test_result_pass(msg: "Setting/clearing PR_SET_MEMORY_MERGE works\n");
440}
441
442/* Verify that prctl ksm flag is inherited. */
443static void test_prctl_fork(void)
444{
445 int ret, status;
446 pid_t child_pid;
447
448 ksft_print_msg(msg: "[RUN] %s\n", __func__);
449
450 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);
451 if (ret < 0 && errno == EINVAL) {
452 ksft_test_result_skip(msg: "PR_SET_MEMORY_MERGE not supported\n");
453 return;
454 } else if (ret) {
455 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=1 failed\n");
456 return;
457 }
458
459 child_pid = fork();
460 if (!child_pid) {
461 exit(prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0));
462 } else if (child_pid < 0) {
463 ksft_test_result_fail(msg: "fork() failed\n");
464 return;
465 }
466
467 if (waitpid(child_pid, &status, 0) < 0) {
468 ksft_test_result_fail(msg: "waitpid() failed\n");
469 return;
470 } else if (WEXITSTATUS(status) != 1) {
471 ksft_test_result_fail(msg: "unexpected PR_GET_MEMORY_MERGE result in child\n");
472 return;
473 }
474
475 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) {
476 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=0 failed\n");
477 return;
478 }
479
480 ksft_test_result_pass(msg: "PR_SET_MEMORY_MERGE value is inherited\n");
481}
482
483static int ksm_fork_exec_child(void)
484{
485 /* Test if KSM is enabled for the process. */
486 return prctl(PR_GET_MEMORY_MERGE, 0, 0, 0, 0) == 1;
487}
488
489static void test_prctl_fork_exec(void)
490{
491 int ret, status;
492 pid_t child_pid;
493
494 ksft_print_msg(msg: "[RUN] %s\n", __func__);
495
496 ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);
497 if (ret < 0 && errno == EINVAL) {
498 ksft_test_result_skip(msg: "PR_SET_MEMORY_MERGE not supported\n");
499 return;
500 } else if (ret) {
501 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=1 failed\n");
502 return;
503 }
504
505 child_pid = fork();
506 if (child_pid == -1) {
507 ksft_test_result_skip(msg: "fork() failed\n");
508 return;
509 } else if (child_pid == 0) {
510 char *prg_name = "./ksm_functional_tests";
511 char *argv_for_program[] = { prg_name, FORK_EXEC_CHILD_PRG_NAME };
512
513 execv(prg_name, argv_for_program);
514 return;
515 }
516
517 if (waitpid(child_pid, &status, 0) > 0) {
518 if (WIFEXITED(status)) {
519 status = WEXITSTATUS(status);
520 if (status) {
521 ksft_test_result_fail(msg: "KSM not enabled\n");
522 return;
523 }
524 } else {
525 ksft_test_result_fail(msg: "program didn't terminate normally\n");
526 return;
527 }
528 } else {
529 ksft_test_result_fail(msg: "waitpid() failed\n");
530 return;
531 }
532
533 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) {
534 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=0 failed\n");
535 return;
536 }
537
538 ksft_test_result_pass(msg: "PR_SET_MEMORY_MERGE value is inherited\n");
539}
540
541static void test_prctl_unmerge(void)
542{
543 const unsigned int size = 2 * MiB;
544 char *map;
545
546 ksft_print_msg(msg: "[RUN] %s\n", __func__);
547
548 map = mmap_and_merge_range(0xcf, size, PROT_READ | PROT_WRITE, true);
549 if (map == MAP_FAILED)
550 return;
551
552 if (prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0)) {
553 ksft_test_result_fail(msg: "PR_SET_MEMORY_MERGE=0 failed\n");
554 goto unmap;
555 }
556
557 ksft_test_result(!range_maps_duplicates(map, size),
558 "Pages were unmerged\n");
559unmap:
560 munmap(map, size);
561}
562
563static void test_prot_none(void)
564{
565 const unsigned int size = 2 * MiB;
566 char *map;
567 int i;
568
569 ksft_print_msg(msg: "[RUN] %s\n", __func__);
570
571 map = mmap_and_merge_range(0x11, size, PROT_NONE, false);
572 if (map == MAP_FAILED)
573 goto unmap;
574
575 /* Store a unique value in each page on one half using ptrace */
576 for (i = 0; i < size / 2; i += pagesize) {
577 lseek(mem_fd, (uintptr_t) map + i, SEEK_SET);
578 if (write(mem_fd, &i, sizeof(i)) != sizeof(i)) {
579 ksft_test_result_fail(msg: "ptrace write failed\n");
580 goto unmap;
581 }
582 }
583
584 /* Trigger unsharing on the other half. */
585 if (madvise(map + size / 2, size / 2, MADV_UNMERGEABLE)) {
586 ksft_test_result_fail(msg: "MADV_UNMERGEABLE failed\n");
587 goto unmap;
588 }
589
590 ksft_test_result(!range_maps_duplicates(map, size),
591 "Pages were unmerged\n");
592unmap:
593 munmap(map, size);
594}
595
596int main(int argc, char **argv)
597{
598 unsigned int tests = 8;
599 int err;
600
601 if (argc > 1 && !strcmp(argv[1], FORK_EXEC_CHILD_PRG_NAME)) {
602 exit(ksm_fork_exec_child() == 1 ? 0 : 1);
603 }
604
605#ifdef __NR_userfaultfd
606 tests++;
607#endif
608
609 ksft_print_header();
610 ksft_set_plan(plan: tests);
611
612 pagesize = getpagesize();
613
614 mem_fd = open("/proc/self/mem", O_RDWR);
615 if (mem_fd < 0)
616 ksft_exit_fail_msg(msg: "opening /proc/self/mem failed\n");
617 ksm_fd = open("/sys/kernel/mm/ksm/run", O_RDWR);
618 if (ksm_fd < 0)
619 ksft_exit_skip(msg: "open(\"/sys/kernel/mm/ksm/run\") failed\n");
620 ksm_full_scans_fd = open("/sys/kernel/mm/ksm/full_scans", O_RDONLY);
621 if (ksm_full_scans_fd < 0)
622 ksft_exit_skip(msg: "open(\"/sys/kernel/mm/ksm/full_scans\") failed\n");
623 pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
624 if (pagemap_fd < 0)
625 ksft_exit_skip(msg: "open(\"/proc/self/pagemap\") failed\n");
626 proc_self_ksm_stat_fd = open("/proc/self/ksm_stat", O_RDONLY);
627 proc_self_ksm_merging_pages_fd = open("/proc/self/ksm_merging_pages",
628 O_RDONLY);
629 ksm_use_zero_pages_fd = open("/sys/kernel/mm/ksm/use_zero_pages", O_RDWR);
630
631 test_unmerge();
632 test_unmerge_zero_pages();
633 test_unmerge_discarded();
634#ifdef __NR_userfaultfd
635 test_unmerge_uffd_wp();
636#endif
637
638 test_prot_none();
639
640 test_prctl();
641 test_prctl_fork();
642 test_prctl_fork_exec();
643 test_prctl_unmerge();
644
645 err = ksft_get_fail_cnt();
646 if (err)
647 ksft_exit_fail_msg(msg: "%d out of %d tests failed\n",
648 err, ksft_test_num());
649 return ksft_exit_pass();
650}
651

source code of linux/tools/testing/selftests/mm/ksm_functional_tests.c