1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* HMM stands for Heterogeneous Memory Management, it is a helper layer inside
4	* the linux kernel to help device drivers mirror a process address space in
5	* the device. This allows the device to use the same address space which
6	* makes communication and data exchange a lot easier.
7	*
8	* This framework's sole purpose is to exercise various code paths inside
9	* the kernel to make sure that HMM performs as expected and to flush out any
10	* bugs.
11	*/
12
13	#include "../kselftest_harness.h"
14
15	#include <errno.h>
16	#include <fcntl.h>
17	#include <stdio.h>
18	#include <stdlib.h>
19	#include <stdint.h>
20	#include <unistd.h>
21	#include <strings.h>
22	#include <time.h>
23	#include <pthread.h>
24	#include <sys/types.h>
25	#include <sys/stat.h>
26	#include <sys/mman.h>
27	#include <sys/ioctl.h>
28
29
30	/*
31	* This is a private UAPI to the kernel test module so it isn't exported
32	* in the usual include/uapi/... directory.
33	*/
34	#include <lib/test_hmm_uapi.h>
35	#include <mm/gup_test.h>
36
37	struct hmm_buffer {
38	void *ptr;
39	void *mirror;
40	unsigned long size;
41	int fd;
42	uint64_t cpages;
43	uint64_t faults;
44	};
45
46	enum {
47	HMM_PRIVATE_DEVICE_ONE,
48	HMM_PRIVATE_DEVICE_TWO,
49	HMM_COHERENCE_DEVICE_ONE,
50	HMM_COHERENCE_DEVICE_TWO,
51	};
52
53	#define TWOMEG (1 << 21)
54	#define HMM_BUFFER_SIZE (1024 << 12)
55	#define HMM_PATH_MAX 64
56	#define NTIMES 10
57
58	#define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
59	/* Just the flags we need, copied from mm.h: */
60
61	#ifndef FOLL_WRITE
62	#define FOLL_WRITE 0x01 /* check pte is writable */
63	#endif
64
65	#ifndef FOLL_LONGTERM
66	#define FOLL_LONGTERM 0x100 /* mapping lifetime is indefinite */
67	#endif
68	FIXTURE(hmm)
69	{
70	int fd;
71	unsigned int page_size;
72	unsigned int page_shift;
73	};
74
75	FIXTURE_VARIANT(hmm)
76	{
77	int device_number;
78	};
79
80	FIXTURE_VARIANT_ADD(hmm, hmm_device_private)
81	{
82	.device_number = HMM_PRIVATE_DEVICE_ONE,
83	};
84
85	FIXTURE_VARIANT_ADD(hmm, hmm_device_coherent)
86	{
87	.device_number = HMM_COHERENCE_DEVICE_ONE,
88	};
89
90	FIXTURE(hmm2)
91	{
92	int fd0;
93	int fd1;
94	unsigned int page_size;
95	unsigned int page_shift;
96	};
97
98	FIXTURE_VARIANT(hmm2)
99	{
100	int device_number0;
101	int device_number1;
102	};
103
104	FIXTURE_VARIANT_ADD(hmm2, hmm2_device_private)
105	{
106	.device_number0 = HMM_PRIVATE_DEVICE_ONE,
107	.device_number1 = HMM_PRIVATE_DEVICE_TWO,
108	};
109
110	FIXTURE_VARIANT_ADD(hmm2, hmm2_device_coherent)
111	{
112	.device_number0 = HMM_COHERENCE_DEVICE_ONE,
113	.device_number1 = HMM_COHERENCE_DEVICE_TWO,
114	};
115
116	static int hmm_open(int unit)
117	{
118	char pathname[HMM_PATH_MAX];
119	int fd;
120
121	snprintf(pathname, sizeof(pathname), "/dev/hmm_dmirror%d", unit);
122	fd = open(pathname, O_RDWR, 0);
123	if (fd < 0)
124	fprintf(stderr, "could not open hmm dmirror driver (%s)\n",
125	pathname);
126	return fd;
127	}
128
129	static bool hmm_is_coherent_type(int dev_num)
130	{
131	return (dev_num >= HMM_COHERENCE_DEVICE_ONE);
132	}
133
134	FIXTURE_SETUP(hmm)
135	{
136	self->page_size = sysconf(_SC_PAGE_SIZE);
137	self->page_shift = ffs(self->page_size) - 1;
138
139	self->fd = hmm_open(unit: variant->device_number);
140	if (self->fd < 0 && hmm_is_coherent_type(variant->device_number))
141	SKIP(return, "DEVICE_COHERENT not available");
142	ASSERT_GE(self->fd, 0);
143	}
144
145	FIXTURE_SETUP(hmm2)
146	{
147	self->page_size = sysconf(_SC_PAGE_SIZE);
148	self->page_shift = ffs(self->page_size) - 1;
149
150	self->fd0 = hmm_open(unit: variant->device_number0);
151	if (self->fd0 < 0 && hmm_is_coherent_type(variant->device_number0))
152	SKIP(return, "DEVICE_COHERENT not available");
153	ASSERT_GE(self->fd0, 0);
154	self->fd1 = hmm_open(unit: variant->device_number1);
155	ASSERT_GE(self->fd1, 0);
156	}
157
158	FIXTURE_TEARDOWN(hmm)
159	{
160	int ret = close(self->fd);
161
162	ASSERT_EQ(ret, 0);
163	self->fd = -1;
164	}
165
166	FIXTURE_TEARDOWN(hmm2)
167	{
168	int ret = close(self->fd0);
169
170	ASSERT_EQ(ret, 0);
171	self->fd0 = -1;
172
173	ret = close(self->fd1);
174	ASSERT_EQ(ret, 0);
175	self->fd1 = -1;
176	}
177
178	static int hmm_dmirror_cmd(int fd,
179	unsigned long request,
180	struct hmm_buffer *buffer,
181	unsigned long npages)
182	{
183	struct hmm_dmirror_cmd cmd;
184	int ret;
185
186	/* Simulate a device reading system memory. */
187	cmd.addr = (__u64)buffer->ptr;
188	cmd.ptr = (__u64)buffer->mirror;
189	cmd.npages = npages;
190
191	for (;;) {
192	ret = ioctl(fd, request, &cmd);
193	if (ret == 0)
194	break;
195	if (errno == EINTR)
196	continue;
197	return -errno;
198	}
199	buffer->cpages = cmd.cpages;
200	buffer->faults = cmd.faults;
201
202	return 0;
203	}
204
205	static void hmm_buffer_free(struct hmm_buffer *buffer)
206	{
207	if (buffer == NULL)
208	return;
209
210	if (buffer->ptr)
211	munmap(buffer->ptr, buffer->size);
212	free(buffer->mirror);
213	free(buffer);
214	}
215
216	/*
217	* Create a temporary file that will be deleted on close.
218	*/
219	static int hmm_create_file(unsigned long size)
220	{
221	char path[HMM_PATH_MAX];
222	int fd;
223
224	strcpy(path, "/tmp");
225	fd = open(path, O_TMPFILE | O_EXCL | O_RDWR, 0600);
226	if (fd >= 0) {
227	int r;
228
229	do {
230	r = ftruncate(fd, size);
231	} while (r == -1 && errno == EINTR);
232	if (!r)
233	return fd;
234	close(fd);
235	}
236	return -1;
237	}
238
239	/*
240	* Return a random unsigned number.
241	*/
242	static unsigned int hmm_random(void)
243	{
244	static int fd = -1;
245	unsigned int r;
246
247	if (fd < 0) {
248	fd = open("/dev/urandom", O_RDONLY);
249	if (fd < 0) {
250	fprintf(stderr, "%s:%d failed to open /dev/urandom\n",
251	__FILE__, __LINE__);
252	return ~0U;
253	}
254	}
255	read(fd, &r, sizeof(r));
256	return r;
257	}
258
259	static void hmm_nanosleep(unsigned int n)
260	{
261	struct timespec t;
262
263	t.tv_sec = 0;
264	t.tv_nsec = n;
265	nanosleep(&t, NULL);
266	}
267
268	static int hmm_migrate_sys_to_dev(int fd,
269	struct hmm_buffer *buffer,
270	unsigned long npages)
271	{
272	return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_DEV, buffer, npages);
273	}
274
275	static int hmm_migrate_dev_to_sys(int fd,
276	struct hmm_buffer *buffer,
277	unsigned long npages)
278	{
279	return hmm_dmirror_cmd(fd, HMM_DMIRROR_MIGRATE_TO_SYS, buffer, npages);
280	}
281
282	/*
283	* Simple NULL test of device open/close.
284	*/
285	TEST_F(hmm, open_close)
286	{
287	}
288
289	/*
290	* Read private anonymous memory.
291	*/
292	TEST_F(hmm, anon_read)
293	{
294	struct hmm_buffer *buffer;
295	unsigned long npages;
296	unsigned long size;
297	unsigned long i;
298	int *ptr;
299	int ret;
300	int val;
301
302	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
303	ASSERT_NE(npages, 0);
304	size = npages << self->page_shift;
305
306	buffer = malloc(sizeof(*buffer));
307	ASSERT_NE(buffer, NULL);
308
309	buffer->fd = -1;
310	buffer->size = size;
311	buffer->mirror = malloc(size);
312	ASSERT_NE(buffer->mirror, NULL);
313
314	buffer->ptr = mmap(NULL, size,
315	PROT_READ | PROT_WRITE,
316	MAP_PRIVATE | MAP_ANONYMOUS,
317	buffer->fd, 0);
318	ASSERT_NE(buffer->ptr, MAP_FAILED);
319
320	/*
321	* Initialize buffer in system memory but leave the first two pages
322	* zero (pte_none and pfn_zero).
323	*/
324	i = 2 * self->page_size / sizeof(*ptr);
325	for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
326	ptr[i] = i;
327
328	/* Set buffer permission to read-only. */
329	ret = mprotect(buffer->ptr, size, PROT_READ);
330	ASSERT_EQ(ret, 0);
331
332	/* Populate the CPU page table with a special zero page. */
333	val = *(int *)(buffer->ptr + self->page_size);
334	ASSERT_EQ(val, 0);
335
336	/* Simulate a device reading system memory. */
337	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
338	ASSERT_EQ(ret, 0);
339	ASSERT_EQ(buffer->cpages, npages);
340	ASSERT_EQ(buffer->faults, 1);
341
342	/* Check what the device read. */
343	ptr = buffer->mirror;
344	for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i)
345	ASSERT_EQ(ptr[i], 0);
346	for (; i < size / sizeof(*ptr); ++i)
347	ASSERT_EQ(ptr[i], i);
348
349	hmm_buffer_free(buffer);
350	}
351
352	/*
353	* Read private anonymous memory which has been protected with
354	* mprotect() PROT_NONE.
355	*/
356	TEST_F(hmm, anon_read_prot)
357	{
358	struct hmm_buffer *buffer;
359	unsigned long npages;
360	unsigned long size;
361	unsigned long i;
362	int *ptr;
363	int ret;
364
365	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
366	ASSERT_NE(npages, 0);
367	size = npages << self->page_shift;
368
369	buffer = malloc(sizeof(*buffer));
370	ASSERT_NE(buffer, NULL);
371
372	buffer->fd = -1;
373	buffer->size = size;
374	buffer->mirror = malloc(size);
375	ASSERT_NE(buffer->mirror, NULL);
376
377	buffer->ptr = mmap(NULL, size,
378	PROT_READ | PROT_WRITE,
379	MAP_PRIVATE | MAP_ANONYMOUS,
380	buffer->fd, 0);
381	ASSERT_NE(buffer->ptr, MAP_FAILED);
382
383	/* Initialize buffer in system memory. */
384	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
385	ptr[i] = i;
386
387	/* Initialize mirror buffer so we can verify it isn't written. */
388	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
389	ptr[i] = -i;
390
391	/* Protect buffer from reading. */
392	ret = mprotect(buffer->ptr, size, PROT_NONE);
393	ASSERT_EQ(ret, 0);
394
395	/* Simulate a device reading system memory. */
396	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
397	ASSERT_EQ(ret, -EFAULT);
398
399	/* Allow CPU to read the buffer so we can check it. */
400	ret = mprotect(buffer->ptr, size, PROT_READ);
401	ASSERT_EQ(ret, 0);
402	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
403	ASSERT_EQ(ptr[i], i);
404
405	/* Check what the device read. */
406	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
407	ASSERT_EQ(ptr[i], -i);
408
409	hmm_buffer_free(buffer);
410	}
411
412	/*
413	* Write private anonymous memory.
414	*/
415	TEST_F(hmm, anon_write)
416	{
417	struct hmm_buffer *buffer;
418	unsigned long npages;
419	unsigned long size;
420	unsigned long i;
421	int *ptr;
422	int ret;
423
424	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
425	ASSERT_NE(npages, 0);
426	size = npages << self->page_shift;
427
428	buffer = malloc(sizeof(*buffer));
429	ASSERT_NE(buffer, NULL);
430
431	buffer->fd = -1;
432	buffer->size = size;
433	buffer->mirror = malloc(size);
434	ASSERT_NE(buffer->mirror, NULL);
435
436	buffer->ptr = mmap(NULL, size,
437	PROT_READ | PROT_WRITE,
438	MAP_PRIVATE | MAP_ANONYMOUS,
439	buffer->fd, 0);
440	ASSERT_NE(buffer->ptr, MAP_FAILED);
441
442	/* Initialize data that the device will write to buffer->ptr. */
443	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
444	ptr[i] = i;
445
446	/* Simulate a device writing system memory. */
447	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
448	ASSERT_EQ(ret, 0);
449	ASSERT_EQ(buffer->cpages, npages);
450	ASSERT_EQ(buffer->faults, 1);
451
452	/* Check what the device wrote. */
453	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
454	ASSERT_EQ(ptr[i], i);
455
456	hmm_buffer_free(buffer);
457	}
458
459	/*
460	* Write private anonymous memory which has been protected with
461	* mprotect() PROT_READ.
462	*/
463	TEST_F(hmm, anon_write_prot)
464	{
465	struct hmm_buffer *buffer;
466	unsigned long npages;
467	unsigned long size;
468	unsigned long i;
469	int *ptr;
470	int ret;
471
472	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
473	ASSERT_NE(npages, 0);
474	size = npages << self->page_shift;
475
476	buffer = malloc(sizeof(*buffer));
477	ASSERT_NE(buffer, NULL);
478
479	buffer->fd = -1;
480	buffer->size = size;
481	buffer->mirror = malloc(size);
482	ASSERT_NE(buffer->mirror, NULL);
483
484	buffer->ptr = mmap(NULL, size,
485	PROT_READ,
486	MAP_PRIVATE | MAP_ANONYMOUS,
487	buffer->fd, 0);
488	ASSERT_NE(buffer->ptr, MAP_FAILED);
489
490	/* Simulate a device reading a zero page of memory. */
491	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1);
492	ASSERT_EQ(ret, 0);
493	ASSERT_EQ(buffer->cpages, 1);
494	ASSERT_EQ(buffer->faults, 1);
495
496	/* Initialize data that the device will write to buffer->ptr. */
497	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
498	ptr[i] = i;
499
500	/* Simulate a device writing system memory. */
501	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
502	ASSERT_EQ(ret, -EPERM);
503
504	/* Check what the device wrote. */
505	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
506	ASSERT_EQ(ptr[i], 0);
507
508	/* Now allow writing and see that the zero page is replaced. */
509	ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ);
510	ASSERT_EQ(ret, 0);
511
512	/* Simulate a device writing system memory. */
513	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
514	ASSERT_EQ(ret, 0);
515	ASSERT_EQ(buffer->cpages, npages);
516	ASSERT_EQ(buffer->faults, 1);
517
518	/* Check what the device wrote. */
519	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
520	ASSERT_EQ(ptr[i], i);
521
522	hmm_buffer_free(buffer);
523	}
524
525	/*
526	* Check that a device writing an anonymous private mapping
527	* will copy-on-write if a child process inherits the mapping.
528	*/
529	TEST_F(hmm, anon_write_child)
530	{
531	struct hmm_buffer *buffer;
532	unsigned long npages;
533	unsigned long size;
534	unsigned long i;
535	int *ptr;
536	pid_t pid;
537	int child_fd;
538	int ret;
539
540	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
541	ASSERT_NE(npages, 0);
542	size = npages << self->page_shift;
543
544	buffer = malloc(sizeof(*buffer));
545	ASSERT_NE(buffer, NULL);
546
547	buffer->fd = -1;
548	buffer->size = size;
549	buffer->mirror = malloc(size);
550	ASSERT_NE(buffer->mirror, NULL);
551
552	buffer->ptr = mmap(NULL, size,
553	PROT_READ | PROT_WRITE,
554	MAP_PRIVATE | MAP_ANONYMOUS,
555	buffer->fd, 0);
556	ASSERT_NE(buffer->ptr, MAP_FAILED);
557
558	/* Initialize buffer->ptr so we can tell if it is written. */
559	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
560	ptr[i] = i;
561
562	/* Initialize data that the device will write to buffer->ptr. */
563	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
564	ptr[i] = -i;
565
566	pid = fork();
567	if (pid == -1)
568	ASSERT_EQ(pid, 0);
569	if (pid != 0) {
570	waitpid(pid, &ret, 0);
571	ASSERT_EQ(WIFEXITED(ret), 1);
572
573	/* Check that the parent's buffer did not change. */
574	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
575	ASSERT_EQ(ptr[i], i);
576	return;
577	}
578
579	/* Check that we see the parent's values. */
580	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
581	ASSERT_EQ(ptr[i], i);
582	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
583	ASSERT_EQ(ptr[i], -i);
584
585	/* The child process needs its own mirror to its own mm. */
586	child_fd = hmm_open(unit: 0);
587	ASSERT_GE(child_fd, 0);
588
589	/* Simulate a device writing system memory. */
590	ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
591	ASSERT_EQ(ret, 0);
592	ASSERT_EQ(buffer->cpages, npages);
593	ASSERT_EQ(buffer->faults, 1);
594
595	/* Check what the device wrote. */
596	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
597	ASSERT_EQ(ptr[i], -i);
598
599	close(child_fd);
600	exit(0);
601	}
602
603	/*
604	* Check that a device writing an anonymous shared mapping
605	* will not copy-on-write if a child process inherits the mapping.
606	*/
607	TEST_F(hmm, anon_write_child_shared)
608	{
609	struct hmm_buffer *buffer;
610	unsigned long npages;
611	unsigned long size;
612	unsigned long i;
613	int *ptr;
614	pid_t pid;
615	int child_fd;
616	int ret;
617
618	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
619	ASSERT_NE(npages, 0);
620	size = npages << self->page_shift;
621
622	buffer = malloc(sizeof(*buffer));
623	ASSERT_NE(buffer, NULL);
624
625	buffer->fd = -1;
626	buffer->size = size;
627	buffer->mirror = malloc(size);
628	ASSERT_NE(buffer->mirror, NULL);
629
630	buffer->ptr = mmap(NULL, size,
631	PROT_READ | PROT_WRITE,
632	MAP_SHARED | MAP_ANONYMOUS,
633	buffer->fd, 0);
634	ASSERT_NE(buffer->ptr, MAP_FAILED);
635
636	/* Initialize buffer->ptr so we can tell if it is written. */
637	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
638	ptr[i] = i;
639
640	/* Initialize data that the device will write to buffer->ptr. */
641	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
642	ptr[i] = -i;
643
644	pid = fork();
645	if (pid == -1)
646	ASSERT_EQ(pid, 0);
647	if (pid != 0) {
648	waitpid(pid, &ret, 0);
649	ASSERT_EQ(WIFEXITED(ret), 1);
650
651	/* Check that the parent's buffer did change. */
652	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
653	ASSERT_EQ(ptr[i], -i);
654	return;
655	}
656
657	/* Check that we see the parent's values. */
658	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
659	ASSERT_EQ(ptr[i], i);
660	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
661	ASSERT_EQ(ptr[i], -i);
662
663	/* The child process needs its own mirror to its own mm. */
664	child_fd = hmm_open(unit: 0);
665	ASSERT_GE(child_fd, 0);
666
667	/* Simulate a device writing system memory. */
668	ret = hmm_dmirror_cmd(child_fd, HMM_DMIRROR_WRITE, buffer, npages);
669	ASSERT_EQ(ret, 0);
670	ASSERT_EQ(buffer->cpages, npages);
671	ASSERT_EQ(buffer->faults, 1);
672
673	/* Check what the device wrote. */
674	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
675	ASSERT_EQ(ptr[i], -i);
676
677	close(child_fd);
678	exit(0);
679	}
680
681	/*
682	* Write private anonymous huge page.
683	*/
684	TEST_F(hmm, anon_write_huge)
685	{
686	struct hmm_buffer *buffer;
687	unsigned long npages;
688	unsigned long size;
689	unsigned long i;
690	void *old_ptr;
691	void *map;
692	int *ptr;
693	int ret;
694
695	size = 2 * TWOMEG;
696
697	buffer = malloc(sizeof(*buffer));
698	ASSERT_NE(buffer, NULL);
699
700	buffer->fd = -1;
701	buffer->size = size;
702	buffer->mirror = malloc(size);
703	ASSERT_NE(buffer->mirror, NULL);
704
705	buffer->ptr = mmap(NULL, size,
706	PROT_READ | PROT_WRITE,
707	MAP_PRIVATE | MAP_ANONYMOUS,
708	buffer->fd, 0);
709	ASSERT_NE(buffer->ptr, MAP_FAILED);
710
711	size = TWOMEG;
712	npages = size >> self->page_shift;
713	map = (void *)ALIGN((uintptr_t)buffer->ptr, size);
714	ret = madvise(map, size, MADV_HUGEPAGE);
715	ASSERT_EQ(ret, 0);
716	old_ptr = buffer->ptr;
717	buffer->ptr = map;
718
719	/* Initialize data that the device will write to buffer->ptr. */
720	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
721	ptr[i] = i;
722
723	/* Simulate a device writing system memory. */
724	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
725	ASSERT_EQ(ret, 0);
726	ASSERT_EQ(buffer->cpages, npages);
727	ASSERT_EQ(buffer->faults, 1);
728
729	/* Check what the device wrote. */
730	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
731	ASSERT_EQ(ptr[i], i);
732
733	buffer->ptr = old_ptr;
734	hmm_buffer_free(buffer);
735	}
736
737	/*
738	* Read numeric data from raw and tagged kernel status files. Used to read
739	* /proc and /sys data (without a tag) and from /proc/meminfo (with a tag).
740	*/
741	static long file_read_ulong(char *file, const char *tag)
742	{
743	int fd;
744	char buf[2048];
745	int len;
746	char *p, *q;
747	long val;
748
749	fd = open(file, O_RDONLY);
750	if (fd < 0) {
751	/* Error opening the file */
752	return -1;
753	}
754
755	len = read(fd, buf, sizeof(buf));
756	close(fd);
757	if (len < 0) {
758	/* Error in reading the file */
759	return -1;
760	}
761	if (len == sizeof(buf)) {
762	/* Error file is too large */
763	return -1;
764	}
765	buf[len] = '\0';
766
767	/* Search for a tag if provided */
768	if (tag) {
769	p = strstr(buf, tag);
770	if (!p)
771	return -1; /* looks like the line we want isn't there */
772	p += strlen(tag);
773	} else
774	p = buf;
775
776	val = strtol(p, &q, 0);
777	if (*q != ' ') {
778	/* Error parsing the file */
779	return -1;
780	}
781
782	return val;
783	}
784
785	/*
786	* Write huge TLBFS page.
787	*/
788	TEST_F(hmm, anon_write_hugetlbfs)
789	{
790	struct hmm_buffer *buffer;
791	unsigned long npages;
792	unsigned long size;
793	unsigned long default_hsize;
794	unsigned long i;
795	int *ptr;
796	int ret;
797
798	default_hsize = file_read_ulong(file: "/proc/meminfo", tag: "Hugepagesize:");
799	if (default_hsize < 0 || default_hsize*1024 < default_hsize)
800	SKIP(return, "Huge page size could not be determined");
801	default_hsize = default_hsize*1024; /* KB to B */
802
803	size = ALIGN(TWOMEG, default_hsize);
804	npages = size >> self->page_shift;
805
806	buffer = malloc(sizeof(*buffer));
807	ASSERT_NE(buffer, NULL);
808
809	buffer->ptr = mmap(NULL, size,
810	PROT_READ | PROT_WRITE,
811	MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
812	-1, 0);
813	if (buffer->ptr == MAP_FAILED) {
814	free(buffer);
815	SKIP(return, "Huge page could not be allocated");
816	}
817
818	buffer->fd = -1;
819	buffer->size = size;
820	buffer->mirror = malloc(size);
821	ASSERT_NE(buffer->mirror, NULL);
822
823	/* Initialize data that the device will write to buffer->ptr. */
824	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
825	ptr[i] = i;
826
827	/* Simulate a device writing system memory. */
828	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
829	ASSERT_EQ(ret, 0);
830	ASSERT_EQ(buffer->cpages, npages);
831	ASSERT_EQ(buffer->faults, 1);
832
833	/* Check what the device wrote. */
834	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
835	ASSERT_EQ(ptr[i], i);
836
837	munmap(buffer->ptr, buffer->size);
838	buffer->ptr = NULL;
839	hmm_buffer_free(buffer);
840	}
841
842	/*
843	* Read mmap'ed file memory.
844	*/
845	TEST_F(hmm, file_read)
846	{
847	struct hmm_buffer *buffer;
848	unsigned long npages;
849	unsigned long size;
850	unsigned long i;
851	int *ptr;
852	int ret;
853	int fd;
854	ssize_t len;
855
856	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
857	ASSERT_NE(npages, 0);
858	size = npages << self->page_shift;
859
860	fd = hmm_create_file(size);
861	ASSERT_GE(fd, 0);
862
863	buffer = malloc(sizeof(*buffer));
864	ASSERT_NE(buffer, NULL);
865
866	buffer->fd = fd;
867	buffer->size = size;
868	buffer->mirror = malloc(size);
869	ASSERT_NE(buffer->mirror, NULL);
870
871	/* Write initial contents of the file. */
872	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
873	ptr[i] = i;
874	len = pwrite(fd, buffer->mirror, size, 0);
875	ASSERT_EQ(len, size);
876	memset(buffer->mirror, 0, size);
877
878	buffer->ptr = mmap(NULL, size,
879	PROT_READ,
880	MAP_SHARED,
881	buffer->fd, 0);
882	ASSERT_NE(buffer->ptr, MAP_FAILED);
883
884	/* Simulate a device reading system memory. */
885	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages);
886	ASSERT_EQ(ret, 0);
887	ASSERT_EQ(buffer->cpages, npages);
888	ASSERT_EQ(buffer->faults, 1);
889
890	/* Check what the device read. */
891	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
892	ASSERT_EQ(ptr[i], i);
893
894	hmm_buffer_free(buffer);
895	}
896
897	/*
898	* Write mmap'ed file memory.
899	*/
900	TEST_F(hmm, file_write)
901	{
902	struct hmm_buffer *buffer;
903	unsigned long npages;
904	unsigned long size;
905	unsigned long i;
906	int *ptr;
907	int ret;
908	int fd;
909	ssize_t len;
910
911	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
912	ASSERT_NE(npages, 0);
913	size = npages << self->page_shift;
914
915	fd = hmm_create_file(size);
916	ASSERT_GE(fd, 0);
917
918	buffer = malloc(sizeof(*buffer));
919	ASSERT_NE(buffer, NULL);
920
921	buffer->fd = fd;
922	buffer->size = size;
923	buffer->mirror = malloc(size);
924	ASSERT_NE(buffer->mirror, NULL);
925
926	buffer->ptr = mmap(NULL, size,
927	PROT_READ | PROT_WRITE,
928	MAP_SHARED,
929	buffer->fd, 0);
930	ASSERT_NE(buffer->ptr, MAP_FAILED);
931
932	/* Initialize data that the device will write to buffer->ptr. */
933	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
934	ptr[i] = i;
935
936	/* Simulate a device writing system memory. */
937	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
938	ASSERT_EQ(ret, 0);
939	ASSERT_EQ(buffer->cpages, npages);
940	ASSERT_EQ(buffer->faults, 1);
941
942	/* Check what the device wrote. */
943	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
944	ASSERT_EQ(ptr[i], i);
945
946	/* Check that the device also wrote the file. */
947	len = pread(fd, buffer->mirror, size, 0);
948	ASSERT_EQ(len, size);
949	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
950	ASSERT_EQ(ptr[i], i);
951
952	hmm_buffer_free(buffer);
953	}
954
955	/*
956	* Migrate anonymous memory to device private memory.
957	*/
958	TEST_F(hmm, migrate)
959	{
960	struct hmm_buffer *buffer;
961	unsigned long npages;
962	unsigned long size;
963	unsigned long i;
964	int *ptr;
965	int ret;
966
967	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
968	ASSERT_NE(npages, 0);
969	size = npages << self->page_shift;
970
971	buffer = malloc(sizeof(*buffer));
972	ASSERT_NE(buffer, NULL);
973
974	buffer->fd = -1;
975	buffer->size = size;
976	buffer->mirror = malloc(size);
977	ASSERT_NE(buffer->mirror, NULL);
978
979	buffer->ptr = mmap(NULL, size,
980	PROT_READ | PROT_WRITE,
981	MAP_PRIVATE | MAP_ANONYMOUS,
982	buffer->fd, 0);
983	ASSERT_NE(buffer->ptr, MAP_FAILED);
984
985	/* Initialize buffer in system memory. */
986	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
987	ptr[i] = i;
988
989	/* Migrate memory to device. */
990	ret = hmm_migrate_sys_to_dev(fd: self->fd, buffer, npages);
991	ASSERT_EQ(ret, 0);
992	ASSERT_EQ(buffer->cpages, npages);
993
994	/* Check what the device read. */
995	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
996	ASSERT_EQ(ptr[i], i);
997
998	hmm_buffer_free(buffer);
999	}
1000
1001	/*
1002	* Migrate anonymous memory to device private memory and fault some of it back
1003	* to system memory, then try migrating the resulting mix of system and device
1004	* private memory to the device.
1005	*/
1006	TEST_F(hmm, migrate_fault)
1007	{
1008	struct hmm_buffer *buffer;
1009	unsigned long npages;
1010	unsigned long size;
1011	unsigned long i;
1012	int *ptr;
1013	int ret;
1014
1015	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1016	ASSERT_NE(npages, 0);
1017	size = npages << self->page_shift;
1018
1019	buffer = malloc(sizeof(*buffer));
1020	ASSERT_NE(buffer, NULL);
1021
1022	buffer->fd = -1;
1023	buffer->size = size;
1024	buffer->mirror = malloc(size);
1025	ASSERT_NE(buffer->mirror, NULL);
1026
1027	buffer->ptr = mmap(NULL, size,
1028	PROT_READ | PROT_WRITE,
1029	MAP_PRIVATE | MAP_ANONYMOUS,
1030	buffer->fd, 0);
1031	ASSERT_NE(buffer->ptr, MAP_FAILED);
1032
1033	/* Initialize buffer in system memory. */
1034	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1035	ptr[i] = i;
1036
1037	/* Migrate memory to device. */
1038	ret = hmm_migrate_sys_to_dev(fd: self->fd, buffer, npages);
1039	ASSERT_EQ(ret, 0);
1040	ASSERT_EQ(buffer->cpages, npages);
1041
1042	/* Check what the device read. */
1043	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1044	ASSERT_EQ(ptr[i], i);
1045
1046	/* Fault half the pages back to system memory and check them. */
1047	for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1048	ASSERT_EQ(ptr[i], i);
1049
1050	/* Migrate memory to the device again. */
1051	ret = hmm_migrate_sys_to_dev(fd: self->fd, buffer, npages);
1052	ASSERT_EQ(ret, 0);
1053	ASSERT_EQ(buffer->cpages, npages);
1054
1055	/* Check what the device read. */
1056	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1057	ASSERT_EQ(ptr[i], i);
1058
1059	hmm_buffer_free(buffer);
1060	}
1061
1062	TEST_F(hmm, migrate_release)
1063	{
1064	struct hmm_buffer *buffer;
1065	unsigned long npages;
1066	unsigned long size;
1067	unsigned long i;
1068	int *ptr;
1069	int ret;
1070
1071	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1072	ASSERT_NE(npages, 0);
1073	size = npages << self->page_shift;
1074
1075	buffer = malloc(sizeof(*buffer));
1076	ASSERT_NE(buffer, NULL);
1077
1078	buffer->fd = -1;
1079	buffer->size = size;
1080	buffer->mirror = malloc(size);
1081	ASSERT_NE(buffer->mirror, NULL);
1082
1083	buffer->ptr = mmap(NULL, size, PROT_READ | PROT_WRITE,
1084	MAP_PRIVATE | MAP_ANONYMOUS, buffer->fd, 0);
1085	ASSERT_NE(buffer->ptr, MAP_FAILED);
1086
1087	/* Initialize buffer in system memory. */
1088	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1089	ptr[i] = i;
1090
1091	/* Migrate memory to device. */
1092	ret = hmm_migrate_sys_to_dev(fd: self->fd, buffer, npages);
1093	ASSERT_EQ(ret, 0);
1094	ASSERT_EQ(buffer->cpages, npages);
1095
1096	/* Check what the device read. */
1097	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1098	ASSERT_EQ(ptr[i], i);
1099
1100	/* Release device memory. */
1101	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_RELEASE, buffer, npages);
1102	ASSERT_EQ(ret, 0);
1103
1104	/* Fault pages back to system memory and check them. */
1105	for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i)
1106	ASSERT_EQ(ptr[i], i);
1107
1108	hmm_buffer_free(buffer);
1109	}
1110
1111	/*
1112	* Migrate anonymous shared memory to device private memory.
1113	*/
1114	TEST_F(hmm, migrate_shared)
1115	{
1116	struct hmm_buffer *buffer;
1117	unsigned long npages;
1118	unsigned long size;
1119	int ret;
1120
1121	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1122	ASSERT_NE(npages, 0);
1123	size = npages << self->page_shift;
1124
1125	buffer = malloc(sizeof(*buffer));
1126	ASSERT_NE(buffer, NULL);
1127
1128	buffer->fd = -1;
1129	buffer->size = size;
1130	buffer->mirror = malloc(size);
1131	ASSERT_NE(buffer->mirror, NULL);
1132
1133	buffer->ptr = mmap(NULL, size,
1134	PROT_READ | PROT_WRITE,
1135	MAP_SHARED | MAP_ANONYMOUS,
1136	buffer->fd, 0);
1137	ASSERT_NE(buffer->ptr, MAP_FAILED);
1138
1139	/* Migrate memory to device. */
1140	ret = hmm_migrate_sys_to_dev(fd: self->fd, buffer, npages);
1141	ASSERT_EQ(ret, -ENOENT);
1142
1143	hmm_buffer_free(buffer);
1144	}
1145
1146	/*
1147	* Try to migrate various memory types to device private memory.
1148	*/
1149	TEST_F(hmm2, migrate_mixed)
1150	{
1151	struct hmm_buffer *buffer;
1152	unsigned long npages;
1153	unsigned long size;
1154	int *ptr;
1155	unsigned char *p;
1156	int ret;
1157	int val;
1158
1159	npages = 6;
1160	size = npages << self->page_shift;
1161
1162	buffer = malloc(sizeof(*buffer));
1163	ASSERT_NE(buffer, NULL);
1164
1165	buffer->fd = -1;
1166	buffer->size = size;
1167	buffer->mirror = malloc(size);
1168	ASSERT_NE(buffer->mirror, NULL);
1169
1170	/* Reserve a range of addresses. */
1171	buffer->ptr = mmap(NULL, size,
1172	PROT_NONE,
1173	MAP_PRIVATE | MAP_ANONYMOUS,
1174	buffer->fd, 0);
1175	ASSERT_NE(buffer->ptr, MAP_FAILED);
1176	p = buffer->ptr;
1177
1178	/* Migrating a protected area should be an error. */
1179	ret = hmm_migrate_sys_to_dev(fd: self->fd1, buffer, npages);
1180	ASSERT_EQ(ret, -EINVAL);
1181
1182	/* Punch a hole after the first page address. */
1183	ret = munmap(buffer->ptr + self->page_size, self->page_size);
1184	ASSERT_EQ(ret, 0);
1185
1186	/* We expect an error if the vma doesn't cover the range. */
1187	ret = hmm_migrate_sys_to_dev(fd: self->fd1, buffer, npages: 3);
1188	ASSERT_EQ(ret, -EINVAL);
1189
1190	/* Page 2 will be a read-only zero page. */
1191	ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1192	PROT_READ);
1193	ASSERT_EQ(ret, 0);
1194	ptr = (int *)(buffer->ptr + 2 * self->page_size);
1195	val = *ptr + 3;
1196	ASSERT_EQ(val, 3);
1197
1198	/* Page 3 will be read-only. */
1199	ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1200	PROT_READ | PROT_WRITE);
1201	ASSERT_EQ(ret, 0);
1202	ptr = (int *)(buffer->ptr + 3 * self->page_size);
1203	*ptr = val;
1204	ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1205	PROT_READ);
1206	ASSERT_EQ(ret, 0);
1207
1208	/* Page 4-5 will be read-write. */
1209	ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size,
1210	PROT_READ | PROT_WRITE);
1211	ASSERT_EQ(ret, 0);
1212	ptr = (int *)(buffer->ptr + 4 * self->page_size);
1213	*ptr = val;
1214	ptr = (int *)(buffer->ptr + 5 * self->page_size);
1215	*ptr = val;
1216
1217	/* Now try to migrate pages 2-5 to device 1. */
1218	buffer->ptr = p + 2 * self->page_size;
1219	ret = hmm_migrate_sys_to_dev(fd: self->fd1, buffer, npages: 4);
1220	ASSERT_EQ(ret, 0);
1221	ASSERT_EQ(buffer->cpages, 4);
1222
1223	/* Page 5 won't be migrated to device 0 because it's on device 1. */
1224	buffer->ptr = p + 5 * self->page_size;
1225	ret = hmm_migrate_sys_to_dev(fd: self->fd0, buffer, npages: 1);
1226	ASSERT_EQ(ret, -ENOENT);
1227	buffer->ptr = p;
1228
1229	buffer->ptr = p;
1230	hmm_buffer_free(buffer);
1231	}
1232
1233	/*
1234	* Migrate anonymous memory to device memory and back to system memory
1235	* multiple times. In case of private zone configuration, this is done
1236	* through fault pages accessed by CPU. In case of coherent zone configuration,
1237	* the pages from the device should be explicitly migrated back to system memory.
1238	* The reason is Coherent device zone has coherent access by CPU, therefore
1239	* it will not generate any page fault.
1240	*/
1241	TEST_F(hmm, migrate_multiple)
1242	{
1243	struct hmm_buffer *buffer;
1244	unsigned long npages;
1245	unsigned long size;
1246	unsigned long i;
1247	unsigned long c;
1248	int *ptr;
1249	int ret;
1250
1251	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1252	ASSERT_NE(npages, 0);
1253	size = npages << self->page_shift;
1254
1255	for (c = 0; c < NTIMES; c++) {
1256	buffer = malloc(sizeof(*buffer));
1257	ASSERT_NE(buffer, NULL);
1258
1259	buffer->fd = -1;
1260	buffer->size = size;
1261	buffer->mirror = malloc(size);
1262	ASSERT_NE(buffer->mirror, NULL);
1263
1264	buffer->ptr = mmap(NULL, size,
1265	PROT_READ | PROT_WRITE,
1266	MAP_PRIVATE | MAP_ANONYMOUS,
1267	buffer->fd, 0);
1268	ASSERT_NE(buffer->ptr, MAP_FAILED);
1269
1270	/* Initialize buffer in system memory. */
1271	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1272	ptr[i] = i;
1273
1274	/* Migrate memory to device. */
1275	ret = hmm_migrate_sys_to_dev(fd: self->fd, buffer, npages);
1276	ASSERT_EQ(ret, 0);
1277	ASSERT_EQ(buffer->cpages, npages);
1278
1279	/* Check what the device read. */
1280	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1281	ASSERT_EQ(ptr[i], i);
1282
1283	/* Migrate back to system memory and check them. */
1284	if (hmm_is_coherent_type(variant->device_number)) {
1285	ret = hmm_migrate_dev_to_sys(fd: self->fd, buffer, npages);
1286	ASSERT_EQ(ret, 0);
1287	ASSERT_EQ(buffer->cpages, npages);
1288	}
1289
1290	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1291	ASSERT_EQ(ptr[i], i);
1292
1293	hmm_buffer_free(buffer);
1294	}
1295	}
1296
1297	/*
1298	* Read anonymous memory multiple times.
1299	*/
1300	TEST_F(hmm, anon_read_multiple)
1301	{
1302	struct hmm_buffer *buffer;
1303	unsigned long npages;
1304	unsigned long size;
1305	unsigned long i;
1306	unsigned long c;
1307	int *ptr;
1308	int ret;
1309
1310	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1311	ASSERT_NE(npages, 0);
1312	size = npages << self->page_shift;
1313
1314	for (c = 0; c < NTIMES; c++) {
1315	buffer = malloc(sizeof(*buffer));
1316	ASSERT_NE(buffer, NULL);
1317
1318	buffer->fd = -1;
1319	buffer->size = size;
1320	buffer->mirror = malloc(size);
1321	ASSERT_NE(buffer->mirror, NULL);
1322
1323	buffer->ptr = mmap(NULL, size,
1324	PROT_READ | PROT_WRITE,
1325	MAP_PRIVATE | MAP_ANONYMOUS,
1326	buffer->fd, 0);
1327	ASSERT_NE(buffer->ptr, MAP_FAILED);
1328
1329	/* Initialize buffer in system memory. */
1330	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1331	ptr[i] = i + c;
1332
1333	/* Simulate a device reading system memory. */
1334	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1335	npages);
1336	ASSERT_EQ(ret, 0);
1337	ASSERT_EQ(buffer->cpages, npages);
1338	ASSERT_EQ(buffer->faults, 1);
1339
1340	/* Check what the device read. */
1341	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1342	ASSERT_EQ(ptr[i], i + c);
1343
1344	hmm_buffer_free(buffer);
1345	}
1346	}
1347
1348	void *unmap_buffer(void *p)
1349	{
1350	struct hmm_buffer *buffer = p;
1351
1352	/* Delay for a bit and then unmap buffer while it is being read. */
1353	hmm_nanosleep(n: hmm_random() % 32000);
1354	munmap(buffer->ptr + buffer->size / 2, buffer->size / 2);
1355	buffer->ptr = NULL;
1356
1357	return NULL;
1358	}
1359
1360	/*
1361	* Try reading anonymous memory while it is being unmapped.
1362	*/
1363	TEST_F(hmm, anon_teardown)
1364	{
1365	unsigned long npages;
1366	unsigned long size;
1367	unsigned long c;
1368	void *ret;
1369
1370	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1371	ASSERT_NE(npages, 0);
1372	size = npages << self->page_shift;
1373
1374	for (c = 0; c < NTIMES; ++c) {
1375	pthread_t thread;
1376	struct hmm_buffer *buffer;
1377	unsigned long i;
1378	int *ptr;
1379	int rc;
1380
1381	buffer = malloc(sizeof(*buffer));
1382	ASSERT_NE(buffer, NULL);
1383
1384	buffer->fd = -1;
1385	buffer->size = size;
1386	buffer->mirror = malloc(size);
1387	ASSERT_NE(buffer->mirror, NULL);
1388
1389	buffer->ptr = mmap(NULL, size,
1390	PROT_READ | PROT_WRITE,
1391	MAP_PRIVATE | MAP_ANONYMOUS,
1392	buffer->fd, 0);
1393	ASSERT_NE(buffer->ptr, MAP_FAILED);
1394
1395	/* Initialize buffer in system memory. */
1396	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1397	ptr[i] = i + c;
1398
1399	rc = pthread_create(&thread, NULL, unmap_buffer, buffer);
1400	ASSERT_EQ(rc, 0);
1401
1402	/* Simulate a device reading system memory. */
1403	rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer,
1404	npages);
1405	if (rc == 0) {
1406	ASSERT_EQ(buffer->cpages, npages);
1407	ASSERT_EQ(buffer->faults, 1);
1408
1409	/* Check what the device read. */
1410	for (i = 0, ptr = buffer->mirror;
1411	i < size / sizeof(*ptr);
1412	++i)
1413	ASSERT_EQ(ptr[i], i + c);
1414	}
1415
1416	pthread_join(thread, &ret);
1417	hmm_buffer_free(buffer);
1418	}
1419	}
1420
1421	/*
1422	* Test memory snapshot without faulting in pages accessed by the device.
1423	*/
1424	TEST_F(hmm, mixedmap)
1425	{
1426	struct hmm_buffer *buffer;
1427	unsigned long npages;
1428	unsigned long size;
1429	unsigned char *m;
1430	int ret;
1431
1432	npages = 1;
1433	size = npages << self->page_shift;
1434
1435	buffer = malloc(sizeof(*buffer));
1436	ASSERT_NE(buffer, NULL);
1437
1438	buffer->fd = -1;
1439	buffer->size = size;
1440	buffer->mirror = malloc(npages);
1441	ASSERT_NE(buffer->mirror, NULL);
1442
1443
1444	/* Reserve a range of addresses. */
1445	buffer->ptr = mmap(NULL, size,
1446	PROT_READ | PROT_WRITE,
1447	MAP_PRIVATE,
1448	self->fd, 0);
1449	ASSERT_NE(buffer->ptr, MAP_FAILED);
1450
1451	/* Simulate a device snapshotting CPU pagetables. */
1452	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1453	ASSERT_EQ(ret, 0);
1454	ASSERT_EQ(buffer->cpages, npages);
1455
1456	/* Check what the device saw. */
1457	m = buffer->mirror;
1458	ASSERT_EQ(m[0], HMM_DMIRROR_PROT_READ);
1459
1460	hmm_buffer_free(buffer);
1461	}
1462
1463	/*
1464	* Test memory snapshot without faulting in pages accessed by the device.
1465	*/
1466	TEST_F(hmm2, snapshot)
1467	{
1468	struct hmm_buffer *buffer;
1469	unsigned long npages;
1470	unsigned long size;
1471	int *ptr;
1472	unsigned char *p;
1473	unsigned char *m;
1474	int ret;
1475	int val;
1476
1477	npages = 7;
1478	size = npages << self->page_shift;
1479
1480	buffer = malloc(sizeof(*buffer));
1481	ASSERT_NE(buffer, NULL);
1482
1483	buffer->fd = -1;
1484	buffer->size = size;
1485	buffer->mirror = malloc(npages);
1486	ASSERT_NE(buffer->mirror, NULL);
1487
1488	/* Reserve a range of addresses. */
1489	buffer->ptr = mmap(NULL, size,
1490	PROT_NONE,
1491	MAP_PRIVATE | MAP_ANONYMOUS,
1492	buffer->fd, 0);
1493	ASSERT_NE(buffer->ptr, MAP_FAILED);
1494	p = buffer->ptr;
1495
1496	/* Punch a hole after the first page address. */
1497	ret = munmap(buffer->ptr + self->page_size, self->page_size);
1498	ASSERT_EQ(ret, 0);
1499
1500	/* Page 2 will be read-only zero page. */
1501	ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size,
1502	PROT_READ);
1503	ASSERT_EQ(ret, 0);
1504	ptr = (int *)(buffer->ptr + 2 * self->page_size);
1505	val = *ptr + 3;
1506	ASSERT_EQ(val, 3);
1507
1508	/* Page 3 will be read-only. */
1509	ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1510	PROT_READ | PROT_WRITE);
1511	ASSERT_EQ(ret, 0);
1512	ptr = (int *)(buffer->ptr + 3 * self->page_size);
1513	*ptr = val;
1514	ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size,
1515	PROT_READ);
1516	ASSERT_EQ(ret, 0);
1517
1518	/* Page 4-6 will be read-write. */
1519	ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size,
1520	PROT_READ | PROT_WRITE);
1521	ASSERT_EQ(ret, 0);
1522	ptr = (int *)(buffer->ptr + 4 * self->page_size);
1523	*ptr = val;
1524
1525	/* Page 5 will be migrated to device 0. */
1526	buffer->ptr = p + 5 * self->page_size;
1527	ret = hmm_migrate_sys_to_dev(fd: self->fd0, buffer, npages: 1);
1528	ASSERT_EQ(ret, 0);
1529	ASSERT_EQ(buffer->cpages, 1);
1530
1531	/* Page 6 will be migrated to device 1. */
1532	buffer->ptr = p + 6 * self->page_size;
1533	ret = hmm_migrate_sys_to_dev(fd: self->fd1, buffer, npages: 1);
1534	ASSERT_EQ(ret, 0);
1535	ASSERT_EQ(buffer->cpages, 1);
1536
1537	/* Simulate a device snapshotting CPU pagetables. */
1538	buffer->ptr = p;
1539	ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1540	ASSERT_EQ(ret, 0);
1541	ASSERT_EQ(buffer->cpages, npages);
1542
1543	/* Check what the device saw. */
1544	m = buffer->mirror;
1545	ASSERT_EQ(m[0], HMM_DMIRROR_PROT_ERROR);
1546	ASSERT_EQ(m[1], HMM_DMIRROR_PROT_ERROR);
1547	ASSERT_EQ(m[2], HMM_DMIRROR_PROT_ZERO | HMM_DMIRROR_PROT_READ);
1548	ASSERT_EQ(m[3], HMM_DMIRROR_PROT_READ);
1549	ASSERT_EQ(m[4], HMM_DMIRROR_PROT_WRITE);
1550	if (!hmm_is_coherent_type(variant->device_number0)) {
1551	ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL |
1552	HMM_DMIRROR_PROT_WRITE);
1553	ASSERT_EQ(m[6], HMM_DMIRROR_PROT_NONE);
1554	} else {
1555	ASSERT_EQ(m[5], HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL |
1556	HMM_DMIRROR_PROT_WRITE);
1557	ASSERT_EQ(m[6], HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE |
1558	HMM_DMIRROR_PROT_WRITE);
1559	}
1560
1561	hmm_buffer_free(buffer);
1562	}
1563
1564	/*
1565	* Test the hmm_range_fault() HMM_PFN_PMD flag for large pages that
1566	* should be mapped by a large page table entry.
1567	*/
1568	TEST_F(hmm, compound)
1569	{
1570	struct hmm_buffer *buffer;
1571	unsigned long npages;
1572	unsigned long size;
1573	unsigned long default_hsize;
1574	int *ptr;
1575	unsigned char *m;
1576	int ret;
1577	unsigned long i;
1578
1579	/* Skip test if we can't allocate a hugetlbfs page. */
1580
1581	default_hsize = file_read_ulong(file: "/proc/meminfo", tag: "Hugepagesize:");
1582	if (default_hsize < 0 || default_hsize*1024 < default_hsize)
1583	SKIP(return, "Huge page size could not be determined");
1584	default_hsize = default_hsize*1024; /* KB to B */
1585
1586	size = ALIGN(TWOMEG, default_hsize);
1587	npages = size >> self->page_shift;
1588
1589	buffer = malloc(sizeof(*buffer));
1590	ASSERT_NE(buffer, NULL);
1591
1592	buffer->ptr = mmap(NULL, size,
1593	PROT_READ | PROT_WRITE,
1594	MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
1595	-1, 0);
1596	if (buffer->ptr == MAP_FAILED) {
1597	free(buffer);
1598	return;
1599	}
1600
1601	buffer->size = size;
1602	buffer->mirror = malloc(npages);
1603	ASSERT_NE(buffer->mirror, NULL);
1604
1605	/* Initialize the pages the device will snapshot in buffer->ptr. */
1606	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1607	ptr[i] = i;
1608
1609	/* Simulate a device snapshotting CPU pagetables. */
1610	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1611	ASSERT_EQ(ret, 0);
1612	ASSERT_EQ(buffer->cpages, npages);
1613
1614	/* Check what the device saw. */
1615	m = buffer->mirror;
1616	for (i = 0; i < npages; ++i)
1617	ASSERT_EQ(m[i], HMM_DMIRROR_PROT_WRITE |
1618	HMM_DMIRROR_PROT_PMD);
1619
1620	/* Make the region read-only. */
1621	ret = mprotect(buffer->ptr, size, PROT_READ);
1622	ASSERT_EQ(ret, 0);
1623
1624	/* Simulate a device snapshotting CPU pagetables. */
1625	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1626	ASSERT_EQ(ret, 0);
1627	ASSERT_EQ(buffer->cpages, npages);
1628
1629	/* Check what the device saw. */
1630	m = buffer->mirror;
1631	for (i = 0; i < npages; ++i)
1632	ASSERT_EQ(m[i], HMM_DMIRROR_PROT_READ |
1633	HMM_DMIRROR_PROT_PMD);
1634
1635	munmap(buffer->ptr, buffer->size);
1636	buffer->ptr = NULL;
1637	hmm_buffer_free(buffer);
1638	}
1639
1640	/*
1641	* Test two devices reading the same memory (double mapped).
1642	*/
1643	TEST_F(hmm2, double_map)
1644	{
1645	struct hmm_buffer *buffer;
1646	unsigned long npages;
1647	unsigned long size;
1648	unsigned long i;
1649	int *ptr;
1650	int ret;
1651
1652	npages = 6;
1653	size = npages << self->page_shift;
1654
1655	buffer = malloc(sizeof(*buffer));
1656	ASSERT_NE(buffer, NULL);
1657
1658	buffer->fd = -1;
1659	buffer->size = size;
1660	buffer->mirror = malloc(npages);
1661	ASSERT_NE(buffer->mirror, NULL);
1662
1663	/* Reserve a range of addresses. */
1664	buffer->ptr = mmap(NULL, size,
1665	PROT_READ | PROT_WRITE,
1666	MAP_PRIVATE | MAP_ANONYMOUS,
1667	buffer->fd, 0);
1668	ASSERT_NE(buffer->ptr, MAP_FAILED);
1669
1670	/* Initialize buffer in system memory. */
1671	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1672	ptr[i] = i;
1673
1674	/* Make region read-only. */
1675	ret = mprotect(buffer->ptr, size, PROT_READ);
1676	ASSERT_EQ(ret, 0);
1677
1678	/* Simulate device 0 reading system memory. */
1679	ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1680	ASSERT_EQ(ret, 0);
1681	ASSERT_EQ(buffer->cpages, npages);
1682	ASSERT_EQ(buffer->faults, 1);
1683
1684	/* Check what the device read. */
1685	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1686	ASSERT_EQ(ptr[i], i);
1687
1688	/* Simulate device 1 reading system memory. */
1689	ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages);
1690	ASSERT_EQ(ret, 0);
1691	ASSERT_EQ(buffer->cpages, npages);
1692	ASSERT_EQ(buffer->faults, 1);
1693
1694	/* Check what the device read. */
1695	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1696	ASSERT_EQ(ptr[i], i);
1697
1698	/* Migrate pages to device 1 and try to read from device 0. */
1699	ret = hmm_migrate_sys_to_dev(fd: self->fd1, buffer, npages);
1700	ASSERT_EQ(ret, 0);
1701	ASSERT_EQ(buffer->cpages, npages);
1702
1703	ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages);
1704	ASSERT_EQ(ret, 0);
1705	ASSERT_EQ(buffer->cpages, npages);
1706	ASSERT_EQ(buffer->faults, 1);
1707
1708	/* Check what device 0 read. */
1709	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1710	ASSERT_EQ(ptr[i], i);
1711
1712	hmm_buffer_free(buffer);
1713	}
1714
1715	/*
1716	* Basic check of exclusive faulting.
1717	*/
1718	TEST_F(hmm, exclusive)
1719	{
1720	struct hmm_buffer *buffer;
1721	unsigned long npages;
1722	unsigned long size;
1723	unsigned long i;
1724	int *ptr;
1725	int ret;
1726
1727	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1728	ASSERT_NE(npages, 0);
1729	size = npages << self->page_shift;
1730
1731	buffer = malloc(sizeof(*buffer));
1732	ASSERT_NE(buffer, NULL);
1733
1734	buffer->fd = -1;
1735	buffer->size = size;
1736	buffer->mirror = malloc(size);
1737	ASSERT_NE(buffer->mirror, NULL);
1738
1739	buffer->ptr = mmap(NULL, size,
1740	PROT_READ | PROT_WRITE,
1741	MAP_PRIVATE | MAP_ANONYMOUS,
1742	buffer->fd, 0);
1743	ASSERT_NE(buffer->ptr, MAP_FAILED);
1744
1745	/* Initialize buffer in system memory. */
1746	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1747	ptr[i] = i;
1748
1749	/* Map memory exclusively for device access. */
1750	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1751	ASSERT_EQ(ret, 0);
1752	ASSERT_EQ(buffer->cpages, npages);
1753
1754	/* Check what the device read. */
1755	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1756	ASSERT_EQ(ptr[i], i);
1757
1758	/* Fault pages back to system memory and check them. */
1759	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1760	ASSERT_EQ(ptr[i]++, i);
1761
1762	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1763	ASSERT_EQ(ptr[i], i+1);
1764
1765	/* Check atomic access revoked */
1766	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_CHECK_EXCLUSIVE, buffer, npages);
1767	ASSERT_EQ(ret, 0);
1768
1769	hmm_buffer_free(buffer);
1770	}
1771
1772	TEST_F(hmm, exclusive_mprotect)
1773	{
1774	struct hmm_buffer *buffer;
1775	unsigned long npages;
1776	unsigned long size;
1777	unsigned long i;
1778	int *ptr;
1779	int ret;
1780
1781	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1782	ASSERT_NE(npages, 0);
1783	size = npages << self->page_shift;
1784
1785	buffer = malloc(sizeof(*buffer));
1786	ASSERT_NE(buffer, NULL);
1787
1788	buffer->fd = -1;
1789	buffer->size = size;
1790	buffer->mirror = malloc(size);
1791	ASSERT_NE(buffer->mirror, NULL);
1792
1793	buffer->ptr = mmap(NULL, size,
1794	PROT_READ | PROT_WRITE,
1795	MAP_PRIVATE | MAP_ANONYMOUS,
1796	buffer->fd, 0);
1797	ASSERT_NE(buffer->ptr, MAP_FAILED);
1798
1799	/* Initialize buffer in system memory. */
1800	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1801	ptr[i] = i;
1802
1803	/* Map memory exclusively for device access. */
1804	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1805	ASSERT_EQ(ret, 0);
1806	ASSERT_EQ(buffer->cpages, npages);
1807
1808	/* Check what the device read. */
1809	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1810	ASSERT_EQ(ptr[i], i);
1811
1812	ret = mprotect(buffer->ptr, size, PROT_READ);
1813	ASSERT_EQ(ret, 0);
1814
1815	/* Simulate a device writing system memory. */
1816	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages);
1817	ASSERT_EQ(ret, -EPERM);
1818
1819	hmm_buffer_free(buffer);
1820	}
1821
1822	/*
1823	* Check copy-on-write works.
1824	*/
1825	TEST_F(hmm, exclusive_cow)
1826	{
1827	struct hmm_buffer *buffer;
1828	unsigned long npages;
1829	unsigned long size;
1830	unsigned long i;
1831	int *ptr;
1832	int ret;
1833
1834	npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift;
1835	ASSERT_NE(npages, 0);
1836	size = npages << self->page_shift;
1837
1838	buffer = malloc(sizeof(*buffer));
1839	ASSERT_NE(buffer, NULL);
1840
1841	buffer->fd = -1;
1842	buffer->size = size;
1843	buffer->mirror = malloc(size);
1844	ASSERT_NE(buffer->mirror, NULL);
1845
1846	buffer->ptr = mmap(NULL, size,
1847	PROT_READ | PROT_WRITE,
1848	MAP_PRIVATE | MAP_ANONYMOUS,
1849	buffer->fd, 0);
1850	ASSERT_NE(buffer->ptr, MAP_FAILED);
1851
1852	/* Initialize buffer in system memory. */
1853	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1854	ptr[i] = i;
1855
1856	/* Map memory exclusively for device access. */
1857	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_EXCLUSIVE, buffer, npages);
1858	ASSERT_EQ(ret, 0);
1859	ASSERT_EQ(buffer->cpages, npages);
1860
1861	fork();
1862
1863	/* Fault pages back to system memory and check them. */
1864	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1865	ASSERT_EQ(ptr[i]++, i);
1866
1867	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1868	ASSERT_EQ(ptr[i], i+1);
1869
1870	hmm_buffer_free(buffer);
1871	}
1872
1873	static int gup_test_exec(int gup_fd, unsigned long addr, int cmd,
1874	int npages, int size, int flags)
1875	{
1876	struct gup_test gup = {
1877	.nr_pages_per_call = npages,
1878	.addr = addr,
1879	.gup_flags = FOLL_WRITE | flags,
1880	.size = size,
1881	};
1882
1883	if (ioctl(gup_fd, cmd, &gup)) {
1884	perror("ioctl on error\n");
1885	return errno;
1886	}
1887
1888	return 0;
1889	}
1890
1891	/*
1892	* Test get user device pages through gup_test. Setting PIN_LONGTERM flag.
1893	* This should trigger a migration back to system memory for both, private
1894	* and coherent type pages.
1895	* This test makes use of gup_test module. Make sure GUP_TEST_CONFIG is added
1896	* to your configuration before you run it.
1897	*/
1898	TEST_F(hmm, hmm_gup_test)
1899	{
1900	struct hmm_buffer *buffer;
1901	int gup_fd;
1902	unsigned long npages;
1903	unsigned long size;
1904	unsigned long i;
1905	int *ptr;
1906	int ret;
1907	unsigned char *m;
1908
1909	gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR);
1910	if (gup_fd == -1)
1911	SKIP(return, "Skipping test, could not find gup_test driver");
1912
1913	npages = 4;
1914	size = npages << self->page_shift;
1915
1916	buffer = malloc(sizeof(*buffer));
1917	ASSERT_NE(buffer, NULL);
1918
1919	buffer->fd = -1;
1920	buffer->size = size;
1921	buffer->mirror = malloc(size);
1922	ASSERT_NE(buffer->mirror, NULL);
1923
1924	buffer->ptr = mmap(NULL, size,
1925	PROT_READ | PROT_WRITE,
1926	MAP_PRIVATE | MAP_ANONYMOUS,
1927	buffer->fd, 0);
1928	ASSERT_NE(buffer->ptr, MAP_FAILED);
1929
1930	/* Initialize buffer in system memory. */
1931	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1932	ptr[i] = i;
1933
1934	/* Migrate memory to device. */
1935	ret = hmm_migrate_sys_to_dev(fd: self->fd, buffer, npages);
1936	ASSERT_EQ(ret, 0);
1937	ASSERT_EQ(buffer->cpages, npages);
1938	/* Check what the device read. */
1939	for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i)
1940	ASSERT_EQ(ptr[i], i);
1941
1942	ASSERT_EQ(gup_test_exec(gup_fd,
1943	(unsigned long)buffer->ptr,
1944	GUP_BASIC_TEST, 1, self->page_size, 0), 0);
1945	ASSERT_EQ(gup_test_exec(gup_fd,
1946	(unsigned long)buffer->ptr + 1 * self->page_size,
1947	GUP_FAST_BENCHMARK, 1, self->page_size, 0), 0);
1948	ASSERT_EQ(gup_test_exec(gup_fd,
1949	(unsigned long)buffer->ptr + 2 * self->page_size,
1950	PIN_FAST_BENCHMARK, 1, self->page_size, FOLL_LONGTERM), 0);
1951	ASSERT_EQ(gup_test_exec(gup_fd,
1952	(unsigned long)buffer->ptr + 3 * self->page_size,
1953	PIN_LONGTERM_BENCHMARK, 1, self->page_size, 0), 0);
1954
1955	/* Take snapshot to CPU pagetables */
1956	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
1957	ASSERT_EQ(ret, 0);
1958	ASSERT_EQ(buffer->cpages, npages);
1959	m = buffer->mirror;
1960	if (hmm_is_coherent_type(variant->device_number)) {
1961	ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[0]);
1962	ASSERT_EQ(HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL | HMM_DMIRROR_PROT_WRITE, m[1]);
1963	} else {
1964	ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[0]);
1965	ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[1]);
1966	}
1967	ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[2]);
1968	ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[3]);
1969	/*
1970	* Check again the content on the pages. Make sure there's no
1971	* corrupted data.
1972	*/
1973	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
1974	ASSERT_EQ(ptr[i], i);
1975
1976	close(gup_fd);
1977	hmm_buffer_free(buffer);
1978	}
1979
1980	/*
1981	* Test copy-on-write in device pages.
1982	* In case of writing to COW private page(s), a page fault will migrate pages
1983	* back to system memory first. Then, these pages will be duplicated. In case
1984	* of COW device coherent type, pages are duplicated directly from device
1985	* memory.
1986	*/
1987	TEST_F(hmm, hmm_cow_in_device)
1988	{
1989	struct hmm_buffer *buffer;
1990	unsigned long npages;
1991	unsigned long size;
1992	unsigned long i;
1993	int *ptr;
1994	int ret;
1995	unsigned char *m;
1996	pid_t pid;
1997	int status;
1998
1999	npages = 4;
2000	size = npages << self->page_shift;
2001
2002	buffer = malloc(sizeof(*buffer));
2003	ASSERT_NE(buffer, NULL);
2004
2005	buffer->fd = -1;
2006	buffer->size = size;
2007	buffer->mirror = malloc(size);
2008	ASSERT_NE(buffer->mirror, NULL);
2009
2010	buffer->ptr = mmap(NULL, size,
2011	PROT_READ | PROT_WRITE,
2012	MAP_PRIVATE | MAP_ANONYMOUS,
2013	buffer->fd, 0);
2014	ASSERT_NE(buffer->ptr, MAP_FAILED);
2015
2016	/* Initialize buffer in system memory. */
2017	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2018	ptr[i] = i;
2019
2020	/* Migrate memory to device. */
2021
2022	ret = hmm_migrate_sys_to_dev(fd: self->fd, buffer, npages);
2023	ASSERT_EQ(ret, 0);
2024	ASSERT_EQ(buffer->cpages, npages);
2025
2026	pid = fork();
2027	if (pid == -1)
2028	ASSERT_EQ(pid, 0);
2029	if (!pid) {
2030	/* Child process waitd for SIGTERM from the parent. */
2031	while (1) {
2032	}
2033	perror("Should not reach this\n");
2034	exit(0);
2035	}
2036	/* Parent process writes to COW pages(s) and gets a
2037	* new copy in system. In case of device private pages,
2038	* this write causes a migration to system mem first.
2039	*/
2040	for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i)
2041	ptr[i] = i;
2042
2043	/* Terminate child and wait */
2044	EXPECT_EQ(0, kill(pid, SIGTERM));
2045	EXPECT_EQ(pid, waitpid(pid, &status, 0));
2046	EXPECT_NE(0, WIFSIGNALED(status));
2047	EXPECT_EQ(SIGTERM, WTERMSIG(status));
2048
2049	/* Take snapshot to CPU pagetables */
2050	ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages);
2051	ASSERT_EQ(ret, 0);
2052	ASSERT_EQ(buffer->cpages, npages);
2053	m = buffer->mirror;
2054	for (i = 0; i < npages; i++)
2055	ASSERT_EQ(HMM_DMIRROR_PROT_WRITE, m[i]);
2056
2057	hmm_buffer_free(buffer);
2058	}
2059	TEST_HARNESS_MAIN
2060