1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * KVM page table test |
4 | * |
5 | * Copyright (C) 2021, Huawei, Inc. |
6 | * |
7 | * Make sure that THP has been enabled or enough HUGETLB pages with specific |
8 | * page size have been pre-allocated on your system, if you are planning to |
9 | * use hugepages to back the guest memory for testing. |
10 | */ |
11 | |
12 | #define _GNU_SOURCE /* for program_invocation_name */ |
13 | |
14 | #include <stdio.h> |
15 | #include <stdlib.h> |
16 | #include <time.h> |
17 | #include <pthread.h> |
18 | #include <semaphore.h> |
19 | |
20 | #include "test_util.h" |
21 | #include "kvm_util.h" |
22 | #include "processor.h" |
23 | #include "guest_modes.h" |
24 | |
25 | #define TEST_MEM_SLOT_INDEX 1 |
26 | |
27 | /* Default size(1GB) of the memory for testing */ |
28 | #define DEFAULT_TEST_MEM_SIZE (1 << 30) |
29 | |
30 | /* Default guest test virtual memory offset */ |
31 | #define DEFAULT_GUEST_TEST_MEM 0xc0000000 |
32 | |
33 | /* Different guest memory accessing stages */ |
34 | enum test_stage { |
35 | KVM_BEFORE_MAPPINGS, |
36 | KVM_CREATE_MAPPINGS, |
37 | KVM_UPDATE_MAPPINGS, |
38 | KVM_ADJUST_MAPPINGS, |
39 | NUM_TEST_STAGES, |
40 | }; |
41 | |
42 | static const char * const test_stage_string[] = { |
43 | "KVM_BEFORE_MAPPINGS" , |
44 | "KVM_CREATE_MAPPINGS" , |
45 | "KVM_UPDATE_MAPPINGS" , |
46 | "KVM_ADJUST_MAPPINGS" , |
47 | }; |
48 | |
49 | struct test_args { |
50 | struct kvm_vm *vm; |
51 | uint64_t guest_test_virt_mem; |
52 | uint64_t host_page_size; |
53 | uint64_t host_num_pages; |
54 | uint64_t large_page_size; |
55 | uint64_t large_num_pages; |
56 | uint64_t host_pages_per_lpage; |
57 | enum vm_mem_backing_src_type src_type; |
58 | struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; |
59 | }; |
60 | |
61 | /* |
62 | * Guest variables. Use addr_gva2hva() if these variables need |
63 | * to be changed in host. |
64 | */ |
65 | static enum test_stage guest_test_stage; |
66 | |
67 | /* Host variables */ |
68 | static uint32_t nr_vcpus = 1; |
69 | static struct test_args test_args; |
70 | static enum test_stage *current_stage; |
71 | static bool host_quit; |
72 | |
73 | /* Whether the test stage is updated, or completed */ |
74 | static sem_t test_stage_updated; |
75 | static sem_t test_stage_completed; |
76 | |
77 | /* |
78 | * Guest physical memory offset of the testing memory slot. |
79 | * This will be set to the topmost valid physical address minus |
80 | * the test memory size. |
81 | */ |
82 | static uint64_t guest_test_phys_mem; |
83 | |
84 | /* |
85 | * Guest virtual memory offset of the testing memory slot. |
86 | * Must not conflict with identity mapped test code. |
87 | */ |
88 | static uint64_t guest_test_virt_mem = DEFAULT_GUEST_TEST_MEM; |
89 | |
90 | static void guest_code(bool do_write) |
91 | { |
92 | struct test_args *p = &test_args; |
93 | enum test_stage *current_stage = &guest_test_stage; |
94 | uint64_t addr; |
95 | int i, j; |
96 | |
97 | while (true) { |
98 | addr = p->guest_test_virt_mem; |
99 | |
100 | switch (READ_ONCE(*current_stage)) { |
101 | /* |
102 | * All vCPU threads will be started in this stage, |
103 | * where guest code of each vCPU will do nothing. |
104 | */ |
105 | case KVM_BEFORE_MAPPINGS: |
106 | break; |
107 | |
108 | /* |
109 | * Before dirty logging, vCPUs concurrently access the first |
110 | * 8 bytes of each page (host page/large page) within the same |
111 | * memory region with different accessing types (read/write). |
112 | * Then KVM will create normal page mappings or huge block |
113 | * mappings for them. |
114 | */ |
115 | case KVM_CREATE_MAPPINGS: |
116 | for (i = 0; i < p->large_num_pages; i++) { |
117 | if (do_write) |
118 | *(uint64_t *)addr = 0x0123456789ABCDEF; |
119 | else |
120 | READ_ONCE(*(uint64_t *)addr); |
121 | |
122 | addr += p->large_page_size; |
123 | } |
124 | break; |
125 | |
126 | /* |
127 | * During dirty logging, KVM will only update attributes of the |
128 | * normal page mappings from RO to RW if memory backing src type |
129 | * is anonymous. In other cases, KVM will split the huge block |
130 | * mappings into normal page mappings if memory backing src type |
131 | * is THP or HUGETLB. |
132 | */ |
133 | case KVM_UPDATE_MAPPINGS: |
134 | if (p->src_type == VM_MEM_SRC_ANONYMOUS) { |
135 | for (i = 0; i < p->host_num_pages; i++) { |
136 | *(uint64_t *)addr = 0x0123456789ABCDEF; |
137 | addr += p->host_page_size; |
138 | } |
139 | break; |
140 | } |
141 | |
142 | for (i = 0; i < p->large_num_pages; i++) { |
143 | /* |
144 | * Write to the first host page in each large |
145 | * page region, and triger break of large pages. |
146 | */ |
147 | *(uint64_t *)addr = 0x0123456789ABCDEF; |
148 | |
149 | /* |
150 | * Access the middle host pages in each large |
151 | * page region. Since dirty logging is enabled, |
152 | * this will create new mappings at the smallest |
153 | * granularity. |
154 | */ |
155 | addr += p->large_page_size / 2; |
156 | for (j = 0; j < p->host_pages_per_lpage / 2; j++) { |
157 | READ_ONCE(*(uint64_t *)addr); |
158 | addr += p->host_page_size; |
159 | } |
160 | } |
161 | break; |
162 | |
163 | /* |
164 | * After dirty logging is stopped, vCPUs concurrently read |
165 | * from every single host page. Then KVM will coalesce the |
166 | * split page mappings back to block mappings. And a TLB |
167 | * conflict abort could occur here if TLB entries of the |
168 | * page mappings are not fully invalidated. |
169 | */ |
170 | case KVM_ADJUST_MAPPINGS: |
171 | for (i = 0; i < p->host_num_pages; i++) { |
172 | READ_ONCE(*(uint64_t *)addr); |
173 | addr += p->host_page_size; |
174 | } |
175 | break; |
176 | |
177 | default: |
178 | GUEST_ASSERT(0); |
179 | } |
180 | |
181 | GUEST_SYNC(1); |
182 | } |
183 | } |
184 | |
185 | static void *vcpu_worker(void *data) |
186 | { |
187 | struct kvm_vcpu *vcpu = data; |
188 | bool do_write = !(vcpu->id % 2); |
189 | struct timespec start; |
190 | struct timespec ts_diff; |
191 | enum test_stage stage; |
192 | int ret; |
193 | |
194 | vcpu_args_set(vcpu, 1, do_write); |
195 | |
196 | while (!READ_ONCE(host_quit)) { |
197 | ret = sem_wait(&test_stage_updated); |
198 | TEST_ASSERT(ret == 0, "Error in sem_wait" ); |
199 | |
200 | if (READ_ONCE(host_quit)) |
201 | return NULL; |
202 | |
203 | clock_gettime(CLOCK_MONOTONIC, &start); |
204 | ret = _vcpu_run(vcpu); |
205 | ts_diff = timespec_elapsed(start); |
206 | |
207 | TEST_ASSERT(ret == 0, "vcpu_run failed: %d" , ret); |
208 | TEST_ASSERT(get_ucall(vcpu, NULL) == UCALL_SYNC, |
209 | "Invalid guest sync status: exit_reason=%s" , |
210 | exit_reason_str(vcpu->run->exit_reason)); |
211 | |
212 | pr_debug("Got sync event from vCPU %d\n" , vcpu->id); |
213 | stage = READ_ONCE(*current_stage); |
214 | |
215 | /* |
216 | * Here we can know the execution time of every |
217 | * single vcpu running in different test stages. |
218 | */ |
219 | pr_debug("vCPU %d has completed stage %s\n" |
220 | "execution time is: %ld.%.9lds\n\n" , |
221 | vcpu->id, test_stage_string[stage], |
222 | ts_diff.tv_sec, ts_diff.tv_nsec); |
223 | |
224 | ret = sem_post(&test_stage_completed); |
225 | TEST_ASSERT(ret == 0, "Error in sem_post" ); |
226 | } |
227 | |
228 | return NULL; |
229 | } |
230 | |
231 | struct test_params { |
232 | uint64_t phys_offset; |
233 | uint64_t test_mem_size; |
234 | enum vm_mem_backing_src_type src_type; |
235 | }; |
236 | |
237 | static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg) |
238 | { |
239 | int ret; |
240 | struct test_params *p = arg; |
241 | enum vm_mem_backing_src_type src_type = p->src_type; |
242 | uint64_t large_page_size = get_backing_src_pagesz(src_type); |
243 | uint64_t guest_page_size = vm_guest_mode_params[mode].page_size; |
244 | uint64_t host_page_size = getpagesize(); |
245 | uint64_t test_mem_size = p->test_mem_size; |
246 | uint64_t guest_num_pages; |
247 | uint64_t alignment; |
248 | void *host_test_mem; |
249 | struct kvm_vm *vm; |
250 | |
251 | /* Align up the test memory size */ |
252 | alignment = max(large_page_size, guest_page_size); |
253 | test_mem_size = (test_mem_size + alignment - 1) & ~(alignment - 1); |
254 | |
255 | /* Create a VM with enough guest pages */ |
256 | guest_num_pages = test_mem_size / guest_page_size; |
257 | vm = __vm_create_with_vcpus(VM_SHAPE(mode), nr_vcpus, guest_num_pages, |
258 | guest_code, test_args.vcpus); |
259 | |
260 | /* Align down GPA of the testing memslot */ |
261 | if (!p->phys_offset) |
262 | guest_test_phys_mem = (vm->max_gfn - guest_num_pages) * |
263 | guest_page_size; |
264 | else |
265 | guest_test_phys_mem = p->phys_offset; |
266 | #ifdef __s390x__ |
267 | alignment = max(0x100000UL, alignment); |
268 | #endif |
269 | guest_test_phys_mem = align_down(guest_test_phys_mem, alignment); |
270 | |
271 | /* Set up the shared data structure test_args */ |
272 | test_args.vm = vm; |
273 | test_args.guest_test_virt_mem = guest_test_virt_mem; |
274 | test_args.host_page_size = host_page_size; |
275 | test_args.host_num_pages = test_mem_size / host_page_size; |
276 | test_args.large_page_size = large_page_size; |
277 | test_args.large_num_pages = test_mem_size / large_page_size; |
278 | test_args.host_pages_per_lpage = large_page_size / host_page_size; |
279 | test_args.src_type = src_type; |
280 | |
281 | /* Add an extra memory slot with specified backing src type */ |
282 | vm_userspace_mem_region_add(vm, src_type, guest_test_phys_mem, |
283 | TEST_MEM_SLOT_INDEX, guest_num_pages, 0); |
284 | |
285 | /* Do mapping(GVA->GPA) for the testing memory slot */ |
286 | virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages); |
287 | |
288 | /* Cache the HVA pointer of the region */ |
289 | host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem); |
290 | |
291 | /* Export shared structure test_args to guest */ |
292 | sync_global_to_guest(vm, test_args); |
293 | |
294 | ret = sem_init(&test_stage_updated, 0, 0); |
295 | TEST_ASSERT(ret == 0, "Error in sem_init" ); |
296 | |
297 | ret = sem_init(&test_stage_completed, 0, 0); |
298 | TEST_ASSERT(ret == 0, "Error in sem_init" ); |
299 | |
300 | current_stage = addr_gva2hva(vm, (vm_vaddr_t)(&guest_test_stage)); |
301 | *current_stage = NUM_TEST_STAGES; |
302 | |
303 | pr_info("Testing guest mode: %s\n" , vm_guest_mode_string(mode)); |
304 | pr_info("Testing memory backing src type: %s\n" , |
305 | vm_mem_backing_src_alias(src_type)->name); |
306 | pr_info("Testing memory backing src granularity: 0x%lx\n" , |
307 | large_page_size); |
308 | pr_info("Testing memory size(aligned): 0x%lx\n" , test_mem_size); |
309 | pr_info("Guest physical test memory offset: 0x%lx\n" , |
310 | guest_test_phys_mem); |
311 | pr_info("Host virtual test memory offset: 0x%lx\n" , |
312 | (uint64_t)host_test_mem); |
313 | pr_info("Number of testing vCPUs: %d\n" , nr_vcpus); |
314 | |
315 | return vm; |
316 | } |
317 | |
318 | static void vcpus_complete_new_stage(enum test_stage stage) |
319 | { |
320 | int ret; |
321 | int vcpus; |
322 | |
323 | /* Wake up all the vcpus to run new test stage */ |
324 | for (vcpus = 0; vcpus < nr_vcpus; vcpus++) { |
325 | ret = sem_post(&test_stage_updated); |
326 | TEST_ASSERT(ret == 0, "Error in sem_post" ); |
327 | } |
328 | pr_debug("All vcpus have been notified to continue\n" ); |
329 | |
330 | /* Wait for all the vcpus to complete new test stage */ |
331 | for (vcpus = 0; vcpus < nr_vcpus; vcpus++) { |
332 | ret = sem_wait(&test_stage_completed); |
333 | TEST_ASSERT(ret == 0, "Error in sem_wait" ); |
334 | |
335 | pr_debug("%d vcpus have completed stage %s\n" , |
336 | vcpus + 1, test_stage_string[stage]); |
337 | } |
338 | |
339 | pr_debug("All vcpus have completed stage %s\n" , |
340 | test_stage_string[stage]); |
341 | } |
342 | |
343 | static void run_test(enum vm_guest_mode mode, void *arg) |
344 | { |
345 | pthread_t *vcpu_threads; |
346 | struct kvm_vm *vm; |
347 | struct timespec start; |
348 | struct timespec ts_diff; |
349 | int ret, i; |
350 | |
351 | /* Create VM with vCPUs and make some pre-initialization */ |
352 | vm = pre_init_before_test(mode: mode, arg); |
353 | |
354 | vcpu_threads = malloc(nr_vcpus * sizeof(*vcpu_threads)); |
355 | TEST_ASSERT(vcpu_threads, "Memory allocation failed" ); |
356 | |
357 | host_quit = false; |
358 | *current_stage = KVM_BEFORE_MAPPINGS; |
359 | |
360 | for (i = 0; i < nr_vcpus; i++) |
361 | pthread_create(&vcpu_threads[i], NULL, vcpu_worker, |
362 | test_args.vcpus[i]); |
363 | |
364 | vcpus_complete_new_stage(stage: *current_stage); |
365 | pr_info("Started all vCPUs successfully\n" ); |
366 | |
367 | /* Test the stage of KVM creating mappings */ |
368 | *current_stage = KVM_CREATE_MAPPINGS; |
369 | |
370 | clock_gettime(CLOCK_MONOTONIC, &start); |
371 | vcpus_complete_new_stage(stage: *current_stage); |
372 | ts_diff = timespec_elapsed(start); |
373 | |
374 | pr_info("KVM_CREATE_MAPPINGS: total execution time: %ld.%.9lds\n\n" , |
375 | ts_diff.tv_sec, ts_diff.tv_nsec); |
376 | |
377 | /* Test the stage of KVM updating mappings */ |
378 | vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, |
379 | KVM_MEM_LOG_DIRTY_PAGES); |
380 | |
381 | *current_stage = KVM_UPDATE_MAPPINGS; |
382 | |
383 | clock_gettime(CLOCK_MONOTONIC, &start); |
384 | vcpus_complete_new_stage(stage: *current_stage); |
385 | ts_diff = timespec_elapsed(start); |
386 | |
387 | pr_info("KVM_UPDATE_MAPPINGS: total execution time: %ld.%.9lds\n\n" , |
388 | ts_diff.tv_sec, ts_diff.tv_nsec); |
389 | |
390 | /* Test the stage of KVM adjusting mappings */ |
391 | vm_mem_region_set_flags(vm, TEST_MEM_SLOT_INDEX, 0); |
392 | |
393 | *current_stage = KVM_ADJUST_MAPPINGS; |
394 | |
395 | clock_gettime(CLOCK_MONOTONIC, &start); |
396 | vcpus_complete_new_stage(stage: *current_stage); |
397 | ts_diff = timespec_elapsed(start); |
398 | |
399 | pr_info("KVM_ADJUST_MAPPINGS: total execution time: %ld.%.9lds\n\n" , |
400 | ts_diff.tv_sec, ts_diff.tv_nsec); |
401 | |
402 | /* Tell the vcpu thread to quit */ |
403 | host_quit = true; |
404 | for (i = 0; i < nr_vcpus; i++) { |
405 | ret = sem_post(&test_stage_updated); |
406 | TEST_ASSERT(ret == 0, "Error in sem_post" ); |
407 | } |
408 | |
409 | for (i = 0; i < nr_vcpus; i++) |
410 | pthread_join(vcpu_threads[i], NULL); |
411 | |
412 | ret = sem_destroy(&test_stage_updated); |
413 | TEST_ASSERT(ret == 0, "Error in sem_destroy" ); |
414 | |
415 | ret = sem_destroy(&test_stage_completed); |
416 | TEST_ASSERT(ret == 0, "Error in sem_destroy" ); |
417 | |
418 | free(vcpu_threads); |
419 | kvm_vm_free(vm); |
420 | } |
421 | |
422 | static void help(char *name) |
423 | { |
424 | puts("" ); |
425 | printf("usage: %s [-h] [-p offset] [-m mode] " |
426 | "[-b mem-size] [-v vcpus] [-s mem-type]\n" , name); |
427 | puts("" ); |
428 | printf(" -p: specify guest physical test memory offset\n" |
429 | " Warning: a low offset can conflict with the loaded test code.\n" ); |
430 | guest_modes_help(); |
431 | printf(" -b: specify size of the memory region for testing. e.g. 10M or 3G.\n" |
432 | " (default: 1G)\n" ); |
433 | printf(" -v: specify the number of vCPUs to run\n" |
434 | " (default: 1)\n" ); |
435 | backing_src_help("-s" ); |
436 | puts("" ); |
437 | } |
438 | |
439 | int main(int argc, char *argv[]) |
440 | { |
441 | int max_vcpus = kvm_check_cap(KVM_CAP_MAX_VCPUS); |
442 | struct test_params p = { |
443 | .test_mem_size = DEFAULT_TEST_MEM_SIZE, |
444 | .src_type = DEFAULT_VM_MEM_SRC, |
445 | }; |
446 | int opt; |
447 | |
448 | guest_modes_append_default(); |
449 | |
450 | while ((opt = getopt(argc, argv, "hp:m:b:v:s:" )) != -1) { |
451 | switch (opt) { |
452 | case 'p': |
453 | p.phys_offset = strtoull(optarg, NULL, 0); |
454 | break; |
455 | case 'm': |
456 | guest_modes_cmdline(optarg); |
457 | break; |
458 | case 'b': |
459 | p.test_mem_size = parse_size(optarg); |
460 | break; |
461 | case 'v': |
462 | nr_vcpus = atoi_positive("Number of vCPUs" , optarg); |
463 | TEST_ASSERT(nr_vcpus <= max_vcpus, |
464 | "Invalid number of vcpus, must be between 1 and %d" , max_vcpus); |
465 | break; |
466 | case 's': |
467 | p.src_type = parse_backing_src_type(optarg); |
468 | break; |
469 | case 'h': |
470 | default: |
471 | help(name: argv[0]); |
472 | exit(0); |
473 | } |
474 | } |
475 | |
476 | for_each_guest_mode(run_test, &p); |
477 | |
478 | return 0; |
479 | } |
480 | |