1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Secure pages management: Migration of pages between normal and secure |
4 | * memory of KVM guests. |
5 | * |
6 | * Copyright 2018 Bharata B Rao, IBM Corp. <bharata@linux.ibm.com> |
7 | */ |
8 | |
9 | /* |
10 | * A pseries guest can be run as secure guest on Ultravisor-enabled |
11 | * POWER platforms. On such platforms, this driver will be used to manage |
12 | * the movement of guest pages between the normal memory managed by |
13 | * hypervisor (HV) and secure memory managed by Ultravisor (UV). |
14 | * |
15 | * The page-in or page-out requests from UV will come to HV as hcalls and |
16 | * HV will call back into UV via ultracalls to satisfy these page requests. |
17 | * |
18 | * Private ZONE_DEVICE memory equal to the amount of secure memory |
19 | * available in the platform for running secure guests is hotplugged. |
20 | * Whenever a page belonging to the guest becomes secure, a page from this |
21 | * private device memory is used to represent and track that secure page |
22 | * on the HV side. Some pages (like virtio buffers, VPA pages etc) are |
23 | * shared between UV and HV. However such pages aren't represented by |
24 | * device private memory and mappings to shared memory exist in both |
25 | * UV and HV page tables. |
26 | */ |
27 | |
28 | /* |
29 | * Notes on locking |
30 | * |
31 | * kvm->arch.uvmem_lock is a per-guest lock that prevents concurrent |
32 | * page-in and page-out requests for the same GPA. Concurrent accesses |
33 | * can either come via UV (guest vCPUs requesting for same page) |
34 | * or when HV and guest simultaneously access the same page. |
35 | * This mutex serializes the migration of page from HV(normal) to |
36 | * UV(secure) and vice versa. So the serialization points are around |
37 | * migrate_vma routines and page-in/out routines. |
38 | * |
39 | * Per-guest mutex comes with a cost though. Mainly it serializes the |
40 | * fault path as page-out can occur when HV faults on accessing secure |
41 | * guest pages. Currently UV issues page-in requests for all the guest |
42 | * PFNs one at a time during early boot (UV_ESM uvcall), so this is |
43 | * not a cause for concern. Also currently the number of page-outs caused |
44 | * by HV touching secure pages is very very low. If an when UV supports |
45 | * overcommitting, then we might see concurrent guest driven page-outs. |
46 | * |
47 | * Locking order |
48 | * |
49 | * 1. kvm->srcu - Protects KVM memslots |
50 | * 2. kvm->mm->mmap_lock - find_vma, migrate_vma_pages and helpers, ksm_madvise |
51 | * 3. kvm->arch.uvmem_lock - protects read/writes to uvmem slots thus acting |
52 | * as sync-points for page-in/out |
53 | */ |
54 | |
55 | /* |
56 | * Notes on page size |
57 | * |
58 | * Currently UV uses 2MB mappings internally, but will issue H_SVM_PAGE_IN |
59 | * and H_SVM_PAGE_OUT hcalls in PAGE_SIZE(64K) granularity. HV tracks |
60 | * secure GPAs at 64K page size and maintains one device PFN for each |
61 | * 64K secure GPA. UV_PAGE_IN and UV_PAGE_OUT calls by HV are also issued |
62 | * for 64K page at a time. |
63 | * |
64 | * HV faulting on secure pages: When HV touches any secure page, it |
65 | * faults and issues a UV_PAGE_OUT request with 64K page size. Currently |
66 | * UV splits and remaps the 2MB page if necessary and copies out the |
67 | * required 64K page contents. |
68 | * |
69 | * Shared pages: Whenever guest shares a secure page, UV will split and |
70 | * remap the 2MB page if required and issue H_SVM_PAGE_IN with 64K page size. |
71 | * |
72 | * HV invalidating a page: When a regular page belonging to secure |
73 | * guest gets unmapped, HV informs UV with UV_PAGE_INVAL of 64K |
74 | * page size. Using 64K page size is correct here because any non-secure |
75 | * page will essentially be of 64K page size. Splitting by UV during sharing |
76 | * and page-out ensures this. |
77 | * |
78 | * Page fault handling: When HV handles page fault of a page belonging |
79 | * to secure guest, it sends that to UV with a 64K UV_PAGE_IN request. |
80 | * Using 64K size is correct here too as UV would have split the 2MB page |
81 | * into 64k mappings and would have done page-outs earlier. |
82 | * |
83 | * In summary, the current secure pages handling code in HV assumes |
84 | * 64K page size and in fact fails any page-in/page-out requests of |
85 | * non-64K size upfront. If and when UV starts supporting multiple |
86 | * page-sizes, we need to break this assumption. |
87 | */ |
88 | |
89 | #include <linux/pagemap.h> |
90 | #include <linux/migrate.h> |
91 | #include <linux/kvm_host.h> |
92 | #include <linux/ksm.h> |
93 | #include <linux/of.h> |
94 | #include <linux/memremap.h> |
95 | #include <asm/ultravisor.h> |
96 | #include <asm/mman.h> |
97 | #include <asm/kvm_ppc.h> |
98 | #include <asm/kvm_book3s_uvmem.h> |
99 | |
100 | static struct dev_pagemap kvmppc_uvmem_pgmap; |
101 | static unsigned long *kvmppc_uvmem_bitmap; |
102 | static DEFINE_SPINLOCK(kvmppc_uvmem_bitmap_lock); |
103 | |
104 | /* |
105 | * States of a GFN |
106 | * --------------- |
107 | * The GFN can be in one of the following states. |
108 | * |
109 | * (a) Secure - The GFN is secure. The GFN is associated with |
110 | * a Secure VM, the contents of the GFN is not accessible |
111 | * to the Hypervisor. This GFN can be backed by a secure-PFN, |
112 | * or can be backed by a normal-PFN with contents encrypted. |
113 | * The former is true when the GFN is paged-in into the |
114 | * ultravisor. The latter is true when the GFN is paged-out |
115 | * of the ultravisor. |
116 | * |
117 | * (b) Shared - The GFN is shared. The GFN is associated with a |
118 | * a secure VM. The contents of the GFN is accessible to |
119 | * Hypervisor. This GFN is backed by a normal-PFN and its |
120 | * content is un-encrypted. |
121 | * |
122 | * (c) Normal - The GFN is a normal. The GFN is associated with |
123 | * a normal VM. The contents of the GFN is accessible to |
124 | * the Hypervisor. Its content is never encrypted. |
125 | * |
126 | * States of a VM. |
127 | * --------------- |
128 | * |
129 | * Normal VM: A VM whose contents are always accessible to |
130 | * the hypervisor. All its GFNs are normal-GFNs. |
131 | * |
132 | * Secure VM: A VM whose contents are not accessible to the |
133 | * hypervisor without the VM's consent. Its GFNs are |
134 | * either Shared-GFN or Secure-GFNs. |
135 | * |
136 | * Transient VM: A Normal VM that is transitioning to secure VM. |
137 | * The transition starts on successful return of |
138 | * H_SVM_INIT_START, and ends on successful return |
139 | * of H_SVM_INIT_DONE. This transient VM, can have GFNs |
140 | * in any of the three states; i.e Secure-GFN, Shared-GFN, |
141 | * and Normal-GFN. The VM never executes in this state |
142 | * in supervisor-mode. |
143 | * |
144 | * Memory slot State. |
145 | * ----------------------------- |
146 | * The state of a memory slot mirrors the state of the |
147 | * VM the memory slot is associated with. |
148 | * |
149 | * VM State transition. |
150 | * -------------------- |
151 | * |
152 | * A VM always starts in Normal Mode. |
153 | * |
154 | * H_SVM_INIT_START moves the VM into transient state. During this |
155 | * time the Ultravisor may request some of its GFNs to be shared or |
156 | * secured. So its GFNs can be in one of the three GFN states. |
157 | * |
158 | * H_SVM_INIT_DONE moves the VM entirely from transient state to |
159 | * secure-state. At this point any left-over normal-GFNs are |
160 | * transitioned to Secure-GFN. |
161 | * |
162 | * H_SVM_INIT_ABORT moves the transient VM back to normal VM. |
163 | * All its GFNs are moved to Normal-GFNs. |
164 | * |
165 | * UV_TERMINATE transitions the secure-VM back to normal-VM. All |
166 | * the secure-GFN and shared-GFNs are tranistioned to normal-GFN |
167 | * Note: The contents of the normal-GFN is undefined at this point. |
168 | * |
169 | * GFN state implementation: |
170 | * ------------------------- |
171 | * |
172 | * Secure GFN is associated with a secure-PFN; also called uvmem_pfn, |
173 | * when the GFN is paged-in. Its pfn[] has KVMPPC_GFN_UVMEM_PFN flag |
174 | * set, and contains the value of the secure-PFN. |
175 | * It is associated with a normal-PFN; also called mem_pfn, when |
176 | * the GFN is pagedout. Its pfn[] has KVMPPC_GFN_MEM_PFN flag set. |
177 | * The value of the normal-PFN is not tracked. |
178 | * |
179 | * Shared GFN is associated with a normal-PFN. Its pfn[] has |
180 | * KVMPPC_UVMEM_SHARED_PFN flag set. The value of the normal-PFN |
181 | * is not tracked. |
182 | * |
183 | * Normal GFN is associated with normal-PFN. Its pfn[] has |
184 | * no flag set. The value of the normal-PFN is not tracked. |
185 | * |
186 | * Life cycle of a GFN |
187 | * -------------------- |
188 | * |
189 | * -------------------------------------------------------------- |
190 | * | | Share | Unshare | SVM |H_SVM_INIT_DONE| |
191 | * | |operation |operation | abort/ | | |
192 | * | | | | terminate | | |
193 | * ------------------------------------------------------------- |
194 | * | | | | | | |
195 | * | Secure | Shared | Secure |Normal |Secure | |
196 | * | | | | | | |
197 | * | Shared | Shared | Secure |Normal |Shared | |
198 | * | | | | | | |
199 | * | Normal | Shared | Secure |Normal |Secure | |
200 | * -------------------------------------------------------------- |
201 | * |
202 | * Life cycle of a VM |
203 | * -------------------- |
204 | * |
205 | * -------------------------------------------------------------------- |
206 | * | | start | H_SVM_ |H_SVM_ |H_SVM_ |UV_SVM_ | |
207 | * | | VM |INIT_START|INIT_DONE|INIT_ABORT |TERMINATE | |
208 | * | | | | | | | |
209 | * --------- ---------------------------------------------------------- |
210 | * | | | | | | | |
211 | * | Normal | Normal | Transient|Error |Error |Normal | |
212 | * | | | | | | | |
213 | * | Secure | Error | Error |Error |Error |Normal | |
214 | * | | | | | | | |
215 | * |Transient| N/A | Error |Secure |Normal |Normal | |
216 | * -------------------------------------------------------------------- |
217 | */ |
218 | |
219 | #define KVMPPC_GFN_UVMEM_PFN (1UL << 63) |
220 | #define KVMPPC_GFN_MEM_PFN (1UL << 62) |
221 | #define KVMPPC_GFN_SHARED (1UL << 61) |
222 | #define KVMPPC_GFN_SECURE (KVMPPC_GFN_UVMEM_PFN | KVMPPC_GFN_MEM_PFN) |
223 | #define KVMPPC_GFN_FLAG_MASK (KVMPPC_GFN_SECURE | KVMPPC_GFN_SHARED) |
224 | #define KVMPPC_GFN_PFN_MASK (~KVMPPC_GFN_FLAG_MASK) |
225 | |
226 | struct kvmppc_uvmem_slot { |
227 | struct list_head list; |
228 | unsigned long nr_pfns; |
229 | unsigned long base_pfn; |
230 | unsigned long *pfns; |
231 | }; |
232 | struct kvmppc_uvmem_page_pvt { |
233 | struct kvm *kvm; |
234 | unsigned long gpa; |
235 | bool skip_page_out; |
236 | bool remove_gfn; |
237 | }; |
238 | |
239 | bool kvmppc_uvmem_available(void) |
240 | { |
241 | /* |
242 | * If kvmppc_uvmem_bitmap != NULL, then there is an ultravisor |
243 | * and our data structures have been initialized successfully. |
244 | */ |
245 | return !!kvmppc_uvmem_bitmap; |
246 | } |
247 | |
248 | int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot) |
249 | { |
250 | struct kvmppc_uvmem_slot *p; |
251 | |
252 | p = kzalloc(size: sizeof(*p), GFP_KERNEL); |
253 | if (!p) |
254 | return -ENOMEM; |
255 | p->pfns = vcalloc(n: slot->npages, size: sizeof(*p->pfns)); |
256 | if (!p->pfns) { |
257 | kfree(objp: p); |
258 | return -ENOMEM; |
259 | } |
260 | p->nr_pfns = slot->npages; |
261 | p->base_pfn = slot->base_gfn; |
262 | |
263 | mutex_lock(&kvm->arch.uvmem_lock); |
264 | list_add(new: &p->list, head: &kvm->arch.uvmem_pfns); |
265 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
266 | |
267 | return 0; |
268 | } |
269 | |
270 | /* |
271 | * All device PFNs are already released by the time we come here. |
272 | */ |
273 | void kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot) |
274 | { |
275 | struct kvmppc_uvmem_slot *p, *next; |
276 | |
277 | mutex_lock(&kvm->arch.uvmem_lock); |
278 | list_for_each_entry_safe(p, next, &kvm->arch.uvmem_pfns, list) { |
279 | if (p->base_pfn == slot->base_gfn) { |
280 | vfree(addr: p->pfns); |
281 | list_del(entry: &p->list); |
282 | kfree(objp: p); |
283 | break; |
284 | } |
285 | } |
286 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
287 | } |
288 | |
289 | static void kvmppc_mark_gfn(unsigned long gfn, struct kvm *kvm, |
290 | unsigned long flag, unsigned long uvmem_pfn) |
291 | { |
292 | struct kvmppc_uvmem_slot *p; |
293 | |
294 | list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) { |
295 | if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) { |
296 | unsigned long index = gfn - p->base_pfn; |
297 | |
298 | if (flag == KVMPPC_GFN_UVMEM_PFN) |
299 | p->pfns[index] = uvmem_pfn | flag; |
300 | else |
301 | p->pfns[index] = flag; |
302 | return; |
303 | } |
304 | } |
305 | } |
306 | |
307 | /* mark the GFN as secure-GFN associated with @uvmem pfn device-PFN. */ |
308 | static void kvmppc_gfn_secure_uvmem_pfn(unsigned long gfn, |
309 | unsigned long uvmem_pfn, struct kvm *kvm) |
310 | { |
311 | kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_UVMEM_PFN, uvmem_pfn); |
312 | } |
313 | |
314 | /* mark the GFN as secure-GFN associated with a memory-PFN. */ |
315 | static void kvmppc_gfn_secure_mem_pfn(unsigned long gfn, struct kvm *kvm) |
316 | { |
317 | kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_MEM_PFN, uvmem_pfn: 0); |
318 | } |
319 | |
320 | /* mark the GFN as a shared GFN. */ |
321 | static void kvmppc_gfn_shared(unsigned long gfn, struct kvm *kvm) |
322 | { |
323 | kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_SHARED, uvmem_pfn: 0); |
324 | } |
325 | |
326 | /* mark the GFN as a non-existent GFN. */ |
327 | static void kvmppc_gfn_remove(unsigned long gfn, struct kvm *kvm) |
328 | { |
329 | kvmppc_mark_gfn(gfn, kvm, flag: 0, uvmem_pfn: 0); |
330 | } |
331 | |
332 | /* return true, if the GFN is a secure-GFN backed by a secure-PFN */ |
333 | static bool kvmppc_gfn_is_uvmem_pfn(unsigned long gfn, struct kvm *kvm, |
334 | unsigned long *uvmem_pfn) |
335 | { |
336 | struct kvmppc_uvmem_slot *p; |
337 | |
338 | list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) { |
339 | if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) { |
340 | unsigned long index = gfn - p->base_pfn; |
341 | |
342 | if (p->pfns[index] & KVMPPC_GFN_UVMEM_PFN) { |
343 | if (uvmem_pfn) |
344 | *uvmem_pfn = p->pfns[index] & |
345 | KVMPPC_GFN_PFN_MASK; |
346 | return true; |
347 | } else |
348 | return false; |
349 | } |
350 | } |
351 | return false; |
352 | } |
353 | |
354 | /* |
355 | * starting from *gfn search for the next available GFN that is not yet |
356 | * transitioned to a secure GFN. return the value of that GFN in *gfn. If a |
357 | * GFN is found, return true, else return false |
358 | * |
359 | * Must be called with kvm->arch.uvmem_lock held. |
360 | */ |
361 | static bool kvmppc_next_nontransitioned_gfn(const struct kvm_memory_slot *memslot, |
362 | struct kvm *kvm, unsigned long *gfn) |
363 | { |
364 | struct kvmppc_uvmem_slot *p = NULL, *iter; |
365 | bool ret = false; |
366 | unsigned long i; |
367 | |
368 | list_for_each_entry(iter, &kvm->arch.uvmem_pfns, list) |
369 | if (*gfn >= iter->base_pfn && *gfn < iter->base_pfn + iter->nr_pfns) { |
370 | p = iter; |
371 | break; |
372 | } |
373 | if (!p) |
374 | return ret; |
375 | /* |
376 | * The code below assumes, one to one correspondence between |
377 | * kvmppc_uvmem_slot and memslot. |
378 | */ |
379 | for (i = *gfn; i < p->base_pfn + p->nr_pfns; i++) { |
380 | unsigned long index = i - p->base_pfn; |
381 | |
382 | if (!(p->pfns[index] & KVMPPC_GFN_FLAG_MASK)) { |
383 | *gfn = i; |
384 | ret = true; |
385 | break; |
386 | } |
387 | } |
388 | return ret; |
389 | } |
390 | |
391 | static int kvmppc_memslot_page_merge(struct kvm *kvm, |
392 | const struct kvm_memory_slot *memslot, bool merge) |
393 | { |
394 | unsigned long gfn = memslot->base_gfn; |
395 | unsigned long end, start = gfn_to_hva(kvm, gfn); |
396 | unsigned long vm_flags; |
397 | int ret = 0; |
398 | struct vm_area_struct *vma; |
399 | int merge_flag = (merge) ? MADV_MERGEABLE : MADV_UNMERGEABLE; |
400 | |
401 | if (kvm_is_error_hva(addr: start)) |
402 | return H_STATE; |
403 | |
404 | end = start + (memslot->npages << PAGE_SHIFT); |
405 | |
406 | mmap_write_lock(mm: kvm->mm); |
407 | do { |
408 | vma = find_vma_intersection(mm: kvm->mm, start_addr: start, end_addr: end); |
409 | if (!vma) { |
410 | ret = H_STATE; |
411 | break; |
412 | } |
413 | vma_start_write(vma); |
414 | /* Copy vm_flags to avoid partial modifications in ksm_madvise */ |
415 | vm_flags = vma->vm_flags; |
416 | ret = ksm_madvise(vma, start: vma->vm_start, end: vma->vm_end, |
417 | advice: merge_flag, vm_flags: &vm_flags); |
418 | if (ret) { |
419 | ret = H_STATE; |
420 | break; |
421 | } |
422 | vm_flags_reset(vma, flags: vm_flags); |
423 | start = vma->vm_end; |
424 | } while (end > vma->vm_end); |
425 | |
426 | mmap_write_unlock(mm: kvm->mm); |
427 | return ret; |
428 | } |
429 | |
430 | static void __kvmppc_uvmem_memslot_delete(struct kvm *kvm, |
431 | const struct kvm_memory_slot *memslot) |
432 | { |
433 | uv_unregister_mem_slot(kvm->arch.lpid, memslot->id); |
434 | kvmppc_uvmem_slot_free(kvm, slot: memslot); |
435 | kvmppc_memslot_page_merge(kvm, memslot, merge: true); |
436 | } |
437 | |
438 | static int __kvmppc_uvmem_memslot_create(struct kvm *kvm, |
439 | const struct kvm_memory_slot *memslot) |
440 | { |
441 | int ret = H_PARAMETER; |
442 | |
443 | if (kvmppc_memslot_page_merge(kvm, memslot, merge: false)) |
444 | return ret; |
445 | |
446 | if (kvmppc_uvmem_slot_init(kvm, slot: memslot)) |
447 | goto out1; |
448 | |
449 | ret = uv_register_mem_slot(kvm->arch.lpid, |
450 | memslot->base_gfn << PAGE_SHIFT, |
451 | memslot->npages * PAGE_SIZE, |
452 | 0, memslot->id); |
453 | if (ret < 0) { |
454 | ret = H_PARAMETER; |
455 | goto out; |
456 | } |
457 | return 0; |
458 | out: |
459 | kvmppc_uvmem_slot_free(kvm, slot: memslot); |
460 | out1: |
461 | kvmppc_memslot_page_merge(kvm, memslot, merge: true); |
462 | return ret; |
463 | } |
464 | |
465 | unsigned long kvmppc_h_svm_init_start(struct kvm *kvm) |
466 | { |
467 | struct kvm_memslots *slots; |
468 | struct kvm_memory_slot *memslot, *m; |
469 | int ret = H_SUCCESS; |
470 | int srcu_idx, bkt; |
471 | |
472 | kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START; |
473 | |
474 | if (!kvmppc_uvmem_bitmap) |
475 | return H_UNSUPPORTED; |
476 | |
477 | /* Only radix guests can be secure guests */ |
478 | if (!kvm_is_radix(kvm)) |
479 | return H_UNSUPPORTED; |
480 | |
481 | /* NAK the transition to secure if not enabled */ |
482 | if (!kvm->arch.svm_enabled) |
483 | return H_AUTHORITY; |
484 | |
485 | srcu_idx = srcu_read_lock(ssp: &kvm->srcu); |
486 | |
487 | /* register the memslot */ |
488 | slots = kvm_memslots(kvm); |
489 | kvm_for_each_memslot(memslot, bkt, slots) { |
490 | ret = __kvmppc_uvmem_memslot_create(kvm, memslot); |
491 | if (ret) |
492 | break; |
493 | } |
494 | |
495 | if (ret) { |
496 | slots = kvm_memslots(kvm); |
497 | kvm_for_each_memslot(m, bkt, slots) { |
498 | if (m == memslot) |
499 | break; |
500 | __kvmppc_uvmem_memslot_delete(kvm, memslot); |
501 | } |
502 | } |
503 | |
504 | srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx); |
505 | return ret; |
506 | } |
507 | |
508 | /* |
509 | * Provision a new page on HV side and copy over the contents |
510 | * from secure memory using UV_PAGE_OUT uvcall. |
511 | * Caller must held kvm->arch.uvmem_lock. |
512 | */ |
513 | static int __kvmppc_svm_page_out(struct vm_area_struct *vma, |
514 | unsigned long start, |
515 | unsigned long end, unsigned long page_shift, |
516 | struct kvm *kvm, unsigned long gpa, struct page *fault_page) |
517 | { |
518 | unsigned long src_pfn, dst_pfn = 0; |
519 | struct migrate_vma mig = { 0 }; |
520 | struct page *dpage, *spage; |
521 | struct kvmppc_uvmem_page_pvt *pvt; |
522 | unsigned long pfn; |
523 | int ret = U_SUCCESS; |
524 | |
525 | memset(&mig, 0, sizeof(mig)); |
526 | mig.vma = vma; |
527 | mig.start = start; |
528 | mig.end = end; |
529 | mig.src = &src_pfn; |
530 | mig.dst = &dst_pfn; |
531 | mig.pgmap_owner = &kvmppc_uvmem_pgmap; |
532 | mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE; |
533 | mig.fault_page = fault_page; |
534 | |
535 | /* The requested page is already paged-out, nothing to do */ |
536 | if (!kvmppc_gfn_is_uvmem_pfn(gfn: gpa >> page_shift, kvm, NULL)) |
537 | return ret; |
538 | |
539 | ret = migrate_vma_setup(args: &mig); |
540 | if (ret) |
541 | return -1; |
542 | |
543 | spage = migrate_pfn_to_page(mpfn: *mig.src); |
544 | if (!spage || !(*mig.src & MIGRATE_PFN_MIGRATE)) |
545 | goto out_finalize; |
546 | |
547 | if (!is_zone_device_page(page: spage)) |
548 | goto out_finalize; |
549 | |
550 | dpage = alloc_page_vma(GFP_HIGHUSER, vma, addr: start); |
551 | if (!dpage) { |
552 | ret = -1; |
553 | goto out_finalize; |
554 | } |
555 | |
556 | lock_page(page: dpage); |
557 | pvt = spage->zone_device_data; |
558 | pfn = page_to_pfn(dpage); |
559 | |
560 | /* |
561 | * This function is used in two cases: |
562 | * - When HV touches a secure page, for which we do UV_PAGE_OUT |
563 | * - When a secure page is converted to shared page, we *get* |
564 | * the page to essentially unmap the device page. In this |
565 | * case we skip page-out. |
566 | */ |
567 | if (!pvt->skip_page_out) |
568 | ret = uv_page_out(kvm->arch.lpid, pfn << page_shift, |
569 | gpa, 0, page_shift); |
570 | |
571 | if (ret == U_SUCCESS) |
572 | *mig.dst = migrate_pfn(pfn); |
573 | else { |
574 | unlock_page(page: dpage); |
575 | __free_page(dpage); |
576 | goto out_finalize; |
577 | } |
578 | |
579 | migrate_vma_pages(migrate: &mig); |
580 | |
581 | out_finalize: |
582 | migrate_vma_finalize(migrate: &mig); |
583 | return ret; |
584 | } |
585 | |
586 | static inline int kvmppc_svm_page_out(struct vm_area_struct *vma, |
587 | unsigned long start, unsigned long end, |
588 | unsigned long page_shift, |
589 | struct kvm *kvm, unsigned long gpa, |
590 | struct page *fault_page) |
591 | { |
592 | int ret; |
593 | |
594 | mutex_lock(&kvm->arch.uvmem_lock); |
595 | ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, |
596 | fault_page); |
597 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
598 | |
599 | return ret; |
600 | } |
601 | |
602 | /* |
603 | * Drop device pages that we maintain for the secure guest |
604 | * |
605 | * We first mark the pages to be skipped from UV_PAGE_OUT when there |
606 | * is HV side fault on these pages. Next we *get* these pages, forcing |
607 | * fault on them, do fault time migration to replace the device PTEs in |
608 | * QEMU page table with normal PTEs from newly allocated pages. |
609 | */ |
610 | void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *slot, |
611 | struct kvm *kvm, bool skip_page_out) |
612 | { |
613 | int i; |
614 | struct kvmppc_uvmem_page_pvt *pvt; |
615 | struct page *uvmem_page; |
616 | struct vm_area_struct *vma = NULL; |
617 | unsigned long uvmem_pfn, gfn; |
618 | unsigned long addr; |
619 | |
620 | mmap_read_lock(mm: kvm->mm); |
621 | |
622 | addr = slot->userspace_addr; |
623 | |
624 | gfn = slot->base_gfn; |
625 | for (i = slot->npages; i; --i, ++gfn, addr += PAGE_SIZE) { |
626 | |
627 | /* Fetch the VMA if addr is not in the latest fetched one */ |
628 | if (!vma || addr >= vma->vm_end) { |
629 | vma = vma_lookup(mm: kvm->mm, addr); |
630 | if (!vma) { |
631 | pr_err("Can't find VMA for gfn:0x%lx\n" , gfn); |
632 | break; |
633 | } |
634 | } |
635 | |
636 | mutex_lock(&kvm->arch.uvmem_lock); |
637 | |
638 | if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, uvmem_pfn: &uvmem_pfn)) { |
639 | uvmem_page = pfn_to_page(uvmem_pfn); |
640 | pvt = uvmem_page->zone_device_data; |
641 | pvt->skip_page_out = skip_page_out; |
642 | pvt->remove_gfn = true; |
643 | |
644 | if (__kvmppc_svm_page_out(vma, start: addr, end: addr + PAGE_SIZE, |
645 | PAGE_SHIFT, kvm, gpa: pvt->gpa, NULL)) |
646 | pr_err("Can't page out gpa:0x%lx addr:0x%lx\n" , |
647 | pvt->gpa, addr); |
648 | } else { |
649 | /* Remove the shared flag if any */ |
650 | kvmppc_gfn_remove(gfn, kvm); |
651 | } |
652 | |
653 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
654 | } |
655 | |
656 | mmap_read_unlock(mm: kvm->mm); |
657 | } |
658 | |
659 | unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm) |
660 | { |
661 | int srcu_idx, bkt; |
662 | struct kvm_memory_slot *memslot; |
663 | |
664 | /* |
665 | * Expect to be called only after INIT_START and before INIT_DONE. |
666 | * If INIT_DONE was completed, use normal VM termination sequence. |
667 | */ |
668 | if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) |
669 | return H_UNSUPPORTED; |
670 | |
671 | if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) |
672 | return H_STATE; |
673 | |
674 | srcu_idx = srcu_read_lock(ssp: &kvm->srcu); |
675 | |
676 | kvm_for_each_memslot(memslot, bkt, kvm_memslots(kvm)) |
677 | kvmppc_uvmem_drop_pages(slot: memslot, kvm, skip_page_out: false); |
678 | |
679 | srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx); |
680 | |
681 | kvm->arch.secure_guest = 0; |
682 | uv_svm_terminate(kvm->arch.lpid); |
683 | |
684 | return H_PARAMETER; |
685 | } |
686 | |
687 | /* |
688 | * Get a free device PFN from the pool |
689 | * |
690 | * Called when a normal page is moved to secure memory (UV_PAGE_IN). Device |
691 | * PFN will be used to keep track of the secure page on HV side. |
692 | * |
693 | * Called with kvm->arch.uvmem_lock held |
694 | */ |
695 | static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm) |
696 | { |
697 | struct page *dpage = NULL; |
698 | unsigned long bit, uvmem_pfn; |
699 | struct kvmppc_uvmem_page_pvt *pvt; |
700 | unsigned long pfn_last, pfn_first; |
701 | |
702 | pfn_first = kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT; |
703 | pfn_last = pfn_first + |
704 | (range_len(range: &kvmppc_uvmem_pgmap.range) >> PAGE_SHIFT); |
705 | |
706 | spin_lock(lock: &kvmppc_uvmem_bitmap_lock); |
707 | bit = find_first_zero_bit(addr: kvmppc_uvmem_bitmap, |
708 | size: pfn_last - pfn_first); |
709 | if (bit >= (pfn_last - pfn_first)) |
710 | goto out; |
711 | bitmap_set(map: kvmppc_uvmem_bitmap, start: bit, nbits: 1); |
712 | spin_unlock(lock: &kvmppc_uvmem_bitmap_lock); |
713 | |
714 | pvt = kzalloc(size: sizeof(*pvt), GFP_KERNEL); |
715 | if (!pvt) |
716 | goto out_clear; |
717 | |
718 | uvmem_pfn = bit + pfn_first; |
719 | kvmppc_gfn_secure_uvmem_pfn(gfn: gpa >> PAGE_SHIFT, uvmem_pfn, kvm); |
720 | |
721 | pvt->gpa = gpa; |
722 | pvt->kvm = kvm; |
723 | |
724 | dpage = pfn_to_page(uvmem_pfn); |
725 | dpage->zone_device_data = pvt; |
726 | zone_device_page_init(page: dpage); |
727 | return dpage; |
728 | out_clear: |
729 | spin_lock(lock: &kvmppc_uvmem_bitmap_lock); |
730 | bitmap_clear(map: kvmppc_uvmem_bitmap, start: bit, nbits: 1); |
731 | out: |
732 | spin_unlock(lock: &kvmppc_uvmem_bitmap_lock); |
733 | return NULL; |
734 | } |
735 | |
736 | /* |
737 | * Alloc a PFN from private device memory pool. If @pagein is true, |
738 | * copy page from normal memory to secure memory using UV_PAGE_IN uvcall. |
739 | */ |
740 | static int kvmppc_svm_page_in(struct vm_area_struct *vma, |
741 | unsigned long start, |
742 | unsigned long end, unsigned long gpa, struct kvm *kvm, |
743 | unsigned long page_shift, |
744 | bool pagein) |
745 | { |
746 | unsigned long src_pfn, dst_pfn = 0; |
747 | struct migrate_vma mig = { 0 }; |
748 | struct page *spage; |
749 | unsigned long pfn; |
750 | struct page *dpage; |
751 | int ret = 0; |
752 | |
753 | memset(&mig, 0, sizeof(mig)); |
754 | mig.vma = vma; |
755 | mig.start = start; |
756 | mig.end = end; |
757 | mig.src = &src_pfn; |
758 | mig.dst = &dst_pfn; |
759 | mig.flags = MIGRATE_VMA_SELECT_SYSTEM; |
760 | |
761 | ret = migrate_vma_setup(args: &mig); |
762 | if (ret) |
763 | return ret; |
764 | |
765 | if (!(*mig.src & MIGRATE_PFN_MIGRATE)) { |
766 | ret = -1; |
767 | goto out_finalize; |
768 | } |
769 | |
770 | dpage = kvmppc_uvmem_get_page(gpa, kvm); |
771 | if (!dpage) { |
772 | ret = -1; |
773 | goto out_finalize; |
774 | } |
775 | |
776 | if (pagein) { |
777 | pfn = *mig.src >> MIGRATE_PFN_SHIFT; |
778 | spage = migrate_pfn_to_page(mpfn: *mig.src); |
779 | if (spage) { |
780 | ret = uv_page_in(kvm->arch.lpid, pfn << page_shift, |
781 | gpa, 0, page_shift); |
782 | if (ret) |
783 | goto out_finalize; |
784 | } |
785 | } |
786 | |
787 | *mig.dst = migrate_pfn(page_to_pfn(dpage)); |
788 | migrate_vma_pages(migrate: &mig); |
789 | out_finalize: |
790 | migrate_vma_finalize(migrate: &mig); |
791 | return ret; |
792 | } |
793 | |
794 | static int kvmppc_uv_migrate_mem_slot(struct kvm *kvm, |
795 | const struct kvm_memory_slot *memslot) |
796 | { |
797 | unsigned long gfn = memslot->base_gfn; |
798 | struct vm_area_struct *vma; |
799 | unsigned long start, end; |
800 | int ret = 0; |
801 | |
802 | mmap_read_lock(mm: kvm->mm); |
803 | mutex_lock(&kvm->arch.uvmem_lock); |
804 | while (kvmppc_next_nontransitioned_gfn(memslot, kvm, gfn: &gfn)) { |
805 | ret = H_STATE; |
806 | start = gfn_to_hva(kvm, gfn); |
807 | if (kvm_is_error_hva(addr: start)) |
808 | break; |
809 | |
810 | end = start + (1UL << PAGE_SHIFT); |
811 | vma = find_vma_intersection(mm: kvm->mm, start_addr: start, end_addr: end); |
812 | if (!vma || vma->vm_start > start || vma->vm_end < end) |
813 | break; |
814 | |
815 | ret = kvmppc_svm_page_in(vma, start, end, |
816 | gpa: (gfn << PAGE_SHIFT), kvm, PAGE_SHIFT, pagein: false); |
817 | if (ret) { |
818 | ret = H_STATE; |
819 | break; |
820 | } |
821 | |
822 | /* relinquish the cpu if needed */ |
823 | cond_resched(); |
824 | } |
825 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
826 | mmap_read_unlock(mm: kvm->mm); |
827 | return ret; |
828 | } |
829 | |
830 | unsigned long kvmppc_h_svm_init_done(struct kvm *kvm) |
831 | { |
832 | struct kvm_memslots *slots; |
833 | struct kvm_memory_slot *memslot; |
834 | int srcu_idx, bkt; |
835 | long ret = H_SUCCESS; |
836 | |
837 | if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) |
838 | return H_UNSUPPORTED; |
839 | |
840 | /* migrate any unmoved normal pfn to device pfns*/ |
841 | srcu_idx = srcu_read_lock(ssp: &kvm->srcu); |
842 | slots = kvm_memslots(kvm); |
843 | kvm_for_each_memslot(memslot, bkt, slots) { |
844 | ret = kvmppc_uv_migrate_mem_slot(kvm, memslot); |
845 | if (ret) { |
846 | /* |
847 | * The pages will remain transitioned. |
848 | * Its the callers responsibility to |
849 | * terminate the VM, which will undo |
850 | * all state of the VM. Till then |
851 | * this VM is in a erroneous state. |
852 | * Its KVMPPC_SECURE_INIT_DONE will |
853 | * remain unset. |
854 | */ |
855 | ret = H_STATE; |
856 | goto out; |
857 | } |
858 | } |
859 | |
860 | kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_DONE; |
861 | pr_info("LPID %lld went secure\n" , kvm->arch.lpid); |
862 | |
863 | out: |
864 | srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx); |
865 | return ret; |
866 | } |
867 | |
868 | /* |
869 | * Shares the page with HV, thus making it a normal page. |
870 | * |
871 | * - If the page is already secure, then provision a new page and share |
872 | * - If the page is a normal page, share the existing page |
873 | * |
874 | * In the former case, uses dev_pagemap_ops.migrate_to_ram handler |
875 | * to unmap the device page from QEMU's page tables. |
876 | */ |
877 | static unsigned long kvmppc_share_page(struct kvm *kvm, unsigned long gpa, |
878 | unsigned long page_shift) |
879 | { |
880 | |
881 | int ret = H_PARAMETER; |
882 | struct page *uvmem_page; |
883 | struct kvmppc_uvmem_page_pvt *pvt; |
884 | unsigned long pfn; |
885 | unsigned long gfn = gpa >> page_shift; |
886 | int srcu_idx; |
887 | unsigned long uvmem_pfn; |
888 | |
889 | srcu_idx = srcu_read_lock(ssp: &kvm->srcu); |
890 | mutex_lock(&kvm->arch.uvmem_lock); |
891 | if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, uvmem_pfn: &uvmem_pfn)) { |
892 | uvmem_page = pfn_to_page(uvmem_pfn); |
893 | pvt = uvmem_page->zone_device_data; |
894 | pvt->skip_page_out = true; |
895 | /* |
896 | * do not drop the GFN. It is a valid GFN |
897 | * that is transitioned to a shared GFN. |
898 | */ |
899 | pvt->remove_gfn = false; |
900 | } |
901 | |
902 | retry: |
903 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
904 | pfn = gfn_to_pfn(kvm, gfn); |
905 | if (is_error_noslot_pfn(pfn)) |
906 | goto out; |
907 | |
908 | mutex_lock(&kvm->arch.uvmem_lock); |
909 | if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, uvmem_pfn: &uvmem_pfn)) { |
910 | uvmem_page = pfn_to_page(uvmem_pfn); |
911 | pvt = uvmem_page->zone_device_data; |
912 | pvt->skip_page_out = true; |
913 | pvt->remove_gfn = false; /* it continues to be a valid GFN */ |
914 | kvm_release_pfn_clean(pfn); |
915 | goto retry; |
916 | } |
917 | |
918 | if (!uv_page_in(kvm->arch.lpid, pfn << page_shift, gpa, 0, |
919 | page_shift)) { |
920 | kvmppc_gfn_shared(gfn, kvm); |
921 | ret = H_SUCCESS; |
922 | } |
923 | kvm_release_pfn_clean(pfn); |
924 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
925 | out: |
926 | srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx); |
927 | return ret; |
928 | } |
929 | |
930 | /* |
931 | * H_SVM_PAGE_IN: Move page from normal memory to secure memory. |
932 | * |
933 | * H_PAGE_IN_SHARED flag makes the page shared which means that the same |
934 | * memory in is visible from both UV and HV. |
935 | */ |
936 | unsigned long kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gpa, |
937 | unsigned long flags, |
938 | unsigned long page_shift) |
939 | { |
940 | unsigned long start, end; |
941 | struct vm_area_struct *vma; |
942 | int srcu_idx; |
943 | unsigned long gfn = gpa >> page_shift; |
944 | int ret; |
945 | |
946 | if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) |
947 | return H_UNSUPPORTED; |
948 | |
949 | if (page_shift != PAGE_SHIFT) |
950 | return H_P3; |
951 | |
952 | if (flags & ~H_PAGE_IN_SHARED) |
953 | return H_P2; |
954 | |
955 | if (flags & H_PAGE_IN_SHARED) |
956 | return kvmppc_share_page(kvm, gpa, page_shift); |
957 | |
958 | ret = H_PARAMETER; |
959 | srcu_idx = srcu_read_lock(ssp: &kvm->srcu); |
960 | mmap_read_lock(mm: kvm->mm); |
961 | |
962 | start = gfn_to_hva(kvm, gfn); |
963 | if (kvm_is_error_hva(addr: start)) |
964 | goto out; |
965 | |
966 | mutex_lock(&kvm->arch.uvmem_lock); |
967 | /* Fail the page-in request of an already paged-in page */ |
968 | if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL)) |
969 | goto out_unlock; |
970 | |
971 | end = start + (1UL << page_shift); |
972 | vma = find_vma_intersection(mm: kvm->mm, start_addr: start, end_addr: end); |
973 | if (!vma || vma->vm_start > start || vma->vm_end < end) |
974 | goto out_unlock; |
975 | |
976 | if (kvmppc_svm_page_in(vma, start, end, gpa, kvm, page_shift, |
977 | pagein: true)) |
978 | goto out_unlock; |
979 | |
980 | ret = H_SUCCESS; |
981 | |
982 | out_unlock: |
983 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
984 | out: |
985 | mmap_read_unlock(mm: kvm->mm); |
986 | srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx); |
987 | return ret; |
988 | } |
989 | |
990 | |
991 | /* |
992 | * Fault handler callback that gets called when HV touches any page that |
993 | * has been moved to secure memory, we ask UV to give back the page by |
994 | * issuing UV_PAGE_OUT uvcall. |
995 | * |
996 | * This eventually results in dropping of device PFN and the newly |
997 | * provisioned page/PFN gets populated in QEMU page tables. |
998 | */ |
999 | static vm_fault_t kvmppc_uvmem_migrate_to_ram(struct vm_fault *vmf) |
1000 | { |
1001 | struct kvmppc_uvmem_page_pvt *pvt = vmf->page->zone_device_data; |
1002 | |
1003 | if (kvmppc_svm_page_out(vma: vmf->vma, start: vmf->address, |
1004 | end: vmf->address + PAGE_SIZE, PAGE_SHIFT, |
1005 | kvm: pvt->kvm, gpa: pvt->gpa, fault_page: vmf->page)) |
1006 | return VM_FAULT_SIGBUS; |
1007 | else |
1008 | return 0; |
1009 | } |
1010 | |
1011 | /* |
1012 | * Release the device PFN back to the pool |
1013 | * |
1014 | * Gets called when secure GFN tranistions from a secure-PFN |
1015 | * to a normal PFN during H_SVM_PAGE_OUT. |
1016 | * Gets called with kvm->arch.uvmem_lock held. |
1017 | */ |
1018 | static void kvmppc_uvmem_page_free(struct page *page) |
1019 | { |
1020 | unsigned long pfn = page_to_pfn(page) - |
1021 | (kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT); |
1022 | struct kvmppc_uvmem_page_pvt *pvt; |
1023 | |
1024 | spin_lock(lock: &kvmppc_uvmem_bitmap_lock); |
1025 | bitmap_clear(map: kvmppc_uvmem_bitmap, start: pfn, nbits: 1); |
1026 | spin_unlock(lock: &kvmppc_uvmem_bitmap_lock); |
1027 | |
1028 | pvt = page->zone_device_data; |
1029 | page->zone_device_data = NULL; |
1030 | if (pvt->remove_gfn) |
1031 | kvmppc_gfn_remove(gfn: pvt->gpa >> PAGE_SHIFT, kvm: pvt->kvm); |
1032 | else |
1033 | kvmppc_gfn_secure_mem_pfn(gfn: pvt->gpa >> PAGE_SHIFT, kvm: pvt->kvm); |
1034 | kfree(objp: pvt); |
1035 | } |
1036 | |
1037 | static const struct dev_pagemap_ops kvmppc_uvmem_ops = { |
1038 | .page_free = kvmppc_uvmem_page_free, |
1039 | .migrate_to_ram = kvmppc_uvmem_migrate_to_ram, |
1040 | }; |
1041 | |
1042 | /* |
1043 | * H_SVM_PAGE_OUT: Move page from secure memory to normal memory. |
1044 | */ |
1045 | unsigned long |
1046 | kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gpa, |
1047 | unsigned long flags, unsigned long page_shift) |
1048 | { |
1049 | unsigned long gfn = gpa >> page_shift; |
1050 | unsigned long start, end; |
1051 | struct vm_area_struct *vma; |
1052 | int srcu_idx; |
1053 | int ret; |
1054 | |
1055 | if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START)) |
1056 | return H_UNSUPPORTED; |
1057 | |
1058 | if (page_shift != PAGE_SHIFT) |
1059 | return H_P3; |
1060 | |
1061 | if (flags) |
1062 | return H_P2; |
1063 | |
1064 | ret = H_PARAMETER; |
1065 | srcu_idx = srcu_read_lock(ssp: &kvm->srcu); |
1066 | mmap_read_lock(mm: kvm->mm); |
1067 | start = gfn_to_hva(kvm, gfn); |
1068 | if (kvm_is_error_hva(addr: start)) |
1069 | goto out; |
1070 | |
1071 | end = start + (1UL << page_shift); |
1072 | vma = find_vma_intersection(mm: kvm->mm, start_addr: start, end_addr: end); |
1073 | if (!vma || vma->vm_start > start || vma->vm_end < end) |
1074 | goto out; |
1075 | |
1076 | if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, NULL)) |
1077 | ret = H_SUCCESS; |
1078 | out: |
1079 | mmap_read_unlock(mm: kvm->mm); |
1080 | srcu_read_unlock(ssp: &kvm->srcu, idx: srcu_idx); |
1081 | return ret; |
1082 | } |
1083 | |
1084 | int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn) |
1085 | { |
1086 | unsigned long pfn; |
1087 | int ret = U_SUCCESS; |
1088 | |
1089 | pfn = gfn_to_pfn(kvm, gfn); |
1090 | if (is_error_noslot_pfn(pfn)) |
1091 | return -EFAULT; |
1092 | |
1093 | mutex_lock(&kvm->arch.uvmem_lock); |
1094 | if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL)) |
1095 | goto out; |
1096 | |
1097 | ret = uv_page_in(kvm->arch.lpid, pfn << PAGE_SHIFT, gfn << PAGE_SHIFT, |
1098 | 0, PAGE_SHIFT); |
1099 | out: |
1100 | kvm_release_pfn_clean(pfn); |
1101 | mutex_unlock(lock: &kvm->arch.uvmem_lock); |
1102 | return (ret == U_SUCCESS) ? RESUME_GUEST : -EFAULT; |
1103 | } |
1104 | |
1105 | int kvmppc_uvmem_memslot_create(struct kvm *kvm, const struct kvm_memory_slot *new) |
1106 | { |
1107 | int ret = __kvmppc_uvmem_memslot_create(kvm, memslot: new); |
1108 | |
1109 | if (!ret) |
1110 | ret = kvmppc_uv_migrate_mem_slot(kvm, memslot: new); |
1111 | |
1112 | return ret; |
1113 | } |
1114 | |
1115 | void kvmppc_uvmem_memslot_delete(struct kvm *kvm, const struct kvm_memory_slot *old) |
1116 | { |
1117 | __kvmppc_uvmem_memslot_delete(kvm, memslot: old); |
1118 | } |
1119 | |
1120 | static u64 kvmppc_get_secmem_size(void) |
1121 | { |
1122 | struct device_node *np; |
1123 | int i, len; |
1124 | const __be32 *prop; |
1125 | u64 size = 0; |
1126 | |
1127 | /* |
1128 | * First try the new ibm,secure-memory nodes which supersede the |
1129 | * secure-memory-ranges property. |
1130 | * If we found some, no need to read the deprecated ones. |
1131 | */ |
1132 | for_each_compatible_node(np, NULL, "ibm,secure-memory" ) { |
1133 | prop = of_get_property(node: np, name: "reg" , lenp: &len); |
1134 | if (!prop) |
1135 | continue; |
1136 | size += of_read_number(cell: prop + 2, size: 2); |
1137 | } |
1138 | if (size) |
1139 | return size; |
1140 | |
1141 | np = of_find_compatible_node(NULL, NULL, compat: "ibm,uv-firmware" ); |
1142 | if (!np) |
1143 | goto out; |
1144 | |
1145 | prop = of_get_property(node: np, name: "secure-memory-ranges" , lenp: &len); |
1146 | if (!prop) |
1147 | goto out_put; |
1148 | |
1149 | for (i = 0; i < len / (sizeof(*prop) * 4); i++) |
1150 | size += of_read_number(cell: prop + (i * 4) + 2, size: 2); |
1151 | |
1152 | out_put: |
1153 | of_node_put(node: np); |
1154 | out: |
1155 | return size; |
1156 | } |
1157 | |
1158 | int kvmppc_uvmem_init(void) |
1159 | { |
1160 | int ret = 0; |
1161 | unsigned long size; |
1162 | struct resource *res; |
1163 | void *addr; |
1164 | unsigned long pfn_last, pfn_first; |
1165 | |
1166 | size = kvmppc_get_secmem_size(); |
1167 | if (!size) { |
1168 | /* |
1169 | * Don't fail the initialization of kvm-hv module if |
1170 | * the platform doesn't export ibm,uv-firmware node. |
1171 | * Let normal guests run on such PEF-disabled platform. |
1172 | */ |
1173 | pr_info("KVMPPC-UVMEM: No support for secure guests\n" ); |
1174 | goto out; |
1175 | } |
1176 | |
1177 | res = request_free_mem_region(base: &iomem_resource, size, name: "kvmppc_uvmem" ); |
1178 | if (IS_ERR(ptr: res)) { |
1179 | ret = PTR_ERR(ptr: res); |
1180 | goto out; |
1181 | } |
1182 | |
1183 | kvmppc_uvmem_pgmap.type = MEMORY_DEVICE_PRIVATE; |
1184 | kvmppc_uvmem_pgmap.range.start = res->start; |
1185 | kvmppc_uvmem_pgmap.range.end = res->end; |
1186 | kvmppc_uvmem_pgmap.nr_range = 1; |
1187 | kvmppc_uvmem_pgmap.ops = &kvmppc_uvmem_ops; |
1188 | /* just one global instance: */ |
1189 | kvmppc_uvmem_pgmap.owner = &kvmppc_uvmem_pgmap; |
1190 | addr = memremap_pages(pgmap: &kvmppc_uvmem_pgmap, NUMA_NO_NODE); |
1191 | if (IS_ERR(ptr: addr)) { |
1192 | ret = PTR_ERR(ptr: addr); |
1193 | goto out_free_region; |
1194 | } |
1195 | |
1196 | pfn_first = res->start >> PAGE_SHIFT; |
1197 | pfn_last = pfn_first + (resource_size(res) >> PAGE_SHIFT); |
1198 | kvmppc_uvmem_bitmap = bitmap_zalloc(nbits: pfn_last - pfn_first, GFP_KERNEL); |
1199 | if (!kvmppc_uvmem_bitmap) { |
1200 | ret = -ENOMEM; |
1201 | goto out_unmap; |
1202 | } |
1203 | |
1204 | pr_info("KVMPPC-UVMEM: Secure Memory size 0x%lx\n" , size); |
1205 | return ret; |
1206 | out_unmap: |
1207 | memunmap_pages(pgmap: &kvmppc_uvmem_pgmap); |
1208 | out_free_region: |
1209 | release_mem_region(res->start, size); |
1210 | out: |
1211 | return ret; |
1212 | } |
1213 | |
1214 | void kvmppc_uvmem_free(void) |
1215 | { |
1216 | if (!kvmppc_uvmem_bitmap) |
1217 | return; |
1218 | |
1219 | memunmap_pages(pgmap: &kvmppc_uvmem_pgmap); |
1220 | release_mem_region(kvmppc_uvmem_pgmap.range.start, |
1221 | range_len(&kvmppc_uvmem_pgmap.range)); |
1222 | bitmap_free(bitmap: kvmppc_uvmem_bitmap); |
1223 | } |
1224 | |