1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Secure pages management: Migration of pages between normal and secure
4 * memory of KVM guests.
5 *
6 * Copyright 2018 Bharata B Rao, IBM Corp. <bharata@linux.ibm.com>
7 */
8
9 /*
10 * A pseries guest can be run as secure guest on Ultravisor-enabled
11 * POWER platforms. On such platforms, this driver will be used to manage
12 * the movement of guest pages between the normal memory managed by
13 * hypervisor (HV) and secure memory managed by Ultravisor (UV).
14 *
15 * The page-in or page-out requests from UV will come to HV as hcalls and
16 * HV will call back into UV via ultracalls to satisfy these page requests.
17 *
18 * Private ZONE_DEVICE memory equal to the amount of secure memory
19 * available in the platform for running secure guests is hotplugged.
20 * Whenever a page belonging to the guest becomes secure, a page from this
21 * private device memory is used to represent and track that secure page
22 * on the HV side. Some pages (like virtio buffers, VPA pages etc) are
23 * shared between UV and HV. However such pages aren't represented by
24 * device private memory and mappings to shared memory exist in both
25 * UV and HV page tables.
26 */
27
28 /*
29 * Notes on locking
30 *
31 * kvm->arch.uvmem_lock is a per-guest lock that prevents concurrent
32 * page-in and page-out requests for the same GPA. Concurrent accesses
33 * can either come via UV (guest vCPUs requesting for same page)
34 * or when HV and guest simultaneously access the same page.
35 * This mutex serializes the migration of page from HV(normal) to
36 * UV(secure) and vice versa. So the serialization points are around
37 * migrate_vma routines and page-in/out routines.
38 *
39 * Per-guest mutex comes with a cost though. Mainly it serializes the
40 * fault path as page-out can occur when HV faults on accessing secure
41 * guest pages. Currently UV issues page-in requests for all the guest
42 * PFNs one at a time during early boot (UV_ESM uvcall), so this is
43 * not a cause for concern. Also currently the number of page-outs caused
44 * by HV touching secure pages is very very low. If an when UV supports
45 * overcommitting, then we might see concurrent guest driven page-outs.
46 *
47 * Locking order
48 *
49 * 1. kvm->srcu - Protects KVM memslots
50 * 2. kvm->mm->mmap_lock - find_vma, migrate_vma_pages and helpers, ksm_madvise
51 * 3. kvm->arch.uvmem_lock - protects read/writes to uvmem slots thus acting
52 * as sync-points for page-in/out
53 */
54
55 /*
56 * Notes on page size
57 *
58 * Currently UV uses 2MB mappings internally, but will issue H_SVM_PAGE_IN
59 * and H_SVM_PAGE_OUT hcalls in PAGE_SIZE(64K) granularity. HV tracks
60 * secure GPAs at 64K page size and maintains one device PFN for each
61 * 64K secure GPA. UV_PAGE_IN and UV_PAGE_OUT calls by HV are also issued
62 * for 64K page at a time.
63 *
64 * HV faulting on secure pages: When HV touches any secure page, it
65 * faults and issues a UV_PAGE_OUT request with 64K page size. Currently
66 * UV splits and remaps the 2MB page if necessary and copies out the
67 * required 64K page contents.
68 *
69 * Shared pages: Whenever guest shares a secure page, UV will split and
70 * remap the 2MB page if required and issue H_SVM_PAGE_IN with 64K page size.
71 *
72 * HV invalidating a page: When a regular page belonging to secure
73 * guest gets unmapped, HV informs UV with UV_PAGE_INVAL of 64K
74 * page size. Using 64K page size is correct here because any non-secure
75 * page will essentially be of 64K page size. Splitting by UV during sharing
76 * and page-out ensures this.
77 *
78 * Page fault handling: When HV handles page fault of a page belonging
79 * to secure guest, it sends that to UV with a 64K UV_PAGE_IN request.
80 * Using 64K size is correct here too as UV would have split the 2MB page
81 * into 64k mappings and would have done page-outs earlier.
82 *
83 * In summary, the current secure pages handling code in HV assumes
84 * 64K page size and in fact fails any page-in/page-out requests of
85 * non-64K size upfront. If and when UV starts supporting multiple
86 * page-sizes, we need to break this assumption.
87 */
88
89 #include <linux/pagemap.h>
90 #include <linux/migrate.h>
91 #include <linux/kvm_host.h>
92 #include <linux/ksm.h>
93 #include <asm/ultravisor.h>
94 #include <asm/mman.h>
95 #include <asm/kvm_ppc.h>
96 #include <asm/kvm_book3s_uvmem.h>
97
98 static struct dev_pagemap kvmppc_uvmem_pgmap;
99 static unsigned long *kvmppc_uvmem_bitmap;
100 static DEFINE_SPINLOCK(kvmppc_uvmem_bitmap_lock);
101
102 /*
103 * States of a GFN
104 * ---------------
105 * The GFN can be in one of the following states.
106 *
107 * (a) Secure - The GFN is secure. The GFN is associated with
108 * a Secure VM, the contents of the GFN is not accessible
109 * to the Hypervisor. This GFN can be backed by a secure-PFN,
110 * or can be backed by a normal-PFN with contents encrypted.
111 * The former is true when the GFN is paged-in into the
112 * ultravisor. The latter is true when the GFN is paged-out
113 * of the ultravisor.
114 *
115 * (b) Shared - The GFN is shared. The GFN is associated with a
116 * a secure VM. The contents of the GFN is accessible to
117 * Hypervisor. This GFN is backed by a normal-PFN and its
118 * content is un-encrypted.
119 *
120 * (c) Normal - The GFN is a normal. The GFN is associated with
121 * a normal VM. The contents of the GFN is accesible to
122 * the Hypervisor. Its content is never encrypted.
123 *
124 * States of a VM.
125 * ---------------
126 *
127 * Normal VM: A VM whose contents are always accessible to
128 * the hypervisor. All its GFNs are normal-GFNs.
129 *
130 * Secure VM: A VM whose contents are not accessible to the
131 * hypervisor without the VM's consent. Its GFNs are
132 * either Shared-GFN or Secure-GFNs.
133 *
134 * Transient VM: A Normal VM that is transitioning to secure VM.
135 * The transition starts on successful return of
136 * H_SVM_INIT_START, and ends on successful return
137 * of H_SVM_INIT_DONE. This transient VM, can have GFNs
138 * in any of the three states; i.e Secure-GFN, Shared-GFN,
139 * and Normal-GFN. The VM never executes in this state
140 * in supervisor-mode.
141 *
142 * Memory slot State.
143 * -----------------------------
144 * The state of a memory slot mirrors the state of the
145 * VM the memory slot is associated with.
146 *
147 * VM State transition.
148 * --------------------
149 *
150 * A VM always starts in Normal Mode.
151 *
152 * H_SVM_INIT_START moves the VM into transient state. During this
153 * time the Ultravisor may request some of its GFNs to be shared or
154 * secured. So its GFNs can be in one of the three GFN states.
155 *
156 * H_SVM_INIT_DONE moves the VM entirely from transient state to
157 * secure-state. At this point any left-over normal-GFNs are
158 * transitioned to Secure-GFN.
159 *
160 * H_SVM_INIT_ABORT moves the transient VM back to normal VM.
161 * All its GFNs are moved to Normal-GFNs.
162 *
163 * UV_TERMINATE transitions the secure-VM back to normal-VM. All
164 * the secure-GFN and shared-GFNs are tranistioned to normal-GFN
165 * Note: The contents of the normal-GFN is undefined at this point.
166 *
167 * GFN state implementation:
168 * -------------------------
169 *
170 * Secure GFN is associated with a secure-PFN; also called uvmem_pfn,
171 * when the GFN is paged-in. Its pfn[] has KVMPPC_GFN_UVMEM_PFN flag
172 * set, and contains the value of the secure-PFN.
173 * It is associated with a normal-PFN; also called mem_pfn, when
174 * the GFN is pagedout. Its pfn[] has KVMPPC_GFN_MEM_PFN flag set.
175 * The value of the normal-PFN is not tracked.
176 *
177 * Shared GFN is associated with a normal-PFN. Its pfn[] has
178 * KVMPPC_UVMEM_SHARED_PFN flag set. The value of the normal-PFN
179 * is not tracked.
180 *
181 * Normal GFN is associated with normal-PFN. Its pfn[] has
182 * no flag set. The value of the normal-PFN is not tracked.
183 *
184 * Life cycle of a GFN
185 * --------------------
186 *
187 * --------------------------------------------------------------
188 * | | Share | Unshare | SVM |H_SVM_INIT_DONE|
189 * | |operation |operation | abort/ | |
190 * | | | | terminate | |
191 * -------------------------------------------------------------
192 * | | | | | |
193 * | Secure | Shared | Secure |Normal |Secure |
194 * | | | | | |
195 * | Shared | Shared | Secure |Normal |Shared |
196 * | | | | | |
197 * | Normal | Shared | Secure |Normal |Secure |
198 * --------------------------------------------------------------
199 *
200 * Life cycle of a VM
201 * --------------------
202 *
203 * --------------------------------------------------------------------
204 * | | start | H_SVM_ |H_SVM_ |H_SVM_ |UV_SVM_ |
205 * | | VM |INIT_START|INIT_DONE|INIT_ABORT |TERMINATE |
206 * | | | | | | |
207 * --------- ----------------------------------------------------------
208 * | | | | | | |
209 * | Normal | Normal | Transient|Error |Error |Normal |
210 * | | | | | | |
211 * | Secure | Error | Error |Error |Error |Normal |
212 * | | | | | | |
213 * |Transient| N/A | Error |Secure |Normal |Normal |
214 * --------------------------------------------------------------------
215 */
216
217 #define KVMPPC_GFN_UVMEM_PFN (1UL << 63)
218 #define KVMPPC_GFN_MEM_PFN (1UL << 62)
219 #define KVMPPC_GFN_SHARED (1UL << 61)
220 #define KVMPPC_GFN_SECURE (KVMPPC_GFN_UVMEM_PFN | KVMPPC_GFN_MEM_PFN)
221 #define KVMPPC_GFN_FLAG_MASK (KVMPPC_GFN_SECURE | KVMPPC_GFN_SHARED)
222 #define KVMPPC_GFN_PFN_MASK (~KVMPPC_GFN_FLAG_MASK)
223
224 struct kvmppc_uvmem_slot {
225 struct list_head list;
226 unsigned long nr_pfns;
227 unsigned long base_pfn;
228 unsigned long *pfns;
229 };
230 struct kvmppc_uvmem_page_pvt {
231 struct kvm *kvm;
232 unsigned long gpa;
233 bool skip_page_out;
234 bool remove_gfn;
235 };
236
kvmppc_uvmem_available(void)237 bool kvmppc_uvmem_available(void)
238 {
239 /*
240 * If kvmppc_uvmem_bitmap != NULL, then there is an ultravisor
241 * and our data structures have been initialized successfully.
242 */
243 return !!kvmppc_uvmem_bitmap;
244 }
245
kvmppc_uvmem_slot_init(struct kvm * kvm,const struct kvm_memory_slot * slot)246 int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot)
247 {
248 struct kvmppc_uvmem_slot *p;
249
250 p = kzalloc(sizeof(*p), GFP_KERNEL);
251 if (!p)
252 return -ENOMEM;
253 p->pfns = vzalloc(array_size(slot->npages, sizeof(*p->pfns)));
254 if (!p->pfns) {
255 kfree(p);
256 return -ENOMEM;
257 }
258 p->nr_pfns = slot->npages;
259 p->base_pfn = slot->base_gfn;
260
261 mutex_lock(&kvm->arch.uvmem_lock);
262 list_add(&p->list, &kvm->arch.uvmem_pfns);
263 mutex_unlock(&kvm->arch.uvmem_lock);
264
265 return 0;
266 }
267
268 /*
269 * All device PFNs are already released by the time we come here.
270 */
kvmppc_uvmem_slot_free(struct kvm * kvm,const struct kvm_memory_slot * slot)271 void kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot)
272 {
273 struct kvmppc_uvmem_slot *p, *next;
274
275 mutex_lock(&kvm->arch.uvmem_lock);
276 list_for_each_entry_safe(p, next, &kvm->arch.uvmem_pfns, list) {
277 if (p->base_pfn == slot->base_gfn) {
278 vfree(p->pfns);
279 list_del(&p->list);
280 kfree(p);
281 break;
282 }
283 }
284 mutex_unlock(&kvm->arch.uvmem_lock);
285 }
286
kvmppc_mark_gfn(unsigned long gfn,struct kvm * kvm,unsigned long flag,unsigned long uvmem_pfn)287 static void kvmppc_mark_gfn(unsigned long gfn, struct kvm *kvm,
288 unsigned long flag, unsigned long uvmem_pfn)
289 {
290 struct kvmppc_uvmem_slot *p;
291
292 list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
293 if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) {
294 unsigned long index = gfn - p->base_pfn;
295
296 if (flag == KVMPPC_GFN_UVMEM_PFN)
297 p->pfns[index] = uvmem_pfn | flag;
298 else
299 p->pfns[index] = flag;
300 return;
301 }
302 }
303 }
304
305 /* mark the GFN as secure-GFN associated with @uvmem pfn device-PFN. */
kvmppc_gfn_secure_uvmem_pfn(unsigned long gfn,unsigned long uvmem_pfn,struct kvm * kvm)306 static void kvmppc_gfn_secure_uvmem_pfn(unsigned long gfn,
307 unsigned long uvmem_pfn, struct kvm *kvm)
308 {
309 kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_UVMEM_PFN, uvmem_pfn);
310 }
311
312 /* mark the GFN as secure-GFN associated with a memory-PFN. */
kvmppc_gfn_secure_mem_pfn(unsigned long gfn,struct kvm * kvm)313 static void kvmppc_gfn_secure_mem_pfn(unsigned long gfn, struct kvm *kvm)
314 {
315 kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_MEM_PFN, 0);
316 }
317
318 /* mark the GFN as a shared GFN. */
kvmppc_gfn_shared(unsigned long gfn,struct kvm * kvm)319 static void kvmppc_gfn_shared(unsigned long gfn, struct kvm *kvm)
320 {
321 kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_SHARED, 0);
322 }
323
324 /* mark the GFN as a non-existent GFN. */
kvmppc_gfn_remove(unsigned long gfn,struct kvm * kvm)325 static void kvmppc_gfn_remove(unsigned long gfn, struct kvm *kvm)
326 {
327 kvmppc_mark_gfn(gfn, kvm, 0, 0);
328 }
329
330 /* return true, if the GFN is a secure-GFN backed by a secure-PFN */
kvmppc_gfn_is_uvmem_pfn(unsigned long gfn,struct kvm * kvm,unsigned long * uvmem_pfn)331 static bool kvmppc_gfn_is_uvmem_pfn(unsigned long gfn, struct kvm *kvm,
332 unsigned long *uvmem_pfn)
333 {
334 struct kvmppc_uvmem_slot *p;
335
336 list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
337 if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) {
338 unsigned long index = gfn - p->base_pfn;
339
340 if (p->pfns[index] & KVMPPC_GFN_UVMEM_PFN) {
341 if (uvmem_pfn)
342 *uvmem_pfn = p->pfns[index] &
343 KVMPPC_GFN_PFN_MASK;
344 return true;
345 } else
346 return false;
347 }
348 }
349 return false;
350 }
351
352 /*
353 * starting from *gfn search for the next available GFN that is not yet
354 * transitioned to a secure GFN. return the value of that GFN in *gfn. If a
355 * GFN is found, return true, else return false
356 *
357 * Must be called with kvm->arch.uvmem_lock held.
358 */
kvmppc_next_nontransitioned_gfn(const struct kvm_memory_slot * memslot,struct kvm * kvm,unsigned long * gfn)359 static bool kvmppc_next_nontransitioned_gfn(const struct kvm_memory_slot *memslot,
360 struct kvm *kvm, unsigned long *gfn)
361 {
362 struct kvmppc_uvmem_slot *p;
363 bool ret = false;
364 unsigned long i;
365
366 list_for_each_entry(p, &kvm->arch.uvmem_pfns, list)
367 if (*gfn >= p->base_pfn && *gfn < p->base_pfn + p->nr_pfns)
368 break;
369 if (!p)
370 return ret;
371 /*
372 * The code below assumes, one to one correspondence between
373 * kvmppc_uvmem_slot and memslot.
374 */
375 for (i = *gfn; i < p->base_pfn + p->nr_pfns; i++) {
376 unsigned long index = i - p->base_pfn;
377
378 if (!(p->pfns[index] & KVMPPC_GFN_FLAG_MASK)) {
379 *gfn = i;
380 ret = true;
381 break;
382 }
383 }
384 return ret;
385 }
386
kvmppc_memslot_page_merge(struct kvm * kvm,const struct kvm_memory_slot * memslot,bool merge)387 static int kvmppc_memslot_page_merge(struct kvm *kvm,
388 const struct kvm_memory_slot *memslot, bool merge)
389 {
390 unsigned long gfn = memslot->base_gfn;
391 unsigned long end, start = gfn_to_hva(kvm, gfn);
392 int ret = 0;
393 struct vm_area_struct *vma;
394 int merge_flag = (merge) ? MADV_MERGEABLE : MADV_UNMERGEABLE;
395
396 if (kvm_is_error_hva(start))
397 return H_STATE;
398
399 end = start + (memslot->npages << PAGE_SHIFT);
400
401 mmap_write_lock(kvm->mm);
402 do {
403 vma = find_vma_intersection(kvm->mm, start, end);
404 if (!vma) {
405 ret = H_STATE;
406 break;
407 }
408 ret = ksm_madvise(vma, vma->vm_start, vma->vm_end,
409 merge_flag, &vma->vm_flags);
410 if (ret) {
411 ret = H_STATE;
412 break;
413 }
414 start = vma->vm_end;
415 } while (end > vma->vm_end);
416
417 mmap_write_unlock(kvm->mm);
418 return ret;
419 }
420
__kvmppc_uvmem_memslot_delete(struct kvm * kvm,const struct kvm_memory_slot * memslot)421 static void __kvmppc_uvmem_memslot_delete(struct kvm *kvm,
422 const struct kvm_memory_slot *memslot)
423 {
424 uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
425 kvmppc_uvmem_slot_free(kvm, memslot);
426 kvmppc_memslot_page_merge(kvm, memslot, true);
427 }
428
__kvmppc_uvmem_memslot_create(struct kvm * kvm,const struct kvm_memory_slot * memslot)429 static int __kvmppc_uvmem_memslot_create(struct kvm *kvm,
430 const struct kvm_memory_slot *memslot)
431 {
432 int ret = H_PARAMETER;
433
434 if (kvmppc_memslot_page_merge(kvm, memslot, false))
435 return ret;
436
437 if (kvmppc_uvmem_slot_init(kvm, memslot))
438 goto out1;
439
440 ret = uv_register_mem_slot(kvm->arch.lpid,
441 memslot->base_gfn << PAGE_SHIFT,
442 memslot->npages * PAGE_SIZE,
443 0, memslot->id);
444 if (ret < 0) {
445 ret = H_PARAMETER;
446 goto out;
447 }
448 return 0;
449 out:
450 kvmppc_uvmem_slot_free(kvm, memslot);
451 out1:
452 kvmppc_memslot_page_merge(kvm, memslot, true);
453 return ret;
454 }
455
kvmppc_h_svm_init_start(struct kvm * kvm)456 unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
457 {
458 struct kvm_memslots *slots;
459 struct kvm_memory_slot *memslot, *m;
460 int ret = H_SUCCESS;
461 int srcu_idx;
462
463 kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START;
464
465 if (!kvmppc_uvmem_bitmap)
466 return H_UNSUPPORTED;
467
468 /* Only radix guests can be secure guests */
469 if (!kvm_is_radix(kvm))
470 return H_UNSUPPORTED;
471
472 /* NAK the transition to secure if not enabled */
473 if (!kvm->arch.svm_enabled)
474 return H_AUTHORITY;
475
476 srcu_idx = srcu_read_lock(&kvm->srcu);
477
478 /* register the memslot */
479 slots = kvm_memslots(kvm);
480 kvm_for_each_memslot(memslot, slots) {
481 ret = __kvmppc_uvmem_memslot_create(kvm, memslot);
482 if (ret)
483 break;
484 }
485
486 if (ret) {
487 slots = kvm_memslots(kvm);
488 kvm_for_each_memslot(m, slots) {
489 if (m == memslot)
490 break;
491 __kvmppc_uvmem_memslot_delete(kvm, memslot);
492 }
493 }
494
495 srcu_read_unlock(&kvm->srcu, srcu_idx);
496 return ret;
497 }
498
499 /*
500 * Provision a new page on HV side and copy over the contents
501 * from secure memory using UV_PAGE_OUT uvcall.
502 * Caller must held kvm->arch.uvmem_lock.
503 */
__kvmppc_svm_page_out(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long page_shift,struct kvm * kvm,unsigned long gpa)504 static int __kvmppc_svm_page_out(struct vm_area_struct *vma,
505 unsigned long start,
506 unsigned long end, unsigned long page_shift,
507 struct kvm *kvm, unsigned long gpa)
508 {
509 unsigned long src_pfn, dst_pfn = 0;
510 struct migrate_vma mig;
511 struct page *dpage, *spage;
512 struct kvmppc_uvmem_page_pvt *pvt;
513 unsigned long pfn;
514 int ret = U_SUCCESS;
515
516 memset(&mig, 0, sizeof(mig));
517 mig.vma = vma;
518 mig.start = start;
519 mig.end = end;
520 mig.src = &src_pfn;
521 mig.dst = &dst_pfn;
522 mig.pgmap_owner = &kvmppc_uvmem_pgmap;
523 mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
524
525 /* The requested page is already paged-out, nothing to do */
526 if (!kvmppc_gfn_is_uvmem_pfn(gpa >> page_shift, kvm, NULL))
527 return ret;
528
529 ret = migrate_vma_setup(&mig);
530 if (ret)
531 return -1;
532
533 spage = migrate_pfn_to_page(*mig.src);
534 if (!spage || !(*mig.src & MIGRATE_PFN_MIGRATE))
535 goto out_finalize;
536
537 if (!is_zone_device_page(spage))
538 goto out_finalize;
539
540 dpage = alloc_page_vma(GFP_HIGHUSER, vma, start);
541 if (!dpage) {
542 ret = -1;
543 goto out_finalize;
544 }
545
546 lock_page(dpage);
547 pvt = spage->zone_device_data;
548 pfn = page_to_pfn(dpage);
549
550 /*
551 * This function is used in two cases:
552 * - When HV touches a secure page, for which we do UV_PAGE_OUT
553 * - When a secure page is converted to shared page, we *get*
554 * the page to essentially unmap the device page. In this
555 * case we skip page-out.
556 */
557 if (!pvt->skip_page_out)
558 ret = uv_page_out(kvm->arch.lpid, pfn << page_shift,
559 gpa, 0, page_shift);
560
561 if (ret == U_SUCCESS)
562 *mig.dst = migrate_pfn(pfn) | MIGRATE_PFN_LOCKED;
563 else {
564 unlock_page(dpage);
565 __free_page(dpage);
566 goto out_finalize;
567 }
568
569 migrate_vma_pages(&mig);
570
571 out_finalize:
572 migrate_vma_finalize(&mig);
573 return ret;
574 }
575
kvmppc_svm_page_out(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long page_shift,struct kvm * kvm,unsigned long gpa)576 static inline int kvmppc_svm_page_out(struct vm_area_struct *vma,
577 unsigned long start, unsigned long end,
578 unsigned long page_shift,
579 struct kvm *kvm, unsigned long gpa)
580 {
581 int ret;
582
583 mutex_lock(&kvm->arch.uvmem_lock);
584 ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa);
585 mutex_unlock(&kvm->arch.uvmem_lock);
586
587 return ret;
588 }
589
590 /*
591 * Drop device pages that we maintain for the secure guest
592 *
593 * We first mark the pages to be skipped from UV_PAGE_OUT when there
594 * is HV side fault on these pages. Next we *get* these pages, forcing
595 * fault on them, do fault time migration to replace the device PTEs in
596 * QEMU page table with normal PTEs from newly allocated pages.
597 */
kvmppc_uvmem_drop_pages(const struct kvm_memory_slot * slot,struct kvm * kvm,bool skip_page_out)598 void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *slot,
599 struct kvm *kvm, bool skip_page_out)
600 {
601 int i;
602 struct kvmppc_uvmem_page_pvt *pvt;
603 struct page *uvmem_page;
604 struct vm_area_struct *vma = NULL;
605 unsigned long uvmem_pfn, gfn;
606 unsigned long addr;
607
608 mmap_read_lock(kvm->mm);
609
610 addr = slot->userspace_addr;
611
612 gfn = slot->base_gfn;
613 for (i = slot->npages; i; --i, ++gfn, addr += PAGE_SIZE) {
614
615 /* Fetch the VMA if addr is not in the latest fetched one */
616 if (!vma || addr >= vma->vm_end) {
617 vma = find_vma_intersection(kvm->mm, addr, addr+1);
618 if (!vma) {
619 pr_err("Can't find VMA for gfn:0x%lx\n", gfn);
620 break;
621 }
622 }
623
624 mutex_lock(&kvm->arch.uvmem_lock);
625
626 if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
627 uvmem_page = pfn_to_page(uvmem_pfn);
628 pvt = uvmem_page->zone_device_data;
629 pvt->skip_page_out = skip_page_out;
630 pvt->remove_gfn = true;
631
632 if (__kvmppc_svm_page_out(vma, addr, addr + PAGE_SIZE,
633 PAGE_SHIFT, kvm, pvt->gpa))
634 pr_err("Can't page out gpa:0x%lx addr:0x%lx\n",
635 pvt->gpa, addr);
636 } else {
637 /* Remove the shared flag if any */
638 kvmppc_gfn_remove(gfn, kvm);
639 }
640
641 mutex_unlock(&kvm->arch.uvmem_lock);
642 }
643
644 mmap_read_unlock(kvm->mm);
645 }
646
kvmppc_h_svm_init_abort(struct kvm * kvm)647 unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm)
648 {
649 int srcu_idx;
650 struct kvm_memory_slot *memslot;
651
652 /*
653 * Expect to be called only after INIT_START and before INIT_DONE.
654 * If INIT_DONE was completed, use normal VM termination sequence.
655 */
656 if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
657 return H_UNSUPPORTED;
658
659 if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
660 return H_STATE;
661
662 srcu_idx = srcu_read_lock(&kvm->srcu);
663
664 kvm_for_each_memslot(memslot, kvm_memslots(kvm))
665 kvmppc_uvmem_drop_pages(memslot, kvm, false);
666
667 srcu_read_unlock(&kvm->srcu, srcu_idx);
668
669 kvm->arch.secure_guest = 0;
670 uv_svm_terminate(kvm->arch.lpid);
671
672 return H_PARAMETER;
673 }
674
675 /*
676 * Get a free device PFN from the pool
677 *
678 * Called when a normal page is moved to secure memory (UV_PAGE_IN). Device
679 * PFN will be used to keep track of the secure page on HV side.
680 *
681 * Called with kvm->arch.uvmem_lock held
682 */
kvmppc_uvmem_get_page(unsigned long gpa,struct kvm * kvm)683 static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm)
684 {
685 struct page *dpage = NULL;
686 unsigned long bit, uvmem_pfn;
687 struct kvmppc_uvmem_page_pvt *pvt;
688 unsigned long pfn_last, pfn_first;
689
690 pfn_first = kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT;
691 pfn_last = pfn_first +
692 (range_len(&kvmppc_uvmem_pgmap.range) >> PAGE_SHIFT);
693
694 spin_lock(&kvmppc_uvmem_bitmap_lock);
695 bit = find_first_zero_bit(kvmppc_uvmem_bitmap,
696 pfn_last - pfn_first);
697 if (bit >= (pfn_last - pfn_first))
698 goto out;
699 bitmap_set(kvmppc_uvmem_bitmap, bit, 1);
700 spin_unlock(&kvmppc_uvmem_bitmap_lock);
701
702 pvt = kzalloc(sizeof(*pvt), GFP_KERNEL);
703 if (!pvt)
704 goto out_clear;
705
706 uvmem_pfn = bit + pfn_first;
707 kvmppc_gfn_secure_uvmem_pfn(gpa >> PAGE_SHIFT, uvmem_pfn, kvm);
708
709 pvt->gpa = gpa;
710 pvt->kvm = kvm;
711
712 dpage = pfn_to_page(uvmem_pfn);
713 dpage->zone_device_data = pvt;
714 get_page(dpage);
715 lock_page(dpage);
716 return dpage;
717 out_clear:
718 spin_lock(&kvmppc_uvmem_bitmap_lock);
719 bitmap_clear(kvmppc_uvmem_bitmap, bit, 1);
720 out:
721 spin_unlock(&kvmppc_uvmem_bitmap_lock);
722 return NULL;
723 }
724
725 /*
726 * Alloc a PFN from private device memory pool. If @pagein is true,
727 * copy page from normal memory to secure memory using UV_PAGE_IN uvcall.
728 */
kvmppc_svm_page_in(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long gpa,struct kvm * kvm,unsigned long page_shift,bool pagein)729 static int kvmppc_svm_page_in(struct vm_area_struct *vma,
730 unsigned long start,
731 unsigned long end, unsigned long gpa, struct kvm *kvm,
732 unsigned long page_shift,
733 bool pagein)
734 {
735 unsigned long src_pfn, dst_pfn = 0;
736 struct migrate_vma mig;
737 struct page *spage;
738 unsigned long pfn;
739 struct page *dpage;
740 int ret = 0;
741
742 memset(&mig, 0, sizeof(mig));
743 mig.vma = vma;
744 mig.start = start;
745 mig.end = end;
746 mig.src = &src_pfn;
747 mig.dst = &dst_pfn;
748 mig.flags = MIGRATE_VMA_SELECT_SYSTEM;
749
750 ret = migrate_vma_setup(&mig);
751 if (ret)
752 return ret;
753
754 if (!(*mig.src & MIGRATE_PFN_MIGRATE)) {
755 ret = -1;
756 goto out_finalize;
757 }
758
759 dpage = kvmppc_uvmem_get_page(gpa, kvm);
760 if (!dpage) {
761 ret = -1;
762 goto out_finalize;
763 }
764
765 if (pagein) {
766 pfn = *mig.src >> MIGRATE_PFN_SHIFT;
767 spage = migrate_pfn_to_page(*mig.src);
768 if (spage) {
769 ret = uv_page_in(kvm->arch.lpid, pfn << page_shift,
770 gpa, 0, page_shift);
771 if (ret)
772 goto out_finalize;
773 }
774 }
775
776 *mig.dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
777 migrate_vma_pages(&mig);
778 out_finalize:
779 migrate_vma_finalize(&mig);
780 return ret;
781 }
782
kvmppc_uv_migrate_mem_slot(struct kvm * kvm,const struct kvm_memory_slot * memslot)783 static int kvmppc_uv_migrate_mem_slot(struct kvm *kvm,
784 const struct kvm_memory_slot *memslot)
785 {
786 unsigned long gfn = memslot->base_gfn;
787 struct vm_area_struct *vma;
788 unsigned long start, end;
789 int ret = 0;
790
791 mmap_read_lock(kvm->mm);
792 mutex_lock(&kvm->arch.uvmem_lock);
793 while (kvmppc_next_nontransitioned_gfn(memslot, kvm, &gfn)) {
794 ret = H_STATE;
795 start = gfn_to_hva(kvm, gfn);
796 if (kvm_is_error_hva(start))
797 break;
798
799 end = start + (1UL << PAGE_SHIFT);
800 vma = find_vma_intersection(kvm->mm, start, end);
801 if (!vma || vma->vm_start > start || vma->vm_end < end)
802 break;
803
804 ret = kvmppc_svm_page_in(vma, start, end,
805 (gfn << PAGE_SHIFT), kvm, PAGE_SHIFT, false);
806 if (ret) {
807 ret = H_STATE;
808 break;
809 }
810
811 /* relinquish the cpu if needed */
812 cond_resched();
813 }
814 mutex_unlock(&kvm->arch.uvmem_lock);
815 mmap_read_unlock(kvm->mm);
816 return ret;
817 }
818
kvmppc_h_svm_init_done(struct kvm * kvm)819 unsigned long kvmppc_h_svm_init_done(struct kvm *kvm)
820 {
821 struct kvm_memslots *slots;
822 struct kvm_memory_slot *memslot;
823 int srcu_idx;
824 long ret = H_SUCCESS;
825
826 if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
827 return H_UNSUPPORTED;
828
829 /* migrate any unmoved normal pfn to device pfns*/
830 srcu_idx = srcu_read_lock(&kvm->srcu);
831 slots = kvm_memslots(kvm);
832 kvm_for_each_memslot(memslot, slots) {
833 ret = kvmppc_uv_migrate_mem_slot(kvm, memslot);
834 if (ret) {
835 /*
836 * The pages will remain transitioned.
837 * Its the callers responsibility to
838 * terminate the VM, which will undo
839 * all state of the VM. Till then
840 * this VM is in a erroneous state.
841 * Its KVMPPC_SECURE_INIT_DONE will
842 * remain unset.
843 */
844 ret = H_STATE;
845 goto out;
846 }
847 }
848
849 kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_DONE;
850 pr_info("LPID %d went secure\n", kvm->arch.lpid);
851
852 out:
853 srcu_read_unlock(&kvm->srcu, srcu_idx);
854 return ret;
855 }
856
857 /*
858 * Shares the page with HV, thus making it a normal page.
859 *
860 * - If the page is already secure, then provision a new page and share
861 * - If the page is a normal page, share the existing page
862 *
863 * In the former case, uses dev_pagemap_ops.migrate_to_ram handler
864 * to unmap the device page from QEMU's page tables.
865 */
kvmppc_share_page(struct kvm * kvm,unsigned long gpa,unsigned long page_shift)866 static unsigned long kvmppc_share_page(struct kvm *kvm, unsigned long gpa,
867 unsigned long page_shift)
868 {
869
870 int ret = H_PARAMETER;
871 struct page *uvmem_page;
872 struct kvmppc_uvmem_page_pvt *pvt;
873 unsigned long pfn;
874 unsigned long gfn = gpa >> page_shift;
875 int srcu_idx;
876 unsigned long uvmem_pfn;
877
878 srcu_idx = srcu_read_lock(&kvm->srcu);
879 mutex_lock(&kvm->arch.uvmem_lock);
880 if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
881 uvmem_page = pfn_to_page(uvmem_pfn);
882 pvt = uvmem_page->zone_device_data;
883 pvt->skip_page_out = true;
884 /*
885 * do not drop the GFN. It is a valid GFN
886 * that is transitioned to a shared GFN.
887 */
888 pvt->remove_gfn = false;
889 }
890
891 retry:
892 mutex_unlock(&kvm->arch.uvmem_lock);
893 pfn = gfn_to_pfn(kvm, gfn);
894 if (is_error_noslot_pfn(pfn))
895 goto out;
896
897 mutex_lock(&kvm->arch.uvmem_lock);
898 if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
899 uvmem_page = pfn_to_page(uvmem_pfn);
900 pvt = uvmem_page->zone_device_data;
901 pvt->skip_page_out = true;
902 pvt->remove_gfn = false; /* it continues to be a valid GFN */
903 kvm_release_pfn_clean(pfn);
904 goto retry;
905 }
906
907 if (!uv_page_in(kvm->arch.lpid, pfn << page_shift, gpa, 0,
908 page_shift)) {
909 kvmppc_gfn_shared(gfn, kvm);
910 ret = H_SUCCESS;
911 }
912 kvm_release_pfn_clean(pfn);
913 mutex_unlock(&kvm->arch.uvmem_lock);
914 out:
915 srcu_read_unlock(&kvm->srcu, srcu_idx);
916 return ret;
917 }
918
919 /*
920 * H_SVM_PAGE_IN: Move page from normal memory to secure memory.
921 *
922 * H_PAGE_IN_SHARED flag makes the page shared which means that the same
923 * memory in is visible from both UV and HV.
924 */
kvmppc_h_svm_page_in(struct kvm * kvm,unsigned long gpa,unsigned long flags,unsigned long page_shift)925 unsigned long kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gpa,
926 unsigned long flags,
927 unsigned long page_shift)
928 {
929 unsigned long start, end;
930 struct vm_area_struct *vma;
931 int srcu_idx;
932 unsigned long gfn = gpa >> page_shift;
933 int ret;
934
935 if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
936 return H_UNSUPPORTED;
937
938 if (page_shift != PAGE_SHIFT)
939 return H_P3;
940
941 if (flags & ~H_PAGE_IN_SHARED)
942 return H_P2;
943
944 if (flags & H_PAGE_IN_SHARED)
945 return kvmppc_share_page(kvm, gpa, page_shift);
946
947 ret = H_PARAMETER;
948 srcu_idx = srcu_read_lock(&kvm->srcu);
949 mmap_read_lock(kvm->mm);
950
951 start = gfn_to_hva(kvm, gfn);
952 if (kvm_is_error_hva(start))
953 goto out;
954
955 mutex_lock(&kvm->arch.uvmem_lock);
956 /* Fail the page-in request of an already paged-in page */
957 if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
958 goto out_unlock;
959
960 end = start + (1UL << page_shift);
961 vma = find_vma_intersection(kvm->mm, start, end);
962 if (!vma || vma->vm_start > start || vma->vm_end < end)
963 goto out_unlock;
964
965 if (kvmppc_svm_page_in(vma, start, end, gpa, kvm, page_shift,
966 true))
967 goto out_unlock;
968
969 ret = H_SUCCESS;
970
971 out_unlock:
972 mutex_unlock(&kvm->arch.uvmem_lock);
973 out:
974 mmap_read_unlock(kvm->mm);
975 srcu_read_unlock(&kvm->srcu, srcu_idx);
976 return ret;
977 }
978
979
980 /*
981 * Fault handler callback that gets called when HV touches any page that
982 * has been moved to secure memory, we ask UV to give back the page by
983 * issuing UV_PAGE_OUT uvcall.
984 *
985 * This eventually results in dropping of device PFN and the newly
986 * provisioned page/PFN gets populated in QEMU page tables.
987 */
kvmppc_uvmem_migrate_to_ram(struct vm_fault * vmf)988 static vm_fault_t kvmppc_uvmem_migrate_to_ram(struct vm_fault *vmf)
989 {
990 struct kvmppc_uvmem_page_pvt *pvt = vmf->page->zone_device_data;
991
992 if (kvmppc_svm_page_out(vmf->vma, vmf->address,
993 vmf->address + PAGE_SIZE, PAGE_SHIFT,
994 pvt->kvm, pvt->gpa))
995 return VM_FAULT_SIGBUS;
996 else
997 return 0;
998 }
999
1000 /*
1001 * Release the device PFN back to the pool
1002 *
1003 * Gets called when secure GFN tranistions from a secure-PFN
1004 * to a normal PFN during H_SVM_PAGE_OUT.
1005 * Gets called with kvm->arch.uvmem_lock held.
1006 */
kvmppc_uvmem_page_free(struct page * page)1007 static void kvmppc_uvmem_page_free(struct page *page)
1008 {
1009 unsigned long pfn = page_to_pfn(page) -
1010 (kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT);
1011 struct kvmppc_uvmem_page_pvt *pvt;
1012
1013 spin_lock(&kvmppc_uvmem_bitmap_lock);
1014 bitmap_clear(kvmppc_uvmem_bitmap, pfn, 1);
1015 spin_unlock(&kvmppc_uvmem_bitmap_lock);
1016
1017 pvt = page->zone_device_data;
1018 page->zone_device_data = NULL;
1019 if (pvt->remove_gfn)
1020 kvmppc_gfn_remove(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
1021 else
1022 kvmppc_gfn_secure_mem_pfn(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
1023 kfree(pvt);
1024 }
1025
1026 static const struct dev_pagemap_ops kvmppc_uvmem_ops = {
1027 .page_free = kvmppc_uvmem_page_free,
1028 .migrate_to_ram = kvmppc_uvmem_migrate_to_ram,
1029 };
1030
1031 /*
1032 * H_SVM_PAGE_OUT: Move page from secure memory to normal memory.
1033 */
1034 unsigned long
kvmppc_h_svm_page_out(struct kvm * kvm,unsigned long gpa,unsigned long flags,unsigned long page_shift)1035 kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gpa,
1036 unsigned long flags, unsigned long page_shift)
1037 {
1038 unsigned long gfn = gpa >> page_shift;
1039 unsigned long start, end;
1040 struct vm_area_struct *vma;
1041 int srcu_idx;
1042 int ret;
1043
1044 if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
1045 return H_UNSUPPORTED;
1046
1047 if (page_shift != PAGE_SHIFT)
1048 return H_P3;
1049
1050 if (flags)
1051 return H_P2;
1052
1053 ret = H_PARAMETER;
1054 srcu_idx = srcu_read_lock(&kvm->srcu);
1055 mmap_read_lock(kvm->mm);
1056 start = gfn_to_hva(kvm, gfn);
1057 if (kvm_is_error_hva(start))
1058 goto out;
1059
1060 end = start + (1UL << page_shift);
1061 vma = find_vma_intersection(kvm->mm, start, end);
1062 if (!vma || vma->vm_start > start || vma->vm_end < end)
1063 goto out;
1064
1065 if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa))
1066 ret = H_SUCCESS;
1067 out:
1068 mmap_read_unlock(kvm->mm);
1069 srcu_read_unlock(&kvm->srcu, srcu_idx);
1070 return ret;
1071 }
1072
kvmppc_send_page_to_uv(struct kvm * kvm,unsigned long gfn)1073 int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn)
1074 {
1075 unsigned long pfn;
1076 int ret = U_SUCCESS;
1077
1078 pfn = gfn_to_pfn(kvm, gfn);
1079 if (is_error_noslot_pfn(pfn))
1080 return -EFAULT;
1081
1082 mutex_lock(&kvm->arch.uvmem_lock);
1083 if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
1084 goto out;
1085
1086 ret = uv_page_in(kvm->arch.lpid, pfn << PAGE_SHIFT, gfn << PAGE_SHIFT,
1087 0, PAGE_SHIFT);
1088 out:
1089 kvm_release_pfn_clean(pfn);
1090 mutex_unlock(&kvm->arch.uvmem_lock);
1091 return (ret == U_SUCCESS) ? RESUME_GUEST : -EFAULT;
1092 }
1093
kvmppc_uvmem_memslot_create(struct kvm * kvm,const struct kvm_memory_slot * new)1094 int kvmppc_uvmem_memslot_create(struct kvm *kvm, const struct kvm_memory_slot *new)
1095 {
1096 int ret = __kvmppc_uvmem_memslot_create(kvm, new);
1097
1098 if (!ret)
1099 ret = kvmppc_uv_migrate_mem_slot(kvm, new);
1100
1101 return ret;
1102 }
1103
kvmppc_uvmem_memslot_delete(struct kvm * kvm,const struct kvm_memory_slot * old)1104 void kvmppc_uvmem_memslot_delete(struct kvm *kvm, const struct kvm_memory_slot *old)
1105 {
1106 __kvmppc_uvmem_memslot_delete(kvm, old);
1107 }
1108
kvmppc_get_secmem_size(void)1109 static u64 kvmppc_get_secmem_size(void)
1110 {
1111 struct device_node *np;
1112 int i, len;
1113 const __be32 *prop;
1114 u64 size = 0;
1115
1116 /*
1117 * First try the new ibm,secure-memory nodes which supersede the
1118 * secure-memory-ranges property.
1119 * If we found some, no need to read the deprecated ones.
1120 */
1121 for_each_compatible_node(np, NULL, "ibm,secure-memory") {
1122 prop = of_get_property(np, "reg", &len);
1123 if (!prop)
1124 continue;
1125 size += of_read_number(prop + 2, 2);
1126 }
1127 if (size)
1128 return size;
1129
1130 np = of_find_compatible_node(NULL, NULL, "ibm,uv-firmware");
1131 if (!np)
1132 goto out;
1133
1134 prop = of_get_property(np, "secure-memory-ranges", &len);
1135 if (!prop)
1136 goto out_put;
1137
1138 for (i = 0; i < len / (sizeof(*prop) * 4); i++)
1139 size += of_read_number(prop + (i * 4) + 2, 2);
1140
1141 out_put:
1142 of_node_put(np);
1143 out:
1144 return size;
1145 }
1146
kvmppc_uvmem_init(void)1147 int kvmppc_uvmem_init(void)
1148 {
1149 int ret = 0;
1150 unsigned long size;
1151 struct resource *res;
1152 void *addr;
1153 unsigned long pfn_last, pfn_first;
1154
1155 size = kvmppc_get_secmem_size();
1156 if (!size) {
1157 /*
1158 * Don't fail the initialization of kvm-hv module if
1159 * the platform doesn't export ibm,uv-firmware node.
1160 * Let normal guests run on such PEF-disabled platform.
1161 */
1162 pr_info("KVMPPC-UVMEM: No support for secure guests\n");
1163 goto out;
1164 }
1165
1166 res = request_free_mem_region(&iomem_resource, size, "kvmppc_uvmem");
1167 if (IS_ERR(res)) {
1168 ret = PTR_ERR(res);
1169 goto out;
1170 }
1171
1172 kvmppc_uvmem_pgmap.type = MEMORY_DEVICE_PRIVATE;
1173 kvmppc_uvmem_pgmap.range.start = res->start;
1174 kvmppc_uvmem_pgmap.range.end = res->end;
1175 kvmppc_uvmem_pgmap.nr_range = 1;
1176 kvmppc_uvmem_pgmap.ops = &kvmppc_uvmem_ops;
1177 /* just one global instance: */
1178 kvmppc_uvmem_pgmap.owner = &kvmppc_uvmem_pgmap;
1179 addr = memremap_pages(&kvmppc_uvmem_pgmap, NUMA_NO_NODE);
1180 if (IS_ERR(addr)) {
1181 ret = PTR_ERR(addr);
1182 goto out_free_region;
1183 }
1184
1185 pfn_first = res->start >> PAGE_SHIFT;
1186 pfn_last = pfn_first + (resource_size(res) >> PAGE_SHIFT);
1187 kvmppc_uvmem_bitmap = kcalloc(BITS_TO_LONGS(pfn_last - pfn_first),
1188 sizeof(unsigned long), GFP_KERNEL);
1189 if (!kvmppc_uvmem_bitmap) {
1190 ret = -ENOMEM;
1191 goto out_unmap;
1192 }
1193
1194 pr_info("KVMPPC-UVMEM: Secure Memory size 0x%lx\n", size);
1195 return ret;
1196 out_unmap:
1197 memunmap_pages(&kvmppc_uvmem_pgmap);
1198 out_free_region:
1199 release_mem_region(res->start, size);
1200 out:
1201 return ret;
1202 }
1203
kvmppc_uvmem_free(void)1204 void kvmppc_uvmem_free(void)
1205 {
1206 if (!kvmppc_uvmem_bitmap)
1207 return;
1208
1209 memunmap_pages(&kvmppc_uvmem_pgmap);
1210 release_mem_region(kvmppc_uvmem_pgmap.range.start,
1211 range_len(&kvmppc_uvmem_pgmap.range));
1212 kfree(kvmppc_uvmem_bitmap);
1213 }
1214