1 // SPDX-License-Identifier: GPL-2.0
3 * Common Ultravisor functions and initialization
5 * Copyright IBM Corp. 2019, 2020
7 #define KMSG_COMPONENT "prot_virt"
8 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
10 #include <linux/kernel.h>
11 #include <linux/types.h>
12 #include <linux/sizes.h>
13 #include <linux/bitmap.h>
14 #include <linux/memblock.h>
15 #include <linux/pagemap.h>
16 #include <linux/swap.h>
17 #include <asm/facility.h>
18 #include <asm/sections.h>
21 /* the bootdata_preserved fields come from ones in arch/s390/boot/uv.c */
22 #ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
23 int __bootdata_preserved(prot_virt_guest);
26 struct uv_info __bootdata_preserved(uv_info);
28 #if IS_ENABLED(CONFIG_KVM)
29 int __bootdata_preserved(prot_virt_host);
30 EXPORT_SYMBOL(prot_virt_host);
31 EXPORT_SYMBOL(uv_info);
33 static int __init uv_init(unsigned long stor_base, unsigned long stor_len)
35 struct uv_cb_init uvcb = {
36 .header.cmd = UVC_CMD_INIT_UV,
37 .header.len = sizeof(uvcb),
38 .stor_origin = stor_base,
42 if (uv_call(0, (uint64_t)&uvcb)) {
43 pr_err("Ultravisor init failed with rc: 0x%x rrc: 0%x\n",
44 uvcb.header.rc, uvcb.header.rrc);
50 void __init setup_uv(void)
52 unsigned long uv_stor_base;
54 if (!is_prot_virt_host())
57 uv_stor_base = (unsigned long)memblock_alloc_try_nid(
58 uv_info.uv_base_stor_len, SZ_1M, SZ_2G,
59 MEMBLOCK_ALLOC_ACCESSIBLE, NUMA_NO_NODE);
61 pr_warn("Failed to reserve %lu bytes for ultravisor base storage\n",
62 uv_info.uv_base_stor_len);
66 if (uv_init(uv_stor_base, uv_info.uv_base_stor_len)) {
67 memblock_free(uv_stor_base, uv_info.uv_base_stor_len);
71 pr_info("Reserving %luMB as ultravisor base storage\n",
72 uv_info.uv_base_stor_len >> 20);
75 pr_info("Disabling support for protected virtualization");
80 * Requests the Ultravisor to pin the page in the shared state. This will
81 * cause an intercept when the guest attempts to unshare the pinned page.
83 static int uv_pin_shared(unsigned long paddr)
85 struct uv_cb_cfs uvcb = {
86 .header.cmd = UVC_CMD_PIN_PAGE_SHARED,
87 .header.len = sizeof(uvcb),
91 if (uv_call(0, (u64)&uvcb))
97 * Requests the Ultravisor to destroy a guest page and make it
98 * accessible to the host. The destroy clears the page instead of
101 * @paddr: Absolute host address of page to be destroyed
103 int uv_destroy_page(unsigned long paddr)
105 struct uv_cb_cfs uvcb = {
106 .header.cmd = UVC_CMD_DESTR_SEC_STOR,
107 .header.len = sizeof(uvcb),
111 if (uv_call(0, (u64)&uvcb)) {
113 * Older firmware uses 107/d as an indication of a non secure
114 * page. Let us emulate the newer variant (no-op).
116 if (uvcb.header.rc == 0x107 && uvcb.header.rrc == 0xd)
124 * Requests the Ultravisor to encrypt a guest page and make it
125 * accessible to the host for paging (export).
127 * @paddr: Absolute host address of page to be exported
129 int uv_convert_from_secure(unsigned long paddr)
131 struct uv_cb_cfs uvcb = {
132 .header.cmd = UVC_CMD_CONV_FROM_SEC_STOR,
133 .header.len = sizeof(uvcb),
137 if (uv_call(0, (u64)&uvcb))
143 * Calculate the expected ref_count for a page that would otherwise have no
144 * further pins. This was cribbed from similar functions in other places in
145 * the kernel, but with some slight modifications. We know that a secure
146 * page can not be a huge page for example.
148 static int expected_page_refs(struct page *page)
152 res = page_mapcount(page);
153 if (PageSwapCache(page)) {
155 } else if (page_mapping(page)) {
157 if (page_has_private(page))
163 static int make_secure_pte(pte_t *ptep, unsigned long addr,
164 struct page *exp_page, struct uv_cb_header *uvcb)
166 pte_t entry = READ_ONCE(*ptep);
168 int expected, rc = 0;
170 if (!pte_present(entry))
172 if (pte_val(entry) & _PAGE_INVALID)
175 page = pte_page(entry);
176 if (page != exp_page)
178 if (PageWriteback(page))
180 expected = expected_page_refs(page);
181 if (!page_ref_freeze(page, expected))
183 set_bit(PG_arch_1, &page->flags);
184 rc = uv_call(0, (u64)uvcb);
185 page_ref_unfreeze(page, expected);
186 /* Return -ENXIO if the page was not mapped, -EINVAL otherwise */
188 rc = uvcb->rc == 0x10a ? -ENXIO : -EINVAL;
193 * Requests the Ultravisor to make a page accessible to a guest.
194 * If it's brought in the first time, it will be cleared. If
195 * it has been exported before, it will be decrypted and integrity
198 int gmap_make_secure(struct gmap *gmap, unsigned long gaddr, void *uvcb)
200 struct vm_area_struct *vma;
201 bool local_drain = false;
210 mmap_read_lock(gmap->mm);
212 uaddr = __gmap_translate(gmap, gaddr);
213 if (IS_ERR_VALUE(uaddr))
215 vma = find_vma(gmap->mm, uaddr);
219 * Secure pages cannot be huge and userspace should not combine both.
220 * In case userspace does it anyway this will result in an -EFAULT for
221 * the unpack. The guest is thus never reaching secure mode. If
222 * userspace is playing dirty tricky with mapping huge pages later
223 * on this will result in a segmentation fault.
225 if (is_vm_hugetlb_page(vma))
229 page = follow_page(vma, uaddr, FOLL_WRITE);
230 if (IS_ERR_OR_NULL(page))
234 ptep = get_locked_pte(gmap->mm, uaddr, &ptelock);
235 rc = make_secure_pte(ptep, uaddr, page, uvcb);
236 pte_unmap_unlock(ptep, ptelock);
239 mmap_read_unlock(gmap->mm);
242 wait_on_page_writeback(page);
243 } else if (rc == -EBUSY) {
245 * If we have tried a local drain and the page refcount
246 * still does not match our expected safe value, try with a
247 * system wide drain. This is needed if the pagevecs holding
248 * the page are on a different CPU.
252 /* We give up here, and let the caller try again */
256 * We are here if the page refcount does not match the
257 * expected safe value. The main culprits are usually
258 * pagevecs. With lru_add_drain() we drain the pagevecs
259 * on the local CPU so that hopefully the refcount will
260 * reach the expected safe value.
264 /* And now we try again immediately after draining */
266 } else if (rc == -ENXIO) {
267 if (gmap_fault(gmap, gaddr, FAULT_FLAG_WRITE))
273 EXPORT_SYMBOL_GPL(gmap_make_secure);
275 int gmap_convert_to_secure(struct gmap *gmap, unsigned long gaddr)
277 struct uv_cb_cts uvcb = {
278 .header.cmd = UVC_CMD_CONV_TO_SEC_STOR,
279 .header.len = sizeof(uvcb),
280 .guest_handle = gmap->guest_handle,
284 return gmap_make_secure(gmap, gaddr, &uvcb);
286 EXPORT_SYMBOL_GPL(gmap_convert_to_secure);
289 * To be called with the page locked or with an extra reference! This will
290 * prevent gmap_make_secure from touching the page concurrently. Having 2
291 * parallel make_page_accessible is fine, as the UV calls will become a
292 * no-op if the page is already exported.
294 int arch_make_page_accessible(struct page *page)
298 /* Hugepage cannot be protected, so nothing to do */
303 * PG_arch_1 is used in 3 places:
304 * 1. for kernel page tables during early boot
305 * 2. for storage keys of huge pages and KVM
306 * 3. As an indication that this page might be secure. This can
307 * overindicate, e.g. we set the bit before calling
309 * As secure pages are never huge, all 3 variants can co-exists.
311 if (!test_bit(PG_arch_1, &page->flags))
314 rc = uv_pin_shared(page_to_phys(page));
316 clear_bit(PG_arch_1, &page->flags);
320 rc = uv_convert_from_secure(page_to_phys(page));
322 clear_bit(PG_arch_1, &page->flags);
328 EXPORT_SYMBOL_GPL(arch_make_page_accessible);
332 #if defined(CONFIG_PROTECTED_VIRTUALIZATION_GUEST) || IS_ENABLED(CONFIG_KVM)
333 static ssize_t uv_query_facilities(struct kobject *kobj,
334 struct kobj_attribute *attr, char *page)
336 return scnprintf(page, PAGE_SIZE, "%lx\n%lx\n%lx\n%lx\n",
337 uv_info.inst_calls_list[0],
338 uv_info.inst_calls_list[1],
339 uv_info.inst_calls_list[2],
340 uv_info.inst_calls_list[3]);
343 static struct kobj_attribute uv_query_facilities_attr =
344 __ATTR(facilities, 0444, uv_query_facilities, NULL);
346 static ssize_t uv_query_feature_indications(struct kobject *kobj,
347 struct kobj_attribute *attr, char *buf)
349 return sysfs_emit(buf, "%lx\n", uv_info.uv_feature_indications);
352 static struct kobj_attribute uv_query_feature_indications_attr =
353 __ATTR(feature_indications, 0444, uv_query_feature_indications, NULL);
355 static ssize_t uv_query_max_guest_cpus(struct kobject *kobj,
356 struct kobj_attribute *attr, char *page)
358 return scnprintf(page, PAGE_SIZE, "%d\n",
359 uv_info.max_guest_cpu_id + 1);
362 static struct kobj_attribute uv_query_max_guest_cpus_attr =
363 __ATTR(max_cpus, 0444, uv_query_max_guest_cpus, NULL);
365 static ssize_t uv_query_max_guest_vms(struct kobject *kobj,
366 struct kobj_attribute *attr, char *page)
368 return scnprintf(page, PAGE_SIZE, "%d\n",
369 uv_info.max_num_sec_conf);
372 static struct kobj_attribute uv_query_max_guest_vms_attr =
373 __ATTR(max_guests, 0444, uv_query_max_guest_vms, NULL);
375 static ssize_t uv_query_max_guest_addr(struct kobject *kobj,
376 struct kobj_attribute *attr, char *page)
378 return scnprintf(page, PAGE_SIZE, "%lx\n",
379 uv_info.max_sec_stor_addr);
382 static struct kobj_attribute uv_query_max_guest_addr_attr =
383 __ATTR(max_address, 0444, uv_query_max_guest_addr, NULL);
385 static struct attribute *uv_query_attrs[] = {
386 &uv_query_facilities_attr.attr,
387 &uv_query_feature_indications_attr.attr,
388 &uv_query_max_guest_cpus_attr.attr,
389 &uv_query_max_guest_vms_attr.attr,
390 &uv_query_max_guest_addr_attr.attr,
394 static struct attribute_group uv_query_attr_group = {
395 .attrs = uv_query_attrs,
398 static ssize_t uv_is_prot_virt_guest(struct kobject *kobj,
399 struct kobj_attribute *attr, char *page)
403 #ifdef CONFIG_PROTECTED_VIRTUALIZATION_GUEST
404 val = prot_virt_guest;
406 return scnprintf(page, PAGE_SIZE, "%d\n", val);
409 static ssize_t uv_is_prot_virt_host(struct kobject *kobj,
410 struct kobj_attribute *attr, char *page)
414 #if IS_ENABLED(CONFIG_KVM)
415 val = prot_virt_host;
418 return scnprintf(page, PAGE_SIZE, "%d\n", val);
421 static struct kobj_attribute uv_prot_virt_guest =
422 __ATTR(prot_virt_guest, 0444, uv_is_prot_virt_guest, NULL);
424 static struct kobj_attribute uv_prot_virt_host =
425 __ATTR(prot_virt_host, 0444, uv_is_prot_virt_host, NULL);
427 static const struct attribute *uv_prot_virt_attrs[] = {
428 &uv_prot_virt_guest.attr,
429 &uv_prot_virt_host.attr,
433 static struct kset *uv_query_kset;
434 static struct kobject *uv_kobj;
436 static int __init uv_info_init(void)
440 if (!test_facility(158))
443 uv_kobj = kobject_create_and_add("uv", firmware_kobj);
447 rc = sysfs_create_files(uv_kobj, uv_prot_virt_attrs);
451 uv_query_kset = kset_create_and_add("query", NULL, uv_kobj);
452 if (!uv_query_kset) {
457 rc = sysfs_create_group(&uv_query_kset->kobj, &uv_query_attr_group);
461 kset_unregister(uv_query_kset);
463 sysfs_remove_files(uv_kobj, uv_prot_virt_attrs);
465 kobject_del(uv_kobj);
466 kobject_put(uv_kobj);
469 device_initcall(uv_info_init);