arch/arm64/kvm/hyp/nvhe/mm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 Google LLC
   4  * Author: Quentin Perret <qperret@google.com>
   5  */
   6
   7 #include <linux/kvm_host.h>
   8 #include <asm/kvm_hyp.h>
   9 #include <asm/kvm_mmu.h>
  10 #include <asm/kvm_pgtable.h>
  11 #include <asm/spectre.h>
  12
  13 #include <nvhe/early_alloc.h>
  14 #include <nvhe/gfp.h>
  15 #include <nvhe/memory.h>
  16 #include <nvhe/mm.h>
  17 #include <nvhe/spinlock.h>
  18
  19 struct kvm_pgtable pkvm_pgtable;
  20 hyp_spinlock_t pkvm_pgd_lock;
  21 u64 __io_map_base;
  22
  23 struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS];
  24 unsigned int hyp_memblock_nr;
  25
  26 int __pkvm_create_mappings(unsigned long start, unsigned long size,
  27                           unsigned long phys, enum kvm_pgtable_prot prot)
  28 {
  29         int err;
  30
  31         hyp_spin_lock(&pkvm_pgd_lock);
  32         err = kvm_pgtable_hyp_map(&pkvm_pgtable, start, size, phys, prot);
  33         hyp_spin_unlock(&pkvm_pgd_lock);
  34
  35         return err;
  36 }
  37
  38 unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
  39                                             enum kvm_pgtable_prot prot)
  40 {
  41         unsigned long addr;
  42         int err;
  43
  44         hyp_spin_lock(&pkvm_pgd_lock);
  45
  46         size = PAGE_ALIGN(size + offset_in_page(phys));
  47         addr = __io_map_base;
  48         __io_map_base += size;
  49
  50         /* Are we overflowing on the vmemmap ? */
  51         if (__io_map_base > __hyp_vmemmap) {
  52                 __io_map_base -= size;
  53                 addr = (unsigned long)ERR_PTR(-ENOMEM);
  54                 goto out;
  55         }
  56
  57         err = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, prot);
  58         if (err) {
  59                 addr = (unsigned long)ERR_PTR(err);
  60                 goto out;
  61         }
  62
  63         addr = addr + offset_in_page(phys);
  64 out:
  65         hyp_spin_unlock(&pkvm_pgd_lock);
  66
  67         return addr;
  68 }
  69
  70 int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
  71 {
  72         unsigned long start = (unsigned long)from;
  73         unsigned long end = (unsigned long)to;
  74         unsigned long virt_addr;
  75         phys_addr_t phys;
  76
  77         start = start & PAGE_MASK;
  78         end = PAGE_ALIGN(end);
  79
  80         for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
  81                 int err;
  82
  83                 phys = hyp_virt_to_phys((void *)virt_addr);
  84                 err = __pkvm_create_mappings(virt_addr, PAGE_SIZE, phys, prot);
  85                 if (err)
  86                         return err;
  87         }
  88
  89         return 0;
  90 }
  91
  92 int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back)
  93 {
  94         unsigned long start, end;
  95
  96         hyp_vmemmap_range(phys, size, &start, &end);
  97
  98         return __pkvm_create_mappings(start, end - start, back, PAGE_HYP);
  99 }
 100
 101 static void *__hyp_bp_vect_base;
 102 int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot)
 103 {
 104         void *vector;
 105
 106         switch (slot) {
 107         case HYP_VECTOR_DIRECT: {
 108                 vector = __kvm_hyp_vector;
 109                 break;
 110         }
 111         case HYP_VECTOR_SPECTRE_DIRECT: {
 112                 vector = __bp_harden_hyp_vecs;
 113                 break;
 114         }
 115         case HYP_VECTOR_INDIRECT:
 116         case HYP_VECTOR_SPECTRE_INDIRECT: {
 117                 vector = (void *)__hyp_bp_vect_base;
 118                 break;
 119         }
 120         default:
 121                 return -EINVAL;
 122         }
 123
 124         vector = __kvm_vector_slot2addr(vector, slot);
 125         *this_cpu_ptr(&kvm_hyp_vector) = (unsigned long)vector;
 126
 127         return 0;
 128 }
 129
 130 int hyp_map_vectors(void)
 131 {
 132         phys_addr_t phys;
 133         void *bp_base;
 134
 135         if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))
 136                 return 0;
 137
 138         phys = __hyp_pa(__bp_harden_hyp_vecs);
 139         bp_base = (void *)__pkvm_create_private_mapping(phys,
 140                                                         __BP_HARDEN_HYP_VECS_SZ,
 141                                                         PAGE_HYP_EXEC);
 142         if (IS_ERR_OR_NULL(bp_base))
 143                 return PTR_ERR(bp_base);
 144
 145         __hyp_bp_vect_base = bp_base;
 146
 147         return 0;
 148 }
 149
 150 int hyp_create_idmap(u32 hyp_va_bits)
 151 {
 152         unsigned long start, end;
 153
 154         start = hyp_virt_to_phys((void *)__hyp_idmap_text_start);
 155         start = ALIGN_DOWN(start, PAGE_SIZE);
 156
 157         end = hyp_virt_to_phys((void *)__hyp_idmap_text_end);
 158         end = ALIGN(end, PAGE_SIZE);
 159
 160         /*
 161          * One half of the VA space is reserved to linearly map portions of
 162          * memory -- see va_layout.c for more details. The other half of the VA
 163          * space contains the trampoline page, and needs some care. Split that
 164          * second half in two and find the quarter of VA space not conflicting
 165          * with the idmap to place the IOs and the vmemmap. IOs use the lower
 166          * half of the quarter and the vmemmap the upper half.
 167          */
 168         __io_map_base = start & BIT(hyp_va_bits - 2);
 169         __io_map_base ^= BIT(hyp_va_bits - 2);
 170         __hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3);
 171
 172         return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC);
 173 }