tools/testing/selftests/kvm/lib/riscv/processor.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * RISC-V code
   4  *
   5  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
   6  */
   7
   8 #include <linux/compiler.h>
   9 #include <assert.h>
  10
  11 #include "kvm_util.h"
  12 #include "../kvm_util_internal.h"
  13 #include "processor.h"
  14
  15 #define DEFAULT_RISCV_GUEST_STACK_VADDR_MIN     0xac0000
  16
  17 static uint64_t page_align(struct kvm_vm *vm, uint64_t v)
  18 {
  19         return (v + vm->page_size) & ~(vm->page_size - 1);
  20 }
  21
  22 static uint64_t pte_addr(struct kvm_vm *vm, uint64_t entry)
  23 {
  24         return ((entry & PGTBL_PTE_ADDR_MASK) >> PGTBL_PTE_ADDR_SHIFT) <<
  25                 PGTBL_PAGE_SIZE_SHIFT;
  26 }
  27
  28 static uint64_t ptrs_per_pte(struct kvm_vm *vm)
  29 {
  30         return PGTBL_PAGE_SIZE / sizeof(uint64_t);
  31 }
  32
  33 static uint64_t pte_index_mask[] = {
  34         PGTBL_L0_INDEX_MASK,
  35         PGTBL_L1_INDEX_MASK,
  36         PGTBL_L2_INDEX_MASK,
  37         PGTBL_L3_INDEX_MASK,
  38 };
  39
  40 static uint32_t pte_index_shift[] = {
  41         PGTBL_L0_INDEX_SHIFT,
  42         PGTBL_L1_INDEX_SHIFT,
  43         PGTBL_L2_INDEX_SHIFT,
  44         PGTBL_L3_INDEX_SHIFT,
  45 };
  46
  47 static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva, int level)
  48 {
  49         TEST_ASSERT(level > -1,
  50                 "Negative page table level (%d) not possible", level);
  51         TEST_ASSERT(level < vm->pgtable_levels,
  52                 "Invalid page table level (%d)", level);
  53
  54         return (gva & pte_index_mask[level]) >> pte_index_shift[level];
  55 }
  56
  57 void virt_pgd_alloc(struct kvm_vm *vm)
  58 {
  59         if (!vm->pgd_created) {
  60                 vm_paddr_t paddr = vm_phy_pages_alloc(vm,
  61                         page_align(vm, ptrs_per_pte(vm) * 8) / vm->page_size,
  62                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);
  63                 vm->pgd = paddr;
  64                 vm->pgd_created = true;
  65         }
  66 }
  67
  68 void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
  69 {
  70         uint64_t *ptep, next_ppn;
  71         int level = vm->pgtable_levels - 1;
  72
  73         TEST_ASSERT((vaddr % vm->page_size) == 0,
  74                 "Virtual address not on page boundary,\n"
  75                 "  vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);
  76         TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
  77                 (vaddr >> vm->page_shift)),
  78                 "Invalid virtual address, vaddr: 0x%lx", vaddr);
  79         TEST_ASSERT((paddr % vm->page_size) == 0,
  80                 "Physical address not on page boundary,\n"
  81                 "  paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);
  82         TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
  83                 "Physical address beyond maximum supported,\n"
  84                 "  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
  85                 paddr, vm->max_gfn, vm->page_size);
  86
  87         ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, vaddr, level) * 8;
  88         if (!*ptep) {
  89                 next_ppn = vm_alloc_page_table(vm) >> PGTBL_PAGE_SIZE_SHIFT;
  90                 *ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) |
  91                         PGTBL_PTE_VALID_MASK;
  92         }
  93         level--;
  94
  95         while (level > -1) {
  96                 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) +
  97                        pte_index(vm, vaddr, level) * 8;
  98                 if (!*ptep && level > 0) {
  99                         next_ppn = vm_alloc_page_table(vm) >>
 100                                    PGTBL_PAGE_SIZE_SHIFT;
 101                         *ptep = (next_ppn << PGTBL_PTE_ADDR_SHIFT) |
 102                                 PGTBL_PTE_VALID_MASK;
 103                 }
 104                 level--;
 105         }
 106
 107         paddr = paddr >> PGTBL_PAGE_SIZE_SHIFT;
 108         *ptep = (paddr << PGTBL_PTE_ADDR_SHIFT) |
 109                 PGTBL_PTE_PERM_MASK | PGTBL_PTE_VALID_MASK;
 110 }
 111
 112 vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
 113 {
 114         uint64_t *ptep;
 115         int level = vm->pgtable_levels - 1;
 116
 117         if (!vm->pgd_created)
 118                 goto unmapped_gva;
 119
 120         ptep = addr_gpa2hva(vm, vm->pgd) + pte_index(vm, gva, level) * 8;
 121         if (!ptep)
 122                 goto unmapped_gva;
 123         level--;
 124
 125         while (level > -1) {
 126                 ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) +
 127                        pte_index(vm, gva, level) * 8;
 128                 if (!ptep)
 129                         goto unmapped_gva;
 130                 level--;
 131         }
 132
 133         return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));
 134
 135 unmapped_gva:
 136         TEST_FAIL("No mapping for vm virtual address gva: 0x%lx level: %d",
 137                   gva, level);
 138         exit(1);
 139 }
 140
 141 static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent,
 142                      uint64_t page, int level)
 143 {
 144 #ifdef DEBUG
 145         static const char *const type[] = { "pte", "pmd", "pud", "p4d"};
 146         uint64_t pte, *ptep;
 147
 148         if (level < 0)
 149                 return;
 150
 151         for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) {
 152                 ptep = addr_gpa2hva(vm, pte);
 153                 if (!*ptep)
 154                         continue;
 155                 fprintf(stream, "%*s%s: %lx: %lx at %p\n", indent, "",
 156                         type[level], pte, *ptep, ptep);
 157                 pte_dump(stream, vm, indent + 1,
 158                          pte_addr(vm, *ptep), level - 1);
 159         }
 160 #endif
 161 }
 162
 163 void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
 164 {
 165         int level = vm->pgtable_levels - 1;
 166         uint64_t pgd, *ptep;
 167
 168         if (!vm->pgd_created)
 169                 return;
 170
 171         for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pte(vm) * 8; pgd += 8) {
 172                 ptep = addr_gpa2hva(vm, pgd);
 173                 if (!*ptep)
 174                         continue;
 175                 fprintf(stream, "%*spgd: %lx: %lx at %p\n", indent, "",
 176                         pgd, *ptep, ptep);
 177                 pte_dump(stream, vm, indent + 1,
 178                          pte_addr(vm, *ptep), level - 1);
 179         }
 180 }
 181
 182 void riscv_vcpu_mmu_setup(struct kvm_vm *vm, int vcpuid)
 183 {
 184         unsigned long satp;
 185
 186         /*
 187          * The RISC-V Sv48 MMU mode supports 56-bit physical address
 188          * for 48-bit virtual address with 4KB last level page size.
 189          */
 190         switch (vm->mode) {
 191         case VM_MODE_P52V48_4K:
 192         case VM_MODE_P48V48_4K:
 193         case VM_MODE_P40V48_4K:
 194                 break;
 195         default:
 196                 TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
 197         }
 198
 199         satp = (vm->pgd >> PGTBL_PAGE_SIZE_SHIFT) & SATP_PPN;
 200         satp |= SATP_MODE_48;
 201
 202         set_reg(vm, vcpuid, RISCV_CSR_REG(satp), satp);
 203 }
 204
 205 void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)
 206 {
 207         struct kvm_riscv_core core;
 208
 209         get_reg(vm, vcpuid, RISCV_CORE_REG(mode), &core.mode);
 210         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.pc), &core.regs.pc);
 211         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.ra), &core.regs.ra);
 212         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.sp), &core.regs.sp);
 213         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.gp), &core.regs.gp);
 214         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.tp), &core.regs.tp);
 215         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.t0), &core.regs.t0);
 216         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.t1), &core.regs.t1);
 217         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.t2), &core.regs.t2);
 218         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s0), &core.regs.s0);
 219         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s1), &core.regs.s1);
 220         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.a0), &core.regs.a0);
 221         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.a1), &core.regs.a1);
 222         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.a2), &core.regs.a2);
 223         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.a3), &core.regs.a3);
 224         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.a4), &core.regs.a4);
 225         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.a5), &core.regs.a5);
 226         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.a6), &core.regs.a6);
 227         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.a7), &core.regs.a7);
 228         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s2), &core.regs.s2);
 229         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s3), &core.regs.s3);
 230         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s4), &core.regs.s4);
 231         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s5), &core.regs.s5);
 232         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s6), &core.regs.s6);
 233         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s7), &core.regs.s7);
 234         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s8), &core.regs.s8);
 235         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s9), &core.regs.s9);
 236         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s10), &core.regs.s10);
 237         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.s11), &core.regs.s11);
 238         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.t3), &core.regs.t3);
 239         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.t4), &core.regs.t4);
 240         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.t5), &core.regs.t5);
 241         get_reg(vm, vcpuid, RISCV_CORE_REG(regs.t6), &core.regs.t6);
 242
 243         fprintf(stream,
 244                 " MODE:  0x%lx\n", core.mode);
 245         fprintf(stream,
 246                 " PC: 0x%016lx   RA: 0x%016lx SP: 0x%016lx GP: 0x%016lx\n",
 247                 core.regs.pc, core.regs.ra, core.regs.sp, core.regs.gp);
 248         fprintf(stream,
 249                 " TP: 0x%016lx   T0: 0x%016lx T1: 0x%016lx T2: 0x%016lx\n",
 250                 core.regs.tp, core.regs.t0, core.regs.t1, core.regs.t2);
 251         fprintf(stream,
 252                 " S0: 0x%016lx   S1: 0x%016lx A0: 0x%016lx A1: 0x%016lx\n",
 253                 core.regs.s0, core.regs.s1, core.regs.a0, core.regs.a1);
 254         fprintf(stream,
 255                 " A2: 0x%016lx   A3: 0x%016lx A4: 0x%016lx A5: 0x%016lx\n",
 256                 core.regs.a2, core.regs.a3, core.regs.a4, core.regs.a5);
 257         fprintf(stream,
 258                 " A6: 0x%016lx   A7: 0x%016lx S2: 0x%016lx S3: 0x%016lx\n",
 259                 core.regs.a6, core.regs.a7, core.regs.s2, core.regs.s3);
 260         fprintf(stream,
 261                 " S4: 0x%016lx   S5: 0x%016lx S6: 0x%016lx S7: 0x%016lx\n",
 262                 core.regs.s4, core.regs.s5, core.regs.s6, core.regs.s7);
 263         fprintf(stream,
 264                 " S8: 0x%016lx   S9: 0x%016lx S10: 0x%016lx S11: 0x%016lx\n",
 265                 core.regs.s8, core.regs.s9, core.regs.s10, core.regs.s11);
 266         fprintf(stream,
 267                 " T3: 0x%016lx   T4: 0x%016lx T5: 0x%016lx T6: 0x%016lx\n",
 268                 core.regs.t3, core.regs.t4, core.regs.t5, core.regs.t6);
 269 }
 270
 271 static void __aligned(16) guest_hang(void)
 272 {
 273         while (1)
 274                 ;
 275 }
 276
 277 void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code)
 278 {
 279         int r;
 280         size_t stack_size = vm->page_size == 4096 ?
 281                                         DEFAULT_STACK_PGS * vm->page_size :
 282                                         vm->page_size;
 283         unsigned long stack_vaddr = vm_vaddr_alloc(vm, stack_size,
 284                                         DEFAULT_RISCV_GUEST_STACK_VADDR_MIN);
 285         unsigned long current_gp = 0;
 286         struct kvm_mp_state mps;
 287
 288         vm_vcpu_add(vm, vcpuid);
 289         riscv_vcpu_mmu_setup(vm, vcpuid);
 290
 291         /*
 292          * With SBI HSM support in KVM RISC-V, all secondary VCPUs are
 293          * powered-off by default so we ensure that all secondary VCPUs
 294          * are powered-on using KVM_SET_MP_STATE ioctl().
 295          */
 296         mps.mp_state = KVM_MP_STATE_RUNNABLE;
 297         r = _vcpu_ioctl(vm, vcpuid, KVM_SET_MP_STATE, &mps);
 298         TEST_ASSERT(!r, "IOCTL KVM_SET_MP_STATE failed (error %d)", r);
 299
 300         /* Setup global pointer of guest to be same as the host */
 301         asm volatile (
 302                 "add %0, gp, zero" : "=r" (current_gp) : : "memory");
 303         set_reg(vm, vcpuid, RISCV_CORE_REG(regs.gp), current_gp);
 304
 305         /* Setup stack pointer and program counter of guest */
 306         set_reg(vm, vcpuid, RISCV_CORE_REG(regs.sp),
 307                 stack_vaddr + stack_size);
 308         set_reg(vm, vcpuid, RISCV_CORE_REG(regs.pc),
 309                 (unsigned long)guest_code);
 310
 311         /* Setup default exception vector of guest */
 312         set_reg(vm, vcpuid, RISCV_CSR_REG(stvec),
 313                 (unsigned long)guest_hang);
 314 }
 315
 316 void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...)
 317 {
 318         va_list ap;
 319         uint64_t id = RISCV_CORE_REG(regs.a0);
 320         int i;
 321
 322         TEST_ASSERT(num >= 1 && num <= 8, "Unsupported number of args,\n"
 323                     "  num: %u\n", num);
 324
 325         va_start(ap, num);
 326
 327         for (i = 0; i < num; i++) {
 328                 switch (i) {
 329                 case 0:
 330                         id = RISCV_CORE_REG(regs.a0);
 331                         break;
 332                 case 1:
 333                         id = RISCV_CORE_REG(regs.a1);
 334                         break;
 335                 case 2:
 336                         id = RISCV_CORE_REG(regs.a2);
 337                         break;
 338                 case 3:
 339                         id = RISCV_CORE_REG(regs.a3);
 340                         break;
 341                 case 4:
 342                         id = RISCV_CORE_REG(regs.a4);
 343                         break;
 344                 case 5:
 345                         id = RISCV_CORE_REG(regs.a5);
 346                         break;
 347                 case 6:
 348                         id = RISCV_CORE_REG(regs.a6);
 349                         break;
 350                 case 7:
 351                         id = RISCV_CORE_REG(regs.a7);
 352                         break;
 353                 };
 354                 set_reg(vm, vcpuid, id, va_arg(ap, uint64_t));
 355         }
 356
 357         va_end(ap);
 358 }
 359
 360 void assert_on_unhandled_exception(struct kvm_vm *vm, uint32_t vcpuid)
 361 {
 362 }