1 // SPDX-License-Identifier: GPL-2.0-only
3 * tools/testing/selftests/kvm/lib/kvm_util.c
5 * Copyright (C) 2018, Google LLC.
8 #define _GNU_SOURCE /* for program_invocation_name */
11 #include "kvm_util_internal.h"
12 #include "processor.h"
16 #include <sys/types.h>
19 #include <linux/kernel.h>
21 #define KVM_UTIL_PGS_PER_HUGEPG 512
22 #define KVM_UTIL_MIN_PFN 2
24 static int vcpu_mmap_sz(void);
26 /* Aligns x up to the next multiple of size. Size must be a power of 2. */
27 static void *align(void *x, size_t size)
29 size_t mask = size - 1;
30 TEST_ASSERT(size != 0 && !(size & (size - 1)),
31 "size not a power of 2: %lu", size);
32 return (void *) (((size_t) x + mask) & ~mask);
44 * On success, the Value corresponding to the capability (KVM_CAP_*)
45 * specified by the value of cap. On failure a TEST_ASSERT failure
48 * Looks up and returns the value corresponding to the capability
49 * (KVM_CAP_*) given by cap.
51 int kvm_check_cap(long cap)
56 kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
60 ret = ioctl(kvm_fd, KVM_CHECK_EXTENSION, cap);
61 TEST_ASSERT(ret != -1, "KVM_CHECK_EXTENSION IOCTL failed,\n"
62 " rc: %i errno: %i", ret, errno);
69 /* VM Enable Capability
72 * vm - Virtual Machine
77 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
79 * Enables a capability (KVM_CAP_*) on the VM.
81 int vm_enable_cap(struct kvm_vm *vm, struct kvm_enable_cap *cap)
85 ret = ioctl(vm->fd, KVM_ENABLE_CAP, cap);
86 TEST_ASSERT(ret == 0, "KVM_ENABLE_CAP IOCTL failed,\n"
87 " rc: %i errno: %i", ret, errno);
92 /* VCPU Enable Capability
95 * vm - Virtual Machine
101 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
103 * Enables a capability (KVM_CAP_*) on the VCPU.
105 int vcpu_enable_cap(struct kvm_vm *vm, uint32_t vcpu_id,
106 struct kvm_enable_cap *cap)
108 struct vcpu *vcpu = vcpu_find(vm, vcpu_id);
111 TEST_ASSERT(vcpu, "cannot find vcpu %d", vcpu_id);
113 r = ioctl(vcpu->fd, KVM_ENABLE_CAP, cap);
114 TEST_ASSERT(!r, "KVM_ENABLE_CAP vCPU ioctl failed,\n"
115 " rc: %i, errno: %i", r, errno);
120 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
122 struct kvm_enable_cap cap = { 0 };
124 cap.cap = KVM_CAP_DIRTY_LOG_RING;
125 cap.args[0] = ring_size;
126 vm_enable_cap(vm, &cap);
127 vm->dirty_ring_size = ring_size;
130 static void vm_open(struct kvm_vm *vm, int perm)
132 vm->kvm_fd = open(KVM_DEV_PATH, perm);
136 if (!kvm_check_cap(KVM_CAP_IMMEDIATE_EXIT)) {
137 print_skip("immediate_exit not available");
141 vm->fd = ioctl(vm->kvm_fd, KVM_CREATE_VM, vm->type);
142 TEST_ASSERT(vm->fd >= 0, "KVM_CREATE_VM ioctl failed, "
143 "rc: %i errno: %i", vm->fd, errno);
146 const char * const vm_guest_mode_string[] = {
147 "PA-bits:52, VA-bits:48, 4K pages",
148 "PA-bits:52, VA-bits:48, 64K pages",
149 "PA-bits:48, VA-bits:48, 4K pages",
150 "PA-bits:48, VA-bits:48, 64K pages",
151 "PA-bits:40, VA-bits:48, 4K pages",
152 "PA-bits:40, VA-bits:48, 64K pages",
153 "PA-bits:ANY, VA-bits:48, 4K pages",
155 _Static_assert(sizeof(vm_guest_mode_string)/sizeof(char *) == NUM_VM_MODES,
156 "Missing new mode strings?");
158 const struct vm_guest_mode_params vm_guest_mode_params[] = {
159 { 52, 48, 0x1000, 12 },
160 { 52, 48, 0x10000, 16 },
161 { 48, 48, 0x1000, 12 },
162 { 48, 48, 0x10000, 16 },
163 { 40, 48, 0x1000, 12 },
164 { 40, 48, 0x10000, 16 },
165 { 0, 0, 0x1000, 12 },
167 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
168 "Missing new mode params?");
174 * mode - VM Mode (e.g. VM_MODE_P52V48_4K)
175 * phy_pages - Physical memory pages
181 * Pointer to opaque structure that describes the created VM.
183 * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
184 * When phy_pages is non-zero, a memory region of phy_pages physical pages
185 * is created and mapped starting at guest physical address 0. The file
186 * descriptor to control the created VM is created with the permissions
187 * given by perm (e.g. O_RDWR).
189 struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
193 pr_debug("%s: mode='%s' pages='%ld' perm='%d'\n", __func__,
194 vm_guest_mode_string(mode), phy_pages, perm);
196 vm = calloc(1, sizeof(*vm));
197 TEST_ASSERT(vm != NULL, "Insufficient Memory");
199 INIT_LIST_HEAD(&vm->vcpus);
200 INIT_LIST_HEAD(&vm->userspace_mem_regions);
205 vm->pa_bits = vm_guest_mode_params[mode].pa_bits;
206 vm->va_bits = vm_guest_mode_params[mode].va_bits;
207 vm->page_size = vm_guest_mode_params[mode].page_size;
208 vm->page_shift = vm_guest_mode_params[mode].page_shift;
210 /* Setup mode specific traits. */
212 case VM_MODE_P52V48_4K:
213 vm->pgtable_levels = 4;
215 case VM_MODE_P52V48_64K:
216 vm->pgtable_levels = 3;
218 case VM_MODE_P48V48_4K:
219 vm->pgtable_levels = 4;
221 case VM_MODE_P48V48_64K:
222 vm->pgtable_levels = 3;
224 case VM_MODE_P40V48_4K:
225 vm->pgtable_levels = 4;
227 case VM_MODE_P40V48_64K:
228 vm->pgtable_levels = 3;
230 case VM_MODE_PXXV48_4K:
232 kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);
234 * Ignore KVM support for 5-level paging (vm->va_bits == 57),
235 * it doesn't take effect unless a CR4.LA57 is set, which it
236 * isn't for this VM_MODE.
238 TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57,
239 "Linear address width (%d bits) not supported",
241 pr_debug("Guest physical address width detected: %d\n",
243 vm->pgtable_levels = 4;
246 TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");
250 TEST_FAIL("Unknown guest mode, mode: 0x%x", mode);
254 if (vm->pa_bits != 40)
255 vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
260 /* Limit to VA-bit canonical virtual addresses. */
261 vm->vpages_valid = sparsebit_alloc();
262 sparsebit_set_num(vm->vpages_valid,
263 0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
264 sparsebit_set_num(vm->vpages_valid,
265 (~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
266 (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
268 /* Limit physical addresses to PA-bits. */
269 vm->max_gfn = ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
271 /* Allocate and setup memory for guest. */
272 vm->vpages_mapped = sparsebit_alloc();
274 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
280 struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus,
281 uint64_t extra_mem_pages, uint32_t num_percpu_pages,
282 void *guest_code, uint32_t vcpuids[])
284 /* The maximum page table size for a memory region will be when the
285 * smallest pages are used. Considering each page contains x page
286 * table descriptors, the total extra size for page tables (for extra
287 * N pages) will be: N/x+N/x^2+N/x^3+... which is definitely smaller
290 uint64_t vcpu_pages = (DEFAULT_STACK_PGS + num_percpu_pages) * nr_vcpus;
291 uint64_t extra_pg_pages = (extra_mem_pages + vcpu_pages) / PTES_PER_MIN_PAGE * 2;
292 uint64_t pages = DEFAULT_GUEST_PHY_PAGES + vcpu_pages + extra_pg_pages;
296 TEST_ASSERT(nr_vcpus <= kvm_check_cap(KVM_CAP_MAX_VCPUS),
297 "nr_vcpus = %d too large for host, max-vcpus = %d",
298 nr_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS));
300 pages = vm_adjust_num_guest_pages(mode, pages);
301 vm = vm_create(mode, pages, O_RDWR);
303 kvm_vm_elf_load(vm, program_invocation_name, 0, 0);
306 vm_create_irqchip(vm);
309 for (i = 0; i < nr_vcpus; ++i) {
310 uint32_t vcpuid = vcpuids ? vcpuids[i] : i;
312 vm_vcpu_add_default(vm, vcpuid, guest_code);
315 vcpu_set_cpuid(vm, vcpuid, kvm_get_supported_cpuid());
322 struct kvm_vm *vm_create_default_with_vcpus(uint32_t nr_vcpus, uint64_t extra_mem_pages,
323 uint32_t num_percpu_pages, void *guest_code,
326 return vm_create_with_vcpus(VM_MODE_DEFAULT, nr_vcpus, extra_mem_pages,
327 num_percpu_pages, guest_code, vcpuids);
330 struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages,
333 return vm_create_default_with_vcpus(1, extra_mem_pages, 0, guest_code,
334 (uint32_t []){ vcpuid });
341 * vm - VM that has been released before
346 * Reopens the file descriptors associated to the VM and reinstates the
347 * global state, such as the irqchip and the memory regions that are mapped
350 void kvm_vm_restart(struct kvm_vm *vmp, int perm)
352 struct userspace_mem_region *region;
355 if (vmp->has_irqchip)
356 vm_create_irqchip(vmp);
358 list_for_each_entry(region, &vmp->userspace_mem_regions, list) {
359 int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
360 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
361 " rc: %i errno: %i\n"
362 " slot: %u flags: 0x%x\n"
363 " guest_phys_addr: 0x%llx size: 0x%llx",
364 ret, errno, region->region.slot,
365 region->region.flags,
366 region->region.guest_phys_addr,
367 region->region.memory_size);
371 void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
373 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
376 ret = ioctl(vm->fd, KVM_GET_DIRTY_LOG, &args);
377 TEST_ASSERT(ret == 0, "%s: KVM_GET_DIRTY_LOG failed: %s",
378 __func__, strerror(-ret));
381 void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
382 uint64_t first_page, uint32_t num_pages)
384 struct kvm_clear_dirty_log args = { .dirty_bitmap = log, .slot = slot,
385 .first_page = first_page,
386 .num_pages = num_pages };
389 ret = ioctl(vm->fd, KVM_CLEAR_DIRTY_LOG, &args);
390 TEST_ASSERT(ret == 0, "%s: KVM_CLEAR_DIRTY_LOG failed: %s",
391 __func__, strerror(-ret));
394 uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
396 return ioctl(vm->fd, KVM_RESET_DIRTY_RINGS);
400 * Userspace Memory Region Find
403 * vm - Virtual Machine
404 * start - Starting VM physical address
405 * end - Ending VM physical address, inclusive.
410 * Pointer to overlapping region, NULL if no such region.
412 * Searches for a region with any physical memory that overlaps with
413 * any portion of the guest physical addresses from start to end
414 * inclusive. If multiple overlapping regions exist, a pointer to any
415 * of the regions is returned. Null is returned only when no overlapping
418 static struct userspace_mem_region *
419 userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
421 struct userspace_mem_region *region;
423 list_for_each_entry(region, &vm->userspace_mem_regions, list) {
424 uint64_t existing_start = region->region.guest_phys_addr;
425 uint64_t existing_end = region->region.guest_phys_addr
426 + region->region.memory_size - 1;
427 if (start <= existing_end && end >= existing_start)
435 * KVM Userspace Memory Region Find
438 * vm - Virtual Machine
439 * start - Starting VM physical address
440 * end - Ending VM physical address, inclusive.
445 * Pointer to overlapping region, NULL if no such region.
447 * Public interface to userspace_mem_region_find. Allows tests to look up
448 * the memslot datastructure for a given range of guest physical memory.
450 struct kvm_userspace_memory_region *
451 kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
454 struct userspace_mem_region *region;
456 region = userspace_mem_region_find(vm, start, end);
460 return ®ion->region;
467 * vm - Virtual Machine
473 * Pointer to VCPU structure
475 * Locates a vcpu structure that describes the VCPU specified by vcpuid and
476 * returns a pointer to it. Returns NULL if the VM doesn't contain a VCPU
477 * for the specified vcpuid.
479 struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid)
483 list_for_each_entry(vcpu, &vm->vcpus, list) {
484 if (vcpu->id == vcpuid)
495 * vcpu - VCPU to remove
499 * Return: None, TEST_ASSERT failures for all error conditions
501 * Removes a vCPU from a VM and frees its resources.
503 static void vm_vcpu_rm(struct kvm_vm *vm, struct vcpu *vcpu)
507 if (vcpu->dirty_gfns) {
508 ret = munmap(vcpu->dirty_gfns, vm->dirty_ring_size);
509 TEST_ASSERT(ret == 0, "munmap of VCPU dirty ring failed, "
510 "rc: %i errno: %i", ret, errno);
511 vcpu->dirty_gfns = NULL;
514 ret = munmap(vcpu->state, vcpu_mmap_sz());
515 TEST_ASSERT(ret == 0, "munmap of VCPU fd failed, rc: %i "
516 "errno: %i", ret, errno);
518 TEST_ASSERT(ret == 0, "Close of VCPU fd failed, rc: %i "
519 "errno: %i", ret, errno);
521 list_del(&vcpu->list);
525 void kvm_vm_release(struct kvm_vm *vmp)
527 struct vcpu *vcpu, *tmp;
530 list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)
531 vm_vcpu_rm(vmp, vcpu);
533 ret = close(vmp->fd);
534 TEST_ASSERT(ret == 0, "Close of vm fd failed,\n"
535 " vmp->fd: %i rc: %i errno: %i", vmp->fd, ret, errno);
538 TEST_ASSERT(ret == 0, "Close of /dev/kvm fd failed,\n"
539 " vmp->kvm_fd: %i rc: %i errno: %i", vmp->kvm_fd, ret, errno);
542 static void __vm_mem_region_delete(struct kvm_vm *vm,
543 struct userspace_mem_region *region)
547 list_del(®ion->list);
549 region->region.memory_size = 0;
550 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
551 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, "
552 "rc: %i errno: %i", ret, errno);
554 sparsebit_free(®ion->unused_phy_pages);
555 ret = munmap(region->mmap_start, region->mmap_size);
556 TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i", ret, errno);
562 * Destroys and frees the VM pointed to by vmp.
564 void kvm_vm_free(struct kvm_vm *vmp)
566 struct userspace_mem_region *region, *tmp;
571 /* Free userspace_mem_regions. */
572 list_for_each_entry_safe(region, tmp, &vmp->userspace_mem_regions, list)
573 __vm_mem_region_delete(vmp, region);
575 /* Free sparsebit arrays. */
576 sparsebit_free(&vmp->vpages_valid);
577 sparsebit_free(&vmp->vpages_mapped);
581 /* Free the structure describing the VM. */
586 * Memory Compare, host virtual to guest virtual
589 * hva - Starting host virtual address
590 * vm - Virtual Machine
591 * gva - Starting guest virtual address
592 * len - number of bytes to compare
596 * Input/Output Args: None
599 * Returns 0 if the bytes starting at hva for a length of len
600 * are equal the guest virtual bytes starting at gva. Returns
601 * a value < 0, if bytes at hva are less than those at gva.
602 * Otherwise a value > 0 is returned.
604 * Compares the bytes starting at the host virtual address hva, for
605 * a length of len, to the guest bytes starting at the guest virtual
606 * address given by gva.
608 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, vm_vaddr_t gva, size_t len)
613 * Compare a batch of bytes until either a match is found
614 * or all the bytes have been compared.
616 for (uintptr_t offset = 0; offset < len; offset += amt) {
617 uintptr_t ptr1 = (uintptr_t)hva + offset;
620 * Determine host address for guest virtual address
623 uintptr_t ptr2 = (uintptr_t)addr_gva2hva(vm, gva + offset);
626 * Determine amount to compare on this pass.
627 * Don't allow the comparsion to cross a page boundary.
630 if ((ptr1 >> vm->page_shift) != ((ptr1 + amt) >> vm->page_shift))
631 amt = vm->page_size - (ptr1 % vm->page_size);
632 if ((ptr2 >> vm->page_shift) != ((ptr2 + amt) >> vm->page_shift))
633 amt = vm->page_size - (ptr2 % vm->page_size);
635 assert((ptr1 >> vm->page_shift) == ((ptr1 + amt - 1) >> vm->page_shift));
636 assert((ptr2 >> vm->page_shift) == ((ptr2 + amt - 1) >> vm->page_shift));
639 * Perform the comparison. If there is a difference
640 * return that result to the caller, otherwise need
641 * to continue on looking for a mismatch.
643 int ret = memcmp((void *)ptr1, (void *)ptr2, amt);
649 * No mismatch found. Let the caller know the two memory
656 * VM Userspace Memory Region Add
659 * vm - Virtual Machine
660 * backing_src - Storage source for this region.
661 * NULL to use anonymous memory.
662 * guest_paddr - Starting guest physical address
663 * slot - KVM region slot
664 * npages - Number of physical pages
665 * flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES)
671 * Allocates a memory area of the number of pages specified by npages
672 * and maps it to the VM specified by vm, at a starting physical address
673 * given by guest_paddr. The region is created with a KVM region slot
674 * given by slot, which must be unique and < KVM_MEM_SLOTS_NUM. The
675 * region is created with the flags given by flags.
677 void vm_userspace_mem_region_add(struct kvm_vm *vm,
678 enum vm_mem_backing_src_type src_type,
679 uint64_t guest_paddr, uint32_t slot, uint64_t npages,
683 struct userspace_mem_region *region;
684 size_t huge_page_size = KVM_UTIL_PGS_PER_HUGEPG * vm->page_size;
687 TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages,
688 "Number of guest pages is not compatible with the host. "
689 "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));
691 TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
692 "address not on a page boundary.\n"
693 " guest_paddr: 0x%lx vm->page_size: 0x%x",
694 guest_paddr, vm->page_size);
695 TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
696 <= vm->max_gfn, "Physical range beyond maximum "
697 "supported physical address,\n"
698 " guest_paddr: 0x%lx npages: 0x%lx\n"
699 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",
700 guest_paddr, npages, vm->max_gfn, vm->page_size);
703 * Confirm a mem region with an overlapping address doesn't
706 region = (struct userspace_mem_region *) userspace_mem_region_find(
707 vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
709 TEST_FAIL("overlapping userspace_mem_region already "
711 " requested guest_paddr: 0x%lx npages: 0x%lx "
713 " existing guest_paddr: 0x%lx size: 0x%lx",
714 guest_paddr, npages, vm->page_size,
715 (uint64_t) region->region.guest_phys_addr,
716 (uint64_t) region->region.memory_size);
718 /* Confirm no region with the requested slot already exists. */
719 list_for_each_entry(region, &vm->userspace_mem_regions, list) {
720 if (region->region.slot != slot)
723 TEST_FAIL("A mem region with the requested slot "
725 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
726 " existing slot: %u paddr: 0x%lx size: 0x%lx",
727 slot, guest_paddr, npages,
729 (uint64_t) region->region.guest_phys_addr,
730 (uint64_t) region->region.memory_size);
733 /* Allocate and initialize new mem region structure. */
734 region = calloc(1, sizeof(*region));
735 TEST_ASSERT(region != NULL, "Insufficient Memory");
736 region->mmap_size = npages * vm->page_size;
739 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
740 alignment = 0x100000;
745 if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
746 alignment = max(huge_page_size, alignment);
748 /* Add enough memory to align up if necessary */
750 region->mmap_size += alignment;
752 region->mmap_start = mmap(NULL, region->mmap_size,
753 PROT_READ | PROT_WRITE,
754 MAP_PRIVATE | MAP_ANONYMOUS
755 | (src_type == VM_MEM_SRC_ANONYMOUS_HUGETLB ? MAP_HUGETLB : 0),
757 TEST_ASSERT(region->mmap_start != MAP_FAILED,
758 "test_malloc failed, mmap_start: %p errno: %i",
759 region->mmap_start, errno);
761 /* Align host address */
762 region->host_mem = align(region->mmap_start, alignment);
764 /* As needed perform madvise */
765 if (src_type == VM_MEM_SRC_ANONYMOUS || src_type == VM_MEM_SRC_ANONYMOUS_THP) {
768 ret = stat("/sys/kernel/mm/transparent_hugepage", &statbuf);
769 TEST_ASSERT(ret == 0 || (ret == -1 && errno == ENOENT),
770 "stat /sys/kernel/mm/transparent_hugepage");
772 TEST_ASSERT(ret == 0 || src_type != VM_MEM_SRC_ANONYMOUS_THP,
773 "VM_MEM_SRC_ANONYMOUS_THP requires THP to be configured in the host kernel");
776 ret = madvise(region->host_mem, npages * vm->page_size,
777 src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
778 TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %x",
779 region->host_mem, npages * vm->page_size, src_type);
783 region->unused_phy_pages = sparsebit_alloc();
784 sparsebit_set_num(region->unused_phy_pages,
785 guest_paddr >> vm->page_shift, npages);
786 region->region.slot = slot;
787 region->region.flags = flags;
788 region->region.guest_phys_addr = guest_paddr;
789 region->region.memory_size = npages * vm->page_size;
790 region->region.userspace_addr = (uintptr_t) region->host_mem;
791 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
792 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
793 " rc: %i errno: %i\n"
794 " slot: %u flags: 0x%x\n"
795 " guest_phys_addr: 0x%lx size: 0x%lx",
796 ret, errno, slot, flags,
797 guest_paddr, (uint64_t) region->region.memory_size);
799 /* Add to linked-list of memory regions. */
800 list_add(®ion->list, &vm->userspace_mem_regions);
807 * vm - Virtual Machine
808 * memslot - KVM memory slot ID
813 * Pointer to memory region structure that describe memory region
814 * using kvm memory slot ID given by memslot. TEST_ASSERT failure
815 * on error (e.g. currently no memory region using memslot as a KVM
818 struct userspace_mem_region *
819 memslot2region(struct kvm_vm *vm, uint32_t memslot)
821 struct userspace_mem_region *region;
823 list_for_each_entry(region, &vm->userspace_mem_regions, list) {
824 if (region->region.slot == memslot)
828 fprintf(stderr, "No mem region with the requested slot found,\n"
829 " requested slot: %u\n", memslot);
830 fputs("---- vm dump ----\n", stderr);
831 vm_dump(stderr, vm, 2);
832 TEST_FAIL("Mem region not found");
837 * VM Memory Region Flags Set
840 * vm - Virtual Machine
841 * flags - Starting guest physical address
847 * Sets the flags of the memory region specified by the value of slot,
848 * to the values given by flags.
850 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
853 struct userspace_mem_region *region;
855 region = memslot2region(vm, slot);
857 region->region.flags = flags;
859 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
861 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
862 " rc: %i errno: %i slot: %u flags: 0x%x",
863 ret, errno, slot, flags);
867 * VM Memory Region Move
870 * vm - Virtual Machine
871 * slot - Slot of the memory region to move
872 * new_gpa - Starting guest physical address
878 * Change the gpa of a memory region.
880 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
882 struct userspace_mem_region *region;
885 region = memslot2region(vm, slot);
887 region->region.guest_phys_addr = new_gpa;
889 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
891 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed\n"
892 "ret: %i errno: %i slot: %u new_gpa: 0x%lx",
893 ret, errno, slot, new_gpa);
897 * VM Memory Region Delete
900 * vm - Virtual Machine
901 * slot - Slot of the memory region to delete
907 * Delete a memory region.
909 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
911 __vm_mem_region_delete(vm, memslot2region(vm, slot));
924 * Returns the size of the structure pointed to by the return value
927 static int vcpu_mmap_sz(void)
931 dev_fd = open(KVM_DEV_PATH, O_RDONLY);
935 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
936 TEST_ASSERT(ret >= sizeof(struct kvm_run),
937 "%s KVM_GET_VCPU_MMAP_SIZE ioctl failed, rc: %i errno: %i",
938 __func__, ret, errno);
949 * vm - Virtual Machine
956 * Adds a virtual CPU to the VM specified by vm with the ID given by vcpuid.
957 * No additional VCPU setup is done.
959 void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid)
963 /* Confirm a vcpu with the specified id doesn't already exist. */
964 vcpu = vcpu_find(vm, vcpuid);
966 TEST_FAIL("vcpu with the specified id "
968 " requested vcpuid: %u\n"
969 " existing vcpuid: %u state: %p",
970 vcpuid, vcpu->id, vcpu->state);
972 /* Allocate and initialize new vcpu structure. */
973 vcpu = calloc(1, sizeof(*vcpu));
974 TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
976 vcpu->fd = ioctl(vm->fd, KVM_CREATE_VCPU, vcpuid);
977 TEST_ASSERT(vcpu->fd >= 0, "KVM_CREATE_VCPU failed, rc: %i errno: %i",
980 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->state), "vcpu mmap size "
981 "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",
982 vcpu_mmap_sz(), sizeof(*vcpu->state));
983 vcpu->state = (struct kvm_run *) mmap(NULL, vcpu_mmap_sz(),
984 PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0);
985 TEST_ASSERT(vcpu->state != MAP_FAILED, "mmap vcpu_state failed, "
986 "vcpu id: %u errno: %i", vcpuid, errno);
988 /* Add to linked-list of VCPUs. */
989 list_add(&vcpu->list, &vm->vcpus);
993 * VM Virtual Address Unused Gap
996 * vm - Virtual Machine
998 * vaddr_min - Minimum Virtual Address
1003 * Lowest virtual address at or below vaddr_min, with at least
1004 * sz unused bytes. TEST_ASSERT failure if no area of at least
1005 * size sz is available.
1007 * Within the VM specified by vm, locates the lowest starting virtual
1008 * address >= vaddr_min, that has at least sz unallocated bytes. A
1009 * TEST_ASSERT failure occurs for invalid input or no area of at least
1010 * sz unallocated bytes >= vaddr_min is available.
1012 static vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
1013 vm_vaddr_t vaddr_min)
1015 uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
1017 /* Determine lowest permitted virtual page index. */
1018 uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
1019 if ((pgidx_start * vm->page_size) < vaddr_min)
1022 /* Loop over section with enough valid virtual page indexes. */
1023 if (!sparsebit_is_set_num(vm->vpages_valid,
1024 pgidx_start, pages))
1025 pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
1026 pgidx_start, pages);
1029 * Are there enough unused virtual pages available at
1030 * the currently proposed starting virtual page index.
1031 * If not, adjust proposed starting index to next
1034 if (sparsebit_is_clear_num(vm->vpages_mapped,
1035 pgidx_start, pages))
1037 pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
1038 pgidx_start, pages);
1039 if (pgidx_start == 0)
1043 * If needed, adjust proposed starting virtual address,
1044 * to next range of valid virtual addresses.
1046 if (!sparsebit_is_set_num(vm->vpages_valid,
1047 pgidx_start, pages)) {
1048 pgidx_start = sparsebit_next_set_num(
1049 vm->vpages_valid, pgidx_start, pages);
1050 if (pgidx_start == 0)
1053 } while (pgidx_start != 0);
1056 TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages);
1062 TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
1063 pgidx_start, pages),
1064 "Unexpected, invalid virtual page index range,\n"
1065 " pgidx_start: 0x%lx\n"
1067 pgidx_start, pages);
1068 TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
1069 pgidx_start, pages),
1070 "Unexpected, pages already mapped,\n"
1071 " pgidx_start: 0x%lx\n"
1073 pgidx_start, pages);
1075 return pgidx_start * vm->page_size;
1079 * VM Virtual Address Allocate
1082 * vm - Virtual Machine
1083 * sz - Size in bytes
1084 * vaddr_min - Minimum starting virtual address
1085 * data_memslot - Memory region slot for data pages
1086 * pgd_memslot - Memory region slot for new virtual translation tables
1091 * Starting guest virtual address
1093 * Allocates at least sz bytes within the virtual address space of the vm
1094 * given by vm. The allocated bytes are mapped to a virtual address >=
1095 * the address given by vaddr_min. Note that each allocation uses a
1096 * a unique set of pages, with the minimum real allocation being at least
1099 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
1100 uint32_t data_memslot, uint32_t pgd_memslot)
1102 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
1104 virt_pgd_alloc(vm, pgd_memslot);
1107 * Find an unused range of virtual page addresses of at least
1110 vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
1112 /* Map the virtual pages. */
1113 for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
1114 pages--, vaddr += vm->page_size) {
1117 paddr = vm_phy_page_alloc(vm,
1118 KVM_UTIL_MIN_PFN * vm->page_size, data_memslot);
1120 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
1122 sparsebit_set(vm->vpages_mapped,
1123 vaddr >> vm->page_shift);
1130 * Map a range of VM virtual address to the VM's physical address
1133 * vm - Virtual Machine
1134 * vaddr - Virtuall address to map
1135 * paddr - VM Physical Address
1136 * npages - The number of pages to map
1137 * pgd_memslot - Memory region slot for new virtual translation tables
1143 * Within the VM given by @vm, creates a virtual translation for
1144 * @npages starting at @vaddr to the page range starting at @paddr.
1146 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
1147 unsigned int npages, uint32_t pgd_memslot)
1149 size_t page_size = vm->page_size;
1150 size_t size = npages * page_size;
1152 TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
1153 TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
1156 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
1163 * Address VM Physical to Host Virtual
1166 * vm - Virtual Machine
1167 * gpa - VM physical address
1172 * Equivalent host virtual address
1174 * Locates the memory region containing the VM physical address given
1175 * by gpa, within the VM given by vm. When found, the host virtual
1176 * address providing the memory to the vm physical address is returned.
1177 * A TEST_ASSERT failure occurs if no region containing gpa exists.
1179 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
1181 struct userspace_mem_region *region;
1183 list_for_each_entry(region, &vm->userspace_mem_regions, list) {
1184 if ((gpa >= region->region.guest_phys_addr)
1185 && (gpa <= (region->region.guest_phys_addr
1186 + region->region.memory_size - 1)))
1187 return (void *) ((uintptr_t) region->host_mem
1188 + (gpa - region->region.guest_phys_addr));
1191 TEST_FAIL("No vm physical memory at 0x%lx", gpa);
1196 * Address Host Virtual to VM Physical
1199 * vm - Virtual Machine
1200 * hva - Host virtual address
1205 * Equivalent VM physical address
1207 * Locates the memory region containing the host virtual address given
1208 * by hva, within the VM given by vm. When found, the equivalent
1209 * VM physical address is returned. A TEST_ASSERT failure occurs if no
1210 * region containing hva exists.
1212 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1214 struct userspace_mem_region *region;
1216 list_for_each_entry(region, &vm->userspace_mem_regions, list) {
1217 if ((hva >= region->host_mem)
1218 && (hva <= (region->host_mem
1219 + region->region.memory_size - 1)))
1220 return (vm_paddr_t) ((uintptr_t)
1221 region->region.guest_phys_addr
1222 + (hva - (uintptr_t) region->host_mem));
1225 TEST_FAIL("No mapping to a guest physical address, hva: %p", hva);
1230 * VM Create IRQ Chip
1233 * vm - Virtual Machine
1239 * Creates an interrupt controller chip for the VM specified by vm.
1241 void vm_create_irqchip(struct kvm_vm *vm)
1245 ret = ioctl(vm->fd, KVM_CREATE_IRQCHIP, 0);
1246 TEST_ASSERT(ret == 0, "KVM_CREATE_IRQCHIP IOCTL failed, "
1247 "rc: %i errno: %i", ret, errno);
1249 vm->has_irqchip = true;
1256 * vm - Virtual Machine
1262 * Pointer to structure that describes the state of the VCPU.
1264 * Locates and returns a pointer to a structure that describes the
1265 * state of the VCPU with the given vcpuid.
1267 struct kvm_run *vcpu_state(struct kvm_vm *vm, uint32_t vcpuid)
1269 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1270 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1279 * vm - Virtual Machine
1286 * Switch to executing the code for the VCPU given by vcpuid, within the VM
1289 void vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1291 int ret = _vcpu_run(vm, vcpuid);
1292 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1293 "rc: %i errno: %i", ret, errno);
1296 int _vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1298 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1301 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1303 rc = ioctl(vcpu->fd, KVM_RUN, NULL);
1304 } while (rc == -1 && errno == EINTR);
1306 assert_on_unhandled_exception(vm, vcpuid);
1311 int vcpu_get_fd(struct kvm_vm *vm, uint32_t vcpuid)
1313 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1315 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1320 void vcpu_run_complete_io(struct kvm_vm *vm, uint32_t vcpuid)
1322 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1325 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1327 vcpu->state->immediate_exit = 1;
1328 ret = ioctl(vcpu->fd, KVM_RUN, NULL);
1329 vcpu->state->immediate_exit = 0;
1331 TEST_ASSERT(ret == -1 && errno == EINTR,
1332 "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1336 void vcpu_set_guest_debug(struct kvm_vm *vm, uint32_t vcpuid,
1337 struct kvm_guest_debug *debug)
1339 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1340 int ret = ioctl(vcpu->fd, KVM_SET_GUEST_DEBUG, debug);
1342 TEST_ASSERT(ret == 0, "KVM_SET_GUEST_DEBUG failed: %d", ret);
1346 * VM VCPU Set MP State
1349 * vm - Virtual Machine
1351 * mp_state - mp_state to be set
1357 * Sets the MP state of the VCPU given by vcpuid, to the state given
1360 void vcpu_set_mp_state(struct kvm_vm *vm, uint32_t vcpuid,
1361 struct kvm_mp_state *mp_state)
1363 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1366 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1368 ret = ioctl(vcpu->fd, KVM_SET_MP_STATE, mp_state);
1369 TEST_ASSERT(ret == 0, "KVM_SET_MP_STATE IOCTL failed, "
1370 "rc: %i errno: %i", ret, errno);
1374 * VM VCPU Get Reg List
1377 * vm - Virtual Machine
1384 * A pointer to an allocated struct kvm_reg_list
1386 * Get the list of guest registers which are supported for
1387 * KVM_GET_ONE_REG/KVM_SET_ONE_REG calls
1389 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vm *vm, uint32_t vcpuid)
1391 struct kvm_reg_list reg_list_n = { .n = 0 }, *reg_list;
1394 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_REG_LIST, ®_list_n);
1395 TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0");
1396 reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64));
1397 reg_list->n = reg_list_n.n;
1398 vcpu_ioctl(vm, vcpuid, KVM_GET_REG_LIST, reg_list);
1406 * vm - Virtual Machine
1410 * regs - current state of VCPU regs
1414 * Obtains the current register state for the VCPU specified by vcpuid
1415 * and stores it at the location given by regs.
1417 void vcpu_regs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1419 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1422 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1424 ret = ioctl(vcpu->fd, KVM_GET_REGS, regs);
1425 TEST_ASSERT(ret == 0, "KVM_GET_REGS failed, rc: %i errno: %i",
1433 * vm - Virtual Machine
1435 * regs - Values to set VCPU regs to
1441 * Sets the regs of the VCPU specified by vcpuid to the values
1444 void vcpu_regs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1446 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1449 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1451 ret = ioctl(vcpu->fd, KVM_SET_REGS, regs);
1452 TEST_ASSERT(ret == 0, "KVM_SET_REGS failed, rc: %i errno: %i",
1456 #ifdef __KVM_HAVE_VCPU_EVENTS
1457 void vcpu_events_get(struct kvm_vm *vm, uint32_t vcpuid,
1458 struct kvm_vcpu_events *events)
1460 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1463 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1465 ret = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, events);
1466 TEST_ASSERT(ret == 0, "KVM_GET_VCPU_EVENTS, failed, rc: %i errno: %i",
1470 void vcpu_events_set(struct kvm_vm *vm, uint32_t vcpuid,
1471 struct kvm_vcpu_events *events)
1473 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1476 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1478 ret = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, events);
1479 TEST_ASSERT(ret == 0, "KVM_SET_VCPU_EVENTS, failed, rc: %i errno: %i",
1485 void vcpu_nested_state_get(struct kvm_vm *vm, uint32_t vcpuid,
1486 struct kvm_nested_state *state)
1488 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1491 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1493 ret = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, state);
1494 TEST_ASSERT(ret == 0,
1495 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1499 int vcpu_nested_state_set(struct kvm_vm *vm, uint32_t vcpuid,
1500 struct kvm_nested_state *state, bool ignore_error)
1502 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1505 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1507 ret = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, state);
1508 if (!ignore_error) {
1509 TEST_ASSERT(ret == 0,
1510 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1519 * VM VCPU System Regs Get
1522 * vm - Virtual Machine
1526 * sregs - current state of VCPU system regs
1530 * Obtains the current system register state for the VCPU specified by
1531 * vcpuid and stores it at the location given by sregs.
1533 void vcpu_sregs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1535 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1538 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1540 ret = ioctl(vcpu->fd, KVM_GET_SREGS, sregs);
1541 TEST_ASSERT(ret == 0, "KVM_GET_SREGS failed, rc: %i errno: %i",
1546 * VM VCPU System Regs Set
1549 * vm - Virtual Machine
1551 * sregs - Values to set VCPU system regs to
1557 * Sets the system regs of the VCPU specified by vcpuid to the values
1560 void vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1562 int ret = _vcpu_sregs_set(vm, vcpuid, sregs);
1563 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1564 "rc: %i errno: %i", ret, errno);
1567 int _vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1569 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1571 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1573 return ioctl(vcpu->fd, KVM_SET_SREGS, sregs);
1576 void vcpu_fpu_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_fpu *fpu)
1580 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_FPU, fpu);
1581 TEST_ASSERT(ret == 0, "KVM_GET_FPU failed, rc: %i errno: %i (%s)",
1582 ret, errno, strerror(errno));
1585 void vcpu_fpu_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_fpu *fpu)
1589 ret = _vcpu_ioctl(vm, vcpuid, KVM_SET_FPU, fpu);
1590 TEST_ASSERT(ret == 0, "KVM_SET_FPU failed, rc: %i errno: %i (%s)",
1591 ret, errno, strerror(errno));
1594 void vcpu_get_reg(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_one_reg *reg)
1598 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_ONE_REG, reg);
1599 TEST_ASSERT(ret == 0, "KVM_GET_ONE_REG failed, rc: %i errno: %i (%s)",
1600 ret, errno, strerror(errno));
1603 void vcpu_set_reg(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_one_reg *reg)
1607 ret = _vcpu_ioctl(vm, vcpuid, KVM_SET_ONE_REG, reg);
1608 TEST_ASSERT(ret == 0, "KVM_SET_ONE_REG failed, rc: %i errno: %i (%s)",
1609 ret, errno, strerror(errno));
1616 * vm - Virtual Machine
1618 * cmd - Ioctl number
1619 * arg - Argument to pass to the ioctl
1623 * Issues an arbitrary ioctl on a VCPU fd.
1625 void vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1626 unsigned long cmd, void *arg)
1630 ret = _vcpu_ioctl(vm, vcpuid, cmd, arg);
1631 TEST_ASSERT(ret == 0, "vcpu ioctl %lu failed, rc: %i errno: %i (%s)",
1632 cmd, ret, errno, strerror(errno));
1635 int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1636 unsigned long cmd, void *arg)
1638 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1641 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1643 ret = ioctl(vcpu->fd, cmd, arg);
1648 void *vcpu_map_dirty_ring(struct kvm_vm *vm, uint32_t vcpuid)
1651 uint32_t size = vm->dirty_ring_size;
1653 TEST_ASSERT(size > 0, "Should enable dirty ring first");
1655 vcpu = vcpu_find(vm, vcpuid);
1657 TEST_ASSERT(vcpu, "Cannot find vcpu %u", vcpuid);
1659 if (!vcpu->dirty_gfns) {
1662 addr = mmap(NULL, size, PROT_READ,
1663 MAP_PRIVATE, vcpu->fd,
1664 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1665 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped private");
1667 addr = mmap(NULL, size, PROT_READ | PROT_EXEC,
1668 MAP_PRIVATE, vcpu->fd,
1669 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1670 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped exec");
1672 addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
1673 MAP_SHARED, vcpu->fd,
1674 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1675 TEST_ASSERT(addr != MAP_FAILED, "Dirty ring map failed");
1677 vcpu->dirty_gfns = addr;
1678 vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);
1681 return vcpu->dirty_gfns;
1688 * vm - Virtual Machine
1689 * cmd - Ioctl number
1690 * arg - Argument to pass to the ioctl
1694 * Issues an arbitrary ioctl on a VM fd.
1696 void vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1700 ret = _vm_ioctl(vm, cmd, arg);
1701 TEST_ASSERT(ret == 0, "vm ioctl %lu failed, rc: %i errno: %i (%s)",
1702 cmd, ret, errno, strerror(errno));
1705 int _vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1707 return ioctl(vm->fd, cmd, arg);
1714 * vm - Virtual Machine
1715 * cmd - Ioctl number
1716 * arg - Argument to pass to the ioctl
1720 * Issues an arbitrary ioctl on a KVM fd.
1722 void kvm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1726 ret = ioctl(vm->kvm_fd, cmd, arg);
1727 TEST_ASSERT(ret == 0, "KVM ioctl %lu failed, rc: %i errno: %i (%s)",
1728 cmd, ret, errno, strerror(errno));
1731 int _kvm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1733 return ioctl(vm->kvm_fd, cmd, arg);
1740 * vm - Virtual Machine
1741 * indent - Left margin indent amount
1744 * stream - Output FILE stream
1748 * Dumps the current state of the VM given by vm, to the FILE stream
1751 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1753 struct userspace_mem_region *region;
1756 fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1757 fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1758 fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1759 fprintf(stream, "%*sMem Regions:\n", indent, "");
1760 list_for_each_entry(region, &vm->userspace_mem_regions, list) {
1761 fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1762 "host_virt: %p\n", indent + 2, "",
1763 (uint64_t) region->region.guest_phys_addr,
1764 (uint64_t) region->region.memory_size,
1766 fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1767 sparsebit_dump(stream, region->unused_phy_pages, 0);
1769 fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1770 sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1771 fprintf(stream, "%*spgd_created: %u\n", indent, "",
1773 if (vm->pgd_created) {
1774 fprintf(stream, "%*sVirtual Translation Tables:\n",
1776 virt_dump(stream, vm, indent + 4);
1778 fprintf(stream, "%*sVCPUs:\n", indent, "");
1779 list_for_each_entry(vcpu, &vm->vcpus, list)
1780 vcpu_dump(stream, vm, vcpu->id, indent + 2);
1783 /* Known KVM exit reasons */
1784 static struct exit_reason {
1785 unsigned int reason;
1787 } exit_reasons_known[] = {
1788 {KVM_EXIT_UNKNOWN, "UNKNOWN"},
1789 {KVM_EXIT_EXCEPTION, "EXCEPTION"},
1790 {KVM_EXIT_IO, "IO"},
1791 {KVM_EXIT_HYPERCALL, "HYPERCALL"},
1792 {KVM_EXIT_DEBUG, "DEBUG"},
1793 {KVM_EXIT_HLT, "HLT"},
1794 {KVM_EXIT_MMIO, "MMIO"},
1795 {KVM_EXIT_IRQ_WINDOW_OPEN, "IRQ_WINDOW_OPEN"},
1796 {KVM_EXIT_SHUTDOWN, "SHUTDOWN"},
1797 {KVM_EXIT_FAIL_ENTRY, "FAIL_ENTRY"},
1798 {KVM_EXIT_INTR, "INTR"},
1799 {KVM_EXIT_SET_TPR, "SET_TPR"},
1800 {KVM_EXIT_TPR_ACCESS, "TPR_ACCESS"},
1801 {KVM_EXIT_S390_SIEIC, "S390_SIEIC"},
1802 {KVM_EXIT_S390_RESET, "S390_RESET"},
1803 {KVM_EXIT_DCR, "DCR"},
1804 {KVM_EXIT_NMI, "NMI"},
1805 {KVM_EXIT_INTERNAL_ERROR, "INTERNAL_ERROR"},
1806 {KVM_EXIT_OSI, "OSI"},
1807 {KVM_EXIT_PAPR_HCALL, "PAPR_HCALL"},
1808 {KVM_EXIT_DIRTY_RING_FULL, "DIRTY_RING_FULL"},
1809 {KVM_EXIT_X86_RDMSR, "RDMSR"},
1810 {KVM_EXIT_X86_WRMSR, "WRMSR"},
1811 {KVM_EXIT_XEN, "XEN"},
1812 #ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1813 {KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},
1818 * Exit Reason String
1821 * exit_reason - Exit reason
1826 * Constant string pointer describing the exit reason.
1828 * Locates and returns a constant string that describes the KVM exit
1829 * reason given by exit_reason. If no such string is found, a constant
1830 * string of "Unknown" is returned.
1832 const char *exit_reason_str(unsigned int exit_reason)
1836 for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
1837 if (exit_reason == exit_reasons_known[n1].reason)
1838 return exit_reasons_known[n1].name;
1845 * Physical Contiguous Page Allocator
1848 * vm - Virtual Machine
1849 * num - number of pages
1850 * paddr_min - Physical address minimum
1851 * memslot - Memory region to allocate page from
1856 * Starting physical address
1858 * Within the VM specified by vm, locates a range of available physical
1859 * pages at or above paddr_min. If found, the pages are marked as in use
1860 * and their base address is returned. A TEST_ASSERT failure occurs if
1861 * not enough pages are available at or above paddr_min.
1863 vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
1864 vm_paddr_t paddr_min, uint32_t memslot)
1866 struct userspace_mem_region *region;
1867 sparsebit_idx_t pg, base;
1869 TEST_ASSERT(num > 0, "Must allocate at least one page");
1871 TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
1872 "not divisible by page size.\n"
1873 " paddr_min: 0x%lx page_size: 0x%x",
1874 paddr_min, vm->page_size);
1876 region = memslot2region(vm, memslot);
1877 base = pg = paddr_min >> vm->page_shift;
1880 for (; pg < base + num; ++pg) {
1881 if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
1882 base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
1886 } while (pg && pg != base + num);
1889 fprintf(stderr, "No guest physical page available, "
1890 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
1891 paddr_min, vm->page_size, memslot);
1892 fputs("---- vm dump ----\n", stderr);
1893 vm_dump(stderr, vm, 2);
1897 for (pg = base; pg < base + num; ++pg)
1898 sparsebit_clear(region->unused_phy_pages, pg);
1900 return base * vm->page_size;
1903 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
1906 return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
1910 * Address Guest Virtual to Host Virtual
1913 * vm - Virtual Machine
1914 * gva - VM virtual address
1919 * Equivalent host virtual address
1921 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
1923 return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
1927 * Is Unrestricted Guest
1930 * vm - Virtual Machine
1934 * Return: True if the unrestricted guest is set to 'Y', otherwise return false.
1936 * Check if the unrestricted guest flag is enabled.
1938 bool vm_is_unrestricted_guest(struct kvm_vm *vm)
1945 /* Ensure that the KVM vendor-specific module is loaded. */
1946 f = fopen(KVM_DEV_PATH, "r");
1947 TEST_ASSERT(f != NULL, "Error in opening KVM dev file: %d",
1952 f = fopen("/sys/module/kvm_intel/parameters/unrestricted_guest", "r");
1954 count = fread(&val, sizeof(char), 1, f);
1955 TEST_ASSERT(count == 1, "Unable to read from param file.");
1962 unsigned int vm_get_page_size(struct kvm_vm *vm)
1964 return vm->page_size;
1967 unsigned int vm_get_page_shift(struct kvm_vm *vm)
1969 return vm->page_shift;
1972 unsigned int vm_get_max_gfn(struct kvm_vm *vm)
1977 int vm_get_fd(struct kvm_vm *vm)
1982 static unsigned int vm_calc_num_pages(unsigned int num_pages,
1983 unsigned int page_shift,
1984 unsigned int new_page_shift,
1987 unsigned int n = 1 << (new_page_shift - page_shift);
1989 if (page_shift >= new_page_shift)
1990 return num_pages * (1 << (page_shift - new_page_shift));
1992 return num_pages / n + !!(ceil && num_pages % n);
1995 static inline int getpageshift(void)
1997 return __builtin_ffs(getpagesize()) - 1;
2001 vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
2003 return vm_calc_num_pages(num_guest_pages,
2004 vm_guest_mode_params[mode].page_shift,
2005 getpageshift(), true);
2009 vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages)
2011 return vm_calc_num_pages(num_host_pages, getpageshift(),
2012 vm_guest_mode_params[mode].page_shift, false);
2015 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size)
2018 n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size);
2019 return vm_adjust_num_guest_pages(mode, n);