1 // SPDX-License-Identifier: GPL-2.0-only
3 * tools/testing/selftests/kvm/lib/kvm_util.c
5 * Copyright (C) 2018, Google LLC.
8 #define _GNU_SOURCE /* for program_invocation_name */
11 #include "kvm_util_internal.h"
12 #include "processor.h"
16 #include <sys/types.h>
19 #include <linux/kernel.h>
21 #define KVM_UTIL_MIN_PFN 2
23 static int vcpu_mmap_sz(void);
25 /* Aligns x up to the next multiple of size. Size must be a power of 2. */
26 static void *align(void *x, size_t size)
28 size_t mask = size - 1;
29 TEST_ASSERT(size != 0 && !(size & (size - 1)),
30 "size not a power of 2: %lu", size);
31 return (void *) (((size_t) x + mask) & ~mask);
35 * Open KVM_DEV_PATH if available, otherwise exit the entire program.
38 * flags - The flags to pass when opening KVM_DEV_PATH.
41 * The opened file descriptor of /dev/kvm.
43 static int _open_kvm_dev_path_or_exit(int flags)
47 fd = open(KVM_DEV_PATH, flags);
49 print_skip("%s not available, is KVM loaded? (errno: %d)",
57 int open_kvm_dev_path_or_exit(void)
59 return _open_kvm_dev_path_or_exit(O_RDONLY);
71 * On success, the Value corresponding to the capability (KVM_CAP_*)
72 * specified by the value of cap. On failure a TEST_ASSERT failure
75 * Looks up and returns the value corresponding to the capability
76 * (KVM_CAP_*) given by cap.
78 int kvm_check_cap(long cap)
83 kvm_fd = open_kvm_dev_path_or_exit();
84 ret = ioctl(kvm_fd, KVM_CHECK_EXTENSION, cap);
85 TEST_ASSERT(ret != -1, "KVM_CHECK_EXTENSION IOCTL failed,\n"
86 " rc: %i errno: %i", ret, errno);
93 /* VM Enable Capability
96 * vm - Virtual Machine
101 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
103 * Enables a capability (KVM_CAP_*) on the VM.
105 int vm_enable_cap(struct kvm_vm *vm, struct kvm_enable_cap *cap)
109 ret = ioctl(vm->fd, KVM_ENABLE_CAP, cap);
110 TEST_ASSERT(ret == 0, "KVM_ENABLE_CAP IOCTL failed,\n"
111 " rc: %i errno: %i", ret, errno);
116 /* VCPU Enable Capability
119 * vm - Virtual Machine
125 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
127 * Enables a capability (KVM_CAP_*) on the VCPU.
129 int vcpu_enable_cap(struct kvm_vm *vm, uint32_t vcpu_id,
130 struct kvm_enable_cap *cap)
132 struct vcpu *vcpu = vcpu_find(vm, vcpu_id);
135 TEST_ASSERT(vcpu, "cannot find vcpu %d", vcpu_id);
137 r = ioctl(vcpu->fd, KVM_ENABLE_CAP, cap);
138 TEST_ASSERT(!r, "KVM_ENABLE_CAP vCPU ioctl failed,\n"
139 " rc: %i, errno: %i", r, errno);
144 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
146 struct kvm_enable_cap cap = { 0 };
148 cap.cap = KVM_CAP_DIRTY_LOG_RING;
149 cap.args[0] = ring_size;
150 vm_enable_cap(vm, &cap);
151 vm->dirty_ring_size = ring_size;
154 static void vm_open(struct kvm_vm *vm, int perm)
156 vm->kvm_fd = _open_kvm_dev_path_or_exit(perm);
158 if (!kvm_check_cap(KVM_CAP_IMMEDIATE_EXIT)) {
159 print_skip("immediate_exit not available");
163 vm->fd = ioctl(vm->kvm_fd, KVM_CREATE_VM, vm->type);
164 TEST_ASSERT(vm->fd >= 0, "KVM_CREATE_VM ioctl failed, "
165 "rc: %i errno: %i", vm->fd, errno);
168 const char *vm_guest_mode_string(uint32_t i)
170 static const char * const strings[] = {
171 [VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages",
172 [VM_MODE_P52V48_64K] = "PA-bits:52, VA-bits:48, 64K pages",
173 [VM_MODE_P48V48_4K] = "PA-bits:48, VA-bits:48, 4K pages",
174 [VM_MODE_P48V48_64K] = "PA-bits:48, VA-bits:48, 64K pages",
175 [VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",
176 [VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",
177 [VM_MODE_PXXV48_4K] = "PA-bits:ANY, VA-bits:48, 4K pages",
179 _Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,
180 "Missing new mode strings?");
182 TEST_ASSERT(i < NUM_VM_MODES, "Guest mode ID %d too big", i);
187 const struct vm_guest_mode_params vm_guest_mode_params[] = {
188 { 52, 48, 0x1000, 12 },
189 { 52, 48, 0x10000, 16 },
190 { 48, 48, 0x1000, 12 },
191 { 48, 48, 0x10000, 16 },
192 { 40, 48, 0x1000, 12 },
193 { 40, 48, 0x10000, 16 },
194 { 0, 0, 0x1000, 12 },
196 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
197 "Missing new mode params?");
203 * mode - VM Mode (e.g. VM_MODE_P52V48_4K)
204 * phy_pages - Physical memory pages
210 * Pointer to opaque structure that describes the created VM.
212 * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
213 * When phy_pages is non-zero, a memory region of phy_pages physical pages
214 * is created and mapped starting at guest physical address 0. The file
215 * descriptor to control the created VM is created with the permissions
216 * given by perm (e.g. O_RDWR).
218 struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
222 pr_debug("%s: mode='%s' pages='%ld' perm='%d'\n", __func__,
223 vm_guest_mode_string(mode), phy_pages, perm);
225 vm = calloc(1, sizeof(*vm));
226 TEST_ASSERT(vm != NULL, "Insufficient Memory");
228 INIT_LIST_HEAD(&vm->vcpus);
229 vm->regions.gpa_tree = RB_ROOT;
230 vm->regions.hva_tree = RB_ROOT;
231 hash_init(vm->regions.slot_hash);
236 vm->pa_bits = vm_guest_mode_params[mode].pa_bits;
237 vm->va_bits = vm_guest_mode_params[mode].va_bits;
238 vm->page_size = vm_guest_mode_params[mode].page_size;
239 vm->page_shift = vm_guest_mode_params[mode].page_shift;
241 /* Setup mode specific traits. */
243 case VM_MODE_P52V48_4K:
244 vm->pgtable_levels = 4;
246 case VM_MODE_P52V48_64K:
247 vm->pgtable_levels = 3;
249 case VM_MODE_P48V48_4K:
250 vm->pgtable_levels = 4;
252 case VM_MODE_P48V48_64K:
253 vm->pgtable_levels = 3;
255 case VM_MODE_P40V48_4K:
256 vm->pgtable_levels = 4;
258 case VM_MODE_P40V48_64K:
259 vm->pgtable_levels = 3;
261 case VM_MODE_PXXV48_4K:
263 kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);
265 * Ignore KVM support for 5-level paging (vm->va_bits == 57),
266 * it doesn't take effect unless a CR4.LA57 is set, which it
267 * isn't for this VM_MODE.
269 TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57,
270 "Linear address width (%d bits) not supported",
272 pr_debug("Guest physical address width detected: %d\n",
274 vm->pgtable_levels = 4;
277 TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");
281 TEST_FAIL("Unknown guest mode, mode: 0x%x", mode);
285 if (vm->pa_bits != 40)
286 vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
291 /* Limit to VA-bit canonical virtual addresses. */
292 vm->vpages_valid = sparsebit_alloc();
293 sparsebit_set_num(vm->vpages_valid,
294 0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
295 sparsebit_set_num(vm->vpages_valid,
296 (~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
297 (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
299 /* Limit physical addresses to PA-bits. */
300 vm->max_gfn = ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
302 /* Allocate and setup memory for guest. */
303 vm->vpages_mapped = sparsebit_alloc();
305 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
311 struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus,
312 uint64_t extra_mem_pages, uint32_t num_percpu_pages,
313 void *guest_code, uint32_t vcpuids[])
315 /* The maximum page table size for a memory region will be when the
316 * smallest pages are used. Considering each page contains x page
317 * table descriptors, the total extra size for page tables (for extra
318 * N pages) will be: N/x+N/x^2+N/x^3+... which is definitely smaller
321 uint64_t vcpu_pages = (DEFAULT_STACK_PGS + num_percpu_pages) * nr_vcpus;
322 uint64_t extra_pg_pages = (extra_mem_pages + vcpu_pages) / PTES_PER_MIN_PAGE * 2;
323 uint64_t pages = DEFAULT_GUEST_PHY_PAGES + extra_mem_pages + vcpu_pages + extra_pg_pages;
327 TEST_ASSERT(nr_vcpus <= kvm_check_cap(KVM_CAP_MAX_VCPUS),
328 "nr_vcpus = %d too large for host, max-vcpus = %d",
329 nr_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS));
331 pages = vm_adjust_num_guest_pages(mode, pages);
332 vm = vm_create(mode, pages, O_RDWR);
334 kvm_vm_elf_load(vm, program_invocation_name, 0, 0);
337 vm_create_irqchip(vm);
340 for (i = 0; i < nr_vcpus; ++i) {
341 uint32_t vcpuid = vcpuids ? vcpuids[i] : i;
343 vm_vcpu_add_default(vm, vcpuid, guest_code);
346 vcpu_set_cpuid(vm, vcpuid, kvm_get_supported_cpuid());
353 struct kvm_vm *vm_create_default_with_vcpus(uint32_t nr_vcpus, uint64_t extra_mem_pages,
354 uint32_t num_percpu_pages, void *guest_code,
357 return vm_create_with_vcpus(VM_MODE_DEFAULT, nr_vcpus, extra_mem_pages,
358 num_percpu_pages, guest_code, vcpuids);
361 struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages,
364 return vm_create_default_with_vcpus(1, extra_mem_pages, 0, guest_code,
365 (uint32_t []){ vcpuid });
372 * vm - VM that has been released before
377 * Reopens the file descriptors associated to the VM and reinstates the
378 * global state, such as the irqchip and the memory regions that are mapped
381 void kvm_vm_restart(struct kvm_vm *vmp, int perm)
384 struct userspace_mem_region *region;
387 if (vmp->has_irqchip)
388 vm_create_irqchip(vmp);
390 hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {
391 int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
392 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
393 " rc: %i errno: %i\n"
394 " slot: %u flags: 0x%x\n"
395 " guest_phys_addr: 0x%llx size: 0x%llx",
396 ret, errno, region->region.slot,
397 region->region.flags,
398 region->region.guest_phys_addr,
399 region->region.memory_size);
403 void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
405 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
408 ret = ioctl(vm->fd, KVM_GET_DIRTY_LOG, &args);
409 TEST_ASSERT(ret == 0, "%s: KVM_GET_DIRTY_LOG failed: %s",
410 __func__, strerror(-ret));
413 void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
414 uint64_t first_page, uint32_t num_pages)
416 struct kvm_clear_dirty_log args = { .dirty_bitmap = log, .slot = slot,
417 .first_page = first_page,
418 .num_pages = num_pages };
421 ret = ioctl(vm->fd, KVM_CLEAR_DIRTY_LOG, &args);
422 TEST_ASSERT(ret == 0, "%s: KVM_CLEAR_DIRTY_LOG failed: %s",
423 __func__, strerror(-ret));
426 uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
428 return ioctl(vm->fd, KVM_RESET_DIRTY_RINGS);
432 * Userspace Memory Region Find
435 * vm - Virtual Machine
436 * start - Starting VM physical address
437 * end - Ending VM physical address, inclusive.
442 * Pointer to overlapping region, NULL if no such region.
444 * Searches for a region with any physical memory that overlaps with
445 * any portion of the guest physical addresses from start to end
446 * inclusive. If multiple overlapping regions exist, a pointer to any
447 * of the regions is returned. Null is returned only when no overlapping
450 static struct userspace_mem_region *
451 userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
453 struct rb_node *node;
455 for (node = vm->regions.gpa_tree.rb_node; node; ) {
456 struct userspace_mem_region *region =
457 container_of(node, struct userspace_mem_region, gpa_node);
458 uint64_t existing_start = region->region.guest_phys_addr;
459 uint64_t existing_end = region->region.guest_phys_addr
460 + region->region.memory_size - 1;
461 if (start <= existing_end && end >= existing_start)
464 if (start < existing_start)
465 node = node->rb_left;
467 node = node->rb_right;
474 * KVM Userspace Memory Region Find
477 * vm - Virtual Machine
478 * start - Starting VM physical address
479 * end - Ending VM physical address, inclusive.
484 * Pointer to overlapping region, NULL if no such region.
486 * Public interface to userspace_mem_region_find. Allows tests to look up
487 * the memslot datastructure for a given range of guest physical memory.
489 struct kvm_userspace_memory_region *
490 kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
493 struct userspace_mem_region *region;
495 region = userspace_mem_region_find(vm, start, end);
499 return ®ion->region;
506 * vm - Virtual Machine
512 * Pointer to VCPU structure
514 * Locates a vcpu structure that describes the VCPU specified by vcpuid and
515 * returns a pointer to it. Returns NULL if the VM doesn't contain a VCPU
516 * for the specified vcpuid.
518 struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid)
522 list_for_each_entry(vcpu, &vm->vcpus, list) {
523 if (vcpu->id == vcpuid)
534 * vcpu - VCPU to remove
538 * Return: None, TEST_ASSERT failures for all error conditions
540 * Removes a vCPU from a VM and frees its resources.
542 static void vm_vcpu_rm(struct kvm_vm *vm, struct vcpu *vcpu)
546 if (vcpu->dirty_gfns) {
547 ret = munmap(vcpu->dirty_gfns, vm->dirty_ring_size);
548 TEST_ASSERT(ret == 0, "munmap of VCPU dirty ring failed, "
549 "rc: %i errno: %i", ret, errno);
550 vcpu->dirty_gfns = NULL;
553 ret = munmap(vcpu->state, vcpu_mmap_sz());
554 TEST_ASSERT(ret == 0, "munmap of VCPU fd failed, rc: %i "
555 "errno: %i", ret, errno);
556 ret = close(vcpu->fd);
557 TEST_ASSERT(ret == 0, "Close of VCPU fd failed, rc: %i "
558 "errno: %i", ret, errno);
560 list_del(&vcpu->list);
564 void kvm_vm_release(struct kvm_vm *vmp)
566 struct vcpu *vcpu, *tmp;
569 list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)
570 vm_vcpu_rm(vmp, vcpu);
572 ret = close(vmp->fd);
573 TEST_ASSERT(ret == 0, "Close of vm fd failed,\n"
574 " vmp->fd: %i rc: %i errno: %i", vmp->fd, ret, errno);
576 ret = close(vmp->kvm_fd);
577 TEST_ASSERT(ret == 0, "Close of /dev/kvm fd failed,\n"
578 " vmp->kvm_fd: %i rc: %i errno: %i", vmp->kvm_fd, ret, errno);
581 static void __vm_mem_region_delete(struct kvm_vm *vm,
582 struct userspace_mem_region *region,
588 rb_erase(®ion->gpa_node, &vm->regions.gpa_tree);
589 rb_erase(®ion->hva_node, &vm->regions.hva_tree);
590 hash_del(®ion->slot_node);
593 region->region.memory_size = 0;
594 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
595 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, "
596 "rc: %i errno: %i", ret, errno);
598 sparsebit_free(®ion->unused_phy_pages);
599 ret = munmap(region->mmap_start, region->mmap_size);
600 TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i", ret, errno);
606 * Destroys and frees the VM pointed to by vmp.
608 void kvm_vm_free(struct kvm_vm *vmp)
611 struct hlist_node *node;
612 struct userspace_mem_region *region;
617 /* Free userspace_mem_regions. */
618 hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)
619 __vm_mem_region_delete(vmp, region, false);
621 /* Free sparsebit arrays. */
622 sparsebit_free(&vmp->vpages_valid);
623 sparsebit_free(&vmp->vpages_mapped);
627 /* Free the structure describing the VM. */
632 * Memory Compare, host virtual to guest virtual
635 * hva - Starting host virtual address
636 * vm - Virtual Machine
637 * gva - Starting guest virtual address
638 * len - number of bytes to compare
642 * Input/Output Args: None
645 * Returns 0 if the bytes starting at hva for a length of len
646 * are equal the guest virtual bytes starting at gva. Returns
647 * a value < 0, if bytes at hva are less than those at gva.
648 * Otherwise a value > 0 is returned.
650 * Compares the bytes starting at the host virtual address hva, for
651 * a length of len, to the guest bytes starting at the guest virtual
652 * address given by gva.
654 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, vm_vaddr_t gva, size_t len)
659 * Compare a batch of bytes until either a match is found
660 * or all the bytes have been compared.
662 for (uintptr_t offset = 0; offset < len; offset += amt) {
663 uintptr_t ptr1 = (uintptr_t)hva + offset;
666 * Determine host address for guest virtual address
669 uintptr_t ptr2 = (uintptr_t)addr_gva2hva(vm, gva + offset);
672 * Determine amount to compare on this pass.
673 * Don't allow the comparsion to cross a page boundary.
676 if ((ptr1 >> vm->page_shift) != ((ptr1 + amt) >> vm->page_shift))
677 amt = vm->page_size - (ptr1 % vm->page_size);
678 if ((ptr2 >> vm->page_shift) != ((ptr2 + amt) >> vm->page_shift))
679 amt = vm->page_size - (ptr2 % vm->page_size);
681 assert((ptr1 >> vm->page_shift) == ((ptr1 + amt - 1) >> vm->page_shift));
682 assert((ptr2 >> vm->page_shift) == ((ptr2 + amt - 1) >> vm->page_shift));
685 * Perform the comparison. If there is a difference
686 * return that result to the caller, otherwise need
687 * to continue on looking for a mismatch.
689 int ret = memcmp((void *)ptr1, (void *)ptr2, amt);
695 * No mismatch found. Let the caller know the two memory
701 static void vm_userspace_mem_region_gpa_insert(struct rb_root *gpa_tree,
702 struct userspace_mem_region *region)
704 struct rb_node **cur, *parent;
706 for (cur = &gpa_tree->rb_node, parent = NULL; *cur; ) {
707 struct userspace_mem_region *cregion;
709 cregion = container_of(*cur, typeof(*cregion), gpa_node);
711 if (region->region.guest_phys_addr <
712 cregion->region.guest_phys_addr)
713 cur = &(*cur)->rb_left;
715 TEST_ASSERT(region->region.guest_phys_addr !=
716 cregion->region.guest_phys_addr,
717 "Duplicate GPA in region tree");
719 cur = &(*cur)->rb_right;
723 rb_link_node(®ion->gpa_node, parent, cur);
724 rb_insert_color(®ion->gpa_node, gpa_tree);
727 static void vm_userspace_mem_region_hva_insert(struct rb_root *hva_tree,
728 struct userspace_mem_region *region)
730 struct rb_node **cur, *parent;
732 for (cur = &hva_tree->rb_node, parent = NULL; *cur; ) {
733 struct userspace_mem_region *cregion;
735 cregion = container_of(*cur, typeof(*cregion), hva_node);
737 if (region->host_mem < cregion->host_mem)
738 cur = &(*cur)->rb_left;
740 TEST_ASSERT(region->host_mem !=
742 "Duplicate HVA in region tree");
744 cur = &(*cur)->rb_right;
748 rb_link_node(®ion->hva_node, parent, cur);
749 rb_insert_color(®ion->hva_node, hva_tree);
753 * VM Userspace Memory Region Add
756 * vm - Virtual Machine
757 * src_type - Storage source for this region.
758 * NULL to use anonymous memory.
759 * guest_paddr - Starting guest physical address
760 * slot - KVM region slot
761 * npages - Number of physical pages
762 * flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES)
768 * Allocates a memory area of the number of pages specified by npages
769 * and maps it to the VM specified by vm, at a starting physical address
770 * given by guest_paddr. The region is created with a KVM region slot
771 * given by slot, which must be unique and < KVM_MEM_SLOTS_NUM. The
772 * region is created with the flags given by flags.
774 void vm_userspace_mem_region_add(struct kvm_vm *vm,
775 enum vm_mem_backing_src_type src_type,
776 uint64_t guest_paddr, uint32_t slot, uint64_t npages,
780 struct userspace_mem_region *region;
781 size_t backing_src_pagesz = get_backing_src_pagesz(src_type);
784 TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages,
785 "Number of guest pages is not compatible with the host. "
786 "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));
788 TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
789 "address not on a page boundary.\n"
790 " guest_paddr: 0x%lx vm->page_size: 0x%x",
791 guest_paddr, vm->page_size);
792 TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
793 <= vm->max_gfn, "Physical range beyond maximum "
794 "supported physical address,\n"
795 " guest_paddr: 0x%lx npages: 0x%lx\n"
796 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",
797 guest_paddr, npages, vm->max_gfn, vm->page_size);
800 * Confirm a mem region with an overlapping address doesn't
803 region = (struct userspace_mem_region *) userspace_mem_region_find(
804 vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
806 TEST_FAIL("overlapping userspace_mem_region already "
808 " requested guest_paddr: 0x%lx npages: 0x%lx "
810 " existing guest_paddr: 0x%lx size: 0x%lx",
811 guest_paddr, npages, vm->page_size,
812 (uint64_t) region->region.guest_phys_addr,
813 (uint64_t) region->region.memory_size);
815 /* Confirm no region with the requested slot already exists. */
816 hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
818 if (region->region.slot != slot)
821 TEST_FAIL("A mem region with the requested slot "
823 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
824 " existing slot: %u paddr: 0x%lx size: 0x%lx",
825 slot, guest_paddr, npages,
827 (uint64_t) region->region.guest_phys_addr,
828 (uint64_t) region->region.memory_size);
831 /* Allocate and initialize new mem region structure. */
832 region = calloc(1, sizeof(*region));
833 TEST_ASSERT(region != NULL, "Insufficient Memory");
834 region->mmap_size = npages * vm->page_size;
837 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
838 alignment = 0x100000;
843 if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
844 alignment = max(backing_src_pagesz, alignment);
846 /* Add enough memory to align up if necessary */
848 region->mmap_size += alignment;
851 if (src_type == VM_MEM_SRC_SHMEM) {
852 region->fd = memfd_create("kvm_selftest", MFD_CLOEXEC);
853 TEST_ASSERT(region->fd != -1,
854 "memfd_create failed, errno: %i", errno);
856 ret = ftruncate(region->fd, region->mmap_size);
857 TEST_ASSERT(ret == 0, "ftruncate failed, errno: %i", errno);
859 ret = fallocate(region->fd,
860 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0,
862 TEST_ASSERT(ret == 0, "fallocate failed, errno: %i", errno);
865 region->mmap_start = mmap(NULL, region->mmap_size,
866 PROT_READ | PROT_WRITE,
867 vm_mem_backing_src_alias(src_type)->flag,
869 TEST_ASSERT(region->mmap_start != MAP_FAILED,
870 "test_malloc failed, mmap_start: %p errno: %i",
871 region->mmap_start, errno);
873 /* Align host address */
874 region->host_mem = align(region->mmap_start, alignment);
876 /* As needed perform madvise */
877 if ((src_type == VM_MEM_SRC_ANONYMOUS ||
878 src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {
879 ret = madvise(region->host_mem, npages * vm->page_size,
880 src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
881 TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",
882 region->host_mem, npages * vm->page_size,
883 vm_mem_backing_src_alias(src_type)->name);
886 region->unused_phy_pages = sparsebit_alloc();
887 sparsebit_set_num(region->unused_phy_pages,
888 guest_paddr >> vm->page_shift, npages);
889 region->region.slot = slot;
890 region->region.flags = flags;
891 region->region.guest_phys_addr = guest_paddr;
892 region->region.memory_size = npages * vm->page_size;
893 region->region.userspace_addr = (uintptr_t) region->host_mem;
894 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
895 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
896 " rc: %i errno: %i\n"
897 " slot: %u flags: 0x%x\n"
898 " guest_phys_addr: 0x%lx size: 0x%lx",
899 ret, errno, slot, flags,
900 guest_paddr, (uint64_t) region->region.memory_size);
902 /* Add to quick lookup data structures */
903 vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);
904 vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);
905 hash_add(vm->regions.slot_hash, ®ion->slot_node, slot);
912 * vm - Virtual Machine
913 * memslot - KVM memory slot ID
918 * Pointer to memory region structure that describe memory region
919 * using kvm memory slot ID given by memslot. TEST_ASSERT failure
920 * on error (e.g. currently no memory region using memslot as a KVM
923 struct userspace_mem_region *
924 memslot2region(struct kvm_vm *vm, uint32_t memslot)
926 struct userspace_mem_region *region;
928 hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
930 if (region->region.slot == memslot)
933 fprintf(stderr, "No mem region with the requested slot found,\n"
934 " requested slot: %u\n", memslot);
935 fputs("---- vm dump ----\n", stderr);
936 vm_dump(stderr, vm, 2);
937 TEST_FAIL("Mem region not found");
942 * VM Memory Region Flags Set
945 * vm - Virtual Machine
946 * flags - Starting guest physical address
952 * Sets the flags of the memory region specified by the value of slot,
953 * to the values given by flags.
955 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
958 struct userspace_mem_region *region;
960 region = memslot2region(vm, slot);
962 region->region.flags = flags;
964 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
966 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
967 " rc: %i errno: %i slot: %u flags: 0x%x",
968 ret, errno, slot, flags);
972 * VM Memory Region Move
975 * vm - Virtual Machine
976 * slot - Slot of the memory region to move
977 * new_gpa - Starting guest physical address
983 * Change the gpa of a memory region.
985 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
987 struct userspace_mem_region *region;
990 region = memslot2region(vm, slot);
992 region->region.guest_phys_addr = new_gpa;
994 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
996 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed\n"
997 "ret: %i errno: %i slot: %u new_gpa: 0x%lx",
998 ret, errno, slot, new_gpa);
1002 * VM Memory Region Delete
1005 * vm - Virtual Machine
1006 * slot - Slot of the memory region to delete
1012 * Delete a memory region.
1014 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
1016 __vm_mem_region_delete(vm, memslot2region(vm, slot), true);
1027 * Size of VCPU state
1029 * Returns the size of the structure pointed to by the return value
1032 static int vcpu_mmap_sz(void)
1036 dev_fd = open_kvm_dev_path_or_exit();
1038 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
1039 TEST_ASSERT(ret >= sizeof(struct kvm_run),
1040 "%s KVM_GET_VCPU_MMAP_SIZE ioctl failed, rc: %i errno: %i",
1041 __func__, ret, errno);
1052 * vm - Virtual Machine
1059 * Adds a virtual CPU to the VM specified by vm with the ID given by vcpuid.
1060 * No additional VCPU setup is done.
1062 void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid)
1066 /* Confirm a vcpu with the specified id doesn't already exist. */
1067 vcpu = vcpu_find(vm, vcpuid);
1069 TEST_FAIL("vcpu with the specified id "
1071 " requested vcpuid: %u\n"
1072 " existing vcpuid: %u state: %p",
1073 vcpuid, vcpu->id, vcpu->state);
1075 /* Allocate and initialize new vcpu structure. */
1076 vcpu = calloc(1, sizeof(*vcpu));
1077 TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
1079 vcpu->fd = ioctl(vm->fd, KVM_CREATE_VCPU, vcpuid);
1080 TEST_ASSERT(vcpu->fd >= 0, "KVM_CREATE_VCPU failed, rc: %i errno: %i",
1083 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->state), "vcpu mmap size "
1084 "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",
1085 vcpu_mmap_sz(), sizeof(*vcpu->state));
1086 vcpu->state = (struct kvm_run *) mmap(NULL, vcpu_mmap_sz(),
1087 PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0);
1088 TEST_ASSERT(vcpu->state != MAP_FAILED, "mmap vcpu_state failed, "
1089 "vcpu id: %u errno: %i", vcpuid, errno);
1091 /* Add to linked-list of VCPUs. */
1092 list_add(&vcpu->list, &vm->vcpus);
1096 * VM Virtual Address Unused Gap
1099 * vm - Virtual Machine
1101 * vaddr_min - Minimum Virtual Address
1106 * Lowest virtual address at or below vaddr_min, with at least
1107 * sz unused bytes. TEST_ASSERT failure if no area of at least
1108 * size sz is available.
1110 * Within the VM specified by vm, locates the lowest starting virtual
1111 * address >= vaddr_min, that has at least sz unallocated bytes. A
1112 * TEST_ASSERT failure occurs for invalid input or no area of at least
1113 * sz unallocated bytes >= vaddr_min is available.
1115 static vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
1116 vm_vaddr_t vaddr_min)
1118 uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
1120 /* Determine lowest permitted virtual page index. */
1121 uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
1122 if ((pgidx_start * vm->page_size) < vaddr_min)
1125 /* Loop over section with enough valid virtual page indexes. */
1126 if (!sparsebit_is_set_num(vm->vpages_valid,
1127 pgidx_start, pages))
1128 pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
1129 pgidx_start, pages);
1132 * Are there enough unused virtual pages available at
1133 * the currently proposed starting virtual page index.
1134 * If not, adjust proposed starting index to next
1137 if (sparsebit_is_clear_num(vm->vpages_mapped,
1138 pgidx_start, pages))
1140 pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
1141 pgidx_start, pages);
1142 if (pgidx_start == 0)
1146 * If needed, adjust proposed starting virtual address,
1147 * to next range of valid virtual addresses.
1149 if (!sparsebit_is_set_num(vm->vpages_valid,
1150 pgidx_start, pages)) {
1151 pgidx_start = sparsebit_next_set_num(
1152 vm->vpages_valid, pgidx_start, pages);
1153 if (pgidx_start == 0)
1156 } while (pgidx_start != 0);
1159 TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages);
1165 TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
1166 pgidx_start, pages),
1167 "Unexpected, invalid virtual page index range,\n"
1168 " pgidx_start: 0x%lx\n"
1170 pgidx_start, pages);
1171 TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
1172 pgidx_start, pages),
1173 "Unexpected, pages already mapped,\n"
1174 " pgidx_start: 0x%lx\n"
1176 pgidx_start, pages);
1178 return pgidx_start * vm->page_size;
1182 * VM Virtual Address Allocate
1185 * vm - Virtual Machine
1186 * sz - Size in bytes
1187 * vaddr_min - Minimum starting virtual address
1188 * data_memslot - Memory region slot for data pages
1189 * pgd_memslot - Memory region slot for new virtual translation tables
1194 * Starting guest virtual address
1196 * Allocates at least sz bytes within the virtual address space of the vm
1197 * given by vm. The allocated bytes are mapped to a virtual address >=
1198 * the address given by vaddr_min. Note that each allocation uses a
1199 * a unique set of pages, with the minimum real allocation being at least
1202 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
1203 uint32_t data_memslot, uint32_t pgd_memslot)
1205 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
1207 virt_pgd_alloc(vm, pgd_memslot);
1208 vm_paddr_t paddr = vm_phy_pages_alloc(vm, pages,
1209 KVM_UTIL_MIN_PFN * vm->page_size,
1213 * Find an unused range of virtual page addresses of at least
1216 vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
1218 /* Map the virtual pages. */
1219 for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
1220 pages--, vaddr += vm->page_size, paddr += vm->page_size) {
1222 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
1224 sparsebit_set(vm->vpages_mapped,
1225 vaddr >> vm->page_shift);
1232 * Map a range of VM virtual address to the VM's physical address
1235 * vm - Virtual Machine
1236 * vaddr - Virtuall address to map
1237 * paddr - VM Physical Address
1238 * npages - The number of pages to map
1239 * pgd_memslot - Memory region slot for new virtual translation tables
1245 * Within the VM given by @vm, creates a virtual translation for
1246 * @npages starting at @vaddr to the page range starting at @paddr.
1248 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
1249 unsigned int npages, uint32_t pgd_memslot)
1251 size_t page_size = vm->page_size;
1252 size_t size = npages * page_size;
1254 TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
1255 TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
1258 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
1265 * Address VM Physical to Host Virtual
1268 * vm - Virtual Machine
1269 * gpa - VM physical address
1274 * Equivalent host virtual address
1276 * Locates the memory region containing the VM physical address given
1277 * by gpa, within the VM given by vm. When found, the host virtual
1278 * address providing the memory to the vm physical address is returned.
1279 * A TEST_ASSERT failure occurs if no region containing gpa exists.
1281 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
1283 struct userspace_mem_region *region;
1285 region = userspace_mem_region_find(vm, gpa, gpa);
1287 TEST_FAIL("No vm physical memory at 0x%lx", gpa);
1291 return (void *)((uintptr_t)region->host_mem
1292 + (gpa - region->region.guest_phys_addr));
1296 * Address Host Virtual to VM Physical
1299 * vm - Virtual Machine
1300 * hva - Host virtual address
1305 * Equivalent VM physical address
1307 * Locates the memory region containing the host virtual address given
1308 * by hva, within the VM given by vm. When found, the equivalent
1309 * VM physical address is returned. A TEST_ASSERT failure occurs if no
1310 * region containing hva exists.
1312 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1314 struct rb_node *node;
1316 for (node = vm->regions.hva_tree.rb_node; node; ) {
1317 struct userspace_mem_region *region =
1318 container_of(node, struct userspace_mem_region, hva_node);
1320 if (hva >= region->host_mem) {
1321 if (hva <= (region->host_mem
1322 + region->region.memory_size - 1))
1323 return (vm_paddr_t)((uintptr_t)
1324 region->region.guest_phys_addr
1325 + (hva - (uintptr_t)region->host_mem));
1327 node = node->rb_right;
1329 node = node->rb_left;
1332 TEST_FAIL("No mapping to a guest physical address, hva: %p", hva);
1337 * VM Create IRQ Chip
1340 * vm - Virtual Machine
1346 * Creates an interrupt controller chip for the VM specified by vm.
1348 void vm_create_irqchip(struct kvm_vm *vm)
1352 ret = ioctl(vm->fd, KVM_CREATE_IRQCHIP, 0);
1353 TEST_ASSERT(ret == 0, "KVM_CREATE_IRQCHIP IOCTL failed, "
1354 "rc: %i errno: %i", ret, errno);
1356 vm->has_irqchip = true;
1363 * vm - Virtual Machine
1369 * Pointer to structure that describes the state of the VCPU.
1371 * Locates and returns a pointer to a structure that describes the
1372 * state of the VCPU with the given vcpuid.
1374 struct kvm_run *vcpu_state(struct kvm_vm *vm, uint32_t vcpuid)
1376 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1377 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1386 * vm - Virtual Machine
1393 * Switch to executing the code for the VCPU given by vcpuid, within the VM
1396 void vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1398 int ret = _vcpu_run(vm, vcpuid);
1399 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1400 "rc: %i errno: %i", ret, errno);
1403 int _vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1405 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1408 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1410 rc = ioctl(vcpu->fd, KVM_RUN, NULL);
1411 } while (rc == -1 && errno == EINTR);
1413 assert_on_unhandled_exception(vm, vcpuid);
1418 int vcpu_get_fd(struct kvm_vm *vm, uint32_t vcpuid)
1420 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1422 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1427 void vcpu_run_complete_io(struct kvm_vm *vm, uint32_t vcpuid)
1429 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1432 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1434 vcpu->state->immediate_exit = 1;
1435 ret = ioctl(vcpu->fd, KVM_RUN, NULL);
1436 vcpu->state->immediate_exit = 0;
1438 TEST_ASSERT(ret == -1 && errno == EINTR,
1439 "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1443 void vcpu_set_guest_debug(struct kvm_vm *vm, uint32_t vcpuid,
1444 struct kvm_guest_debug *debug)
1446 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1447 int ret = ioctl(vcpu->fd, KVM_SET_GUEST_DEBUG, debug);
1449 TEST_ASSERT(ret == 0, "KVM_SET_GUEST_DEBUG failed: %d", ret);
1453 * VM VCPU Set MP State
1456 * vm - Virtual Machine
1458 * mp_state - mp_state to be set
1464 * Sets the MP state of the VCPU given by vcpuid, to the state given
1467 void vcpu_set_mp_state(struct kvm_vm *vm, uint32_t vcpuid,
1468 struct kvm_mp_state *mp_state)
1470 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1473 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1475 ret = ioctl(vcpu->fd, KVM_SET_MP_STATE, mp_state);
1476 TEST_ASSERT(ret == 0, "KVM_SET_MP_STATE IOCTL failed, "
1477 "rc: %i errno: %i", ret, errno);
1481 * VM VCPU Get Reg List
1484 * vm - Virtual Machine
1491 * A pointer to an allocated struct kvm_reg_list
1493 * Get the list of guest registers which are supported for
1494 * KVM_GET_ONE_REG/KVM_SET_ONE_REG calls
1496 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vm *vm, uint32_t vcpuid)
1498 struct kvm_reg_list reg_list_n = { .n = 0 }, *reg_list;
1501 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_REG_LIST, ®_list_n);
1502 TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0");
1503 reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64));
1504 reg_list->n = reg_list_n.n;
1505 vcpu_ioctl(vm, vcpuid, KVM_GET_REG_LIST, reg_list);
1513 * vm - Virtual Machine
1517 * regs - current state of VCPU regs
1521 * Obtains the current register state for the VCPU specified by vcpuid
1522 * and stores it at the location given by regs.
1524 void vcpu_regs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1526 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1529 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1531 ret = ioctl(vcpu->fd, KVM_GET_REGS, regs);
1532 TEST_ASSERT(ret == 0, "KVM_GET_REGS failed, rc: %i errno: %i",
1540 * vm - Virtual Machine
1542 * regs - Values to set VCPU regs to
1548 * Sets the regs of the VCPU specified by vcpuid to the values
1551 void vcpu_regs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1553 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1556 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1558 ret = ioctl(vcpu->fd, KVM_SET_REGS, regs);
1559 TEST_ASSERT(ret == 0, "KVM_SET_REGS failed, rc: %i errno: %i",
1563 #ifdef __KVM_HAVE_VCPU_EVENTS
1564 void vcpu_events_get(struct kvm_vm *vm, uint32_t vcpuid,
1565 struct kvm_vcpu_events *events)
1567 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1570 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1572 ret = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, events);
1573 TEST_ASSERT(ret == 0, "KVM_GET_VCPU_EVENTS, failed, rc: %i errno: %i",
1577 void vcpu_events_set(struct kvm_vm *vm, uint32_t vcpuid,
1578 struct kvm_vcpu_events *events)
1580 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1583 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1585 ret = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, events);
1586 TEST_ASSERT(ret == 0, "KVM_SET_VCPU_EVENTS, failed, rc: %i errno: %i",
1592 void vcpu_nested_state_get(struct kvm_vm *vm, uint32_t vcpuid,
1593 struct kvm_nested_state *state)
1595 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1598 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1600 ret = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, state);
1601 TEST_ASSERT(ret == 0,
1602 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1606 int vcpu_nested_state_set(struct kvm_vm *vm, uint32_t vcpuid,
1607 struct kvm_nested_state *state, bool ignore_error)
1609 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1612 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1614 ret = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, state);
1615 if (!ignore_error) {
1616 TEST_ASSERT(ret == 0,
1617 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1626 * VM VCPU System Regs Get
1629 * vm - Virtual Machine
1633 * sregs - current state of VCPU system regs
1637 * Obtains the current system register state for the VCPU specified by
1638 * vcpuid and stores it at the location given by sregs.
1640 void vcpu_sregs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1642 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1645 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1647 ret = ioctl(vcpu->fd, KVM_GET_SREGS, sregs);
1648 TEST_ASSERT(ret == 0, "KVM_GET_SREGS failed, rc: %i errno: %i",
1653 * VM VCPU System Regs Set
1656 * vm - Virtual Machine
1658 * sregs - Values to set VCPU system regs to
1664 * Sets the system regs of the VCPU specified by vcpuid to the values
1667 void vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1669 int ret = _vcpu_sregs_set(vm, vcpuid, sregs);
1670 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1671 "rc: %i errno: %i", ret, errno);
1674 int _vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1676 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1678 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1680 return ioctl(vcpu->fd, KVM_SET_SREGS, sregs);
1683 void vcpu_fpu_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_fpu *fpu)
1687 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_FPU, fpu);
1688 TEST_ASSERT(ret == 0, "KVM_GET_FPU failed, rc: %i errno: %i (%s)",
1689 ret, errno, strerror(errno));
1692 void vcpu_fpu_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_fpu *fpu)
1696 ret = _vcpu_ioctl(vm, vcpuid, KVM_SET_FPU, fpu);
1697 TEST_ASSERT(ret == 0, "KVM_SET_FPU failed, rc: %i errno: %i (%s)",
1698 ret, errno, strerror(errno));
1701 void vcpu_get_reg(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_one_reg *reg)
1705 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_ONE_REG, reg);
1706 TEST_ASSERT(ret == 0, "KVM_GET_ONE_REG failed, rc: %i errno: %i (%s)",
1707 ret, errno, strerror(errno));
1710 void vcpu_set_reg(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_one_reg *reg)
1714 ret = _vcpu_ioctl(vm, vcpuid, KVM_SET_ONE_REG, reg);
1715 TEST_ASSERT(ret == 0, "KVM_SET_ONE_REG failed, rc: %i errno: %i (%s)",
1716 ret, errno, strerror(errno));
1723 * vm - Virtual Machine
1725 * cmd - Ioctl number
1726 * arg - Argument to pass to the ioctl
1730 * Issues an arbitrary ioctl on a VCPU fd.
1732 void vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1733 unsigned long cmd, void *arg)
1737 ret = _vcpu_ioctl(vm, vcpuid, cmd, arg);
1738 TEST_ASSERT(ret == 0, "vcpu ioctl %lu failed, rc: %i errno: %i (%s)",
1739 cmd, ret, errno, strerror(errno));
1742 int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1743 unsigned long cmd, void *arg)
1745 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1748 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1750 ret = ioctl(vcpu->fd, cmd, arg);
1755 void *vcpu_map_dirty_ring(struct kvm_vm *vm, uint32_t vcpuid)
1758 uint32_t size = vm->dirty_ring_size;
1760 TEST_ASSERT(size > 0, "Should enable dirty ring first");
1762 vcpu = vcpu_find(vm, vcpuid);
1764 TEST_ASSERT(vcpu, "Cannot find vcpu %u", vcpuid);
1766 if (!vcpu->dirty_gfns) {
1769 addr = mmap(NULL, size, PROT_READ,
1770 MAP_PRIVATE, vcpu->fd,
1771 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1772 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped private");
1774 addr = mmap(NULL, size, PROT_READ | PROT_EXEC,
1775 MAP_PRIVATE, vcpu->fd,
1776 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1777 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped exec");
1779 addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
1780 MAP_SHARED, vcpu->fd,
1781 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1782 TEST_ASSERT(addr != MAP_FAILED, "Dirty ring map failed");
1784 vcpu->dirty_gfns = addr;
1785 vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);
1788 return vcpu->dirty_gfns;
1795 * vm - Virtual Machine
1796 * cmd - Ioctl number
1797 * arg - Argument to pass to the ioctl
1801 * Issues an arbitrary ioctl on a VM fd.
1803 void vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1807 ret = _vm_ioctl(vm, cmd, arg);
1808 TEST_ASSERT(ret == 0, "vm ioctl %lu failed, rc: %i errno: %i (%s)",
1809 cmd, ret, errno, strerror(errno));
1812 int _vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1814 return ioctl(vm->fd, cmd, arg);
1821 * vm - Virtual Machine
1822 * cmd - Ioctl number
1823 * arg - Argument to pass to the ioctl
1827 * Issues an arbitrary ioctl on a KVM fd.
1829 void kvm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1833 ret = ioctl(vm->kvm_fd, cmd, arg);
1834 TEST_ASSERT(ret == 0, "KVM ioctl %lu failed, rc: %i errno: %i (%s)",
1835 cmd, ret, errno, strerror(errno));
1838 int _kvm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1840 return ioctl(vm->kvm_fd, cmd, arg);
1847 int _kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
1849 struct kvm_device_attr attribute = {
1855 return ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute);
1858 int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
1860 int ret = _kvm_device_check_attr(dev_fd, group, attr);
1862 TEST_ASSERT(ret >= 0, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno);
1866 int _kvm_create_device(struct kvm_vm *vm, uint64_t type, bool test, int *fd)
1868 struct kvm_create_device create_dev;
1871 create_dev.type = type;
1873 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
1874 ret = ioctl(vm_get_fd(vm), KVM_CREATE_DEVICE, &create_dev);
1875 *fd = create_dev.fd;
1879 int kvm_create_device(struct kvm_vm *vm, uint64_t type, bool test)
1883 ret = _kvm_create_device(vm, type, test, &fd);
1886 TEST_ASSERT(ret >= 0,
1887 "KVM_CREATE_DEVICE IOCTL failed, rc: %i errno: %i", ret, errno);
1893 int _kvm_device_access(int dev_fd, uint32_t group, uint64_t attr,
1894 void *val, bool write)
1896 struct kvm_device_attr kvmattr = {
1900 .addr = (uintptr_t)val,
1904 ret = ioctl(dev_fd, write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
1909 int kvm_device_access(int dev_fd, uint32_t group, uint64_t attr,
1910 void *val, bool write)
1912 int ret = _kvm_device_access(dev_fd, group, attr, val, write);
1914 TEST_ASSERT(ret >= 0, "KVM_SET|GET_DEVICE_ATTR IOCTL failed, rc: %i errno: %i", ret, errno);
1922 * vm - Virtual Machine
1923 * indent - Left margin indent amount
1926 * stream - Output FILE stream
1930 * Dumps the current state of the VM given by vm, to the FILE stream
1933 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1936 struct userspace_mem_region *region;
1939 fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1940 fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1941 fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1942 fprintf(stream, "%*sMem Regions:\n", indent, "");
1943 hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {
1944 fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1945 "host_virt: %p\n", indent + 2, "",
1946 (uint64_t) region->region.guest_phys_addr,
1947 (uint64_t) region->region.memory_size,
1949 fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1950 sparsebit_dump(stream, region->unused_phy_pages, 0);
1952 fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1953 sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1954 fprintf(stream, "%*spgd_created: %u\n", indent, "",
1956 if (vm->pgd_created) {
1957 fprintf(stream, "%*sVirtual Translation Tables:\n",
1959 virt_dump(stream, vm, indent + 4);
1961 fprintf(stream, "%*sVCPUs:\n", indent, "");
1962 list_for_each_entry(vcpu, &vm->vcpus, list)
1963 vcpu_dump(stream, vm, vcpu->id, indent + 2);
1966 /* Known KVM exit reasons */
1967 static struct exit_reason {
1968 unsigned int reason;
1970 } exit_reasons_known[] = {
1971 {KVM_EXIT_UNKNOWN, "UNKNOWN"},
1972 {KVM_EXIT_EXCEPTION, "EXCEPTION"},
1973 {KVM_EXIT_IO, "IO"},
1974 {KVM_EXIT_HYPERCALL, "HYPERCALL"},
1975 {KVM_EXIT_DEBUG, "DEBUG"},
1976 {KVM_EXIT_HLT, "HLT"},
1977 {KVM_EXIT_MMIO, "MMIO"},
1978 {KVM_EXIT_IRQ_WINDOW_OPEN, "IRQ_WINDOW_OPEN"},
1979 {KVM_EXIT_SHUTDOWN, "SHUTDOWN"},
1980 {KVM_EXIT_FAIL_ENTRY, "FAIL_ENTRY"},
1981 {KVM_EXIT_INTR, "INTR"},
1982 {KVM_EXIT_SET_TPR, "SET_TPR"},
1983 {KVM_EXIT_TPR_ACCESS, "TPR_ACCESS"},
1984 {KVM_EXIT_S390_SIEIC, "S390_SIEIC"},
1985 {KVM_EXIT_S390_RESET, "S390_RESET"},
1986 {KVM_EXIT_DCR, "DCR"},
1987 {KVM_EXIT_NMI, "NMI"},
1988 {KVM_EXIT_INTERNAL_ERROR, "INTERNAL_ERROR"},
1989 {KVM_EXIT_OSI, "OSI"},
1990 {KVM_EXIT_PAPR_HCALL, "PAPR_HCALL"},
1991 {KVM_EXIT_DIRTY_RING_FULL, "DIRTY_RING_FULL"},
1992 {KVM_EXIT_X86_RDMSR, "RDMSR"},
1993 {KVM_EXIT_X86_WRMSR, "WRMSR"},
1994 {KVM_EXIT_XEN, "XEN"},
1995 #ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1996 {KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},
2001 * Exit Reason String
2004 * exit_reason - Exit reason
2009 * Constant string pointer describing the exit reason.
2011 * Locates and returns a constant string that describes the KVM exit
2012 * reason given by exit_reason. If no such string is found, a constant
2013 * string of "Unknown" is returned.
2015 const char *exit_reason_str(unsigned int exit_reason)
2019 for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
2020 if (exit_reason == exit_reasons_known[n1].reason)
2021 return exit_reasons_known[n1].name;
2028 * Physical Contiguous Page Allocator
2031 * vm - Virtual Machine
2032 * num - number of pages
2033 * paddr_min - Physical address minimum
2034 * memslot - Memory region to allocate page from
2039 * Starting physical address
2041 * Within the VM specified by vm, locates a range of available physical
2042 * pages at or above paddr_min. If found, the pages are marked as in use
2043 * and their base address is returned. A TEST_ASSERT failure occurs if
2044 * not enough pages are available at or above paddr_min.
2046 vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
2047 vm_paddr_t paddr_min, uint32_t memslot)
2049 struct userspace_mem_region *region;
2050 sparsebit_idx_t pg, base;
2052 TEST_ASSERT(num > 0, "Must allocate at least one page");
2054 TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
2055 "not divisible by page size.\n"
2056 " paddr_min: 0x%lx page_size: 0x%x",
2057 paddr_min, vm->page_size);
2059 region = memslot2region(vm, memslot);
2060 base = pg = paddr_min >> vm->page_shift;
2063 for (; pg < base + num; ++pg) {
2064 if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
2065 base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
2069 } while (pg && pg != base + num);
2072 fprintf(stderr, "No guest physical page available, "
2073 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
2074 paddr_min, vm->page_size, memslot);
2075 fputs("---- vm dump ----\n", stderr);
2076 vm_dump(stderr, vm, 2);
2080 for (pg = base; pg < base + num; ++pg)
2081 sparsebit_clear(region->unused_phy_pages, pg);
2083 return base * vm->page_size;
2086 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
2089 return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
2093 * Address Guest Virtual to Host Virtual
2096 * vm - Virtual Machine
2097 * gva - VM virtual address
2102 * Equivalent host virtual address
2104 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
2106 return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
2110 * Is Unrestricted Guest
2113 * vm - Virtual Machine
2117 * Return: True if the unrestricted guest is set to 'Y', otherwise return false.
2119 * Check if the unrestricted guest flag is enabled.
2121 bool vm_is_unrestricted_guest(struct kvm_vm *vm)
2128 /* Ensure that the KVM vendor-specific module is loaded. */
2129 close(open_kvm_dev_path_or_exit());
2132 f = fopen("/sys/module/kvm_intel/parameters/unrestricted_guest", "r");
2134 count = fread(&val, sizeof(char), 1, f);
2135 TEST_ASSERT(count == 1, "Unable to read from param file.");
2142 unsigned int vm_get_page_size(struct kvm_vm *vm)
2144 return vm->page_size;
2147 unsigned int vm_get_page_shift(struct kvm_vm *vm)
2149 return vm->page_shift;
2152 uint64_t vm_get_max_gfn(struct kvm_vm *vm)
2157 int vm_get_fd(struct kvm_vm *vm)
2162 static unsigned int vm_calc_num_pages(unsigned int num_pages,
2163 unsigned int page_shift,
2164 unsigned int new_page_shift,
2167 unsigned int n = 1 << (new_page_shift - page_shift);
2169 if (page_shift >= new_page_shift)
2170 return num_pages * (1 << (page_shift - new_page_shift));
2172 return num_pages / n + !!(ceil && num_pages % n);
2175 static inline int getpageshift(void)
2177 return __builtin_ffs(getpagesize()) - 1;
2181 vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
2183 return vm_calc_num_pages(num_guest_pages,
2184 vm_guest_mode_params[mode].page_shift,
2185 getpageshift(), true);
2189 vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages)
2191 return vm_calc_num_pages(num_host_pages, getpageshift(),
2192 vm_guest_mode_params[mode].page_shift, false);
2195 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size)
2198 n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size);
2199 return vm_adjust_num_guest_pages(mode, n);