1 // SPDX-License-Identifier: GPL-2.0-only
3 * tools/testing/selftests/kvm/lib/kvm_util.c
5 * Copyright (C) 2018, Google LLC.
8 #define _GNU_SOURCE /* for program_invocation_name */
11 #include "kvm_util_internal.h"
12 #include "processor.h"
16 #include <sys/types.h>
19 #include <linux/kernel.h>
21 #define KVM_UTIL_MIN_PFN 2
23 static int vcpu_mmap_sz(void);
25 /* Aligns x up to the next multiple of size. Size must be a power of 2. */
26 static void *align(void *x, size_t size)
28 size_t mask = size - 1;
29 TEST_ASSERT(size != 0 && !(size & (size - 1)),
30 "size not a power of 2: %lu", size);
31 return (void *) (((size_t) x + mask) & ~mask);
43 * On success, the Value corresponding to the capability (KVM_CAP_*)
44 * specified by the value of cap. On failure a TEST_ASSERT failure
47 * Looks up and returns the value corresponding to the capability
48 * (KVM_CAP_*) given by cap.
50 int kvm_check_cap(long cap)
55 kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
59 ret = ioctl(kvm_fd, KVM_CHECK_EXTENSION, cap);
60 TEST_ASSERT(ret != -1, "KVM_CHECK_EXTENSION IOCTL failed,\n"
61 " rc: %i errno: %i", ret, errno);
68 /* VM Enable Capability
71 * vm - Virtual Machine
76 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
78 * Enables a capability (KVM_CAP_*) on the VM.
80 int vm_enable_cap(struct kvm_vm *vm, struct kvm_enable_cap *cap)
84 ret = ioctl(vm->fd, KVM_ENABLE_CAP, cap);
85 TEST_ASSERT(ret == 0, "KVM_ENABLE_CAP IOCTL failed,\n"
86 " rc: %i errno: %i", ret, errno);
91 /* VCPU Enable Capability
94 * vm - Virtual Machine
100 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
102 * Enables a capability (KVM_CAP_*) on the VCPU.
104 int vcpu_enable_cap(struct kvm_vm *vm, uint32_t vcpu_id,
105 struct kvm_enable_cap *cap)
107 struct vcpu *vcpu = vcpu_find(vm, vcpu_id);
110 TEST_ASSERT(vcpu, "cannot find vcpu %d", vcpu_id);
112 r = ioctl(vcpu->fd, KVM_ENABLE_CAP, cap);
113 TEST_ASSERT(!r, "KVM_ENABLE_CAP vCPU ioctl failed,\n"
114 " rc: %i, errno: %i", r, errno);
119 void vm_enable_dirty_ring(struct kvm_vm *vm, uint32_t ring_size)
121 struct kvm_enable_cap cap = { 0 };
123 cap.cap = KVM_CAP_DIRTY_LOG_RING;
124 cap.args[0] = ring_size;
125 vm_enable_cap(vm, &cap);
126 vm->dirty_ring_size = ring_size;
129 static void vm_open(struct kvm_vm *vm, int perm)
131 vm->kvm_fd = open(KVM_DEV_PATH, perm);
135 if (!kvm_check_cap(KVM_CAP_IMMEDIATE_EXIT)) {
136 print_skip("immediate_exit not available");
140 vm->fd = ioctl(vm->kvm_fd, KVM_CREATE_VM, vm->type);
141 TEST_ASSERT(vm->fd >= 0, "KVM_CREATE_VM ioctl failed, "
142 "rc: %i errno: %i", vm->fd, errno);
145 const char *vm_guest_mode_string(uint32_t i)
147 static const char * const strings[] = {
148 [VM_MODE_P52V48_4K] = "PA-bits:52, VA-bits:48, 4K pages",
149 [VM_MODE_P52V48_64K] = "PA-bits:52, VA-bits:48, 64K pages",
150 [VM_MODE_P48V48_4K] = "PA-bits:48, VA-bits:48, 4K pages",
151 [VM_MODE_P48V48_64K] = "PA-bits:48, VA-bits:48, 64K pages",
152 [VM_MODE_P40V48_4K] = "PA-bits:40, VA-bits:48, 4K pages",
153 [VM_MODE_P40V48_64K] = "PA-bits:40, VA-bits:48, 64K pages",
154 [VM_MODE_PXXV48_4K] = "PA-bits:ANY, VA-bits:48, 4K pages",
156 _Static_assert(sizeof(strings)/sizeof(char *) == NUM_VM_MODES,
157 "Missing new mode strings?");
159 TEST_ASSERT(i < NUM_VM_MODES, "Guest mode ID %d too big", i);
164 const struct vm_guest_mode_params vm_guest_mode_params[] = {
165 { 52, 48, 0x1000, 12 },
166 { 52, 48, 0x10000, 16 },
167 { 48, 48, 0x1000, 12 },
168 { 48, 48, 0x10000, 16 },
169 { 40, 48, 0x1000, 12 },
170 { 40, 48, 0x10000, 16 },
171 { 0, 0, 0x1000, 12 },
173 _Static_assert(sizeof(vm_guest_mode_params)/sizeof(struct vm_guest_mode_params) == NUM_VM_MODES,
174 "Missing new mode params?");
180 * mode - VM Mode (e.g. VM_MODE_P52V48_4K)
181 * phy_pages - Physical memory pages
187 * Pointer to opaque structure that describes the created VM.
189 * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
190 * When phy_pages is non-zero, a memory region of phy_pages physical pages
191 * is created and mapped starting at guest physical address 0. The file
192 * descriptor to control the created VM is created with the permissions
193 * given by perm (e.g. O_RDWR).
195 struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
199 pr_debug("%s: mode='%s' pages='%ld' perm='%d'\n", __func__,
200 vm_guest_mode_string(mode), phy_pages, perm);
202 vm = calloc(1, sizeof(*vm));
203 TEST_ASSERT(vm != NULL, "Insufficient Memory");
205 INIT_LIST_HEAD(&vm->vcpus);
206 vm->regions.gpa_tree = RB_ROOT;
207 vm->regions.hva_tree = RB_ROOT;
208 hash_init(vm->regions.slot_hash);
213 vm->pa_bits = vm_guest_mode_params[mode].pa_bits;
214 vm->va_bits = vm_guest_mode_params[mode].va_bits;
215 vm->page_size = vm_guest_mode_params[mode].page_size;
216 vm->page_shift = vm_guest_mode_params[mode].page_shift;
218 /* Setup mode specific traits. */
220 case VM_MODE_P52V48_4K:
221 vm->pgtable_levels = 4;
223 case VM_MODE_P52V48_64K:
224 vm->pgtable_levels = 3;
226 case VM_MODE_P48V48_4K:
227 vm->pgtable_levels = 4;
229 case VM_MODE_P48V48_64K:
230 vm->pgtable_levels = 3;
232 case VM_MODE_P40V48_4K:
233 vm->pgtable_levels = 4;
235 case VM_MODE_P40V48_64K:
236 vm->pgtable_levels = 3;
238 case VM_MODE_PXXV48_4K:
240 kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits);
242 * Ignore KVM support for 5-level paging (vm->va_bits == 57),
243 * it doesn't take effect unless a CR4.LA57 is set, which it
244 * isn't for this VM_MODE.
246 TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57,
247 "Linear address width (%d bits) not supported",
249 pr_debug("Guest physical address width detected: %d\n",
251 vm->pgtable_levels = 4;
254 TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms");
258 TEST_FAIL("Unknown guest mode, mode: 0x%x", mode);
262 if (vm->pa_bits != 40)
263 vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
268 /* Limit to VA-bit canonical virtual addresses. */
269 vm->vpages_valid = sparsebit_alloc();
270 sparsebit_set_num(vm->vpages_valid,
271 0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
272 sparsebit_set_num(vm->vpages_valid,
273 (~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
274 (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
276 /* Limit physical addresses to PA-bits. */
277 vm->max_gfn = ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
279 /* Allocate and setup memory for guest. */
280 vm->vpages_mapped = sparsebit_alloc();
282 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
288 struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus,
289 uint64_t extra_mem_pages, uint32_t num_percpu_pages,
290 void *guest_code, uint32_t vcpuids[])
292 /* The maximum page table size for a memory region will be when the
293 * smallest pages are used. Considering each page contains x page
294 * table descriptors, the total extra size for page tables (for extra
295 * N pages) will be: N/x+N/x^2+N/x^3+... which is definitely smaller
298 uint64_t vcpu_pages = (DEFAULT_STACK_PGS + num_percpu_pages) * nr_vcpus;
299 uint64_t extra_pg_pages = (extra_mem_pages + vcpu_pages) / PTES_PER_MIN_PAGE * 2;
300 uint64_t pages = DEFAULT_GUEST_PHY_PAGES + extra_mem_pages + vcpu_pages + extra_pg_pages;
304 TEST_ASSERT(nr_vcpus <= kvm_check_cap(KVM_CAP_MAX_VCPUS),
305 "nr_vcpus = %d too large for host, max-vcpus = %d",
306 nr_vcpus, kvm_check_cap(KVM_CAP_MAX_VCPUS));
308 pages = vm_adjust_num_guest_pages(mode, pages);
309 vm = vm_create(mode, pages, O_RDWR);
311 kvm_vm_elf_load(vm, program_invocation_name, 0, 0);
314 vm_create_irqchip(vm);
317 for (i = 0; i < nr_vcpus; ++i) {
318 uint32_t vcpuid = vcpuids ? vcpuids[i] : i;
320 vm_vcpu_add_default(vm, vcpuid, guest_code);
323 vcpu_set_cpuid(vm, vcpuid, kvm_get_supported_cpuid());
330 struct kvm_vm *vm_create_default_with_vcpus(uint32_t nr_vcpus, uint64_t extra_mem_pages,
331 uint32_t num_percpu_pages, void *guest_code,
334 return vm_create_with_vcpus(VM_MODE_DEFAULT, nr_vcpus, extra_mem_pages,
335 num_percpu_pages, guest_code, vcpuids);
338 struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages,
341 return vm_create_default_with_vcpus(1, extra_mem_pages, 0, guest_code,
342 (uint32_t []){ vcpuid });
349 * vm - VM that has been released before
354 * Reopens the file descriptors associated to the VM and reinstates the
355 * global state, such as the irqchip and the memory regions that are mapped
358 void kvm_vm_restart(struct kvm_vm *vmp, int perm)
361 struct userspace_mem_region *region;
364 if (vmp->has_irqchip)
365 vm_create_irqchip(vmp);
367 hash_for_each(vmp->regions.slot_hash, ctr, region, slot_node) {
368 int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
369 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
370 " rc: %i errno: %i\n"
371 " slot: %u flags: 0x%x\n"
372 " guest_phys_addr: 0x%llx size: 0x%llx",
373 ret, errno, region->region.slot,
374 region->region.flags,
375 region->region.guest_phys_addr,
376 region->region.memory_size);
380 void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
382 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
385 ret = ioctl(vm->fd, KVM_GET_DIRTY_LOG, &args);
386 TEST_ASSERT(ret == 0, "%s: KVM_GET_DIRTY_LOG failed: %s",
387 __func__, strerror(-ret));
390 void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
391 uint64_t first_page, uint32_t num_pages)
393 struct kvm_clear_dirty_log args = { .dirty_bitmap = log, .slot = slot,
394 .first_page = first_page,
395 .num_pages = num_pages };
398 ret = ioctl(vm->fd, KVM_CLEAR_DIRTY_LOG, &args);
399 TEST_ASSERT(ret == 0, "%s: KVM_CLEAR_DIRTY_LOG failed: %s",
400 __func__, strerror(-ret));
403 uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm)
405 return ioctl(vm->fd, KVM_RESET_DIRTY_RINGS);
409 * Userspace Memory Region Find
412 * vm - Virtual Machine
413 * start - Starting VM physical address
414 * end - Ending VM physical address, inclusive.
419 * Pointer to overlapping region, NULL if no such region.
421 * Searches for a region with any physical memory that overlaps with
422 * any portion of the guest physical addresses from start to end
423 * inclusive. If multiple overlapping regions exist, a pointer to any
424 * of the regions is returned. Null is returned only when no overlapping
427 static struct userspace_mem_region *
428 userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
430 struct rb_node *node;
432 for (node = vm->regions.gpa_tree.rb_node; node; ) {
433 struct userspace_mem_region *region =
434 container_of(node, struct userspace_mem_region, gpa_node);
435 uint64_t existing_start = region->region.guest_phys_addr;
436 uint64_t existing_end = region->region.guest_phys_addr
437 + region->region.memory_size - 1;
438 if (start <= existing_end && end >= existing_start)
441 if (start < existing_start)
442 node = node->rb_left;
444 node = node->rb_right;
451 * KVM Userspace Memory Region Find
454 * vm - Virtual Machine
455 * start - Starting VM physical address
456 * end - Ending VM physical address, inclusive.
461 * Pointer to overlapping region, NULL if no such region.
463 * Public interface to userspace_mem_region_find. Allows tests to look up
464 * the memslot datastructure for a given range of guest physical memory.
466 struct kvm_userspace_memory_region *
467 kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
470 struct userspace_mem_region *region;
472 region = userspace_mem_region_find(vm, start, end);
476 return ®ion->region;
483 * vm - Virtual Machine
489 * Pointer to VCPU structure
491 * Locates a vcpu structure that describes the VCPU specified by vcpuid and
492 * returns a pointer to it. Returns NULL if the VM doesn't contain a VCPU
493 * for the specified vcpuid.
495 struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid)
499 list_for_each_entry(vcpu, &vm->vcpus, list) {
500 if (vcpu->id == vcpuid)
511 * vcpu - VCPU to remove
515 * Return: None, TEST_ASSERT failures for all error conditions
517 * Removes a vCPU from a VM and frees its resources.
519 static void vm_vcpu_rm(struct kvm_vm *vm, struct vcpu *vcpu)
523 if (vcpu->dirty_gfns) {
524 ret = munmap(vcpu->dirty_gfns, vm->dirty_ring_size);
525 TEST_ASSERT(ret == 0, "munmap of VCPU dirty ring failed, "
526 "rc: %i errno: %i", ret, errno);
527 vcpu->dirty_gfns = NULL;
530 ret = munmap(vcpu->state, vcpu_mmap_sz());
531 TEST_ASSERT(ret == 0, "munmap of VCPU fd failed, rc: %i "
532 "errno: %i", ret, errno);
533 ret = close(vcpu->fd);
534 TEST_ASSERT(ret == 0, "Close of VCPU fd failed, rc: %i "
535 "errno: %i", ret, errno);
537 list_del(&vcpu->list);
541 void kvm_vm_release(struct kvm_vm *vmp)
543 struct vcpu *vcpu, *tmp;
546 list_for_each_entry_safe(vcpu, tmp, &vmp->vcpus, list)
547 vm_vcpu_rm(vmp, vcpu);
549 ret = close(vmp->fd);
550 TEST_ASSERT(ret == 0, "Close of vm fd failed,\n"
551 " vmp->fd: %i rc: %i errno: %i", vmp->fd, ret, errno);
553 ret = close(vmp->kvm_fd);
554 TEST_ASSERT(ret == 0, "Close of /dev/kvm fd failed,\n"
555 " vmp->kvm_fd: %i rc: %i errno: %i", vmp->kvm_fd, ret, errno);
558 static void __vm_mem_region_delete(struct kvm_vm *vm,
559 struct userspace_mem_region *region,
565 rb_erase(®ion->gpa_node, &vm->regions.gpa_tree);
566 rb_erase(®ion->hva_node, &vm->regions.hva_tree);
567 hash_del(®ion->slot_node);
570 region->region.memory_size = 0;
571 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
572 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, "
573 "rc: %i errno: %i", ret, errno);
575 sparsebit_free(®ion->unused_phy_pages);
576 ret = munmap(region->mmap_start, region->mmap_size);
577 TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i", ret, errno);
583 * Destroys and frees the VM pointed to by vmp.
585 void kvm_vm_free(struct kvm_vm *vmp)
588 struct hlist_node *node;
589 struct userspace_mem_region *region;
594 /* Free userspace_mem_regions. */
595 hash_for_each_safe(vmp->regions.slot_hash, ctr, node, region, slot_node)
596 __vm_mem_region_delete(vmp, region, false);
598 /* Free sparsebit arrays. */
599 sparsebit_free(&vmp->vpages_valid);
600 sparsebit_free(&vmp->vpages_mapped);
604 /* Free the structure describing the VM. */
609 * Memory Compare, host virtual to guest virtual
612 * hva - Starting host virtual address
613 * vm - Virtual Machine
614 * gva - Starting guest virtual address
615 * len - number of bytes to compare
619 * Input/Output Args: None
622 * Returns 0 if the bytes starting at hva for a length of len
623 * are equal the guest virtual bytes starting at gva. Returns
624 * a value < 0, if bytes at hva are less than those at gva.
625 * Otherwise a value > 0 is returned.
627 * Compares the bytes starting at the host virtual address hva, for
628 * a length of len, to the guest bytes starting at the guest virtual
629 * address given by gva.
631 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, vm_vaddr_t gva, size_t len)
636 * Compare a batch of bytes until either a match is found
637 * or all the bytes have been compared.
639 for (uintptr_t offset = 0; offset < len; offset += amt) {
640 uintptr_t ptr1 = (uintptr_t)hva + offset;
643 * Determine host address for guest virtual address
646 uintptr_t ptr2 = (uintptr_t)addr_gva2hva(vm, gva + offset);
649 * Determine amount to compare on this pass.
650 * Don't allow the comparsion to cross a page boundary.
653 if ((ptr1 >> vm->page_shift) != ((ptr1 + amt) >> vm->page_shift))
654 amt = vm->page_size - (ptr1 % vm->page_size);
655 if ((ptr2 >> vm->page_shift) != ((ptr2 + amt) >> vm->page_shift))
656 amt = vm->page_size - (ptr2 % vm->page_size);
658 assert((ptr1 >> vm->page_shift) == ((ptr1 + amt - 1) >> vm->page_shift));
659 assert((ptr2 >> vm->page_shift) == ((ptr2 + amt - 1) >> vm->page_shift));
662 * Perform the comparison. If there is a difference
663 * return that result to the caller, otherwise need
664 * to continue on looking for a mismatch.
666 int ret = memcmp((void *)ptr1, (void *)ptr2, amt);
672 * No mismatch found. Let the caller know the two memory
678 static void vm_userspace_mem_region_gpa_insert(struct rb_root *gpa_tree,
679 struct userspace_mem_region *region)
681 struct rb_node **cur, *parent;
683 for (cur = &gpa_tree->rb_node, parent = NULL; *cur; ) {
684 struct userspace_mem_region *cregion;
686 cregion = container_of(*cur, typeof(*cregion), gpa_node);
688 if (region->region.guest_phys_addr <
689 cregion->region.guest_phys_addr)
690 cur = &(*cur)->rb_left;
692 TEST_ASSERT(region->region.guest_phys_addr !=
693 cregion->region.guest_phys_addr,
694 "Duplicate GPA in region tree");
696 cur = &(*cur)->rb_right;
700 rb_link_node(®ion->gpa_node, parent, cur);
701 rb_insert_color(®ion->gpa_node, gpa_tree);
704 static void vm_userspace_mem_region_hva_insert(struct rb_root *hva_tree,
705 struct userspace_mem_region *region)
707 struct rb_node **cur, *parent;
709 for (cur = &hva_tree->rb_node, parent = NULL; *cur; ) {
710 struct userspace_mem_region *cregion;
712 cregion = container_of(*cur, typeof(*cregion), hva_node);
714 if (region->host_mem < cregion->host_mem)
715 cur = &(*cur)->rb_left;
717 TEST_ASSERT(region->host_mem !=
719 "Duplicate HVA in region tree");
721 cur = &(*cur)->rb_right;
725 rb_link_node(®ion->hva_node, parent, cur);
726 rb_insert_color(®ion->hva_node, hva_tree);
730 * VM Userspace Memory Region Add
733 * vm - Virtual Machine
734 * backing_src - Storage source for this region.
735 * NULL to use anonymous memory.
736 * guest_paddr - Starting guest physical address
737 * slot - KVM region slot
738 * npages - Number of physical pages
739 * flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES)
745 * Allocates a memory area of the number of pages specified by npages
746 * and maps it to the VM specified by vm, at a starting physical address
747 * given by guest_paddr. The region is created with a KVM region slot
748 * given by slot, which must be unique and < KVM_MEM_SLOTS_NUM. The
749 * region is created with the flags given by flags.
751 void vm_userspace_mem_region_add(struct kvm_vm *vm,
752 enum vm_mem_backing_src_type src_type,
753 uint64_t guest_paddr, uint32_t slot, uint64_t npages,
757 struct userspace_mem_region *region;
758 size_t backing_src_pagesz = get_backing_src_pagesz(src_type);
761 TEST_ASSERT(vm_adjust_num_guest_pages(vm->mode, npages) == npages,
762 "Number of guest pages is not compatible with the host. "
763 "Try npages=%d", vm_adjust_num_guest_pages(vm->mode, npages));
765 TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
766 "address not on a page boundary.\n"
767 " guest_paddr: 0x%lx vm->page_size: 0x%x",
768 guest_paddr, vm->page_size);
769 TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
770 <= vm->max_gfn, "Physical range beyond maximum "
771 "supported physical address,\n"
772 " guest_paddr: 0x%lx npages: 0x%lx\n"
773 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",
774 guest_paddr, npages, vm->max_gfn, vm->page_size);
777 * Confirm a mem region with an overlapping address doesn't
780 region = (struct userspace_mem_region *) userspace_mem_region_find(
781 vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
783 TEST_FAIL("overlapping userspace_mem_region already "
785 " requested guest_paddr: 0x%lx npages: 0x%lx "
787 " existing guest_paddr: 0x%lx size: 0x%lx",
788 guest_paddr, npages, vm->page_size,
789 (uint64_t) region->region.guest_phys_addr,
790 (uint64_t) region->region.memory_size);
792 /* Confirm no region with the requested slot already exists. */
793 hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
795 if (region->region.slot != slot)
798 TEST_FAIL("A mem region with the requested slot "
800 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
801 " existing slot: %u paddr: 0x%lx size: 0x%lx",
802 slot, guest_paddr, npages,
804 (uint64_t) region->region.guest_phys_addr,
805 (uint64_t) region->region.memory_size);
808 /* Allocate and initialize new mem region structure. */
809 region = calloc(1, sizeof(*region));
810 TEST_ASSERT(region != NULL, "Insufficient Memory");
811 region->mmap_size = npages * vm->page_size;
814 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
815 alignment = 0x100000;
820 if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
821 alignment = max(backing_src_pagesz, alignment);
823 /* Add enough memory to align up if necessary */
825 region->mmap_size += alignment;
827 region->mmap_start = mmap(NULL, region->mmap_size,
828 PROT_READ | PROT_WRITE,
829 MAP_PRIVATE | MAP_ANONYMOUS
830 | vm_mem_backing_src_alias(src_type)->flag,
832 TEST_ASSERT(region->mmap_start != MAP_FAILED,
833 "test_malloc failed, mmap_start: %p errno: %i",
834 region->mmap_start, errno);
836 /* Align host address */
837 region->host_mem = align(region->mmap_start, alignment);
839 /* As needed perform madvise */
840 if ((src_type == VM_MEM_SRC_ANONYMOUS ||
841 src_type == VM_MEM_SRC_ANONYMOUS_THP) && thp_configured()) {
842 ret = madvise(region->host_mem, npages * vm->page_size,
843 src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
844 TEST_ASSERT(ret == 0, "madvise failed, addr: %p length: 0x%lx src_type: %s",
845 region->host_mem, npages * vm->page_size,
846 vm_mem_backing_src_alias(src_type)->name);
849 region->unused_phy_pages = sparsebit_alloc();
850 sparsebit_set_num(region->unused_phy_pages,
851 guest_paddr >> vm->page_shift, npages);
852 region->region.slot = slot;
853 region->region.flags = flags;
854 region->region.guest_phys_addr = guest_paddr;
855 region->region.memory_size = npages * vm->page_size;
856 region->region.userspace_addr = (uintptr_t) region->host_mem;
857 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
858 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
859 " rc: %i errno: %i\n"
860 " slot: %u flags: 0x%x\n"
861 " guest_phys_addr: 0x%lx size: 0x%lx",
862 ret, errno, slot, flags,
863 guest_paddr, (uint64_t) region->region.memory_size);
865 /* Add to quick lookup data structures */
866 vm_userspace_mem_region_gpa_insert(&vm->regions.gpa_tree, region);
867 vm_userspace_mem_region_hva_insert(&vm->regions.hva_tree, region);
868 hash_add(vm->regions.slot_hash, ®ion->slot_node, slot);
875 * vm - Virtual Machine
876 * memslot - KVM memory slot ID
881 * Pointer to memory region structure that describe memory region
882 * using kvm memory slot ID given by memslot. TEST_ASSERT failure
883 * on error (e.g. currently no memory region using memslot as a KVM
886 struct userspace_mem_region *
887 memslot2region(struct kvm_vm *vm, uint32_t memslot)
889 struct userspace_mem_region *region;
891 hash_for_each_possible(vm->regions.slot_hash, region, slot_node,
893 if (region->region.slot == memslot)
896 fprintf(stderr, "No mem region with the requested slot found,\n"
897 " requested slot: %u\n", memslot);
898 fputs("---- vm dump ----\n", stderr);
899 vm_dump(stderr, vm, 2);
900 TEST_FAIL("Mem region not found");
905 * VM Memory Region Flags Set
908 * vm - Virtual Machine
909 * flags - Starting guest physical address
915 * Sets the flags of the memory region specified by the value of slot,
916 * to the values given by flags.
918 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
921 struct userspace_mem_region *region;
923 region = memslot2region(vm, slot);
925 region->region.flags = flags;
927 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
929 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
930 " rc: %i errno: %i slot: %u flags: 0x%x",
931 ret, errno, slot, flags);
935 * VM Memory Region Move
938 * vm - Virtual Machine
939 * slot - Slot of the memory region to move
940 * new_gpa - Starting guest physical address
946 * Change the gpa of a memory region.
948 void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa)
950 struct userspace_mem_region *region;
953 region = memslot2region(vm, slot);
955 region->region.guest_phys_addr = new_gpa;
957 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
959 TEST_ASSERT(!ret, "KVM_SET_USER_MEMORY_REGION failed\n"
960 "ret: %i errno: %i slot: %u new_gpa: 0x%lx",
961 ret, errno, slot, new_gpa);
965 * VM Memory Region Delete
968 * vm - Virtual Machine
969 * slot - Slot of the memory region to delete
975 * Delete a memory region.
977 void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot)
979 __vm_mem_region_delete(vm, memslot2region(vm, slot), true);
992 * Returns the size of the structure pointed to by the return value
995 static int vcpu_mmap_sz(void)
999 dev_fd = open(KVM_DEV_PATH, O_RDONLY);
1003 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
1004 TEST_ASSERT(ret >= sizeof(struct kvm_run),
1005 "%s KVM_GET_VCPU_MMAP_SIZE ioctl failed, rc: %i errno: %i",
1006 __func__, ret, errno);
1017 * vm - Virtual Machine
1024 * Adds a virtual CPU to the VM specified by vm with the ID given by vcpuid.
1025 * No additional VCPU setup is done.
1027 void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid)
1031 /* Confirm a vcpu with the specified id doesn't already exist. */
1032 vcpu = vcpu_find(vm, vcpuid);
1034 TEST_FAIL("vcpu with the specified id "
1036 " requested vcpuid: %u\n"
1037 " existing vcpuid: %u state: %p",
1038 vcpuid, vcpu->id, vcpu->state);
1040 /* Allocate and initialize new vcpu structure. */
1041 vcpu = calloc(1, sizeof(*vcpu));
1042 TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
1044 vcpu->fd = ioctl(vm->fd, KVM_CREATE_VCPU, vcpuid);
1045 TEST_ASSERT(vcpu->fd >= 0, "KVM_CREATE_VCPU failed, rc: %i errno: %i",
1048 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->state), "vcpu mmap size "
1049 "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",
1050 vcpu_mmap_sz(), sizeof(*vcpu->state));
1051 vcpu->state = (struct kvm_run *) mmap(NULL, vcpu_mmap_sz(),
1052 PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0);
1053 TEST_ASSERT(vcpu->state != MAP_FAILED, "mmap vcpu_state failed, "
1054 "vcpu id: %u errno: %i", vcpuid, errno);
1056 /* Add to linked-list of VCPUs. */
1057 list_add(&vcpu->list, &vm->vcpus);
1061 * VM Virtual Address Unused Gap
1064 * vm - Virtual Machine
1066 * vaddr_min - Minimum Virtual Address
1071 * Lowest virtual address at or below vaddr_min, with at least
1072 * sz unused bytes. TEST_ASSERT failure if no area of at least
1073 * size sz is available.
1075 * Within the VM specified by vm, locates the lowest starting virtual
1076 * address >= vaddr_min, that has at least sz unallocated bytes. A
1077 * TEST_ASSERT failure occurs for invalid input or no area of at least
1078 * sz unallocated bytes >= vaddr_min is available.
1080 static vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
1081 vm_vaddr_t vaddr_min)
1083 uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
1085 /* Determine lowest permitted virtual page index. */
1086 uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
1087 if ((pgidx_start * vm->page_size) < vaddr_min)
1090 /* Loop over section with enough valid virtual page indexes. */
1091 if (!sparsebit_is_set_num(vm->vpages_valid,
1092 pgidx_start, pages))
1093 pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
1094 pgidx_start, pages);
1097 * Are there enough unused virtual pages available at
1098 * the currently proposed starting virtual page index.
1099 * If not, adjust proposed starting index to next
1102 if (sparsebit_is_clear_num(vm->vpages_mapped,
1103 pgidx_start, pages))
1105 pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
1106 pgidx_start, pages);
1107 if (pgidx_start == 0)
1111 * If needed, adjust proposed starting virtual address,
1112 * to next range of valid virtual addresses.
1114 if (!sparsebit_is_set_num(vm->vpages_valid,
1115 pgidx_start, pages)) {
1116 pgidx_start = sparsebit_next_set_num(
1117 vm->vpages_valid, pgidx_start, pages);
1118 if (pgidx_start == 0)
1121 } while (pgidx_start != 0);
1124 TEST_FAIL("No vaddr of specified pages available, pages: 0x%lx", pages);
1130 TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
1131 pgidx_start, pages),
1132 "Unexpected, invalid virtual page index range,\n"
1133 " pgidx_start: 0x%lx\n"
1135 pgidx_start, pages);
1136 TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
1137 pgidx_start, pages),
1138 "Unexpected, pages already mapped,\n"
1139 " pgidx_start: 0x%lx\n"
1141 pgidx_start, pages);
1143 return pgidx_start * vm->page_size;
1147 * VM Virtual Address Allocate
1150 * vm - Virtual Machine
1151 * sz - Size in bytes
1152 * vaddr_min - Minimum starting virtual address
1153 * data_memslot - Memory region slot for data pages
1154 * pgd_memslot - Memory region slot for new virtual translation tables
1159 * Starting guest virtual address
1161 * Allocates at least sz bytes within the virtual address space of the vm
1162 * given by vm. The allocated bytes are mapped to a virtual address >=
1163 * the address given by vaddr_min. Note that each allocation uses a
1164 * a unique set of pages, with the minimum real allocation being at least
1167 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
1168 uint32_t data_memslot, uint32_t pgd_memslot)
1170 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
1172 virt_pgd_alloc(vm, pgd_memslot);
1173 vm_paddr_t paddr = vm_phy_pages_alloc(vm, pages,
1174 KVM_UTIL_MIN_PFN * vm->page_size,
1178 * Find an unused range of virtual page addresses of at least
1181 vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
1183 /* Map the virtual pages. */
1184 for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
1185 pages--, vaddr += vm->page_size, paddr += vm->page_size) {
1187 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
1189 sparsebit_set(vm->vpages_mapped,
1190 vaddr >> vm->page_shift);
1197 * Map a range of VM virtual address to the VM's physical address
1200 * vm - Virtual Machine
1201 * vaddr - Virtuall address to map
1202 * paddr - VM Physical Address
1203 * npages - The number of pages to map
1204 * pgd_memslot - Memory region slot for new virtual translation tables
1210 * Within the VM given by @vm, creates a virtual translation for
1211 * @npages starting at @vaddr to the page range starting at @paddr.
1213 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
1214 unsigned int npages, uint32_t pgd_memslot)
1216 size_t page_size = vm->page_size;
1217 size_t size = npages * page_size;
1219 TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
1220 TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
1223 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
1230 * Address VM Physical to Host Virtual
1233 * vm - Virtual Machine
1234 * gpa - VM physical address
1239 * Equivalent host virtual address
1241 * Locates the memory region containing the VM physical address given
1242 * by gpa, within the VM given by vm. When found, the host virtual
1243 * address providing the memory to the vm physical address is returned.
1244 * A TEST_ASSERT failure occurs if no region containing gpa exists.
1246 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
1248 struct userspace_mem_region *region;
1250 region = userspace_mem_region_find(vm, gpa, gpa);
1252 TEST_FAIL("No vm physical memory at 0x%lx", gpa);
1256 return (void *)((uintptr_t)region->host_mem
1257 + (gpa - region->region.guest_phys_addr));
1261 * Address Host Virtual to VM Physical
1264 * vm - Virtual Machine
1265 * hva - Host virtual address
1270 * Equivalent VM physical address
1272 * Locates the memory region containing the host virtual address given
1273 * by hva, within the VM given by vm. When found, the equivalent
1274 * VM physical address is returned. A TEST_ASSERT failure occurs if no
1275 * region containing hva exists.
1277 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1279 struct rb_node *node;
1281 for (node = vm->regions.hva_tree.rb_node; node; ) {
1282 struct userspace_mem_region *region =
1283 container_of(node, struct userspace_mem_region, hva_node);
1285 if (hva >= region->host_mem) {
1286 if (hva <= (region->host_mem
1287 + region->region.memory_size - 1))
1288 return (vm_paddr_t)((uintptr_t)
1289 region->region.guest_phys_addr
1290 + (hva - (uintptr_t)region->host_mem));
1292 node = node->rb_right;
1294 node = node->rb_left;
1297 TEST_FAIL("No mapping to a guest physical address, hva: %p", hva);
1302 * VM Create IRQ Chip
1305 * vm - Virtual Machine
1311 * Creates an interrupt controller chip for the VM specified by vm.
1313 void vm_create_irqchip(struct kvm_vm *vm)
1317 ret = ioctl(vm->fd, KVM_CREATE_IRQCHIP, 0);
1318 TEST_ASSERT(ret == 0, "KVM_CREATE_IRQCHIP IOCTL failed, "
1319 "rc: %i errno: %i", ret, errno);
1321 vm->has_irqchip = true;
1328 * vm - Virtual Machine
1334 * Pointer to structure that describes the state of the VCPU.
1336 * Locates and returns a pointer to a structure that describes the
1337 * state of the VCPU with the given vcpuid.
1339 struct kvm_run *vcpu_state(struct kvm_vm *vm, uint32_t vcpuid)
1341 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1342 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1351 * vm - Virtual Machine
1358 * Switch to executing the code for the VCPU given by vcpuid, within the VM
1361 void vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1363 int ret = _vcpu_run(vm, vcpuid);
1364 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1365 "rc: %i errno: %i", ret, errno);
1368 int _vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1370 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1373 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1375 rc = ioctl(vcpu->fd, KVM_RUN, NULL);
1376 } while (rc == -1 && errno == EINTR);
1378 assert_on_unhandled_exception(vm, vcpuid);
1383 int vcpu_get_fd(struct kvm_vm *vm, uint32_t vcpuid)
1385 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1387 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1392 void vcpu_run_complete_io(struct kvm_vm *vm, uint32_t vcpuid)
1394 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1397 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1399 vcpu->state->immediate_exit = 1;
1400 ret = ioctl(vcpu->fd, KVM_RUN, NULL);
1401 vcpu->state->immediate_exit = 0;
1403 TEST_ASSERT(ret == -1 && errno == EINTR,
1404 "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1408 void vcpu_set_guest_debug(struct kvm_vm *vm, uint32_t vcpuid,
1409 struct kvm_guest_debug *debug)
1411 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1412 int ret = ioctl(vcpu->fd, KVM_SET_GUEST_DEBUG, debug);
1414 TEST_ASSERT(ret == 0, "KVM_SET_GUEST_DEBUG failed: %d", ret);
1418 * VM VCPU Set MP State
1421 * vm - Virtual Machine
1423 * mp_state - mp_state to be set
1429 * Sets the MP state of the VCPU given by vcpuid, to the state given
1432 void vcpu_set_mp_state(struct kvm_vm *vm, uint32_t vcpuid,
1433 struct kvm_mp_state *mp_state)
1435 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1438 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1440 ret = ioctl(vcpu->fd, KVM_SET_MP_STATE, mp_state);
1441 TEST_ASSERT(ret == 0, "KVM_SET_MP_STATE IOCTL failed, "
1442 "rc: %i errno: %i", ret, errno);
1446 * VM VCPU Get Reg List
1449 * vm - Virtual Machine
1456 * A pointer to an allocated struct kvm_reg_list
1458 * Get the list of guest registers which are supported for
1459 * KVM_GET_ONE_REG/KVM_SET_ONE_REG calls
1461 struct kvm_reg_list *vcpu_get_reg_list(struct kvm_vm *vm, uint32_t vcpuid)
1463 struct kvm_reg_list reg_list_n = { .n = 0 }, *reg_list;
1466 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_REG_LIST, ®_list_n);
1467 TEST_ASSERT(ret == -1 && errno == E2BIG, "KVM_GET_REG_LIST n=0");
1468 reg_list = calloc(1, sizeof(*reg_list) + reg_list_n.n * sizeof(__u64));
1469 reg_list->n = reg_list_n.n;
1470 vcpu_ioctl(vm, vcpuid, KVM_GET_REG_LIST, reg_list);
1478 * vm - Virtual Machine
1482 * regs - current state of VCPU regs
1486 * Obtains the current register state for the VCPU specified by vcpuid
1487 * and stores it at the location given by regs.
1489 void vcpu_regs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1491 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1494 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1496 ret = ioctl(vcpu->fd, KVM_GET_REGS, regs);
1497 TEST_ASSERT(ret == 0, "KVM_GET_REGS failed, rc: %i errno: %i",
1505 * vm - Virtual Machine
1507 * regs - Values to set VCPU regs to
1513 * Sets the regs of the VCPU specified by vcpuid to the values
1516 void vcpu_regs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1518 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1521 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1523 ret = ioctl(vcpu->fd, KVM_SET_REGS, regs);
1524 TEST_ASSERT(ret == 0, "KVM_SET_REGS failed, rc: %i errno: %i",
1528 #ifdef __KVM_HAVE_VCPU_EVENTS
1529 void vcpu_events_get(struct kvm_vm *vm, uint32_t vcpuid,
1530 struct kvm_vcpu_events *events)
1532 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1535 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1537 ret = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, events);
1538 TEST_ASSERT(ret == 0, "KVM_GET_VCPU_EVENTS, failed, rc: %i errno: %i",
1542 void vcpu_events_set(struct kvm_vm *vm, uint32_t vcpuid,
1543 struct kvm_vcpu_events *events)
1545 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1548 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1550 ret = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, events);
1551 TEST_ASSERT(ret == 0, "KVM_SET_VCPU_EVENTS, failed, rc: %i errno: %i",
1557 void vcpu_nested_state_get(struct kvm_vm *vm, uint32_t vcpuid,
1558 struct kvm_nested_state *state)
1560 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1563 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1565 ret = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, state);
1566 TEST_ASSERT(ret == 0,
1567 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1571 int vcpu_nested_state_set(struct kvm_vm *vm, uint32_t vcpuid,
1572 struct kvm_nested_state *state, bool ignore_error)
1574 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1577 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1579 ret = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, state);
1580 if (!ignore_error) {
1581 TEST_ASSERT(ret == 0,
1582 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1591 * VM VCPU System Regs Get
1594 * vm - Virtual Machine
1598 * sregs - current state of VCPU system regs
1602 * Obtains the current system register state for the VCPU specified by
1603 * vcpuid and stores it at the location given by sregs.
1605 void vcpu_sregs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1607 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1610 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1612 ret = ioctl(vcpu->fd, KVM_GET_SREGS, sregs);
1613 TEST_ASSERT(ret == 0, "KVM_GET_SREGS failed, rc: %i errno: %i",
1618 * VM VCPU System Regs Set
1621 * vm - Virtual Machine
1623 * sregs - Values to set VCPU system regs to
1629 * Sets the system regs of the VCPU specified by vcpuid to the values
1632 void vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1634 int ret = _vcpu_sregs_set(vm, vcpuid, sregs);
1635 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1636 "rc: %i errno: %i", ret, errno);
1639 int _vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1641 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1643 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1645 return ioctl(vcpu->fd, KVM_SET_SREGS, sregs);
1648 void vcpu_fpu_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_fpu *fpu)
1652 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_FPU, fpu);
1653 TEST_ASSERT(ret == 0, "KVM_GET_FPU failed, rc: %i errno: %i (%s)",
1654 ret, errno, strerror(errno));
1657 void vcpu_fpu_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_fpu *fpu)
1661 ret = _vcpu_ioctl(vm, vcpuid, KVM_SET_FPU, fpu);
1662 TEST_ASSERT(ret == 0, "KVM_SET_FPU failed, rc: %i errno: %i (%s)",
1663 ret, errno, strerror(errno));
1666 void vcpu_get_reg(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_one_reg *reg)
1670 ret = _vcpu_ioctl(vm, vcpuid, KVM_GET_ONE_REG, reg);
1671 TEST_ASSERT(ret == 0, "KVM_GET_ONE_REG failed, rc: %i errno: %i (%s)",
1672 ret, errno, strerror(errno));
1675 void vcpu_set_reg(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_one_reg *reg)
1679 ret = _vcpu_ioctl(vm, vcpuid, KVM_SET_ONE_REG, reg);
1680 TEST_ASSERT(ret == 0, "KVM_SET_ONE_REG failed, rc: %i errno: %i (%s)",
1681 ret, errno, strerror(errno));
1688 * vm - Virtual Machine
1690 * cmd - Ioctl number
1691 * arg - Argument to pass to the ioctl
1695 * Issues an arbitrary ioctl on a VCPU fd.
1697 void vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1698 unsigned long cmd, void *arg)
1702 ret = _vcpu_ioctl(vm, vcpuid, cmd, arg);
1703 TEST_ASSERT(ret == 0, "vcpu ioctl %lu failed, rc: %i errno: %i (%s)",
1704 cmd, ret, errno, strerror(errno));
1707 int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1708 unsigned long cmd, void *arg)
1710 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1713 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1715 ret = ioctl(vcpu->fd, cmd, arg);
1720 void *vcpu_map_dirty_ring(struct kvm_vm *vm, uint32_t vcpuid)
1723 uint32_t size = vm->dirty_ring_size;
1725 TEST_ASSERT(size > 0, "Should enable dirty ring first");
1727 vcpu = vcpu_find(vm, vcpuid);
1729 TEST_ASSERT(vcpu, "Cannot find vcpu %u", vcpuid);
1731 if (!vcpu->dirty_gfns) {
1734 addr = mmap(NULL, size, PROT_READ,
1735 MAP_PRIVATE, vcpu->fd,
1736 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1737 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped private");
1739 addr = mmap(NULL, size, PROT_READ | PROT_EXEC,
1740 MAP_PRIVATE, vcpu->fd,
1741 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1742 TEST_ASSERT(addr == MAP_FAILED, "Dirty ring mapped exec");
1744 addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
1745 MAP_SHARED, vcpu->fd,
1746 vm->page_size * KVM_DIRTY_LOG_PAGE_OFFSET);
1747 TEST_ASSERT(addr != MAP_FAILED, "Dirty ring map failed");
1749 vcpu->dirty_gfns = addr;
1750 vcpu->dirty_gfns_count = size / sizeof(struct kvm_dirty_gfn);
1753 return vcpu->dirty_gfns;
1760 * vm - Virtual Machine
1761 * cmd - Ioctl number
1762 * arg - Argument to pass to the ioctl
1766 * Issues an arbitrary ioctl on a VM fd.
1768 void vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1772 ret = _vm_ioctl(vm, cmd, arg);
1773 TEST_ASSERT(ret == 0, "vm ioctl %lu failed, rc: %i errno: %i (%s)",
1774 cmd, ret, errno, strerror(errno));
1777 int _vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1779 return ioctl(vm->fd, cmd, arg);
1786 * vm - Virtual Machine
1787 * cmd - Ioctl number
1788 * arg - Argument to pass to the ioctl
1792 * Issues an arbitrary ioctl on a KVM fd.
1794 void kvm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1798 ret = ioctl(vm->kvm_fd, cmd, arg);
1799 TEST_ASSERT(ret == 0, "KVM ioctl %lu failed, rc: %i errno: %i (%s)",
1800 cmd, ret, errno, strerror(errno));
1803 int _kvm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1805 return ioctl(vm->kvm_fd, cmd, arg);
1812 int _kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
1814 struct kvm_device_attr attribute = {
1820 return ioctl(dev_fd, KVM_HAS_DEVICE_ATTR, &attribute);
1823 int kvm_device_check_attr(int dev_fd, uint32_t group, uint64_t attr)
1825 int ret = _kvm_device_check_attr(dev_fd, group, attr);
1827 TEST_ASSERT(ret >= 0, "KVM_HAS_DEVICE_ATTR failed, rc: %i errno: %i", ret, errno);
1831 int _kvm_create_device(struct kvm_vm *vm, uint64_t type, bool test, int *fd)
1833 struct kvm_create_device create_dev;
1836 create_dev.type = type;
1838 create_dev.flags = test ? KVM_CREATE_DEVICE_TEST : 0;
1839 ret = ioctl(vm_get_fd(vm), KVM_CREATE_DEVICE, &create_dev);
1840 *fd = create_dev.fd;
1844 int kvm_create_device(struct kvm_vm *vm, uint64_t type, bool test)
1848 ret = _kvm_create_device(vm, type, test, &fd);
1851 TEST_ASSERT(ret >= 0,
1852 "KVM_CREATE_DEVICE IOCTL failed, rc: %i errno: %i", ret, errno);
1858 int _kvm_device_access(int dev_fd, uint32_t group, uint64_t attr,
1859 void *val, bool write)
1861 struct kvm_device_attr kvmattr = {
1865 .addr = (uintptr_t)val,
1869 ret = ioctl(dev_fd, write ? KVM_SET_DEVICE_ATTR : KVM_GET_DEVICE_ATTR,
1874 int kvm_device_access(int dev_fd, uint32_t group, uint64_t attr,
1875 void *val, bool write)
1877 int ret = _kvm_device_access(dev_fd, group, attr, val, write);
1879 TEST_ASSERT(ret >= 0, "KVM_SET|GET_DEVICE_ATTR IOCTL failed, rc: %i errno: %i", ret, errno);
1887 * vm - Virtual Machine
1888 * indent - Left margin indent amount
1891 * stream - Output FILE stream
1895 * Dumps the current state of the VM given by vm, to the FILE stream
1898 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1901 struct userspace_mem_region *region;
1904 fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1905 fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1906 fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1907 fprintf(stream, "%*sMem Regions:\n", indent, "");
1908 hash_for_each(vm->regions.slot_hash, ctr, region, slot_node) {
1909 fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1910 "host_virt: %p\n", indent + 2, "",
1911 (uint64_t) region->region.guest_phys_addr,
1912 (uint64_t) region->region.memory_size,
1914 fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1915 sparsebit_dump(stream, region->unused_phy_pages, 0);
1917 fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1918 sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1919 fprintf(stream, "%*spgd_created: %u\n", indent, "",
1921 if (vm->pgd_created) {
1922 fprintf(stream, "%*sVirtual Translation Tables:\n",
1924 virt_dump(stream, vm, indent + 4);
1926 fprintf(stream, "%*sVCPUs:\n", indent, "");
1927 list_for_each_entry(vcpu, &vm->vcpus, list)
1928 vcpu_dump(stream, vm, vcpu->id, indent + 2);
1931 /* Known KVM exit reasons */
1932 static struct exit_reason {
1933 unsigned int reason;
1935 } exit_reasons_known[] = {
1936 {KVM_EXIT_UNKNOWN, "UNKNOWN"},
1937 {KVM_EXIT_EXCEPTION, "EXCEPTION"},
1938 {KVM_EXIT_IO, "IO"},
1939 {KVM_EXIT_HYPERCALL, "HYPERCALL"},
1940 {KVM_EXIT_DEBUG, "DEBUG"},
1941 {KVM_EXIT_HLT, "HLT"},
1942 {KVM_EXIT_MMIO, "MMIO"},
1943 {KVM_EXIT_IRQ_WINDOW_OPEN, "IRQ_WINDOW_OPEN"},
1944 {KVM_EXIT_SHUTDOWN, "SHUTDOWN"},
1945 {KVM_EXIT_FAIL_ENTRY, "FAIL_ENTRY"},
1946 {KVM_EXIT_INTR, "INTR"},
1947 {KVM_EXIT_SET_TPR, "SET_TPR"},
1948 {KVM_EXIT_TPR_ACCESS, "TPR_ACCESS"},
1949 {KVM_EXIT_S390_SIEIC, "S390_SIEIC"},
1950 {KVM_EXIT_S390_RESET, "S390_RESET"},
1951 {KVM_EXIT_DCR, "DCR"},
1952 {KVM_EXIT_NMI, "NMI"},
1953 {KVM_EXIT_INTERNAL_ERROR, "INTERNAL_ERROR"},
1954 {KVM_EXIT_OSI, "OSI"},
1955 {KVM_EXIT_PAPR_HCALL, "PAPR_HCALL"},
1956 {KVM_EXIT_DIRTY_RING_FULL, "DIRTY_RING_FULL"},
1957 {KVM_EXIT_X86_RDMSR, "RDMSR"},
1958 {KVM_EXIT_X86_WRMSR, "WRMSR"},
1959 {KVM_EXIT_XEN, "XEN"},
1960 #ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1961 {KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},
1966 * Exit Reason String
1969 * exit_reason - Exit reason
1974 * Constant string pointer describing the exit reason.
1976 * Locates and returns a constant string that describes the KVM exit
1977 * reason given by exit_reason. If no such string is found, a constant
1978 * string of "Unknown" is returned.
1980 const char *exit_reason_str(unsigned int exit_reason)
1984 for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
1985 if (exit_reason == exit_reasons_known[n1].reason)
1986 return exit_reasons_known[n1].name;
1993 * Physical Contiguous Page Allocator
1996 * vm - Virtual Machine
1997 * num - number of pages
1998 * paddr_min - Physical address minimum
1999 * memslot - Memory region to allocate page from
2004 * Starting physical address
2006 * Within the VM specified by vm, locates a range of available physical
2007 * pages at or above paddr_min. If found, the pages are marked as in use
2008 * and their base address is returned. A TEST_ASSERT failure occurs if
2009 * not enough pages are available at or above paddr_min.
2011 vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
2012 vm_paddr_t paddr_min, uint32_t memslot)
2014 struct userspace_mem_region *region;
2015 sparsebit_idx_t pg, base;
2017 TEST_ASSERT(num > 0, "Must allocate at least one page");
2019 TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
2020 "not divisible by page size.\n"
2021 " paddr_min: 0x%lx page_size: 0x%x",
2022 paddr_min, vm->page_size);
2024 region = memslot2region(vm, memslot);
2025 base = pg = paddr_min >> vm->page_shift;
2028 for (; pg < base + num; ++pg) {
2029 if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
2030 base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
2034 } while (pg && pg != base + num);
2037 fprintf(stderr, "No guest physical page available, "
2038 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
2039 paddr_min, vm->page_size, memslot);
2040 fputs("---- vm dump ----\n", stderr);
2041 vm_dump(stderr, vm, 2);
2045 for (pg = base; pg < base + num; ++pg)
2046 sparsebit_clear(region->unused_phy_pages, pg);
2048 return base * vm->page_size;
2051 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
2054 return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
2058 * Address Guest Virtual to Host Virtual
2061 * vm - Virtual Machine
2062 * gva - VM virtual address
2067 * Equivalent host virtual address
2069 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
2071 return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
2075 * Is Unrestricted Guest
2078 * vm - Virtual Machine
2082 * Return: True if the unrestricted guest is set to 'Y', otherwise return false.
2084 * Check if the unrestricted guest flag is enabled.
2086 bool vm_is_unrestricted_guest(struct kvm_vm *vm)
2093 /* Ensure that the KVM vendor-specific module is loaded. */
2094 f = fopen(KVM_DEV_PATH, "r");
2095 TEST_ASSERT(f != NULL, "Error in opening KVM dev file: %d",
2100 f = fopen("/sys/module/kvm_intel/parameters/unrestricted_guest", "r");
2102 count = fread(&val, sizeof(char), 1, f);
2103 TEST_ASSERT(count == 1, "Unable to read from param file.");
2110 unsigned int vm_get_page_size(struct kvm_vm *vm)
2112 return vm->page_size;
2115 unsigned int vm_get_page_shift(struct kvm_vm *vm)
2117 return vm->page_shift;
2120 unsigned int vm_get_max_gfn(struct kvm_vm *vm)
2125 int vm_get_fd(struct kvm_vm *vm)
2130 static unsigned int vm_calc_num_pages(unsigned int num_pages,
2131 unsigned int page_shift,
2132 unsigned int new_page_shift,
2135 unsigned int n = 1 << (new_page_shift - page_shift);
2137 if (page_shift >= new_page_shift)
2138 return num_pages * (1 << (page_shift - new_page_shift));
2140 return num_pages / n + !!(ceil && num_pages % n);
2143 static inline int getpageshift(void)
2145 return __builtin_ffs(getpagesize()) - 1;
2149 vm_num_host_pages(enum vm_guest_mode mode, unsigned int num_guest_pages)
2151 return vm_calc_num_pages(num_guest_pages,
2152 vm_guest_mode_params[mode].page_shift,
2153 getpageshift(), true);
2157 vm_num_guest_pages(enum vm_guest_mode mode, unsigned int num_host_pages)
2159 return vm_calc_num_pages(num_host_pages, getpageshift(),
2160 vm_guest_mode_params[mode].page_shift, false);
2163 unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size)
2166 n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size);
2167 return vm_adjust_num_guest_pages(mode, n);