1 // SPDX-License-Identifier: GPL-2.0-only
3 * tools/testing/selftests/kvm/lib/kvm_util.c
5 * Copyright (C) 2018, Google LLC.
10 #include "kvm_util_internal.h"
14 #include <sys/types.h>
16 #include <linux/kernel.h>
18 #define KVM_UTIL_PGS_PER_HUGEPG 512
19 #define KVM_UTIL_MIN_PFN 2
21 /* Aligns x up to the next multiple of size. Size must be a power of 2. */
22 static void *align(void *x, size_t size)
24 size_t mask = size - 1;
25 TEST_ASSERT(size != 0 && !(size & (size - 1)),
26 "size not a power of 2: %lu", size);
27 return (void *) (((size_t) x + mask) & ~mask);
39 * On success, the Value corresponding to the capability (KVM_CAP_*)
40 * specified by the value of cap. On failure a TEST_ASSERT failure
43 * Looks up and returns the value corresponding to the capability
44 * (KVM_CAP_*) given by cap.
46 int kvm_check_cap(long cap)
51 kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
55 ret = ioctl(kvm_fd, KVM_CHECK_EXTENSION, cap);
56 TEST_ASSERT(ret != -1, "KVM_CHECK_EXTENSION IOCTL failed,\n"
57 " rc: %i errno: %i", ret, errno);
64 /* VM Enable Capability
67 * vm - Virtual Machine
72 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
74 * Enables a capability (KVM_CAP_*) on the VM.
76 int vm_enable_cap(struct kvm_vm *vm, struct kvm_enable_cap *cap)
80 ret = ioctl(vm->fd, KVM_ENABLE_CAP, cap);
81 TEST_ASSERT(ret == 0, "KVM_ENABLE_CAP IOCTL failed,\n"
82 " rc: %i errno: %i", ret, errno);
87 static void vm_open(struct kvm_vm *vm, int perm)
89 vm->kvm_fd = open(KVM_DEV_PATH, perm);
93 if (!kvm_check_cap(KVM_CAP_IMMEDIATE_EXIT)) {
94 fprintf(stderr, "immediate_exit not available, skipping test\n");
98 vm->fd = ioctl(vm->kvm_fd, KVM_CREATE_VM, vm->type);
99 TEST_ASSERT(vm->fd >= 0, "KVM_CREATE_VM ioctl failed, "
100 "rc: %i errno: %i", vm->fd, errno);
103 const char * const vm_guest_mode_string[] = {
104 "PA-bits:52, VA-bits:48, 4K pages",
105 "PA-bits:52, VA-bits:48, 64K pages",
106 "PA-bits:48, VA-bits:48, 4K pages",
107 "PA-bits:48, VA-bits:48, 64K pages",
108 "PA-bits:40, VA-bits:48, 4K pages",
109 "PA-bits:40, VA-bits:48, 64K pages",
111 _Static_assert(sizeof(vm_guest_mode_string)/sizeof(char *) == NUM_VM_MODES,
112 "Missing new mode strings?");
118 * mode - VM Mode (e.g. VM_MODE_P52V48_4K)
119 * phy_pages - Physical memory pages
125 * Pointer to opaque structure that describes the created VM.
127 * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
128 * When phy_pages is non-zero, a memory region of phy_pages physical pages
129 * is created and mapped starting at guest physical address 0. The file
130 * descriptor to control the created VM is created with the permissions
131 * given by perm (e.g. O_RDWR).
133 struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
137 vm = calloc(1, sizeof(*vm));
138 TEST_ASSERT(vm != NULL, "Insufficient Memory");
143 /* Setup mode specific traits. */
145 case VM_MODE_P52V48_4K:
146 vm->pgtable_levels = 4;
149 vm->page_size = 0x1000;
152 case VM_MODE_P52V48_64K:
153 vm->pgtable_levels = 3;
156 vm->page_size = 0x10000;
159 case VM_MODE_P48V48_4K:
160 vm->pgtable_levels = 4;
163 vm->page_size = 0x1000;
166 case VM_MODE_P48V48_64K:
167 vm->pgtable_levels = 3;
170 vm->page_size = 0x10000;
173 case VM_MODE_P40V48_4K:
174 vm->pgtable_levels = 4;
177 vm->page_size = 0x1000;
180 case VM_MODE_P40V48_64K:
181 vm->pgtable_levels = 3;
184 vm->page_size = 0x10000;
188 TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", mode);
192 if (vm->pa_bits != 40)
193 vm->type = KVM_VM_TYPE_ARM_IPA_SIZE(vm->pa_bits);
198 /* Limit to VA-bit canonical virtual addresses. */
199 vm->vpages_valid = sparsebit_alloc();
200 sparsebit_set_num(vm->vpages_valid,
201 0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
202 sparsebit_set_num(vm->vpages_valid,
203 (~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
204 (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
206 /* Limit physical addresses to PA-bits. */
207 vm->max_gfn = ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
209 /* Allocate and setup memory for guest. */
210 vm->vpages_mapped = sparsebit_alloc();
212 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
218 struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
220 return _vm_create(mode, phy_pages, perm);
227 * vm - VM that has been released before
232 * Reopens the file descriptors associated to the VM and reinstates the
233 * global state, such as the irqchip and the memory regions that are mapped
236 void kvm_vm_restart(struct kvm_vm *vmp, int perm)
238 struct userspace_mem_region *region;
241 if (vmp->has_irqchip)
242 vm_create_irqchip(vmp);
244 for (region = vmp->userspace_mem_region_head; region;
245 region = region->next) {
246 int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
247 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
248 " rc: %i errno: %i\n"
249 " slot: %u flags: 0x%x\n"
250 " guest_phys_addr: 0x%lx size: 0x%lx",
251 ret, errno, region->region.slot,
252 region->region.flags,
253 region->region.guest_phys_addr,
254 region->region.memory_size);
258 void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
260 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
263 ret = ioctl(vm->fd, KVM_GET_DIRTY_LOG, &args);
264 TEST_ASSERT(ret == 0, "%s: KVM_GET_DIRTY_LOG failed: %s",
268 void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
269 uint64_t first_page, uint32_t num_pages)
271 struct kvm_clear_dirty_log args = { .dirty_bitmap = log, .slot = slot,
272 .first_page = first_page,
273 .num_pages = num_pages };
276 ret = ioctl(vm->fd, KVM_CLEAR_DIRTY_LOG, &args);
277 TEST_ASSERT(ret == 0, "%s: KVM_CLEAR_DIRTY_LOG failed: %s",
282 * Userspace Memory Region Find
285 * vm - Virtual Machine
286 * start - Starting VM physical address
287 * end - Ending VM physical address, inclusive.
292 * Pointer to overlapping region, NULL if no such region.
294 * Searches for a region with any physical memory that overlaps with
295 * any portion of the guest physical addresses from start to end
296 * inclusive. If multiple overlapping regions exist, a pointer to any
297 * of the regions is returned. Null is returned only when no overlapping
300 static struct userspace_mem_region *
301 userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
303 struct userspace_mem_region *region;
305 for (region = vm->userspace_mem_region_head; region;
306 region = region->next) {
307 uint64_t existing_start = region->region.guest_phys_addr;
308 uint64_t existing_end = region->region.guest_phys_addr
309 + region->region.memory_size - 1;
310 if (start <= existing_end && end >= existing_start)
318 * KVM Userspace Memory Region Find
321 * vm - Virtual Machine
322 * start - Starting VM physical address
323 * end - Ending VM physical address, inclusive.
328 * Pointer to overlapping region, NULL if no such region.
330 * Public interface to userspace_mem_region_find. Allows tests to look up
331 * the memslot datastructure for a given range of guest physical memory.
333 struct kvm_userspace_memory_region *
334 kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
337 struct userspace_mem_region *region;
339 region = userspace_mem_region_find(vm, start, end);
343 return ®ion->region;
350 * vm - Virtual Machine
356 * Pointer to VCPU structure
358 * Locates a vcpu structure that describes the VCPU specified by vcpuid and
359 * returns a pointer to it. Returns NULL if the VM doesn't contain a VCPU
360 * for the specified vcpuid.
362 struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid)
366 for (vcpup = vm->vcpu_head; vcpup; vcpup = vcpup->next) {
367 if (vcpup->id == vcpuid)
378 * vm - Virtual Machine
383 * Return: None, TEST_ASSERT failures for all error conditions
385 * Within the VM specified by vm, removes the VCPU given by vcpuid.
387 static void vm_vcpu_rm(struct kvm_vm *vm, uint32_t vcpuid)
389 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
392 ret = munmap(vcpu->state, sizeof(*vcpu->state));
393 TEST_ASSERT(ret == 0, "munmap of VCPU fd failed, rc: %i "
394 "errno: %i", ret, errno);
396 TEST_ASSERT(ret == 0, "Close of VCPU fd failed, rc: %i "
397 "errno: %i", ret, errno);
400 vcpu->next->prev = vcpu->prev;
402 vcpu->prev->next = vcpu->next;
404 vm->vcpu_head = vcpu->next;
408 void kvm_vm_release(struct kvm_vm *vmp)
412 while (vmp->vcpu_head)
413 vm_vcpu_rm(vmp, vmp->vcpu_head->id);
415 ret = close(vmp->fd);
416 TEST_ASSERT(ret == 0, "Close of vm fd failed,\n"
417 " vmp->fd: %i rc: %i errno: %i", vmp->fd, ret, errno);
420 TEST_ASSERT(ret == 0, "Close of /dev/kvm fd failed,\n"
421 " vmp->kvm_fd: %i rc: %i errno: %i", vmp->kvm_fd, ret, errno);
425 * Destroys and frees the VM pointed to by vmp.
427 void kvm_vm_free(struct kvm_vm *vmp)
434 /* Free userspace_mem_regions. */
435 while (vmp->userspace_mem_region_head) {
436 struct userspace_mem_region *region
437 = vmp->userspace_mem_region_head;
439 region->region.memory_size = 0;
440 ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION,
442 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, "
443 "rc: %i errno: %i", ret, errno);
445 vmp->userspace_mem_region_head = region->next;
446 sparsebit_free(®ion->unused_phy_pages);
447 ret = munmap(region->mmap_start, region->mmap_size);
448 TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i",
454 /* Free sparsebit arrays. */
455 sparsebit_free(&vmp->vpages_valid);
456 sparsebit_free(&vmp->vpages_mapped);
460 /* Free the structure describing the VM. */
465 * Memory Compare, host virtual to guest virtual
468 * hva - Starting host virtual address
469 * vm - Virtual Machine
470 * gva - Starting guest virtual address
471 * len - number of bytes to compare
475 * Input/Output Args: None
478 * Returns 0 if the bytes starting at hva for a length of len
479 * are equal the guest virtual bytes starting at gva. Returns
480 * a value < 0, if bytes at hva are less than those at gva.
481 * Otherwise a value > 0 is returned.
483 * Compares the bytes starting at the host virtual address hva, for
484 * a length of len, to the guest bytes starting at the guest virtual
485 * address given by gva.
487 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, vm_vaddr_t gva, size_t len)
492 * Compare a batch of bytes until either a match is found
493 * or all the bytes have been compared.
495 for (uintptr_t offset = 0; offset < len; offset += amt) {
496 uintptr_t ptr1 = (uintptr_t)hva + offset;
499 * Determine host address for guest virtual address
502 uintptr_t ptr2 = (uintptr_t)addr_gva2hva(vm, gva + offset);
505 * Determine amount to compare on this pass.
506 * Don't allow the comparsion to cross a page boundary.
509 if ((ptr1 >> vm->page_shift) != ((ptr1 + amt) >> vm->page_shift))
510 amt = vm->page_size - (ptr1 % vm->page_size);
511 if ((ptr2 >> vm->page_shift) != ((ptr2 + amt) >> vm->page_shift))
512 amt = vm->page_size - (ptr2 % vm->page_size);
514 assert((ptr1 >> vm->page_shift) == ((ptr1 + amt - 1) >> vm->page_shift));
515 assert((ptr2 >> vm->page_shift) == ((ptr2 + amt - 1) >> vm->page_shift));
518 * Perform the comparison. If there is a difference
519 * return that result to the caller, otherwise need
520 * to continue on looking for a mismatch.
522 int ret = memcmp((void *)ptr1, (void *)ptr2, amt);
528 * No mismatch found. Let the caller know the two memory
535 * VM Userspace Memory Region Add
538 * vm - Virtual Machine
539 * backing_src - Storage source for this region.
540 * NULL to use anonymous memory.
541 * guest_paddr - Starting guest physical address
542 * slot - KVM region slot
543 * npages - Number of physical pages
544 * flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES)
550 * Allocates a memory area of the number of pages specified by npages
551 * and maps it to the VM specified by vm, at a starting physical address
552 * given by guest_paddr. The region is created with a KVM region slot
553 * given by slot, which must be unique and < KVM_MEM_SLOTS_NUM. The
554 * region is created with the flags given by flags.
556 void vm_userspace_mem_region_add(struct kvm_vm *vm,
557 enum vm_mem_backing_src_type src_type,
558 uint64_t guest_paddr, uint32_t slot, uint64_t npages,
562 struct userspace_mem_region *region;
563 size_t huge_page_size = KVM_UTIL_PGS_PER_HUGEPG * vm->page_size;
566 TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
567 "address not on a page boundary.\n"
568 " guest_paddr: 0x%lx vm->page_size: 0x%x",
569 guest_paddr, vm->page_size);
570 TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
571 <= vm->max_gfn, "Physical range beyond maximum "
572 "supported physical address,\n"
573 " guest_paddr: 0x%lx npages: 0x%lx\n"
574 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",
575 guest_paddr, npages, vm->max_gfn, vm->page_size);
578 * Confirm a mem region with an overlapping address doesn't
581 region = (struct userspace_mem_region *) userspace_mem_region_find(
582 vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
584 TEST_ASSERT(false, "overlapping userspace_mem_region already "
586 " requested guest_paddr: 0x%lx npages: 0x%lx "
588 " existing guest_paddr: 0x%lx size: 0x%lx",
589 guest_paddr, npages, vm->page_size,
590 (uint64_t) region->region.guest_phys_addr,
591 (uint64_t) region->region.memory_size);
593 /* Confirm no region with the requested slot already exists. */
594 for (region = vm->userspace_mem_region_head; region;
595 region = region->next) {
596 if (region->region.slot == slot)
600 TEST_ASSERT(false, "A mem region with the requested slot "
602 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
603 " existing slot: %u paddr: 0x%lx size: 0x%lx",
604 slot, guest_paddr, npages,
606 (uint64_t) region->region.guest_phys_addr,
607 (uint64_t) region->region.memory_size);
609 /* Allocate and initialize new mem region structure. */
610 region = calloc(1, sizeof(*region));
611 TEST_ASSERT(region != NULL, "Insufficient Memory");
612 region->mmap_size = npages * vm->page_size;
615 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
616 alignment = 0x100000;
621 if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
622 alignment = max(huge_page_size, alignment);
624 /* Add enough memory to align up if necessary */
626 region->mmap_size += alignment;
628 region->mmap_start = mmap(NULL, region->mmap_size,
629 PROT_READ | PROT_WRITE,
630 MAP_PRIVATE | MAP_ANONYMOUS
631 | (src_type == VM_MEM_SRC_ANONYMOUS_HUGETLB ? MAP_HUGETLB : 0),
633 TEST_ASSERT(region->mmap_start != MAP_FAILED,
634 "test_malloc failed, mmap_start: %p errno: %i",
635 region->mmap_start, errno);
637 /* Align host address */
638 region->host_mem = align(region->mmap_start, alignment);
640 /* As needed perform madvise */
641 if (src_type == VM_MEM_SRC_ANONYMOUS || src_type == VM_MEM_SRC_ANONYMOUS_THP) {
642 ret = madvise(region->host_mem, npages * vm->page_size,
643 src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
644 TEST_ASSERT(ret == 0, "madvise failed,\n"
648 region->host_mem, npages * vm->page_size, src_type);
651 region->unused_phy_pages = sparsebit_alloc();
652 sparsebit_set_num(region->unused_phy_pages,
653 guest_paddr >> vm->page_shift, npages);
654 region->region.slot = slot;
655 region->region.flags = flags;
656 region->region.guest_phys_addr = guest_paddr;
657 region->region.memory_size = npages * vm->page_size;
658 region->region.userspace_addr = (uintptr_t) region->host_mem;
659 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
660 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
661 " rc: %i errno: %i\n"
662 " slot: %u flags: 0x%x\n"
663 " guest_phys_addr: 0x%lx size: 0x%lx",
664 ret, errno, slot, flags,
665 guest_paddr, (uint64_t) region->region.memory_size);
667 /* Add to linked-list of memory regions. */
668 if (vm->userspace_mem_region_head)
669 vm->userspace_mem_region_head->prev = region;
670 region->next = vm->userspace_mem_region_head;
671 vm->userspace_mem_region_head = region;
678 * vm - Virtual Machine
679 * memslot - KVM memory slot ID
684 * Pointer to memory region structure that describe memory region
685 * using kvm memory slot ID given by memslot. TEST_ASSERT failure
686 * on error (e.g. currently no memory region using memslot as a KVM
689 static struct userspace_mem_region *
690 memslot2region(struct kvm_vm *vm, uint32_t memslot)
692 struct userspace_mem_region *region;
694 for (region = vm->userspace_mem_region_head; region;
695 region = region->next) {
696 if (region->region.slot == memslot)
699 if (region == NULL) {
700 fprintf(stderr, "No mem region with the requested slot found,\n"
701 " requested slot: %u\n", memslot);
702 fputs("---- vm dump ----\n", stderr);
703 vm_dump(stderr, vm, 2);
704 TEST_ASSERT(false, "Mem region not found");
711 * VM Memory Region Flags Set
714 * vm - Virtual Machine
715 * flags - Starting guest physical address
721 * Sets the flags of the memory region specified by the value of slot,
722 * to the values given by flags.
724 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
727 struct userspace_mem_region *region;
729 region = memslot2region(vm, slot);
731 region->region.flags = flags;
733 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
735 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
736 " rc: %i errno: %i slot: %u flags: 0x%x",
737 ret, errno, slot, flags);
750 * Returns the size of the structure pointed to by the return value
753 static int vcpu_mmap_sz(void)
757 dev_fd = open(KVM_DEV_PATH, O_RDONLY);
761 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
762 TEST_ASSERT(ret >= sizeof(struct kvm_run),
763 "%s KVM_GET_VCPU_MMAP_SIZE ioctl failed, rc: %i errno: %i",
764 __func__, ret, errno);
775 * vm - Virtual Machine
782 * Adds a virtual CPU to the VM specified by vm with the ID given by vcpuid.
783 * No additional VCPU setup is done.
785 void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid)
789 /* Confirm a vcpu with the specified id doesn't already exist. */
790 vcpu = vcpu_find(vm, vcpuid);
792 TEST_ASSERT(false, "vcpu with the specified id "
794 " requested vcpuid: %u\n"
795 " existing vcpuid: %u state: %p",
796 vcpuid, vcpu->id, vcpu->state);
798 /* Allocate and initialize new vcpu structure. */
799 vcpu = calloc(1, sizeof(*vcpu));
800 TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
802 vcpu->fd = ioctl(vm->fd, KVM_CREATE_VCPU, vcpuid);
803 TEST_ASSERT(vcpu->fd >= 0, "KVM_CREATE_VCPU failed, rc: %i errno: %i",
806 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->state), "vcpu mmap size "
807 "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",
808 vcpu_mmap_sz(), sizeof(*vcpu->state));
809 vcpu->state = (struct kvm_run *) mmap(NULL, sizeof(*vcpu->state),
810 PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0);
811 TEST_ASSERT(vcpu->state != MAP_FAILED, "mmap vcpu_state failed, "
812 "vcpu id: %u errno: %i", vcpuid, errno);
814 /* Add to linked-list of VCPUs. */
816 vm->vcpu_head->prev = vcpu;
817 vcpu->next = vm->vcpu_head;
818 vm->vcpu_head = vcpu;
822 * VM Virtual Address Unused Gap
825 * vm - Virtual Machine
827 * vaddr_min - Minimum Virtual Address
832 * Lowest virtual address at or below vaddr_min, with at least
833 * sz unused bytes. TEST_ASSERT failure if no area of at least
834 * size sz is available.
836 * Within the VM specified by vm, locates the lowest starting virtual
837 * address >= vaddr_min, that has at least sz unallocated bytes. A
838 * TEST_ASSERT failure occurs for invalid input or no area of at least
839 * sz unallocated bytes >= vaddr_min is available.
841 static vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
842 vm_vaddr_t vaddr_min)
844 uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
846 /* Determine lowest permitted virtual page index. */
847 uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
848 if ((pgidx_start * vm->page_size) < vaddr_min)
851 /* Loop over section with enough valid virtual page indexes. */
852 if (!sparsebit_is_set_num(vm->vpages_valid,
854 pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
858 * Are there enough unused virtual pages available at
859 * the currently proposed starting virtual page index.
860 * If not, adjust proposed starting index to next
863 if (sparsebit_is_clear_num(vm->vpages_mapped,
866 pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
868 if (pgidx_start == 0)
872 * If needed, adjust proposed starting virtual address,
873 * to next range of valid virtual addresses.
875 if (!sparsebit_is_set_num(vm->vpages_valid,
876 pgidx_start, pages)) {
877 pgidx_start = sparsebit_next_set_num(
878 vm->vpages_valid, pgidx_start, pages);
879 if (pgidx_start == 0)
882 } while (pgidx_start != 0);
885 TEST_ASSERT(false, "No vaddr of specified pages available, "
886 "pages: 0x%lx", pages);
892 TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
894 "Unexpected, invalid virtual page index range,\n"
895 " pgidx_start: 0x%lx\n"
898 TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
900 "Unexpected, pages already mapped,\n"
901 " pgidx_start: 0x%lx\n"
905 return pgidx_start * vm->page_size;
909 * VM Virtual Address Allocate
912 * vm - Virtual Machine
914 * vaddr_min - Minimum starting virtual address
915 * data_memslot - Memory region slot for data pages
916 * pgd_memslot - Memory region slot for new virtual translation tables
921 * Starting guest virtual address
923 * Allocates at least sz bytes within the virtual address space of the vm
924 * given by vm. The allocated bytes are mapped to a virtual address >=
925 * the address given by vaddr_min. Note that each allocation uses a
926 * a unique set of pages, with the minimum real allocation being at least
929 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
930 uint32_t data_memslot, uint32_t pgd_memslot)
932 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
934 virt_pgd_alloc(vm, pgd_memslot);
937 * Find an unused range of virtual page addresses of at least
940 vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
942 /* Map the virtual pages. */
943 for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
944 pages--, vaddr += vm->page_size) {
947 paddr = vm_phy_page_alloc(vm,
948 KVM_UTIL_MIN_PFN * vm->page_size, data_memslot);
950 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
952 sparsebit_set(vm->vpages_mapped,
953 vaddr >> vm->page_shift);
960 * Map a range of VM virtual address to the VM's physical address
963 * vm - Virtual Machine
964 * vaddr - Virtuall address to map
965 * paddr - VM Physical Address
966 * size - The size of the range to map
967 * pgd_memslot - Memory region slot for new virtual translation tables
973 * Within the VM given by vm, creates a virtual translation for the
974 * page range starting at vaddr to the page range starting at paddr.
976 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
977 size_t size, uint32_t pgd_memslot)
979 size_t page_size = vm->page_size;
980 size_t npages = size / page_size;
982 TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
983 TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
986 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
993 * Address VM Physical to Host Virtual
996 * vm - Virtual Machine
997 * gpa - VM physical address
1002 * Equivalent host virtual address
1004 * Locates the memory region containing the VM physical address given
1005 * by gpa, within the VM given by vm. When found, the host virtual
1006 * address providing the memory to the vm physical address is returned.
1007 * A TEST_ASSERT failure occurs if no region containing gpa exists.
1009 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
1011 struct userspace_mem_region *region;
1012 for (region = vm->userspace_mem_region_head; region;
1013 region = region->next) {
1014 if ((gpa >= region->region.guest_phys_addr)
1015 && (gpa <= (region->region.guest_phys_addr
1016 + region->region.memory_size - 1)))
1017 return (void *) ((uintptr_t) region->host_mem
1018 + (gpa - region->region.guest_phys_addr));
1021 TEST_ASSERT(false, "No vm physical memory at 0x%lx", gpa);
1026 * Address Host Virtual to VM Physical
1029 * vm - Virtual Machine
1030 * hva - Host virtual address
1035 * Equivalent VM physical address
1037 * Locates the memory region containing the host virtual address given
1038 * by hva, within the VM given by vm. When found, the equivalent
1039 * VM physical address is returned. A TEST_ASSERT failure occurs if no
1040 * region containing hva exists.
1042 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1044 struct userspace_mem_region *region;
1045 for (region = vm->userspace_mem_region_head; region;
1046 region = region->next) {
1047 if ((hva >= region->host_mem)
1048 && (hva <= (region->host_mem
1049 + region->region.memory_size - 1)))
1050 return (vm_paddr_t) ((uintptr_t)
1051 region->region.guest_phys_addr
1052 + (hva - (uintptr_t) region->host_mem));
1055 TEST_ASSERT(false, "No mapping to a guest physical address, "
1061 * VM Create IRQ Chip
1064 * vm - Virtual Machine
1070 * Creates an interrupt controller chip for the VM specified by vm.
1072 void vm_create_irqchip(struct kvm_vm *vm)
1076 ret = ioctl(vm->fd, KVM_CREATE_IRQCHIP, 0);
1077 TEST_ASSERT(ret == 0, "KVM_CREATE_IRQCHIP IOCTL failed, "
1078 "rc: %i errno: %i", ret, errno);
1080 vm->has_irqchip = true;
1087 * vm - Virtual Machine
1093 * Pointer to structure that describes the state of the VCPU.
1095 * Locates and returns a pointer to a structure that describes the
1096 * state of the VCPU with the given vcpuid.
1098 struct kvm_run *vcpu_state(struct kvm_vm *vm, uint32_t vcpuid)
1100 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1101 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1110 * vm - Virtual Machine
1117 * Switch to executing the code for the VCPU given by vcpuid, within the VM
1120 void vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1122 int ret = _vcpu_run(vm, vcpuid);
1123 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1124 "rc: %i errno: %i", ret, errno);
1127 int _vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1129 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1132 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1134 rc = ioctl(vcpu->fd, KVM_RUN, NULL);
1135 } while (rc == -1 && errno == EINTR);
1139 void vcpu_run_complete_io(struct kvm_vm *vm, uint32_t vcpuid)
1141 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1144 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1146 vcpu->state->immediate_exit = 1;
1147 ret = ioctl(vcpu->fd, KVM_RUN, NULL);
1148 vcpu->state->immediate_exit = 0;
1150 TEST_ASSERT(ret == -1 && errno == EINTR,
1151 "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1156 * VM VCPU Set MP State
1159 * vm - Virtual Machine
1161 * mp_state - mp_state to be set
1167 * Sets the MP state of the VCPU given by vcpuid, to the state given
1170 void vcpu_set_mp_state(struct kvm_vm *vm, uint32_t vcpuid,
1171 struct kvm_mp_state *mp_state)
1173 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1176 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1178 ret = ioctl(vcpu->fd, KVM_SET_MP_STATE, mp_state);
1179 TEST_ASSERT(ret == 0, "KVM_SET_MP_STATE IOCTL failed, "
1180 "rc: %i errno: %i", ret, errno);
1187 * vm - Virtual Machine
1191 * regs - current state of VCPU regs
1195 * Obtains the current register state for the VCPU specified by vcpuid
1196 * and stores it at the location given by regs.
1198 void vcpu_regs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1200 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1203 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1205 ret = ioctl(vcpu->fd, KVM_GET_REGS, regs);
1206 TEST_ASSERT(ret == 0, "KVM_GET_REGS failed, rc: %i errno: %i",
1214 * vm - Virtual Machine
1216 * regs - Values to set VCPU regs to
1222 * Sets the regs of the VCPU specified by vcpuid to the values
1225 void vcpu_regs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1227 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1230 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1232 ret = ioctl(vcpu->fd, KVM_SET_REGS, regs);
1233 TEST_ASSERT(ret == 0, "KVM_SET_REGS failed, rc: %i errno: %i",
1237 #ifdef __KVM_HAVE_VCPU_EVENTS
1238 void vcpu_events_get(struct kvm_vm *vm, uint32_t vcpuid,
1239 struct kvm_vcpu_events *events)
1241 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1244 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1246 ret = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, events);
1247 TEST_ASSERT(ret == 0, "KVM_GET_VCPU_EVENTS, failed, rc: %i errno: %i",
1251 void vcpu_events_set(struct kvm_vm *vm, uint32_t vcpuid,
1252 struct kvm_vcpu_events *events)
1254 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1257 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1259 ret = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, events);
1260 TEST_ASSERT(ret == 0, "KVM_SET_VCPU_EVENTS, failed, rc: %i errno: %i",
1266 void vcpu_nested_state_get(struct kvm_vm *vm, uint32_t vcpuid,
1267 struct kvm_nested_state *state)
1269 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1272 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1274 ret = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, state);
1275 TEST_ASSERT(ret == 0,
1276 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1280 int vcpu_nested_state_set(struct kvm_vm *vm, uint32_t vcpuid,
1281 struct kvm_nested_state *state, bool ignore_error)
1283 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1286 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1288 ret = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, state);
1289 if (!ignore_error) {
1290 TEST_ASSERT(ret == 0,
1291 "KVM_SET_NESTED_STATE failed, ret: %i errno: %i",
1300 * VM VCPU System Regs Get
1303 * vm - Virtual Machine
1307 * sregs - current state of VCPU system regs
1311 * Obtains the current system register state for the VCPU specified by
1312 * vcpuid and stores it at the location given by sregs.
1314 void vcpu_sregs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1316 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1319 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1321 ret = ioctl(vcpu->fd, KVM_GET_SREGS, sregs);
1322 TEST_ASSERT(ret == 0, "KVM_GET_SREGS failed, rc: %i errno: %i",
1327 * VM VCPU System Regs Set
1330 * vm - Virtual Machine
1332 * sregs - Values to set VCPU system regs to
1338 * Sets the system regs of the VCPU specified by vcpuid to the values
1341 void vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1343 int ret = _vcpu_sregs_set(vm, vcpuid, sregs);
1344 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1345 "rc: %i errno: %i", ret, errno);
1348 int _vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1350 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1352 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1354 return ioctl(vcpu->fd, KVM_SET_SREGS, sregs);
1361 * vm - Virtual Machine
1363 * cmd - Ioctl number
1364 * arg - Argument to pass to the ioctl
1368 * Issues an arbitrary ioctl on a VCPU fd.
1370 void vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1371 unsigned long cmd, void *arg)
1375 ret = _vcpu_ioctl(vm, vcpuid, cmd, arg);
1376 TEST_ASSERT(ret == 0, "vcpu ioctl %lu failed, rc: %i errno: %i (%s)",
1377 cmd, ret, errno, strerror(errno));
1380 int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1381 unsigned long cmd, void *arg)
1383 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1386 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1388 ret = ioctl(vcpu->fd, cmd, arg);
1397 * vm - Virtual Machine
1398 * cmd - Ioctl number
1399 * arg - Argument to pass to the ioctl
1403 * Issues an arbitrary ioctl on a VM fd.
1405 void vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1409 ret = ioctl(vm->fd, cmd, arg);
1410 TEST_ASSERT(ret == 0, "vm ioctl %lu failed, rc: %i errno: %i (%s)",
1411 cmd, ret, errno, strerror(errno));
1418 * vm - Virtual Machine
1419 * indent - Left margin indent amount
1422 * stream - Output FILE stream
1426 * Dumps the current state of the VM given by vm, to the FILE stream
1429 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1431 struct userspace_mem_region *region;
1434 fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1435 fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1436 fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1437 fprintf(stream, "%*sMem Regions:\n", indent, "");
1438 for (region = vm->userspace_mem_region_head; region;
1439 region = region->next) {
1440 fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1441 "host_virt: %p\n", indent + 2, "",
1442 (uint64_t) region->region.guest_phys_addr,
1443 (uint64_t) region->region.memory_size,
1445 fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1446 sparsebit_dump(stream, region->unused_phy_pages, 0);
1448 fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1449 sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1450 fprintf(stream, "%*spgd_created: %u\n", indent, "",
1452 if (vm->pgd_created) {
1453 fprintf(stream, "%*sVirtual Translation Tables:\n",
1455 virt_dump(stream, vm, indent + 4);
1457 fprintf(stream, "%*sVCPUs:\n", indent, "");
1458 for (vcpu = vm->vcpu_head; vcpu; vcpu = vcpu->next)
1459 vcpu_dump(stream, vm, vcpu->id, indent + 2);
1462 /* Known KVM exit reasons */
1463 static struct exit_reason {
1464 unsigned int reason;
1466 } exit_reasons_known[] = {
1467 {KVM_EXIT_UNKNOWN, "UNKNOWN"},
1468 {KVM_EXIT_EXCEPTION, "EXCEPTION"},
1469 {KVM_EXIT_IO, "IO"},
1470 {KVM_EXIT_HYPERCALL, "HYPERCALL"},
1471 {KVM_EXIT_DEBUG, "DEBUG"},
1472 {KVM_EXIT_HLT, "HLT"},
1473 {KVM_EXIT_MMIO, "MMIO"},
1474 {KVM_EXIT_IRQ_WINDOW_OPEN, "IRQ_WINDOW_OPEN"},
1475 {KVM_EXIT_SHUTDOWN, "SHUTDOWN"},
1476 {KVM_EXIT_FAIL_ENTRY, "FAIL_ENTRY"},
1477 {KVM_EXIT_INTR, "INTR"},
1478 {KVM_EXIT_SET_TPR, "SET_TPR"},
1479 {KVM_EXIT_TPR_ACCESS, "TPR_ACCESS"},
1480 {KVM_EXIT_S390_SIEIC, "S390_SIEIC"},
1481 {KVM_EXIT_S390_RESET, "S390_RESET"},
1482 {KVM_EXIT_DCR, "DCR"},
1483 {KVM_EXIT_NMI, "NMI"},
1484 {KVM_EXIT_INTERNAL_ERROR, "INTERNAL_ERROR"},
1485 {KVM_EXIT_OSI, "OSI"},
1486 {KVM_EXIT_PAPR_HCALL, "PAPR_HCALL"},
1487 #ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1488 {KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},
1493 * Exit Reason String
1496 * exit_reason - Exit reason
1501 * Constant string pointer describing the exit reason.
1503 * Locates and returns a constant string that describes the KVM exit
1504 * reason given by exit_reason. If no such string is found, a constant
1505 * string of "Unknown" is returned.
1507 const char *exit_reason_str(unsigned int exit_reason)
1511 for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
1512 if (exit_reason == exit_reasons_known[n1].reason)
1513 return exit_reasons_known[n1].name;
1520 * Physical Contiguous Page Allocator
1523 * vm - Virtual Machine
1524 * num - number of pages
1525 * paddr_min - Physical address minimum
1526 * memslot - Memory region to allocate page from
1531 * Starting physical address
1533 * Within the VM specified by vm, locates a range of available physical
1534 * pages at or above paddr_min. If found, the pages are marked as in use
1535 * and their base address is returned. A TEST_ASSERT failure occurs if
1536 * not enough pages are available at or above paddr_min.
1538 vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
1539 vm_paddr_t paddr_min, uint32_t memslot)
1541 struct userspace_mem_region *region;
1542 sparsebit_idx_t pg, base;
1544 TEST_ASSERT(num > 0, "Must allocate at least one page");
1546 TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
1547 "not divisible by page size.\n"
1548 " paddr_min: 0x%lx page_size: 0x%x",
1549 paddr_min, vm->page_size);
1551 region = memslot2region(vm, memslot);
1552 base = pg = paddr_min >> vm->page_shift;
1555 for (; pg < base + num; ++pg) {
1556 if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
1557 base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
1561 } while (pg && pg != base + num);
1564 fprintf(stderr, "No guest physical page available, "
1565 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
1566 paddr_min, vm->page_size, memslot);
1567 fputs("---- vm dump ----\n", stderr);
1568 vm_dump(stderr, vm, 2);
1572 for (pg = base; pg < base + num; ++pg)
1573 sparsebit_clear(region->unused_phy_pages, pg);
1575 return base * vm->page_size;
1578 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
1581 return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
1585 * Address Guest Virtual to Host Virtual
1588 * vm - Virtual Machine
1589 * gva - VM virtual address
1594 * Equivalent host virtual address
1596 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
1598 return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));
1602 * Is Unrestricted Guest
1605 * vm - Virtual Machine
1609 * Return: True if the unrestricted guest is set to 'Y', otherwise return false.
1611 * Check if the unrestricted guest flag is enabled.
1613 bool vm_is_unrestricted_guest(struct kvm_vm *vm)
1620 /* Ensure that the KVM vendor-specific module is loaded. */
1621 f = fopen(KVM_DEV_PATH, "r");
1622 TEST_ASSERT(f != NULL, "Error in opening KVM dev file: %d",
1627 f = fopen("/sys/module/kvm_intel/parameters/unrestricted_guest", "r");
1629 count = fread(&val, sizeof(char), 1, f);
1630 TEST_ASSERT(count == 1, "Unable to read from param file.");