1 // SPDX-License-Identifier: GPL-2.0
2 #define _GNU_SOURCE /* for program_invocation_short_name */
11 #include <sys/ioctl.h>
14 #include <linux/compiler.h>
16 #include <test_util.h>
18 #include <processor.h>
23 * s390x needs at least 1MB alignment, and the x86_64 MOVE/DELETE tests need a
24 * 2MB sized and aligned region so that the initial region corresponds to
25 * exactly one large page.
27 #define MEM_REGION_SIZE 0x200000
31 * Somewhat arbitrary location and slot, intended to not overlap anything.
33 #define MEM_REGION_GPA 0xc0000000
34 #define MEM_REGION_SLOT 10
36 static const uint64_t MMIO_VAL = 0xbeefull;
38 extern const uint64_t final_rip_start;
39 extern const uint64_t final_rip_end;
41 static sem_t vcpu_ready;
43 static inline uint64_t guest_spin_on_val(uint64_t spin_val)
48 val = READ_ONCE(*((uint64_t *)MEM_REGION_GPA));
49 } while (val == spin_val);
55 static void *vcpu_worker(void *data)
57 struct kvm_vm *vm = data;
63 * Loop until the guest is done. Re-enter the guest on all MMIO exits,
64 * which will occur if the guest attempts to access a memslot after it
65 * has been deleted or while it is being moved .
67 run = vcpu_state(vm, VCPU_ID);
70 vcpu_run(vm, VCPU_ID);
72 if (run->exit_reason == KVM_EXIT_IO) {
73 cmd = get_ucall(vm, VCPU_ID, &uc);
74 if (cmd != UCALL_SYNC)
77 sem_post(&vcpu_ready);
81 if (run->exit_reason != KVM_EXIT_MMIO)
84 TEST_ASSERT(!run->mmio.is_write, "Unexpected exit mmio write");
85 TEST_ASSERT(run->mmio.len == 8,
86 "Unexpected exit mmio size = %u", run->mmio.len);
88 TEST_ASSERT(run->mmio.phys_addr == MEM_REGION_GPA,
89 "Unexpected exit mmio address = 0x%llx",
91 memcpy(run->mmio.data, &MMIO_VAL, 8);
94 if (run->exit_reason == KVM_EXIT_IO && cmd == UCALL_ABORT)
95 TEST_FAIL("%s at %s:%ld, val = %lu", (const char *)uc.args[0],
96 __FILE__, uc.args[1], uc.args[2]);
101 static void wait_for_vcpu(void)
105 TEST_ASSERT(!clock_gettime(CLOCK_REALTIME, &ts),
106 "clock_gettime() failed: %d\n", errno);
109 TEST_ASSERT(!sem_timedwait(&vcpu_ready, &ts),
110 "sem_timedwait() failed: %d\n", errno);
112 /* Wait for the vCPU thread to reenter the guest. */
116 static struct kvm_vm *spawn_vm(pthread_t *vcpu_thread, void *guest_code)
122 vm = vm_create_default(VCPU_ID, 0, guest_code);
124 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,
125 MEM_REGION_GPA, MEM_REGION_SLOT,
126 MEM_REGION_SIZE / getpagesize(), 0);
129 * Allocate and map two pages so that the GPA accessed by guest_code()
130 * stays valid across the memslot move.
132 gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT);
133 TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n");
135 virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2, 0);
137 /* Ditto for the host mapping so that both pages can be zeroed. */
138 hva = addr_gpa2hva(vm, MEM_REGION_GPA);
139 memset(hva, 0, 2 * 4096);
141 pthread_create(vcpu_thread, NULL, vcpu_worker, vm);
143 /* Ensure the guest thread is spun up. */
150 static void guest_code_move_memory_region(void)
157 * Spin until the memory region starts getting moved to a
158 * misaligned address.
159 * Every region move may or may not trigger MMIO, as the
160 * window where the memslot is invalid is usually quite small.
162 val = guest_spin_on_val(0);
163 GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val);
165 /* Spin until the misaligning memory region move completes. */
166 val = guest_spin_on_val(MMIO_VAL);
167 GUEST_ASSERT_1(val == 1 || val == 0, val);
169 /* Spin until the memory region starts to get re-aligned. */
170 val = guest_spin_on_val(0);
171 GUEST_ASSERT_1(val == 1 || val == MMIO_VAL, val);
173 /* Spin until the re-aligning memory region move completes. */
174 val = guest_spin_on_val(MMIO_VAL);
175 GUEST_ASSERT_1(val == 1, val);
180 static void test_move_memory_region(void)
182 pthread_t vcpu_thread;
186 vm = spawn_vm(&vcpu_thread, guest_code_move_memory_region);
188 hva = addr_gpa2hva(vm, MEM_REGION_GPA);
191 * Shift the region's base GPA. The guest should not see "2" as the
192 * hva->gpa translation is misaligned, i.e. the guest is accessing a
193 * different host pfn.
195 vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA - 4096);
199 * The guest _might_ see an invalid memslot and trigger MMIO, but it's
200 * a tiny window. Spin and defer the sync until the memslot is
201 * restored and guest behavior is once again deterministic.
206 * Note, value in memory needs to be changed *before* restoring the
207 * memslot, else the guest could race the update and see "2".
211 /* Restore the original base, the guest should see "1". */
212 vm_mem_region_move(vm, MEM_REGION_SLOT, MEM_REGION_GPA);
214 /* Defered sync from when the memslot was misaligned (above). */
217 pthread_join(vcpu_thread, NULL);
222 static void guest_code_delete_memory_region(void)
228 /* Spin until the memory region is deleted. */
229 val = guest_spin_on_val(0);
230 GUEST_ASSERT_1(val == MMIO_VAL, val);
232 /* Spin until the memory region is recreated. */
233 val = guest_spin_on_val(MMIO_VAL);
234 GUEST_ASSERT_1(val == 0, val);
236 /* Spin until the memory region is deleted. */
237 val = guest_spin_on_val(0);
238 GUEST_ASSERT_1(val == MMIO_VAL, val);
241 ".pushsection .rodata\n\t"
242 ".global final_rip_start\n\t"
243 "final_rip_start: .quad 1b\n\t"
246 /* Spin indefinitely (until the code memslot is deleted). */
247 guest_spin_on_val(MMIO_VAL);
250 ".pushsection .rodata\n\t"
251 ".global final_rip_end\n\t"
252 "final_rip_end: .quad 1b\n\t"
255 GUEST_ASSERT_1(0, 0);
258 static void test_delete_memory_region(void)
260 pthread_t vcpu_thread;
261 struct kvm_regs regs;
265 vm = spawn_vm(&vcpu_thread, guest_code_delete_memory_region);
267 /* Delete the memory region, the guest should not die. */
268 vm_mem_region_delete(vm, MEM_REGION_SLOT);
271 /* Recreate the memory region. The guest should see "0". */
272 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_THP,
273 MEM_REGION_GPA, MEM_REGION_SLOT,
274 MEM_REGION_SIZE / getpagesize(), 0);
277 /* Delete the region again so that there's only one memslot left. */
278 vm_mem_region_delete(vm, MEM_REGION_SLOT);
282 * Delete the primary memslot. This should cause an emulation error or
283 * shutdown due to the page tables getting nuked.
285 vm_mem_region_delete(vm, 0);
287 pthread_join(vcpu_thread, NULL);
289 run = vcpu_state(vm, VCPU_ID);
291 TEST_ASSERT(run->exit_reason == KVM_EXIT_SHUTDOWN ||
292 run->exit_reason == KVM_EXIT_INTERNAL_ERROR,
293 "Unexpected exit reason = %d", run->exit_reason);
295 vcpu_regs_get(vm, VCPU_ID, ®s);
298 * On AMD, after KVM_EXIT_SHUTDOWN the VMCB has been reinitialized already,
299 * so the instruction pointer would point to the reset vector.
301 if (run->exit_reason == KVM_EXIT_INTERNAL_ERROR)
302 TEST_ASSERT(regs.rip >= final_rip_start &&
303 regs.rip < final_rip_end,
304 "Bad rip, expected 0x%lx - 0x%lx, got 0x%llx\n",
305 final_rip_start, final_rip_end, regs.rip);
310 static void test_zero_memory_regions(void)
315 pr_info("Testing KVM_RUN with zero added memory regions\n");
317 vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
318 vm_vcpu_add(vm, VCPU_ID);
320 TEST_ASSERT(!ioctl(vm_get_fd(vm), KVM_SET_NR_MMU_PAGES, 64),
321 "KVM_SET_NR_MMU_PAGES failed, errno = %d\n", errno);
322 vcpu_run(vm, VCPU_ID);
324 run = vcpu_state(vm, VCPU_ID);
325 TEST_ASSERT(run->exit_reason == KVM_EXIT_INTERNAL_ERROR,
326 "Unexpected exit_reason = %u\n", run->exit_reason);
330 #endif /* __x86_64__ */
332 static int test_memory_region_add(struct kvm_vm *vm, void *mem, uint32_t slot,
333 uint32_t size, uint64_t guest_addr)
335 struct kvm_userspace_memory_region region;
340 region.guest_phys_addr = guest_addr;
341 region.memory_size = size;
342 region.userspace_addr = (uintptr_t) mem;
343 ret = ioctl(vm_get_fd(vm), KVM_SET_USER_MEMORY_REGION, ®ion);
349 * Test it can be added memory slots up to KVM_CAP_NR_MEMSLOTS, then any
350 * tentative to add further slots should fail.
352 static void test_add_max_memory_regions(void)
356 uint32_t max_mem_slots;
358 void *mem, *mem_aligned, *mem_extra;
362 /* On s390x, the host address must be aligned to 1M (due to PGSTEs) */
363 alignment = 0x100000;
368 max_mem_slots = kvm_check_cap(KVM_CAP_NR_MEMSLOTS);
369 TEST_ASSERT(max_mem_slots > 0,
370 "KVM_CAP_NR_MEMSLOTS should be greater than 0");
371 pr_info("Allowed number of memory slots: %i\n", max_mem_slots);
373 vm = vm_create(VM_MODE_DEFAULT, 0, O_RDWR);
375 /* Check it can be added memory slots up to the maximum allowed */
376 pr_info("Adding slots 0..%i, each memory region with %dK size\n",
377 (max_mem_slots - 1), MEM_REGION_SIZE >> 10);
379 mem = mmap(NULL, MEM_REGION_SIZE * max_mem_slots + alignment,
380 PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
381 TEST_ASSERT(mem != MAP_FAILED, "Failed to mmap() host");
382 mem_aligned = (void *)(((size_t) mem + alignment - 1) & ~(alignment - 1));
384 for (slot = 0; slot < max_mem_slots; slot++) {
385 ret = test_memory_region_add(vm, mem_aligned +
386 ((uint64_t)slot * MEM_REGION_SIZE),
387 slot, MEM_REGION_SIZE,
388 (uint64_t)slot * MEM_REGION_SIZE);
389 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
390 " rc: %i errno: %i slot: %i\n",
394 /* Check it cannot be added memory slots beyond the limit */
395 mem_extra = mmap(NULL, MEM_REGION_SIZE, PROT_READ | PROT_WRITE,
396 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
397 TEST_ASSERT(mem_extra != MAP_FAILED, "Failed to mmap() host");
399 ret = test_memory_region_add(vm, mem_extra, max_mem_slots, MEM_REGION_SIZE,
400 (uint64_t)max_mem_slots * MEM_REGION_SIZE);
401 TEST_ASSERT(ret == -1 && errno == EINVAL,
402 "Adding one more memory slot should fail with EINVAL");
404 munmap(mem, MEM_REGION_SIZE * max_mem_slots + alignment);
405 munmap(mem_extra, MEM_REGION_SIZE);
409 int main(int argc, char *argv[])
415 /* Tell stdout not to buffer its content */
416 setbuf(stdout, NULL);
420 * FIXME: the zero-memslot test fails on aarch64 and s390x because
421 * KVM_RUN fails with ENOEXEC or EFAULT.
423 test_zero_memory_regions();
426 test_add_max_memory_regions();
430 loops = atoi(argv[1]);
434 pr_info("Testing MOVE of in-use region, %d loops\n", loops);
435 for (i = 0; i < loops; i++)
436 test_move_memory_region();
438 pr_info("Testing DELETE of in-use region, %d loops\n", loops);
439 for (i = 0; i < loops; i++)
440 test_delete_memory_region();