virt/kvm/async_pf.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * kvm asynchronous fault support
   4  *
   5  * Copyright 2010 Red Hat, Inc.
   6  *
   7  * Author:
   8  *      Gleb Natapov <gleb@redhat.com>
   9  */
  10
  11 #include <linux/kvm_host.h>
  12 #include <linux/slab.h>
  13 #include <linux/module.h>
  14 #include <linux/mmu_context.h>
  15 #include <linux/sched/mm.h>
  16
  17 #include "async_pf.h"
  18 #include <trace/events/kvm.h>
  19
  20 static struct kmem_cache *async_pf_cache;
  21
  22 int kvm_async_pf_init(void)
  23 {
  24         async_pf_cache = KMEM_CACHE(kvm_async_pf, 0);
  25
  26         if (!async_pf_cache)
  27                 return -ENOMEM;
  28
  29         return 0;
  30 }
  31
  32 void kvm_async_pf_deinit(void)
  33 {
  34         kmem_cache_destroy(async_pf_cache);
  35         async_pf_cache = NULL;
  36 }
  37
  38 void kvm_async_pf_vcpu_init(struct kvm_vcpu *vcpu)
  39 {
  40         INIT_LIST_HEAD(&vcpu->async_pf.done);
  41         INIT_LIST_HEAD(&vcpu->async_pf.queue);
  42         spin_lock_init(&vcpu->async_pf.lock);
  43 }
  44
  45 static void async_pf_execute(struct work_struct *work)
  46 {
  47         struct kvm_async_pf *apf =
  48                 container_of(work, struct kvm_async_pf, work);
  49         struct mm_struct *mm = apf->mm;
  50         struct kvm_vcpu *vcpu = apf->vcpu;
  51         unsigned long addr = apf->addr;
  52         gpa_t cr2_or_gpa = apf->cr2_or_gpa;
  53         int locked = 1;
  54
  55         might_sleep();
  56
  57         /*
  58          * This work is run asynchronously to the task which owns
  59          * mm and might be done in another context, so we must
  60          * access remotely.
  61          */
  62         down_read(&mm->mmap_sem);
  63         get_user_pages_remote(NULL, mm, addr, 1, FOLL_WRITE, NULL, NULL,
  64                         &locked);
  65         if (locked)
  66                 up_read(&mm->mmap_sem);
  67
  68         if (IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC))
  69                 kvm_arch_async_page_present(vcpu, apf);
  70
  71         spin_lock(&vcpu->async_pf.lock);
  72         list_add_tail(&apf->link, &vcpu->async_pf.done);
  73         apf->vcpu = NULL;
  74         spin_unlock(&vcpu->async_pf.lock);
  75
  76         /*
  77          * apf may be freed by kvm_check_async_pf_completion() after
  78          * this point
  79          */
  80
  81         trace_kvm_async_pf_completed(addr, cr2_or_gpa);
  82
  83         if (swq_has_sleeper(&vcpu->wq))
  84                 swake_up_one(&vcpu->wq);
  85
  86         mmput(mm);
  87         kvm_put_kvm(vcpu->kvm);
  88 }
  89
  90 void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu)
  91 {
  92         spin_lock(&vcpu->async_pf.lock);
  93
  94         /* cancel outstanding work queue item */
  95         while (!list_empty(&vcpu->async_pf.queue)) {
  96                 struct kvm_async_pf *work =
  97                         list_first_entry(&vcpu->async_pf.queue,
  98                                          typeof(*work), queue);
  99                 list_del(&work->queue);
 100
 101                 /*
 102                  * We know it's present in vcpu->async_pf.done, do
 103                  * nothing here.
 104                  */
 105                 if (!work->vcpu)
 106                         continue;
 107
 108                 spin_unlock(&vcpu->async_pf.lock);
 109 #ifdef CONFIG_KVM_ASYNC_PF_SYNC
 110                 flush_work(&work->work);
 111 #else
 112                 if (cancel_work_sync(&work->work)) {
 113                         mmput(work->mm);
 114                         kvm_put_kvm(vcpu->kvm); /* == work->vcpu->kvm */
 115                         kmem_cache_free(async_pf_cache, work);
 116                 }
 117 #endif
 118                 spin_lock(&vcpu->async_pf.lock);
 119         }
 120
 121         while (!list_empty(&vcpu->async_pf.done)) {
 122                 struct kvm_async_pf *work =
 123                         list_first_entry(&vcpu->async_pf.done,
 124                                          typeof(*work), link);
 125                 list_del(&work->link);
 126                 kmem_cache_free(async_pf_cache, work);
 127         }
 128         spin_unlock(&vcpu->async_pf.lock);
 129
 130         vcpu->async_pf.queued = 0;
 131 }
 132
 133 void kvm_check_async_pf_completion(struct kvm_vcpu *vcpu)
 134 {
 135         struct kvm_async_pf *work;
 136
 137         while (!list_empty_careful(&vcpu->async_pf.done) &&
 138               kvm_arch_can_inject_async_page_present(vcpu)) {
 139                 spin_lock(&vcpu->async_pf.lock);
 140                 work = list_first_entry(&vcpu->async_pf.done, typeof(*work),
 141                                               link);
 142                 list_del(&work->link);
 143                 spin_unlock(&vcpu->async_pf.lock);
 144
 145                 kvm_arch_async_page_ready(vcpu, work);
 146                 if (!IS_ENABLED(CONFIG_KVM_ASYNC_PF_SYNC))
 147                         kvm_arch_async_page_present(vcpu, work);
 148
 149                 list_del(&work->queue);
 150                 vcpu->async_pf.queued--;
 151                 kmem_cache_free(async_pf_cache, work);
 152         }
 153 }
 154
 155 int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
 156                        unsigned long hva, struct kvm_arch_async_pf *arch)
 157 {
 158         struct kvm_async_pf *work;
 159
 160         if (vcpu->async_pf.queued >= ASYNC_PF_PER_VCPU)
 161                 return 0;
 162
 163         /* setup delayed work */
 164
 165         /*
 166          * do alloc nowait since if we are going to sleep anyway we
 167          * may as well sleep faulting in page
 168          */
 169         work = kmem_cache_zalloc(async_pf_cache, GFP_NOWAIT | __GFP_NOWARN);
 170         if (!work)
 171                 return 0;
 172
 173         work->wakeup_all = false;
 174         work->vcpu = vcpu;
 175         work->cr2_or_gpa = cr2_or_gpa;
 176         work->addr = hva;
 177         work->arch = *arch;
 178         work->mm = current->mm;
 179         mmget(work->mm);
 180         kvm_get_kvm(work->vcpu->kvm);
 181
 182         /* this can't really happen otherwise gfn_to_pfn_async
 183            would succeed */
 184         if (unlikely(kvm_is_error_hva(work->addr)))
 185                 goto retry_sync;
 186
 187         INIT_WORK(&work->work, async_pf_execute);
 188         if (!schedule_work(&work->work))
 189                 goto retry_sync;
 190
 191         list_add_tail(&work->queue, &vcpu->async_pf.queue);
 192         vcpu->async_pf.queued++;
 193         kvm_arch_async_page_not_present(vcpu, work);
 194         return 1;
 195 retry_sync:
 196         kvm_put_kvm(work->vcpu->kvm);
 197         mmput(work->mm);
 198         kmem_cache_free(async_pf_cache, work);
 199         return 0;
 200 }
 201
 202 int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu)
 203 {
 204         struct kvm_async_pf *work;
 205
 206         if (!list_empty_careful(&vcpu->async_pf.done))
 207                 return 0;
 208
 209         work = kmem_cache_zalloc(async_pf_cache, GFP_ATOMIC);
 210         if (!work)
 211                 return -ENOMEM;
 212
 213         work->wakeup_all = true;
 214         INIT_LIST_HEAD(&work->queue); /* for list_del to work */
 215
 216         spin_lock(&vcpu->async_pf.lock);
 217         list_add_tail(&work->link, &vcpu->async_pf.done);
 218         spin_unlock(&vcpu->async_pf.lock);
 219
 220         vcpu->async_pf.queued++;
 221         return 0;
 222 }