arch/x86/kvm/svm/avic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * AMD SVM support
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *   Avi Kivity   <avi@qumranet.com>
  13  */
  14
  15 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  16
  17 #include <linux/kvm_types.h>
  18 #include <linux/hashtable.h>
  19 #include <linux/amd-iommu.h>
  20 #include <linux/kvm_host.h>
  21
  22 #include <asm/irq_remapping.h>
  23
  24 #include "trace.h"
  25 #include "lapic.h"
  26 #include "x86.h"
  27 #include "irq.h"
  28 #include "svm.h"
  29
  30 /*
  31  * Encode the arbitrary VM ID and the vCPU's default APIC ID, i.e the vCPU ID,
  32  * into the GATag so that KVM can retrieve the correct vCPU from a GALog entry
  33  * if an interrupt can't be delivered, e.g. because the vCPU isn't running.
  34  *
  35  * For the vCPU ID, use however many bits are currently allowed for the max
  36  * guest physical APIC ID (limited by the size of the physical ID table), and
  37  * use whatever bits remain to assign arbitrary AVIC IDs to VMs.  Note, the
  38  * size of the GATag is defined by hardware (32 bits), but is an opaque value
  39  * as far as hardware is concerned.
  40  */
  41 #define AVIC_VCPU_ID_MASK               AVIC_PHYSICAL_MAX_INDEX_MASK
  42
  43 #define AVIC_VM_ID_SHIFT                HWEIGHT32(AVIC_PHYSICAL_MAX_INDEX_MASK)
  44 #define AVIC_VM_ID_MASK                 (GENMASK(31, AVIC_VM_ID_SHIFT) >> AVIC_VM_ID_SHIFT)
  45
  46 #define AVIC_GATAG_TO_VMID(x)           ((x >> AVIC_VM_ID_SHIFT) & AVIC_VM_ID_MASK)
  47 #define AVIC_GATAG_TO_VCPUID(x)         (x & AVIC_VCPU_ID_MASK)
  48
  49 #define __AVIC_GATAG(vm_id, vcpu_id)    ((((vm_id) & AVIC_VM_ID_MASK) << AVIC_VM_ID_SHIFT) | \
  50                                          ((vcpu_id) & AVIC_VCPU_ID_MASK))
  51 #define AVIC_GATAG(vm_id, vcpu_id)                                      \
  52 ({                                                                      \
  53         u32 ga_tag = __AVIC_GATAG(vm_id, vcpu_id);                      \
  54                                                                         \
  55         WARN_ON_ONCE(AVIC_GATAG_TO_VCPUID(ga_tag) != (vcpu_id));        \
  56         WARN_ON_ONCE(AVIC_GATAG_TO_VMID(ga_tag) != (vm_id));            \
  57         ga_tag;                                                         \
  58 })
  59
  60 static_assert(__AVIC_GATAG(AVIC_VM_ID_MASK, AVIC_VCPU_ID_MASK) == -1u);
  61
  62 static bool force_avic;
  63 module_param_unsafe(force_avic, bool, 0444);
  64
  65 /* Note:
  66  * This hash table is used to map VM_ID to a struct kvm_svm,
  67  * when handling AMD IOMMU GALOG notification to schedule in
  68  * a particular vCPU.
  69  */
  70 #define SVM_VM_DATA_HASH_BITS   8
  71 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
  72 static u32 next_vm_id = 0;
  73 static bool next_vm_id_wrapped = 0;
  74 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
  75 bool x2avic_enabled;
  76
  77 /*
  78  * This is a wrapper of struct amd_iommu_ir_data.
  79  */
  80 struct amd_svm_iommu_ir {
  81         struct list_head node;  /* Used by SVM for per-vcpu ir_list */
  82         void *data;             /* Storing pointer to struct amd_ir_data */
  83 };
  84
  85 static void avic_activate_vmcb(struct vcpu_svm *svm)
  86 {
  87         struct vmcb *vmcb = svm->vmcb01.ptr;
  88
  89         vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK);
  90         vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK;
  91
  92         vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
  93
  94         /*
  95          * Note: KVM supports hybrid-AVIC mode, where KVM emulates x2APIC MSR
  96          * accesses, while interrupt injection to a running vCPU can be
  97          * achieved using AVIC doorbell.  KVM disables the APIC access page
  98          * (deletes the memslot) if any vCPU has x2APIC enabled, thus enabling
  99          * AVIC in hybrid mode activates only the doorbell mechanism.
 100          */
 101         if (x2avic_enabled && apic_x2apic_mode(svm->vcpu.arch.apic)) {
 102                 vmcb->control.int_ctl |= X2APIC_MODE_MASK;
 103                 vmcb->control.avic_physical_id |= X2AVIC_MAX_PHYSICAL_ID;
 104                 /* Disabling MSR intercept for x2APIC registers */
 105                 svm_set_x2apic_msr_interception(svm, false);
 106         } else {
 107                 /*
 108                  * Flush the TLB, the guest may have inserted a non-APIC
 109                  * mapping into the TLB while AVIC was disabled.
 110                  */
 111                 kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, &svm->vcpu);
 112
 113                 /* For xAVIC and hybrid-xAVIC modes */
 114                 vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID;
 115                 /* Enabling MSR intercept for x2APIC registers */
 116                 svm_set_x2apic_msr_interception(svm, true);
 117         }
 118 }
 119
 120 static void avic_deactivate_vmcb(struct vcpu_svm *svm)
 121 {
 122         struct vmcb *vmcb = svm->vmcb01.ptr;
 123
 124         vmcb->control.int_ctl &= ~(AVIC_ENABLE_MASK | X2APIC_MODE_MASK);
 125         vmcb->control.avic_physical_id &= ~AVIC_PHYSICAL_MAX_INDEX_MASK;
 126
 127         /*
 128          * If running nested and the guest uses its own MSR bitmap, there
 129          * is no need to update L0's msr bitmap
 130          */
 131         if (is_guest_mode(&svm->vcpu) &&
 132             vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_MSR_PROT))
 133                 return;
 134
 135         /* Enabling MSR intercept for x2APIC registers */
 136         svm_set_x2apic_msr_interception(svm, true);
 137 }
 138
 139 /* Note:
 140  * This function is called from IOMMU driver to notify
 141  * SVM to schedule in a particular vCPU of a particular VM.
 142  */
 143 int avic_ga_log_notifier(u32 ga_tag)
 144 {
 145         unsigned long flags;
 146         struct kvm_svm *kvm_svm;
 147         struct kvm_vcpu *vcpu = NULL;
 148         u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
 149         u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
 150
 151         pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
 152         trace_kvm_avic_ga_log(vm_id, vcpu_id);
 153
 154         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
 155         hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
 156                 if (kvm_svm->avic_vm_id != vm_id)
 157                         continue;
 158                 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
 159                 break;
 160         }
 161         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
 162
 163         /* Note:
 164          * At this point, the IOMMU should have already set the pending
 165          * bit in the vAPIC backing page. So, we just need to schedule
 166          * in the vcpu.
 167          */
 168         if (vcpu)
 169                 kvm_vcpu_wake_up(vcpu);
 170
 171         return 0;
 172 }
 173
 174 void avic_vm_destroy(struct kvm *kvm)
 175 {
 176         unsigned long flags;
 177         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 178
 179         if (!enable_apicv)
 180                 return;
 181
 182         if (kvm_svm->avic_logical_id_table_page)
 183                 __free_page(kvm_svm->avic_logical_id_table_page);
 184         if (kvm_svm->avic_physical_id_table_page)
 185                 __free_page(kvm_svm->avic_physical_id_table_page);
 186
 187         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
 188         hash_del(&kvm_svm->hnode);
 189         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
 190 }
 191
 192 int avic_vm_init(struct kvm *kvm)
 193 {
 194         unsigned long flags;
 195         int err = -ENOMEM;
 196         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 197         struct kvm_svm *k2;
 198         struct page *p_page;
 199         struct page *l_page;
 200         u32 vm_id;
 201
 202         if (!enable_apicv)
 203                 return 0;
 204
 205         /* Allocating physical APIC ID table (4KB) */
 206         p_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 207         if (!p_page)
 208                 goto free_avic;
 209
 210         kvm_svm->avic_physical_id_table_page = p_page;
 211
 212         /* Allocating logical APIC ID table (4KB) */
 213         l_page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
 214         if (!l_page)
 215                 goto free_avic;
 216
 217         kvm_svm->avic_logical_id_table_page = l_page;
 218
 219         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
 220  again:
 221         vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
 222         if (vm_id == 0) { /* id is 1-based, zero is not okay */
 223                 next_vm_id_wrapped = 1;
 224                 goto again;
 225         }
 226         /* Is it still in use? Only possible if wrapped at least once */
 227         if (next_vm_id_wrapped) {
 228                 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
 229                         if (k2->avic_vm_id == vm_id)
 230                                 goto again;
 231                 }
 232         }
 233         kvm_svm->avic_vm_id = vm_id;
 234         hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
 235         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
 236
 237         return 0;
 238
 239 free_avic:
 240         avic_vm_destroy(kvm);
 241         return err;
 242 }
 243
 244 void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb)
 245 {
 246         struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
 247         phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
 248         phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
 249         phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
 250
 251         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
 252         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
 253         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
 254         vmcb->control.avic_vapic_bar = APIC_DEFAULT_PHYS_BASE & VMCB_AVIC_APIC_BAR_MASK;
 255
 256         if (kvm_apicv_activated(svm->vcpu.kvm))
 257                 avic_activate_vmcb(svm);
 258         else
 259                 avic_deactivate_vmcb(svm);
 260 }
 261
 262 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
 263                                        unsigned int index)
 264 {
 265         u64 *avic_physical_id_table;
 266         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
 267
 268         if ((!x2avic_enabled && index > AVIC_MAX_PHYSICAL_ID) ||
 269             (index > X2AVIC_MAX_PHYSICAL_ID))
 270                 return NULL;
 271
 272         avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
 273
 274         return &avic_physical_id_table[index];
 275 }
 276
 277 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
 278 {
 279         u64 *entry, new_entry;
 280         int id = vcpu->vcpu_id;
 281         struct vcpu_svm *svm = to_svm(vcpu);
 282
 283         if ((!x2avic_enabled && id > AVIC_MAX_PHYSICAL_ID) ||
 284             (id > X2AVIC_MAX_PHYSICAL_ID))
 285                 return -EINVAL;
 286
 287         if (!vcpu->arch.apic->regs)
 288                 return -EINVAL;
 289
 290         if (kvm_apicv_activated(vcpu->kvm)) {
 291                 int ret;
 292
 293                 /*
 294                  * Note, AVIC hardware walks the nested page table to check
 295                  * permissions, but does not use the SPA address specified in
 296                  * the leaf SPTE since it uses address in the AVIC_BACKING_PAGE
 297                  * pointer field of the VMCB.
 298                  */
 299                 ret = kvm_alloc_apic_access_page(vcpu->kvm);
 300                 if (ret)
 301                         return ret;
 302         }
 303
 304         svm->avic_backing_page = virt_to_page(vcpu->arch.apic->regs);
 305
 306         /* Setting AVIC backing page address in the phy APIC ID table */
 307         entry = avic_get_physical_id_entry(vcpu, id);
 308         if (!entry)
 309                 return -EINVAL;
 310
 311         new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
 312                               AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
 313                               AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
 314         WRITE_ONCE(*entry, new_entry);
 315
 316         svm->avic_physical_id_cache = entry;
 317
 318         return 0;
 319 }
 320
 321 void avic_ring_doorbell(struct kvm_vcpu *vcpu)
 322 {
 323         /*
 324          * Note, the vCPU could get migrated to a different pCPU at any point,
 325          * which could result in signalling the wrong/previous pCPU.  But if
 326          * that happens the vCPU is guaranteed to do a VMRUN (after being
 327          * migrated) and thus will process pending interrupts, i.e. a doorbell
 328          * is not needed (and the spurious one is harmless).
 329          */
 330         int cpu = READ_ONCE(vcpu->cpu);
 331
 332         if (cpu != get_cpu()) {
 333                 wrmsrl(MSR_AMD64_SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpu));
 334                 trace_kvm_avic_doorbell(vcpu->vcpu_id, kvm_cpu_get_apicid(cpu));
 335         }
 336         put_cpu();
 337 }
 338
 339
 340 static void avic_kick_vcpu(struct kvm_vcpu *vcpu, u32 icrl)
 341 {
 342         vcpu->arch.apic->irr_pending = true;
 343         svm_complete_interrupt_delivery(vcpu,
 344                                         icrl & APIC_MODE_MASK,
 345                                         icrl & APIC_INT_LEVELTRIG,
 346                                         icrl & APIC_VECTOR_MASK);
 347 }
 348
 349 static void avic_kick_vcpu_by_physical_id(struct kvm *kvm, u32 physical_id,
 350                                           u32 icrl)
 351 {
 352         /*
 353          * KVM inhibits AVIC if any vCPU ID diverges from the vCPUs APIC ID,
 354          * i.e. APIC ID == vCPU ID.
 355          */
 356         struct kvm_vcpu *target_vcpu = kvm_get_vcpu_by_id(kvm, physical_id);
 357
 358         /* Once again, nothing to do if the target vCPU doesn't exist. */
 359         if (unlikely(!target_vcpu))
 360                 return;
 361
 362         avic_kick_vcpu(target_vcpu, icrl);
 363 }
 364
 365 static void avic_kick_vcpu_by_logical_id(struct kvm *kvm, u32 *avic_logical_id_table,
 366                                          u32 logid_index, u32 icrl)
 367 {
 368         u32 physical_id;
 369
 370         if (avic_logical_id_table) {
 371                 u32 logid_entry = avic_logical_id_table[logid_index];
 372
 373                 /* Nothing to do if the logical destination is invalid. */
 374                 if (unlikely(!(logid_entry & AVIC_LOGICAL_ID_ENTRY_VALID_MASK)))
 375                         return;
 376
 377                 physical_id = logid_entry &
 378                               AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
 379         } else {
 380                 /*
 381                  * For x2APIC, the logical APIC ID is a read-only value that is
 382                  * derived from the x2APIC ID, thus the x2APIC ID can be found
 383                  * by reversing the calculation (stored in logid_index).  Note,
 384                  * bits 31:20 of the x2APIC ID aren't propagated to the logical
 385                  * ID, but KVM limits the x2APIC ID limited to KVM_MAX_VCPU_IDS.
 386                  */
 387                 physical_id = logid_index;
 388         }
 389
 390         avic_kick_vcpu_by_physical_id(kvm, physical_id, icrl);
 391 }
 392
 393 /*
 394  * A fast-path version of avic_kick_target_vcpus(), which attempts to match
 395  * destination APIC ID to vCPU without looping through all vCPUs.
 396  */
 397 static int avic_kick_target_vcpus_fast(struct kvm *kvm, struct kvm_lapic *source,
 398                                        u32 icrl, u32 icrh, u32 index)
 399 {
 400         int dest_mode = icrl & APIC_DEST_MASK;
 401         int shorthand = icrl & APIC_SHORT_MASK;
 402         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
 403         u32 dest;
 404
 405         if (shorthand != APIC_DEST_NOSHORT)
 406                 return -EINVAL;
 407
 408         if (apic_x2apic_mode(source))
 409                 dest = icrh;
 410         else
 411                 dest = GET_XAPIC_DEST_FIELD(icrh);
 412
 413         if (dest_mode == APIC_DEST_PHYSICAL) {
 414                 /* broadcast destination, use slow path */
 415                 if (apic_x2apic_mode(source) && dest == X2APIC_BROADCAST)
 416                         return -EINVAL;
 417                 if (!apic_x2apic_mode(source) && dest == APIC_BROADCAST)
 418                         return -EINVAL;
 419
 420                 if (WARN_ON_ONCE(dest != index))
 421                         return -EINVAL;
 422
 423                 avic_kick_vcpu_by_physical_id(kvm, dest, icrl);
 424         } else {
 425                 u32 *avic_logical_id_table;
 426                 unsigned long bitmap, i;
 427                 u32 cluster;
 428
 429                 if (apic_x2apic_mode(source)) {
 430                         /* 16 bit dest mask, 16 bit cluster id */
 431                         bitmap = dest & 0xFFFF;
 432                         cluster = (dest >> 16) << 4;
 433                 } else if (kvm_lapic_get_reg(source, APIC_DFR) == APIC_DFR_FLAT) {
 434                         /* 8 bit dest mask*/
 435                         bitmap = dest;
 436                         cluster = 0;
 437                 } else {
 438                         /* 4 bit desk mask, 4 bit cluster id */
 439                         bitmap = dest & 0xF;
 440                         cluster = (dest >> 4) << 2;
 441                 }
 442
 443                 /* Nothing to do if there are no destinations in the cluster. */
 444                 if (unlikely(!bitmap))
 445                         return 0;
 446
 447                 if (apic_x2apic_mode(source))
 448                         avic_logical_id_table = NULL;
 449                 else
 450                         avic_logical_id_table = page_address(kvm_svm->avic_logical_id_table_page);
 451
 452                 /*
 453                  * AVIC is inhibited if vCPUs aren't mapped 1:1 with logical
 454                  * IDs, thus each bit in the destination is guaranteed to map
 455                  * to at most one vCPU.
 456                  */
 457                 for_each_set_bit(i, &bitmap, 16)
 458                         avic_kick_vcpu_by_logical_id(kvm, avic_logical_id_table,
 459                                                      cluster + i, icrl);
 460         }
 461
 462         return 0;
 463 }
 464
 465 static void avic_kick_target_vcpus(struct kvm *kvm, struct kvm_lapic *source,
 466                                    u32 icrl, u32 icrh, u32 index)
 467 {
 468         u32 dest = apic_x2apic_mode(source) ? icrh : GET_XAPIC_DEST_FIELD(icrh);
 469         unsigned long i;
 470         struct kvm_vcpu *vcpu;
 471
 472         if (!avic_kick_target_vcpus_fast(kvm, source, icrl, icrh, index))
 473                 return;
 474
 475         trace_kvm_avic_kick_vcpu_slowpath(icrh, icrl, index);
 476
 477         /*
 478          * Wake any target vCPUs that are blocking, i.e. waiting for a wake
 479          * event.  There's no need to signal doorbells, as hardware has handled
 480          * vCPUs that were in guest at the time of the IPI, and vCPUs that have
 481          * since entered the guest will have processed pending IRQs at VMRUN.
 482          */
 483         kvm_for_each_vcpu(i, vcpu, kvm) {
 484                 if (kvm_apic_match_dest(vcpu, source, icrl & APIC_SHORT_MASK,
 485                                         dest, icrl & APIC_DEST_MASK))
 486                         avic_kick_vcpu(vcpu, icrl);
 487         }
 488 }
 489
 490 int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu)
 491 {
 492         struct vcpu_svm *svm = to_svm(vcpu);
 493         u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
 494         u32 icrl = svm->vmcb->control.exit_info_1;
 495         u32 id = svm->vmcb->control.exit_info_2 >> 32;
 496         u32 index = svm->vmcb->control.exit_info_2 & 0x1FF;
 497         struct kvm_lapic *apic = vcpu->arch.apic;
 498
 499         trace_kvm_avic_incomplete_ipi(vcpu->vcpu_id, icrh, icrl, id, index);
 500
 501         switch (id) {
 502         case AVIC_IPI_FAILURE_INVALID_TARGET:
 503         case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
 504                 /*
 505                  * Emulate IPIs that are not handled by AVIC hardware, which
 506                  * only virtualizes Fixed, Edge-Triggered INTRs, and falls over
 507                  * if _any_ targets are invalid, e.g. if the logical mode mask
 508                  * is a superset of running vCPUs.
 509                  *
 510                  * The exit is a trap, e.g. ICR holds the correct value and RIP
 511                  * has been advanced, KVM is responsible only for emulating the
 512                  * IPI.  Sadly, hardware may sometimes leave the BUSY flag set,
 513                  * in which case KVM needs to emulate the ICR write as well in
 514                  * order to clear the BUSY flag.
 515                  */
 516                 if (icrl & APIC_ICR_BUSY)
 517                         kvm_apic_write_nodecode(vcpu, APIC_ICR);
 518                 else
 519                         kvm_apic_send_ipi(apic, icrl, icrh);
 520                 break;
 521         case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING:
 522                 /*
 523                  * At this point, we expect that the AVIC HW has already
 524                  * set the appropriate IRR bits on the valid target
 525                  * vcpus. So, we just need to kick the appropriate vcpu.
 526                  */
 527                 avic_kick_target_vcpus(vcpu->kvm, apic, icrl, icrh, index);
 528                 break;
 529         case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
 530                 WARN_ONCE(1, "Invalid backing page\n");
 531                 break;
 532         case AVIC_IPI_FAILURE_INVALID_IPI_VECTOR:
 533                 /* Invalid IPI with vector < 16 */
 534                 break;
 535         default:
 536                 vcpu_unimpl(vcpu, "Unknown avic incomplete IPI interception\n");
 537         }
 538
 539         return 1;
 540 }
 541
 542 unsigned long avic_vcpu_get_apicv_inhibit_reasons(struct kvm_vcpu *vcpu)
 543 {
 544         if (is_guest_mode(vcpu))
 545                 return APICV_INHIBIT_REASON_NESTED;
 546         return 0;
 547 }
 548
 549 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
 550 {
 551         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
 552         u32 *logical_apic_id_table;
 553         u32 cluster, index;
 554
 555         ldr = GET_APIC_LOGICAL_ID(ldr);
 556
 557         if (flat) {
 558                 cluster = 0;
 559         } else {
 560                 cluster = (ldr >> 4);
 561                 if (cluster >= 0xf)
 562                         return NULL;
 563                 ldr &= 0xf;
 564         }
 565         if (!ldr || !is_power_of_2(ldr))
 566                 return NULL;
 567
 568         index = __ffs(ldr);
 569         if (WARN_ON_ONCE(index > 7))
 570                 return NULL;
 571         index += (cluster << 2);
 572
 573         logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
 574
 575         return &logical_apic_id_table[index];
 576 }
 577
 578 static void avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr)
 579 {
 580         bool flat;
 581         u32 *entry, new_entry;
 582
 583         flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
 584         entry = avic_get_logical_id_entry(vcpu, ldr, flat);
 585         if (!entry)
 586                 return;
 587
 588         new_entry = READ_ONCE(*entry);
 589         new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
 590         new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
 591         new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
 592         WRITE_ONCE(*entry, new_entry);
 593 }
 594
 595 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu)
 596 {
 597         struct vcpu_svm *svm = to_svm(vcpu);
 598         bool flat = svm->dfr_reg == APIC_DFR_FLAT;
 599         u32 *entry;
 600
 601         /* Note: x2AVIC does not use logical APIC ID table */
 602         if (apic_x2apic_mode(vcpu->arch.apic))
 603                 return;
 604
 605         entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat);
 606         if (entry)
 607                 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry);
 608 }
 609
 610 static void avic_handle_ldr_update(struct kvm_vcpu *vcpu)
 611 {
 612         struct vcpu_svm *svm = to_svm(vcpu);
 613         u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
 614         u32 id = kvm_xapic_id(vcpu->arch.apic);
 615
 616         /* AVIC does not support LDR update for x2APIC */
 617         if (apic_x2apic_mode(vcpu->arch.apic))
 618                 return;
 619
 620         if (ldr == svm->ldr_reg)
 621                 return;
 622
 623         avic_invalidate_logical_id_entry(vcpu);
 624
 625         svm->ldr_reg = ldr;
 626         avic_ldr_write(vcpu, id, ldr);
 627 }
 628
 629 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu)
 630 {
 631         struct vcpu_svm *svm = to_svm(vcpu);
 632         u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
 633
 634         if (svm->dfr_reg == dfr)
 635                 return;
 636
 637         avic_invalidate_logical_id_entry(vcpu);
 638         svm->dfr_reg = dfr;
 639 }
 640
 641 static int avic_unaccel_trap_write(struct kvm_vcpu *vcpu)
 642 {
 643         u32 offset = to_svm(vcpu)->vmcb->control.exit_info_1 &
 644                                 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
 645
 646         switch (offset) {
 647         case APIC_LDR:
 648                 avic_handle_ldr_update(vcpu);
 649                 break;
 650         case APIC_DFR:
 651                 avic_handle_dfr_update(vcpu);
 652                 break;
 653         case APIC_RRR:
 654                 /* Ignore writes to Read Remote Data, it's read-only. */
 655                 return 1;
 656         default:
 657                 break;
 658         }
 659
 660         kvm_apic_write_nodecode(vcpu, offset);
 661         return 1;
 662 }
 663
 664 static bool is_avic_unaccelerated_access_trap(u32 offset)
 665 {
 666         bool ret = false;
 667
 668         switch (offset) {
 669         case APIC_ID:
 670         case APIC_EOI:
 671         case APIC_RRR:
 672         case APIC_LDR:
 673         case APIC_DFR:
 674         case APIC_SPIV:
 675         case APIC_ESR:
 676         case APIC_ICR:
 677         case APIC_LVTT:
 678         case APIC_LVTTHMR:
 679         case APIC_LVTPC:
 680         case APIC_LVT0:
 681         case APIC_LVT1:
 682         case APIC_LVTERR:
 683         case APIC_TMICT:
 684         case APIC_TDCR:
 685                 ret = true;
 686                 break;
 687         default:
 688                 break;
 689         }
 690         return ret;
 691 }
 692
 693 int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu)
 694 {
 695         struct vcpu_svm *svm = to_svm(vcpu);
 696         int ret = 0;
 697         u32 offset = svm->vmcb->control.exit_info_1 &
 698                      AVIC_UNACCEL_ACCESS_OFFSET_MASK;
 699         u32 vector = svm->vmcb->control.exit_info_2 &
 700                      AVIC_UNACCEL_ACCESS_VECTOR_MASK;
 701         bool write = (svm->vmcb->control.exit_info_1 >> 32) &
 702                      AVIC_UNACCEL_ACCESS_WRITE_MASK;
 703         bool trap = is_avic_unaccelerated_access_trap(offset);
 704
 705         trace_kvm_avic_unaccelerated_access(vcpu->vcpu_id, offset,
 706                                             trap, write, vector);
 707         if (trap) {
 708                 /* Handling Trap */
 709                 WARN_ONCE(!write, "svm: Handling trap read.\n");
 710                 ret = avic_unaccel_trap_write(vcpu);
 711         } else {
 712                 /* Handling Fault */
 713                 ret = kvm_emulate_instruction(vcpu, 0);
 714         }
 715
 716         return ret;
 717 }
 718
 719 int avic_init_vcpu(struct vcpu_svm *svm)
 720 {
 721         int ret;
 722         struct kvm_vcpu *vcpu = &svm->vcpu;
 723
 724         if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm))
 725                 return 0;
 726
 727         ret = avic_init_backing_page(vcpu);
 728         if (ret)
 729                 return ret;
 730
 731         INIT_LIST_HEAD(&svm->ir_list);
 732         spin_lock_init(&svm->ir_list_lock);
 733         svm->dfr_reg = APIC_DFR_FLAT;
 734
 735         return ret;
 736 }
 737
 738 void avic_apicv_post_state_restore(struct kvm_vcpu *vcpu)
 739 {
 740         avic_handle_dfr_update(vcpu);
 741         avic_handle_ldr_update(vcpu);
 742 }
 743
 744 static int avic_set_pi_irte_mode(struct kvm_vcpu *vcpu, bool activate)
 745 {
 746         int ret = 0;
 747         unsigned long flags;
 748         struct amd_svm_iommu_ir *ir;
 749         struct vcpu_svm *svm = to_svm(vcpu);
 750
 751         if (!kvm_arch_has_assigned_device(vcpu->kvm))
 752                 return 0;
 753
 754         /*
 755          * Here, we go through the per-vcpu ir_list to update all existing
 756          * interrupt remapping table entry targeting this vcpu.
 757          */
 758         spin_lock_irqsave(&svm->ir_list_lock, flags);
 759
 760         if (list_empty(&svm->ir_list))
 761                 goto out;
 762
 763         list_for_each_entry(ir, &svm->ir_list, node) {
 764                 if (activate)
 765                         ret = amd_iommu_activate_guest_mode(ir->data);
 766                 else
 767                         ret = amd_iommu_deactivate_guest_mode(ir->data);
 768                 if (ret)
 769                         break;
 770         }
 771 out:
 772         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
 773         return ret;
 774 }
 775
 776 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
 777 {
 778         unsigned long flags;
 779         struct amd_svm_iommu_ir *cur;
 780
 781         spin_lock_irqsave(&svm->ir_list_lock, flags);
 782         list_for_each_entry(cur, &svm->ir_list, node) {
 783                 if (cur->data != pi->ir_data)
 784                         continue;
 785                 list_del(&cur->node);
 786                 kfree(cur);
 787                 break;
 788         }
 789         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
 790 }
 791
 792 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
 793 {
 794         int ret = 0;
 795         unsigned long flags;
 796         struct amd_svm_iommu_ir *ir;
 797         u64 entry;
 798
 799         /**
 800          * In some cases, the existing irte is updated and re-set,
 801          * so we need to check here if it's already been * added
 802          * to the ir_list.
 803          */
 804         if (pi->ir_data && (pi->prev_ga_tag != 0)) {
 805                 struct kvm *kvm = svm->vcpu.kvm;
 806                 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
 807                 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
 808                 struct vcpu_svm *prev_svm;
 809
 810                 if (!prev_vcpu) {
 811                         ret = -EINVAL;
 812                         goto out;
 813                 }
 814
 815                 prev_svm = to_svm(prev_vcpu);
 816                 svm_ir_list_del(prev_svm, pi);
 817         }
 818
 819         /**
 820          * Allocating new amd_iommu_pi_data, which will get
 821          * add to the per-vcpu ir_list.
 822          */
 823         ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT);
 824         if (!ir) {
 825                 ret = -ENOMEM;
 826                 goto out;
 827         }
 828         ir->data = pi->ir_data;
 829
 830         spin_lock_irqsave(&svm->ir_list_lock, flags);
 831
 832         /*
 833          * Update the target pCPU for IOMMU doorbells if the vCPU is running.
 834          * If the vCPU is NOT running, i.e. is blocking or scheduled out, KVM
 835          * will update the pCPU info when the vCPU awkened and/or scheduled in.
 836          * See also avic_vcpu_load().
 837          */
 838         entry = READ_ONCE(*(svm->avic_physical_id_cache));
 839         if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
 840                 amd_iommu_update_ga(entry & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK,
 841                                     true, pi->ir_data);
 842
 843         list_add(&ir->node, &svm->ir_list);
 844         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
 845 out:
 846         return ret;
 847 }
 848
 849 /*
 850  * Note:
 851  * The HW cannot support posting multicast/broadcast
 852  * interrupts to a vCPU. So, we still use legacy interrupt
 853  * remapping for these kind of interrupts.
 854  *
 855  * For lowest-priority interrupts, we only support
 856  * those with single CPU as the destination, e.g. user
 857  * configures the interrupts via /proc/irq or uses
 858  * irqbalance to make the interrupts single-CPU.
 859  */
 860 static int
 861 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
 862                  struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
 863 {
 864         struct kvm_lapic_irq irq;
 865         struct kvm_vcpu *vcpu = NULL;
 866
 867         kvm_set_msi_irq(kvm, e, &irq);
 868
 869         if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
 870             !kvm_irq_is_postable(&irq)) {
 871                 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
 872                          __func__, irq.vector);
 873                 return -1;
 874         }
 875
 876         pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
 877                  irq.vector);
 878         *svm = to_svm(vcpu);
 879         vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
 880         vcpu_info->vector = irq.vector;
 881
 882         return 0;
 883 }
 884
 885 /*
 886  * avic_pi_update_irte - set IRTE for Posted-Interrupts
 887  *
 888  * @kvm: kvm
 889  * @host_irq: host irq of the interrupt
 890  * @guest_irq: gsi of the interrupt
 891  * @set: set or unset PI
 892  * returns 0 on success, < 0 on failure
 893  */
 894 int avic_pi_update_irte(struct kvm *kvm, unsigned int host_irq,
 895                         uint32_t guest_irq, bool set)
 896 {
 897         struct kvm_kernel_irq_routing_entry *e;
 898         struct kvm_irq_routing_table *irq_rt;
 899         int idx, ret = 0;
 900
 901         if (!kvm_arch_has_assigned_device(kvm) ||
 902             !irq_remapping_cap(IRQ_POSTING_CAP))
 903                 return 0;
 904
 905         pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
 906                  __func__, host_irq, guest_irq, set);
 907
 908         idx = srcu_read_lock(&kvm->irq_srcu);
 909         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
 910
 911         if (guest_irq >= irq_rt->nr_rt_entries ||
 912                 hlist_empty(&irq_rt->map[guest_irq])) {
 913                 pr_warn_once("no route for guest_irq %u/%u (broken user space?)\n",
 914                              guest_irq, irq_rt->nr_rt_entries);
 915                 goto out;
 916         }
 917
 918         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
 919                 struct vcpu_data vcpu_info;
 920                 struct vcpu_svm *svm = NULL;
 921
 922                 if (e->type != KVM_IRQ_ROUTING_MSI)
 923                         continue;
 924
 925                 /**
 926                  * Here, we setup with legacy mode in the following cases:
 927                  * 1. When cannot target interrupt to a specific vcpu.
 928                  * 2. Unsetting posted interrupt.
 929                  * 3. APIC virtualization is disabled for the vcpu.
 930                  * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
 931                  */
 932                 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
 933                     kvm_vcpu_apicv_active(&svm->vcpu)) {
 934                         struct amd_iommu_pi_data pi;
 935
 936                         /* Try to enable guest_mode in IRTE */
 937                         pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
 938                                             AVIC_HPA_MASK);
 939                         pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
 940                                                      svm->vcpu.vcpu_id);
 941                         pi.is_guest_mode = true;
 942                         pi.vcpu_data = &vcpu_info;
 943                         ret = irq_set_vcpu_affinity(host_irq, &pi);
 944
 945                         /**
 946                          * Here, we successfully setting up vcpu affinity in
 947                          * IOMMU guest mode. Now, we need to store the posted
 948                          * interrupt information in a per-vcpu ir_list so that
 949                          * we can reference to them directly when we update vcpu
 950                          * scheduling information in IOMMU irte.
 951                          */
 952                         if (!ret && pi.is_guest_mode)
 953                                 svm_ir_list_add(svm, &pi);
 954                 } else {
 955                         /* Use legacy mode in IRTE */
 956                         struct amd_iommu_pi_data pi;
 957
 958                         /**
 959                          * Here, pi is used to:
 960                          * - Tell IOMMU to use legacy mode for this interrupt.
 961                          * - Retrieve ga_tag of prior interrupt remapping data.
 962                          */
 963                         pi.prev_ga_tag = 0;
 964                         pi.is_guest_mode = false;
 965                         ret = irq_set_vcpu_affinity(host_irq, &pi);
 966
 967                         /**
 968                          * Check if the posted interrupt was previously
 969                          * setup with the guest_mode by checking if the ga_tag
 970                          * was cached. If so, we need to clean up the per-vcpu
 971                          * ir_list.
 972                          */
 973                         if (!ret && pi.prev_ga_tag) {
 974                                 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
 975                                 struct kvm_vcpu *vcpu;
 976
 977                                 vcpu = kvm_get_vcpu_by_id(kvm, id);
 978                                 if (vcpu)
 979                                         svm_ir_list_del(to_svm(vcpu), &pi);
 980                         }
 981                 }
 982
 983                 if (!ret && svm) {
 984                         trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
 985                                                  e->gsi, vcpu_info.vector,
 986                                                  vcpu_info.pi_desc_addr, set);
 987                 }
 988
 989                 if (ret < 0) {
 990                         pr_err("%s: failed to update PI IRTE\n", __func__);
 991                         goto out;
 992                 }
 993         }
 994
 995         ret = 0;
 996 out:
 997         srcu_read_unlock(&kvm->irq_srcu, idx);
 998         return ret;
 999 }
1000
1001 static inline int
1002 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
1003 {
1004         int ret = 0;
1005         struct amd_svm_iommu_ir *ir;
1006         struct vcpu_svm *svm = to_svm(vcpu);
1007
1008         lockdep_assert_held(&svm->ir_list_lock);
1009
1010         if (!kvm_arch_has_assigned_device(vcpu->kvm))
1011                 return 0;
1012
1013         /*
1014          * Here, we go through the per-vcpu ir_list to update all existing
1015          * interrupt remapping table entry targeting this vcpu.
1016          */
1017         if (list_empty(&svm->ir_list))
1018                 return 0;
1019
1020         list_for_each_entry(ir, &svm->ir_list, node) {
1021                 ret = amd_iommu_update_ga(cpu, r, ir->data);
1022                 if (ret)
1023                         return ret;
1024         }
1025         return 0;
1026 }
1027
1028 void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1029 {
1030         u64 entry;
1031         int h_physical_id = kvm_cpu_get_apicid(cpu);
1032         struct vcpu_svm *svm = to_svm(vcpu);
1033         unsigned long flags;
1034
1035         lockdep_assert_preemption_disabled();
1036
1037         if (WARN_ON(h_physical_id & ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
1038                 return;
1039
1040         /*
1041          * No need to update anything if the vCPU is blocking, i.e. if the vCPU
1042          * is being scheduled in after being preempted.  The CPU entries in the
1043          * Physical APIC table and IRTE are consumed iff IsRun{ning} is '1'.
1044          * If the vCPU was migrated, its new CPU value will be stuffed when the
1045          * vCPU unblocks.
1046          */
1047         if (kvm_vcpu_is_blocking(vcpu))
1048                 return;
1049
1050         /*
1051          * Grab the per-vCPU interrupt remapping lock even if the VM doesn't
1052          * _currently_ have assigned devices, as that can change.  Holding
1053          * ir_list_lock ensures that either svm_ir_list_add() will consume
1054          * up-to-date entry information, or that this task will wait until
1055          * svm_ir_list_add() completes to set the new target pCPU.
1056          */
1057         spin_lock_irqsave(&svm->ir_list_lock, flags);
1058
1059         entry = READ_ONCE(*(svm->avic_physical_id_cache));
1060         WARN_ON_ONCE(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
1061
1062         entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
1063         entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
1064         entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
1065
1066         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
1067         avic_update_iommu_vcpu_affinity(vcpu, h_physical_id, true);
1068
1069         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
1070 }
1071
1072 void avic_vcpu_put(struct kvm_vcpu *vcpu)
1073 {
1074         u64 entry;
1075         struct vcpu_svm *svm = to_svm(vcpu);
1076         unsigned long flags;
1077
1078         lockdep_assert_preemption_disabled();
1079
1080         /*
1081          * Note, reading the Physical ID entry outside of ir_list_lock is safe
1082          * as only the pCPU that has loaded (or is loading) the vCPU is allowed
1083          * to modify the entry, and preemption is disabled.  I.e. the vCPU
1084          * can't be scheduled out and thus avic_vcpu_{put,load}() can't run
1085          * recursively.
1086          */
1087         entry = READ_ONCE(*(svm->avic_physical_id_cache));
1088
1089         /* Nothing to do if IsRunning == '0' due to vCPU blocking. */
1090         if (!(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK))
1091                 return;
1092
1093         /*
1094          * Take and hold the per-vCPU interrupt remapping lock while updating
1095          * the Physical ID entry even though the lock doesn't protect against
1096          * multiple writers (see above).  Holding ir_list_lock ensures that
1097          * either svm_ir_list_add() will consume up-to-date entry information,
1098          * or that this task will wait until svm_ir_list_add() completes to
1099          * mark the vCPU as not running.
1100          */
1101         spin_lock_irqsave(&svm->ir_list_lock, flags);
1102
1103         avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
1104
1105         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
1106         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
1107
1108         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
1109
1110 }
1111
1112 void avic_refresh_virtual_apic_mode(struct kvm_vcpu *vcpu)
1113 {
1114         struct vcpu_svm *svm = to_svm(vcpu);
1115         struct vmcb *vmcb = svm->vmcb01.ptr;
1116
1117         if (!lapic_in_kernel(vcpu) || !enable_apicv)
1118                 return;
1119
1120         if (kvm_vcpu_apicv_active(vcpu)) {
1121                 /**
1122                  * During AVIC temporary deactivation, guest could update
1123                  * APIC ID, DFR and LDR registers, which would not be trapped
1124                  * by avic_unaccelerated_access_interception(). In this case,
1125                  * we need to check and update the AVIC logical APIC ID table
1126                  * accordingly before re-activating.
1127                  */
1128                 avic_apicv_post_state_restore(vcpu);
1129                 avic_activate_vmcb(svm);
1130         } else {
1131                 avic_deactivate_vmcb(svm);
1132         }
1133         vmcb_mark_dirty(vmcb, VMCB_AVIC);
1134 }
1135
1136 void avic_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
1137 {
1138         bool activated = kvm_vcpu_apicv_active(vcpu);
1139
1140         if (!enable_apicv)
1141                 return;
1142
1143         avic_refresh_virtual_apic_mode(vcpu);
1144
1145         if (activated)
1146                 avic_vcpu_load(vcpu, vcpu->cpu);
1147         else
1148                 avic_vcpu_put(vcpu);
1149
1150         avic_set_pi_irte_mode(vcpu, activated);
1151 }
1152
1153 void avic_vcpu_blocking(struct kvm_vcpu *vcpu)
1154 {
1155         if (!kvm_vcpu_apicv_active(vcpu))
1156                 return;
1157
1158        /*
1159         * Unload the AVIC when the vCPU is about to block, _before_
1160         * the vCPU actually blocks.
1161         *
1162         * Any IRQs that arrive before IsRunning=0 will not cause an
1163         * incomplete IPI vmexit on the source, therefore vIRR will also
1164         * be checked by kvm_vcpu_check_block() before blocking.  The
1165         * memory barrier implicit in set_current_state orders writing
1166         * IsRunning=0 before reading the vIRR.  The processor needs a
1167         * matching memory barrier on interrupt delivery between writing
1168         * IRR and reading IsRunning; the lack of this barrier might be
1169         * the cause of errata #1235).
1170         */
1171         avic_vcpu_put(vcpu);
1172 }
1173
1174 void avic_vcpu_unblocking(struct kvm_vcpu *vcpu)
1175 {
1176         if (!kvm_vcpu_apicv_active(vcpu))
1177                 return;
1178
1179         avic_vcpu_load(vcpu, vcpu->cpu);
1180 }
1181
1182 /*
1183  * Note:
1184  * - The module param avic enable both xAPIC and x2APIC mode.
1185  * - Hypervisor can support both xAVIC and x2AVIC in the same guest.
1186  * - The mode can be switched at run-time.
1187  */
1188 bool avic_hardware_setup(void)
1189 {
1190         if (!npt_enabled)
1191                 return false;
1192
1193         /* AVIC is a prerequisite for x2AVIC. */
1194         if (!boot_cpu_has(X86_FEATURE_AVIC) && !force_avic) {
1195                 if (boot_cpu_has(X86_FEATURE_X2AVIC)) {
1196                         pr_warn(FW_BUG "Cannot support x2AVIC due to AVIC is disabled");
1197                         pr_warn(FW_BUG "Try enable AVIC using force_avic option");
1198                 }
1199                 return false;
1200         }
1201
1202         if (boot_cpu_has(X86_FEATURE_AVIC)) {
1203                 pr_info("AVIC enabled\n");
1204         } else if (force_avic) {
1205                 /*
1206                  * Some older systems does not advertise AVIC support.
1207                  * See Revision Guide for specific AMD processor for more detail.
1208                  */
1209                 pr_warn("AVIC is not supported in CPUID but force enabled");
1210                 pr_warn("Your system might crash and burn");
1211         }
1212
1213         /* AVIC is a prerequisite for x2AVIC. */
1214         x2avic_enabled = boot_cpu_has(X86_FEATURE_X2AVIC);
1215         if (x2avic_enabled)
1216                 pr_info("x2AVIC enabled\n");
1217
1218         amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1219
1220         return true;
1221 }