arch/x86/kvm/lapic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /*
   4  * Local APIC virtualization
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright (C) 2007 Novell
   8  * Copyright (C) 2007 Intel
   9  * Copyright 2009 Red Hat, Inc. and/or its affiliates.
  10  *
  11  * Authors:
  12  *   Dor Laor <dor.laor@qumranet.com>
  13  *   Gregory Haskins <ghaskins@novell.com>
  14  *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
  15  *
  16  * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
  17  */
  18
  19 #include <linux/kvm_host.h>
  20 #include <linux/kvm.h>
  21 #include <linux/mm.h>
  22 #include <linux/highmem.h>
  23 #include <linux/smp.h>
  24 #include <linux/hrtimer.h>
  25 #include <linux/io.h>
  26 #include <linux/export.h>
  27 #include <linux/math64.h>
  28 #include <linux/slab.h>
  29 #include <asm/processor.h>
  30 #include <asm/msr.h>
  31 #include <asm/page.h>
  32 #include <asm/current.h>
  33 #include <asm/apicdef.h>
  34 #include <asm/delay.h>
  35 #include <linux/atomic.h>
  36 #include <linux/jump_label.h>
  37 #include "kvm_cache_regs.h"
  38 #include "irq.h"
  39 #include "trace.h"
  40 #include "x86.h"
  41 #include "cpuid.h"
  42 #include "hyperv.h"
  43
  44 #ifndef CONFIG_X86_64
  45 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
  46 #else
  47 #define mod_64(x, y) ((x) % (y))
  48 #endif
  49
  50 #define PRId64 "d"
  51 #define PRIx64 "llx"
  52 #define PRIu64 "u"
  53 #define PRIo64 "o"
  54
  55 /* 14 is the version for Xeon and Pentium 8.4.8*/
  56 #define APIC_VERSION                    (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
  57 #define LAPIC_MMIO_LENGTH               (1 << 12)
  58 /* followed define is not in apicdef.h */
  59 #define MAX_APIC_VECTOR                 256
  60 #define APIC_VECTORS_PER_REG            32
  61
  62 #define APIC_BROADCAST                  0xFF
  63 #define X2APIC_BROADCAST                0xFFFFFFFFul
  64
  65 static bool lapic_timer_advance_dynamic __read_mostly;
  66 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN  100     /* clock cycles */
  67 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX  10000   /* clock cycles */
  68 #define LAPIC_TIMER_ADVANCE_NS_INIT     1000
  69 #define LAPIC_TIMER_ADVANCE_NS_MAX     5000
  70 /* step-by-step approximation to mitigate fluctuation */
  71 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
  72
  73 static inline int apic_test_vector(int vec, void *bitmap)
  74 {
  75         return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  76 }
  77
  78 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
  79 {
  80         struct kvm_lapic *apic = vcpu->arch.apic;
  81
  82         return apic_test_vector(vector, apic->regs + APIC_ISR) ||
  83                 apic_test_vector(vector, apic->regs + APIC_IRR);
  84 }
  85
  86 static inline int __apic_test_and_set_vector(int vec, void *bitmap)
  87 {
  88         return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  89 }
  90
  91 static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
  92 {
  93         return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  94 }
  95
  96 struct static_key_deferred apic_hw_disabled __read_mostly;
  97 struct static_key_deferred apic_sw_disabled __read_mostly;
  98
  99 static inline int apic_enabled(struct kvm_lapic *apic)
 100 {
 101         return kvm_apic_sw_enabled(apic) &&     kvm_apic_hw_enabled(apic);
 102 }
 103
 104 #define LVT_MASK        \
 105         (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
 106
 107 #define LINT_MASK       \
 108         (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
 109          APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
 110
 111 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
 112 {
 113         return apic->vcpu->vcpu_id;
 114 }
 115
 116 bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
 117 {
 118         return pi_inject_timer && kvm_vcpu_apicv_active(vcpu);
 119 }
 120 EXPORT_SYMBOL_GPL(kvm_can_post_timer_interrupt);
 121
 122 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
 123 {
 124         return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
 125 }
 126
 127 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
 128                 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
 129         switch (map->mode) {
 130         case KVM_APIC_MODE_X2APIC: {
 131                 u32 offset = (dest_id >> 16) * 16;
 132                 u32 max_apic_id = map->max_apic_id;
 133
 134                 if (offset <= max_apic_id) {
 135                         u8 cluster_size = min(max_apic_id - offset + 1, 16U);
 136
 137                         offset = array_index_nospec(offset, map->max_apic_id + 1);
 138                         *cluster = &map->phys_map[offset];
 139                         *mask = dest_id & (0xffff >> (16 - cluster_size));
 140                 } else {
 141                         *mask = 0;
 142                 }
 143
 144                 return true;
 145                 }
 146         case KVM_APIC_MODE_XAPIC_FLAT:
 147                 *cluster = map->xapic_flat_map;
 148                 *mask = dest_id & 0xff;
 149                 return true;
 150         case KVM_APIC_MODE_XAPIC_CLUSTER:
 151                 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
 152                 *mask = dest_id & 0xf;
 153                 return true;
 154         default:
 155                 /* Not optimized. */
 156                 return false;
 157         }
 158 }
 159
 160 static void kvm_apic_map_free(struct rcu_head *rcu)
 161 {
 162         struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
 163
 164         kvfree(map);
 165 }
 166
 167 static void recalculate_apic_map(struct kvm *kvm)
 168 {
 169         struct kvm_apic_map *new, *old = NULL;
 170         struct kvm_vcpu *vcpu;
 171         int i;
 172         u32 max_id = 255; /* enough space for any xAPIC ID */
 173
 174         mutex_lock(&kvm->arch.apic_map_lock);
 175
 176         kvm_for_each_vcpu(i, vcpu, kvm)
 177                 if (kvm_apic_present(vcpu))
 178                         max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
 179
 180         new = kvzalloc(sizeof(struct kvm_apic_map) +
 181                            sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
 182                            GFP_KERNEL_ACCOUNT);
 183
 184         if (!new)
 185                 goto out;
 186
 187         new->max_apic_id = max_id;
 188
 189         kvm_for_each_vcpu(i, vcpu, kvm) {
 190                 struct kvm_lapic *apic = vcpu->arch.apic;
 191                 struct kvm_lapic **cluster;
 192                 u16 mask;
 193                 u32 ldr;
 194                 u8 xapic_id;
 195                 u32 x2apic_id;
 196
 197                 if (!kvm_apic_present(vcpu))
 198                         continue;
 199
 200                 xapic_id = kvm_xapic_id(apic);
 201                 x2apic_id = kvm_x2apic_id(apic);
 202
 203                 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
 204                 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
 205                                 x2apic_id <= new->max_apic_id)
 206                         new->phys_map[x2apic_id] = apic;
 207                 /*
 208                  * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
 209                  * prevent them from masking VCPUs with APIC ID <= 0xff.
 210                  */
 211                 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
 212                         new->phys_map[xapic_id] = apic;
 213
 214                 if (!kvm_apic_sw_enabled(apic))
 215                         continue;
 216
 217                 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
 218
 219                 if (apic_x2apic_mode(apic)) {
 220                         new->mode |= KVM_APIC_MODE_X2APIC;
 221                 } else if (ldr) {
 222                         ldr = GET_APIC_LOGICAL_ID(ldr);
 223                         if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
 224                                 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
 225                         else
 226                                 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
 227                 }
 228
 229                 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
 230                         continue;
 231
 232                 if (mask)
 233                         cluster[ffs(mask) - 1] = apic;
 234         }
 235 out:
 236         old = rcu_dereference_protected(kvm->arch.apic_map,
 237                         lockdep_is_held(&kvm->arch.apic_map_lock));
 238         rcu_assign_pointer(kvm->arch.apic_map, new);
 239         mutex_unlock(&kvm->arch.apic_map_lock);
 240
 241         if (old)
 242                 call_rcu(&old->rcu, kvm_apic_map_free);
 243
 244         kvm_make_scan_ioapic_request(kvm);
 245 }
 246
 247 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
 248 {
 249         bool enabled = val & APIC_SPIV_APIC_ENABLED;
 250
 251         kvm_lapic_set_reg(apic, APIC_SPIV, val);
 252
 253         if (enabled != apic->sw_enabled) {
 254                 apic->sw_enabled = enabled;
 255                 if (enabled)
 256                         static_key_slow_dec_deferred(&apic_sw_disabled);
 257                 else
 258                         static_key_slow_inc(&apic_sw_disabled.key);
 259
 260                 recalculate_apic_map(apic->vcpu->kvm);
 261         }
 262 }
 263
 264 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
 265 {
 266         kvm_lapic_set_reg(apic, APIC_ID, id << 24);
 267         recalculate_apic_map(apic->vcpu->kvm);
 268 }
 269
 270 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
 271 {
 272         kvm_lapic_set_reg(apic, APIC_LDR, id);
 273         recalculate_apic_map(apic->vcpu->kvm);
 274 }
 275
 276 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
 277 {
 278         return ((id >> 4) << 16) | (1 << (id & 0xf));
 279 }
 280
 281 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
 282 {
 283         u32 ldr = kvm_apic_calc_x2apic_ldr(id);
 284
 285         WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
 286
 287         kvm_lapic_set_reg(apic, APIC_ID, id);
 288         kvm_lapic_set_reg(apic, APIC_LDR, ldr);
 289         recalculate_apic_map(apic->vcpu->kvm);
 290 }
 291
 292 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
 293 {
 294         return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
 295 }
 296
 297 static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
 298 {
 299         return kvm_lapic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
 300 }
 301
 302 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
 303 {
 304         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
 305 }
 306
 307 static inline int apic_lvtt_period(struct kvm_lapic *apic)
 308 {
 309         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
 310 }
 311
 312 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
 313 {
 314         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
 315 }
 316
 317 static inline int apic_lvt_nmi_mode(u32 lvt_val)
 318 {
 319         return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
 320 }
 321
 322 void kvm_apic_set_version(struct kvm_vcpu *vcpu)
 323 {
 324         struct kvm_lapic *apic = vcpu->arch.apic;
 325         struct kvm_cpuid_entry2 *feat;
 326         u32 v = APIC_VERSION;
 327
 328         if (!lapic_in_kernel(vcpu))
 329                 return;
 330
 331         /*
 332          * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
 333          * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
 334          * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
 335          * version first and level-triggered interrupts never get EOIed in
 336          * IOAPIC.
 337          */
 338         feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
 339         if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) &&
 340             !ioapic_in_kernel(vcpu->kvm))
 341                 v |= APIC_LVR_DIRECTED_EOI;
 342         kvm_lapic_set_reg(apic, APIC_LVR, v);
 343 }
 344
 345 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
 346         LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
 347         LVT_MASK | APIC_MODE_MASK,      /* LVTTHMR */
 348         LVT_MASK | APIC_MODE_MASK,      /* LVTPC */
 349         LINT_MASK, LINT_MASK,   /* LVT0-1 */
 350         LVT_MASK                /* LVTERR */
 351 };
 352
 353 static int find_highest_vector(void *bitmap)
 354 {
 355         int vec;
 356         u32 *reg;
 357
 358         for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
 359              vec >= 0; vec -= APIC_VECTORS_PER_REG) {
 360                 reg = bitmap + REG_POS(vec);
 361                 if (*reg)
 362                         return __fls(*reg) + vec;
 363         }
 364
 365         return -1;
 366 }
 367
 368 static u8 count_vectors(void *bitmap)
 369 {
 370         int vec;
 371         u32 *reg;
 372         u8 count = 0;
 373
 374         for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
 375                 reg = bitmap + REG_POS(vec);
 376                 count += hweight32(*reg);
 377         }
 378
 379         return count;
 380 }
 381
 382 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
 383 {
 384         u32 i, vec;
 385         u32 pir_val, irr_val, prev_irr_val;
 386         int max_updated_irr;
 387
 388         max_updated_irr = -1;
 389         *max_irr = -1;
 390
 391         for (i = vec = 0; i <= 7; i++, vec += 32) {
 392                 pir_val = READ_ONCE(pir[i]);
 393                 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
 394                 if (pir_val) {
 395                         prev_irr_val = irr_val;
 396                         irr_val |= xchg(&pir[i], 0);
 397                         *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
 398                         if (prev_irr_val != irr_val) {
 399                                 max_updated_irr =
 400                                         __fls(irr_val ^ prev_irr_val) + vec;
 401                         }
 402                 }
 403                 if (irr_val)
 404                         *max_irr = __fls(irr_val) + vec;
 405         }
 406
 407         return ((max_updated_irr != -1) &&
 408                 (max_updated_irr == *max_irr));
 409 }
 410 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
 411
 412 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
 413 {
 414         struct kvm_lapic *apic = vcpu->arch.apic;
 415
 416         return __kvm_apic_update_irr(pir, apic->regs, max_irr);
 417 }
 418 EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
 419
 420 static inline int apic_search_irr(struct kvm_lapic *apic)
 421 {
 422         return find_highest_vector(apic->regs + APIC_IRR);
 423 }
 424
 425 static inline int apic_find_highest_irr(struct kvm_lapic *apic)
 426 {
 427         int result;
 428
 429         /*
 430          * Note that irr_pending is just a hint. It will be always
 431          * true with virtual interrupt delivery enabled.
 432          */
 433         if (!apic->irr_pending)
 434                 return -1;
 435
 436         result = apic_search_irr(apic);
 437         ASSERT(result == -1 || result >= 16);
 438
 439         return result;
 440 }
 441
 442 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
 443 {
 444         struct kvm_vcpu *vcpu;
 445
 446         vcpu = apic->vcpu;
 447
 448         if (unlikely(vcpu->arch.apicv_active)) {
 449                 /* need to update RVI */
 450                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 451                 kvm_x86_ops->hwapic_irr_update(vcpu,
 452                                 apic_find_highest_irr(apic));
 453         } else {
 454                 apic->irr_pending = false;
 455                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 456                 if (apic_search_irr(apic) != -1)
 457                         apic->irr_pending = true;
 458         }
 459 }
 460
 461 static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
 462 {
 463         struct kvm_vcpu *vcpu;
 464
 465         if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
 466                 return;
 467
 468         vcpu = apic->vcpu;
 469
 470         /*
 471          * With APIC virtualization enabled, all caching is disabled
 472          * because the processor can modify ISR under the hood.  Instead
 473          * just set SVI.
 474          */
 475         if (unlikely(vcpu->arch.apicv_active))
 476                 kvm_x86_ops->hwapic_isr_update(vcpu, vec);
 477         else {
 478                 ++apic->isr_count;
 479                 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
 480                 /*
 481                  * ISR (in service register) bit is set when injecting an interrupt.
 482                  * The highest vector is injected. Thus the latest bit set matches
 483                  * the highest bit in ISR.
 484                  */
 485                 apic->highest_isr_cache = vec;
 486         }
 487 }
 488
 489 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
 490 {
 491         int result;
 492
 493         /*
 494          * Note that isr_count is always 1, and highest_isr_cache
 495          * is always -1, with APIC virtualization enabled.
 496          */
 497         if (!apic->isr_count)
 498                 return -1;
 499         if (likely(apic->highest_isr_cache != -1))
 500                 return apic->highest_isr_cache;
 501
 502         result = find_highest_vector(apic->regs + APIC_ISR);
 503         ASSERT(result == -1 || result >= 16);
 504
 505         return result;
 506 }
 507
 508 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
 509 {
 510         struct kvm_vcpu *vcpu;
 511         if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
 512                 return;
 513
 514         vcpu = apic->vcpu;
 515
 516         /*
 517          * We do get here for APIC virtualization enabled if the guest
 518          * uses the Hyper-V APIC enlightenment.  In this case we may need
 519          * to trigger a new interrupt delivery by writing the SVI field;
 520          * on the other hand isr_count and highest_isr_cache are unused
 521          * and must be left alone.
 522          */
 523         if (unlikely(vcpu->arch.apicv_active))
 524                 kvm_x86_ops->hwapic_isr_update(vcpu,
 525                                                apic_find_highest_isr(apic));
 526         else {
 527                 --apic->isr_count;
 528                 BUG_ON(apic->isr_count < 0);
 529                 apic->highest_isr_cache = -1;
 530         }
 531 }
 532
 533 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 534 {
 535         /* This may race with setting of irr in __apic_accept_irq() and
 536          * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
 537          * will cause vmexit immediately and the value will be recalculated
 538          * on the next vmentry.
 539          */
 540         return apic_find_highest_irr(vcpu->arch.apic);
 541 }
 542 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
 543
 544 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
 545                              int vector, int level, int trig_mode,
 546                              struct dest_map *dest_map);
 547
 548 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
 549                      struct dest_map *dest_map)
 550 {
 551         struct kvm_lapic *apic = vcpu->arch.apic;
 552
 553         return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
 554                         irq->level, irq->trig_mode, dest_map);
 555 }
 556
 557 static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map,
 558                          struct kvm_lapic_irq *irq, u32 min)
 559 {
 560         int i, count = 0;
 561         struct kvm_vcpu *vcpu;
 562
 563         if (min > map->max_apic_id)
 564                 return 0;
 565
 566         for_each_set_bit(i, ipi_bitmap,
 567                 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
 568                 if (map->phys_map[min + i]) {
 569                         vcpu = map->phys_map[min + i]->vcpu;
 570                         count += kvm_apic_set_irq(vcpu, irq, NULL);
 571                 }
 572         }
 573
 574         return count;
 575 }
 576
 577 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
 578                     unsigned long ipi_bitmap_high, u32 min,
 579                     unsigned long icr, int op_64_bit)
 580 {
 581         struct kvm_apic_map *map;
 582         struct kvm_lapic_irq irq = {0};
 583         int cluster_size = op_64_bit ? 64 : 32;
 584         int count;
 585
 586         if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK))
 587                 return -KVM_EINVAL;
 588
 589         irq.vector = icr & APIC_VECTOR_MASK;
 590         irq.delivery_mode = icr & APIC_MODE_MASK;
 591         irq.level = (icr & APIC_INT_ASSERT) != 0;
 592         irq.trig_mode = icr & APIC_INT_LEVELTRIG;
 593
 594         rcu_read_lock();
 595         map = rcu_dereference(kvm->arch.apic_map);
 596
 597         count = -EOPNOTSUPP;
 598         if (likely(map)) {
 599                 count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min);
 600                 min += cluster_size;
 601                 count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min);
 602         }
 603
 604         rcu_read_unlock();
 605         return count;
 606 }
 607
 608 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
 609 {
 610
 611         return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
 612                                       sizeof(val));
 613 }
 614
 615 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
 616 {
 617
 618         return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
 619                                       sizeof(*val));
 620 }
 621
 622 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
 623 {
 624         return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
 625 }
 626
 627 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
 628 {
 629         u8 val;
 630         if (pv_eoi_get_user(vcpu, &val) < 0) {
 631                 printk(KERN_WARNING "Can't read EOI MSR value: 0x%llx\n",
 632                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 633                 return false;
 634         }
 635         return val & 0x1;
 636 }
 637
 638 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
 639 {
 640         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
 641                 printk(KERN_WARNING "Can't set EOI MSR value: 0x%llx\n",
 642                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 643                 return;
 644         }
 645         __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 646 }
 647
 648 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
 649 {
 650         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
 651                 printk(KERN_WARNING "Can't clear EOI MSR value: 0x%llx\n",
 652                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 653                 return;
 654         }
 655         __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 656 }
 657
 658 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
 659 {
 660         int highest_irr;
 661         if (apic->vcpu->arch.apicv_active)
 662                 highest_irr = kvm_x86_ops->sync_pir_to_irr(apic->vcpu);
 663         else
 664                 highest_irr = apic_find_highest_irr(apic);
 665         if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
 666                 return -1;
 667         return highest_irr;
 668 }
 669
 670 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
 671 {
 672         u32 tpr, isrv, ppr, old_ppr;
 673         int isr;
 674
 675         old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
 676         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
 677         isr = apic_find_highest_isr(apic);
 678         isrv = (isr != -1) ? isr : 0;
 679
 680         if ((tpr & 0xf0) >= (isrv & 0xf0))
 681                 ppr = tpr & 0xff;
 682         else
 683                 ppr = isrv & 0xf0;
 684
 685         *new_ppr = ppr;
 686         if (old_ppr != ppr)
 687                 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
 688
 689         return ppr < old_ppr;
 690 }
 691
 692 static void apic_update_ppr(struct kvm_lapic *apic)
 693 {
 694         u32 ppr;
 695
 696         if (__apic_update_ppr(apic, &ppr) &&
 697             apic_has_interrupt_for_ppr(apic, ppr) != -1)
 698                 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
 699 }
 700
 701 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
 702 {
 703         apic_update_ppr(vcpu->arch.apic);
 704 }
 705 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
 706
 707 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
 708 {
 709         kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
 710         apic_update_ppr(apic);
 711 }
 712
 713 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
 714 {
 715         return mda == (apic_x2apic_mode(apic) ?
 716                         X2APIC_BROADCAST : APIC_BROADCAST);
 717 }
 718
 719 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
 720 {
 721         if (kvm_apic_broadcast(apic, mda))
 722                 return true;
 723
 724         if (apic_x2apic_mode(apic))
 725                 return mda == kvm_x2apic_id(apic);
 726
 727         /*
 728          * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
 729          * it were in x2APIC mode.  Hotplugged VCPUs start in xAPIC mode and
 730          * this allows unique addressing of VCPUs with APIC ID over 0xff.
 731          * The 0xff condition is needed because writeable xAPIC ID.
 732          */
 733         if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
 734                 return true;
 735
 736         return mda == kvm_xapic_id(apic);
 737 }
 738
 739 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
 740 {
 741         u32 logical_id;
 742
 743         if (kvm_apic_broadcast(apic, mda))
 744                 return true;
 745
 746         logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
 747
 748         if (apic_x2apic_mode(apic))
 749                 return ((logical_id >> 16) == (mda >> 16))
 750                        && (logical_id & mda & 0xffff) != 0;
 751
 752         logical_id = GET_APIC_LOGICAL_ID(logical_id);
 753
 754         switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
 755         case APIC_DFR_FLAT:
 756                 return (logical_id & mda) != 0;
 757         case APIC_DFR_CLUSTER:
 758                 return ((logical_id >> 4) == (mda >> 4))
 759                        && (logical_id & mda & 0xf) != 0;
 760         default:
 761                 return false;
 762         }
 763 }
 764
 765 /* The KVM local APIC implementation has two quirks:
 766  *
 767  *  - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
 768  *    in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
 769  *    KVM doesn't do that aliasing.
 770  *
 771  *  - in-kernel IOAPIC messages have to be delivered directly to
 772  *    x2APIC, because the kernel does not support interrupt remapping.
 773  *    In order to support broadcast without interrupt remapping, x2APIC
 774  *    rewrites the destination of non-IPI messages from APIC_BROADCAST
 775  *    to X2APIC_BROADCAST.
 776  *
 777  * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
 778  * important when userspace wants to use x2APIC-format MSIs, because
 779  * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
 780  */
 781 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
 782                 struct kvm_lapic *source, struct kvm_lapic *target)
 783 {
 784         bool ipi = source != NULL;
 785
 786         if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
 787             !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
 788                 return X2APIC_BROADCAST;
 789
 790         return dest_id;
 791 }
 792
 793 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 794                            int shorthand, unsigned int dest, int dest_mode)
 795 {
 796         struct kvm_lapic *target = vcpu->arch.apic;
 797         u32 mda = kvm_apic_mda(vcpu, dest, source, target);
 798
 799         ASSERT(target);
 800         switch (shorthand) {
 801         case APIC_DEST_NOSHORT:
 802                 if (dest_mode == APIC_DEST_PHYSICAL)
 803                         return kvm_apic_match_physical_addr(target, mda);
 804                 else
 805                         return kvm_apic_match_logical_addr(target, mda);
 806         case APIC_DEST_SELF:
 807                 return target == source;
 808         case APIC_DEST_ALLINC:
 809                 return true;
 810         case APIC_DEST_ALLBUT:
 811                 return target != source;
 812         default:
 813                 return false;
 814         }
 815 }
 816 EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
 817
 818 int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
 819                        const unsigned long *bitmap, u32 bitmap_size)
 820 {
 821         u32 mod;
 822         int i, idx = -1;
 823
 824         mod = vector % dest_vcpus;
 825
 826         for (i = 0; i <= mod; i++) {
 827                 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
 828                 BUG_ON(idx == bitmap_size);
 829         }
 830
 831         return idx;
 832 }
 833
 834 static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
 835 {
 836         if (!kvm->arch.disabled_lapic_found) {
 837                 kvm->arch.disabled_lapic_found = true;
 838                 printk(KERN_INFO
 839                        "Disabled LAPIC found during irq injection\n");
 840         }
 841 }
 842
 843 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
 844                 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
 845 {
 846         if (kvm->arch.x2apic_broadcast_quirk_disabled) {
 847                 if ((irq->dest_id == APIC_BROADCAST &&
 848                                 map->mode != KVM_APIC_MODE_X2APIC))
 849                         return true;
 850                 if (irq->dest_id == X2APIC_BROADCAST)
 851                         return true;
 852         } else {
 853                 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
 854                 if (irq->dest_id == (x2apic_ipi ?
 855                                      X2APIC_BROADCAST : APIC_BROADCAST))
 856                         return true;
 857         }
 858
 859         return false;
 860 }
 861
 862 /* Return true if the interrupt can be handled by using *bitmap as index mask
 863  * for valid destinations in *dst array.
 864  * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
 865  * Note: we may have zero kvm_lapic destinations when we return true, which
 866  * means that the interrupt should be dropped.  In this case, *bitmap would be
 867  * zero and *dst undefined.
 868  */
 869 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
 870                 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
 871                 struct kvm_apic_map *map, struct kvm_lapic ***dst,
 872                 unsigned long *bitmap)
 873 {
 874         int i, lowest;
 875
 876         if (irq->shorthand == APIC_DEST_SELF && src) {
 877                 *dst = src;
 878                 *bitmap = 1;
 879                 return true;
 880         } else if (irq->shorthand)
 881                 return false;
 882
 883         if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
 884                 return false;
 885
 886         if (irq->dest_mode == APIC_DEST_PHYSICAL) {
 887                 if (irq->dest_id > map->max_apic_id) {
 888                         *bitmap = 0;
 889                 } else {
 890                         u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
 891                         *dst = &map->phys_map[dest_id];
 892                         *bitmap = 1;
 893                 }
 894                 return true;
 895         }
 896
 897         *bitmap = 0;
 898         if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
 899                                 (u16 *)bitmap))
 900                 return false;
 901
 902         if (!kvm_lowest_prio_delivery(irq))
 903                 return true;
 904
 905         if (!kvm_vector_hashing_enabled()) {
 906                 lowest = -1;
 907                 for_each_set_bit(i, bitmap, 16) {
 908                         if (!(*dst)[i])
 909                                 continue;
 910                         if (lowest < 0)
 911                                 lowest = i;
 912                         else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
 913                                                 (*dst)[lowest]->vcpu) < 0)
 914                                 lowest = i;
 915                 }
 916         } else {
 917                 if (!*bitmap)
 918                         return true;
 919
 920                 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
 921                                 bitmap, 16);
 922
 923                 if (!(*dst)[lowest]) {
 924                         kvm_apic_disabled_lapic_found(kvm);
 925                         *bitmap = 0;
 926                         return true;
 927                 }
 928         }
 929
 930         *bitmap = (lowest >= 0) ? 1 << lowest : 0;
 931
 932         return true;
 933 }
 934
 935 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
 936                 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
 937 {
 938         struct kvm_apic_map *map;
 939         unsigned long bitmap;
 940         struct kvm_lapic **dst = NULL;
 941         int i;
 942         bool ret;
 943
 944         *r = -1;
 945
 946         if (irq->shorthand == APIC_DEST_SELF) {
 947                 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
 948                 return true;
 949         }
 950
 951         rcu_read_lock();
 952         map = rcu_dereference(kvm->arch.apic_map);
 953
 954         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
 955         if (ret) {
 956                 *r = 0;
 957                 for_each_set_bit(i, &bitmap, 16) {
 958                         if (!dst[i])
 959                                 continue;
 960                         *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
 961                 }
 962         }
 963
 964         rcu_read_unlock();
 965         return ret;
 966 }
 967
 968 /*
 969  * This routine tries to handle interrupts in posted mode, here is how
 970  * it deals with different cases:
 971  * - For single-destination interrupts, handle it in posted mode
 972  * - Else if vector hashing is enabled and it is a lowest-priority
 973  *   interrupt, handle it in posted mode and use the following mechanism
 974  *   to find the destination vCPU.
 975  *      1. For lowest-priority interrupts, store all the possible
 976  *         destination vCPUs in an array.
 977  *      2. Use "guest vector % max number of destination vCPUs" to find
 978  *         the right destination vCPU in the array for the lowest-priority
 979  *         interrupt.
 980  * - Otherwise, use remapped mode to inject the interrupt.
 981  */
 982 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
 983                         struct kvm_vcpu **dest_vcpu)
 984 {
 985         struct kvm_apic_map *map;
 986         unsigned long bitmap;
 987         struct kvm_lapic **dst = NULL;
 988         bool ret = false;
 989
 990         if (irq->shorthand)
 991                 return false;
 992
 993         rcu_read_lock();
 994         map = rcu_dereference(kvm->arch.apic_map);
 995
 996         if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
 997                         hweight16(bitmap) == 1) {
 998                 unsigned long i = find_first_bit(&bitmap, 16);
 999
1000                 if (dst[i]) {
1001                         *dest_vcpu = dst[i]->vcpu;
1002                         ret = true;
1003                 }
1004         }
1005
1006         rcu_read_unlock();
1007         return ret;
1008 }
1009
1010 /*
1011  * Add a pending IRQ into lapic.
1012  * Return 1 if successfully added and 0 if discarded.
1013  */
1014 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
1015                              int vector, int level, int trig_mode,
1016                              struct dest_map *dest_map)
1017 {
1018         int result = 0;
1019         struct kvm_vcpu *vcpu = apic->vcpu;
1020
1021         trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
1022                                   trig_mode, vector);
1023         switch (delivery_mode) {
1024         case APIC_DM_LOWEST:
1025                 vcpu->arch.apic_arb_prio++;
1026                 /* fall through */
1027         case APIC_DM_FIXED:
1028                 if (unlikely(trig_mode && !level))
1029                         break;
1030
1031                 /* FIXME add logic for vcpu on reset */
1032                 if (unlikely(!apic_enabled(apic)))
1033                         break;
1034
1035                 result = 1;
1036
1037                 if (dest_map) {
1038                         __set_bit(vcpu->vcpu_id, dest_map->map);
1039                         dest_map->vectors[vcpu->vcpu_id] = vector;
1040                 }
1041
1042                 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
1043                         if (trig_mode)
1044                                 kvm_lapic_set_vector(vector,
1045                                                      apic->regs + APIC_TMR);
1046                         else
1047                                 kvm_lapic_clear_vector(vector,
1048                                                        apic->regs + APIC_TMR);
1049                 }
1050
1051                 if (kvm_x86_ops->deliver_posted_interrupt(vcpu, vector)) {
1052                         kvm_lapic_set_irr(vector, apic);
1053                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1054                         kvm_vcpu_kick(vcpu);
1055                 }
1056                 break;
1057
1058         case APIC_DM_REMRD:
1059                 result = 1;
1060                 vcpu->arch.pv.pv_unhalted = 1;
1061                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1062                 kvm_vcpu_kick(vcpu);
1063                 break;
1064
1065         case APIC_DM_SMI:
1066                 result = 1;
1067                 kvm_make_request(KVM_REQ_SMI, vcpu);
1068                 kvm_vcpu_kick(vcpu);
1069                 break;
1070
1071         case APIC_DM_NMI:
1072                 result = 1;
1073                 kvm_inject_nmi(vcpu);
1074                 kvm_vcpu_kick(vcpu);
1075                 break;
1076
1077         case APIC_DM_INIT:
1078                 if (!trig_mode || level) {
1079                         result = 1;
1080                         /* assumes that there are only KVM_APIC_INIT/SIPI */
1081                         apic->pending_events = (1UL << KVM_APIC_INIT);
1082                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1083                         kvm_vcpu_kick(vcpu);
1084                 }
1085                 break;
1086
1087         case APIC_DM_STARTUP:
1088                 result = 1;
1089                 apic->sipi_vector = vector;
1090                 /* make sure sipi_vector is visible for the receiver */
1091                 smp_wmb();
1092                 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1093                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1094                 kvm_vcpu_kick(vcpu);
1095                 break;
1096
1097         case APIC_DM_EXTINT:
1098                 /*
1099                  * Should only be called by kvm_apic_local_deliver() with LVT0,
1100                  * before NMI watchdog was enabled. Already handled by
1101                  * kvm_apic_accept_pic_intr().
1102                  */
1103                 break;
1104
1105         default:
1106                 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1107                        delivery_mode);
1108                 break;
1109         }
1110         return result;
1111 }
1112
1113 /*
1114  * This routine identifies the destination vcpus mask meant to receive the
1115  * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find
1116  * out the destination vcpus array and set the bitmap or it traverses to
1117  * each available vcpu to identify the same.
1118  */
1119 void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq,
1120                               unsigned long *vcpu_bitmap)
1121 {
1122         struct kvm_lapic **dest_vcpu = NULL;
1123         struct kvm_lapic *src = NULL;
1124         struct kvm_apic_map *map;
1125         struct kvm_vcpu *vcpu;
1126         unsigned long bitmap;
1127         int i, vcpu_idx;
1128         bool ret;
1129
1130         rcu_read_lock();
1131         map = rcu_dereference(kvm->arch.apic_map);
1132
1133         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu,
1134                                           &bitmap);
1135         if (ret) {
1136                 for_each_set_bit(i, &bitmap, 16) {
1137                         if (!dest_vcpu[i])
1138                                 continue;
1139                         vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx;
1140                         __set_bit(vcpu_idx, vcpu_bitmap);
1141                 }
1142         } else {
1143                 kvm_for_each_vcpu(i, vcpu, kvm) {
1144                         if (!kvm_apic_present(vcpu))
1145                                 continue;
1146                         if (!kvm_apic_match_dest(vcpu, NULL,
1147                                                  irq->shorthand,
1148                                                  irq->dest_id,
1149                                                  irq->dest_mode))
1150                                 continue;
1151                         __set_bit(i, vcpu_bitmap);
1152                 }
1153         }
1154         rcu_read_unlock();
1155 }
1156
1157 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1158 {
1159         return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1160 }
1161
1162 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1163 {
1164         return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1165 }
1166
1167 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1168 {
1169         int trigger_mode;
1170
1171         /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1172         if (!kvm_ioapic_handles_vector(apic, vector))
1173                 return;
1174
1175         /* Request a KVM exit to inform the userspace IOAPIC. */
1176         if (irqchip_split(apic->vcpu->kvm)) {
1177                 apic->vcpu->arch.pending_ioapic_eoi = vector;
1178                 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1179                 return;
1180         }
1181
1182         if (apic_test_vector(vector, apic->regs + APIC_TMR))
1183                 trigger_mode = IOAPIC_LEVEL_TRIG;
1184         else
1185                 trigger_mode = IOAPIC_EDGE_TRIG;
1186
1187         kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1188 }
1189
1190 static int apic_set_eoi(struct kvm_lapic *apic)
1191 {
1192         int vector = apic_find_highest_isr(apic);
1193
1194         trace_kvm_eoi(apic, vector);
1195
1196         /*
1197          * Not every write EOI will has corresponding ISR,
1198          * one example is when Kernel check timer on setup_IO_APIC
1199          */
1200         if (vector == -1)
1201                 return vector;
1202
1203         apic_clear_isr(vector, apic);
1204         apic_update_ppr(apic);
1205
1206         if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1207                 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1208
1209         kvm_ioapic_send_eoi(apic, vector);
1210         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1211         return vector;
1212 }
1213
1214 /*
1215  * this interface assumes a trap-like exit, which has already finished
1216  * desired side effect including vISR and vPPR update.
1217  */
1218 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1219 {
1220         struct kvm_lapic *apic = vcpu->arch.apic;
1221
1222         trace_kvm_eoi(apic, vector);
1223
1224         kvm_ioapic_send_eoi(apic, vector);
1225         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1226 }
1227 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1228
1229 static void apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
1230 {
1231         struct kvm_lapic_irq irq;
1232
1233         irq.vector = icr_low & APIC_VECTOR_MASK;
1234         irq.delivery_mode = icr_low & APIC_MODE_MASK;
1235         irq.dest_mode = icr_low & APIC_DEST_MASK;
1236         irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1237         irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1238         irq.shorthand = icr_low & APIC_SHORT_MASK;
1239         irq.msi_redir_hint = false;
1240         if (apic_x2apic_mode(apic))
1241                 irq.dest_id = icr_high;
1242         else
1243                 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
1244
1245         trace_kvm_apic_ipi(icr_low, irq.dest_id);
1246
1247         kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
1248 }
1249
1250 static u32 apic_get_tmcct(struct kvm_lapic *apic)
1251 {
1252         ktime_t remaining, now;
1253         s64 ns;
1254         u32 tmcct;
1255
1256         ASSERT(apic != NULL);
1257
1258         /* if initial count is 0, current count should also be 0 */
1259         if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1260                 apic->lapic_timer.period == 0)
1261                 return 0;
1262
1263         now = ktime_get();
1264         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1265         if (ktime_to_ns(remaining) < 0)
1266                 remaining = 0;
1267
1268         ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1269         tmcct = div64_u64(ns,
1270                          (APIC_BUS_CYCLE_NS * apic->divide_count));
1271
1272         return tmcct;
1273 }
1274
1275 static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1276 {
1277         struct kvm_vcpu *vcpu = apic->vcpu;
1278         struct kvm_run *run = vcpu->run;
1279
1280         kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1281         run->tpr_access.rip = kvm_rip_read(vcpu);
1282         run->tpr_access.is_write = write;
1283 }
1284
1285 static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1286 {
1287         if (apic->vcpu->arch.tpr_access_reporting)
1288                 __report_tpr_access(apic, write);
1289 }
1290
1291 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1292 {
1293         u32 val = 0;
1294
1295         if (offset >= LAPIC_MMIO_LENGTH)
1296                 return 0;
1297
1298         switch (offset) {
1299         case APIC_ARBPRI:
1300                 break;
1301
1302         case APIC_TMCCT:        /* Timer CCR */
1303                 if (apic_lvtt_tscdeadline(apic))
1304                         return 0;
1305
1306                 val = apic_get_tmcct(apic);
1307                 break;
1308         case APIC_PROCPRI:
1309                 apic_update_ppr(apic);
1310                 val = kvm_lapic_get_reg(apic, offset);
1311                 break;
1312         case APIC_TASKPRI:
1313                 report_tpr_access(apic, false);
1314                 /* fall thru */
1315         default:
1316                 val = kvm_lapic_get_reg(apic, offset);
1317                 break;
1318         }
1319
1320         return val;
1321 }
1322
1323 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1324 {
1325         return container_of(dev, struct kvm_lapic, dev);
1326 }
1327
1328 #define APIC_REG_MASK(reg)      (1ull << ((reg) >> 4))
1329 #define APIC_REGS_MASK(first, count) \
1330         (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1331
1332 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
1333                 void *data)
1334 {
1335         unsigned char alignment = offset & 0xf;
1336         u32 result;
1337         /* this bitmask has a bit cleared for each reserved register */
1338         u64 valid_reg_mask =
1339                 APIC_REG_MASK(APIC_ID) |
1340                 APIC_REG_MASK(APIC_LVR) |
1341                 APIC_REG_MASK(APIC_TASKPRI) |
1342                 APIC_REG_MASK(APIC_PROCPRI) |
1343                 APIC_REG_MASK(APIC_LDR) |
1344                 APIC_REG_MASK(APIC_DFR) |
1345                 APIC_REG_MASK(APIC_SPIV) |
1346                 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
1347                 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
1348                 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
1349                 APIC_REG_MASK(APIC_ESR) |
1350                 APIC_REG_MASK(APIC_ICR) |
1351                 APIC_REG_MASK(APIC_ICR2) |
1352                 APIC_REG_MASK(APIC_LVTT) |
1353                 APIC_REG_MASK(APIC_LVTTHMR) |
1354                 APIC_REG_MASK(APIC_LVTPC) |
1355                 APIC_REG_MASK(APIC_LVT0) |
1356                 APIC_REG_MASK(APIC_LVT1) |
1357                 APIC_REG_MASK(APIC_LVTERR) |
1358                 APIC_REG_MASK(APIC_TMICT) |
1359                 APIC_REG_MASK(APIC_TMCCT) |
1360                 APIC_REG_MASK(APIC_TDCR);
1361
1362         /* ARBPRI is not valid on x2APIC */
1363         if (!apic_x2apic_mode(apic))
1364                 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI);
1365
1366         if (offset > 0x3f0 || !(valid_reg_mask & APIC_REG_MASK(offset)))
1367                 return 1;
1368
1369         result = __apic_read(apic, offset & ~0xf);
1370
1371         trace_kvm_apic_read(offset, result);
1372
1373         switch (len) {
1374         case 1:
1375         case 2:
1376         case 4:
1377                 memcpy(data, (char *)&result + alignment, len);
1378                 break;
1379         default:
1380                 printk(KERN_ERR "Local APIC read with len = %x, "
1381                        "should be 1,2, or 4 instead\n", len);
1382                 break;
1383         }
1384         return 0;
1385 }
1386 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
1387
1388 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1389 {
1390         return addr >= apic->base_address &&
1391                 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1392 }
1393
1394 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1395                            gpa_t address, int len, void *data)
1396 {
1397         struct kvm_lapic *apic = to_lapic(this);
1398         u32 offset = address - apic->base_address;
1399
1400         if (!apic_mmio_in_range(apic, address))
1401                 return -EOPNOTSUPP;
1402
1403         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1404                 if (!kvm_check_has_quirk(vcpu->kvm,
1405                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1406                         return -EOPNOTSUPP;
1407
1408                 memset(data, 0xff, len);
1409                 return 0;
1410         }
1411
1412         kvm_lapic_reg_read(apic, offset, len, data);
1413
1414         return 0;
1415 }
1416
1417 static void update_divide_count(struct kvm_lapic *apic)
1418 {
1419         u32 tmp1, tmp2, tdcr;
1420
1421         tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
1422         tmp1 = tdcr & 0xf;
1423         tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1424         apic->divide_count = 0x1 << (tmp2 & 0x7);
1425 }
1426
1427 static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1428 {
1429         /*
1430          * Do not allow the guest to program periodic timers with small
1431          * interval, since the hrtimers are not throttled by the host
1432          * scheduler.
1433          */
1434         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1435                 s64 min_period = min_timer_period_us * 1000LL;
1436
1437                 if (apic->lapic_timer.period < min_period) {
1438                         pr_info_ratelimited(
1439                             "kvm: vcpu %i: requested %lld ns "
1440                             "lapic timer period limited to %lld ns\n",
1441                             apic->vcpu->vcpu_id,
1442                             apic->lapic_timer.period, min_period);
1443                         apic->lapic_timer.period = min_period;
1444                 }
1445         }
1446 }
1447
1448 static void apic_update_lvtt(struct kvm_lapic *apic)
1449 {
1450         u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1451                         apic->lapic_timer.timer_mode_mask;
1452
1453         if (apic->lapic_timer.timer_mode != timer_mode) {
1454                 if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1455                                 APIC_LVT_TIMER_TSCDEADLINE)) {
1456                         hrtimer_cancel(&apic->lapic_timer.timer);
1457                         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
1458                         apic->lapic_timer.period = 0;
1459                         apic->lapic_timer.tscdeadline = 0;
1460                 }
1461                 apic->lapic_timer.timer_mode = timer_mode;
1462                 limit_periodic_timer_frequency(apic);
1463         }
1464 }
1465
1466 /*
1467  * On APICv, this test will cause a busy wait
1468  * during a higher-priority task.
1469  */
1470
1471 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1472 {
1473         struct kvm_lapic *apic = vcpu->arch.apic;
1474         u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1475
1476         if (kvm_apic_hw_enabled(apic)) {
1477                 int vec = reg & APIC_VECTOR_MASK;
1478                 void *bitmap = apic->regs + APIC_ISR;
1479
1480                 if (vcpu->arch.apicv_active)
1481                         bitmap = apic->regs + APIC_IRR;
1482
1483                 if (apic_test_vector(vec, bitmap))
1484                         return true;
1485         }
1486         return false;
1487 }
1488
1489 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
1490 {
1491         u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
1492
1493         /*
1494          * If the guest TSC is running at a different ratio than the host, then
1495          * convert the delay to nanoseconds to achieve an accurate delay.  Note
1496          * that __delay() uses delay_tsc whenever the hardware has TSC, thus
1497          * always for VMX enabled hardware.
1498          */
1499         if (vcpu->arch.tsc_scaling_ratio == kvm_default_tsc_scaling_ratio) {
1500                 __delay(min(guest_cycles,
1501                         nsec_to_cycles(vcpu, timer_advance_ns)));
1502         } else {
1503                 u64 delay_ns = guest_cycles * 1000000ULL;
1504                 do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
1505                 ndelay(min_t(u32, delay_ns, timer_advance_ns));
1506         }
1507 }
1508
1509 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
1510                                               s64 advance_expire_delta)
1511 {
1512         struct kvm_lapic *apic = vcpu->arch.apic;
1513         u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
1514         u64 ns;
1515
1516         /* Do not adjust for tiny fluctuations or large random spikes. */
1517         if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX ||
1518             abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN)
1519                 return;
1520
1521         /* too early */
1522         if (advance_expire_delta < 0) {
1523                 ns = -advance_expire_delta * 1000000ULL;
1524                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1525                 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1526         } else {
1527         /* too late */
1528                 ns = advance_expire_delta * 1000000ULL;
1529                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1530                 timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1531         }
1532
1533         if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX))
1534                 timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
1535         apic->lapic_timer.timer_advance_ns = timer_advance_ns;
1536 }
1537
1538 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1539 {
1540         struct kvm_lapic *apic = vcpu->arch.apic;
1541         u64 guest_tsc, tsc_deadline;
1542
1543         if (apic->lapic_timer.expired_tscdeadline == 0)
1544                 return;
1545
1546         tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1547         apic->lapic_timer.expired_tscdeadline = 0;
1548         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1549         apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline;
1550
1551         if (guest_tsc < tsc_deadline)
1552                 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
1553
1554         if (lapic_timer_advance_dynamic)
1555                 adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
1556 }
1557
1558 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1559 {
1560         if (lapic_timer_int_injected(vcpu))
1561                 __kvm_wait_lapic_expire(vcpu);
1562 }
1563 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
1564
1565 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
1566 {
1567         struct kvm_timer *ktimer = &apic->lapic_timer;
1568
1569         kvm_apic_local_deliver(apic, APIC_LVTT);
1570         if (apic_lvtt_tscdeadline(apic)) {
1571                 ktimer->tscdeadline = 0;
1572         } else if (apic_lvtt_oneshot(apic)) {
1573                 ktimer->tscdeadline = 0;
1574                 ktimer->target_expiration = 0;
1575         }
1576 }
1577
1578 static void apic_timer_expired(struct kvm_lapic *apic)
1579 {
1580         struct kvm_vcpu *vcpu = apic->vcpu;
1581         struct kvm_timer *ktimer = &apic->lapic_timer;
1582
1583         if (atomic_read(&apic->lapic_timer.pending))
1584                 return;
1585
1586         if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
1587                 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1588
1589         if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
1590                 if (apic->lapic_timer.timer_advance_ns)
1591                         __kvm_wait_lapic_expire(vcpu);
1592                 kvm_apic_inject_pending_timer_irqs(apic);
1593                 return;
1594         }
1595
1596         atomic_inc(&apic->lapic_timer.pending);
1597         kvm_set_pending_timer(vcpu);
1598 }
1599
1600 static void start_sw_tscdeadline(struct kvm_lapic *apic)
1601 {
1602         struct kvm_timer *ktimer = &apic->lapic_timer;
1603         u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
1604         u64 ns = 0;
1605         ktime_t expire;
1606         struct kvm_vcpu *vcpu = apic->vcpu;
1607         unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1608         unsigned long flags;
1609         ktime_t now;
1610
1611         if (unlikely(!tscdeadline || !this_tsc_khz))
1612                 return;
1613
1614         local_irq_save(flags);
1615
1616         now = ktime_get();
1617         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1618
1619         ns = (tscdeadline - guest_tsc) * 1000000ULL;
1620         do_div(ns, this_tsc_khz);
1621
1622         if (likely(tscdeadline > guest_tsc) &&
1623             likely(ns > apic->lapic_timer.timer_advance_ns)) {
1624                 expire = ktime_add_ns(now, ns);
1625                 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
1626                 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
1627         } else
1628                 apic_timer_expired(apic);
1629
1630         local_irq_restore(flags);
1631 }
1632
1633 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1634 {
1635         ktime_t now, remaining;
1636         u64 ns_remaining_old, ns_remaining_new;
1637
1638         apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1639                 * APIC_BUS_CYCLE_NS * apic->divide_count;
1640         limit_periodic_timer_frequency(apic);
1641
1642         now = ktime_get();
1643         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1644         if (ktime_to_ns(remaining) < 0)
1645                 remaining = 0;
1646
1647         ns_remaining_old = ktime_to_ns(remaining);
1648         ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1649                                            apic->divide_count, old_divisor);
1650
1651         apic->lapic_timer.tscdeadline +=
1652                 nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1653                 nsec_to_cycles(apic->vcpu, ns_remaining_old);
1654         apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1655 }
1656
1657 static bool set_target_expiration(struct kvm_lapic *apic)
1658 {
1659         ktime_t now;
1660         u64 tscl = rdtsc();
1661
1662         now = ktime_get();
1663         apic->lapic_timer.period = (u64)kvm_lapic_get_reg(apic, APIC_TMICT)
1664                 * APIC_BUS_CYCLE_NS * apic->divide_count;
1665
1666         if (!apic->lapic_timer.period) {
1667                 apic->lapic_timer.tscdeadline = 0;
1668                 return false;
1669         }
1670
1671         limit_periodic_timer_frequency(apic);
1672
1673         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1674                 nsec_to_cycles(apic->vcpu, apic->lapic_timer.period);
1675         apic->lapic_timer.target_expiration = ktime_add_ns(now, apic->lapic_timer.period);
1676
1677         return true;
1678 }
1679
1680 static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1681 {
1682         ktime_t now = ktime_get();
1683         u64 tscl = rdtsc();
1684         ktime_t delta;
1685
1686         /*
1687          * Synchronize both deadlines to the same time source or
1688          * differences in the periods (caused by differences in the
1689          * underlying clocks or numerical approximation errors) will
1690          * cause the two to drift apart over time as the errors
1691          * accumulate.
1692          */
1693         apic->lapic_timer.target_expiration =
1694                 ktime_add_ns(apic->lapic_timer.target_expiration,
1695                                 apic->lapic_timer.period);
1696         delta = ktime_sub(apic->lapic_timer.target_expiration, now);
1697         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1698                 nsec_to_cycles(apic->vcpu, delta);
1699 }
1700
1701 static void start_sw_period(struct kvm_lapic *apic)
1702 {
1703         if (!apic->lapic_timer.period)
1704                 return;
1705
1706         if (ktime_after(ktime_get(),
1707                         apic->lapic_timer.target_expiration)) {
1708                 apic_timer_expired(apic);
1709
1710                 if (apic_lvtt_oneshot(apic))
1711                         return;
1712
1713                 advance_periodic_target_expiration(apic);
1714         }
1715
1716         hrtimer_start(&apic->lapic_timer.timer,
1717                 apic->lapic_timer.target_expiration,
1718                 HRTIMER_MODE_ABS);
1719 }
1720
1721 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1722 {
1723         if (!lapic_in_kernel(vcpu))
1724                 return false;
1725
1726         return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1727 }
1728 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1729
1730 static void cancel_hv_timer(struct kvm_lapic *apic)
1731 {
1732         WARN_ON(preemptible());
1733         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1734         kvm_x86_ops->cancel_hv_timer(apic->vcpu);
1735         apic->lapic_timer.hv_timer_in_use = false;
1736 }
1737
1738 static bool start_hv_timer(struct kvm_lapic *apic)
1739 {
1740         struct kvm_timer *ktimer = &apic->lapic_timer;
1741         struct kvm_vcpu *vcpu = apic->vcpu;
1742         bool expired;
1743
1744         WARN_ON(preemptible());
1745         if (!kvm_x86_ops->set_hv_timer)
1746                 return false;
1747
1748         if (!ktimer->tscdeadline)
1749                 return false;
1750
1751         if (kvm_x86_ops->set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
1752                 return false;
1753
1754         ktimer->hv_timer_in_use = true;
1755         hrtimer_cancel(&ktimer->timer);
1756
1757         /*
1758          * To simplify handling the periodic timer, leave the hv timer running
1759          * even if the deadline timer has expired, i.e. rely on the resulting
1760          * VM-Exit to recompute the periodic timer's target expiration.
1761          */
1762         if (!apic_lvtt_period(apic)) {
1763                 /*
1764                  * Cancel the hv timer if the sw timer fired while the hv timer
1765                  * was being programmed, or if the hv timer itself expired.
1766                  */
1767                 if (atomic_read(&ktimer->pending)) {
1768                         cancel_hv_timer(apic);
1769                 } else if (expired) {
1770                         apic_timer_expired(apic);
1771                         cancel_hv_timer(apic);
1772                 }
1773         }
1774
1775         trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
1776
1777         return true;
1778 }
1779
1780 static void start_sw_timer(struct kvm_lapic *apic)
1781 {
1782         struct kvm_timer *ktimer = &apic->lapic_timer;
1783
1784         WARN_ON(preemptible());
1785         if (apic->lapic_timer.hv_timer_in_use)
1786                 cancel_hv_timer(apic);
1787         if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1788                 return;
1789
1790         if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1791                 start_sw_period(apic);
1792         else if (apic_lvtt_tscdeadline(apic))
1793                 start_sw_tscdeadline(apic);
1794         trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
1795 }
1796
1797 static void restart_apic_timer(struct kvm_lapic *apic)
1798 {
1799         preempt_disable();
1800
1801         if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
1802                 goto out;
1803
1804         if (!start_hv_timer(apic))
1805                 start_sw_timer(apic);
1806 out:
1807         preempt_enable();
1808 }
1809
1810 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1811 {
1812         struct kvm_lapic *apic = vcpu->arch.apic;
1813
1814         preempt_disable();
1815         /* If the preempt notifier has already run, it also called apic_timer_expired */
1816         if (!apic->lapic_timer.hv_timer_in_use)
1817                 goto out;
1818         WARN_ON(swait_active(&vcpu->wq));
1819         cancel_hv_timer(apic);
1820         apic_timer_expired(apic);
1821
1822         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1823                 advance_periodic_target_expiration(apic);
1824                 restart_apic_timer(apic);
1825         }
1826 out:
1827         preempt_enable();
1828 }
1829 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1830
1831 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1832 {
1833         restart_apic_timer(vcpu->arch.apic);
1834 }
1835 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1836
1837 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1838 {
1839         struct kvm_lapic *apic = vcpu->arch.apic;
1840
1841         preempt_disable();
1842         /* Possibly the TSC deadline timer is not enabled yet */
1843         if (apic->lapic_timer.hv_timer_in_use)
1844                 start_sw_timer(apic);
1845         preempt_enable();
1846 }
1847 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1848
1849 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
1850 {
1851         struct kvm_lapic *apic = vcpu->arch.apic;
1852
1853         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1854         restart_apic_timer(apic);
1855 }
1856
1857 static void start_apic_timer(struct kvm_lapic *apic)
1858 {
1859         atomic_set(&apic->lapic_timer.pending, 0);
1860
1861         if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1862             && !set_target_expiration(apic))
1863                 return;
1864
1865         restart_apic_timer(apic);
1866 }
1867
1868 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1869 {
1870         bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
1871
1872         if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1873                 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1874                 if (lvt0_in_nmi_mode) {
1875                         atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1876                 } else
1877                         atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1878         }
1879 }
1880
1881 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
1882 {
1883         int ret = 0;
1884
1885         trace_kvm_apic_write(reg, val);
1886
1887         switch (reg) {
1888         case APIC_ID:           /* Local APIC ID */
1889                 if (!apic_x2apic_mode(apic))
1890                         kvm_apic_set_xapic_id(apic, val >> 24);
1891                 else
1892                         ret = 1;
1893                 break;
1894
1895         case APIC_TASKPRI:
1896                 report_tpr_access(apic, true);
1897                 apic_set_tpr(apic, val & 0xff);
1898                 break;
1899
1900         case APIC_EOI:
1901                 apic_set_eoi(apic);
1902                 break;
1903
1904         case APIC_LDR:
1905                 if (!apic_x2apic_mode(apic))
1906                         kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
1907                 else
1908                         ret = 1;
1909                 break;
1910
1911         case APIC_DFR:
1912                 if (!apic_x2apic_mode(apic)) {
1913                         kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1914                         recalculate_apic_map(apic->vcpu->kvm);
1915                 } else
1916                         ret = 1;
1917                 break;
1918
1919         case APIC_SPIV: {
1920                 u32 mask = 0x3ff;
1921                 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
1922                         mask |= APIC_SPIV_DIRECTED_EOI;
1923                 apic_set_spiv(apic, val & mask);
1924                 if (!(val & APIC_SPIV_APIC_ENABLED)) {
1925                         int i;
1926                         u32 lvt_val;
1927
1928                         for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
1929                                 lvt_val = kvm_lapic_get_reg(apic,
1930                                                        APIC_LVTT + 0x10 * i);
1931                                 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
1932                                              lvt_val | APIC_LVT_MASKED);
1933                         }
1934                         apic_update_lvtt(apic);
1935                         atomic_set(&apic->lapic_timer.pending, 0);
1936
1937                 }
1938                 break;
1939         }
1940         case APIC_ICR:
1941                 /* No delay here, so we always clear the pending bit */
1942                 val &= ~(1 << 12);
1943                 apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
1944                 kvm_lapic_set_reg(apic, APIC_ICR, val);
1945                 break;
1946
1947         case APIC_ICR2:
1948                 if (!apic_x2apic_mode(apic))
1949                         val &= 0xff000000;
1950                 kvm_lapic_set_reg(apic, APIC_ICR2, val);
1951                 break;
1952
1953         case APIC_LVT0:
1954                 apic_manage_nmi_watchdog(apic, val);
1955                 /* fall through */
1956         case APIC_LVTTHMR:
1957         case APIC_LVTPC:
1958         case APIC_LVT1:
1959         case APIC_LVTERR: {
1960                 /* TODO: Check vector */
1961                 size_t size;
1962                 u32 index;
1963
1964                 if (!kvm_apic_sw_enabled(apic))
1965                         val |= APIC_LVT_MASKED;
1966                 size = ARRAY_SIZE(apic_lvt_mask);
1967                 index = array_index_nospec(
1968                                 (reg - APIC_LVTT) >> 4, size);
1969                 val &= apic_lvt_mask[index];
1970                 kvm_lapic_set_reg(apic, reg, val);
1971                 break;
1972         }
1973
1974         case APIC_LVTT:
1975                 if (!kvm_apic_sw_enabled(apic))
1976                         val |= APIC_LVT_MASKED;
1977                 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
1978                 kvm_lapic_set_reg(apic, APIC_LVTT, val);
1979                 apic_update_lvtt(apic);
1980                 break;
1981
1982         case APIC_TMICT:
1983                 if (apic_lvtt_tscdeadline(apic))
1984                         break;
1985
1986                 hrtimer_cancel(&apic->lapic_timer.timer);
1987                 kvm_lapic_set_reg(apic, APIC_TMICT, val);
1988                 start_apic_timer(apic);
1989                 break;
1990
1991         case APIC_TDCR: {
1992                 uint32_t old_divisor = apic->divide_count;
1993
1994                 kvm_lapic_set_reg(apic, APIC_TDCR, val);
1995                 update_divide_count(apic);
1996                 if (apic->divide_count != old_divisor &&
1997                                 apic->lapic_timer.period) {
1998                         hrtimer_cancel(&apic->lapic_timer.timer);
1999                         update_target_expiration(apic, old_divisor);
2000                         restart_apic_timer(apic);
2001                 }
2002                 break;
2003         }
2004         case APIC_ESR:
2005                 if (apic_x2apic_mode(apic) && val != 0)
2006                         ret = 1;
2007                 break;
2008
2009         case APIC_SELF_IPI:
2010                 if (apic_x2apic_mode(apic)) {
2011                         kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
2012                 } else
2013                         ret = 1;
2014                 break;
2015         default:
2016                 ret = 1;
2017                 break;
2018         }
2019
2020         return ret;
2021 }
2022 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
2023
2024 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
2025                             gpa_t address, int len, const void *data)
2026 {
2027         struct kvm_lapic *apic = to_lapic(this);
2028         unsigned int offset = address - apic->base_address;
2029         u32 val;
2030
2031         if (!apic_mmio_in_range(apic, address))
2032                 return -EOPNOTSUPP;
2033
2034         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
2035                 if (!kvm_check_has_quirk(vcpu->kvm,
2036                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
2037                         return -EOPNOTSUPP;
2038
2039                 return 0;
2040         }
2041
2042         /*
2043          * APIC register must be aligned on 128-bits boundary.
2044          * 32/64/128 bits registers must be accessed thru 32 bits.
2045          * Refer SDM 8.4.1
2046          */
2047         if (len != 4 || (offset & 0xf))
2048                 return 0;
2049
2050         val = *(u32*)data;
2051
2052         kvm_lapic_reg_write(apic, offset & 0xff0, val);
2053
2054         return 0;
2055 }
2056
2057 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
2058 {
2059         kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
2060 }
2061 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
2062
2063 /* emulate APIC access in a trap manner */
2064 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
2065 {
2066         u32 val = 0;
2067
2068         /* hw has done the conditional check and inst decode */
2069         offset &= 0xff0;
2070
2071         kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
2072
2073         /* TODO: optimize to just emulate side effect w/o one more write */
2074         kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
2075 }
2076 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
2077
2078 void kvm_free_lapic(struct kvm_vcpu *vcpu)
2079 {
2080         struct kvm_lapic *apic = vcpu->arch.apic;
2081
2082         if (!vcpu->arch.apic)
2083                 return;
2084
2085         hrtimer_cancel(&apic->lapic_timer.timer);
2086
2087         if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2088                 static_key_slow_dec_deferred(&apic_hw_disabled);
2089
2090         if (!apic->sw_enabled)
2091                 static_key_slow_dec_deferred(&apic_sw_disabled);
2092
2093         if (apic->regs)
2094                 free_page((unsigned long)apic->regs);
2095
2096         kfree(apic);
2097 }
2098
2099 /*
2100  *----------------------------------------------------------------------
2101  * LAPIC interface
2102  *----------------------------------------------------------------------
2103  */
2104 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
2105 {
2106         struct kvm_lapic *apic = vcpu->arch.apic;
2107
2108         if (!lapic_in_kernel(vcpu) ||
2109                 !apic_lvtt_tscdeadline(apic))
2110                 return 0;
2111
2112         return apic->lapic_timer.tscdeadline;
2113 }
2114
2115 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
2116 {
2117         struct kvm_lapic *apic = vcpu->arch.apic;
2118
2119         if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
2120                         apic_lvtt_period(apic))
2121                 return;
2122
2123         hrtimer_cancel(&apic->lapic_timer.timer);
2124         apic->lapic_timer.tscdeadline = data;
2125         start_apic_timer(apic);
2126 }
2127
2128 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
2129 {
2130         struct kvm_lapic *apic = vcpu->arch.apic;
2131
2132         apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
2133                      | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
2134 }
2135
2136 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
2137 {
2138         u64 tpr;
2139
2140         tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
2141
2142         return (tpr & 0xf0) >> 4;
2143 }
2144
2145 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
2146 {
2147         u64 old_value = vcpu->arch.apic_base;
2148         struct kvm_lapic *apic = vcpu->arch.apic;
2149
2150         if (!apic)
2151                 value |= MSR_IA32_APICBASE_BSP;
2152
2153         vcpu->arch.apic_base = value;
2154
2155         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
2156                 kvm_update_cpuid(vcpu);
2157
2158         if (!apic)
2159                 return;
2160
2161         /* update jump label if enable bit changes */
2162         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2163                 if (value & MSR_IA32_APICBASE_ENABLE) {
2164                         kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2165                         static_key_slow_dec_deferred(&apic_hw_disabled);
2166                 } else {
2167                         static_key_slow_inc(&apic_hw_disabled.key);
2168                         recalculate_apic_map(vcpu->kvm);
2169                 }
2170         }
2171
2172         if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
2173                 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2174
2175         if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
2176                 kvm_x86_ops->set_virtual_apic_mode(vcpu);
2177
2178         apic->base_address = apic->vcpu->arch.apic_base &
2179                              MSR_IA32_APICBASE_BASE;
2180
2181         if ((value & MSR_IA32_APICBASE_ENABLE) &&
2182              apic->base_address != APIC_DEFAULT_PHYS_BASE)
2183                 pr_warn_once("APIC base relocation is unsupported by KVM");
2184 }
2185
2186 void kvm_apic_update_apicv(struct kvm_vcpu *vcpu)
2187 {
2188         struct kvm_lapic *apic = vcpu->arch.apic;
2189
2190         if (vcpu->arch.apicv_active) {
2191                 /* irr_pending is always true when apicv is activated. */
2192                 apic->irr_pending = true;
2193                 apic->isr_count = 1;
2194         } else {
2195                 apic->irr_pending = (apic_search_irr(apic) != -1);
2196                 apic->isr_count = count_vectors(apic->regs + APIC_ISR);
2197         }
2198 }
2199 EXPORT_SYMBOL_GPL(kvm_apic_update_apicv);
2200
2201 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
2202 {
2203         struct kvm_lapic *apic = vcpu->arch.apic;
2204         int i;
2205
2206         if (!apic)
2207                 return;
2208
2209         /* Stop the timer in case it's a reset to an active apic */
2210         hrtimer_cancel(&apic->lapic_timer.timer);
2211
2212         if (!init_event) {
2213                 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
2214                                          MSR_IA32_APICBASE_ENABLE);
2215                 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2216         }
2217         kvm_apic_set_version(apic->vcpu);
2218
2219         for (i = 0; i < KVM_APIC_LVT_NUM; i++)
2220                 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
2221         apic_update_lvtt(apic);
2222         if (kvm_vcpu_is_reset_bsp(vcpu) &&
2223             kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2224                 kvm_lapic_set_reg(apic, APIC_LVT0,
2225                              SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2226         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2227
2228         kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
2229         apic_set_spiv(apic, 0xff);
2230         kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2231         if (!apic_x2apic_mode(apic))
2232                 kvm_apic_set_ldr(apic, 0);
2233         kvm_lapic_set_reg(apic, APIC_ESR, 0);
2234         kvm_lapic_set_reg(apic, APIC_ICR, 0);
2235         kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2236         kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2237         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
2238         for (i = 0; i < 8; i++) {
2239                 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2240                 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2241                 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
2242         }
2243         kvm_apic_update_apicv(vcpu);
2244         apic->highest_isr_cache = -1;
2245         update_divide_count(apic);
2246         atomic_set(&apic->lapic_timer.pending, 0);
2247         if (kvm_vcpu_is_bsp(vcpu))
2248                 kvm_lapic_set_base(vcpu,
2249                                 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
2250         vcpu->arch.pv_eoi.msr_val = 0;
2251         apic_update_ppr(apic);
2252         if (vcpu->arch.apicv_active) {
2253                 kvm_x86_ops->apicv_post_state_restore(vcpu);
2254                 kvm_x86_ops->hwapic_irr_update(vcpu, -1);
2255                 kvm_x86_ops->hwapic_isr_update(vcpu, -1);
2256         }
2257
2258         vcpu->arch.apic_arb_prio = 0;
2259         vcpu->arch.apic_attention = 0;
2260 }
2261
2262 /*
2263  *----------------------------------------------------------------------
2264  * timer interface
2265  *----------------------------------------------------------------------
2266  */
2267
2268 static bool lapic_is_periodic(struct kvm_lapic *apic)
2269 {
2270         return apic_lvtt_period(apic);
2271 }
2272
2273 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2274 {
2275         struct kvm_lapic *apic = vcpu->arch.apic;
2276
2277         if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2278                 return atomic_read(&apic->lapic_timer.pending);
2279
2280         return 0;
2281 }
2282
2283 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2284 {
2285         u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2286         int vector, mode, trig_mode;
2287
2288         if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2289                 vector = reg & APIC_VECTOR_MASK;
2290                 mode = reg & APIC_MODE_MASK;
2291                 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2292                 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2293                                         NULL);
2294         }
2295         return 0;
2296 }
2297
2298 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2299 {
2300         struct kvm_lapic *apic = vcpu->arch.apic;
2301
2302         if (apic)
2303                 kvm_apic_local_deliver(apic, APIC_LVT0);
2304 }
2305
2306 static const struct kvm_io_device_ops apic_mmio_ops = {
2307         .read     = apic_mmio_read,
2308         .write    = apic_mmio_write,
2309 };
2310
2311 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2312 {
2313         struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2314         struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2315
2316         apic_timer_expired(apic);
2317
2318         if (lapic_is_periodic(apic)) {
2319                 advance_periodic_target_expiration(apic);
2320                 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2321                 return HRTIMER_RESTART;
2322         } else
2323                 return HRTIMER_NORESTART;
2324 }
2325
2326 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
2327 {
2328         struct kvm_lapic *apic;
2329
2330         ASSERT(vcpu != NULL);
2331
2332         apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
2333         if (!apic)
2334                 goto nomem;
2335
2336         vcpu->arch.apic = apic;
2337
2338         apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
2339         if (!apic->regs) {
2340                 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2341                        vcpu->vcpu_id);
2342                 goto nomem_free_apic;
2343         }
2344         apic->vcpu = vcpu;
2345
2346         hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2347                      HRTIMER_MODE_ABS_HARD);
2348         apic->lapic_timer.timer.function = apic_timer_fn;
2349         if (timer_advance_ns == -1) {
2350                 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
2351                 lapic_timer_advance_dynamic = true;
2352         } else {
2353                 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2354                 lapic_timer_advance_dynamic = false;
2355         }
2356
2357         /*
2358          * APIC is created enabled. This will prevent kvm_lapic_set_base from
2359          * thinking that APIC state has changed.
2360          */
2361         vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2362         static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
2363         kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
2364
2365         return 0;
2366 nomem_free_apic:
2367         kfree(apic);
2368         vcpu->arch.apic = NULL;
2369 nomem:
2370         return -ENOMEM;
2371 }
2372
2373 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2374 {
2375         struct kvm_lapic *apic = vcpu->arch.apic;
2376         u32 ppr;
2377
2378         if (!kvm_apic_hw_enabled(apic))
2379                 return -1;
2380
2381         __apic_update_ppr(apic, &ppr);
2382         return apic_has_interrupt_for_ppr(apic, ppr);
2383 }
2384
2385 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2386 {
2387         u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
2388
2389         if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2390                 return 1;
2391         if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2392             GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2393                 return 1;
2394         return 0;
2395 }
2396
2397 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2398 {
2399         struct kvm_lapic *apic = vcpu->arch.apic;
2400
2401         if (atomic_read(&apic->lapic_timer.pending) > 0) {
2402                 kvm_apic_inject_pending_timer_irqs(apic);
2403                 atomic_set(&apic->lapic_timer.pending, 0);
2404         }
2405 }
2406
2407 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2408 {
2409         int vector = kvm_apic_has_interrupt(vcpu);
2410         struct kvm_lapic *apic = vcpu->arch.apic;
2411         u32 ppr;
2412
2413         if (vector == -1)
2414                 return -1;
2415
2416         /*
2417          * We get here even with APIC virtualization enabled, if doing
2418          * nested virtualization and L1 runs with the "acknowledge interrupt
2419          * on exit" mode.  Then we cannot inject the interrupt via RVI,
2420          * because the process would deliver it through the IDT.
2421          */
2422
2423         apic_clear_irr(vector, apic);
2424         if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
2425                 /*
2426                  * For auto-EOI interrupts, there might be another pending
2427                  * interrupt above PPR, so check whether to raise another
2428                  * KVM_REQ_EVENT.
2429                  */
2430                 apic_update_ppr(apic);
2431         } else {
2432                 /*
2433                  * For normal interrupts, PPR has been raised and there cannot
2434                  * be a higher-priority pending interrupt---except if there was
2435                  * a concurrent interrupt injection, but that would have
2436                  * triggered KVM_REQ_EVENT already.
2437                  */
2438                 apic_set_isr(vector, apic);
2439                 __apic_update_ppr(apic, &ppr);
2440         }
2441
2442         return vector;
2443 }
2444
2445 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2446                 struct kvm_lapic_state *s, bool set)
2447 {
2448         if (apic_x2apic_mode(vcpu->arch.apic)) {
2449                 u32 *id = (u32 *)(s->regs + APIC_ID);
2450                 u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2451
2452                 if (vcpu->kvm->arch.x2apic_format) {
2453                         if (*id != vcpu->vcpu_id)
2454                                 return -EINVAL;
2455                 } else {
2456                         if (set)
2457                                 *id >>= 24;
2458                         else
2459                                 *id <<= 24;
2460                 }
2461
2462                 /* In x2APIC mode, the LDR is fixed and based on the id */
2463                 if (set)
2464                         *ldr = kvm_apic_calc_x2apic_ldr(*id);
2465         }
2466
2467         return 0;
2468 }
2469
2470 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2471 {
2472         memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2473         return kvm_apic_state_fixup(vcpu, s, false);
2474 }
2475
2476 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2477 {
2478         struct kvm_lapic *apic = vcpu->arch.apic;
2479         int r;
2480
2481
2482         kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
2483         /* set SPIV separately to get count of SW disabled APICs right */
2484         apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
2485
2486         r = kvm_apic_state_fixup(vcpu, s, true);
2487         if (r)
2488                 return r;
2489         memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
2490
2491         recalculate_apic_map(vcpu->kvm);
2492         kvm_apic_set_version(vcpu);
2493
2494         apic_update_ppr(apic);
2495         hrtimer_cancel(&apic->lapic_timer.timer);
2496         apic_update_lvtt(apic);
2497         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2498         update_divide_count(apic);
2499         start_apic_timer(apic);
2500         kvm_apic_update_apicv(vcpu);
2501         apic->highest_isr_cache = -1;
2502         if (vcpu->arch.apicv_active) {
2503                 kvm_x86_ops->apicv_post_state_restore(vcpu);
2504                 kvm_x86_ops->hwapic_irr_update(vcpu,
2505                                 apic_find_highest_irr(apic));
2506                 kvm_x86_ops->hwapic_isr_update(vcpu,
2507                                 apic_find_highest_isr(apic));
2508         }
2509         kvm_make_request(KVM_REQ_EVENT, vcpu);
2510         if (ioapic_in_kernel(vcpu->kvm))
2511                 kvm_rtc_eoi_tracking_restore_one(vcpu);
2512
2513         vcpu->arch.apic_arb_prio = 0;
2514
2515         return 0;
2516 }
2517
2518 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
2519 {
2520         struct hrtimer *timer;
2521
2522         if (!lapic_in_kernel(vcpu) ||
2523                 kvm_can_post_timer_interrupt(vcpu))
2524                 return;
2525
2526         timer = &vcpu->arch.apic->lapic_timer.timer;
2527         if (hrtimer_cancel(timer))
2528                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
2529 }
2530
2531 /*
2532  * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2533  *
2534  * Detect whether guest triggered PV EOI since the
2535  * last entry. If yes, set EOI on guests's behalf.
2536  * Clear PV EOI in guest memory in any case.
2537  */
2538 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2539                                         struct kvm_lapic *apic)
2540 {
2541         bool pending;
2542         int vector;
2543         /*
2544          * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2545          * and KVM_PV_EOI_ENABLED in guest memory as follows:
2546          *
2547          * KVM_APIC_PV_EOI_PENDING is unset:
2548          *      -> host disabled PV EOI.
2549          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2550          *      -> host enabled PV EOI, guest did not execute EOI yet.
2551          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2552          *      -> host enabled PV EOI, guest executed EOI.
2553          */
2554         BUG_ON(!pv_eoi_enabled(vcpu));
2555         pending = pv_eoi_get_pending(vcpu);
2556         /*
2557          * Clear pending bit in any case: it will be set again on vmentry.
2558          * While this might not be ideal from performance point of view,
2559          * this makes sure pv eoi is only enabled when we know it's safe.
2560          */
2561         pv_eoi_clr_pending(vcpu);
2562         if (pending)
2563                 return;
2564         vector = apic_set_eoi(apic);
2565         trace_kvm_pv_eoi(apic, vector);
2566 }
2567
2568 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2569 {
2570         u32 data;
2571
2572         if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2573                 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2574
2575         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2576                 return;
2577
2578         if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2579                                   sizeof(u32)))
2580                 return;
2581
2582         apic_set_tpr(vcpu->arch.apic, data & 0xff);
2583 }
2584
2585 /*
2586  * apic_sync_pv_eoi_to_guest - called before vmentry
2587  *
2588  * Detect whether it's safe to enable PV EOI and
2589  * if yes do so.
2590  */
2591 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2592                                         struct kvm_lapic *apic)
2593 {
2594         if (!pv_eoi_enabled(vcpu) ||
2595             /* IRR set or many bits in ISR: could be nested. */
2596             apic->irr_pending ||
2597             /* Cache not set: could be safe but we don't bother. */
2598             apic->highest_isr_cache == -1 ||
2599             /* Need EOI to update ioapic. */
2600             kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
2601                 /*
2602                  * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2603                  * so we need not do anything here.
2604                  */
2605                 return;
2606         }
2607
2608         pv_eoi_set_pending(apic->vcpu);
2609 }
2610
2611 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2612 {
2613         u32 data, tpr;
2614         int max_irr, max_isr;
2615         struct kvm_lapic *apic = vcpu->arch.apic;
2616
2617         apic_sync_pv_eoi_to_guest(vcpu, apic);
2618
2619         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2620                 return;
2621
2622         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
2623         max_irr = apic_find_highest_irr(apic);
2624         if (max_irr < 0)
2625                 max_irr = 0;
2626         max_isr = apic_find_highest_isr(apic);
2627         if (max_isr < 0)
2628                 max_isr = 0;
2629         data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2630
2631         kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2632                                 sizeof(u32));
2633 }
2634
2635 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
2636 {
2637         if (vapic_addr) {
2638                 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2639                                         &vcpu->arch.apic->vapic_cache,
2640                                         vapic_addr, sizeof(u32)))
2641                         return -EINVAL;
2642                 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2643         } else {
2644                 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2645         }
2646
2647         vcpu->arch.apic->vapic_addr = vapic_addr;
2648         return 0;
2649 }
2650
2651 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2652 {
2653         struct kvm_lapic *apic = vcpu->arch.apic;
2654         u32 reg = (msr - APIC_BASE_MSR) << 4;
2655
2656         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2657                 return 1;
2658
2659         if (reg == APIC_ICR2)
2660                 return 1;
2661
2662         /* if this is ICR write vector before command */
2663         if (reg == APIC_ICR)
2664                 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2665         return kvm_lapic_reg_write(apic, reg, (u32)data);
2666 }
2667
2668 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2669 {
2670         struct kvm_lapic *apic = vcpu->arch.apic;
2671         u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2672
2673         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2674                 return 1;
2675
2676         if (reg == APIC_DFR || reg == APIC_ICR2)
2677                 return 1;
2678
2679         if (kvm_lapic_reg_read(apic, reg, 4, &low))
2680                 return 1;
2681         if (reg == APIC_ICR)
2682                 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2683
2684         *data = (((u64)high) << 32) | low;
2685
2686         return 0;
2687 }
2688
2689 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2690 {
2691         struct kvm_lapic *apic = vcpu->arch.apic;
2692
2693         if (!lapic_in_kernel(vcpu))
2694                 return 1;
2695
2696         /* if this is ICR write vector before command */
2697         if (reg == APIC_ICR)
2698                 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2699         return kvm_lapic_reg_write(apic, reg, (u32)data);
2700 }
2701
2702 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2703 {
2704         struct kvm_lapic *apic = vcpu->arch.apic;
2705         u32 low, high = 0;
2706
2707         if (!lapic_in_kernel(vcpu))
2708                 return 1;
2709
2710         if (kvm_lapic_reg_read(apic, reg, 4, &low))
2711                 return 1;
2712         if (reg == APIC_ICR)
2713                 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2714
2715         *data = (((u64)high) << 32) | low;
2716
2717         return 0;
2718 }
2719
2720 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
2721 {
2722         u64 addr = data & ~KVM_MSR_ENABLED;
2723         struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
2724         unsigned long new_len;
2725
2726         if (!IS_ALIGNED(addr, 4))
2727                 return 1;
2728
2729         vcpu->arch.pv_eoi.msr_val = data;
2730         if (!pv_eoi_enabled(vcpu))
2731                 return 0;
2732
2733         if (addr == ghc->gpa && len <= ghc->len)
2734                 new_len = ghc->len;
2735         else
2736                 new_len = len;
2737
2738         return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
2739 }
2740
2741 void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2742 {
2743         struct kvm_lapic *apic = vcpu->arch.apic;
2744         u8 sipi_vector;
2745         unsigned long pe;
2746
2747         if (!lapic_in_kernel(vcpu) || !apic->pending_events)
2748                 return;
2749
2750         /*
2751          * INITs are latched while CPU is in specific states
2752          * (SMM, VMX non-root mode, SVM with GIF=0).
2753          * Because a CPU cannot be in these states immediately
2754          * after it has processed an INIT signal (and thus in
2755          * KVM_MP_STATE_INIT_RECEIVED state), just eat SIPIs
2756          * and leave the INIT pending.
2757          */
2758         if (kvm_vcpu_latch_init(vcpu)) {
2759                 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2760                 if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2761                         clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2762                 return;
2763         }
2764
2765         pe = xchg(&apic->pending_events, 0);
2766         if (test_bit(KVM_APIC_INIT, &pe)) {
2767                 kvm_vcpu_reset(vcpu, true);
2768                 if (kvm_vcpu_is_bsp(apic->vcpu))
2769                         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2770                 else
2771                         vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2772         }
2773         if (test_bit(KVM_APIC_SIPI, &pe) &&
2774             vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2775                 /* evaluate pending_events before reading the vector */
2776                 smp_rmb();
2777                 sipi_vector = apic->sipi_vector;
2778                 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2779                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2780         }
2781 }
2782
2783 void kvm_lapic_init(void)
2784 {
2785         /* do not patch jump label more than once per second */
2786         jump_label_rate_limit(&apic_hw_disabled, HZ);
2787         jump_label_rate_limit(&apic_sw_disabled, HZ);
2788 }
2789
2790 void kvm_lapic_exit(void)
2791 {
2792         static_key_deferred_flush(&apic_hw_disabled);
2793         static_key_deferred_flush(&apic_sw_disabled);
2794 }