arch/x86/kvm/lapic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /*
   4  * Local APIC virtualization
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright (C) 2007 Novell
   8  * Copyright (C) 2007 Intel
   9  * Copyright 2009 Red Hat, Inc. and/or its affiliates.
  10  *
  11  * Authors:
  12  *   Dor Laor <dor.laor@qumranet.com>
  13  *   Gregory Haskins <ghaskins@novell.com>
  14  *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
  15  *
  16  * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
  17  */
  18
  19 #include <linux/kvm_host.h>
  20 #include <linux/kvm.h>
  21 #include <linux/mm.h>
  22 #include <linux/highmem.h>
  23 #include <linux/smp.h>
  24 #include <linux/hrtimer.h>
  25 #include <linux/io.h>
  26 #include <linux/export.h>
  27 #include <linux/math64.h>
  28 #include <linux/slab.h>
  29 #include <asm/processor.h>
  30 #include <asm/msr.h>
  31 #include <asm/page.h>
  32 #include <asm/current.h>
  33 #include <asm/apicdef.h>
  34 #include <asm/delay.h>
  35 #include <linux/atomic.h>
  36 #include <linux/jump_label.h>
  37 #include "kvm_cache_regs.h"
  38 #include "irq.h"
  39 #include "trace.h"
  40 #include "x86.h"
  41 #include "cpuid.h"
  42 #include "hyperv.h"
  43
  44 #ifndef CONFIG_X86_64
  45 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
  46 #else
  47 #define mod_64(x, y) ((x) % (y))
  48 #endif
  49
  50 #define PRId64 "d"
  51 #define PRIx64 "llx"
  52 #define PRIu64 "u"
  53 #define PRIo64 "o"
  54
  55 /* 14 is the version for Xeon and Pentium 8.4.8*/
  56 #define APIC_VERSION                    (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
  57 #define LAPIC_MMIO_LENGTH               (1 << 12)
  58 /* followed define is not in apicdef.h */
  59 #define MAX_APIC_VECTOR                 256
  60 #define APIC_VECTORS_PER_REG            32
  61
  62 static bool lapic_timer_advance_dynamic __read_mostly;
  63 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN  100     /* clock cycles */
  64 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX  10000   /* clock cycles */
  65 #define LAPIC_TIMER_ADVANCE_NS_INIT     1000
  66 #define LAPIC_TIMER_ADVANCE_NS_MAX     5000
  67 /* step-by-step approximation to mitigate fluctuation */
  68 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
  69
  70 static inline int apic_test_vector(int vec, void *bitmap)
  71 {
  72         return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  73 }
  74
  75 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
  76 {
  77         struct kvm_lapic *apic = vcpu->arch.apic;
  78
  79         return apic_test_vector(vector, apic->regs + APIC_ISR) ||
  80                 apic_test_vector(vector, apic->regs + APIC_IRR);
  81 }
  82
  83 static inline int __apic_test_and_set_vector(int vec, void *bitmap)
  84 {
  85         return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  86 }
  87
  88 static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
  89 {
  90         return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  91 }
  92
  93 struct static_key_deferred apic_hw_disabled __read_mostly;
  94 struct static_key_deferred apic_sw_disabled __read_mostly;
  95
  96 static inline int apic_enabled(struct kvm_lapic *apic)
  97 {
  98         return kvm_apic_sw_enabled(apic) &&     kvm_apic_hw_enabled(apic);
  99 }
 100
 101 #define LVT_MASK        \
 102         (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
 103
 104 #define LINT_MASK       \
 105         (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
 106          APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
 107
 108 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
 109 {
 110         return apic->vcpu->vcpu_id;
 111 }
 112
 113 static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
 114 {
 115         return pi_inject_timer && kvm_vcpu_apicv_active(vcpu);
 116 }
 117
 118 bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu)
 119 {
 120         return kvm_x86_ops.set_hv_timer
 121                && !(kvm_mwait_in_guest(vcpu->kvm) ||
 122                     kvm_can_post_timer_interrupt(vcpu));
 123 }
 124 EXPORT_SYMBOL_GPL(kvm_can_use_hv_timer);
 125
 126 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
 127 {
 128         return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
 129 }
 130
 131 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
 132                 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
 133         switch (map->mode) {
 134         case KVM_APIC_MODE_X2APIC: {
 135                 u32 offset = (dest_id >> 16) * 16;
 136                 u32 max_apic_id = map->max_apic_id;
 137
 138                 if (offset <= max_apic_id) {
 139                         u8 cluster_size = min(max_apic_id - offset + 1, 16U);
 140
 141                         offset = array_index_nospec(offset, map->max_apic_id + 1);
 142                         *cluster = &map->phys_map[offset];
 143                         *mask = dest_id & (0xffff >> (16 - cluster_size));
 144                 } else {
 145                         *mask = 0;
 146                 }
 147
 148                 return true;
 149                 }
 150         case KVM_APIC_MODE_XAPIC_FLAT:
 151                 *cluster = map->xapic_flat_map;
 152                 *mask = dest_id & 0xff;
 153                 return true;
 154         case KVM_APIC_MODE_XAPIC_CLUSTER:
 155                 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
 156                 *mask = dest_id & 0xf;
 157                 return true;
 158         default:
 159                 /* Not optimized. */
 160                 return false;
 161         }
 162 }
 163
 164 static void kvm_apic_map_free(struct rcu_head *rcu)
 165 {
 166         struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
 167
 168         kvfree(map);
 169 }
 170
 171 void kvm_recalculate_apic_map(struct kvm *kvm)
 172 {
 173         struct kvm_apic_map *new, *old = NULL;
 174         struct kvm_vcpu *vcpu;
 175         int i;
 176         u32 max_id = 255; /* enough space for any xAPIC ID */
 177
 178         if (!kvm->arch.apic_map_dirty) {
 179                 /*
 180                  * Read kvm->arch.apic_map_dirty before
 181                  * kvm->arch.apic_map
 182                  */
 183                 smp_rmb();
 184                 return;
 185         }
 186
 187         mutex_lock(&kvm->arch.apic_map_lock);
 188         if (!kvm->arch.apic_map_dirty) {
 189                 /* Someone else has updated the map. */
 190                 mutex_unlock(&kvm->arch.apic_map_lock);
 191                 return;
 192         }
 193
 194         kvm_for_each_vcpu(i, vcpu, kvm)
 195                 if (kvm_apic_present(vcpu))
 196                         max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
 197
 198         new = kvzalloc(sizeof(struct kvm_apic_map) +
 199                            sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
 200                            GFP_KERNEL_ACCOUNT);
 201
 202         if (!new)
 203                 goto out;
 204
 205         new->max_apic_id = max_id;
 206
 207         kvm_for_each_vcpu(i, vcpu, kvm) {
 208                 struct kvm_lapic *apic = vcpu->arch.apic;
 209                 struct kvm_lapic **cluster;
 210                 u16 mask;
 211                 u32 ldr;
 212                 u8 xapic_id;
 213                 u32 x2apic_id;
 214
 215                 if (!kvm_apic_present(vcpu))
 216                         continue;
 217
 218                 xapic_id = kvm_xapic_id(apic);
 219                 x2apic_id = kvm_x2apic_id(apic);
 220
 221                 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
 222                 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
 223                                 x2apic_id <= new->max_apic_id)
 224                         new->phys_map[x2apic_id] = apic;
 225                 /*
 226                  * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
 227                  * prevent them from masking VCPUs with APIC ID <= 0xff.
 228                  */
 229                 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
 230                         new->phys_map[xapic_id] = apic;
 231
 232                 if (!kvm_apic_sw_enabled(apic))
 233                         continue;
 234
 235                 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
 236
 237                 if (apic_x2apic_mode(apic)) {
 238                         new->mode |= KVM_APIC_MODE_X2APIC;
 239                 } else if (ldr) {
 240                         ldr = GET_APIC_LOGICAL_ID(ldr);
 241                         if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
 242                                 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
 243                         else
 244                                 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
 245                 }
 246
 247                 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
 248                         continue;
 249
 250                 if (mask)
 251                         cluster[ffs(mask) - 1] = apic;
 252         }
 253 out:
 254         old = rcu_dereference_protected(kvm->arch.apic_map,
 255                         lockdep_is_held(&kvm->arch.apic_map_lock));
 256         rcu_assign_pointer(kvm->arch.apic_map, new);
 257         /*
 258          * Write kvm->arch.apic_map before
 259          * clearing apic->apic_map_dirty
 260          */
 261         smp_wmb();
 262         kvm->arch.apic_map_dirty = false;
 263         mutex_unlock(&kvm->arch.apic_map_lock);
 264
 265         if (old)
 266                 call_rcu(&old->rcu, kvm_apic_map_free);
 267
 268         kvm_make_scan_ioapic_request(kvm);
 269 }
 270
 271 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
 272 {
 273         bool enabled = val & APIC_SPIV_APIC_ENABLED;
 274
 275         kvm_lapic_set_reg(apic, APIC_SPIV, val);
 276
 277         if (enabled != apic->sw_enabled) {
 278                 apic->sw_enabled = enabled;
 279                 if (enabled)
 280                         static_key_slow_dec_deferred(&apic_sw_disabled);
 281                 else
 282                         static_key_slow_inc(&apic_sw_disabled.key);
 283
 284                 apic->vcpu->kvm->arch.apic_map_dirty = true;
 285         }
 286 }
 287
 288 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
 289 {
 290         kvm_lapic_set_reg(apic, APIC_ID, id << 24);
 291         apic->vcpu->kvm->arch.apic_map_dirty = true;
 292 }
 293
 294 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
 295 {
 296         kvm_lapic_set_reg(apic, APIC_LDR, id);
 297         apic->vcpu->kvm->arch.apic_map_dirty = true;
 298 }
 299
 300 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
 301 {
 302         return ((id >> 4) << 16) | (1 << (id & 0xf));
 303 }
 304
 305 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
 306 {
 307         u32 ldr = kvm_apic_calc_x2apic_ldr(id);
 308
 309         WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
 310
 311         kvm_lapic_set_reg(apic, APIC_ID, id);
 312         kvm_lapic_set_reg(apic, APIC_LDR, ldr);
 313         apic->vcpu->kvm->arch.apic_map_dirty = true;
 314 }
 315
 316 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
 317 {
 318         return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
 319 }
 320
 321 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
 322 {
 323         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
 324 }
 325
 326 static inline int apic_lvtt_period(struct kvm_lapic *apic)
 327 {
 328         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
 329 }
 330
 331 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
 332 {
 333         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
 334 }
 335
 336 static inline int apic_lvt_nmi_mode(u32 lvt_val)
 337 {
 338         return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
 339 }
 340
 341 void kvm_apic_set_version(struct kvm_vcpu *vcpu)
 342 {
 343         struct kvm_lapic *apic = vcpu->arch.apic;
 344         struct kvm_cpuid_entry2 *feat;
 345         u32 v = APIC_VERSION;
 346
 347         if (!lapic_in_kernel(vcpu))
 348                 return;
 349
 350         /*
 351          * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
 352          * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
 353          * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
 354          * version first and level-triggered interrupts never get EOIed in
 355          * IOAPIC.
 356          */
 357         feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
 358         if (feat && (feat->ecx & (1 << (X86_FEATURE_X2APIC & 31))) &&
 359             !ioapic_in_kernel(vcpu->kvm))
 360                 v |= APIC_LVR_DIRECTED_EOI;
 361         kvm_lapic_set_reg(apic, APIC_LVR, v);
 362 }
 363
 364 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
 365         LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
 366         LVT_MASK | APIC_MODE_MASK,      /* LVTTHMR */
 367         LVT_MASK | APIC_MODE_MASK,      /* LVTPC */
 368         LINT_MASK, LINT_MASK,   /* LVT0-1 */
 369         LVT_MASK                /* LVTERR */
 370 };
 371
 372 static int find_highest_vector(void *bitmap)
 373 {
 374         int vec;
 375         u32 *reg;
 376
 377         for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
 378              vec >= 0; vec -= APIC_VECTORS_PER_REG) {
 379                 reg = bitmap + REG_POS(vec);
 380                 if (*reg)
 381                         return __fls(*reg) + vec;
 382         }
 383
 384         return -1;
 385 }
 386
 387 static u8 count_vectors(void *bitmap)
 388 {
 389         int vec;
 390         u32 *reg;
 391         u8 count = 0;
 392
 393         for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
 394                 reg = bitmap + REG_POS(vec);
 395                 count += hweight32(*reg);
 396         }
 397
 398         return count;
 399 }
 400
 401 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
 402 {
 403         u32 i, vec;
 404         u32 pir_val, irr_val, prev_irr_val;
 405         int max_updated_irr;
 406
 407         max_updated_irr = -1;
 408         *max_irr = -1;
 409
 410         for (i = vec = 0; i <= 7; i++, vec += 32) {
 411                 pir_val = READ_ONCE(pir[i]);
 412                 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
 413                 if (pir_val) {
 414                         prev_irr_val = irr_val;
 415                         irr_val |= xchg(&pir[i], 0);
 416                         *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
 417                         if (prev_irr_val != irr_val) {
 418                                 max_updated_irr =
 419                                         __fls(irr_val ^ prev_irr_val) + vec;
 420                         }
 421                 }
 422                 if (irr_val)
 423                         *max_irr = __fls(irr_val) + vec;
 424         }
 425
 426         return ((max_updated_irr != -1) &&
 427                 (max_updated_irr == *max_irr));
 428 }
 429 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
 430
 431 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
 432 {
 433         struct kvm_lapic *apic = vcpu->arch.apic;
 434
 435         return __kvm_apic_update_irr(pir, apic->regs, max_irr);
 436 }
 437 EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
 438
 439 static inline int apic_search_irr(struct kvm_lapic *apic)
 440 {
 441         return find_highest_vector(apic->regs + APIC_IRR);
 442 }
 443
 444 static inline int apic_find_highest_irr(struct kvm_lapic *apic)
 445 {
 446         int result;
 447
 448         /*
 449          * Note that irr_pending is just a hint. It will be always
 450          * true with virtual interrupt delivery enabled.
 451          */
 452         if (!apic->irr_pending)
 453                 return -1;
 454
 455         result = apic_search_irr(apic);
 456         ASSERT(result == -1 || result >= 16);
 457
 458         return result;
 459 }
 460
 461 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
 462 {
 463         struct kvm_vcpu *vcpu;
 464
 465         vcpu = apic->vcpu;
 466
 467         if (unlikely(vcpu->arch.apicv_active)) {
 468                 /* need to update RVI */
 469                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 470                 kvm_x86_ops.hwapic_irr_update(vcpu,
 471                                 apic_find_highest_irr(apic));
 472         } else {
 473                 apic->irr_pending = false;
 474                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 475                 if (apic_search_irr(apic) != -1)
 476                         apic->irr_pending = true;
 477         }
 478 }
 479
 480 static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
 481 {
 482         struct kvm_vcpu *vcpu;
 483
 484         if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
 485                 return;
 486
 487         vcpu = apic->vcpu;
 488
 489         /*
 490          * With APIC virtualization enabled, all caching is disabled
 491          * because the processor can modify ISR under the hood.  Instead
 492          * just set SVI.
 493          */
 494         if (unlikely(vcpu->arch.apicv_active))
 495                 kvm_x86_ops.hwapic_isr_update(vcpu, vec);
 496         else {
 497                 ++apic->isr_count;
 498                 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
 499                 /*
 500                  * ISR (in service register) bit is set when injecting an interrupt.
 501                  * The highest vector is injected. Thus the latest bit set matches
 502                  * the highest bit in ISR.
 503                  */
 504                 apic->highest_isr_cache = vec;
 505         }
 506 }
 507
 508 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
 509 {
 510         int result;
 511
 512         /*
 513          * Note that isr_count is always 1, and highest_isr_cache
 514          * is always -1, with APIC virtualization enabled.
 515          */
 516         if (!apic->isr_count)
 517                 return -1;
 518         if (likely(apic->highest_isr_cache != -1))
 519                 return apic->highest_isr_cache;
 520
 521         result = find_highest_vector(apic->regs + APIC_ISR);
 522         ASSERT(result == -1 || result >= 16);
 523
 524         return result;
 525 }
 526
 527 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
 528 {
 529         struct kvm_vcpu *vcpu;
 530         if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
 531                 return;
 532
 533         vcpu = apic->vcpu;
 534
 535         /*
 536          * We do get here for APIC virtualization enabled if the guest
 537          * uses the Hyper-V APIC enlightenment.  In this case we may need
 538          * to trigger a new interrupt delivery by writing the SVI field;
 539          * on the other hand isr_count and highest_isr_cache are unused
 540          * and must be left alone.
 541          */
 542         if (unlikely(vcpu->arch.apicv_active))
 543                 kvm_x86_ops.hwapic_isr_update(vcpu,
 544                                                apic_find_highest_isr(apic));
 545         else {
 546                 --apic->isr_count;
 547                 BUG_ON(apic->isr_count < 0);
 548                 apic->highest_isr_cache = -1;
 549         }
 550 }
 551
 552 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 553 {
 554         /* This may race with setting of irr in __apic_accept_irq() and
 555          * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
 556          * will cause vmexit immediately and the value will be recalculated
 557          * on the next vmentry.
 558          */
 559         return apic_find_highest_irr(vcpu->arch.apic);
 560 }
 561 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
 562
 563 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
 564                              int vector, int level, int trig_mode,
 565                              struct dest_map *dest_map);
 566
 567 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
 568                      struct dest_map *dest_map)
 569 {
 570         struct kvm_lapic *apic = vcpu->arch.apic;
 571
 572         return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
 573                         irq->level, irq->trig_mode, dest_map);
 574 }
 575
 576 static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map,
 577                          struct kvm_lapic_irq *irq, u32 min)
 578 {
 579         int i, count = 0;
 580         struct kvm_vcpu *vcpu;
 581
 582         if (min > map->max_apic_id)
 583                 return 0;
 584
 585         for_each_set_bit(i, ipi_bitmap,
 586                 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
 587                 if (map->phys_map[min + i]) {
 588                         vcpu = map->phys_map[min + i]->vcpu;
 589                         count += kvm_apic_set_irq(vcpu, irq, NULL);
 590                 }
 591         }
 592
 593         return count;
 594 }
 595
 596 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
 597                     unsigned long ipi_bitmap_high, u32 min,
 598                     unsigned long icr, int op_64_bit)
 599 {
 600         struct kvm_apic_map *map;
 601         struct kvm_lapic_irq irq = {0};
 602         int cluster_size = op_64_bit ? 64 : 32;
 603         int count;
 604
 605         if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK))
 606                 return -KVM_EINVAL;
 607
 608         irq.vector = icr & APIC_VECTOR_MASK;
 609         irq.delivery_mode = icr & APIC_MODE_MASK;
 610         irq.level = (icr & APIC_INT_ASSERT) != 0;
 611         irq.trig_mode = icr & APIC_INT_LEVELTRIG;
 612
 613         rcu_read_lock();
 614         map = rcu_dereference(kvm->arch.apic_map);
 615
 616         count = -EOPNOTSUPP;
 617         if (likely(map)) {
 618                 count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min);
 619                 min += cluster_size;
 620                 count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min);
 621         }
 622
 623         rcu_read_unlock();
 624         return count;
 625 }
 626
 627 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
 628 {
 629
 630         return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
 631                                       sizeof(val));
 632 }
 633
 634 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
 635 {
 636
 637         return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
 638                                       sizeof(*val));
 639 }
 640
 641 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
 642 {
 643         return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
 644 }
 645
 646 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
 647 {
 648         u8 val;
 649         if (pv_eoi_get_user(vcpu, &val) < 0) {
 650                 printk(KERN_WARNING "Can't read EOI MSR value: 0x%llx\n",
 651                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 652                 return false;
 653         }
 654         return val & 0x1;
 655 }
 656
 657 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
 658 {
 659         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0) {
 660                 printk(KERN_WARNING "Can't set EOI MSR value: 0x%llx\n",
 661                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 662                 return;
 663         }
 664         __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 665 }
 666
 667 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
 668 {
 669         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0) {
 670                 printk(KERN_WARNING "Can't clear EOI MSR value: 0x%llx\n",
 671                            (unsigned long long)vcpu->arch.pv_eoi.msr_val);
 672                 return;
 673         }
 674         __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 675 }
 676
 677 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
 678 {
 679         int highest_irr;
 680         if (apic->vcpu->arch.apicv_active)
 681                 highest_irr = kvm_x86_ops.sync_pir_to_irr(apic->vcpu);
 682         else
 683                 highest_irr = apic_find_highest_irr(apic);
 684         if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
 685                 return -1;
 686         return highest_irr;
 687 }
 688
 689 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
 690 {
 691         u32 tpr, isrv, ppr, old_ppr;
 692         int isr;
 693
 694         old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
 695         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
 696         isr = apic_find_highest_isr(apic);
 697         isrv = (isr != -1) ? isr : 0;
 698
 699         if ((tpr & 0xf0) >= (isrv & 0xf0))
 700                 ppr = tpr & 0xff;
 701         else
 702                 ppr = isrv & 0xf0;
 703
 704         *new_ppr = ppr;
 705         if (old_ppr != ppr)
 706                 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
 707
 708         return ppr < old_ppr;
 709 }
 710
 711 static void apic_update_ppr(struct kvm_lapic *apic)
 712 {
 713         u32 ppr;
 714
 715         if (__apic_update_ppr(apic, &ppr) &&
 716             apic_has_interrupt_for_ppr(apic, ppr) != -1)
 717                 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
 718 }
 719
 720 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
 721 {
 722         apic_update_ppr(vcpu->arch.apic);
 723 }
 724 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
 725
 726 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
 727 {
 728         kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
 729         apic_update_ppr(apic);
 730 }
 731
 732 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
 733 {
 734         return mda == (apic_x2apic_mode(apic) ?
 735                         X2APIC_BROADCAST : APIC_BROADCAST);
 736 }
 737
 738 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
 739 {
 740         if (kvm_apic_broadcast(apic, mda))
 741                 return true;
 742
 743         if (apic_x2apic_mode(apic))
 744                 return mda == kvm_x2apic_id(apic);
 745
 746         /*
 747          * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
 748          * it were in x2APIC mode.  Hotplugged VCPUs start in xAPIC mode and
 749          * this allows unique addressing of VCPUs with APIC ID over 0xff.
 750          * The 0xff condition is needed because writeable xAPIC ID.
 751          */
 752         if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
 753                 return true;
 754
 755         return mda == kvm_xapic_id(apic);
 756 }
 757
 758 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
 759 {
 760         u32 logical_id;
 761
 762         if (kvm_apic_broadcast(apic, mda))
 763                 return true;
 764
 765         logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
 766
 767         if (apic_x2apic_mode(apic))
 768                 return ((logical_id >> 16) == (mda >> 16))
 769                        && (logical_id & mda & 0xffff) != 0;
 770
 771         logical_id = GET_APIC_LOGICAL_ID(logical_id);
 772
 773         switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
 774         case APIC_DFR_FLAT:
 775                 return (logical_id & mda) != 0;
 776         case APIC_DFR_CLUSTER:
 777                 return ((logical_id >> 4) == (mda >> 4))
 778                        && (logical_id & mda & 0xf) != 0;
 779         default:
 780                 return false;
 781         }
 782 }
 783
 784 /* The KVM local APIC implementation has two quirks:
 785  *
 786  *  - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
 787  *    in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
 788  *    KVM doesn't do that aliasing.
 789  *
 790  *  - in-kernel IOAPIC messages have to be delivered directly to
 791  *    x2APIC, because the kernel does not support interrupt remapping.
 792  *    In order to support broadcast without interrupt remapping, x2APIC
 793  *    rewrites the destination of non-IPI messages from APIC_BROADCAST
 794  *    to X2APIC_BROADCAST.
 795  *
 796  * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
 797  * important when userspace wants to use x2APIC-format MSIs, because
 798  * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
 799  */
 800 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
 801                 struct kvm_lapic *source, struct kvm_lapic *target)
 802 {
 803         bool ipi = source != NULL;
 804
 805         if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
 806             !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
 807                 return X2APIC_BROADCAST;
 808
 809         return dest_id;
 810 }
 811
 812 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 813                            int shorthand, unsigned int dest, int dest_mode)
 814 {
 815         struct kvm_lapic *target = vcpu->arch.apic;
 816         u32 mda = kvm_apic_mda(vcpu, dest, source, target);
 817
 818         ASSERT(target);
 819         switch (shorthand) {
 820         case APIC_DEST_NOSHORT:
 821                 if (dest_mode == APIC_DEST_PHYSICAL)
 822                         return kvm_apic_match_physical_addr(target, mda);
 823                 else
 824                         return kvm_apic_match_logical_addr(target, mda);
 825         case APIC_DEST_SELF:
 826                 return target == source;
 827         case APIC_DEST_ALLINC:
 828                 return true;
 829         case APIC_DEST_ALLBUT:
 830                 return target != source;
 831         default:
 832                 return false;
 833         }
 834 }
 835 EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
 836
 837 int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
 838                        const unsigned long *bitmap, u32 bitmap_size)
 839 {
 840         u32 mod;
 841         int i, idx = -1;
 842
 843         mod = vector % dest_vcpus;
 844
 845         for (i = 0; i <= mod; i++) {
 846                 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
 847                 BUG_ON(idx == bitmap_size);
 848         }
 849
 850         return idx;
 851 }
 852
 853 static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
 854 {
 855         if (!kvm->arch.disabled_lapic_found) {
 856                 kvm->arch.disabled_lapic_found = true;
 857                 printk(KERN_INFO
 858                        "Disabled LAPIC found during irq injection\n");
 859         }
 860 }
 861
 862 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
 863                 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
 864 {
 865         if (kvm->arch.x2apic_broadcast_quirk_disabled) {
 866                 if ((irq->dest_id == APIC_BROADCAST &&
 867                                 map->mode != KVM_APIC_MODE_X2APIC))
 868                         return true;
 869                 if (irq->dest_id == X2APIC_BROADCAST)
 870                         return true;
 871         } else {
 872                 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
 873                 if (irq->dest_id == (x2apic_ipi ?
 874                                      X2APIC_BROADCAST : APIC_BROADCAST))
 875                         return true;
 876         }
 877
 878         return false;
 879 }
 880
 881 /* Return true if the interrupt can be handled by using *bitmap as index mask
 882  * for valid destinations in *dst array.
 883  * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
 884  * Note: we may have zero kvm_lapic destinations when we return true, which
 885  * means that the interrupt should be dropped.  In this case, *bitmap would be
 886  * zero and *dst undefined.
 887  */
 888 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
 889                 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
 890                 struct kvm_apic_map *map, struct kvm_lapic ***dst,
 891                 unsigned long *bitmap)
 892 {
 893         int i, lowest;
 894
 895         if (irq->shorthand == APIC_DEST_SELF && src) {
 896                 *dst = src;
 897                 *bitmap = 1;
 898                 return true;
 899         } else if (irq->shorthand)
 900                 return false;
 901
 902         if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
 903                 return false;
 904
 905         if (irq->dest_mode == APIC_DEST_PHYSICAL) {
 906                 if (irq->dest_id > map->max_apic_id) {
 907                         *bitmap = 0;
 908                 } else {
 909                         u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
 910                         *dst = &map->phys_map[dest_id];
 911                         *bitmap = 1;
 912                 }
 913                 return true;
 914         }
 915
 916         *bitmap = 0;
 917         if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
 918                                 (u16 *)bitmap))
 919                 return false;
 920
 921         if (!kvm_lowest_prio_delivery(irq))
 922                 return true;
 923
 924         if (!kvm_vector_hashing_enabled()) {
 925                 lowest = -1;
 926                 for_each_set_bit(i, bitmap, 16) {
 927                         if (!(*dst)[i])
 928                                 continue;
 929                         if (lowest < 0)
 930                                 lowest = i;
 931                         else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
 932                                                 (*dst)[lowest]->vcpu) < 0)
 933                                 lowest = i;
 934                 }
 935         } else {
 936                 if (!*bitmap)
 937                         return true;
 938
 939                 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
 940                                 bitmap, 16);
 941
 942                 if (!(*dst)[lowest]) {
 943                         kvm_apic_disabled_lapic_found(kvm);
 944                         *bitmap = 0;
 945                         return true;
 946                 }
 947         }
 948
 949         *bitmap = (lowest >= 0) ? 1 << lowest : 0;
 950
 951         return true;
 952 }
 953
 954 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
 955                 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
 956 {
 957         struct kvm_apic_map *map;
 958         unsigned long bitmap;
 959         struct kvm_lapic **dst = NULL;
 960         int i;
 961         bool ret;
 962
 963         *r = -1;
 964
 965         if (irq->shorthand == APIC_DEST_SELF) {
 966                 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
 967                 return true;
 968         }
 969
 970         rcu_read_lock();
 971         map = rcu_dereference(kvm->arch.apic_map);
 972
 973         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
 974         if (ret) {
 975                 *r = 0;
 976                 for_each_set_bit(i, &bitmap, 16) {
 977                         if (!dst[i])
 978                                 continue;
 979                         *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
 980                 }
 981         }
 982
 983         rcu_read_unlock();
 984         return ret;
 985 }
 986
 987 /*
 988  * This routine tries to handle interrupts in posted mode, here is how
 989  * it deals with different cases:
 990  * - For single-destination interrupts, handle it in posted mode
 991  * - Else if vector hashing is enabled and it is a lowest-priority
 992  *   interrupt, handle it in posted mode and use the following mechanism
 993  *   to find the destination vCPU.
 994  *      1. For lowest-priority interrupts, store all the possible
 995  *         destination vCPUs in an array.
 996  *      2. Use "guest vector % max number of destination vCPUs" to find
 997  *         the right destination vCPU in the array for the lowest-priority
 998  *         interrupt.
 999  * - Otherwise, use remapped mode to inject the interrupt.
1000  */
1001 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
1002                         struct kvm_vcpu **dest_vcpu)
1003 {
1004         struct kvm_apic_map *map;
1005         unsigned long bitmap;
1006         struct kvm_lapic **dst = NULL;
1007         bool ret = false;
1008
1009         if (irq->shorthand)
1010                 return false;
1011
1012         rcu_read_lock();
1013         map = rcu_dereference(kvm->arch.apic_map);
1014
1015         if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
1016                         hweight16(bitmap) == 1) {
1017                 unsigned long i = find_first_bit(&bitmap, 16);
1018
1019                 if (dst[i]) {
1020                         *dest_vcpu = dst[i]->vcpu;
1021                         ret = true;
1022                 }
1023         }
1024
1025         rcu_read_unlock();
1026         return ret;
1027 }
1028
1029 /*
1030  * Add a pending IRQ into lapic.
1031  * Return 1 if successfully added and 0 if discarded.
1032  */
1033 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
1034                              int vector, int level, int trig_mode,
1035                              struct dest_map *dest_map)
1036 {
1037         int result = 0;
1038         struct kvm_vcpu *vcpu = apic->vcpu;
1039
1040         trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
1041                                   trig_mode, vector);
1042         switch (delivery_mode) {
1043         case APIC_DM_LOWEST:
1044                 vcpu->arch.apic_arb_prio++;
1045                 /* fall through */
1046         case APIC_DM_FIXED:
1047                 if (unlikely(trig_mode && !level))
1048                         break;
1049
1050                 /* FIXME add logic for vcpu on reset */
1051                 if (unlikely(!apic_enabled(apic)))
1052                         break;
1053
1054                 result = 1;
1055
1056                 if (dest_map) {
1057                         __set_bit(vcpu->vcpu_id, dest_map->map);
1058                         dest_map->vectors[vcpu->vcpu_id] = vector;
1059                 }
1060
1061                 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
1062                         if (trig_mode)
1063                                 kvm_lapic_set_vector(vector,
1064                                                      apic->regs + APIC_TMR);
1065                         else
1066                                 kvm_lapic_clear_vector(vector,
1067                                                        apic->regs + APIC_TMR);
1068                 }
1069
1070                 if (kvm_x86_ops.deliver_posted_interrupt(vcpu, vector)) {
1071                         kvm_lapic_set_irr(vector, apic);
1072                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1073                         kvm_vcpu_kick(vcpu);
1074                 }
1075                 break;
1076
1077         case APIC_DM_REMRD:
1078                 result = 1;
1079                 vcpu->arch.pv.pv_unhalted = 1;
1080                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1081                 kvm_vcpu_kick(vcpu);
1082                 break;
1083
1084         case APIC_DM_SMI:
1085                 result = 1;
1086                 kvm_make_request(KVM_REQ_SMI, vcpu);
1087                 kvm_vcpu_kick(vcpu);
1088                 break;
1089
1090         case APIC_DM_NMI:
1091                 result = 1;
1092                 kvm_inject_nmi(vcpu);
1093                 kvm_vcpu_kick(vcpu);
1094                 break;
1095
1096         case APIC_DM_INIT:
1097                 if (!trig_mode || level) {
1098                         result = 1;
1099                         /* assumes that there are only KVM_APIC_INIT/SIPI */
1100                         apic->pending_events = (1UL << KVM_APIC_INIT);
1101                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1102                         kvm_vcpu_kick(vcpu);
1103                 }
1104                 break;
1105
1106         case APIC_DM_STARTUP:
1107                 result = 1;
1108                 apic->sipi_vector = vector;
1109                 /* make sure sipi_vector is visible for the receiver */
1110                 smp_wmb();
1111                 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1112                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1113                 kvm_vcpu_kick(vcpu);
1114                 break;
1115
1116         case APIC_DM_EXTINT:
1117                 /*
1118                  * Should only be called by kvm_apic_local_deliver() with LVT0,
1119                  * before NMI watchdog was enabled. Already handled by
1120                  * kvm_apic_accept_pic_intr().
1121                  */
1122                 break;
1123
1124         default:
1125                 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1126                        delivery_mode);
1127                 break;
1128         }
1129         return result;
1130 }
1131
1132 /*
1133  * This routine identifies the destination vcpus mask meant to receive the
1134  * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find
1135  * out the destination vcpus array and set the bitmap or it traverses to
1136  * each available vcpu to identify the same.
1137  */
1138 void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq,
1139                               unsigned long *vcpu_bitmap)
1140 {
1141         struct kvm_lapic **dest_vcpu = NULL;
1142         struct kvm_lapic *src = NULL;
1143         struct kvm_apic_map *map;
1144         struct kvm_vcpu *vcpu;
1145         unsigned long bitmap;
1146         int i, vcpu_idx;
1147         bool ret;
1148
1149         rcu_read_lock();
1150         map = rcu_dereference(kvm->arch.apic_map);
1151
1152         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu,
1153                                           &bitmap);
1154         if (ret) {
1155                 for_each_set_bit(i, &bitmap, 16) {
1156                         if (!dest_vcpu[i])
1157                                 continue;
1158                         vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx;
1159                         __set_bit(vcpu_idx, vcpu_bitmap);
1160                 }
1161         } else {
1162                 kvm_for_each_vcpu(i, vcpu, kvm) {
1163                         if (!kvm_apic_present(vcpu))
1164                                 continue;
1165                         if (!kvm_apic_match_dest(vcpu, NULL,
1166                                                  irq->shorthand,
1167                                                  irq->dest_id,
1168                                                  irq->dest_mode))
1169                                 continue;
1170                         __set_bit(i, vcpu_bitmap);
1171                 }
1172         }
1173         rcu_read_unlock();
1174 }
1175
1176 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1177 {
1178         return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1179 }
1180
1181 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1182 {
1183         return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1184 }
1185
1186 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1187 {
1188         int trigger_mode;
1189
1190         /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1191         if (!kvm_ioapic_handles_vector(apic, vector))
1192                 return;
1193
1194         /* Request a KVM exit to inform the userspace IOAPIC. */
1195         if (irqchip_split(apic->vcpu->kvm)) {
1196                 apic->vcpu->arch.pending_ioapic_eoi = vector;
1197                 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1198                 return;
1199         }
1200
1201         if (apic_test_vector(vector, apic->regs + APIC_TMR))
1202                 trigger_mode = IOAPIC_LEVEL_TRIG;
1203         else
1204                 trigger_mode = IOAPIC_EDGE_TRIG;
1205
1206         kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1207 }
1208
1209 static int apic_set_eoi(struct kvm_lapic *apic)
1210 {
1211         int vector = apic_find_highest_isr(apic);
1212
1213         trace_kvm_eoi(apic, vector);
1214
1215         /*
1216          * Not every write EOI will has corresponding ISR,
1217          * one example is when Kernel check timer on setup_IO_APIC
1218          */
1219         if (vector == -1)
1220                 return vector;
1221
1222         apic_clear_isr(vector, apic);
1223         apic_update_ppr(apic);
1224
1225         if (test_bit(vector, vcpu_to_synic(apic->vcpu)->vec_bitmap))
1226                 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1227
1228         kvm_ioapic_send_eoi(apic, vector);
1229         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1230         return vector;
1231 }
1232
1233 /*
1234  * this interface assumes a trap-like exit, which has already finished
1235  * desired side effect including vISR and vPPR update.
1236  */
1237 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1238 {
1239         struct kvm_lapic *apic = vcpu->arch.apic;
1240
1241         trace_kvm_eoi(apic, vector);
1242
1243         kvm_ioapic_send_eoi(apic, vector);
1244         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1245 }
1246 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1247
1248 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
1249 {
1250         struct kvm_lapic_irq irq;
1251
1252         irq.vector = icr_low & APIC_VECTOR_MASK;
1253         irq.delivery_mode = icr_low & APIC_MODE_MASK;
1254         irq.dest_mode = icr_low & APIC_DEST_MASK;
1255         irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1256         irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1257         irq.shorthand = icr_low & APIC_SHORT_MASK;
1258         irq.msi_redir_hint = false;
1259         if (apic_x2apic_mode(apic))
1260                 irq.dest_id = icr_high;
1261         else
1262                 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
1263
1264         trace_kvm_apic_ipi(icr_low, irq.dest_id);
1265
1266         kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
1267 }
1268
1269 static u32 apic_get_tmcct(struct kvm_lapic *apic)
1270 {
1271         ktime_t remaining, now;
1272         s64 ns;
1273         u32 tmcct;
1274
1275         ASSERT(apic != NULL);
1276
1277         /* if initial count is 0, current count should also be 0 */
1278         if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1279                 apic->lapic_timer.period == 0)
1280                 return 0;
1281
1282         now = ktime_get();
1283         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1284         if (ktime_to_ns(remaining) < 0)
1285                 remaining = 0;
1286
1287         ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1288         tmcct = div64_u64(ns,
1289                          (APIC_BUS_CYCLE_NS * apic->divide_count));
1290
1291         return tmcct;
1292 }
1293
1294 static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1295 {
1296         struct kvm_vcpu *vcpu = apic->vcpu;
1297         struct kvm_run *run = vcpu->run;
1298
1299         kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1300         run->tpr_access.rip = kvm_rip_read(vcpu);
1301         run->tpr_access.is_write = write;
1302 }
1303
1304 static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1305 {
1306         if (apic->vcpu->arch.tpr_access_reporting)
1307                 __report_tpr_access(apic, write);
1308 }
1309
1310 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1311 {
1312         u32 val = 0;
1313
1314         if (offset >= LAPIC_MMIO_LENGTH)
1315                 return 0;
1316
1317         switch (offset) {
1318         case APIC_ARBPRI:
1319                 break;
1320
1321         case APIC_TMCCT:        /* Timer CCR */
1322                 if (apic_lvtt_tscdeadline(apic))
1323                         return 0;
1324
1325                 val = apic_get_tmcct(apic);
1326                 break;
1327         case APIC_PROCPRI:
1328                 apic_update_ppr(apic);
1329                 val = kvm_lapic_get_reg(apic, offset);
1330                 break;
1331         case APIC_TASKPRI:
1332                 report_tpr_access(apic, false);
1333                 /* fall thru */
1334         default:
1335                 val = kvm_lapic_get_reg(apic, offset);
1336                 break;
1337         }
1338
1339         return val;
1340 }
1341
1342 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1343 {
1344         return container_of(dev, struct kvm_lapic, dev);
1345 }
1346
1347 #define APIC_REG_MASK(reg)      (1ull << ((reg) >> 4))
1348 #define APIC_REGS_MASK(first, count) \
1349         (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1350
1351 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
1352                 void *data)
1353 {
1354         unsigned char alignment = offset & 0xf;
1355         u32 result;
1356         /* this bitmask has a bit cleared for each reserved register */
1357         u64 valid_reg_mask =
1358                 APIC_REG_MASK(APIC_ID) |
1359                 APIC_REG_MASK(APIC_LVR) |
1360                 APIC_REG_MASK(APIC_TASKPRI) |
1361                 APIC_REG_MASK(APIC_PROCPRI) |
1362                 APIC_REG_MASK(APIC_LDR) |
1363                 APIC_REG_MASK(APIC_DFR) |
1364                 APIC_REG_MASK(APIC_SPIV) |
1365                 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
1366                 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
1367                 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
1368                 APIC_REG_MASK(APIC_ESR) |
1369                 APIC_REG_MASK(APIC_ICR) |
1370                 APIC_REG_MASK(APIC_ICR2) |
1371                 APIC_REG_MASK(APIC_LVTT) |
1372                 APIC_REG_MASK(APIC_LVTTHMR) |
1373                 APIC_REG_MASK(APIC_LVTPC) |
1374                 APIC_REG_MASK(APIC_LVT0) |
1375                 APIC_REG_MASK(APIC_LVT1) |
1376                 APIC_REG_MASK(APIC_LVTERR) |
1377                 APIC_REG_MASK(APIC_TMICT) |
1378                 APIC_REG_MASK(APIC_TMCCT) |
1379                 APIC_REG_MASK(APIC_TDCR);
1380
1381         /* ARBPRI is not valid on x2APIC */
1382         if (!apic_x2apic_mode(apic))
1383                 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI);
1384
1385         if (offset > 0x3f0 || !(valid_reg_mask & APIC_REG_MASK(offset)))
1386                 return 1;
1387
1388         result = __apic_read(apic, offset & ~0xf);
1389
1390         trace_kvm_apic_read(offset, result);
1391
1392         switch (len) {
1393         case 1:
1394         case 2:
1395         case 4:
1396                 memcpy(data, (char *)&result + alignment, len);
1397                 break;
1398         default:
1399                 printk(KERN_ERR "Local APIC read with len = %x, "
1400                        "should be 1,2, or 4 instead\n", len);
1401                 break;
1402         }
1403         return 0;
1404 }
1405 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
1406
1407 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1408 {
1409         return addr >= apic->base_address &&
1410                 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1411 }
1412
1413 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1414                            gpa_t address, int len, void *data)
1415 {
1416         struct kvm_lapic *apic = to_lapic(this);
1417         u32 offset = address - apic->base_address;
1418
1419         if (!apic_mmio_in_range(apic, address))
1420                 return -EOPNOTSUPP;
1421
1422         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1423                 if (!kvm_check_has_quirk(vcpu->kvm,
1424                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1425                         return -EOPNOTSUPP;
1426
1427                 memset(data, 0xff, len);
1428                 return 0;
1429         }
1430
1431         kvm_lapic_reg_read(apic, offset, len, data);
1432
1433         return 0;
1434 }
1435
1436 static void update_divide_count(struct kvm_lapic *apic)
1437 {
1438         u32 tmp1, tmp2, tdcr;
1439
1440         tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
1441         tmp1 = tdcr & 0xf;
1442         tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1443         apic->divide_count = 0x1 << (tmp2 & 0x7);
1444 }
1445
1446 static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1447 {
1448         /*
1449          * Do not allow the guest to program periodic timers with small
1450          * interval, since the hrtimers are not throttled by the host
1451          * scheduler.
1452          */
1453         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1454                 s64 min_period = min_timer_period_us * 1000LL;
1455
1456                 if (apic->lapic_timer.period < min_period) {
1457                         pr_info_ratelimited(
1458                             "kvm: vcpu %i: requested %lld ns "
1459                             "lapic timer period limited to %lld ns\n",
1460                             apic->vcpu->vcpu_id,
1461                             apic->lapic_timer.period, min_period);
1462                         apic->lapic_timer.period = min_period;
1463                 }
1464         }
1465 }
1466
1467 static void cancel_hv_timer(struct kvm_lapic *apic);
1468
1469 static void apic_update_lvtt(struct kvm_lapic *apic)
1470 {
1471         u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1472                         apic->lapic_timer.timer_mode_mask;
1473
1474         if (apic->lapic_timer.timer_mode != timer_mode) {
1475                 if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1476                                 APIC_LVT_TIMER_TSCDEADLINE)) {
1477                         hrtimer_cancel(&apic->lapic_timer.timer);
1478                         preempt_disable();
1479                         if (apic->lapic_timer.hv_timer_in_use)
1480                                 cancel_hv_timer(apic);
1481                         preempt_enable();
1482                         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
1483                         apic->lapic_timer.period = 0;
1484                         apic->lapic_timer.tscdeadline = 0;
1485                 }
1486                 apic->lapic_timer.timer_mode = timer_mode;
1487                 limit_periodic_timer_frequency(apic);
1488         }
1489 }
1490
1491 /*
1492  * On APICv, this test will cause a busy wait
1493  * during a higher-priority task.
1494  */
1495
1496 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1497 {
1498         struct kvm_lapic *apic = vcpu->arch.apic;
1499         u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1500
1501         if (kvm_apic_hw_enabled(apic)) {
1502                 int vec = reg & APIC_VECTOR_MASK;
1503                 void *bitmap = apic->regs + APIC_ISR;
1504
1505                 if (vcpu->arch.apicv_active)
1506                         bitmap = apic->regs + APIC_IRR;
1507
1508                 if (apic_test_vector(vec, bitmap))
1509                         return true;
1510         }
1511         return false;
1512 }
1513
1514 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
1515 {
1516         u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
1517
1518         /*
1519          * If the guest TSC is running at a different ratio than the host, then
1520          * convert the delay to nanoseconds to achieve an accurate delay.  Note
1521          * that __delay() uses delay_tsc whenever the hardware has TSC, thus
1522          * always for VMX enabled hardware.
1523          */
1524         if (vcpu->arch.tsc_scaling_ratio == kvm_default_tsc_scaling_ratio) {
1525                 __delay(min(guest_cycles,
1526                         nsec_to_cycles(vcpu, timer_advance_ns)));
1527         } else {
1528                 u64 delay_ns = guest_cycles * 1000000ULL;
1529                 do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
1530                 ndelay(min_t(u32, delay_ns, timer_advance_ns));
1531         }
1532 }
1533
1534 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
1535                                               s64 advance_expire_delta)
1536 {
1537         struct kvm_lapic *apic = vcpu->arch.apic;
1538         u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
1539         u64 ns;
1540
1541         /* Do not adjust for tiny fluctuations or large random spikes. */
1542         if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX ||
1543             abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN)
1544                 return;
1545
1546         /* too early */
1547         if (advance_expire_delta < 0) {
1548                 ns = -advance_expire_delta * 1000000ULL;
1549                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1550                 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1551         } else {
1552         /* too late */
1553                 ns = advance_expire_delta * 1000000ULL;
1554                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1555                 timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1556         }
1557
1558         if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX))
1559                 timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
1560         apic->lapic_timer.timer_advance_ns = timer_advance_ns;
1561 }
1562
1563 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1564 {
1565         struct kvm_lapic *apic = vcpu->arch.apic;
1566         u64 guest_tsc, tsc_deadline;
1567
1568         if (apic->lapic_timer.expired_tscdeadline == 0)
1569                 return;
1570
1571         tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1572         apic->lapic_timer.expired_tscdeadline = 0;
1573         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1574         apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline;
1575
1576         if (guest_tsc < tsc_deadline)
1577                 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
1578
1579         if (lapic_timer_advance_dynamic)
1580                 adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
1581 }
1582
1583 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1584 {
1585         if (lapic_timer_int_injected(vcpu))
1586                 __kvm_wait_lapic_expire(vcpu);
1587 }
1588 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
1589
1590 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
1591 {
1592         struct kvm_timer *ktimer = &apic->lapic_timer;
1593
1594         kvm_apic_local_deliver(apic, APIC_LVTT);
1595         if (apic_lvtt_tscdeadline(apic)) {
1596                 ktimer->tscdeadline = 0;
1597         } else if (apic_lvtt_oneshot(apic)) {
1598                 ktimer->tscdeadline = 0;
1599                 ktimer->target_expiration = 0;
1600         }
1601 }
1602
1603 static void apic_timer_expired(struct kvm_lapic *apic)
1604 {
1605         struct kvm_vcpu *vcpu = apic->vcpu;
1606         struct kvm_timer *ktimer = &apic->lapic_timer;
1607
1608         if (atomic_read(&apic->lapic_timer.pending))
1609                 return;
1610
1611         if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
1612                 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1613
1614         if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
1615                 if (apic->lapic_timer.timer_advance_ns)
1616                         __kvm_wait_lapic_expire(vcpu);
1617                 kvm_apic_inject_pending_timer_irqs(apic);
1618                 return;
1619         }
1620
1621         atomic_inc(&apic->lapic_timer.pending);
1622         kvm_set_pending_timer(vcpu);
1623 }
1624
1625 static void start_sw_tscdeadline(struct kvm_lapic *apic)
1626 {
1627         struct kvm_timer *ktimer = &apic->lapic_timer;
1628         u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
1629         u64 ns = 0;
1630         ktime_t expire;
1631         struct kvm_vcpu *vcpu = apic->vcpu;
1632         unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1633         unsigned long flags;
1634         ktime_t now;
1635
1636         if (unlikely(!tscdeadline || !this_tsc_khz))
1637                 return;
1638
1639         local_irq_save(flags);
1640
1641         now = ktime_get();
1642         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1643
1644         ns = (tscdeadline - guest_tsc) * 1000000ULL;
1645         do_div(ns, this_tsc_khz);
1646
1647         if (likely(tscdeadline > guest_tsc) &&
1648             likely(ns > apic->lapic_timer.timer_advance_ns)) {
1649                 expire = ktime_add_ns(now, ns);
1650                 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
1651                 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
1652         } else
1653                 apic_timer_expired(apic);
1654
1655         local_irq_restore(flags);
1656 }
1657
1658 static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict)
1659 {
1660         return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count;
1661 }
1662
1663 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1664 {
1665         ktime_t now, remaining;
1666         u64 ns_remaining_old, ns_remaining_new;
1667
1668         apic->lapic_timer.period =
1669                         tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
1670         limit_periodic_timer_frequency(apic);
1671
1672         now = ktime_get();
1673         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1674         if (ktime_to_ns(remaining) < 0)
1675                 remaining = 0;
1676
1677         ns_remaining_old = ktime_to_ns(remaining);
1678         ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1679                                            apic->divide_count, old_divisor);
1680
1681         apic->lapic_timer.tscdeadline +=
1682                 nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1683                 nsec_to_cycles(apic->vcpu, ns_remaining_old);
1684         apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1685 }
1686
1687 static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg)
1688 {
1689         ktime_t now;
1690         u64 tscl = rdtsc();
1691         s64 deadline;
1692
1693         now = ktime_get();
1694         apic->lapic_timer.period =
1695                         tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
1696
1697         if (!apic->lapic_timer.period) {
1698                 apic->lapic_timer.tscdeadline = 0;
1699                 return false;
1700         }
1701
1702         limit_periodic_timer_frequency(apic);
1703         deadline = apic->lapic_timer.period;
1704
1705         if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
1706                 if (unlikely(count_reg != APIC_TMICT)) {
1707                         deadline = tmict_to_ns(apic,
1708                                      kvm_lapic_get_reg(apic, count_reg));
1709                         if (unlikely(deadline <= 0))
1710                                 deadline = apic->lapic_timer.period;
1711                         else if (unlikely(deadline > apic->lapic_timer.period)) {
1712                                 pr_info_ratelimited(
1713                                     "kvm: vcpu %i: requested lapic timer restore with "
1714                                     "starting count register %#x=%u (%lld ns) > initial count (%lld ns). "
1715                                     "Using initial count to start timer.\n",
1716                                     apic->vcpu->vcpu_id,
1717                                     count_reg,
1718                                     kvm_lapic_get_reg(apic, count_reg),
1719                                     deadline, apic->lapic_timer.period);
1720                                 kvm_lapic_set_reg(apic, count_reg, 0);
1721                                 deadline = apic->lapic_timer.period;
1722                         }
1723                 }
1724         }
1725
1726         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1727                 nsec_to_cycles(apic->vcpu, deadline);
1728         apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline);
1729
1730         return true;
1731 }
1732
1733 static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1734 {
1735         ktime_t now = ktime_get();
1736         u64 tscl = rdtsc();
1737         ktime_t delta;
1738
1739         /*
1740          * Synchronize both deadlines to the same time source or
1741          * differences in the periods (caused by differences in the
1742          * underlying clocks or numerical approximation errors) will
1743          * cause the two to drift apart over time as the errors
1744          * accumulate.
1745          */
1746         apic->lapic_timer.target_expiration =
1747                 ktime_add_ns(apic->lapic_timer.target_expiration,
1748                                 apic->lapic_timer.period);
1749         delta = ktime_sub(apic->lapic_timer.target_expiration, now);
1750         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1751                 nsec_to_cycles(apic->vcpu, delta);
1752 }
1753
1754 static void start_sw_period(struct kvm_lapic *apic)
1755 {
1756         if (!apic->lapic_timer.period)
1757                 return;
1758
1759         if (ktime_after(ktime_get(),
1760                         apic->lapic_timer.target_expiration)) {
1761                 apic_timer_expired(apic);
1762
1763                 if (apic_lvtt_oneshot(apic))
1764                         return;
1765
1766                 advance_periodic_target_expiration(apic);
1767         }
1768
1769         hrtimer_start(&apic->lapic_timer.timer,
1770                 apic->lapic_timer.target_expiration,
1771                 HRTIMER_MODE_ABS_HARD);
1772 }
1773
1774 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1775 {
1776         if (!lapic_in_kernel(vcpu))
1777                 return false;
1778
1779         return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1780 }
1781 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1782
1783 static void cancel_hv_timer(struct kvm_lapic *apic)
1784 {
1785         WARN_ON(preemptible());
1786         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1787         kvm_x86_ops.cancel_hv_timer(apic->vcpu);
1788         apic->lapic_timer.hv_timer_in_use = false;
1789 }
1790
1791 static bool start_hv_timer(struct kvm_lapic *apic)
1792 {
1793         struct kvm_timer *ktimer = &apic->lapic_timer;
1794         struct kvm_vcpu *vcpu = apic->vcpu;
1795         bool expired;
1796
1797         WARN_ON(preemptible());
1798         if (!kvm_can_use_hv_timer(vcpu))
1799                 return false;
1800
1801         if (!ktimer->tscdeadline)
1802                 return false;
1803
1804         if (kvm_x86_ops.set_hv_timer(vcpu, ktimer->tscdeadline, &expired))
1805                 return false;
1806
1807         ktimer->hv_timer_in_use = true;
1808         hrtimer_cancel(&ktimer->timer);
1809
1810         /*
1811          * To simplify handling the periodic timer, leave the hv timer running
1812          * even if the deadline timer has expired, i.e. rely on the resulting
1813          * VM-Exit to recompute the periodic timer's target expiration.
1814          */
1815         if (!apic_lvtt_period(apic)) {
1816                 /*
1817                  * Cancel the hv timer if the sw timer fired while the hv timer
1818                  * was being programmed, or if the hv timer itself expired.
1819                  */
1820                 if (atomic_read(&ktimer->pending)) {
1821                         cancel_hv_timer(apic);
1822                 } else if (expired) {
1823                         apic_timer_expired(apic);
1824                         cancel_hv_timer(apic);
1825                 }
1826         }
1827
1828         trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
1829
1830         return true;
1831 }
1832
1833 static void start_sw_timer(struct kvm_lapic *apic)
1834 {
1835         struct kvm_timer *ktimer = &apic->lapic_timer;
1836
1837         WARN_ON(preemptible());
1838         if (apic->lapic_timer.hv_timer_in_use)
1839                 cancel_hv_timer(apic);
1840         if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1841                 return;
1842
1843         if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1844                 start_sw_period(apic);
1845         else if (apic_lvtt_tscdeadline(apic))
1846                 start_sw_tscdeadline(apic);
1847         trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
1848 }
1849
1850 static void restart_apic_timer(struct kvm_lapic *apic)
1851 {
1852         preempt_disable();
1853
1854         if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
1855                 goto out;
1856
1857         if (!start_hv_timer(apic))
1858                 start_sw_timer(apic);
1859 out:
1860         preempt_enable();
1861 }
1862
1863 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1864 {
1865         struct kvm_lapic *apic = vcpu->arch.apic;
1866
1867         preempt_disable();
1868         /* If the preempt notifier has already run, it also called apic_timer_expired */
1869         if (!apic->lapic_timer.hv_timer_in_use)
1870                 goto out;
1871         WARN_ON(rcuwait_active(&vcpu->wait));
1872         cancel_hv_timer(apic);
1873         apic_timer_expired(apic);
1874
1875         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1876                 advance_periodic_target_expiration(apic);
1877                 restart_apic_timer(apic);
1878         }
1879 out:
1880         preempt_enable();
1881 }
1882 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1883
1884 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1885 {
1886         restart_apic_timer(vcpu->arch.apic);
1887 }
1888 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1889
1890 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1891 {
1892         struct kvm_lapic *apic = vcpu->arch.apic;
1893
1894         preempt_disable();
1895         /* Possibly the TSC deadline timer is not enabled yet */
1896         if (apic->lapic_timer.hv_timer_in_use)
1897                 start_sw_timer(apic);
1898         preempt_enable();
1899 }
1900 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1901
1902 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
1903 {
1904         struct kvm_lapic *apic = vcpu->arch.apic;
1905
1906         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1907         restart_apic_timer(apic);
1908 }
1909
1910 static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg)
1911 {
1912         atomic_set(&apic->lapic_timer.pending, 0);
1913
1914         if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1915             && !set_target_expiration(apic, count_reg))
1916                 return;
1917
1918         restart_apic_timer(apic);
1919 }
1920
1921 static void start_apic_timer(struct kvm_lapic *apic)
1922 {
1923         __start_apic_timer(apic, APIC_TMICT);
1924 }
1925
1926 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1927 {
1928         bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
1929
1930         if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1931                 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1932                 if (lvt0_in_nmi_mode) {
1933                         atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1934                 } else
1935                         atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1936         }
1937 }
1938
1939 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
1940 {
1941         int ret = 0;
1942
1943         trace_kvm_apic_write(reg, val);
1944
1945         switch (reg) {
1946         case APIC_ID:           /* Local APIC ID */
1947                 if (!apic_x2apic_mode(apic))
1948                         kvm_apic_set_xapic_id(apic, val >> 24);
1949                 else
1950                         ret = 1;
1951                 break;
1952
1953         case APIC_TASKPRI:
1954                 report_tpr_access(apic, true);
1955                 apic_set_tpr(apic, val & 0xff);
1956                 break;
1957
1958         case APIC_EOI:
1959                 apic_set_eoi(apic);
1960                 break;
1961
1962         case APIC_LDR:
1963                 if (!apic_x2apic_mode(apic))
1964                         kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
1965                 else
1966                         ret = 1;
1967                 break;
1968
1969         case APIC_DFR:
1970                 if (!apic_x2apic_mode(apic)) {
1971                         kvm_lapic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
1972                         apic->vcpu->kvm->arch.apic_map_dirty = true;
1973                 } else
1974                         ret = 1;
1975                 break;
1976
1977         case APIC_SPIV: {
1978                 u32 mask = 0x3ff;
1979                 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
1980                         mask |= APIC_SPIV_DIRECTED_EOI;
1981                 apic_set_spiv(apic, val & mask);
1982                 if (!(val & APIC_SPIV_APIC_ENABLED)) {
1983                         int i;
1984                         u32 lvt_val;
1985
1986                         for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
1987                                 lvt_val = kvm_lapic_get_reg(apic,
1988                                                        APIC_LVTT + 0x10 * i);
1989                                 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
1990                                              lvt_val | APIC_LVT_MASKED);
1991                         }
1992                         apic_update_lvtt(apic);
1993                         atomic_set(&apic->lapic_timer.pending, 0);
1994
1995                 }
1996                 break;
1997         }
1998         case APIC_ICR:
1999                 /* No delay here, so we always clear the pending bit */
2000                 val &= ~(1 << 12);
2001                 kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
2002                 kvm_lapic_set_reg(apic, APIC_ICR, val);
2003                 break;
2004
2005         case APIC_ICR2:
2006                 if (!apic_x2apic_mode(apic))
2007                         val &= 0xff000000;
2008                 kvm_lapic_set_reg(apic, APIC_ICR2, val);
2009                 break;
2010
2011         case APIC_LVT0:
2012                 apic_manage_nmi_watchdog(apic, val);
2013                 /* fall through */
2014         case APIC_LVTTHMR:
2015         case APIC_LVTPC:
2016         case APIC_LVT1:
2017         case APIC_LVTERR: {
2018                 /* TODO: Check vector */
2019                 size_t size;
2020                 u32 index;
2021
2022                 if (!kvm_apic_sw_enabled(apic))
2023                         val |= APIC_LVT_MASKED;
2024                 size = ARRAY_SIZE(apic_lvt_mask);
2025                 index = array_index_nospec(
2026                                 (reg - APIC_LVTT) >> 4, size);
2027                 val &= apic_lvt_mask[index];
2028                 kvm_lapic_set_reg(apic, reg, val);
2029                 break;
2030         }
2031
2032         case APIC_LVTT:
2033                 if (!kvm_apic_sw_enabled(apic))
2034                         val |= APIC_LVT_MASKED;
2035                 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
2036                 kvm_lapic_set_reg(apic, APIC_LVTT, val);
2037                 apic_update_lvtt(apic);
2038                 break;
2039
2040         case APIC_TMICT:
2041                 if (apic_lvtt_tscdeadline(apic))
2042                         break;
2043
2044                 hrtimer_cancel(&apic->lapic_timer.timer);
2045                 kvm_lapic_set_reg(apic, APIC_TMICT, val);
2046                 start_apic_timer(apic);
2047                 break;
2048
2049         case APIC_TDCR: {
2050                 uint32_t old_divisor = apic->divide_count;
2051
2052                 kvm_lapic_set_reg(apic, APIC_TDCR, val);
2053                 update_divide_count(apic);
2054                 if (apic->divide_count != old_divisor &&
2055                                 apic->lapic_timer.period) {
2056                         hrtimer_cancel(&apic->lapic_timer.timer);
2057                         update_target_expiration(apic, old_divisor);
2058                         restart_apic_timer(apic);
2059                 }
2060                 break;
2061         }
2062         case APIC_ESR:
2063                 if (apic_x2apic_mode(apic) && val != 0)
2064                         ret = 1;
2065                 break;
2066
2067         case APIC_SELF_IPI:
2068                 if (apic_x2apic_mode(apic)) {
2069                         kvm_lapic_reg_write(apic, APIC_ICR, 0x40000 | (val & 0xff));
2070                 } else
2071                         ret = 1;
2072                 break;
2073         default:
2074                 ret = 1;
2075                 break;
2076         }
2077
2078         kvm_recalculate_apic_map(apic->vcpu->kvm);
2079
2080         return ret;
2081 }
2082 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
2083
2084 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
2085                             gpa_t address, int len, const void *data)
2086 {
2087         struct kvm_lapic *apic = to_lapic(this);
2088         unsigned int offset = address - apic->base_address;
2089         u32 val;
2090
2091         if (!apic_mmio_in_range(apic, address))
2092                 return -EOPNOTSUPP;
2093
2094         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
2095                 if (!kvm_check_has_quirk(vcpu->kvm,
2096                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
2097                         return -EOPNOTSUPP;
2098
2099                 return 0;
2100         }
2101
2102         /*
2103          * APIC register must be aligned on 128-bits boundary.
2104          * 32/64/128 bits registers must be accessed thru 32 bits.
2105          * Refer SDM 8.4.1
2106          */
2107         if (len != 4 || (offset & 0xf))
2108                 return 0;
2109
2110         val = *(u32*)data;
2111
2112         kvm_lapic_reg_write(apic, offset & 0xff0, val);
2113
2114         return 0;
2115 }
2116
2117 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
2118 {
2119         kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
2120 }
2121 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
2122
2123 /* emulate APIC access in a trap manner */
2124 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
2125 {
2126         u32 val = 0;
2127
2128         /* hw has done the conditional check and inst decode */
2129         offset &= 0xff0;
2130
2131         kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
2132
2133         /* TODO: optimize to just emulate side effect w/o one more write */
2134         kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
2135 }
2136 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
2137
2138 void kvm_free_lapic(struct kvm_vcpu *vcpu)
2139 {
2140         struct kvm_lapic *apic = vcpu->arch.apic;
2141
2142         if (!vcpu->arch.apic)
2143                 return;
2144
2145         hrtimer_cancel(&apic->lapic_timer.timer);
2146
2147         if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2148                 static_key_slow_dec_deferred(&apic_hw_disabled);
2149
2150         if (!apic->sw_enabled)
2151                 static_key_slow_dec_deferred(&apic_sw_disabled);
2152
2153         if (apic->regs)
2154                 free_page((unsigned long)apic->regs);
2155
2156         kfree(apic);
2157 }
2158
2159 /*
2160  *----------------------------------------------------------------------
2161  * LAPIC interface
2162  *----------------------------------------------------------------------
2163  */
2164 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
2165 {
2166         struct kvm_lapic *apic = vcpu->arch.apic;
2167
2168         if (!lapic_in_kernel(vcpu) ||
2169                 !apic_lvtt_tscdeadline(apic))
2170                 return 0;
2171
2172         return apic->lapic_timer.tscdeadline;
2173 }
2174
2175 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
2176 {
2177         struct kvm_lapic *apic = vcpu->arch.apic;
2178
2179         if (!lapic_in_kernel(vcpu) || apic_lvtt_oneshot(apic) ||
2180                         apic_lvtt_period(apic))
2181                 return;
2182
2183         hrtimer_cancel(&apic->lapic_timer.timer);
2184         apic->lapic_timer.tscdeadline = data;
2185         start_apic_timer(apic);
2186 }
2187
2188 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
2189 {
2190         struct kvm_lapic *apic = vcpu->arch.apic;
2191
2192         apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
2193                      | (kvm_lapic_get_reg(apic, APIC_TASKPRI) & 4));
2194 }
2195
2196 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
2197 {
2198         u64 tpr;
2199
2200         tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
2201
2202         return (tpr & 0xf0) >> 4;
2203 }
2204
2205 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
2206 {
2207         u64 old_value = vcpu->arch.apic_base;
2208         struct kvm_lapic *apic = vcpu->arch.apic;
2209
2210         if (!apic)
2211                 value |= MSR_IA32_APICBASE_BSP;
2212
2213         vcpu->arch.apic_base = value;
2214
2215         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
2216                 kvm_update_cpuid(vcpu);
2217
2218         if (!apic)
2219                 return;
2220
2221         /* update jump label if enable bit changes */
2222         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2223                 if (value & MSR_IA32_APICBASE_ENABLE) {
2224                         kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2225                         static_key_slow_dec_deferred(&apic_hw_disabled);
2226                 } else {
2227                         static_key_slow_inc(&apic_hw_disabled.key);
2228                         vcpu->kvm->arch.apic_map_dirty = true;
2229                 }
2230         }
2231
2232         if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
2233                 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2234
2235         if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
2236                 kvm_x86_ops.set_virtual_apic_mode(vcpu);
2237
2238         apic->base_address = apic->vcpu->arch.apic_base &
2239                              MSR_IA32_APICBASE_BASE;
2240
2241         if ((value & MSR_IA32_APICBASE_ENABLE) &&
2242              apic->base_address != APIC_DEFAULT_PHYS_BASE)
2243                 pr_warn_once("APIC base relocation is unsupported by KVM");
2244 }
2245
2246 void kvm_apic_update_apicv(struct kvm_vcpu *vcpu)
2247 {
2248         struct kvm_lapic *apic = vcpu->arch.apic;
2249
2250         if (vcpu->arch.apicv_active) {
2251                 /* irr_pending is always true when apicv is activated. */
2252                 apic->irr_pending = true;
2253                 apic->isr_count = 1;
2254         } else {
2255                 apic->irr_pending = (apic_search_irr(apic) != -1);
2256                 apic->isr_count = count_vectors(apic->regs + APIC_ISR);
2257         }
2258 }
2259 EXPORT_SYMBOL_GPL(kvm_apic_update_apicv);
2260
2261 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
2262 {
2263         struct kvm_lapic *apic = vcpu->arch.apic;
2264         int i;
2265
2266         if (!apic)
2267                 return;
2268
2269         vcpu->kvm->arch.apic_map_dirty = false;
2270         /* Stop the timer in case it's a reset to an active apic */
2271         hrtimer_cancel(&apic->lapic_timer.timer);
2272
2273         if (!init_event) {
2274                 kvm_lapic_set_base(vcpu, APIC_DEFAULT_PHYS_BASE |
2275                                          MSR_IA32_APICBASE_ENABLE);
2276                 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2277         }
2278         kvm_apic_set_version(apic->vcpu);
2279
2280         for (i = 0; i < KVM_APIC_LVT_NUM; i++)
2281                 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
2282         apic_update_lvtt(apic);
2283         if (kvm_vcpu_is_reset_bsp(vcpu) &&
2284             kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2285                 kvm_lapic_set_reg(apic, APIC_LVT0,
2286                              SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2287         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2288
2289         kvm_lapic_set_reg(apic, APIC_DFR, 0xffffffffU);
2290         apic_set_spiv(apic, 0xff);
2291         kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2292         if (!apic_x2apic_mode(apic))
2293                 kvm_apic_set_ldr(apic, 0);
2294         kvm_lapic_set_reg(apic, APIC_ESR, 0);
2295         kvm_lapic_set_reg(apic, APIC_ICR, 0);
2296         kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2297         kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2298         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
2299         for (i = 0; i < 8; i++) {
2300                 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2301                 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2302                 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
2303         }
2304         kvm_apic_update_apicv(vcpu);
2305         apic->highest_isr_cache = -1;
2306         update_divide_count(apic);
2307         atomic_set(&apic->lapic_timer.pending, 0);
2308         if (kvm_vcpu_is_bsp(vcpu))
2309                 kvm_lapic_set_base(vcpu,
2310                                 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
2311         vcpu->arch.pv_eoi.msr_val = 0;
2312         apic_update_ppr(apic);
2313         if (vcpu->arch.apicv_active) {
2314                 kvm_x86_ops.apicv_post_state_restore(vcpu);
2315                 kvm_x86_ops.hwapic_irr_update(vcpu, -1);
2316                 kvm_x86_ops.hwapic_isr_update(vcpu, -1);
2317         }
2318
2319         vcpu->arch.apic_arb_prio = 0;
2320         vcpu->arch.apic_attention = 0;
2321
2322         kvm_recalculate_apic_map(vcpu->kvm);
2323 }
2324
2325 /*
2326  *----------------------------------------------------------------------
2327  * timer interface
2328  *----------------------------------------------------------------------
2329  */
2330
2331 static bool lapic_is_periodic(struct kvm_lapic *apic)
2332 {
2333         return apic_lvtt_period(apic);
2334 }
2335
2336 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2337 {
2338         struct kvm_lapic *apic = vcpu->arch.apic;
2339
2340         if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2341                 return atomic_read(&apic->lapic_timer.pending);
2342
2343         return 0;
2344 }
2345
2346 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2347 {
2348         u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2349         int vector, mode, trig_mode;
2350
2351         if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2352                 vector = reg & APIC_VECTOR_MASK;
2353                 mode = reg & APIC_MODE_MASK;
2354                 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2355                 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2356                                         NULL);
2357         }
2358         return 0;
2359 }
2360
2361 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2362 {
2363         struct kvm_lapic *apic = vcpu->arch.apic;
2364
2365         if (apic)
2366                 kvm_apic_local_deliver(apic, APIC_LVT0);
2367 }
2368
2369 static const struct kvm_io_device_ops apic_mmio_ops = {
2370         .read     = apic_mmio_read,
2371         .write    = apic_mmio_write,
2372 };
2373
2374 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2375 {
2376         struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2377         struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2378
2379         apic_timer_expired(apic);
2380
2381         if (lapic_is_periodic(apic)) {
2382                 advance_periodic_target_expiration(apic);
2383                 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2384                 return HRTIMER_RESTART;
2385         } else
2386                 return HRTIMER_NORESTART;
2387 }
2388
2389 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
2390 {
2391         struct kvm_lapic *apic;
2392
2393         ASSERT(vcpu != NULL);
2394
2395         apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
2396         if (!apic)
2397                 goto nomem;
2398
2399         vcpu->arch.apic = apic;
2400
2401         apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
2402         if (!apic->regs) {
2403                 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2404                        vcpu->vcpu_id);
2405                 goto nomem_free_apic;
2406         }
2407         apic->vcpu = vcpu;
2408
2409         hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2410                      HRTIMER_MODE_ABS_HARD);
2411         apic->lapic_timer.timer.function = apic_timer_fn;
2412         if (timer_advance_ns == -1) {
2413                 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
2414                 lapic_timer_advance_dynamic = true;
2415         } else {
2416                 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2417                 lapic_timer_advance_dynamic = false;
2418         }
2419
2420         /*
2421          * APIC is created enabled. This will prevent kvm_lapic_set_base from
2422          * thinking that APIC state has changed.
2423          */
2424         vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2425         static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
2426         kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
2427
2428         return 0;
2429 nomem_free_apic:
2430         kfree(apic);
2431         vcpu->arch.apic = NULL;
2432 nomem:
2433         return -ENOMEM;
2434 }
2435
2436 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2437 {
2438         struct kvm_lapic *apic = vcpu->arch.apic;
2439         u32 ppr;
2440
2441         if (!kvm_apic_hw_enabled(apic))
2442                 return -1;
2443
2444         __apic_update_ppr(apic, &ppr);
2445         return apic_has_interrupt_for_ppr(apic, ppr);
2446 }
2447
2448 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2449 {
2450         u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
2451
2452         if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2453                 return 1;
2454         if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2455             GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2456                 return 1;
2457         return 0;
2458 }
2459
2460 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2461 {
2462         struct kvm_lapic *apic = vcpu->arch.apic;
2463
2464         if (atomic_read(&apic->lapic_timer.pending) > 0) {
2465                 kvm_apic_inject_pending_timer_irqs(apic);
2466                 atomic_set(&apic->lapic_timer.pending, 0);
2467         }
2468 }
2469
2470 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2471 {
2472         int vector = kvm_apic_has_interrupt(vcpu);
2473         struct kvm_lapic *apic = vcpu->arch.apic;
2474         u32 ppr;
2475
2476         if (vector == -1)
2477                 return -1;
2478
2479         /*
2480          * We get here even with APIC virtualization enabled, if doing
2481          * nested virtualization and L1 runs with the "acknowledge interrupt
2482          * on exit" mode.  Then we cannot inject the interrupt via RVI,
2483          * because the process would deliver it through the IDT.
2484          */
2485
2486         apic_clear_irr(vector, apic);
2487         if (test_bit(vector, vcpu_to_synic(vcpu)->auto_eoi_bitmap)) {
2488                 /*
2489                  * For auto-EOI interrupts, there might be another pending
2490                  * interrupt above PPR, so check whether to raise another
2491                  * KVM_REQ_EVENT.
2492                  */
2493                 apic_update_ppr(apic);
2494         } else {
2495                 /*
2496                  * For normal interrupts, PPR has been raised and there cannot
2497                  * be a higher-priority pending interrupt---except if there was
2498                  * a concurrent interrupt injection, but that would have
2499                  * triggered KVM_REQ_EVENT already.
2500                  */
2501                 apic_set_isr(vector, apic);
2502                 __apic_update_ppr(apic, &ppr);
2503         }
2504
2505         return vector;
2506 }
2507
2508 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2509                 struct kvm_lapic_state *s, bool set)
2510 {
2511         if (apic_x2apic_mode(vcpu->arch.apic)) {
2512                 u32 *id = (u32 *)(s->regs + APIC_ID);
2513                 u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2514
2515                 if (vcpu->kvm->arch.x2apic_format) {
2516                         if (*id != vcpu->vcpu_id)
2517                                 return -EINVAL;
2518                 } else {
2519                         if (set)
2520                                 *id >>= 24;
2521                         else
2522                                 *id <<= 24;
2523                 }
2524
2525                 /* In x2APIC mode, the LDR is fixed and based on the id */
2526                 if (set)
2527                         *ldr = kvm_apic_calc_x2apic_ldr(*id);
2528         }
2529
2530         return 0;
2531 }
2532
2533 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2534 {
2535         memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2536
2537         /*
2538          * Get calculated timer current count for remaining timer period (if
2539          * any) and store it in the returned register set.
2540          */
2541         __kvm_lapic_set_reg(s->regs, APIC_TMCCT,
2542                             __apic_read(vcpu->arch.apic, APIC_TMCCT));
2543
2544         return kvm_apic_state_fixup(vcpu, s, false);
2545 }
2546
2547 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2548 {
2549         struct kvm_lapic *apic = vcpu->arch.apic;
2550         int r;
2551
2552         kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
2553         /* set SPIV separately to get count of SW disabled APICs right */
2554         apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
2555
2556         r = kvm_apic_state_fixup(vcpu, s, true);
2557         if (r) {
2558                 kvm_recalculate_apic_map(vcpu->kvm);
2559                 return r;
2560         }
2561         memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
2562
2563         kvm_recalculate_apic_map(vcpu->kvm);
2564         kvm_apic_set_version(vcpu);
2565
2566         apic_update_ppr(apic);
2567         hrtimer_cancel(&apic->lapic_timer.timer);
2568         apic_update_lvtt(apic);
2569         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2570         update_divide_count(apic);
2571         __start_apic_timer(apic, APIC_TMCCT);
2572         kvm_apic_update_apicv(vcpu);
2573         apic->highest_isr_cache = -1;
2574         if (vcpu->arch.apicv_active) {
2575                 kvm_x86_ops.apicv_post_state_restore(vcpu);
2576                 kvm_x86_ops.hwapic_irr_update(vcpu,
2577                                 apic_find_highest_irr(apic));
2578                 kvm_x86_ops.hwapic_isr_update(vcpu,
2579                                 apic_find_highest_isr(apic));
2580         }
2581         kvm_make_request(KVM_REQ_EVENT, vcpu);
2582         if (ioapic_in_kernel(vcpu->kvm))
2583                 kvm_rtc_eoi_tracking_restore_one(vcpu);
2584
2585         vcpu->arch.apic_arb_prio = 0;
2586
2587         return 0;
2588 }
2589
2590 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
2591 {
2592         struct hrtimer *timer;
2593
2594         if (!lapic_in_kernel(vcpu) ||
2595                 kvm_can_post_timer_interrupt(vcpu))
2596                 return;
2597
2598         timer = &vcpu->arch.apic->lapic_timer.timer;
2599         if (hrtimer_cancel(timer))
2600                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
2601 }
2602
2603 /*
2604  * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2605  *
2606  * Detect whether guest triggered PV EOI since the
2607  * last entry. If yes, set EOI on guests's behalf.
2608  * Clear PV EOI in guest memory in any case.
2609  */
2610 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2611                                         struct kvm_lapic *apic)
2612 {
2613         bool pending;
2614         int vector;
2615         /*
2616          * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2617          * and KVM_PV_EOI_ENABLED in guest memory as follows:
2618          *
2619          * KVM_APIC_PV_EOI_PENDING is unset:
2620          *      -> host disabled PV EOI.
2621          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2622          *      -> host enabled PV EOI, guest did not execute EOI yet.
2623          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2624          *      -> host enabled PV EOI, guest executed EOI.
2625          */
2626         BUG_ON(!pv_eoi_enabled(vcpu));
2627         pending = pv_eoi_get_pending(vcpu);
2628         /*
2629          * Clear pending bit in any case: it will be set again on vmentry.
2630          * While this might not be ideal from performance point of view,
2631          * this makes sure pv eoi is only enabled when we know it's safe.
2632          */
2633         pv_eoi_clr_pending(vcpu);
2634         if (pending)
2635                 return;
2636         vector = apic_set_eoi(apic);
2637         trace_kvm_pv_eoi(apic, vector);
2638 }
2639
2640 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2641 {
2642         u32 data;
2643
2644         if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2645                 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2646
2647         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2648                 return;
2649
2650         if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2651                                   sizeof(u32)))
2652                 return;
2653
2654         apic_set_tpr(vcpu->arch.apic, data & 0xff);
2655 }
2656
2657 /*
2658  * apic_sync_pv_eoi_to_guest - called before vmentry
2659  *
2660  * Detect whether it's safe to enable PV EOI and
2661  * if yes do so.
2662  */
2663 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2664                                         struct kvm_lapic *apic)
2665 {
2666         if (!pv_eoi_enabled(vcpu) ||
2667             /* IRR set or many bits in ISR: could be nested. */
2668             apic->irr_pending ||
2669             /* Cache not set: could be safe but we don't bother. */
2670             apic->highest_isr_cache == -1 ||
2671             /* Need EOI to update ioapic. */
2672             kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
2673                 /*
2674                  * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2675                  * so we need not do anything here.
2676                  */
2677                 return;
2678         }
2679
2680         pv_eoi_set_pending(apic->vcpu);
2681 }
2682
2683 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2684 {
2685         u32 data, tpr;
2686         int max_irr, max_isr;
2687         struct kvm_lapic *apic = vcpu->arch.apic;
2688
2689         apic_sync_pv_eoi_to_guest(vcpu, apic);
2690
2691         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2692                 return;
2693
2694         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
2695         max_irr = apic_find_highest_irr(apic);
2696         if (max_irr < 0)
2697                 max_irr = 0;
2698         max_isr = apic_find_highest_isr(apic);
2699         if (max_isr < 0)
2700                 max_isr = 0;
2701         data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2702
2703         kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2704                                 sizeof(u32));
2705 }
2706
2707 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
2708 {
2709         if (vapic_addr) {
2710                 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2711                                         &vcpu->arch.apic->vapic_cache,
2712                                         vapic_addr, sizeof(u32)))
2713                         return -EINVAL;
2714                 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2715         } else {
2716                 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2717         }
2718
2719         vcpu->arch.apic->vapic_addr = vapic_addr;
2720         return 0;
2721 }
2722
2723 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2724 {
2725         struct kvm_lapic *apic = vcpu->arch.apic;
2726         u32 reg = (msr - APIC_BASE_MSR) << 4;
2727
2728         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2729                 return 1;
2730
2731         if (reg == APIC_ICR2)
2732                 return 1;
2733
2734         /* if this is ICR write vector before command */
2735         if (reg == APIC_ICR)
2736                 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2737         return kvm_lapic_reg_write(apic, reg, (u32)data);
2738 }
2739
2740 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2741 {
2742         struct kvm_lapic *apic = vcpu->arch.apic;
2743         u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2744
2745         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2746                 return 1;
2747
2748         if (reg == APIC_DFR || reg == APIC_ICR2)
2749                 return 1;
2750
2751         if (kvm_lapic_reg_read(apic, reg, 4, &low))
2752                 return 1;
2753         if (reg == APIC_ICR)
2754                 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2755
2756         *data = (((u64)high) << 32) | low;
2757
2758         return 0;
2759 }
2760
2761 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2762 {
2763         struct kvm_lapic *apic = vcpu->arch.apic;
2764
2765         if (!lapic_in_kernel(vcpu))
2766                 return 1;
2767
2768         /* if this is ICR write vector before command */
2769         if (reg == APIC_ICR)
2770                 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2771         return kvm_lapic_reg_write(apic, reg, (u32)data);
2772 }
2773
2774 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2775 {
2776         struct kvm_lapic *apic = vcpu->arch.apic;
2777         u32 low, high = 0;
2778
2779         if (!lapic_in_kernel(vcpu))
2780                 return 1;
2781
2782         if (kvm_lapic_reg_read(apic, reg, 4, &low))
2783                 return 1;
2784         if (reg == APIC_ICR)
2785                 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2786
2787         *data = (((u64)high) << 32) | low;
2788
2789         return 0;
2790 }
2791
2792 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
2793 {
2794         u64 addr = data & ~KVM_MSR_ENABLED;
2795         struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
2796         unsigned long new_len;
2797
2798         if (!IS_ALIGNED(addr, 4))
2799                 return 1;
2800
2801         vcpu->arch.pv_eoi.msr_val = data;
2802         if (!pv_eoi_enabled(vcpu))
2803                 return 0;
2804
2805         if (addr == ghc->gpa && len <= ghc->len)
2806                 new_len = ghc->len;
2807         else
2808                 new_len = len;
2809
2810         return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
2811 }
2812
2813 void kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2814 {
2815         struct kvm_lapic *apic = vcpu->arch.apic;
2816         u8 sipi_vector;
2817         unsigned long pe;
2818
2819         if (!lapic_in_kernel(vcpu) || !apic->pending_events)
2820                 return;
2821
2822         /*
2823          * INITs are latched while CPU is in specific states
2824          * (SMM, VMX non-root mode, SVM with GIF=0).
2825          * Because a CPU cannot be in these states immediately
2826          * after it has processed an INIT signal (and thus in
2827          * KVM_MP_STATE_INIT_RECEIVED state), just eat SIPIs
2828          * and leave the INIT pending.
2829          */
2830         if (kvm_vcpu_latch_init(vcpu)) {
2831                 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2832                 if (test_bit(KVM_APIC_SIPI, &apic->pending_events))
2833                         clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2834                 return;
2835         }
2836
2837         pe = xchg(&apic->pending_events, 0);
2838         if (test_bit(KVM_APIC_INIT, &pe)) {
2839                 kvm_vcpu_reset(vcpu, true);
2840                 if (kvm_vcpu_is_bsp(apic->vcpu))
2841                         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2842                 else
2843                         vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2844         }
2845         if (test_bit(KVM_APIC_SIPI, &pe) &&
2846             vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2847                 /* evaluate pending_events before reading the vector */
2848                 smp_rmb();
2849                 sipi_vector = apic->sipi_vector;
2850                 kvm_vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2851                 vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2852         }
2853 }
2854
2855 void kvm_lapic_init(void)
2856 {
2857         /* do not patch jump label more than once per second */
2858         jump_label_rate_limit(&apic_hw_disabled, HZ);
2859         jump_label_rate_limit(&apic_sw_disabled, HZ);
2860 }
2861
2862 void kvm_lapic_exit(void)
2863 {
2864         static_key_deferred_flush(&apic_hw_disabled);
2865         static_key_deferred_flush(&apic_sw_disabled);
2866 }