virt/kvm/arm/arch_timer.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2012 ARM Ltd.
   4  * Author: Marc Zyngier <marc.zyngier@arm.com>
   5  */
   6
   7 #include <linux/cpu.h>
   8 #include <linux/kvm.h>
   9 #include <linux/kvm_host.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/irq.h>
  12 #include <linux/uaccess.h>
  13
  14 #include <clocksource/arm_arch_timer.h>
  15 #include <asm/arch_timer.h>
  16 #include <asm/kvm_emulate.h>
  17 #include <asm/kvm_hyp.h>
  18
  19 #include <kvm/arm_vgic.h>
  20 #include <kvm/arm_arch_timer.h>
  21
  22 #include "trace.h"
  23
  24 static struct timecounter *timecounter;
  25 static unsigned int host_vtimer_irq;
  26 static unsigned int host_ptimer_irq;
  27 static u32 host_vtimer_irq_flags;
  28 static u32 host_ptimer_irq_flags;
  29
  30 static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);
  31
  32 static const struct kvm_irq_level default_ptimer_irq = {
  33         .irq    = 30,
  34         .level  = 1,
  35 };
  36
  37 static const struct kvm_irq_level default_vtimer_irq = {
  38         .irq    = 27,
  39         .level  = 1,
  40 };
  41
  42 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
  43 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
  44                                  struct arch_timer_context *timer_ctx);
  45 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
  46 static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
  47                                 struct arch_timer_context *timer,
  48                                 enum kvm_arch_timer_regs treg,
  49                                 u64 val);
  50 static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
  51                               struct arch_timer_context *timer,
  52                               enum kvm_arch_timer_regs treg);
  53
  54 u64 kvm_phys_timer_read(void)
  55 {
  56         return timecounter->cc->read(timecounter->cc);
  57 }
  58
  59 static void get_timer_map(struct kvm_vcpu *vcpu, struct timer_map *map)
  60 {
  61         if (has_vhe()) {
  62                 map->direct_vtimer = vcpu_vtimer(vcpu);
  63                 map->direct_ptimer = vcpu_ptimer(vcpu);
  64                 map->emul_ptimer = NULL;
  65         } else {
  66                 map->direct_vtimer = vcpu_vtimer(vcpu);
  67                 map->direct_ptimer = NULL;
  68                 map->emul_ptimer = vcpu_ptimer(vcpu);
  69         }
  70
  71         trace_kvm_get_timer_map(vcpu->vcpu_id, map);
  72 }
  73
  74 static inline bool userspace_irqchip(struct kvm *kvm)
  75 {
  76         return static_branch_unlikely(&userspace_irqchip_in_use) &&
  77                 unlikely(!irqchip_in_kernel(kvm));
  78 }
  79
  80 static void soft_timer_start(struct hrtimer *hrt, u64 ns)
  81 {
  82         hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
  83                       HRTIMER_MODE_ABS);
  84 }
  85
  86 static void soft_timer_cancel(struct hrtimer *hrt)
  87 {
  88         hrtimer_cancel(hrt);
  89 }
  90
  91 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  92 {
  93         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  94         struct arch_timer_context *ctx;
  95         struct timer_map map;
  96
  97         /*
  98          * We may see a timer interrupt after vcpu_put() has been called which
  99          * sets the CPU's vcpu pointer to NULL, because even though the timer
 100          * has been disabled in timer_save_state(), the hardware interrupt
 101          * signal may not have been retired from the interrupt controller yet.
 102          */
 103         if (!vcpu)
 104                 return IRQ_HANDLED;
 105
 106         get_timer_map(vcpu, &map);
 107
 108         if (irq == host_vtimer_irq)
 109                 ctx = map.direct_vtimer;
 110         else
 111                 ctx = map.direct_ptimer;
 112
 113         if (kvm_timer_should_fire(ctx))
 114                 kvm_timer_update_irq(vcpu, true, ctx);
 115
 116         if (userspace_irqchip(vcpu->kvm) &&
 117             !static_branch_unlikely(&has_gic_active_state))
 118                 disable_percpu_irq(host_vtimer_irq);
 119
 120         return IRQ_HANDLED;
 121 }
 122
 123 static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
 124 {
 125         u64 cval, now;
 126
 127         cval = timer_ctx->cnt_cval;
 128         now = kvm_phys_timer_read() - timer_ctx->cntvoff;
 129
 130         if (now < cval) {
 131                 u64 ns;
 132
 133                 ns = cyclecounter_cyc2ns(timecounter->cc,
 134                                          cval - now,
 135                                          timecounter->mask,
 136                                          &timecounter->frac);
 137                 return ns;
 138         }
 139
 140         return 0;
 141 }
 142
 143 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
 144 {
 145         WARN_ON(timer_ctx && timer_ctx->loaded);
 146         return timer_ctx &&
 147                !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 148                 (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE);
 149 }
 150
 151 /*
 152  * Returns the earliest expiration time in ns among guest timers.
 153  * Note that it will return 0 if none of timers can fire.
 154  */
 155 static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
 156 {
 157         u64 min_delta = ULLONG_MAX;
 158         int i;
 159
 160         for (i = 0; i < NR_KVM_TIMERS; i++) {
 161                 struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
 162
 163                 WARN(ctx->loaded, "timer %d loaded\n", i);
 164                 if (kvm_timer_irq_can_fire(ctx))
 165                         min_delta = min(min_delta, kvm_timer_compute_delta(ctx));
 166         }
 167
 168         /* If none of timers can fire, then return 0 */
 169         if (min_delta == ULLONG_MAX)
 170                 return 0;
 171
 172         return min_delta;
 173 }
 174
 175 static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
 176 {
 177         struct arch_timer_cpu *timer;
 178         struct kvm_vcpu *vcpu;
 179         u64 ns;
 180
 181         timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
 182         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 183
 184         /*
 185          * Check that the timer has really expired from the guest's
 186          * PoV (NTP on the host may have forced it to expire
 187          * early). If we should have slept longer, restart it.
 188          */
 189         ns = kvm_timer_earliest_exp(vcpu);
 190         if (unlikely(ns)) {
 191                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 192                 return HRTIMER_RESTART;
 193         }
 194
 195         kvm_vcpu_wake_up(vcpu);
 196         return HRTIMER_NORESTART;
 197 }
 198
 199 static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt)
 200 {
 201         struct arch_timer_context *ctx;
 202         struct kvm_vcpu *vcpu;
 203         u64 ns;
 204
 205         ctx = container_of(hrt, struct arch_timer_context, hrtimer);
 206         vcpu = ctx->vcpu;
 207
 208         trace_kvm_timer_hrtimer_expire(ctx);
 209
 210         /*
 211          * Check that the timer has really expired from the guest's
 212          * PoV (NTP on the host may have forced it to expire
 213          * early). If not ready, schedule for a later time.
 214          */
 215         ns = kvm_timer_compute_delta(ctx);
 216         if (unlikely(ns)) {
 217                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 218                 return HRTIMER_RESTART;
 219         }
 220
 221         kvm_timer_update_irq(vcpu, true, ctx);
 222         return HRTIMER_NORESTART;
 223 }
 224
 225 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
 226 {
 227         enum kvm_arch_timers index;
 228         u64 cval, now;
 229
 230         if (!timer_ctx)
 231                 return false;
 232
 233         index = arch_timer_ctx_index(timer_ctx);
 234
 235         if (timer_ctx->loaded) {
 236                 u32 cnt_ctl = 0;
 237
 238                 switch (index) {
 239                 case TIMER_VTIMER:
 240                         cnt_ctl = read_sysreg_el0(cntv_ctl);
 241                         break;
 242                 case TIMER_PTIMER:
 243                         cnt_ctl = read_sysreg_el0(cntp_ctl);
 244                         break;
 245                 case NR_KVM_TIMERS:
 246                         /* GCC is braindead */
 247                         cnt_ctl = 0;
 248                         break;
 249                 }
 250
 251                 return  (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
 252                         (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
 253                        !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
 254         }
 255
 256         if (!kvm_timer_irq_can_fire(timer_ctx))
 257                 return false;
 258
 259         cval = timer_ctx->cnt_cval;
 260         now = kvm_phys_timer_read() - timer_ctx->cntvoff;
 261
 262         return cval <= now;
 263 }
 264
 265 bool kvm_timer_is_pending(struct kvm_vcpu *vcpu)
 266 {
 267         struct timer_map map;
 268
 269         get_timer_map(vcpu, &map);
 270
 271         return kvm_timer_should_fire(map.direct_vtimer) ||
 272                kvm_timer_should_fire(map.direct_ptimer) ||
 273                kvm_timer_should_fire(map.emul_ptimer);
 274 }
 275
 276 /*
 277  * Reflect the timer output level into the kvm_run structure
 278  */
 279 void kvm_timer_update_run(struct kvm_vcpu *vcpu)
 280 {
 281         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 282         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 283         struct kvm_sync_regs *regs = &vcpu->run->s.regs;
 284
 285         /* Populate the device bitmap with the timer states */
 286         regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
 287                                     KVM_ARM_DEV_EL1_PTIMER);
 288         if (kvm_timer_should_fire(vtimer))
 289                 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
 290         if (kvm_timer_should_fire(ptimer))
 291                 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
 292 }
 293
 294 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 295                                  struct arch_timer_context *timer_ctx)
 296 {
 297         int ret;
 298
 299         timer_ctx->irq.level = new_level;
 300         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
 301                                    timer_ctx->irq.level);
 302
 303         if (!userspace_irqchip(vcpu->kvm)) {
 304                 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
 305                                           timer_ctx->irq.irq,
 306                                           timer_ctx->irq.level,
 307                                           timer_ctx);
 308                 WARN_ON(ret);
 309         }
 310 }
 311
 312 static void timer_emulate(struct arch_timer_context *ctx)
 313 {
 314         bool should_fire = kvm_timer_should_fire(ctx);
 315
 316         trace_kvm_timer_emulate(ctx, should_fire);
 317
 318         if (should_fire) {
 319                 kvm_timer_update_irq(ctx->vcpu, true, ctx);
 320                 return;
 321         }
 322
 323         /*
 324          * If the timer can fire now, we don't need to have a soft timer
 325          * scheduled for the future.  If the timer cannot fire at all,
 326          * then we also don't need a soft timer.
 327          */
 328         if (!kvm_timer_irq_can_fire(ctx)) {
 329                 soft_timer_cancel(&ctx->hrtimer);
 330                 return;
 331         }
 332
 333         soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));
 334 }
 335
 336 static void timer_save_state(struct arch_timer_context *ctx)
 337 {
 338         struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
 339         enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
 340         unsigned long flags;
 341
 342         if (!timer->enabled)
 343                 return;
 344
 345         local_irq_save(flags);
 346
 347         if (!ctx->loaded)
 348                 goto out;
 349
 350         switch (index) {
 351         case TIMER_VTIMER:
 352                 ctx->cnt_ctl = read_sysreg_el0(cntv_ctl);
 353                 ctx->cnt_cval = read_sysreg_el0(cntv_cval);
 354
 355                 /* Disable the timer */
 356                 write_sysreg_el0(0, cntv_ctl);
 357                 isb();
 358
 359                 break;
 360         case TIMER_PTIMER:
 361                 ctx->cnt_ctl = read_sysreg_el0(cntp_ctl);
 362                 ctx->cnt_cval = read_sysreg_el0(cntp_cval);
 363
 364                 /* Disable the timer */
 365                 write_sysreg_el0(0, cntp_ctl);
 366                 isb();
 367
 368                 break;
 369         case NR_KVM_TIMERS:
 370                 BUG();
 371         }
 372
 373         trace_kvm_timer_save_state(ctx);
 374
 375         ctx->loaded = false;
 376 out:
 377         local_irq_restore(flags);
 378 }
 379
 380 /*
 381  * Schedule the background timer before calling kvm_vcpu_block, so that this
 382  * thread is removed from its waitqueue and made runnable when there's a timer
 383  * interrupt to handle.
 384  */
 385 static void kvm_timer_blocking(struct kvm_vcpu *vcpu)
 386 {
 387         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 388         struct timer_map map;
 389
 390         get_timer_map(vcpu, &map);
 391
 392         /*
 393          * If no timers are capable of raising interrupts (disabled or
 394          * masked), then there's no more work for us to do.
 395          */
 396         if (!kvm_timer_irq_can_fire(map.direct_vtimer) &&
 397             !kvm_timer_irq_can_fire(map.direct_ptimer) &&
 398             !kvm_timer_irq_can_fire(map.emul_ptimer))
 399                 return;
 400
 401         /*
 402          * At least one guest time will expire. Schedule a background timer.
 403          * Set the earliest expiration time among the guest timers.
 404          */
 405         soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
 406 }
 407
 408 static void kvm_timer_unblocking(struct kvm_vcpu *vcpu)
 409 {
 410         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 411
 412         soft_timer_cancel(&timer->bg_timer);
 413 }
 414
 415 static void timer_restore_state(struct arch_timer_context *ctx)
 416 {
 417         struct arch_timer_cpu *timer = vcpu_timer(ctx->vcpu);
 418         enum kvm_arch_timers index = arch_timer_ctx_index(ctx);
 419         unsigned long flags;
 420
 421         if (!timer->enabled)
 422                 return;
 423
 424         local_irq_save(flags);
 425
 426         if (ctx->loaded)
 427                 goto out;
 428
 429         switch (index) {
 430         case TIMER_VTIMER:
 431                 write_sysreg_el0(ctx->cnt_cval, cntv_cval);
 432                 isb();
 433                 write_sysreg_el0(ctx->cnt_ctl, cntv_ctl);
 434                 break;
 435         case TIMER_PTIMER:
 436                 write_sysreg_el0(ctx->cnt_cval, cntp_cval);
 437                 isb();
 438                 write_sysreg_el0(ctx->cnt_ctl, cntp_ctl);
 439                 break;
 440         case NR_KVM_TIMERS:
 441                 BUG();
 442         }
 443
 444         trace_kvm_timer_restore_state(ctx);
 445
 446         ctx->loaded = true;
 447 out:
 448         local_irq_restore(flags);
 449 }
 450
 451 static void set_cntvoff(u64 cntvoff)
 452 {
 453         u32 low = lower_32_bits(cntvoff);
 454         u32 high = upper_32_bits(cntvoff);
 455
 456         /*
 457          * Since kvm_call_hyp doesn't fully support the ARM PCS especially on
 458          * 32-bit systems, but rather passes register by register shifted one
 459          * place (we put the function address in r0/x0), we cannot simply pass
 460          * a 64-bit value as an argument, but have to split the value in two
 461          * 32-bit halves.
 462          */
 463         kvm_call_hyp(__kvm_timer_set_cntvoff, low, high);
 464 }
 465
 466 static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, bool active)
 467 {
 468         int r;
 469         r = irq_set_irqchip_state(ctx->host_timer_irq, IRQCHIP_STATE_ACTIVE, active);
 470         WARN_ON(r);
 471 }
 472
 473 static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
 474 {
 475         struct kvm_vcpu *vcpu = ctx->vcpu;
 476         bool phys_active = false;
 477
 478         /*
 479          * Update the timer output so that it is likely to match the
 480          * state we're about to restore. If the timer expires between
 481          * this point and the register restoration, we'll take the
 482          * interrupt anyway.
 483          */
 484         kvm_timer_update_irq(ctx->vcpu, kvm_timer_should_fire(ctx), ctx);
 485
 486         if (irqchip_in_kernel(vcpu->kvm))
 487                 phys_active = kvm_vgic_map_is_active(vcpu, ctx->irq.irq);
 488
 489         phys_active |= ctx->irq.level;
 490
 491         set_timer_irq_phys_active(ctx, phys_active);
 492 }
 493
 494 static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
 495 {
 496         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 497
 498         /*
 499          * Update the timer output so that it is likely to match the
 500          * state we're about to restore. If the timer expires between
 501          * this point and the register restoration, we'll take the
 502          * interrupt anyway.
 503          */
 504         kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer);
 505
 506         /*
 507          * When using a userspace irqchip with the architected timers and a
 508          * host interrupt controller that doesn't support an active state, we
 509          * must still prevent continuously exiting from the guest, and
 510          * therefore mask the physical interrupt by disabling it on the host
 511          * interrupt controller when the virtual level is high, such that the
 512          * guest can make forward progress.  Once we detect the output level
 513          * being de-asserted, we unmask the interrupt again so that we exit
 514          * from the guest when the timer fires.
 515          */
 516         if (vtimer->irq.level)
 517                 disable_percpu_irq(host_vtimer_irq);
 518         else
 519                 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 520 }
 521
 522 void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
 523 {
 524         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 525         struct timer_map map;
 526
 527         if (unlikely(!timer->enabled))
 528                 return;
 529
 530         get_timer_map(vcpu, &map);
 531
 532         if (static_branch_likely(&has_gic_active_state)) {
 533                 kvm_timer_vcpu_load_gic(map.direct_vtimer);
 534                 if (map.direct_ptimer)
 535                         kvm_timer_vcpu_load_gic(map.direct_ptimer);
 536         } else {
 537                 kvm_timer_vcpu_load_nogic(vcpu);
 538         }
 539
 540         set_cntvoff(map.direct_vtimer->cntvoff);
 541
 542         kvm_timer_unblocking(vcpu);
 543
 544         timer_restore_state(map.direct_vtimer);
 545         if (map.direct_ptimer)
 546                 timer_restore_state(map.direct_ptimer);
 547
 548         if (map.emul_ptimer)
 549                 timer_emulate(map.emul_ptimer);
 550 }
 551
 552 bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
 553 {
 554         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 555         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 556         struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
 557         bool vlevel, plevel;
 558
 559         if (likely(irqchip_in_kernel(vcpu->kvm)))
 560                 return false;
 561
 562         vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
 563         plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
 564
 565         return kvm_timer_should_fire(vtimer) != vlevel ||
 566                kvm_timer_should_fire(ptimer) != plevel;
 567 }
 568
 569 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
 570 {
 571         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 572         struct timer_map map;
 573
 574         if (unlikely(!timer->enabled))
 575                 return;
 576
 577         get_timer_map(vcpu, &map);
 578
 579         timer_save_state(map.direct_vtimer);
 580         if (map.direct_ptimer)
 581                 timer_save_state(map.direct_ptimer);
 582
 583         /*
 584          * Cancel soft timer emulation, because the only case where we
 585          * need it after a vcpu_put is in the context of a sleeping VCPU, and
 586          * in that case we already factor in the deadline for the physical
 587          * timer when scheduling the bg_timer.
 588          *
 589          * In any case, we re-schedule the hrtimer for the physical timer when
 590          * coming back to the VCPU thread in kvm_timer_vcpu_load().
 591          */
 592         if (map.emul_ptimer)
 593                 soft_timer_cancel(&map.emul_ptimer->hrtimer);
 594
 595         if (swait_active(kvm_arch_vcpu_wq(vcpu)))
 596                 kvm_timer_blocking(vcpu);
 597
 598         /*
 599          * The kernel may decide to run userspace after calling vcpu_put, so
 600          * we reset cntvoff to 0 to ensure a consistent read between user
 601          * accesses to the virtual counter and kernel access to the physical
 602          * counter of non-VHE case. For VHE, the virtual counter uses a fixed
 603          * virtual offset of zero, so no need to zero CNTVOFF_EL2 register.
 604          */
 605         set_cntvoff(0);
 606 }
 607
 608 /*
 609  * With a userspace irqchip we have to check if the guest de-asserted the
 610  * timer and if so, unmask the timer irq signal on the host interrupt
 611  * controller to ensure that we see future timer signals.
 612  */
 613 static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
 614 {
 615         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 616
 617         if (!kvm_timer_should_fire(vtimer)) {
 618                 kvm_timer_update_irq(vcpu, false, vtimer);
 619                 if (static_branch_likely(&has_gic_active_state))
 620                         set_timer_irq_phys_active(vtimer, false);
 621                 else
 622                         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 623         }
 624 }
 625
 626 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 627 {
 628         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 629
 630         if (unlikely(!timer->enabled))
 631                 return;
 632
 633         if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
 634                 unmask_vtimer_irq_user(vcpu);
 635 }
 636
 637 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
 638 {
 639         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 640         struct timer_map map;
 641
 642         get_timer_map(vcpu, &map);
 643
 644         /*
 645          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 646          * and to 0 for ARMv7.  We provide an implementation that always
 647          * resets the timer to be disabled and unmasked and is compliant with
 648          * the ARMv7 architecture.
 649          */
 650         vcpu_vtimer(vcpu)->cnt_ctl = 0;
 651         vcpu_ptimer(vcpu)->cnt_ctl = 0;
 652
 653         if (timer->enabled) {
 654                 kvm_timer_update_irq(vcpu, false, vcpu_vtimer(vcpu));
 655                 kvm_timer_update_irq(vcpu, false, vcpu_ptimer(vcpu));
 656
 657                 if (irqchip_in_kernel(vcpu->kvm)) {
 658                         kvm_vgic_reset_mapped_irq(vcpu, map.direct_vtimer->irq.irq);
 659                         if (map.direct_ptimer)
 660                                 kvm_vgic_reset_mapped_irq(vcpu, map.direct_ptimer->irq.irq);
 661                 }
 662         }
 663
 664         if (map.emul_ptimer)
 665                 soft_timer_cancel(&map.emul_ptimer->hrtimer);
 666
 667         return 0;
 668 }
 669
 670 /* Make the updates of cntvoff for all vtimer contexts atomic */
 671 static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
 672 {
 673         int i;
 674         struct kvm *kvm = vcpu->kvm;
 675         struct kvm_vcpu *tmp;
 676
 677         mutex_lock(&kvm->lock);
 678         kvm_for_each_vcpu(i, tmp, kvm)
 679                 vcpu_vtimer(tmp)->cntvoff = cntvoff;
 680
 681         /*
 682          * When called from the vcpu create path, the CPU being created is not
 683          * included in the loop above, so we just set it here as well.
 684          */
 685         vcpu_vtimer(vcpu)->cntvoff = cntvoff;
 686         mutex_unlock(&kvm->lock);
 687 }
 688
 689 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 690 {
 691         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 692         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 693         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 694
 695         /* Synchronize cntvoff across all vtimers of a VM. */
 696         update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
 697         ptimer->cntvoff = 0;
 698
 699         hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 700         timer->bg_timer.function = kvm_bg_timer_expire;
 701
 702         hrtimer_init(&vtimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 703         hrtimer_init(&ptimer->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 704         vtimer->hrtimer.function = kvm_hrtimer_expire;
 705         ptimer->hrtimer.function = kvm_hrtimer_expire;
 706
 707         vtimer->irq.irq = default_vtimer_irq.irq;
 708         ptimer->irq.irq = default_ptimer_irq.irq;
 709
 710         vtimer->host_timer_irq = host_vtimer_irq;
 711         ptimer->host_timer_irq = host_ptimer_irq;
 712
 713         vtimer->host_timer_irq_flags = host_vtimer_irq_flags;
 714         ptimer->host_timer_irq_flags = host_ptimer_irq_flags;
 715
 716         vtimer->vcpu = vcpu;
 717         ptimer->vcpu = vcpu;
 718 }
 719
 720 static void kvm_timer_init_interrupt(void *info)
 721 {
 722         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 723         enable_percpu_irq(host_ptimer_irq, host_ptimer_irq_flags);
 724 }
 725
 726 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 727 {
 728         struct arch_timer_context *timer;
 729
 730         switch (regid) {
 731         case KVM_REG_ARM_TIMER_CTL:
 732                 timer = vcpu_vtimer(vcpu);
 733                 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
 734                 break;
 735         case KVM_REG_ARM_TIMER_CNT:
 736                 timer = vcpu_vtimer(vcpu);
 737                 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
 738                 break;
 739         case KVM_REG_ARM_TIMER_CVAL:
 740                 timer = vcpu_vtimer(vcpu);
 741                 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
 742                 break;
 743         case KVM_REG_ARM_PTIMER_CTL:
 744                 timer = vcpu_ptimer(vcpu);
 745                 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CTL, value);
 746                 break;
 747         case KVM_REG_ARM_PTIMER_CVAL:
 748                 timer = vcpu_ptimer(vcpu);
 749                 kvm_arm_timer_write(vcpu, timer, TIMER_REG_CVAL, value);
 750                 break;
 751
 752         default:
 753                 return -1;
 754         }
 755
 756         return 0;
 757 }
 758
 759 static u64 read_timer_ctl(struct arch_timer_context *timer)
 760 {
 761         /*
 762          * Set ISTATUS bit if it's expired.
 763          * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is
 764          * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit
 765          * regardless of ENABLE bit for our implementation convenience.
 766          */
 767         if (!kvm_timer_compute_delta(timer))
 768                 return timer->cnt_ctl | ARCH_TIMER_CTRL_IT_STAT;
 769         else
 770                 return timer->cnt_ctl;
 771 }
 772
 773 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 774 {
 775         switch (regid) {
 776         case KVM_REG_ARM_TIMER_CTL:
 777                 return kvm_arm_timer_read(vcpu,
 778                                           vcpu_vtimer(vcpu), TIMER_REG_CTL);
 779         case KVM_REG_ARM_TIMER_CNT:
 780                 return kvm_arm_timer_read(vcpu,
 781                                           vcpu_vtimer(vcpu), TIMER_REG_CNT);
 782         case KVM_REG_ARM_TIMER_CVAL:
 783                 return kvm_arm_timer_read(vcpu,
 784                                           vcpu_vtimer(vcpu), TIMER_REG_CVAL);
 785         case KVM_REG_ARM_PTIMER_CTL:
 786                 return kvm_arm_timer_read(vcpu,
 787                                           vcpu_ptimer(vcpu), TIMER_REG_CTL);
 788         case KVM_REG_ARM_PTIMER_CNT:
 789                 return kvm_arm_timer_read(vcpu,
 790                                           vcpu_vtimer(vcpu), TIMER_REG_CNT);
 791         case KVM_REG_ARM_PTIMER_CVAL:
 792                 return kvm_arm_timer_read(vcpu,
 793                                           vcpu_ptimer(vcpu), TIMER_REG_CVAL);
 794         }
 795         return (u64)-1;
 796 }
 797
 798 static u64 kvm_arm_timer_read(struct kvm_vcpu *vcpu,
 799                               struct arch_timer_context *timer,
 800                               enum kvm_arch_timer_regs treg)
 801 {
 802         u64 val;
 803
 804         switch (treg) {
 805         case TIMER_REG_TVAL:
 806                 val = timer->cnt_cval - kvm_phys_timer_read() + timer->cntvoff;
 807                 break;
 808
 809         case TIMER_REG_CTL:
 810                 val = read_timer_ctl(timer);
 811                 break;
 812
 813         case TIMER_REG_CVAL:
 814                 val = timer->cnt_cval;
 815                 break;
 816
 817         case TIMER_REG_CNT:
 818                 val = kvm_phys_timer_read() - timer->cntvoff;
 819                 break;
 820
 821         default:
 822                 BUG();
 823         }
 824
 825         return val;
 826 }
 827
 828 u64 kvm_arm_timer_read_sysreg(struct kvm_vcpu *vcpu,
 829                               enum kvm_arch_timers tmr,
 830                               enum kvm_arch_timer_regs treg)
 831 {
 832         u64 val;
 833
 834         preempt_disable();
 835         kvm_timer_vcpu_put(vcpu);
 836
 837         val = kvm_arm_timer_read(vcpu, vcpu_get_timer(vcpu, tmr), treg);
 838
 839         kvm_timer_vcpu_load(vcpu);
 840         preempt_enable();
 841
 842         return val;
 843 }
 844
 845 static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
 846                                 struct arch_timer_context *timer,
 847                                 enum kvm_arch_timer_regs treg,
 848                                 u64 val)
 849 {
 850         switch (treg) {
 851         case TIMER_REG_TVAL:
 852                 timer->cnt_cval = kvm_phys_timer_read() - timer->cntvoff + val;
 853                 break;
 854
 855         case TIMER_REG_CTL:
 856                 timer->cnt_ctl = val & ~ARCH_TIMER_CTRL_IT_STAT;
 857                 break;
 858
 859         case TIMER_REG_CVAL:
 860                 timer->cnt_cval = val;
 861                 break;
 862
 863         default:
 864                 BUG();
 865         }
 866 }
 867
 868 void kvm_arm_timer_write_sysreg(struct kvm_vcpu *vcpu,
 869                                 enum kvm_arch_timers tmr,
 870                                 enum kvm_arch_timer_regs treg,
 871                                 u64 val)
 872 {
 873         preempt_disable();
 874         kvm_timer_vcpu_put(vcpu);
 875
 876         kvm_arm_timer_write(vcpu, vcpu_get_timer(vcpu, tmr), treg, val);
 877
 878         kvm_timer_vcpu_load(vcpu);
 879         preempt_enable();
 880 }
 881
 882 static int kvm_timer_starting_cpu(unsigned int cpu)
 883 {
 884         kvm_timer_init_interrupt(NULL);
 885         return 0;
 886 }
 887
 888 static int kvm_timer_dying_cpu(unsigned int cpu)
 889 {
 890         disable_percpu_irq(host_vtimer_irq);
 891         return 0;
 892 }
 893
 894 int kvm_timer_hyp_init(bool has_gic)
 895 {
 896         struct arch_timer_kvm_info *info;
 897         int err;
 898
 899         info = arch_timer_get_kvm_info();
 900         timecounter = &info->timecounter;
 901
 902         if (!timecounter->cc) {
 903                 kvm_err("kvm_arch_timer: uninitialized timecounter\n");
 904                 return -ENODEV;
 905         }
 906
 907         /* First, do the virtual EL1 timer irq */
 908
 909         if (info->virtual_irq <= 0) {
 910                 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
 911                         info->virtual_irq);
 912                 return -ENODEV;
 913         }
 914         host_vtimer_irq = info->virtual_irq;
 915
 916         host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
 917         if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
 918             host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
 919                 kvm_err("Invalid trigger for vtimer IRQ%d, assuming level low\n",
 920                         host_vtimer_irq);
 921                 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
 922         }
 923
 924         err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
 925                                  "kvm guest vtimer", kvm_get_running_vcpus());
 926         if (err) {
 927                 kvm_err("kvm_arch_timer: can't request vtimer interrupt %d (%d)\n",
 928                         host_vtimer_irq, err);
 929                 return err;
 930         }
 931
 932         if (has_gic) {
 933                 err = irq_set_vcpu_affinity(host_vtimer_irq,
 934                                             kvm_get_running_vcpus());
 935                 if (err) {
 936                         kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
 937                         goto out_free_irq;
 938                 }
 939
 940                 static_branch_enable(&has_gic_active_state);
 941         }
 942
 943         kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
 944
 945         /* Now let's do the physical EL1 timer irq */
 946
 947         if (info->physical_irq > 0) {
 948                 host_ptimer_irq = info->physical_irq;
 949                 host_ptimer_irq_flags = irq_get_trigger_type(host_ptimer_irq);
 950                 if (host_ptimer_irq_flags != IRQF_TRIGGER_HIGH &&
 951                     host_ptimer_irq_flags != IRQF_TRIGGER_LOW) {
 952                         kvm_err("Invalid trigger for ptimer IRQ%d, assuming level low\n",
 953                                 host_ptimer_irq);
 954                         host_ptimer_irq_flags = IRQF_TRIGGER_LOW;
 955                 }
 956
 957                 err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler,
 958                                          "kvm guest ptimer", kvm_get_running_vcpus());
 959                 if (err) {
 960                         kvm_err("kvm_arch_timer: can't request ptimer interrupt %d (%d)\n",
 961                                 host_ptimer_irq, err);
 962                         return err;
 963                 }
 964
 965                 if (has_gic) {
 966                         err = irq_set_vcpu_affinity(host_ptimer_irq,
 967                                                     kvm_get_running_vcpus());
 968                         if (err) {
 969                                 kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
 970                                 goto out_free_irq;
 971                         }
 972                 }
 973
 974                 kvm_debug("physical timer IRQ%d\n", host_ptimer_irq);
 975         } else if (has_vhe()) {
 976                 kvm_err("kvm_arch_timer: invalid physical timer IRQ: %d\n",
 977                         info->physical_irq);
 978                 err = -ENODEV;
 979                 goto out_free_irq;
 980         }
 981
 982         cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
 983                           "kvm/arm/timer:starting", kvm_timer_starting_cpu,
 984                           kvm_timer_dying_cpu);
 985         return 0;
 986 out_free_irq:
 987         free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
 988         return err;
 989 }
 990
 991 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 992 {
 993         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
 994
 995         soft_timer_cancel(&timer->bg_timer);
 996 }
 997
 998 static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
 999 {
1000         int vtimer_irq, ptimer_irq;
1001         int i, ret;
1002
1003         vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
1004         ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
1005         if (ret)
1006                 return false;
1007
1008         ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
1009         ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
1010         if (ret)
1011                 return false;
1012
1013         kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
1014                 if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
1015                     vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
1016                         return false;
1017         }
1018
1019         return true;
1020 }
1021
1022 bool kvm_arch_timer_get_input_level(int vintid)
1023 {
1024         struct kvm_vcpu *vcpu = kvm_arm_get_running_vcpu();
1025         struct arch_timer_context *timer;
1026
1027         if (vintid == vcpu_vtimer(vcpu)->irq.irq)
1028                 timer = vcpu_vtimer(vcpu);
1029         else if (vintid == vcpu_ptimer(vcpu)->irq.irq)
1030                 timer = vcpu_ptimer(vcpu);
1031         else
1032                 BUG();
1033
1034         return kvm_timer_should_fire(timer);
1035 }
1036
1037 int kvm_timer_enable(struct kvm_vcpu *vcpu)
1038 {
1039         struct arch_timer_cpu *timer = vcpu_timer(vcpu);
1040         struct timer_map map;
1041         int ret;
1042
1043         if (timer->enabled)
1044                 return 0;
1045
1046         /* Without a VGIC we do not map virtual IRQs to physical IRQs */
1047         if (!irqchip_in_kernel(vcpu->kvm))
1048                 goto no_vgic;
1049
1050         if (!vgic_initialized(vcpu->kvm))
1051                 return -ENODEV;
1052
1053         if (!timer_irqs_are_valid(vcpu)) {
1054                 kvm_debug("incorrectly configured timer irqs\n");
1055                 return -EINVAL;
1056         }
1057
1058         get_timer_map(vcpu, &map);
1059
1060         ret = kvm_vgic_map_phys_irq(vcpu,
1061                                     map.direct_vtimer->host_timer_irq,
1062                                     map.direct_vtimer->irq.irq,
1063                                     kvm_arch_timer_get_input_level);
1064         if (ret)
1065                 return ret;
1066
1067         if (map.direct_ptimer) {
1068                 ret = kvm_vgic_map_phys_irq(vcpu,
1069                                             map.direct_ptimer->host_timer_irq,
1070                                             map.direct_ptimer->irq.irq,
1071                                             kvm_arch_timer_get_input_level);
1072         }
1073
1074         if (ret)
1075                 return ret;
1076
1077 no_vgic:
1078         timer->enabled = 1;
1079         return 0;
1080 }
1081
1082 /*
1083  * On VHE system, we only need to configure the EL2 timer trap register once,
1084  * not for every world switch.
1085  * The host kernel runs at EL2 with HCR_EL2.TGE == 1,
1086  * and this makes those bits have no effect for the host kernel execution.
1087  */
1088 void kvm_timer_init_vhe(void)
1089 {
1090         /* When HCR_EL2.E2H ==1, EL1PCEN and EL1PCTEN are shifted by 10 */
1091         u32 cnthctl_shift = 10;
1092         u64 val;
1093
1094         /*
1095          * VHE systems allow the guest direct access to the EL1 physical
1096          * timer/counter.
1097          */
1098         val = read_sysreg(cnthctl_el2);
1099         val |= (CNTHCTL_EL1PCEN << cnthctl_shift);
1100         val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
1101         write_sysreg(val, cnthctl_el2);
1102 }
1103
1104 static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
1105 {
1106         struct kvm_vcpu *vcpu;
1107         int i;
1108
1109         kvm_for_each_vcpu(i, vcpu, kvm) {
1110                 vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
1111                 vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
1112         }
1113 }
1114
1115 int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1116 {
1117         int __user *uaddr = (int __user *)(long)attr->addr;
1118         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
1119         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
1120         int irq;
1121
1122         if (!irqchip_in_kernel(vcpu->kvm))
1123                 return -EINVAL;
1124
1125         if (get_user(irq, uaddr))
1126                 return -EFAULT;
1127
1128         if (!(irq_is_ppi(irq)))
1129                 return -EINVAL;
1130
1131         if (vcpu->arch.timer_cpu.enabled)
1132                 return -EBUSY;
1133
1134         switch (attr->attr) {
1135         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1136                 set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
1137                 break;
1138         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1139                 set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
1140                 break;
1141         default:
1142                 return -ENXIO;
1143         }
1144
1145         return 0;
1146 }
1147
1148 int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1149 {
1150         int __user *uaddr = (int __user *)(long)attr->addr;
1151         struct arch_timer_context *timer;
1152         int irq;
1153
1154         switch (attr->attr) {
1155         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1156                 timer = vcpu_vtimer(vcpu);
1157                 break;
1158         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1159                 timer = vcpu_ptimer(vcpu);
1160                 break;
1161         default:
1162                 return -ENXIO;
1163         }
1164
1165         irq = timer->irq.irq;
1166         return put_user(irq, uaddr);
1167 }
1168
1169 int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
1170 {
1171         switch (attr->attr) {
1172         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
1173         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
1174                 return 0;
1175         }
1176
1177         return -ENXIO;
1178 }