drivers/clocksource/arm_arch_timer.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/drivers/clocksource/arm_arch_timer.c
   4  *
   5  *  Copyright (C) 2011 ARM Ltd.
   6  *  All Rights Reserved
   7  */
   8
   9 #define pr_fmt(fmt)     "arch_timer: " fmt
  10
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/clocksource.h>
  19 #include <linux/clocksource_ids.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/kstrtox.h>
  22 #include <linux/of_irq.h>
  23 #include <linux/of_address.h>
  24 #include <linux/io.h>
  25 #include <linux/slab.h>
  26 #include <linux/sched/clock.h>
  27 #include <linux/sched_clock.h>
  28 #include <linux/acpi.h>
  29 #include <linux/arm-smccc.h>
  30 #include <linux/ptp_kvm.h>
  31
  32 #include <asm/arch_timer.h>
  33 #include <asm/virt.h>
  34
  35 #include <clocksource/arm_arch_timer.h>
  36
  37 #define CNTTIDR         0x08
  38 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  39
  40 #define CNTACR(n)       (0x40 + ((n) * 4))
  41 #define CNTACR_RPCT     BIT(0)
  42 #define CNTACR_RVCT     BIT(1)
  43 #define CNTACR_RFRQ     BIT(2)
  44 #define CNTACR_RVOFF    BIT(3)
  45 #define CNTACR_RWVT     BIT(4)
  46 #define CNTACR_RWPT     BIT(5)
  47
  48 #define CNTPCT_LO       0x00
  49 #define CNTVCT_LO       0x08
  50 #define CNTFRQ          0x10
  51 #define CNTP_CVAL_LO    0x20
  52 #define CNTP_CTL        0x2c
  53 #define CNTV_CVAL_LO    0x30
  54 #define CNTV_CTL        0x3c
  55
  56 /*
  57  * The minimum amount of time a generic counter is guaranteed to not roll over
  58  * (40 years)
  59  */
  60 #define MIN_ROLLOVER_SECS       (40ULL * 365 * 24 * 3600)
  61
  62 static unsigned arch_timers_present __initdata;
  63
  64 struct arch_timer {
  65         void __iomem *base;
  66         struct clock_event_device evt;
  67 };
  68
  69 static struct arch_timer *arch_timer_mem __ro_after_init;
  70
  71 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  72
  73 static u32 arch_timer_rate __ro_after_init;
  74 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI] __ro_after_init;
  75
  76 static const char *arch_timer_ppi_names[ARCH_TIMER_MAX_TIMER_PPI] = {
  77         [ARCH_TIMER_PHYS_SECURE_PPI]    = "sec-phys",
  78         [ARCH_TIMER_PHYS_NONSECURE_PPI] = "phys",
  79         [ARCH_TIMER_VIRT_PPI]           = "virt",
  80         [ARCH_TIMER_HYP_PPI]            = "hyp-phys",
  81         [ARCH_TIMER_HYP_VIRT_PPI]       = "hyp-virt",
  82 };
  83
  84 static struct clock_event_device __percpu *arch_timer_evt;
  85
  86 static enum arch_timer_ppi_nr arch_timer_uses_ppi __ro_after_init = ARCH_TIMER_VIRT_PPI;
  87 static bool arch_timer_c3stop __ro_after_init;
  88 static bool arch_timer_mem_use_virtual __ro_after_init;
  89 static bool arch_counter_suspend_stop __ro_after_init;
  90 #ifdef CONFIG_GENERIC_GETTIMEOFDAY
  91 static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;
  92 #else
  93 static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;
  94 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
  95
  96 static cpumask_t evtstrm_available = CPU_MASK_NONE;
  97 static bool evtstrm_enable __ro_after_init = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  98
  99 static int __init early_evtstrm_cfg(char *buf)
 100 {
 101         return kstrtobool(buf, &evtstrm_enable);
 102 }
 103 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
 104
 105 /*
 106  * Makes an educated guess at a valid counter width based on the Generic Timer
 107  * specification. Of note:
 108  *   1) the system counter is at least 56 bits wide
 109  *   2) a roll-over time of not less than 40 years
 110  *
 111  * See 'ARM DDI 0487G.a D11.1.2 ("The system counter")' for more details.
 112  */
 113 static int arch_counter_get_width(void)
 114 {
 115         u64 min_cycles = MIN_ROLLOVER_SECS * arch_timer_rate;
 116
 117         /* guarantee the returned width is within the valid range */
 118         return clamp_val(ilog2(min_cycles - 1) + 1, 56, 64);
 119 }
 120
 121 /*
 122  * Architected system timer support.
 123  */
 124
 125 static __always_inline
 126 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u64 val,
 127                           struct clock_event_device *clk)
 128 {
 129         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 130                 struct arch_timer *timer = to_arch_timer(clk);
 131                 switch (reg) {
 132                 case ARCH_TIMER_REG_CTRL:
 133                         writel_relaxed((u32)val, timer->base + CNTP_CTL);
 134                         break;
 135                 case ARCH_TIMER_REG_CVAL:
 136                         /*
 137                          * Not guaranteed to be atomic, so the timer
 138                          * must be disabled at this point.
 139                          */
 140                         writeq_relaxed(val, timer->base + CNTP_CVAL_LO);
 141                         break;
 142                 default:
 143                         BUILD_BUG();
 144                 }
 145         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 146                 struct arch_timer *timer = to_arch_timer(clk);
 147                 switch (reg) {
 148                 case ARCH_TIMER_REG_CTRL:
 149                         writel_relaxed((u32)val, timer->base + CNTV_CTL);
 150                         break;
 151                 case ARCH_TIMER_REG_CVAL:
 152                         /* Same restriction as above */
 153                         writeq_relaxed(val, timer->base + CNTV_CVAL_LO);
 154                         break;
 155                 default:
 156                         BUILD_BUG();
 157                 }
 158         } else {
 159                 arch_timer_reg_write_cp15(access, reg, val);
 160         }
 161 }
 162
 163 static __always_inline
 164 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 165                         struct clock_event_device *clk)
 166 {
 167         u32 val;
 168
 169         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 170                 struct arch_timer *timer = to_arch_timer(clk);
 171                 switch (reg) {
 172                 case ARCH_TIMER_REG_CTRL:
 173                         val = readl_relaxed(timer->base + CNTP_CTL);
 174                         break;
 175                 default:
 176                         BUILD_BUG();
 177                 }
 178         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 179                 struct arch_timer *timer = to_arch_timer(clk);
 180                 switch (reg) {
 181                 case ARCH_TIMER_REG_CTRL:
 182                         val = readl_relaxed(timer->base + CNTV_CTL);
 183                         break;
 184                 default:
 185                         BUILD_BUG();
 186                 }
 187         } else {
 188                 val = arch_timer_reg_read_cp15(access, reg);
 189         }
 190
 191         return val;
 192 }
 193
 194 static noinstr u64 raw_counter_get_cntpct_stable(void)
 195 {
 196         return __arch_counter_get_cntpct_stable();
 197 }
 198
 199 static notrace u64 arch_counter_get_cntpct_stable(void)
 200 {
 201         u64 val;
 202         preempt_disable_notrace();
 203         val = __arch_counter_get_cntpct_stable();
 204         preempt_enable_notrace();
 205         return val;
 206 }
 207
 208 static noinstr u64 arch_counter_get_cntpct(void)
 209 {
 210         return __arch_counter_get_cntpct();
 211 }
 212
 213 static noinstr u64 raw_counter_get_cntvct_stable(void)
 214 {
 215         return __arch_counter_get_cntvct_stable();
 216 }
 217
 218 static notrace u64 arch_counter_get_cntvct_stable(void)
 219 {
 220         u64 val;
 221         preempt_disable_notrace();
 222         val = __arch_counter_get_cntvct_stable();
 223         preempt_enable_notrace();
 224         return val;
 225 }
 226
 227 static noinstr u64 arch_counter_get_cntvct(void)
 228 {
 229         return __arch_counter_get_cntvct();
 230 }
 231
 232 /*
 233  * Default to cp15 based access because arm64 uses this function for
 234  * sched_clock() before DT is probed and the cp15 method is guaranteed
 235  * to exist on arm64. arm doesn't use this before DT is probed so even
 236  * if we don't have the cp15 accessors we won't have a problem.
 237  */
 238 u64 (*arch_timer_read_counter)(void) __ro_after_init = arch_counter_get_cntvct;
 239 EXPORT_SYMBOL_GPL(arch_timer_read_counter);
 240
 241 static u64 arch_counter_read(struct clocksource *cs)
 242 {
 243         return arch_timer_read_counter();
 244 }
 245
 246 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 247 {
 248         return arch_timer_read_counter();
 249 }
 250
 251 static struct clocksource clocksource_counter = {
 252         .name   = "arch_sys_counter",
 253         .id     = CSID_ARM_ARCH_COUNTER,
 254         .rating = 400,
 255         .read   = arch_counter_read,
 256         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 257 };
 258
 259 static struct cyclecounter cyclecounter __ro_after_init = {
 260         .read   = arch_counter_read_cc,
 261 };
 262
 263 struct ate_acpi_oem_info {
 264         char oem_id[ACPI_OEM_ID_SIZE + 1];
 265         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
 266         u32 oem_revision;
 267 };
 268
 269 #ifdef CONFIG_FSL_ERRATUM_A008585
 270 /*
 271  * The number of retries is an arbitrary value well beyond the highest number
 272  * of iterations the loop has been observed to take.
 273  */
 274 #define __fsl_a008585_read_reg(reg) ({                  \
 275         u64 _old, _new;                                 \
 276         int _retries = 200;                             \
 277                                                         \
 278         do {                                            \
 279                 _old = read_sysreg(reg);                \
 280                 _new = read_sysreg(reg);                \
 281                 _retries--;                             \
 282         } while (unlikely(_old != _new) && _retries);   \
 283                                                         \
 284         WARN_ON_ONCE(!_retries);                        \
 285         _new;                                           \
 286 })
 287
 288 static u64 notrace fsl_a008585_read_cntpct_el0(void)
 289 {
 290         return __fsl_a008585_read_reg(cntpct_el0);
 291 }
 292
 293 static u64 notrace fsl_a008585_read_cntvct_el0(void)
 294 {
 295         return __fsl_a008585_read_reg(cntvct_el0);
 296 }
 297 #endif
 298
 299 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 300 /*
 301  * Verify whether the value of the second read is larger than the first by
 302  * less than 32 is the only way to confirm the value is correct, so clear the
 303  * lower 5 bits to check whether the difference is greater than 32 or not.
 304  * Theoretically the erratum should not occur more than twice in succession
 305  * when reading the system counter, but it is possible that some interrupts
 306  * may lead to more than twice read errors, triggering the warning, so setting
 307  * the number of retries far beyond the number of iterations the loop has been
 308  * observed to take.
 309  */
 310 #define __hisi_161010101_read_reg(reg) ({                               \
 311         u64 _old, _new;                                         \
 312         int _retries = 50;                                      \
 313                                                                 \
 314         do {                                                    \
 315                 _old = read_sysreg(reg);                        \
 316                 _new = read_sysreg(reg);                        \
 317                 _retries--;                                     \
 318         } while (unlikely((_new - _old) >> 5) && _retries);     \
 319                                                                 \
 320         WARN_ON_ONCE(!_retries);                                \
 321         _new;                                                   \
 322 })
 323
 324 static u64 notrace hisi_161010101_read_cntpct_el0(void)
 325 {
 326         return __hisi_161010101_read_reg(cntpct_el0);
 327 }
 328
 329 static u64 notrace hisi_161010101_read_cntvct_el0(void)
 330 {
 331         return __hisi_161010101_read_reg(cntvct_el0);
 332 }
 333
 334 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
 335         /*
 336          * Note that trailing spaces are required to properly match
 337          * the OEM table information.
 338          */
 339         {
 340                 .oem_id         = "HISI  ",
 341                 .oem_table_id   = "HIP05   ",
 342                 .oem_revision   = 0,
 343         },
 344         {
 345                 .oem_id         = "HISI  ",
 346                 .oem_table_id   = "HIP06   ",
 347                 .oem_revision   = 0,
 348         },
 349         {
 350                 .oem_id         = "HISI  ",
 351                 .oem_table_id   = "HIP07   ",
 352                 .oem_revision   = 0,
 353         },
 354         { /* Sentinel indicating the end of the OEM array */ },
 355 };
 356 #endif
 357
 358 #ifdef CONFIG_ARM64_ERRATUM_858921
 359 static u64 notrace arm64_858921_read_cntpct_el0(void)
 360 {
 361         u64 old, new;
 362
 363         old = read_sysreg(cntpct_el0);
 364         new = read_sysreg(cntpct_el0);
 365         return (((old ^ new) >> 32) & 1) ? old : new;
 366 }
 367
 368 static u64 notrace arm64_858921_read_cntvct_el0(void)
 369 {
 370         u64 old, new;
 371
 372         old = read_sysreg(cntvct_el0);
 373         new = read_sysreg(cntvct_el0);
 374         return (((old ^ new) >> 32) & 1) ? old : new;
 375 }
 376 #endif
 377
 378 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 379 /*
 380  * The low bits of the counter registers are indeterminate while bit 10 or
 381  * greater is rolling over. Since the counter value can jump both backward
 382  * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
 383  * with all ones or all zeros in the low bits. Bound the loop by the maximum
 384  * number of CPU cycles in 3 consecutive 24 MHz counter periods.
 385  */
 386 #define __sun50i_a64_read_reg(reg) ({                                   \
 387         u64 _val;                                                       \
 388         int _retries = 150;                                             \
 389                                                                         \
 390         do {                                                            \
 391                 _val = read_sysreg(reg);                                \
 392                 _retries--;                                             \
 393         } while (((_val + 1) & GENMASK(8, 0)) <= 1 && _retries);        \
 394                                                                         \
 395         WARN_ON_ONCE(!_retries);                                        \
 396         _val;                                                           \
 397 })
 398
 399 static u64 notrace sun50i_a64_read_cntpct_el0(void)
 400 {
 401         return __sun50i_a64_read_reg(cntpct_el0);
 402 }
 403
 404 static u64 notrace sun50i_a64_read_cntvct_el0(void)
 405 {
 406         return __sun50i_a64_read_reg(cntvct_el0);
 407 }
 408 #endif
 409
 410 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
 411 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
 412 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
 413
 414 static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
 415
 416 /*
 417  * Force the inlining of this function so that the register accesses
 418  * can be themselves correctly inlined.
 419  */
 420 static __always_inline
 421 void erratum_set_next_event_generic(const int access, unsigned long evt,
 422                                     struct clock_event_device *clk)
 423 {
 424         unsigned long ctrl;
 425         u64 cval;
 426
 427         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 428         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 429         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 430
 431         if (access == ARCH_TIMER_PHYS_ACCESS) {
 432                 cval = evt + arch_counter_get_cntpct_stable();
 433                 write_sysreg(cval, cntp_cval_el0);
 434         } else {
 435                 cval = evt + arch_counter_get_cntvct_stable();
 436                 write_sysreg(cval, cntv_cval_el0);
 437         }
 438
 439         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 440 }
 441
 442 static __maybe_unused int erratum_set_next_event_virt(unsigned long evt,
 443                                             struct clock_event_device *clk)
 444 {
 445         erratum_set_next_event_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 446         return 0;
 447 }
 448
 449 static __maybe_unused int erratum_set_next_event_phys(unsigned long evt,
 450                                             struct clock_event_device *clk)
 451 {
 452         erratum_set_next_event_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 453         return 0;
 454 }
 455
 456 static const struct arch_timer_erratum_workaround ool_workarounds[] = {
 457 #ifdef CONFIG_FSL_ERRATUM_A008585
 458         {
 459                 .match_type = ate_match_dt,
 460                 .id = "fsl,erratum-a008585",
 461                 .desc = "Freescale erratum a005858",
 462                 .read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
 463                 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
 464                 .set_next_event_phys = erratum_set_next_event_phys,
 465                 .set_next_event_virt = erratum_set_next_event_virt,
 466         },
 467 #endif
 468 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 469         {
 470                 .match_type = ate_match_dt,
 471                 .id = "hisilicon,erratum-161010101",
 472                 .desc = "HiSilicon erratum 161010101",
 473                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 474                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 475                 .set_next_event_phys = erratum_set_next_event_phys,
 476                 .set_next_event_virt = erratum_set_next_event_virt,
 477         },
 478         {
 479                 .match_type = ate_match_acpi_oem_info,
 480                 .id = hisi_161010101_oem_info,
 481                 .desc = "HiSilicon erratum 161010101",
 482                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 483                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 484                 .set_next_event_phys = erratum_set_next_event_phys,
 485                 .set_next_event_virt = erratum_set_next_event_virt,
 486         },
 487 #endif
 488 #ifdef CONFIG_ARM64_ERRATUM_858921
 489         {
 490                 .match_type = ate_match_local_cap_id,
 491                 .id = (void *)ARM64_WORKAROUND_858921,
 492                 .desc = "ARM erratum 858921",
 493                 .read_cntpct_el0 = arm64_858921_read_cntpct_el0,
 494                 .read_cntvct_el0 = arm64_858921_read_cntvct_el0,
 495                 .set_next_event_phys = erratum_set_next_event_phys,
 496                 .set_next_event_virt = erratum_set_next_event_virt,
 497         },
 498 #endif
 499 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 500         {
 501                 .match_type = ate_match_dt,
 502                 .id = "allwinner,erratum-unknown1",
 503                 .desc = "Allwinner erratum UNKNOWN1",
 504                 .read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
 505                 .read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
 506                 .set_next_event_phys = erratum_set_next_event_phys,
 507                 .set_next_event_virt = erratum_set_next_event_virt,
 508         },
 509 #endif
 510 #ifdef CONFIG_ARM64_ERRATUM_1418040
 511         {
 512                 .match_type = ate_match_local_cap_id,
 513                 .id = (void *)ARM64_WORKAROUND_1418040,
 514                 .desc = "ARM erratum 1418040",
 515                 .disable_compat_vdso = true,
 516         },
 517 #endif
 518 };
 519
 520 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
 521                                const void *);
 522
 523 static
 524 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
 525                                  const void *arg)
 526 {
 527         const struct device_node *np = arg;
 528
 529         return of_property_read_bool(np, wa->id);
 530 }
 531
 532 static
 533 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
 534                                         const void *arg)
 535 {
 536         return this_cpu_has_cap((uintptr_t)wa->id);
 537 }
 538
 539
 540 static
 541 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
 542                                        const void *arg)
 543 {
 544         static const struct ate_acpi_oem_info empty_oem_info = {};
 545         const struct ate_acpi_oem_info *info = wa->id;
 546         const struct acpi_table_header *table = arg;
 547
 548         /* Iterate over the ACPI OEM info array, looking for a match */
 549         while (memcmp(info, &empty_oem_info, sizeof(*info))) {
 550                 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
 551                     !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
 552                     info->oem_revision == table->oem_revision)
 553                         return true;
 554
 555                 info++;
 556         }
 557
 558         return false;
 559 }
 560
 561 static const struct arch_timer_erratum_workaround *
 562 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
 563                           ate_match_fn_t match_fn,
 564                           void *arg)
 565 {
 566         int i;
 567
 568         for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
 569                 if (ool_workarounds[i].match_type != type)
 570                         continue;
 571
 572                 if (match_fn(&ool_workarounds[i], arg))
 573                         return &ool_workarounds[i];
 574         }
 575
 576         return NULL;
 577 }
 578
 579 static
 580 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
 581                                   bool local)
 582 {
 583         int i;
 584
 585         if (local) {
 586                 __this_cpu_write(timer_unstable_counter_workaround, wa);
 587         } else {
 588                 for_each_possible_cpu(i)
 589                         per_cpu(timer_unstable_counter_workaround, i) = wa;
 590         }
 591
 592         if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
 593                 atomic_set(&timer_unstable_counter_workaround_in_use, 1);
 594
 595         /*
 596          * Don't use the vdso fastpath if errata require using the
 597          * out-of-line counter accessor. We may change our mind pretty
 598          * late in the game (with a per-CPU erratum, for example), so
 599          * change both the default value and the vdso itself.
 600          */
 601         if (wa->read_cntvct_el0) {
 602                 clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
 603                 vdso_default = VDSO_CLOCKMODE_NONE;
 604         } else if (wa->disable_compat_vdso && vdso_default != VDSO_CLOCKMODE_NONE) {
 605                 vdso_default = VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT;
 606                 clocksource_counter.vdso_clock_mode = vdso_default;
 607         }
 608 }
 609
 610 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
 611                                             void *arg)
 612 {
 613         const struct arch_timer_erratum_workaround *wa, *__wa;
 614         ate_match_fn_t match_fn = NULL;
 615         bool local = false;
 616
 617         switch (type) {
 618         case ate_match_dt:
 619                 match_fn = arch_timer_check_dt_erratum;
 620                 break;
 621         case ate_match_local_cap_id:
 622                 match_fn = arch_timer_check_local_cap_erratum;
 623                 local = true;
 624                 break;
 625         case ate_match_acpi_oem_info:
 626                 match_fn = arch_timer_check_acpi_oem_erratum;
 627                 break;
 628         default:
 629                 WARN_ON(1);
 630                 return;
 631         }
 632
 633         wa = arch_timer_iterate_errata(type, match_fn, arg);
 634         if (!wa)
 635                 return;
 636
 637         __wa = __this_cpu_read(timer_unstable_counter_workaround);
 638         if (__wa && wa != __wa)
 639                 pr_warn("Can't enable workaround for %s (clashes with %s\n)",
 640                         wa->desc, __wa->desc);
 641
 642         if (__wa)
 643                 return;
 644
 645         arch_timer_enable_workaround(wa, local);
 646         pr_info("Enabling %s workaround for %s\n",
 647                 local ? "local" : "global", wa->desc);
 648 }
 649
 650 static bool arch_timer_this_cpu_has_cntvct_wa(void)
 651 {
 652         return has_erratum_handler(read_cntvct_el0);
 653 }
 654
 655 static bool arch_timer_counter_has_wa(void)
 656 {
 657         return atomic_read(&timer_unstable_counter_workaround_in_use);
 658 }
 659 #else
 660 #define arch_timer_check_ool_workaround(t,a)            do { } while(0)
 661 #define arch_timer_this_cpu_has_cntvct_wa()             ({false;})
 662 #define arch_timer_counter_has_wa()                     ({false;})
 663 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
 664
 665 static __always_inline irqreturn_t timer_handler(const int access,
 666                                         struct clock_event_device *evt)
 667 {
 668         unsigned long ctrl;
 669
 670         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 671         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 672                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 673                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 674                 evt->event_handler(evt);
 675                 return IRQ_HANDLED;
 676         }
 677
 678         return IRQ_NONE;
 679 }
 680
 681 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 682 {
 683         struct clock_event_device *evt = dev_id;
 684
 685         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 686 }
 687
 688 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 689 {
 690         struct clock_event_device *evt = dev_id;
 691
 692         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 693 }
 694
 695 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 696 {
 697         struct clock_event_device *evt = dev_id;
 698
 699         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 700 }
 701
 702 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 703 {
 704         struct clock_event_device *evt = dev_id;
 705
 706         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 707 }
 708
 709 static __always_inline int arch_timer_shutdown(const int access,
 710                                                struct clock_event_device *clk)
 711 {
 712         unsigned long ctrl;
 713
 714         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 715         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 716         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 717
 718         return 0;
 719 }
 720
 721 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 722 {
 723         return arch_timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 724 }
 725
 726 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 727 {
 728         return arch_timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 729 }
 730
 731 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 732 {
 733         return arch_timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 734 }
 735
 736 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 737 {
 738         return arch_timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 739 }
 740
 741 static __always_inline void set_next_event(const int access, unsigned long evt,
 742                                            struct clock_event_device *clk)
 743 {
 744         unsigned long ctrl;
 745         u64 cnt;
 746
 747         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 748         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 749         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 750
 751         if (access == ARCH_TIMER_PHYS_ACCESS)
 752                 cnt = __arch_counter_get_cntpct();
 753         else
 754                 cnt = __arch_counter_get_cntvct();
 755
 756         arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
 757         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 758 }
 759
 760 static int arch_timer_set_next_event_virt(unsigned long evt,
 761                                           struct clock_event_device *clk)
 762 {
 763         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 764         return 0;
 765 }
 766
 767 static int arch_timer_set_next_event_phys(unsigned long evt,
 768                                           struct clock_event_device *clk)
 769 {
 770         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 771         return 0;
 772 }
 773
 774 static noinstr u64 arch_counter_get_cnt_mem(struct arch_timer *t, int offset_lo)
 775 {
 776         u32 cnt_lo, cnt_hi, tmp_hi;
 777
 778         do {
 779                 cnt_hi = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo + 4));
 780                 cnt_lo = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo));
 781                 tmp_hi = __le32_to_cpu((__le32 __force)__raw_readl(t->base + offset_lo + 4));
 782         } while (cnt_hi != tmp_hi);
 783
 784         return ((u64) cnt_hi << 32) | cnt_lo;
 785 }
 786
 787 static __always_inline void set_next_event_mem(const int access, unsigned long evt,
 788                                            struct clock_event_device *clk)
 789 {
 790         struct arch_timer *timer = to_arch_timer(clk);
 791         unsigned long ctrl;
 792         u64 cnt;
 793
 794         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 795         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 796         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 797
 798         if (access ==  ARCH_TIMER_MEM_VIRT_ACCESS)
 799                 cnt = arch_counter_get_cnt_mem(timer, CNTVCT_LO);
 800         else
 801                 cnt = arch_counter_get_cnt_mem(timer, CNTPCT_LO);
 802
 803         arch_timer_reg_write(access, ARCH_TIMER_REG_CVAL, evt + cnt, clk);
 804         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 805 }
 806
 807 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 808                                               struct clock_event_device *clk)
 809 {
 810         set_next_event_mem(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 811         return 0;
 812 }
 813
 814 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 815                                               struct clock_event_device *clk)
 816 {
 817         set_next_event_mem(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 818         return 0;
 819 }
 820
 821 static u64 __arch_timer_check_delta(void)
 822 {
 823 #ifdef CONFIG_ARM64
 824         const struct midr_range broken_cval_midrs[] = {
 825                 /*
 826                  * XGene-1 implements CVAL in terms of TVAL, meaning
 827                  * that the maximum timer range is 32bit. Shame on them.
 828                  *
 829                  * Note that TVAL is signed, thus has only 31 of its
 830                  * 32 bits to express magnitude.
 831                  */
 832                 MIDR_ALL_VERSIONS(MIDR_CPU_MODEL(ARM_CPU_IMP_APM,
 833                                                  APM_CPU_PART_POTENZA)),
 834                 {},
 835         };
 836
 837         if (is_midr_in_range_list(read_cpuid_id(), broken_cval_midrs)) {
 838                 pr_warn_once("Broken CNTx_CVAL_EL1, using 31 bit TVAL instead.\n");
 839                 return CLOCKSOURCE_MASK(31);
 840         }
 841 #endif
 842         return CLOCKSOURCE_MASK(arch_counter_get_width());
 843 }
 844
 845 static void __arch_timer_setup(unsigned type,
 846                                struct clock_event_device *clk)
 847 {
 848         u64 max_delta;
 849
 850         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 851
 852         if (type == ARCH_TIMER_TYPE_CP15) {
 853                 typeof(clk->set_next_event) sne;
 854
 855                 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
 856
 857                 if (arch_timer_c3stop)
 858                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 859                 clk->name = "arch_sys_timer";
 860                 clk->rating = 450;
 861                 clk->cpumask = cpumask_of(smp_processor_id());
 862                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 863                 switch (arch_timer_uses_ppi) {
 864                 case ARCH_TIMER_VIRT_PPI:
 865                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 866                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 867                         sne = erratum_handler(set_next_event_virt);
 868                         break;
 869                 case ARCH_TIMER_PHYS_SECURE_PPI:
 870                 case ARCH_TIMER_PHYS_NONSECURE_PPI:
 871                 case ARCH_TIMER_HYP_PPI:
 872                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 873                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 874                         sne = erratum_handler(set_next_event_phys);
 875                         break;
 876                 default:
 877                         BUG();
 878                 }
 879
 880                 clk->set_next_event = sne;
 881                 max_delta = __arch_timer_check_delta();
 882         } else {
 883                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 884                 clk->name = "arch_mem_timer";
 885                 clk->rating = 400;
 886                 clk->cpumask = cpu_possible_mask;
 887                 if (arch_timer_mem_use_virtual) {
 888                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 889                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 890                         clk->set_next_event =
 891                                 arch_timer_set_next_event_virt_mem;
 892                 } else {
 893                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 894                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 895                         clk->set_next_event =
 896                                 arch_timer_set_next_event_phys_mem;
 897                 }
 898
 899                 max_delta = CLOCKSOURCE_MASK(56);
 900         }
 901
 902         clk->set_state_shutdown(clk);
 903
 904         clockevents_config_and_register(clk, arch_timer_rate, 0xf, max_delta);
 905 }
 906
 907 static void arch_timer_evtstrm_enable(unsigned int divider)
 908 {
 909         u32 cntkctl = arch_timer_get_cntkctl();
 910
 911 #ifdef CONFIG_ARM64
 912         /* ECV is likely to require a large divider. Use the EVNTIS flag. */
 913         if (cpus_have_const_cap(ARM64_HAS_ECV) && divider > 15) {
 914                 cntkctl |= ARCH_TIMER_EVT_INTERVAL_SCALE;
 915                 divider -= 8;
 916         }
 917 #endif
 918
 919         divider = min(divider, 15U);
 920         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 921         /* Set the divider and enable virtual event stream */
 922         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 923                         | ARCH_TIMER_VIRT_EVT_EN;
 924         arch_timer_set_cntkctl(cntkctl);
 925         arch_timer_set_evtstrm_feature();
 926         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
 927 }
 928
 929 static void arch_timer_configure_evtstream(void)
 930 {
 931         int evt_stream_div, lsb;
 932
 933         /*
 934          * As the event stream can at most be generated at half the frequency
 935          * of the counter, use half the frequency when computing the divider.
 936          */
 937         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ / 2;
 938
 939         /*
 940          * Find the closest power of two to the divisor. If the adjacent bit
 941          * of lsb (last set bit, starts from 0) is set, then we use (lsb + 1).
 942          */
 943         lsb = fls(evt_stream_div) - 1;
 944         if (lsb > 0 && (evt_stream_div & BIT(lsb - 1)))
 945                 lsb++;
 946
 947         /* enable event stream */
 948         arch_timer_evtstrm_enable(max(0, lsb));
 949 }
 950
 951 static void arch_counter_set_user_access(void)
 952 {
 953         u32 cntkctl = arch_timer_get_cntkctl();
 954
 955         /* Disable user access to the timers and both counters */
 956         /* Also disable virtual event stream */
 957         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 958                         | ARCH_TIMER_USR_VT_ACCESS_EN
 959                         | ARCH_TIMER_USR_VCT_ACCESS_EN
 960                         | ARCH_TIMER_VIRT_EVT_EN
 961                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 962
 963         /*
 964          * Enable user access to the virtual counter if it doesn't
 965          * need to be workaround. The vdso may have been already
 966          * disabled though.
 967          */
 968         if (arch_timer_this_cpu_has_cntvct_wa())
 969                 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
 970         else
 971                 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 972
 973         arch_timer_set_cntkctl(cntkctl);
 974 }
 975
 976 static bool arch_timer_has_nonsecure_ppi(void)
 977 {
 978         return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
 979                 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 980 }
 981
 982 static u32 check_ppi_trigger(int irq)
 983 {
 984         u32 flags = irq_get_trigger_type(irq);
 985
 986         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 987                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 988                 pr_warn("WARNING: Please fix your firmware\n");
 989                 flags = IRQF_TRIGGER_LOW;
 990         }
 991
 992         return flags;
 993 }
 994
 995 static int arch_timer_starting_cpu(unsigned int cpu)
 996 {
 997         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 998         u32 flags;
 999
1000         __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
1001
1002         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
1003         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
1004
1005         if (arch_timer_has_nonsecure_ppi()) {
1006                 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1007                 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1008                                   flags);
1009         }
1010
1011         arch_counter_set_user_access();
1012         if (evtstrm_enable)
1013                 arch_timer_configure_evtstream();
1014
1015         return 0;
1016 }
1017
1018 static int validate_timer_rate(void)
1019 {
1020         if (!arch_timer_rate)
1021                 return -EINVAL;
1022
1023         /* Arch timer frequency < 1MHz can cause trouble */
1024         WARN_ON(arch_timer_rate < 1000000);
1025
1026         return 0;
1027 }
1028
1029 /*
1030  * For historical reasons, when probing with DT we use whichever (non-zero)
1031  * rate was probed first, and don't verify that others match. If the first node
1032  * probed has a clock-frequency property, this overrides the HW register.
1033  */
1034 static void __init arch_timer_of_configure_rate(u32 rate, struct device_node *np)
1035 {
1036         /* Who has more than one independent system counter? */
1037         if (arch_timer_rate)
1038                 return;
1039
1040         if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
1041                 arch_timer_rate = rate;
1042
1043         /* Check the timer frequency. */
1044         if (validate_timer_rate())
1045                 pr_warn("frequency not available\n");
1046 }
1047
1048 static void __init arch_timer_banner(unsigned type)
1049 {
1050         pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
1051                 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
1052                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
1053                         " and " : "",
1054                 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
1055                 (unsigned long)arch_timer_rate / 1000000,
1056                 (unsigned long)(arch_timer_rate / 10000) % 100,
1057                 type & ARCH_TIMER_TYPE_CP15 ?
1058                         (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
1059                         "",
1060                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
1061                 type & ARCH_TIMER_TYPE_MEM ?
1062                         arch_timer_mem_use_virtual ? "virt" : "phys" :
1063                         "");
1064 }
1065
1066 u32 arch_timer_get_rate(void)
1067 {
1068         return arch_timer_rate;
1069 }
1070
1071 bool arch_timer_evtstrm_available(void)
1072 {
1073         /*
1074          * We might get called from a preemptible context. This is fine
1075          * because availability of the event stream should be always the same
1076          * for a preemptible context and context where we might resume a task.
1077          */
1078         return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
1079 }
1080
1081 static noinstr u64 arch_counter_get_cntvct_mem(void)
1082 {
1083         return arch_counter_get_cnt_mem(arch_timer_mem, CNTVCT_LO);
1084 }
1085
1086 static struct arch_timer_kvm_info arch_timer_kvm_info;
1087
1088 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
1089 {
1090         return &arch_timer_kvm_info;
1091 }
1092
1093 static void __init arch_counter_register(unsigned type)
1094 {
1095         u64 (*scr)(void);
1096         u64 start_count;
1097         int width;
1098
1099         /* Register the CP15 based counter if we have one */
1100         if (type & ARCH_TIMER_TYPE_CP15) {
1101                 u64 (*rd)(void);
1102
1103                 if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
1104                     arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
1105                         if (arch_timer_counter_has_wa()) {
1106                                 rd = arch_counter_get_cntvct_stable;
1107                                 scr = raw_counter_get_cntvct_stable;
1108                         } else {
1109                                 rd = arch_counter_get_cntvct;
1110                                 scr = arch_counter_get_cntvct;
1111                         }
1112                 } else {
1113                         if (arch_timer_counter_has_wa()) {
1114                                 rd = arch_counter_get_cntpct_stable;
1115                                 scr = raw_counter_get_cntpct_stable;
1116                         } else {
1117                                 rd = arch_counter_get_cntpct;
1118                                 scr = arch_counter_get_cntpct;
1119                         }
1120                 }
1121
1122                 arch_timer_read_counter = rd;
1123                 clocksource_counter.vdso_clock_mode = vdso_default;
1124         } else {
1125                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
1126                 scr = arch_counter_get_cntvct_mem;
1127         }
1128
1129         width = arch_counter_get_width();
1130         clocksource_counter.mask = CLOCKSOURCE_MASK(width);
1131         cyclecounter.mask = CLOCKSOURCE_MASK(width);
1132
1133         if (!arch_counter_suspend_stop)
1134                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
1135         start_count = arch_timer_read_counter();
1136         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
1137         cyclecounter.mult = clocksource_counter.mult;
1138         cyclecounter.shift = clocksource_counter.shift;
1139         timecounter_init(&arch_timer_kvm_info.timecounter,
1140                          &cyclecounter, start_count);
1141
1142         sched_clock_register(scr, width, arch_timer_rate);
1143 }
1144
1145 static void arch_timer_stop(struct clock_event_device *clk)
1146 {
1147         pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1148
1149         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
1150         if (arch_timer_has_nonsecure_ppi())
1151                 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1152
1153         clk->set_state_shutdown(clk);
1154 }
1155
1156 static int arch_timer_dying_cpu(unsigned int cpu)
1157 {
1158         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1159
1160         cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1161
1162         arch_timer_stop(clk);
1163         return 0;
1164 }
1165
1166 #ifdef CONFIG_CPU_PM
1167 static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
1168 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
1169                                     unsigned long action, void *hcpu)
1170 {
1171         if (action == CPU_PM_ENTER) {
1172                 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1173
1174                 cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1175         } else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1176                 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1177
1178                 if (arch_timer_have_evtstrm_feature())
1179                         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
1180         }
1181         return NOTIFY_OK;
1182 }
1183
1184 static struct notifier_block arch_timer_cpu_pm_notifier = {
1185         .notifier_call = arch_timer_cpu_pm_notify,
1186 };
1187
1188 static int __init arch_timer_cpu_pm_init(void)
1189 {
1190         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
1191 }
1192
1193 static void __init arch_timer_cpu_pm_deinit(void)
1194 {
1195         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
1196 }
1197
1198 #else
1199 static int __init arch_timer_cpu_pm_init(void)
1200 {
1201         return 0;
1202 }
1203
1204 static void __init arch_timer_cpu_pm_deinit(void)
1205 {
1206 }
1207 #endif
1208
1209 static int __init arch_timer_register(void)
1210 {
1211         int err;
1212         int ppi;
1213
1214         arch_timer_evt = alloc_percpu(struct clock_event_device);
1215         if (!arch_timer_evt) {
1216                 err = -ENOMEM;
1217                 goto out;
1218         }
1219
1220         ppi = arch_timer_ppi[arch_timer_uses_ppi];
1221         switch (arch_timer_uses_ppi) {
1222         case ARCH_TIMER_VIRT_PPI:
1223                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
1224                                          "arch_timer", arch_timer_evt);
1225                 break;
1226         case ARCH_TIMER_PHYS_SECURE_PPI:
1227         case ARCH_TIMER_PHYS_NONSECURE_PPI:
1228                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1229                                          "arch_timer", arch_timer_evt);
1230                 if (!err && arch_timer_has_nonsecure_ppi()) {
1231                         ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1232                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
1233                                                  "arch_timer", arch_timer_evt);
1234                         if (err)
1235                                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1236                                                 arch_timer_evt);
1237                 }
1238                 break;
1239         case ARCH_TIMER_HYP_PPI:
1240                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1241                                          "arch_timer", arch_timer_evt);
1242                 break;
1243         default:
1244                 BUG();
1245         }
1246
1247         if (err) {
1248                 pr_err("can't register interrupt %d (%d)\n", ppi, err);
1249                 goto out_free;
1250         }
1251
1252         err = arch_timer_cpu_pm_init();
1253         if (err)
1254                 goto out_unreg_notify;
1255
1256         /* Register and immediately configure the timer on the boot CPU */
1257         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1258                                 "clockevents/arm/arch_timer:starting",
1259                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
1260         if (err)
1261                 goto out_unreg_cpupm;
1262         return 0;
1263
1264 out_unreg_cpupm:
1265         arch_timer_cpu_pm_deinit();
1266
1267 out_unreg_notify:
1268         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1269         if (arch_timer_has_nonsecure_ppi())
1270                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1271                                 arch_timer_evt);
1272
1273 out_free:
1274         free_percpu(arch_timer_evt);
1275 out:
1276         return err;
1277 }
1278
1279 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1280 {
1281         int ret;
1282         irq_handler_t func;
1283
1284         arch_timer_mem = kzalloc(sizeof(*arch_timer_mem), GFP_KERNEL);
1285         if (!arch_timer_mem)
1286                 return -ENOMEM;
1287
1288         arch_timer_mem->base = base;
1289         arch_timer_mem->evt.irq = irq;
1290         __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &arch_timer_mem->evt);
1291
1292         if (arch_timer_mem_use_virtual)
1293                 func = arch_timer_handler_virt_mem;
1294         else
1295                 func = arch_timer_handler_phys_mem;
1296
1297         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &arch_timer_mem->evt);
1298         if (ret) {
1299                 pr_err("Failed to request mem timer irq\n");
1300                 kfree(arch_timer_mem);
1301                 arch_timer_mem = NULL;
1302         }
1303
1304         return ret;
1305 }
1306
1307 static const struct of_device_id arch_timer_of_match[] __initconst = {
1308         { .compatible   = "arm,armv7-timer",    },
1309         { .compatible   = "arm,armv8-timer",    },
1310         {},
1311 };
1312
1313 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1314         { .compatible   = "arm,armv7-timer-mem", },
1315         {},
1316 };
1317
1318 static bool __init arch_timer_needs_of_probing(void)
1319 {
1320         struct device_node *dn;
1321         bool needs_probing = false;
1322         unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1323
1324         /* We have two timers, and both device-tree nodes are probed. */
1325         if ((arch_timers_present & mask) == mask)
1326                 return false;
1327
1328         /*
1329          * Only one type of timer is probed,
1330          * check if we have another type of timer node in device-tree.
1331          */
1332         if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1333                 dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1334         else
1335                 dn = of_find_matching_node(NULL, arch_timer_of_match);
1336
1337         if (dn && of_device_is_available(dn))
1338                 needs_probing = true;
1339
1340         of_node_put(dn);
1341
1342         return needs_probing;
1343 }
1344
1345 static int __init arch_timer_common_init(void)
1346 {
1347         arch_timer_banner(arch_timers_present);
1348         arch_counter_register(arch_timers_present);
1349         return arch_timer_arch_init();
1350 }
1351
1352 /**
1353  * arch_timer_select_ppi() - Select suitable PPI for the current system.
1354  *
1355  * If HYP mode is available, we know that the physical timer
1356  * has been configured to be accessible from PL1. Use it, so
1357  * that a guest can use the virtual timer instead.
1358  *
1359  * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1360  * accesses to CNTP_*_EL1 registers are silently redirected to
1361  * their CNTHP_*_EL2 counterparts, and use a different PPI
1362  * number.
1363  *
1364  * If no interrupt provided for virtual timer, we'll have to
1365  * stick to the physical timer. It'd better be accessible...
1366  * For arm64 we never use the secure interrupt.
1367  *
1368  * Return: a suitable PPI type for the current system.
1369  */
1370 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1371 {
1372         if (is_kernel_in_hyp_mode())
1373                 return ARCH_TIMER_HYP_PPI;
1374
1375         if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1376                 return ARCH_TIMER_VIRT_PPI;
1377
1378         if (IS_ENABLED(CONFIG_ARM64))
1379                 return ARCH_TIMER_PHYS_NONSECURE_PPI;
1380
1381         return ARCH_TIMER_PHYS_SECURE_PPI;
1382 }
1383
1384 static void __init arch_timer_populate_kvm_info(void)
1385 {
1386         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1387         if (is_kernel_in_hyp_mode())
1388                 arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1389 }
1390
1391 static int __init arch_timer_of_init(struct device_node *np)
1392 {
1393         int i, irq, ret;
1394         u32 rate;
1395         bool has_names;
1396
1397         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1398                 pr_warn("multiple nodes in dt, skipping\n");
1399                 return 0;
1400         }
1401
1402         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1403
1404         has_names = of_property_read_bool(np, "interrupt-names");
1405
1406         for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++) {
1407                 if (has_names)
1408                         irq = of_irq_get_byname(np, arch_timer_ppi_names[i]);
1409                 else
1410                         irq = of_irq_get(np, i);
1411                 if (irq > 0)
1412                         arch_timer_ppi[i] = irq;
1413         }
1414
1415         arch_timer_populate_kvm_info();
1416
1417         rate = arch_timer_get_cntfrq();
1418         arch_timer_of_configure_rate(rate, np);
1419
1420         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1421
1422         /* Check for globally applicable workarounds */
1423         arch_timer_check_ool_workaround(ate_match_dt, np);
1424
1425         /*
1426          * If we cannot rely on firmware initializing the timer registers then
1427          * we should use the physical timers instead.
1428          */
1429         if (IS_ENABLED(CONFIG_ARM) &&
1430             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1431                 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1432         else
1433                 arch_timer_uses_ppi = arch_timer_select_ppi();
1434
1435         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1436                 pr_err("No interrupt available, giving up\n");
1437                 return -EINVAL;
1438         }
1439
1440         /* On some systems, the counter stops ticking when in suspend. */
1441         arch_counter_suspend_stop = of_property_read_bool(np,
1442                                                          "arm,no-tick-in-suspend");
1443
1444         ret = arch_timer_register();
1445         if (ret)
1446                 return ret;
1447
1448         if (arch_timer_needs_of_probing())
1449                 return 0;
1450
1451         return arch_timer_common_init();
1452 }
1453 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1454 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1455
1456 static u32 __init
1457 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1458 {
1459         void __iomem *base;
1460         u32 rate;
1461
1462         base = ioremap(frame->cntbase, frame->size);
1463         if (!base) {
1464                 pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1465                 return 0;
1466         }
1467
1468         rate = readl_relaxed(base + CNTFRQ);
1469
1470         iounmap(base);
1471
1472         return rate;
1473 }
1474
1475 static struct arch_timer_mem_frame * __init
1476 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1477 {
1478         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1479         void __iomem *cntctlbase;
1480         u32 cnttidr;
1481         int i;
1482
1483         cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1484         if (!cntctlbase) {
1485                 pr_err("Can't map CNTCTLBase @ %pa\n",
1486                         &timer_mem->cntctlbase);
1487                 return NULL;
1488         }
1489
1490         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1491
1492         /*
1493          * Try to find a virtual capable frame. Otherwise fall back to a
1494          * physical capable frame.
1495          */
1496         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1497                 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1498                              CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1499
1500                 frame = &timer_mem->frame[i];
1501                 if (!frame->valid)
1502                         continue;
1503
1504                 /* Try enabling everything, and see what sticks */
1505                 writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1506                 cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1507
1508                 if ((cnttidr & CNTTIDR_VIRT(i)) &&
1509                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1510                         best_frame = frame;
1511                         arch_timer_mem_use_virtual = true;
1512                         break;
1513                 }
1514
1515                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1516                         continue;
1517
1518                 best_frame = frame;
1519         }
1520
1521         iounmap(cntctlbase);
1522
1523         return best_frame;
1524 }
1525
1526 static int __init
1527 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1528 {
1529         void __iomem *base;
1530         int ret, irq = 0;
1531
1532         if (arch_timer_mem_use_virtual)
1533                 irq = frame->virt_irq;
1534         else
1535                 irq = frame->phys_irq;
1536
1537         if (!irq) {
1538                 pr_err("Frame missing %s irq.\n",
1539                        arch_timer_mem_use_virtual ? "virt" : "phys");
1540                 return -EINVAL;
1541         }
1542
1543         if (!request_mem_region(frame->cntbase, frame->size,
1544                                 "arch_mem_timer"))
1545                 return -EBUSY;
1546
1547         base = ioremap(frame->cntbase, frame->size);
1548         if (!base) {
1549                 pr_err("Can't map frame's registers\n");
1550                 return -ENXIO;
1551         }
1552
1553         ret = arch_timer_mem_register(base, irq);
1554         if (ret) {
1555                 iounmap(base);
1556                 return ret;
1557         }
1558
1559         arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1560
1561         return 0;
1562 }
1563
1564 static int __init arch_timer_mem_of_init(struct device_node *np)
1565 {
1566         struct arch_timer_mem *timer_mem;
1567         struct arch_timer_mem_frame *frame;
1568         struct device_node *frame_node;
1569         struct resource res;
1570         int ret = -EINVAL;
1571         u32 rate;
1572
1573         timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1574         if (!timer_mem)
1575                 return -ENOMEM;
1576
1577         if (of_address_to_resource(np, 0, &res))
1578                 goto out;
1579         timer_mem->cntctlbase = res.start;
1580         timer_mem->size = resource_size(&res);
1581
1582         for_each_available_child_of_node(np, frame_node) {
1583                 u32 n;
1584                 struct arch_timer_mem_frame *frame;
1585
1586                 if (of_property_read_u32(frame_node, "frame-number", &n)) {
1587                         pr_err(FW_BUG "Missing frame-number.\n");
1588                         of_node_put(frame_node);
1589                         goto out;
1590                 }
1591                 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1592                         pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1593                                ARCH_TIMER_MEM_MAX_FRAMES - 1);
1594                         of_node_put(frame_node);
1595                         goto out;
1596                 }
1597                 frame = &timer_mem->frame[n];
1598
1599                 if (frame->valid) {
1600                         pr_err(FW_BUG "Duplicated frame-number.\n");
1601                         of_node_put(frame_node);
1602                         goto out;
1603                 }
1604
1605                 if (of_address_to_resource(frame_node, 0, &res)) {
1606                         of_node_put(frame_node);
1607                         goto out;
1608                 }
1609                 frame->cntbase = res.start;
1610                 frame->size = resource_size(&res);
1611
1612                 frame->virt_irq = irq_of_parse_and_map(frame_node,
1613                                                        ARCH_TIMER_VIRT_SPI);
1614                 frame->phys_irq = irq_of_parse_and_map(frame_node,
1615                                                        ARCH_TIMER_PHYS_SPI);
1616
1617                 frame->valid = true;
1618         }
1619
1620         frame = arch_timer_mem_find_best_frame(timer_mem);
1621         if (!frame) {
1622                 pr_err("Unable to find a suitable frame in timer @ %pa\n",
1623                         &timer_mem->cntctlbase);
1624                 ret = -EINVAL;
1625                 goto out;
1626         }
1627
1628         rate = arch_timer_mem_frame_get_cntfrq(frame);
1629         arch_timer_of_configure_rate(rate, np);
1630
1631         ret = arch_timer_mem_frame_register(frame);
1632         if (!ret && !arch_timer_needs_of_probing())
1633                 ret = arch_timer_common_init();
1634 out:
1635         kfree(timer_mem);
1636         return ret;
1637 }
1638 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1639                        arch_timer_mem_of_init);
1640
1641 #ifdef CONFIG_ACPI_GTDT
1642 static int __init
1643 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1644 {
1645         struct arch_timer_mem_frame *frame;
1646         u32 rate;
1647         int i;
1648
1649         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1650                 frame = &timer_mem->frame[i];
1651
1652                 if (!frame->valid)
1653                         continue;
1654
1655                 rate = arch_timer_mem_frame_get_cntfrq(frame);
1656                 if (rate == arch_timer_rate)
1657                         continue;
1658
1659                 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1660                         &frame->cntbase,
1661                         (unsigned long)rate, (unsigned long)arch_timer_rate);
1662
1663                 return -EINVAL;
1664         }
1665
1666         return 0;
1667 }
1668
1669 static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1670 {
1671         struct arch_timer_mem *timers, *timer;
1672         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1673         int timer_count, i, ret = 0;
1674
1675         timers = kcalloc(platform_timer_count, sizeof(*timers),
1676                             GFP_KERNEL);
1677         if (!timers)
1678                 return -ENOMEM;
1679
1680         ret = acpi_arch_timer_mem_init(timers, &timer_count);
1681         if (ret || !timer_count)
1682                 goto out;
1683
1684         /*
1685          * While unlikely, it's theoretically possible that none of the frames
1686          * in a timer expose the combination of feature we want.
1687          */
1688         for (i = 0; i < timer_count; i++) {
1689                 timer = &timers[i];
1690
1691                 frame = arch_timer_mem_find_best_frame(timer);
1692                 if (!best_frame)
1693                         best_frame = frame;
1694
1695                 ret = arch_timer_mem_verify_cntfrq(timer);
1696                 if (ret) {
1697                         pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1698                         goto out;
1699                 }
1700
1701                 if (!best_frame) /* implies !frame */
1702                         /*
1703                          * Only complain about missing suitable frames if we
1704                          * haven't already found one in a previous iteration.
1705                          */
1706                         pr_err("Unable to find a suitable frame in timer @ %pa\n",
1707                                 &timer->cntctlbase);
1708         }
1709
1710         if (best_frame)
1711                 ret = arch_timer_mem_frame_register(best_frame);
1712 out:
1713         kfree(timers);
1714         return ret;
1715 }
1716
1717 /* Initialize per-processor generic timer and memory-mapped timer(if present) */
1718 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1719 {
1720         int ret, platform_timer_count;
1721
1722         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1723                 pr_warn("already initialized, skipping\n");
1724                 return -EINVAL;
1725         }
1726
1727         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1728
1729         ret = acpi_gtdt_init(table, &platform_timer_count);
1730         if (ret)
1731                 return ret;
1732
1733         arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1734                 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1735
1736         arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1737                 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1738
1739         arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1740                 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1741
1742         arch_timer_populate_kvm_info();
1743
1744         /*
1745          * When probing via ACPI, we have no mechanism to override the sysreg
1746          * CNTFRQ value. This *must* be correct.
1747          */
1748         arch_timer_rate = arch_timer_get_cntfrq();
1749         ret = validate_timer_rate();
1750         if (ret) {
1751                 pr_err(FW_BUG "frequency not available.\n");
1752                 return ret;
1753         }
1754
1755         arch_timer_uses_ppi = arch_timer_select_ppi();
1756         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1757                 pr_err("No interrupt available, giving up\n");
1758                 return -EINVAL;
1759         }
1760
1761         /* Always-on capability */
1762         arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1763
1764         /* Check for globally applicable workarounds */
1765         arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1766
1767         ret = arch_timer_register();
1768         if (ret)
1769                 return ret;
1770
1771         if (platform_timer_count &&
1772             arch_timer_mem_acpi_init(platform_timer_count))
1773                 pr_err("Failed to initialize memory-mapped timer.\n");
1774
1775         return arch_timer_common_init();
1776 }
1777 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1778 #endif
1779
1780 int kvm_arch_ptp_get_crosststamp(u64 *cycle, struct timespec64 *ts,
1781                                  struct clocksource **cs)
1782 {
1783         struct arm_smccc_res hvc_res;
1784         u32 ptp_counter;
1785         ktime_t ktime;
1786
1787         if (!IS_ENABLED(CONFIG_HAVE_ARM_SMCCC_DISCOVERY))
1788                 return -EOPNOTSUPP;
1789
1790         if (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI)
1791                 ptp_counter = KVM_PTP_VIRT_COUNTER;
1792         else
1793                 ptp_counter = KVM_PTP_PHYS_COUNTER;
1794
1795         arm_smccc_1_1_invoke(ARM_SMCCC_VENDOR_HYP_KVM_PTP_FUNC_ID,
1796                              ptp_counter, &hvc_res);
1797
1798         if ((int)(hvc_res.a0) < 0)
1799                 return -EOPNOTSUPP;
1800
1801         ktime = (u64)hvc_res.a0 << 32 | hvc_res.a1;
1802         *ts = ktime_to_timespec64(ktime);
1803         if (cycle)
1804                 *cycle = (u64)hvc_res.a2 << 32 | hvc_res.a3;
1805         if (cs)
1806                 *cs = &clocksource_counter;
1807
1808         return 0;
1809 }
1810 EXPORT_SYMBOL_GPL(kvm_arch_ptp_get_crosststamp);