kernel/time/tick-broadcast.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file contains functions which emulate a local clock-event
   4  * device via a broadcast event source.
   5  *
   6  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
   7  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
   8  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
   9  */
  10 #include <linux/cpu.h>
  11 #include <linux/err.h>
  12 #include <linux/hrtimer.h>
  13 #include <linux/interrupt.h>
  14 #include <linux/percpu.h>
  15 #include <linux/profile.h>
  16 #include <linux/sched.h>
  17 #include <linux/smp.h>
  18 #include <linux/module.h>
  19
  20 #include "tick-internal.h"
  21
  22 /*
  23  * Broadcast support for broken x86 hardware, where the local apic
  24  * timer stops in C3 state.
  25  */
  26
  27 static struct tick_device tick_broadcast_device;
  28 static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;
  29 static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;
  30 static cpumask_var_t tmpmask __cpumask_var_read_mostly;
  31 static int tick_broadcast_forced;
  32
  33 static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
  34
  35 #ifdef CONFIG_TICK_ONESHOT
  36 static void tick_broadcast_setup_oneshot(struct clock_event_device *bc);
  37 static void tick_broadcast_clear_oneshot(int cpu);
  38 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
  39 # ifdef CONFIG_HOTPLUG_CPU
  40 static void tick_broadcast_oneshot_offline(unsigned int cpu);
  41 # endif
  42 #else
  43 static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) { BUG(); }
  44 static inline void tick_broadcast_clear_oneshot(int cpu) { }
  45 static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
  46 # ifdef CONFIG_HOTPLUG_CPU
  47 static inline void tick_broadcast_oneshot_offline(unsigned int cpu) { }
  48 # endif
  49 #endif
  50
  51 /*
  52  * Debugging: see timer_list.c
  53  */
  54 struct tick_device *tick_get_broadcast_device(void)
  55 {
  56         return &tick_broadcast_device;
  57 }
  58
  59 struct cpumask *tick_get_broadcast_mask(void)
  60 {
  61         return tick_broadcast_mask;
  62 }
  63
  64 /*
  65  * Start the device in periodic mode
  66  */
  67 static void tick_broadcast_start_periodic(struct clock_event_device *bc)
  68 {
  69         if (bc)
  70                 tick_setup_periodic(bc, 1);
  71 }
  72
  73 /*
  74  * Check, if the device can be utilized as broadcast device:
  75  */
  76 static bool tick_check_broadcast_device(struct clock_event_device *curdev,
  77                                         struct clock_event_device *newdev)
  78 {
  79         if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
  80             (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
  81             (newdev->features & CLOCK_EVT_FEAT_C3STOP))
  82                 return false;
  83
  84         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
  85             !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
  86                 return false;
  87
  88         return !curdev || newdev->rating > curdev->rating;
  89 }
  90
  91 /*
  92  * Conditionally install/replace broadcast device
  93  */
  94 void tick_install_broadcast_device(struct clock_event_device *dev)
  95 {
  96         struct clock_event_device *cur = tick_broadcast_device.evtdev;
  97
  98         if (!tick_check_broadcast_device(cur, dev))
  99                 return;
 100
 101         if (!try_module_get(dev->owner))
 102                 return;
 103
 104         clockevents_exchange_device(cur, dev);
 105         if (cur)
 106                 cur->event_handler = clockevents_handle_noop;
 107         tick_broadcast_device.evtdev = dev;
 108         if (!cpumask_empty(tick_broadcast_mask))
 109                 tick_broadcast_start_periodic(dev);
 110         /*
 111          * Inform all cpus about this. We might be in a situation
 112          * where we did not switch to oneshot mode because the per cpu
 113          * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
 114          * of a oneshot capable broadcast device. Without that
 115          * notification the systems stays stuck in periodic mode
 116          * forever.
 117          */
 118         if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
 119                 tick_clock_notify();
 120 }
 121
 122 /*
 123  * Check, if the device is the broadcast device
 124  */
 125 int tick_is_broadcast_device(struct clock_event_device *dev)
 126 {
 127         return (dev && tick_broadcast_device.evtdev == dev);
 128 }
 129
 130 int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq)
 131 {
 132         int ret = -ENODEV;
 133
 134         if (tick_is_broadcast_device(dev)) {
 135                 raw_spin_lock(&tick_broadcast_lock);
 136                 ret = __clockevents_update_freq(dev, freq);
 137                 raw_spin_unlock(&tick_broadcast_lock);
 138         }
 139         return ret;
 140 }
 141
 142
 143 static void err_broadcast(const struct cpumask *mask)
 144 {
 145         pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
 146 }
 147
 148 static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
 149 {
 150         if (!dev->broadcast)
 151                 dev->broadcast = tick_broadcast;
 152         if (!dev->broadcast) {
 153                 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
 154                              dev->name);
 155                 dev->broadcast = err_broadcast;
 156         }
 157 }
 158
 159 /*
 160  * Check, if the device is disfunctional and a place holder, which
 161  * needs to be handled by the broadcast device.
 162  */
 163 int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
 164 {
 165         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 166         unsigned long flags;
 167         int ret = 0;
 168
 169         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 170
 171         /*
 172          * Devices might be registered with both periodic and oneshot
 173          * mode disabled. This signals, that the device needs to be
 174          * operated from the broadcast device and is a placeholder for
 175          * the cpu local device.
 176          */
 177         if (!tick_device_is_functional(dev)) {
 178                 dev->event_handler = tick_handle_periodic;
 179                 tick_device_setup_broadcast_func(dev);
 180                 cpumask_set_cpu(cpu, tick_broadcast_mask);
 181                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 182                         tick_broadcast_start_periodic(bc);
 183                 else
 184                         tick_broadcast_setup_oneshot(bc);
 185                 ret = 1;
 186         } else {
 187                 /*
 188                  * Clear the broadcast bit for this cpu if the
 189                  * device is not power state affected.
 190                  */
 191                 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
 192                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 193                 else
 194                         tick_device_setup_broadcast_func(dev);
 195
 196                 /*
 197                  * Clear the broadcast bit if the CPU is not in
 198                  * periodic broadcast on state.
 199                  */
 200                 if (!cpumask_test_cpu(cpu, tick_broadcast_on))
 201                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 202
 203                 switch (tick_broadcast_device.mode) {
 204                 case TICKDEV_MODE_ONESHOT:
 205                         /*
 206                          * If the system is in oneshot mode we can
 207                          * unconditionally clear the oneshot mask bit,
 208                          * because the CPU is running and therefore
 209                          * not in an idle state which causes the power
 210                          * state affected device to stop. Let the
 211                          * caller initialize the device.
 212                          */
 213                         tick_broadcast_clear_oneshot(cpu);
 214                         ret = 0;
 215                         break;
 216
 217                 case TICKDEV_MODE_PERIODIC:
 218                         /*
 219                          * If the system is in periodic mode, check
 220                          * whether the broadcast device can be
 221                          * switched off now.
 222                          */
 223                         if (cpumask_empty(tick_broadcast_mask) && bc)
 224                                 clockevents_shutdown(bc);
 225                         /*
 226                          * If we kept the cpu in the broadcast mask,
 227                          * tell the caller to leave the per cpu device
 228                          * in shutdown state. The periodic interrupt
 229                          * is delivered by the broadcast device, if
 230                          * the broadcast device exists and is not
 231                          * hrtimer based.
 232                          */
 233                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 234                                 ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
 235                         break;
 236                 default:
 237                         break;
 238                 }
 239         }
 240         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 241         return ret;
 242 }
 243
 244 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 245 int tick_receive_broadcast(void)
 246 {
 247         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 248         struct clock_event_device *evt = td->evtdev;
 249
 250         if (!evt)
 251                 return -ENODEV;
 252
 253         if (!evt->event_handler)
 254                 return -EINVAL;
 255
 256         evt->event_handler(evt);
 257         return 0;
 258 }
 259 #endif
 260
 261 /*
 262  * Broadcast the event to the cpus, which are set in the mask (mangled).
 263  */
 264 static bool tick_do_broadcast(struct cpumask *mask)
 265 {
 266         int cpu = smp_processor_id();
 267         struct tick_device *td;
 268         bool local = false;
 269
 270         /*
 271          * Check, if the current cpu is in the mask
 272          */
 273         if (cpumask_test_cpu(cpu, mask)) {
 274                 struct clock_event_device *bc = tick_broadcast_device.evtdev;
 275
 276                 cpumask_clear_cpu(cpu, mask);
 277                 /*
 278                  * We only run the local handler, if the broadcast
 279                  * device is not hrtimer based. Otherwise we run into
 280                  * a hrtimer recursion.
 281                  *
 282                  * local timer_interrupt()
 283                  *   local_handler()
 284                  *     expire_hrtimers()
 285                  *       bc_handler()
 286                  *         local_handler()
 287                  *           expire_hrtimers()
 288                  */
 289                 local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);
 290         }
 291
 292         if (!cpumask_empty(mask)) {
 293                 /*
 294                  * It might be necessary to actually check whether the devices
 295                  * have different broadcast functions. For now, just use the
 296                  * one of the first device. This works as long as we have this
 297                  * misfeature only on x86 (lapic)
 298                  */
 299                 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
 300                 td->evtdev->broadcast(mask);
 301         }
 302         return local;
 303 }
 304
 305 /*
 306  * Periodic broadcast:
 307  * - invoke the broadcast handlers
 308  */
 309 static bool tick_do_periodic_broadcast(void)
 310 {
 311         cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
 312         return tick_do_broadcast(tmpmask);
 313 }
 314
 315 /*
 316  * Event handler for periodic broadcast ticks
 317  */
 318 static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
 319 {
 320         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 321         bool bc_local;
 322
 323         raw_spin_lock(&tick_broadcast_lock);
 324
 325         /* Handle spurious interrupts gracefully */
 326         if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {
 327                 raw_spin_unlock(&tick_broadcast_lock);
 328                 return;
 329         }
 330
 331         bc_local = tick_do_periodic_broadcast();
 332
 333         if (clockevent_state_oneshot(dev)) {
 334                 ktime_t next = ktime_add(dev->next_event, tick_period);
 335
 336                 clockevents_program_event(dev, next, true);
 337         }
 338         raw_spin_unlock(&tick_broadcast_lock);
 339
 340         /*
 341          * We run the handler of the local cpu after dropping
 342          * tick_broadcast_lock because the handler might deadlock when
 343          * trying to switch to oneshot mode.
 344          */
 345         if (bc_local)
 346                 td->evtdev->event_handler(td->evtdev);
 347 }
 348
 349 /**
 350  * tick_broadcast_control - Enable/disable or force broadcast mode
 351  * @mode:       The selected broadcast mode
 352  *
 353  * Called when the system enters a state where affected tick devices
 354  * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
 355  */
 356 void tick_broadcast_control(enum tick_broadcast_mode mode)
 357 {
 358         struct clock_event_device *bc, *dev;
 359         struct tick_device *td;
 360         int cpu, bc_stopped;
 361         unsigned long flags;
 362
 363         /* Protects also the local clockevent device. */
 364         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 365         td = this_cpu_ptr(&tick_cpu_device);
 366         dev = td->evtdev;
 367
 368         /*
 369          * Is the device not affected by the powerstate ?
 370          */
 371         if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
 372                 goto out;
 373
 374         if (!tick_device_is_functional(dev))
 375                 goto out;
 376
 377         cpu = smp_processor_id();
 378         bc = tick_broadcast_device.evtdev;
 379         bc_stopped = cpumask_empty(tick_broadcast_mask);
 380
 381         switch (mode) {
 382         case TICK_BROADCAST_FORCE:
 383                 tick_broadcast_forced = 1;
 384                 fallthrough;
 385         case TICK_BROADCAST_ON:
 386                 cpumask_set_cpu(cpu, tick_broadcast_on);
 387                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
 388                         /*
 389                          * Only shutdown the cpu local device, if:
 390                          *
 391                          * - the broadcast device exists
 392                          * - the broadcast device is not a hrtimer based one
 393                          * - the broadcast device is in periodic mode to
 394                          *   avoid a hickup during switch to oneshot mode
 395                          */
 396                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&
 397                             tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 398                                 clockevents_shutdown(dev);
 399                 }
 400                 break;
 401
 402         case TICK_BROADCAST_OFF:
 403                 if (tick_broadcast_forced)
 404                         break;
 405                 cpumask_clear_cpu(cpu, tick_broadcast_on);
 406                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
 407                         if (tick_broadcast_device.mode ==
 408                             TICKDEV_MODE_PERIODIC)
 409                                 tick_setup_periodic(dev, 0);
 410                 }
 411                 break;
 412         }
 413
 414         if (bc) {
 415                 if (cpumask_empty(tick_broadcast_mask)) {
 416                         if (!bc_stopped)
 417                                 clockevents_shutdown(bc);
 418                 } else if (bc_stopped) {
 419                         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
 420                                 tick_broadcast_start_periodic(bc);
 421                         else
 422                                 tick_broadcast_setup_oneshot(bc);
 423                 }
 424         }
 425 out:
 426         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 427 }
 428 EXPORT_SYMBOL_GPL(tick_broadcast_control);
 429
 430 /*
 431  * Set the periodic handler depending on broadcast on/off
 432  */
 433 void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
 434 {
 435         if (!broadcast)
 436                 dev->event_handler = tick_handle_periodic;
 437         else
 438                 dev->event_handler = tick_handle_periodic_broadcast;
 439 }
 440
 441 #ifdef CONFIG_HOTPLUG_CPU
 442 static void tick_shutdown_broadcast(void)
 443 {
 444         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 445
 446         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 447                 if (bc && cpumask_empty(tick_broadcast_mask))
 448                         clockevents_shutdown(bc);
 449         }
 450 }
 451
 452 /*
 453  * Remove a CPU from broadcasting
 454  */
 455 void tick_broadcast_offline(unsigned int cpu)
 456 {
 457         raw_spin_lock(&tick_broadcast_lock);
 458         cpumask_clear_cpu(cpu, tick_broadcast_mask);
 459         cpumask_clear_cpu(cpu, tick_broadcast_on);
 460         tick_broadcast_oneshot_offline(cpu);
 461         tick_shutdown_broadcast();
 462         raw_spin_unlock(&tick_broadcast_lock);
 463 }
 464
 465 #endif
 466
 467 void tick_suspend_broadcast(void)
 468 {
 469         struct clock_event_device *bc;
 470         unsigned long flags;
 471
 472         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 473
 474         bc = tick_broadcast_device.evtdev;
 475         if (bc)
 476                 clockevents_shutdown(bc);
 477
 478         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 479 }
 480
 481 /*
 482  * This is called from tick_resume_local() on a resuming CPU. That's
 483  * called from the core resume function, tick_unfreeze() and the magic XEN
 484  * resume hackery.
 485  *
 486  * In none of these cases the broadcast device mode can change and the
 487  * bit of the resuming CPU in the broadcast mask is safe as well.
 488  */
 489 bool tick_resume_check_broadcast(void)
 490 {
 491         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT)
 492                 return false;
 493         else
 494                 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_mask);
 495 }
 496
 497 void tick_resume_broadcast(void)
 498 {
 499         struct clock_event_device *bc;
 500         unsigned long flags;
 501
 502         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 503
 504         bc = tick_broadcast_device.evtdev;
 505
 506         if (bc) {
 507                 clockevents_tick_resume(bc);
 508
 509                 switch (tick_broadcast_device.mode) {
 510                 case TICKDEV_MODE_PERIODIC:
 511                         if (!cpumask_empty(tick_broadcast_mask))
 512                                 tick_broadcast_start_periodic(bc);
 513                         break;
 514                 case TICKDEV_MODE_ONESHOT:
 515                         if (!cpumask_empty(tick_broadcast_mask))
 516                                 tick_resume_broadcast_oneshot(bc);
 517                         break;
 518                 }
 519         }
 520         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 521 }
 522
 523 #ifdef CONFIG_TICK_ONESHOT
 524
 525 static cpumask_var_t tick_broadcast_oneshot_mask __cpumask_var_read_mostly;
 526 static cpumask_var_t tick_broadcast_pending_mask __cpumask_var_read_mostly;
 527 static cpumask_var_t tick_broadcast_force_mask __cpumask_var_read_mostly;
 528
 529 /*
 530  * Exposed for debugging: see timer_list.c
 531  */
 532 struct cpumask *tick_get_broadcast_oneshot_mask(void)
 533 {
 534         return tick_broadcast_oneshot_mask;
 535 }
 536
 537 /*
 538  * Called before going idle with interrupts disabled. Checks whether a
 539  * broadcast event from the other core is about to happen. We detected
 540  * that in tick_broadcast_oneshot_control(). The callsite can use this
 541  * to avoid a deep idle transition as we are about to get the
 542  * broadcast IPI right away.
 543  */
 544 int tick_check_broadcast_expired(void)
 545 {
 546         return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
 547 }
 548
 549 /*
 550  * Set broadcast interrupt affinity
 551  */
 552 static void tick_broadcast_set_affinity(struct clock_event_device *bc,
 553                                         const struct cpumask *cpumask)
 554 {
 555         if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
 556                 return;
 557
 558         if (cpumask_equal(bc->cpumask, cpumask))
 559                 return;
 560
 561         bc->cpumask = cpumask;
 562         irq_set_affinity(bc->irq, bc->cpumask);
 563 }
 564
 565 static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
 566                                      ktime_t expires)
 567 {
 568         if (!clockevent_state_oneshot(bc))
 569                 clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 570
 571         clockevents_program_event(bc, expires, 1);
 572         tick_broadcast_set_affinity(bc, cpumask_of(cpu));
 573 }
 574
 575 static void tick_resume_broadcast_oneshot(struct clock_event_device *bc)
 576 {
 577         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 578 }
 579
 580 /*
 581  * Called from irq_enter() when idle was interrupted to reenable the
 582  * per cpu device.
 583  */
 584 void tick_check_oneshot_broadcast_this_cpu(void)
 585 {
 586         if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
 587                 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
 588
 589                 /*
 590                  * We might be in the middle of switching over from
 591                  * periodic to oneshot. If the CPU has not yet
 592                  * switched over, leave the device alone.
 593                  */
 594                 if (td->mode == TICKDEV_MODE_ONESHOT) {
 595                         clockevents_switch_state(td->evtdev,
 596                                               CLOCK_EVT_STATE_ONESHOT);
 597                 }
 598         }
 599 }
 600
 601 /*
 602  * Handle oneshot mode broadcasting
 603  */
 604 static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
 605 {
 606         struct tick_device *td;
 607         ktime_t now, next_event;
 608         int cpu, next_cpu = 0;
 609         bool bc_local;
 610
 611         raw_spin_lock(&tick_broadcast_lock);
 612         dev->next_event = KTIME_MAX;
 613         next_event = KTIME_MAX;
 614         cpumask_clear(tmpmask);
 615         now = ktime_get();
 616         /* Find all expired events */
 617         for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
 618                 /*
 619                  * Required for !SMP because for_each_cpu() reports
 620                  * unconditionally CPU0 as set on UP kernels.
 621                  */
 622                 if (!IS_ENABLED(CONFIG_SMP) &&
 623                     cpumask_empty(tick_broadcast_oneshot_mask))
 624                         break;
 625
 626                 td = &per_cpu(tick_cpu_device, cpu);
 627                 if (td->evtdev->next_event <= now) {
 628                         cpumask_set_cpu(cpu, tmpmask);
 629                         /*
 630                          * Mark the remote cpu in the pending mask, so
 631                          * it can avoid reprogramming the cpu local
 632                          * timer in tick_broadcast_oneshot_control().
 633                          */
 634                         cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
 635                 } else if (td->evtdev->next_event < next_event) {
 636                         next_event = td->evtdev->next_event;
 637                         next_cpu = cpu;
 638                 }
 639         }
 640
 641         /*
 642          * Remove the current cpu from the pending mask. The event is
 643          * delivered immediately in tick_do_broadcast() !
 644          */
 645         cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
 646
 647         /* Take care of enforced broadcast requests */
 648         cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
 649         cpumask_clear(tick_broadcast_force_mask);
 650
 651         /*
 652          * Sanity check. Catch the case where we try to broadcast to
 653          * offline cpus.
 654          */
 655         if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
 656                 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
 657
 658         /*
 659          * Wakeup the cpus which have an expired event.
 660          */
 661         bc_local = tick_do_broadcast(tmpmask);
 662
 663         /*
 664          * Two reasons for reprogram:
 665          *
 666          * - The global event did not expire any CPU local
 667          * events. This happens in dyntick mode, as the maximum PIT
 668          * delta is quite small.
 669          *
 670          * - There are pending events on sleeping CPUs which were not
 671          * in the event mask
 672          */
 673         if (next_event != KTIME_MAX)
 674                 tick_broadcast_set_event(dev, next_cpu, next_event);
 675
 676         raw_spin_unlock(&tick_broadcast_lock);
 677
 678         if (bc_local) {
 679                 td = this_cpu_ptr(&tick_cpu_device);
 680                 td->evtdev->event_handler(td->evtdev);
 681         }
 682 }
 683
 684 static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
 685 {
 686         if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
 687                 return 0;
 688         if (bc->next_event == KTIME_MAX)
 689                 return 0;
 690         return bc->bound_on == cpu ? -EBUSY : 0;
 691 }
 692
 693 static void broadcast_shutdown_local(struct clock_event_device *bc,
 694                                      struct clock_event_device *dev)
 695 {
 696         /*
 697          * For hrtimer based broadcasting we cannot shutdown the cpu
 698          * local device if our own event is the first one to expire or
 699          * if we own the broadcast timer.
 700          */
 701         if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
 702                 if (broadcast_needs_cpu(bc, smp_processor_id()))
 703                         return;
 704                 if (dev->next_event < bc->next_event)
 705                         return;
 706         }
 707         clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
 708 }
 709
 710 int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
 711 {
 712         struct clock_event_device *bc, *dev;
 713         int cpu, ret = 0;
 714         ktime_t now;
 715
 716         /*
 717          * If there is no broadcast device, tell the caller not to go
 718          * into deep idle.
 719          */
 720         if (!tick_broadcast_device.evtdev)
 721                 return -EBUSY;
 722
 723         dev = this_cpu_ptr(&tick_cpu_device)->evtdev;
 724
 725         raw_spin_lock(&tick_broadcast_lock);
 726         bc = tick_broadcast_device.evtdev;
 727         cpu = smp_processor_id();
 728
 729         if (state == TICK_BROADCAST_ENTER) {
 730                 /*
 731                  * If the current CPU owns the hrtimer broadcast
 732                  * mechanism, it cannot go deep idle and we do not add
 733                  * the CPU to the broadcast mask. We don't have to go
 734                  * through the EXIT path as the local timer is not
 735                  * shutdown.
 736                  */
 737                 ret = broadcast_needs_cpu(bc, cpu);
 738                 if (ret)
 739                         goto out;
 740
 741                 /*
 742                  * If the broadcast device is in periodic mode, we
 743                  * return.
 744                  */
 745                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
 746                         /* If it is a hrtimer based broadcast, return busy */
 747                         if (bc->features & CLOCK_EVT_FEAT_HRTIMER)
 748                                 ret = -EBUSY;
 749                         goto out;
 750                 }
 751
 752                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
 753                         WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
 754
 755                         /* Conditionally shut down the local timer. */
 756                         broadcast_shutdown_local(bc, dev);
 757
 758                         /*
 759                          * We only reprogram the broadcast timer if we
 760                          * did not mark ourself in the force mask and
 761                          * if the cpu local event is earlier than the
 762                          * broadcast event. If the current CPU is in
 763                          * the force mask, then we are going to be
 764                          * woken by the IPI right away; we return
 765                          * busy, so the CPU does not try to go deep
 766                          * idle.
 767                          */
 768                         if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) {
 769                                 ret = -EBUSY;
 770                         } else if (dev->next_event < bc->next_event) {
 771                                 tick_broadcast_set_event(bc, cpu, dev->next_event);
 772                                 /*
 773                                  * In case of hrtimer broadcasts the
 774                                  * programming might have moved the
 775                                  * timer to this cpu. If yes, remove
 776                                  * us from the broadcast mask and
 777                                  * return busy.
 778                                  */
 779                                 ret = broadcast_needs_cpu(bc, cpu);
 780                                 if (ret) {
 781                                         cpumask_clear_cpu(cpu,
 782                                                 tick_broadcast_oneshot_mask);
 783                                 }
 784                         }
 785                 }
 786         } else {
 787                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
 788                         clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
 789                         /*
 790                          * The cpu which was handling the broadcast
 791                          * timer marked this cpu in the broadcast
 792                          * pending mask and fired the broadcast
 793                          * IPI. So we are going to handle the expired
 794                          * event anyway via the broadcast IPI
 795                          * handler. No need to reprogram the timer
 796                          * with an already expired event.
 797                          */
 798                         if (cpumask_test_and_clear_cpu(cpu,
 799                                        tick_broadcast_pending_mask))
 800                                 goto out;
 801
 802                         /*
 803                          * Bail out if there is no next event.
 804                          */
 805                         if (dev->next_event == KTIME_MAX)
 806                                 goto out;
 807                         /*
 808                          * If the pending bit is not set, then we are
 809                          * either the CPU handling the broadcast
 810                          * interrupt or we got woken by something else.
 811                          *
 812                          * We are no longer in the broadcast mask, so
 813                          * if the cpu local expiry time is already
 814                          * reached, we would reprogram the cpu local
 815                          * timer with an already expired event.
 816                          *
 817                          * This can lead to a ping-pong when we return
 818                          * to idle and therefore rearm the broadcast
 819                          * timer before the cpu local timer was able
 820                          * to fire. This happens because the forced
 821                          * reprogramming makes sure that the event
 822                          * will happen in the future and depending on
 823                          * the min_delta setting this might be far
 824                          * enough out that the ping-pong starts.
 825                          *
 826                          * If the cpu local next_event has expired
 827                          * then we know that the broadcast timer
 828                          * next_event has expired as well and
 829                          * broadcast is about to be handled. So we
 830                          * avoid reprogramming and enforce that the
 831                          * broadcast handler, which did not run yet,
 832                          * will invoke the cpu local handler.
 833                          *
 834                          * We cannot call the handler directly from
 835                          * here, because we might be in a NOHZ phase
 836                          * and we did not go through the irq_enter()
 837                          * nohz fixups.
 838                          */
 839                         now = ktime_get();
 840                         if (dev->next_event <= now) {
 841                                 cpumask_set_cpu(cpu, tick_broadcast_force_mask);
 842                                 goto out;
 843                         }
 844                         /*
 845                          * We got woken by something else. Reprogram
 846                          * the cpu local timer device.
 847                          */
 848                         tick_program_event(dev->next_event, 1);
 849                 }
 850         }
 851 out:
 852         raw_spin_unlock(&tick_broadcast_lock);
 853         return ret;
 854 }
 855
 856 /*
 857  * Reset the one shot broadcast for a cpu
 858  *
 859  * Called with tick_broadcast_lock held
 860  */
 861 static void tick_broadcast_clear_oneshot(int cpu)
 862 {
 863         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 864         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 865 }
 866
 867 static void tick_broadcast_init_next_event(struct cpumask *mask,
 868                                            ktime_t expires)
 869 {
 870         struct tick_device *td;
 871         int cpu;
 872
 873         for_each_cpu(cpu, mask) {
 874                 td = &per_cpu(tick_cpu_device, cpu);
 875                 if (td->evtdev)
 876                         td->evtdev->next_event = expires;
 877         }
 878 }
 879
 880 /**
 881  * tick_broadcast_setup_oneshot - setup the broadcast device
 882  */
 883 static void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
 884 {
 885         int cpu = smp_processor_id();
 886
 887         if (!bc)
 888                 return;
 889
 890         /* Set it up only once ! */
 891         if (bc->event_handler != tick_handle_oneshot_broadcast) {
 892                 int was_periodic = clockevent_state_periodic(bc);
 893
 894                 bc->event_handler = tick_handle_oneshot_broadcast;
 895
 896                 /*
 897                  * We must be careful here. There might be other CPUs
 898                  * waiting for periodic broadcast. We need to set the
 899                  * oneshot_mask bits for those and program the
 900                  * broadcast device to fire.
 901                  */
 902                 cpumask_copy(tmpmask, tick_broadcast_mask);
 903                 cpumask_clear_cpu(cpu, tmpmask);
 904                 cpumask_or(tick_broadcast_oneshot_mask,
 905                            tick_broadcast_oneshot_mask, tmpmask);
 906
 907                 if (was_periodic && !cpumask_empty(tmpmask)) {
 908                         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
 909                         tick_broadcast_init_next_event(tmpmask,
 910                                                        tick_next_period);
 911                         tick_broadcast_set_event(bc, cpu, tick_next_period);
 912                 } else
 913                         bc->next_event = KTIME_MAX;
 914         } else {
 915                 /*
 916                  * The first cpu which switches to oneshot mode sets
 917                  * the bit for all other cpus which are in the general
 918                  * (periodic) broadcast mask. So the bit is set and
 919                  * would prevent the first broadcast enter after this
 920                  * to program the bc device.
 921                  */
 922                 tick_broadcast_clear_oneshot(cpu);
 923         }
 924 }
 925
 926 /*
 927  * Select oneshot operating mode for the broadcast device
 928  */
 929 void tick_broadcast_switch_to_oneshot(void)
 930 {
 931         struct clock_event_device *bc;
 932         unsigned long flags;
 933
 934         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 935
 936         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
 937         bc = tick_broadcast_device.evtdev;
 938         if (bc)
 939                 tick_broadcast_setup_oneshot(bc);
 940
 941         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 942 }
 943
 944 #ifdef CONFIG_HOTPLUG_CPU
 945 void hotplug_cpu__broadcast_tick_pull(int deadcpu)
 946 {
 947         struct clock_event_device *bc;
 948         unsigned long flags;
 949
 950         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
 951         bc = tick_broadcast_device.evtdev;
 952
 953         if (bc && broadcast_needs_cpu(bc, deadcpu)) {
 954                 /* This moves the broadcast assignment to this CPU: */
 955                 clockevents_program_event(bc, bc->next_event, 1);
 956         }
 957         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
 958 }
 959
 960 /*
 961  * Remove a dying CPU from broadcasting
 962  */
 963 static void tick_broadcast_oneshot_offline(unsigned int cpu)
 964 {
 965         /*
 966          * Clear the broadcast masks for the dead cpu, but do not stop
 967          * the broadcast device!
 968          */
 969         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
 970         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
 971         cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
 972 }
 973 #endif
 974
 975 /*
 976  * Check, whether the broadcast device is in one shot mode
 977  */
 978 int tick_broadcast_oneshot_active(void)
 979 {
 980         return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
 981 }
 982
 983 /*
 984  * Check whether the broadcast device supports oneshot.
 985  */
 986 bool tick_broadcast_oneshot_available(void)
 987 {
 988         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 989
 990         return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
 991 }
 992
 993 #else
 994 int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
 995 {
 996         struct clock_event_device *bc = tick_broadcast_device.evtdev;
 997
 998         if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
 999                 return -EBUSY;
1000
1001         return 0;
1002 }
1003 #endif
1004
1005 void __init tick_broadcast_init(void)
1006 {
1007         zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
1008         zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
1009         zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
1010 #ifdef CONFIG_TICK_ONESHOT
1011         zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
1012         zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
1013         zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
1014 #endif
1015 }