kernel/cpu.c

   1 /* CPU control.
   2  * (C) 2001, 2002, 2003, 2004 Rusty Russell
   3  *
   4  * This code is licenced under the GPL.
   5  */
   6 #include <linux/proc_fs.h>
   7 #include <linux/smp.h>
   8 #include <linux/init.h>
   9 #include <linux/notifier.h>
  10 #include <linux/sched/signal.h>
  11 #include <linux/sched/hotplug.h>
  12 #include <linux/sched/task.h>
  13 #include <linux/unistd.h>
  14 #include <linux/cpu.h>
  15 #include <linux/oom.h>
  16 #include <linux/rcupdate.h>
  17 #include <linux/export.h>
  18 #include <linux/bug.h>
  19 #include <linux/kthread.h>
  20 #include <linux/stop_machine.h>
  21 #include <linux/mutex.h>
  22 #include <linux/gfp.h>
  23 #include <linux/suspend.h>
  24 #include <linux/lockdep.h>
  25 #include <linux/tick.h>
  26 #include <linux/irq.h>
  27 #include <linux/nmi.h>
  28 #include <linux/smpboot.h>
  29 #include <linux/relay.h>
  30 #include <linux/slab.h>
  31 #include <linux/percpu-rwsem.h>
  32
  33 #include <trace/events/power.h>
  34 #define CREATE_TRACE_POINTS
  35 #include <trace/events/cpuhp.h>
  36
  37 #include "smpboot.h"
  38
  39 /**
  40  * cpuhp_cpu_state - Per cpu hotplug state storage
  41  * @state:      The current cpu state
  42  * @target:     The target state
  43  * @thread:     Pointer to the hotplug thread
  44  * @should_run: Thread should execute
  45  * @rollback:   Perform a rollback
  46  * @single:     Single callback invocation
  47  * @bringup:    Single callback bringup or teardown selector
  48  * @cb_state:   The state for a single callback (install/uninstall)
  49  * @result:     Result of the operation
  50  * @done_up:    Signal completion to the issuer of the task for cpu-up
  51  * @done_down:  Signal completion to the issuer of the task for cpu-down
  52  */
  53 struct cpuhp_cpu_state {
  54         enum cpuhp_state        state;
  55         enum cpuhp_state        target;
  56         enum cpuhp_state        fail;
  57 #ifdef CONFIG_SMP
  58         struct task_struct      *thread;
  59         bool                    should_run;
  60         bool                    rollback;
  61         bool                    single;
  62         bool                    bringup;
  63         struct hlist_node       *node;
  64         struct hlist_node       *last;
  65         enum cpuhp_state        cb_state;
  66         int                     result;
  67         struct completion       done_up;
  68         struct completion       done_down;
  69 #endif
  70 };
  71
  72 static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  73         .fail = CPUHP_INVALID,
  74 };
  75
  76 #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  77 static struct lockdep_map cpuhp_state_up_map =
  78         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  79 static struct lockdep_map cpuhp_state_down_map =
  80         STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
  81
  82
  83 static inline void cpuhp_lock_acquire(bool bringup)
  84 {
  85         lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  86 }
  87
  88 static inline void cpuhp_lock_release(bool bringup)
  89 {
  90         lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  91 }
  92 #else
  93
  94 static inline void cpuhp_lock_acquire(bool bringup) { }
  95 static inline void cpuhp_lock_release(bool bringup) { }
  96
  97 #endif
  98
  99 /**
 100  * cpuhp_step - Hotplug state machine step
 101  * @name:       Name of the step
 102  * @startup:    Startup function of the step
 103  * @teardown:   Teardown function of the step
 104  * @skip_onerr: Do not invoke the functions on error rollback
 105  *              Will go away once the notifiers are gone
 106  * @cant_stop:  Bringup/teardown can't be stopped at this step
 107  */
 108 struct cpuhp_step {
 109         const char              *name;
 110         union {
 111                 int             (*single)(unsigned int cpu);
 112                 int             (*multi)(unsigned int cpu,
 113                                          struct hlist_node *node);
 114         } startup;
 115         union {
 116                 int             (*single)(unsigned int cpu);
 117                 int             (*multi)(unsigned int cpu,
 118                                          struct hlist_node *node);
 119         } teardown;
 120         struct hlist_head       list;
 121         bool                    skip_onerr;
 122         bool                    cant_stop;
 123         bool                    multi_instance;
 124 };
 125
 126 static DEFINE_MUTEX(cpuhp_state_mutex);
 127 static struct cpuhp_step cpuhp_hp_states[];
 128
 129 static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
 130 {
 131         return cpuhp_hp_states + state;
 132 }
 133
 134 /**
 135  * cpuhp_invoke_callback _ Invoke the callbacks for a given state
 136  * @cpu:        The cpu for which the callback should be invoked
 137  * @state:      The state to do callbacks for
 138  * @bringup:    True if the bringup callback should be invoked
 139  * @node:       For multi-instance, do a single entry callback for install/remove
 140  * @lastp:      For multi-instance rollback, remember how far we got
 141  *
 142  * Called from cpu hotplug and from the state register machinery.
 143  */
 144 static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
 145                                  bool bringup, struct hlist_node *node,
 146                                  struct hlist_node **lastp)
 147 {
 148         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 149         struct cpuhp_step *step = cpuhp_get_step(state);
 150         int (*cbm)(unsigned int cpu, struct hlist_node *node);
 151         int (*cb)(unsigned int cpu);
 152         int ret, cnt;
 153
 154         if (st->fail == state) {
 155                 st->fail = CPUHP_INVALID;
 156
 157                 if (!(bringup ? step->startup.single : step->teardown.single))
 158                         return 0;
 159
 160                 return -EAGAIN;
 161         }
 162
 163         if (!step->multi_instance) {
 164                 WARN_ON_ONCE(lastp && *lastp);
 165                 cb = bringup ? step->startup.single : step->teardown.single;
 166                 if (!cb)
 167                         return 0;
 168                 trace_cpuhp_enter(cpu, st->target, state, cb);
 169                 ret = cb(cpu);
 170                 trace_cpuhp_exit(cpu, st->state, state, ret);
 171                 return ret;
 172         }
 173         cbm = bringup ? step->startup.multi : step->teardown.multi;
 174         if (!cbm)
 175                 return 0;
 176
 177         /* Single invocation for instance add/remove */
 178         if (node) {
 179                 WARN_ON_ONCE(lastp && *lastp);
 180                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 181                 ret = cbm(cpu, node);
 182                 trace_cpuhp_exit(cpu, st->state, state, ret);
 183                 return ret;
 184         }
 185
 186         /* State transition. Invoke on all instances */
 187         cnt = 0;
 188         hlist_for_each(node, &step->list) {
 189                 if (lastp && node == *lastp)
 190                         break;
 191
 192                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 193                 ret = cbm(cpu, node);
 194                 trace_cpuhp_exit(cpu, st->state, state, ret);
 195                 if (ret) {
 196                         if (!lastp)
 197                                 goto err;
 198
 199                         *lastp = node;
 200                         return ret;
 201                 }
 202                 cnt++;
 203         }
 204         if (lastp)
 205                 *lastp = NULL;
 206         return 0;
 207 err:
 208         /* Rollback the instances if one failed */
 209         cbm = !bringup ? step->startup.multi : step->teardown.multi;
 210         if (!cbm)
 211                 return ret;
 212
 213         hlist_for_each(node, &step->list) {
 214                 if (!cnt--)
 215                         break;
 216
 217                 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
 218                 ret = cbm(cpu, node);
 219                 trace_cpuhp_exit(cpu, st->state, state, ret);
 220                 /*
 221                  * Rollback must not fail,
 222                  */
 223                 WARN_ON_ONCE(ret);
 224         }
 225         return ret;
 226 }
 227
 228 #ifdef CONFIG_SMP
 229 static bool cpuhp_is_ap_state(enum cpuhp_state state)
 230 {
 231         /*
 232          * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
 233          * purposes as that state is handled explicitly in cpu_down.
 234          */
 235         return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
 236 }
 237
 238 static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 239 {
 240         struct completion *done = bringup ? &st->done_up : &st->done_down;
 241         wait_for_completion(done);
 242 }
 243
 244 static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
 245 {
 246         struct completion *done = bringup ? &st->done_up : &st->done_down;
 247         complete(done);
 248 }
 249
 250 /*
 251  * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
 252  */
 253 static bool cpuhp_is_atomic_state(enum cpuhp_state state)
 254 {
 255         return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
 256 }
 257
 258 /* Serializes the updates to cpu_online_mask, cpu_present_mask */
 259 static DEFINE_MUTEX(cpu_add_remove_lock);
 260 bool cpuhp_tasks_frozen;
 261 EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
 262
 263 /*
 264  * The following two APIs (cpu_maps_update_begin/done) must be used when
 265  * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
 266  */
 267 void cpu_maps_update_begin(void)
 268 {
 269         mutex_lock(&cpu_add_remove_lock);
 270 }
 271
 272 void cpu_maps_update_done(void)
 273 {
 274         mutex_unlock(&cpu_add_remove_lock);
 275 }
 276
 277 /*
 278  * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
 279  * Should always be manipulated under cpu_add_remove_lock
 280  */
 281 static int cpu_hotplug_disabled;
 282
 283 #ifdef CONFIG_HOTPLUG_CPU
 284
 285 DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
 286
 287 void cpus_read_lock(void)
 288 {
 289         percpu_down_read(&cpu_hotplug_lock);
 290 }
 291 EXPORT_SYMBOL_GPL(cpus_read_lock);
 292
 293 int cpus_read_trylock(void)
 294 {
 295         return percpu_down_read_trylock(&cpu_hotplug_lock);
 296 }
 297 EXPORT_SYMBOL_GPL(cpus_read_trylock);
 298
 299 void cpus_read_unlock(void)
 300 {
 301         percpu_up_read(&cpu_hotplug_lock);
 302 }
 303 EXPORT_SYMBOL_GPL(cpus_read_unlock);
 304
 305 void cpus_write_lock(void)
 306 {
 307         percpu_down_write(&cpu_hotplug_lock);
 308 }
 309
 310 void cpus_write_unlock(void)
 311 {
 312         percpu_up_write(&cpu_hotplug_lock);
 313 }
 314
 315 void lockdep_assert_cpus_held(void)
 316 {
 317         percpu_rwsem_assert_held(&cpu_hotplug_lock);
 318 }
 319
 320 /*
 321  * Wait for currently running CPU hotplug operations to complete (if any) and
 322  * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
 323  * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
 324  * hotplug path before performing hotplug operations. So acquiring that lock
 325  * guarantees mutual exclusion from any currently running hotplug operations.
 326  */
 327 void cpu_hotplug_disable(void)
 328 {
 329         cpu_maps_update_begin();
 330         cpu_hotplug_disabled++;
 331         cpu_maps_update_done();
 332 }
 333 EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
 334
 335 static void __cpu_hotplug_enable(void)
 336 {
 337         if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
 338                 return;
 339         cpu_hotplug_disabled--;
 340 }
 341
 342 void cpu_hotplug_enable(void)
 343 {
 344         cpu_maps_update_begin();
 345         __cpu_hotplug_enable();
 346         cpu_maps_update_done();
 347 }
 348 EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
 349 #endif  /* CONFIG_HOTPLUG_CPU */
 350
 351 static inline enum cpuhp_state
 352 cpuhp_set_state(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 353 {
 354         enum cpuhp_state prev_state = st->state;
 355
 356         st->rollback = false;
 357         st->last = NULL;
 358
 359         st->target = target;
 360         st->single = false;
 361         st->bringup = st->state < target;
 362
 363         return prev_state;
 364 }
 365
 366 static inline void
 367 cpuhp_reset_state(struct cpuhp_cpu_state *st, enum cpuhp_state prev_state)
 368 {
 369         st->rollback = true;
 370
 371         /*
 372          * If we have st->last we need to undo partial multi_instance of this
 373          * state first. Otherwise start undo at the previous state.
 374          */
 375         if (!st->last) {
 376                 if (st->bringup)
 377                         st->state--;
 378                 else
 379                         st->state++;
 380         }
 381
 382         st->target = prev_state;
 383         st->bringup = !st->bringup;
 384 }
 385
 386 /* Regular hotplug invocation of the AP hotplug thread */
 387 static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
 388 {
 389         if (!st->single && st->state == st->target)
 390                 return;
 391
 392         st->result = 0;
 393         /*
 394          * Make sure the above stores are visible before should_run becomes
 395          * true. Paired with the mb() above in cpuhp_thread_fun()
 396          */
 397         smp_mb();
 398         st->should_run = true;
 399         wake_up_process(st->thread);
 400         wait_for_ap_thread(st, st->bringup);
 401 }
 402
 403 static int cpuhp_kick_ap(struct cpuhp_cpu_state *st, enum cpuhp_state target)
 404 {
 405         enum cpuhp_state prev_state;
 406         int ret;
 407
 408         prev_state = cpuhp_set_state(st, target);
 409         __cpuhp_kick_ap(st);
 410         if ((ret = st->result)) {
 411                 cpuhp_reset_state(st, prev_state);
 412                 __cpuhp_kick_ap(st);
 413         }
 414
 415         return ret;
 416 }
 417
 418 static int bringup_wait_for_ap(unsigned int cpu)
 419 {
 420         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 421
 422         /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
 423         wait_for_ap_thread(st, true);
 424         if (WARN_ON_ONCE((!cpu_online(cpu))))
 425                 return -ECANCELED;
 426
 427         /* Unpark the stopper thread and the hotplug thread of the target cpu */
 428         stop_machine_unpark(cpu);
 429         kthread_unpark(st->thread);
 430
 431         if (st->target <= CPUHP_AP_ONLINE_IDLE)
 432                 return 0;
 433
 434         return cpuhp_kick_ap(st, st->target);
 435 }
 436
 437 static int bringup_cpu(unsigned int cpu)
 438 {
 439         struct task_struct *idle = idle_thread_get(cpu);
 440         int ret;
 441
 442         /*
 443          * Some architectures have to walk the irq descriptors to
 444          * setup the vector space for the cpu which comes online.
 445          * Prevent irq alloc/free across the bringup.
 446          */
 447         irq_lock_sparse();
 448
 449         /* Arch-specific enabling code. */
 450         ret = __cpu_up(cpu, idle);
 451         irq_unlock_sparse();
 452         if (ret)
 453                 return ret;
 454         return bringup_wait_for_ap(cpu);
 455 }
 456
 457 /*
 458  * Hotplug state machine related functions
 459  */
 460
 461 static void undo_cpu_up(unsigned int cpu, struct cpuhp_cpu_state *st)
 462 {
 463         for (st->state--; st->state > st->target; st->state--) {
 464                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 465
 466                 if (!step->skip_onerr)
 467                         cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 468         }
 469 }
 470
 471 static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 472                               enum cpuhp_state target)
 473 {
 474         enum cpuhp_state prev_state = st->state;
 475         int ret = 0;
 476
 477         while (st->state < target) {
 478                 st->state++;
 479                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 480                 if (ret) {
 481                         st->target = prev_state;
 482                         undo_cpu_up(cpu, st);
 483                         break;
 484                 }
 485         }
 486         return ret;
 487 }
 488
 489 /*
 490  * The cpu hotplug threads manage the bringup and teardown of the cpus
 491  */
 492 static void cpuhp_create(unsigned int cpu)
 493 {
 494         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 495
 496         init_completion(&st->done_up);
 497         init_completion(&st->done_down);
 498 }
 499
 500 static int cpuhp_should_run(unsigned int cpu)
 501 {
 502         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 503
 504         return st->should_run;
 505 }
 506
 507 /*
 508  * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
 509  * callbacks when a state gets [un]installed at runtime.
 510  *
 511  * Each invocation of this function by the smpboot thread does a single AP
 512  * state callback.
 513  *
 514  * It has 3 modes of operation:
 515  *  - single: runs st->cb_state
 516  *  - up:     runs ++st->state, while st->state < st->target
 517  *  - down:   runs st->state--, while st->state > st->target
 518  *
 519  * When complete or on error, should_run is cleared and the completion is fired.
 520  */
 521 static void cpuhp_thread_fun(unsigned int cpu)
 522 {
 523         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 524         bool bringup = st->bringup;
 525         enum cpuhp_state state;
 526
 527         /*
 528          * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
 529          * that if we see ->should_run we also see the rest of the state.
 530          */
 531         smp_mb();
 532
 533         if (WARN_ON_ONCE(!st->should_run))
 534                 return;
 535
 536         cpuhp_lock_acquire(bringup);
 537
 538         if (st->single) {
 539                 state = st->cb_state;
 540                 st->should_run = false;
 541         } else {
 542                 if (bringup) {
 543                         st->state++;
 544                         state = st->state;
 545                         st->should_run = (st->state < st->target);
 546                         WARN_ON_ONCE(st->state > st->target);
 547                 } else {
 548                         state = st->state;
 549                         st->state--;
 550                         st->should_run = (st->state > st->target);
 551                         WARN_ON_ONCE(st->state < st->target);
 552                 }
 553         }
 554
 555         WARN_ON_ONCE(!cpuhp_is_ap_state(state));
 556
 557         if (st->rollback) {
 558                 struct cpuhp_step *step = cpuhp_get_step(state);
 559                 if (step->skip_onerr)
 560                         goto next;
 561         }
 562
 563         if (cpuhp_is_atomic_state(state)) {
 564                 local_irq_disable();
 565                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 566                 local_irq_enable();
 567
 568                 /*
 569                  * STARTING/DYING must not fail!
 570                  */
 571                 WARN_ON_ONCE(st->result);
 572         } else {
 573                 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
 574         }
 575
 576         if (st->result) {
 577                 /*
 578                  * If we fail on a rollback, we're up a creek without no
 579                  * paddle, no way forward, no way back. We loose, thanks for
 580                  * playing.
 581                  */
 582                 WARN_ON_ONCE(st->rollback);
 583                 st->should_run = false;
 584         }
 585
 586 next:
 587         cpuhp_lock_release(bringup);
 588
 589         if (!st->should_run)
 590                 complete_ap_thread(st, bringup);
 591 }
 592
 593 /* Invoke a single callback on a remote cpu */
 594 static int
 595 cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
 596                          struct hlist_node *node)
 597 {
 598         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 599         int ret;
 600
 601         if (!cpu_online(cpu))
 602                 return 0;
 603
 604         cpuhp_lock_acquire(false);
 605         cpuhp_lock_release(false);
 606
 607         cpuhp_lock_acquire(true);
 608         cpuhp_lock_release(true);
 609
 610         /*
 611          * If we are up and running, use the hotplug thread. For early calls
 612          * we invoke the thread function directly.
 613          */
 614         if (!st->thread)
 615                 return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
 616
 617         st->rollback = false;
 618         st->last = NULL;
 619
 620         st->node = node;
 621         st->bringup = bringup;
 622         st->cb_state = state;
 623         st->single = true;
 624
 625         __cpuhp_kick_ap(st);
 626
 627         /*
 628          * If we failed and did a partial, do a rollback.
 629          */
 630         if ((ret = st->result) && st->last) {
 631                 st->rollback = true;
 632                 st->bringup = !bringup;
 633
 634                 __cpuhp_kick_ap(st);
 635         }
 636
 637         /*
 638          * Clean up the leftovers so the next hotplug operation wont use stale
 639          * data.
 640          */
 641         st->node = st->last = NULL;
 642         return ret;
 643 }
 644
 645 static int cpuhp_kick_ap_work(unsigned int cpu)
 646 {
 647         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 648         enum cpuhp_state prev_state = st->state;
 649         int ret;
 650
 651         cpuhp_lock_acquire(false);
 652         cpuhp_lock_release(false);
 653
 654         cpuhp_lock_acquire(true);
 655         cpuhp_lock_release(true);
 656
 657         trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
 658         ret = cpuhp_kick_ap(st, st->target);
 659         trace_cpuhp_exit(cpu, st->state, prev_state, ret);
 660
 661         return ret;
 662 }
 663
 664 static struct smp_hotplug_thread cpuhp_threads = {
 665         .store                  = &cpuhp_state.thread,
 666         .create                 = &cpuhp_create,
 667         .thread_should_run      = cpuhp_should_run,
 668         .thread_fn              = cpuhp_thread_fun,
 669         .thread_comm            = "cpuhp/%u",
 670         .selfparking            = true,
 671 };
 672
 673 void __init cpuhp_threads_init(void)
 674 {
 675         BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
 676         kthread_unpark(this_cpu_read(cpuhp_state.thread));
 677 }
 678
 679 #ifdef CONFIG_HOTPLUG_CPU
 680 /**
 681  * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
 682  * @cpu: a CPU id
 683  *
 684  * This function walks all processes, finds a valid mm struct for each one and
 685  * then clears a corresponding bit in mm's cpumask.  While this all sounds
 686  * trivial, there are various non-obvious corner cases, which this function
 687  * tries to solve in a safe manner.
 688  *
 689  * Also note that the function uses a somewhat relaxed locking scheme, so it may
 690  * be called only for an already offlined CPU.
 691  */
 692 void clear_tasks_mm_cpumask(int cpu)
 693 {
 694         struct task_struct *p;
 695
 696         /*
 697          * This function is called after the cpu is taken down and marked
 698          * offline, so its not like new tasks will ever get this cpu set in
 699          * their mm mask. -- Peter Zijlstra
 700          * Thus, we may use rcu_read_lock() here, instead of grabbing
 701          * full-fledged tasklist_lock.
 702          */
 703         WARN_ON(cpu_online(cpu));
 704         rcu_read_lock();
 705         for_each_process(p) {
 706                 struct task_struct *t;
 707
 708                 /*
 709                  * Main thread might exit, but other threads may still have
 710                  * a valid mm. Find one.
 711                  */
 712                 t = find_lock_task_mm(p);
 713                 if (!t)
 714                         continue;
 715                 cpumask_clear_cpu(cpu, mm_cpumask(t->mm));
 716                 task_unlock(t);
 717         }
 718         rcu_read_unlock();
 719 }
 720
 721 /* Take this CPU down. */
 722 static int take_cpu_down(void *_param)
 723 {
 724         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 725         enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
 726         int err, cpu = smp_processor_id();
 727         int ret;
 728
 729         /* Ensure this CPU doesn't handle any more interrupts. */
 730         err = __cpu_disable();
 731         if (err < 0)
 732                 return err;
 733
 734         /*
 735          * We get here while we are in CPUHP_TEARDOWN_CPU state and we must not
 736          * do this step again.
 737          */
 738         WARN_ON(st->state != CPUHP_TEARDOWN_CPU);
 739         st->state--;
 740         /* Invoke the former CPU_DYING callbacks */
 741         for (; st->state > target; st->state--) {
 742                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 743                 /*
 744                  * DYING must not fail!
 745                  */
 746                 WARN_ON_ONCE(ret);
 747         }
 748
 749         /* Give up timekeeping duties */
 750         tick_handover_do_timer();
 751         /* Park the stopper thread */
 752         stop_machine_park(cpu);
 753         return 0;
 754 }
 755
 756 static int takedown_cpu(unsigned int cpu)
 757 {
 758         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 759         int err;
 760
 761         /* Park the smpboot threads */
 762         kthread_park(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 763         smpboot_park_threads(cpu);
 764
 765         /*
 766          * Prevent irq alloc/free while the dying cpu reorganizes the
 767          * interrupt affinities.
 768          */
 769         irq_lock_sparse();
 770
 771         /*
 772          * So now all preempt/rcu users must observe !cpu_active().
 773          */
 774         err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
 775         if (err) {
 776                 /* CPU refused to die */
 777                 irq_unlock_sparse();
 778                 /* Unpark the hotplug thread so we can rollback there */
 779                 kthread_unpark(per_cpu_ptr(&cpuhp_state, cpu)->thread);
 780                 return err;
 781         }
 782         BUG_ON(cpu_online(cpu));
 783
 784         /*
 785          * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
 786          * all runnable tasks from the CPU, there's only the idle task left now
 787          * that the migration thread is done doing the stop_machine thing.
 788          *
 789          * Wait for the stop thread to go away.
 790          */
 791         wait_for_ap_thread(st, false);
 792         BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
 793
 794         /* Interrupts are moved away from the dying cpu, reenable alloc/free */
 795         irq_unlock_sparse();
 796
 797         hotplug_cpu__broadcast_tick_pull(cpu);
 798         /* This actually kills the CPU. */
 799         __cpu_die(cpu);
 800
 801         tick_cleanup_dead_cpu(cpu);
 802         rcutree_migrate_callbacks(cpu);
 803         return 0;
 804 }
 805
 806 static void cpuhp_complete_idle_dead(void *arg)
 807 {
 808         struct cpuhp_cpu_state *st = arg;
 809
 810         complete_ap_thread(st, false);
 811 }
 812
 813 void cpuhp_report_idle_dead(void)
 814 {
 815         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 816
 817         BUG_ON(st->state != CPUHP_AP_OFFLINE);
 818         rcu_report_dead(smp_processor_id());
 819         st->state = CPUHP_AP_IDLE_DEAD;
 820         /*
 821          * We cannot call complete after rcu_report_dead() so we delegate it
 822          * to an online cpu.
 823          */
 824         smp_call_function_single(cpumask_first(cpu_online_mask),
 825                                  cpuhp_complete_idle_dead, st, 0);
 826 }
 827
 828 static void undo_cpu_down(unsigned int cpu, struct cpuhp_cpu_state *st)
 829 {
 830         for (st->state++; st->state < st->target; st->state++) {
 831                 struct cpuhp_step *step = cpuhp_get_step(st->state);
 832
 833                 if (!step->skip_onerr)
 834                         cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 835         }
 836 }
 837
 838 static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
 839                                 enum cpuhp_state target)
 840 {
 841         enum cpuhp_state prev_state = st->state;
 842         int ret = 0;
 843
 844         for (; st->state > target; st->state--) {
 845                 ret = cpuhp_invoke_callback(cpu, st->state, false, NULL, NULL);
 846                 if (ret) {
 847                         st->target = prev_state;
 848                         undo_cpu_down(cpu, st);
 849                         break;
 850                 }
 851         }
 852         return ret;
 853 }
 854
 855 /* Requires cpu_add_remove_lock to be held */
 856 static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
 857                            enum cpuhp_state target)
 858 {
 859         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 860         int prev_state, ret = 0;
 861
 862         if (num_online_cpus() == 1)
 863                 return -EBUSY;
 864
 865         if (!cpu_present(cpu))
 866                 return -EINVAL;
 867
 868         cpus_write_lock();
 869
 870         cpuhp_tasks_frozen = tasks_frozen;
 871
 872         prev_state = cpuhp_set_state(st, target);
 873         /*
 874          * If the current CPU state is in the range of the AP hotplug thread,
 875          * then we need to kick the thread.
 876          */
 877         if (st->state > CPUHP_TEARDOWN_CPU) {
 878                 st->target = max((int)target, CPUHP_TEARDOWN_CPU);
 879                 ret = cpuhp_kick_ap_work(cpu);
 880                 /*
 881                  * The AP side has done the error rollback already. Just
 882                  * return the error code..
 883                  */
 884                 if (ret)
 885                         goto out;
 886
 887                 /*
 888                  * We might have stopped still in the range of the AP hotplug
 889                  * thread. Nothing to do anymore.
 890                  */
 891                 if (st->state > CPUHP_TEARDOWN_CPU)
 892                         goto out;
 893
 894                 st->target = target;
 895         }
 896         /*
 897          * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
 898          * to do the further cleanups.
 899          */
 900         ret = cpuhp_down_callbacks(cpu, st, target);
 901         if (ret && st->state > CPUHP_TEARDOWN_CPU && st->state < prev_state) {
 902                 cpuhp_reset_state(st, prev_state);
 903                 __cpuhp_kick_ap(st);
 904         }
 905
 906 out:
 907         cpus_write_unlock();
 908         /*
 909          * Do post unplug cleanup. This is still protected against
 910          * concurrent CPU hotplug via cpu_add_remove_lock.
 911          */
 912         lockup_detector_cleanup();
 913         return ret;
 914 }
 915
 916 static int do_cpu_down(unsigned int cpu, enum cpuhp_state target)
 917 {
 918         int err;
 919
 920         cpu_maps_update_begin();
 921
 922         if (cpu_hotplug_disabled) {
 923                 err = -EBUSY;
 924                 goto out;
 925         }
 926
 927         err = _cpu_down(cpu, 0, target);
 928
 929 out:
 930         cpu_maps_update_done();
 931         return err;
 932 }
 933
 934 int cpu_down(unsigned int cpu)
 935 {
 936         return do_cpu_down(cpu, CPUHP_OFFLINE);
 937 }
 938 EXPORT_SYMBOL(cpu_down);
 939
 940 #else
 941 #define takedown_cpu            NULL
 942 #endif /*CONFIG_HOTPLUG_CPU*/
 943
 944 /**
 945  * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
 946  * @cpu: cpu that just started
 947  *
 948  * It must be called by the arch code on the new cpu, before the new cpu
 949  * enables interrupts and before the "boot" cpu returns from __cpu_up().
 950  */
 951 void notify_cpu_starting(unsigned int cpu)
 952 {
 953         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 954         enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
 955         int ret;
 956
 957         rcu_cpu_starting(cpu);  /* Enables RCU usage on this CPU. */
 958         while (st->state < target) {
 959                 st->state++;
 960                 ret = cpuhp_invoke_callback(cpu, st->state, true, NULL, NULL);
 961                 /*
 962                  * STARTING must not fail!
 963                  */
 964                 WARN_ON_ONCE(ret);
 965         }
 966 }
 967
 968 /*
 969  * Called from the idle task. Wake up the controlling task which brings the
 970  * stopper and the hotplug thread of the upcoming CPU up and then delegates
 971  * the rest of the online bringup to the hotplug thread.
 972  */
 973 void cpuhp_online_idle(enum cpuhp_state state)
 974 {
 975         struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
 976
 977         /* Happens for the boot cpu */
 978         if (state != CPUHP_AP_ONLINE_IDLE)
 979                 return;
 980
 981         st->state = CPUHP_AP_ONLINE_IDLE;
 982         complete_ap_thread(st, true);
 983 }
 984
 985 /* Requires cpu_add_remove_lock to be held */
 986 static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
 987 {
 988         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
 989         struct task_struct *idle;
 990         int ret = 0;
 991
 992         cpus_write_lock();
 993
 994         if (!cpu_present(cpu)) {
 995                 ret = -EINVAL;
 996                 goto out;
 997         }
 998
 999         /*
1000          * The caller of do_cpu_up might have raced with another
1001          * caller. Ignore it for now.
1002          */
1003         if (st->state >= target)
1004                 goto out;
1005
1006         if (st->state == CPUHP_OFFLINE) {
1007                 /* Let it fail before we try to bring the cpu up */
1008                 idle = idle_thread_get(cpu);
1009                 if (IS_ERR(idle)) {
1010                         ret = PTR_ERR(idle);
1011                         goto out;
1012                 }
1013         }
1014
1015         cpuhp_tasks_frozen = tasks_frozen;
1016
1017         cpuhp_set_state(st, target);
1018         /*
1019          * If the current CPU state is in the range of the AP hotplug thread,
1020          * then we need to kick the thread once more.
1021          */
1022         if (st->state > CPUHP_BRINGUP_CPU) {
1023                 ret = cpuhp_kick_ap_work(cpu);
1024                 /*
1025                  * The AP side has done the error rollback already. Just
1026                  * return the error code..
1027                  */
1028                 if (ret)
1029                         goto out;
1030         }
1031
1032         /*
1033          * Try to reach the target state. We max out on the BP at
1034          * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
1035          * responsible for bringing it up to the target state.
1036          */
1037         target = min((int)target, CPUHP_BRINGUP_CPU);
1038         ret = cpuhp_up_callbacks(cpu, st, target);
1039 out:
1040         cpus_write_unlock();
1041         return ret;
1042 }
1043
1044 static int do_cpu_up(unsigned int cpu, enum cpuhp_state target)
1045 {
1046         int err = 0;
1047
1048         if (!cpu_possible(cpu)) {
1049                 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
1050                        cpu);
1051 #if defined(CONFIG_IA64)
1052                 pr_err("please check additional_cpus= boot parameter\n");
1053 #endif
1054                 return -EINVAL;
1055         }
1056
1057         err = try_online_node(cpu_to_node(cpu));
1058         if (err)
1059                 return err;
1060
1061         cpu_maps_update_begin();
1062
1063         if (cpu_hotplug_disabled) {
1064                 err = -EBUSY;
1065                 goto out;
1066         }
1067
1068         err = _cpu_up(cpu, 0, target);
1069 out:
1070         cpu_maps_update_done();
1071         return err;
1072 }
1073
1074 int cpu_up(unsigned int cpu)
1075 {
1076         return do_cpu_up(cpu, CPUHP_ONLINE);
1077 }
1078 EXPORT_SYMBOL_GPL(cpu_up);
1079
1080 #ifdef CONFIG_PM_SLEEP_SMP
1081 static cpumask_var_t frozen_cpus;
1082
1083 int freeze_secondary_cpus(int primary)
1084 {
1085         int cpu, error = 0;
1086
1087         cpu_maps_update_begin();
1088         if (!cpu_online(primary))
1089                 primary = cpumask_first(cpu_online_mask);
1090         /*
1091          * We take down all of the non-boot CPUs in one shot to avoid races
1092          * with the userspace trying to use the CPU hotplug at the same time
1093          */
1094         cpumask_clear(frozen_cpus);
1095
1096         pr_info("Disabling non-boot CPUs ...\n");
1097         for_each_online_cpu(cpu) {
1098                 if (cpu == primary)
1099                         continue;
1100                 trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
1101                 error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
1102                 trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
1103                 if (!error)
1104                         cpumask_set_cpu(cpu, frozen_cpus);
1105                 else {
1106                         pr_err("Error taking CPU%d down: %d\n", cpu, error);
1107                         break;
1108                 }
1109         }
1110
1111         if (!error)
1112                 BUG_ON(num_online_cpus() > 1);
1113         else
1114                 pr_err("Non-boot CPUs are not disabled\n");
1115
1116         /*
1117          * Make sure the CPUs won't be enabled by someone else. We need to do
1118          * this even in case of failure as all disable_nonboot_cpus() users are
1119          * supposed to do enable_nonboot_cpus() on the failure path.
1120          */
1121         cpu_hotplug_disabled++;
1122
1123         cpu_maps_update_done();
1124         return error;
1125 }
1126
1127 void __weak arch_enable_nonboot_cpus_begin(void)
1128 {
1129 }
1130
1131 void __weak arch_enable_nonboot_cpus_end(void)
1132 {
1133 }
1134
1135 void enable_nonboot_cpus(void)
1136 {
1137         int cpu, error;
1138
1139         /* Allow everyone to use the CPU hotplug again */
1140         cpu_maps_update_begin();
1141         __cpu_hotplug_enable();
1142         if (cpumask_empty(frozen_cpus))
1143                 goto out;
1144
1145         pr_info("Enabling non-boot CPUs ...\n");
1146
1147         arch_enable_nonboot_cpus_begin();
1148
1149         for_each_cpu(cpu, frozen_cpus) {
1150                 trace_suspend_resume(TPS("CPU_ON"), cpu, true);
1151                 error = _cpu_up(cpu, 1, CPUHP_ONLINE);
1152                 trace_suspend_resume(TPS("CPU_ON"), cpu, false);
1153                 if (!error) {
1154                         pr_info("CPU%d is up\n", cpu);
1155                         continue;
1156                 }
1157                 pr_warn("Error taking CPU%d up: %d\n", cpu, error);
1158         }
1159
1160         arch_enable_nonboot_cpus_end();
1161
1162         cpumask_clear(frozen_cpus);
1163 out:
1164         cpu_maps_update_done();
1165 }
1166
1167 static int __init alloc_frozen_cpus(void)
1168 {
1169         if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
1170                 return -ENOMEM;
1171         return 0;
1172 }
1173 core_initcall(alloc_frozen_cpus);
1174
1175 /*
1176  * When callbacks for CPU hotplug notifications are being executed, we must
1177  * ensure that the state of the system with respect to the tasks being frozen
1178  * or not, as reported by the notification, remains unchanged *throughout the
1179  * duration* of the execution of the callbacks.
1180  * Hence we need to prevent the freezer from racing with regular CPU hotplug.
1181  *
1182  * This synchronization is implemented by mutually excluding regular CPU
1183  * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
1184  * Hibernate notifications.
1185  */
1186 static int
1187 cpu_hotplug_pm_callback(struct notifier_block *nb,
1188                         unsigned long action, void *ptr)
1189 {
1190         switch (action) {
1191
1192         case PM_SUSPEND_PREPARE:
1193         case PM_HIBERNATION_PREPARE:
1194                 cpu_hotplug_disable();
1195                 break;
1196
1197         case PM_POST_SUSPEND:
1198         case PM_POST_HIBERNATION:
1199                 cpu_hotplug_enable();
1200                 break;
1201
1202         default:
1203                 return NOTIFY_DONE;
1204         }
1205
1206         return NOTIFY_OK;
1207 }
1208
1209
1210 static int __init cpu_hotplug_pm_sync_init(void)
1211 {
1212         /*
1213          * cpu_hotplug_pm_callback has higher priority than x86
1214          * bsp_pm_callback which depends on cpu_hotplug_pm_callback
1215          * to disable cpu hotplug to avoid cpu hotplug race.
1216          */
1217         pm_notifier(cpu_hotplug_pm_callback, 0);
1218         return 0;
1219 }
1220 core_initcall(cpu_hotplug_pm_sync_init);
1221
1222 #endif /* CONFIG_PM_SLEEP_SMP */
1223
1224 int __boot_cpu_id;
1225
1226 #endif /* CONFIG_SMP */
1227
1228 /* Boot processor state steps */
1229 static struct cpuhp_step cpuhp_hp_states[] = {
1230         [CPUHP_OFFLINE] = {
1231                 .name                   = "offline",
1232                 .startup.single         = NULL,
1233                 .teardown.single        = NULL,
1234         },
1235 #ifdef CONFIG_SMP
1236         [CPUHP_CREATE_THREADS]= {
1237                 .name                   = "threads:prepare",
1238                 .startup.single         = smpboot_create_threads,
1239                 .teardown.single        = NULL,
1240                 .cant_stop              = true,
1241         },
1242         [CPUHP_PERF_PREPARE] = {
1243                 .name                   = "perf:prepare",
1244                 .startup.single         = perf_event_init_cpu,
1245                 .teardown.single        = perf_event_exit_cpu,
1246         },
1247         [CPUHP_WORKQUEUE_PREP] = {
1248                 .name                   = "workqueue:prepare",
1249                 .startup.single         = workqueue_prepare_cpu,
1250                 .teardown.single        = NULL,
1251         },
1252         [CPUHP_HRTIMERS_PREPARE] = {
1253                 .name                   = "hrtimers:prepare",
1254                 .startup.single         = hrtimers_prepare_cpu,
1255                 .teardown.single        = hrtimers_dead_cpu,
1256         },
1257         [CPUHP_SMPCFD_PREPARE] = {
1258                 .name                   = "smpcfd:prepare",
1259                 .startup.single         = smpcfd_prepare_cpu,
1260                 .teardown.single        = smpcfd_dead_cpu,
1261         },
1262         [CPUHP_RELAY_PREPARE] = {
1263                 .name                   = "relay:prepare",
1264                 .startup.single         = relay_prepare_cpu,
1265                 .teardown.single        = NULL,
1266         },
1267         [CPUHP_SLAB_PREPARE] = {
1268                 .name                   = "slab:prepare",
1269                 .startup.single         = slab_prepare_cpu,
1270                 .teardown.single        = slab_dead_cpu,
1271         },
1272         [CPUHP_RCUTREE_PREP] = {
1273                 .name                   = "RCU/tree:prepare",
1274                 .startup.single         = rcutree_prepare_cpu,
1275                 .teardown.single        = rcutree_dead_cpu,
1276         },
1277         /*
1278          * On the tear-down path, timers_dead_cpu() must be invoked
1279          * before blk_mq_queue_reinit_notify() from notify_dead(),
1280          * otherwise a RCU stall occurs.
1281          */
1282         [CPUHP_TIMERS_PREPARE] = {
1283                 .name                   = "timers:dead",
1284                 .startup.single         = timers_prepare_cpu,
1285                 .teardown.single        = timers_dead_cpu,
1286         },
1287         /* Kicks the plugged cpu into life */
1288         [CPUHP_BRINGUP_CPU] = {
1289                 .name                   = "cpu:bringup",
1290                 .startup.single         = bringup_cpu,
1291                 .teardown.single        = NULL,
1292                 .cant_stop              = true,
1293         },
1294         /* Final state before CPU kills itself */
1295         [CPUHP_AP_IDLE_DEAD] = {
1296                 .name                   = "idle:dead",
1297         },
1298         /*
1299          * Last state before CPU enters the idle loop to die. Transient state
1300          * for synchronization.
1301          */
1302         [CPUHP_AP_OFFLINE] = {
1303                 .name                   = "ap:offline",
1304                 .cant_stop              = true,
1305         },
1306         /* First state is scheduler control. Interrupts are disabled */
1307         [CPUHP_AP_SCHED_STARTING] = {
1308                 .name                   = "sched:starting",
1309                 .startup.single         = sched_cpu_starting,
1310                 .teardown.single        = sched_cpu_dying,
1311         },
1312         [CPUHP_AP_RCUTREE_DYING] = {
1313                 .name                   = "RCU/tree:dying",
1314                 .startup.single         = NULL,
1315                 .teardown.single        = rcutree_dying_cpu,
1316         },
1317         [CPUHP_AP_SMPCFD_DYING] = {
1318                 .name                   = "smpcfd:dying",
1319                 .startup.single         = NULL,
1320                 .teardown.single        = smpcfd_dying_cpu,
1321         },
1322         /* Entry state on starting. Interrupts enabled from here on. Transient
1323          * state for synchronsization */
1324         [CPUHP_AP_ONLINE] = {
1325                 .name                   = "ap:online",
1326         },
1327         /*
1328          * Handled on controll processor until the plugged processor manages
1329          * this itself.
1330          */
1331         [CPUHP_TEARDOWN_CPU] = {
1332                 .name                   = "cpu:teardown",
1333                 .startup.single         = NULL,
1334                 .teardown.single        = takedown_cpu,
1335                 .cant_stop              = true,
1336         },
1337         /* Handle smpboot threads park/unpark */
1338         [CPUHP_AP_SMPBOOT_THREADS] = {
1339                 .name                   = "smpboot/threads:online",
1340                 .startup.single         = smpboot_unpark_threads,
1341                 .teardown.single        = NULL,
1342         },
1343         [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1344                 .name                   = "irq/affinity:online",
1345                 .startup.single         = irq_affinity_online_cpu,
1346                 .teardown.single        = NULL,
1347         },
1348         [CPUHP_AP_PERF_ONLINE] = {
1349                 .name                   = "perf:online",
1350                 .startup.single         = perf_event_init_cpu,
1351                 .teardown.single        = perf_event_exit_cpu,
1352         },
1353         [CPUHP_AP_WORKQUEUE_ONLINE] = {
1354                 .name                   = "workqueue:online",
1355                 .startup.single         = workqueue_online_cpu,
1356                 .teardown.single        = workqueue_offline_cpu,
1357         },
1358         [CPUHP_AP_RCUTREE_ONLINE] = {
1359                 .name                   = "RCU/tree:online",
1360                 .startup.single         = rcutree_online_cpu,
1361                 .teardown.single        = rcutree_offline_cpu,
1362         },
1363 #endif
1364         /*
1365          * The dynamically registered state space is here
1366          */
1367
1368 #ifdef CONFIG_SMP
1369         /* Last state is scheduler control setting the cpu active */
1370         [CPUHP_AP_ACTIVE] = {
1371                 .name                   = "sched:active",
1372                 .startup.single         = sched_cpu_activate,
1373                 .teardown.single        = sched_cpu_deactivate,
1374         },
1375 #endif
1376
1377         /* CPU is fully up and running. */
1378         [CPUHP_ONLINE] = {
1379                 .name                   = "online",
1380                 .startup.single         = NULL,
1381                 .teardown.single        = NULL,
1382         },
1383 };
1384
1385 /* Sanity check for callbacks */
1386 static int cpuhp_cb_check(enum cpuhp_state state)
1387 {
1388         if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
1389                 return -EINVAL;
1390         return 0;
1391 }
1392
1393 /*
1394  * Returns a free for dynamic slot assignment of the Online state. The states
1395  * are protected by the cpuhp_slot_states mutex and an empty slot is identified
1396  * by having no name assigned.
1397  */
1398 static int cpuhp_reserve_state(enum cpuhp_state state)
1399 {
1400         enum cpuhp_state i, end;
1401         struct cpuhp_step *step;
1402
1403         switch (state) {
1404         case CPUHP_AP_ONLINE_DYN:
1405                 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
1406                 end = CPUHP_AP_ONLINE_DYN_END;
1407                 break;
1408         case CPUHP_BP_PREPARE_DYN:
1409                 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
1410                 end = CPUHP_BP_PREPARE_DYN_END;
1411                 break;
1412         default:
1413                 return -EINVAL;
1414         }
1415
1416         for (i = state; i <= end; i++, step++) {
1417                 if (!step->name)
1418                         return i;
1419         }
1420         WARN(1, "No more dynamic states available for CPU hotplug\n");
1421         return -ENOSPC;
1422 }
1423
1424 static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
1425                                  int (*startup)(unsigned int cpu),
1426                                  int (*teardown)(unsigned int cpu),
1427                                  bool multi_instance)
1428 {
1429         /* (Un)Install the callbacks for further cpu hotplug operations */
1430         struct cpuhp_step *sp;
1431         int ret = 0;
1432
1433         /*
1434          * If name is NULL, then the state gets removed.
1435          *
1436          * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
1437          * the first allocation from these dynamic ranges, so the removal
1438          * would trigger a new allocation and clear the wrong (already
1439          * empty) state, leaving the callbacks of the to be cleared state
1440          * dangling, which causes wreckage on the next hotplug operation.
1441          */
1442         if (name && (state == CPUHP_AP_ONLINE_DYN ||
1443                      state == CPUHP_BP_PREPARE_DYN)) {
1444                 ret = cpuhp_reserve_state(state);
1445                 if (ret < 0)
1446                         return ret;
1447                 state = ret;
1448         }
1449         sp = cpuhp_get_step(state);
1450         if (name && sp->name)
1451                 return -EBUSY;
1452
1453         sp->startup.single = startup;
1454         sp->teardown.single = teardown;
1455         sp->name = name;
1456         sp->multi_instance = multi_instance;
1457         INIT_HLIST_HEAD(&sp->list);
1458         return ret;
1459 }
1460
1461 static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
1462 {
1463         return cpuhp_get_step(state)->teardown.single;
1464 }
1465
1466 /*
1467  * Call the startup/teardown function for a step either on the AP or
1468  * on the current CPU.
1469  */
1470 static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
1471                             struct hlist_node *node)
1472 {
1473         struct cpuhp_step *sp = cpuhp_get_step(state);
1474         int ret;
1475
1476         /*
1477          * If there's nothing to do, we done.
1478          * Relies on the union for multi_instance.
1479          */
1480         if ((bringup && !sp->startup.single) ||
1481             (!bringup && !sp->teardown.single))
1482                 return 0;
1483         /*
1484          * The non AP bound callbacks can fail on bringup. On teardown
1485          * e.g. module removal we crash for now.
1486          */
1487 #ifdef CONFIG_SMP
1488         if (cpuhp_is_ap_state(state))
1489                 ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
1490         else
1491                 ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1492 #else
1493         ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
1494 #endif
1495         BUG_ON(ret && !bringup);
1496         return ret;
1497 }
1498
1499 /*
1500  * Called from __cpuhp_setup_state on a recoverable failure.
1501  *
1502  * Note: The teardown callbacks for rollback are not allowed to fail!
1503  */
1504 static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
1505                                    struct hlist_node *node)
1506 {
1507         int cpu;
1508
1509         /* Roll back the already executed steps on the other cpus */
1510         for_each_present_cpu(cpu) {
1511                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1512                 int cpustate = st->state;
1513
1514                 if (cpu >= failedcpu)
1515                         break;
1516
1517                 /* Did we invoke the startup call on that cpu ? */
1518                 if (cpustate >= state)
1519                         cpuhp_issue_call(cpu, state, false, node);
1520         }
1521 }
1522
1523 int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
1524                                           struct hlist_node *node,
1525                                           bool invoke)
1526 {
1527         struct cpuhp_step *sp;
1528         int cpu;
1529         int ret;
1530
1531         lockdep_assert_cpus_held();
1532
1533         sp = cpuhp_get_step(state);
1534         if (sp->multi_instance == false)
1535                 return -EINVAL;
1536
1537         mutex_lock(&cpuhp_state_mutex);
1538
1539         if (!invoke || !sp->startup.multi)
1540                 goto add_node;
1541
1542         /*
1543          * Try to call the startup callback for each present cpu
1544          * depending on the hotplug state of the cpu.
1545          */
1546         for_each_present_cpu(cpu) {
1547                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1548                 int cpustate = st->state;
1549
1550                 if (cpustate < state)
1551                         continue;
1552
1553                 ret = cpuhp_issue_call(cpu, state, true, node);
1554                 if (ret) {
1555                         if (sp->teardown.multi)
1556                                 cpuhp_rollback_install(cpu, state, node);
1557                         goto unlock;
1558                 }
1559         }
1560 add_node:
1561         ret = 0;
1562         hlist_add_head(node, &sp->list);
1563 unlock:
1564         mutex_unlock(&cpuhp_state_mutex);
1565         return ret;
1566 }
1567
1568 int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
1569                                bool invoke)
1570 {
1571         int ret;
1572
1573         cpus_read_lock();
1574         ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
1575         cpus_read_unlock();
1576         return ret;
1577 }
1578 EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
1579
1580 /**
1581  * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
1582  * @state:              The state to setup
1583  * @invoke:             If true, the startup function is invoked for cpus where
1584  *                      cpu state >= @state
1585  * @startup:            startup callback function
1586  * @teardown:           teardown callback function
1587  * @multi_instance:     State is set up for multiple instances which get
1588  *                      added afterwards.
1589  *
1590  * The caller needs to hold cpus read locked while calling this function.
1591  * Returns:
1592  *   On success:
1593  *      Positive state number if @state is CPUHP_AP_ONLINE_DYN
1594  *      0 for all other states
1595  *   On failure: proper (negative) error code
1596  */
1597 int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
1598                                    const char *name, bool invoke,
1599                                    int (*startup)(unsigned int cpu),
1600                                    int (*teardown)(unsigned int cpu),
1601                                    bool multi_instance)
1602 {
1603         int cpu, ret = 0;
1604         bool dynstate;
1605
1606         lockdep_assert_cpus_held();
1607
1608         if (cpuhp_cb_check(state) || !name)
1609                 return -EINVAL;
1610
1611         mutex_lock(&cpuhp_state_mutex);
1612
1613         ret = cpuhp_store_callbacks(state, name, startup, teardown,
1614                                     multi_instance);
1615
1616         dynstate = state == CPUHP_AP_ONLINE_DYN;
1617         if (ret > 0 && dynstate) {
1618                 state = ret;
1619                 ret = 0;
1620         }
1621
1622         if (ret || !invoke || !startup)
1623                 goto out;
1624
1625         /*
1626          * Try to call the startup callback for each present cpu
1627          * depending on the hotplug state of the cpu.
1628          */
1629         for_each_present_cpu(cpu) {
1630                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1631                 int cpustate = st->state;
1632
1633                 if (cpustate < state)
1634                         continue;
1635
1636                 ret = cpuhp_issue_call(cpu, state, true, NULL);
1637                 if (ret) {
1638                         if (teardown)
1639                                 cpuhp_rollback_install(cpu, state, NULL);
1640                         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1641                         goto out;
1642                 }
1643         }
1644 out:
1645         mutex_unlock(&cpuhp_state_mutex);
1646         /*
1647          * If the requested state is CPUHP_AP_ONLINE_DYN, return the
1648          * dynamically allocated state in case of success.
1649          */
1650         if (!ret && dynstate)
1651                 return state;
1652         return ret;
1653 }
1654 EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
1655
1656 int __cpuhp_setup_state(enum cpuhp_state state,
1657                         const char *name, bool invoke,
1658                         int (*startup)(unsigned int cpu),
1659                         int (*teardown)(unsigned int cpu),
1660                         bool multi_instance)
1661 {
1662         int ret;
1663
1664         cpus_read_lock();
1665         ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
1666                                              teardown, multi_instance);
1667         cpus_read_unlock();
1668         return ret;
1669 }
1670 EXPORT_SYMBOL(__cpuhp_setup_state);
1671
1672 int __cpuhp_state_remove_instance(enum cpuhp_state state,
1673                                   struct hlist_node *node, bool invoke)
1674 {
1675         struct cpuhp_step *sp = cpuhp_get_step(state);
1676         int cpu;
1677
1678         BUG_ON(cpuhp_cb_check(state));
1679
1680         if (!sp->multi_instance)
1681                 return -EINVAL;
1682
1683         cpus_read_lock();
1684         mutex_lock(&cpuhp_state_mutex);
1685
1686         if (!invoke || !cpuhp_get_teardown_cb(state))
1687                 goto remove;
1688         /*
1689          * Call the teardown callback for each present cpu depending
1690          * on the hotplug state of the cpu. This function is not
1691          * allowed to fail currently!
1692          */
1693         for_each_present_cpu(cpu) {
1694                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1695                 int cpustate = st->state;
1696
1697                 if (cpustate >= state)
1698                         cpuhp_issue_call(cpu, state, false, node);
1699         }
1700
1701 remove:
1702         hlist_del(node);
1703         mutex_unlock(&cpuhp_state_mutex);
1704         cpus_read_unlock();
1705
1706         return 0;
1707 }
1708 EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
1709
1710 /**
1711  * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
1712  * @state:      The state to remove
1713  * @invoke:     If true, the teardown function is invoked for cpus where
1714  *              cpu state >= @state
1715  *
1716  * The caller needs to hold cpus read locked while calling this function.
1717  * The teardown callback is currently not allowed to fail. Think
1718  * about module removal!
1719  */
1720 void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
1721 {
1722         struct cpuhp_step *sp = cpuhp_get_step(state);
1723         int cpu;
1724
1725         BUG_ON(cpuhp_cb_check(state));
1726
1727         lockdep_assert_cpus_held();
1728
1729         mutex_lock(&cpuhp_state_mutex);
1730         if (sp->multi_instance) {
1731                 WARN(!hlist_empty(&sp->list),
1732                      "Error: Removing state %d which has instances left.\n",
1733                      state);
1734                 goto remove;
1735         }
1736
1737         if (!invoke || !cpuhp_get_teardown_cb(state))
1738                 goto remove;
1739
1740         /*
1741          * Call the teardown callback for each present cpu depending
1742          * on the hotplug state of the cpu. This function is not
1743          * allowed to fail currently!
1744          */
1745         for_each_present_cpu(cpu) {
1746                 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
1747                 int cpustate = st->state;
1748
1749                 if (cpustate >= state)
1750                         cpuhp_issue_call(cpu, state, false, NULL);
1751         }
1752 remove:
1753         cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
1754         mutex_unlock(&cpuhp_state_mutex);
1755 }
1756 EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
1757
1758 void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
1759 {
1760         cpus_read_lock();
1761         __cpuhp_remove_state_cpuslocked(state, invoke);
1762         cpus_read_unlock();
1763 }
1764 EXPORT_SYMBOL(__cpuhp_remove_state);
1765
1766 #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
1767 static ssize_t show_cpuhp_state(struct device *dev,
1768                                 struct device_attribute *attr, char *buf)
1769 {
1770         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1771
1772         return sprintf(buf, "%d\n", st->state);
1773 }
1774 static DEVICE_ATTR(state, 0444, show_cpuhp_state, NULL);
1775
1776 static ssize_t write_cpuhp_target(struct device *dev,
1777                                   struct device_attribute *attr,
1778                                   const char *buf, size_t count)
1779 {
1780         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1781         struct cpuhp_step *sp;
1782         int target, ret;
1783
1784         ret = kstrtoint(buf, 10, &target);
1785         if (ret)
1786                 return ret;
1787
1788 #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
1789         if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
1790                 return -EINVAL;
1791 #else
1792         if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
1793                 return -EINVAL;
1794 #endif
1795
1796         ret = lock_device_hotplug_sysfs();
1797         if (ret)
1798                 return ret;
1799
1800         mutex_lock(&cpuhp_state_mutex);
1801         sp = cpuhp_get_step(target);
1802         ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
1803         mutex_unlock(&cpuhp_state_mutex);
1804         if (ret)
1805                 goto out;
1806
1807         if (st->state < target)
1808                 ret = do_cpu_up(dev->id, target);
1809         else
1810                 ret = do_cpu_down(dev->id, target);
1811 out:
1812         unlock_device_hotplug();
1813         return ret ? ret : count;
1814 }
1815
1816 static ssize_t show_cpuhp_target(struct device *dev,
1817                                  struct device_attribute *attr, char *buf)
1818 {
1819         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1820
1821         return sprintf(buf, "%d\n", st->target);
1822 }
1823 static DEVICE_ATTR(target, 0644, show_cpuhp_target, write_cpuhp_target);
1824
1825
1826 static ssize_t write_cpuhp_fail(struct device *dev,
1827                                 struct device_attribute *attr,
1828                                 const char *buf, size_t count)
1829 {
1830         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1831         struct cpuhp_step *sp;
1832         int fail, ret;
1833
1834         ret = kstrtoint(buf, 10, &fail);
1835         if (ret)
1836                 return ret;
1837
1838         /*
1839          * Cannot fail STARTING/DYING callbacks.
1840          */
1841         if (cpuhp_is_atomic_state(fail))
1842                 return -EINVAL;
1843
1844         /*
1845          * Cannot fail anything that doesn't have callbacks.
1846          */
1847         mutex_lock(&cpuhp_state_mutex);
1848         sp = cpuhp_get_step(fail);
1849         if (!sp->startup.single && !sp->teardown.single)
1850                 ret = -EINVAL;
1851         mutex_unlock(&cpuhp_state_mutex);
1852         if (ret)
1853                 return ret;
1854
1855         st->fail = fail;
1856
1857         return count;
1858 }
1859
1860 static ssize_t show_cpuhp_fail(struct device *dev,
1861                                struct device_attribute *attr, char *buf)
1862 {
1863         struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
1864
1865         return sprintf(buf, "%d\n", st->fail);
1866 }
1867
1868 static DEVICE_ATTR(fail, 0644, show_cpuhp_fail, write_cpuhp_fail);
1869
1870 static struct attribute *cpuhp_cpu_attrs[] = {
1871         &dev_attr_state.attr,
1872         &dev_attr_target.attr,
1873         &dev_attr_fail.attr,
1874         NULL
1875 };
1876
1877 static const struct attribute_group cpuhp_cpu_attr_group = {
1878         .attrs = cpuhp_cpu_attrs,
1879         .name = "hotplug",
1880         NULL
1881 };
1882
1883 static ssize_t show_cpuhp_states(struct device *dev,
1884                                  struct device_attribute *attr, char *buf)
1885 {
1886         ssize_t cur, res = 0;
1887         int i;
1888
1889         mutex_lock(&cpuhp_state_mutex);
1890         for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
1891                 struct cpuhp_step *sp = cpuhp_get_step(i);
1892
1893                 if (sp->name) {
1894                         cur = sprintf(buf, "%3d: %s\n", i, sp->name);
1895                         buf += cur;
1896                         res += cur;
1897                 }
1898         }
1899         mutex_unlock(&cpuhp_state_mutex);
1900         return res;
1901 }
1902 static DEVICE_ATTR(states, 0444, show_cpuhp_states, NULL);
1903
1904 static struct attribute *cpuhp_cpu_root_attrs[] = {
1905         &dev_attr_states.attr,
1906         NULL
1907 };
1908
1909 static const struct attribute_group cpuhp_cpu_root_attr_group = {
1910         .attrs = cpuhp_cpu_root_attrs,
1911         .name = "hotplug",
1912         NULL
1913 };
1914
1915 static int __init cpuhp_sysfs_init(void)
1916 {
1917         int cpu, ret;
1918
1919         ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
1920                                  &cpuhp_cpu_root_attr_group);
1921         if (ret)
1922                 return ret;
1923
1924         for_each_possible_cpu(cpu) {
1925                 struct device *dev = get_cpu_device(cpu);
1926
1927                 if (!dev)
1928                         continue;
1929                 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
1930                 if (ret)
1931                         return ret;
1932         }
1933         return 0;
1934 }
1935 device_initcall(cpuhp_sysfs_init);
1936 #endif
1937
1938 /*
1939  * cpu_bit_bitmap[] is a special, "compressed" data structure that
1940  * represents all NR_CPUS bits binary values of 1<<nr.
1941  *
1942  * It is used by cpumask_of() to get a constant address to a CPU
1943  * mask value that has a single bit set only.
1944  */
1945
1946 /* cpu_bit_bitmap[0] is empty - so we can back into it */
1947 #define MASK_DECLARE_1(x)       [x+1][0] = (1UL << (x))
1948 #define MASK_DECLARE_2(x)       MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
1949 #define MASK_DECLARE_4(x)       MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
1950 #define MASK_DECLARE_8(x)       MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
1951
1952 const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
1953
1954         MASK_DECLARE_8(0),      MASK_DECLARE_8(8),
1955         MASK_DECLARE_8(16),     MASK_DECLARE_8(24),
1956 #if BITS_PER_LONG > 32
1957         MASK_DECLARE_8(32),     MASK_DECLARE_8(40),
1958         MASK_DECLARE_8(48),     MASK_DECLARE_8(56),
1959 #endif
1960 };
1961 EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
1962
1963 const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
1964 EXPORT_SYMBOL(cpu_all_bits);
1965
1966 #ifdef CONFIG_INIT_ALL_POSSIBLE
1967 struct cpumask __cpu_possible_mask __read_mostly
1968         = {CPU_BITS_ALL};
1969 #else
1970 struct cpumask __cpu_possible_mask __read_mostly;
1971 #endif
1972 EXPORT_SYMBOL(__cpu_possible_mask);
1973
1974 struct cpumask __cpu_online_mask __read_mostly;
1975 EXPORT_SYMBOL(__cpu_online_mask);
1976
1977 struct cpumask __cpu_present_mask __read_mostly;
1978 EXPORT_SYMBOL(__cpu_present_mask);
1979
1980 struct cpumask __cpu_active_mask __read_mostly;
1981 EXPORT_SYMBOL(__cpu_active_mask);
1982
1983 void init_cpu_present(const struct cpumask *src)
1984 {
1985         cpumask_copy(&__cpu_present_mask, src);
1986 }
1987
1988 void init_cpu_possible(const struct cpumask *src)
1989 {
1990         cpumask_copy(&__cpu_possible_mask, src);
1991 }
1992
1993 void init_cpu_online(const struct cpumask *src)
1994 {
1995         cpumask_copy(&__cpu_online_mask, src);
1996 }
1997
1998 /*
1999  * Activate the first processor.
2000  */
2001 void __init boot_cpu_init(void)
2002 {
2003         int cpu = smp_processor_id();
2004
2005         /* Mark the boot cpu "present", "online" etc for SMP and UP case */
2006         set_cpu_online(cpu, true);
2007         set_cpu_active(cpu, true);
2008         set_cpu_present(cpu, true);
2009         set_cpu_possible(cpu, true);
2010
2011 #ifdef CONFIG_SMP
2012         __boot_cpu_id = cpu;
2013 #endif
2014 }
2015
2016 /*
2017  * Must be called _AFTER_ setting up the per_cpu areas
2018  */
2019 void __init boot_cpu_state_init(void)
2020 {
2021         per_cpu_ptr(&cpuhp_state, smp_processor_id())->state = CPUHP_ONLINE;
2022 }