kernel/softirq.c

   1 /*
   2  *      linux/kernel/softirq.c
   3  *
   4  *      Copyright (C) 1992 Linus Torvalds
   5  *
   6  *      Distribute under GPLv2.
   7  *
   8  *      Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   9  *
  10  *      Remote softirq infrastructure is by Jens Axboe.
  11  */
  12
  13 #include <linux/export.h>
  14 #include <linux/kernel_stat.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/init.h>
  17 #include <linux/mm.h>
  18 #include <linux/notifier.h>
  19 #include <linux/percpu.h>
  20 #include <linux/cpu.h>
  21 #include <linux/freezer.h>
  22 #include <linux/kthread.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/ftrace.h>
  25 #include <linux/smp.h>
  26 #include <linux/smpboot.h>
  27 #include <linux/tick.h>
  28
  29 #define CREATE_TRACE_POINTS
  30 #include <trace/events/irq.h>
  31
  32 #include <asm/irq.h>
  33 /*
  34    - No shared variables, all the data are CPU local.
  35    - If a softirq needs serialization, let it serialize itself
  36      by its own spinlocks.
  37    - Even if softirq is serialized, only local cpu is marked for
  38      execution. Hence, we get something sort of weak cpu binding.
  39      Though it is still not clear, will it result in better locality
  40      or will not.
  41
  42    Examples:
  43    - NET RX softirq. It is multithreaded and does not require
  44      any global serialization.
  45    - NET TX softirq. It kicks software netdevice queues, hence
  46      it is logically serialized per device, but this serialization
  47      is invisible to common code.
  48    - Tasklets: serialized wrt itself.
  49  */
  50
  51 #ifndef __ARCH_IRQ_STAT
  52 irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned;
  53 EXPORT_SYMBOL(irq_stat);
  54 #endif
  55
  56 static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  57
  58 DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  59
  60 char *softirq_to_name[NR_SOFTIRQS] = {
  61         "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL",
  62         "TASKLET", "SCHED", "HRTIMER", "RCU"
  63 };
  64
  65 /*
  66  * we cannot loop indefinitely here to avoid userspace starvation,
  67  * but we also don't want to introduce a worst case 1/HZ latency
  68  * to the pending events, so lets the scheduler to balance
  69  * the softirq load for us.
  70  */
  71 static void wakeup_softirqd(void)
  72 {
  73         /* Interrupts are disabled: no need to stop preemption */
  74         struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  75
  76         if (tsk && tsk->state != TASK_RUNNING)
  77                 wake_up_process(tsk);
  78 }
  79
  80 /*
  81  * preempt_count and SOFTIRQ_OFFSET usage:
  82  * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving
  83  *   softirq processing.
  84  * - preempt_count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
  85  *   on local_bh_disable or local_bh_enable.
  86  * This lets us distinguish between whether we are currently processing
  87  * softirq and whether we just have bh disabled.
  88  */
  89
  90 /*
  91  * This one is for softirq.c-internal use,
  92  * where hardirqs are disabled legitimately:
  93  */
  94 #ifdef CONFIG_TRACE_IRQFLAGS
  95 static void __local_bh_disable(unsigned long ip, unsigned int cnt)
  96 {
  97         unsigned long flags;
  98
  99         WARN_ON_ONCE(in_irq());
 100
 101         raw_local_irq_save(flags);
 102         /*
 103          * The preempt tracer hooks into add_preempt_count and will break
 104          * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 105          * is set and before current->softirq_enabled is cleared.
 106          * We must manually increment preempt_count here and manually
 107          * call the trace_preempt_off later.
 108          */
 109         preempt_count() += cnt;
 110         /*
 111          * Were softirqs turned off above:
 112          */
 113         if (softirq_count() == cnt)
 114                 trace_softirqs_off(ip);
 115         raw_local_irq_restore(flags);
 116
 117         if (preempt_count() == cnt)
 118                 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
 119 }
 120 #else /* !CONFIG_TRACE_IRQFLAGS */
 121 static inline void __local_bh_disable(unsigned long ip, unsigned int cnt)
 122 {
 123         add_preempt_count(cnt);
 124         barrier();
 125 }
 126 #endif /* CONFIG_TRACE_IRQFLAGS */
 127
 128 void local_bh_disable(void)
 129 {
 130         __local_bh_disable((unsigned long)__builtin_return_address(0),
 131                                 SOFTIRQ_DISABLE_OFFSET);
 132 }
 133
 134 EXPORT_SYMBOL(local_bh_disable);
 135
 136 static void __local_bh_enable(unsigned int cnt)
 137 {
 138         WARN_ON_ONCE(in_irq());
 139         WARN_ON_ONCE(!irqs_disabled());
 140
 141         if (softirq_count() == cnt)
 142                 trace_softirqs_on((unsigned long)__builtin_return_address(0));
 143         sub_preempt_count(cnt);
 144 }
 145
 146 /*
 147  * Special-case - softirqs can safely be enabled in
 148  * cond_resched_softirq(), or by __do_softirq(),
 149  * without processing still-pending softirqs:
 150  */
 151 void _local_bh_enable(void)
 152 {
 153         __local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 154 }
 155
 156 EXPORT_SYMBOL(_local_bh_enable);
 157
 158 static inline void _local_bh_enable_ip(unsigned long ip)
 159 {
 160         WARN_ON_ONCE(in_irq() || irqs_disabled());
 161 #ifdef CONFIG_TRACE_IRQFLAGS
 162         local_irq_disable();
 163 #endif
 164         /*
 165          * Are softirqs going to be turned on now:
 166          */
 167         if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 168                 trace_softirqs_on(ip);
 169         /*
 170          * Keep preemption disabled until we are done with
 171          * softirq processing:
 172          */
 173         sub_preempt_count(SOFTIRQ_DISABLE_OFFSET - 1);
 174
 175         if (unlikely(!in_interrupt() && local_softirq_pending()))
 176                 do_softirq();
 177
 178         dec_preempt_count();
 179 #ifdef CONFIG_TRACE_IRQFLAGS
 180         local_irq_enable();
 181 #endif
 182         preempt_check_resched();
 183 }
 184
 185 void local_bh_enable(void)
 186 {
 187         _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
 188 }
 189 EXPORT_SYMBOL(local_bh_enable);
 190
 191 void local_bh_enable_ip(unsigned long ip)
 192 {
 193         _local_bh_enable_ip(ip);
 194 }
 195 EXPORT_SYMBOL(local_bh_enable_ip);
 196
 197 /*
 198  * We restart softirq processing for at most 2 ms,
 199  * and if need_resched() is not set.
 200  *
 201  * These limits have been established via experimentation.
 202  * The two things to balance is latency against fairness -
 203  * we want to handle softirqs as soon as possible, but they
 204  * should not be able to lock up the box.
 205  */
 206 #define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 207
 208 asmlinkage void __do_softirq(void)
 209 {
 210         struct softirq_action *h;
 211         __u32 pending;
 212         unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 213         int cpu;
 214         unsigned long old_flags = current->flags;
 215
 216         /*
 217          * Mask out PF_MEMALLOC s current task context is borrowed for the
 218          * softirq. A softirq handled such as network RX might set PF_MEMALLOC
 219          * again if the socket is related to swap
 220          */
 221         current->flags &= ~PF_MEMALLOC;
 222
 223         pending = local_softirq_pending();
 224         account_irq_enter_time(current);
 225
 226         __local_bh_disable((unsigned long)__builtin_return_address(0),
 227                                 SOFTIRQ_OFFSET);
 228         lockdep_softirq_enter();
 229
 230         cpu = smp_processor_id();
 231 restart:
 232         /* Reset the pending bitmask before enabling irqs */
 233         set_softirq_pending(0);
 234
 235         local_irq_enable();
 236
 237         h = softirq_vec;
 238
 239         do {
 240                 if (pending & 1) {
 241                         unsigned int vec_nr = h - softirq_vec;
 242                         int prev_count = preempt_count();
 243
 244                         kstat_incr_softirqs_this_cpu(vec_nr);
 245
 246                         trace_softirq_entry(vec_nr);
 247                         h->action(h);
 248                         trace_softirq_exit(vec_nr);
 249                         if (unlikely(prev_count != preempt_count())) {
 250                                 printk(KERN_ERR "huh, entered softirq %u %s %p"
 251                                        "with preempt_count %08x,"
 252                                        " exited with %08x?\n", vec_nr,
 253                                        softirq_to_name[vec_nr], h->action,
 254                                        prev_count, preempt_count());
 255                                 preempt_count() = prev_count;
 256                         }
 257
 258                         rcu_bh_qs(cpu);
 259                 }
 260                 h++;
 261                 pending >>= 1;
 262         } while (pending);
 263
 264         local_irq_disable();
 265
 266         pending = local_softirq_pending();
 267         if (pending) {
 268                 if (time_before(jiffies, end) && !need_resched())
 269                         goto restart;
 270
 271                 wakeup_softirqd();
 272         }
 273
 274         lockdep_softirq_exit();
 275
 276         account_irq_exit_time(current);
 277         __local_bh_enable(SOFTIRQ_OFFSET);
 278         tsk_restore_flags(current, old_flags, PF_MEMALLOC);
 279 }
 280
 281 #ifndef __ARCH_HAS_DO_SOFTIRQ
 282
 283 asmlinkage void do_softirq(void)
 284 {
 285         __u32 pending;
 286         unsigned long flags;
 287
 288         if (in_interrupt())
 289                 return;
 290
 291         local_irq_save(flags);
 292
 293         pending = local_softirq_pending();
 294
 295         if (pending)
 296                 __do_softirq();
 297
 298         local_irq_restore(flags);
 299 }
 300
 301 #endif
 302
 303 /*
 304  * Enter an interrupt context.
 305  */
 306 void irq_enter(void)
 307 {
 308         int cpu = smp_processor_id();
 309
 310         rcu_irq_enter();
 311         if (is_idle_task(current) && !in_interrupt()) {
 312                 /*
 313                  * Prevent raise_softirq from needlessly waking up ksoftirqd
 314                  * here, as softirq will be serviced on return from interrupt.
 315                  */
 316                 local_bh_disable();
 317                 tick_check_idle(cpu);
 318                 _local_bh_enable();
 319         }
 320
 321         __irq_enter();
 322 }
 323
 324 static inline void invoke_softirq(void)
 325 {
 326         if (!force_irqthreads)
 327                 __do_softirq();
 328         else
 329                 wakeup_softirqd();
 330 }
 331
 332 /*
 333  * Exit an interrupt context. Process softirqs if needed and possible:
 334  */
 335 void irq_exit(void)
 336 {
 337 #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 338         local_irq_disable();
 339 #else
 340         WARN_ON_ONCE(!irqs_disabled());
 341 #endif
 342
 343         account_irq_exit_time(current);
 344         trace_hardirq_exit();
 345         sub_preempt_count(HARDIRQ_OFFSET);
 346         if (!in_interrupt() && local_softirq_pending())
 347                 invoke_softirq();
 348
 349 #ifdef CONFIG_NO_HZ
 350         /* Make sure that timer wheel updates are propagated */
 351         if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
 352                 tick_nohz_irq_exit();
 353 #endif
 354         rcu_irq_exit();
 355 }
 356
 357 /*
 358  * This function must run with irqs disabled!
 359  */
 360 inline void raise_softirq_irqoff(unsigned int nr)
 361 {
 362         __raise_softirq_irqoff(nr);
 363
 364         /*
 365          * If we're in an interrupt or softirq, we're done
 366          * (this also catches softirq-disabled code). We will
 367          * actually run the softirq once we return from
 368          * the irq or softirq.
 369          *
 370          * Otherwise we wake up ksoftirqd to make sure we
 371          * schedule the softirq soon.
 372          */
 373         if (!in_interrupt())
 374                 wakeup_softirqd();
 375 }
 376
 377 void raise_softirq(unsigned int nr)
 378 {
 379         unsigned long flags;
 380
 381         local_irq_save(flags);
 382         raise_softirq_irqoff(nr);
 383         local_irq_restore(flags);
 384 }
 385
 386 void __raise_softirq_irqoff(unsigned int nr)
 387 {
 388         trace_softirq_raise(nr);
 389         or_softirq_pending(1UL << nr);
 390 }
 391
 392 void open_softirq(int nr, void (*action)(struct softirq_action *))
 393 {
 394         softirq_vec[nr].action = action;
 395 }
 396
 397 /*
 398  * Tasklets
 399  */
 400 struct tasklet_head
 401 {
 402         struct tasklet_struct *head;
 403         struct tasklet_struct **tail;
 404 };
 405
 406 static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 407 static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 408
 409 void __tasklet_schedule(struct tasklet_struct *t)
 410 {
 411         unsigned long flags;
 412
 413         local_irq_save(flags);
 414         t->next = NULL;
 415         *__this_cpu_read(tasklet_vec.tail) = t;
 416         __this_cpu_write(tasklet_vec.tail, &(t->next));
 417         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 418         local_irq_restore(flags);
 419 }
 420
 421 EXPORT_SYMBOL(__tasklet_schedule);
 422
 423 void __tasklet_hi_schedule(struct tasklet_struct *t)
 424 {
 425         unsigned long flags;
 426
 427         local_irq_save(flags);
 428         t->next = NULL;
 429         *__this_cpu_read(tasklet_hi_vec.tail) = t;
 430         __this_cpu_write(tasklet_hi_vec.tail,  &(t->next));
 431         raise_softirq_irqoff(HI_SOFTIRQ);
 432         local_irq_restore(flags);
 433 }
 434
 435 EXPORT_SYMBOL(__tasklet_hi_schedule);
 436
 437 void __tasklet_hi_schedule_first(struct tasklet_struct *t)
 438 {
 439         BUG_ON(!irqs_disabled());
 440
 441         t->next = __this_cpu_read(tasklet_hi_vec.head);
 442         __this_cpu_write(tasklet_hi_vec.head, t);
 443         __raise_softirq_irqoff(HI_SOFTIRQ);
 444 }
 445
 446 EXPORT_SYMBOL(__tasklet_hi_schedule_first);
 447
 448 static void tasklet_action(struct softirq_action *a)
 449 {
 450         struct tasklet_struct *list;
 451
 452         local_irq_disable();
 453         list = __this_cpu_read(tasklet_vec.head);
 454         __this_cpu_write(tasklet_vec.head, NULL);
 455         __this_cpu_write(tasklet_vec.tail, &__get_cpu_var(tasklet_vec).head);
 456         local_irq_enable();
 457
 458         while (list) {
 459                 struct tasklet_struct *t = list;
 460
 461                 list = list->next;
 462
 463                 if (tasklet_trylock(t)) {
 464                         if (!atomic_read(&t->count)) {
 465                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 466                                         BUG();
 467                                 t->func(t->data);
 468                                 tasklet_unlock(t);
 469                                 continue;
 470                         }
 471                         tasklet_unlock(t);
 472                 }
 473
 474                 local_irq_disable();
 475                 t->next = NULL;
 476                 *__this_cpu_read(tasklet_vec.tail) = t;
 477                 __this_cpu_write(tasklet_vec.tail, &(t->next));
 478                 __raise_softirq_irqoff(TASKLET_SOFTIRQ);
 479                 local_irq_enable();
 480         }
 481 }
 482
 483 static void tasklet_hi_action(struct softirq_action *a)
 484 {
 485         struct tasklet_struct *list;
 486
 487         local_irq_disable();
 488         list = __this_cpu_read(tasklet_hi_vec.head);
 489         __this_cpu_write(tasklet_hi_vec.head, NULL);
 490         __this_cpu_write(tasklet_hi_vec.tail, &__get_cpu_var(tasklet_hi_vec).head);
 491         local_irq_enable();
 492
 493         while (list) {
 494                 struct tasklet_struct *t = list;
 495
 496                 list = list->next;
 497
 498                 if (tasklet_trylock(t)) {
 499                         if (!atomic_read(&t->count)) {
 500                                 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state))
 501                                         BUG();
 502                                 t->func(t->data);
 503                                 tasklet_unlock(t);
 504                                 continue;
 505                         }
 506                         tasklet_unlock(t);
 507                 }
 508
 509                 local_irq_disable();
 510                 t->next = NULL;
 511                 *__this_cpu_read(tasklet_hi_vec.tail) = t;
 512                 __this_cpu_write(tasklet_hi_vec.tail, &(t->next));
 513                 __raise_softirq_irqoff(HI_SOFTIRQ);
 514                 local_irq_enable();
 515         }
 516 }
 517
 518
 519 void tasklet_init(struct tasklet_struct *t,
 520                   void (*func)(unsigned long), unsigned long data)
 521 {
 522         t->next = NULL;
 523         t->state = 0;
 524         atomic_set(&t->count, 0);
 525         t->func = func;
 526         t->data = data;
 527 }
 528
 529 EXPORT_SYMBOL(tasklet_init);
 530
 531 void tasklet_kill(struct tasklet_struct *t)
 532 {
 533         if (in_interrupt())
 534                 printk("Attempt to kill tasklet from interrupt\n");
 535
 536         while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) {
 537                 do {
 538                         yield();
 539                 } while (test_bit(TASKLET_STATE_SCHED, &t->state));
 540         }
 541         tasklet_unlock_wait(t);
 542         clear_bit(TASKLET_STATE_SCHED, &t->state);
 543 }
 544
 545 EXPORT_SYMBOL(tasklet_kill);
 546
 547 /*
 548  * tasklet_hrtimer
 549  */
 550
 551 /*
 552  * The trampoline is called when the hrtimer expires. It schedules a tasklet
 553  * to run __tasklet_hrtimer_trampoline() which in turn will call the intended
 554  * hrtimer callback, but from softirq context.
 555  */
 556 static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer)
 557 {
 558         struct tasklet_hrtimer *ttimer =
 559                 container_of(timer, struct tasklet_hrtimer, timer);
 560
 561         tasklet_hi_schedule(&ttimer->tasklet);
 562         return HRTIMER_NORESTART;
 563 }
 564
 565 /*
 566  * Helper function which calls the hrtimer callback from
 567  * tasklet/softirq context
 568  */
 569 static void __tasklet_hrtimer_trampoline(unsigned long data)
 570 {
 571         struct tasklet_hrtimer *ttimer = (void *)data;
 572         enum hrtimer_restart restart;
 573
 574         restart = ttimer->function(&ttimer->timer);
 575         if (restart != HRTIMER_NORESTART)
 576                 hrtimer_restart(&ttimer->timer);
 577 }
 578
 579 /**
 580  * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks
 581  * @ttimer:      tasklet_hrtimer which is initialized
 582  * @function:    hrtimer callback function which gets called from softirq context
 583  * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME)
 584  * @mode:        hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL)
 585  */
 586 void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer,
 587                           enum hrtimer_restart (*function)(struct hrtimer *),
 588                           clockid_t which_clock, enum hrtimer_mode mode)
 589 {
 590         hrtimer_init(&ttimer->timer, which_clock, mode);
 591         ttimer->timer.function = __hrtimer_tasklet_trampoline;
 592         tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline,
 593                      (unsigned long)ttimer);
 594         ttimer->function = function;
 595 }
 596 EXPORT_SYMBOL_GPL(tasklet_hrtimer_init);
 597
 598 /*
 599  * Remote softirq bits
 600  */
 601
 602 DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list);
 603 EXPORT_PER_CPU_SYMBOL(softirq_work_list);
 604
 605 static void __local_trigger(struct call_single_data *cp, int softirq)
 606 {
 607         struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]);
 608
 609         list_add_tail(&cp->list, head);
 610
 611         /* Trigger the softirq only if the list was previously empty.  */
 612         if (head->next == &cp->list)
 613                 raise_softirq_irqoff(softirq);
 614 }
 615
 616 #ifdef CONFIG_USE_GENERIC_SMP_HELPERS
 617 static void remote_softirq_receive(void *data)
 618 {
 619         struct call_single_data *cp = data;
 620         unsigned long flags;
 621         int softirq;
 622
 623         softirq = cp->priv;
 624
 625         local_irq_save(flags);
 626         __local_trigger(cp, softirq);
 627         local_irq_restore(flags);
 628 }
 629
 630 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 631 {
 632         if (cpu_online(cpu)) {
 633                 cp->func = remote_softirq_receive;
 634                 cp->info = cp;
 635                 cp->flags = 0;
 636                 cp->priv = softirq;
 637
 638                 __smp_call_function_single(cpu, cp, 0);
 639                 return 0;
 640         }
 641         return 1;
 642 }
 643 #else /* CONFIG_USE_GENERIC_SMP_HELPERS */
 644 static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 645 {
 646         return 1;
 647 }
 648 #endif
 649
 650 /**
 651  * __send_remote_softirq - try to schedule softirq work on a remote cpu
 652  * @cp: private SMP call function data area
 653  * @cpu: the remote cpu
 654  * @this_cpu: the currently executing cpu
 655  * @softirq: the softirq for the work
 656  *
 657  * Attempt to schedule softirq work on a remote cpu.  If this cannot be
 658  * done, the work is instead queued up on the local cpu.
 659  *
 660  * Interrupts must be disabled.
 661  */
 662 void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq)
 663 {
 664         if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq))
 665                 __local_trigger(cp, softirq);
 666 }
 667 EXPORT_SYMBOL(__send_remote_softirq);
 668
 669 /**
 670  * send_remote_softirq - try to schedule softirq work on a remote cpu
 671  * @cp: private SMP call function data area
 672  * @cpu: the remote cpu
 673  * @softirq: the softirq for the work
 674  *
 675  * Like __send_remote_softirq except that disabling interrupts and
 676  * computing the current cpu is done for the caller.
 677  */
 678 void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
 679 {
 680         unsigned long flags;
 681         int this_cpu;
 682
 683         local_irq_save(flags);
 684         this_cpu = smp_processor_id();
 685         __send_remote_softirq(cp, cpu, this_cpu, softirq);
 686         local_irq_restore(flags);
 687 }
 688 EXPORT_SYMBOL(send_remote_softirq);
 689
 690 static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self,
 691                                                unsigned long action, void *hcpu)
 692 {
 693         /*
 694          * If a CPU goes away, splice its entries to the current CPU
 695          * and trigger a run of the softirq
 696          */
 697         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
 698                 int cpu = (unsigned long) hcpu;
 699                 int i;
 700
 701                 local_irq_disable();
 702                 for (i = 0; i < NR_SOFTIRQS; i++) {
 703                         struct list_head *head = &per_cpu(softirq_work_list[i], cpu);
 704                         struct list_head *local_head;
 705
 706                         if (list_empty(head))
 707                                 continue;
 708
 709                         local_head = &__get_cpu_var(softirq_work_list[i]);
 710                         list_splice_init(head, local_head);
 711                         raise_softirq_irqoff(i);
 712                 }
 713                 local_irq_enable();
 714         }
 715
 716         return NOTIFY_OK;
 717 }
 718
 719 static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = {
 720         .notifier_call  = remote_softirq_cpu_notify,
 721 };
 722
 723 void __init softirq_init(void)
 724 {
 725         int cpu;
 726
 727         for_each_possible_cpu(cpu) {
 728                 int i;
 729
 730                 per_cpu(tasklet_vec, cpu).tail =
 731                         &per_cpu(tasklet_vec, cpu).head;
 732                 per_cpu(tasklet_hi_vec, cpu).tail =
 733                         &per_cpu(tasklet_hi_vec, cpu).head;
 734                 for (i = 0; i < NR_SOFTIRQS; i++)
 735                         INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu));
 736         }
 737
 738         register_hotcpu_notifier(&remote_softirq_cpu_notifier);
 739
 740         open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 741         open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 742 }
 743
 744 static int ksoftirqd_should_run(unsigned int cpu)
 745 {
 746         return local_softirq_pending();
 747 }
 748
 749 static void run_ksoftirqd(unsigned int cpu)
 750 {
 751         local_irq_disable();
 752         if (local_softirq_pending()) {
 753                 __do_softirq();
 754                 rcu_note_context_switch(cpu);
 755                 local_irq_enable();
 756                 cond_resched();
 757                 return;
 758         }
 759         local_irq_enable();
 760 }
 761
 762 #ifdef CONFIG_HOTPLUG_CPU
 763 /*
 764  * tasklet_kill_immediate is called to remove a tasklet which can already be
 765  * scheduled for execution on @cpu.
 766  *
 767  * Unlike tasklet_kill, this function removes the tasklet
 768  * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state.
 769  *
 770  * When this function is called, @cpu must be in the CPU_DEAD state.
 771  */
 772 void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu)
 773 {
 774         struct tasklet_struct **i;
 775
 776         BUG_ON(cpu_online(cpu));
 777         BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state));
 778
 779         if (!test_bit(TASKLET_STATE_SCHED, &t->state))
 780                 return;
 781
 782         /* CPU is dead, so no lock needed. */
 783         for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) {
 784                 if (*i == t) {
 785                         *i = t->next;
 786                         /* If this was the tail element, move the tail ptr */
 787                         if (*i == NULL)
 788                                 per_cpu(tasklet_vec, cpu).tail = i;
 789                         return;
 790                 }
 791         }
 792         BUG();
 793 }
 794
 795 static void takeover_tasklets(unsigned int cpu)
 796 {
 797         /* CPU is dead, so no lock needed. */
 798         local_irq_disable();
 799
 800         /* Find end, append list for that CPU. */
 801         if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 802                 *__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 803                 this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 804                 per_cpu(tasklet_vec, cpu).head = NULL;
 805                 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 806         }
 807         raise_softirq_irqoff(TASKLET_SOFTIRQ);
 808
 809         if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 810                 *__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 811                 __this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 812                 per_cpu(tasklet_hi_vec, cpu).head = NULL;
 813                 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 814         }
 815         raise_softirq_irqoff(HI_SOFTIRQ);
 816
 817         local_irq_enable();
 818 }
 819 #endif /* CONFIG_HOTPLUG_CPU */
 820
 821 static int __cpuinit cpu_callback(struct notifier_block *nfb,
 822                                   unsigned long action,
 823                                   void *hcpu)
 824 {
 825         switch (action) {
 826 #ifdef CONFIG_HOTPLUG_CPU
 827         case CPU_DEAD:
 828         case CPU_DEAD_FROZEN:
 829                 takeover_tasklets((unsigned long)hcpu);
 830                 break;
 831 #endif /* CONFIG_HOTPLUG_CPU */
 832         }
 833         return NOTIFY_OK;
 834 }
 835
 836 static struct notifier_block __cpuinitdata cpu_nfb = {
 837         .notifier_call = cpu_callback
 838 };
 839
 840 static struct smp_hotplug_thread softirq_threads = {
 841         .store                  = &ksoftirqd,
 842         .thread_should_run      = ksoftirqd_should_run,
 843         .thread_fn              = run_ksoftirqd,
 844         .thread_comm            = "ksoftirqd/%u",
 845 };
 846
 847 static __init int spawn_ksoftirqd(void)
 848 {
 849         register_cpu_notifier(&cpu_nfb);
 850
 851         BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
 852
 853         return 0;
 854 }
 855 early_initcall(spawn_ksoftirqd);
 856
 857 /*
 858  * [ These __weak aliases are kept in a separate compilation unit, so that
 859  *   GCC does not inline them incorrectly. ]
 860  */
 861
 862 int __init __weak early_irq_init(void)
 863 {
 864         return 0;
 865 }
 866
 867 #ifdef CONFIG_GENERIC_HARDIRQS
 868 int __init __weak arch_probe_nr_irqs(void)
 869 {
 870         return NR_IRQS_LEGACY;
 871 }
 872
 873 int __init __weak arch_early_irq_init(void)
 874 {
 875         return 0;
 876 }
 877 #endif