drivers/acpi/processor_idle.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * processor_idle - idle state submodule to the ACPI processor driver
   4  *
   5  *  Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com>
   6  *  Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com>
   7  *  Copyright (C) 2004, 2005 Dominik Brodowski <linux@brodo.de>
   8  *  Copyright (C) 2004  Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
   9  *                      - Added processor hotplug support
  10  *  Copyright (C) 2005  Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
  11  *                      - Added support for C3 on SMP
  12  */
  13 #define pr_fmt(fmt) "ACPI: " fmt
  14
  15 #include <linux/module.h>
  16 #include <linux/acpi.h>
  17 #include <linux/dmi.h>
  18 #include <linux/sched.h>       /* need_resched() */
  19 #include <linux/tick.h>
  20 #include <linux/cpuidle.h>
  21 #include <linux/cpu.h>
  22 #include <acpi/processor.h>
  23
  24 /*
  25  * Include the apic definitions for x86 to have the APIC timer related defines
  26  * available also for UP (on SMP it gets magically included via linux/smp.h).
  27  * asm/acpi.h is not an option, as it would require more include magic. Also
  28  * creating an empty asm-ia64/apic.h would just trade pest vs. cholera.
  29  */
  30 #ifdef CONFIG_X86
  31 #include <asm/apic.h>
  32 #endif
  33
  34 #define ACPI_PROCESSOR_CLASS            "processor"
  35 #define _COMPONENT              ACPI_PROCESSOR_COMPONENT
  36 ACPI_MODULE_NAME("processor_idle");
  37
  38 #define ACPI_IDLE_STATE_START   (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX) ? 1 : 0)
  39
  40 static unsigned int max_cstate __read_mostly = ACPI_PROCESSOR_MAX_POWER;
  41 module_param(max_cstate, uint, 0000);
  42 static unsigned int nocst __read_mostly;
  43 module_param(nocst, uint, 0000);
  44 static int bm_check_disable __read_mostly;
  45 module_param(bm_check_disable, uint, 0000);
  46
  47 static unsigned int latency_factor __read_mostly = 2;
  48 module_param(latency_factor, uint, 0644);
  49
  50 static DEFINE_PER_CPU(struct cpuidle_device *, acpi_cpuidle_device);
  51
  52 struct cpuidle_driver acpi_idle_driver = {
  53         .name =         "acpi_idle",
  54         .owner =        THIS_MODULE,
  55 };
  56
  57 #ifdef CONFIG_ACPI_PROCESSOR_CSTATE
  58 static
  59 DEFINE_PER_CPU(struct acpi_processor_cx * [CPUIDLE_STATE_MAX], acpi_cstate);
  60
  61 static int disabled_by_idle_boot_param(void)
  62 {
  63         return boot_option_idle_override == IDLE_POLL ||
  64                 boot_option_idle_override == IDLE_HALT;
  65 }
  66
  67 /*
  68  * IBM ThinkPad R40e crashes mysteriously when going into C2 or C3.
  69  * For now disable this. Probably a bug somewhere else.
  70  *
  71  * To skip this limit, boot/load with a large max_cstate limit.
  72  */
  73 static int set_max_cstate(const struct dmi_system_id *id)
  74 {
  75         if (max_cstate > ACPI_PROCESSOR_MAX_POWER)
  76                 return 0;
  77
  78         pr_notice("%s detected - limiting to C%ld max_cstate."
  79                   " Override with \"processor.max_cstate=%d\"\n", id->ident,
  80                   (long)id->driver_data, ACPI_PROCESSOR_MAX_POWER + 1);
  81
  82         max_cstate = (long)id->driver_data;
  83
  84         return 0;
  85 }
  86
  87 static const struct dmi_system_id processor_power_dmi_table[] = {
  88         { set_max_cstate, "Clevo 5600D", {
  89           DMI_MATCH(DMI_BIOS_VENDOR,"Phoenix Technologies LTD"),
  90           DMI_MATCH(DMI_BIOS_VERSION,"SHE845M0.86C.0013.D.0302131307")},
  91          (void *)2},
  92         { set_max_cstate, "Pavilion zv5000", {
  93           DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
  94           DMI_MATCH(DMI_PRODUCT_NAME,"Pavilion zv5000 (DS502A#ABA)")},
  95          (void *)1},
  96         { set_max_cstate, "Asus L8400B", {
  97           DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK Computer Inc."),
  98           DMI_MATCH(DMI_PRODUCT_NAME,"L8400B series Notebook PC")},
  99          (void *)1},
 100         {},
 101 };
 102
 103
 104 /*
 105  * Callers should disable interrupts before the call and enable
 106  * interrupts after return.
 107  */
 108 static void __cpuidle acpi_safe_halt(void)
 109 {
 110         if (!tif_need_resched()) {
 111                 safe_halt();
 112                 local_irq_disable();
 113         }
 114 }
 115
 116 #ifdef ARCH_APICTIMER_STOPS_ON_C3
 117
 118 /*
 119  * Some BIOS implementations switch to C3 in the published C2 state.
 120  * This seems to be a common problem on AMD boxen, but other vendors
 121  * are affected too. We pick the most conservative approach: we assume
 122  * that the local APIC stops in both C2 and C3.
 123  */
 124 static void lapic_timer_check_state(int state, struct acpi_processor *pr,
 125                                    struct acpi_processor_cx *cx)
 126 {
 127         struct acpi_processor_power *pwr = &pr->power;
 128         u8 type = local_apic_timer_c2_ok ? ACPI_STATE_C3 : ACPI_STATE_C2;
 129
 130         if (cpu_has(&cpu_data(pr->id), X86_FEATURE_ARAT))
 131                 return;
 132
 133         if (boot_cpu_has_bug(X86_BUG_AMD_APIC_C1E))
 134                 type = ACPI_STATE_C1;
 135
 136         /*
 137          * Check, if one of the previous states already marked the lapic
 138          * unstable
 139          */
 140         if (pwr->timer_broadcast_on_state < state)
 141                 return;
 142
 143         if (cx->type >= type)
 144                 pr->power.timer_broadcast_on_state = state;
 145 }
 146
 147 static void __lapic_timer_propagate_broadcast(void *arg)
 148 {
 149         struct acpi_processor *pr = (struct acpi_processor *) arg;
 150
 151         if (pr->power.timer_broadcast_on_state < INT_MAX)
 152                 tick_broadcast_enable();
 153         else
 154                 tick_broadcast_disable();
 155 }
 156
 157 static void lapic_timer_propagate_broadcast(struct acpi_processor *pr)
 158 {
 159         smp_call_function_single(pr->id, __lapic_timer_propagate_broadcast,
 160                                  (void *)pr, 1);
 161 }
 162
 163 /* Power(C) State timer broadcast control */
 164 static void lapic_timer_state_broadcast(struct acpi_processor *pr,
 165                                        struct acpi_processor_cx *cx,
 166                                        int broadcast)
 167 {
 168         int state = cx - pr->power.states;
 169
 170         if (state >= pr->power.timer_broadcast_on_state) {
 171                 if (broadcast)
 172                         tick_broadcast_enter();
 173                 else
 174                         tick_broadcast_exit();
 175         }
 176 }
 177
 178 #else
 179
 180 static void lapic_timer_check_state(int state, struct acpi_processor *pr,
 181                                    struct acpi_processor_cx *cstate) { }
 182 static void lapic_timer_propagate_broadcast(struct acpi_processor *pr) { }
 183 static void lapic_timer_state_broadcast(struct acpi_processor *pr,
 184                                        struct acpi_processor_cx *cx,
 185                                        int broadcast)
 186 {
 187 }
 188
 189 #endif
 190
 191 #if defined(CONFIG_X86)
 192 static void tsc_check_state(int state)
 193 {
 194         switch (boot_cpu_data.x86_vendor) {
 195         case X86_VENDOR_HYGON:
 196         case X86_VENDOR_AMD:
 197         case X86_VENDOR_INTEL:
 198         case X86_VENDOR_CENTAUR:
 199         case X86_VENDOR_ZHAOXIN:
 200                 /*
 201                  * AMD Fam10h TSC will tick in all
 202                  * C/P/S0/S1 states when this bit is set.
 203                  */
 204                 if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
 205                         return;
 206                 fallthrough;
 207         default:
 208                 /* TSC could halt in idle, so notify users */
 209                 if (state > ACPI_STATE_C1)
 210                         mark_tsc_unstable("TSC halts in idle");
 211         }
 212 }
 213 #else
 214 static void tsc_check_state(int state) { return; }
 215 #endif
 216
 217 static int acpi_processor_get_power_info_fadt(struct acpi_processor *pr)
 218 {
 219
 220         if (!pr->pblk)
 221                 return -ENODEV;
 222
 223         /* if info is obtained from pblk/fadt, type equals state */
 224         pr->power.states[ACPI_STATE_C2].type = ACPI_STATE_C2;
 225         pr->power.states[ACPI_STATE_C3].type = ACPI_STATE_C3;
 226
 227 #ifndef CONFIG_HOTPLUG_CPU
 228         /*
 229          * Check for P_LVL2_UP flag before entering C2 and above on
 230          * an SMP system.
 231          */
 232         if ((num_online_cpus() > 1) &&
 233             !(acpi_gbl_FADT.flags & ACPI_FADT_C2_MP_SUPPORTED))
 234                 return -ENODEV;
 235 #endif
 236
 237         /* determine C2 and C3 address from pblk */
 238         pr->power.states[ACPI_STATE_C2].address = pr->pblk + 4;
 239         pr->power.states[ACPI_STATE_C3].address = pr->pblk + 5;
 240
 241         /* determine latencies from FADT */
 242         pr->power.states[ACPI_STATE_C2].latency = acpi_gbl_FADT.c2_latency;
 243         pr->power.states[ACPI_STATE_C3].latency = acpi_gbl_FADT.c3_latency;
 244
 245         /*
 246          * FADT specified C2 latency must be less than or equal to
 247          * 100 microseconds.
 248          */
 249         if (acpi_gbl_FADT.c2_latency > ACPI_PROCESSOR_MAX_C2_LATENCY) {
 250                 ACPI_DEBUG_PRINT((ACPI_DB_INFO,
 251                         "C2 latency too large [%d]\n", acpi_gbl_FADT.c2_latency));
 252                 /* invalidate C2 */
 253                 pr->power.states[ACPI_STATE_C2].address = 0;
 254         }
 255
 256         /*
 257          * FADT supplied C3 latency must be less than or equal to
 258          * 1000 microseconds.
 259          */
 260         if (acpi_gbl_FADT.c3_latency > ACPI_PROCESSOR_MAX_C3_LATENCY) {
 261                 ACPI_DEBUG_PRINT((ACPI_DB_INFO,
 262                         "C3 latency too large [%d]\n", acpi_gbl_FADT.c3_latency));
 263                 /* invalidate C3 */
 264                 pr->power.states[ACPI_STATE_C3].address = 0;
 265         }
 266
 267         ACPI_DEBUG_PRINT((ACPI_DB_INFO,
 268                           "lvl2[0x%08x] lvl3[0x%08x]\n",
 269                           pr->power.states[ACPI_STATE_C2].address,
 270                           pr->power.states[ACPI_STATE_C3].address));
 271
 272         snprintf(pr->power.states[ACPI_STATE_C2].desc,
 273                          ACPI_CX_DESC_LEN, "ACPI P_LVL2 IOPORT 0x%x",
 274                          pr->power.states[ACPI_STATE_C2].address);
 275         snprintf(pr->power.states[ACPI_STATE_C3].desc,
 276                          ACPI_CX_DESC_LEN, "ACPI P_LVL3 IOPORT 0x%x",
 277                          pr->power.states[ACPI_STATE_C3].address);
 278
 279         return 0;
 280 }
 281
 282 static int acpi_processor_get_power_info_default(struct acpi_processor *pr)
 283 {
 284         if (!pr->power.states[ACPI_STATE_C1].valid) {
 285                 /* set the first C-State to C1 */
 286                 /* all processors need to support C1 */
 287                 pr->power.states[ACPI_STATE_C1].type = ACPI_STATE_C1;
 288                 pr->power.states[ACPI_STATE_C1].valid = 1;
 289                 pr->power.states[ACPI_STATE_C1].entry_method = ACPI_CSTATE_HALT;
 290
 291                 snprintf(pr->power.states[ACPI_STATE_C1].desc,
 292                          ACPI_CX_DESC_LEN, "ACPI HLT");
 293         }
 294         /* the C0 state only exists as a filler in our array */
 295         pr->power.states[ACPI_STATE_C0].valid = 1;
 296         return 0;
 297 }
 298
 299 static int acpi_processor_get_power_info_cst(struct acpi_processor *pr)
 300 {
 301         int ret;
 302
 303         if (nocst)
 304                 return -ENODEV;
 305
 306         ret = acpi_processor_evaluate_cst(pr->handle, pr->id, &pr->power);
 307         if (ret)
 308                 return ret;
 309
 310         if (!pr->power.count)
 311                 return -EFAULT;
 312
 313         pr->flags.has_cst = 1;
 314         return 0;
 315 }
 316
 317 static void acpi_processor_power_verify_c3(struct acpi_processor *pr,
 318                                            struct acpi_processor_cx *cx)
 319 {
 320         static int bm_check_flag = -1;
 321         static int bm_control_flag = -1;
 322
 323
 324         if (!cx->address)
 325                 return;
 326
 327         /*
 328          * PIIX4 Erratum #18: We don't support C3 when Type-F (fast)
 329          * DMA transfers are used by any ISA device to avoid livelock.
 330          * Note that we could disable Type-F DMA (as recommended by
 331          * the erratum), but this is known to disrupt certain ISA
 332          * devices thus we take the conservative approach.
 333          */
 334         else if (errata.piix4.fdma) {
 335                 ACPI_DEBUG_PRINT((ACPI_DB_INFO,
 336                                   "C3 not supported on PIIX4 with Type-F DMA\n"));
 337                 return;
 338         }
 339
 340         /* All the logic here assumes flags.bm_check is same across all CPUs */
 341         if (bm_check_flag == -1) {
 342                 /* Determine whether bm_check is needed based on CPU  */
 343                 acpi_processor_power_init_bm_check(&(pr->flags), pr->id);
 344                 bm_check_flag = pr->flags.bm_check;
 345                 bm_control_flag = pr->flags.bm_control;
 346         } else {
 347                 pr->flags.bm_check = bm_check_flag;
 348                 pr->flags.bm_control = bm_control_flag;
 349         }
 350
 351         if (pr->flags.bm_check) {
 352                 if (!pr->flags.bm_control) {
 353                         if (pr->flags.has_cst != 1) {
 354                                 /* bus mastering control is necessary */
 355                                 ACPI_DEBUG_PRINT((ACPI_DB_INFO,
 356                                         "C3 support requires BM control\n"));
 357                                 return;
 358                         } else {
 359                                 /* Here we enter C3 without bus mastering */
 360                                 ACPI_DEBUG_PRINT((ACPI_DB_INFO,
 361                                         "C3 support without BM control\n"));
 362                         }
 363                 }
 364         } else {
 365                 /*
 366                  * WBINVD should be set in fadt, for C3 state to be
 367                  * supported on when bm_check is not required.
 368                  */
 369                 if (!(acpi_gbl_FADT.flags & ACPI_FADT_WBINVD)) {
 370                         ACPI_DEBUG_PRINT((ACPI_DB_INFO,
 371                                           "Cache invalidation should work properly"
 372                                           " for C3 to be enabled on SMP systems\n"));
 373                         return;
 374                 }
 375         }
 376
 377         /*
 378          * Otherwise we've met all of our C3 requirements.
 379          * Normalize the C3 latency to expidite policy.  Enable
 380          * checking of bus mastering status (bm_check) so we can
 381          * use this in our C3 policy
 382          */
 383         cx->valid = 1;
 384
 385         /*
 386          * On older chipsets, BM_RLD needs to be set
 387          * in order for Bus Master activity to wake the
 388          * system from C3.  Newer chipsets handle DMA
 389          * during C3 automatically and BM_RLD is a NOP.
 390          * In either case, the proper way to
 391          * handle BM_RLD is to set it and leave it set.
 392          */
 393         acpi_write_bit_register(ACPI_BITREG_BUS_MASTER_RLD, 1);
 394
 395         return;
 396 }
 397
 398 static int acpi_processor_power_verify(struct acpi_processor *pr)
 399 {
 400         unsigned int i;
 401         unsigned int working = 0;
 402
 403         pr->power.timer_broadcast_on_state = INT_MAX;
 404
 405         for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
 406                 struct acpi_processor_cx *cx = &pr->power.states[i];
 407
 408                 switch (cx->type) {
 409                 case ACPI_STATE_C1:
 410                         cx->valid = 1;
 411                         break;
 412
 413                 case ACPI_STATE_C2:
 414                         if (!cx->address)
 415                                 break;
 416                         cx->valid = 1;
 417                         break;
 418
 419                 case ACPI_STATE_C3:
 420                         acpi_processor_power_verify_c3(pr, cx);
 421                         break;
 422                 }
 423                 if (!cx->valid)
 424                         continue;
 425
 426                 lapic_timer_check_state(i, pr, cx);
 427                 tsc_check_state(cx->type);
 428                 working++;
 429         }
 430
 431         lapic_timer_propagate_broadcast(pr);
 432
 433         return (working);
 434 }
 435
 436 static int acpi_processor_get_cstate_info(struct acpi_processor *pr)
 437 {
 438         unsigned int i;
 439         int result;
 440
 441
 442         /* NOTE: the idle thread may not be running while calling
 443          * this function */
 444
 445         /* Zero initialize all the C-states info. */
 446         memset(pr->power.states, 0, sizeof(pr->power.states));
 447
 448         result = acpi_processor_get_power_info_cst(pr);
 449         if (result == -ENODEV)
 450                 result = acpi_processor_get_power_info_fadt(pr);
 451
 452         if (result)
 453                 return result;
 454
 455         acpi_processor_get_power_info_default(pr);
 456
 457         pr->power.count = acpi_processor_power_verify(pr);
 458
 459         /*
 460          * if one state of type C2 or C3 is available, mark this
 461          * CPU as being "idle manageable"
 462          */
 463         for (i = 1; i < ACPI_PROCESSOR_MAX_POWER; i++) {
 464                 if (pr->power.states[i].valid) {
 465                         pr->power.count = i;
 466                         pr->flags.power = 1;
 467                 }
 468         }
 469
 470         return 0;
 471 }
 472
 473 /**
 474  * acpi_idle_bm_check - checks if bus master activity was detected
 475  */
 476 static int acpi_idle_bm_check(void)
 477 {
 478         u32 bm_status = 0;
 479
 480         if (bm_check_disable)
 481                 return 0;
 482
 483         acpi_read_bit_register(ACPI_BITREG_BUS_MASTER_STATUS, &bm_status);
 484         if (bm_status)
 485                 acpi_write_bit_register(ACPI_BITREG_BUS_MASTER_STATUS, 1);
 486         /*
 487          * PIIX4 Erratum #18: Note that BM_STS doesn't always reflect
 488          * the true state of bus mastering activity; forcing us to
 489          * manually check the BMIDEA bit of each IDE channel.
 490          */
 491         else if (errata.piix4.bmisx) {
 492                 if ((inb_p(errata.piix4.bmisx + 0x02) & 0x01)
 493                     || (inb_p(errata.piix4.bmisx + 0x0A) & 0x01))
 494                         bm_status = 1;
 495         }
 496         return bm_status;
 497 }
 498
 499 static void wait_for_freeze(void)
 500 {
 501 #ifdef  CONFIG_X86
 502         /* No delay is needed if we are in guest */
 503         if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 504                 return;
 505 #endif
 506         /* Dummy wait op - must do something useless after P_LVL2 read
 507            because chipsets cannot guarantee that STPCLK# signal
 508            gets asserted in time to freeze execution properly. */
 509         inl(acpi_gbl_FADT.xpm_timer_block.address);
 510 }
 511
 512 /**
 513  * acpi_idle_do_entry - enter idle state using the appropriate method
 514  * @cx: cstate data
 515  *
 516  * Caller disables interrupt before call and enables interrupt after return.
 517  */
 518 static void __cpuidle acpi_idle_do_entry(struct acpi_processor_cx *cx)
 519 {
 520         if (cx->entry_method == ACPI_CSTATE_FFH) {
 521                 /* Call into architectural FFH based C-state */
 522                 acpi_processor_ffh_cstate_enter(cx);
 523         } else if (cx->entry_method == ACPI_CSTATE_HALT) {
 524                 acpi_safe_halt();
 525         } else {
 526                 /* IO port based C-state */
 527                 inb(cx->address);
 528                 wait_for_freeze();
 529         }
 530 }
 531
 532 /**
 533  * acpi_idle_play_dead - enters an ACPI state for long-term idle (i.e. off-lining)
 534  * @dev: the target CPU
 535  * @index: the index of suggested state
 536  */
 537 static int acpi_idle_play_dead(struct cpuidle_device *dev, int index)
 538 {
 539         struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
 540
 541         ACPI_FLUSH_CPU_CACHE();
 542
 543         while (1) {
 544
 545                 if (cx->entry_method == ACPI_CSTATE_HALT)
 546                         safe_halt();
 547                 else if (cx->entry_method == ACPI_CSTATE_SYSTEMIO) {
 548                         inb(cx->address);
 549                         wait_for_freeze();
 550                 } else
 551                         return -ENODEV;
 552         }
 553
 554         /* Never reached */
 555         return 0;
 556 }
 557
 558 static bool acpi_idle_fallback_to_c1(struct acpi_processor *pr)
 559 {
 560         return IS_ENABLED(CONFIG_HOTPLUG_CPU) && !pr->flags.has_cst &&
 561                 !(acpi_gbl_FADT.flags & ACPI_FADT_C2_MP_SUPPORTED);
 562 }
 563
 564 static int c3_cpu_count;
 565 static DEFINE_RAW_SPINLOCK(c3_lock);
 566
 567 /**
 568  * acpi_idle_enter_bm - enters C3 with proper BM handling
 569  * @pr: Target processor
 570  * @cx: Target state context
 571  * @timer_bc: Whether or not to change timer mode to broadcast
 572  */
 573 static void acpi_idle_enter_bm(struct acpi_processor *pr,
 574                                struct acpi_processor_cx *cx, bool timer_bc)
 575 {
 576         acpi_unlazy_tlb(smp_processor_id());
 577
 578         /*
 579          * Must be done before busmaster disable as we might need to
 580          * access HPET !
 581          */
 582         if (timer_bc)
 583                 lapic_timer_state_broadcast(pr, cx, 1);
 584
 585         /*
 586          * disable bus master
 587          * bm_check implies we need ARB_DIS
 588          * bm_control implies whether we can do ARB_DIS
 589          *
 590          * That leaves a case where bm_check is set and bm_control is
 591          * not set. In that case we cannot do much, we enter C3
 592          * without doing anything.
 593          */
 594         if (pr->flags.bm_control) {
 595                 raw_spin_lock(&c3_lock);
 596                 c3_cpu_count++;
 597                 /* Disable bus master arbitration when all CPUs are in C3 */
 598                 if (c3_cpu_count == num_online_cpus())
 599                         acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 1);
 600                 raw_spin_unlock(&c3_lock);
 601         }
 602
 603         acpi_idle_do_entry(cx);
 604
 605         /* Re-enable bus master arbitration */
 606         if (pr->flags.bm_control) {
 607                 raw_spin_lock(&c3_lock);
 608                 acpi_write_bit_register(ACPI_BITREG_ARB_DISABLE, 0);
 609                 c3_cpu_count--;
 610                 raw_spin_unlock(&c3_lock);
 611         }
 612
 613         if (timer_bc)
 614                 lapic_timer_state_broadcast(pr, cx, 0);
 615 }
 616
 617 static int acpi_idle_enter(struct cpuidle_device *dev,
 618                            struct cpuidle_driver *drv, int index)
 619 {
 620         struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
 621         struct acpi_processor *pr;
 622
 623         pr = __this_cpu_read(processors);
 624         if (unlikely(!pr))
 625                 return -EINVAL;
 626
 627         if (cx->type != ACPI_STATE_C1) {
 628                 if (acpi_idle_fallback_to_c1(pr) && num_online_cpus() > 1) {
 629                         index = ACPI_IDLE_STATE_START;
 630                         cx = per_cpu(acpi_cstate[index], dev->cpu);
 631                 } else if (cx->type == ACPI_STATE_C3 && pr->flags.bm_check) {
 632                         if (cx->bm_sts_skip || !acpi_idle_bm_check()) {
 633                                 acpi_idle_enter_bm(pr, cx, true);
 634                                 return index;
 635                         } else if (drv->safe_state_index >= 0) {
 636                                 index = drv->safe_state_index;
 637                                 cx = per_cpu(acpi_cstate[index], dev->cpu);
 638                         } else {
 639                                 acpi_safe_halt();
 640                                 return -EBUSY;
 641                         }
 642                 }
 643         }
 644
 645         lapic_timer_state_broadcast(pr, cx, 1);
 646
 647         if (cx->type == ACPI_STATE_C3)
 648                 ACPI_FLUSH_CPU_CACHE();
 649
 650         acpi_idle_do_entry(cx);
 651
 652         lapic_timer_state_broadcast(pr, cx, 0);
 653
 654         return index;
 655 }
 656
 657 static int acpi_idle_enter_s2idle(struct cpuidle_device *dev,
 658                                   struct cpuidle_driver *drv, int index)
 659 {
 660         struct acpi_processor_cx *cx = per_cpu(acpi_cstate[index], dev->cpu);
 661
 662         if (cx->type == ACPI_STATE_C3) {
 663                 struct acpi_processor *pr = __this_cpu_read(processors);
 664
 665                 if (unlikely(!pr))
 666                         return 0;
 667
 668                 if (pr->flags.bm_check) {
 669                         acpi_idle_enter_bm(pr, cx, false);
 670                         return 0;
 671                 } else {
 672                         ACPI_FLUSH_CPU_CACHE();
 673                 }
 674         }
 675         acpi_idle_do_entry(cx);
 676
 677         return 0;
 678 }
 679
 680 static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
 681                                            struct cpuidle_device *dev)
 682 {
 683         int i, count = ACPI_IDLE_STATE_START;
 684         struct acpi_processor_cx *cx;
 685
 686         if (max_cstate == 0)
 687                 max_cstate = 1;
 688
 689         for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
 690                 cx = &pr->power.states[i];
 691
 692                 if (!cx->valid)
 693                         continue;
 694
 695                 per_cpu(acpi_cstate[count], dev->cpu) = cx;
 696
 697                 count++;
 698                 if (count == CPUIDLE_STATE_MAX)
 699                         break;
 700         }
 701
 702         if (!count)
 703                 return -EINVAL;
 704
 705         return 0;
 706 }
 707
 708 static int acpi_processor_setup_cstates(struct acpi_processor *pr)
 709 {
 710         int i, count;
 711         struct acpi_processor_cx *cx;
 712         struct cpuidle_state *state;
 713         struct cpuidle_driver *drv = &acpi_idle_driver;
 714
 715         if (max_cstate == 0)
 716                 max_cstate = 1;
 717
 718         if (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX)) {
 719                 cpuidle_poll_state_init(drv);
 720                 count = 1;
 721         } else {
 722                 count = 0;
 723         }
 724
 725         for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) {
 726                 cx = &pr->power.states[i];
 727
 728                 if (!cx->valid)
 729                         continue;
 730
 731                 state = &drv->states[count];
 732                 snprintf(state->name, CPUIDLE_NAME_LEN, "C%d", i);
 733                 strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
 734                 state->exit_latency = cx->latency;
 735                 state->target_residency = cx->latency * latency_factor;
 736                 state->enter = acpi_idle_enter;
 737
 738                 state->flags = 0;
 739                 if (cx->type == ACPI_STATE_C1 || cx->type == ACPI_STATE_C2) {
 740                         state->enter_dead = acpi_idle_play_dead;
 741                         drv->safe_state_index = count;
 742                 }
 743                 /*
 744                  * Halt-induced C1 is not good for ->enter_s2idle, because it
 745                  * re-enables interrupts on exit.  Moreover, C1 is generally not
 746                  * particularly interesting from the suspend-to-idle angle, so
 747                  * avoid C1 and the situations in which we may need to fall back
 748                  * to it altogether.
 749                  */
 750                 if (cx->type != ACPI_STATE_C1 && !acpi_idle_fallback_to_c1(pr))
 751                         state->enter_s2idle = acpi_idle_enter_s2idle;
 752
 753                 count++;
 754                 if (count == CPUIDLE_STATE_MAX)
 755                         break;
 756         }
 757
 758         drv->state_count = count;
 759
 760         if (!count)
 761                 return -EINVAL;
 762
 763         return 0;
 764 }
 765
 766 static inline void acpi_processor_cstate_first_run_checks(void)
 767 {
 768         static int first_run;
 769
 770         if (first_run)
 771                 return;
 772         dmi_check_system(processor_power_dmi_table);
 773         max_cstate = acpi_processor_cstate_check(max_cstate);
 774         if (max_cstate < ACPI_C_STATES_MAX)
 775                 pr_notice("ACPI: processor limited to max C-state %d\n",
 776                           max_cstate);
 777         first_run++;
 778
 779         if (nocst)
 780                 return;
 781
 782         acpi_processor_claim_cst_control();
 783 }
 784 #else
 785
 786 static inline int disabled_by_idle_boot_param(void) { return 0; }
 787 static inline void acpi_processor_cstate_first_run_checks(void) { }
 788 static int acpi_processor_get_cstate_info(struct acpi_processor *pr)
 789 {
 790         return -ENODEV;
 791 }
 792
 793 static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr,
 794                                            struct cpuidle_device *dev)
 795 {
 796         return -EINVAL;
 797 }
 798
 799 static int acpi_processor_setup_cstates(struct acpi_processor *pr)
 800 {
 801         return -EINVAL;
 802 }
 803
 804 #endif /* CONFIG_ACPI_PROCESSOR_CSTATE */
 805
 806 struct acpi_lpi_states_array {
 807         unsigned int size;
 808         unsigned int composite_states_size;
 809         struct acpi_lpi_state *entries;
 810         struct acpi_lpi_state *composite_states[ACPI_PROCESSOR_MAX_POWER];
 811 };
 812
 813 static int obj_get_integer(union acpi_object *obj, u32 *value)
 814 {
 815         if (obj->type != ACPI_TYPE_INTEGER)
 816                 return -EINVAL;
 817
 818         *value = obj->integer.value;
 819         return 0;
 820 }
 821
 822 static int acpi_processor_evaluate_lpi(acpi_handle handle,
 823                                        struct acpi_lpi_states_array *info)
 824 {
 825         acpi_status status;
 826         int ret = 0;
 827         int pkg_count, state_idx = 1, loop;
 828         struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 829         union acpi_object *lpi_data;
 830         struct acpi_lpi_state *lpi_state;
 831
 832         status = acpi_evaluate_object(handle, "_LPI", NULL, &buffer);
 833         if (ACPI_FAILURE(status)) {
 834                 ACPI_DEBUG_PRINT((ACPI_DB_INFO, "No _LPI, giving up\n"));
 835                 return -ENODEV;
 836         }
 837
 838         lpi_data = buffer.pointer;
 839
 840         /* There must be at least 4 elements = 3 elements + 1 package */
 841         if (!lpi_data || lpi_data->type != ACPI_TYPE_PACKAGE ||
 842             lpi_data->package.count < 4) {
 843                 pr_debug("not enough elements in _LPI\n");
 844                 ret = -ENODATA;
 845                 goto end;
 846         }
 847
 848         pkg_count = lpi_data->package.elements[2].integer.value;
 849
 850         /* Validate number of power states. */
 851         if (pkg_count < 1 || pkg_count != lpi_data->package.count - 3) {
 852                 pr_debug("count given by _LPI is not valid\n");
 853                 ret = -ENODATA;
 854                 goto end;
 855         }
 856
 857         lpi_state = kcalloc(pkg_count, sizeof(*lpi_state), GFP_KERNEL);
 858         if (!lpi_state) {
 859                 ret = -ENOMEM;
 860                 goto end;
 861         }
 862
 863         info->size = pkg_count;
 864         info->entries = lpi_state;
 865
 866         /* LPI States start at index 3 */
 867         for (loop = 3; state_idx <= pkg_count; loop++, state_idx++, lpi_state++) {
 868                 union acpi_object *element, *pkg_elem, *obj;
 869
 870                 element = &lpi_data->package.elements[loop];
 871                 if (element->type != ACPI_TYPE_PACKAGE || element->package.count < 7)
 872                         continue;
 873
 874                 pkg_elem = element->package.elements;
 875
 876                 obj = pkg_elem + 6;
 877                 if (obj->type == ACPI_TYPE_BUFFER) {
 878                         struct acpi_power_register *reg;
 879
 880                         reg = (struct acpi_power_register *)obj->buffer.pointer;
 881                         if (reg->space_id != ACPI_ADR_SPACE_SYSTEM_IO &&
 882                             reg->space_id != ACPI_ADR_SPACE_FIXED_HARDWARE)
 883                                 continue;
 884
 885                         lpi_state->address = reg->address;
 886                         lpi_state->entry_method =
 887                                 reg->space_id == ACPI_ADR_SPACE_FIXED_HARDWARE ?
 888                                 ACPI_CSTATE_FFH : ACPI_CSTATE_SYSTEMIO;
 889                 } else if (obj->type == ACPI_TYPE_INTEGER) {
 890                         lpi_state->entry_method = ACPI_CSTATE_INTEGER;
 891                         lpi_state->address = obj->integer.value;
 892                 } else {
 893                         continue;
 894                 }
 895
 896                 /* elements[7,8] skipped for now i.e. Residency/Usage counter*/
 897
 898                 obj = pkg_elem + 9;
 899                 if (obj->type == ACPI_TYPE_STRING)
 900                         strlcpy(lpi_state->desc, obj->string.pointer,
 901                                 ACPI_CX_DESC_LEN);
 902
 903                 lpi_state->index = state_idx;
 904                 if (obj_get_integer(pkg_elem + 0, &lpi_state->min_residency)) {
 905                         pr_debug("No min. residency found, assuming 10 us\n");
 906                         lpi_state->min_residency = 10;
 907                 }
 908
 909                 if (obj_get_integer(pkg_elem + 1, &lpi_state->wake_latency)) {
 910                         pr_debug("No wakeup residency found, assuming 10 us\n");
 911                         lpi_state->wake_latency = 10;
 912                 }
 913
 914                 if (obj_get_integer(pkg_elem + 2, &lpi_state->flags))
 915                         lpi_state->flags = 0;
 916
 917                 if (obj_get_integer(pkg_elem + 3, &lpi_state->arch_flags))
 918                         lpi_state->arch_flags = 0;
 919
 920                 if (obj_get_integer(pkg_elem + 4, &lpi_state->res_cnt_freq))
 921                         lpi_state->res_cnt_freq = 1;
 922
 923                 if (obj_get_integer(pkg_elem + 5, &lpi_state->enable_parent_state))
 924                         lpi_state->enable_parent_state = 0;
 925         }
 926
 927         acpi_handle_debug(handle, "Found %d power states\n", state_idx);
 928 end:
 929         kfree(buffer.pointer);
 930         return ret;
 931 }
 932
 933 /*
 934  * flat_state_cnt - the number of composite LPI states after the process of flattening
 935  */
 936 static int flat_state_cnt;
 937
 938 /**
 939  * combine_lpi_states - combine local and parent LPI states to form a composite LPI state
 940  *
 941  * @local: local LPI state
 942  * @parent: parent LPI state
 943  * @result: composite LPI state
 944  */
 945 static bool combine_lpi_states(struct acpi_lpi_state *local,
 946                                struct acpi_lpi_state *parent,
 947                                struct acpi_lpi_state *result)
 948 {
 949         if (parent->entry_method == ACPI_CSTATE_INTEGER) {
 950                 if (!parent->address) /* 0 means autopromotable */
 951                         return false;
 952                 result->address = local->address + parent->address;
 953         } else {
 954                 result->address = parent->address;
 955         }
 956
 957         result->min_residency = max(local->min_residency, parent->min_residency);
 958         result->wake_latency = local->wake_latency + parent->wake_latency;
 959         result->enable_parent_state = parent->enable_parent_state;
 960         result->entry_method = local->entry_method;
 961
 962         result->flags = parent->flags;
 963         result->arch_flags = parent->arch_flags;
 964         result->index = parent->index;
 965
 966         strlcpy(result->desc, local->desc, ACPI_CX_DESC_LEN);
 967         strlcat(result->desc, "+", ACPI_CX_DESC_LEN);
 968         strlcat(result->desc, parent->desc, ACPI_CX_DESC_LEN);
 969         return true;
 970 }
 971
 972 #define ACPI_LPI_STATE_FLAGS_ENABLED                    BIT(0)
 973
 974 static void stash_composite_state(struct acpi_lpi_states_array *curr_level,
 975                                   struct acpi_lpi_state *t)
 976 {
 977         curr_level->composite_states[curr_level->composite_states_size++] = t;
 978 }
 979
 980 static int flatten_lpi_states(struct acpi_processor *pr,
 981                               struct acpi_lpi_states_array *curr_level,
 982                               struct acpi_lpi_states_array *prev_level)
 983 {
 984         int i, j, state_count = curr_level->size;
 985         struct acpi_lpi_state *p, *t = curr_level->entries;
 986
 987         curr_level->composite_states_size = 0;
 988         for (j = 0; j < state_count; j++, t++) {
 989                 struct acpi_lpi_state *flpi;
 990
 991                 if (!(t->flags & ACPI_LPI_STATE_FLAGS_ENABLED))
 992                         continue;
 993
 994                 if (flat_state_cnt >= ACPI_PROCESSOR_MAX_POWER) {
 995                         pr_warn("Limiting number of LPI states to max (%d)\n",
 996                                 ACPI_PROCESSOR_MAX_POWER);
 997                         pr_warn("Please increase ACPI_PROCESSOR_MAX_POWER if needed.\n");
 998                         break;
 999                 }
1000
1001                 flpi = &pr->power.lpi_states[flat_state_cnt];
1002
1003                 if (!prev_level) { /* leaf/processor node */
1004                         memcpy(flpi, t, sizeof(*t));
1005                         stash_composite_state(curr_level, flpi);
1006                         flat_state_cnt++;
1007                         continue;
1008                 }
1009
1010                 for (i = 0; i < prev_level->composite_states_size; i++) {
1011                         p = prev_level->composite_states[i];
1012                         if (t->index <= p->enable_parent_state &&
1013                             combine_lpi_states(p, t, flpi)) {
1014                                 stash_composite_state(curr_level, flpi);
1015                                 flat_state_cnt++;
1016                                 flpi++;
1017                         }
1018                 }
1019         }
1020
1021         kfree(curr_level->entries);
1022         return 0;
1023 }
1024
1025 static int acpi_processor_get_lpi_info(struct acpi_processor *pr)
1026 {
1027         int ret, i;
1028         acpi_status status;
1029         acpi_handle handle = pr->handle, pr_ahandle;
1030         struct acpi_device *d = NULL;
1031         struct acpi_lpi_states_array info[2], *tmp, *prev, *curr;
1032
1033         if (!osc_pc_lpi_support_confirmed)
1034                 return -EOPNOTSUPP;
1035
1036         if (!acpi_has_method(handle, "_LPI"))
1037                 return -EINVAL;
1038
1039         flat_state_cnt = 0;
1040         prev = &info[0];
1041         curr = &info[1];
1042         handle = pr->handle;
1043         ret = acpi_processor_evaluate_lpi(handle, prev);
1044         if (ret)
1045                 return ret;
1046         flatten_lpi_states(pr, prev, NULL);
1047
1048         status = acpi_get_parent(handle, &pr_ahandle);
1049         while (ACPI_SUCCESS(status)) {
1050                 acpi_bus_get_device(pr_ahandle, &d);
1051                 handle = pr_ahandle;
1052
1053                 if (strcmp(acpi_device_hid(d), ACPI_PROCESSOR_CONTAINER_HID))
1054                         break;
1055
1056                 /* can be optional ? */
1057                 if (!acpi_has_method(handle, "_LPI"))
1058                         break;
1059
1060                 ret = acpi_processor_evaluate_lpi(handle, curr);
1061                 if (ret)
1062                         break;
1063
1064                 /* flatten all the LPI states in this level of hierarchy */
1065                 flatten_lpi_states(pr, curr, prev);
1066
1067                 tmp = prev, prev = curr, curr = tmp;
1068
1069                 status = acpi_get_parent(handle, &pr_ahandle);
1070         }
1071
1072         pr->power.count = flat_state_cnt;
1073         /* reset the index after flattening */
1074         for (i = 0; i < pr->power.count; i++)
1075                 pr->power.lpi_states[i].index = i;
1076
1077         /* Tell driver that _LPI is supported. */
1078         pr->flags.has_lpi = 1;
1079         pr->flags.power = 1;
1080
1081         return 0;
1082 }
1083
1084 int __weak acpi_processor_ffh_lpi_probe(unsigned int cpu)
1085 {
1086         return -ENODEV;
1087 }
1088
1089 int __weak acpi_processor_ffh_lpi_enter(struct acpi_lpi_state *lpi)
1090 {
1091         return -ENODEV;
1092 }
1093
1094 /**
1095  * acpi_idle_lpi_enter - enters an ACPI any LPI state
1096  * @dev: the target CPU
1097  * @drv: cpuidle driver containing cpuidle state info
1098  * @index: index of target state
1099  *
1100  * Return: 0 for success or negative value for error
1101  */
1102 static int acpi_idle_lpi_enter(struct cpuidle_device *dev,
1103                                struct cpuidle_driver *drv, int index)
1104 {
1105         struct acpi_processor *pr;
1106         struct acpi_lpi_state *lpi;
1107
1108         pr = __this_cpu_read(processors);
1109
1110         if (unlikely(!pr))
1111                 return -EINVAL;
1112
1113         lpi = &pr->power.lpi_states[index];
1114         if (lpi->entry_method == ACPI_CSTATE_FFH)
1115                 return acpi_processor_ffh_lpi_enter(lpi);
1116
1117         return -EINVAL;
1118 }
1119
1120 static int acpi_processor_setup_lpi_states(struct acpi_processor *pr)
1121 {
1122         int i;
1123         struct acpi_lpi_state *lpi;
1124         struct cpuidle_state *state;
1125         struct cpuidle_driver *drv = &acpi_idle_driver;
1126
1127         if (!pr->flags.has_lpi)
1128                 return -EOPNOTSUPP;
1129
1130         for (i = 0; i < pr->power.count && i < CPUIDLE_STATE_MAX; i++) {
1131                 lpi = &pr->power.lpi_states[i];
1132
1133                 state = &drv->states[i];
1134                 snprintf(state->name, CPUIDLE_NAME_LEN, "LPI-%d", i);
1135                 strlcpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN);
1136                 state->exit_latency = lpi->wake_latency;
1137                 state->target_residency = lpi->min_residency;
1138                 if (lpi->arch_flags)
1139                         state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1140                 state->enter = acpi_idle_lpi_enter;
1141                 drv->safe_state_index = i;
1142         }
1143
1144         drv->state_count = i;
1145
1146         return 0;
1147 }
1148
1149 /**
1150  * acpi_processor_setup_cpuidle_states- prepares and configures cpuidle
1151  * global state data i.e. idle routines
1152  *
1153  * @pr: the ACPI processor
1154  */
1155 static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr)
1156 {
1157         int i;
1158         struct cpuidle_driver *drv = &acpi_idle_driver;
1159
1160         if (!pr->flags.power_setup_done || !pr->flags.power)
1161                 return -EINVAL;
1162
1163         drv->safe_state_index = -1;
1164         for (i = ACPI_IDLE_STATE_START; i < CPUIDLE_STATE_MAX; i++) {
1165                 drv->states[i].name[0] = '\0';
1166                 drv->states[i].desc[0] = '\0';
1167         }
1168
1169         if (pr->flags.has_lpi)
1170                 return acpi_processor_setup_lpi_states(pr);
1171
1172         return acpi_processor_setup_cstates(pr);
1173 }
1174
1175 /**
1176  * acpi_processor_setup_cpuidle_dev - prepares and configures CPUIDLE
1177  * device i.e. per-cpu data
1178  *
1179  * @pr: the ACPI processor
1180  * @dev : the cpuidle device
1181  */
1182 static int acpi_processor_setup_cpuidle_dev(struct acpi_processor *pr,
1183                                             struct cpuidle_device *dev)
1184 {
1185         if (!pr->flags.power_setup_done || !pr->flags.power || !dev)
1186                 return -EINVAL;
1187
1188         dev->cpu = pr->id;
1189         if (pr->flags.has_lpi)
1190                 return acpi_processor_ffh_lpi_probe(pr->id);
1191
1192         return acpi_processor_setup_cpuidle_cx(pr, dev);
1193 }
1194
1195 static int acpi_processor_get_power_info(struct acpi_processor *pr)
1196 {
1197         int ret;
1198
1199         ret = acpi_processor_get_lpi_info(pr);
1200         if (ret)
1201                 ret = acpi_processor_get_cstate_info(pr);
1202
1203         return ret;
1204 }
1205
1206 int acpi_processor_hotplug(struct acpi_processor *pr)
1207 {
1208         int ret = 0;
1209         struct cpuidle_device *dev;
1210
1211         if (disabled_by_idle_boot_param())
1212                 return 0;
1213
1214         if (!pr->flags.power_setup_done)
1215                 return -ENODEV;
1216
1217         dev = per_cpu(acpi_cpuidle_device, pr->id);
1218         cpuidle_pause_and_lock();
1219         cpuidle_disable_device(dev);
1220         ret = acpi_processor_get_power_info(pr);
1221         if (!ret && pr->flags.power) {
1222                 acpi_processor_setup_cpuidle_dev(pr, dev);
1223                 ret = cpuidle_enable_device(dev);
1224         }
1225         cpuidle_resume_and_unlock();
1226
1227         return ret;
1228 }
1229
1230 int acpi_processor_power_state_has_changed(struct acpi_processor *pr)
1231 {
1232         int cpu;
1233         struct acpi_processor *_pr;
1234         struct cpuidle_device *dev;
1235
1236         if (disabled_by_idle_boot_param())
1237                 return 0;
1238
1239         if (!pr->flags.power_setup_done)
1240                 return -ENODEV;
1241
1242         /*
1243          * FIXME:  Design the ACPI notification to make it once per
1244          * system instead of once per-cpu.  This condition is a hack
1245          * to make the code that updates C-States be called once.
1246          */
1247
1248         if (pr->id == 0 && cpuidle_get_driver() == &acpi_idle_driver) {
1249
1250                 /* Protect against cpu-hotplug */
1251                 get_online_cpus();
1252                 cpuidle_pause_and_lock();
1253
1254                 /* Disable all cpuidle devices */
1255                 for_each_online_cpu(cpu) {
1256                         _pr = per_cpu(processors, cpu);
1257                         if (!_pr || !_pr->flags.power_setup_done)
1258                                 continue;
1259                         dev = per_cpu(acpi_cpuidle_device, cpu);
1260                         cpuidle_disable_device(dev);
1261                 }
1262
1263                 /* Populate Updated C-state information */
1264                 acpi_processor_get_power_info(pr);
1265                 acpi_processor_setup_cpuidle_states(pr);
1266
1267                 /* Enable all cpuidle devices */
1268                 for_each_online_cpu(cpu) {
1269                         _pr = per_cpu(processors, cpu);
1270                         if (!_pr || !_pr->flags.power_setup_done)
1271                                 continue;
1272                         acpi_processor_get_power_info(_pr);
1273                         if (_pr->flags.power) {
1274                                 dev = per_cpu(acpi_cpuidle_device, cpu);
1275                                 acpi_processor_setup_cpuidle_dev(_pr, dev);
1276                                 cpuidle_enable_device(dev);
1277                         }
1278                 }
1279                 cpuidle_resume_and_unlock();
1280                 put_online_cpus();
1281         }
1282
1283         return 0;
1284 }
1285
1286 static int acpi_processor_registered;
1287
1288 int acpi_processor_power_init(struct acpi_processor *pr)
1289 {
1290         int retval;
1291         struct cpuidle_device *dev;
1292
1293         if (disabled_by_idle_boot_param())
1294                 return 0;
1295
1296         acpi_processor_cstate_first_run_checks();
1297
1298         if (!acpi_processor_get_power_info(pr))
1299                 pr->flags.power_setup_done = 1;
1300
1301         /*
1302          * Install the idle handler if processor power management is supported.
1303          * Note that we use previously set idle handler will be used on
1304          * platforms that only support C1.
1305          */
1306         if (pr->flags.power) {
1307                 /* Register acpi_idle_driver if not already registered */
1308                 if (!acpi_processor_registered) {
1309                         acpi_processor_setup_cpuidle_states(pr);
1310                         retval = cpuidle_register_driver(&acpi_idle_driver);
1311                         if (retval)
1312                                 return retval;
1313                         pr_debug("%s registered with cpuidle\n",
1314                                  acpi_idle_driver.name);
1315                 }
1316
1317                 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1318                 if (!dev)
1319                         return -ENOMEM;
1320                 per_cpu(acpi_cpuidle_device, pr->id) = dev;
1321
1322                 acpi_processor_setup_cpuidle_dev(pr, dev);
1323
1324                 /* Register per-cpu cpuidle_device. Cpuidle driver
1325                  * must already be registered before registering device
1326                  */
1327                 retval = cpuidle_register_device(dev);
1328                 if (retval) {
1329                         if (acpi_processor_registered == 0)
1330                                 cpuidle_unregister_driver(&acpi_idle_driver);
1331                         return retval;
1332                 }
1333                 acpi_processor_registered++;
1334         }
1335         return 0;
1336 }
1337
1338 int acpi_processor_power_exit(struct acpi_processor *pr)
1339 {
1340         struct cpuidle_device *dev = per_cpu(acpi_cpuidle_device, pr->id);
1341
1342         if (disabled_by_idle_boot_param())
1343                 return 0;
1344
1345         if (pr->flags.power) {
1346                 cpuidle_unregister_device(dev);
1347                 acpi_processor_registered--;
1348                 if (acpi_processor_registered == 0)
1349                         cpuidle_unregister_driver(&acpi_idle_driver);
1350         }
1351
1352         pr->flags.power_setup_done = 0;
1353         return 0;
1354 }