drivers/cpufreq/cpufreq.c

   1 /*
   2  *  linux/drivers/cpufreq/cpufreq.c
   3  *
   4  *  Copyright (C) 2001 Russell King
   5  *            (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
   6  *
   7  *  Oct 2005 - Ashok Raj <ashok.raj@intel.com>
   8  *      Added handling for CPU hotplug
   9  *  Feb 2006 - Jacob Shin <jacob.shin@amd.com>
  10  *      Fix handling for CPU hotplug -- affected CPUs
  11  *
  12  * This program is free software; you can redistribute it and/or modify
  13  * it under the terms of the GNU General Public License version 2 as
  14  * published by the Free Software Foundation.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  19
  20 #include <linux/kernel.h>
  21 #include <linux/module.h>
  22 #include <linux/init.h>
  23 #include <linux/notifier.h>
  24 #include <linux/cpufreq.h>
  25 #include <linux/delay.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/spinlock.h>
  28 #include <linux/device.h>
  29 #include <linux/slab.h>
  30 #include <linux/cpu.h>
  31 #include <linux/completion.h>
  32 #include <linux/mutex.h>
  33 #include <linux/syscore_ops.h>
  34
  35 #include <trace/events/power.h>
  36
  37 /**
  38  * The "cpufreq driver" - the arch- or hardware-dependent low
  39  * level driver of CPUFreq support, and its spinlock. This lock
  40  * also protects the cpufreq_cpu_data array.
  41  */
  42 static struct cpufreq_driver *cpufreq_driver;
  43 static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
  44 #ifdef CONFIG_HOTPLUG_CPU
  45 /* This one keeps track of the previously set governor of a removed CPU */
  46 static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor);
  47 #endif
  48 static DEFINE_SPINLOCK(cpufreq_driver_lock);
  49
  50 /*
  51  * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure
  52  * all cpufreq/hotplug/workqueue/etc related lock issues.
  53  *
  54  * The rules for this semaphore:
  55  * - Any routine that wants to read from the policy structure will
  56  *   do a down_read on this semaphore.
  57  * - Any routine that will write to the policy structure and/or may take away
  58  *   the policy altogether (eg. CPU hotplug), will hold this lock in write
  59  *   mode before doing so.
  60  *
  61  * Additional rules:
  62  * - All holders of the lock should check to make sure that the CPU they
  63  *   are concerned with are online after they get the lock.
  64  * - Governor routines that can be called in cpufreq hotplug path should not
  65  *   take this sem as top level hotplug notifier handler takes this.
  66  * - Lock should not be held across
  67  *     __cpufreq_governor(data, CPUFREQ_GOV_STOP);
  68  */
  69 static DEFINE_PER_CPU(int, cpufreq_policy_cpu);
  70 static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem);
  71
  72 #define lock_policy_rwsem(mode, cpu)                                    \
  73 static int lock_policy_rwsem_##mode                                     \
  74 (int cpu)                                                               \
  75 {                                                                       \
  76         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);              \
  77         BUG_ON(policy_cpu == -1);                                       \
  78         down_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));            \
  79         if (unlikely(!cpu_online(cpu))) {                               \
  80                 up_##mode(&per_cpu(cpu_policy_rwsem, policy_cpu));      \
  81                 return -1;                                              \
  82         }                                                               \
  83                                                                         \
  84         return 0;                                                       \
  85 }
  86
  87 lock_policy_rwsem(read, cpu);
  88
  89 lock_policy_rwsem(write, cpu);
  90
  91 static void unlock_policy_rwsem_read(int cpu)
  92 {
  93         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
  94         BUG_ON(policy_cpu == -1);
  95         up_read(&per_cpu(cpu_policy_rwsem, policy_cpu));
  96 }
  97
  98 static void unlock_policy_rwsem_write(int cpu)
  99 {
 100         int policy_cpu = per_cpu(cpufreq_policy_cpu, cpu);
 101         BUG_ON(policy_cpu == -1);
 102         up_write(&per_cpu(cpu_policy_rwsem, policy_cpu));
 103 }
 104
 105
 106 /* internal prototypes */
 107 static int __cpufreq_governor(struct cpufreq_policy *policy,
 108                 unsigned int event);
 109 static unsigned int __cpufreq_get(unsigned int cpu);
 110 static void handle_update(struct work_struct *work);
 111
 112 /**
 113  * Two notifier lists: the "policy" list is involved in the
 114  * validation process for a new CPU frequency policy; the
 115  * "transition" list for kernel code that needs to handle
 116  * changes to devices when the CPU clock speed changes.
 117  * The mutex locks both lists.
 118  */
 119 static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
 120 static struct srcu_notifier_head cpufreq_transition_notifier_list;
 121
 122 static bool init_cpufreq_transition_notifier_list_called;
 123 static int __init init_cpufreq_transition_notifier_list(void)
 124 {
 125         srcu_init_notifier_head(&cpufreq_transition_notifier_list);
 126         init_cpufreq_transition_notifier_list_called = true;
 127         return 0;
 128 }
 129 pure_initcall(init_cpufreq_transition_notifier_list);
 130
 131 static int off __read_mostly;
 132 static int cpufreq_disabled(void)
 133 {
 134         return off;
 135 }
 136 void disable_cpufreq(void)
 137 {
 138         off = 1;
 139 }
 140 static LIST_HEAD(cpufreq_governor_list);
 141 static DEFINE_MUTEX(cpufreq_governor_mutex);
 142
 143 static struct cpufreq_policy *__cpufreq_cpu_get(unsigned int cpu, bool sysfs)
 144 {
 145         struct cpufreq_policy *data;
 146         unsigned long flags;
 147
 148         if (cpu >= nr_cpu_ids)
 149                 goto err_out;
 150
 151         /* get the cpufreq driver */
 152         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 153
 154         if (!cpufreq_driver)
 155                 goto err_out_unlock;
 156
 157         if (!try_module_get(cpufreq_driver->owner))
 158                 goto err_out_unlock;
 159
 160
 161         /* get the CPU */
 162         data = per_cpu(cpufreq_cpu_data, cpu);
 163
 164         if (!data)
 165                 goto err_out_put_module;
 166
 167         if (!sysfs && !kobject_get(&data->kobj))
 168                 goto err_out_put_module;
 169
 170         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 171         return data;
 172
 173 err_out_put_module:
 174         module_put(cpufreq_driver->owner);
 175 err_out_unlock:
 176         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 177 err_out:
 178         return NULL;
 179 }
 180
 181 struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
 182 {
 183         return __cpufreq_cpu_get(cpu, false);
 184 }
 185 EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
 186
 187 static struct cpufreq_policy *cpufreq_cpu_get_sysfs(unsigned int cpu)
 188 {
 189         return __cpufreq_cpu_get(cpu, true);
 190 }
 191
 192 static void __cpufreq_cpu_put(struct cpufreq_policy *data, bool sysfs)
 193 {
 194         if (!sysfs)
 195                 kobject_put(&data->kobj);
 196         module_put(cpufreq_driver->owner);
 197 }
 198
 199 void cpufreq_cpu_put(struct cpufreq_policy *data)
 200 {
 201         __cpufreq_cpu_put(data, false);
 202 }
 203 EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
 204
 205 static void cpufreq_cpu_put_sysfs(struct cpufreq_policy *data)
 206 {
 207         __cpufreq_cpu_put(data, true);
 208 }
 209
 210 /*********************************************************************
 211  *            EXTERNALLY AFFECTING FREQUENCY CHANGES                 *
 212  *********************************************************************/
 213
 214 /**
 215  * adjust_jiffies - adjust the system "loops_per_jiffy"
 216  *
 217  * This function alters the system "loops_per_jiffy" for the clock
 218  * speed change. Note that loops_per_jiffy cannot be updated on SMP
 219  * systems as each CPU might be scaled differently. So, use the arch
 220  * per-CPU loops_per_jiffy value wherever possible.
 221  */
 222 #ifndef CONFIG_SMP
 223 static unsigned long l_p_j_ref;
 224 static unsigned int  l_p_j_ref_freq;
 225
 226 static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 227 {
 228         if (ci->flags & CPUFREQ_CONST_LOOPS)
 229                 return;
 230
 231         if (!l_p_j_ref_freq) {
 232                 l_p_j_ref = loops_per_jiffy;
 233                 l_p_j_ref_freq = ci->old;
 234                 pr_debug("saving %lu as reference value for loops_per_jiffy; "
 235                         "freq is %u kHz\n", l_p_j_ref, l_p_j_ref_freq);
 236         }
 237         if ((val == CPUFREQ_POSTCHANGE  && ci->old != ci->new) ||
 238             (val == CPUFREQ_RESUMECHANGE || val == CPUFREQ_SUSPENDCHANGE)) {
 239                 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
 240                                                                 ci->new);
 241                 pr_debug("scaling loops_per_jiffy to %lu "
 242                         "for frequency %u kHz\n", loops_per_jiffy, ci->new);
 243         }
 244 }
 245 #else
 246 static inline void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
 247 {
 248         return;
 249 }
 250 #endif
 251
 252
 253 /**
 254  * cpufreq_notify_transition - call notifier chain and adjust_jiffies
 255  * on frequency transition.
 256  *
 257  * This function calls the transition notifiers and the "adjust_jiffies"
 258  * function. It is called twice on all CPU frequency changes that have
 259  * external effects.
 260  */
 261 void cpufreq_notify_transition(struct cpufreq_freqs *freqs, unsigned int state)
 262 {
 263         struct cpufreq_policy *policy;
 264
 265         BUG_ON(irqs_disabled());
 266
 267         freqs->flags = cpufreq_driver->flags;
 268         pr_debug("notification %u of frequency transition to %u kHz\n",
 269                 state, freqs->new);
 270
 271         policy = per_cpu(cpufreq_cpu_data, freqs->cpu);
 272         switch (state) {
 273
 274         case CPUFREQ_PRECHANGE:
 275                 /* detect if the driver reported a value as "old frequency"
 276                  * which is not equal to what the cpufreq core thinks is
 277                  * "old frequency".
 278                  */
 279                 if (!(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
 280                         if ((policy) && (policy->cpu == freqs->cpu) &&
 281                             (policy->cur) && (policy->cur != freqs->old)) {
 282                                 pr_debug("Warning: CPU frequency is"
 283                                         " %u, cpufreq assumed %u kHz.\n",
 284                                         freqs->old, policy->cur);
 285                                 freqs->old = policy->cur;
 286                         }
 287                 }
 288                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 289                                 CPUFREQ_PRECHANGE, freqs);
 290                 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
 291                 break;
 292
 293         case CPUFREQ_POSTCHANGE:
 294                 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
 295                 pr_debug("FREQ: %lu - CPU: %lu", (unsigned long)freqs->new,
 296                         (unsigned long)freqs->cpu);
 297                 trace_cpu_frequency(freqs->new, freqs->cpu);
 298                 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
 299                                 CPUFREQ_POSTCHANGE, freqs);
 300                 if (likely(policy) && likely(policy->cpu == freqs->cpu))
 301                         policy->cur = freqs->new;
 302                 break;
 303         }
 304 }
 305 EXPORT_SYMBOL_GPL(cpufreq_notify_transition);
 306
 307
 308
 309 /*********************************************************************
 310  *                          SYSFS INTERFACE                          *
 311  *********************************************************************/
 312
 313 static struct cpufreq_governor *__find_governor(const char *str_governor)
 314 {
 315         struct cpufreq_governor *t;
 316
 317         list_for_each_entry(t, &cpufreq_governor_list, governor_list)
 318                 if (!strnicmp(str_governor, t->name, CPUFREQ_NAME_LEN))
 319                         return t;
 320
 321         return NULL;
 322 }
 323
 324 /**
 325  * cpufreq_parse_governor - parse a governor string
 326  */
 327 static int cpufreq_parse_governor(char *str_governor, unsigned int *policy,
 328                                 struct cpufreq_governor **governor)
 329 {
 330         int err = -EINVAL;
 331
 332         if (!cpufreq_driver)
 333                 goto out;
 334
 335         if (cpufreq_driver->setpolicy) {
 336                 if (!strnicmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
 337                         *policy = CPUFREQ_POLICY_PERFORMANCE;
 338                         err = 0;
 339                 } else if (!strnicmp(str_governor, "powersave",
 340                                                 CPUFREQ_NAME_LEN)) {
 341                         *policy = CPUFREQ_POLICY_POWERSAVE;
 342                         err = 0;
 343                 }
 344         } else if (cpufreq_driver->target) {
 345                 struct cpufreq_governor *t;
 346
 347                 mutex_lock(&cpufreq_governor_mutex);
 348
 349                 t = __find_governor(str_governor);
 350
 351                 if (t == NULL) {
 352                         int ret;
 353
 354                         mutex_unlock(&cpufreq_governor_mutex);
 355                         ret = request_module("cpufreq_%s", str_governor);
 356                         mutex_lock(&cpufreq_governor_mutex);
 357
 358                         if (ret == 0)
 359                                 t = __find_governor(str_governor);
 360                 }
 361
 362                 if (t != NULL) {
 363                         *governor = t;
 364                         err = 0;
 365                 }
 366
 367                 mutex_unlock(&cpufreq_governor_mutex);
 368         }
 369 out:
 370         return err;
 371 }
 372
 373
 374 /**
 375  * cpufreq_per_cpu_attr_read() / show_##file_name() -
 376  * print out cpufreq information
 377  *
 378  * Write out information from cpufreq_driver->policy[cpu]; object must be
 379  * "unsigned int".
 380  */
 381
 382 #define show_one(file_name, object)                     \
 383 static ssize_t show_##file_name                         \
 384 (struct cpufreq_policy *policy, char *buf)              \
 385 {                                                       \
 386         return sprintf(buf, "%u\n", policy->object);    \
 387 }
 388
 389 show_one(cpuinfo_min_freq, cpuinfo.min_freq);
 390 show_one(cpuinfo_max_freq, cpuinfo.max_freq);
 391 show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
 392 show_one(scaling_min_freq, min);
 393 show_one(scaling_max_freq, max);
 394 show_one(scaling_cur_freq, cur);
 395
 396 static int __cpufreq_set_policy(struct cpufreq_policy *data,
 397                                 struct cpufreq_policy *policy);
 398
 399 /**
 400  * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
 401  */
 402 #define store_one(file_name, object)                    \
 403 static ssize_t store_##file_name                                        \
 404 (struct cpufreq_policy *policy, const char *buf, size_t count)          \
 405 {                                                                       \
 406         unsigned int ret;                                               \
 407         struct cpufreq_policy new_policy;                               \
 408                                                                         \
 409         ret = cpufreq_get_policy(&new_policy, policy->cpu);             \
 410         if (ret)                                                        \
 411                 return -EINVAL;                                         \
 412                                                                         \
 413         ret = sscanf(buf, "%u", &new_policy.object);                    \
 414         if (ret != 1)                                                   \
 415                 return -EINVAL;                                         \
 416                                                                         \
 417         ret = __cpufreq_set_policy(policy, &new_policy);                \
 418         policy->user_policy.object = policy->object;                    \
 419                                                                         \
 420         return ret ? ret : count;                                       \
 421 }
 422
 423 store_one(scaling_min_freq, min);
 424 store_one(scaling_max_freq, max);
 425
 426 /**
 427  * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
 428  */
 429 static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
 430                                         char *buf)
 431 {
 432         unsigned int cur_freq = __cpufreq_get(policy->cpu);
 433         if (!cur_freq)
 434                 return sprintf(buf, "<unknown>");
 435         return sprintf(buf, "%u\n", cur_freq);
 436 }
 437
 438
 439 /**
 440  * show_scaling_governor - show the current policy for the specified CPU
 441  */
 442 static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
 443 {
 444         if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
 445                 return sprintf(buf, "powersave\n");
 446         else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
 447                 return sprintf(buf, "performance\n");
 448         else if (policy->governor)
 449                 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
 450                                 policy->governor->name);
 451         return -EINVAL;
 452 }
 453
 454
 455 /**
 456  * store_scaling_governor - store policy for the specified CPU
 457  */
 458 static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
 459                                         const char *buf, size_t count)
 460 {
 461         unsigned int ret;
 462         char    str_governor[16];
 463         struct cpufreq_policy new_policy;
 464
 465         ret = cpufreq_get_policy(&new_policy, policy->cpu);
 466         if (ret)
 467                 return ret;
 468
 469         ret = sscanf(buf, "%15s", str_governor);
 470         if (ret != 1)
 471                 return -EINVAL;
 472
 473         if (cpufreq_parse_governor(str_governor, &new_policy.policy,
 474                                                 &new_policy.governor))
 475                 return -EINVAL;
 476
 477         /* Do not use cpufreq_set_policy here or the user_policy.max
 478            will be wrongly overridden */
 479         ret = __cpufreq_set_policy(policy, &new_policy);
 480
 481         policy->user_policy.policy = policy->policy;
 482         policy->user_policy.governor = policy->governor;
 483
 484         if (ret)
 485                 return ret;
 486         else
 487                 return count;
 488 }
 489
 490 /**
 491  * show_scaling_driver - show the cpufreq driver currently loaded
 492  */
 493 static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
 494 {
 495         return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
 496 }
 497
 498 /**
 499  * show_scaling_available_governors - show the available CPUfreq governors
 500  */
 501 static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
 502                                                 char *buf)
 503 {
 504         ssize_t i = 0;
 505         struct cpufreq_governor *t;
 506
 507         if (!cpufreq_driver->target) {
 508                 i += sprintf(buf, "performance powersave");
 509                 goto out;
 510         }
 511
 512         list_for_each_entry(t, &cpufreq_governor_list, governor_list) {
 513                 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
 514                     - (CPUFREQ_NAME_LEN + 2)))
 515                         goto out;
 516                 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
 517         }
 518 out:
 519         i += sprintf(&buf[i], "\n");
 520         return i;
 521 }
 522
 523 static ssize_t show_cpus(const struct cpumask *mask, char *buf)
 524 {
 525         ssize_t i = 0;
 526         unsigned int cpu;
 527
 528         for_each_cpu(cpu, mask) {
 529                 if (i)
 530                         i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
 531                 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
 532                 if (i >= (PAGE_SIZE - 5))
 533                         break;
 534         }
 535         i += sprintf(&buf[i], "\n");
 536         return i;
 537 }
 538
 539 /**
 540  * show_related_cpus - show the CPUs affected by each transition even if
 541  * hw coordination is in use
 542  */
 543 static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
 544 {
 545         if (cpumask_empty(policy->related_cpus))
 546                 return show_cpus(policy->cpus, buf);
 547         return show_cpus(policy->related_cpus, buf);
 548 }
 549
 550 /**
 551  * show_affected_cpus - show the CPUs affected by each transition
 552  */
 553 static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
 554 {
 555         return show_cpus(policy->cpus, buf);
 556 }
 557
 558 static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
 559                                         const char *buf, size_t count)
 560 {
 561         unsigned int freq = 0;
 562         unsigned int ret;
 563
 564         if (!policy->governor || !policy->governor->store_setspeed)
 565                 return -EINVAL;
 566
 567         ret = sscanf(buf, "%u", &freq);
 568         if (ret != 1)
 569                 return -EINVAL;
 570
 571         policy->governor->store_setspeed(policy, freq);
 572
 573         return count;
 574 }
 575
 576 static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
 577 {
 578         if (!policy->governor || !policy->governor->show_setspeed)
 579                 return sprintf(buf, "<unsupported>\n");
 580
 581         return policy->governor->show_setspeed(policy, buf);
 582 }
 583
 584 /**
 585  * show_bios_limit - show the current cpufreq HW/BIOS limitation
 586  */
 587 static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
 588 {
 589         unsigned int limit;
 590         int ret;
 591         if (cpufreq_driver->bios_limit) {
 592                 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
 593                 if (!ret)
 594                         return sprintf(buf, "%u\n", limit);
 595         }
 596         return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
 597 }
 598
 599 cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
 600 cpufreq_freq_attr_ro(cpuinfo_min_freq);
 601 cpufreq_freq_attr_ro(cpuinfo_max_freq);
 602 cpufreq_freq_attr_ro(cpuinfo_transition_latency);
 603 cpufreq_freq_attr_ro(scaling_available_governors);
 604 cpufreq_freq_attr_ro(scaling_driver);
 605 cpufreq_freq_attr_ro(scaling_cur_freq);
 606 cpufreq_freq_attr_ro(bios_limit);
 607 cpufreq_freq_attr_ro(related_cpus);
 608 cpufreq_freq_attr_ro(affected_cpus);
 609 cpufreq_freq_attr_rw(scaling_min_freq);
 610 cpufreq_freq_attr_rw(scaling_max_freq);
 611 cpufreq_freq_attr_rw(scaling_governor);
 612 cpufreq_freq_attr_rw(scaling_setspeed);
 613
 614 static struct attribute *default_attrs[] = {
 615         &cpuinfo_min_freq.attr,
 616         &cpuinfo_max_freq.attr,
 617         &cpuinfo_transition_latency.attr,
 618         &scaling_min_freq.attr,
 619         &scaling_max_freq.attr,
 620         &affected_cpus.attr,
 621         &related_cpus.attr,
 622         &scaling_governor.attr,
 623         &scaling_driver.attr,
 624         &scaling_available_governors.attr,
 625         &scaling_setspeed.attr,
 626         NULL
 627 };
 628
 629 struct kobject *cpufreq_global_kobject;
 630 EXPORT_SYMBOL(cpufreq_global_kobject);
 631
 632 #define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
 633 #define to_attr(a) container_of(a, struct freq_attr, attr)
 634
 635 static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
 636 {
 637         struct cpufreq_policy *policy = to_policy(kobj);
 638         struct freq_attr *fattr = to_attr(attr);
 639         ssize_t ret = -EINVAL;
 640         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 641         if (!policy)
 642                 goto no_policy;
 643
 644         if (lock_policy_rwsem_read(policy->cpu) < 0)
 645                 goto fail;
 646
 647         if (fattr->show)
 648                 ret = fattr->show(policy, buf);
 649         else
 650                 ret = -EIO;
 651
 652         unlock_policy_rwsem_read(policy->cpu);
 653 fail:
 654         cpufreq_cpu_put_sysfs(policy);
 655 no_policy:
 656         return ret;
 657 }
 658
 659 static ssize_t store(struct kobject *kobj, struct attribute *attr,
 660                      const char *buf, size_t count)
 661 {
 662         struct cpufreq_policy *policy = to_policy(kobj);
 663         struct freq_attr *fattr = to_attr(attr);
 664         ssize_t ret = -EINVAL;
 665         policy = cpufreq_cpu_get_sysfs(policy->cpu);
 666         if (!policy)
 667                 goto no_policy;
 668
 669         if (lock_policy_rwsem_write(policy->cpu) < 0)
 670                 goto fail;
 671
 672         if (fattr->store)
 673                 ret = fattr->store(policy, buf, count);
 674         else
 675                 ret = -EIO;
 676
 677         unlock_policy_rwsem_write(policy->cpu);
 678 fail:
 679         cpufreq_cpu_put_sysfs(policy);
 680 no_policy:
 681         return ret;
 682 }
 683
 684 static void cpufreq_sysfs_release(struct kobject *kobj)
 685 {
 686         struct cpufreq_policy *policy = to_policy(kobj);
 687         pr_debug("last reference is dropped\n");
 688         complete(&policy->kobj_unregister);
 689 }
 690
 691 static const struct sysfs_ops sysfs_ops = {
 692         .show   = show,
 693         .store  = store,
 694 };
 695
 696 static struct kobj_type ktype_cpufreq = {
 697         .sysfs_ops      = &sysfs_ops,
 698         .default_attrs  = default_attrs,
 699         .release        = cpufreq_sysfs_release,
 700 };
 701
 702 /*
 703  * Returns:
 704  *   Negative: Failure
 705  *   0:        Success
 706  *   Positive: When we have a managed CPU and the sysfs got symlinked
 707  */
 708 static int cpufreq_add_dev_policy(unsigned int cpu,
 709                                   struct cpufreq_policy *policy,
 710                                   struct device *dev)
 711 {
 712         int ret = 0;
 713 #ifdef CONFIG_SMP
 714         unsigned long flags;
 715         unsigned int j;
 716 #ifdef CONFIG_HOTPLUG_CPU
 717         struct cpufreq_governor *gov;
 718
 719         gov = __find_governor(per_cpu(cpufreq_cpu_governor, cpu));
 720         if (gov) {
 721                 policy->governor = gov;
 722                 pr_debug("Restoring governor %s for cpu %d\n",
 723                        policy->governor->name, cpu);
 724         }
 725 #endif
 726
 727         for_each_cpu(j, policy->cpus) {
 728                 struct cpufreq_policy *managed_policy;
 729
 730                 if (cpu == j)
 731                         continue;
 732
 733                 /* Check for existing affected CPUs.
 734                  * They may not be aware of it due to CPU Hotplug.
 735                  * cpufreq_cpu_put is called when the device is removed
 736                  * in __cpufreq_remove_dev()
 737                  */
 738                 managed_policy = cpufreq_cpu_get(j);
 739                 if (unlikely(managed_policy)) {
 740
 741                         /* Set proper policy_cpu */
 742                         unlock_policy_rwsem_write(cpu);
 743                         per_cpu(cpufreq_policy_cpu, cpu) = managed_policy->cpu;
 744
 745                         if (lock_policy_rwsem_write(cpu) < 0) {
 746                                 /* Should not go through policy unlock path */
 747                                 if (cpufreq_driver->exit)
 748                                         cpufreq_driver->exit(policy);
 749                                 cpufreq_cpu_put(managed_policy);
 750                                 return -EBUSY;
 751                         }
 752
 753                         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 754                         cpumask_copy(managed_policy->cpus, policy->cpus);
 755                         per_cpu(cpufreq_cpu_data, cpu) = managed_policy;
 756                         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 757
 758                         pr_debug("CPU already managed, adding link\n");
 759                         ret = sysfs_create_link(&dev->kobj,
 760                                                 &managed_policy->kobj,
 761                                                 "cpufreq");
 762                         if (ret)
 763                                 cpufreq_cpu_put(managed_policy);
 764                         /*
 765                          * Success. We only needed to be added to the mask.
 766                          * Call driver->exit() because only the cpu parent of
 767                          * the kobj needed to call init().
 768                          */
 769                         if (cpufreq_driver->exit)
 770                                 cpufreq_driver->exit(policy);
 771
 772                         if (!ret)
 773                                 return 1;
 774                         else
 775                                 return ret;
 776                 }
 777         }
 778 #endif
 779         return ret;
 780 }
 781
 782
 783 /* symlink affected CPUs */
 784 static int cpufreq_add_dev_symlink(unsigned int cpu,
 785                                    struct cpufreq_policy *policy)
 786 {
 787         unsigned int j;
 788         int ret = 0;
 789
 790         for_each_cpu(j, policy->cpus) {
 791                 struct cpufreq_policy *managed_policy;
 792                 struct device *cpu_dev;
 793
 794                 if (j == cpu)
 795                         continue;
 796                 if (!cpu_online(j))
 797                         continue;
 798
 799                 pr_debug("CPU %u already managed, adding link\n", j);
 800                 managed_policy = cpufreq_cpu_get(cpu);
 801                 cpu_dev = get_cpu_device(j);
 802                 ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj,
 803                                         "cpufreq");
 804                 if (ret) {
 805                         cpufreq_cpu_put(managed_policy);
 806                         return ret;
 807                 }
 808         }
 809         return ret;
 810 }
 811
 812 static int cpufreq_add_dev_interface(unsigned int cpu,
 813                                      struct cpufreq_policy *policy,
 814                                      struct device *dev)
 815 {
 816         struct cpufreq_policy new_policy;
 817         struct freq_attr **drv_attr;
 818         unsigned long flags;
 819         int ret = 0;
 820         unsigned int j;
 821
 822         /* prepare interface data */
 823         ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
 824                                    &dev->kobj, "cpufreq");
 825         if (ret)
 826                 return ret;
 827
 828         /* set up files for this cpu device */
 829         drv_attr = cpufreq_driver->attr;
 830         while ((drv_attr) && (*drv_attr)) {
 831                 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
 832                 if (ret)
 833                         goto err_out_kobj_put;
 834                 drv_attr++;
 835         }
 836         if (cpufreq_driver->get) {
 837                 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
 838                 if (ret)
 839                         goto err_out_kobj_put;
 840         }
 841         if (cpufreq_driver->target) {
 842                 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
 843                 if (ret)
 844                         goto err_out_kobj_put;
 845         }
 846         if (cpufreq_driver->bios_limit) {
 847                 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
 848                 if (ret)
 849                         goto err_out_kobj_put;
 850         }
 851
 852         spin_lock_irqsave(&cpufreq_driver_lock, flags);
 853         for_each_cpu(j, policy->cpus) {
 854                 if (!cpu_online(j))
 855                         continue;
 856                 per_cpu(cpufreq_cpu_data, j) = policy;
 857                 per_cpu(cpufreq_policy_cpu, j) = policy->cpu;
 858         }
 859         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
 860
 861         ret = cpufreq_add_dev_symlink(cpu, policy);
 862         if (ret)
 863                 goto err_out_kobj_put;
 864
 865         memcpy(&new_policy, policy, sizeof(struct cpufreq_policy));
 866         /* assure that the starting sequence is run in __cpufreq_set_policy */
 867         policy->governor = NULL;
 868
 869         /* set default policy */
 870         ret = __cpufreq_set_policy(policy, &new_policy);
 871         policy->user_policy.policy = policy->policy;
 872         policy->user_policy.governor = policy->governor;
 873
 874         if (ret) {
 875                 pr_debug("setting policy failed\n");
 876                 if (cpufreq_driver->exit)
 877                         cpufreq_driver->exit(policy);
 878         }
 879         return ret;
 880
 881 err_out_kobj_put:
 882         kobject_put(&policy->kobj);
 883         wait_for_completion(&policy->kobj_unregister);
 884         return ret;
 885 }
 886
 887
 888 /**
 889  * cpufreq_add_dev - add a CPU device
 890  *
 891  * Adds the cpufreq interface for a CPU device.
 892  *
 893  * The Oracle says: try running cpufreq registration/unregistration concurrently
 894  * with with cpu hotplugging and all hell will break loose. Tried to clean this
 895  * mess up, but more thorough testing is needed. - Mathieu
 896  */
 897 static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
 898 {
 899         unsigned int cpu = dev->id;
 900         int ret = 0, found = 0;
 901         struct cpufreq_policy *policy;
 902         unsigned long flags;
 903         unsigned int j;
 904 #ifdef CONFIG_HOTPLUG_CPU
 905         int sibling;
 906 #endif
 907
 908         if (cpu_is_offline(cpu))
 909                 return 0;
 910
 911         pr_debug("adding CPU %u\n", cpu);
 912
 913 #ifdef CONFIG_SMP
 914         /* check whether a different CPU already registered this
 915          * CPU because it is in the same boat. */
 916         policy = cpufreq_cpu_get(cpu);
 917         if (unlikely(policy)) {
 918                 cpufreq_cpu_put(policy);
 919                 return 0;
 920         }
 921 #endif
 922
 923         if (!try_module_get(cpufreq_driver->owner)) {
 924                 ret = -EINVAL;
 925                 goto module_out;
 926         }
 927
 928         ret = -ENOMEM;
 929         policy = kzalloc(sizeof(struct cpufreq_policy), GFP_KERNEL);
 930         if (!policy)
 931                 goto nomem_out;
 932
 933         if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
 934                 goto err_free_policy;
 935
 936         if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
 937                 goto err_free_cpumask;
 938
 939         policy->cpu = cpu;
 940         cpumask_copy(policy->cpus, cpumask_of(cpu));
 941
 942         /* Initially set CPU itself as the policy_cpu */
 943         per_cpu(cpufreq_policy_cpu, cpu) = cpu;
 944         ret = (lock_policy_rwsem_write(cpu) < 0);
 945         WARN_ON(ret);
 946
 947         init_completion(&policy->kobj_unregister);
 948         INIT_WORK(&policy->update, handle_update);
 949
 950         /* Set governor before ->init, so that driver could check it */
 951 #ifdef CONFIG_HOTPLUG_CPU
 952         for_each_online_cpu(sibling) {
 953                 struct cpufreq_policy *cp = per_cpu(cpufreq_cpu_data, sibling);
 954                 if (cp && cp->governor &&
 955                     (cpumask_test_cpu(cpu, cp->related_cpus))) {
 956                         policy->governor = cp->governor;
 957                         found = 1;
 958                         break;
 959                 }
 960         }
 961 #endif
 962         if (!found)
 963                 policy->governor = CPUFREQ_DEFAULT_GOVERNOR;
 964         /* call driver. From then on the cpufreq must be able
 965          * to accept all calls to ->verify and ->setpolicy for this CPU
 966          */
 967         ret = cpufreq_driver->init(policy);
 968         if (ret) {
 969                 pr_debug("initialization failed\n");
 970                 goto err_unlock_policy;
 971         }
 972         policy->user_policy.min = policy->min;
 973         policy->user_policy.max = policy->max;
 974
 975         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
 976                                      CPUFREQ_START, policy);
 977
 978         ret = cpufreq_add_dev_policy(cpu, policy, dev);
 979         if (ret) {
 980                 if (ret > 0)
 981                         /* This is a managed cpu, symlink created,
 982                            exit with 0 */
 983                         ret = 0;
 984                 goto err_unlock_policy;
 985         }
 986
 987         ret = cpufreq_add_dev_interface(cpu, policy, dev);
 988         if (ret)
 989                 goto err_out_unregister;
 990
 991         unlock_policy_rwsem_write(cpu);
 992
 993         kobject_uevent(&policy->kobj, KOBJ_ADD);
 994         module_put(cpufreq_driver->owner);
 995         pr_debug("initialization complete\n");
 996
 997         return 0;
 998
 999
1000 err_out_unregister:
1001         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1002         for_each_cpu(j, policy->cpus)
1003                 per_cpu(cpufreq_cpu_data, j) = NULL;
1004         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1005
1006         kobject_put(&policy->kobj);
1007         wait_for_completion(&policy->kobj_unregister);
1008
1009 err_unlock_policy:
1010         unlock_policy_rwsem_write(cpu);
1011         free_cpumask_var(policy->related_cpus);
1012 err_free_cpumask:
1013         free_cpumask_var(policy->cpus);
1014 err_free_policy:
1015         kfree(policy);
1016 nomem_out:
1017         module_put(cpufreq_driver->owner);
1018 module_out:
1019         return ret;
1020 }
1021
1022
1023 /**
1024  * __cpufreq_remove_dev - remove a CPU device
1025  *
1026  * Removes the cpufreq interface for a CPU device.
1027  * Caller should already have policy_rwsem in write mode for this CPU.
1028  * This routine frees the rwsem before returning.
1029  */
1030 static int __cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1031 {
1032         unsigned int cpu = dev->id;
1033         unsigned long flags;
1034         struct cpufreq_policy *data;
1035         struct kobject *kobj;
1036         struct completion *cmp;
1037 #ifdef CONFIG_SMP
1038         struct device *cpu_dev;
1039         unsigned int j;
1040 #endif
1041
1042         pr_debug("unregistering CPU %u\n", cpu);
1043
1044         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1045         data = per_cpu(cpufreq_cpu_data, cpu);
1046
1047         if (!data) {
1048                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1049                 unlock_policy_rwsem_write(cpu);
1050                 return -EINVAL;
1051         }
1052         per_cpu(cpufreq_cpu_data, cpu) = NULL;
1053
1054
1055 #ifdef CONFIG_SMP
1056         /* if this isn't the CPU which is the parent of the kobj, we
1057          * only need to unlink, put and exit
1058          */
1059         if (unlikely(cpu != data->cpu)) {
1060                 pr_debug("removing link\n");
1061                 cpumask_clear_cpu(cpu, data->cpus);
1062                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1063                 kobj = &dev->kobj;
1064                 cpufreq_cpu_put(data);
1065                 unlock_policy_rwsem_write(cpu);
1066                 sysfs_remove_link(kobj, "cpufreq");
1067                 return 0;
1068         }
1069 #endif
1070
1071 #ifdef CONFIG_SMP
1072
1073 #ifdef CONFIG_HOTPLUG_CPU
1074         strncpy(per_cpu(cpufreq_cpu_governor, cpu), data->governor->name,
1075                         CPUFREQ_NAME_LEN);
1076 #endif
1077
1078         /* if we have other CPUs still registered, we need to unlink them,
1079          * or else wait_for_completion below will lock up. Clean the
1080          * per_cpu(cpufreq_cpu_data) while holding the lock, and remove
1081          * the sysfs links afterwards.
1082          */
1083         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1084                 for_each_cpu(j, data->cpus) {
1085                         if (j == cpu)
1086                                 continue;
1087                         per_cpu(cpufreq_cpu_data, j) = NULL;
1088                 }
1089         }
1090
1091         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1092
1093         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1094                 for_each_cpu(j, data->cpus) {
1095                         if (j == cpu)
1096                                 continue;
1097                         pr_debug("removing link for cpu %u\n", j);
1098 #ifdef CONFIG_HOTPLUG_CPU
1099                         strncpy(per_cpu(cpufreq_cpu_governor, j),
1100                                 data->governor->name, CPUFREQ_NAME_LEN);
1101 #endif
1102                         cpu_dev = get_cpu_device(j);
1103                         kobj = &cpu_dev->kobj;
1104                         unlock_policy_rwsem_write(cpu);
1105                         sysfs_remove_link(kobj, "cpufreq");
1106                         lock_policy_rwsem_write(cpu);
1107                         cpufreq_cpu_put(data);
1108                 }
1109         }
1110 #else
1111         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1112 #endif
1113
1114         if (cpufreq_driver->target)
1115                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1116
1117         kobj = &data->kobj;
1118         cmp = &data->kobj_unregister;
1119         unlock_policy_rwsem_write(cpu);
1120         kobject_put(kobj);
1121
1122         /* we need to make sure that the underlying kobj is actually
1123          * not referenced anymore by anybody before we proceed with
1124          * unloading.
1125          */
1126         pr_debug("waiting for dropping of refcount\n");
1127         wait_for_completion(cmp);
1128         pr_debug("wait complete\n");
1129
1130         lock_policy_rwsem_write(cpu);
1131         if (cpufreq_driver->exit)
1132                 cpufreq_driver->exit(data);
1133         unlock_policy_rwsem_write(cpu);
1134
1135 #ifdef CONFIG_HOTPLUG_CPU
1136         /* when the CPU which is the parent of the kobj is hotplugged
1137          * offline, check for siblings, and create cpufreq sysfs interface
1138          * and symlinks
1139          */
1140         if (unlikely(cpumask_weight(data->cpus) > 1)) {
1141                 /* first sibling now owns the new sysfs dir */
1142                 cpumask_clear_cpu(cpu, data->cpus);
1143                 cpufreq_add_dev(get_cpu_device(cpumask_first(data->cpus)), NULL);
1144
1145                 /* finally remove our own symlink */
1146                 lock_policy_rwsem_write(cpu);
1147                 __cpufreq_remove_dev(dev, sif);
1148         }
1149 #endif
1150
1151         free_cpumask_var(data->related_cpus);
1152         free_cpumask_var(data->cpus);
1153         kfree(data);
1154
1155         return 0;
1156 }
1157
1158
1159 static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
1160 {
1161         unsigned int cpu = dev->id;
1162         int retval;
1163
1164         if (cpu_is_offline(cpu))
1165                 return 0;
1166
1167         if (unlikely(lock_policy_rwsem_write(cpu)))
1168                 BUG();
1169
1170         retval = __cpufreq_remove_dev(dev, sif);
1171         return retval;
1172 }
1173
1174
1175 static void handle_update(struct work_struct *work)
1176 {
1177         struct cpufreq_policy *policy =
1178                 container_of(work, struct cpufreq_policy, update);
1179         unsigned int cpu = policy->cpu;
1180         pr_debug("handle_update for cpu %u called\n", cpu);
1181         cpufreq_update_policy(cpu);
1182 }
1183
1184 /**
1185  *      cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're in deep trouble.
1186  *      @cpu: cpu number
1187  *      @old_freq: CPU frequency the kernel thinks the CPU runs at
1188  *      @new_freq: CPU frequency the CPU actually runs at
1189  *
1190  *      We adjust to current frequency first, and need to clean up later.
1191  *      So either call to cpufreq_update_policy() or schedule handle_update()).
1192  */
1193 static void cpufreq_out_of_sync(unsigned int cpu, unsigned int old_freq,
1194                                 unsigned int new_freq)
1195 {
1196         struct cpufreq_freqs freqs;
1197
1198         pr_debug("Warning: CPU frequency out of sync: cpufreq and timing "
1199                "core thinks of %u, is %u kHz.\n", old_freq, new_freq);
1200
1201         freqs.cpu = cpu;
1202         freqs.old = old_freq;
1203         freqs.new = new_freq;
1204         cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE);
1205         cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE);
1206 }
1207
1208
1209 /**
1210  * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
1211  * @cpu: CPU number
1212  *
1213  * This is the last known freq, without actually getting it from the driver.
1214  * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1215  */
1216 unsigned int cpufreq_quick_get(unsigned int cpu)
1217 {
1218         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1219         unsigned int ret_freq = 0;
1220
1221         if (policy) {
1222                 ret_freq = policy->cur;
1223                 cpufreq_cpu_put(policy);
1224         }
1225
1226         return ret_freq;
1227 }
1228 EXPORT_SYMBOL(cpufreq_quick_get);
1229
1230 /**
1231  * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1232  * @cpu: CPU number
1233  *
1234  * Just return the max possible frequency for a given CPU.
1235  */
1236 unsigned int cpufreq_quick_get_max(unsigned int cpu)
1237 {
1238         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1239         unsigned int ret_freq = 0;
1240
1241         if (policy) {
1242                 ret_freq = policy->max;
1243                 cpufreq_cpu_put(policy);
1244         }
1245
1246         return ret_freq;
1247 }
1248 EXPORT_SYMBOL(cpufreq_quick_get_max);
1249
1250
1251 static unsigned int __cpufreq_get(unsigned int cpu)
1252 {
1253         struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
1254         unsigned int ret_freq = 0;
1255
1256         if (!cpufreq_driver->get)
1257                 return ret_freq;
1258
1259         ret_freq = cpufreq_driver->get(cpu);
1260
1261         if (ret_freq && policy->cur &&
1262                 !(cpufreq_driver->flags & CPUFREQ_CONST_LOOPS)) {
1263                 /* verify no discrepancy between actual and
1264                                         saved value exists */
1265                 if (unlikely(ret_freq != policy->cur)) {
1266                         cpufreq_out_of_sync(cpu, policy->cur, ret_freq);
1267                         schedule_work(&policy->update);
1268                 }
1269         }
1270
1271         return ret_freq;
1272 }
1273
1274 /**
1275  * cpufreq_get - get the current CPU frequency (in kHz)
1276  * @cpu: CPU number
1277  *
1278  * Get the CPU current (static) CPU frequency
1279  */
1280 unsigned int cpufreq_get(unsigned int cpu)
1281 {
1282         unsigned int ret_freq = 0;
1283         struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1284
1285         if (!policy)
1286                 goto out;
1287
1288         if (unlikely(lock_policy_rwsem_read(cpu)))
1289                 goto out_policy;
1290
1291         ret_freq = __cpufreq_get(cpu);
1292
1293         unlock_policy_rwsem_read(cpu);
1294
1295 out_policy:
1296         cpufreq_cpu_put(policy);
1297 out:
1298         return ret_freq;
1299 }
1300 EXPORT_SYMBOL(cpufreq_get);
1301
1302 static struct subsys_interface cpufreq_interface = {
1303         .name           = "cpufreq",
1304         .subsys         = &cpu_subsys,
1305         .add_dev        = cpufreq_add_dev,
1306         .remove_dev     = cpufreq_remove_dev,
1307 };
1308
1309
1310 /**
1311  * cpufreq_bp_suspend - Prepare the boot CPU for system suspend.
1312  *
1313  * This function is only executed for the boot processor.  The other CPUs
1314  * have been put offline by means of CPU hotplug.
1315  */
1316 static int cpufreq_bp_suspend(void)
1317 {
1318         int ret = 0;
1319
1320         int cpu = smp_processor_id();
1321         struct cpufreq_policy *cpu_policy;
1322
1323         pr_debug("suspending cpu %u\n", cpu);
1324
1325         /* If there's no policy for the boot CPU, we have nothing to do. */
1326         cpu_policy = cpufreq_cpu_get(cpu);
1327         if (!cpu_policy)
1328                 return 0;
1329
1330         if (cpufreq_driver->suspend) {
1331                 ret = cpufreq_driver->suspend(cpu_policy);
1332                 if (ret)
1333                         printk(KERN_ERR "cpufreq: suspend failed in ->suspend "
1334                                         "step on CPU %u\n", cpu_policy->cpu);
1335         }
1336
1337         cpufreq_cpu_put(cpu_policy);
1338         return ret;
1339 }
1340
1341 /**
1342  * cpufreq_bp_resume - Restore proper frequency handling of the boot CPU.
1343  *
1344  *      1.) resume CPUfreq hardware support (cpufreq_driver->resume())
1345  *      2.) schedule call cpufreq_update_policy() ASAP as interrupts are
1346  *          restored. It will verify that the current freq is in sync with
1347  *          what we believe it to be. This is a bit later than when it
1348  *          should be, but nonethteless it's better than calling
1349  *          cpufreq_driver->get() here which might re-enable interrupts...
1350  *
1351  * This function is only executed for the boot CPU.  The other CPUs have not
1352  * been turned on yet.
1353  */
1354 static void cpufreq_bp_resume(void)
1355 {
1356         int ret = 0;
1357
1358         int cpu = smp_processor_id();
1359         struct cpufreq_policy *cpu_policy;
1360
1361         pr_debug("resuming cpu %u\n", cpu);
1362
1363         /* If there's no policy for the boot CPU, we have nothing to do. */
1364         cpu_policy = cpufreq_cpu_get(cpu);
1365         if (!cpu_policy)
1366                 return;
1367
1368         if (cpufreq_driver->resume) {
1369                 ret = cpufreq_driver->resume(cpu_policy);
1370                 if (ret) {
1371                         printk(KERN_ERR "cpufreq: resume failed in ->resume "
1372                                         "step on CPU %u\n", cpu_policy->cpu);
1373                         goto fail;
1374                 }
1375         }
1376
1377         schedule_work(&cpu_policy->update);
1378
1379 fail:
1380         cpufreq_cpu_put(cpu_policy);
1381 }
1382
1383 static struct syscore_ops cpufreq_syscore_ops = {
1384         .suspend        = cpufreq_bp_suspend,
1385         .resume         = cpufreq_bp_resume,
1386 };
1387
1388
1389 /*********************************************************************
1390  *                     NOTIFIER LISTS INTERFACE                      *
1391  *********************************************************************/
1392
1393 /**
1394  *      cpufreq_register_notifier - register a driver with cpufreq
1395  *      @nb: notifier function to register
1396  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1397  *
1398  *      Add a driver to one of two lists: either a list of drivers that
1399  *      are notified about clock rate changes (once before and once after
1400  *      the transition), or a list of drivers that are notified about
1401  *      changes in cpufreq policy.
1402  *
1403  *      This function may sleep, and has the same return conditions as
1404  *      blocking_notifier_chain_register.
1405  */
1406 int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1407 {
1408         int ret;
1409
1410         WARN_ON(!init_cpufreq_transition_notifier_list_called);
1411
1412         switch (list) {
1413         case CPUFREQ_TRANSITION_NOTIFIER:
1414                 ret = srcu_notifier_chain_register(
1415                                 &cpufreq_transition_notifier_list, nb);
1416                 break;
1417         case CPUFREQ_POLICY_NOTIFIER:
1418                 ret = blocking_notifier_chain_register(
1419                                 &cpufreq_policy_notifier_list, nb);
1420                 break;
1421         default:
1422                 ret = -EINVAL;
1423         }
1424
1425         return ret;
1426 }
1427 EXPORT_SYMBOL(cpufreq_register_notifier);
1428
1429
1430 /**
1431  *      cpufreq_unregister_notifier - unregister a driver with cpufreq
1432  *      @nb: notifier block to be unregistered
1433  *      @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1434  *
1435  *      Remove a driver from the CPU frequency notifier list.
1436  *
1437  *      This function may sleep, and has the same return conditions as
1438  *      blocking_notifier_chain_unregister.
1439  */
1440 int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1441 {
1442         int ret;
1443
1444         switch (list) {
1445         case CPUFREQ_TRANSITION_NOTIFIER:
1446                 ret = srcu_notifier_chain_unregister(
1447                                 &cpufreq_transition_notifier_list, nb);
1448                 break;
1449         case CPUFREQ_POLICY_NOTIFIER:
1450                 ret = blocking_notifier_chain_unregister(
1451                                 &cpufreq_policy_notifier_list, nb);
1452                 break;
1453         default:
1454                 ret = -EINVAL;
1455         }
1456
1457         return ret;
1458 }
1459 EXPORT_SYMBOL(cpufreq_unregister_notifier);
1460
1461
1462 /*********************************************************************
1463  *                              GOVERNORS                            *
1464  *********************************************************************/
1465
1466
1467 int __cpufreq_driver_target(struct cpufreq_policy *policy,
1468                             unsigned int target_freq,
1469                             unsigned int relation)
1470 {
1471         int retval = -EINVAL;
1472         unsigned int old_target_freq = target_freq;
1473
1474         if (cpufreq_disabled())
1475                 return -ENODEV;
1476
1477         /* Make sure that target_freq is within supported range */
1478         if (target_freq > policy->max)
1479                 target_freq = policy->max;
1480         if (target_freq < policy->min)
1481                 target_freq = policy->min;
1482
1483         pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
1484                         policy->cpu, target_freq, relation, old_target_freq);
1485
1486         if (target_freq == policy->cur)
1487                 return 0;
1488
1489         if (cpu_online(policy->cpu) && cpufreq_driver->target)
1490                 retval = cpufreq_driver->target(policy, target_freq, relation);
1491
1492         return retval;
1493 }
1494 EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
1495
1496 int cpufreq_driver_target(struct cpufreq_policy *policy,
1497                           unsigned int target_freq,
1498                           unsigned int relation)
1499 {
1500         int ret = -EINVAL;
1501
1502         policy = cpufreq_cpu_get(policy->cpu);
1503         if (!policy)
1504                 goto no_policy;
1505
1506         if (unlikely(lock_policy_rwsem_write(policy->cpu)))
1507                 goto fail;
1508
1509         ret = __cpufreq_driver_target(policy, target_freq, relation);
1510
1511         unlock_policy_rwsem_write(policy->cpu);
1512
1513 fail:
1514         cpufreq_cpu_put(policy);
1515 no_policy:
1516         return ret;
1517 }
1518 EXPORT_SYMBOL_GPL(cpufreq_driver_target);
1519
1520 int __cpufreq_driver_getavg(struct cpufreq_policy *policy, unsigned int cpu)
1521 {
1522         int ret = 0;
1523
1524         if (!(cpu_online(cpu) && cpufreq_driver->getavg))
1525                 return 0;
1526
1527         policy = cpufreq_cpu_get(policy->cpu);
1528         if (!policy)
1529                 return -EINVAL;
1530
1531         ret = cpufreq_driver->getavg(policy, cpu);
1532
1533         cpufreq_cpu_put(policy);
1534         return ret;
1535 }
1536 EXPORT_SYMBOL_GPL(__cpufreq_driver_getavg);
1537
1538 /*
1539  * when "event" is CPUFREQ_GOV_LIMITS
1540  */
1541
1542 static int __cpufreq_governor(struct cpufreq_policy *policy,
1543                                         unsigned int event)
1544 {
1545         int ret;
1546
1547         /* Only must be defined when default governor is known to have latency
1548            restrictions, like e.g. conservative or ondemand.
1549            That this is the case is already ensured in Kconfig
1550         */
1551 #ifdef CONFIG_CPU_FREQ_GOV_PERFORMANCE
1552         struct cpufreq_governor *gov = &cpufreq_gov_performance;
1553 #else
1554         struct cpufreq_governor *gov = NULL;
1555 #endif
1556
1557         if (policy->governor->max_transition_latency &&
1558             policy->cpuinfo.transition_latency >
1559             policy->governor->max_transition_latency) {
1560                 if (!gov)
1561                         return -EINVAL;
1562                 else {
1563                         printk(KERN_WARNING "%s governor failed, too long"
1564                                " transition latency of HW, fallback"
1565                                " to %s governor\n",
1566                                policy->governor->name,
1567                                gov->name);
1568                         policy->governor = gov;
1569                 }
1570         }
1571
1572         if (!try_module_get(policy->governor->owner))
1573                 return -EINVAL;
1574
1575         pr_debug("__cpufreq_governor for CPU %u, event %u\n",
1576                                                 policy->cpu, event);
1577         ret = policy->governor->governor(policy, event);
1578
1579         /* we keep one module reference alive for
1580                         each CPU governed by this CPU */
1581         if ((event != CPUFREQ_GOV_START) || ret)
1582                 module_put(policy->governor->owner);
1583         if ((event == CPUFREQ_GOV_STOP) && !ret)
1584                 module_put(policy->governor->owner);
1585
1586         return ret;
1587 }
1588
1589
1590 int cpufreq_register_governor(struct cpufreq_governor *governor)
1591 {
1592         int err;
1593
1594         if (!governor)
1595                 return -EINVAL;
1596
1597         if (cpufreq_disabled())
1598                 return -ENODEV;
1599
1600         mutex_lock(&cpufreq_governor_mutex);
1601
1602         err = -EBUSY;
1603         if (__find_governor(governor->name) == NULL) {
1604                 err = 0;
1605                 list_add(&governor->governor_list, &cpufreq_governor_list);
1606         }
1607
1608         mutex_unlock(&cpufreq_governor_mutex);
1609         return err;
1610 }
1611 EXPORT_SYMBOL_GPL(cpufreq_register_governor);
1612
1613
1614 void cpufreq_unregister_governor(struct cpufreq_governor *governor)
1615 {
1616 #ifdef CONFIG_HOTPLUG_CPU
1617         int cpu;
1618 #endif
1619
1620         if (!governor)
1621                 return;
1622
1623         if (cpufreq_disabled())
1624                 return;
1625
1626 #ifdef CONFIG_HOTPLUG_CPU
1627         for_each_present_cpu(cpu) {
1628                 if (cpu_online(cpu))
1629                         continue;
1630                 if (!strcmp(per_cpu(cpufreq_cpu_governor, cpu), governor->name))
1631                         strcpy(per_cpu(cpufreq_cpu_governor, cpu), "\0");
1632         }
1633 #endif
1634
1635         mutex_lock(&cpufreq_governor_mutex);
1636         list_del(&governor->governor_list);
1637         mutex_unlock(&cpufreq_governor_mutex);
1638         return;
1639 }
1640 EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
1641
1642
1643
1644 /*********************************************************************
1645  *                          POLICY INTERFACE                         *
1646  *********************************************************************/
1647
1648 /**
1649  * cpufreq_get_policy - get the current cpufreq_policy
1650  * @policy: struct cpufreq_policy into which the current cpufreq_policy
1651  *      is written
1652  *
1653  * Reads the current cpufreq policy.
1654  */
1655 int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
1656 {
1657         struct cpufreq_policy *cpu_policy;
1658         if (!policy)
1659                 return -EINVAL;
1660
1661         cpu_policy = cpufreq_cpu_get(cpu);
1662         if (!cpu_policy)
1663                 return -EINVAL;
1664
1665         memcpy(policy, cpu_policy, sizeof(struct cpufreq_policy));
1666
1667         cpufreq_cpu_put(cpu_policy);
1668         return 0;
1669 }
1670 EXPORT_SYMBOL(cpufreq_get_policy);
1671
1672
1673 /*
1674  * data   : current policy.
1675  * policy : policy to be set.
1676  */
1677 static int __cpufreq_set_policy(struct cpufreq_policy *data,
1678                                 struct cpufreq_policy *policy)
1679 {
1680         int ret = 0;
1681
1682         pr_debug("setting new policy for CPU %u: %u - %u kHz\n", policy->cpu,
1683                 policy->min, policy->max);
1684
1685         memcpy(&policy->cpuinfo, &data->cpuinfo,
1686                                 sizeof(struct cpufreq_cpuinfo));
1687
1688         if (policy->min > data->max || policy->max < data->min) {
1689                 ret = -EINVAL;
1690                 goto error_out;
1691         }
1692
1693         /* verify the cpu speed can be set within this limit */
1694         ret = cpufreq_driver->verify(policy);
1695         if (ret)
1696                 goto error_out;
1697
1698         /* adjust if necessary - all reasons */
1699         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1700                         CPUFREQ_ADJUST, policy);
1701
1702         /* adjust if necessary - hardware incompatibility*/
1703         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1704                         CPUFREQ_INCOMPATIBLE, policy);
1705
1706         /* verify the cpu speed can be set within this limit,
1707            which might be different to the first one */
1708         ret = cpufreq_driver->verify(policy);
1709         if (ret)
1710                 goto error_out;
1711
1712         /* notification of the new policy */
1713         blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1714                         CPUFREQ_NOTIFY, policy);
1715
1716         data->min = policy->min;
1717         data->max = policy->max;
1718
1719         pr_debug("new min and max freqs are %u - %u kHz\n",
1720                                         data->min, data->max);
1721
1722         if (cpufreq_driver->setpolicy) {
1723                 data->policy = policy->policy;
1724                 pr_debug("setting range\n");
1725                 ret = cpufreq_driver->setpolicy(policy);
1726         } else {
1727                 if (policy->governor != data->governor) {
1728                         /* save old, working values */
1729                         struct cpufreq_governor *old_gov = data->governor;
1730
1731                         pr_debug("governor switch\n");
1732
1733                         /* end old governor */
1734                         if (data->governor)
1735                                 __cpufreq_governor(data, CPUFREQ_GOV_STOP);
1736
1737                         /* start new governor */
1738                         data->governor = policy->governor;
1739                         if (__cpufreq_governor(data, CPUFREQ_GOV_START)) {
1740                                 /* new governor failed, so re-start old one */
1741                                 pr_debug("starting governor %s failed\n",
1742                                                         data->governor->name);
1743                                 if (old_gov) {
1744                                         data->governor = old_gov;
1745                                         __cpufreq_governor(data,
1746                                                            CPUFREQ_GOV_START);
1747                                 }
1748                                 ret = -EINVAL;
1749                                 goto error_out;
1750                         }
1751                         /* might be a policy change, too, so fall through */
1752                 }
1753                 pr_debug("governor: change or update limits\n");
1754                 __cpufreq_governor(data, CPUFREQ_GOV_LIMITS);
1755         }
1756
1757 error_out:
1758         return ret;
1759 }
1760
1761 /**
1762  *      cpufreq_update_policy - re-evaluate an existing cpufreq policy
1763  *      @cpu: CPU which shall be re-evaluated
1764  *
1765  *      Useful for policy notifiers which have different necessities
1766  *      at different times.
1767  */
1768 int cpufreq_update_policy(unsigned int cpu)
1769 {
1770         struct cpufreq_policy *data = cpufreq_cpu_get(cpu);
1771         struct cpufreq_policy policy;
1772         int ret;
1773
1774         if (!data) {
1775                 ret = -ENODEV;
1776                 goto no_policy;
1777         }
1778
1779         if (unlikely(lock_policy_rwsem_write(cpu))) {
1780                 ret = -EINVAL;
1781                 goto fail;
1782         }
1783
1784         pr_debug("updating policy for CPU %u\n", cpu);
1785         memcpy(&policy, data, sizeof(struct cpufreq_policy));
1786         policy.min = data->user_policy.min;
1787         policy.max = data->user_policy.max;
1788         policy.policy = data->user_policy.policy;
1789         policy.governor = data->user_policy.governor;
1790
1791         /* BIOS might change freq behind our back
1792           -> ask driver for current freq and notify governors about a change */
1793         if (cpufreq_driver->get) {
1794                 policy.cur = cpufreq_driver->get(cpu);
1795                 if (!data->cur) {
1796                         pr_debug("Driver did not initialize current freq");
1797                         data->cur = policy.cur;
1798                 } else {
1799                         if (data->cur != policy.cur)
1800                                 cpufreq_out_of_sync(cpu, data->cur,
1801                                                                 policy.cur);
1802                 }
1803         }
1804
1805         ret = __cpufreq_set_policy(data, &policy);
1806
1807         unlock_policy_rwsem_write(cpu);
1808
1809 fail:
1810         cpufreq_cpu_put(data);
1811 no_policy:
1812         return ret;
1813 }
1814 EXPORT_SYMBOL(cpufreq_update_policy);
1815
1816 static int __cpuinit cpufreq_cpu_callback(struct notifier_block *nfb,
1817                                         unsigned long action, void *hcpu)
1818 {
1819         unsigned int cpu = (unsigned long)hcpu;
1820         struct device *dev;
1821
1822         dev = get_cpu_device(cpu);
1823         if (dev) {
1824                 switch (action) {
1825                 case CPU_ONLINE:
1826                 case CPU_ONLINE_FROZEN:
1827                         cpufreq_add_dev(dev, NULL);
1828                         break;
1829                 case CPU_DOWN_PREPARE:
1830                 case CPU_DOWN_PREPARE_FROZEN:
1831                         if (unlikely(lock_policy_rwsem_write(cpu)))
1832                                 BUG();
1833
1834                         __cpufreq_remove_dev(dev, NULL);
1835                         break;
1836                 case CPU_DOWN_FAILED:
1837                 case CPU_DOWN_FAILED_FROZEN:
1838                         cpufreq_add_dev(dev, NULL);
1839                         break;
1840                 }
1841         }
1842         return NOTIFY_OK;
1843 }
1844
1845 static struct notifier_block __refdata cpufreq_cpu_notifier = {
1846     .notifier_call = cpufreq_cpu_callback,
1847 };
1848
1849 /*********************************************************************
1850  *               REGISTER / UNREGISTER CPUFREQ DRIVER                *
1851  *********************************************************************/
1852
1853 /**
1854  * cpufreq_register_driver - register a CPU Frequency driver
1855  * @driver_data: A struct cpufreq_driver containing the values#
1856  * submitted by the CPU Frequency driver.
1857  *
1858  *   Registers a CPU Frequency driver to this core code. This code
1859  * returns zero on success, -EBUSY when another driver got here first
1860  * (and isn't unregistered in the meantime).
1861  *
1862  */
1863 int cpufreq_register_driver(struct cpufreq_driver *driver_data)
1864 {
1865         unsigned long flags;
1866         int ret;
1867
1868         if (cpufreq_disabled())
1869                 return -ENODEV;
1870
1871         if (!driver_data || !driver_data->verify || !driver_data->init ||
1872             ((!driver_data->setpolicy) && (!driver_data->target)))
1873                 return -EINVAL;
1874
1875         pr_debug("trying to register driver %s\n", driver_data->name);
1876
1877         if (driver_data->setpolicy)
1878                 driver_data->flags |= CPUFREQ_CONST_LOOPS;
1879
1880         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1881         if (cpufreq_driver) {
1882                 spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1883                 return -EBUSY;
1884         }
1885         cpufreq_driver = driver_data;
1886         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1887
1888         ret = subsys_interface_register(&cpufreq_interface);
1889         if (ret)
1890                 goto err_null_driver;
1891
1892         if (!(cpufreq_driver->flags & CPUFREQ_STICKY)) {
1893                 int i;
1894                 ret = -ENODEV;
1895
1896                 /* check for at least one working CPU */
1897                 for (i = 0; i < nr_cpu_ids; i++)
1898                         if (cpu_possible(i) && per_cpu(cpufreq_cpu_data, i)) {
1899                                 ret = 0;
1900                                 break;
1901                         }
1902
1903                 /* if all ->init() calls failed, unregister */
1904                 if (ret) {
1905                         pr_debug("no CPU initialized for driver %s\n",
1906                                                         driver_data->name);
1907                         goto err_if_unreg;
1908                 }
1909         }
1910
1911         register_hotcpu_notifier(&cpufreq_cpu_notifier);
1912         pr_debug("driver %s up and running\n", driver_data->name);
1913
1914         return 0;
1915 err_if_unreg:
1916         subsys_interface_unregister(&cpufreq_interface);
1917 err_null_driver:
1918         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1919         cpufreq_driver = NULL;
1920         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1921         return ret;
1922 }
1923 EXPORT_SYMBOL_GPL(cpufreq_register_driver);
1924
1925
1926 /**
1927  * cpufreq_unregister_driver - unregister the current CPUFreq driver
1928  *
1929  *    Unregister the current CPUFreq driver. Only call this if you have
1930  * the right to do so, i.e. if you have succeeded in initialising before!
1931  * Returns zero if successful, and -EINVAL if the cpufreq_driver is
1932  * currently not initialised.
1933  */
1934 int cpufreq_unregister_driver(struct cpufreq_driver *driver)
1935 {
1936         unsigned long flags;
1937
1938         if (!cpufreq_driver || (driver != cpufreq_driver))
1939                 return -EINVAL;
1940
1941         pr_debug("unregistering driver %s\n", driver->name);
1942
1943         subsys_interface_unregister(&cpufreq_interface);
1944         unregister_hotcpu_notifier(&cpufreq_cpu_notifier);
1945
1946         spin_lock_irqsave(&cpufreq_driver_lock, flags);
1947         cpufreq_driver = NULL;
1948         spin_unlock_irqrestore(&cpufreq_driver_lock, flags);
1949
1950         return 0;
1951 }
1952 EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
1953
1954 static int __init cpufreq_core_init(void)
1955 {
1956         int cpu;
1957
1958         if (cpufreq_disabled())
1959                 return -ENODEV;
1960
1961         for_each_possible_cpu(cpu) {
1962                 per_cpu(cpufreq_policy_cpu, cpu) = -1;
1963                 init_rwsem(&per_cpu(cpu_policy_rwsem, cpu));
1964         }
1965
1966         cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
1967         BUG_ON(!cpufreq_global_kobject);
1968         register_syscore_ops(&cpufreq_syscore_ops);
1969
1970         return 0;
1971 }
1972 core_initcall(cpufreq_core_init);