drivers/thermal/intel/intel_powerclamp.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * intel_powerclamp.c - package c-state idle injection
   4  *
   5  * Copyright (c) 2012, Intel Corporation.
   6  *
   7  * Authors:
   8  *     Arjan van de Ven <arjan@linux.intel.com>
   9  *     Jacob Pan <jacob.jun.pan@linux.intel.com>
  10  *
  11  *      TODO:
  12  *           1. better handle wakeup from external interrupts, currently a fixed
  13  *              compensation is added to clamping duration when excessive amount
  14  *              of wakeups are observed during idle time. the reason is that in
  15  *              case of external interrupts without need for ack, clamping down
  16  *              cpu in non-irq context does not reduce irq. for majority of the
  17  *              cases, clamping down cpu does help reduce irq as well, we should
  18  *              be able to differentiate the two cases and give a quantitative
  19  *              solution for the irqs that we can control. perhaps based on
  20  *              get_cpu_iowait_time_us()
  21  *
  22  *           2. synchronization with other hw blocks
  23  */
  24
  25 #define pr_fmt(fmt)     KBUILD_MODNAME ": " fmt
  26
  27 #include <linux/module.h>
  28 #include <linux/kernel.h>
  29 #include <linux/delay.h>
  30 #include <linux/kthread.h>
  31 #include <linux/cpu.h>
  32 #include <linux/thermal.h>
  33 #include <linux/slab.h>
  34 #include <linux/tick.h>
  35 #include <linux/debugfs.h>
  36 #include <linux/seq_file.h>
  37 #include <linux/sched/rt.h>
  38 #include <uapi/linux/sched/types.h>
  39
  40 #include <asm/nmi.h>
  41 #include <asm/msr.h>
  42 #include <asm/mwait.h>
  43 #include <asm/cpu_device_id.h>
  44 #include <asm/hardirq.h>
  45
  46 #define MAX_TARGET_RATIO (50U)
  47 /* For each undisturbed clamping period (no extra wake ups during idle time),
  48  * we increment the confidence counter for the given target ratio.
  49  * CONFIDENCE_OK defines the level where runtime calibration results are
  50  * valid.
  51  */
  52 #define CONFIDENCE_OK (3)
  53 /* Default idle injection duration, driver adjust sleep time to meet target
  54  * idle ratio. Similar to frequency modulation.
  55  */
  56 #define DEFAULT_DURATION_JIFFIES (6)
  57
  58 static unsigned int target_mwait;
  59 static struct dentry *debug_dir;
  60
  61 /* user selected target */
  62 static unsigned int set_target_ratio;
  63 static unsigned int current_ratio;
  64 static bool should_skip;
  65 static bool reduce_irq;
  66 static atomic_t idle_wakeup_counter;
  67 static unsigned int control_cpu; /* The cpu assigned to collect stat and update
  68                                   * control parameters. default to BSP but BSP
  69                                   * can be offlined.
  70                                   */
  71 static bool clamping;
  72
  73 struct powerclamp_worker_data {
  74         struct kthread_worker *worker;
  75         struct kthread_work balancing_work;
  76         struct kthread_delayed_work idle_injection_work;
  77         unsigned int cpu;
  78         unsigned int count;
  79         unsigned int guard;
  80         unsigned int window_size_now;
  81         unsigned int target_ratio;
  82         unsigned int duration_jiffies;
  83         bool clamping;
  84 };
  85
  86 static struct powerclamp_worker_data __percpu *worker_data;
  87 static struct thermal_cooling_device *cooling_dev;
  88 static unsigned long *cpu_clamping_mask;  /* bit map for tracking per cpu
  89                                            * clamping kthread worker
  90                                            */
  91
  92 static unsigned int duration;
  93 static unsigned int pkg_cstate_ratio_cur;
  94 static unsigned int window_size;
  95
  96 static int duration_set(const char *arg, const struct kernel_param *kp)
  97 {
  98         int ret = 0;
  99         unsigned long new_duration;
 100
 101         ret = kstrtoul(arg, 10, &new_duration);
 102         if (ret)
 103                 goto exit;
 104         if (new_duration > 25 || new_duration < 6) {
 105                 pr_err("Out of recommended range %lu, between 6-25ms\n",
 106                         new_duration);
 107                 ret = -EINVAL;
 108         }
 109
 110         duration = clamp(new_duration, 6ul, 25ul);
 111         smp_mb();
 112
 113 exit:
 114
 115         return ret;
 116 }
 117
 118 static const struct kernel_param_ops duration_ops = {
 119         .set = duration_set,
 120         .get = param_get_int,
 121 };
 122
 123
 124 module_param_cb(duration, &duration_ops, &duration, 0644);
 125 MODULE_PARM_DESC(duration, "forced idle time for each attempt in msec.");
 126
 127 struct powerclamp_calibration_data {
 128         unsigned long confidence;  /* used for calibration, basically a counter
 129                                     * gets incremented each time a clamping
 130                                     * period is completed without extra wakeups
 131                                     * once that counter is reached given level,
 132                                     * compensation is deemed usable.
 133                                     */
 134         unsigned long steady_comp; /* steady state compensation used when
 135                                     * no extra wakeups occurred.
 136                                     */
 137         unsigned long dynamic_comp; /* compensate excessive wakeup from idle
 138                                      * mostly from external interrupts.
 139                                      */
 140 };
 141
 142 static struct powerclamp_calibration_data cal_data[MAX_TARGET_RATIO];
 143
 144 static int window_size_set(const char *arg, const struct kernel_param *kp)
 145 {
 146         int ret = 0;
 147         unsigned long new_window_size;
 148
 149         ret = kstrtoul(arg, 10, &new_window_size);
 150         if (ret)
 151                 goto exit_win;
 152         if (new_window_size > 10 || new_window_size < 2) {
 153                 pr_err("Out of recommended window size %lu, between 2-10\n",
 154                         new_window_size);
 155                 ret = -EINVAL;
 156         }
 157
 158         window_size = clamp(new_window_size, 2ul, 10ul);
 159         smp_mb();
 160
 161 exit_win:
 162
 163         return ret;
 164 }
 165
 166 static const struct kernel_param_ops window_size_ops = {
 167         .set = window_size_set,
 168         .get = param_get_int,
 169 };
 170
 171 module_param_cb(window_size, &window_size_ops, &window_size, 0644);
 172 MODULE_PARM_DESC(window_size, "sliding window in number of clamping cycles\n"
 173         "\tpowerclamp controls idle ratio within this window. larger\n"
 174         "\twindow size results in slower response time but more smooth\n"
 175         "\tclamping results. default to 2.");
 176
 177 static void find_target_mwait(void)
 178 {
 179         unsigned int eax, ebx, ecx, edx;
 180         unsigned int highest_cstate = 0;
 181         unsigned int highest_subcstate = 0;
 182         int i;
 183
 184         if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
 185                 return;
 186
 187         cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
 188
 189         if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
 190             !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
 191                 return;
 192
 193         edx >>= MWAIT_SUBSTATE_SIZE;
 194         for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
 195                 if (edx & MWAIT_SUBSTATE_MASK) {
 196                         highest_cstate = i;
 197                         highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
 198                 }
 199         }
 200         target_mwait = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
 201                 (highest_subcstate - 1);
 202
 203 }
 204
 205 struct pkg_cstate_info {
 206         bool skip;
 207         int msr_index;
 208         int cstate_id;
 209 };
 210
 211 #define PKG_CSTATE_INIT(id) {                           \
 212                 .msr_index = MSR_PKG_C##id##_RESIDENCY, \
 213                 .cstate_id = id                         \
 214                         }
 215
 216 static struct pkg_cstate_info pkg_cstates[] = {
 217         PKG_CSTATE_INIT(2),
 218         PKG_CSTATE_INIT(3),
 219         PKG_CSTATE_INIT(6),
 220         PKG_CSTATE_INIT(7),
 221         PKG_CSTATE_INIT(8),
 222         PKG_CSTATE_INIT(9),
 223         PKG_CSTATE_INIT(10),
 224         {NULL},
 225 };
 226
 227 static bool has_pkg_state_counter(void)
 228 {
 229         u64 val;
 230         struct pkg_cstate_info *info = pkg_cstates;
 231
 232         /* check if any one of the counter msrs exists */
 233         while (info->msr_index) {
 234                 if (!rdmsrl_safe(info->msr_index, &val))
 235                         return true;
 236                 info++;
 237         }
 238
 239         return false;
 240 }
 241
 242 static u64 pkg_state_counter(void)
 243 {
 244         u64 val;
 245         u64 count = 0;
 246         struct pkg_cstate_info *info = pkg_cstates;
 247
 248         while (info->msr_index) {
 249                 if (!info->skip) {
 250                         if (!rdmsrl_safe(info->msr_index, &val))
 251                                 count += val;
 252                         else
 253                                 info->skip = true;
 254                 }
 255                 info++;
 256         }
 257
 258         return count;
 259 }
 260
 261 static unsigned int get_compensation(int ratio)
 262 {
 263         unsigned int comp = 0;
 264
 265         /* we only use compensation if all adjacent ones are good */
 266         if (ratio == 1 &&
 267                 cal_data[ratio].confidence >= CONFIDENCE_OK &&
 268                 cal_data[ratio + 1].confidence >= CONFIDENCE_OK &&
 269                 cal_data[ratio + 2].confidence >= CONFIDENCE_OK) {
 270                 comp = (cal_data[ratio].steady_comp +
 271                         cal_data[ratio + 1].steady_comp +
 272                         cal_data[ratio + 2].steady_comp) / 3;
 273         } else if (ratio == MAX_TARGET_RATIO - 1 &&
 274                 cal_data[ratio].confidence >= CONFIDENCE_OK &&
 275                 cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
 276                 cal_data[ratio - 2].confidence >= CONFIDENCE_OK) {
 277                 comp = (cal_data[ratio].steady_comp +
 278                         cal_data[ratio - 1].steady_comp +
 279                         cal_data[ratio - 2].steady_comp) / 3;
 280         } else if (cal_data[ratio].confidence >= CONFIDENCE_OK &&
 281                 cal_data[ratio - 1].confidence >= CONFIDENCE_OK &&
 282                 cal_data[ratio + 1].confidence >= CONFIDENCE_OK) {
 283                 comp = (cal_data[ratio].steady_comp +
 284                         cal_data[ratio - 1].steady_comp +
 285                         cal_data[ratio + 1].steady_comp) / 3;
 286         }
 287
 288         /* REVISIT: simple penalty of double idle injection */
 289         if (reduce_irq)
 290                 comp = ratio;
 291         /* do not exceed limit */
 292         if (comp + ratio >= MAX_TARGET_RATIO)
 293                 comp = MAX_TARGET_RATIO - ratio - 1;
 294
 295         return comp;
 296 }
 297
 298 static void adjust_compensation(int target_ratio, unsigned int win)
 299 {
 300         int delta;
 301         struct powerclamp_calibration_data *d = &cal_data[target_ratio];
 302
 303         /*
 304          * adjust compensations if confidence level has not been reached or
 305          * there are too many wakeups during the last idle injection period, we
 306          * cannot trust the data for compensation.
 307          */
 308         if (d->confidence >= CONFIDENCE_OK ||
 309                 atomic_read(&idle_wakeup_counter) >
 310                 win * num_online_cpus())
 311                 return;
 312
 313         delta = set_target_ratio - current_ratio;
 314         /* filter out bad data */
 315         if (delta >= 0 && delta <= (1+target_ratio/10)) {
 316                 if (d->steady_comp)
 317                         d->steady_comp =
 318                                 roundup(delta+d->steady_comp, 2)/2;
 319                 else
 320                         d->steady_comp = delta;
 321                 d->confidence++;
 322         }
 323 }
 324
 325 static bool powerclamp_adjust_controls(unsigned int target_ratio,
 326                                 unsigned int guard, unsigned int win)
 327 {
 328         static u64 msr_last, tsc_last;
 329         u64 msr_now, tsc_now;
 330         u64 val64;
 331
 332         /* check result for the last window */
 333         msr_now = pkg_state_counter();
 334         tsc_now = rdtsc();
 335
 336         /* calculate pkg cstate vs tsc ratio */
 337         if (!msr_last || !tsc_last)
 338                 current_ratio = 1;
 339         else if (tsc_now-tsc_last) {
 340                 val64 = 100*(msr_now-msr_last);
 341                 do_div(val64, (tsc_now-tsc_last));
 342                 current_ratio = val64;
 343         }
 344
 345         /* update record */
 346         msr_last = msr_now;
 347         tsc_last = tsc_now;
 348
 349         adjust_compensation(target_ratio, win);
 350         /*
 351          * too many external interrupts, set flag such
 352          * that we can take measure later.
 353          */
 354         reduce_irq = atomic_read(&idle_wakeup_counter) >=
 355                 2 * win * num_online_cpus();
 356
 357         atomic_set(&idle_wakeup_counter, 0);
 358         /* if we are above target+guard, skip */
 359         return set_target_ratio + guard <= current_ratio;
 360 }
 361
 362 static void clamp_balancing_func(struct kthread_work *work)
 363 {
 364         struct powerclamp_worker_data *w_data;
 365         int sleeptime;
 366         unsigned long target_jiffies;
 367         unsigned int compensated_ratio;
 368         int interval; /* jiffies to sleep for each attempt */
 369
 370         w_data = container_of(work, struct powerclamp_worker_data,
 371                               balancing_work);
 372
 373         /*
 374          * make sure user selected ratio does not take effect until
 375          * the next round. adjust target_ratio if user has changed
 376          * target such that we can converge quickly.
 377          */
 378         w_data->target_ratio = READ_ONCE(set_target_ratio);
 379         w_data->guard = 1 + w_data->target_ratio / 20;
 380         w_data->window_size_now = window_size;
 381         w_data->duration_jiffies = msecs_to_jiffies(duration);
 382         w_data->count++;
 383
 384         /*
 385          * systems may have different ability to enter package level
 386          * c-states, thus we need to compensate the injected idle ratio
 387          * to achieve the actual target reported by the HW.
 388          */
 389         compensated_ratio = w_data->target_ratio +
 390                 get_compensation(w_data->target_ratio);
 391         if (compensated_ratio <= 0)
 392                 compensated_ratio = 1;
 393         interval = w_data->duration_jiffies * 100 / compensated_ratio;
 394
 395         /* align idle time */
 396         target_jiffies = roundup(jiffies, interval);
 397         sleeptime = target_jiffies - jiffies;
 398         if (sleeptime <= 0)
 399                 sleeptime = 1;
 400
 401         if (clamping && w_data->clamping && cpu_online(w_data->cpu))
 402                 kthread_queue_delayed_work(w_data->worker,
 403                                            &w_data->idle_injection_work,
 404                                            sleeptime);
 405 }
 406
 407 static void clamp_idle_injection_func(struct kthread_work *work)
 408 {
 409         struct powerclamp_worker_data *w_data;
 410
 411         w_data = container_of(work, struct powerclamp_worker_data,
 412                               idle_injection_work.work);
 413
 414         /*
 415          * only elected controlling cpu can collect stats and update
 416          * control parameters.
 417          */
 418         if (w_data->cpu == control_cpu &&
 419             !(w_data->count % w_data->window_size_now)) {
 420                 should_skip =
 421                         powerclamp_adjust_controls(w_data->target_ratio,
 422                                                    w_data->guard,
 423                                                    w_data->window_size_now);
 424                 smp_mb();
 425         }
 426
 427         if (should_skip)
 428                 goto balance;
 429
 430         play_idle(jiffies_to_usecs(w_data->duration_jiffies));
 431
 432 balance:
 433         if (clamping && w_data->clamping && cpu_online(w_data->cpu))
 434                 kthread_queue_work(w_data->worker, &w_data->balancing_work);
 435 }
 436
 437 /*
 438  * 1 HZ polling while clamping is active, useful for userspace
 439  * to monitor actual idle ratio.
 440  */
 441 static void poll_pkg_cstate(struct work_struct *dummy);
 442 static DECLARE_DELAYED_WORK(poll_pkg_cstate_work, poll_pkg_cstate);
 443 static void poll_pkg_cstate(struct work_struct *dummy)
 444 {
 445         static u64 msr_last;
 446         static u64 tsc_last;
 447
 448         u64 msr_now;
 449         u64 tsc_now;
 450         u64 val64;
 451
 452         msr_now = pkg_state_counter();
 453         tsc_now = rdtsc();
 454
 455         /* calculate pkg cstate vs tsc ratio */
 456         if (!msr_last || !tsc_last)
 457                 pkg_cstate_ratio_cur = 1;
 458         else {
 459                 if (tsc_now - tsc_last) {
 460                         val64 = 100 * (msr_now - msr_last);
 461                         do_div(val64, (tsc_now - tsc_last));
 462                         pkg_cstate_ratio_cur = val64;
 463                 }
 464         }
 465
 466         /* update record */
 467         msr_last = msr_now;
 468         tsc_last = tsc_now;
 469
 470         if (true == clamping)
 471                 schedule_delayed_work(&poll_pkg_cstate_work, HZ);
 472 }
 473
 474 static void start_power_clamp_worker(unsigned long cpu)
 475 {
 476         struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
 477         struct kthread_worker *worker;
 478
 479         worker = kthread_create_worker_on_cpu(cpu, 0, "kidle_inj/%ld", cpu);
 480         if (IS_ERR(worker))
 481                 return;
 482
 483         w_data->worker = worker;
 484         w_data->count = 0;
 485         w_data->cpu = cpu;
 486         w_data->clamping = true;
 487         set_bit(cpu, cpu_clamping_mask);
 488         sched_set_fifo(worker->task);
 489         kthread_init_work(&w_data->balancing_work, clamp_balancing_func);
 490         kthread_init_delayed_work(&w_data->idle_injection_work,
 491                                   clamp_idle_injection_func);
 492         kthread_queue_work(w_data->worker, &w_data->balancing_work);
 493 }
 494
 495 static void stop_power_clamp_worker(unsigned long cpu)
 496 {
 497         struct powerclamp_worker_data *w_data = per_cpu_ptr(worker_data, cpu);
 498
 499         if (!w_data->worker)
 500                 return;
 501
 502         w_data->clamping = false;
 503         /*
 504          * Make sure that all works that get queued after this point see
 505          * the clamping disabled. The counter part is not needed because
 506          * there is an implicit memory barrier when the queued work
 507          * is proceed.
 508          */
 509         smp_wmb();
 510         kthread_cancel_work_sync(&w_data->balancing_work);
 511         kthread_cancel_delayed_work_sync(&w_data->idle_injection_work);
 512         /*
 513          * The balancing work still might be queued here because
 514          * the handling of the "clapming" variable, cancel, and queue
 515          * operations are not synchronized via a lock. But it is not
 516          * a big deal. The balancing work is fast and destroy kthread
 517          * will wait for it.
 518          */
 519         clear_bit(w_data->cpu, cpu_clamping_mask);
 520         kthread_destroy_worker(w_data->worker);
 521
 522         w_data->worker = NULL;
 523 }
 524
 525 static int start_power_clamp(void)
 526 {
 527         unsigned long cpu;
 528
 529         set_target_ratio = clamp(set_target_ratio, 0U, MAX_TARGET_RATIO - 1);
 530         /* prevent cpu hotplug */
 531         cpus_read_lock();
 532
 533         /* prefer BSP */
 534         control_cpu = 0;
 535         if (!cpu_online(control_cpu))
 536                 control_cpu = smp_processor_id();
 537
 538         clamping = true;
 539         schedule_delayed_work(&poll_pkg_cstate_work, 0);
 540
 541         /* start one kthread worker per online cpu */
 542         for_each_online_cpu(cpu) {
 543                 start_power_clamp_worker(cpu);
 544         }
 545         cpus_read_unlock();
 546
 547         return 0;
 548 }
 549
 550 static void end_power_clamp(void)
 551 {
 552         int i;
 553
 554         /*
 555          * Block requeuing in all the kthread workers. They will flush and
 556          * stop faster.
 557          */
 558         clamping = false;
 559         if (bitmap_weight(cpu_clamping_mask, num_possible_cpus())) {
 560                 for_each_set_bit(i, cpu_clamping_mask, num_possible_cpus()) {
 561                         pr_debug("clamping worker for cpu %d alive, destroy\n",
 562                                  i);
 563                         stop_power_clamp_worker(i);
 564                 }
 565         }
 566 }
 567
 568 static int powerclamp_cpu_online(unsigned int cpu)
 569 {
 570         if (clamping == false)
 571                 return 0;
 572         start_power_clamp_worker(cpu);
 573         /* prefer BSP as controlling CPU */
 574         if (cpu == 0) {
 575                 control_cpu = 0;
 576                 smp_mb();
 577         }
 578         return 0;
 579 }
 580
 581 static int powerclamp_cpu_predown(unsigned int cpu)
 582 {
 583         if (clamping == false)
 584                 return 0;
 585
 586         stop_power_clamp_worker(cpu);
 587         if (cpu != control_cpu)
 588                 return 0;
 589
 590         control_cpu = cpumask_first(cpu_online_mask);
 591         if (control_cpu == cpu)
 592                 control_cpu = cpumask_next(cpu, cpu_online_mask);
 593         smp_mb();
 594         return 0;
 595 }
 596
 597 static int powerclamp_get_max_state(struct thermal_cooling_device *cdev,
 598                                  unsigned long *state)
 599 {
 600         *state = MAX_TARGET_RATIO;
 601
 602         return 0;
 603 }
 604
 605 static int powerclamp_get_cur_state(struct thermal_cooling_device *cdev,
 606                                  unsigned long *state)
 607 {
 608         if (true == clamping)
 609                 *state = pkg_cstate_ratio_cur;
 610         else
 611                 /* to save power, do not poll idle ratio while not clamping */
 612                 *state = -1; /* indicates invalid state */
 613
 614         return 0;
 615 }
 616
 617 static int powerclamp_set_cur_state(struct thermal_cooling_device *cdev,
 618                                  unsigned long new_target_ratio)
 619 {
 620         int ret = 0;
 621
 622         new_target_ratio = clamp(new_target_ratio, 0UL,
 623                                 (unsigned long) (MAX_TARGET_RATIO-1));
 624         if (set_target_ratio == 0 && new_target_ratio > 0) {
 625                 pr_info("Start idle injection to reduce power\n");
 626                 set_target_ratio = new_target_ratio;
 627                 ret = start_power_clamp();
 628                 goto exit_set;
 629         } else  if (set_target_ratio > 0 && new_target_ratio == 0) {
 630                 pr_info("Stop forced idle injection\n");
 631                 end_power_clamp();
 632                 set_target_ratio = 0;
 633         } else  /* adjust currently running */ {
 634                 set_target_ratio = new_target_ratio;
 635                 /* make new set_target_ratio visible to other cpus */
 636                 smp_mb();
 637         }
 638
 639 exit_set:
 640         return ret;
 641 }
 642
 643 /* bind to generic thermal layer as cooling device*/
 644 static struct thermal_cooling_device_ops powerclamp_cooling_ops = {
 645         .get_max_state = powerclamp_get_max_state,
 646         .get_cur_state = powerclamp_get_cur_state,
 647         .set_cur_state = powerclamp_set_cur_state,
 648 };
 649
 650 static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = {
 651         X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL),
 652         {}
 653 };
 654 MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids);
 655
 656 static int __init powerclamp_probe(void)
 657 {
 658
 659         if (!x86_match_cpu(intel_powerclamp_ids)) {
 660                 pr_err("CPU does not support MWAIT\n");
 661                 return -ENODEV;
 662         }
 663
 664         /* The goal for idle time alignment is to achieve package cstate. */
 665         if (!has_pkg_state_counter()) {
 666                 pr_info("No package C-state available\n");
 667                 return -ENODEV;
 668         }
 669
 670         /* find the deepest mwait value */
 671         find_target_mwait();
 672
 673         return 0;
 674 }
 675
 676 static int powerclamp_debug_show(struct seq_file *m, void *unused)
 677 {
 678         int i = 0;
 679
 680         seq_printf(m, "controlling cpu: %d\n", control_cpu);
 681         seq_printf(m, "pct confidence steady dynamic (compensation)\n");
 682         for (i = 0; i < MAX_TARGET_RATIO; i++) {
 683                 seq_printf(m, "%d\t%lu\t%lu\t%lu\n",
 684                         i,
 685                         cal_data[i].confidence,
 686                         cal_data[i].steady_comp,
 687                         cal_data[i].dynamic_comp);
 688         }
 689
 690         return 0;
 691 }
 692
 693 DEFINE_SHOW_ATTRIBUTE(powerclamp_debug);
 694
 695 static inline void powerclamp_create_debug_files(void)
 696 {
 697         debug_dir = debugfs_create_dir("intel_powerclamp", NULL);
 698
 699         debugfs_create_file("powerclamp_calib", S_IRUGO, debug_dir, cal_data,
 700                             &powerclamp_debug_fops);
 701 }
 702
 703 static enum cpuhp_state hp_state;
 704
 705 static int __init powerclamp_init(void)
 706 {
 707         int retval;
 708         int bitmap_size;
 709
 710         bitmap_size = BITS_TO_LONGS(num_possible_cpus()) * sizeof(long);
 711         cpu_clamping_mask = kzalloc(bitmap_size, GFP_KERNEL);
 712         if (!cpu_clamping_mask)
 713                 return -ENOMEM;
 714
 715         /* probe cpu features and ids here */
 716         retval = powerclamp_probe();
 717         if (retval)
 718                 goto exit_free;
 719
 720         /* set default limit, maybe adjusted during runtime based on feedback */
 721         window_size = 2;
 722         retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 723                                            "thermal/intel_powerclamp:online",
 724                                            powerclamp_cpu_online,
 725                                            powerclamp_cpu_predown);
 726         if (retval < 0)
 727                 goto exit_free;
 728
 729         hp_state = retval;
 730
 731         worker_data = alloc_percpu(struct powerclamp_worker_data);
 732         if (!worker_data) {
 733                 retval = -ENOMEM;
 734                 goto exit_unregister;
 735         }
 736
 737         cooling_dev = thermal_cooling_device_register("intel_powerclamp", NULL,
 738                                                 &powerclamp_cooling_ops);
 739         if (IS_ERR(cooling_dev)) {
 740                 retval = -ENODEV;
 741                 goto exit_free_thread;
 742         }
 743
 744         if (!duration)
 745                 duration = jiffies_to_msecs(DEFAULT_DURATION_JIFFIES);
 746
 747         powerclamp_create_debug_files();
 748
 749         return 0;
 750
 751 exit_free_thread:
 752         free_percpu(worker_data);
 753 exit_unregister:
 754         cpuhp_remove_state_nocalls(hp_state);
 755 exit_free:
 756         kfree(cpu_clamping_mask);
 757         return retval;
 758 }
 759 module_init(powerclamp_init);
 760
 761 static void __exit powerclamp_exit(void)
 762 {
 763         end_power_clamp();
 764         cpuhp_remove_state_nocalls(hp_state);
 765         free_percpu(worker_data);
 766         thermal_cooling_device_unregister(cooling_dev);
 767         kfree(cpu_clamping_mask);
 768
 769         cancel_delayed_work_sync(&poll_pkg_cstate_work);
 770         debugfs_remove_recursive(debug_dir);
 771 }
 772 module_exit(powerclamp_exit);
 773
 774 MODULE_LICENSE("GPL");
 775 MODULE_AUTHOR("Arjan van de Ven <arjan@linux.intel.com>");
 776 MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@linux.intel.com>");
 777 MODULE_DESCRIPTION("Package Level C-state Idle Injection for Intel CPUs");