1 // SPDX-License-Identifier: GPL-2.0-only
3 * x86_pkg_temp_thermal driver
4 * Copyright (c) 2013, Intel Corporation.
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/err.h>
11 #include <linux/param.h>
12 #include <linux/device.h>
13 #include <linux/platform_device.h>
14 #include <linux/cpu.h>
15 #include <linux/smp.h>
16 #include <linux/slab.h>
18 #include <linux/thermal.h>
19 #include <linux/debugfs.h>
21 #include <asm/cpu_device_id.h>
23 #include "thermal_interrupt.h"
26 * Rate control delay: Idea is to introduce denounce effect
27 * This should be long enough to avoid reduce events, when
28 * threshold is set to a temperature, which is constantly
29 * violated, but at the short enough to take any action.
30 * The action can be remove threshold or change it to next
31 * interesting setting. Based on experiments, in around
32 * every 5 seconds under load will give us a significant
35 #define PKG_TEMP_THERMAL_NOTIFY_DELAY 5000
36 static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
37 module_param(notify_delay_ms, int, 0644);
38 MODULE_PARM_DESC(notify_delay_ms,
39 "User space notification delay in milli seconds.");
41 /* Number of trip points in thermal zone. Currently it can't
42 * be more than 2. MSR can allow setting and getting notifications
43 * for only 2 thresholds. This define enforces this, if there
44 * is some wrong values returned by cpuid for number of thresholds.
46 #define MAX_NUMBER_OF_TRIPS 2
52 u32 msr_pkg_therm_low;
53 u32 msr_pkg_therm_high;
54 struct delayed_work work;
55 struct thermal_zone_device *tzone;
56 struct cpumask cpumask;
59 static struct thermal_zone_params pkg_temp_tz_params = {
63 /* Keep track of how many zone pointers we allocated in init() */
64 static int max_id __read_mostly;
65 /* Array of zone pointers */
66 static struct zone_device **zones;
67 /* Serializes interrupt notification, work and hotplug */
68 static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
69 /* Protects zone operation in the work function against hotplug removal */
70 static DEFINE_MUTEX(thermal_zone_mutex);
72 /* The dynamically assigned cpu hotplug state for module_exit() */
73 static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
75 /* Debug counters to show using debugfs */
76 static struct dentry *debugfs;
77 static unsigned int pkg_interrupt_cnt;
78 static unsigned int pkg_work_cnt;
80 static void pkg_temp_debugfs_init(void)
82 debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
84 debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
86 debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
93 * - cpu hotplug: Read serialized by cpu hotplug lock
94 * Write must hold pkg_temp_lock
96 * - Other callsites: Must hold pkg_temp_lock
98 static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
100 int id = topology_logical_die_id(cpu);
102 if (id >= 0 && id < max_id)
108 * tj-max is is interesting because threshold is set relative to this
111 static int get_tj_max(int cpu, u32 *tj_max)
116 err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx);
120 val = (eax >> 16) & 0xff;
121 *tj_max = val * 1000;
123 return val ? 0 : -EINVAL;
126 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
128 struct zone_device *zonedev = tzd->devdata;
131 rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_STATUS,
133 if (eax & 0x80000000) {
134 *temp = zonedev->tj_max - ((eax >> 16) & 0x7f) * 1000;
135 pr_debug("sys_get_curr_temp %d\n", *temp);
141 static int sys_get_trip_temp(struct thermal_zone_device *tzd,
144 struct zone_device *zonedev = tzd->devdata;
145 unsigned long thres_reg_value;
146 u32 mask, shift, eax, edx;
149 if (trip >= MAX_NUMBER_OF_TRIPS)
153 mask = THERM_MASK_THRESHOLD1;
154 shift = THERM_SHIFT_THRESHOLD1;
156 mask = THERM_MASK_THRESHOLD0;
157 shift = THERM_SHIFT_THRESHOLD0;
160 ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
165 thres_reg_value = (eax & mask) >> shift;
167 *temp = zonedev->tj_max - thres_reg_value * 1000;
169 *temp = THERMAL_TEMP_INVALID;
170 pr_debug("sys_get_trip_temp %d\n", *temp);
176 sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
178 struct zone_device *zonedev = tzd->devdata;
179 u32 l, h, mask, shift, intr;
182 if (trip >= MAX_NUMBER_OF_TRIPS || temp >= zonedev->tj_max)
185 ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
191 mask = THERM_MASK_THRESHOLD1;
192 shift = THERM_SHIFT_THRESHOLD1;
193 intr = THERM_INT_THRESHOLD1_ENABLE;
195 mask = THERM_MASK_THRESHOLD0;
196 shift = THERM_SHIFT_THRESHOLD0;
197 intr = THERM_INT_THRESHOLD0_ENABLE;
201 * When users space sets a trip temperature == 0, which is indication
202 * that, it is no longer interested in receiving notifications.
207 l |= (zonedev->tj_max - temp)/1000 << shift;
211 return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
215 static int sys_get_trip_type(struct thermal_zone_device *thermal, int trip,
216 enum thermal_trip_type *type)
218 *type = THERMAL_TRIP_PASSIVE;
222 /* Thermal zone callback registry */
223 static struct thermal_zone_device_ops tzone_ops = {
224 .get_temp = sys_get_curr_temp,
225 .get_trip_temp = sys_get_trip_temp,
226 .get_trip_type = sys_get_trip_type,
227 .set_trip_temp = sys_set_trip_temp,
230 static bool pkg_thermal_rate_control(void)
235 /* Enable threshold interrupt on local package/cpu */
236 static inline void enable_pkg_thres_interrupt(void)
241 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
242 /* only enable/disable if it had valid threshold value */
243 thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
244 thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
246 l |= THERM_INT_THRESHOLD0_ENABLE;
248 l |= THERM_INT_THRESHOLD1_ENABLE;
249 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
252 /* Disable threshold interrupt on local package/cpu */
253 static inline void disable_pkg_thres_interrupt(void)
257 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
259 l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
260 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
263 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
265 struct thermal_zone_device *tzone = NULL;
266 int cpu = smp_processor_id();
267 struct zone_device *zonedev;
270 mutex_lock(&thermal_zone_mutex);
271 raw_spin_lock_irq(&pkg_temp_lock);
274 zonedev = pkg_temp_thermal_get_dev(cpu);
276 raw_spin_unlock_irq(&pkg_temp_lock);
277 mutex_unlock(&thermal_zone_mutex);
280 zonedev->work_scheduled = false;
282 rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
283 wr_val = msr_val & ~(THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
284 if (wr_val != msr_val) {
285 wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, wr_val);
286 tzone = zonedev->tzone;
289 enable_pkg_thres_interrupt();
290 raw_spin_unlock_irq(&pkg_temp_lock);
293 * If tzone is not NULL, then thermal_zone_mutex will prevent the
294 * concurrent removal in the cpu offline callback.
297 thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
299 mutex_unlock(&thermal_zone_mutex);
302 static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
304 unsigned long ms = msecs_to_jiffies(notify_delay_ms);
306 schedule_delayed_work_on(cpu, work, ms);
309 static int pkg_thermal_notify(u64 msr_val)
311 int cpu = smp_processor_id();
312 struct zone_device *zonedev;
315 raw_spin_lock_irqsave(&pkg_temp_lock, flags);
318 disable_pkg_thres_interrupt();
320 /* Work is per package, so scheduling it once is enough. */
321 zonedev = pkg_temp_thermal_get_dev(cpu);
322 if (zonedev && !zonedev->work_scheduled) {
323 zonedev->work_scheduled = true;
324 pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
327 raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
331 static int pkg_temp_thermal_device_add(unsigned int cpu)
333 int id = topology_logical_die_id(cpu);
334 u32 tj_max, eax, ebx, ecx, edx;
335 struct zone_device *zonedev;
336 int thres_count, err;
341 cpuid(6, &eax, &ebx, &ecx, &edx);
342 thres_count = ebx & 0x07;
346 thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
348 err = get_tj_max(cpu, &tj_max);
352 zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL);
356 INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
358 zonedev->tj_max = tj_max;
359 zonedev->tzone = thermal_zone_device_register("x86_pkg_temp",
361 (thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
362 zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
363 if (IS_ERR(zonedev->tzone)) {
364 err = PTR_ERR(zonedev->tzone);
368 err = thermal_zone_device_enable(zonedev->tzone);
370 thermal_zone_device_unregister(zonedev->tzone);
374 /* Store MSR value for package thermal interrupt, to restore at exit */
375 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
376 zonedev->msr_pkg_therm_high);
378 cpumask_set_cpu(cpu, &zonedev->cpumask);
379 raw_spin_lock_irq(&pkg_temp_lock);
381 raw_spin_unlock_irq(&pkg_temp_lock);
385 static int pkg_thermal_cpu_offline(unsigned int cpu)
387 struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
388 bool lastcpu, was_target;
394 target = cpumask_any_but(&zonedev->cpumask, cpu);
395 cpumask_clear_cpu(cpu, &zonedev->cpumask);
396 lastcpu = target >= nr_cpu_ids;
398 * Remove the sysfs files, if this is the last cpu in the package
399 * before doing further cleanups.
402 struct thermal_zone_device *tzone = zonedev->tzone;
405 * We must protect against a work function calling
406 * thermal_zone_update, after/while unregister. We null out
407 * the pointer under the zone mutex, so the worker function
410 mutex_lock(&thermal_zone_mutex);
411 zonedev->tzone = NULL;
412 mutex_unlock(&thermal_zone_mutex);
414 thermal_zone_device_unregister(tzone);
417 /* Protect against work and interrupts */
418 raw_spin_lock_irq(&pkg_temp_lock);
421 * Check whether this cpu was the current target and store the new
422 * one. When we drop the lock, then the interrupt notify function
423 * will see the new target.
425 was_target = zonedev->cpu == cpu;
426 zonedev->cpu = target;
429 * If this is the last CPU in the package remove the package
430 * reference from the array and restore the interrupt MSR. When we
431 * drop the lock neither the interrupt notify function nor the
432 * worker will see the package anymore.
435 zones[topology_logical_die_id(cpu)] = NULL;
436 /* After this point nothing touches the MSR anymore. */
437 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
438 zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
442 * Check whether there is work scheduled and whether the work is
443 * targeted at the outgoing CPU.
445 if (zonedev->work_scheduled && was_target) {
447 * To cancel the work we need to drop the lock, otherwise
448 * we might deadlock if the work needs to be flushed.
450 raw_spin_unlock_irq(&pkg_temp_lock);
451 cancel_delayed_work_sync(&zonedev->work);
452 raw_spin_lock_irq(&pkg_temp_lock);
454 * If this is not the last cpu in the package and the work
455 * did not run after we dropped the lock above, then we
456 * need to reschedule the work, otherwise the interrupt
457 * stays disabled forever.
459 if (!lastcpu && zonedev->work_scheduled)
460 pkg_thermal_schedule_work(target, &zonedev->work);
463 raw_spin_unlock_irq(&pkg_temp_lock);
465 /* Final cleanup if this is the last cpu */
471 static int pkg_thermal_cpu_online(unsigned int cpu)
473 struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
474 struct cpuinfo_x86 *c = &cpu_data(cpu);
477 if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
480 /* If the package exists, nothing to do */
482 cpumask_set_cpu(cpu, &zonedev->cpumask);
485 return pkg_temp_thermal_device_add(cpu);
488 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
489 X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
492 MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
494 static int __init pkg_temp_thermal_init(void)
498 if (!x86_match_cpu(pkg_temp_thermal_ids))
501 max_id = topology_max_packages() * topology_max_die_per_package();
502 zones = kcalloc(max_id, sizeof(struct zone_device *),
507 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
508 pkg_thermal_cpu_online, pkg_thermal_cpu_offline);
512 /* Store the state for module exit */
513 pkg_thermal_hp_state = ret;
515 platform_thermal_package_notify = pkg_thermal_notify;
516 platform_thermal_package_rate_control = pkg_thermal_rate_control;
518 /* Don't care if it fails */
519 pkg_temp_debugfs_init();
526 module_init(pkg_temp_thermal_init)
528 static void __exit pkg_temp_thermal_exit(void)
530 platform_thermal_package_notify = NULL;
531 platform_thermal_package_rate_control = NULL;
533 cpuhp_remove_state(pkg_thermal_hp_state);
534 debugfs_remove_recursive(debugfs);
537 module_exit(pkg_temp_thermal_exit)
539 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
540 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
541 MODULE_LICENSE("GPL v2");