1 // SPDX-License-Identifier: GPL-2.0-only
3 * Hardware Feedback Interface Driver
5 * Copyright (c) 2021, Intel Corporation.
7 * Authors: Aubrey Li <aubrey.li@linux.intel.com>
8 * Ricardo Neri <ricardo.neri-calderon@linux.intel.com>
11 * The Hardware Feedback Interface provides a performance and energy efficiency
12 * capability information for each CPU in the system. Depending on the processor
13 * model, hardware may periodically update these capabilities as a result of
14 * changes in the operating conditions (e.g., power limits or thermal
15 * constraints). On other processor models, there is a single HFI update
18 * This file provides functionality to process HFI updates and relay these
19 * updates to userspace.
22 #define pr_fmt(fmt) "intel-hfi: " fmt
24 #include <linux/bitops.h>
25 #include <linux/cpufeature.h>
26 #include <linux/cpumask.h>
27 #include <linux/gfp.h>
29 #include <linux/kernel.h>
30 #include <linux/math.h>
31 #include <linux/mutex.h>
32 #include <linux/percpu-defs.h>
33 #include <linux/printk.h>
34 #include <linux/processor.h>
35 #include <linux/slab.h>
36 #include <linux/spinlock.h>
37 #include <linux/string.h>
38 #include <linux/topology.h>
39 #include <linux/workqueue.h>
43 #include "../thermal_core.h"
44 #include "intel_hfi.h"
46 #define THERM_STATUS_CLEAR_PKG_MASK (BIT(1) | BIT(3) | BIT(5) | BIT(7) | \
47 BIT(9) | BIT(11) | BIT(26))
49 /* Hardware Feedback Interface MSR configuration bits */
50 #define HW_FEEDBACK_PTR_VALID_BIT BIT(0)
51 #define HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT BIT(0)
53 /* CPUID detection and enumeration definitions for HFI */
55 #define CPUID_HFI_LEAF 6
57 union hfi_capabilities {
60 u8 energy_efficiency:1;
68 union hfi_capabilities capabilities;
77 * struct hfi_cpu_data - HFI capabilities per CPU
78 * @perf_cap: Performance capability
79 * @ee_cap: Energy efficiency capability
81 * Capabilities of a logical processor in the HFI table. These capabilities are
90 * struct hfi_hdr - Header of the HFI table
91 * @perf_updated: Hardware updated performance capabilities
92 * @ee_updated: Hardware updated energy efficiency capabilities
94 * Properties of the data in an HFI table.
102 * struct hfi_instance - Representation of an HFI instance (i.e., a table)
103 * @local_table: Base of the local copy of the HFI table
104 * @timestamp: Timestamp of the last update of the local table.
105 * Located at the base of the local table.
106 * @hdr: Base address of the header of the local table
107 * @data: Base address of the data of the local table
108 * @cpus: CPUs represented in this HFI table instance
109 * @hw_table: Pointer to the HFI table of this instance
110 * @update_work: Delayed work to process HFI updates
111 * @table_lock: Lock to protect acceses to the table of this instance
112 * @event_lock: Lock to process HFI interrupts
114 * A set of parameters to parse and navigate a specific HFI table.
116 struct hfi_instance {
125 struct delayed_work update_work;
126 raw_spinlock_t table_lock;
127 raw_spinlock_t event_lock;
131 * struct hfi_features - Supported HFI features
132 * @nr_table_pages: Size of the HFI table in 4KB pages
133 * @cpu_stride: Stride size to locate the capability data of a logical
134 * processor within the table (i.e., row stride)
135 * @hdr_size: Size of the table header
137 * Parameters and supported features that are common to all HFI instances
139 struct hfi_features {
140 unsigned int nr_table_pages;
141 unsigned int cpu_stride;
142 unsigned int hdr_size;
146 * struct hfi_cpu_info - Per-CPU attributes to consume HFI data
147 * @index: Row of this CPU in its HFI table
148 * @hfi_instance: Attributes of the HFI table to which this CPU belongs
150 * Parameters to link a logical processor to an HFI table and a row within it.
152 struct hfi_cpu_info {
154 struct hfi_instance *hfi_instance;
157 static DEFINE_PER_CPU(struct hfi_cpu_info, hfi_cpu_info) = { .index = -1 };
159 static int max_hfi_instances;
160 static struct hfi_instance *hfi_instances;
162 static struct hfi_features hfi_features;
163 static DEFINE_MUTEX(hfi_instance_lock);
165 static struct workqueue_struct *hfi_updates_wq;
166 #define HFI_UPDATE_INTERVAL HZ
167 #define HFI_MAX_THERM_NOTIFY_COUNT 16
169 static void get_hfi_caps(struct hfi_instance *hfi_instance,
170 struct thermal_genl_cpu_caps *cpu_caps)
174 raw_spin_lock_irq(&hfi_instance->table_lock);
175 for_each_cpu(cpu, hfi_instance->cpus) {
176 struct hfi_cpu_data *caps;
179 index = per_cpu(hfi_cpu_info, cpu).index;
180 caps = hfi_instance->data + index * hfi_features.cpu_stride;
181 cpu_caps[i].cpu = cpu;
184 * Scale performance and energy efficiency to
185 * the [0, 1023] interval that thermal netlink uses.
187 cpu_caps[i].performance = caps->perf_cap << 2;
188 cpu_caps[i].efficiency = caps->ee_cap << 2;
192 raw_spin_unlock_irq(&hfi_instance->table_lock);
196 * Call update_capabilities() when there are changes in the HFI table.
198 static void update_capabilities(struct hfi_instance *hfi_instance)
200 struct thermal_genl_cpu_caps *cpu_caps;
201 int i = 0, cpu_count;
203 /* CPUs may come online/offline while processing an HFI update. */
204 mutex_lock(&hfi_instance_lock);
206 cpu_count = cpumask_weight(hfi_instance->cpus);
208 /* No CPUs to report in this hfi_instance. */
212 cpu_caps = kcalloc(cpu_count, sizeof(*cpu_caps), GFP_KERNEL);
216 get_hfi_caps(hfi_instance, cpu_caps);
218 if (cpu_count < HFI_MAX_THERM_NOTIFY_COUNT)
221 /* Process complete chunks of HFI_MAX_THERM_NOTIFY_COUNT capabilities. */
223 (i + HFI_MAX_THERM_NOTIFY_COUNT) <= cpu_count;
224 i += HFI_MAX_THERM_NOTIFY_COUNT)
225 thermal_genl_cpu_capability_event(HFI_MAX_THERM_NOTIFY_COUNT,
228 cpu_count = cpu_count - i;
231 /* Process the remaining capabilities if any. */
233 thermal_genl_cpu_capability_event(cpu_count, &cpu_caps[i]);
237 mutex_unlock(&hfi_instance_lock);
240 static void hfi_update_work_fn(struct work_struct *work)
242 struct hfi_instance *hfi_instance;
244 hfi_instance = container_of(to_delayed_work(work), struct hfi_instance,
247 update_capabilities(hfi_instance);
250 void intel_hfi_process_event(__u64 pkg_therm_status_msr_val)
252 struct hfi_instance *hfi_instance;
253 int cpu = smp_processor_id();
254 struct hfi_cpu_info *info;
257 if (!pkg_therm_status_msr_val)
260 info = &per_cpu(hfi_cpu_info, cpu);
265 * A CPU is linked to its HFI instance before the thermal vector in the
266 * local APIC is unmasked. Hence, info->hfi_instance cannot be NULL
267 * when receiving an HFI event.
269 hfi_instance = info->hfi_instance;
270 if (unlikely(!hfi_instance)) {
271 pr_debug("Received event on CPU %d but instance was null", cpu);
276 * On most systems, all CPUs in the package receive a package-level
277 * thermal interrupt when there is an HFI update. It is sufficient to
278 * let a single CPU to acknowledge the update and queue work to
279 * process it. The remaining CPUs can resume their work.
281 if (!raw_spin_trylock(&hfi_instance->event_lock))
284 /* Skip duplicated updates. */
285 new_timestamp = *(u64 *)hfi_instance->hw_table;
286 if (*hfi_instance->timestamp == new_timestamp) {
287 raw_spin_unlock(&hfi_instance->event_lock);
291 raw_spin_lock(&hfi_instance->table_lock);
294 * Copy the updated table into our local copy. This includes the new
297 memcpy(hfi_instance->local_table, hfi_instance->hw_table,
298 hfi_features.nr_table_pages << PAGE_SHIFT);
300 raw_spin_unlock(&hfi_instance->table_lock);
301 raw_spin_unlock(&hfi_instance->event_lock);
304 * Let hardware know that we are done reading the HFI table and it is
305 * free to update it again.
307 pkg_therm_status_msr_val &= THERM_STATUS_CLEAR_PKG_MASK &
308 ~PACKAGE_THERM_STATUS_HFI_UPDATED;
309 wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, pkg_therm_status_msr_val);
311 queue_delayed_work(hfi_updates_wq, &hfi_instance->update_work,
312 HFI_UPDATE_INTERVAL);
315 static void init_hfi_cpu_index(struct hfi_cpu_info *info)
317 union cpuid6_edx edx;
319 /* Do not re-read @cpu's index if it has already been initialized. */
320 if (info->index > -1)
323 edx.full = cpuid_edx(CPUID_HFI_LEAF);
324 info->index = edx.split.index;
328 * The format of the HFI table depends on the number of capabilities that the
329 * hardware supports. Keep a data structure to navigate the table.
331 static void init_hfi_instance(struct hfi_instance *hfi_instance)
333 /* The HFI header is below the time-stamp. */
334 hfi_instance->hdr = hfi_instance->local_table +
335 sizeof(*hfi_instance->timestamp);
337 /* The HFI data starts below the header. */
338 hfi_instance->data = hfi_instance->hdr + hfi_features.hdr_size;
342 * intel_hfi_online() - Enable HFI on @cpu
343 * @cpu: CPU in which the HFI will be enabled
345 * Enable the HFI to be used in @cpu. The HFI is enabled at the die/package
346 * level. The first CPU in the die/package to come online does the full HFI
347 * initialization. Subsequent CPUs will just link themselves to the HFI
348 * instance of their die/package.
350 * This function is called before enabling the thermal vector in the local APIC
351 * in order to ensure that @cpu has an associated HFI instance when it receives
354 void intel_hfi_online(unsigned int cpu)
356 struct hfi_instance *hfi_instance;
357 struct hfi_cpu_info *info;
358 phys_addr_t hw_table_pa;
362 /* Nothing to do if hfi_instances are missing. */
367 * Link @cpu to the HFI instance of its package/die. It does not
368 * matter whether the instance has been initialized.
370 info = &per_cpu(hfi_cpu_info, cpu);
371 die_id = topology_logical_die_id(cpu);
372 hfi_instance = info->hfi_instance;
374 if (die_id < 0 || die_id >= max_hfi_instances)
377 hfi_instance = &hfi_instances[die_id];
378 info->hfi_instance = hfi_instance;
381 init_hfi_cpu_index(info);
384 * Now check if the HFI instance of the package/die of @cpu has been
385 * initialized (by checking its header). In such case, all we have to
386 * do is to add @cpu to this instance's cpumask.
388 mutex_lock(&hfi_instance_lock);
389 if (hfi_instance->hdr) {
390 cpumask_set_cpu(cpu, hfi_instance->cpus);
395 * Hardware is programmed with the physical address of the first page
396 * frame of the table. Hence, the allocated memory must be page-aligned.
398 hfi_instance->hw_table = alloc_pages_exact(hfi_features.nr_table_pages,
399 GFP_KERNEL | __GFP_ZERO);
400 if (!hfi_instance->hw_table)
403 hw_table_pa = virt_to_phys(hfi_instance->hw_table);
406 * Allocate memory to keep a local copy of the table that
407 * hardware generates.
409 hfi_instance->local_table = kzalloc(hfi_features.nr_table_pages << PAGE_SHIFT,
411 if (!hfi_instance->local_table)
415 * Program the address of the feedback table of this die/package. On
416 * some processors, hardware remembers the old address of the HFI table
417 * even after having been reprogrammed and re-enabled. Thus, do not free
418 * the pages allocated for the table or reprogram the hardware with a
419 * new base address. Namely, program the hardware only once.
421 msr_val = hw_table_pa | HW_FEEDBACK_PTR_VALID_BIT;
422 wrmsrl(MSR_IA32_HW_FEEDBACK_PTR, msr_val);
424 init_hfi_instance(hfi_instance);
426 INIT_DELAYED_WORK(&hfi_instance->update_work, hfi_update_work_fn);
427 raw_spin_lock_init(&hfi_instance->table_lock);
428 raw_spin_lock_init(&hfi_instance->event_lock);
430 cpumask_set_cpu(cpu, hfi_instance->cpus);
433 * Enable the hardware feedback interface and never disable it. See
434 * comment on programming the address of the table.
436 rdmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val);
437 msr_val |= HW_FEEDBACK_CONFIG_HFI_ENABLE_BIT;
438 wrmsrl(MSR_IA32_HW_FEEDBACK_CONFIG, msr_val);
441 mutex_unlock(&hfi_instance_lock);
445 free_pages_exact(hfi_instance->hw_table, hfi_features.nr_table_pages);
450 * intel_hfi_offline() - Disable HFI on @cpu
451 * @cpu: CPU in which the HFI will be disabled
453 * Remove @cpu from those covered by its HFI instance.
455 * On some processors, hardware remembers previous programming settings even
456 * after being reprogrammed. Thus, keep HFI enabled even if all CPUs in the
457 * die/package of @cpu are offline. See note in intel_hfi_online().
459 void intel_hfi_offline(unsigned int cpu)
461 struct hfi_cpu_info *info = &per_cpu(hfi_cpu_info, cpu);
462 struct hfi_instance *hfi_instance;
465 * Check if @cpu as an associated, initialized (i.e., with a non-NULL
466 * header). Also, HFI instances are only initialized if X86_FEATURE_HFI
469 hfi_instance = info->hfi_instance;
473 if (!hfi_instance->hdr)
476 mutex_lock(&hfi_instance_lock);
477 cpumask_clear_cpu(cpu, hfi_instance->cpus);
478 mutex_unlock(&hfi_instance_lock);
481 static __init int hfi_parse_features(void)
483 unsigned int nr_capabilities;
484 union cpuid6_edx edx;
486 if (!boot_cpu_has(X86_FEATURE_HFI))
490 * If we are here we know that CPUID_HFI_LEAF exists. Parse the
491 * supported capabilities and the size of the HFI table.
493 edx.full = cpuid_edx(CPUID_HFI_LEAF);
495 if (!edx.split.capabilities.split.performance) {
496 pr_debug("Performance reporting not supported! Not using HFI\n");
501 * The number of supported capabilities determines the number of
502 * columns in the HFI table. Exclude the reserved bits.
504 edx.split.capabilities.split.__reserved = 0;
505 nr_capabilities = hweight8(edx.split.capabilities.bits);
507 /* The number of 4KB pages required by the table */
508 hfi_features.nr_table_pages = edx.split.table_pages + 1;
511 * The header contains change indications for each supported feature.
512 * The size of the table header is rounded up to be a multiple of 8
515 hfi_features.hdr_size = DIV_ROUND_UP(nr_capabilities, 8) * 8;
518 * Data of each logical processor is also rounded up to be a multiple
521 hfi_features.cpu_stride = DIV_ROUND_UP(nr_capabilities, 8) * 8;
526 void __init intel_hfi_init(void)
528 struct hfi_instance *hfi_instance;
531 if (hfi_parse_features())
534 /* There is one HFI instance per die/package. */
535 max_hfi_instances = topology_max_packages() *
536 topology_max_die_per_package();
539 * This allocation may fail. CPU hotplug callbacks must check
540 * for a null pointer.
542 hfi_instances = kcalloc(max_hfi_instances, sizeof(*hfi_instances),
547 for (i = 0; i < max_hfi_instances; i++) {
548 hfi_instance = &hfi_instances[i];
549 if (!zalloc_cpumask_var(&hfi_instance->cpus, GFP_KERNEL))
553 hfi_updates_wq = create_singlethread_workqueue("hfi-updates");
560 for (j = 0; j < i; ++j) {
561 hfi_instance = &hfi_instances[j];
562 free_cpumask_var(hfi_instance->cpus);
565 kfree(hfi_instances);
566 hfi_instances = NULL;