1 // SPDX-License-Identifier: GPL-2.0-only
3 * CPU idle driver for Tegra CPUs
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 * Copyright (c) 2011 Google, Inc.
7 * Author: Colin Cross <ccross@android.com>
8 * Gary King <gking@nvidia.com>
10 * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com>
12 * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com>
15 #define pr_fmt(fmt) "tegra-cpuidle: " fmt
17 #include <linux/atomic.h>
18 #include <linux/cpuidle.h>
19 #include <linux/cpumask.h>
20 #include <linux/cpu_pm.h>
21 #include <linux/delay.h>
22 #include <linux/errno.h>
23 #include <linux/platform_device.h>
24 #include <linux/types.h>
26 #include <linux/clk/tegra.h>
27 #include <linux/firmware/trusted_foundations.h>
29 #include <soc/tegra/cpuidle.h>
30 #include <soc/tegra/flowctrl.h>
31 #include <soc/tegra/fuse.h>
32 #include <soc/tegra/irq.h>
33 #include <soc/tegra/pm.h>
34 #include <soc/tegra/pmc.h>
36 #include <asm/cpuidle.h>
37 #include <asm/firmware.h>
38 #include <asm/smp_plat.h>
39 #include <asm/suspend.h>
48 static atomic_t tegra_idle_barrier;
49 static atomic_t tegra_abort_flag;
51 static inline bool tegra_cpuidle_using_firmware(void)
53 return firmware_ops->prepare_idle && firmware_ops->do_idle;
56 static void tegra_cpuidle_report_cpus_state(void)
58 unsigned long cpu, lcpu, csr;
60 for_each_cpu(lcpu, cpu_possible_mask) {
61 cpu = cpu_logical_map(lcpu);
62 csr = flowctrl_read_cpu_csr(cpu);
64 pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n",
65 cpu, cpu_online(lcpu), csr);
69 static int tegra_cpuidle_wait_for_secondary_cpus_parking(void)
71 unsigned int retries = 3;
74 unsigned int delay_us = 10;
75 unsigned int timeout_us = 500 * 1000 / delay_us;
78 * The primary CPU0 core shall wait for the secondaries
79 * shutdown in order to power-off CPU's cluster safely.
80 * The timeout value depends on the current CPU frequency,
81 * it takes about 40-150us in average and over 1000us in
82 * a worst case scenario.
85 if (tegra_cpu_rail_off_ready())
90 } while (timeout_us--);
92 pr_err("secondary CPU taking too long to park\n");
94 tegra_cpuidle_report_cpus_state();
97 pr_err("timed out waiting secondaries to park\n");
102 static void tegra_cpuidle_unpark_secondary_cpus(void)
104 unsigned int cpu, lcpu;
106 for_each_cpu(lcpu, cpu_online_mask) {
107 cpu = cpu_logical_map(lcpu);
110 tegra_enable_cpu_clock(cpu);
111 tegra_cpu_out_of_reset(cpu);
112 flowctrl_write_cpu_halt(cpu, 0);
117 static int tegra_cpuidle_cc6_enter(unsigned int cpu)
122 ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu);
124 ret = tegra_cpuidle_wait_for_secondary_cpus_parking();
126 ret = tegra_pm_enter_lp2();
128 tegra_cpuidle_unpark_secondary_cpus();
134 static int tegra_cpuidle_c7_enter(void)
138 if (tegra_cpuidle_using_firmware()) {
139 err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2);
143 return call_firmware_op(do_idle, 0);
146 return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend);
149 static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev)
151 if (tegra_pending_sgi()) {
153 * CPU got local interrupt that will be lost after GIC's
154 * shutdown because GIC driver doesn't save/restore the
155 * pending SGI state across CPU cluster PM. Abort and retry
158 atomic_set(&tegra_abort_flag, 1);
161 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);
163 if (atomic_read(&tegra_abort_flag)) {
164 cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);
165 atomic_set(&tegra_abort_flag, 0);
172 static int tegra_cpuidle_state_enter(struct cpuidle_device *dev,
173 int index, unsigned int cpu)
178 * CC6 state is the "CPU cluster power-off" state. In order to
179 * enter this state, at first the secondary CPU cores need to be
180 * parked into offline mode, then the last CPU should clean out
181 * remaining dirty cache lines into DRAM and trigger Flow Controller
182 * logic that turns off the cluster's power domain (which includes
183 * CPU cores, GIC and L2 cache).
185 if (index == TEGRA_CC6) {
186 ret = tegra_cpuidle_coupled_barrier(dev);
192 tegra_pm_set_cpu_in_lp2();
197 ret = tegra_cpuidle_c7_enter();
201 ret = tegra_cpuidle_cc6_enter(cpu);
210 tegra_pm_clear_cpu_in_lp2();
216 static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu)
219 * On Tegra30 CPU0 can't be power-gated separately from secondary
220 * cores because it gates the whole CPU cluster.
222 if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30)
225 /* put CPU0 into C1 if C7 is requested and secondaries are online */
226 if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1)
234 static int tegra_cpuidle_enter(struct cpuidle_device *dev,
235 struct cpuidle_driver *drv,
238 unsigned int cpu = cpu_logical_map(dev->cpu);
241 index = tegra_cpuidle_adjust_state_index(index, cpu);
242 if (dev->states_usage[index].disable)
245 if (index == TEGRA_C1)
246 err = arm_cpuidle_simple_enter(dev, drv, index);
248 err = tegra_cpuidle_state_enter(dev, index, cpu);
250 if (err && (err != -EINTR || index != TEGRA_CC6))
251 pr_err_once("failed to enter state %d err: %d\n", index, err);
253 return err ? -1 : index;
256 static void tegra114_enter_s2idle(struct cpuidle_device *dev,
257 struct cpuidle_driver *drv,
260 tegra_cpuidle_enter(dev, drv, index);
264 * The previous versions of Tegra CPUIDLE driver used a different "legacy"
265 * terminology for naming of the idling states, while this driver uses the
268 * Mapping of the old terms into the new ones:
272 * LP3 | C1 (CPU core clock gating)
273 * LP2 | C7 (CPU core power gating)
274 * LP2 | CC6 (CPU cluster power gating)
276 * Note that that the older CPUIDLE driver versions didn't explicitly
277 * differentiate the LP2 states because these states either used the same
278 * code path or because CC6 wasn't supported.
280 static struct cpuidle_driver tegra_idle_driver = {
281 .name = "tegra_idle",
283 [TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600),
285 .enter = tegra_cpuidle_enter,
286 .exit_latency = 2000,
287 .target_residency = 2200,
289 .flags = CPUIDLE_FLAG_TIMER_STOP,
291 .desc = "CPU core powered off",
294 .enter = tegra_cpuidle_enter,
295 .exit_latency = 5000,
296 .target_residency = 10000,
298 .flags = CPUIDLE_FLAG_TIMER_STOP |
299 CPUIDLE_FLAG_COUPLED,
301 .desc = "CPU cluster powered off",
304 .state_count = TEGRA_STATE_COUNT,
305 .safe_state_index = TEGRA_C1,
308 static inline void tegra_cpuidle_disable_state(enum tegra_state state)
310 cpuidle_driver_state_disabled(&tegra_idle_driver, state, true);
314 * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether
315 * they are legacy IRQs or MSI, are lost when CC6 is enabled. To work around
316 * this, simply disable CC6 if the PCI driver and DT node are both enabled.
318 void tegra_cpuidle_pcie_irqs_in_use(void)
320 struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6];
322 if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) ||
323 tegra_get_chip_id() != TEGRA20)
326 pr_info("disabling CC6 state, since PCIe IRQs are in use\n");
327 tegra_cpuidle_disable_state(TEGRA_CC6);
330 static void tegra_cpuidle_setup_tegra114_c7_state(void)
332 struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7];
334 s->enter_s2idle = tegra114_enter_s2idle;
335 s->target_residency = 1000;
336 s->exit_latency = 500;
339 static int tegra_cpuidle_probe(struct platform_device *pdev)
341 /* LP2 could be disabled in device-tree */
342 if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2)
343 tegra_cpuidle_disable_state(TEGRA_CC6);
346 * Required suspend-resume functionality, which is provided by the
347 * Tegra-arch core and PMC driver, is unavailable if PM-sleep option
350 if (!IS_ENABLED(CONFIG_PM_SLEEP)) {
351 if (!tegra_cpuidle_using_firmware())
352 tegra_cpuidle_disable_state(TEGRA_C7);
354 tegra_cpuidle_disable_state(TEGRA_CC6);
358 * Generic WFI state (also known as C1 or LP3) and the coupled CPU
359 * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs.
361 switch (tegra_get_chip_id()) {
363 /* Tegra20 isn't capable to power-off individual CPU cores */
364 tegra_cpuidle_disable_state(TEGRA_C7);
372 tegra_cpuidle_setup_tegra114_c7_state();
374 /* coupled CC6 (LP2) state isn't implemented yet */
375 tegra_cpuidle_disable_state(TEGRA_CC6);
382 return cpuidle_register(&tegra_idle_driver, cpu_possible_mask);
385 static struct platform_driver tegra_cpuidle_driver = {
386 .probe = tegra_cpuidle_probe,
388 .name = "tegra-cpuidle",
391 builtin_platform_driver(tegra_cpuidle_driver);