1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright 2018 Linaro Limited
5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org>
7 * The idle injection framework provides a way to force CPUs to enter idle
8 * states for a specified fraction of time over a specified period.
10 * It relies on the smpboot kthreads feature providing common code for CPU
11 * hotplug and thread [un]parking.
13 * All of the kthreads used for idle injection are created at init time.
15 * Next, the users of the idle injection framework provide a cpumask via
16 * its register function. The kthreads will be synchronized with respect to
19 * The idle + run duration is specified via separate helpers and that allows
20 * idle injection to be started.
22 * The idle injection kthreads will call play_idle_precise() with the idle
23 * duration and max allowed latency specified as per the above.
25 * After all of them have been woken up, a timer is set to start the next idle
28 * The timer interrupt handler will wake up the idle injection kthreads for
29 * all of the CPUs in the cpumask provided by the user.
31 * Idle injection is stopped synchronously and no leftover idle injection
32 * kthread activity after its completion is guaranteed.
34 * It is up to the user of this framework to provide a lock for higher-level
35 * synchronization to prevent race conditions like starting idle injection
36 * while unregistering from the framework.
38 #define pr_fmt(fmt) "ii_dev: " fmt
40 #include <linux/cpu.h>
41 #include <linux/hrtimer.h>
42 #include <linux/kthread.h>
43 #include <linux/sched.h>
44 #include <linux/slab.h>
45 #include <linux/smpboot.h>
46 #include <linux/idle_inject.h>
48 #include <uapi/linux/sched/types.h>
51 * struct idle_inject_thread - task on/off switch structure
52 * @tsk: task injecting the idle cycles
53 * @should_run: whether or not to run the task (for the smpboot kthread API)
55 struct idle_inject_thread {
56 struct task_struct *tsk;
61 * struct idle_inject_device - idle injection data
62 * @timer: idle injection period timer
63 * @idle_duration_us: duration of CPU idle time to inject
64 * @run_duration_us: duration of CPU run time to allow
65 * @latency_us: max allowed latency
66 * @cpumask: mask of CPUs affected by idle injection
68 struct idle_inject_device {
70 unsigned int idle_duration_us;
71 unsigned int run_duration_us;
72 unsigned int latency_us;
73 unsigned long cpumask[];
76 static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);
77 static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);
80 * idle_inject_wakeup - Wake up idle injection threads
81 * @ii_dev: target idle injection device
83 * Every idle injection task associated with the given idle injection device
84 * and running on an online CPU will be woken up.
86 static void idle_inject_wakeup(struct idle_inject_device *ii_dev)
88 struct idle_inject_thread *iit;
91 for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {
92 iit = per_cpu_ptr(&idle_inject_thread, cpu);
94 wake_up_process(iit->tsk);
99 * idle_inject_timer_fn - idle injection timer function
100 * @timer: idle injection hrtimer
102 * This function is called when the idle injection timer expires. It wakes up
103 * idle injection tasks associated with the timer and they, in turn, invoke
104 * play_idle_precise() to inject a specified amount of CPU idle time.
106 * Return: HRTIMER_RESTART.
108 static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)
110 unsigned int duration_us;
111 struct idle_inject_device *ii_dev =
112 container_of(timer, struct idle_inject_device, timer);
114 duration_us = READ_ONCE(ii_dev->run_duration_us);
115 duration_us += READ_ONCE(ii_dev->idle_duration_us);
117 idle_inject_wakeup(ii_dev);
119 hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));
121 return HRTIMER_RESTART;
125 * idle_inject_fn - idle injection work function
126 * @cpu: the CPU owning the task
128 * This function calls play_idle_precise() to inject a specified amount of CPU
131 static void idle_inject_fn(unsigned int cpu)
133 struct idle_inject_device *ii_dev;
134 struct idle_inject_thread *iit;
136 ii_dev = per_cpu(idle_inject_device, cpu);
137 iit = per_cpu_ptr(&idle_inject_thread, cpu);
140 * Let the smpboot main loop know that the task should not run again.
144 play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,
145 READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);
149 * idle_inject_set_duration - idle and run duration update helper
150 * @ii_dev: idle injection control device structure
151 * @run_duration_us: CPU run time to allow in microseconds
152 * @idle_duration_us: CPU idle time to inject in microseconds
154 void idle_inject_set_duration(struct idle_inject_device *ii_dev,
155 unsigned int run_duration_us,
156 unsigned int idle_duration_us)
158 if (run_duration_us && idle_duration_us) {
159 WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);
160 WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);
165 * idle_inject_get_duration - idle and run duration retrieval helper
166 * @ii_dev: idle injection control device structure
167 * @run_duration_us: memory location to store the current CPU run time
168 * @idle_duration_us: memory location to store the current CPU idle time
170 void idle_inject_get_duration(struct idle_inject_device *ii_dev,
171 unsigned int *run_duration_us,
172 unsigned int *idle_duration_us)
174 *run_duration_us = READ_ONCE(ii_dev->run_duration_us);
175 *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
179 * idle_inject_set_latency - set the maximum latency allowed
180 * @ii_dev: idle injection control device structure
181 * @latency_us: set the latency requirement for the idle state
183 void idle_inject_set_latency(struct idle_inject_device *ii_dev,
184 unsigned int latency_us)
186 WRITE_ONCE(ii_dev->latency_us, latency_us);
190 * idle_inject_start - start idle injections
191 * @ii_dev: idle injection control device structure
193 * The function starts idle injection by first waking up all of the idle
194 * injection kthreads associated with @ii_dev to let them inject CPU idle time
195 * sets up a timer to start the next idle injection period.
197 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.
199 int idle_inject_start(struct idle_inject_device *ii_dev)
201 unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);
202 unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);
204 if (!idle_duration_us || !run_duration_us)
207 pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",
208 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
210 idle_inject_wakeup(ii_dev);
212 hrtimer_start(&ii_dev->timer,
213 ns_to_ktime((idle_duration_us + run_duration_us) *
221 * idle_inject_stop - stops idle injections
222 * @ii_dev: idle injection control device structure
224 * The function stops idle injection and waits for the threads to finish work.
225 * If CPU idle time is being injected when this function runs, then it will
226 * wait until the end of the cycle.
228 * When it returns, there is no more idle injection kthread activity. The
229 * kthreads are scheduled out and the periodic timer is off.
231 void idle_inject_stop(struct idle_inject_device *ii_dev)
233 struct idle_inject_thread *iit;
236 pr_debug("Stopping idle injection on CPUs '%*pbl'\n",
237 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));
239 hrtimer_cancel(&ii_dev->timer);
242 * Stopping idle injection requires all of the idle injection kthreads
243 * associated with the given cpumask to be parked and stay that way, so
244 * prevent CPUs from going online at this point. Any CPUs going online
245 * after the loop below will be covered by clearing the should_run flag
246 * that will cause the smpboot main loop to schedule them out.
248 cpu_hotplug_disable();
251 * Iterate over all (online + offline) CPUs here in case one of them
252 * goes offline with the should_run flag set so as to prevent its idle
253 * injection kthread from running when the CPU goes online again after
254 * the ii_dev has been freed.
256 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
257 iit = per_cpu_ptr(&idle_inject_thread, cpu);
260 wait_task_inactive(iit->tsk, TASK_ANY);
263 cpu_hotplug_enable();
267 * idle_inject_setup - prepare the current task for idle injection
270 * Called once, this function is in charge of setting the current task's
271 * scheduler parameters to make it an RT task.
273 static void idle_inject_setup(unsigned int cpu)
275 sched_set_fifo(current);
279 * idle_inject_should_run - function helper for the smpboot API
280 * @cpu: CPU the kthread is running on
282 * Return: whether or not the thread can run.
284 static int idle_inject_should_run(unsigned int cpu)
286 struct idle_inject_thread *iit =
287 per_cpu_ptr(&idle_inject_thread, cpu);
289 return iit->should_run;
293 * idle_inject_register - initialize idle injection on a set of CPUs
294 * @cpumask: CPUs to be affected by idle injection
296 * This function creates an idle injection control device structure for the
297 * given set of CPUs and initializes the timer associated with it. It does not
298 * start any injection cycles.
300 * Return: NULL if memory allocation fails, idle injection control device
301 * pointer on success.
303 struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)
305 struct idle_inject_device *ii_dev;
308 ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);
312 cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);
313 hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
314 ii_dev->timer.function = idle_inject_timer_fn;
315 ii_dev->latency_us = UINT_MAX;
317 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {
319 if (per_cpu(idle_inject_device, cpu)) {
320 pr_err("cpu%d is already registered\n", cpu);
324 per_cpu(idle_inject_device, cpu) = ii_dev;
330 for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {
333 per_cpu(idle_inject_device, cpu_rb) = NULL;
342 * idle_inject_unregister - unregister idle injection control device
343 * @ii_dev: idle injection control device to unregister
345 * The function stops idle injection for the given control device,
346 * unregisters its kthreads and frees memory allocated when that device was
349 void idle_inject_unregister(struct idle_inject_device *ii_dev)
353 idle_inject_stop(ii_dev);
355 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))
356 per_cpu(idle_inject_device, cpu) = NULL;
361 static struct smp_hotplug_thread idle_inject_threads = {
362 .store = &idle_inject_thread.tsk,
363 .setup = idle_inject_setup,
364 .thread_fn = idle_inject_fn,
365 .thread_comm = "idle_inject/%u",
366 .thread_should_run = idle_inject_should_run,
369 static int __init idle_inject_init(void)
371 return smpboot_register_percpu_thread(&idle_inject_threads);
373 early_initcall(idle_inject_init);