kernel/delayacct.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* delayacct.c - per-task delay accounting
   3  *
   4  * Copyright (C) Shailabh Nagar, IBM Corp. 2006
   5  */
   6
   7 #include <linux/sched.h>
   8 #include <linux/sched/task.h>
   9 #include <linux/sched/cputime.h>
  10 #include <linux/sched/clock.h>
  11 #include <linux/slab.h>
  12 #include <linux/taskstats.h>
  13 #include <linux/sysctl.h>
  14 #include <linux/delayacct.h>
  15 #include <linux/module.h>
  16
  17 DEFINE_STATIC_KEY_FALSE(delayacct_key);
  18 int delayacct_on __read_mostly; /* Delay accounting turned on/off */
  19 struct kmem_cache *delayacct_cache;
  20
  21 static int __init delayacct_setup_enable(char *str)
  22 {
  23         delayacct_on = 1;
  24         return 1;
  25 }
  26 __setup("delayacct", delayacct_setup_enable);
  27
  28 void delayacct_init(void)
  29 {
  30         delayacct_cache = KMEM_CACHE(task_delay_info, SLAB_PANIC|SLAB_ACCOUNT);
  31         delayacct_tsk_init(&init_task);
  32         if (delayacct_on)
  33                 static_branch_enable(&delayacct_key);
  34 }
  35
  36 void __delayacct_tsk_init(struct task_struct *tsk)
  37 {
  38         tsk->delays = kmem_cache_zalloc(delayacct_cache, GFP_KERNEL);
  39         if (tsk->delays)
  40                 raw_spin_lock_init(&tsk->delays->lock);
  41 }
  42
  43 /*
  44  * Finish delay accounting for a statistic using its timestamps (@start),
  45  * accumalator (@total) and @count
  46  */
  47 static void delayacct_end(raw_spinlock_t *lock, u64 *start, u64 *total, u32 *count)
  48 {
  49         s64 ns = local_clock() - *start;
  50         unsigned long flags;
  51
  52         if (ns > 0) {
  53                 raw_spin_lock_irqsave(lock, flags);
  54                 *total += ns;
  55                 (*count)++;
  56                 raw_spin_unlock_irqrestore(lock, flags);
  57         }
  58 }
  59
  60 void __delayacct_blkio_start(void)
  61 {
  62         current->delays->blkio_start = local_clock();
  63 }
  64
  65 /*
  66  * We cannot rely on the `current` macro, as we haven't yet switched back to
  67  * the process being woken.
  68  */
  69 void __delayacct_blkio_end(struct task_struct *p)
  70 {
  71         struct task_delay_info *delays = p->delays;
  72         u64 *total;
  73         u32 *count;
  74
  75         if (p->delays->flags & DELAYACCT_PF_SWAPIN) {
  76                 total = &delays->swapin_delay;
  77                 count = &delays->swapin_count;
  78         } else {
  79                 total = &delays->blkio_delay;
  80                 count = &delays->blkio_count;
  81         }
  82
  83         delayacct_end(&delays->lock, &delays->blkio_start, total, count);
  84 }
  85
  86 int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
  87 {
  88         u64 utime, stime, stimescaled, utimescaled;
  89         unsigned long long t2, t3;
  90         unsigned long flags, t1;
  91         s64 tmp;
  92
  93         task_cputime(tsk, &utime, &stime);
  94         tmp = (s64)d->cpu_run_real_total;
  95         tmp += utime + stime;
  96         d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp;
  97
  98         task_cputime_scaled(tsk, &utimescaled, &stimescaled);
  99         tmp = (s64)d->cpu_scaled_run_real_total;
 100         tmp += utimescaled + stimescaled;
 101         d->cpu_scaled_run_real_total =
 102                 (tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp;
 103
 104         /*
 105          * No locking available for sched_info (and too expensive to add one)
 106          * Mitigate by taking snapshot of values
 107          */
 108         t1 = tsk->sched_info.pcount;
 109         t2 = tsk->sched_info.run_delay;
 110         t3 = tsk->se.sum_exec_runtime;
 111
 112         d->cpu_count += t1;
 113
 114         tmp = (s64)d->cpu_delay_total + t2;
 115         d->cpu_delay_total = (tmp < (s64)d->cpu_delay_total) ? 0 : tmp;
 116
 117         tmp = (s64)d->cpu_run_virtual_total + t3;
 118         d->cpu_run_virtual_total =
 119                 (tmp < (s64)d->cpu_run_virtual_total) ? 0 : tmp;
 120
 121         if (!tsk->delays)
 122                 return 0;
 123
 124         /* zero XXX_total, non-zero XXX_count implies XXX stat overflowed */
 125
 126         raw_spin_lock_irqsave(&tsk->delays->lock, flags);
 127         tmp = d->blkio_delay_total + tsk->delays->blkio_delay;
 128         d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp;
 129         tmp = d->swapin_delay_total + tsk->delays->swapin_delay;
 130         d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp;
 131         tmp = d->freepages_delay_total + tsk->delays->freepages_delay;
 132         d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp;
 133         tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay;
 134         d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp;
 135         d->blkio_count += tsk->delays->blkio_count;
 136         d->swapin_count += tsk->delays->swapin_count;
 137         d->freepages_count += tsk->delays->freepages_count;
 138         d->thrashing_count += tsk->delays->thrashing_count;
 139         raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
 140
 141         return 0;
 142 }
 143
 144 __u64 __delayacct_blkio_ticks(struct task_struct *tsk)
 145 {
 146         __u64 ret;
 147         unsigned long flags;
 148
 149         raw_spin_lock_irqsave(&tsk->delays->lock, flags);
 150         ret = nsec_to_clock_t(tsk->delays->blkio_delay +
 151                                 tsk->delays->swapin_delay);
 152         raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
 153         return ret;
 154 }
 155
 156 void __delayacct_freepages_start(void)
 157 {
 158         current->delays->freepages_start = local_clock();
 159 }
 160
 161 void __delayacct_freepages_end(void)
 162 {
 163         delayacct_end(&current->delays->lock,
 164                       &current->delays->freepages_start,
 165                       &current->delays->freepages_delay,
 166                       &current->delays->freepages_count);
 167 }
 168
 169 void __delayacct_thrashing_start(void)
 170 {
 171         current->delays->thrashing_start = local_clock();
 172 }
 173
 174 void __delayacct_thrashing_end(void)
 175 {
 176         delayacct_end(&current->delays->lock,
 177                       &current->delays->thrashing_start,
 178                       &current->delays->thrashing_delay,
 179                       &current->delays->thrashing_count);
 180 }