tools/cgroup/iocost_monitor.py

   1 #!/usr/bin/env drgn
   2 #
   3 # Copyright (C) 2019 Tejun Heo <tj@kernel.org>
   4 # Copyright (C) 2019 Facebook
   5
   6 desc = """
   7 This is a drgn script to monitor the blk-iocost cgroup controller.
   8 See the comment at the top of block/blk-iocost.c for more details.
   9 For drgn, visit https://github.com/osandov/drgn.
  10 """
  11
  12 import sys
  13 import re
  14 import time
  15 import json
  16 import math
  17
  18 import drgn
  19 from drgn import container_of
  20 from drgn.helpers.linux.list import list_for_each_entry,list_empty
  21 from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup
  22
  23 import argparse
  24 parser = argparse.ArgumentParser(description=desc,
  25                                  formatter_class=argparse.RawTextHelpFormatter)
  26 parser.add_argument('devname', metavar='DEV',
  27                     help='Target block device name (e.g. sda)')
  28 parser.add_argument('--cgroup', action='append', metavar='REGEX',
  29                     help='Regex for target cgroups, ')
  30 parser.add_argument('--interval', '-i', metavar='SECONDS', type=float, default=1,
  31                     help='Monitoring interval in seconds')
  32 parser.add_argument('--json', action='store_true',
  33                     help='Output in json')
  34 args = parser.parse_args()
  35
  36 def err(s):
  37     print(s, file=sys.stderr, flush=True)
  38     sys.exit(1)
  39
  40 try:
  41     blkcg_root = prog['blkcg_root']
  42     plid = prog['blkcg_policy_iocost'].plid.value_()
  43 except:
  44     err('The kernel does not have iocost enabled')
  45
  46 IOC_RUNNING     = prog['IOC_RUNNING'].value_()
  47 NR_USAGE_SLOTS  = prog['NR_USAGE_SLOTS'].value_()
  48 HWEIGHT_WHOLE   = prog['HWEIGHT_WHOLE'].value_()
  49 VTIME_PER_SEC   = prog['VTIME_PER_SEC'].value_()
  50 VTIME_PER_USEC  = prog['VTIME_PER_USEC'].value_()
  51 AUTOP_SSD_FAST  = prog['AUTOP_SSD_FAST'].value_()
  52 AUTOP_SSD_DFL   = prog['AUTOP_SSD_DFL'].value_()
  53 AUTOP_SSD_QD1   = prog['AUTOP_SSD_QD1'].value_()
  54 AUTOP_HDD       = prog['AUTOP_HDD'].value_()
  55
  56 autop_names = {
  57     AUTOP_SSD_FAST:        'ssd_fast',
  58     AUTOP_SSD_DFL:         'ssd_dfl',
  59     AUTOP_SSD_QD1:         'ssd_qd1',
  60     AUTOP_HDD:             'hdd',
  61 }
  62
  63 class BlkgIterator:
  64     def blkcg_name(blkcg):
  65         return blkcg.css.cgroup.kn.name.string_().decode('utf-8')
  66
  67     def walk(self, blkcg, q_id, parent_path):
  68         if not self.include_dying and \
  69            not (blkcg.css.flags.value_() & prog['CSS_ONLINE'].value_()):
  70             return
  71
  72         name = BlkgIterator.blkcg_name(blkcg)
  73         path = parent_path + '/' + name if parent_path else name
  74         blkg = drgn.Object(prog, 'struct blkcg_gq',
  75                            address=radix_tree_lookup(blkcg.blkg_tree.address_of_(), q_id))
  76         if not blkg.address_:
  77             return
  78
  79         self.blkgs.append((path if path else '/', blkg))
  80
  81         for c in list_for_each_entry('struct blkcg',
  82                                      blkcg.css.children.address_of_(), 'css.sibling'):
  83             self.walk(c, q_id, path)
  84
  85     def __init__(self, root_blkcg, q_id, include_dying=False):
  86         self.include_dying = include_dying
  87         self.blkgs = []
  88         self.walk(root_blkcg, q_id, '')
  89
  90     def __iter__(self):
  91         return iter(self.blkgs)
  92
  93 class IocStat:
  94     def __init__(self, ioc):
  95         global autop_names
  96
  97         self.enabled = ioc.enabled.value_()
  98         self.running = ioc.running.value_() == IOC_RUNNING
  99         self.period_ms = ioc.period_us.value_() / 1_000
 100         self.period_at = ioc.period_at.value_() / 1_000_000
 101         self.vperiod_at = ioc.period_at_vtime.value_() / VTIME_PER_SEC
 102         self.vrate_pct = ioc.vtime_rate.counter.value_() * 100 / VTIME_PER_USEC
 103         self.busy_level = ioc.busy_level.value_()
 104         self.autop_idx = ioc.autop_idx.value_()
 105         self.user_cost_model = ioc.user_cost_model.value_()
 106         self.user_qos_params = ioc.user_qos_params.value_()
 107
 108         if self.autop_idx in autop_names:
 109             self.autop_name = autop_names[self.autop_idx]
 110         else:
 111             self.autop_name = '?'
 112
 113     def dict(self, now):
 114         return { 'device'               : devname,
 115                  'timestamp'            : str(now),
 116                  'enabled'              : str(int(self.enabled)),
 117                  'running'              : str(int(self.running)),
 118                  'period_ms'            : str(self.period_ms),
 119                  'period_at'            : str(self.period_at),
 120                  'period_vtime_at'      : str(self.vperiod_at),
 121                  'busy_level'           : str(self.busy_level),
 122                  'vrate_pct'            : str(self.vrate_pct), }
 123
 124     def table_preamble_str(self):
 125         state = ('RUN' if self.running else 'IDLE') if self.enabled else 'OFF'
 126         output = f'{devname} {state:4} ' \
 127                  f'per={self.period_ms}ms ' \
 128                  f'cur_per={self.period_at:.3f}:v{self.vperiod_at:.3f} ' \
 129                  f'busy={self.busy_level:+3} ' \
 130                  f'vrate={self.vrate_pct:6.2f}% ' \
 131                  f'params={self.autop_name}'
 132         if self.user_cost_model or self.user_qos_params:
 133             output += f'({"C" if self.user_cost_model else ""}{"Q" if self.user_qos_params else ""})'
 134         return output
 135
 136     def table_header_str(self):
 137         return f'{"":25} active {"weight":>9} {"hweight%":>13} {"inflt%":>6} ' \
 138                f'{"dbt":>3} {"delay":>6} {"usages%"}'
 139
 140 class IocgStat:
 141     def __init__(self, iocg):
 142         ioc = iocg.ioc
 143         blkg = iocg.pd.blkg
 144
 145         self.is_active = not list_empty(iocg.active_list.address_of_())
 146         self.weight = iocg.weight.value_()
 147         self.active = iocg.active.value_()
 148         self.inuse = iocg.inuse.value_()
 149         self.hwa_pct = iocg.hweight_active.value_() * 100 / HWEIGHT_WHOLE
 150         self.hwi_pct = iocg.hweight_inuse.value_() * 100 / HWEIGHT_WHOLE
 151         self.address = iocg.value_()
 152
 153         vdone = iocg.done_vtime.counter.value_()
 154         vtime = iocg.vtime.counter.value_()
 155         vrate = ioc.vtime_rate.counter.value_()
 156         period_vtime = ioc.period_us.value_() * vrate
 157         if period_vtime:
 158             self.inflight_pct = (vtime - vdone) * 100 / period_vtime
 159         else:
 160             self.inflight_pct = 0
 161
 162         # vdebt used to be an atomic64_t and is now u64, support both
 163         try:
 164             self.debt_ms = iocg.abs_vdebt.counter.value_() / VTIME_PER_USEC / 1000
 165         except:
 166             self.debt_ms = iocg.abs_vdebt.value_() / VTIME_PER_USEC / 1000
 167
 168         self.use_delay = blkg.use_delay.counter.value_()
 169         self.delay_ms = blkg.delay_nsec.counter.value_() / 1_000_000
 170
 171         usage_idx = iocg.usage_idx.value_()
 172         self.usages = []
 173         self.usage = 0
 174         for i in range(NR_USAGE_SLOTS):
 175             usage = iocg.usages[(usage_idx + i) % NR_USAGE_SLOTS].value_()
 176             upct = usage * 100 / HWEIGHT_WHOLE
 177             self.usages.append(upct)
 178             self.usage = max(self.usage, upct)
 179
 180     def dict(self, now, path):
 181         out = { 'cgroup'                : path,
 182                 'timestamp'             : str(now),
 183                 'is_active'             : str(int(self.is_active)),
 184                 'weight'                : str(self.weight),
 185                 'weight_active'         : str(self.active),
 186                 'weight_inuse'          : str(self.inuse),
 187                 'hweight_active_pct'    : str(self.hwa_pct),
 188                 'hweight_inuse_pct'     : str(self.hwi_pct),
 189                 'inflight_pct'          : str(self.inflight_pct),
 190                 'debt_ms'               : str(self.debt_ms),
 191                 'use_delay'             : str(self.use_delay),
 192                 'delay_ms'              : str(self.delay_ms),
 193                 'usage_pct'             : str(self.usage),
 194                 'address'               : str(hex(self.address)) }
 195         for i in range(len(self.usages)):
 196             out[f'usage_pct_{i}'] = str(self.usages[i])
 197         return out
 198
 199     def table_row_str(self, path):
 200         out = f'{path[-28:]:28} ' \
 201               f'{"*" if self.is_active else " "} ' \
 202               f'{self.inuse:5}/{self.active:5} ' \
 203               f'{self.hwi_pct:6.2f}/{self.hwa_pct:6.2f} ' \
 204               f'{self.inflight_pct:6.2f} ' \
 205               f'{min(math.ceil(self.debt_ms), 999):3} ' \
 206               f'{min(self.use_delay, 99):2}*'\
 207               f'{min(math.ceil(self.delay_ms), 999):03} '
 208         for u in self.usages:
 209             out += f'{min(round(u), 999):03d}:'
 210         out = out.rstrip(':')
 211         return out
 212
 213 # handle args
 214 table_fmt = not args.json
 215 interval = args.interval
 216 devname = args.devname
 217
 218 if args.json:
 219     table_fmt = False
 220
 221 re_str = None
 222 if args.cgroup:
 223     for r in args.cgroup:
 224         if re_str is None:
 225             re_str = r
 226         else:
 227             re_str += '|' + r
 228
 229 filter_re = re.compile(re_str) if re_str else None
 230
 231 # Locate the roots
 232 q_id = None
 233 root_iocg = None
 234 ioc = None
 235
 236 for i, ptr in radix_tree_for_each(blkcg_root.blkg_tree.address_of_()):
 237     blkg = drgn.Object(prog, 'struct blkcg_gq', address=ptr)
 238     try:
 239         if devname == blkg.q.kobj.parent.name.string_().decode('utf-8'):
 240             q_id = blkg.q.id.value_()
 241             if blkg.pd[plid]:
 242                 root_iocg = container_of(blkg.pd[plid], 'struct ioc_gq', 'pd')
 243                 ioc = root_iocg.ioc
 244             break
 245     except:
 246         pass
 247
 248 if ioc is None:
 249     err(f'Could not find ioc for {devname}');
 250
 251 # Keep printing
 252 while True:
 253     now = time.time()
 254     iocstat = IocStat(ioc)
 255     output = ''
 256
 257     if table_fmt:
 258         output += '\n' + iocstat.table_preamble_str()
 259         output += '\n' + iocstat.table_header_str()
 260     else:
 261         output += json.dumps(iocstat.dict(now))
 262
 263     for path, blkg in BlkgIterator(blkcg_root, q_id):
 264         if filter_re and not filter_re.match(path):
 265             continue
 266         if not blkg.pd[plid]:
 267             continue
 268
 269         iocg = container_of(blkg.pd[plid], 'struct ioc_gq', 'pd')
 270         iocg_stat = IocgStat(iocg)
 271
 272         if not filter_re and not iocg_stat.is_active:
 273             continue
 274
 275         if table_fmt:
 276             output += '\n' + iocg_stat.table_row_str(path)
 277         else:
 278             output += '\n' + json.dumps(iocg_stat.dict(now, path))
 279
 280     print(output)
 281     sys.stdout.flush()
 282     time.sleep(interval)