tools/cgroup/iocost_monitor.py

   1 #!/usr/bin/env drgn
   2 #
   3 # Copyright (C) 2019 Tejun Heo <tj@kernel.org>
   4 # Copyright (C) 2019 Facebook
   5
   6 desc = """
   7 This is a drgn script to monitor the blk-iocost cgroup controller.
   8 See the comment at the top of block/blk-iocost.c for more details.
   9 For drgn, visit https://github.com/osandov/drgn.
  10 """
  11
  12 import sys
  13 import re
  14 import time
  15 import json
  16 import math
  17
  18 import drgn
  19 from drgn import container_of
  20 from drgn.helpers.linux.list import list_for_each_entry,list_empty
  21 from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup
  22
  23 import argparse
  24 parser = argparse.ArgumentParser(description=desc,
  25                                  formatter_class=argparse.RawTextHelpFormatter)
  26 parser.add_argument('devname', metavar='DEV',
  27                     help='Target block device name (e.g. sda)')
  28 parser.add_argument('--cgroup', action='append', metavar='REGEX',
  29                     help='Regex for target cgroups, ')
  30 parser.add_argument('--interval', '-i', metavar='SECONDS', type=float, default=1,
  31                     help='Monitoring interval in seconds (0 exits immediately '
  32                     'after checking requirements)')
  33 parser.add_argument('--json', action='store_true',
  34                     help='Output in json')
  35 args = parser.parse_args()
  36
  37 def err(s):
  38     print(s, file=sys.stderr, flush=True)
  39     sys.exit(1)
  40
  41 try:
  42     blkcg_root = prog['blkcg_root']
  43     plid = prog['blkcg_policy_iocost'].plid.value_()
  44 except:
  45     err('The kernel does not have iocost enabled')
  46
  47 IOC_RUNNING     = prog['IOC_RUNNING'].value_()
  48 WEIGHT_ONE      = prog['WEIGHT_ONE'].value_()
  49 VTIME_PER_SEC   = prog['VTIME_PER_SEC'].value_()
  50 VTIME_PER_USEC  = prog['VTIME_PER_USEC'].value_()
  51 AUTOP_SSD_FAST  = prog['AUTOP_SSD_FAST'].value_()
  52 AUTOP_SSD_DFL   = prog['AUTOP_SSD_DFL'].value_()
  53 AUTOP_SSD_QD1   = prog['AUTOP_SSD_QD1'].value_()
  54 AUTOP_HDD       = prog['AUTOP_HDD'].value_()
  55
  56 autop_names = {
  57     AUTOP_SSD_FAST:        'ssd_fast',
  58     AUTOP_SSD_DFL:         'ssd_dfl',
  59     AUTOP_SSD_QD1:         'ssd_qd1',
  60     AUTOP_HDD:             'hdd',
  61 }
  62
  63 class BlkgIterator:
  64     def __init__(self, root_blkcg, q_id, include_dying=False):
  65         self.include_dying = include_dying
  66         self.blkgs = []
  67         self.walk(root_blkcg, q_id, '')
  68
  69     def blkcg_name(blkcg):
  70         return blkcg.css.cgroup.kn.name.string_().decode('utf-8')
  71
  72     def walk(self, blkcg, q_id, parent_path):
  73         if not self.include_dying and \
  74            not (blkcg.css.flags.value_() & prog['CSS_ONLINE'].value_()):
  75             return
  76
  77         name = BlkgIterator.blkcg_name(blkcg)
  78         path = parent_path + '/' + name if parent_path else name
  79         blkg = drgn.Object(prog, 'struct blkcg_gq',
  80                            address=radix_tree_lookup(blkcg.blkg_tree.address_of_(), q_id))
  81         if not blkg.address_:
  82             return
  83
  84         self.blkgs.append((path if path else '/', blkg))
  85
  86         for c in list_for_each_entry('struct blkcg',
  87                                      blkcg.css.children.address_of_(), 'css.sibling'):
  88             self.walk(c, q_id, path)
  89
  90     def __iter__(self):
  91         return iter(self.blkgs)
  92
  93 class IocStat:
  94     def __init__(self, ioc):
  95         global autop_names
  96
  97         self.enabled = ioc.enabled.value_()
  98         self.running = ioc.running.value_() == IOC_RUNNING
  99         self.period_ms = ioc.period_us.value_() / 1_000
 100         self.period_at = ioc.period_at.value_() / 1_000_000
 101         self.vperiod_at = ioc.period_at_vtime.value_() / VTIME_PER_SEC
 102         self.vrate_pct = ioc.vtime_base_rate.value_() * 100 / VTIME_PER_USEC
 103         self.ivrate_pct = ioc.vtime_rate.counter.value_() * 100 / VTIME_PER_USEC
 104         self.busy_level = ioc.busy_level.value_()
 105         self.autop_idx = ioc.autop_idx.value_()
 106         self.user_cost_model = ioc.user_cost_model.value_()
 107         self.user_qos_params = ioc.user_qos_params.value_()
 108
 109         if self.autop_idx in autop_names:
 110             self.autop_name = autop_names[self.autop_idx]
 111         else:
 112             self.autop_name = '?'
 113
 114     def dict(self, now):
 115         return { 'device'               : devname,
 116                  'timestamp'            : now,
 117                  'enabled'              : self.enabled,
 118                  'running'              : self.running,
 119                  'period_ms'            : self.period_ms,
 120                  'period_at'            : self.period_at,
 121                  'period_vtime_at'      : self.vperiod_at,
 122                  'busy_level'           : self.busy_level,
 123                  'vrate_pct'            : self.vrate_pct,
 124                  'ivrate_pct'           : self.ivrate_pct,
 125                 }
 126
 127     def table_preamble_str(self):
 128         state = ('RUN' if self.running else 'IDLE') if self.enabled else 'OFF'
 129         output = f'{devname} {state:4} ' \
 130                  f'per={self.period_ms}ms ' \
 131                  f'cur_per={self.period_at:.3f}:v{self.vperiod_at:.3f} ' \
 132                  f'busy={self.busy_level:+3} ' \
 133                  f'vrate={self.vrate_pct:6.2f}%:{self.ivrate_pct:6.2f}% ' \
 134                  f'params={self.autop_name}'
 135         if self.user_cost_model or self.user_qos_params:
 136             output += f'({"C" if self.user_cost_model else ""}{"Q" if self.user_qos_params else ""})'
 137         return output
 138
 139     def table_header_str(self):
 140         return f'{"":25} active {"weight":>9} {"hweight%":>13} {"inflt%":>6} ' \
 141                f'{"usage%":>6} {"wait":>7} {"debt":>7} {"delay":>7}'
 142
 143 class IocgStat:
 144     def __init__(self, iocg):
 145         ioc = iocg.ioc
 146         blkg = iocg.pd.blkg
 147
 148         self.is_active = not list_empty(iocg.active_list.address_of_())
 149         self.weight = iocg.weight.value_() / WEIGHT_ONE
 150         self.active = iocg.active.value_() / WEIGHT_ONE
 151         self.inuse = iocg.inuse.value_() / WEIGHT_ONE
 152         self.hwa_pct = iocg.hweight_active.value_() * 100 / WEIGHT_ONE
 153         self.hwi_pct = iocg.hweight_inuse.value_() * 100 / WEIGHT_ONE
 154         self.address = iocg.value_()
 155
 156         vdone = iocg.done_vtime.counter.value_()
 157         vtime = iocg.vtime.counter.value_()
 158         vrate = ioc.vtime_rate.counter.value_()
 159         period_vtime = ioc.period_us.value_() * vrate
 160         if period_vtime:
 161             self.inflight_pct = (vtime - vdone) * 100 / period_vtime
 162         else:
 163             self.inflight_pct = 0
 164
 165         self.usage = (100 * iocg.usage_delta_us.value_() /
 166                       ioc.period_us.value_()) if self.active else 0
 167         self.wait_ms = (iocg.stat.wait_us.value_() -
 168                         iocg.last_stat.wait_us.value_()) / 1000
 169         self.debt_ms = iocg.abs_vdebt.value_() / VTIME_PER_USEC / 1000
 170         if blkg.use_delay.counter.value_() != 0:
 171             self.delay_ms = blkg.delay_nsec.counter.value_() / 1_000_000
 172         else:
 173             self.delay_ms = 0
 174
 175     def dict(self, now, path):
 176         out = { 'cgroup'                : path,
 177                 'timestamp'             : now,
 178                 'is_active'             : self.is_active,
 179                 'weight'                : self.weight,
 180                 'weight_active'         : self.active,
 181                 'weight_inuse'          : self.inuse,
 182                 'hweight_active_pct'    : self.hwa_pct,
 183                 'hweight_inuse_pct'     : self.hwi_pct,
 184                 'inflight_pct'          : self.inflight_pct,
 185                 'usage_pct'             : self.usage,
 186                 'wait_ms'               : self.wait_ms,
 187                 'debt_ms'               : self.debt_ms,
 188                 'delay_ms'              : self.delay_ms,
 189                 'address'               : self.address }
 190         return out
 191
 192     def table_row_str(self, path):
 193         out = f'{path[-28:]:28} ' \
 194               f'{"*" if self.is_active else " "} ' \
 195               f'{round(self.inuse):5}/{round(self.active):5} ' \
 196               f'{self.hwi_pct:6.2f}/{self.hwa_pct:6.2f} ' \
 197               f'{self.inflight_pct:6.2f} ' \
 198               f'{min(self.usage, 999):6.2f} ' \
 199               f'{self.wait_ms:7.2f} ' \
 200               f'{self.debt_ms:7.2f} ' \
 201               f'{self.delay_ms:7.2f}'
 202         out = out.rstrip(':')
 203         return out
 204
 205 # handle args
 206 table_fmt = not args.json
 207 interval = args.interval
 208 devname = args.devname
 209
 210 if args.json:
 211     table_fmt = False
 212
 213 re_str = None
 214 if args.cgroup:
 215     for r in args.cgroup:
 216         if re_str is None:
 217             re_str = r
 218         else:
 219             re_str += '|' + r
 220
 221 filter_re = re.compile(re_str) if re_str else None
 222
 223 # Locate the roots
 224 q_id = None
 225 root_iocg = None
 226 ioc = None
 227
 228 for i, ptr in radix_tree_for_each(blkcg_root.blkg_tree.address_of_()):
 229     blkg = drgn.Object(prog, 'struct blkcg_gq', address=ptr)
 230     try:
 231         if devname == blkg.q.mq_kobj.parent.name.string_().decode('utf-8'):
 232             q_id = blkg.q.id.value_()
 233             if blkg.pd[plid]:
 234                 root_iocg = container_of(blkg.pd[plid], 'struct ioc_gq', 'pd')
 235                 ioc = root_iocg.ioc
 236             break
 237     except:
 238         pass
 239
 240 if ioc is None:
 241     err(f'Could not find ioc for {devname}');
 242
 243 if interval == 0:
 244     sys.exit(0)
 245
 246 # Keep printing
 247 while True:
 248     now = time.time()
 249     iocstat = IocStat(ioc)
 250     output = ''
 251
 252     if table_fmt:
 253         output += '\n' + iocstat.table_preamble_str()
 254         output += '\n' + iocstat.table_header_str()
 255     else:
 256         output += json.dumps(iocstat.dict(now))
 257
 258     for path, blkg in BlkgIterator(blkcg_root, q_id):
 259         if filter_re and not filter_re.match(path):
 260             continue
 261         if not blkg.pd[plid]:
 262             continue
 263
 264         iocg = container_of(blkg.pd[plid], 'struct ioc_gq', 'pd')
 265         iocg_stat = IocgStat(iocg)
 266
 267         if not filter_re and not iocg_stat.is_active:
 268             continue
 269
 270         if table_fmt:
 271             output += '\n' + iocg_stat.table_row_str(path)
 272         else:
 273             output += '\n' + json.dumps(iocg_stat.dict(now, path))
 274
 275     print(output)
 276     sys.stdout.flush()
 277     time.sleep(interval)