2 # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
4 # Copyright (C) 2021 Isovalent, Inc.
10 LINUX_ROOT = os.path.abspath(os.path.join(__file__,
11 os.pardir, os.pardir, os.pardir, os.pardir, os.pardir))
12 BPFTOOL_DIR = os.getenv('BPFTOOL_DIR',
13 os.path.join(LINUX_ROOT, 'tools/bpf/bpftool'))
14 BPFTOOL_BASHCOMP_DIR = os.getenv('BPFTOOL_BASHCOMP_DIR',
15 os.path.join(BPFTOOL_DIR, 'bash-completion'))
16 BPFTOOL_DOC_DIR = os.getenv('BPFTOOL_DOC_DIR',
17 os.path.join(BPFTOOL_DIR, 'Documentation'))
18 INCLUDE_DIR = os.getenv('INCLUDE_DIR',
19 os.path.join(LINUX_ROOT, 'tools/include'))
23 class BlockParser(object):
25 A parser for extracting set of values from blocks such as enums.
26 @reader: a pointer to the open file to parse
28 def __init__(self, reader):
31 def search_block(self, start_marker):
33 Search for a given structure in a file.
34 @start_marker: regex marking the beginning of a structure to parse
36 offset = self.reader.tell()
37 array_start = re.search(start_marker, self.reader.read())
38 if array_start is None:
39 raise Exception('Failed to find start of block')
40 self.reader.seek(offset + array_start.start())
42 def parse(self, pattern, end_marker):
44 Parse a block and return a set of values. Values to extract must be
45 on separate lines in the file.
46 @pattern: pattern used to identify the values to extract
47 @end_marker: regex marking the end of the block to parse
51 line = self.reader.readline()
52 if not line or re.match(end_marker, line):
54 capture = pattern.search(line)
55 if capture and pattern.groups >= 1:
56 entries.add(capture.group(1))
59 class ArrayParser(BlockParser):
61 A parser for extracting dicionaries of values from some BPF-related arrays.
62 @reader: a pointer to the open file to parse
63 @array_name: name of the array to parse
65 end_marker = re.compile('^};')
67 def __init__(self, reader, array_name):
68 self.array_name = array_name
69 self.start_marker = re.compile(f'(static )?const char \* const {self.array_name}\[.*\] = {{\n')
70 super().__init__(reader)
72 def search_block(self):
74 Search for the given array in a file.
76 super().search_block(self.start_marker);
80 Parse a block and return data as a dictionary. Items to extract must be
81 on separate lines in the file.
83 pattern = re.compile('\[(BPF_\w*)\]\s*= "(.*)",?$')
86 line = self.reader.readline()
87 if line == '' or re.match(self.end_marker, line):
89 capture = pattern.search(line)
91 entries[capture.group(1)] = capture.group(2)
94 class InlineListParser(BlockParser):
96 A parser for extracting set of values from inline lists.
98 def parse(self, pattern, end_marker):
100 Parse a block and return a set of values. Multiple values to extract
101 can be on a same line in the file.
102 @pattern: pattern used to identify the values to extract
103 @end_marker: regex marking the end of the block to parse
107 line = self.reader.readline()
110 entries.update(pattern.findall(line))
111 if re.search(end_marker, line):
115 class FileExtractor(object):
117 A generic reader for extracting data from a given file. This class contains
118 several helper methods that wrap arround parser objects to extract values
119 from different structures.
120 This class does not offer a way to set a filename, which is expected to be
121 defined in children classes.
124 self.reader = open(self.filename, 'r')
128 Close the file used by the parser.
132 def reset_read(self):
134 Reset the file position indicator for this parser. This is useful when
135 parsing several structures in the file without respecting the order in
136 which those structures appear in the file.
140 def get_types_from_array(self, array_name):
142 Search for and parse an array associating names to BPF_* enum members,
145 const char * const prog_type_name[] = {
146 [BPF_PROG_TYPE_UNSPEC] = "unspec",
147 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter",
148 [BPF_PROG_TYPE_KPROBE] = "kprobe",
151 Return a dictionary with the enum member names as keys and the
152 associated names as values, for example:
154 {'BPF_PROG_TYPE_UNSPEC': 'unspec',
155 'BPF_PROG_TYPE_SOCKET_FILTER': 'socket_filter',
156 'BPF_PROG_TYPE_KPROBE': 'kprobe'}
158 @array_name: name of the array to parse
160 array_parser = ArrayParser(self.reader, array_name)
161 array_parser.search_block()
162 return array_parser.parse()
164 def get_enum(self, enum_name):
166 Search for and parse an enum containing BPF_* members, for example:
169 BPF_PROG_TYPE_UNSPEC,
170 BPF_PROG_TYPE_SOCKET_FILTER,
171 BPF_PROG_TYPE_KPROBE,
174 Return a set containing all member names, for example:
176 {'BPF_PROG_TYPE_UNSPEC',
177 'BPF_PROG_TYPE_SOCKET_FILTER',
178 'BPF_PROG_TYPE_KPROBE'}
180 @enum_name: name of the enum to parse
182 start_marker = re.compile(f'enum {enum_name} {{\n')
183 pattern = re.compile('^\s*(BPF_\w+),?(\s+/\*.*\*/)?$')
184 end_marker = re.compile('^};')
185 parser = BlockParser(self.reader)
186 parser.search_block(start_marker)
187 return parser.parse(pattern, end_marker)
189 def __get_description_list(self, start_marker, pattern, end_marker):
190 parser = InlineListParser(self.reader)
191 parser.search_block(start_marker)
192 return parser.parse(pattern, end_marker)
194 def get_rst_list(self, block_name):
196 Search for and parse a list of type names from RST documentation, for
200 | **socket** | **kprobe** |
204 Return a set containing all type names, for example:
206 {'socket', 'kprobe', 'kretprobe'}
208 @block_name: name of the blog to parse, 'TYPE' in the example
210 start_marker = re.compile(f'\*{block_name}\* := {{')
211 pattern = re.compile('\*\*([\w/-]+)\*\*')
212 end_marker = re.compile('}\n')
213 return self.__get_description_list(start_marker, pattern, end_marker)
215 def get_help_list(self, block_name):
217 Search for and parse a list of type names from a help message in
218 bpftool, for example:
220 " TYPE := { socket | kprobe |\\n"
223 Return a set containing all type names, for example:
225 {'socket', 'kprobe', 'kretprobe'}
227 @block_name: name of the blog to parse, 'TYPE' in the example
229 start_marker = re.compile(f'"\s*{block_name} := {{')
230 pattern = re.compile('([\w/]+) [|}]')
231 end_marker = re.compile('}')
232 return self.__get_description_list(start_marker, pattern, end_marker)
234 def get_help_list_macro(self, macro):
236 Search for and parse a list of values from a help message starting with
237 a macro in bpftool, for example:
239 " " HELP_SPEC_OPTIONS " |\\n"
240 " {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} }\\n"
242 Return a set containing all item names, for example:
244 {'-f', '--bpffs', '-m', '--mapcompat', '-n', '--nomount'}
246 @macro: macro starting the block, 'HELP_SPEC_OPTIONS' in the example
248 start_marker = re.compile(f'"\s*{macro}\s*" [|}}]')
249 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]])')
250 end_marker = re.compile('}\\\\n')
251 return self.__get_description_list(start_marker, pattern, end_marker)
253 def get_bashcomp_list(self, block_name):
255 Search for and parse a list of type names from a variable in bash
256 completion file, for example:
258 local BPFTOOL_PROG_LOAD_TYPES='socket kprobe \\
261 Return a set containing all type names, for example:
263 {'socket', 'kprobe', 'kretprobe'}
265 @block_name: name of the blog to parse, 'TYPE' in the example
267 start_marker = re.compile(f'local {block_name}=\'')
268 pattern = re.compile('(?:.*=\')?([\w/]+)')
269 end_marker = re.compile('\'$')
270 return self.__get_description_list(start_marker, pattern, end_marker)
272 class SourceFileExtractor(FileExtractor):
274 An abstract extractor for a source file with usage message.
275 This class does not offer a way to set a filename, which is expected to be
276 defined in children classes.
278 def get_options(self):
279 return self.get_help_list_macro('HELP_SPEC_OPTIONS')
281 class MainHeaderFileExtractor(SourceFileExtractor):
283 An extractor for bpftool's main.h
285 filename = os.path.join(BPFTOOL_DIR, 'main.h')
287 def get_common_options(self):
289 Parse the list of common options in main.h (options that apply to all
290 commands), which looks to the lists of options in other source files
291 but has different start and end markers:
293 "OPTIONS := { {-j|--json} [{-p|--pretty}] | {-d|--debug} | {-l|--legacy}"
295 Return a set containing all options, such as:
297 {'-p', '-d', '--legacy', '--pretty', '--debug', '--json', '-l', '-j'}
299 start_marker = re.compile(f'"OPTIONS :=')
300 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]"])')
301 end_marker = re.compile('#define')
303 parser = InlineListParser(self.reader)
304 parser.search_block(start_marker)
305 return parser.parse(pattern, end_marker)
307 class ManSubstitutionsExtractor(SourceFileExtractor):
309 An extractor for substitutions.rst
311 filename = os.path.join(BPFTOOL_DOC_DIR, 'substitutions.rst')
313 def get_common_options(self):
315 Parse the list of common options in substitutions.rst (options that
316 apply to all commands).
318 Return a set containing all options, such as:
320 {'-p', '-d', '--legacy', '--pretty', '--debug', '--json', '-l', '-j'}
322 start_marker = re.compile('\|COMMON_OPTIONS\| replace:: {')
323 pattern = re.compile('\*\*([\w/-]+)\*\*')
324 end_marker = re.compile('}$')
326 parser = InlineListParser(self.reader)
327 parser.search_block(start_marker)
328 return parser.parse(pattern, end_marker)
330 class ProgFileExtractor(SourceFileExtractor):
332 An extractor for bpftool's prog.c.
334 filename = os.path.join(BPFTOOL_DIR, 'prog.c')
336 def get_prog_types(self):
337 return self.get_types_from_array('prog_type_name')
339 def get_attach_types(self):
340 return self.get_types_from_array('attach_type_strings')
342 def get_prog_attach_help(self):
343 return self.get_help_list('ATTACH_TYPE')
345 class MapFileExtractor(SourceFileExtractor):
347 An extractor for bpftool's map.c.
349 filename = os.path.join(BPFTOOL_DIR, 'map.c')
351 def get_map_types(self):
352 return self.get_types_from_array('map_type_name')
354 def get_map_help(self):
355 return self.get_help_list('TYPE')
357 class CgroupFileExtractor(SourceFileExtractor):
359 An extractor for bpftool's cgroup.c.
361 filename = os.path.join(BPFTOOL_DIR, 'cgroup.c')
363 def get_prog_attach_help(self):
364 return self.get_help_list('ATTACH_TYPE')
366 class CommonFileExtractor(SourceFileExtractor):
368 An extractor for bpftool's common.c.
370 filename = os.path.join(BPFTOOL_DIR, 'common.c')
374 self.attach_types = {}
376 def get_attach_types(self):
377 if not self.attach_types:
378 self.attach_types = self.get_types_from_array('attach_type_name')
379 return self.attach_types
381 def get_cgroup_attach_types(self):
382 if not self.attach_types:
383 self.get_attach_types()
385 for (key, value) in self.attach_types.items():
386 if key.find('BPF_CGROUP') != -1:
387 cgroup_types[key] = value
390 class GenericSourceExtractor(SourceFileExtractor):
392 An extractor for generic source code files.
396 def __init__(self, filename):
397 self.filename = os.path.join(BPFTOOL_DIR, filename)
400 class BpfHeaderExtractor(FileExtractor):
402 An extractor for the UAPI BPF header.
404 filename = os.path.join(INCLUDE_DIR, 'uapi/linux/bpf.h')
406 def get_prog_types(self):
407 return self.get_enum('bpf_prog_type')
409 def get_map_types(self):
410 return self.get_enum('bpf_map_type')
412 def get_attach_types(self):
413 return self.get_enum('bpf_attach_type')
415 class ManPageExtractor(FileExtractor):
417 An abstract extractor for an RST documentation page.
418 This class does not offer a way to set a filename, which is expected to be
419 defined in children classes.
421 def get_options(self):
422 return self.get_rst_list('OPTIONS')
424 class ManProgExtractor(ManPageExtractor):
426 An extractor for bpftool-prog.rst.
428 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-prog.rst')
430 def get_attach_types(self):
431 return self.get_rst_list('ATTACH_TYPE')
433 class ManMapExtractor(ManPageExtractor):
435 An extractor for bpftool-map.rst.
437 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-map.rst')
439 def get_map_types(self):
440 return self.get_rst_list('TYPE')
442 class ManCgroupExtractor(ManPageExtractor):
444 An extractor for bpftool-cgroup.rst.
446 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-cgroup.rst')
448 def get_attach_types(self):
449 return self.get_rst_list('ATTACH_TYPE')
451 class ManGenericExtractor(ManPageExtractor):
453 An extractor for generic RST documentation pages.
457 def __init__(self, filename):
458 self.filename = os.path.join(BPFTOOL_DIR, filename)
461 class BashcompExtractor(FileExtractor):
463 An extractor for bpftool's bash completion file.
465 filename = os.path.join(BPFTOOL_BASHCOMP_DIR, 'bpftool')
467 def get_prog_attach_types(self):
468 return self.get_bashcomp_list('BPFTOOL_PROG_ATTACH_TYPES')
470 def get_map_types(self):
471 return self.get_bashcomp_list('BPFTOOL_MAP_CREATE_TYPES')
473 def get_cgroup_attach_types(self):
474 return self.get_bashcomp_list('BPFTOOL_CGROUP_ATTACH_TYPES')
476 def verify(first_set, second_set, message):
478 Print all values that differ between two sets.
479 @first_set: one set to compare
480 @second_set: another set to compare
481 @message: message to print for values belonging to only one of the sets
484 diff = first_set.symmetric_difference(second_set)
490 # No arguments supported at this time, but print usage for -h|--help
491 argParser = argparse.ArgumentParser(description="""
492 Verify that bpftool's code, help messages, documentation and bash
493 completion are all in sync on program types, map types, attach types, and
494 options. Also check that bpftool is in sync with the UAPI BPF header.
496 args = argParser.parse_args()
500 bpf_info = BpfHeaderExtractor()
501 ref = bpf_info.get_map_types()
503 map_info = MapFileExtractor()
504 source_map_items = map_info.get_map_types()
505 map_types_enum = set(source_map_items.keys())
507 verify(ref, map_types_enum,
508 f'Comparing BPF header (enum bpf_map_type) and {MapFileExtractor.filename} (map_type_name):')
512 source_map_types = set(source_map_items.values())
513 source_map_types.discard('unspec')
515 help_map_types = map_info.get_map_help()
516 help_map_options = map_info.get_options()
519 man_map_info = ManMapExtractor()
520 man_map_options = man_map_info.get_options()
521 man_map_types = man_map_info.get_map_types()
524 bashcomp_info = BashcompExtractor()
525 bashcomp_map_types = bashcomp_info.get_map_types()
527 verify(source_map_types, help_map_types,
528 f'Comparing {MapFileExtractor.filename} (map_type_name) and {MapFileExtractor.filename} (do_help() TYPE):')
529 verify(source_map_types, man_map_types,
530 f'Comparing {MapFileExtractor.filename} (map_type_name) and {ManMapExtractor.filename} (TYPE):')
531 verify(help_map_options, man_map_options,
532 f'Comparing {MapFileExtractor.filename} (do_help() OPTIONS) and {ManMapExtractor.filename} (OPTIONS):')
533 verify(source_map_types, bashcomp_map_types,
534 f'Comparing {MapFileExtractor.filename} (map_type_name) and {BashcompExtractor.filename} (BPFTOOL_MAP_CREATE_TYPES):')
536 # Program types (enum)
538 ref = bpf_info.get_prog_types()
540 prog_info = ProgFileExtractor()
541 prog_types = set(prog_info.get_prog_types().keys())
543 verify(ref, prog_types,
544 f'Comparing BPF header (enum bpf_prog_type) and {ProgFileExtractor.filename} (prog_type_name):')
546 # Attach types (enum)
548 ref = bpf_info.get_attach_types()
551 common_info = CommonFileExtractor()
552 attach_types = common_info.get_attach_types()
554 verify(ref, attach_types,
555 f'Comparing BPF header (enum bpf_attach_type) and {CommonFileExtractor.filename} (attach_type_name):')
557 # Attach types (names)
559 source_prog_attach_types = set(prog_info.get_attach_types().values())
561 help_prog_attach_types = prog_info.get_prog_attach_help()
562 help_prog_options = prog_info.get_options()
565 man_prog_info = ManProgExtractor()
566 man_prog_options = man_prog_info.get_options()
567 man_prog_attach_types = man_prog_info.get_attach_types()
568 man_prog_info.close()
570 bashcomp_info.reset_read() # We stopped at map types, rewind
571 bashcomp_prog_attach_types = bashcomp_info.get_prog_attach_types()
573 verify(source_prog_attach_types, help_prog_attach_types,
574 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ProgFileExtractor.filename} (do_help() ATTACH_TYPE):')
575 verify(source_prog_attach_types, man_prog_attach_types,
576 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ManProgExtractor.filename} (ATTACH_TYPE):')
577 verify(help_prog_options, man_prog_options,
578 f'Comparing {ProgFileExtractor.filename} (do_help() OPTIONS) and {ManProgExtractor.filename} (OPTIONS):')
579 verify(source_prog_attach_types, bashcomp_prog_attach_types,
580 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_PROG_ATTACH_TYPES):')
582 # Cgroup attach types
584 source_cgroup_attach_types = set(common_info.get_cgroup_attach_types().values())
587 cgroup_info = CgroupFileExtractor()
588 help_cgroup_attach_types = cgroup_info.get_prog_attach_help()
589 help_cgroup_options = cgroup_info.get_options()
592 man_cgroup_info = ManCgroupExtractor()
593 man_cgroup_options = man_cgroup_info.get_options()
594 man_cgroup_attach_types = man_cgroup_info.get_attach_types()
595 man_cgroup_info.close()
597 bashcomp_cgroup_attach_types = bashcomp_info.get_cgroup_attach_types()
598 bashcomp_info.close()
600 verify(source_cgroup_attach_types, help_cgroup_attach_types,
601 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {CgroupFileExtractor.filename} (do_help() ATTACH_TYPE):')
602 verify(source_cgroup_attach_types, man_cgroup_attach_types,
603 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {ManCgroupExtractor.filename} (ATTACH_TYPE):')
604 verify(help_cgroup_options, man_cgroup_options,
605 f'Comparing {CgroupFileExtractor.filename} (do_help() OPTIONS) and {ManCgroupExtractor.filename} (OPTIONS):')
606 verify(source_cgroup_attach_types, bashcomp_cgroup_attach_types,
607 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_CGROUP_ATTACH_TYPES):')
609 # Options for remaining commands
611 for cmd in [ 'btf', 'feature', 'gen', 'iter', 'link', 'net', 'perf', 'struct_ops', ]:
612 source_info = GenericSourceExtractor(cmd + '.c')
613 help_cmd_options = source_info.get_options()
616 man_cmd_info = ManGenericExtractor(os.path.join(BPFTOOL_DOC_DIR, 'bpftool-' + cmd + '.rst'))
617 man_cmd_options = man_cmd_info.get_options()
620 verify(help_cmd_options, man_cmd_options,
621 f'Comparing {source_info.filename} (do_help() OPTIONS) and {man_cmd_info.filename} (OPTIONS):')
623 source_main_info = GenericSourceExtractor('main.c')
624 help_main_options = source_main_info.get_options()
625 source_main_info.close()
627 man_main_info = ManGenericExtractor(os.path.join(BPFTOOL_DOC_DIR, 'bpftool.rst'))
628 man_main_options = man_main_info.get_options()
629 man_main_info.close()
631 verify(help_main_options, man_main_options,
632 f'Comparing {source_main_info.filename} (do_help() OPTIONS) and {man_main_info.filename} (OPTIONS):')
634 # Compare common options (options that apply to all commands)
636 main_hdr_info = MainHeaderFileExtractor()
637 source_common_options = main_hdr_info.get_common_options()
638 main_hdr_info.close()
640 man_substitutions = ManSubstitutionsExtractor()
641 man_common_options = man_substitutions.get_common_options()
642 man_substitutions.close()
644 verify(source_common_options, man_common_options,
645 f'Comparing common options from {main_hdr_info.filename} (HELP_SPEC_OPTIONS) and {man_substitutions.filename}:')
649 if __name__ == "__main__":