1#!/usr/bin/env python3 2# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 3# 4# Copyright (C) 2021 Isovalent, Inc. 5 6import argparse 7import re 8import os, sys 9 10LINUX_ROOT = os.path.abspath(os.path.join(__file__, 11 os.pardir, os.pardir, os.pardir, os.pardir, os.pardir)) 12BPFTOOL_DIR = os.getenv('BPFTOOL_DIR', 13 os.path.join(LINUX_ROOT, 'tools/bpf/bpftool')) 14BPFTOOL_BASHCOMP_DIR = os.getenv('BPFTOOL_BASHCOMP_DIR', 15 os.path.join(BPFTOOL_DIR, 'bash-completion')) 16BPFTOOL_DOC_DIR = os.getenv('BPFTOOL_DOC_DIR', 17 os.path.join(BPFTOOL_DIR, 'Documentation')) 18INCLUDE_DIR = os.getenv('INCLUDE_DIR', 19 os.path.join(LINUX_ROOT, 'tools/include')) 20 21retval = 0 22 23class BlockParser(object): 24 """ 25 A parser for extracting set of values from blocks such as enums. 26 @reader: a pointer to the open file to parse 27 """ 28 def __init__(self, reader): 29 self.reader = reader 30 31 def search_block(self, start_marker): 32 """ 33 Search for a given structure in a file. 34 @start_marker: regex marking the beginning of a structure to parse 35 """ 36 offset = self.reader.tell() 37 array_start = re.search(start_marker, self.reader.read()) 38 if array_start is None: 39 raise Exception('Failed to find start of block') 40 self.reader.seek(offset + array_start.start()) 41 42 def parse(self, pattern, end_marker): 43 """ 44 Parse a block and return a set of values. Values to extract must be 45 on separate lines in the file. 46 @pattern: pattern used to identify the values to extract 47 @end_marker: regex marking the end of the block to parse 48 """ 49 entries = set() 50 while True: 51 line = self.reader.readline() 52 if not line or re.match(end_marker, line): 53 break 54 capture = pattern.search(line) 55 if capture and pattern.groups >= 1: 56 entries.add(capture.group(1)) 57 return entries 58 59class ArrayParser(BlockParser): 60 """ 61 A parser for extracting dicionaries of values from some BPF-related arrays. 62 @reader: a pointer to the open file to parse 63 @array_name: name of the array to parse 64 """ 65 end_marker = re.compile('^};') 66 67 def __init__(self, reader, array_name): 68 self.array_name = array_name 69 self.start_marker = re.compile(f'(static )?const char \* const {self.array_name}\[.*\] = {{\n') 70 super().__init__(reader) 71 72 def search_block(self): 73 """ 74 Search for the given array in a file. 75 """ 76 super().search_block(self.start_marker); 77 78 def parse(self): 79 """ 80 Parse a block and return data as a dictionary. Items to extract must be 81 on separate lines in the file. 82 """ 83 pattern = re.compile('\[(BPF_\w*)\]\s*= "(.*)",?$') 84 entries = {} 85 while True: 86 line = self.reader.readline() 87 if line == '' or re.match(self.end_marker, line): 88 break 89 capture = pattern.search(line) 90 if capture: 91 entries[capture.group(1)] = capture.group(2) 92 return entries 93 94class InlineListParser(BlockParser): 95 """ 96 A parser for extracting set of values from inline lists. 97 """ 98 def parse(self, pattern, end_marker): 99 """ 100 Parse a block and return a set of values. Multiple values to extract 101 can be on a same line in the file. 102 @pattern: pattern used to identify the values to extract 103 @end_marker: regex marking the end of the block to parse 104 """ 105 entries = set() 106 while True: 107 line = self.reader.readline() 108 if not line: 109 break 110 entries.update(pattern.findall(line)) 111 if re.search(end_marker, line): 112 break 113 return entries 114 115class FileExtractor(object): 116 """ 117 A generic reader for extracting data from a given file. This class contains 118 several helper methods that wrap arround parser objects to extract values 119 from different structures. 120 This class does not offer a way to set a filename, which is expected to be 121 defined in children classes. 122 """ 123 def __init__(self): 124 self.reader = open(self.filename, 'r') 125 126 def close(self): 127 """ 128 Close the file used by the parser. 129 """ 130 self.reader.close() 131 132 def reset_read(self): 133 """ 134 Reset the file position indicator for this parser. This is useful when 135 parsing several structures in the file without respecting the order in 136 which those structures appear in the file. 137 """ 138 self.reader.seek(0) 139 140 def get_types_from_array(self, array_name): 141 """ 142 Search for and parse an array associating names to BPF_* enum members, 143 for example: 144 145 const char * const prog_type_name[] = { 146 [BPF_PROG_TYPE_UNSPEC] = "unspec", 147 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", 148 [BPF_PROG_TYPE_KPROBE] = "kprobe", 149 }; 150 151 Return a dictionary with the enum member names as keys and the 152 associated names as values, for example: 153 154 {'BPF_PROG_TYPE_UNSPEC': 'unspec', 155 'BPF_PROG_TYPE_SOCKET_FILTER': 'socket_filter', 156 'BPF_PROG_TYPE_KPROBE': 'kprobe'} 157 158 @array_name: name of the array to parse 159 """ 160 array_parser = ArrayParser(self.reader, array_name) 161 array_parser.search_block() 162 return array_parser.parse() 163 164 def get_enum(self, enum_name): 165 """ 166 Search for and parse an enum containing BPF_* members, for example: 167 168 enum bpf_prog_type { 169 BPF_PROG_TYPE_UNSPEC, 170 BPF_PROG_TYPE_SOCKET_FILTER, 171 BPF_PROG_TYPE_KPROBE, 172 }; 173 174 Return a set containing all member names, for example: 175 176 {'BPF_PROG_TYPE_UNSPEC', 177 'BPF_PROG_TYPE_SOCKET_FILTER', 178 'BPF_PROG_TYPE_KPROBE'} 179 180 @enum_name: name of the enum to parse 181 """ 182 start_marker = re.compile(f'enum {enum_name} {{\n') 183 pattern = re.compile('^\s*(BPF_\w+),?(\s+/\*.*\*/)?$') 184 end_marker = re.compile('^};') 185 parser = BlockParser(self.reader) 186 parser.search_block(start_marker) 187 return parser.parse(pattern, end_marker) 188 189 def __get_description_list(self, start_marker, pattern, end_marker): 190 parser = InlineListParser(self.reader) 191 parser.search_block(start_marker) 192 return parser.parse(pattern, end_marker) 193 194 def get_rst_list(self, block_name): 195 """ 196 Search for and parse a list of type names from RST documentation, for 197 example: 198 199 | *TYPE* := { 200 | **socket** | **kprobe** | 201 | **kretprobe** 202 | } 203 204 Return a set containing all type names, for example: 205 206 {'socket', 'kprobe', 'kretprobe'} 207 208 @block_name: name of the blog to parse, 'TYPE' in the example 209 """ 210 start_marker = re.compile(f'\*{block_name}\* := {{') 211 pattern = re.compile('\*\*([\w/-]+)\*\*') 212 end_marker = re.compile('}\n') 213 return self.__get_description_list(start_marker, pattern, end_marker) 214 215 def get_help_list(self, block_name): 216 """ 217 Search for and parse a list of type names from a help message in 218 bpftool, for example: 219 220 " TYPE := { socket | kprobe |\\n" 221 " kretprobe }\\n" 222 223 Return a set containing all type names, for example: 224 225 {'socket', 'kprobe', 'kretprobe'} 226 227 @block_name: name of the blog to parse, 'TYPE' in the example 228 """ 229 start_marker = re.compile(f'"\s*{block_name} := {{') 230 pattern = re.compile('([\w/]+) [|}]') 231 end_marker = re.compile('}') 232 return self.__get_description_list(start_marker, pattern, end_marker) 233 234 def get_help_list_macro(self, macro): 235 """ 236 Search for and parse a list of values from a help message starting with 237 a macro in bpftool, for example: 238 239 " " HELP_SPEC_OPTIONS " |\\n" 240 " {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} }\\n" 241 242 Return a set containing all item names, for example: 243 244 {'-f', '--bpffs', '-m', '--mapcompat', '-n', '--nomount'} 245 246 @macro: macro starting the block, 'HELP_SPEC_OPTIONS' in the example 247 """ 248 start_marker = re.compile(f'"\s*{macro}\s*" [|}}]') 249 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]])') 250 end_marker = re.compile('}\\\\n') 251 return self.__get_description_list(start_marker, pattern, end_marker) 252 253 def get_bashcomp_list(self, block_name): 254 """ 255 Search for and parse a list of type names from a variable in bash 256 completion file, for example: 257 258 local BPFTOOL_PROG_LOAD_TYPES='socket kprobe \\ 259 kretprobe' 260 261 Return a set containing all type names, for example: 262 263 {'socket', 'kprobe', 'kretprobe'} 264 265 @block_name: name of the blog to parse, 'TYPE' in the example 266 """ 267 start_marker = re.compile(f'local {block_name}=\'') 268 pattern = re.compile('(?:.*=\')?([\w/]+)') 269 end_marker = re.compile('\'$') 270 return self.__get_description_list(start_marker, pattern, end_marker) 271 272class SourceFileExtractor(FileExtractor): 273 """ 274 An abstract extractor for a source file with usage message. 275 This class does not offer a way to set a filename, which is expected to be 276 defined in children classes. 277 """ 278 def get_options(self): 279 return self.get_help_list_macro('HELP_SPEC_OPTIONS') 280 281class MainHeaderFileExtractor(SourceFileExtractor): 282 """ 283 An extractor for bpftool's main.h 284 """ 285 filename = os.path.join(BPFTOOL_DIR, 'main.h') 286 287 def get_common_options(self): 288 """ 289 Parse the list of common options in main.h (options that apply to all 290 commands), which looks to the lists of options in other source files 291 but has different start and end markers: 292 293 "OPTIONS := { {-j|--json} [{-p|--pretty}] | {-d|--debug} | {-l|--legacy}" 294 295 Return a set containing all options, such as: 296 297 {'-p', '-d', '--legacy', '--pretty', '--debug', '--json', '-l', '-j'} 298 """ 299 start_marker = re.compile(f'"OPTIONS :=') 300 pattern = re.compile('([\w-]+) ?(?:\||}[ }\]"])') 301 end_marker = re.compile('#define') 302 303 parser = InlineListParser(self.reader) 304 parser.search_block(start_marker) 305 return parser.parse(pattern, end_marker) 306 307class ManSubstitutionsExtractor(SourceFileExtractor): 308 """ 309 An extractor for substitutions.rst 310 """ 311 filename = os.path.join(BPFTOOL_DOC_DIR, 'substitutions.rst') 312 313 def get_common_options(self): 314 """ 315 Parse the list of common options in substitutions.rst (options that 316 apply to all commands). 317 318 Return a set containing all options, such as: 319 320 {'-p', '-d', '--legacy', '--pretty', '--debug', '--json', '-l', '-j'} 321 """ 322 start_marker = re.compile('\|COMMON_OPTIONS\| replace:: {') 323 pattern = re.compile('\*\*([\w/-]+)\*\*') 324 end_marker = re.compile('}$') 325 326 parser = InlineListParser(self.reader) 327 parser.search_block(start_marker) 328 return parser.parse(pattern, end_marker) 329 330class ProgFileExtractor(SourceFileExtractor): 331 """ 332 An extractor for bpftool's prog.c. 333 """ 334 filename = os.path.join(BPFTOOL_DIR, 'prog.c') 335 336 def get_prog_types(self): 337 return self.get_types_from_array('prog_type_name') 338 339 def get_attach_types(self): 340 return self.get_types_from_array('attach_type_strings') 341 342 def get_prog_attach_help(self): 343 return self.get_help_list('ATTACH_TYPE') 344 345class MapFileExtractor(SourceFileExtractor): 346 """ 347 An extractor for bpftool's map.c. 348 """ 349 filename = os.path.join(BPFTOOL_DIR, 'map.c') 350 351 def get_map_types(self): 352 return self.get_types_from_array('map_type_name') 353 354 def get_map_help(self): 355 return self.get_help_list('TYPE') 356 357class CgroupFileExtractor(SourceFileExtractor): 358 """ 359 An extractor for bpftool's cgroup.c. 360 """ 361 filename = os.path.join(BPFTOOL_DIR, 'cgroup.c') 362 363 def get_prog_attach_help(self): 364 return self.get_help_list('ATTACH_TYPE') 365 366class CommonFileExtractor(SourceFileExtractor): 367 """ 368 An extractor for bpftool's common.c. 369 """ 370 filename = os.path.join(BPFTOOL_DIR, 'common.c') 371 372 def __init__(self): 373 super().__init__() 374 self.attach_types = {} 375 376 def get_attach_types(self): 377 if not self.attach_types: 378 self.attach_types = self.get_types_from_array('attach_type_name') 379 return self.attach_types 380 381 def get_cgroup_attach_types(self): 382 if not self.attach_types: 383 self.get_attach_types() 384 cgroup_types = {} 385 for (key, value) in self.attach_types.items(): 386 if key.find('BPF_CGROUP') != -1: 387 cgroup_types[key] = value 388 return cgroup_types 389 390class GenericSourceExtractor(SourceFileExtractor): 391 """ 392 An extractor for generic source code files. 393 """ 394 filename = "" 395 396 def __init__(self, filename): 397 self.filename = os.path.join(BPFTOOL_DIR, filename) 398 super().__init__() 399 400class BpfHeaderExtractor(FileExtractor): 401 """ 402 An extractor for the UAPI BPF header. 403 """ 404 filename = os.path.join(INCLUDE_DIR, 'uapi/linux/bpf.h') 405 406 def get_prog_types(self): 407 return self.get_enum('bpf_prog_type') 408 409 def get_map_types(self): 410 return self.get_enum('bpf_map_type') 411 412 def get_attach_types(self): 413 return self.get_enum('bpf_attach_type') 414 415class ManPageExtractor(FileExtractor): 416 """ 417 An abstract extractor for an RST documentation page. 418 This class does not offer a way to set a filename, which is expected to be 419 defined in children classes. 420 """ 421 def get_options(self): 422 return self.get_rst_list('OPTIONS') 423 424class ManProgExtractor(ManPageExtractor): 425 """ 426 An extractor for bpftool-prog.rst. 427 """ 428 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-prog.rst') 429 430 def get_attach_types(self): 431 return self.get_rst_list('ATTACH_TYPE') 432 433class ManMapExtractor(ManPageExtractor): 434 """ 435 An extractor for bpftool-map.rst. 436 """ 437 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-map.rst') 438 439 def get_map_types(self): 440 return self.get_rst_list('TYPE') 441 442class ManCgroupExtractor(ManPageExtractor): 443 """ 444 An extractor for bpftool-cgroup.rst. 445 """ 446 filename = os.path.join(BPFTOOL_DOC_DIR, 'bpftool-cgroup.rst') 447 448 def get_attach_types(self): 449 return self.get_rst_list('ATTACH_TYPE') 450 451class ManGenericExtractor(ManPageExtractor): 452 """ 453 An extractor for generic RST documentation pages. 454 """ 455 filename = "" 456 457 def __init__(self, filename): 458 self.filename = os.path.join(BPFTOOL_DIR, filename) 459 super().__init__() 460 461class BashcompExtractor(FileExtractor): 462 """ 463 An extractor for bpftool's bash completion file. 464 """ 465 filename = os.path.join(BPFTOOL_BASHCOMP_DIR, 'bpftool') 466 467 def get_prog_attach_types(self): 468 return self.get_bashcomp_list('BPFTOOL_PROG_ATTACH_TYPES') 469 470 def get_map_types(self): 471 return self.get_bashcomp_list('BPFTOOL_MAP_CREATE_TYPES') 472 473 def get_cgroup_attach_types(self): 474 return self.get_bashcomp_list('BPFTOOL_CGROUP_ATTACH_TYPES') 475 476def verify(first_set, second_set, message): 477 """ 478 Print all values that differ between two sets. 479 @first_set: one set to compare 480 @second_set: another set to compare 481 @message: message to print for values belonging to only one of the sets 482 """ 483 global retval 484 diff = first_set.symmetric_difference(second_set) 485 if diff: 486 print(message, diff) 487 retval = 1 488 489def main(): 490 # No arguments supported at this time, but print usage for -h|--help 491 argParser = argparse.ArgumentParser(description=""" 492 Verify that bpftool's code, help messages, documentation and bash 493 completion are all in sync on program types, map types, attach types, and 494 options. Also check that bpftool is in sync with the UAPI BPF header. 495 """) 496 args = argParser.parse_args() 497 498 # Map types (enum) 499 500 bpf_info = BpfHeaderExtractor() 501 ref = bpf_info.get_map_types() 502 503 map_info = MapFileExtractor() 504 source_map_items = map_info.get_map_types() 505 map_types_enum = set(source_map_items.keys()) 506 507 verify(ref, map_types_enum, 508 f'Comparing BPF header (enum bpf_map_type) and {MapFileExtractor.filename} (map_type_name):') 509 510 # Map types (names) 511 512 source_map_types = set(source_map_items.values()) 513 source_map_types.discard('unspec') 514 515 help_map_types = map_info.get_map_help() 516 help_map_options = map_info.get_options() 517 map_info.close() 518 519 man_map_info = ManMapExtractor() 520 man_map_options = man_map_info.get_options() 521 man_map_types = man_map_info.get_map_types() 522 man_map_info.close() 523 524 bashcomp_info = BashcompExtractor() 525 bashcomp_map_types = bashcomp_info.get_map_types() 526 527 verify(source_map_types, help_map_types, 528 f'Comparing {MapFileExtractor.filename} (map_type_name) and {MapFileExtractor.filename} (do_help() TYPE):') 529 verify(source_map_types, man_map_types, 530 f'Comparing {MapFileExtractor.filename} (map_type_name) and {ManMapExtractor.filename} (TYPE):') 531 verify(help_map_options, man_map_options, 532 f'Comparing {MapFileExtractor.filename} (do_help() OPTIONS) and {ManMapExtractor.filename} (OPTIONS):') 533 verify(source_map_types, bashcomp_map_types, 534 f'Comparing {MapFileExtractor.filename} (map_type_name) and {BashcompExtractor.filename} (BPFTOOL_MAP_CREATE_TYPES):') 535 536 # Program types (enum) 537 538 ref = bpf_info.get_prog_types() 539 540 prog_info = ProgFileExtractor() 541 prog_types = set(prog_info.get_prog_types().keys()) 542 543 verify(ref, prog_types, 544 f'Comparing BPF header (enum bpf_prog_type) and {ProgFileExtractor.filename} (prog_type_name):') 545 546 # Attach types (enum) 547 548 ref = bpf_info.get_attach_types() 549 bpf_info.close() 550 551 common_info = CommonFileExtractor() 552 attach_types = common_info.get_attach_types() 553 554 verify(ref, attach_types, 555 f'Comparing BPF header (enum bpf_attach_type) and {CommonFileExtractor.filename} (attach_type_name):') 556 557 # Attach types (names) 558 559 source_prog_attach_types = set(prog_info.get_attach_types().values()) 560 561 help_prog_attach_types = prog_info.get_prog_attach_help() 562 help_prog_options = prog_info.get_options() 563 prog_info.close() 564 565 man_prog_info = ManProgExtractor() 566 man_prog_options = man_prog_info.get_options() 567 man_prog_attach_types = man_prog_info.get_attach_types() 568 man_prog_info.close() 569 570 bashcomp_info.reset_read() # We stopped at map types, rewind 571 bashcomp_prog_attach_types = bashcomp_info.get_prog_attach_types() 572 573 verify(source_prog_attach_types, help_prog_attach_types, 574 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ProgFileExtractor.filename} (do_help() ATTACH_TYPE):') 575 verify(source_prog_attach_types, man_prog_attach_types, 576 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {ManProgExtractor.filename} (ATTACH_TYPE):') 577 verify(help_prog_options, man_prog_options, 578 f'Comparing {ProgFileExtractor.filename} (do_help() OPTIONS) and {ManProgExtractor.filename} (OPTIONS):') 579 verify(source_prog_attach_types, bashcomp_prog_attach_types, 580 f'Comparing {ProgFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_PROG_ATTACH_TYPES):') 581 582 # Cgroup attach types 583 584 source_cgroup_attach_types = set(common_info.get_cgroup_attach_types().values()) 585 common_info.close() 586 587 cgroup_info = CgroupFileExtractor() 588 help_cgroup_attach_types = cgroup_info.get_prog_attach_help() 589 help_cgroup_options = cgroup_info.get_options() 590 cgroup_info.close() 591 592 man_cgroup_info = ManCgroupExtractor() 593 man_cgroup_options = man_cgroup_info.get_options() 594 man_cgroup_attach_types = man_cgroup_info.get_attach_types() 595 man_cgroup_info.close() 596 597 bashcomp_cgroup_attach_types = bashcomp_info.get_cgroup_attach_types() 598 bashcomp_info.close() 599 600 verify(source_cgroup_attach_types, help_cgroup_attach_types, 601 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {CgroupFileExtractor.filename} (do_help() ATTACH_TYPE):') 602 verify(source_cgroup_attach_types, man_cgroup_attach_types, 603 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {ManCgroupExtractor.filename} (ATTACH_TYPE):') 604 verify(help_cgroup_options, man_cgroup_options, 605 f'Comparing {CgroupFileExtractor.filename} (do_help() OPTIONS) and {ManCgroupExtractor.filename} (OPTIONS):') 606 verify(source_cgroup_attach_types, bashcomp_cgroup_attach_types, 607 f'Comparing {CommonFileExtractor.filename} (attach_type_strings) and {BashcompExtractor.filename} (BPFTOOL_CGROUP_ATTACH_TYPES):') 608 609 # Options for remaining commands 610 611 for cmd in [ 'btf', 'feature', 'gen', 'iter', 'link', 'net', 'perf', 'struct_ops', ]: 612 source_info = GenericSourceExtractor(cmd + '.c') 613 help_cmd_options = source_info.get_options() 614 source_info.close() 615 616 man_cmd_info = ManGenericExtractor(os.path.join(BPFTOOL_DOC_DIR, 'bpftool-' + cmd + '.rst')) 617 man_cmd_options = man_cmd_info.get_options() 618 man_cmd_info.close() 619 620 verify(help_cmd_options, man_cmd_options, 621 f'Comparing {source_info.filename} (do_help() OPTIONS) and {man_cmd_info.filename} (OPTIONS):') 622 623 source_main_info = GenericSourceExtractor('main.c') 624 help_main_options = source_main_info.get_options() 625 source_main_info.close() 626 627 man_main_info = ManGenericExtractor(os.path.join(BPFTOOL_DOC_DIR, 'bpftool.rst')) 628 man_main_options = man_main_info.get_options() 629 man_main_info.close() 630 631 verify(help_main_options, man_main_options, 632 f'Comparing {source_main_info.filename} (do_help() OPTIONS) and {man_main_info.filename} (OPTIONS):') 633 634 # Compare common options (options that apply to all commands) 635 636 main_hdr_info = MainHeaderFileExtractor() 637 source_common_options = main_hdr_info.get_common_options() 638 main_hdr_info.close() 639 640 man_substitutions = ManSubstitutionsExtractor() 641 man_common_options = man_substitutions.get_common_options() 642 man_substitutions.close() 643 644 verify(source_common_options, man_common_options, 645 f'Comparing common options from {main_hdr_info.filename} (HELP_SPEC_OPTIONS) and {man_substitutions.filename}:') 646 647 sys.exit(retval) 648 649if __name__ == "__main__": 650 main() 651