1import logging 2import sys 3 4from c_common.fsutil import expand_filenames, iter_files_by_suffix 5from c_common.scriptutil import ( 6 VERBOSITY, 7 add_verbosity_cli, 8 add_traceback_cli, 9 add_commands_cli, 10 add_kind_filtering_cli, 11 add_files_cli, 12 add_progress_cli, 13 main_for_filenames, 14 process_args_by_key, 15 configure_logger, 16 get_prog, 17) 18from c_parser.info import KIND 19import c_parser.__main__ as c_parser 20import c_analyzer.__main__ as c_analyzer 21import c_analyzer as _c_analyzer 22from c_analyzer.info import UNKNOWN 23from . import _analyzer, _capi, _files, _parser, REPO_ROOT 24 25 26logger = logging.getLogger(__name__) 27 28 29def _resolve_filenames(filenames): 30 if filenames: 31 resolved = (_files.resolve_filename(f) for f in filenames) 32 else: 33 resolved = _files.iter_filenames() 34 return resolved 35 36 37####################################### 38# the formats 39 40def fmt_summary(analysis): 41 # XXX Support sorting and grouping. 42 supported = [] 43 unsupported = [] 44 for item in analysis: 45 if item.supported: 46 supported.append(item) 47 else: 48 unsupported.append(item) 49 total = 0 50 51 def section(name, groupitems): 52 nonlocal total 53 items, render = c_analyzer.build_section(name, groupitems, 54 relroot=REPO_ROOT) 55 yield from render() 56 total += len(items) 57 58 yield '' 59 yield '====================' 60 yield 'supported' 61 yield '====================' 62 63 yield from section('types', supported) 64 yield from section('variables', supported) 65 66 yield '' 67 yield '====================' 68 yield 'unsupported' 69 yield '====================' 70 71 yield from section('types', unsupported) 72 yield from section('variables', unsupported) 73 74 yield '' 75 yield f'grand total: {total}' 76 77 78####################################### 79# the checks 80 81CHECKS = dict(c_analyzer.CHECKS, **{ 82 'globals': _analyzer.check_globals, 83}) 84 85####################################### 86# the commands 87 88FILES_KWARGS = dict(excluded=_parser.EXCLUDED, nargs='*') 89 90 91def _cli_parse(parser): 92 process_output = c_parser.add_output_cli(parser) 93 process_kind = add_kind_filtering_cli(parser) 94 process_preprocessor = c_parser.add_preprocessor_cli( 95 parser, 96 get_preprocessor=_parser.get_preprocessor, 97 ) 98 process_files = add_files_cli(parser, **FILES_KWARGS) 99 return [ 100 process_output, 101 process_kind, 102 process_preprocessor, 103 process_files, 104 ] 105 106 107def cmd_parse(filenames=None, **kwargs): 108 filenames = _resolve_filenames(filenames) 109 if 'get_file_preprocessor' not in kwargs: 110 kwargs['get_file_preprocessor'] = _parser.get_preprocessor() 111 c_parser.cmd_parse( 112 filenames, 113 relroot=REPO_ROOT, 114 file_maxsizes=_parser.MAX_SIZES, 115 **kwargs 116 ) 117 118 119def _cli_check(parser, **kwargs): 120 return c_analyzer._cli_check(parser, CHECKS, **kwargs, **FILES_KWARGS) 121 122 123def cmd_check(filenames=None, **kwargs): 124 filenames = _resolve_filenames(filenames) 125 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) 126 c_analyzer.cmd_check( 127 filenames, 128 relroot=REPO_ROOT, 129 _analyze=_analyzer.analyze, 130 _CHECKS=CHECKS, 131 file_maxsizes=_parser.MAX_SIZES, 132 **kwargs 133 ) 134 135 136def cmd_analyze(filenames=None, **kwargs): 137 formats = dict(c_analyzer.FORMATS) 138 formats['summary'] = fmt_summary 139 filenames = _resolve_filenames(filenames) 140 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) 141 c_analyzer.cmd_analyze( 142 filenames, 143 relroot=REPO_ROOT, 144 _analyze=_analyzer.analyze, 145 formats=formats, 146 file_maxsizes=_parser.MAX_SIZES, 147 **kwargs 148 ) 149 150 151def _cli_data(parser): 152 filenames = False 153 known = True 154 return c_analyzer._cli_data(parser, filenames, known) 155 156 157def cmd_data(datacmd, **kwargs): 158 formats = dict(c_analyzer.FORMATS) 159 formats['summary'] = fmt_summary 160 filenames = (file 161 for file in _resolve_filenames(None) 162 if file not in _parser.EXCLUDED) 163 kwargs['get_file_preprocessor'] = _parser.get_preprocessor(log_err=print) 164 if datacmd == 'show': 165 types = _analyzer.read_known() 166 results = [] 167 for decl, info in types.items(): 168 if info is UNKNOWN: 169 if decl.kind in (KIND.STRUCT, KIND.UNION): 170 extra = {'unsupported': ['type unknown'] * len(decl.members)} 171 else: 172 extra = {'unsupported': ['type unknown']} 173 info = (info, extra) 174 results.append((decl, info)) 175 if decl.shortkey == 'struct _object': 176 tempinfo = info 177 known = _analyzer.Analysis.from_results(results) 178 analyze = None 179 elif datacmd == 'dump': 180 known = _analyzer.KNOWN_FILE 181 def analyze(files, **kwargs): 182 decls = [] 183 for decl in _analyzer.iter_decls(files, **kwargs): 184 if not KIND.is_type_decl(decl.kind): 185 continue 186 if not decl.filename.endswith('.h'): 187 if decl.shortkey not in _analyzer.KNOWN_IN_DOT_C: 188 continue 189 decls.append(decl) 190 results = _c_analyzer.analyze_decls( 191 decls, 192 known={}, 193 analyze_resolved=_analyzer.analyze_resolved, 194 ) 195 return _analyzer.Analysis.from_results(results) 196 else: # check 197 known = _analyzer.read_known() 198 def analyze(files, **kwargs): 199 return _analyzer.iter_decls(files, **kwargs) 200 extracolumns = None 201 c_analyzer.cmd_data( 202 datacmd, 203 filenames, 204 known, 205 _analyze=analyze, 206 formats=formats, 207 extracolumns=extracolumns, 208 relroot=REPO_ROOT, 209 **kwargs 210 ) 211 212 213def _cli_capi(parser): 214 parser.add_argument('--levels', action='append', metavar='LEVEL[,...]') 215 parser.add_argument(f'--public', dest='levels', 216 action='append_const', const='public') 217 parser.add_argument(f'--no-public', dest='levels', 218 action='append_const', const='no-public') 219 for level in _capi.LEVELS: 220 parser.add_argument(f'--{level}', dest='levels', 221 action='append_const', const=level) 222 def process_levels(args, *, argv=None): 223 levels = [] 224 for raw in args.levels or (): 225 for level in raw.replace(',', ' ').strip().split(): 226 if level == 'public': 227 levels.append('stable') 228 levels.append('cpython') 229 elif level == 'no-public': 230 levels.append('private') 231 levels.append('internal') 232 elif level in _capi.LEVELS: 233 levels.append(level) 234 else: 235 parser.error(f'expected LEVEL to be one of {sorted(_capi.LEVELS)}, got {level!r}') 236 args.levels = set(levels) 237 238 parser.add_argument('--kinds', action='append', metavar='KIND[,...]') 239 for kind in _capi.KINDS: 240 parser.add_argument(f'--{kind}', dest='kinds', 241 action='append_const', const=kind) 242 def process_kinds(args, *, argv=None): 243 kinds = [] 244 for raw in args.kinds or (): 245 for kind in raw.replace(',', ' ').strip().split(): 246 if kind in _capi.KINDS: 247 kinds.append(kind) 248 else: 249 parser.error(f'expected KIND to be one of {sorted(_capi.KINDS)}, got {kind!r}') 250 args.kinds = set(kinds) 251 252 parser.add_argument('--group-by', dest='groupby', 253 choices=['level', 'kind']) 254 255 parser.add_argument('--format', default='table') 256 parser.add_argument('--summary', dest='format', 257 action='store_const', const='summary') 258 def process_format(args, *, argv=None): 259 orig = args.format 260 args.format = _capi.resolve_format(args.format) 261 if isinstance(args.format, str): 262 if args.format not in _capi._FORMATS: 263 parser.error(f'unsupported format {orig!r}') 264 265 parser.add_argument('--show-empty', dest='showempty', action='store_true') 266 parser.add_argument('--no-show-empty', dest='showempty', action='store_false') 267 parser.set_defaults(showempty=None) 268 269 # XXX Add --sort-by, --sort and --no-sort. 270 271 parser.add_argument('--ignore', dest='ignored', action='append') 272 def process_ignored(args, *, argv=None): 273 ignored = [] 274 for raw in args.ignored or (): 275 ignored.extend(raw.replace(',', ' ').strip().split()) 276 args.ignored = ignored or None 277 278 parser.add_argument('filenames', nargs='*', metavar='FILENAME') 279 process_progress = add_progress_cli(parser) 280 281 return [ 282 process_levels, 283 process_kinds, 284 process_format, 285 process_ignored, 286 process_progress, 287 ] 288 289 290def cmd_capi(filenames=None, *, 291 levels=None, 292 kinds=None, 293 groupby='kind', 294 format='table', 295 showempty=None, 296 ignored=None, 297 track_progress=None, 298 verbosity=VERBOSITY, 299 **kwargs 300 ): 301 render = _capi.get_renderer(format) 302 303 filenames = _files.iter_header_files(filenames, levels=levels) 304 #filenames = (file for file, _ in main_for_filenames(filenames)) 305 if track_progress: 306 filenames = track_progress(filenames) 307 items = _capi.iter_capi(filenames) 308 if levels: 309 items = (item for item in items if item.level in levels) 310 if kinds: 311 items = (item for item in items if item.kind in kinds) 312 313 filter = _capi.resolve_filter(ignored) 314 if filter: 315 items = (item for item in items if filter(item, log=lambda msg: logger.log(1, msg))) 316 317 lines = render( 318 items, 319 groupby=groupby, 320 showempty=showempty, 321 verbose=verbosity > VERBOSITY, 322 ) 323 print() 324 for line in lines: 325 print(line) 326 327 328# We do not define any other cmd_*() handlers here, 329# favoring those defined elsewhere. 330 331COMMANDS = { 332 'check': ( 333 'analyze and fail if the CPython source code has any problems', 334 [_cli_check], 335 cmd_check, 336 ), 337 'analyze': ( 338 'report on the state of the CPython source code', 339 [(lambda p: c_analyzer._cli_analyze(p, **FILES_KWARGS))], 340 cmd_analyze, 341 ), 342 'parse': ( 343 'parse the CPython source files', 344 [_cli_parse], 345 cmd_parse, 346 ), 347 'data': ( 348 'check/manage local data (e.g. known types, ignored vars, caches)', 349 [_cli_data], 350 cmd_data, 351 ), 352 'capi': ( 353 'inspect the C-API', 354 [_cli_capi], 355 cmd_capi, 356 ), 357} 358 359 360####################################### 361# the script 362 363def parse_args(argv=sys.argv[1:], prog=None, *, subset=None): 364 import argparse 365 parser = argparse.ArgumentParser( 366 prog=prog or get_prog(), 367 ) 368 369# if subset == 'check' or subset == ['check']: 370# if checks is not None: 371# commands = dict(COMMANDS) 372# commands['check'] = list(commands['check']) 373# cli = commands['check'][1][0] 374# commands['check'][1][0] = (lambda p: cli(p, checks=checks)) 375 processors = add_commands_cli( 376 parser, 377 commands=COMMANDS, 378 commonspecs=[ 379 add_verbosity_cli, 380 add_traceback_cli, 381 ], 382 subset=subset, 383 ) 384 385 args = parser.parse_args(argv) 386 ns = vars(args) 387 388 cmd = ns.pop('cmd') 389 390 verbosity, traceback_cm = process_args_by_key( 391 args, 392 argv, 393 processors[cmd], 394 ['verbosity', 'traceback_cm'], 395 ) 396 if cmd != 'parse': 397 # "verbosity" is sent to the commands, so we put it back. 398 args.verbosity = verbosity 399 400 return cmd, ns, verbosity, traceback_cm 401 402 403def main(cmd, cmd_kwargs): 404 try: 405 run_cmd = COMMANDS[cmd][-1] 406 except KeyError: 407 raise ValueError(f'unsupported cmd {cmd!r}') 408 run_cmd(**cmd_kwargs) 409 410 411if __name__ == '__main__': 412 cmd, cmd_kwargs, verbosity, traceback_cm = parse_args() 413 configure_logger(verbosity) 414 with traceback_cm: 415 main(cmd, cmd_kwargs) 416