1#!/usr/bin/env python3.8
2
3import argparse
4import ast
5import os
6import sys
7import time
8import traceback
9import tokenize
10from glob import glob, escape
11from pathlib import PurePath
12
13from typing import List, Optional, Any, Tuple
14
15sys.path.insert(0, os.getcwd())
16from pegen.ast_dump import ast_dump
17from pegen.testutil import print_memstats
18
19SUCCESS = "\033[92m"
20FAIL = "\033[91m"
21ENDC = "\033[0m"
22
23COMPILE = 2
24PARSE = 1
25NOTREE = 0
26
27argparser = argparse.ArgumentParser(
28    prog="test_parse_directory",
29    description="Helper program to test directories or files for pegen",
30)
31argparser.add_argument("-d", "--directory", help="Directory path containing files to test")
32argparser.add_argument(
33    "-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude"
34)
35argparser.add_argument(
36    "-s", "--short", action="store_true", help="Only show errors, in a more Emacs-friendly format"
37)
38argparser.add_argument(
39    "-v", "--verbose", action="store_true", help="Display detailed errors for failures"
40)
41
42
43def report_status(
44    succeeded: bool,
45    file: str,
46    verbose: bool,
47    error: Optional[Exception] = None,
48    short: bool = False,
49) -> None:
50    if short and succeeded:
51        return
52
53    if succeeded is True:
54        status = "OK"
55        COLOR = SUCCESS
56    else:
57        status = "Fail"
58        COLOR = FAIL
59
60    if short:
61        lineno = 0
62        offset = 0
63        if isinstance(error, SyntaxError):
64            lineno = error.lineno or 1
65            offset = error.offset or 1
66            message = error.args[0]
67        else:
68            message = f"{error.__class__.__name__}: {error}"
69        print(f"{file}:{lineno}:{offset}: {message}")
70    else:
71        print(f"{COLOR}{file:60} {status}{ENDC}")
72
73        if error and verbose:
74            print(f"  {str(error.__class__.__name__)}: {error}")
75
76
77def parse_file(source: str, file: str) -> Tuple[Any, float]:
78    t0 = time.time()
79    result = ast.parse(source, filename=file)
80    t1 = time.time()
81    return result, t1 - t0
82
83
84def generate_time_stats(files, total_seconds) -> None:
85    total_files = len(files)
86    total_bytes = 0
87    total_lines = 0
88    for file in files:
89        # Count lines and bytes separately
90        with open(file, "rb") as f:
91            total_lines += sum(1 for _ in f)
92            total_bytes += f.tell()
93
94    print(
95        f"Checked {total_files:,} files, {total_lines:,} lines,",
96        f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
97    )
98    if total_seconds > 0:
99        print(
100            f"That's {total_lines / total_seconds :,.0f} lines/sec,",
101            f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
102        )
103
104
105def parse_directory(directory: str, verbose: bool, excluded_files: List[str], short: bool) -> int:
106    # For a given directory, traverse files and attempt to parse each one
107    # - Output success/failure for each file
108    errors = 0
109    files = []
110    total_seconds = 0
111
112    for file in sorted(glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)):
113        # Only attempt to parse Python files and files that are not excluded
114        if any(PurePath(file).match(pattern) for pattern in excluded_files):
115            continue
116
117        with tokenize.open(file) as f:
118            source = f.read()
119
120        try:
121            result, dt = parse_file(source, file)
122            total_seconds += dt
123            report_status(succeeded=True, file=file, verbose=verbose, short=short)
124        except SyntaxError as error:
125            report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short)
126            errors += 1
127        files.append(file)
128
129    generate_time_stats(files, total_seconds)
130    if short:
131        print_memstats()
132
133    if errors:
134        print(f"Encountered {errors} failures.", file=sys.stderr)
135        return 1
136
137    return 0
138
139
140def main() -> None:
141    args = argparser.parse_args()
142    directory = args.directory
143    verbose = args.verbose
144    excluded_files = args.exclude
145    short = args.short
146    sys.exit(parse_directory(directory, verbose, excluded_files, short))
147
148
149if __name__ == "__main__":
150    main()
151