1#!/usr/bin/env python
2# emacs-mode: -*-python-*-
3
4"""
5test_pythonlib.py -- disassemble Python libraries
6
7Usage-Examples:
8
9  # disassemble base set of python 2.7 byte-compiled files
10  test_pythonlib.py --base-2.7 --verify
11
12  # Same as above but compile the base set first
13  test_pythonlib.py --base-2.7 --verify --compile
14
15  # Same as above but use a longer set from the python 2.7 library
16  test_pythonlib.py --ok-2.7 --verify --compile
17
18  # Just deompile the longer set of files
19  test_pythonlib.py --ok-2.7
20
21Adding own test-trees:
22
23Step 1) Edit this file and add a new entry to 'test_options', eg.
24  test_options['mylib'] = ('/usr/lib/mylib', PYOC, 'mylib')
25Step 2: Run the test:
26  test_pythonlib.py --mylib	  # decompile 'mylib'
27  test_pythonlib.py --mylib --verify # decompile verify 'mylib'
28"""
29
30from __future__ import print_function
31import getopt, os, py_compile, sys, shutil, tempfile, time
32
33from xdis import PYTHON_VERSION, disassemble_file
34from fnmatch import fnmatch
35
36def get_srcdir():
37    filename = os.path.normcase(os.path.dirname(__file__))
38    return os.path.realpath(filename)
39
40src_dir = get_srcdir()
41
42
43#----- configure this for your needs
44
45lib_prefix = '/usr/lib'
46#lib_prefix = [src_dir, '/usr/lib/', '/usr/local/lib/']
47
48target_base = tempfile.mkdtemp(prefix='py-dis-')
49
50PY = ('*.py', )
51PYC = ('*.pyc', )
52PYO = ('*.pyo', )
53PYOC = ('*.pyc', '*.pyo')
54
55test_options = {
56    # name:   (src_basedir, pattern, output_base_suffix, python_version)
57    'test':
58        ('test', PYC, 'test'),
59
60    'ok-2.6':
61        (os.path.join(src_dir, 'ok_2.6'),
62         PYOC, 'ok-2.6', 2.6),
63
64    'ok-2.7':    (os.path.join(src_dir, 'ok_lib2.7'),
65                 PYOC, 'ok-2.7', 2.7),
66
67    'ok-3.2':    (os.path.join(src_dir, 'ok_lib3.2'),
68                 PYOC, 'ok-3.2', 3.5),
69
70    'base-2.7':  (os.path.join(src_dir, 'base_tests', 'python2.7'),
71                  PYOC, 'base_2.7', 2.7),
72}
73
74for vers in (1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6,
75             2.1, 2.2, 2.3, 2.4, 2.5, '2.5dropbox', 2.6, 2.7,
76             3.0, 3.1, 3.2, 3.3, 3.4, 3.5, '3.2pypy', '2.7pypy',
77             '3.5pypy', '3.6pypy',
78             3.6, 3.7, 3.8):
79    bytecode = "bytecode_%s" % vers
80    key = "bytecode-%s" % vers
81    test_options[key] = (os.path.join(src_dir, bytecode), PYC, bytecode, vers)
82    key = "%s" % vers
83    pythonlib = "python%s" % vers
84    if isinstance(vers, float) and vers >= 3.0:
85        pythonlib = os.path.join(src_dir, pythonlib, '__pycache__')
86    test_options[key] =  (os.path.join(lib_prefix, pythonlib), PYOC, pythonlib, vers)
87
88#-----
89
90def help():
91    print("""Usage-Examples:
92
93  # compile, decompyle and verify short tests for Python 2.7:
94  test_pythonlib.py --bytecode-2.7 --verify --compile
95
96  # decompile all of Python's installed lib files
97  test_pythonlib.py --2.7
98
99  # decompile and verify known good python 2.7
100  test_pythonlib.py --ok-2.7 --verify
101""")
102    sys.exit(1)
103
104
105def do_tests(src_dir, obj_patterns, target_dir, opts):
106
107    def file_matches(files, root, basenames, patterns):
108        files.extend(
109            [os.path.normpath(os.path.join(root, n))
110                 for n in basenames
111                    for pat in patterns
112                        if fnmatch(n, pat)])
113
114    files = []
115    # Change directories so use relative rather than
116    # absolute paths. This speeds up things, and allows
117    # main() to write to a relative-path destination.
118    cwd = os.getcwd()
119    os.chdir(src_dir)
120
121    if opts['do_compile']:
122        compiled_version = opts['compiled_version']
123        if compiled_version and PYTHON_VERSION != compiled_version:
124            sys.stderr.write("Not compiling: desired Python version is %s "
125                                 "but we are running %s\n" %
126                                 (compiled_version, PYTHON_VERSION))
127        else:
128            for root, dirs, basenames in os.walk(src_dir):
129                file_matches(files, root, basenames, PY)
130                for sfile in files:
131                    py_compile.compile(sfile)
132                    pass
133                pass
134            files = []
135            pass
136        pass
137
138    for root, dirs, basenames in os.walk('.'):
139        # Turn root into a relative path
140        dirname = root[2:]  # 2 = len('.') + 1
141        file_matches(files, dirname, basenames, obj_patterns)
142
143    if not files:
144        sys.stderr.write("Didn't come up with any files to test! Try with --compile?\n")
145        exit(1)
146
147    os.chdir(cwd)
148    files.sort()
149
150    if opts['start_with']:
151        try:
152            start_with = files.index(opts['start_with'])
153            files = files[start_with:]
154            print('>>> starting with file', files[0])
155        except ValueError:
156            pass
157
158    output = open(os.devnull,"w")
159    # output = sys.stdout
160    print(time.ctime())
161    print('Source directory: ', src_dir)
162    cwd = os.getcwd()
163    os.chdir(src_dir)
164    try:
165        for infile in files:
166            disassemble_file(infile, output)
167            if opts['do_verify']:
168                pass
169            # print("Need to do something here to verify %s" % infile)
170            # msg = verify.verify_file(infile, outfile)
171
172        # if failed_files != 0:
173        #     exit(2)
174        # elif failed_verify != 0:
175        #     exit(3)
176
177    except (KeyboardInterrupt, OSError):
178        print()
179        exit(1)
180    os.chdir(cwd)
181    # if test_opts['rmtree']:
182    #     parent_dir = os.path.dirname(target_dir)
183    #     print("Everything good, removing %s" % parent_dir)
184    #     shutil.rmtree(parent_dir)
185
186if __name__ == '__main__':
187    test_dirs = []
188    checked_dirs = []
189    start_with = None
190
191    test_options_keys = list(test_options.keys())
192    test_options_keys.sort()
193    opts, args = getopt.getopt(sys.argv[1:], '',
194                               ['start-with=', 'verify', 'all', 'compile',
195                                'no-rm'] \
196                               + test_options_keys )
197    if not opts: help()
198
199    test_opts = {
200        'do_compile': False,
201        'do_verify': False,
202        'start_with': None,
203        'rmtree' : True
204        }
205
206    for opt, val in opts:
207        if opt == '--verify':
208            test_opts['do_verify'] = True
209        elif opt == '--compile':
210            test_opts['do_compile'] = True
211        elif opt == '--start-with':
212            test_opts['start_with'] = val
213        elif opt == '--no-rm':
214            test_opts['rmtree'] = False
215        elif opt[2:] in test_options_keys:
216            test_dirs.append(test_options[opt[2:]])
217        elif opt == '--all':
218            for val in test_options_keys:
219                test_dirs.append(test_options[val])
220        else:
221            help()
222            pass
223        pass
224
225    last_compile_version = None
226    for src_dir, pattern, target_dir, compiled_version in test_dirs:
227        if os.path.isdir(src_dir):
228            checked_dirs.append([src_dir, pattern, target_dir])
229        else:
230            sys.stderr.write("Can't find directory %s. Skipping\n" % src_dir)
231            continue
232        last_compile_version = compiled_version
233        pass
234
235    if not checked_dirs:
236        sys.stderr.write("No directories found to check\n")
237        sys.exit(1)
238
239    test_opts['compiled_version'] = last_compile_version
240
241    for src_dir, pattern, target_dir in checked_dirs:
242        target_dir = os.path.join(target_base, target_dir)
243        if os.path.exists(target_dir):
244            shutil.rmtree(target_dir, ignore_errors=1)
245        do_tests(src_dir, pattern, target_dir, test_opts)
246