1#-----------------------------------------------------------------------------
2# Copyright (c) 2013-2019, PyInstaller Development Team.
3#
4# Distributed under the terms of the GNU General Public License with exception
5# for distributing bootloader.
6#
7# The full license is in the file COPYING.txt, distributed with this software.
8#-----------------------------------------------------------------------------
9
10
11"""
12This module is for the miscellaneous routines which do not fit somewhere else.
13"""
14
15import glob
16import os
17import pprint
18import py_compile
19import sys
20
21from PyInstaller import log as logging
22from PyInstaller.compat import BYTECODE_MAGIC, is_py2, text_read_mode
23
24logger = logging.getLogger(__name__)
25
26
27def dlls_in_subdirs(directory):
28    """Returns a list *.dll, *.so, *.dylib in given directories and subdirectories."""
29    filelist = []
30    for root, dirs, files in os.walk(directory):
31        filelist.extend(dlls_in_dir(root))
32    return filelist
33
34
35def dlls_in_dir(directory):
36    """Returns a list of *.dll, *.so, *.dylib in given directory."""
37    return files_in_dir(directory, ["*.so", "*.dll", "*.dylib"])
38
39
40def files_in_dir(directory, file_patterns=[]):
41    """Returns a list of files which match a pattern in given directory."""
42    files = []
43    for file_pattern in file_patterns:
44        files.extend(glob.glob(os.path.join(directory, file_pattern)))
45    return files
46
47
48def get_unicode_modules():
49    """
50    Try importing codecs and encodings to include unicode support
51    in created binary.
52    """
53    modules = []
54    try:
55        # `codecs` depends on `encodings` and this is then included.
56        import codecs
57        modules.append('codecs')
58    except ImportError:
59        logger.error("Cannot detect modules 'codecs'.")
60
61    return modules
62
63
64def get_path_to_toplevel_modules(filename):
65    """
66    Return the path to top-level directory that contains Python modules.
67
68    It will look in parent directories for __init__.py files. The first parent
69    directory without __init__.py is the top-level directory.
70
71    Returned directory might be used to extend the PYTHONPATH.
72    """
73    curr_dir = os.path.dirname(os.path.abspath(filename))
74    pattern = '__init__.py'
75
76    # Try max. 10 levels up.
77    try:
78        for i in range(10):
79            files = set(os.listdir(curr_dir))
80            # 'curr_dir' is still not top-leve go to parent dir.
81            if pattern in files:
82                curr_dir = os.path.dirname(curr_dir)
83            # Top-level dir found - return it.
84            else:
85                return curr_dir
86    except IOError:
87        pass
88    # No top-level directory found or any error.
89    return None
90
91
92def mtime(fnm):
93    try:
94        # TODO: explain why this doesn't use os.path.getmtime() ?
95        #       - It is probably not used because it returns fload and not int.
96        return os.stat(fnm)[8]
97    except:
98        return 0
99
100
101def compile_py_files(toc, workpath):
102    """
103    Given a TOC or equivalent list of tuples, generates all the required
104    pyc/pyo files, writing in a local directory if required, and returns the
105    list of tuples with the updated pathnames.
106
107    In the old system using ImpTracker, the generated TOC of "pure" modules
108    already contains paths to nm.pyc or nm.pyo and it is only necessary
109    to check that these files are not older than the source.
110    In the new system using ModuleGraph, the path given is to nm.py
111    and we do not know if nm.pyc/.pyo exists. The following logic works
112    with both (so if at some time modulegraph starts returning filenames
113    of .pyc, it will cope).
114    """
115
116    # For those modules that need to be rebuilt, use the build directory
117    # PyInstaller creates during the build process.
118    basepath = os.path.join(workpath, "localpycos")
119
120    # Copy everything from toc to this new TOC, possibly unchanged.
121    new_toc = []
122    for (nm, fnm, typ) in toc:
123        # Keep unrelevant items unchanged.
124        if typ != 'PYMODULE':
125            new_toc.append((nm, fnm, typ))
126            continue
127
128        if fnm.endswith('.py') :
129            # we are given a source path, determine the object path if any
130            src_fnm = fnm
131            # assume we want pyo only when now running -O or -OO
132            obj_fnm = src_fnm + ('o' if sys.flags.optimize else 'c')
133            if not os.path.exists(obj_fnm) :
134                # alas that one is not there so assume the other choice
135                obj_fnm = src_fnm + ('c' if sys.flags.optimize else 'o')
136        else:
137            # fnm is not "name.py" so assume we are given name.pyc/.pyo
138            obj_fnm = fnm # take that namae to be the desired object
139            src_fnm = fnm[:-1] # drop the 'c' or 'o' to make a source name
140
141        # We need to perform a build ourselves if obj_fnm doesn't exist,
142        # or if src_fnm is newer than obj_fnm, or if obj_fnm was created
143        # by a different Python version.
144        # TODO: explain why this does read()[:4] (reading all the file)
145        # instead of just read(4)? Yes for many a .pyc file, it is all
146        # in one sector so there's no difference in I/O but still it
147        # seems inelegant to copy it all then subscript 4 bytes.
148        needs_compile = mtime(src_fnm) > mtime(obj_fnm)
149        if not needs_compile:
150            with open(obj_fnm, 'rb') as fh:
151                needs_compile = fh.read()[:4] != BYTECODE_MAGIC
152        if needs_compile:
153            try:
154                # TODO: there should be no need to repeat the compile,
155                # because ModuleGraph does a compile and stores the result
156                # in the .code member of the graph node. Should be possible
157                # to get the node and write the code to obj_fnm
158                py_compile.compile(src_fnm, obj_fnm)
159                logger.debug("compiled %s", src_fnm)
160            except IOError:
161                # If we're compiling on a system directory, probably we don't
162                # have write permissions; thus we compile to a local directory
163                # and change the TOC entry accordingly.
164                ext = os.path.splitext(obj_fnm)[1]
165
166                if "__init__" not in obj_fnm:
167                    # If it's a normal module, use last part of the qualified
168                    # name as module name and the first as leading path
169                    leading, mod_name = nm.split(".")[:-1], nm.split(".")[-1]
170                else:
171                    # In case of a __init__ module, use all the qualified name
172                    # as leading path and use "__init__" as the module name
173                    leading, mod_name = nm.split("."), "__init__"
174
175                leading = os.path.join(basepath, *leading)
176
177                if not os.path.exists(leading):
178                    os.makedirs(leading)
179
180                obj_fnm = os.path.join(leading, mod_name + ext)
181                # TODO see above regarding read()[:4] versus read(4)
182                needs_compile = mtime(src_fnm) > mtime(obj_fnm)
183                if not needs_compile:
184                    with open(obj_fnm, 'rb') as fh:
185                        needs_compile = fh.read()[:4] != BYTECODE_MAGIC
186                if needs_compile:
187                    # TODO see above todo regarding using node.code
188                    py_compile.compile(src_fnm, obj_fnm)
189                    logger.debug("compiled %s", src_fnm)
190        # if we get to here, obj_fnm is the path to the compiled module nm.py
191        new_toc.append((nm, obj_fnm, typ))
192
193    return new_toc
194
195
196def save_py_data_struct(filename, data):
197    """
198    Save data into text file as Python data structure.
199    :param filename:
200    :param data:
201    :return:
202    """
203    dirname = os.path.dirname(filename)
204    if not os.path.exists(dirname):
205        os.makedirs(dirname)
206    if is_py2:
207        import codecs
208        f = codecs.open(filename, 'w', encoding='utf-8')
209    else:
210        f = open(filename, 'w', encoding='utf-8')
211    with f:
212        pprint.pprint(data, f)
213
214
215def load_py_data_struct(filename):
216    """
217    Load data saved as python code and interpret that code.
218    :param filename:
219    :return:
220    """
221    if is_py2:
222        import codecs
223        f = codecs.open(filename, text_read_mode, encoding='utf-8')
224    else:
225        f = open(filename, text_read_mode, encoding='utf-8')
226    with f:
227        # Binding redirects are stored as a named tuple, so bring the namedtuple
228        # class into scope for parsing the TOC.
229        from ..depend.bindepend import BindingRedirect
230
231        return eval(f.read())
232
233
234def absnormpath(apath):
235    return os.path.abspath(os.path.normpath(apath))
236
237
238def module_parent_packages(full_modname):
239    """
240    Return list of parent package names.
241        'aaa.bb.c.dddd' ->  ['aaa', 'aaa.bb', 'aaa.bb.c']
242    :param full_modname: Full name of a module.
243    :return: List of parent module names.
244    """
245    prefix = ''
246    parents = []
247    # Ignore the last component in module name and get really just
248    # parent, grand parent, grandgrand parent, etc.
249    for pkg in full_modname.split('.')[0:-1]:
250        # Ensure first item does not start with dot '.'
251        prefix += '.' + pkg if prefix else pkg
252        parents.append(prefix)
253    return parents
254