1#----------------------------------------------------------------------------- 2# Copyright (c) 2013-2019, PyInstaller Development Team. 3# 4# Distributed under the terms of the GNU General Public License with exception 5# for distributing bootloader. 6# 7# The full license is in the file COPYING.txt, distributed with this software. 8#----------------------------------------------------------------------------- 9 10 11""" 12This module is for the miscellaneous routines which do not fit somewhere else. 13""" 14 15import glob 16import os 17import pprint 18import py_compile 19import sys 20 21from PyInstaller import log as logging 22from PyInstaller.compat import BYTECODE_MAGIC, is_py2, text_read_mode 23 24logger = logging.getLogger(__name__) 25 26 27def dlls_in_subdirs(directory): 28 """Returns a list *.dll, *.so, *.dylib in given directories and subdirectories.""" 29 filelist = [] 30 for root, dirs, files in os.walk(directory): 31 filelist.extend(dlls_in_dir(root)) 32 return filelist 33 34 35def dlls_in_dir(directory): 36 """Returns a list of *.dll, *.so, *.dylib in given directory.""" 37 return files_in_dir(directory, ["*.so", "*.dll", "*.dylib"]) 38 39 40def files_in_dir(directory, file_patterns=[]): 41 """Returns a list of files which match a pattern in given directory.""" 42 files = [] 43 for file_pattern in file_patterns: 44 files.extend(glob.glob(os.path.join(directory, file_pattern))) 45 return files 46 47 48def get_unicode_modules(): 49 """ 50 Try importing codecs and encodings to include unicode support 51 in created binary. 52 """ 53 modules = [] 54 try: 55 # `codecs` depends on `encodings` and this is then included. 56 import codecs 57 modules.append('codecs') 58 except ImportError: 59 logger.error("Cannot detect modules 'codecs'.") 60 61 return modules 62 63 64def get_path_to_toplevel_modules(filename): 65 """ 66 Return the path to top-level directory that contains Python modules. 67 68 It will look in parent directories for __init__.py files. The first parent 69 directory without __init__.py is the top-level directory. 70 71 Returned directory might be used to extend the PYTHONPATH. 72 """ 73 curr_dir = os.path.dirname(os.path.abspath(filename)) 74 pattern = '__init__.py' 75 76 # Try max. 10 levels up. 77 try: 78 for i in range(10): 79 files = set(os.listdir(curr_dir)) 80 # 'curr_dir' is still not top-leve go to parent dir. 81 if pattern in files: 82 curr_dir = os.path.dirname(curr_dir) 83 # Top-level dir found - return it. 84 else: 85 return curr_dir 86 except IOError: 87 pass 88 # No top-level directory found or any error. 89 return None 90 91 92def mtime(fnm): 93 try: 94 # TODO: explain why this doesn't use os.path.getmtime() ? 95 # - It is probably not used because it returns fload and not int. 96 return os.stat(fnm)[8] 97 except: 98 return 0 99 100 101def compile_py_files(toc, workpath): 102 """ 103 Given a TOC or equivalent list of tuples, generates all the required 104 pyc/pyo files, writing in a local directory if required, and returns the 105 list of tuples with the updated pathnames. 106 107 In the old system using ImpTracker, the generated TOC of "pure" modules 108 already contains paths to nm.pyc or nm.pyo and it is only necessary 109 to check that these files are not older than the source. 110 In the new system using ModuleGraph, the path given is to nm.py 111 and we do not know if nm.pyc/.pyo exists. The following logic works 112 with both (so if at some time modulegraph starts returning filenames 113 of .pyc, it will cope). 114 """ 115 116 # For those modules that need to be rebuilt, use the build directory 117 # PyInstaller creates during the build process. 118 basepath = os.path.join(workpath, "localpycos") 119 120 # Copy everything from toc to this new TOC, possibly unchanged. 121 new_toc = [] 122 for (nm, fnm, typ) in toc: 123 # Keep unrelevant items unchanged. 124 if typ != 'PYMODULE': 125 new_toc.append((nm, fnm, typ)) 126 continue 127 128 if fnm.endswith('.py') : 129 # we are given a source path, determine the object path if any 130 src_fnm = fnm 131 # assume we want pyo only when now running -O or -OO 132 obj_fnm = src_fnm + ('o' if sys.flags.optimize else 'c') 133 if not os.path.exists(obj_fnm) : 134 # alas that one is not there so assume the other choice 135 obj_fnm = src_fnm + ('c' if sys.flags.optimize else 'o') 136 else: 137 # fnm is not "name.py" so assume we are given name.pyc/.pyo 138 obj_fnm = fnm # take that namae to be the desired object 139 src_fnm = fnm[:-1] # drop the 'c' or 'o' to make a source name 140 141 # We need to perform a build ourselves if obj_fnm doesn't exist, 142 # or if src_fnm is newer than obj_fnm, or if obj_fnm was created 143 # by a different Python version. 144 # TODO: explain why this does read()[:4] (reading all the file) 145 # instead of just read(4)? Yes for many a .pyc file, it is all 146 # in one sector so there's no difference in I/O but still it 147 # seems inelegant to copy it all then subscript 4 bytes. 148 needs_compile = mtime(src_fnm) > mtime(obj_fnm) 149 if not needs_compile: 150 with open(obj_fnm, 'rb') as fh: 151 needs_compile = fh.read()[:4] != BYTECODE_MAGIC 152 if needs_compile: 153 try: 154 # TODO: there should be no need to repeat the compile, 155 # because ModuleGraph does a compile and stores the result 156 # in the .code member of the graph node. Should be possible 157 # to get the node and write the code to obj_fnm 158 py_compile.compile(src_fnm, obj_fnm) 159 logger.debug("compiled %s", src_fnm) 160 except IOError: 161 # If we're compiling on a system directory, probably we don't 162 # have write permissions; thus we compile to a local directory 163 # and change the TOC entry accordingly. 164 ext = os.path.splitext(obj_fnm)[1] 165 166 if "__init__" not in obj_fnm: 167 # If it's a normal module, use last part of the qualified 168 # name as module name and the first as leading path 169 leading, mod_name = nm.split(".")[:-1], nm.split(".")[-1] 170 else: 171 # In case of a __init__ module, use all the qualified name 172 # as leading path and use "__init__" as the module name 173 leading, mod_name = nm.split("."), "__init__" 174 175 leading = os.path.join(basepath, *leading) 176 177 if not os.path.exists(leading): 178 os.makedirs(leading) 179 180 obj_fnm = os.path.join(leading, mod_name + ext) 181 # TODO see above regarding read()[:4] versus read(4) 182 needs_compile = mtime(src_fnm) > mtime(obj_fnm) 183 if not needs_compile: 184 with open(obj_fnm, 'rb') as fh: 185 needs_compile = fh.read()[:4] != BYTECODE_MAGIC 186 if needs_compile: 187 # TODO see above todo regarding using node.code 188 py_compile.compile(src_fnm, obj_fnm) 189 logger.debug("compiled %s", src_fnm) 190 # if we get to here, obj_fnm is the path to the compiled module nm.py 191 new_toc.append((nm, obj_fnm, typ)) 192 193 return new_toc 194 195 196def save_py_data_struct(filename, data): 197 """ 198 Save data into text file as Python data structure. 199 :param filename: 200 :param data: 201 :return: 202 """ 203 dirname = os.path.dirname(filename) 204 if not os.path.exists(dirname): 205 os.makedirs(dirname) 206 if is_py2: 207 import codecs 208 f = codecs.open(filename, 'w', encoding='utf-8') 209 else: 210 f = open(filename, 'w', encoding='utf-8') 211 with f: 212 pprint.pprint(data, f) 213 214 215def load_py_data_struct(filename): 216 """ 217 Load data saved as python code and interpret that code. 218 :param filename: 219 :return: 220 """ 221 if is_py2: 222 import codecs 223 f = codecs.open(filename, text_read_mode, encoding='utf-8') 224 else: 225 f = open(filename, text_read_mode, encoding='utf-8') 226 with f: 227 # Binding redirects are stored as a named tuple, so bring the namedtuple 228 # class into scope for parsing the TOC. 229 from ..depend.bindepend import BindingRedirect 230 231 return eval(f.read()) 232 233 234def absnormpath(apath): 235 return os.path.abspath(os.path.normpath(apath)) 236 237 238def module_parent_packages(full_modname): 239 """ 240 Return list of parent package names. 241 'aaa.bb.c.dddd' -> ['aaa', 'aaa.bb', 'aaa.bb.c'] 242 :param full_modname: Full name of a module. 243 :return: List of parent module names. 244 """ 245 prefix = '' 246 parents = [] 247 # Ignore the last component in module name and get really just 248 # parent, grand parent, grandgrand parent, etc. 249 for pkg in full_modname.split('.')[0:-1]: 250 # Ensure first item does not start with dot '.' 251 prefix += '.' + pkg if prefix else pkg 252 parents.append(prefix) 253 return parents 254