1############################################################################# 2## 3## Copyright (C) 2019 The Qt Company Ltd. 4## Contact: https://www.qt.io/licensing/ 5## 6## This file is part of PySide2. 7## 8## $QT_BEGIN_LICENSE:LGPL$ 9## Commercial License Usage 10## Licensees holding valid commercial Qt licenses may use this file in 11## accordance with the commercial license agreement provided with the 12## Software or, alternatively, in accordance with the terms contained in 13## a written agreement between you and The Qt Company. For licensing terms 14## and conditions see https://www.qt.io/terms-conditions. For further 15## information use the contact form at https://www.qt.io/contact-us. 16## 17## GNU Lesser General Public License Usage 18## Alternatively, this file may be used under the terms of the GNU Lesser 19## General Public License version 3 as published by the Free Software 20## Foundation and appearing in the file LICENSE.LGPL3 included in the 21## packaging of this file. Please review the following information to 22## ensure the GNU Lesser General Public License version 3 requirements 23## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. 24## 25## GNU General Public License Usage 26## Alternatively, this file may be used under the terms of the GNU 27## General Public License version 2.0 or (at your option) the GNU General 28## Public license version 3 or any later version approved by the KDE Free 29## Qt Foundation. The licenses are as published by the Free Software 30## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 31## included in the packaging of this file. Please review the following 32## information to ensure the GNU General Public License requirements will 33## be met: https://www.gnu.org/licenses/gpl-2.0.html and 34## https://www.gnu.org/licenses/gpl-3.0.html. 35## 36## $QT_END_LICENSE$ 37## 38############################################################################# 39 40""" 41embedding_generator.py 42 43This file takes the content of the two supported directories and inserts 44it into a zip file. The zip file is then converted into a C++ source 45file that can easily be unpacked again with Python (see signature.cpp, 46constant 'PySide_PythonCode'). 47 48Note that this _is_ a zipfile, but since it is embedded into the shiboken 49binary, we cannot use the zipimport module from Python. 50But a similar solution is possible that allows for normal imports. 51 52See signature_bootstrap.py for details. 53""" 54 55from __future__ import print_function, absolute_import 56 57import sys 58import os 59import subprocess 60import textwrap 61import tempfile 62import argparse 63import marshal 64import traceback 65 66# work_dir is set to the source for testing, onl. 67# It can be overridden in the command line. 68work_dir = os.path.abspath(os.path.dirname(__file__)) 69embed_dir = work_dir 70cur_dir = os.getcwd() 71source_dir = os.path.normpath(os.path.join(work_dir, "..", "..", "..")) 72assert os.path.basename(source_dir) == "sources" 73build_script_dir = os.path.normpath(os.path.join(work_dir, "..", "..", "..", "..")) 74assert os.path.exists(os.path.join(build_script_dir, "build_scripts")) 75 76sys.path.insert(0, build_script_dir) 77 78from build_scripts import utils 79 80 81def runpy(cmd, **kw): 82 subprocess.call([sys.executable, '-E'] + cmd.split(), **kw) 83 84 85def create_zipfile(limited_api): 86 """ 87 Collect all Python files, compile them, create a zip file 88 and make a chunked base64 encoded file from it. 89 """ 90 zip_name = "signature.zip" 91 inc_name = "signature_inc.h" 92 flag = '-b' if sys.version_info >= (3,) else '' 93 os.chdir(work_dir) 94 95 # Remove all left-over py[co] and other files first, in case we use '--reuse-build'. 96 # Note that we could improve that with the PyZipfile function to use .pyc files 97 # in different folders, but that makes only sense when COIN allows us to have 98 # multiple Python versions in parallel. 99 from os.path import join, getsize 100 for root, dirs, files in os.walk(work_dir): 101 for name in files: 102 fpath = os.path.join(root, name) 103 ew = name.endswith 104 if ew(".pyc") or ew(".pyo") or ew(".zip") or ew(".inc"): 105 os.remove(fpath) 106 # We copy every Python file into this dir, but only for the right version. 107 # For testing in the source dir, we need to filter. 108 if sys.version_info[0] == 3: 109 ignore = "backport_inspect.py typing27.py".split() 110 else: 111 ignore = "".split() 112 utils.copydir(os.path.join(source_dir, "shiboken2", "shibokenmodule", "files.dir", "shibokensupport"), 113 os.path.join(work_dir, "shibokensupport"), 114 ignore=ignore, file_filter_function=lambda name, n2: name.endswith(".py")) 115 if embed_dir != work_dir: 116 utils.copyfile(os.path.join(embed_dir, "signature_bootstrap.py"), work_dir) 117 118 if limited_api: 119 pass # We cannot compile, unless we have folders per Python version 120 else: 121 files = ' '.join(fn for fn in os.listdir('.')) 122 runpy('-m compileall -q {flag} {files}'.format(**locals())) 123 files = ' '.join(fn for fn in os.listdir('.') if not fn == zip_name) 124 runpy('-m zipfile -c {zip_name} {files}'.format(**locals())) 125 tmp = tempfile.TemporaryFile(mode="w+") 126 runpy('-m base64 {zip_name}'.format(**locals()), stdout=tmp) 127 # now generate the include file 128 tmp.seek(0) 129 with open(inc_name, "w") as inc: 130 _embed_file(tmp, inc) 131 tmp.close() 132 # also generate a simple embeddable .pyc file for signature_bootstrap.pyc 133 boot_name = "signature_bootstrap.py" if limited_api else "signature_bootstrap.pyc" 134 with open(boot_name, "rb") as ldr, open("signature_bootstrap_inc.h", "w") as inc: 135 _embed_bytefile(ldr, inc, limited_api) 136 os.chdir(cur_dir) 137 138 139def _embed_file(fin, fout): 140 """ 141 Format a text file for embedding in a C++ source file. 142 """ 143 # MSVC has a 64k string limitation. In C, it would be easy to create an 144 # array of 64 byte strings and use them as one big array. In C++ this does 145 # not work, since C++ insists in having the terminating nullbyte. 146 # Therefore, we split the string after an arbitrary number of lines 147 # (chunked file). 148 limit = 50 149 text = fin.readlines() 150 print(textwrap.dedent(""" 151 /* 152 * This is a ZIP archive of all Python files in the directory 153 * "shiboken2/shibokenmodule/files.dir/shibokensupport/signature" 154 * There is also a toplevel file "signature_bootstrap.py[c]" that will be 155 * directly executed from C++ as a bootstrap loader. 156 */ 157 """).strip(), file=fout) 158 block, blocks = 0, len(text) // limit + 1 159 for idx, line in enumerate(text): 160 if idx % limit == 0: 161 comma = "," if block else "" 162 block += 1 163 print(file=fout) 164 print('/* Block {block} of {blocks} */{comma}'.format(**locals()), file=fout) 165 print('\"{}\"'.format(line.strip()), file=fout) 166 print('/* Sentinel */, \"\"', file=fout) 167 168 169def _embed_bytefile(fin, fout, is_text): 170 """ 171 Format a binary file for embedding in a C++ source file. 172 This version works directly with a single .pyc file. 173 """ 174 fname = fin.name 175 remark = ("No .pyc file because '--LIMITED-API=yes'" if is_text else 176 "The .pyc header is stripped away") 177 print(textwrap.dedent(""" 178 /* 179 * This is the file "{fname}" as a simple byte array. 180 * It can be directly embedded without any further processing. 181 * {remark}. 182 */ 183 """).format(**locals()).strip(), file=fout) 184 headsize = ( 0 if is_text else 185 16 if sys.version_info >= (3, 7) else 12 if sys.version_info >= (3, 3) else 8) 186 binstr = fin.read()[headsize:] 187 if is_text: 188 try: 189 compile(binstr, fin.name, "exec") 190 except SyntaxError as e: 191 print(e) 192 traceback.print_exc(file=sys.stdout) 193 print(textwrap.dedent(""" 194 ************************************************************************* 195 *** 196 *** Could not compile the boot loader '{fname}'! 197 *** 198 ************************************************************************* 199 """).format(version=sys.version_info[:3], **locals())) 200 raise SystemError 201 else: 202 try: 203 marshal.loads(binstr) 204 except ValueError as e: 205 print(e) 206 traceback.print_exc(file=sys.stdout) 207 print(textwrap.dedent(""" 208 ************************************************************************* 209 *** 210 *** This Python version {version} seems to have a new .pyc header size. 211 *** Please correct the 'headsize' constant ({headsize}). 212 *** 213 ************************************************************************* 214 """).format(version=sys.version_info[:3], **locals())) 215 raise SystemError 216 217 print(file=fout) 218 use_ord = sys.version_info[0] == 2 219 for i in range(0, len(binstr), 16): 220 for c in bytes(binstr[i : i + 16]): 221 print("{:#4},".format(ord(c) if use_ord else c), file=fout, end="") 222 print(file=fout) 223 print("/* End Of File */", file=fout) 224 225 226def str2bool(v): 227 if v.lower() in ('yes', 'true', 't', 'y', '1'): 228 return True 229 elif v.lower() in ('no', 'false', 'f', 'n', '0'): 230 return False 231 else: 232 raise argparse.ArgumentTypeError('Boolean value expected.') 233 234 235if __name__ == "__main__": 236 parser = argparse.ArgumentParser() 237 parser.add_argument('--cmake-dir', nargs="?") 238 parser.add_argument('--limited-api', type=str2bool) 239 args = parser.parse_args() 240 if args.cmake_dir: 241 work_dir = os.path.abspath(args.cmake_dir) 242 create_zipfile(args.limited_api) 243