1#############################################################################
2##
3## Copyright (C) 2019 The Qt Company Ltd.
4## Contact: https://www.qt.io/licensing/
5##
6## This file is part of PySide2.
7##
8## $QT_BEGIN_LICENSE:LGPL$
9## Commercial License Usage
10## Licensees holding valid commercial Qt licenses may use this file in
11## accordance with the commercial license agreement provided with the
12## Software or, alternatively, in accordance with the terms contained in
13## a written agreement between you and The Qt Company. For licensing terms
14## and conditions see https://www.qt.io/terms-conditions. For further
15## information use the contact form at https://www.qt.io/contact-us.
16##
17## GNU Lesser General Public License Usage
18## Alternatively, this file may be used under the terms of the GNU Lesser
19## General Public License version 3 as published by the Free Software
20## Foundation and appearing in the file LICENSE.LGPL3 included in the
21## packaging of this file. Please review the following information to
22## ensure the GNU Lesser General Public License version 3 requirements
23## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24##
25## GNU General Public License Usage
26## Alternatively, this file may be used under the terms of the GNU
27## General Public License version 2.0 or (at your option) the GNU General
28## Public license version 3 or any later version approved by the KDE Free
29## Qt Foundation. The licenses are as published by the Free Software
30## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31## included in the packaging of this file. Please review the following
32## information to ensure the GNU General Public License requirements will
33## be met: https://www.gnu.org/licenses/gpl-2.0.html and
34## https://www.gnu.org/licenses/gpl-3.0.html.
35##
36## $QT_END_LICENSE$
37##
38#############################################################################
39
40"""
41embedding_generator.py
42
43This file takes the content of the two supported directories and inserts
44it into a zip file. The zip file is then converted into a C++ source
45file that can easily be unpacked again with Python (see signature.cpp,
46constant 'PySide_PythonCode').
47
48Note that this _is_ a zipfile, but since it is embedded into the shiboken
49binary, we cannot use the zipimport module from Python.
50But a similar solution is possible that allows for normal imports.
51
52See signature_bootstrap.py for details.
53"""
54
55from __future__ import print_function, absolute_import
56
57import sys
58import os
59import subprocess
60import textwrap
61import tempfile
62import argparse
63import marshal
64import traceback
65
66# work_dir is set to the source for testing, onl.
67# It can be overridden in the command line.
68work_dir = os.path.abspath(os.path.dirname(__file__))
69embed_dir = work_dir
70cur_dir = os.getcwd()
71source_dir = os.path.normpath(os.path.join(work_dir, "..", "..", ".."))
72assert os.path.basename(source_dir) == "sources"
73build_script_dir = os.path.normpath(os.path.join(work_dir, "..", "..", "..", ".."))
74assert os.path.exists(os.path.join(build_script_dir, "build_scripts"))
75
76sys.path.insert(0, build_script_dir)
77
78from build_scripts import utils
79
80
81def runpy(cmd, **kw):
82    subprocess.call([sys.executable, '-E'] + cmd.split(), **kw)
83
84
85def create_zipfile(limited_api):
86    """
87    Collect all Python files, compile them, create a zip file
88    and make a chunked base64 encoded file from it.
89    """
90    zip_name = "signature.zip"
91    inc_name = "signature_inc.h"
92    flag = '-b' if sys.version_info >= (3,) else ''
93    os.chdir(work_dir)
94
95    # Remove all left-over py[co] and other files first, in case we use '--reuse-build'.
96    # Note that we could improve that with the PyZipfile function to use .pyc files
97    # in different folders, but that makes only sense when COIN allows us to have
98    # multiple Python versions in parallel.
99    from os.path import join, getsize
100    for root, dirs, files in os.walk(work_dir):
101        for name in files:
102            fpath = os.path.join(root, name)
103            ew = name.endswith
104            if ew(".pyc") or ew(".pyo") or ew(".zip") or ew(".inc"):
105                os.remove(fpath)
106    # We copy every Python file into this dir, but only for the right version.
107    # For testing in the source dir, we need to filter.
108    if sys.version_info[0] == 3:
109        ignore = "backport_inspect.py typing27.py".split()
110    else:
111        ignore = "".split()
112    utils.copydir(os.path.join(source_dir, "shiboken2", "shibokenmodule", "files.dir", "shibokensupport"),
113                  os.path.join(work_dir, "shibokensupport"),
114                  ignore=ignore, file_filter_function=lambda name, n2: name.endswith(".py"))
115    if embed_dir != work_dir:
116        utils.copyfile(os.path.join(embed_dir, "signature_bootstrap.py"), work_dir)
117
118    if limited_api:
119        pass   # We cannot compile, unless we have folders per Python version
120    else:
121        files = ' '.join(fn for fn in os.listdir('.'))
122        runpy('-m compileall -q {flag} {files}'.format(**locals()))
123    files = ' '.join(fn for fn in os.listdir('.') if not fn == zip_name)
124    runpy('-m zipfile -c {zip_name} {files}'.format(**locals()))
125    tmp = tempfile.TemporaryFile(mode="w+")
126    runpy('-m base64 {zip_name}'.format(**locals()), stdout=tmp)
127    # now generate the include file
128    tmp.seek(0)
129    with open(inc_name, "w") as inc:
130        _embed_file(tmp, inc)
131    tmp.close()
132    # also generate a simple embeddable .pyc file for signature_bootstrap.pyc
133    boot_name = "signature_bootstrap.py" if limited_api else "signature_bootstrap.pyc"
134    with open(boot_name, "rb") as ldr, open("signature_bootstrap_inc.h", "w") as inc:
135        _embed_bytefile(ldr, inc, limited_api)
136    os.chdir(cur_dir)
137
138
139def _embed_file(fin, fout):
140    """
141    Format a text file for embedding in a C++ source file.
142    """
143    # MSVC has a 64k string limitation. In C, it would be easy to create an
144    # array of 64 byte strings and use them as one big array. In C++ this does
145    # not work, since C++ insists in having the terminating nullbyte.
146    # Therefore, we split the string after an arbitrary number of lines
147    # (chunked file).
148    limit = 50
149    text = fin.readlines()
150    print(textwrap.dedent("""
151        /*
152         * This is a ZIP archive of all Python files in the directory
153         *         "shiboken2/shibokenmodule/files.dir/shibokensupport/signature"
154         * There is also a toplevel file "signature_bootstrap.py[c]" that will be
155         * directly executed from C++ as a bootstrap loader.
156         */
157         """).strip(), file=fout)
158    block, blocks = 0, len(text) // limit + 1
159    for idx, line in enumerate(text):
160        if idx % limit == 0:
161            comma = "," if block else ""
162            block += 1
163            print(file=fout)
164            print('/* Block {block} of {blocks} */{comma}'.format(**locals()), file=fout)
165        print('\"{}\"'.format(line.strip()), file=fout)
166    print('/* Sentinel */, \"\"', file=fout)
167
168
169def _embed_bytefile(fin, fout, is_text):
170    """
171    Format a binary file for embedding in a C++ source file.
172    This version works directly with a single .pyc file.
173    """
174    fname = fin.name
175    remark = ("No .pyc file because '--LIMITED-API=yes'" if is_text else
176              "The .pyc header is stripped away")
177    print(textwrap.dedent("""
178        /*
179         * This is the file "{fname}" as a simple byte array.
180         * It can be directly embedded without any further processing.
181         * {remark}.
182         */
183         """).format(**locals()).strip(), file=fout)
184    headsize = ( 0 if is_text else
185                16 if sys.version_info >= (3, 7) else 12 if sys.version_info >= (3, 3) else 8)
186    binstr = fin.read()[headsize:]
187    if is_text:
188        try:
189            compile(binstr, fin.name, "exec")
190        except SyntaxError as e:
191            print(e)
192            traceback.print_exc(file=sys.stdout)
193            print(textwrap.dedent("""
194                *************************************************************************
195                ***
196                *** Could not compile the boot loader '{fname}'!
197                ***
198                *************************************************************************
199                """).format(version=sys.version_info[:3], **locals()))
200            raise SystemError
201    else:
202        try:
203            marshal.loads(binstr)
204        except ValueError as e:
205            print(e)
206            traceback.print_exc(file=sys.stdout)
207            print(textwrap.dedent("""
208                *************************************************************************
209                ***
210                *** This Python version {version} seems to have a new .pyc header size.
211                *** Please correct the 'headsize' constant ({headsize}).
212                ***
213                *************************************************************************
214                """).format(version=sys.version_info[:3], **locals()))
215            raise SystemError
216
217    print(file=fout)
218    use_ord = sys.version_info[0] == 2
219    for i in range(0, len(binstr), 16):
220        for c in bytes(binstr[i : i + 16]):
221            print("{:#4},".format(ord(c) if use_ord else c), file=fout, end="")
222        print(file=fout)
223    print("/* End Of File */", file=fout)
224
225
226def str2bool(v):
227    if v.lower() in ('yes', 'true', 't', 'y', '1'):
228        return True
229    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
230        return False
231    else:
232        raise argparse.ArgumentTypeError('Boolean value expected.')
233
234
235if __name__ == "__main__":
236    parser = argparse.ArgumentParser()
237    parser.add_argument('--cmake-dir', nargs="?")
238    parser.add_argument('--limited-api', type=str2bool)
239    args = parser.parse_args()
240    if args.cmake_dir:
241        work_dir = os.path.abspath(args.cmake_dir)
242    create_zipfile(args.limited_api)
243