1#############################################################################
2##
3## Copyright (C) 2019 The Qt Company Ltd.
4## Contact: https://www.qt.io/licensing/
5##
6## This file is part of Qt for Python.
7##
8## $QT_BEGIN_LICENSE:LGPL$
9## Commercial License Usage
10## Licensees holding valid commercial Qt licenses may use this file in
11## accordance with the commercial license agreement provided with the
12## Software or, alternatively, in accordance with the terms contained in
13## a written agreement between you and The Qt Company. For licensing terms
14## and conditions see https://www.qt.io/terms-conditions. For further
15## information use the contact form at https://www.qt.io/contact-us.
16##
17## GNU Lesser General Public License Usage
18## Alternatively, this file may be used under the terms of the GNU Lesser
19## General Public License version 3 as published by the Free Software
20## Foundation and appearing in the file LICENSE.LGPL3 included in the
21## packaging of this file. Please review the following information to
22## ensure the GNU Lesser General Public License version 3 requirements
23## will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24##
25## GNU General Public License Usage
26## Alternatively, this file may be used under the terms of the GNU
27## General Public License version 2.0 or (at your option) the GNU General
28## Public license version 3 or any later version approved by the KDE Free
29## Qt Foundation. The licenses are as published by the Free Software
30## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31## included in the packaging of this file. Please review the following
32## information to ensure the GNU General Public License requirements will
33## be met: https://www.gnu.org/licenses/gpl-2.0.html and
34## https://www.gnu.org/licenses/gpl-3.0.html.
35##
36## $QT_END_LICENSE$
37##
38#############################################################################
39
40from __future__ import print_function
41
42from argparse import ArgumentParser, RawTextHelpFormatter
43import datetime
44from enum import Enum
45import os
46import re
47import subprocess
48import sys
49import time
50import warnings
51
52
53DESC = """
54Utility script for working with Qt for Python.
55
56Feel free to extend!
57
58Typical Usage:
59Update and build a repository: python qp5_tool -p -b
60
61qp5_tool.py uses a configuration file "%CONFIGFILE%"
62in the format key=value.
63
64It is possible to use repository-specific values by adding a key postfixed by
65a dash and the repository folder base name, eg:
66Modules-pyside-setup512=Core,Gui,Widgets,Network,Test
67
68Configuration keys:
69Acceleration     Incredibuild or unset
70BuildArguments   Arguments to setup.py
71Generator        Generator to be used for CMake. Currently, only Ninja is
72                 supported.
73Jobs             Number of jobs to be run simultaneously
74Modules          Comma separated list of modules to be built
75                 (for --module-subset=)
76Python           Python executable (Use python_d for debug builds on Windows)
77
78Arbitrary keys can be defined and referenced by $(name):
79
80MinimalModules=Core,Gui,Widgets,Network,Test
81Modules=$(MinimalModules),Multimedia
82Modules-pyside-setup-minimal=$(MinimalModules)
83"""
84
85
86class Acceleration(Enum):
87    NONE = 0
88    INCREDIBUILD = 1
89
90
91class BuildMode(Enum):
92    NONE = 0
93    BUILD = 1
94    RECONFIGURE = 2
95    MAKE = 3
96
97
98DEFAULT_BUILD_ARGS = ['--build-tests', '--skip-docs', '--quiet']
99IS_WINDOWS = sys.platform == 'win32'
100INCREDIBUILD_CONSOLE = 'BuildConsole' if IS_WINDOWS else '/opt/incredibuild/bin/ib_console'
101# Config file keys
102ACCELERATION_KEY = 'Acceleration'
103BUILDARGUMENTS_KEY = 'BuildArguments'
104GENERATOR_KEY = 'Generator'
105JOBS_KEY = 'Jobs'
106MODULES_KEY = 'Modules'
107PYTHON_KEY = 'Python'
108
109DEFAULT_MODULES = "Core,Gui,Widgets,Network,Test,Qml,Quick,Multimedia,MultimediaWidgets"
110DEFAULT_CONFIG_FILE = "Modules={}\n".format(DEFAULT_MODULES)
111
112build_mode = BuildMode.NONE
113opt_dry_run = False
114
115
116def which(needle):
117    """Perform a path search"""
118    needles = [needle]
119    if IS_WINDOWS:
120        for ext in ("exe", "bat", "cmd"):
121            needles.append("{}.{}".format(needle, ext))
122
123    for path in os.environ.get("PATH", "").split(os.pathsep):
124        for n in needles:
125            binary = os.path.join(path, n)
126            if os.path.isfile(binary):
127                return binary
128    return None
129
130
131def command_log_string(args, dir):
132    result = '[{}]'.format(os.path.basename(dir))
133    for arg in args:
134        result += ' "{}"'.format(arg) if ' ' in arg else ' {}'.format(arg)
135    return result
136
137
138def execute(args):
139    """Execute a command and print to log"""
140    log_string = command_log_string(args, os.getcwd())
141    print(log_string)
142    if opt_dry_run:
143        return
144    exit_code = subprocess.call(args)
145    if exit_code != 0:
146        raise RuntimeError('FAIL({}): {}'.format(exit_code, log_string))
147
148
149def run_process_output(args):
150    """Run a process and return its output. Also run in dry_run mode"""
151    std_out = subprocess.Popen(args, universal_newlines=1,
152                               stdout=subprocess.PIPE).stdout
153    result = [line.rstrip() for line in std_out.readlines()]
154    std_out.close()
155    return result
156
157
158def run_git(args):
159    """Run git in the current directory and its submodules"""
160    args.insert(0, git)  # run in repo
161    execute(args)  # run for submodules
162    module_args = [git, "submodule", "foreach"]
163    module_args.extend(args)
164    execute(module_args)
165
166
167def expand_reference(cache_dict, value):
168    """Expand references to other keys in config files $(name) by value."""
169    pattern = re.compile(r"\$\([^)]+\)")
170    while True:
171        match = pattern.match(value)
172        if not match:
173            break
174        key = match.group(0)[2:-1]
175        value = value[:match.start(0)] + cache_dict[key] + value[match.end(0):]
176    return value
177
178
179def editor():
180    editor = os.getenv('EDITOR')
181    if not editor:
182        return 'notepad' if IS_WINDOWS else 'vi'
183    editor = editor.strip()
184    if IS_WINDOWS:
185        # Windows: git requires quotes in the variable
186        if editor.startswith('"') and editor.endswith('"'):
187            editor = editor[1:-1]
188        editor = editor.replace('/', '\\')
189    return editor
190
191
192def edit_config_file():
193    exit_code = -1
194    try:
195        exit_code = subprocess.call([editor(), config_file])
196    except Exception as e:
197        reason = str(e)
198        print('Unable to launch: {}: {}'.format(editor(), reason))
199    return exit_code
200
201
202"""
203Config file handling, cache and read function
204"""
205config_dict = {}
206
207
208def read_config_file(file_name):
209    """Read the config file into config_dict, expanding continuation lines"""
210    global config_dict
211    keyPattern = re.compile(r'^\s*([A-Za-z0-9\_\-]+)\s*=\s*(.*)$')
212    with open(file_name) as f:
213        while True:
214            line = f.readline()
215            if not line:
216                break
217            line = line.rstrip()
218            match = keyPattern.match(line)
219            if match:
220                key = match.group(1)
221                value = match.group(2)
222                while value.endswith('\\'):
223                    value = value.rstrip('\\')
224                    value += f.readline().rstrip()
225                config_dict[key] = expand_reference(config_dict, value)
226
227
228def read_config(key):
229    """
230    Read a value from the '$HOME/.qp5_tool' configuration file. When given
231    a key 'key' for the repository directory '/foo/qt-5', check for the
232    repo-specific value 'key-qt5' and then for the general 'key'.
233    """
234    if not config_dict:
235        read_config_file(config_file)
236    repo_value = config_dict.get(key + '-' + base_dir)
237    return repo_value if repo_value else config_dict.get(key)
238
239
240def read_bool_config(key):
241    value = read_config(key)
242    return value and value in ['1', 'true', 'True']
243
244
245def read_int_config(key, default=-1):
246    value = read_config(key)
247    return int(value) if value else default
248
249
250def read_acceleration_config():
251    value = read_config(ACCELERATION_KEY)
252    if value:
253        value = value.lower()
254        if value == 'incredibuild':
255            return Acceleration.INCREDIBUILD
256    return Acceleration.NONE
257
258
259def read_config_build_arguments():
260    value = read_config(BUILDARGUMENTS_KEY)
261    if value:
262        return re.split(r'\s+', value)
263    return DEFAULT_BUILD_ARGS
264
265
266def read_config_modules_argument():
267    value = read_config(MODULES_KEY)
268    if value and value != '' and value != 'all':
269        return '--module-subset=' + value
270    return None
271
272
273def read_config_python_binary():
274    binary = read_config(PYTHON_KEY)
275    if binary:
276        return binary
277    # Use 'python3' unless virtualenv is set
278    use_py3 = (not os.environ.get('VIRTUAL_ENV') and which('python3'))
279    return 'python3' if use_py3 else 'python'
280
281
282def get_config_file(base_name):
283    home = os.getenv('HOME')
284    if IS_WINDOWS:
285        # Set a HOME variable on Windows such that scp. etc.
286        # feel at home (locating .ssh).
287        if not home:
288            home = os.getenv('HOMEDRIVE') + os.getenv('HOMEPATH')
289            os.environ['HOME'] = home
290        user = os.getenv('USERNAME')
291        config_file = os.path.join(os.getenv('APPDATA'), base_name)
292    else:
293        user = os.getenv('USER')
294        config_dir = os.path.join(home, '.config')
295        if os.path.exists(config_dir):
296            config_file = os.path.join(config_dir, base_name)
297        else:
298            config_file = os.path.join(home, '.' + base_name)
299    return config_file
300
301
302def build(target):
303    """Run configure and build steps"""
304    start_time = time.time()
305
306    arguments = []
307    acceleration = read_acceleration_config()
308    if not IS_WINDOWS and acceleration == Acceleration.INCREDIBUILD:
309        arguments.append(INCREDIBUILD_CONSOLE)
310        arguments.append('--avoid')  # caching, v0.96.74
311    arguments.extend([read_config_python_binary(), 'setup.py', target])
312    arguments.extend(read_config_build_arguments())
313    generator = read_config(GENERATOR_KEY)
314    if generator == 'Ninja':
315        arguments.extend(['--make-spec', 'ninja'])
316    jobs = read_int_config(JOBS_KEY)
317    if jobs > 1:
318        arguments.extend(['-j', str(jobs)])
319    if build_mode != BuildMode.BUILD:
320        arguments.extend(['--reuse-build', '--ignore-git'])
321        if build_mode != BuildMode.RECONFIGURE:
322            arguments.append('--skip-cmake')
323    modules = read_config_modules_argument()
324    if modules:
325        arguments.append(modules)
326    if IS_WINDOWS and acceleration == Acceleration.INCREDIBUILD:
327        arg_string = ' '.join(arguments)
328        arguments = [INCREDIBUILD_CONSOLE, '/command={}'.format(arg_string)]
329
330    execute(arguments)
331
332    elapsed_time = int(time.time() - start_time)
333    print('--- Done({}s) ---'.format(elapsed_time))
334
335
336def run_tests():
337    """Run tests redirected into a log file with a time stamp"""
338    logfile_name = datetime.datetime.today().strftime("test_%Y%m%d_%H%M.txt")
339    binary = sys.executable
340    command = '"{}" testrunner.py test > {}'.format(binary, logfile_name)
341    print(command_log_string([command], os.getcwd()))
342    start_time = time.time()
343    result = 0 if opt_dry_run else os.system(command)
344    elapsed_time = int(time.time() - start_time)
345    print('--- Done({}s) ---'.format(elapsed_time))
346    return result
347
348
349def create_argument_parser(desc):
350    parser = ArgumentParser(description=desc, formatter_class=RawTextHelpFormatter)
351    parser.add_argument('--dry-run', '-d', action='store_true',
352                        help='Dry run, print commands')
353    parser.add_argument('--edit', '-e', action='store_true',
354                        help='Edit config file')
355    parser.add_argument('--reset', '-r', action='store_true',
356                        help='Git reset hard to upstream state')
357    parser.add_argument('--clean', '-c', action='store_true',
358                        help='Git clean')
359    parser.add_argument('--pull', '-p', action='store_true',
360                        help='Git pull')
361    parser.add_argument('--build', '-b', action='store_true',
362                        help='Build (configure + build)')
363    parser.add_argument('--make', '-m', action='store_true', help='Make')
364    parser.add_argument('--no-install', '-n', action='store_true',
365                        help='Run --build only, do not install')
366    parser.add_argument('--Make', '-M', action='store_true',
367                        help='cmake + Make (continue broken build)')
368    parser.add_argument('--test', '-t', action='store_true',
369                        help='Run tests')
370    parser.add_argument('--version', '-v', action='version', version='%(prog)s 1.0')
371    return parser
372
373
374if __name__ == '__main__':
375    git = None
376    base_dir = None
377    config_file = None
378    user = None
379
380    config_file = get_config_file('qp5_tool.conf')
381    argument_parser = create_argument_parser(DESC.replace('%CONFIGFILE%', config_file))
382    options = argument_parser.parse_args()
383    opt_dry_run = options.dry_run
384
385    if options.edit:
386        sys.exit(edit_config_file())
387
388    if options.build:
389        build_mode = BuildMode.BUILD
390    elif options.make:
391        build_mode = BuildMode.MAKE
392    elif options.Make:
393        build_mode = BuildMode.RECONFIGURE
394
395    if build_mode == BuildMode.NONE and not (options.clean or options.reset
396                                             or options.pull or options.test):
397        argument_parser.print_help()
398        sys.exit(0)
399
400    git = 'git'
401    if which(git) is None:
402        warnings.warn('Unable to find git', RuntimeWarning)
403        sys.exit(-1)
404
405    if not os.path.exists(config_file):
406        print('Create initial config file ', config_file, " ..")
407        with open(config_file, 'w') as f:
408            f.write(DEFAULT_CONFIG_FILE.format(' '.join(DEFAULT_BUILD_ARGS)))
409
410    while not os.path.exists('.gitmodules'):
411        cwd = os.getcwd()
412        if cwd == '/' or (IS_WINDOWS and len(cwd) < 4):
413            warnings.warn('Unable to find git root', RuntimeWarning)
414            sys.exit(-1)
415        os.chdir(os.path.dirname(cwd))
416
417    base_dir = os.path.basename(os.getcwd())
418
419    if options.clean:
420        run_git(['clean', '-dxf'])
421
422    if options.reset:
423        run_git(['reset', '--hard', '@{upstream}'])
424
425    if options.pull:
426        run_git(['pull', '--rebase'])
427
428    if build_mode != BuildMode.NONE:
429        target = 'build' if options.no_install else 'install'
430        build(target)
431
432    if options.test:
433        sys.exit(run_tests())
434
435    sys.exit(0)
436