1############################################################################# 2## 3## Copyright (C) 2019 The Qt Company Ltd. 4## Contact: https://www.qt.io/licensing/ 5## 6## This file is part of Qt for Python. 7## 8## $QT_BEGIN_LICENSE:LGPL$ 9## Commercial License Usage 10## Licensees holding valid commercial Qt licenses may use this file in 11## accordance with the commercial license agreement provided with the 12## Software or, alternatively, in accordance with the terms contained in 13## a written agreement between you and The Qt Company. For licensing terms 14## and conditions see https://www.qt.io/terms-conditions. For further 15## information use the contact form at https://www.qt.io/contact-us. 16## 17## GNU Lesser General Public License Usage 18## Alternatively, this file may be used under the terms of the GNU Lesser 19## General Public License version 3 as published by the Free Software 20## Foundation and appearing in the file LICENSE.LGPL3 included in the 21## packaging of this file. Please review the following information to 22## ensure the GNU Lesser General Public License version 3 requirements 23## will be met: https://www.gnu.org/licenses/lgpl-3.0.html. 24## 25## GNU General Public License Usage 26## Alternatively, this file may be used under the terms of the GNU 27## General Public License version 2.0 or (at your option) the GNU General 28## Public license version 3 or any later version approved by the KDE Free 29## Qt Foundation. The licenses are as published by the Free Software 30## Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 31## included in the packaging of this file. Please review the following 32## information to ensure the GNU General Public License requirements will 33## be met: https://www.gnu.org/licenses/gpl-2.0.html and 34## https://www.gnu.org/licenses/gpl-3.0.html. 35## 36## $QT_END_LICENSE$ 37## 38############################################################################# 39 40from __future__ import print_function 41 42from argparse import ArgumentParser, RawTextHelpFormatter 43import datetime 44from enum import Enum 45import os 46import re 47import subprocess 48import sys 49import time 50import warnings 51 52 53DESC = """ 54Utility script for working with Qt for Python. 55 56Feel free to extend! 57 58Typical Usage: 59Update and build a repository: python qp5_tool -p -b 60 61qp5_tool.py uses a configuration file "%CONFIGFILE%" 62in the format key=value. 63 64It is possible to use repository-specific values by adding a key postfixed by 65a dash and the repository folder base name, eg: 66Modules-pyside-setup512=Core,Gui,Widgets,Network,Test 67 68Configuration keys: 69Acceleration Incredibuild or unset 70BuildArguments Arguments to setup.py 71Generator Generator to be used for CMake. Currently, only Ninja is 72 supported. 73Jobs Number of jobs to be run simultaneously 74Modules Comma separated list of modules to be built 75 (for --module-subset=) 76Python Python executable (Use python_d for debug builds on Windows) 77 78Arbitrary keys can be defined and referenced by $(name): 79 80MinimalModules=Core,Gui,Widgets,Network,Test 81Modules=$(MinimalModules),Multimedia 82Modules-pyside-setup-minimal=$(MinimalModules) 83""" 84 85 86class Acceleration(Enum): 87 NONE = 0 88 INCREDIBUILD = 1 89 90 91class BuildMode(Enum): 92 NONE = 0 93 BUILD = 1 94 RECONFIGURE = 2 95 MAKE = 3 96 97 98DEFAULT_BUILD_ARGS = ['--build-tests', '--skip-docs', '--quiet'] 99IS_WINDOWS = sys.platform == 'win32' 100INCREDIBUILD_CONSOLE = 'BuildConsole' if IS_WINDOWS else '/opt/incredibuild/bin/ib_console' 101# Config file keys 102ACCELERATION_KEY = 'Acceleration' 103BUILDARGUMENTS_KEY = 'BuildArguments' 104GENERATOR_KEY = 'Generator' 105JOBS_KEY = 'Jobs' 106MODULES_KEY = 'Modules' 107PYTHON_KEY = 'Python' 108 109DEFAULT_MODULES = "Core,Gui,Widgets,Network,Test,Qml,Quick,Multimedia,MultimediaWidgets" 110DEFAULT_CONFIG_FILE = "Modules={}\n".format(DEFAULT_MODULES) 111 112build_mode = BuildMode.NONE 113opt_dry_run = False 114 115 116def which(needle): 117 """Perform a path search""" 118 needles = [needle] 119 if IS_WINDOWS: 120 for ext in ("exe", "bat", "cmd"): 121 needles.append("{}.{}".format(needle, ext)) 122 123 for path in os.environ.get("PATH", "").split(os.pathsep): 124 for n in needles: 125 binary = os.path.join(path, n) 126 if os.path.isfile(binary): 127 return binary 128 return None 129 130 131def command_log_string(args, dir): 132 result = '[{}]'.format(os.path.basename(dir)) 133 for arg in args: 134 result += ' "{}"'.format(arg) if ' ' in arg else ' {}'.format(arg) 135 return result 136 137 138def execute(args): 139 """Execute a command and print to log""" 140 log_string = command_log_string(args, os.getcwd()) 141 print(log_string) 142 if opt_dry_run: 143 return 144 exit_code = subprocess.call(args) 145 if exit_code != 0: 146 raise RuntimeError('FAIL({}): {}'.format(exit_code, log_string)) 147 148 149def run_process_output(args): 150 """Run a process and return its output. Also run in dry_run mode""" 151 std_out = subprocess.Popen(args, universal_newlines=1, 152 stdout=subprocess.PIPE).stdout 153 result = [line.rstrip() for line in std_out.readlines()] 154 std_out.close() 155 return result 156 157 158def run_git(args): 159 """Run git in the current directory and its submodules""" 160 args.insert(0, git) # run in repo 161 execute(args) # run for submodules 162 module_args = [git, "submodule", "foreach"] 163 module_args.extend(args) 164 execute(module_args) 165 166 167def expand_reference(cache_dict, value): 168 """Expand references to other keys in config files $(name) by value.""" 169 pattern = re.compile(r"\$\([^)]+\)") 170 while True: 171 match = pattern.match(value) 172 if not match: 173 break 174 key = match.group(0)[2:-1] 175 value = value[:match.start(0)] + cache_dict[key] + value[match.end(0):] 176 return value 177 178 179def editor(): 180 editor = os.getenv('EDITOR') 181 if not editor: 182 return 'notepad' if IS_WINDOWS else 'vi' 183 editor = editor.strip() 184 if IS_WINDOWS: 185 # Windows: git requires quotes in the variable 186 if editor.startswith('"') and editor.endswith('"'): 187 editor = editor[1:-1] 188 editor = editor.replace('/', '\\') 189 return editor 190 191 192def edit_config_file(): 193 exit_code = -1 194 try: 195 exit_code = subprocess.call([editor(), config_file]) 196 except Exception as e: 197 reason = str(e) 198 print('Unable to launch: {}: {}'.format(editor(), reason)) 199 return exit_code 200 201 202""" 203Config file handling, cache and read function 204""" 205config_dict = {} 206 207 208def read_config_file(file_name): 209 """Read the config file into config_dict, expanding continuation lines""" 210 global config_dict 211 keyPattern = re.compile(r'^\s*([A-Za-z0-9\_\-]+)\s*=\s*(.*)$') 212 with open(file_name) as f: 213 while True: 214 line = f.readline() 215 if not line: 216 break 217 line = line.rstrip() 218 match = keyPattern.match(line) 219 if match: 220 key = match.group(1) 221 value = match.group(2) 222 while value.endswith('\\'): 223 value = value.rstrip('\\') 224 value += f.readline().rstrip() 225 config_dict[key] = expand_reference(config_dict, value) 226 227 228def read_config(key): 229 """ 230 Read a value from the '$HOME/.qp5_tool' configuration file. When given 231 a key 'key' for the repository directory '/foo/qt-5', check for the 232 repo-specific value 'key-qt5' and then for the general 'key'. 233 """ 234 if not config_dict: 235 read_config_file(config_file) 236 repo_value = config_dict.get(key + '-' + base_dir) 237 return repo_value if repo_value else config_dict.get(key) 238 239 240def read_bool_config(key): 241 value = read_config(key) 242 return value and value in ['1', 'true', 'True'] 243 244 245def read_int_config(key, default=-1): 246 value = read_config(key) 247 return int(value) if value else default 248 249 250def read_acceleration_config(): 251 value = read_config(ACCELERATION_KEY) 252 if value: 253 value = value.lower() 254 if value == 'incredibuild': 255 return Acceleration.INCREDIBUILD 256 return Acceleration.NONE 257 258 259def read_config_build_arguments(): 260 value = read_config(BUILDARGUMENTS_KEY) 261 if value: 262 return re.split(r'\s+', value) 263 return DEFAULT_BUILD_ARGS 264 265 266def read_config_modules_argument(): 267 value = read_config(MODULES_KEY) 268 if value and value != '' and value != 'all': 269 return '--module-subset=' + value 270 return None 271 272 273def read_config_python_binary(): 274 binary = read_config(PYTHON_KEY) 275 if binary: 276 return binary 277 # Use 'python3' unless virtualenv is set 278 use_py3 = (not os.environ.get('VIRTUAL_ENV') and which('python3')) 279 return 'python3' if use_py3 else 'python' 280 281 282def get_config_file(base_name): 283 home = os.getenv('HOME') 284 if IS_WINDOWS: 285 # Set a HOME variable on Windows such that scp. etc. 286 # feel at home (locating .ssh). 287 if not home: 288 home = os.getenv('HOMEDRIVE') + os.getenv('HOMEPATH') 289 os.environ['HOME'] = home 290 user = os.getenv('USERNAME') 291 config_file = os.path.join(os.getenv('APPDATA'), base_name) 292 else: 293 user = os.getenv('USER') 294 config_dir = os.path.join(home, '.config') 295 if os.path.exists(config_dir): 296 config_file = os.path.join(config_dir, base_name) 297 else: 298 config_file = os.path.join(home, '.' + base_name) 299 return config_file 300 301 302def build(target): 303 """Run configure and build steps""" 304 start_time = time.time() 305 306 arguments = [] 307 acceleration = read_acceleration_config() 308 if not IS_WINDOWS and acceleration == Acceleration.INCREDIBUILD: 309 arguments.append(INCREDIBUILD_CONSOLE) 310 arguments.append('--avoid') # caching, v0.96.74 311 arguments.extend([read_config_python_binary(), 'setup.py', target]) 312 arguments.extend(read_config_build_arguments()) 313 generator = read_config(GENERATOR_KEY) 314 if generator == 'Ninja': 315 arguments.extend(['--make-spec', 'ninja']) 316 jobs = read_int_config(JOBS_KEY) 317 if jobs > 1: 318 arguments.extend(['-j', str(jobs)]) 319 if build_mode != BuildMode.BUILD: 320 arguments.extend(['--reuse-build', '--ignore-git']) 321 if build_mode != BuildMode.RECONFIGURE: 322 arguments.append('--skip-cmake') 323 modules = read_config_modules_argument() 324 if modules: 325 arguments.append(modules) 326 if IS_WINDOWS and acceleration == Acceleration.INCREDIBUILD: 327 arg_string = ' '.join(arguments) 328 arguments = [INCREDIBUILD_CONSOLE, '/command={}'.format(arg_string)] 329 330 execute(arguments) 331 332 elapsed_time = int(time.time() - start_time) 333 print('--- Done({}s) ---'.format(elapsed_time)) 334 335 336def run_tests(): 337 """Run tests redirected into a log file with a time stamp""" 338 logfile_name = datetime.datetime.today().strftime("test_%Y%m%d_%H%M.txt") 339 binary = sys.executable 340 command = '"{}" testrunner.py test > {}'.format(binary, logfile_name) 341 print(command_log_string([command], os.getcwd())) 342 start_time = time.time() 343 result = 0 if opt_dry_run else os.system(command) 344 elapsed_time = int(time.time() - start_time) 345 print('--- Done({}s) ---'.format(elapsed_time)) 346 return result 347 348 349def create_argument_parser(desc): 350 parser = ArgumentParser(description=desc, formatter_class=RawTextHelpFormatter) 351 parser.add_argument('--dry-run', '-d', action='store_true', 352 help='Dry run, print commands') 353 parser.add_argument('--edit', '-e', action='store_true', 354 help='Edit config file') 355 parser.add_argument('--reset', '-r', action='store_true', 356 help='Git reset hard to upstream state') 357 parser.add_argument('--clean', '-c', action='store_true', 358 help='Git clean') 359 parser.add_argument('--pull', '-p', action='store_true', 360 help='Git pull') 361 parser.add_argument('--build', '-b', action='store_true', 362 help='Build (configure + build)') 363 parser.add_argument('--make', '-m', action='store_true', help='Make') 364 parser.add_argument('--no-install', '-n', action='store_true', 365 help='Run --build only, do not install') 366 parser.add_argument('--Make', '-M', action='store_true', 367 help='cmake + Make (continue broken build)') 368 parser.add_argument('--test', '-t', action='store_true', 369 help='Run tests') 370 parser.add_argument('--version', '-v', action='version', version='%(prog)s 1.0') 371 return parser 372 373 374if __name__ == '__main__': 375 git = None 376 base_dir = None 377 config_file = None 378 user = None 379 380 config_file = get_config_file('qp5_tool.conf') 381 argument_parser = create_argument_parser(DESC.replace('%CONFIGFILE%', config_file)) 382 options = argument_parser.parse_args() 383 opt_dry_run = options.dry_run 384 385 if options.edit: 386 sys.exit(edit_config_file()) 387 388 if options.build: 389 build_mode = BuildMode.BUILD 390 elif options.make: 391 build_mode = BuildMode.MAKE 392 elif options.Make: 393 build_mode = BuildMode.RECONFIGURE 394 395 if build_mode == BuildMode.NONE and not (options.clean or options.reset 396 or options.pull or options.test): 397 argument_parser.print_help() 398 sys.exit(0) 399 400 git = 'git' 401 if which(git) is None: 402 warnings.warn('Unable to find git', RuntimeWarning) 403 sys.exit(-1) 404 405 if not os.path.exists(config_file): 406 print('Create initial config file ', config_file, " ..") 407 with open(config_file, 'w') as f: 408 f.write(DEFAULT_CONFIG_FILE.format(' '.join(DEFAULT_BUILD_ARGS))) 409 410 while not os.path.exists('.gitmodules'): 411 cwd = os.getcwd() 412 if cwd == '/' or (IS_WINDOWS and len(cwd) < 4): 413 warnings.warn('Unable to find git root', RuntimeWarning) 414 sys.exit(-1) 415 os.chdir(os.path.dirname(cwd)) 416 417 base_dir = os.path.basename(os.getcwd()) 418 419 if options.clean: 420 run_git(['clean', '-dxf']) 421 422 if options.reset: 423 run_git(['reset', '--hard', '@{upstream}']) 424 425 if options.pull: 426 run_git(['pull', '--rebase']) 427 428 if build_mode != BuildMode.NONE: 429 target = 'build' if options.no_install else 'install' 430 build(target) 431 432 if options.test: 433 sys.exit(run_tests()) 434 435 sys.exit(0) 436