1#!/usr/bin/python 2 3# Copyright (c) 2017 nexB Inc. and others. All rights reserved. 4# http://nexb.com and http://aboutcode.org 5# 6# This software is licensed under the Apache License version 2.0. 7# 8# You may not use this software except in compliance with the License. 9# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 10# Unless required by applicable law or agreed to in writing, software distributed 11# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12# CONDITIONS OF ANY KIND, either express or implied. See the License for the 13# specific language governing permissions and limitations under the License. 14 15""" 16This script a configuration helper to select pip requirement files to install 17and python and shell configuration scripts to execute based on provided config 18directories paths arguments and the operating system platform. To use, create 19a configuration directory tree that contains any of these: 20 21 * Requirements files named with this convention: 22 - base.txt contains common requirements installed on all platforms. 23 - win.txt, linux.txt, mac.txt, posix.txt are os-specific requirements. 24 25 * Python scripts files named with this convention: 26 - base.py is a common script executed on all os before os-specific scripts. 27 - win.py, linux.py, mac.py, posix.py are os-specific scripts to execute. 28 29 * Shell or Windows CMD scripts files named with this convention: 30 - win.bat is a windows bat file to execute 31 - posix.sh, linux.sh, mac.sh are os-specific scripts to execute. 32 33The config directory structure contains one or more directories paths. This 34way you can have a main configuration and additional sub-configurations of a 35product such as for prod, test, ci, dev, or anything else. 36 37All scripts and requirements are optional and only used if presents. Scripts 38are executed in sequence, one after the other after all requirements are 39installed, so they may import from any installed requirement. 40 41The execution order is: 42 - requirements installation 43 - python scripts execution 44 - shell scripts execution 45 46On posix, posix Python and shell scripts are executed before mac or linux 47scripts. 48 49The base scripts or packages are always installed first before platform- 50specific ones. 51 52For example a tree could be looking like this:: 53 etc/conf 54 base.txt : base pip requirements for all platforms 55 linux.txt : linux-only pip requirements 56 base.py : base config script for all platforms 57 win.py : windows-only config script 58 posix.sh: posix-only shell script 59 60 etc/conf/prod 61 base.txt : base pip requirements for all platforms 62 linux.txt : linux-only pip requirements 63 linux.sh : linux-only script 64 base.py : base config script for all platforms 65 mac.py : mac-only config script 66""" 67 68from __future__ import print_function 69 70import os 71import stat 72import sys 73import shutil 74import subprocess 75 76# platform-specific file base names 77sys_platform = str(sys.platform).lower() 78on_win = False 79if 'linux' in sys_platform: 80 platform_names = ('posix', 'linux',) 81elif'win32' in sys_platform: 82 platform_names = ('win',) 83 on_win = True 84elif 'darwin' in sys_platform: 85 platform_names = ('posix', 'mac',) 86else: 87 raise Exception('Unsupported OS/platform') 88 platform_names = tuple() 89 90# common file basenames for requirements and scripts 91base = ('base',) 92 93# known full file names with txt extension for requirements 94# base is always last 95requirements = tuple(p + '.txt' for p in platform_names + base) 96 97# known full file names with py extensions for scripts 98# base is always last 99python_scripts = tuple(p + '.py' for p in platform_names + base) 100 101# known full file names of shell scripts 102# there is no base for scripts: they cannot work cross OS (cmd vs. sh) 103shell_scripts = tuple(p + '.sh' for p in platform_names) 104if on_win: 105 shell_scripts = ('win.bat',) 106 107 108def call(cmd, root_dir, quiet=True): 109 """ Run a `cmd` command (as a list of args) with all env vars.""" 110 cmd = ' '.join(cmd) 111 # if not quiet: 112 # print(' Running command:', repr(cmd)) 113 114 if subprocess.Popen(cmd, shell=True, env=dict(os.environ), cwd=root_dir).wait() != 0: 115 print() 116 print('Failed to execute command:\n%(cmd)s. Aborting...' % locals()) 117 sys.exit(1) 118 119 120def find_pycache(root_dir): 121 """ 122 Yield __pycache__ directory paths found in root_dir as paths relative to 123 root_dir. 124 """ 125 for top, dirs, _files in os.walk(root_dir): 126 for d in dirs: 127 if d == '__pycache__': 128 dir_path = os.path.join(top, d) 129 dir_path = dir_path.replace(root_dir, '', 1) 130 dir_path = dir_path.strip(os.path.sep) 131 yield dir_path 132 133 134def clean(root_dir): 135 """ 136 Remove cleanable directories and files in root_dir. 137 """ 138 print('* Cleaning ...') 139 cleanable = '''build bin lib Lib include Include Scripts local 140 django_background_task.log 141 develop-eggs eggs parts .installed.cfg 142 .Python 143 .cache 144 pip-selfcheck.json 145 '''.split() 146 147 # also clean __pycache__ if any 148 cleanable.extend(find_pycache(root_dir)) 149 150 for d in cleanable: 151 try: 152 loc = os.path.join(root_dir, d) 153 if os.path.exists(loc): 154 if os.path.isdir(loc): 155 shutil.rmtree(loc) 156 else: 157 os.remove(loc) 158 except: 159 pass 160 161 162def build_pip_dirs_args(paths, root_dir, option='--extra-search-dir='): 163 """ 164 Return an iterable of pip command line options for `option` of pip using a 165 list of `paths` to directories. 166 """ 167 for path in paths: 168 if not os.path.isabs(path): 169 path = os.path.join(root_dir, path) 170 if os.path.exists(path): 171 yield option + '"' + path + '"' 172 173 174def create_virtualenv(std_python, root_dir, tpp_dirs, quiet=False): 175 """ 176 Create a virtualenv in `root_dir` using the `std_python` Python 177 executable. One of the `tpp_dirs` must contain a vendored virtualenv.py and 178 virtualenv dependencies such as setuptools and pip packages. 179 180 @std_python: Path or name of the Python executable to use. 181 182 @root_dir: directory in which the virtualenv will be created. This is also 183 the root directory for the project and the base directory for vendored 184 components directory paths. 185 186 @tpp_dirs: list of directory paths relative to `root_dir` containing 187 vendored Python distributions that pip will use to find required 188 components. 189 """ 190 if not quiet: 191 print('* Configuring Python ...') 192 # search virtualenv.py in the tpp_dirs. keep the first found 193 venv_py = None 194 for tpd in tpp_dirs: 195 venv = os.path.join(root_dir, tpd, 'virtualenv.py') 196 if os.path.exists(venv): 197 venv_py = '"' + venv + '"' 198 break 199 200 # error out if venv_py not found 201 if not venv_py: 202 print('Configuration Error ... aborting.') 203 exit(1) 204 205 vcmd = [std_python, venv_py, '--never-download'] 206 if quiet: 207 vcmd += ['--quiet'] 208 # third parties may be in more than one directory 209 vcmd.extend(build_pip_dirs_args(tpp_dirs, root_dir)) 210 # we create the virtualenv in the root_dir 211 vcmd.append('"' + root_dir + '"') 212 call(vcmd, root_dir, quiet) 213 214 215def activate(root_dir): 216 """ Activate a virtualenv in the current process.""" 217 print('* Activating ...') 218 bin_dir = os.path.join(root_dir, 'bin') 219 activate_this = os.path.join(bin_dir, 'activate_this.py') 220 with open(activate_this) as f: 221 code = compile(f.read(), activate_this, 'exec') 222 exec(code, dict(__file__=activate_this)) 223 224 225def install_3pp(configs, root_dir, tpp_dirs, quiet=False): 226 """ 227 Install requirements from requirement files found in `configs` with pip, 228 using the vendored components in `tpp_dirs`. 229 """ 230 if not quiet: 231 print('* Installing components ...') 232 requirement_files = get_conf_files(configs, root_dir, requirements) 233 for req_file in requirement_files: 234 if on_win: 235 pcmd = ['python', '-m'] 236 else: 237 pcmd = [] 238 pcmd += ['pip', 'install', '--no-index', '--no-cache-dir'] 239 if quiet: 240 pcmd += ['--quiet'] 241 242 pip_dir_args = list(build_pip_dirs_args(tpp_dirs, root_dir, '--find-links=')) 243 pcmd.extend(pip_dir_args) 244 req_loc = os.path.join(root_dir, req_file) 245 pcmd.extend(['-r' , '"' + req_loc + '"']) 246 call(pcmd, root_dir, quiet) 247 248 249def run_scripts(configs, root_dir, configured_python, quiet=False): 250 """ 251 Run Python scripts and shell scripts found in `configs`. 252 """ 253 if not quiet: 254 print('* Configuring ...') 255 # Run Python scripts for each configurations 256 for py_script in get_conf_files(configs, root_dir, python_scripts): 257 cmd = [configured_python, '"' + os.path.join(root_dir, py_script) + '"'] 258 call(cmd, root_dir, quiet) 259 260 # Run sh_script scripts for each configurations 261 for sh_script in get_conf_files(configs, root_dir, shell_scripts): 262 # we source the scripts on posix 263 cmd = ['.'] 264 if on_win: 265 cmd = [] 266 cmd = cmd + [os.path.join(root_dir, sh_script)] 267 call(cmd, root_dir, quiet) 268 269 270def chmod_bin(directory): 271 """ 272 Makes the directory and its children executable recursively. 273 """ 274 rwx = (stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR 275 | stat.S_IXGRP | stat.S_IXOTH) 276 for path, _, files in os.walk(directory): 277 for f in files: 278 os.chmod(os.path.join(path, f), rwx) 279 280 281def get_conf_files(config_dir_paths, root_dir, file_names=requirements): 282 """ 283 Return a list of collected path-prefixed file paths matching names in a 284 file_names tuple, based on config_dir_paths, root_dir and the types of 285 file_names requested. Returned paths are posix paths. 286 287 @config_dir_paths: Each config_dir_path is a relative from the project 288 root to a config dir. This script should always be called from the project 289 root dir. 290 291 @root_dir: The project absolute root dir. 292 293 @file_names: get requirements, python or shell files based on list of 294 supported file names provided as a tuple of supported file_names. 295 296 Scripts or requirements are optional and only used if presents. Unknown 297 scripts or requirements file_names are ignored (but they could be used 298 indirectly by known requirements with -r requirements inclusion, or 299 scripts with python imports.) 300 301 Since Python scripts are executed after requirements are installed they 302 can import from any requirement-installed component such as Fabric. 303 """ 304 # collect files for each requested dir path 305 collected = [] 306 for config_dir_path in config_dir_paths: 307 abs_config_dir_path = os.path.join(root_dir, config_dir_path) 308 if not os.path.exists(abs_config_dir_path): 309 print('Configuration directory %(config_dir_path)s ' 310 'does not exists. Skipping.' % locals()) 311 continue 312 # Support args like enterprise or enterprise/dev 313 paths = config_dir_path.strip('/').replace('\\', '/').split('/') 314 # a tuple of (relative path, location,) 315 current = None 316 for path in paths: 317 if not current: 318 current = (path, os.path.join(root_dir, path),) 319 else: 320 base_path, base_loc = current 321 current = (os.path.join(base_path, path), 322 os.path.join(base_loc, path),) 323 path, loc = current 324 # we iterate on known filenames to ensure the defined precedence 325 # is respected (posix over mac, linux), etc 326 for n in file_names: 327 for f in os.listdir(loc): 328 if f == n: 329 f_loc = os.path.join(loc, f) 330 if f_loc not in collected: 331 collected.append(f_loc) 332 333 return collected 334 335 336if __name__ == '__main__': 337 # define/setup common directories 338 etc_dir = os.path.abspath(os.path.dirname(__file__)) 339 root_dir = os.path.dirname(etc_dir) 340 341 args = sys.argv[1:] 342 if args[0] == '--clean': 343 clean(root_dir) 344 sys.exit(0) 345 346 sys.path.insert(0, root_dir) 347 bin_dir = os.path.join(root_dir, 'bin') 348 standard_python = sys.executable 349 350 # you must create a CONFIGURE_QUIET env var if you want to run quietly 351 run_quiet = 'CONFIGURE_QUIET' in os.environ 352 353 if on_win: 354 configured_python = os.path.join(bin_dir, 'python.exe') 355 scripts_dir = os.path.join(root_dir, 'Scripts') 356 bin_dir = os.path.join(root_dir, 'bin') 357 if not os.path.exists(scripts_dir): 358 os.makedirs(scripts_dir) 359 if not os.path.exists(bin_dir): 360 cmd = [ 361 'mklink', '/J', 362 '"%(bin_dir)s"' % locals(), 363 '"%(scripts_dir)s"' % locals()] 364 call(cmd, root_dir, run_quiet) 365 else: 366 configured_python = os.path.join(bin_dir, 'python') 367 scripts_dir = bin_dir 368 369 # Get requested configuration paths to collect components and scripts later 370 configs = [] 371 for path in args[:]: 372 if not os.path.isabs(path): 373 abs_path = os.path.join(root_dir, path) 374 if os.path.exists(abs_path): 375 configs.append(path) 376 else: 377 print() 378 print('WARNING: Skipping missing Configuration directory:\n' 379 ' %(path)s does not exist.' % locals()) 380 print() 381 382 # Collect vendor directories from environment variables: one or more third- 383 # party directories may exist as environment variables prefixed with TPP_DIR 384 thirdparty_dirs = [] 385 for envvar, path in os.environ.items(): 386 if not envvar.startswith('TPP_DIR'): 387 continue 388 if not os.path.isabs(path): 389 abs_path = os.path.join(root_dir, path) 390 if os.path.exists(abs_path): 391 thirdparty_dirs.append(path) 392 else: 393 print() 394 print('WARNING: Skipping missing Python thirdparty directory:\n' 395 ' %(path)s does not exist.\n' 396 ' Provided by environment variable:\n' 397 ' set %(envvar)s=%(path)s' % locals()) 398 print() 399 400 # Finally execute our three steps: venv, install and scripts 401 if not os.path.exists(configured_python): 402 create_virtualenv(standard_python, root_dir, thirdparty_dirs, quiet=run_quiet) 403 activate(root_dir) 404 405 install_3pp(configs, root_dir, thirdparty_dirs, quiet=run_quiet) 406 run_scripts(configs, root_dir, configured_python, quiet=run_quiet) 407 chmod_bin(bin_dir) 408 if not run_quiet: 409 print('* Configuration completed.') 410 print() 411