1# Copyright 2017 The Meson development team 2 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6 7# http://www.apache.org/licenses/LICENSE-2.0 8 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15 16import gzip 17import os 18import sys 19import shutil 20import subprocess 21import tarfile 22import tempfile 23import hashlib 24import json 25from glob import glob 26from pathlib import Path 27from mesonbuild.environment import detect_ninja 28from mesonbuild.mesonlib import (MesonException, RealPathAction, quiet_git, 29 windows_proof_rmtree, setup_vsenv) 30from mesonbuild.wrap import wrap 31from mesonbuild import mlog, build 32from .scripts.meson_exe import run_exe 33 34archive_choices = ['gztar', 'xztar', 'zip'] 35 36archive_extension = {'gztar': '.tar.gz', 37 'xztar': '.tar.xz', 38 'zip': '.zip'} 39 40def add_arguments(parser): 41 parser.add_argument('-C', dest='wd', action=RealPathAction, 42 help='directory to cd into before running') 43 parser.add_argument('--formats', default='xztar', 44 help='Comma separated list of archive types to create. Supports xztar (default), gztar, and zip.') 45 parser.add_argument('--include-subprojects', action='store_true', 46 help='Include source code of subprojects that have been used for the build.') 47 parser.add_argument('--no-tests', action='store_true', 48 help='Do not build and test generated packages.') 49 50 51def create_hash(fname): 52 hashname = fname + '.sha256sum' 53 m = hashlib.sha256() 54 m.update(open(fname, 'rb').read()) 55 with open(hashname, 'w', encoding='utf-8') as f: 56 # A space and an asterisk because that is the format defined by GNU coreutils 57 # and accepted by busybox and the Perl shasum tool. 58 f.write('{} *{}\n'.format(m.hexdigest(), os.path.basename(fname))) 59 60 61def copy_git(src, distdir, revision='HEAD', prefix=None, subdir=None): 62 cmd = ['git', 'archive', '--format', 'tar', revision] 63 if prefix is not None: 64 cmd.insert(2, f'--prefix={prefix}/') 65 if subdir is not None: 66 cmd.extend(['--', subdir]) 67 with tempfile.TemporaryFile() as f: 68 subprocess.check_call(cmd, cwd=src, stdout=f) 69 f.seek(0) 70 t = tarfile.open(fileobj=f) # [ignore encoding] 71 t.extractall(path=distdir) 72 73def process_submodules(src, distdir): 74 module_file = os.path.join(src, '.gitmodules') 75 if not os.path.exists(module_file): 76 return 77 cmd = ['git', 'submodule', 'status', '--cached', '--recursive'] 78 modlist = subprocess.check_output(cmd, cwd=src, universal_newlines=True).splitlines() 79 for submodule in modlist: 80 status = submodule[:1] 81 sha1, rest = submodule[1:].split(' ', 1) 82 subpath = rest.rsplit(' ', 1)[0] 83 84 if status == '-': 85 mlog.warning(f'Submodule {subpath!r} is not checked out and cannot be added to the dist') 86 continue 87 elif status in {'+', 'U'}: 88 mlog.warning(f'Submodule {subpath!r} has uncommitted changes that will not be included in the dist tarball') 89 90 copy_git(os.path.join(src, subpath), distdir, revision=sha1, prefix=subpath) 91 92 93def run_dist_scripts(src_root, bld_root, dist_root, dist_scripts, subprojects): 94 assert os.path.isabs(dist_root) 95 env = {} 96 env['MESON_DIST_ROOT'] = dist_root 97 env['MESON_SOURCE_ROOT'] = src_root 98 env['MESON_BUILD_ROOT'] = bld_root 99 for d in dist_scripts: 100 if d.subproject and d.subproject not in subprojects: 101 continue 102 subdir = subprojects.get(d.subproject, '') 103 env['MESON_PROJECT_DIST_ROOT'] = os.path.join(dist_root, subdir) 104 env['MESON_PROJECT_SOURCE_ROOT'] = os.path.join(src_root, subdir) 105 env['MESON_PROJECT_BUILD_ROOT'] = os.path.join(bld_root, subdir) 106 name = ' '.join(d.cmd_args) 107 print(f'Running custom dist script {name!r}') 108 try: 109 rc = run_exe(d, env) 110 if rc != 0: 111 sys.exit('Dist script errored out') 112 except OSError: 113 print(f'Failed to run dist script {name!r}') 114 sys.exit(1) 115 116def git_root(src_root): 117 # Cannot use --show-toplevel here because git in our CI prints cygwin paths 118 # that python cannot resolve. Workaround this by taking parent of src_root. 119 prefix = quiet_git(['rev-parse', '--show-prefix'], src_root, check=True)[1].strip() 120 if not prefix: 121 return Path(src_root) 122 prefix_level = len(Path(prefix).parents) 123 return Path(src_root).parents[prefix_level - 1] 124 125def is_git(src_root): 126 ''' 127 Checks if meson.build file at the root source directory is tracked by git. 128 It could be a subproject part of the parent project git repository. 129 ''' 130 return quiet_git(['ls-files', '--error-unmatch', 'meson.build'], src_root)[0] 131 132def git_have_dirty_index(src_root): 133 '''Check whether there are uncommitted changes in git''' 134 ret = subprocess.call(['git', '-C', src_root, 'diff-index', '--quiet', 'HEAD']) 135 return ret == 1 136 137def process_git_project(src_root, distdir): 138 if git_have_dirty_index(src_root): 139 mlog.warning('Repository has uncommitted changes that will not be included in the dist tarball') 140 if os.path.exists(distdir): 141 windows_proof_rmtree(distdir) 142 repo_root = git_root(src_root) 143 if repo_root.samefile(src_root): 144 os.makedirs(distdir) 145 copy_git(src_root, distdir) 146 else: 147 subdir = Path(src_root).relative_to(repo_root) 148 tmp_distdir = distdir + '-tmp' 149 if os.path.exists(tmp_distdir): 150 windows_proof_rmtree(tmp_distdir) 151 os.makedirs(tmp_distdir) 152 copy_git(repo_root, tmp_distdir, subdir=str(subdir)) 153 Path(tmp_distdir, subdir).rename(distdir) 154 windows_proof_rmtree(tmp_distdir) 155 process_submodules(src_root, distdir) 156 157def create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, dist_scripts, subprojects): 158 distdir = os.path.join(dist_sub, dist_name) 159 process_git_project(src_root, distdir) 160 for path in subprojects.values(): 161 sub_src_root = os.path.join(src_root, path) 162 sub_distdir = os.path.join(distdir, path) 163 if os.path.exists(sub_distdir): 164 continue 165 if is_git(sub_src_root): 166 process_git_project(sub_src_root, sub_distdir) 167 else: 168 shutil.copytree(sub_src_root, sub_distdir) 169 run_dist_scripts(src_root, bld_root, distdir, dist_scripts, subprojects) 170 output_names = [] 171 for a in archives: 172 compressed_name = distdir + archive_extension[a] 173 shutil.make_archive(distdir, a, root_dir=dist_sub, base_dir=dist_name) 174 output_names.append(compressed_name) 175 windows_proof_rmtree(distdir) 176 return output_names 177 178def is_hg(src_root): 179 return os.path.isdir(os.path.join(src_root, '.hg')) 180 181def hg_have_dirty_index(src_root): 182 '''Check whether there are uncommitted changes in hg''' 183 out = subprocess.check_output(['hg', '-R', src_root, 'summary']) 184 return b'commit: (clean)' not in out 185 186def create_dist_hg(dist_name, archives, src_root, bld_root, dist_sub, dist_scripts): 187 if hg_have_dirty_index(src_root): 188 mlog.warning('Repository has uncommitted changes that will not be included in the dist tarball') 189 if dist_scripts: 190 mlog.warning('dist scripts are not supported in Mercurial projects') 191 192 os.makedirs(dist_sub, exist_ok=True) 193 tarname = os.path.join(dist_sub, dist_name + '.tar') 194 xzname = tarname + '.xz' 195 gzname = tarname + '.gz' 196 zipname = os.path.join(dist_sub, dist_name + '.zip') 197 # Note that -X interprets relative paths using the current working 198 # directory, not the repository root, so this must be an absolute path: 199 # https://bz.mercurial-scm.org/show_bug.cgi?id=6267 200 # 201 # .hg[a-z]* is used instead of .hg* to keep .hg_archival.txt, which may 202 # be useful to link the tarball to the Mercurial revision for either 203 # manual inspection or in case any code interprets it for a --version or 204 # similar. 205 subprocess.check_call(['hg', 'archive', '-R', src_root, '-S', '-t', 'tar', 206 '-X', src_root + '/.hg[a-z]*', tarname]) 207 output_names = [] 208 if 'xztar' in archives: 209 import lzma 210 with lzma.open(xzname, 'wb') as xf, open(tarname, 'rb') as tf: 211 shutil.copyfileobj(tf, xf) 212 output_names.append(xzname) 213 if 'gztar' in archives: 214 with gzip.open(gzname, 'wb') as zf, open(tarname, 'rb') as tf: 215 shutil.copyfileobj(tf, zf) 216 output_names.append(gzname) 217 os.unlink(tarname) 218 if 'zip' in archives: 219 subprocess.check_call(['hg', 'archive', '-R', src_root, '-S', '-t', 'zip', zipname]) 220 output_names.append(zipname) 221 return output_names 222 223def run_dist_steps(meson_command, unpacked_src_dir, builddir, installdir, ninja_args): 224 if subprocess.call(meson_command + ['--backend=ninja', unpacked_src_dir, builddir]) != 0: 225 print('Running Meson on distribution package failed') 226 return 1 227 if subprocess.call(ninja_args, cwd=builddir) != 0: 228 print('Compiling the distribution package failed') 229 return 1 230 if subprocess.call(ninja_args + ['test'], cwd=builddir) != 0: 231 print('Running unit tests on the distribution package failed') 232 return 1 233 myenv = os.environ.copy() 234 myenv['DESTDIR'] = installdir 235 if subprocess.call(ninja_args + ['install'], cwd=builddir, env=myenv) != 0: 236 print('Installing the distribution package failed') 237 return 1 238 return 0 239 240def check_dist(packagename, meson_command, extra_meson_args, bld_root, privdir): 241 print(f'Testing distribution package {packagename}') 242 unpackdir = os.path.join(privdir, 'dist-unpack') 243 builddir = os.path.join(privdir, 'dist-build') 244 installdir = os.path.join(privdir, 'dist-install') 245 for p in (unpackdir, builddir, installdir): 246 if os.path.exists(p): 247 windows_proof_rmtree(p) 248 os.mkdir(p) 249 ninja_args = detect_ninja() 250 shutil.unpack_archive(packagename, unpackdir) 251 unpacked_files = glob(os.path.join(unpackdir, '*')) 252 assert len(unpacked_files) == 1 253 unpacked_src_dir = unpacked_files[0] 254 with open(os.path.join(bld_root, 'meson-info', 'intro-buildoptions.json'), encoding='utf-8') as boptions: 255 meson_command += ['-D{name}={value}'.format(**o) for o in json.load(boptions) 256 if o['name'] not in ['backend', 'install_umask', 'buildtype']] 257 meson_command += extra_meson_args 258 259 ret = run_dist_steps(meson_command, unpacked_src_dir, builddir, installdir, ninja_args) 260 if ret > 0: 261 print(f'Dist check build directory was {builddir}') 262 else: 263 windows_proof_rmtree(unpackdir) 264 windows_proof_rmtree(builddir) 265 windows_proof_rmtree(installdir) 266 print(f'Distribution package {packagename} tested') 267 return ret 268 269def determine_archives_to_generate(options): 270 result = [] 271 for i in options.formats.split(','): 272 if i not in archive_choices: 273 sys.exit(f'Value "{i}" not one of permitted values {archive_choices}.') 274 result.append(i) 275 if len(i) == 0: 276 sys.exit('No archive types specified.') 277 return result 278 279def run(options): 280 buildfile = Path(options.wd) / 'meson-private' / 'build.dat' 281 if not buildfile.is_file(): 282 raise MesonException(f'Directory {options.wd!r} does not seem to be a Meson build directory.') 283 b = build.load(options.wd) 284 setup_vsenv(b.need_vsenv) 285 # This import must be load delayed, otherwise it will get the default 286 # value of None. 287 from mesonbuild.mesonlib import get_meson_command 288 src_root = b.environment.source_dir 289 bld_root = b.environment.build_dir 290 priv_dir = os.path.join(bld_root, 'meson-private') 291 dist_sub = os.path.join(bld_root, 'meson-dist') 292 293 dist_name = b.project_name + '-' + b.project_version 294 295 archives = determine_archives_to_generate(options) 296 297 subprojects = {} 298 extra_meson_args = [] 299 if options.include_subprojects: 300 subproject_dir = os.path.join(src_root, b.subproject_dir) 301 for sub in b.subprojects: 302 directory = wrap.get_directory(subproject_dir, sub) 303 subprojects[sub] = os.path.join(b.subproject_dir, directory) 304 extra_meson_args.append('-Dwrap_mode=nodownload') 305 306 if is_git(src_root): 307 names = create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, b.dist_scripts, subprojects) 308 elif is_hg(src_root): 309 if subprojects: 310 print('--include-subprojects option currently not supported with Mercurial') 311 return 1 312 names = create_dist_hg(dist_name, archives, src_root, bld_root, dist_sub, b.dist_scripts) 313 else: 314 print('Dist currently only works with Git or Mercurial repos') 315 return 1 316 if names is None: 317 return 1 318 rc = 0 319 if not options.no_tests: 320 # Check only one. 321 rc = check_dist(names[0], get_meson_command(), extra_meson_args, bld_root, priv_dir) 322 if rc == 0: 323 for name in names: 324 create_hash(name) 325 print('Created', name) 326 return rc 327