1# Copyright 2017 The Meson development team
2
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6
7#     http://www.apache.org/licenses/LICENSE-2.0
8
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15
16import gzip
17import os
18import sys
19import shutil
20import subprocess
21import tarfile
22import tempfile
23import hashlib
24import json
25from glob import glob
26from pathlib import Path
27from mesonbuild.environment import detect_ninja
28from mesonbuild.mesonlib import (MesonException, RealPathAction, quiet_git,
29                                 windows_proof_rmtree, setup_vsenv)
30from mesonbuild.wrap import wrap
31from mesonbuild import mlog, build
32from .scripts.meson_exe import run_exe
33
34archive_choices = ['gztar', 'xztar', 'zip']
35
36archive_extension = {'gztar': '.tar.gz',
37                     'xztar': '.tar.xz',
38                     'zip': '.zip'}
39
40def add_arguments(parser):
41    parser.add_argument('-C', dest='wd', action=RealPathAction,
42                        help='directory to cd into before running')
43    parser.add_argument('--formats', default='xztar',
44                        help='Comma separated list of archive types to create. Supports xztar (default), gztar, and zip.')
45    parser.add_argument('--include-subprojects', action='store_true',
46                        help='Include source code of subprojects that have been used for the build.')
47    parser.add_argument('--no-tests', action='store_true',
48                        help='Do not build and test generated packages.')
49
50
51def create_hash(fname):
52    hashname = fname + '.sha256sum'
53    m = hashlib.sha256()
54    m.update(open(fname, 'rb').read())
55    with open(hashname, 'w', encoding='utf-8') as f:
56        # A space and an asterisk because that is the format defined by GNU coreutils
57        # and accepted by busybox and the Perl shasum tool.
58        f.write('{} *{}\n'.format(m.hexdigest(), os.path.basename(fname)))
59
60
61def copy_git(src, distdir, revision='HEAD', prefix=None, subdir=None):
62    cmd = ['git', 'archive', '--format', 'tar', revision]
63    if prefix is not None:
64        cmd.insert(2, f'--prefix={prefix}/')
65    if subdir is not None:
66        cmd.extend(['--', subdir])
67    with tempfile.TemporaryFile() as f:
68        subprocess.check_call(cmd, cwd=src, stdout=f)
69        f.seek(0)
70        t = tarfile.open(fileobj=f) # [ignore encoding]
71        t.extractall(path=distdir)
72
73def process_submodules(src, distdir):
74    module_file = os.path.join(src, '.gitmodules')
75    if not os.path.exists(module_file):
76        return
77    cmd = ['git', 'submodule', 'status', '--cached', '--recursive']
78    modlist = subprocess.check_output(cmd, cwd=src, universal_newlines=True).splitlines()
79    for submodule in modlist:
80        status = submodule[:1]
81        sha1, rest = submodule[1:].split(' ', 1)
82        subpath = rest.rsplit(' ', 1)[0]
83
84        if status == '-':
85            mlog.warning(f'Submodule {subpath!r} is not checked out and cannot be added to the dist')
86            continue
87        elif status in {'+', 'U'}:
88            mlog.warning(f'Submodule {subpath!r} has uncommitted changes that will not be included in the dist tarball')
89
90        copy_git(os.path.join(src, subpath), distdir, revision=sha1, prefix=subpath)
91
92
93def run_dist_scripts(src_root, bld_root, dist_root, dist_scripts, subprojects):
94    assert os.path.isabs(dist_root)
95    env = {}
96    env['MESON_DIST_ROOT'] = dist_root
97    env['MESON_SOURCE_ROOT'] = src_root
98    env['MESON_BUILD_ROOT'] = bld_root
99    for d in dist_scripts:
100        if d.subproject and d.subproject not in subprojects:
101            continue
102        subdir = subprojects.get(d.subproject, '')
103        env['MESON_PROJECT_DIST_ROOT'] = os.path.join(dist_root, subdir)
104        env['MESON_PROJECT_SOURCE_ROOT'] = os.path.join(src_root, subdir)
105        env['MESON_PROJECT_BUILD_ROOT'] = os.path.join(bld_root, subdir)
106        name = ' '.join(d.cmd_args)
107        print(f'Running custom dist script {name!r}')
108        try:
109            rc = run_exe(d, env)
110            if rc != 0:
111                sys.exit('Dist script errored out')
112        except OSError:
113            print(f'Failed to run dist script {name!r}')
114            sys.exit(1)
115
116def git_root(src_root):
117    # Cannot use --show-toplevel here because git in our CI prints cygwin paths
118    # that python cannot resolve. Workaround this by taking parent of src_root.
119    prefix = quiet_git(['rev-parse', '--show-prefix'], src_root, check=True)[1].strip()
120    if not prefix:
121        return Path(src_root)
122    prefix_level = len(Path(prefix).parents)
123    return Path(src_root).parents[prefix_level - 1]
124
125def is_git(src_root):
126    '''
127    Checks if meson.build file at the root source directory is tracked by git.
128    It could be a subproject part of the parent project git repository.
129    '''
130    return quiet_git(['ls-files', '--error-unmatch', 'meson.build'], src_root)[0]
131
132def git_have_dirty_index(src_root):
133    '''Check whether there are uncommitted changes in git'''
134    ret = subprocess.call(['git', '-C', src_root, 'diff-index', '--quiet', 'HEAD'])
135    return ret == 1
136
137def process_git_project(src_root, distdir):
138    if git_have_dirty_index(src_root):
139        mlog.warning('Repository has uncommitted changes that will not be included in the dist tarball')
140    if os.path.exists(distdir):
141        windows_proof_rmtree(distdir)
142    repo_root = git_root(src_root)
143    if repo_root.samefile(src_root):
144        os.makedirs(distdir)
145        copy_git(src_root, distdir)
146    else:
147        subdir = Path(src_root).relative_to(repo_root)
148        tmp_distdir = distdir + '-tmp'
149        if os.path.exists(tmp_distdir):
150            windows_proof_rmtree(tmp_distdir)
151        os.makedirs(tmp_distdir)
152        copy_git(repo_root, tmp_distdir, subdir=str(subdir))
153        Path(tmp_distdir, subdir).rename(distdir)
154        windows_proof_rmtree(tmp_distdir)
155    process_submodules(src_root, distdir)
156
157def create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, dist_scripts, subprojects):
158    distdir = os.path.join(dist_sub, dist_name)
159    process_git_project(src_root, distdir)
160    for path in subprojects.values():
161        sub_src_root = os.path.join(src_root, path)
162        sub_distdir = os.path.join(distdir, path)
163        if os.path.exists(sub_distdir):
164            continue
165        if is_git(sub_src_root):
166            process_git_project(sub_src_root, sub_distdir)
167        else:
168            shutil.copytree(sub_src_root, sub_distdir)
169    run_dist_scripts(src_root, bld_root, distdir, dist_scripts, subprojects)
170    output_names = []
171    for a in archives:
172        compressed_name = distdir + archive_extension[a]
173        shutil.make_archive(distdir, a, root_dir=dist_sub, base_dir=dist_name)
174        output_names.append(compressed_name)
175    windows_proof_rmtree(distdir)
176    return output_names
177
178def is_hg(src_root):
179    return os.path.isdir(os.path.join(src_root, '.hg'))
180
181def hg_have_dirty_index(src_root):
182    '''Check whether there are uncommitted changes in hg'''
183    out = subprocess.check_output(['hg', '-R', src_root, 'summary'])
184    return b'commit: (clean)' not in out
185
186def create_dist_hg(dist_name, archives, src_root, bld_root, dist_sub, dist_scripts):
187    if hg_have_dirty_index(src_root):
188        mlog.warning('Repository has uncommitted changes that will not be included in the dist tarball')
189    if dist_scripts:
190        mlog.warning('dist scripts are not supported in Mercurial projects')
191
192    os.makedirs(dist_sub, exist_ok=True)
193    tarname = os.path.join(dist_sub, dist_name + '.tar')
194    xzname = tarname + '.xz'
195    gzname = tarname + '.gz'
196    zipname = os.path.join(dist_sub, dist_name + '.zip')
197    # Note that -X interprets relative paths using the current working
198    # directory, not the repository root, so this must be an absolute path:
199    # https://bz.mercurial-scm.org/show_bug.cgi?id=6267
200    #
201    # .hg[a-z]* is used instead of .hg* to keep .hg_archival.txt, which may
202    # be useful to link the tarball to the Mercurial revision for either
203    # manual inspection or in case any code interprets it for a --version or
204    # similar.
205    subprocess.check_call(['hg', 'archive', '-R', src_root, '-S', '-t', 'tar',
206                           '-X', src_root + '/.hg[a-z]*', tarname])
207    output_names = []
208    if 'xztar' in archives:
209        import lzma
210        with lzma.open(xzname, 'wb') as xf, open(tarname, 'rb') as tf:
211            shutil.copyfileobj(tf, xf)
212        output_names.append(xzname)
213    if 'gztar' in archives:
214        with gzip.open(gzname, 'wb') as zf, open(tarname, 'rb') as tf:
215            shutil.copyfileobj(tf, zf)
216        output_names.append(gzname)
217    os.unlink(tarname)
218    if 'zip' in archives:
219        subprocess.check_call(['hg', 'archive', '-R', src_root, '-S', '-t', 'zip', zipname])
220        output_names.append(zipname)
221    return output_names
222
223def run_dist_steps(meson_command, unpacked_src_dir, builddir, installdir, ninja_args):
224    if subprocess.call(meson_command + ['--backend=ninja', unpacked_src_dir, builddir]) != 0:
225        print('Running Meson on distribution package failed')
226        return 1
227    if subprocess.call(ninja_args, cwd=builddir) != 0:
228        print('Compiling the distribution package failed')
229        return 1
230    if subprocess.call(ninja_args + ['test'], cwd=builddir) != 0:
231        print('Running unit tests on the distribution package failed')
232        return 1
233    myenv = os.environ.copy()
234    myenv['DESTDIR'] = installdir
235    if subprocess.call(ninja_args + ['install'], cwd=builddir, env=myenv) != 0:
236        print('Installing the distribution package failed')
237        return 1
238    return 0
239
240def check_dist(packagename, meson_command, extra_meson_args, bld_root, privdir):
241    print(f'Testing distribution package {packagename}')
242    unpackdir = os.path.join(privdir, 'dist-unpack')
243    builddir = os.path.join(privdir, 'dist-build')
244    installdir = os.path.join(privdir, 'dist-install')
245    for p in (unpackdir, builddir, installdir):
246        if os.path.exists(p):
247            windows_proof_rmtree(p)
248        os.mkdir(p)
249    ninja_args = detect_ninja()
250    shutil.unpack_archive(packagename, unpackdir)
251    unpacked_files = glob(os.path.join(unpackdir, '*'))
252    assert len(unpacked_files) == 1
253    unpacked_src_dir = unpacked_files[0]
254    with open(os.path.join(bld_root, 'meson-info', 'intro-buildoptions.json'), encoding='utf-8') as boptions:
255        meson_command += ['-D{name}={value}'.format(**o) for o in json.load(boptions)
256                          if o['name'] not in ['backend', 'install_umask', 'buildtype']]
257    meson_command += extra_meson_args
258
259    ret = run_dist_steps(meson_command, unpacked_src_dir, builddir, installdir, ninja_args)
260    if ret > 0:
261        print(f'Dist check build directory was {builddir}')
262    else:
263        windows_proof_rmtree(unpackdir)
264        windows_proof_rmtree(builddir)
265        windows_proof_rmtree(installdir)
266        print(f'Distribution package {packagename} tested')
267    return ret
268
269def determine_archives_to_generate(options):
270    result = []
271    for i in options.formats.split(','):
272        if i not in archive_choices:
273            sys.exit(f'Value "{i}" not one of permitted values {archive_choices}.')
274        result.append(i)
275    if len(i) == 0:
276        sys.exit('No archive types specified.')
277    return result
278
279def run(options):
280    buildfile = Path(options.wd) / 'meson-private' / 'build.dat'
281    if not buildfile.is_file():
282        raise MesonException(f'Directory {options.wd!r} does not seem to be a Meson build directory.')
283    b = build.load(options.wd)
284    setup_vsenv(b.need_vsenv)
285    # This import must be load delayed, otherwise it will get the default
286    # value of None.
287    from mesonbuild.mesonlib import get_meson_command
288    src_root = b.environment.source_dir
289    bld_root = b.environment.build_dir
290    priv_dir = os.path.join(bld_root, 'meson-private')
291    dist_sub = os.path.join(bld_root, 'meson-dist')
292
293    dist_name = b.project_name + '-' + b.project_version
294
295    archives = determine_archives_to_generate(options)
296
297    subprojects = {}
298    extra_meson_args = []
299    if options.include_subprojects:
300        subproject_dir = os.path.join(src_root, b.subproject_dir)
301        for sub in b.subprojects:
302            directory = wrap.get_directory(subproject_dir, sub)
303            subprojects[sub] = os.path.join(b.subproject_dir, directory)
304        extra_meson_args.append('-Dwrap_mode=nodownload')
305
306    if is_git(src_root):
307        names = create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, b.dist_scripts, subprojects)
308    elif is_hg(src_root):
309        if subprojects:
310            print('--include-subprojects option currently not supported with Mercurial')
311            return 1
312        names = create_dist_hg(dist_name, archives, src_root, bld_root, dist_sub, b.dist_scripts)
313    else:
314        print('Dist currently only works with Git or Mercurial repos')
315        return 1
316    if names is None:
317        return 1
318    rc = 0
319    if not options.no_tests:
320        # Check only one.
321        rc = check_dist(names[0], get_meson_command(), extra_meson_args, bld_root, priv_dir)
322    if rc == 0:
323        for name in names:
324            create_hash(name)
325            print('Created', name)
326    return rc
327