1#!/usr/bin/python3
2# This Source Code Form is subject to the terms of the Mozilla Public
3# License, v. 2.0. If a copy of the MPL was not distributed with this
4# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6# Only necessary for flake8 to be happy...
7from __future__ import print_function
8
9import os
10import os.path
11import shutil
12import subprocess
13import platform
14import json
15import argparse
16import fnmatch
17import glob
18import errno
19import re
20import sys
21import tarfile
22from contextlib import contextmanager
23from distutils.dir_util import copy_tree
24
25from shutil import which
26
27import zstandard
28
29
30def symlink(source, link_name):
31    os_symlink = getattr(os, "symlink", None)
32    if callable(os_symlink):
33        os_symlink(source, link_name)
34    else:
35        if os.path.isdir(source):
36            # Fall back to copying the directory :(
37            copy_tree(source, link_name)
38
39
40def check_run(args):
41    print(" ".join(args), file=sys.stderr, flush=True)
42    if args[0] == "cmake":
43        # CMake `message(STATUS)` messages, as appearing in failed source code
44        # compiles, appear on stdout, so we only capture that.
45        p = subprocess.Popen(args, stdout=subprocess.PIPE)
46        lines = []
47        for line in p.stdout:
48            lines.append(line)
49            sys.stdout.write(line.decode())
50            sys.stdout.flush()
51        r = p.wait()
52        if r != 0 and os.environ.get("UPLOAD_DIR"):
53            cmake_output_re = re.compile(b'See also "(.*/CMakeOutput.log)"')
54            cmake_error_re = re.compile(b'See also "(.*/CMakeError.log)"')
55
56            def find_first_match(re):
57                for l in lines:
58                    match = re.search(l)
59                    if match:
60                        return match
61
62            output_match = find_first_match(cmake_output_re)
63            error_match = find_first_match(cmake_error_re)
64
65            upload_dir = os.environ["UPLOAD_DIR"].encode("utf-8")
66            if output_match or error_match:
67                mkdir_p(upload_dir)
68            if output_match:
69                shutil.copy2(output_match.group(1), upload_dir)
70            if error_match:
71                shutil.copy2(error_match.group(1), upload_dir)
72    else:
73        r = subprocess.call(args)
74    assert r == 0
75
76
77def run_in(path, args):
78    with chdir(path):
79        check_run(args)
80
81
82@contextmanager
83def chdir(path):
84    d = os.getcwd()
85    print('cd "%s"' % path, file=sys.stderr)
86    os.chdir(path)
87    try:
88        yield
89    finally:
90        print('cd "%s"' % d, file=sys.stderr)
91        os.chdir(d)
92
93
94def patch(patch, srcdir):
95    patch = os.path.realpath(patch)
96    check_run(["patch", "-d", srcdir, "-p1", "-i", patch, "--fuzz=0", "-s"])
97
98
99def import_clang_tidy(source_dir, build_clang_tidy_alpha, build_clang_tidy_external):
100    clang_plugin_path = os.path.join(os.path.dirname(sys.argv[0]), "..", "clang-plugin")
101    clang_tidy_path = os.path.join(source_dir, "clang-tools-extra/clang-tidy")
102    sys.path.append(clang_plugin_path)
103    from import_mozilla_checks import do_import
104
105    import_options = {
106        "alpha": build_clang_tidy_alpha,
107        "external": build_clang_tidy_external,
108    }
109    do_import(clang_plugin_path, clang_tidy_path, import_options)
110
111
112def build_package(package_build_dir, cmake_args):
113    if not os.path.exists(package_build_dir):
114        os.mkdir(package_build_dir)
115    # If CMake has already been run, it may have been run with different
116    # arguments, so we need to re-run it.  Make sure the cached copy of the
117    # previous CMake run is cleared before running it again.
118    if os.path.exists(package_build_dir + "/CMakeCache.txt"):
119        os.remove(package_build_dir + "/CMakeCache.txt")
120    if os.path.exists(package_build_dir + "/CMakeFiles"):
121        shutil.rmtree(package_build_dir + "/CMakeFiles")
122
123    run_in(package_build_dir, ["cmake"] + cmake_args)
124    run_in(package_build_dir, ["ninja", "install", "-v"])
125
126
127@contextmanager
128def updated_env(env):
129    old_env = os.environ.copy()
130    os.environ.update(env)
131    yield
132    os.environ.clear()
133    os.environ.update(old_env)
134
135
136def build_tar_package(name, base, directory):
137    name = os.path.realpath(name)
138    print("tarring {} from {}/{}".format(name, base, directory), file=sys.stderr)
139    assert name.endswith(".tar.zst")
140
141    cctx = zstandard.ZstdCompressor()
142    with open(name, "wb") as f, cctx.stream_writer(f) as z:
143        with tarfile.open(mode="w|", fileobj=z) as tf:
144            with chdir(base):
145                tf.add(directory)
146
147
148def mkdir_p(path):
149    try:
150        os.makedirs(path)
151    except OSError as e:
152        if e.errno != errno.EEXIST or not os.path.isdir(path):
153            raise
154
155
156def delete(path):
157    if os.path.isdir(path):
158        shutil.rmtree(path)
159    else:
160        try:
161            os.unlink(path)
162        except Exception:
163            pass
164
165
166def install_import_library(build_dir, clang_dir):
167    shutil.copy2(
168        os.path.join(build_dir, "lib", "clang.lib"), os.path.join(clang_dir, "lib")
169    )
170
171
172def is_darwin():
173    return platform.system() == "Darwin"
174
175
176def is_linux():
177    return platform.system() == "Linux"
178
179
180def is_windows():
181    return platform.system() == "Windows"
182
183
184def build_one_stage(
185    cc,
186    cxx,
187    asm,
188    ld,
189    ar,
190    ranlib,
191    libtool,
192    src_dir,
193    stage_dir,
194    package_name,
195    build_libcxx,
196    osx_cross_compile,
197    build_type,
198    assertions,
199    libcxx_include_dir,
200    build_wasm,
201    is_final_stage=False,
202    profile=None,
203):
204    if not os.path.exists(stage_dir):
205        os.mkdir(stage_dir)
206
207    build_dir = stage_dir + "/build"
208    inst_dir = stage_dir + "/" + package_name
209
210    # cmake doesn't deal well with backslashes in paths.
211    def slashify_path(path):
212        return path.replace("\\", "/")
213
214    def cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir):
215        machine_targets = "X86;ARM;AArch64" if is_final_stage else "X86"
216        if build_wasm and is_final_stage:
217            machine_targets += ";WebAssembly"
218
219        cmake_args = [
220            "-GNinja",
221            "-DCMAKE_C_COMPILER=%s" % slashify_path(cc[0]),
222            "-DCMAKE_CXX_COMPILER=%s" % slashify_path(cxx[0]),
223            "-DCMAKE_ASM_COMPILER=%s" % slashify_path(asm[0]),
224            "-DCMAKE_LINKER=%s" % slashify_path(ld[0]),
225            "-DCMAKE_AR=%s" % slashify_path(ar),
226            "-DCMAKE_C_FLAGS=%s" % " ".join(cc[1:]),
227            "-DCMAKE_CXX_FLAGS=%s" % " ".join(cxx[1:]),
228            "-DCMAKE_ASM_FLAGS=%s" % " ".join(asm[1:]),
229            "-DCMAKE_EXE_LINKER_FLAGS=%s" % " ".join(ld[1:]),
230            "-DCMAKE_SHARED_LINKER_FLAGS=%s" % " ".join(ld[1:]),
231            "-DCMAKE_BUILD_TYPE=%s" % build_type,
232            "-DCMAKE_INSTALL_PREFIX=%s" % inst_dir,
233            "-DLLVM_TARGETS_TO_BUILD=%s" % machine_targets,
234            "-DLLVM_ENABLE_ASSERTIONS=%s" % ("ON" if assertions else "OFF"),
235            "-DLLVM_ENABLE_BINDINGS=OFF",
236            "-DLLVM_ENABLE_CURL=OFF",
237        ]
238        if "TASK_ID" in os.environ:
239            cmake_args += [
240                "-DCLANG_REPOSITORY_STRING=taskcluster-%s" % os.environ["TASK_ID"],
241            ]
242        # libc++ doesn't build with MSVC because of the use of #include_next.
243        if is_final_stage and os.path.basename(cc[0]).lower() != "cl.exe":
244            cmake_args += [
245                "-DLLVM_TOOL_LIBCXX_BUILD=%s" % ("ON" if build_libcxx else "OFF"),
246                # libc++abi has conflicting definitions between the shared and static
247                # library on Windows because of the import library for the dll having
248                # the same name as the static library. libc++abi is not necessary on
249                # Windows anyways.
250                "-DLLVM_TOOL_LIBCXXABI_BUILD=%s" % ("OFF" if is_windows() else "ON"),
251            ]
252        if not is_final_stage:
253            cmake_args += [
254                "-DLLVM_ENABLE_PROJECTS=clang",
255                "-DLLVM_INCLUDE_TESTS=OFF",
256                "-DLLVM_TOOL_LLI_BUILD=OFF",
257            ]
258
259        # There is no libxml2 on Windows except if we build one ourselves.
260        # libxml2 is only necessary for llvm-mt, but Windows can just use the
261        # native MT tool.
262        if not is_windows() and is_final_stage:
263            cmake_args += ["-DLLVM_ENABLE_LIBXML2=FORCE_ON"]
264        if is_linux() and not osx_cross_compile and is_final_stage:
265            sysroot = os.path.join(os.environ.get("MOZ_FETCHES_DIR", ""), "sysroot")
266            if os.path.exists(sysroot):
267                cmake_args += ["-DLLVM_BINUTILS_INCDIR=/usr/include"]
268                cmake_args += ["-DCMAKE_SYSROOT=%s" % sysroot]
269                # Work around the LLVM build system not building the i386 compiler-rt
270                # because it doesn't allow to use a sysroot for that during the cmake
271                # checks.
272                cmake_args += ["-DCAN_TARGET_i386=1"]
273        if is_windows():
274            cmake_args.insert(-1, "-DLLVM_EXPORT_SYMBOLS_FOR_PLUGINS=ON")
275            cmake_args.insert(-1, "-DLLVM_USE_CRT_RELEASE=MT")
276        else:
277            # libllvm as a shared library is not supported on Windows
278            cmake_args += ["-DLLVM_LINK_LLVM_DYLIB=ON"]
279        if ranlib is not None:
280            cmake_args += ["-DCMAKE_RANLIB=%s" % slashify_path(ranlib)]
281        if libtool is not None:
282            cmake_args += ["-DCMAKE_LIBTOOL=%s" % slashify_path(libtool)]
283        if osx_cross_compile:
284            arch = "arm64" if os.environ.get("OSX_ARCH") == "arm64" else "x86_64"
285            target_cpu = (
286                "aarch64" if os.environ.get("OSX_ARCH") == "arm64" else "x86_64"
287            )
288            cmake_args += [
289                "-DCMAKE_SYSTEM_NAME=Darwin",
290                "-DCMAKE_SYSTEM_VERSION=%s" % os.environ["MACOSX_DEPLOYMENT_TARGET"],
291                "-DLIBCXXABI_LIBCXX_INCLUDES=%s" % libcxx_include_dir,
292                "-DCMAKE_OSX_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
293                "-DCMAKE_FIND_ROOT_PATH=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
294                "-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER",
295                "-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY",
296                "-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY",
297                "-DCMAKE_MACOSX_RPATH=ON",
298                "-DCMAKE_OSX_ARCHITECTURES=%s" % arch,
299                "-DDARWIN_osx_ARCHS=%s" % arch,
300                "-DDARWIN_osx_SYSROOT=%s" % slashify_path(os.getenv("CROSS_SYSROOT")),
301                "-DLLVM_DEFAULT_TARGET_TRIPLE=%s-apple-darwin" % target_cpu,
302            ]
303            if os.environ.get("OSX_ARCH") == "arm64":
304                cmake_args += [
305                    "-DDARWIN_osx_BUILTIN_ARCHS=arm64",
306                ]
307            # Starting in LLVM 11 (which requires SDK 10.12) the build tries to
308            # detect the SDK version by calling xcrun. Cross-compiles don't have
309            # an xcrun, so we have to set the version explicitly.
310            cmake_args += [
311                "-DDARWIN_macosx_OVERRIDE_SDK_VERSION=%s"
312                % os.environ["MACOSX_DEPLOYMENT_TARGET"],
313            ]
314        if profile == "gen":
315            # Per https://releases.llvm.org/10.0.0/docs/HowToBuildWithPGO.html
316            cmake_args += [
317                "-DLLVM_BUILD_INSTRUMENTED=IR",
318                "-DLLVM_BUILD_RUNTIME=No",
319            ]
320        elif profile:
321            cmake_args += [
322                "-DLLVM_PROFDATA_FILE=%s" % profile,
323            ]
324        return cmake_args
325
326    cmake_args = []
327    cmake_args += cmake_base_args(cc, cxx, asm, ld, ar, ranlib, libtool, inst_dir)
328    cmake_args += [src_dir]
329    build_package(build_dir, cmake_args)
330
331    # For some reasons the import library clang.lib of clang.exe is not
332    # installed, so we copy it by ourselves.
333    if is_windows() and is_final_stage:
334        install_import_library(build_dir, inst_dir)
335
336
337# Return the absolute path of a build tool.  We first look to see if the
338# variable is defined in the config file, and if so we make sure it's an
339# absolute path to an existing tool, otherwise we look for a program in
340# $PATH named "key".
341#
342# This expects the name of the key in the config file to match the name of
343# the tool in the default toolchain on the system (for example, "ld" on Unix
344# and "link" on Windows).
345def get_tool(config, key):
346    f = None
347    if key in config:
348        f = config[key].format(**os.environ)
349        if os.path.isabs(f):
350            if not os.path.exists(f):
351                raise ValueError("%s must point to an existing path" % key)
352            return f
353
354    # Assume that we have the name of some program that should be on PATH.
355    tool = which(f) if f else which(key)
356    if not tool:
357        raise ValueError("%s not found on PATH" % (f or key))
358    return tool
359
360
361# This function is intended to be called on the final build directory when
362# building clang-tidy. Also clang-format binaries are included that can be used
363# in conjunction with clang-tidy.
364# As a separate binary we also ship clangd for the language server protocol that
365# can be used as a plugin in `vscode`.
366# Its job is to remove all of the files which won't be used for clang-tidy or
367# clang-format to reduce the download size.  Currently when this function
368# finishes its job, it will leave final_dir with a layout like this:
369#
370# clang/
371#   bin/
372#     clang-apply-replacements
373#     clang-format
374#     clang-tidy
375#     clangd
376#     run-clang-tidy
377#   include/
378#     * (nothing will be deleted here)
379#   lib/
380#     clang/
381#       4.0.0/
382#         include/
383#           * (nothing will be deleted here)
384#   share/
385#     clang/
386#       clang-format-diff.py
387#       clang-tidy-diff.py
388#       run-clang-tidy.py
389def prune_final_dir_for_clang_tidy(final_dir, osx_cross_compile):
390    # Make sure we only have what we expect.
391    dirs = [
392        "bin",
393        "include",
394        "lib",
395        "lib32",
396        "libexec",
397        "msbuild-bin",
398        "share",
399        "tools",
400    ]
401    if is_linux():
402        dirs.append("x86_64-unknown-linux-gnu")
403    for f in glob.glob("%s/*" % final_dir):
404        if os.path.basename(f) not in dirs:
405            raise Exception("Found unknown file %s in the final directory" % f)
406        if not os.path.isdir(f):
407            raise Exception("Expected %s to be a directory" % f)
408
409    kept_binaries = [
410        "clang-apply-replacements",
411        "clang-format",
412        "clang-tidy",
413        "clangd",
414        "clang-query",
415        "run-clang-tidy",
416    ]
417    re_clang_tidy = re.compile(r"^(" + "|".join(kept_binaries) + r")(\.exe)?$", re.I)
418    for f in glob.glob("%s/bin/*" % final_dir):
419        if re_clang_tidy.search(os.path.basename(f)) is None:
420            delete(f)
421
422    # Keep include/ intact.
423
424    # Remove the target-specific files.
425    if is_linux():
426        if os.path.exists(os.path.join(final_dir, "x86_64-unknown-linux-gnu")):
427            shutil.rmtree(os.path.join(final_dir, "x86_64-unknown-linux-gnu"))
428
429    # In lib/, only keep lib/clang/N.M.O/include and the LLVM shared library.
430    re_ver_num = re.compile(r"^\d+\.\d+\.\d+$", re.I)
431    for f in glob.glob("%s/lib/*" % final_dir):
432        name = os.path.basename(f)
433        if name == "clang":
434            continue
435        if osx_cross_compile and name in ["libLLVM.dylib", "libclang-cpp.dylib"]:
436            continue
437        if is_linux() and (
438            fnmatch.fnmatch(name, "libLLVM*.so")
439            or fnmatch.fnmatch(name, "libclang-cpp.so*")
440        ):
441            continue
442        delete(f)
443    for f in glob.glob("%s/lib/clang/*" % final_dir):
444        if re_ver_num.search(os.path.basename(f)) is None:
445            delete(f)
446    for f in glob.glob("%s/lib/clang/*/*" % final_dir):
447        if os.path.basename(f) != "include":
448            delete(f)
449
450    # Completely remove libexec/, msbuild-bin and tools, if it exists.
451    shutil.rmtree(os.path.join(final_dir, "libexec"))
452    for d in ("msbuild-bin", "tools"):
453        d = os.path.join(final_dir, d)
454        if os.path.exists(d):
455            shutil.rmtree(d)
456
457    # In share/, only keep share/clang/*tidy*
458    re_clang_tidy = re.compile(r"format|tidy", re.I)
459    for f in glob.glob("%s/share/*" % final_dir):
460        if os.path.basename(f) != "clang":
461            delete(f)
462    for f in glob.glob("%s/share/clang/*" % final_dir):
463        if re_clang_tidy.search(os.path.basename(f)) is None:
464            delete(f)
465
466
467def main():
468    parser = argparse.ArgumentParser()
469    parser.add_argument(
470        "-c",
471        "--config",
472        action="append",
473        required=True,
474        type=argparse.FileType("r"),
475        help="Clang configuration file",
476    )
477    parser.add_argument(
478        "--clean", required=False, action="store_true", help="Clean the build directory"
479    )
480    parser.add_argument(
481        "--skip-tar",
482        required=False,
483        action="store_true",
484        help="Skip tar packaging stage",
485    )
486    parser.add_argument(
487        "--skip-patch",
488        required=False,
489        action="store_true",
490        help="Do not patch source",
491    )
492
493    args = parser.parse_args()
494
495    if not os.path.exists("llvm/README.txt"):
496        raise Exception(
497            "The script must be run from the root directory of the llvm-project tree"
498        )
499    source_dir = os.getcwd()
500    build_dir = source_dir + "/build"
501
502    if args.clean:
503        shutil.rmtree(build_dir)
504        os.sys.exit(0)
505
506    llvm_source_dir = source_dir + "/llvm"
507    extra_source_dir = source_dir + "/clang-tools-extra"
508    clang_source_dir = source_dir + "/clang"
509    lld_source_dir = source_dir + "/lld"
510    libcxx_source_dir = source_dir + "/libcxx"
511    libcxxabi_source_dir = source_dir + "/libcxxabi"
512
513    exe_ext = ""
514    if is_windows():
515        exe_ext = ".exe"
516
517    cc_name = "clang"
518    cxx_name = "clang++"
519    if is_windows():
520        cc_name = "clang-cl"
521        cxx_name = "clang-cl"
522
523    config = {}
524    # Merge all the configs we got from the command line.
525    for c in args.config:
526        this_config_dir = os.path.dirname(c.name)
527        this_config = json.load(c)
528        patches = this_config.get("patches")
529        if patches:
530            this_config["patches"] = [os.path.join(this_config_dir, p) for p in patches]
531        for key, value in this_config.items():
532            old_value = config.get(key)
533            if old_value is None:
534                config[key] = value
535            elif value is None:
536                if key in config:
537                    del config[key]
538            elif type(old_value) != type(value):
539                raise Exception(
540                    "{} is overriding `{}` with a value of the wrong type".format(
541                        c.name, key
542                    )
543                )
544            elif isinstance(old_value, list):
545                for v in value:
546                    if v not in old_value:
547                        old_value.append(v)
548            elif isinstance(old_value, dict):
549                raise Exception("{} is setting `{}` to a dict?".format(c.name, key))
550            else:
551                config[key] = value
552
553    stages = 2
554    if "stages" in config:
555        stages = int(config["stages"])
556        if stages not in (1, 2, 3, 4):
557            raise ValueError("We only know how to build 1, 2, 3, or 4 stages.")
558    skip_stages = 0
559    if "skip_stages" in config:
560        # The assumption here is that the compiler given in `cc` and other configs
561        # is the result of the last skip stage, built somewhere else.
562        skip_stages = int(config["skip_stages"])
563        if skip_stages >= stages:
564            raise ValueError("Cannot skip more stages than are built.")
565    pgo = False
566    if "pgo" in config:
567        pgo = config["pgo"]
568        if pgo not in (True, False):
569            raise ValueError("Only boolean values are accepted for pgo.")
570    build_type = "Release"
571    if "build_type" in config:
572        build_type = config["build_type"]
573        if build_type not in ("Release", "Debug", "RelWithDebInfo", "MinSizeRel"):
574            raise ValueError(
575                "We only know how to do Release, Debug, RelWithDebInfo or "
576                "MinSizeRel builds"
577            )
578    build_libcxx = True
579    if "build_libcxx" in config:
580        build_libcxx = config["build_libcxx"]
581        if build_libcxx not in (True, False):
582            raise ValueError("Only boolean values are accepted for build_libcxx.")
583    build_wasm = False
584    if "build_wasm" in config:
585        build_wasm = config["build_wasm"]
586        if build_wasm not in (True, False):
587            raise ValueError("Only boolean values are accepted for build_wasm.")
588    build_clang_tidy = False
589    if "build_clang_tidy" in config:
590        build_clang_tidy = config["build_clang_tidy"]
591        if build_clang_tidy not in (True, False):
592            raise ValueError("Only boolean values are accepted for build_clang_tidy.")
593    build_clang_tidy_alpha = False
594    # check for build_clang_tidy_alpha only if build_clang_tidy is true
595    if build_clang_tidy and "build_clang_tidy_alpha" in config:
596        build_clang_tidy_alpha = config["build_clang_tidy_alpha"]
597        if build_clang_tidy_alpha not in (True, False):
598            raise ValueError(
599                "Only boolean values are accepted for build_clang_tidy_alpha."
600            )
601    build_clang_tidy_external = False
602    # check for build_clang_tidy_external only if build_clang_tidy is true
603    if build_clang_tidy and "build_clang_tidy_external" in config:
604        build_clang_tidy_external = config["build_clang_tidy_external"]
605        if build_clang_tidy_external not in (True, False):
606            raise ValueError(
607                "Only boolean values are accepted for build_clang_tidy_external."
608            )
609    osx_cross_compile = False
610    if "osx_cross_compile" in config:
611        osx_cross_compile = config["osx_cross_compile"]
612        if osx_cross_compile not in (True, False):
613            raise ValueError("Only boolean values are accepted for osx_cross_compile.")
614        if osx_cross_compile and not is_linux():
615            raise ValueError("osx_cross_compile can only be used on Linux.")
616    assertions = False
617    if "assertions" in config:
618        assertions = config["assertions"]
619        if assertions not in (True, False):
620            raise ValueError("Only boolean values are accepted for assertions.")
621
622    if is_darwin() or osx_cross_compile:
623        os.environ["MACOSX_DEPLOYMENT_TARGET"] = (
624            "11.0" if os.environ.get("OSX_ARCH") == "arm64" else "10.12"
625        )
626
627    cc = get_tool(config, "cc")
628    cxx = get_tool(config, "cxx")
629    asm = get_tool(config, "ml" if is_windows() else "as")
630    ld = get_tool(config, "link" if is_windows() else "ld")
631    ar = get_tool(config, "lib" if is_windows() else "ar")
632    ranlib = None if is_windows() else get_tool(config, "ranlib")
633    libtool = None
634    if "libtool" in config:
635        libtool = get_tool(config, "libtool")
636
637    if not os.path.exists(source_dir):
638        os.makedirs(source_dir)
639
640    if not args.skip_patch:
641        for p in config.get("patches", []):
642            patch(p, source_dir)
643
644    symlinks = [
645        (clang_source_dir, llvm_source_dir + "/tools/clang"),
646        (extra_source_dir, llvm_source_dir + "/tools/clang/tools/extra"),
647        (lld_source_dir, llvm_source_dir + "/tools/lld"),
648        (libcxx_source_dir, llvm_source_dir + "/projects/libcxx"),
649        (libcxxabi_source_dir, llvm_source_dir + "/projects/libcxxabi"),
650        (source_dir + "/cmake", llvm_source_dir + "/projects/cmake"),
651    ]
652    for l in symlinks:
653        # On Windows, we have to re-copy the whole directory every time.
654        if not is_windows() and os.path.islink(l[1]):
655            continue
656        delete(l[1])
657        if os.path.exists(l[0]):
658            symlink(l[0], l[1])
659
660    package_name = "clang"
661    if build_clang_tidy:
662        package_name = "clang-tidy"
663        if not args.skip_patch:
664            import_clang_tidy(
665                source_dir, build_clang_tidy_alpha, build_clang_tidy_external
666            )
667
668    if not os.path.exists(build_dir):
669        os.makedirs(build_dir)
670
671    libcxx_include_dir = os.path.join(llvm_source_dir, "projects", "libcxx", "include")
672
673    stage1_dir = build_dir + "/stage1"
674    stage1_inst_dir = stage1_dir + "/" + package_name
675
676    final_stage_dir = stage1_dir
677
678    if is_darwin():
679        extra_cflags = []
680        extra_cxxflags = ["-stdlib=libc++"]
681        extra_cflags2 = []
682        extra_cxxflags2 = ["-stdlib=libc++"]
683        extra_asmflags = []
684        extra_ldflags = []
685    elif is_linux():
686        extra_cflags = []
687        extra_cxxflags = []
688        extra_cflags2 = ["-fPIC"]
689        # Silence clang's warnings about arguments not being used in compilation.
690        extra_cxxflags2 = [
691            "-fPIC",
692            "-Qunused-arguments",
693        ]
694        extra_asmflags = []
695        # Avoid libLLVM internal function calls going through the PLT.
696        extra_ldflags = ["-Wl,-Bsymbolic-functions"]
697        # For whatever reason, LLVM's build system will set things up to turn
698        # on -ffunction-sections and -fdata-sections, but won't turn on the
699        # corresponding option to strip unused sections.  We do it explicitly
700        # here.  LLVM's build system is also picky about turning on ICF, so
701        # we do that explicitly here, too.
702        extra_ldflags += ["-fuse-ld=gold", "-Wl,--gc-sections", "-Wl,--icf=safe"]
703    elif is_windows():
704        extra_cflags = []
705        extra_cxxflags = []
706        # clang-cl would like to figure out what it's supposed to be emulating
707        # by looking at an MSVC install, but we don't really have that here.
708        # Force things on.
709        extra_cflags2 = []
710        extra_cxxflags2 = [
711            "-fms-compatibility-version=19.15.26726",
712        ]
713        extra_asmflags = []
714        extra_ldflags = []
715
716    if osx_cross_compile:
717        # undo the damage done in the is_linux() block above, and also simulate
718        # the is_darwin() block above.
719        extra_cflags = []
720        extra_cxxflags = ["-stdlib=libc++"]
721        extra_cxxflags2 = ["-stdlib=libc++"]
722
723        extra_flags = [
724            "-target",
725            "x86_64-apple-darwin",
726            "-mlinker-version=137",
727            "-B",
728            "%s/bin" % os.getenv("CROSS_CCTOOLS_PATH"),
729            "-isysroot",
730            os.getenv("CROSS_SYSROOT"),
731            # technically the sysroot flag there should be enough to deduce this,
732            # but clang needs some help to figure this out.
733            "-I%s/usr/include" % os.getenv("CROSS_SYSROOT"),
734            "-iframework",
735            "%s/System/Library/Frameworks" % os.getenv("CROSS_SYSROOT"),
736        ]
737        extra_cflags += extra_flags
738        extra_cxxflags += extra_flags
739        extra_cflags2 += extra_flags
740        extra_cxxflags2 += extra_flags
741        extra_asmflags += extra_flags
742        extra_ldflags = [
743            "-Wl,-syslibroot,%s" % os.getenv("CROSS_SYSROOT"),
744            "-Wl,-dead_strip",
745        ]
746
747    upload_dir = os.getenv("UPLOAD_DIR")
748    if assertions and upload_dir:
749        extra_cflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
750        extra_cxxflags2 += ["-fcrash-diagnostics-dir=%s" % upload_dir]
751
752    if skip_stages < 1:
753        build_one_stage(
754            [cc] + extra_cflags,
755            [cxx] + extra_cxxflags,
756            [asm] + extra_asmflags,
757            [ld] + extra_ldflags,
758            ar,
759            ranlib,
760            libtool,
761            llvm_source_dir,
762            stage1_dir,
763            package_name,
764            build_libcxx,
765            osx_cross_compile,
766            build_type,
767            assertions,
768            libcxx_include_dir,
769            build_wasm,
770            is_final_stage=(stages == 1),
771        )
772
773    if stages >= 2 and skip_stages < 2:
774        stage2_dir = build_dir + "/stage2"
775        stage2_inst_dir = stage2_dir + "/" + package_name
776        final_stage_dir = stage2_dir
777        if skip_stages < 1:
778            cc = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
779            cxx = stage1_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
780            asm = stage1_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
781        build_one_stage(
782            [cc] + extra_cflags2,
783            [cxx] + extra_cxxflags2,
784            [asm] + extra_asmflags,
785            [ld] + extra_ldflags,
786            ar,
787            ranlib,
788            libtool,
789            llvm_source_dir,
790            stage2_dir,
791            package_name,
792            build_libcxx,
793            osx_cross_compile,
794            build_type,
795            assertions,
796            libcxx_include_dir,
797            build_wasm,
798            is_final_stage=(stages == 2),
799            profile="gen" if pgo else None,
800        )
801
802    if stages >= 3 and skip_stages < 3:
803        stage3_dir = build_dir + "/stage3"
804        stage3_inst_dir = stage3_dir + "/" + package_name
805        final_stage_dir = stage3_dir
806        if skip_stages < 2:
807            cc = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
808            cxx = stage2_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
809            asm = stage2_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
810        build_one_stage(
811            [cc] + extra_cflags2,
812            [cxx] + extra_cxxflags2,
813            [asm] + extra_asmflags,
814            [ld] + extra_ldflags,
815            ar,
816            ranlib,
817            libtool,
818            llvm_source_dir,
819            stage3_dir,
820            package_name,
821            build_libcxx,
822            osx_cross_compile,
823            build_type,
824            assertions,
825            libcxx_include_dir,
826            build_wasm,
827            (stages == 3),
828        )
829        if pgo:
830            llvm_profdata = stage2_inst_dir + "/bin/llvm-profdata%s" % exe_ext
831            merge_cmd = [llvm_profdata, "merge", "-o", "merged.profdata"]
832            profraw_files = glob.glob(
833                os.path.join(stage2_dir, "build", "profiles", "*.profraw")
834            )
835            run_in(stage3_dir, merge_cmd + profraw_files)
836            if stages == 3:
837                mkdir_p(upload_dir)
838                shutil.copy2(os.path.join(stage3_dir, "merged.profdata"), upload_dir)
839                return
840
841    if stages >= 4 and skip_stages < 4:
842        stage4_dir = build_dir + "/stage4"
843        final_stage_dir = stage4_dir
844        profile = None
845        if pgo:
846            if skip_stages == 3:
847                profile_dir = os.environ.get("MOZ_FETCHES_DIR", "")
848            else:
849                profile_dir = stage3_dir
850            profile = os.path.join(profile_dir, "merged.profdata")
851        if skip_stages < 3:
852            cc = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
853            cxx = stage3_inst_dir + "/bin/%s%s" % (cxx_name, exe_ext)
854            asm = stage3_inst_dir + "/bin/%s%s" % (cc_name, exe_ext)
855        build_one_stage(
856            [cc] + extra_cflags2,
857            [cxx] + extra_cxxflags2,
858            [asm] + extra_asmflags,
859            [ld] + extra_ldflags,
860            ar,
861            ranlib,
862            libtool,
863            llvm_source_dir,
864            stage4_dir,
865            package_name,
866            build_libcxx,
867            osx_cross_compile,
868            build_type,
869            assertions,
870            libcxx_include_dir,
871            build_wasm,
872            (stages == 4),
873            profile=profile,
874        )
875
876    if build_clang_tidy:
877        prune_final_dir_for_clang_tidy(
878            os.path.join(final_stage_dir, package_name), osx_cross_compile
879        )
880
881    if not args.skip_tar:
882        build_tar_package("%s.tar.zst" % package_name, final_stage_dir, package_name)
883
884
885if __name__ == "__main__":
886    main()
887