1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, unicode_literals, print_function
6
7from mozpack.packager.formats import (
8    FlatFormatter,
9    JarFormatter,
10    OmniJarFormatter,
11)
12from mozpack.packager import (
13    preprocess_manifest,
14    preprocess,
15    Component,
16    SimpleManifestSink,
17)
18from mozpack.files import (
19    GeneratedFile,
20    FileFinder,
21    File,
22)
23from mozpack.copier import (
24    FileCopier,
25    Jarrer,
26)
27from mozpack.errors import errors
28from mozpack.files import ExecutableFile
29import mozpack.path as mozpath
30import buildconfig
31from argparse import ArgumentParser
32from collections import OrderedDict
33from createprecomplete import generate_precomplete
34import os
35import plistlib
36import six
37from six import StringIO
38import subprocess
39
40
41class PackagerFileFinder(FileFinder):
42    def get(self, path):
43        f = super(PackagerFileFinder, self).get(path)
44        # Normalize Info.plist files, and remove the MozillaDeveloper*Path
45        # entries which are only needed on unpackaged builds.
46        if mozpath.basename(path) == "Info.plist":
47            info = plistlib.load(f.open(), dict_type=OrderedDict)
48            info.pop("MozillaDeveloperObjPath", None)
49            info.pop("MozillaDeveloperRepoPath", None)
50            return GeneratedFile(plistlib.dumps(info, sort_keys=False))
51        return f
52
53
54class RemovedFiles(GeneratedFile):
55    """
56    File class for removed-files. Is used as a preprocessor parser.
57    """
58
59    def __init__(self, copier):
60        self.copier = copier
61        GeneratedFile.__init__(self, b"")
62
63    def handle_line(self, f):
64        f = f.strip()
65        if not f:
66            return
67        if self.copier.contains(f):
68            errors.error("Removal of packaged file(s): %s" % f)
69        self.content += six.ensure_binary(f) + b"\n"
70
71
72def split_define(define):
73    """
74    Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to
75    1. Numeric VALs are returned as ints.
76    """
77    if "=" in define:
78        name, value = define.split("=", 1)
79        try:
80            value = int(value)
81        except ValueError:
82            pass
83        return (name, value)
84    return (define, 1)
85
86
87class NoPkgFilesRemover(object):
88    """
89    Formatter wrapper to handle NO_PKG_FILES.
90    """
91
92    def __init__(self, formatter, has_manifest):
93        assert "NO_PKG_FILES" in os.environ
94        self._formatter = formatter
95        self._files = os.environ["NO_PKG_FILES"].split()
96        if has_manifest:
97            self._error = errors.error
98            self._msg = "NO_PKG_FILES contains file listed in manifest: %s"
99        else:
100            self._error = errors.warn
101            self._msg = "Skipping %s"
102
103    def add_base(self, base, *args):
104        self._formatter.add_base(base, *args)
105
106    def add(self, path, content):
107        if not any(mozpath.match(path, spec) for spec in self._files):
108            self._formatter.add(path, content)
109        else:
110            self._error(self._msg % path)
111
112    def add_manifest(self, entry):
113        self._formatter.add_manifest(entry)
114
115    def contains(self, path):
116        return self._formatter.contains(path)
117
118
119def main():
120    parser = ArgumentParser()
121    parser.add_argument(
122        "-D",
123        dest="defines",
124        action="append",
125        metavar="VAR[=VAL]",
126        help="Define a variable",
127    )
128    parser.add_argument(
129        "--format",
130        default="omni",
131        help="Choose the chrome format for packaging "
132        + "(omni, jar or flat ; default: %(default)s)",
133    )
134    parser.add_argument("--removals", default=None, help="removed-files source file")
135    parser.add_argument(
136        "--ignore-errors",
137        action="store_true",
138        default=False,
139        help="Transform errors into warnings.",
140    )
141    parser.add_argument(
142        "--ignore-broken-symlinks",
143        action="store_true",
144        default=False,
145        help="Do not fail when processing broken symlinks.",
146    )
147    parser.add_argument(
148        "--minify",
149        action="store_true",
150        default=False,
151        help="Make some files more compact while packaging",
152    )
153    parser.add_argument(
154        "--minify-js",
155        action="store_true",
156        help="Minify JavaScript files while packaging.",
157    )
158    parser.add_argument(
159        "--js-binary",
160        help="Path to js binary. This is used to verify "
161        "minified JavaScript. If this is not defined, "
162        "minification verification will not be performed.",
163    )
164    parser.add_argument(
165        "--jarlog", default="", help="File containing jar " + "access logs"
166    )
167    parser.add_argument(
168        "--compress",
169        choices=("none", "deflate"),
170        default="deflate",
171        help="Use given jar compression (default: deflate)",
172    )
173    parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name")
174    parser.add_argument("source", help="Source directory")
175    parser.add_argument("destination", help="Destination directory")
176    parser.add_argument(
177        "--non-resource",
178        nargs="+",
179        metavar="PATTERN",
180        default=[],
181        help="Extra files not to be considered as resources",
182    )
183    args = parser.parse_args()
184
185    defines = dict(buildconfig.defines["ALLDEFINES"])
186    if args.ignore_errors:
187        errors.ignore_errors()
188
189    if args.defines:
190        for name, value in [split_define(d) for d in args.defines]:
191            defines[name] = value
192
193    compress = {
194        "none": False,
195        "deflate": True,
196    }[args.compress]
197
198    copier = FileCopier()
199    if args.format == "flat":
200        formatter = FlatFormatter(copier)
201    elif args.format == "jar":
202        formatter = JarFormatter(copier, compress=compress)
203    elif args.format == "omni":
204        formatter = OmniJarFormatter(
205            copier,
206            buildconfig.substs["OMNIJAR_NAME"],
207            compress=compress,
208            non_resources=args.non_resource,
209        )
210    else:
211        errors.fatal("Unknown format: %s" % args.format)
212
213    # Adjust defines according to the requested format.
214    if isinstance(formatter, OmniJarFormatter):
215        defines["MOZ_OMNIJAR"] = 1
216    elif "MOZ_OMNIJAR" in defines:
217        del defines["MOZ_OMNIJAR"]
218
219    respath = ""
220    if "RESPATH" in defines:
221        respath = SimpleManifestSink.normalize_path(defines["RESPATH"])
222    while respath.startswith("/"):
223        respath = respath[1:]
224
225    with errors.accumulate():
226        finder_args = dict(
227            minify=args.minify,
228            minify_js=args.minify_js,
229            ignore_broken_symlinks=args.ignore_broken_symlinks,
230        )
231        if args.js_binary:
232            finder_args["minify_js_verify_command"] = [
233                args.js_binary,
234                os.path.join(
235                    os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js"
236                ),
237            ]
238        finder = PackagerFileFinder(args.source, find_executables=True, **finder_args)
239        if "NO_PKG_FILES" in os.environ:
240            sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None)
241        else:
242            sinkformatter = formatter
243        sink = SimpleManifestSink(finder, sinkformatter)
244        if args.manifest:
245            preprocess_manifest(sink, args.manifest, defines)
246        else:
247            sink.add(Component(""), "bin/*")
248        sink.close(args.manifest is not None)
249
250        if args.removals:
251            removals_in = StringIO(open(args.removals).read())
252            removals_in.name = args.removals
253            removals = RemovedFiles(copier)
254            preprocess(removals_in, removals, defines)
255            copier.add(mozpath.join(respath, "removed-files"), removals)
256
257    # If a pdb file is present and we were instructed to copy it, include it.
258    # Run on all OSes to capture MinGW builds
259    if buildconfig.substs.get("MOZ_COPY_PDBS"):
260        # We want to mutate the copier while we're iterating through it, so copy
261        # the items to a list first.
262        copier_items = [(p, f) for p, f in copier]
263        for p, f in copier_items:
264            if isinstance(f, ExecutableFile):
265                pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb"
266                if os.path.exists(pdbname):
267                    copier.add(os.path.basename(pdbname), File(pdbname))
268
269    # Setup preloading
270    if args.jarlog:
271        if not os.path.exists(args.jarlog):
272            raise Exception("Cannot find jar log: %s" % args.jarlog)
273        omnijars = []
274        if isinstance(formatter, OmniJarFormatter):
275            omnijars = [
276                mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"])
277                for base in sink.packager.get_bases(addons=False)
278            ]
279
280        from mozpack.mozjar import JarLog
281
282        log = JarLog(args.jarlog)
283        for p, f in copier:
284            if not isinstance(f, Jarrer):
285                continue
286            if respath:
287                p = mozpath.relpath(p, respath)
288            if p in log:
289                f.preload(log[p])
290            elif p in omnijars:
291                raise Exception("No jar log data for %s" % p)
292
293    copier.copy(args.destination)
294    generate_precomplete(os.path.normpath(os.path.join(args.destination, respath)))
295
296
297if __name__ == "__main__":
298    main()
299