1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, unicode_literals, print_function 6 7from mozpack.packager.formats import ( 8 FlatFormatter, 9 JarFormatter, 10 OmniJarFormatter, 11) 12from mozpack.packager import ( 13 preprocess_manifest, 14 preprocess, 15 Component, 16 SimpleManifestSink, 17) 18from mozpack.files import ( 19 GeneratedFile, 20 FileFinder, 21 File, 22) 23from mozpack.copier import ( 24 FileCopier, 25 Jarrer, 26) 27from mozpack.errors import errors 28from mozpack.files import ExecutableFile 29import mozpack.path as mozpath 30import buildconfig 31from argparse import ArgumentParser 32from collections import OrderedDict 33from createprecomplete import generate_precomplete 34import os 35import plistlib 36import six 37from six import StringIO 38import subprocess 39 40 41class PackagerFileFinder(FileFinder): 42 def get(self, path): 43 f = super(PackagerFileFinder, self).get(path) 44 # Normalize Info.plist files, and remove the MozillaDeveloper*Path 45 # entries which are only needed on unpackaged builds. 46 if mozpath.basename(path) == "Info.plist": 47 info = plistlib.load(f.open(), dict_type=OrderedDict) 48 info.pop("MozillaDeveloperObjPath", None) 49 info.pop("MozillaDeveloperRepoPath", None) 50 return GeneratedFile(plistlib.dumps(info, sort_keys=False)) 51 return f 52 53 54class RemovedFiles(GeneratedFile): 55 """ 56 File class for removed-files. Is used as a preprocessor parser. 57 """ 58 59 def __init__(self, copier): 60 self.copier = copier 61 GeneratedFile.__init__(self, b"") 62 63 def handle_line(self, f): 64 f = f.strip() 65 if not f: 66 return 67 if self.copier.contains(f): 68 errors.error("Removal of packaged file(s): %s" % f) 69 self.content += six.ensure_binary(f) + b"\n" 70 71 72def split_define(define): 73 """ 74 Give a VAR[=VAL] string, returns a (VAR, VAL) tuple, where VAL defaults to 75 1. Numeric VALs are returned as ints. 76 """ 77 if "=" in define: 78 name, value = define.split("=", 1) 79 try: 80 value = int(value) 81 except ValueError: 82 pass 83 return (name, value) 84 return (define, 1) 85 86 87class NoPkgFilesRemover(object): 88 """ 89 Formatter wrapper to handle NO_PKG_FILES. 90 """ 91 92 def __init__(self, formatter, has_manifest): 93 assert "NO_PKG_FILES" in os.environ 94 self._formatter = formatter 95 self._files = os.environ["NO_PKG_FILES"].split() 96 if has_manifest: 97 self._error = errors.error 98 self._msg = "NO_PKG_FILES contains file listed in manifest: %s" 99 else: 100 self._error = errors.warn 101 self._msg = "Skipping %s" 102 103 def add_base(self, base, *args): 104 self._formatter.add_base(base, *args) 105 106 def add(self, path, content): 107 if not any(mozpath.match(path, spec) for spec in self._files): 108 self._formatter.add(path, content) 109 else: 110 self._error(self._msg % path) 111 112 def add_manifest(self, entry): 113 self._formatter.add_manifest(entry) 114 115 def contains(self, path): 116 return self._formatter.contains(path) 117 118 119def main(): 120 parser = ArgumentParser() 121 parser.add_argument( 122 "-D", 123 dest="defines", 124 action="append", 125 metavar="VAR[=VAL]", 126 help="Define a variable", 127 ) 128 parser.add_argument( 129 "--format", 130 default="omni", 131 help="Choose the chrome format for packaging " 132 + "(omni, jar or flat ; default: %(default)s)", 133 ) 134 parser.add_argument("--removals", default=None, help="removed-files source file") 135 parser.add_argument( 136 "--ignore-errors", 137 action="store_true", 138 default=False, 139 help="Transform errors into warnings.", 140 ) 141 parser.add_argument( 142 "--ignore-broken-symlinks", 143 action="store_true", 144 default=False, 145 help="Do not fail when processing broken symlinks.", 146 ) 147 parser.add_argument( 148 "--minify", 149 action="store_true", 150 default=False, 151 help="Make some files more compact while packaging", 152 ) 153 parser.add_argument( 154 "--minify-js", 155 action="store_true", 156 help="Minify JavaScript files while packaging.", 157 ) 158 parser.add_argument( 159 "--js-binary", 160 help="Path to js binary. This is used to verify " 161 "minified JavaScript. If this is not defined, " 162 "minification verification will not be performed.", 163 ) 164 parser.add_argument( 165 "--jarlog", default="", help="File containing jar " + "access logs" 166 ) 167 parser.add_argument( 168 "--compress", 169 choices=("none", "deflate"), 170 default="deflate", 171 help="Use given jar compression (default: deflate)", 172 ) 173 parser.add_argument("manifest", default=None, nargs="?", help="Manifest file name") 174 parser.add_argument("source", help="Source directory") 175 parser.add_argument("destination", help="Destination directory") 176 parser.add_argument( 177 "--non-resource", 178 nargs="+", 179 metavar="PATTERN", 180 default=[], 181 help="Extra files not to be considered as resources", 182 ) 183 args = parser.parse_args() 184 185 defines = dict(buildconfig.defines["ALLDEFINES"]) 186 if args.ignore_errors: 187 errors.ignore_errors() 188 189 if args.defines: 190 for name, value in [split_define(d) for d in args.defines]: 191 defines[name] = value 192 193 compress = { 194 "none": False, 195 "deflate": True, 196 }[args.compress] 197 198 copier = FileCopier() 199 if args.format == "flat": 200 formatter = FlatFormatter(copier) 201 elif args.format == "jar": 202 formatter = JarFormatter(copier, compress=compress) 203 elif args.format == "omni": 204 formatter = OmniJarFormatter( 205 copier, 206 buildconfig.substs["OMNIJAR_NAME"], 207 compress=compress, 208 non_resources=args.non_resource, 209 ) 210 else: 211 errors.fatal("Unknown format: %s" % args.format) 212 213 # Adjust defines according to the requested format. 214 if isinstance(formatter, OmniJarFormatter): 215 defines["MOZ_OMNIJAR"] = 1 216 elif "MOZ_OMNIJAR" in defines: 217 del defines["MOZ_OMNIJAR"] 218 219 respath = "" 220 if "RESPATH" in defines: 221 respath = SimpleManifestSink.normalize_path(defines["RESPATH"]) 222 while respath.startswith("/"): 223 respath = respath[1:] 224 225 with errors.accumulate(): 226 finder_args = dict( 227 minify=args.minify, 228 minify_js=args.minify_js, 229 ignore_broken_symlinks=args.ignore_broken_symlinks, 230 ) 231 if args.js_binary: 232 finder_args["minify_js_verify_command"] = [ 233 args.js_binary, 234 os.path.join( 235 os.path.abspath(os.path.dirname(__file__)), "js-compare-ast.js" 236 ), 237 ] 238 finder = PackagerFileFinder(args.source, find_executables=True, **finder_args) 239 if "NO_PKG_FILES" in os.environ: 240 sinkformatter = NoPkgFilesRemover(formatter, args.manifest is not None) 241 else: 242 sinkformatter = formatter 243 sink = SimpleManifestSink(finder, sinkformatter) 244 if args.manifest: 245 preprocess_manifest(sink, args.manifest, defines) 246 else: 247 sink.add(Component(""), "bin/*") 248 sink.close(args.manifest is not None) 249 250 if args.removals: 251 removals_in = StringIO(open(args.removals).read()) 252 removals_in.name = args.removals 253 removals = RemovedFiles(copier) 254 preprocess(removals_in, removals, defines) 255 copier.add(mozpath.join(respath, "removed-files"), removals) 256 257 # If a pdb file is present and we were instructed to copy it, include it. 258 # Run on all OSes to capture MinGW builds 259 if buildconfig.substs.get("MOZ_COPY_PDBS"): 260 # We want to mutate the copier while we're iterating through it, so copy 261 # the items to a list first. 262 copier_items = [(p, f) for p, f in copier] 263 for p, f in copier_items: 264 if isinstance(f, ExecutableFile): 265 pdbname = os.path.splitext(f.inputs()[0])[0] + ".pdb" 266 if os.path.exists(pdbname): 267 copier.add(os.path.basename(pdbname), File(pdbname)) 268 269 # Setup preloading 270 if args.jarlog: 271 if not os.path.exists(args.jarlog): 272 raise Exception("Cannot find jar log: %s" % args.jarlog) 273 omnijars = [] 274 if isinstance(formatter, OmniJarFormatter): 275 omnijars = [ 276 mozpath.join(base, buildconfig.substs["OMNIJAR_NAME"]) 277 for base in sink.packager.get_bases(addons=False) 278 ] 279 280 from mozpack.mozjar import JarLog 281 282 log = JarLog(args.jarlog) 283 for p, f in copier: 284 if not isinstance(f, Jarrer): 285 continue 286 if respath: 287 p = mozpath.relpath(p, respath) 288 if p in log: 289 f.preload(log[p]) 290 elif p in omnijars: 291 raise Exception("No jar log data for %s" % p) 292 293 copier.copy(args.destination) 294 generate_precomplete(os.path.normpath(os.path.join(args.destination, respath))) 295 296 297if __name__ == "__main__": 298 main() 299