1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import 6 7from mozbuild.preprocessor import Preprocessor 8import re 9import os 10from mozpack.errors import errors 11from mozpack.chrome.manifest import ( 12 Manifest, 13 ManifestBinaryComponent, 14 ManifestChrome, 15 ManifestInterfaces, 16 is_manifest, 17 parse_manifest, 18) 19import mozpack.path as mozpath 20from collections import deque 21 22 23class Component(object): 24 ''' 25 Class that represents a component in a package manifest. 26 ''' 27 def __init__(self, name, destdir=''): 28 if name.find(' ') > 0: 29 errors.fatal('Malformed manifest: space in component name "%s"' 30 % component) 31 self._name = name 32 self._destdir = destdir 33 34 def __repr__(self): 35 s = self.name 36 if self.destdir: 37 s += ' destdir="%s"' % self.destdir 38 return s 39 40 @property 41 def name(self): 42 return self._name 43 44 @property 45 def destdir(self): 46 return self._destdir 47 48 @staticmethod 49 def _triples(lst): 50 ''' 51 Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)]. 52 ''' 53 return zip(*[iter(lst)] * 3) 54 55 KEY_VALUE_RE = re.compile(r''' 56 \s* # optional whitespace. 57 ([a-zA-Z0-9_]+) # key. 58 \s*=\s* # optional space around =. 59 "([^"]*)" # value without surrounding quotes. 60 (?:\s+|$) 61 ''', re.VERBOSE) 62 63 @staticmethod 64 def _split_options(string): 65 ''' 66 Split 'key1="value1" key2="value2"' into 67 {'key1':'value1', 'key2':'value2'}. 68 69 Returned keys and values are all strings. 70 71 Throws ValueError if the input is malformed. 72 ''' 73 options = {} 74 splits = Component.KEY_VALUE_RE.split(string) 75 if len(splits) % 3 != 1: 76 # This should never happen -- we expect to always split 77 # into ['', ('key', 'val', '')*]. 78 raise ValueError("Bad input") 79 if splits[0]: 80 raise ValueError('Unrecognized input ' + splits[0]) 81 for key, val, no_match in Component._triples(splits[1:]): 82 if no_match: 83 raise ValueError('Unrecognized input ' + no_match) 84 options[key] = val 85 return options 86 87 @staticmethod 88 def _split_component_and_options(string): 89 ''' 90 Split 'name key1="value1" key2="value2"' into 91 ('name', {'key1':'value1', 'key2':'value2'}). 92 93 Returned name, keys and values are all strings. 94 95 Raises ValueError if the input is malformed. 96 ''' 97 splits = string.strip().split(None, 1) 98 if not splits: 99 raise ValueError('No component found') 100 component = splits[0].strip() 101 if not component: 102 raise ValueError('No component found') 103 if not re.match('[a-zA-Z0-9_\-]+$', component): 104 raise ValueError('Bad component name ' + component) 105 options = Component._split_options(splits[1]) if len(splits) > 1 else {} 106 return component, options 107 108 @staticmethod 109 def from_string(string): 110 ''' 111 Create a component from a string. 112 ''' 113 try: 114 name, options = Component._split_component_and_options(string) 115 except ValueError as e: 116 errors.fatal('Malformed manifest: %s' % e) 117 return 118 destdir = options.pop('destdir', '') 119 if options: 120 errors.fatal('Malformed manifest: options %s not recognized' 121 % options.keys()) 122 return Component(name, destdir=destdir) 123 124 125class PackageManifestParser(object): 126 ''' 127 Class for parsing of a package manifest, after preprocessing. 128 129 A package manifest is a list of file paths, with some syntaxic sugar: 130 [] designates a toplevel component. Example: [xpcom] 131 - in front of a file specifies it to be removed 132 * wildcard support 133 ** expands to all files and zero or more directories 134 ; file comment 135 136 The parser takes input from the preprocessor line by line, and pushes 137 parsed information to a sink object. 138 139 The add and remove methods of the sink object are called with the 140 current Component instance and a path. 141 ''' 142 def __init__(self, sink): 143 ''' 144 Initialize the package manifest parser with the given sink. 145 ''' 146 self._component = Component('') 147 self._sink = sink 148 149 def handle_line(self, str): 150 ''' 151 Handle a line of input and push the parsed information to the sink 152 object. 153 ''' 154 # Remove comments. 155 str = str.strip() 156 if not str or str.startswith(';'): 157 return 158 if str.startswith('[') and str.endswith(']'): 159 self._component = Component.from_string(str[1:-1]) 160 elif str.startswith('-'): 161 str = str[1:] 162 self._sink.remove(self._component, str) 163 elif ',' in str: 164 errors.fatal('Incompatible syntax') 165 else: 166 self._sink.add(self._component, str) 167 168 169class PreprocessorOutputWrapper(object): 170 ''' 171 File-like helper to handle the preprocessor output and send it to a parser. 172 The parser's handle_line method is called in the relevant errors.context. 173 ''' 174 def __init__(self, preprocessor, parser): 175 self._parser = parser 176 self._pp = preprocessor 177 178 def write(self, str): 179 file = os.path.normpath(os.path.abspath(self._pp.context['FILE'])) 180 with errors.context(file, self._pp.context['LINE']): 181 self._parser.handle_line(str) 182 183 184def preprocess(input, parser, defines={}): 185 ''' 186 Preprocess the file-like input with the given defines, and send the 187 preprocessed output line by line to the given parser. 188 ''' 189 pp = Preprocessor() 190 pp.context.update(defines) 191 pp.do_filter('substitution') 192 pp.out = PreprocessorOutputWrapper(pp, parser) 193 pp.do_include(input) 194 195 196def preprocess_manifest(sink, manifest, defines={}): 197 ''' 198 Preprocess the given file-like manifest with the given defines, and push 199 the parsed information to a sink. See PackageManifestParser documentation 200 for more details on the sink. 201 ''' 202 preprocess(manifest, PackageManifestParser(sink), defines) 203 204 205class CallDeque(deque): 206 ''' 207 Queue of function calls to make. 208 ''' 209 def append(self, function, *args): 210 deque.append(self, (errors.get_context(), function, args)) 211 212 def execute(self): 213 while True: 214 try: 215 context, function, args = self.popleft() 216 except IndexError: 217 return 218 if context: 219 with errors.context(context[0], context[1]): 220 function(*args) 221 else: 222 function(*args) 223 224 225class SimplePackager(object): 226 ''' 227 Helper used to translate and buffer instructions from the 228 SimpleManifestSink to a formatter. Formatters expect some information to be 229 given first that the simple manifest contents can't guarantee before the 230 end of the input. 231 ''' 232 def __init__(self, formatter): 233 self.formatter = formatter 234 # Queue for formatter.add_interfaces()/add_manifest() calls. 235 self._queue = CallDeque() 236 # Queue for formatter.add_manifest() calls for ManifestChrome. 237 self._chrome_queue = CallDeque() 238 # Queue for formatter.add() calls. 239 self._file_queue = CallDeque() 240 # All paths containing addons. (key is path, value is whether it 241 # should be packed or unpacked) 242 self._addons = {} 243 # All manifest paths imported. 244 self._manifests = set() 245 # All manifest paths included from some other manifest. 246 self._included_manifests = {} 247 self._closed = False 248 249 # Parsing RDF is complex, and would require an external library to do 250 # properly. Just go with some hackish but probably sufficient regexp 251 UNPACK_ADDON_RE = re.compile(r'''(?: 252 <em:unpack>true</em:unpack> 253 |em:unpack=(?P<quote>["']?)true(?P=quote) 254 )''', re.VERBOSE) 255 256 def add(self, path, file): 257 ''' 258 Add the given BaseFile instance with the given path. 259 ''' 260 assert not self._closed 261 if is_manifest(path): 262 self._add_manifest_file(path, file) 263 elif path.endswith('.xpt'): 264 self._queue.append(self.formatter.add_interfaces, path, file) 265 else: 266 self._file_queue.append(self.formatter.add, path, file) 267 if mozpath.basename(path) == 'install.rdf': 268 addon = True 269 install_rdf = file.open().read() 270 if self.UNPACK_ADDON_RE.search(install_rdf): 271 addon = 'unpacked' 272 self._addons[mozpath.dirname(path)] = addon 273 274 def _add_manifest_file(self, path, file): 275 ''' 276 Add the given BaseFile with manifest file contents with the given path. 277 ''' 278 self._manifests.add(path) 279 base = '' 280 if hasattr(file, 'path'): 281 # Find the directory the given path is relative to. 282 b = mozpath.normsep(file.path) 283 if b.endswith('/' + path) or b == path: 284 base = os.path.normpath(b[:-len(path)]) 285 for e in parse_manifest(base, path, file.open()): 286 # ManifestResources need to be given after ManifestChrome, so just 287 # put all ManifestChrome in a separate queue to make them first. 288 if isinstance(e, ManifestChrome): 289 # e.move(e.base) just returns a clone of the entry. 290 self._chrome_queue.append(self.formatter.add_manifest, 291 e.move(e.base)) 292 elif not isinstance(e, (Manifest, ManifestInterfaces)): 293 self._queue.append(self.formatter.add_manifest, e.move(e.base)) 294 # If a binary component is added to an addon, prevent the addon 295 # from being packed. 296 if isinstance(e, ManifestBinaryComponent): 297 addon = mozpath.basedir(e.base, self._addons) 298 if addon: 299 self._addons[addon] = 'unpacked' 300 if isinstance(e, Manifest): 301 if e.flags: 302 errors.fatal('Flags are not supported on ' + 303 '"manifest" entries') 304 self._included_manifests[e.path] = path 305 306 def get_bases(self, addons=True): 307 ''' 308 Return all paths under which root manifests have been found. Root 309 manifests are manifests that are included in no other manifest. 310 `addons` indicates whether to include addon bases as well. 311 ''' 312 all_bases = set(mozpath.dirname(m) 313 for m in self._manifests 314 - set(self._included_manifests)) 315 if not addons: 316 all_bases -= set(self._addons) 317 else: 318 # If for some reason some detected addon doesn't have a 319 # non-included manifest. 320 all_bases |= set(self._addons) 321 return all_bases 322 323 def close(self): 324 ''' 325 Push all instructions to the formatter. 326 ''' 327 self._closed = True 328 329 bases = self.get_bases() 330 broken_bases = sorted( 331 m for m, includer in self._included_manifests.iteritems() 332 if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases)) 333 for m in broken_bases: 334 errors.fatal('"%s" is included from "%s", which is outside "%s"' % 335 (m, self._included_manifests[m], 336 mozpath.basedir(m, bases))) 337 for base in sorted(bases): 338 self.formatter.add_base(base, self._addons.get(base, False)) 339 self._chrome_queue.execute() 340 self._queue.execute() 341 self._file_queue.execute() 342 343 344class SimpleManifestSink(object): 345 ''' 346 Parser sink for "simple" package manifests. Simple package manifests use 347 the format described in the PackageManifestParser documentation, but don't 348 support file removals, and require manifests, interfaces and chrome data to 349 be explicitely listed. 350 Entries starting with bin/ are searched under bin/ in the FileFinder, but 351 are packaged without the bin/ prefix. 352 ''' 353 def __init__(self, finder, formatter): 354 ''' 355 Initialize the SimpleManifestSink. The given FileFinder is used to 356 get files matching the patterns given in the manifest. The given 357 formatter does the packaging job. 358 ''' 359 self._finder = finder 360 self.packager = SimplePackager(formatter) 361 self._closed = False 362 self._manifests = set() 363 364 @staticmethod 365 def normalize_path(path): 366 ''' 367 Remove any bin/ prefix. 368 ''' 369 if mozpath.basedir(path, ['bin']) == 'bin': 370 return mozpath.relpath(path, 'bin') 371 return path 372 373 def add(self, component, pattern): 374 ''' 375 Add files with the given pattern in the given component. 376 ''' 377 assert not self._closed 378 added = False 379 for p, f in self._finder.find(pattern): 380 added = True 381 if is_manifest(p): 382 self._manifests.add(p) 383 dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p)) 384 self.packager.add(dest, f) 385 if not added: 386 errors.error('Missing file(s): %s' % pattern) 387 388 def remove(self, component, pattern): 389 ''' 390 Remove files with the given pattern in the given component. 391 ''' 392 assert not self._closed 393 errors.fatal('Removal is unsupported') 394 395 def close(self, auto_root_manifest=True): 396 ''' 397 Add possibly missing bits and push all instructions to the formatter. 398 ''' 399 if auto_root_manifest: 400 # Simple package manifests don't contain the root manifests, so 401 # find and add them. 402 paths = [mozpath.dirname(m) for m in self._manifests] 403 path = mozpath.dirname(mozpath.commonprefix(paths)) 404 for p, f in self._finder.find(mozpath.join(path, 405 'chrome.manifest')): 406 if not p in self._manifests: 407 self.packager.add(SimpleManifestSink.normalize_path(p), f) 408 self.packager.close() 409