1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7from contextlib import contextmanager 8import json 9import six 10 11from .files import ( 12 AbsoluteSymlinkFile, 13 ExistingFile, 14 File, 15 FileFinder, 16 GeneratedFile, 17 HardlinkFile, 18 PreprocessedFile, 19) 20import mozpack.path as mozpath 21 22 23# This probably belongs in a more generic module. Where? 24@contextmanager 25def _auto_fileobj(path, fileobj, mode="r"): 26 if path and fileobj: 27 raise AssertionError("Only 1 of path or fileobj may be defined.") 28 29 if not path and not fileobj: 30 raise AssertionError("Must specified 1 of path or fileobj.") 31 32 if path: 33 fileobj = open(path, mode) 34 35 try: 36 yield fileobj 37 finally: 38 if path: 39 fileobj.close() 40 41 42class UnreadableInstallManifest(Exception): 43 """Raised when an invalid install manifest is parsed.""" 44 45 46class InstallManifest(object): 47 """Describes actions to be used with a copier.FileCopier instance. 48 49 This class facilitates serialization and deserialization of data used to 50 construct a copier.FileCopier and to perform copy operations. 51 52 The manifest defines source paths, destination paths, and a mechanism by 53 which the destination file should come into existence. 54 55 Entries in the manifest correspond to the following types: 56 57 copy -- The file specified as the source path will be copied to the 58 destination path. 59 60 link -- The destination path will be a symlink or hardlink to the source 61 path. If symlinks are not supported, a copy will be performed. 62 63 exists -- The destination path is accounted for and won't be deleted by 64 the FileCopier. If the destination path doesn't exist, an error is 65 raised. 66 67 optional -- The destination path is accounted for and won't be deleted by 68 the FileCopier. No error is raised if the destination path does not 69 exist. 70 71 patternlink -- Paths matched by the expression in the source path 72 will be symlinked or hardlinked to the destination directory. 73 74 patterncopy -- Similar to patternlink except files are copied, not 75 symlinked/hardlinked. 76 77 preprocess -- The file specified at the source path will be run through 78 the preprocessor, and the output will be written to the destination 79 path. 80 81 content -- The destination file will be created with the given content. 82 83 Version 1 of the manifest was the initial version. 84 Version 2 added optional path support 85 Version 3 added support for pattern entries. 86 Version 4 added preprocessed file support. 87 Version 5 added content support. 88 """ 89 90 CURRENT_VERSION = 5 91 92 FIELD_SEPARATOR = "\x1f" 93 94 # Negative values are reserved for non-actionable items, that is, metadata 95 # that doesn't describe files in the destination. 96 LINK = 1 97 COPY = 2 98 REQUIRED_EXISTS = 3 99 OPTIONAL_EXISTS = 4 100 PATTERN_LINK = 5 101 PATTERN_COPY = 6 102 PREPROCESS = 7 103 CONTENT = 8 104 105 def __init__(self, path=None, fileobj=None): 106 """Create a new InstallManifest entry. 107 108 If path is defined, the manifest will be populated with data from the 109 file path. 110 111 If fileobj is defined, the manifest will be populated with data read 112 from the specified file object. 113 114 Both path and fileobj cannot be defined. 115 """ 116 self._dests = {} 117 self._source_files = set() 118 119 if path or fileobj: 120 with _auto_fileobj(path, fileobj, "r") as fh: 121 self._source_files.add(fh.name) 122 self._load_from_fileobj(fh) 123 124 def _load_from_fileobj(self, fileobj): 125 version = fileobj.readline().rstrip() 126 if version not in ("1", "2", "3", "4", "5"): 127 raise UnreadableInstallManifest("Unknown manifest version: %s" % version) 128 129 for line in fileobj: 130 # Explicitly strip on \n so we don't strip out the FIELD_SEPARATOR 131 # as well. 132 line = line.rstrip("\n") 133 134 fields = line.split(self.FIELD_SEPARATOR) 135 136 record_type = int(fields[0]) 137 138 if record_type == self.LINK: 139 dest, source = fields[1:] 140 self.add_link(source, dest) 141 continue 142 143 if record_type == self.COPY: 144 dest, source = fields[1:] 145 self.add_copy(source, dest) 146 continue 147 148 if record_type == self.REQUIRED_EXISTS: 149 _, path = fields 150 self.add_required_exists(path) 151 continue 152 153 if record_type == self.OPTIONAL_EXISTS: 154 _, path = fields 155 self.add_optional_exists(path) 156 continue 157 158 if record_type == self.PATTERN_LINK: 159 _, base, pattern, dest = fields[1:] 160 self.add_pattern_link(base, pattern, dest) 161 continue 162 163 if record_type == self.PATTERN_COPY: 164 _, base, pattern, dest = fields[1:] 165 self.add_pattern_copy(base, pattern, dest) 166 continue 167 168 if record_type == self.PREPROCESS: 169 dest, source, deps, marker, defines, warnings = fields[1:] 170 171 self.add_preprocess( 172 source, 173 dest, 174 deps, 175 marker, 176 self._decode_field_entry(defines), 177 silence_missing_directive_warnings=bool(int(warnings)), 178 ) 179 continue 180 181 if record_type == self.CONTENT: 182 dest, content = fields[1:] 183 184 self.add_content( 185 six.ensure_text(self._decode_field_entry(content)), dest 186 ) 187 continue 188 189 # Don't fail for non-actionable items, allowing 190 # forward-compatibility with those we will add in the future. 191 if record_type >= 0: 192 raise UnreadableInstallManifest("Unknown record type: %d" % record_type) 193 194 def __len__(self): 195 return len(self._dests) 196 197 def __contains__(self, item): 198 return item in self._dests 199 200 def __eq__(self, other): 201 return isinstance(other, InstallManifest) and self._dests == other._dests 202 203 def __neq__(self, other): 204 return not self.__eq__(other) 205 206 def __ior__(self, other): 207 if not isinstance(other, InstallManifest): 208 raise ValueError("Can only | with another instance of InstallManifest.") 209 210 self.add_entries_from(other) 211 212 return self 213 214 def _encode_field_entry(self, data): 215 """Converts an object into a format that can be stored in the manifest file. 216 217 Complex data types, such as ``dict``, need to be converted into a text 218 representation before they can be written to a file. 219 """ 220 return json.dumps(data, sort_keys=True) 221 222 def _decode_field_entry(self, data): 223 """Restores an object from a format that can be stored in the manifest file. 224 225 Complex data types, such as ``dict``, need to be converted into a text 226 representation before they can be written to a file. 227 """ 228 return json.loads(data) 229 230 def write(self, path=None, fileobj=None, expand_pattern=False): 231 """Serialize this manifest to a file or file object. 232 233 If path is specified, that file will be written to. If fileobj is specified, 234 the serialized content will be written to that file object. 235 236 It is an error if both are specified. 237 """ 238 with _auto_fileobj(path, fileobj, "wt") as fh: 239 fh.write("%d\n" % self.CURRENT_VERSION) 240 241 for dest in sorted(self._dests): 242 entry = self._dests[dest] 243 244 if expand_pattern and entry[0] in ( 245 self.PATTERN_LINK, 246 self.PATTERN_COPY, 247 ): 248 type, base, pattern, dest = entry 249 type = self.LINK if type == self.PATTERN_LINK else self.COPY 250 finder = FileFinder(base) 251 paths = [f[0] for f in finder.find(pattern)] 252 for path in paths: 253 source = mozpath.join(base, path) 254 parts = ["%d" % type, mozpath.join(dest, path), source] 255 fh.write( 256 "%s\n" 257 % self.FIELD_SEPARATOR.join( 258 six.ensure_text(p) for p in parts 259 ) 260 ) 261 else: 262 parts = ["%d" % entry[0], dest] 263 parts.extend(entry[1:]) 264 fh.write( 265 "%s\n" 266 % self.FIELD_SEPARATOR.join(six.ensure_text(p) for p in parts) 267 ) 268 269 def add_link(self, source, dest): 270 """Add a link to this manifest. 271 272 dest will be either a symlink or hardlink to source. 273 """ 274 self._add_entry(dest, (self.LINK, source)) 275 276 def add_copy(self, source, dest): 277 """Add a copy to this manifest. 278 279 source will be copied to dest. 280 """ 281 self._add_entry(dest, (self.COPY, source)) 282 283 def add_required_exists(self, dest): 284 """Record that a destination file must exist. 285 286 This effectively prevents the listed file from being deleted. 287 """ 288 self._add_entry(dest, (self.REQUIRED_EXISTS,)) 289 290 def add_optional_exists(self, dest): 291 """Record that a destination file may exist. 292 293 This effectively prevents the listed file from being deleted. Unlike a 294 "required exists" file, files of this type do not raise errors if the 295 destination file does not exist. 296 """ 297 self._add_entry(dest, (self.OPTIONAL_EXISTS,)) 298 299 def add_pattern_link(self, base, pattern, dest): 300 """Add a pattern match that results in links being created. 301 302 A ``FileFinder`` will be created with its base set to ``base`` 303 and ``FileFinder.find()`` will be called with ``pattern`` to discover 304 source files. Each source file will be either symlinked or hardlinked 305 under ``dest``. 306 307 Filenames under ``dest`` are constructed by taking the path fragment 308 after ``base`` and concatenating it with ``dest``. e.g. 309 310 <base>/foo/bar.h -> <dest>/foo/bar.h 311 """ 312 self._add_entry( 313 mozpath.join(dest, pattern), (self.PATTERN_LINK, base, pattern, dest) 314 ) 315 316 def add_pattern_copy(self, base, pattern, dest): 317 """Add a pattern match that results in copies. 318 319 See ``add_pattern_link()`` for usage. 320 """ 321 self._add_entry( 322 mozpath.join(dest, pattern), (self.PATTERN_COPY, base, pattern, dest) 323 ) 324 325 def add_preprocess( 326 self, 327 source, 328 dest, 329 deps, 330 marker="#", 331 defines={}, 332 silence_missing_directive_warnings=False, 333 ): 334 """Add a preprocessed file to this manifest. 335 336 ``source`` will be passed through preprocessor.py, and the output will be 337 written to ``dest``. 338 """ 339 self._add_entry( 340 dest, 341 ( 342 self.PREPROCESS, 343 source, 344 deps, 345 marker, 346 self._encode_field_entry(defines), 347 "1" if silence_missing_directive_warnings else "0", 348 ), 349 ) 350 351 def add_content(self, content, dest): 352 """Add a file with the given content.""" 353 self._add_entry( 354 dest, 355 ( 356 self.CONTENT, 357 self._encode_field_entry(content), 358 ), 359 ) 360 361 def _add_entry(self, dest, entry): 362 if dest in self._dests: 363 raise ValueError("Item already in manifest: %s" % dest) 364 365 self._dests[dest] = entry 366 367 def add_entries_from(self, other, base=""): 368 """ 369 Copy data from another mozpack.copier.InstallManifest 370 instance, adding an optional base prefix to the destination. 371 372 This allows to merge two manifests into a single manifest, or 373 two take the tagged union of two manifests. 374 """ 375 # We must copy source files to ourselves so extra dependencies from 376 # the preprocessor are taken into account. Ideally, we would track 377 # which source file each entry came from. However, this is more 378 # complicated and not yet implemented. The current implementation 379 # will result in over invalidation, possibly leading to performance 380 # loss. 381 self._source_files |= other._source_files 382 383 for dest in sorted(other._dests): 384 new_dest = mozpath.join(base, dest) if base else dest 385 entry = other._dests[dest] 386 if entry[0] in (self.PATTERN_LINK, self.PATTERN_COPY): 387 entry_type, entry_base, entry_pattern, entry_dest = entry 388 new_entry_dest = mozpath.join(base, entry_dest) if base else entry_dest 389 new_entry = (entry_type, entry_base, entry_pattern, new_entry_dest) 390 else: 391 new_entry = tuple(entry) 392 393 self._add_entry(new_dest, new_entry) 394 395 def populate_registry(self, registry, defines_override={}, link_policy="symlink"): 396 """Populate a mozpack.copier.FileRegistry instance with data from us. 397 398 The caller supplied a FileRegistry instance (or at least something that 399 conforms to its interface) and that instance is populated with data 400 from this manifest. 401 402 Defines can be given to override the ones in the manifest for 403 preprocessing. 404 405 The caller can set a link policy. This determines whether symlinks, 406 hardlinks, or copies are used for LINK and PATTERN_LINK. 407 """ 408 assert link_policy in ("symlink", "hardlink", "copy") 409 for dest in sorted(self._dests): 410 entry = self._dests[dest] 411 install_type = entry[0] 412 413 if install_type == self.LINK: 414 if link_policy == "symlink": 415 cls = AbsoluteSymlinkFile 416 elif link_policy == "hardlink": 417 cls = HardlinkFile 418 else: 419 cls = File 420 registry.add(dest, cls(entry[1])) 421 continue 422 423 if install_type == self.COPY: 424 registry.add(dest, File(entry[1])) 425 continue 426 427 if install_type == self.REQUIRED_EXISTS: 428 registry.add(dest, ExistingFile(required=True)) 429 continue 430 431 if install_type == self.OPTIONAL_EXISTS: 432 registry.add(dest, ExistingFile(required=False)) 433 continue 434 435 if install_type in (self.PATTERN_LINK, self.PATTERN_COPY): 436 _, base, pattern, dest = entry 437 finder = FileFinder(base) 438 paths = [f[0] for f in finder.find(pattern)] 439 440 if install_type == self.PATTERN_LINK: 441 if link_policy == "symlink": 442 cls = AbsoluteSymlinkFile 443 elif link_policy == "hardlink": 444 cls = HardlinkFile 445 else: 446 cls = File 447 else: 448 cls = File 449 450 for path in paths: 451 source = mozpath.join(base, path) 452 registry.add(mozpath.join(dest, path), cls(source)) 453 454 continue 455 456 if install_type == self.PREPROCESS: 457 defines = self._decode_field_entry(entry[4]) 458 if defines_override: 459 defines.update(defines_override) 460 registry.add( 461 dest, 462 PreprocessedFile( 463 entry[1], 464 depfile_path=entry[2], 465 marker=entry[3], 466 defines=defines, 467 extra_depends=self._source_files, 468 silence_missing_directive_warnings=bool(int(entry[5])), 469 ), 470 ) 471 472 continue 473 474 if install_type == self.CONTENT: 475 # GeneratedFile expect the buffer interface, which the unicode 476 # type doesn't have, so encode to a str. 477 content = self._decode_field_entry(entry[1]).encode("utf-8") 478 registry.add(dest, GeneratedFile(content)) 479 continue 480 481 raise Exception( 482 "Unknown install type defined in manifest: %d" % install_type 483 ) 484