1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, print_function, unicode_literals
6
7from contextlib import contextmanager
8import json
9import six
10
11from .files import (
12    AbsoluteSymlinkFile,
13    ExistingFile,
14    File,
15    FileFinder,
16    GeneratedFile,
17    HardlinkFile,
18    PreprocessedFile,
19)
20import mozpack.path as mozpath
21
22
23# This probably belongs in a more generic module. Where?
24@contextmanager
25def _auto_fileobj(path, fileobj, mode="r"):
26    if path and fileobj:
27        raise AssertionError("Only 1 of path or fileobj may be defined.")
28
29    if not path and not fileobj:
30        raise AssertionError("Must specified 1 of path or fileobj.")
31
32    if path:
33        fileobj = open(path, mode)
34
35    try:
36        yield fileobj
37    finally:
38        if path:
39            fileobj.close()
40
41
42class UnreadableInstallManifest(Exception):
43    """Raised when an invalid install manifest is parsed."""
44
45
46class InstallManifest(object):
47    """Describes actions to be used with a copier.FileCopier instance.
48
49    This class facilitates serialization and deserialization of data used to
50    construct a copier.FileCopier and to perform copy operations.
51
52    The manifest defines source paths, destination paths, and a mechanism by
53    which the destination file should come into existence.
54
55    Entries in the manifest correspond to the following types:
56
57      copy -- The file specified as the source path will be copied to the
58          destination path.
59
60      link -- The destination path will be a symlink or hardlink to the source
61          path. If symlinks are not supported, a copy will be performed.
62
63      exists -- The destination path is accounted for and won't be deleted by
64          the FileCopier. If the destination path doesn't exist, an error is
65          raised.
66
67      optional -- The destination path is accounted for and won't be deleted by
68          the FileCopier. No error is raised if the destination path does not
69          exist.
70
71      patternlink -- Paths matched by the expression in the source path
72          will be symlinked or hardlinked to the destination directory.
73
74      patterncopy -- Similar to patternlink except files are copied, not
75          symlinked/hardlinked.
76
77      preprocess -- The file specified at the source path will be run through
78          the preprocessor, and the output will be written to the destination
79          path.
80
81      content -- The destination file will be created with the given content.
82
83    Version 1 of the manifest was the initial version.
84    Version 2 added optional path support
85    Version 3 added support for pattern entries.
86    Version 4 added preprocessed file support.
87    Version 5 added content support.
88    """
89
90    CURRENT_VERSION = 5
91
92    FIELD_SEPARATOR = "\x1f"
93
94    # Negative values are reserved for non-actionable items, that is, metadata
95    # that doesn't describe files in the destination.
96    LINK = 1
97    COPY = 2
98    REQUIRED_EXISTS = 3
99    OPTIONAL_EXISTS = 4
100    PATTERN_LINK = 5
101    PATTERN_COPY = 6
102    PREPROCESS = 7
103    CONTENT = 8
104
105    def __init__(self, path=None, fileobj=None):
106        """Create a new InstallManifest entry.
107
108        If path is defined, the manifest will be populated with data from the
109        file path.
110
111        If fileobj is defined, the manifest will be populated with data read
112        from the specified file object.
113
114        Both path and fileobj cannot be defined.
115        """
116        self._dests = {}
117        self._source_files = set()
118
119        if path or fileobj:
120            with _auto_fileobj(path, fileobj, "r") as fh:
121                self._source_files.add(fh.name)
122                self._load_from_fileobj(fh)
123
124    def _load_from_fileobj(self, fileobj):
125        version = fileobj.readline().rstrip()
126        if version not in ("1", "2", "3", "4", "5"):
127            raise UnreadableInstallManifest("Unknown manifest version: %s" % version)
128
129        for line in fileobj:
130            # Explicitly strip on \n so we don't strip out the FIELD_SEPARATOR
131            # as well.
132            line = line.rstrip("\n")
133
134            fields = line.split(self.FIELD_SEPARATOR)
135
136            record_type = int(fields[0])
137
138            if record_type == self.LINK:
139                dest, source = fields[1:]
140                self.add_link(source, dest)
141                continue
142
143            if record_type == self.COPY:
144                dest, source = fields[1:]
145                self.add_copy(source, dest)
146                continue
147
148            if record_type == self.REQUIRED_EXISTS:
149                _, path = fields
150                self.add_required_exists(path)
151                continue
152
153            if record_type == self.OPTIONAL_EXISTS:
154                _, path = fields
155                self.add_optional_exists(path)
156                continue
157
158            if record_type == self.PATTERN_LINK:
159                _, base, pattern, dest = fields[1:]
160                self.add_pattern_link(base, pattern, dest)
161                continue
162
163            if record_type == self.PATTERN_COPY:
164                _, base, pattern, dest = fields[1:]
165                self.add_pattern_copy(base, pattern, dest)
166                continue
167
168            if record_type == self.PREPROCESS:
169                dest, source, deps, marker, defines, warnings = fields[1:]
170
171                self.add_preprocess(
172                    source,
173                    dest,
174                    deps,
175                    marker,
176                    self._decode_field_entry(defines),
177                    silence_missing_directive_warnings=bool(int(warnings)),
178                )
179                continue
180
181            if record_type == self.CONTENT:
182                dest, content = fields[1:]
183
184                self.add_content(
185                    six.ensure_text(self._decode_field_entry(content)), dest
186                )
187                continue
188
189            # Don't fail for non-actionable items, allowing
190            # forward-compatibility with those we will add in the future.
191            if record_type >= 0:
192                raise UnreadableInstallManifest("Unknown record type: %d" % record_type)
193
194    def __len__(self):
195        return len(self._dests)
196
197    def __contains__(self, item):
198        return item in self._dests
199
200    def __eq__(self, other):
201        return isinstance(other, InstallManifest) and self._dests == other._dests
202
203    def __neq__(self, other):
204        return not self.__eq__(other)
205
206    def __ior__(self, other):
207        if not isinstance(other, InstallManifest):
208            raise ValueError("Can only | with another instance of InstallManifest.")
209
210        self.add_entries_from(other)
211
212        return self
213
214    def _encode_field_entry(self, data):
215        """Converts an object into a format that can be stored in the manifest file.
216
217        Complex data types, such as ``dict``, need to be converted into a text
218        representation before they can be written to a file.
219        """
220        return json.dumps(data, sort_keys=True)
221
222    def _decode_field_entry(self, data):
223        """Restores an object from a format that can be stored in the manifest file.
224
225        Complex data types, such as ``dict``, need to be converted into a text
226        representation before they can be written to a file.
227        """
228        return json.loads(data)
229
230    def write(self, path=None, fileobj=None, expand_pattern=False):
231        """Serialize this manifest to a file or file object.
232
233        If path is specified, that file will be written to. If fileobj is specified,
234        the serialized content will be written to that file object.
235
236        It is an error if both are specified.
237        """
238        with _auto_fileobj(path, fileobj, "wt") as fh:
239            fh.write("%d\n" % self.CURRENT_VERSION)
240
241            for dest in sorted(self._dests):
242                entry = self._dests[dest]
243
244                if expand_pattern and entry[0] in (
245                    self.PATTERN_LINK,
246                    self.PATTERN_COPY,
247                ):
248                    type, base, pattern, dest = entry
249                    type = self.LINK if type == self.PATTERN_LINK else self.COPY
250                    finder = FileFinder(base)
251                    paths = [f[0] for f in finder.find(pattern)]
252                    for path in paths:
253                        source = mozpath.join(base, path)
254                        parts = ["%d" % type, mozpath.join(dest, path), source]
255                        fh.write(
256                            "%s\n"
257                            % self.FIELD_SEPARATOR.join(
258                                six.ensure_text(p) for p in parts
259                            )
260                        )
261                else:
262                    parts = ["%d" % entry[0], dest]
263                    parts.extend(entry[1:])
264                    fh.write(
265                        "%s\n"
266                        % self.FIELD_SEPARATOR.join(six.ensure_text(p) for p in parts)
267                    )
268
269    def add_link(self, source, dest):
270        """Add a link to this manifest.
271
272        dest will be either a symlink or hardlink to source.
273        """
274        self._add_entry(dest, (self.LINK, source))
275
276    def add_copy(self, source, dest):
277        """Add a copy to this manifest.
278
279        source will be copied to dest.
280        """
281        self._add_entry(dest, (self.COPY, source))
282
283    def add_required_exists(self, dest):
284        """Record that a destination file must exist.
285
286        This effectively prevents the listed file from being deleted.
287        """
288        self._add_entry(dest, (self.REQUIRED_EXISTS,))
289
290    def add_optional_exists(self, dest):
291        """Record that a destination file may exist.
292
293        This effectively prevents the listed file from being deleted. Unlike a
294        "required exists" file, files of this type do not raise errors if the
295        destination file does not exist.
296        """
297        self._add_entry(dest, (self.OPTIONAL_EXISTS,))
298
299    def add_pattern_link(self, base, pattern, dest):
300        """Add a pattern match that results in links being created.
301
302        A ``FileFinder`` will be created with its base set to ``base``
303        and ``FileFinder.find()`` will be called with ``pattern`` to discover
304        source files. Each source file will be either symlinked or hardlinked
305        under ``dest``.
306
307        Filenames under ``dest`` are constructed by taking the path fragment
308        after ``base`` and concatenating it with ``dest``. e.g.
309
310           <base>/foo/bar.h -> <dest>/foo/bar.h
311        """
312        self._add_entry(
313            mozpath.join(dest, pattern), (self.PATTERN_LINK, base, pattern, dest)
314        )
315
316    def add_pattern_copy(self, base, pattern, dest):
317        """Add a pattern match that results in copies.
318
319        See ``add_pattern_link()`` for usage.
320        """
321        self._add_entry(
322            mozpath.join(dest, pattern), (self.PATTERN_COPY, base, pattern, dest)
323        )
324
325    def add_preprocess(
326        self,
327        source,
328        dest,
329        deps,
330        marker="#",
331        defines={},
332        silence_missing_directive_warnings=False,
333    ):
334        """Add a preprocessed file to this manifest.
335
336        ``source`` will be passed through preprocessor.py, and the output will be
337        written to ``dest``.
338        """
339        self._add_entry(
340            dest,
341            (
342                self.PREPROCESS,
343                source,
344                deps,
345                marker,
346                self._encode_field_entry(defines),
347                "1" if silence_missing_directive_warnings else "0",
348            ),
349        )
350
351    def add_content(self, content, dest):
352        """Add a file with the given content."""
353        self._add_entry(
354            dest,
355            (
356                self.CONTENT,
357                self._encode_field_entry(content),
358            ),
359        )
360
361    def _add_entry(self, dest, entry):
362        if dest in self._dests:
363            raise ValueError("Item already in manifest: %s" % dest)
364
365        self._dests[dest] = entry
366
367    def add_entries_from(self, other, base=""):
368        """
369        Copy data from another mozpack.copier.InstallManifest
370        instance, adding an optional base prefix to the destination.
371
372        This allows to merge two manifests into a single manifest, or
373        two take the tagged union of two manifests.
374        """
375        # We must copy source files to ourselves so extra dependencies from
376        # the preprocessor are taken into account. Ideally, we would track
377        # which source file each entry came from. However, this is more
378        # complicated and not yet implemented. The current implementation
379        # will result in over invalidation, possibly leading to performance
380        # loss.
381        self._source_files |= other._source_files
382
383        for dest in sorted(other._dests):
384            new_dest = mozpath.join(base, dest) if base else dest
385            entry = other._dests[dest]
386            if entry[0] in (self.PATTERN_LINK, self.PATTERN_COPY):
387                entry_type, entry_base, entry_pattern, entry_dest = entry
388                new_entry_dest = mozpath.join(base, entry_dest) if base else entry_dest
389                new_entry = (entry_type, entry_base, entry_pattern, new_entry_dest)
390            else:
391                new_entry = tuple(entry)
392
393            self._add_entry(new_dest, new_entry)
394
395    def populate_registry(self, registry, defines_override={}, link_policy="symlink"):
396        """Populate a mozpack.copier.FileRegistry instance with data from us.
397
398        The caller supplied a FileRegistry instance (or at least something that
399        conforms to its interface) and that instance is populated with data
400        from this manifest.
401
402        Defines can be given to override the ones in the manifest for
403        preprocessing.
404
405        The caller can set a link policy. This determines whether symlinks,
406        hardlinks, or copies are used for LINK and PATTERN_LINK.
407        """
408        assert link_policy in ("symlink", "hardlink", "copy")
409        for dest in sorted(self._dests):
410            entry = self._dests[dest]
411            install_type = entry[0]
412
413            if install_type == self.LINK:
414                if link_policy == "symlink":
415                    cls = AbsoluteSymlinkFile
416                elif link_policy == "hardlink":
417                    cls = HardlinkFile
418                else:
419                    cls = File
420                registry.add(dest, cls(entry[1]))
421                continue
422
423            if install_type == self.COPY:
424                registry.add(dest, File(entry[1]))
425                continue
426
427            if install_type == self.REQUIRED_EXISTS:
428                registry.add(dest, ExistingFile(required=True))
429                continue
430
431            if install_type == self.OPTIONAL_EXISTS:
432                registry.add(dest, ExistingFile(required=False))
433                continue
434
435            if install_type in (self.PATTERN_LINK, self.PATTERN_COPY):
436                _, base, pattern, dest = entry
437                finder = FileFinder(base)
438                paths = [f[0] for f in finder.find(pattern)]
439
440                if install_type == self.PATTERN_LINK:
441                    if link_policy == "symlink":
442                        cls = AbsoluteSymlinkFile
443                    elif link_policy == "hardlink":
444                        cls = HardlinkFile
445                    else:
446                        cls = File
447                else:
448                    cls = File
449
450                for path in paths:
451                    source = mozpath.join(base, path)
452                    registry.add(mozpath.join(dest, path), cls(source))
453
454                continue
455
456            if install_type == self.PREPROCESS:
457                defines = self._decode_field_entry(entry[4])
458                if defines_override:
459                    defines.update(defines_override)
460                registry.add(
461                    dest,
462                    PreprocessedFile(
463                        entry[1],
464                        depfile_path=entry[2],
465                        marker=entry[3],
466                        defines=defines,
467                        extra_depends=self._source_files,
468                        silence_missing_directive_warnings=bool(int(entry[5])),
469                    ),
470                )
471
472                continue
473
474            if install_type == self.CONTENT:
475                # GeneratedFile expect the buffer interface, which the unicode
476                # type doesn't have, so encode to a str.
477                content = self._decode_field_entry(entry[1]).encode("utf-8")
478                registry.add(dest, GeneratedFile(content))
479                continue
480
481            raise Exception(
482                "Unknown install type defined in manifest: %d" % install_type
483            )
484