1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import
6
7from mozbuild.preprocessor import Preprocessor
8import re
9import os
10from mozpack.errors import errors
11from mozpack.chrome.manifest import (
12    Manifest,
13    ManifestBinaryComponent,
14    ManifestChrome,
15    ManifestInterfaces,
16    is_manifest,
17    parse_manifest,
18)
19import mozpack.path as mozpath
20from collections import deque
21
22
23class Component(object):
24    '''
25    Class that represents a component in a package manifest.
26    '''
27    def __init__(self, name, destdir=''):
28        if name.find(' ') > 0:
29            errors.fatal('Malformed manifest: space in component name "%s"'
30                         % component)
31        self._name = name
32        self._destdir = destdir
33
34    def __repr__(self):
35        s = self.name
36        if self.destdir:
37            s += ' destdir="%s"' % self.destdir
38        return s
39
40    @property
41    def name(self):
42        return self._name
43
44    @property
45    def destdir(self):
46        return self._destdir
47
48    @staticmethod
49    def _triples(lst):
50        '''
51        Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)].
52        '''
53        return zip(*[iter(lst)] * 3)
54
55    KEY_VALUE_RE = re.compile(r'''
56        \s*                 # optional whitespace.
57        ([a-zA-Z0-9_]+)     # key.
58        \s*=\s*             # optional space around =.
59        "([^"]*)"           # value without surrounding quotes.
60        (?:\s+|$)
61        ''', re.VERBOSE)
62
63    @staticmethod
64    def _split_options(string):
65        '''
66        Split 'key1="value1" key2="value2"' into
67        {'key1':'value1', 'key2':'value2'}.
68
69        Returned keys and values are all strings.
70
71        Throws ValueError if the input is malformed.
72        '''
73        options = {}
74        splits = Component.KEY_VALUE_RE.split(string)
75        if len(splits) % 3 != 1:
76            # This should never happen -- we expect to always split
77            # into ['', ('key', 'val', '')*].
78            raise ValueError("Bad input")
79        if splits[0]:
80            raise ValueError('Unrecognized input ' + splits[0])
81        for key, val, no_match in Component._triples(splits[1:]):
82            if no_match:
83                raise ValueError('Unrecognized input ' + no_match)
84            options[key] = val
85        return options
86
87    @staticmethod
88    def _split_component_and_options(string):
89        '''
90        Split 'name key1="value1" key2="value2"' into
91        ('name', {'key1':'value1', 'key2':'value2'}).
92
93        Returned name, keys and values are all strings.
94
95        Raises ValueError if the input is malformed.
96        '''
97        splits = string.strip().split(None, 1)
98        if not splits:
99            raise ValueError('No component found')
100        component = splits[0].strip()
101        if not component:
102            raise ValueError('No component found')
103        if not re.match('[a-zA-Z0-9_\-]+$', component):
104            raise ValueError('Bad component name ' + component)
105        options = Component._split_options(splits[1]) if len(splits) > 1 else {}
106        return component, options
107
108    @staticmethod
109    def from_string(string):
110        '''
111        Create a component from a string.
112        '''
113        try:
114            name, options = Component._split_component_and_options(string)
115        except ValueError as e:
116            errors.fatal('Malformed manifest: %s' % e)
117            return
118        destdir = options.pop('destdir', '')
119        if options:
120            errors.fatal('Malformed manifest: options %s not recognized'
121                         % options.keys())
122        return Component(name, destdir=destdir)
123
124
125class PackageManifestParser(object):
126    '''
127    Class for parsing of a package manifest, after preprocessing.
128
129    A package manifest is a list of file paths, with some syntaxic sugar:
130        [] designates a toplevel component. Example: [xpcom]
131        - in front of a file specifies it to be removed
132        * wildcard support
133        ** expands to all files and zero or more directories
134        ; file comment
135
136    The parser takes input from the preprocessor line by line, and pushes
137    parsed information to a sink object.
138
139    The add and remove methods of the sink object are called with the
140    current Component instance and a path.
141    '''
142    def __init__(self, sink):
143        '''
144        Initialize the package manifest parser with the given sink.
145        '''
146        self._component = Component('')
147        self._sink = sink
148
149    def handle_line(self, str):
150        '''
151        Handle a line of input and push the parsed information to the sink
152        object.
153        '''
154        # Remove comments.
155        str = str.strip()
156        if not str or str.startswith(';'):
157            return
158        if str.startswith('[') and str.endswith(']'):
159            self._component = Component.from_string(str[1:-1])
160        elif str.startswith('-'):
161            str = str[1:]
162            self._sink.remove(self._component, str)
163        elif ',' in str:
164            errors.fatal('Incompatible syntax')
165        else:
166            self._sink.add(self._component, str)
167
168
169class PreprocessorOutputWrapper(object):
170    '''
171    File-like helper to handle the preprocessor output and send it to a parser.
172    The parser's handle_line method is called in the relevant errors.context.
173    '''
174    def __init__(self, preprocessor, parser):
175        self._parser = parser
176        self._pp = preprocessor
177
178    def write(self, str):
179        file = os.path.normpath(os.path.abspath(self._pp.context['FILE']))
180        with errors.context(file, self._pp.context['LINE']):
181            self._parser.handle_line(str)
182
183
184def preprocess(input, parser, defines={}):
185    '''
186    Preprocess the file-like input with the given defines, and send the
187    preprocessed output line by line to the given parser.
188    '''
189    pp = Preprocessor()
190    pp.context.update(defines)
191    pp.do_filter('substitution')
192    pp.out = PreprocessorOutputWrapper(pp, parser)
193    pp.do_include(input)
194
195
196def preprocess_manifest(sink, manifest, defines={}):
197    '''
198    Preprocess the given file-like manifest with the given defines, and push
199    the parsed information to a sink. See PackageManifestParser documentation
200    for more details on the sink.
201    '''
202    preprocess(manifest, PackageManifestParser(sink), defines)
203
204
205class CallDeque(deque):
206    '''
207    Queue of function calls to make.
208    '''
209    def append(self, function, *args):
210        deque.append(self, (errors.get_context(), function, args))
211
212    def execute(self):
213        while True:
214            try:
215                context, function, args = self.popleft()
216            except IndexError:
217                return
218            if context:
219                with errors.context(context[0], context[1]):
220                    function(*args)
221            else:
222                function(*args)
223
224
225class SimplePackager(object):
226    '''
227    Helper used to translate and buffer instructions from the
228    SimpleManifestSink to a formatter. Formatters expect some information to be
229    given first that the simple manifest contents can't guarantee before the
230    end of the input.
231    '''
232    def __init__(self, formatter):
233        self.formatter = formatter
234        # Queue for formatter.add_interfaces()/add_manifest() calls.
235        self._queue = CallDeque()
236        # Queue for formatter.add_manifest() calls for ManifestChrome.
237        self._chrome_queue = CallDeque()
238        # Queue for formatter.add() calls.
239        self._file_queue = CallDeque()
240        # All paths containing addons. (key is path, value is whether it
241        # should be packed or unpacked)
242        self._addons = {}
243        # All manifest paths imported.
244        self._manifests = set()
245        # All manifest paths included from some other manifest.
246        self._included_manifests = {}
247        self._closed = False
248
249    # Parsing RDF is complex, and would require an external library to do
250    # properly. Just go with some hackish but probably sufficient regexp
251    UNPACK_ADDON_RE = re.compile(r'''(?:
252        <em:unpack>true</em:unpack>
253        |em:unpack=(?P<quote>["']?)true(?P=quote)
254    )''', re.VERBOSE)
255
256    def add(self, path, file):
257        '''
258        Add the given BaseFile instance with the given path.
259        '''
260        assert not self._closed
261        if is_manifest(path):
262            self._add_manifest_file(path, file)
263        elif path.endswith('.xpt'):
264            self._queue.append(self.formatter.add_interfaces, path, file)
265        else:
266            self._file_queue.append(self.formatter.add, path, file)
267            if mozpath.basename(path) == 'install.rdf':
268                addon = True
269                install_rdf = file.open().read()
270                if self.UNPACK_ADDON_RE.search(install_rdf):
271                    addon = 'unpacked'
272                self._addons[mozpath.dirname(path)] = addon
273
274    def _add_manifest_file(self, path, file):
275        '''
276        Add the given BaseFile with manifest file contents with the given path.
277        '''
278        self._manifests.add(path)
279        base = ''
280        if hasattr(file, 'path'):
281            # Find the directory the given path is relative to.
282            b = mozpath.normsep(file.path)
283            if b.endswith('/' + path) or b == path:
284                base = os.path.normpath(b[:-len(path)])
285        for e in parse_manifest(base, path, file.open()):
286            # ManifestResources need to be given after ManifestChrome, so just
287            # put all ManifestChrome in a separate queue to make them first.
288            if isinstance(e, ManifestChrome):
289                # e.move(e.base) just returns a clone of the entry.
290                self._chrome_queue.append(self.formatter.add_manifest,
291                                          e.move(e.base))
292            elif not isinstance(e, (Manifest, ManifestInterfaces)):
293                self._queue.append(self.formatter.add_manifest, e.move(e.base))
294            # If a binary component is added to an addon, prevent the addon
295            # from being packed.
296            if isinstance(e, ManifestBinaryComponent):
297                addon = mozpath.basedir(e.base, self._addons)
298                if addon:
299                    self._addons[addon] = 'unpacked'
300            if isinstance(e, Manifest):
301                if e.flags:
302                    errors.fatal('Flags are not supported on ' +
303                                 '"manifest" entries')
304                self._included_manifests[e.path] = path
305
306    def get_bases(self, addons=True):
307        '''
308        Return all paths under which root manifests have been found. Root
309        manifests are manifests that are included in no other manifest.
310        `addons` indicates whether to include addon bases as well.
311        '''
312        all_bases = set(mozpath.dirname(m)
313                        for m in self._manifests
314                                 - set(self._included_manifests))
315        if not addons:
316            all_bases -= set(self._addons)
317        else:
318            # If for some reason some detected addon doesn't have a
319            # non-included manifest.
320            all_bases |= set(self._addons)
321        return all_bases
322
323    def close(self):
324        '''
325        Push all instructions to the formatter.
326        '''
327        self._closed = True
328
329        bases = self.get_bases()
330        broken_bases = sorted(
331            m for m, includer in self._included_manifests.iteritems()
332            if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases))
333        for m in broken_bases:
334            errors.fatal('"%s" is included from "%s", which is outside "%s"' %
335                         (m, self._included_manifests[m],
336                          mozpath.basedir(m, bases)))
337        for base in sorted(bases):
338            self.formatter.add_base(base, self._addons.get(base, False))
339        self._chrome_queue.execute()
340        self._queue.execute()
341        self._file_queue.execute()
342
343
344class SimpleManifestSink(object):
345    '''
346    Parser sink for "simple" package manifests. Simple package manifests use
347    the format described in the PackageManifestParser documentation, but don't
348    support file removals, and require manifests, interfaces and chrome data to
349    be explicitely listed.
350    Entries starting with bin/ are searched under bin/ in the FileFinder, but
351    are packaged without the bin/ prefix.
352    '''
353    def __init__(self, finder, formatter):
354        '''
355        Initialize the SimpleManifestSink. The given FileFinder is used to
356        get files matching the patterns given in the manifest. The given
357        formatter does the packaging job.
358        '''
359        self._finder = finder
360        self.packager = SimplePackager(formatter)
361        self._closed = False
362        self._manifests = set()
363
364    @staticmethod
365    def normalize_path(path):
366        '''
367        Remove any bin/ prefix.
368        '''
369        if mozpath.basedir(path, ['bin']) == 'bin':
370            return mozpath.relpath(path, 'bin')
371        return path
372
373    def add(self, component, pattern):
374        '''
375        Add files with the given pattern in the given component.
376        '''
377        assert not self._closed
378        added = False
379        for p, f in self._finder.find(pattern):
380            added = True
381            if is_manifest(p):
382                self._manifests.add(p)
383            dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p))
384            self.packager.add(dest, f)
385        if not added:
386            errors.error('Missing file(s): %s' % pattern)
387
388    def remove(self, component, pattern):
389        '''
390        Remove files with the given pattern in the given component.
391        '''
392        assert not self._closed
393        errors.fatal('Removal is unsupported')
394
395    def close(self, auto_root_manifest=True):
396        '''
397        Add possibly missing bits and push all instructions to the formatter.
398        '''
399        if auto_root_manifest:
400            # Simple package manifests don't contain the root manifests, so
401            # find and add them.
402            paths = [mozpath.dirname(m) for m in self._manifests]
403            path = mozpath.dirname(mozpath.commonprefix(paths))
404            for p, f in self._finder.find(mozpath.join(path,
405                                          'chrome.manifest')):
406                if not p in self._manifests:
407                    self.packager.add(SimpleManifestSink.normalize_path(p), f)
408        self.packager.close()
409