1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from mozpack.files import (
6    BaseFinder,
7    ExecutableFile,
8    BaseFile,
9    GeneratedFile,
10)
11from mozpack.executables import (
12    MACHO_SIGNATURES,
13)
14from mozpack.errors import errors
15from mozbuild.util import hexdump
16from tempfile import mkstemp
17import mozpack.path as mozpath
18import struct
19import os
20import re
21import subprocess
22import buildconfig
23
24# Regular expressions for unifying install.rdf
25FIND_TARGET_PLATFORM = re.compile(
26    r"""
27    <(?P<ns>[-._0-9A-Za-z]+:)?targetPlatform>  # The targetPlatform tag, with any namespace
28    (?P<platform>[^<]*)                        # The actual platform value
29    </(?P=ns)?targetPlatform>                  # The closing tag
30    """,
31    re.X,
32)
33FIND_TARGET_PLATFORM_ATTR = re.compile(
34    r"""
35    (?P<tag><(?:[-._0-9A-Za-z]+:)?Description) # The opening part of the <Description> tag
36    (?P<attrs>[^>]*?)\s+                       # The initial attributes
37    (?P<ns>[-._0-9A-Za-z]+:)?targetPlatform=   # The targetPlatform attribute, with any namespace
38    [\'"](?P<platform>[^\'"]+)[\'"]            # The actual platform value
39    (?P<otherattrs>[^>]*?>)                    # The remaining attributes and closing angle bracket
40    """,
41    re.X,
42)
43
44
45def may_unify_binary(file):
46    """
47    Return whether the given BaseFile instance is an ExecutableFile that
48    may be unified. Only non-fat Mach-O binaries are to be unified.
49    """
50    if isinstance(file, ExecutableFile):
51        signature = file.open().read(4)
52        if len(signature) < 4:
53            return False
54        signature = struct.unpack(">L", signature)[0]
55        if signature in MACHO_SIGNATURES:
56            return True
57    return False
58
59
60class UnifiedExecutableFile(BaseFile):
61    """
62    File class for executable and library files that to be unified with 'lipo'.
63    """
64
65    def __init__(self, executable1, executable2):
66        """
67        Initialize a UnifiedExecutableFile with a pair of ExecutableFiles to
68        be unified. They are expected to be non-fat Mach-O executables.
69        """
70        assert isinstance(executable1, ExecutableFile)
71        assert isinstance(executable2, ExecutableFile)
72        self._executables = (executable1, executable2)
73
74    def copy(self, dest, skip_if_older=True):
75        """
76        Create a fat executable from the two Mach-O executable given when
77        creating the instance.
78        skip_if_older is ignored.
79        """
80        assert isinstance(dest, str)
81        tmpfiles = []
82        try:
83            for e in self._executables:
84                fd, f = mkstemp()
85                os.close(fd)
86                tmpfiles.append(f)
87                e.copy(f, skip_if_older=False)
88            lipo = buildconfig.substs.get("LIPO") or "lipo"
89            subprocess.check_call([lipo, "-create"] + tmpfiles + ["-output", dest])
90        except Exception as e:
91            errors.error(
92                "Failed to unify %s and %s: %s"
93                % (self._executables[0].path, self._executables[1].path, str(e))
94            )
95        finally:
96            for f in tmpfiles:
97                os.unlink(f)
98
99
100class UnifiedFinder(BaseFinder):
101    """
102    Helper to get unified BaseFile instances from two distinct trees on the
103    file system.
104    """
105
106    def __init__(self, finder1, finder2, sorted=[], **kargs):
107        """
108        Initialize a UnifiedFinder. finder1 and finder2 are BaseFinder
109        instances from which files are picked. UnifiedFinder.find() will act as
110        FileFinder.find() but will error out when matches can only be found in
111        one of the two trees and not the other. It will also error out if
112        matches can be found on both ends but their contents are not identical.
113
114        The sorted argument gives a list of mozpath.match patterns. File
115        paths matching one of these patterns will have their contents compared
116        with their lines sorted.
117        """
118        assert isinstance(finder1, BaseFinder)
119        assert isinstance(finder2, BaseFinder)
120        self._finder1 = finder1
121        self._finder2 = finder2
122        self._sorted = sorted
123        BaseFinder.__init__(self, finder1.base, **kargs)
124
125    def _find(self, path):
126        """
127        UnifiedFinder.find() implementation.
128        """
129        files1 = {p: f for p, f in self._finder1.find(path)}
130        files2 = {p: f for p, f in self._finder2.find(path)}
131        all_paths = set(files1) | set(files2)
132        for p in sorted(all_paths):
133            err = errors.count
134            unified = self.unify_file(p, files1.get(p), files2.get(p))
135            if unified:
136                yield p, unified
137            elif err == errors.count:  # No errors have already been reported.
138                self._report_difference(p, files1.get(p), files2.get(p))
139
140    def _report_difference(self, path, file1, file2):
141        """
142        Report differences between files in both trees.
143        """
144        if not file1:
145            errors.error("File missing in %s: %s" % (self._finder1.base, path))
146            return
147        if not file2:
148            errors.error("File missing in %s: %s" % (self._finder2.base, path))
149            return
150
151        errors.error(
152            "Can't unify %s: file differs between %s and %s"
153            % (path, self._finder1.base, self._finder2.base)
154        )
155        if not isinstance(file1, ExecutableFile) and not isinstance(
156            file2, ExecutableFile
157        ):
158            from difflib import unified_diff
159
160            try:
161                lines1 = [l.decode("utf-8") for l in file1.open().readlines()]
162                lines2 = [l.decode("utf-8") for l in file2.open().readlines()]
163            except UnicodeDecodeError:
164                lines1 = hexdump(file1.open().read())
165                lines2 = hexdump(file2.open().read())
166
167            for line in unified_diff(
168                lines1,
169                lines2,
170                os.path.join(self._finder1.base, path),
171                os.path.join(self._finder2.base, path),
172            ):
173                errors.out.write(line)
174
175    def unify_file(self, path, file1, file2):
176        """
177        Given two BaseFiles and the path they were found at, return a
178        unified version of the files. If the files match, the first BaseFile
179        may be returned.
180        If the files don't match or one of them is `None`, the method returns
181        `None`.
182        Subclasses may decide to unify by using one of the files in that case.
183        """
184        if not file1 or not file2:
185            return None
186
187        if may_unify_binary(file1) and may_unify_binary(file2):
188            return UnifiedExecutableFile(file1, file2)
189
190        content1 = file1.open().readlines()
191        content2 = file2.open().readlines()
192        if content1 == content2:
193            return file1
194        for pattern in self._sorted:
195            if mozpath.match(path, pattern):
196                if sorted(content1) == sorted(content2):
197                    return file1
198                break
199        return None
200
201
202class UnifiedBuildFinder(UnifiedFinder):
203    """
204    Specialized UnifiedFinder for Mozilla applications packaging. It allows
205    "*.manifest" files to differ in their order, and unifies "buildconfig.html"
206    files by merging their content.
207    """
208
209    def __init__(self, finder1, finder2, **kargs):
210        UnifiedFinder.__init__(
211            self, finder1, finder2, sorted=["**/*.manifest"], **kargs
212        )
213
214    def unify_file(self, path, file1, file2):
215        """
216        Unify files taking Mozilla application special cases into account.
217        Otherwise defer to UnifiedFinder.unify_file.
218        """
219        basename = mozpath.basename(path)
220        if file1 and file2 and basename == "buildconfig.html":
221            content1 = file1.open().readlines()
222            content2 = file2.open().readlines()
223            # Copy everything from the first file up to the end of its <div>,
224            # insert a <hr> between the two files and copy the second file's
225            # content beginning after its leading <h1>.
226            return GeneratedFile(
227                b"".join(
228                    content1[: content1.index(b"    </div>\n")]
229                    + [b"      <hr> </hr>\n"]
230                    + content2[
231                        content2.index(b"      <h1>Build Configuration</h1>\n") + 1 :
232                    ]
233                )
234            )
235        elif file1 and file2 and basename == "install.rdf":
236            # install.rdf files often have em:targetPlatform (either as
237            # attribute or as tag) that will differ between platforms. The
238            # unified install.rdf should contain both em:targetPlatforms if
239            # they exist, or strip them if only one file has a target platform.
240            content1, content2 = (
241                FIND_TARGET_PLATFORM_ATTR.sub(
242                    lambda m: m.group("tag")
243                    + m.group("attrs")
244                    + m.group("otherattrs")
245                    + "<%stargetPlatform>%s</%stargetPlatform>"
246                    % (m.group("ns") or "", m.group("platform"), m.group("ns") or ""),
247                    f.open().read().decode("utf-8"),
248                )
249                for f in (file1, file2)
250            )
251
252            platform2 = FIND_TARGET_PLATFORM.search(content2)
253            return GeneratedFile(
254                FIND_TARGET_PLATFORM.sub(
255                    lambda m: m.group(0) + platform2.group(0) if platform2 else "",
256                    content1,
257                )
258            )
259        return UnifiedFinder.unify_file(self, path, file1, file2)
260