1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from mozpack.files import ( 6 BaseFinder, 7 ExecutableFile, 8 BaseFile, 9 GeneratedFile, 10) 11from mozpack.executables import ( 12 MACHO_SIGNATURES, 13) 14from mozpack.errors import errors 15from mozbuild.util import hexdump 16from tempfile import mkstemp 17import mozpack.path as mozpath 18import struct 19import os 20import re 21import subprocess 22import buildconfig 23 24# Regular expressions for unifying install.rdf 25FIND_TARGET_PLATFORM = re.compile( 26 r""" 27 <(?P<ns>[-._0-9A-Za-z]+:)?targetPlatform> # The targetPlatform tag, with any namespace 28 (?P<platform>[^<]*) # The actual platform value 29 </(?P=ns)?targetPlatform> # The closing tag 30 """, 31 re.X, 32) 33FIND_TARGET_PLATFORM_ATTR = re.compile( 34 r""" 35 (?P<tag><(?:[-._0-9A-Za-z]+:)?Description) # The opening part of the <Description> tag 36 (?P<attrs>[^>]*?)\s+ # The initial attributes 37 (?P<ns>[-._0-9A-Za-z]+:)?targetPlatform= # The targetPlatform attribute, with any namespace 38 [\'"](?P<platform>[^\'"]+)[\'"] # The actual platform value 39 (?P<otherattrs>[^>]*?>) # The remaining attributes and closing angle bracket 40 """, 41 re.X, 42) 43 44 45def may_unify_binary(file): 46 """ 47 Return whether the given BaseFile instance is an ExecutableFile that 48 may be unified. Only non-fat Mach-O binaries are to be unified. 49 """ 50 if isinstance(file, ExecutableFile): 51 signature = file.open().read(4) 52 if len(signature) < 4: 53 return False 54 signature = struct.unpack(">L", signature)[0] 55 if signature in MACHO_SIGNATURES: 56 return True 57 return False 58 59 60class UnifiedExecutableFile(BaseFile): 61 """ 62 File class for executable and library files that to be unified with 'lipo'. 63 """ 64 65 def __init__(self, executable1, executable2): 66 """ 67 Initialize a UnifiedExecutableFile with a pair of ExecutableFiles to 68 be unified. They are expected to be non-fat Mach-O executables. 69 """ 70 assert isinstance(executable1, ExecutableFile) 71 assert isinstance(executable2, ExecutableFile) 72 self._executables = (executable1, executable2) 73 74 def copy(self, dest, skip_if_older=True): 75 """ 76 Create a fat executable from the two Mach-O executable given when 77 creating the instance. 78 skip_if_older is ignored. 79 """ 80 assert isinstance(dest, str) 81 tmpfiles = [] 82 try: 83 for e in self._executables: 84 fd, f = mkstemp() 85 os.close(fd) 86 tmpfiles.append(f) 87 e.copy(f, skip_if_older=False) 88 lipo = buildconfig.substs.get("LIPO") or "lipo" 89 subprocess.check_call([lipo, "-create"] + tmpfiles + ["-output", dest]) 90 except Exception as e: 91 errors.error( 92 "Failed to unify %s and %s: %s" 93 % (self._executables[0].path, self._executables[1].path, str(e)) 94 ) 95 finally: 96 for f in tmpfiles: 97 os.unlink(f) 98 99 100class UnifiedFinder(BaseFinder): 101 """ 102 Helper to get unified BaseFile instances from two distinct trees on the 103 file system. 104 """ 105 106 def __init__(self, finder1, finder2, sorted=[], **kargs): 107 """ 108 Initialize a UnifiedFinder. finder1 and finder2 are BaseFinder 109 instances from which files are picked. UnifiedFinder.find() will act as 110 FileFinder.find() but will error out when matches can only be found in 111 one of the two trees and not the other. It will also error out if 112 matches can be found on both ends but their contents are not identical. 113 114 The sorted argument gives a list of mozpath.match patterns. File 115 paths matching one of these patterns will have their contents compared 116 with their lines sorted. 117 """ 118 assert isinstance(finder1, BaseFinder) 119 assert isinstance(finder2, BaseFinder) 120 self._finder1 = finder1 121 self._finder2 = finder2 122 self._sorted = sorted 123 BaseFinder.__init__(self, finder1.base, **kargs) 124 125 def _find(self, path): 126 """ 127 UnifiedFinder.find() implementation. 128 """ 129 files1 = {p: f for p, f in self._finder1.find(path)} 130 files2 = {p: f for p, f in self._finder2.find(path)} 131 all_paths = set(files1) | set(files2) 132 for p in sorted(all_paths): 133 err = errors.count 134 unified = self.unify_file(p, files1.get(p), files2.get(p)) 135 if unified: 136 yield p, unified 137 elif err == errors.count: # No errors have already been reported. 138 self._report_difference(p, files1.get(p), files2.get(p)) 139 140 def _report_difference(self, path, file1, file2): 141 """ 142 Report differences between files in both trees. 143 """ 144 if not file1: 145 errors.error("File missing in %s: %s" % (self._finder1.base, path)) 146 return 147 if not file2: 148 errors.error("File missing in %s: %s" % (self._finder2.base, path)) 149 return 150 151 errors.error( 152 "Can't unify %s: file differs between %s and %s" 153 % (path, self._finder1.base, self._finder2.base) 154 ) 155 if not isinstance(file1, ExecutableFile) and not isinstance( 156 file2, ExecutableFile 157 ): 158 from difflib import unified_diff 159 160 try: 161 lines1 = [l.decode("utf-8") for l in file1.open().readlines()] 162 lines2 = [l.decode("utf-8") for l in file2.open().readlines()] 163 except UnicodeDecodeError: 164 lines1 = hexdump(file1.open().read()) 165 lines2 = hexdump(file2.open().read()) 166 167 for line in unified_diff( 168 lines1, 169 lines2, 170 os.path.join(self._finder1.base, path), 171 os.path.join(self._finder2.base, path), 172 ): 173 errors.out.write(line) 174 175 def unify_file(self, path, file1, file2): 176 """ 177 Given two BaseFiles and the path they were found at, return a 178 unified version of the files. If the files match, the first BaseFile 179 may be returned. 180 If the files don't match or one of them is `None`, the method returns 181 `None`. 182 Subclasses may decide to unify by using one of the files in that case. 183 """ 184 if not file1 or not file2: 185 return None 186 187 if may_unify_binary(file1) and may_unify_binary(file2): 188 return UnifiedExecutableFile(file1, file2) 189 190 content1 = file1.open().readlines() 191 content2 = file2.open().readlines() 192 if content1 == content2: 193 return file1 194 for pattern in self._sorted: 195 if mozpath.match(path, pattern): 196 if sorted(content1) == sorted(content2): 197 return file1 198 break 199 return None 200 201 202class UnifiedBuildFinder(UnifiedFinder): 203 """ 204 Specialized UnifiedFinder for Mozilla applications packaging. It allows 205 "*.manifest" files to differ in their order, and unifies "buildconfig.html" 206 files by merging their content. 207 """ 208 209 def __init__(self, finder1, finder2, **kargs): 210 UnifiedFinder.__init__( 211 self, finder1, finder2, sorted=["**/*.manifest"], **kargs 212 ) 213 214 def unify_file(self, path, file1, file2): 215 """ 216 Unify files taking Mozilla application special cases into account. 217 Otherwise defer to UnifiedFinder.unify_file. 218 """ 219 basename = mozpath.basename(path) 220 if file1 and file2 and basename == "buildconfig.html": 221 content1 = file1.open().readlines() 222 content2 = file2.open().readlines() 223 # Copy everything from the first file up to the end of its <div>, 224 # insert a <hr> between the two files and copy the second file's 225 # content beginning after its leading <h1>. 226 return GeneratedFile( 227 b"".join( 228 content1[: content1.index(b" </div>\n")] 229 + [b" <hr> </hr>\n"] 230 + content2[ 231 content2.index(b" <h1>Build Configuration</h1>\n") + 1 : 232 ] 233 ) 234 ) 235 elif file1 and file2 and basename == "install.rdf": 236 # install.rdf files often have em:targetPlatform (either as 237 # attribute or as tag) that will differ between platforms. The 238 # unified install.rdf should contain both em:targetPlatforms if 239 # they exist, or strip them if only one file has a target platform. 240 content1, content2 = ( 241 FIND_TARGET_PLATFORM_ATTR.sub( 242 lambda m: m.group("tag") 243 + m.group("attrs") 244 + m.group("otherattrs") 245 + "<%stargetPlatform>%s</%stargetPlatform>" 246 % (m.group("ns") or "", m.group("platform"), m.group("ns") or ""), 247 f.open().read().decode("utf-8"), 248 ) 249 for f in (file1, file2) 250 ) 251 252 platform2 = FIND_TARGET_PLATFORM.search(content2) 253 return GeneratedFile( 254 FIND_TARGET_PLATFORM.sub( 255 lambda m: m.group(0) + platform2.group(0) if platform2 else "", 256 content1, 257 ) 258 ) 259 return UnifiedFinder.unify_file(self, path, file1, file2) 260