1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5'''jarmaker.py provides a python class to package up chrome content by 6processing jar.mn files. 7 8See the documentation for jar.mn on MDC for further details on the format. 9''' 10 11from __future__ import absolute_import 12 13import sys 14import os 15import errno 16import re 17import logging 18from time import localtime 19from MozZipFile import ZipFile 20from cStringIO import StringIO 21from collections import defaultdict 22 23from mozbuild.preprocessor import Preprocessor 24from mozbuild.action.buildlist import addEntriesToListFile 25from mozpack.files import FileFinder 26import mozpack.path as mozpath 27if sys.platform == 'win32': 28 from ctypes import windll, WinError 29 CreateHardLink = windll.kernel32.CreateHardLinkA 30 31__all__ = ['JarMaker'] 32 33 34class ZipEntry(object): 35 '''Helper class for jar output. 36 37 This class defines a simple file-like object for a zipfile.ZipEntry 38 so that we can consecutively write to it and then close it. 39 This methods hooks into ZipFile.writestr on close(). 40 ''' 41 42 def __init__(self, name, zipfile): 43 self._zipfile = zipfile 44 self._name = name 45 self._inner = StringIO() 46 47 def write(self, content): 48 '''Append the given content to this zip entry''' 49 50 self._inner.write(content) 51 return 52 53 def close(self): 54 '''The close method writes the content back to the zip file.''' 55 56 self._zipfile.writestr(self._name, self._inner.getvalue()) 57 58 59def getModTime(aPath): 60 if not os.path.isfile(aPath): 61 return 0 62 mtime = os.stat(aPath).st_mtime 63 return localtime(mtime) 64 65 66class JarManifestEntry(object): 67 def __init__(self, output, source, is_locale=False, preprocess=False): 68 self.output = output 69 self.source = source 70 self.is_locale = is_locale 71 self.preprocess = preprocess 72 73 74class JarInfo(object): 75 def __init__(self, base_or_jarinfo, name=None): 76 if name is None: 77 assert isinstance(base_or_jarinfo, JarInfo) 78 self.base = base_or_jarinfo.base 79 self.name = base_or_jarinfo.name 80 else: 81 assert not isinstance(base_or_jarinfo, JarInfo) 82 self.base = base_or_jarinfo or '' 83 self.name = name 84 # For compatibility with existing jar.mn files, if there is no 85 # base, the jar name is under chrome/ 86 if not self.base: 87 self.name = mozpath.join('chrome', self.name) 88 self.relativesrcdir = None 89 self.chrome_manifests = [] 90 self.entries = [] 91 92 93class DeprecatedJarManifest(Exception): pass 94 95 96class JarManifestParser(object): 97 98 ignore = re.compile('\s*(\#.*)?$') 99 jarline = re.compile(''' 100 (?: 101 (?:\[(?P<base>[\w\d.\-\_\\\/{}@]+)\]\s*)? # optional [base/path] 102 (?P<jarfile>[\w\d.\-\_\\\/{}]+).jar\: # filename.jar: 103 | 104 (?:\s*(\#.*)?) # comment 105 )\s*$ # whitespaces 106 ''', re.VERBOSE) 107 relsrcline = re.compile('relativesrcdir\s+(?P<relativesrcdir>.+?):') 108 regline = re.compile('\%\s+(.*)$') 109 entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+' 110 entryline = re.compile(entryre 111 + '(?P<output>[\w\d.\-\_\\\/\+\@]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/\@\*]+)\))?\s*$' 112 ) 113 114 def __init__(self): 115 self._current_jar = None 116 self._jars = [] 117 118 def write(self, line): 119 # A Preprocessor instance feeds the parser through calls to this method. 120 121 # Ignore comments and empty lines 122 if self.ignore.match(line): 123 return 124 125 # A jar manifest file can declare several different sections, each of 126 # which applies to a given "jar file". Each of those sections starts 127 # with "<name>.jar:", in which case the path is assumed relative to 128 # a "chrome" directory, or "[<base/path>] <subpath/name>.jar:", where 129 # a base directory is given (usually pointing at the root of the 130 # application or addon) and the jar path is given relative to the base 131 # directory. 132 if self._current_jar is None: 133 m = self.jarline.match(line) 134 if not m: 135 raise RuntimeError(line) 136 if m.group('jarfile'): 137 self._current_jar = JarInfo(m.group('base'), 138 m.group('jarfile')) 139 self._jars.append(self._current_jar) 140 return 141 142 # Within each section, there can be three different types of entries: 143 144 # - indications of the relative source directory we pretend to be in 145 # when considering localization files, in the following form; 146 # "relativesrcdir <path>:" 147 m = self.relsrcline.match(line) 148 if m: 149 if self._current_jar.chrome_manifests or self._current_jar.entries: 150 self._current_jar = JarInfo(self._current_jar) 151 self._jars.append(self._current_jar) 152 self._current_jar.relativesrcdir = m.group('relativesrcdir') 153 return 154 155 # - chrome manifest entries, prefixed with "%". 156 m = self.regline.match(line) 157 if m: 158 rline = ' '.join(m.group(1).split()) 159 if rline not in self._current_jar.chrome_manifests: 160 self._current_jar.chrome_manifests.append(rline) 161 return 162 163 # - entries indicating files to be part of the given jar. They are 164 # formed thusly: 165 # "<dest_path>" 166 # or 167 # "<dest_path> (<source_path>)" 168 # The <dest_path> is where the file(s) will be put in the chrome jar. 169 # The <source_path> is where the file(s) can be found in the source 170 # directory. The <source_path> may start with a "%" for files part 171 # of a localization directory, in which case the "%" counts as the 172 # locale. 173 # Each entry can be prefixed with "*" for preprocessing. 174 m = self.entryline.match(line) 175 if m: 176 if m.group('optOverwrite'): 177 raise DeprecatedJarManifest( 178 'The "+" prefix is not supported anymore') 179 self._current_jar.entries.append(JarManifestEntry( 180 m.group('output'), 181 m.group('source') or mozpath.basename(m.group('output')), 182 is_locale=bool(m.group('locale')), 183 preprocess=bool(m.group('optPreprocess')), 184 )) 185 return 186 187 self._current_jar = None 188 self.write(line) 189 190 def __iter__(self): 191 return iter(self._jars) 192 193 194class JarMaker(object): 195 '''JarMaker reads jar.mn files and process those into jar files or 196 flat directories, along with chrome.manifest files. 197 ''' 198 199 def __init__(self, outputFormat='flat', useJarfileManifest=True, 200 useChromeManifest=False): 201 202 self.outputFormat = outputFormat 203 self.useJarfileManifest = useJarfileManifest 204 self.useChromeManifest = useChromeManifest 205 self.pp = Preprocessor() 206 self.topsourcedir = None 207 self.sourcedirs = [] 208 self.localedirs = None 209 self.l10nbase = None 210 self.l10nmerge = None 211 self.relativesrcdir = None 212 self.rootManifestAppId = None 213 self._seen_output = set() 214 215 def getCommandLineParser(self): 216 '''Get a optparse.OptionParser for jarmaker. 217 218 This OptionParser has the options for jarmaker as well as 219 the options for the inner PreProcessor. 220 ''' 221 222 # HACK, we need to unescape the string variables we get, 223 # the perl versions didn't grok strings right 224 225 p = self.pp.getCommandLineParser(unescapeDefines=True) 226 p.add_option('-f', type='choice', default='jar', 227 choices=('jar', 'flat', 'symlink'), 228 help='fileformat used for output', 229 metavar='[jar, flat, symlink]', 230 ) 231 p.add_option('-v', action='store_true', dest='verbose', 232 help='verbose output') 233 p.add_option('-q', action='store_false', dest='verbose', 234 help='verbose output') 235 p.add_option('-e', action='store_true', 236 help='create chrome.manifest instead of jarfile.manifest' 237 ) 238 p.add_option('-s', type='string', action='append', default=[], 239 help='source directory') 240 p.add_option('-t', type='string', help='top source directory') 241 p.add_option('-c', '--l10n-src', type='string', action='append' 242 , help='localization directory') 243 p.add_option('--l10n-base', type='string', action='store', 244 help='base directory to be used for localization (requires relativesrcdir)' 245 ) 246 p.add_option('--locale-mergedir', type='string', action='store' 247 , 248 help='base directory to be used for l10n-merge (requires l10n-base and relativesrcdir)' 249 ) 250 p.add_option('--relativesrcdir', type='string', 251 help='relativesrcdir to be used for localization') 252 p.add_option('-d', type='string', help='base directory') 253 p.add_option('--root-manifest-entry-appid', type='string', 254 help='add an app id specific root chrome manifest entry.' 255 ) 256 return p 257 258 def finalizeJar(self, jardir, jarbase, jarname, chromebasepath, register, doZip=True): 259 '''Helper method to write out the chrome registration entries to 260 jarfile.manifest or chrome.manifest, or both. 261 262 The actual file processing is done in updateManifest. 263 ''' 264 265 # rewrite the manifest, if entries given 266 if not register: 267 return 268 269 chromeManifest = os.path.join(jardir, jarbase, 'chrome.manifest') 270 271 if self.useJarfileManifest: 272 self.updateManifest(os.path.join(jardir, jarbase, 273 jarname + '.manifest'), 274 chromebasepath.format(''), register) 275 if jarname != 'chrome': 276 addEntriesToListFile(chromeManifest, 277 ['manifest {0}.manifest'.format(jarname)]) 278 if self.useChromeManifest: 279 chromebase = os.path.dirname(jarname) + '/' 280 self.updateManifest(chromeManifest, 281 chromebasepath.format(chromebase), register) 282 283 # If requested, add a root chrome manifest entry (assumed to be in the parent directory 284 # of chromeManifest) with the application specific id. In cases where we're building 285 # lang packs, the root manifest must know about application sub directories. 286 287 if self.rootManifestAppId: 288 rootChromeManifest = \ 289 os.path.join(os.path.normpath(os.path.dirname(chromeManifest)), 290 '..', 'chrome.manifest') 291 rootChromeManifest = os.path.normpath(rootChromeManifest) 292 chromeDir = \ 293 os.path.basename(os.path.dirname(os.path.normpath(chromeManifest))) 294 logging.info("adding '%s' entry to root chrome manifest appid=%s" 295 % (chromeDir, self.rootManifestAppId)) 296 addEntriesToListFile(rootChromeManifest, 297 ['manifest %s/chrome.manifest application=%s' 298 % (chromeDir, 299 self.rootManifestAppId)]) 300 301 def updateManifest(self, manifestPath, chromebasepath, register): 302 '''updateManifest replaces the % in the chrome registration entries 303 with the given chrome base path, and updates the given manifest file. 304 ''' 305 myregister = dict.fromkeys(map(lambda s: s.replace('%', 306 chromebasepath), register)) 307 addEntriesToListFile(manifestPath, myregister.iterkeys()) 308 309 def makeJar(self, infile, jardir): 310 '''makeJar is the main entry point to JarMaker. 311 312 It takes the input file, the output directory, the source dirs and the 313 top source dir as argument, and optionally the l10n dirs. 314 ''' 315 316 # making paths absolute, guess srcdir if file and add to sourcedirs 317 _normpath = lambda p: os.path.normpath(os.path.abspath(p)) 318 self.topsourcedir = _normpath(self.topsourcedir) 319 self.sourcedirs = [_normpath(p) for p in self.sourcedirs] 320 if self.localedirs: 321 self.localedirs = [_normpath(p) for p in self.localedirs] 322 elif self.relativesrcdir: 323 self.localedirs = \ 324 self.generateLocaleDirs(self.relativesrcdir) 325 if isinstance(infile, basestring): 326 logging.info('processing ' + infile) 327 self.sourcedirs.append(_normpath(os.path.dirname(infile))) 328 pp = self.pp.clone() 329 pp.out = JarManifestParser() 330 pp.do_include(infile) 331 332 for info in pp.out: 333 self.processJarSection(info, jardir) 334 335 def generateLocaleDirs(self, relativesrcdir): 336 if os.path.basename(relativesrcdir) == 'locales': 337 # strip locales 338 l10nrelsrcdir = os.path.dirname(relativesrcdir) 339 else: 340 l10nrelsrcdir = relativesrcdir 341 locdirs = [] 342 343 # generate locales dirs, merge, l10nbase, en-US 344 if self.l10nmerge: 345 locdirs.append(os.path.join(self.l10nmerge, l10nrelsrcdir)) 346 if self.l10nbase: 347 locdirs.append(os.path.join(self.l10nbase, l10nrelsrcdir)) 348 if self.l10nmerge or not self.l10nbase: 349 # add en-US if we merge, or if it's not l10n 350 locdirs.append(os.path.join(self.topsourcedir, 351 relativesrcdir, 'en-US')) 352 return locdirs 353 354 def processJarSection(self, jarinfo, jardir): 355 '''Internal method called by makeJar to actually process a section 356 of a jar.mn file. 357 ''' 358 359 # chromebasepath is used for chrome registration manifests 360 # {0} is getting replaced with chrome/ for chrome.manifest, and with 361 # an empty string for jarfile.manifest 362 363 chromebasepath = '{0}' + os.path.basename(jarinfo.name) 364 if self.outputFormat == 'jar': 365 chromebasepath = 'jar:' + chromebasepath + '.jar!' 366 chromebasepath += '/' 367 368 jarfile = os.path.join(jardir, jarinfo.base, jarinfo.name) 369 jf = None 370 if self.outputFormat == 'jar': 371 # jar 372 jarfilepath = jarfile + '.jar' 373 try: 374 os.makedirs(os.path.dirname(jarfilepath)) 375 except OSError as error: 376 if error.errno != errno.EEXIST: 377 raise 378 jf = ZipFile(jarfilepath, 'a', lock=True) 379 outHelper = self.OutputHelper_jar(jf) 380 else: 381 outHelper = getattr(self, 'OutputHelper_' 382 + self.outputFormat)(jarfile) 383 384 if jarinfo.relativesrcdir: 385 self.localedirs = self.generateLocaleDirs(jarinfo.relativesrcdir) 386 387 for e in jarinfo.entries: 388 self._processEntryLine(e, outHelper, jf) 389 390 self.finalizeJar(jardir, jarinfo.base, jarinfo.name, chromebasepath, 391 jarinfo.chrome_manifests) 392 if jf is not None: 393 jf.close() 394 395 def _processEntryLine(self, e, outHelper, jf): 396 out = e.output 397 src = e.source 398 399 # pick the right sourcedir -- l10n, topsrc or src 400 401 if e.is_locale: 402 # If the file is a Fluent l10n resource, we want to skip the 403 # 'en-US' fallbacking. 404 # 405 # To achieve that, we're testing if we have more than one localedir, 406 # and if the last of those has 'en-US' in it. 407 # If that's the case, we're removing the last one. 408 if (e.source.endswith('.ftl') and 409 len(self.localedirs) > 1 and 410 'en-US' in self.localedirs[-1]): 411 src_base = self.localedirs[:-1] 412 else: 413 src_base = self.localedirs 414 elif src.startswith('/'): 415 # path/in/jar/file_name.xul (/path/in/sourcetree/file_name.xul) 416 # refers to a path relative to topsourcedir, use that as base 417 # and strip the leading '/' 418 src_base = [self.topsourcedir] 419 src = src[1:] 420 else: 421 # use srcdirs and the objdir (current working dir) for relative paths 422 src_base = self.sourcedirs + [os.getcwd()] 423 424 if '*' in src: 425 def _prefix(s): 426 for p in s.split('/'): 427 if '*' not in p: 428 yield p + '/' 429 prefix = ''.join(_prefix(src)) 430 emitted = set() 431 for _srcdir in src_base: 432 finder = FileFinder(_srcdir) 433 for path, _ in finder.find(src): 434 # If the path was already seen in one of the other source 435 # directories, skip it. That matches the non-wildcard case 436 # below, where we pick the first existing file. 437 reduced_path = path[len(prefix):] 438 if reduced_path in emitted: 439 continue 440 emitted.add(reduced_path) 441 e = JarManifestEntry( 442 mozpath.join(out, reduced_path), 443 path, 444 is_locale=e.is_locale, 445 preprocess=e.preprocess, 446 ) 447 self._processEntryLine(e, outHelper, jf) 448 return 449 450 # check if the source file exists 451 realsrc = None 452 for _srcdir in src_base: 453 if os.path.isfile(os.path.join(_srcdir, src)): 454 realsrc = os.path.join(_srcdir, src) 455 break 456 if realsrc is None: 457 if jf is not None: 458 jf.close() 459 raise RuntimeError('File "{0}" not found in {1}'.format(src, 460 ', '.join(src_base))) 461 462 if out in self._seen_output: 463 raise RuntimeError('%s already added' % out) 464 self._seen_output.add(out) 465 466 if e.preprocess: 467 outf = outHelper.getOutput(out) 468 inf = open(realsrc) 469 pp = self.pp.clone() 470 if src[-4:] == '.css': 471 pp.setMarker('%') 472 pp.out = outf 473 pp.do_include(inf) 474 pp.failUnused(realsrc) 475 outf.close() 476 inf.close() 477 return 478 479 # copy or symlink if newer 480 481 if getModTime(realsrc) > outHelper.getDestModTime(e.output): 482 if self.outputFormat == 'symlink': 483 outHelper.symlink(realsrc, out) 484 return 485 outf = outHelper.getOutput(out) 486 487 # open in binary mode, this can be images etc 488 489 inf = open(realsrc, 'rb') 490 outf.write(inf.read()) 491 outf.close() 492 inf.close() 493 494 class OutputHelper_jar(object): 495 '''Provide getDestModTime and getOutput for a given jarfile.''' 496 497 def __init__(self, jarfile): 498 self.jarfile = jarfile 499 500 def getDestModTime(self, aPath): 501 try: 502 info = self.jarfile.getinfo(aPath) 503 return info.date_time 504 except: 505 return 0 506 507 def getOutput(self, name): 508 return ZipEntry(name, self.jarfile) 509 510 class OutputHelper_flat(object): 511 '''Provide getDestModTime and getOutput for a given flat 512 output directory. The helper method ensureDirFor is used by 513 the symlink subclass. 514 ''' 515 516 def __init__(self, basepath): 517 self.basepath = basepath 518 519 def getDestModTime(self, aPath): 520 return getModTime(os.path.join(self.basepath, aPath)) 521 522 def getOutput(self, name): 523 out = self.ensureDirFor(name) 524 525 # remove previous link or file 526 try: 527 os.remove(out) 528 except OSError as e: 529 if e.errno != errno.ENOENT: 530 raise 531 return open(out, 'wb') 532 533 def ensureDirFor(self, name): 534 out = os.path.join(self.basepath, name) 535 outdir = os.path.dirname(out) 536 if not os.path.isdir(outdir): 537 try: 538 os.makedirs(outdir) 539 except OSError as error: 540 if error.errno != errno.EEXIST: 541 raise 542 return out 543 544 class OutputHelper_symlink(OutputHelper_flat): 545 '''Subclass of OutputHelper_flat that provides a helper for 546 creating a symlink including creating the parent directories. 547 ''' 548 549 def symlink(self, src, dest): 550 out = self.ensureDirFor(dest) 551 552 # remove previous link or file 553 try: 554 os.remove(out) 555 except OSError as e: 556 if e.errno != errno.ENOENT: 557 raise 558 if sys.platform != 'win32': 559 os.symlink(src, out) 560 else: 561 # On Win32, use ctypes to create a hardlink 562 rv = CreateHardLink(out, src, None) 563 if rv == 0: 564 raise WinError() 565 566 567def main(args=None): 568 args = args or sys.argv 569 jm = JarMaker() 570 p = jm.getCommandLineParser() 571 (options, args) = p.parse_args(args) 572 jm.outputFormat = options.f 573 jm.sourcedirs = options.s 574 jm.topsourcedir = options.t 575 if options.e: 576 jm.useChromeManifest = True 577 jm.useJarfileManifest = False 578 if options.l10n_base: 579 if not options.relativesrcdir: 580 p.error('relativesrcdir required when using l10n-base') 581 if options.l10n_src: 582 p.error('both l10n-src and l10n-base are not supported') 583 jm.l10nbase = options.l10n_base 584 jm.relativesrcdir = options.relativesrcdir 585 jm.l10nmerge = options.locale_mergedir 586 if jm.l10nmerge and not os.path.isdir(jm.l10nmerge): 587 logging.warning("WARNING: --locale-mergedir passed, but '%s' does not exist. " 588 "Ignore this message if the locale is complete." % jm.l10nmerge) 589 elif options.locale_mergedir: 590 p.error('l10n-base required when using locale-mergedir') 591 jm.localedirs = options.l10n_src 592 if options.root_manifest_entry_appid: 593 jm.rootManifestAppId = options.root_manifest_entry_appid 594 noise = logging.INFO 595 if options.verbose is not None: 596 noise = options.verbose and logging.DEBUG or logging.WARN 597 if sys.version_info[:2] > (2, 3): 598 logging.basicConfig(format='%(message)s') 599 else: 600 logging.basicConfig() 601 logging.getLogger().setLevel(noise) 602 topsrc = options.t 603 topsrc = os.path.normpath(os.path.abspath(topsrc)) 604 if not args: 605 infile = sys.stdin 606 else: 607 (infile, ) = args 608 jm.makeJar(infile, options.d) 609