1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, # You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function, unicode_literals 6 7from distutils.version import LooseVersion 8import logging 9from mozbuild.base import ( 10 BuildEnvironmentNotFoundException, 11 MozbuildObject, 12) 13import mozfile 14import mozpack.path as mozpath 15import os 16import requests 17import re 18import sys 19import tarfile 20from urlparse import urlparse 21 22class VendorAOM(MozbuildObject): 23 def upstream_snapshot(self, revision): 24 '''Construct a url for a tarball snapshot of the given revision.''' 25 if 'googlesource' in self.repo_url: 26 return mozpath.join(self.repo_url, '+archive', revision + '.tar.gz') 27 elif 'github' in self.repo_url: 28 return mozpath.join(self.repo_url, 'archive', revision + '.tar.gz') 29 else: 30 raise ValueError('Unknown git host, no snapshot lookup method') 31 32 def upstream_commit(self, revision): 33 '''Convert a revision to a git commit and timestamp. 34 35 Ask the upstream repo to convert the requested revision to 36 a git commit id and timestamp, so we can be precise in 37 what we're vendoring.''' 38 if 'googlesource' in self.repo_url: 39 return self.upstream_googlesource_commit(revision) 40 elif 'github' in self.repo_url: 41 return self.upstream_github_commit(revision) 42 else: 43 raise ValueError('Unknown git host, no commit lookup method') 44 45 def upstream_validate(self, url): 46 '''Validate repository urls to make sure we can handle them.''' 47 host = urlparse(url).netloc 48 valid_domains = ('googlesource.com', 'github.com') 49 if not any(filter(lambda domain: domain in host, valid_domains)): 50 self.log(logging.ERROR, 'upstream_url', {}, 51 '''Unsupported git host %s; cannot fetch snapshots. 52 53Please set a repository url with --repo on either googlesource or github.''' % host) 54 sys.exit(1) 55 56 def upstream_googlesource_commit(self, revision): 57 '''Query gitiles for a git commit and timestamp.''' 58 url = mozpath.join(self.repo_url, '+', revision + '?format=JSON') 59 self.log(logging.INFO, 'fetch', {'url': url}, 60 'Fetching commit id from {url}') 61 req = requests.get(url) 62 req.raise_for_status() 63 try: 64 info = req.json() 65 except ValueError as e: 66 if 'No JSON object' in e.message: 67 # As of 2017 May, googlesource sends 4 garbage characters 68 # at the beginning of the json response. Work around this. 69 # https://bugs.chromium.org/p/chromium/issues/detail?id=718550 70 import json 71 info = json.loads(req.text[4:]) 72 else: 73 raise 74 return (info['commit'], info['committer']['time']) 75 76 def upstream_github_commit(self, revision): 77 '''Query the github api for a git commit id and timestamp.''' 78 github_api = 'https://api.github.com/' 79 repo = urlparse(self.repo_url).path[1:] 80 url = mozpath.join(github_api, 'repos', repo, 'commits', revision) 81 self.log(logging.INFO, 'fetch', {'url': url}, 82 'Fetching commit id from {url}') 83 req = requests.get(url) 84 req.raise_for_status() 85 info = req.json() 86 return (info['sha'], info['commit']['committer']['date']) 87 88 def fetch_and_unpack(self, revision, target): 89 '''Fetch and unpack upstream source''' 90 url = self.upstream_snapshot(revision) 91 self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}') 92 prefix = 'aom-' + revision 93 filename = prefix + '.tar.gz' 94 with open(filename, 'wb') as f: 95 req = requests.get(url, stream=True) 96 for data in req.iter_content(4096): 97 f.write(data) 98 tar = tarfile.open(filename) 99 bad_paths = filter(lambda name: name.startswith('/') or '..' in name, 100 tar.getnames()) 101 if any(bad_paths): 102 raise Exception("Tar archive contains non-local paths," 103 "e.g. '%s'" % bad_paths[0]) 104 self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target) 105 mozfile.remove(target) 106 self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.') 107 tar.extractall(target) 108 # Github puts everything properly down a directory; move it up. 109 if all(map(lambda name: name.startswith(prefix), tar.getnames())): 110 tardir = mozpath.join(target, prefix) 111 os.system('mv %s/* %s/.* %s' % (tardir, tardir, target)) 112 os.rmdir(tardir) 113 # Remove the tarball. 114 mozfile.remove(filename) 115 116 def update_readme(self, revision, timestamp, target): 117 filename = mozpath.join(target, 'README_MOZILLA') 118 with open(filename) as f: 119 readme = f.read() 120 121 prefix = 'The git commit ID used was' 122 if prefix in readme: 123 new_readme = re.sub(prefix + ' [v\.a-f0-9]+.*$', 124 prefix + ' %s (%s).' % (revision, timestamp), 125 readme) 126 else: 127 new_readme = '%s\n\n%s %s.' % (readme, prefix, revision) 128 129 prefix = 'The last update was pulled from' 130 new_readme = re.sub(prefix + ' https*://.*', 131 prefix + ' %s' % self.repo_url, 132 new_readme) 133 134 if readme != new_readme: 135 with open(filename, 'w') as f: 136 f.write(new_readme) 137 138 def update_mimetype(self, revision): 139 '''Update source tree references to the aom revision. 140 141 While the av1 bitstream is unstable, we track the git revision 142 of the reference implementation we're building, and are careful 143 to build with default feature flags. This lets us answer whether 144 a particular bitstream will be playable by comparing the encode- 145 side hash as part of the mime type. 146 ''' 147 filename = mozpath.join(self.topsrcdir, 148 'dom/media/platforms/agnostic/AOMDecoder.cpp') 149 with open(filename) as f: 150 source = f.read() 151 152 new_source = '' 153 pattern = re.compile('version.AppendLiteral\("([a-f0-9]{40})"\);') 154 match = pattern.search(source) 155 if match: 156 old_revision = match.group(1) 157 if old_revision == revision: 158 # Nothing to update. 159 return 160 new_source = pattern.sub('version.AppendLiteral("%s");' % revision, 161 source) 162 if not match or new_source == source: 163 self.log(logging.ERROR, 'hash_update', {}, 164 '''Couldn't update commit hash in 165 {file}. 166Please check manually and update the vendor script. 167 '''.format(file=filename)) 168 sys.exit(1) 169 170 with open(filename, 'w') as f: 171 f.write(new_source) 172 173 174 def clean_upstream(self, target): 175 '''Remove files we don't want to import.''' 176 mozfile.remove(mozpath.join(target, '.gitattributes')) 177 mozfile.remove(mozpath.join(target, '.gitignore')) 178 mozfile.remove(mozpath.join(target, 'build', '.gitattributes')) 179 mozfile.remove(mozpath.join(target, 'build' ,'.gitignore')) 180 181 def generate_sources(self, target): 182 ''' 183 Run the library's native build system to update ours. 184 185 Invoke configure for each supported platform to generate 186 appropriate config and header files, then invoke the 187 makefile to obtain a list of source files, writing 188 these out in the appropriate format for our build 189 system to use. 190 ''' 191 config_dir = mozpath.join(target, 'config') 192 self.log(logging.INFO, 'rm_confg_dir', {}, 'rm -rf %s' % config_dir) 193 mozfile.remove(config_dir) 194 self.run_process(args=['./generate_sources_mozbuild.sh'], 195 cwd=target, log_name='generate_sources') 196 197 def check_modified_files(self): 198 ''' 199 Ensure that there aren't any uncommitted changes to files 200 in the working copy, since we're going to change some state 201 on the user. 202 ''' 203 modified = self.repository.get_changed_files('M') 204 if modified: 205 self.log(logging.ERROR, 'modified_files', {}, 206 '''You have uncommitted changes to the following files: 207 208{files} 209 210Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`. 211'''.format(files='\n'.join(sorted(modified)))) 212 sys.exit(1) 213 214 def vendor(self, revision, repo, ignore_modified=False): 215 self.populate_logger() 216 self.log_manager.enable_unstructured() 217 218 if not ignore_modified: 219 self.check_modified_files() 220 if not revision: 221 revision = 'master' 222 if repo: 223 self.repo_url = repo 224 else: 225 self.repo_url = 'https://aomedia.googlesource.com/aom/' 226 self.upstream_validate(self.repo_url) 227 228 commit, timestamp = self.upstream_commit(revision) 229 230 vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom') 231 self.fetch_and_unpack(commit, vendor_dir) 232 self.log(logging.INFO, 'clean_upstream', {}, 233 '''Removing unnecessary files.''') 234 self.clean_upstream(vendor_dir) 235 glue_dir = mozpath.join(self.topsrcdir, 'media/libaom') 236 self.log(logging.INFO, 'generate_sources', {}, 237 '''Generating build files...''') 238 self.generate_sources(glue_dir) 239 self.log(logging.INFO, 'update_source', {}, 240 '''Updating mimetype extension.''') 241 self.update_mimetype(commit) 242 self.log(logging.INFO, 'update_readme', {}, 243 '''Updating README_MOZILLA.''') 244 self.update_readme(commit, timestamp, glue_dir) 245 self.repository.add_remove_files(vendor_dir) 246 self.log(logging.INFO, 'add_remove_files', {}, 247 '''Registering changes with version control.''') 248 self.repository.add_remove_files(vendor_dir) 249 self.repository.add_remove_files(glue_dir) 250 self.log(logging.INFO, 'done', {'revision': revision}, 251 '''Update to aom version '{revision}' ready to commit.''') 252