1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
4
5from __future__ import absolute_import, print_function, unicode_literals
6
7from distutils.version import LooseVersion
8import logging
9from mozbuild.base import (
10    BuildEnvironmentNotFoundException,
11    MozbuildObject,
12)
13import mozfile
14import mozpack.path as mozpath
15import os
16import requests
17import re
18import sys
19import tarfile
20from urlparse import urlparse
21
22class VendorAOM(MozbuildObject):
23    def upstream_snapshot(self, revision):
24        '''Construct a url for a tarball snapshot of the given revision.'''
25        if 'googlesource' in self.repo_url:
26            return mozpath.join(self.repo_url, '+archive', revision + '.tar.gz')
27        elif 'github' in self.repo_url:
28            return mozpath.join(self.repo_url, 'archive', revision + '.tar.gz')
29        else:
30            raise ValueError('Unknown git host, no snapshot lookup method')
31
32    def upstream_commit(self, revision):
33        '''Convert a revision to a git commit and timestamp.
34
35        Ask the upstream repo to convert the requested revision to
36        a git commit id and timestamp, so we can be precise in
37        what we're vendoring.'''
38        if 'googlesource' in self.repo_url:
39            return self.upstream_googlesource_commit(revision)
40        elif 'github' in self.repo_url:
41            return self.upstream_github_commit(revision)
42        else:
43            raise ValueError('Unknown git host, no commit lookup method')
44
45    def upstream_validate(self, url):
46        '''Validate repository urls to make sure we can handle them.'''
47        host = urlparse(url).netloc
48        valid_domains = ('googlesource.com', 'github.com')
49        if not any(filter(lambda domain: domain in host, valid_domains)):
50            self.log(logging.ERROR, 'upstream_url', {},
51                     '''Unsupported git host %s; cannot fetch snapshots.
52
53Please set a repository url with --repo on either googlesource or github.''' % host)
54            sys.exit(1)
55
56    def upstream_googlesource_commit(self, revision):
57        '''Query gitiles for a git commit and timestamp.'''
58        url = mozpath.join(self.repo_url, '+', revision + '?format=JSON')
59        self.log(logging.INFO, 'fetch', {'url': url},
60                 'Fetching commit id from {url}')
61        req = requests.get(url)
62        req.raise_for_status()
63        try:
64            info = req.json()
65        except ValueError as e:
66            if 'No JSON object' in e.message:
67                # As of 2017 May, googlesource sends 4 garbage characters
68                # at the beginning of the json response. Work around this.
69                # https://bugs.chromium.org/p/chromium/issues/detail?id=718550
70                import json
71                info = json.loads(req.text[4:])
72            else:
73                raise
74        return (info['commit'], info['committer']['time'])
75
76    def upstream_github_commit(self, revision):
77        '''Query the github api for a git commit id and timestamp.'''
78        github_api = 'https://api.github.com/'
79        repo = urlparse(self.repo_url).path[1:]
80        url = mozpath.join(github_api, 'repos', repo, 'commits', revision)
81        self.log(logging.INFO, 'fetch', {'url': url},
82                 'Fetching commit id from {url}')
83        req = requests.get(url)
84        req.raise_for_status()
85        info = req.json()
86        return (info['sha'], info['commit']['committer']['date'])
87
88    def fetch_and_unpack(self, revision, target):
89        '''Fetch and unpack upstream source'''
90        url = self.upstream_snapshot(revision)
91        self.log(logging.INFO, 'fetch', {'url': url}, 'Fetching {url}')
92        prefix = 'aom-' + revision
93        filename = prefix + '.tar.gz'
94        with open(filename, 'wb') as f:
95            req = requests.get(url, stream=True)
96            for data in req.iter_content(4096):
97                f.write(data)
98        tar = tarfile.open(filename)
99        bad_paths = filter(lambda name: name.startswith('/') or '..' in name,
100                           tar.getnames())
101        if any(bad_paths):
102            raise Exception("Tar archive contains non-local paths,"
103                            "e.g. '%s'" % bad_paths[0])
104        self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % target)
105        mozfile.remove(target)
106        self.log(logging.INFO, 'unpack', {}, 'Unpacking upstream files.')
107        tar.extractall(target)
108        # Github puts everything properly down a directory; move it up.
109        if all(map(lambda name: name.startswith(prefix), tar.getnames())):
110            tardir = mozpath.join(target, prefix)
111            os.system('mv %s/* %s/.* %s' % (tardir, tardir, target))
112            os.rmdir(tardir)
113        # Remove the tarball.
114        mozfile.remove(filename)
115
116    def update_readme(self, revision, timestamp, target):
117        filename = mozpath.join(target, 'README_MOZILLA')
118        with open(filename) as f:
119            readme = f.read()
120
121        prefix = 'The git commit ID used was'
122        if prefix in readme:
123            new_readme = re.sub(prefix + ' [v\.a-f0-9]+.*$',
124                                prefix + ' %s (%s).' % (revision, timestamp),
125                                readme)
126        else:
127            new_readme = '%s\n\n%s %s.' % (readme, prefix, revision)
128
129        prefix = 'The last update was pulled from'
130        new_readme = re.sub(prefix + ' https*://.*',
131                            prefix + ' %s' % self.repo_url,
132                            new_readme)
133
134        if readme != new_readme:
135            with open(filename, 'w') as f:
136                f.write(new_readme)
137
138    def update_mimetype(self, revision):
139        '''Update source tree references to the aom revision.
140
141        While the av1 bitstream is unstable, we track the git revision
142        of the reference implementation we're building, and are careful
143        to build with default feature flags. This lets us answer whether
144        a particular bitstream will be playable by comparing the encode-
145        side hash as part of the mime type.
146        '''
147        filename = mozpath.join(self.topsrcdir,
148                'dom/media/platforms/agnostic/AOMDecoder.cpp')
149        with open(filename) as f:
150            source = f.read()
151
152        new_source = ''
153        pattern = re.compile('version.AppendLiteral\("([a-f0-9]{40})"\);')
154        match = pattern.search(source)
155        if match:
156            old_revision = match.group(1)
157            if old_revision == revision:
158                # Nothing to update.
159                return
160            new_source = pattern.sub('version.AppendLiteral("%s");' % revision,
161                                     source)
162        if not match or new_source == source:
163            self.log(logging.ERROR, 'hash_update', {},
164                     '''Couldn't update commit hash in
165    {file}.
166Please check manually and update the vendor script.
167                     '''.format(file=filename))
168            sys.exit(1)
169
170        with open(filename, 'w') as f:
171            f.write(new_source)
172
173
174    def clean_upstream(self, target):
175        '''Remove files we don't want to import.'''
176        mozfile.remove(mozpath.join(target, '.gitattributes'))
177        mozfile.remove(mozpath.join(target, '.gitignore'))
178        mozfile.remove(mozpath.join(target, 'build', '.gitattributes'))
179        mozfile.remove(mozpath.join(target, 'build' ,'.gitignore'))
180
181    def generate_sources(self, target):
182        '''
183        Run the library's native build system to update ours.
184
185        Invoke configure for each supported platform to generate
186        appropriate config and header files, then invoke the
187        makefile to obtain a list of source files, writing
188        these out in the appropriate format for our build
189        system to use.
190        '''
191        config_dir = mozpath.join(target, 'config')
192        self.log(logging.INFO, 'rm_confg_dir', {}, 'rm -rf %s' % config_dir)
193        mozfile.remove(config_dir)
194        self.run_process(args=['./generate_sources_mozbuild.sh'],
195                         cwd=target, log_name='generate_sources')
196
197    def check_modified_files(self):
198        '''
199        Ensure that there aren't any uncommitted changes to files
200        in the working copy, since we're going to change some state
201        on the user.
202        '''
203        modified = self.repository.get_changed_files('M')
204        if modified:
205            self.log(logging.ERROR, 'modified_files', {},
206                     '''You have uncommitted changes to the following files:
207
208{files}
209
210Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`.
211'''.format(files='\n'.join(sorted(modified))))
212            sys.exit(1)
213
214    def vendor(self, revision, repo, ignore_modified=False):
215        self.populate_logger()
216        self.log_manager.enable_unstructured()
217
218        if not ignore_modified:
219            self.check_modified_files()
220        if not revision:
221            revision = 'master'
222        if repo:
223            self.repo_url = repo
224        else:
225            self.repo_url = 'https://aomedia.googlesource.com/aom/'
226        self.upstream_validate(self.repo_url)
227
228        commit, timestamp = self.upstream_commit(revision)
229
230        vendor_dir = mozpath.join(self.topsrcdir, 'third_party/aom')
231        self.fetch_and_unpack(commit, vendor_dir)
232        self.log(logging.INFO, 'clean_upstream', {},
233                 '''Removing unnecessary files.''')
234        self.clean_upstream(vendor_dir)
235        glue_dir = mozpath.join(self.topsrcdir, 'media/libaom')
236        self.log(logging.INFO, 'generate_sources', {},
237                 '''Generating build files...''')
238        self.generate_sources(glue_dir)
239        self.log(logging.INFO, 'update_source', {},
240                 '''Updating mimetype extension.''')
241        self.update_mimetype(commit)
242        self.log(logging.INFO, 'update_readme', {},
243                 '''Updating README_MOZILLA.''')
244        self.update_readme(commit, timestamp, glue_dir)
245        self.repository.add_remove_files(vendor_dir)
246        self.log(logging.INFO, 'add_remove_files', {},
247                 '''Registering changes with version control.''')
248        self.repository.add_remove_files(vendor_dir)
249        self.repository.add_remove_files(glue_dir)
250        self.log(logging.INFO, 'done', {'revision': revision},
251                 '''Update to aom version '{revision}' ready to commit.''')
252