1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5'''
6Use pywatchman to watch source directories and perform partial |mach
7build faster| builds.
8'''
9
10from __future__ import absolute_import, print_function, unicode_literals
11
12import datetime
13import sys
14import time
15
16import mozbuild.util
17import mozpack.path as mozpath
18from mozpack.manifests import (
19    InstallManifest,
20)
21from mozpack.copier import (
22    FileCopier,
23)
24from mozbuild.backend import (
25    get_backend_class,
26)
27
28# Watchman integration cribbed entirely from
29# https://github.com/facebook/watchman/blob/19aebfebb0b5b0b5174b3914a879370ffc5dac37/python/bin/watchman-wait
30import pywatchman
31
32
33def print_line(prefix, m, now=None):
34    now = now or datetime.datetime.utcnow()
35    print(b'[%s %sZ] %s' % (prefix, now.isoformat(), m))
36
37
38def print_copy_result(elapsed, destdir, result, verbose=True):
39    COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
40        'Added/updated {updated}; ' \
41        'Removed {rm_files} files and {rm_dirs} directories.'
42
43    print_line('watch', COMPLETE.format(
44        elapsed=elapsed,
45        dest=destdir,
46        existing=result.existing_files_count,
47        updated=result.updated_files_count,
48        rm_files=result.removed_files_count,
49        rm_dirs=result.removed_directories_count))
50
51
52class FasterBuildException(Exception):
53    def __init__(self, message, cause):
54        Exception.__init__(self, message)
55        self.cause = cause
56
57
58class FasterBuildChange(object):
59    def __init__(self):
60        self.unrecognized = set()
61        self.input_to_outputs = {}
62        self.output_to_inputs = {}
63
64
65class Daemon(object):
66    def __init__(self, config_environment):
67        self.config_environment = config_environment
68        self._client = None
69
70    @property
71    def defines(self):
72        defines = dict(self.config_environment.acdefines)
73        # These additions work around warts in the build system: see
74        # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/config/faster/rules.mk#92-93
75        # and
76        # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/python/mozbuild/mozbuild/backend/tup.py#244-253.
77        defines.update({
78            'AB_CD': 'en-US',
79            'BUILD_FASTER': '1',
80        })
81        defines.update({
82            'BOOKMARKS_INCLUDE_DIR': mozpath.join(self.config_environment.topsrcdir,
83                                                  'browser', 'locales', 'en-US', 'profile'),
84        })
85        return defines
86
87    @mozbuild.util.memoized_property
88    def file_copier(self):
89        # TODO: invalidate the file copier when the build system
90        # itself changes, i.e., the underlying unified manifest
91        # changes.
92        file_copier = FileCopier()
93
94        unified_manifest = InstallManifest(
95            mozpath.join(self.config_environment.topobjdir,
96                         'faster', 'unified_install_dist_bin'))
97
98        unified_manifest.populate_registry(file_copier, defines_override=self.defines)
99
100        return file_copier
101
102    def subscribe_to_topsrcdir(self):
103        self.subscribe_to_dir('topsrcdir', self.config_environment.topsrcdir)
104
105    def subscribe_to_dir(self, name, dir_to_watch):
106        query = {
107            'empty_on_fresh_instance': True,
108            'expression': [
109                'allof',
110                ['type', 'f'],
111                ['not',
112                 ['anyof',
113                  ['dirname', '.hg'],
114                  ['name', '.hg', 'wholename'],
115                  ['dirname', '.git'],
116                  ['name', '.git', 'wholename'],
117                 ],
118                ],
119            ],
120            'fields': ['name'],
121        }
122        watch = self.client.query('watch-project', dir_to_watch)
123        if 'warning' in watch:
124            print('WARNING: ', watch['warning'], file=sys.stderr)
125
126        root = watch['watch']
127        if 'relative_path' in watch:
128            query['relative_root'] = watch['relative_path']
129
130        # Get the initial clock value so that we only get updates.
131        # Wait 30s to allow for slow Windows IO.  See
132        # https://facebook.github.io/watchman/docs/cmd/clock.html.
133        query['since'] = self.client.query('clock', root, {'sync_timeout': 30000})['clock']
134
135        return self.client.query('subscribe', root, name, query)
136
137    def changed_files(self):
138        # In theory we can parse just the result variable here, but
139        # the client object will accumulate all subscription results
140        # over time, so we ask it to remove and return those values.
141        files = set()
142
143        data = self.client.getSubscription('topsrcdir')
144        if data:
145            for dat in data:
146                files |= set([mozpath.normpath(mozpath.join(self.config_environment.topsrcdir, f))
147                              for f in dat.get('files', [])])
148
149        return files
150
151    def incremental_copy(self, copier, force=False, verbose=True):
152        # Just like the 'repackage' target in browser/app/Makefile.in.
153        if 'cocoa' == self.config_environment.substs['MOZ_WIDGET_TOOLKIT']:
154            bundledir = mozpath.join(self.config_environment.topobjdir, 'dist',
155                                     self.config_environment.substs['MOZ_MACBUNDLE_NAME'],
156                                     'Contents', 'Resources')
157            start = time.time()
158            result = copier.copy(bundledir,
159                                 skip_if_older=not force,
160                                 remove_unaccounted=False,
161                                 remove_all_directory_symlinks=False,
162                                 remove_empty_directories=False)
163            print_copy_result(time.time() - start, bundledir, result, verbose=verbose)
164
165        destdir = mozpath.join(self.config_environment.topobjdir, 'dist', 'bin')
166        start = time.time()
167        result = copier.copy(destdir,
168                             skip_if_older=not force,
169                             remove_unaccounted=False,
170                             remove_all_directory_symlinks=False,
171                             remove_empty_directories=False)
172        print_copy_result(time.time() - start, destdir, result, verbose=verbose)
173
174    def input_changes(self, verbose=True):
175        '''
176        Return an iterator of `FasterBuildChange` instances as inputs
177        to the faster build system change.
178        '''
179
180        # TODO: provide the debug diagnostics we want: this print is
181        # not immediately before the watch.
182        if verbose:
183            print_line('watch', 'Connecting to watchman')
184        # TODO: figure out why a large timeout is required for the
185        # client, and a robust strategy for retrying timed out
186        # requests.
187        self.client = pywatchman.client(timeout=5.0)
188
189        try:
190            if verbose:
191                print_line('watch', 'Checking watchman capabilities')
192            # TODO: restrict these capabilities to the minimal set.
193            self.client.capabilityCheck(required=[
194                'clock-sync-timeout',
195                'cmd-watch-project',
196                'term-dirname',
197                'wildmatch',
198            ])
199
200            if verbose:
201                print_line('watch', 'Subscribing to {}'.format(self.config_environment.topsrcdir))
202            self.subscribe_to_topsrcdir()
203            if verbose:
204                print_line('watch', 'Watching {}'.format(self.config_environment.topsrcdir))
205
206            input_to_outputs = self.file_copier.input_to_outputs_tree()
207            for input, outputs in input_to_outputs.items():
208                if not outputs:
209                    raise Exception("Refusing to watch input ({}) with no outputs".format(input))
210
211            while True:
212                try:
213                    _watch_result = self.client.receive()
214
215                    changed = self.changed_files()
216                    if not changed:
217                        continue
218
219                    result = FasterBuildChange()
220
221                    for change in changed:
222                        if change in input_to_outputs:
223                            result.input_to_outputs[change] = set(input_to_outputs[change])
224                        else:
225                            result.unrecognized.add(change)
226
227                    for input, outputs in result.input_to_outputs.items():
228                        for output in outputs:
229                            if output not in result.output_to_inputs:
230                                result.output_to_inputs[output] = set()
231                            result.output_to_inputs[output].add(input)
232
233                    yield result
234
235                except pywatchman.SocketTimeout:
236                    # Let's check to see if we're still functional.
237                    _version = self.client.query('version')
238
239        except pywatchman.CommandError as e:
240            # Abstract away pywatchman errors.
241            raise FasterBuildException(e, 'Command error using pywatchman to watch {}'.format(
242                self.config_environment.topsrcdir))
243
244        except pywatchman.SocketTimeout as e:
245            # Abstract away pywatchman errors.
246            raise FasterBuildException(e, 'Socket timeout using pywatchman to watch {}'.format(
247                self.config_environment.topsrcdir))
248
249        finally:
250            self.client.close()
251
252    def output_changes(self, verbose=True):
253        '''
254        Return an iterator of `FasterBuildChange` instances as outputs
255        from the faster build system are updated.
256        '''
257        for change in self.input_changes(verbose=verbose):
258            now = datetime.datetime.utcnow()
259
260            for unrecognized in sorted(change.unrecognized):
261                print_line('watch', '! {}'.format(unrecognized), now=now)
262
263            all_outputs = set()
264            for input in sorted(change.input_to_outputs):
265                outputs = change.input_to_outputs[input]
266
267                print_line('watch', '< {}'.format(input), now=now)
268                for output in sorted(outputs):
269                    print_line('watch', '> {}'.format(output), now=now)
270                all_outputs |= outputs
271
272            if all_outputs:
273                partial_copier = FileCopier()
274                for output in all_outputs:
275                    partial_copier.add(output, self.file_copier[output])
276
277                self.incremental_copy(partial_copier, force=True, verbose=verbose)
278                yield change
279
280    def watch(self, verbose=True):
281        try:
282            active_backend = self.config_environment.substs.get('BUILD_BACKENDS', [None])[0]
283            if active_backend:
284                backend_cls = get_backend_class(active_backend)(self.config_environment)
285        except Exception:
286            backend_cls = None
287
288        for change in self.output_changes(verbose=verbose):
289            # Try to run the active build backend's post-build step, if possible.
290            if backend_cls:
291                backend_cls.post_build(self.config_environment, None, 1, False, 0)
292