1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5'''
6Use pywatchman to watch source directories and perform partial |mach
7build faster| builds.
8'''
9
10from __future__ import absolute_import, print_function, unicode_literals
11
12import datetime
13import sys
14import time
15
16import mozbuild.util
17import mozpack.path as mozpath
18from mozpack.manifests import (
19    InstallManifest,
20)
21from mozpack.copier import (
22    FileCopier,
23)
24from mozbuild.backend import (
25    get_backend_class,
26)
27
28# Watchman integration cribbed entirely from
29# https://github.com/facebook/watchman/blob/19aebfebb0b5b0b5174b3914a879370ffc5dac37/python/bin/watchman-wait
30import pywatchman
31
32
33def print_line(prefix, m, now=None):
34    now = now or datetime.datetime.utcnow()
35    print('[%s %sZ] %s' % (prefix, now.isoformat(), m))
36
37
38def print_copy_result(elapsed, destdir, result, verbose=True):
39    COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
40        'Added/updated {updated}; ' \
41        'Removed {rm_files} files and {rm_dirs} directories.'
42
43    print_line('watch', COMPLETE.format(
44        elapsed=elapsed,
45        dest=destdir,
46        existing=result.existing_files_count,
47        updated=result.updated_files_count,
48        rm_files=result.removed_files_count,
49        rm_dirs=result.removed_directories_count))
50
51
52class FasterBuildException(Exception):
53    def __init__(self, message, cause):
54        Exception.__init__(self, message)
55        self.cause = cause
56
57
58class FasterBuildChange(object):
59    def __init__(self):
60        self.unrecognized = set()
61        self.input_to_outputs = {}
62        self.output_to_inputs = {}
63
64
65class Daemon(object):
66    def __init__(self, config_environment):
67        self.config_environment = config_environment
68        self._client = None
69
70    @property
71    def defines(self):
72        defines = dict(self.config_environment.acdefines)
73        # These additions work around warts in the build system: see
74        # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/config/faster/rules.mk#92-93
75        defines.update({
76            'AB_CD': 'en-US',
77        })
78        return defines
79
80    @mozbuild.util.memoized_property
81    def file_copier(self):
82        # TODO: invalidate the file copier when the build system
83        # itself changes, i.e., the underlying unified manifest
84        # changes.
85        file_copier = FileCopier()
86
87        unified_manifest = InstallManifest(
88            mozpath.join(self.config_environment.topobjdir,
89                         'faster', 'unified_install_dist_bin'))
90
91        unified_manifest.populate_registry(file_copier, defines_override=self.defines)
92
93        return file_copier
94
95    def subscribe_to_topsrcdir(self):
96        self.subscribe_to_dir('topsrcdir', self.config_environment.topsrcdir)
97
98    def subscribe_to_dir(self, name, dir_to_watch):
99        query = {
100            'empty_on_fresh_instance': True,
101            'expression': [
102                'allof',
103                ['type', 'f'],
104                ['not',
105                 ['anyof',
106                  ['dirname', '.hg'],
107                  ['name', '.hg', 'wholename'],
108                  ['dirname', '.git'],
109                  ['name', '.git', 'wholename'],
110                  ],
111                 ],
112            ],
113            'fields': ['name'],
114        }
115        watch = self.client.query('watch-project', dir_to_watch)
116        if 'warning' in watch:
117            print('WARNING: ', watch['warning'], file=sys.stderr)
118
119        root = watch['watch']
120        if 'relative_path' in watch:
121            query['relative_root'] = watch['relative_path']
122
123        # Get the initial clock value so that we only get updates.
124        # Wait 30s to allow for slow Windows IO.  See
125        # https://facebook.github.io/watchman/docs/cmd/clock.html.
126        query['since'] = self.client.query('clock', root, {'sync_timeout': 30000})['clock']
127
128        return self.client.query('subscribe', root, name, query)
129
130    def changed_files(self):
131        # In theory we can parse just the result variable here, but
132        # the client object will accumulate all subscription results
133        # over time, so we ask it to remove and return those values.
134        files = set()
135
136        data = self.client.getSubscription('topsrcdir')
137        if data:
138            for dat in data:
139                files |= set([mozpath.normpath(mozpath.join(self.config_environment.topsrcdir, f))
140                              for f in dat.get('files', [])])
141
142        return files
143
144    def incremental_copy(self, copier, force=False, verbose=True):
145        # Just like the 'repackage' target in browser/app/Makefile.in.
146        if 'cocoa' == self.config_environment.substs['MOZ_WIDGET_TOOLKIT']:
147            bundledir = mozpath.join(self.config_environment.topobjdir, 'dist',
148                                     self.config_environment.substs['MOZ_MACBUNDLE_NAME'],
149                                     'Contents', 'Resources')
150            start = time.time()
151            result = copier.copy(bundledir,
152                                 skip_if_older=not force,
153                                 remove_unaccounted=False,
154                                 remove_all_directory_symlinks=False,
155                                 remove_empty_directories=False)
156            print_copy_result(time.time() - start, bundledir, result, verbose=verbose)
157
158        destdir = mozpath.join(self.config_environment.topobjdir, 'dist', 'bin')
159        start = time.time()
160        result = copier.copy(destdir,
161                             skip_if_older=not force,
162                             remove_unaccounted=False,
163                             remove_all_directory_symlinks=False,
164                             remove_empty_directories=False)
165        print_copy_result(time.time() - start, destdir, result, verbose=verbose)
166
167    def input_changes(self, verbose=True):
168        '''
169        Return an iterator of `FasterBuildChange` instances as inputs
170        to the faster build system change.
171        '''
172
173        # TODO: provide the debug diagnostics we want: this print is
174        # not immediately before the watch.
175        if verbose:
176            print_line('watch', 'Connecting to watchman')
177        # TODO: figure out why a large timeout is required for the
178        # client, and a robust strategy for retrying timed out
179        # requests.
180        self.client = pywatchman.client(timeout=5.0)
181
182        try:
183            if verbose:
184                print_line('watch', 'Checking watchman capabilities')
185            # TODO: restrict these capabilities to the minimal set.
186            self.client.capabilityCheck(required=[
187                'clock-sync-timeout',
188                'cmd-watch-project',
189                'term-dirname',
190                'wildmatch',
191            ])
192
193            if verbose:
194                print_line('watch', 'Subscribing to {}'.format(self.config_environment.topsrcdir))
195            self.subscribe_to_topsrcdir()
196            if verbose:
197                print_line('watch', 'Watching {}'.format(self.config_environment.topsrcdir))
198
199            input_to_outputs = self.file_copier.input_to_outputs_tree()
200            for input, outputs in input_to_outputs.items():
201                if not outputs:
202                    raise Exception("Refusing to watch input ({}) with no outputs".format(input))
203
204            while True:
205                try:
206                    self.client.receive()
207
208                    changed = self.changed_files()
209                    if not changed:
210                        continue
211
212                    result = FasterBuildChange()
213
214                    for change in changed:
215                        if change in input_to_outputs:
216                            result.input_to_outputs[change] = set(input_to_outputs[change])
217                        else:
218                            result.unrecognized.add(change)
219
220                    for input, outputs in result.input_to_outputs.items():
221                        for output in outputs:
222                            if output not in result.output_to_inputs:
223                                result.output_to_inputs[output] = set()
224                            result.output_to_inputs[output].add(input)
225
226                    yield result
227
228                except pywatchman.SocketTimeout:
229                    # Let's check to see if we're still functional.
230                    self.client.query('version')
231
232        except pywatchman.CommandError as e:
233            # Abstract away pywatchman errors.
234            raise FasterBuildException(e, 'Command error using pywatchman to watch {}'.format(
235                self.config_environment.topsrcdir))
236
237        except pywatchman.SocketTimeout as e:
238            # Abstract away pywatchman errors.
239            raise FasterBuildException(e, 'Socket timeout using pywatchman to watch {}'.format(
240                self.config_environment.topsrcdir))
241
242        finally:
243            self.client.close()
244
245    def output_changes(self, verbose=True):
246        '''
247        Return an iterator of `FasterBuildChange` instances as outputs
248        from the faster build system are updated.
249        '''
250        for change in self.input_changes(verbose=verbose):
251            now = datetime.datetime.utcnow()
252
253            for unrecognized in sorted(change.unrecognized):
254                print_line('watch', '! {}'.format(unrecognized), now=now)
255
256            all_outputs = set()
257            for input in sorted(change.input_to_outputs):
258                outputs = change.input_to_outputs[input]
259
260                print_line('watch', '< {}'.format(input), now=now)
261                for output in sorted(outputs):
262                    print_line('watch', '> {}'.format(output), now=now)
263                all_outputs |= outputs
264
265            if all_outputs:
266                partial_copier = FileCopier()
267                for output in all_outputs:
268                    partial_copier.add(output, self.file_copier[output])
269
270                self.incremental_copy(partial_copier, force=True, verbose=verbose)
271                yield change
272
273    def watch(self, verbose=True):
274        try:
275            active_backend = self.config_environment.substs.get('BUILD_BACKENDS', [None])[0]
276            if active_backend:
277                backend_cls = get_backend_class(active_backend)(self.config_environment)
278        except Exception:
279            backend_cls = None
280
281        for change in self.output_changes(verbose=verbose):
282            # Try to run the active build backend's post-build step, if possible.
283            if backend_cls:
284                backend_cls.post_build(self.config_environment, None, 1, False, 0)
285