1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, You can obtain one at http://mozilla.org/MPL/2.0/.
4
5"""
6Use pywatchman to watch source directories and perform partial |mach
7build faster| builds.
8"""
9
10from __future__ import absolute_import, print_function, unicode_literals
11
12import datetime
13import sys
14import time
15
16import mozbuild.util
17import mozpack.path as mozpath
18from mozpack.manifests import InstallManifest
19from mozpack.copier import FileCopier
20from mozbuild.backend import get_backend_class
21
22# Watchman integration cribbed entirely from
23# https://github.com/facebook/watchman/blob/19aebfebb0b5b0b5174b3914a879370ffc5dac37/python/bin/watchman-wait
24import pywatchman
25
26
27def print_line(prefix, m, now=None):
28    now = now or datetime.datetime.utcnow()
29    print("[%s %sZ] %s" % (prefix, now.isoformat(), m))
30
31
32def print_copy_result(elapsed, destdir, result, verbose=True):
33    COMPLETE = (
34        "Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; "
35        "Added/updated {updated}; "
36        "Removed {rm_files} files and {rm_dirs} directories."
37    )
38
39    print_line(
40        "watch",
41        COMPLETE.format(
42            elapsed=elapsed,
43            dest=destdir,
44            existing=result.existing_files_count,
45            updated=result.updated_files_count,
46            rm_files=result.removed_files_count,
47            rm_dirs=result.removed_directories_count,
48        ),
49    )
50
51
52class FasterBuildException(Exception):
53    def __init__(self, message, cause):
54        Exception.__init__(self, message)
55        self.cause = cause
56
57
58class FasterBuildChange(object):
59    def __init__(self):
60        self.unrecognized = set()
61        self.input_to_outputs = {}
62        self.output_to_inputs = {}
63
64
65class Daemon(object):
66    def __init__(self, config_environment):
67        self.config_environment = config_environment
68        self._client = None
69
70    @property
71    def defines(self):
72        defines = dict(self.config_environment.acdefines)
73        # These additions work around warts in the build system: see
74        # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/config/faster/rules.mk#92-93
75        defines.update(
76            {
77                "AB_CD": "en-US",
78            }
79        )
80        return defines
81
82    @mozbuild.util.memoized_property
83    def file_copier(self):
84        # TODO: invalidate the file copier when the build system
85        # itself changes, i.e., the underlying unified manifest
86        # changes.
87        file_copier = FileCopier()
88
89        unified_manifest = InstallManifest(
90            mozpath.join(
91                self.config_environment.topobjdir, "faster", "unified_install_dist_bin"
92            )
93        )
94
95        unified_manifest.populate_registry(file_copier, defines_override=self.defines)
96
97        return file_copier
98
99    def subscribe_to_topsrcdir(self):
100        self.subscribe_to_dir("topsrcdir", self.config_environment.topsrcdir)
101
102    def subscribe_to_dir(self, name, dir_to_watch):
103        query = {
104            "empty_on_fresh_instance": True,
105            "expression": [
106                "allof",
107                ["type", "f"],
108                [
109                    "not",
110                    [
111                        "anyof",
112                        ["dirname", ".hg"],
113                        ["name", ".hg", "wholename"],
114                        ["dirname", ".git"],
115                        ["name", ".git", "wholename"],
116                    ],
117                ],
118            ],
119            "fields": ["name"],
120        }
121        watch = self.client.query("watch-project", dir_to_watch)
122        if "warning" in watch:
123            print("WARNING: ", watch["warning"], file=sys.stderr)
124
125        root = watch["watch"]
126        if "relative_path" in watch:
127            query["relative_root"] = watch["relative_path"]
128
129        # Get the initial clock value so that we only get updates.
130        # Wait 30s to allow for slow Windows IO.  See
131        # https://facebook.github.io/watchman/docs/cmd/clock.html.
132        query["since"] = self.client.query("clock", root, {"sync_timeout": 30000})[
133            "clock"
134        ]
135
136        return self.client.query("subscribe", root, name, query)
137
138    def changed_files(self):
139        # In theory we can parse just the result variable here, but
140        # the client object will accumulate all subscription results
141        # over time, so we ask it to remove and return those values.
142        files = set()
143
144        data = self.client.getSubscription("topsrcdir")
145        if data:
146            for dat in data:
147                files |= set(
148                    [
149                        mozpath.normpath(
150                            mozpath.join(self.config_environment.topsrcdir, f)
151                        )
152                        for f in dat.get("files", [])
153                    ]
154                )
155
156        return files
157
158    def incremental_copy(self, copier, force=False, verbose=True):
159        # Just like the 'repackage' target in browser/app/Makefile.in.
160        if "cocoa" == self.config_environment.substs["MOZ_WIDGET_TOOLKIT"]:
161            bundledir = mozpath.join(
162                self.config_environment.topobjdir,
163                "dist",
164                self.config_environment.substs["MOZ_MACBUNDLE_NAME"],
165                "Contents",
166                "Resources",
167            )
168            start = time.time()
169            result = copier.copy(
170                bundledir,
171                skip_if_older=not force,
172                remove_unaccounted=False,
173                remove_all_directory_symlinks=False,
174                remove_empty_directories=False,
175            )
176            print_copy_result(time.time() - start, bundledir, result, verbose=verbose)
177
178        destdir = mozpath.join(self.config_environment.topobjdir, "dist", "bin")
179        start = time.time()
180        result = copier.copy(
181            destdir,
182            skip_if_older=not force,
183            remove_unaccounted=False,
184            remove_all_directory_symlinks=False,
185            remove_empty_directories=False,
186        )
187        print_copy_result(time.time() - start, destdir, result, verbose=verbose)
188
189    def input_changes(self, verbose=True):
190        """
191        Return an iterator of `FasterBuildChange` instances as inputs
192        to the faster build system change.
193        """
194
195        # TODO: provide the debug diagnostics we want: this print is
196        # not immediately before the watch.
197        if verbose:
198            print_line("watch", "Connecting to watchman")
199        # TODO: figure out why a large timeout is required for the
200        # client, and a robust strategy for retrying timed out
201        # requests.
202        self.client = pywatchman.client(timeout=5.0)
203
204        try:
205            if verbose:
206                print_line("watch", "Checking watchman capabilities")
207            # TODO: restrict these capabilities to the minimal set.
208            self.client.capabilityCheck(
209                required=[
210                    "clock-sync-timeout",
211                    "cmd-watch-project",
212                    "term-dirname",
213                    "wildmatch",
214                ]
215            )
216
217            if verbose:
218                print_line(
219                    "watch",
220                    "Subscribing to {}".format(self.config_environment.topsrcdir),
221                )
222            self.subscribe_to_topsrcdir()
223            if verbose:
224                print_line(
225                    "watch", "Watching {}".format(self.config_environment.topsrcdir)
226                )
227
228            input_to_outputs = self.file_copier.input_to_outputs_tree()
229            for input, outputs in input_to_outputs.items():
230                if not outputs:
231                    raise Exception(
232                        "Refusing to watch input ({}) with no outputs".format(input)
233                    )
234
235            while True:
236                try:
237                    self.client.receive()
238
239                    changed = self.changed_files()
240                    if not changed:
241                        continue
242
243                    result = FasterBuildChange()
244
245                    for change in changed:
246                        if change in input_to_outputs:
247                            result.input_to_outputs[change] = set(
248                                input_to_outputs[change]
249                            )
250                        else:
251                            result.unrecognized.add(change)
252
253                    for input, outputs in result.input_to_outputs.items():
254                        for output in outputs:
255                            if output not in result.output_to_inputs:
256                                result.output_to_inputs[output] = set()
257                            result.output_to_inputs[output].add(input)
258
259                    yield result
260
261                except pywatchman.SocketTimeout:
262                    # Let's check to see if we're still functional.
263                    self.client.query("version")
264
265        except pywatchman.CommandError as e:
266            # Abstract away pywatchman errors.
267            raise FasterBuildException(
268                e,
269                "Command error using pywatchman to watch {}".format(
270                    self.config_environment.topsrcdir
271                ),
272            )
273
274        except pywatchman.SocketTimeout as e:
275            # Abstract away pywatchman errors.
276            raise FasterBuildException(
277                e,
278                "Socket timeout using pywatchman to watch {}".format(
279                    self.config_environment.topsrcdir
280                ),
281            )
282
283        finally:
284            self.client.close()
285
286    def output_changes(self, verbose=True):
287        """
288        Return an iterator of `FasterBuildChange` instances as outputs
289        from the faster build system are updated.
290        """
291        for change in self.input_changes(verbose=verbose):
292            now = datetime.datetime.utcnow()
293
294            for unrecognized in sorted(change.unrecognized):
295                print_line("watch", "! {}".format(unrecognized), now=now)
296
297            all_outputs = set()
298            for input in sorted(change.input_to_outputs):
299                outputs = change.input_to_outputs[input]
300
301                print_line("watch", "< {}".format(input), now=now)
302                for output in sorted(outputs):
303                    print_line("watch", "> {}".format(output), now=now)
304                all_outputs |= outputs
305
306            if all_outputs:
307                partial_copier = FileCopier()
308                for output in all_outputs:
309                    partial_copier.add(output, self.file_copier[output])
310
311                self.incremental_copy(partial_copier, force=True, verbose=verbose)
312                yield change
313
314    def watch(self, verbose=True):
315        try:
316            active_backend = self.config_environment.substs.get(
317                "BUILD_BACKENDS", [None]
318            )[0]
319            if active_backend:
320                backend_cls = get_backend_class(active_backend)(self.config_environment)
321        except Exception:
322            backend_cls = None
323
324        for change in self.output_changes(verbose=verbose):
325            # Try to run the active build backend's post-build step, if possible.
326            if backend_cls:
327                backend_cls.post_build(self.config_environment, None, 1, False, 0)
328