1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5""" 6Use pywatchman to watch source directories and perform partial |mach 7build faster| builds. 8""" 9 10from __future__ import absolute_import, print_function, unicode_literals 11 12import datetime 13import sys 14import time 15 16import mozbuild.util 17import mozpack.path as mozpath 18from mozpack.manifests import InstallManifest 19from mozpack.copier import FileCopier 20from mozbuild.backend import get_backend_class 21 22# Watchman integration cribbed entirely from 23# https://github.com/facebook/watchman/blob/19aebfebb0b5b0b5174b3914a879370ffc5dac37/python/bin/watchman-wait 24import pywatchman 25 26 27def print_line(prefix, m, now=None): 28 now = now or datetime.datetime.utcnow() 29 print("[%s %sZ] %s" % (prefix, now.isoformat(), m)) 30 31 32def print_copy_result(elapsed, destdir, result, verbose=True): 33 COMPLETE = ( 34 "Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; " 35 "Added/updated {updated}; " 36 "Removed {rm_files} files and {rm_dirs} directories." 37 ) 38 39 print_line( 40 "watch", 41 COMPLETE.format( 42 elapsed=elapsed, 43 dest=destdir, 44 existing=result.existing_files_count, 45 updated=result.updated_files_count, 46 rm_files=result.removed_files_count, 47 rm_dirs=result.removed_directories_count, 48 ), 49 ) 50 51 52class FasterBuildException(Exception): 53 def __init__(self, message, cause): 54 Exception.__init__(self, message) 55 self.cause = cause 56 57 58class FasterBuildChange(object): 59 def __init__(self): 60 self.unrecognized = set() 61 self.input_to_outputs = {} 62 self.output_to_inputs = {} 63 64 65class Daemon(object): 66 def __init__(self, config_environment): 67 self.config_environment = config_environment 68 self._client = None 69 70 @property 71 def defines(self): 72 defines = dict(self.config_environment.acdefines) 73 # These additions work around warts in the build system: see 74 # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/config/faster/rules.mk#92-93 75 defines.update( 76 { 77 "AB_CD": "en-US", 78 } 79 ) 80 return defines 81 82 @mozbuild.util.memoized_property 83 def file_copier(self): 84 # TODO: invalidate the file copier when the build system 85 # itself changes, i.e., the underlying unified manifest 86 # changes. 87 file_copier = FileCopier() 88 89 unified_manifest = InstallManifest( 90 mozpath.join( 91 self.config_environment.topobjdir, "faster", "unified_install_dist_bin" 92 ) 93 ) 94 95 unified_manifest.populate_registry(file_copier, defines_override=self.defines) 96 97 return file_copier 98 99 def subscribe_to_topsrcdir(self): 100 self.subscribe_to_dir("topsrcdir", self.config_environment.topsrcdir) 101 102 def subscribe_to_dir(self, name, dir_to_watch): 103 query = { 104 "empty_on_fresh_instance": True, 105 "expression": [ 106 "allof", 107 ["type", "f"], 108 [ 109 "not", 110 [ 111 "anyof", 112 ["dirname", ".hg"], 113 ["name", ".hg", "wholename"], 114 ["dirname", ".git"], 115 ["name", ".git", "wholename"], 116 ], 117 ], 118 ], 119 "fields": ["name"], 120 } 121 watch = self.client.query("watch-project", dir_to_watch) 122 if "warning" in watch: 123 print("WARNING: ", watch["warning"], file=sys.stderr) 124 125 root = watch["watch"] 126 if "relative_path" in watch: 127 query["relative_root"] = watch["relative_path"] 128 129 # Get the initial clock value so that we only get updates. 130 # Wait 30s to allow for slow Windows IO. See 131 # https://facebook.github.io/watchman/docs/cmd/clock.html. 132 query["since"] = self.client.query("clock", root, {"sync_timeout": 30000})[ 133 "clock" 134 ] 135 136 return self.client.query("subscribe", root, name, query) 137 138 def changed_files(self): 139 # In theory we can parse just the result variable here, but 140 # the client object will accumulate all subscription results 141 # over time, so we ask it to remove and return those values. 142 files = set() 143 144 data = self.client.getSubscription("topsrcdir") 145 if data: 146 for dat in data: 147 files |= set( 148 [ 149 mozpath.normpath( 150 mozpath.join(self.config_environment.topsrcdir, f) 151 ) 152 for f in dat.get("files", []) 153 ] 154 ) 155 156 return files 157 158 def incremental_copy(self, copier, force=False, verbose=True): 159 # Just like the 'repackage' target in browser/app/Makefile.in. 160 if "cocoa" == self.config_environment.substs["MOZ_WIDGET_TOOLKIT"]: 161 bundledir = mozpath.join( 162 self.config_environment.topobjdir, 163 "dist", 164 self.config_environment.substs["MOZ_MACBUNDLE_NAME"], 165 "Contents", 166 "Resources", 167 ) 168 start = time.time() 169 result = copier.copy( 170 bundledir, 171 skip_if_older=not force, 172 remove_unaccounted=False, 173 remove_all_directory_symlinks=False, 174 remove_empty_directories=False, 175 ) 176 print_copy_result(time.time() - start, bundledir, result, verbose=verbose) 177 178 destdir = mozpath.join(self.config_environment.topobjdir, "dist", "bin") 179 start = time.time() 180 result = copier.copy( 181 destdir, 182 skip_if_older=not force, 183 remove_unaccounted=False, 184 remove_all_directory_symlinks=False, 185 remove_empty_directories=False, 186 ) 187 print_copy_result(time.time() - start, destdir, result, verbose=verbose) 188 189 def input_changes(self, verbose=True): 190 """ 191 Return an iterator of `FasterBuildChange` instances as inputs 192 to the faster build system change. 193 """ 194 195 # TODO: provide the debug diagnostics we want: this print is 196 # not immediately before the watch. 197 if verbose: 198 print_line("watch", "Connecting to watchman") 199 # TODO: figure out why a large timeout is required for the 200 # client, and a robust strategy for retrying timed out 201 # requests. 202 self.client = pywatchman.client(timeout=5.0) 203 204 try: 205 if verbose: 206 print_line("watch", "Checking watchman capabilities") 207 # TODO: restrict these capabilities to the minimal set. 208 self.client.capabilityCheck( 209 required=[ 210 "clock-sync-timeout", 211 "cmd-watch-project", 212 "term-dirname", 213 "wildmatch", 214 ] 215 ) 216 217 if verbose: 218 print_line( 219 "watch", 220 "Subscribing to {}".format(self.config_environment.topsrcdir), 221 ) 222 self.subscribe_to_topsrcdir() 223 if verbose: 224 print_line( 225 "watch", "Watching {}".format(self.config_environment.topsrcdir) 226 ) 227 228 input_to_outputs = self.file_copier.input_to_outputs_tree() 229 for input, outputs in input_to_outputs.items(): 230 if not outputs: 231 raise Exception( 232 "Refusing to watch input ({}) with no outputs".format(input) 233 ) 234 235 while True: 236 try: 237 self.client.receive() 238 239 changed = self.changed_files() 240 if not changed: 241 continue 242 243 result = FasterBuildChange() 244 245 for change in changed: 246 if change in input_to_outputs: 247 result.input_to_outputs[change] = set( 248 input_to_outputs[change] 249 ) 250 else: 251 result.unrecognized.add(change) 252 253 for input, outputs in result.input_to_outputs.items(): 254 for output in outputs: 255 if output not in result.output_to_inputs: 256 result.output_to_inputs[output] = set() 257 result.output_to_inputs[output].add(input) 258 259 yield result 260 261 except pywatchman.SocketTimeout: 262 # Let's check to see if we're still functional. 263 self.client.query("version") 264 265 except pywatchman.CommandError as e: 266 # Abstract away pywatchman errors. 267 raise FasterBuildException( 268 e, 269 "Command error using pywatchman to watch {}".format( 270 self.config_environment.topsrcdir 271 ), 272 ) 273 274 except pywatchman.SocketTimeout as e: 275 # Abstract away pywatchman errors. 276 raise FasterBuildException( 277 e, 278 "Socket timeout using pywatchman to watch {}".format( 279 self.config_environment.topsrcdir 280 ), 281 ) 282 283 finally: 284 self.client.close() 285 286 def output_changes(self, verbose=True): 287 """ 288 Return an iterator of `FasterBuildChange` instances as outputs 289 from the faster build system are updated. 290 """ 291 for change in self.input_changes(verbose=verbose): 292 now = datetime.datetime.utcnow() 293 294 for unrecognized in sorted(change.unrecognized): 295 print_line("watch", "! {}".format(unrecognized), now=now) 296 297 all_outputs = set() 298 for input in sorted(change.input_to_outputs): 299 outputs = change.input_to_outputs[input] 300 301 print_line("watch", "< {}".format(input), now=now) 302 for output in sorted(outputs): 303 print_line("watch", "> {}".format(output), now=now) 304 all_outputs |= outputs 305 306 if all_outputs: 307 partial_copier = FileCopier() 308 for output in all_outputs: 309 partial_copier.add(output, self.file_copier[output]) 310 311 self.incremental_copy(partial_copier, force=True, verbose=verbose) 312 yield change 313 314 def watch(self, verbose=True): 315 try: 316 active_backend = self.config_environment.substs.get( 317 "BUILD_BACKENDS", [None] 318 )[0] 319 if active_backend: 320 backend_cls = get_backend_class(active_backend)(self.config_environment) 321 except Exception: 322 backend_cls = None 323 324 for change in self.output_changes(verbose=verbose): 325 # Try to run the active build backend's post-build step, if possible. 326 if backend_cls: 327 backend_cls.post_build(self.config_environment, None, 1, False, 0) 328