1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5''' 6Use pywatchman to watch source directories and perform partial |mach 7build faster| builds. 8''' 9 10from __future__ import absolute_import, print_function, unicode_literals 11 12import datetime 13import sys 14import time 15 16import mozbuild.util 17import mozpack.path as mozpath 18from mozpack.manifests import ( 19 InstallManifest, 20) 21from mozpack.copier import ( 22 FileCopier, 23) 24from mozbuild.backend import ( 25 get_backend_class, 26) 27 28# Watchman integration cribbed entirely from 29# https://github.com/facebook/watchman/blob/19aebfebb0b5b0b5174b3914a879370ffc5dac37/python/bin/watchman-wait 30import pywatchman 31 32 33def print_line(prefix, m, now=None): 34 now = now or datetime.datetime.utcnow() 35 print(b'[%s %sZ] %s' % (prefix, now.isoformat(), m)) 36 37 38def print_copy_result(elapsed, destdir, result, verbose=True): 39 COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \ 40 'Added/updated {updated}; ' \ 41 'Removed {rm_files} files and {rm_dirs} directories.' 42 43 print_line('watch', COMPLETE.format( 44 elapsed=elapsed, 45 dest=destdir, 46 existing=result.existing_files_count, 47 updated=result.updated_files_count, 48 rm_files=result.removed_files_count, 49 rm_dirs=result.removed_directories_count)) 50 51 52class FasterBuildException(Exception): 53 def __init__(self, message, cause): 54 Exception.__init__(self, message) 55 self.cause = cause 56 57 58class FasterBuildChange(object): 59 def __init__(self): 60 self.unrecognized = set() 61 self.input_to_outputs = {} 62 self.output_to_inputs = {} 63 64 65class Daemon(object): 66 def __init__(self, config_environment): 67 self.config_environment = config_environment 68 self._client = None 69 70 @property 71 def defines(self): 72 defines = dict(self.config_environment.acdefines) 73 # These additions work around warts in the build system: see 74 # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/config/faster/rules.mk#92-93 75 # and 76 # http://searchfox.org/mozilla-central/rev/ad093e98f42338effe2e2513e26c3a311dd96422/python/mozbuild/mozbuild/backend/tup.py#244-253. 77 defines.update({ 78 'AB_CD': 'en-US', 79 'BUILD_FASTER': '1', 80 }) 81 defines.update({ 82 'BOOKMARKS_INCLUDE_DIR': mozpath.join(self.config_environment.topsrcdir, 83 'browser', 'locales', 'en-US', 'profile'), 84 }) 85 return defines 86 87 @mozbuild.util.memoized_property 88 def file_copier(self): 89 # TODO: invalidate the file copier when the build system 90 # itself changes, i.e., the underlying unified manifest 91 # changes. 92 file_copier = FileCopier() 93 94 unified_manifest = InstallManifest( 95 mozpath.join(self.config_environment.topobjdir, 96 'faster', 'unified_install_dist_bin')) 97 98 unified_manifest.populate_registry(file_copier, defines_override=self.defines) 99 100 return file_copier 101 102 def subscribe_to_topsrcdir(self): 103 self.subscribe_to_dir('topsrcdir', self.config_environment.topsrcdir) 104 105 def subscribe_to_dir(self, name, dir_to_watch): 106 query = { 107 'empty_on_fresh_instance': True, 108 'expression': [ 109 'allof', 110 ['type', 'f'], 111 ['not', 112 ['anyof', 113 ['dirname', '.hg'], 114 ['name', '.hg', 'wholename'], 115 ['dirname', '.git'], 116 ['name', '.git', 'wholename'], 117 ], 118 ], 119 ], 120 'fields': ['name'], 121 } 122 watch = self.client.query('watch-project', dir_to_watch) 123 if 'warning' in watch: 124 print('WARNING: ', watch['warning'], file=sys.stderr) 125 126 root = watch['watch'] 127 if 'relative_path' in watch: 128 query['relative_root'] = watch['relative_path'] 129 130 # Get the initial clock value so that we only get updates. 131 # Wait 30s to allow for slow Windows IO. See 132 # https://facebook.github.io/watchman/docs/cmd/clock.html. 133 query['since'] = self.client.query('clock', root, {'sync_timeout': 30000})['clock'] 134 135 return self.client.query('subscribe', root, name, query) 136 137 def changed_files(self): 138 # In theory we can parse just the result variable here, but 139 # the client object will accumulate all subscription results 140 # over time, so we ask it to remove and return those values. 141 files = set() 142 143 data = self.client.getSubscription('topsrcdir') 144 if data: 145 for dat in data: 146 files |= set([mozpath.normpath(mozpath.join(self.config_environment.topsrcdir, f)) 147 for f in dat.get('files', [])]) 148 149 return files 150 151 def incremental_copy(self, copier, force=False, verbose=True): 152 # Just like the 'repackage' target in browser/app/Makefile.in. 153 if 'cocoa' == self.config_environment.substs['MOZ_WIDGET_TOOLKIT']: 154 bundledir = mozpath.join(self.config_environment.topobjdir, 'dist', 155 self.config_environment.substs['MOZ_MACBUNDLE_NAME'], 156 'Contents', 'Resources') 157 start = time.time() 158 result = copier.copy(bundledir, 159 skip_if_older=not force, 160 remove_unaccounted=False, 161 remove_all_directory_symlinks=False, 162 remove_empty_directories=False) 163 print_copy_result(time.time() - start, bundledir, result, verbose=verbose) 164 165 destdir = mozpath.join(self.config_environment.topobjdir, 'dist', 'bin') 166 start = time.time() 167 result = copier.copy(destdir, 168 skip_if_older=not force, 169 remove_unaccounted=False, 170 remove_all_directory_symlinks=False, 171 remove_empty_directories=False) 172 print_copy_result(time.time() - start, destdir, result, verbose=verbose) 173 174 def input_changes(self, verbose=True): 175 ''' 176 Return an iterator of `FasterBuildChange` instances as inputs 177 to the faster build system change. 178 ''' 179 180 # TODO: provide the debug diagnostics we want: this print is 181 # not immediately before the watch. 182 if verbose: 183 print_line('watch', 'Connecting to watchman') 184 # TODO: figure out why a large timeout is required for the 185 # client, and a robust strategy for retrying timed out 186 # requests. 187 self.client = pywatchman.client(timeout=5.0) 188 189 try: 190 if verbose: 191 print_line('watch', 'Checking watchman capabilities') 192 # TODO: restrict these capabilities to the minimal set. 193 self.client.capabilityCheck(required=[ 194 'clock-sync-timeout', 195 'cmd-watch-project', 196 'term-dirname', 197 'wildmatch', 198 ]) 199 200 if verbose: 201 print_line('watch', 'Subscribing to {}'.format(self.config_environment.topsrcdir)) 202 self.subscribe_to_topsrcdir() 203 if verbose: 204 print_line('watch', 'Watching {}'.format(self.config_environment.topsrcdir)) 205 206 input_to_outputs = self.file_copier.input_to_outputs_tree() 207 for input, outputs in input_to_outputs.items(): 208 if not outputs: 209 raise Exception("Refusing to watch input ({}) with no outputs".format(input)) 210 211 while True: 212 try: 213 _watch_result = self.client.receive() 214 215 changed = self.changed_files() 216 if not changed: 217 continue 218 219 result = FasterBuildChange() 220 221 for change in changed: 222 if change in input_to_outputs: 223 result.input_to_outputs[change] = set(input_to_outputs[change]) 224 else: 225 result.unrecognized.add(change) 226 227 for input, outputs in result.input_to_outputs.items(): 228 for output in outputs: 229 if output not in result.output_to_inputs: 230 result.output_to_inputs[output] = set() 231 result.output_to_inputs[output].add(input) 232 233 yield result 234 235 except pywatchman.SocketTimeout: 236 # Let's check to see if we're still functional. 237 _version = self.client.query('version') 238 239 except pywatchman.CommandError as e: 240 # Abstract away pywatchman errors. 241 raise FasterBuildException(e, 'Command error using pywatchman to watch {}'.format( 242 self.config_environment.topsrcdir)) 243 244 except pywatchman.SocketTimeout as e: 245 # Abstract away pywatchman errors. 246 raise FasterBuildException(e, 'Socket timeout using pywatchman to watch {}'.format( 247 self.config_environment.topsrcdir)) 248 249 finally: 250 self.client.close() 251 252 def output_changes(self, verbose=True): 253 ''' 254 Return an iterator of `FasterBuildChange` instances as outputs 255 from the faster build system are updated. 256 ''' 257 for change in self.input_changes(verbose=verbose): 258 now = datetime.datetime.utcnow() 259 260 for unrecognized in sorted(change.unrecognized): 261 print_line('watch', '! {}'.format(unrecognized), now=now) 262 263 all_outputs = set() 264 for input in sorted(change.input_to_outputs): 265 outputs = change.input_to_outputs[input] 266 267 print_line('watch', '< {}'.format(input), now=now) 268 for output in sorted(outputs): 269 print_line('watch', '> {}'.format(output), now=now) 270 all_outputs |= outputs 271 272 if all_outputs: 273 partial_copier = FileCopier() 274 for output in all_outputs: 275 partial_copier.add(output, self.file_copier[output]) 276 277 self.incremental_copy(partial_copier, force=True, verbose=verbose) 278 yield change 279 280 def watch(self, verbose=True): 281 try: 282 active_backend = self.config_environment.substs.get('BUILD_BACKENDS', [None])[0] 283 if active_backend: 284 backend_cls = get_backend_class(active_backend)(self.config_environment) 285 except Exception: 286 backend_cls = None 287 288 for change in self.output_changes(verbose=verbose): 289 # Try to run the active build backend's post-build step, if possible. 290 if backend_cls: 291 backend_cls.post_build(self.config_environment, None, 1, False, 0) 292