1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, unicode_literals 6 7import filecmp 8import os 9import re 10import sys 11import subprocess 12import traceback 13 14from collections import defaultdict 15from mozpack import path as mozpath 16 17 18MOZ_MYCONFIG_ERROR = ''' 19The MOZ_MYCONFIG environment variable to define the location of mozconfigs 20is deprecated. If you wish to define the mozconfig path via an environment 21variable, use MOZCONFIG instead. 22'''.strip() 23 24MOZCONFIG_LEGACY_PATH = ''' 25You currently have a mozconfig at %s. This implicit location is no longer 26supported. Please move it to %s/.mozconfig or set an explicit path 27via the $MOZCONFIG environment variable. 28'''.strip() 29 30MOZCONFIG_BAD_EXIT_CODE = ''' 31Evaluation of your mozconfig exited with an error. This could be triggered 32by a command inside your mozconfig failing. Please change your mozconfig 33to not error and/or to catch errors in executed commands. 34'''.strip() 35 36MOZCONFIG_BAD_OUTPUT = ''' 37Evaluation of your mozconfig produced unexpected output. This could be 38triggered by a command inside your mozconfig failing or producing some warnings 39or error messages. Please change your mozconfig to not error and/or to catch 40errors in executed commands. 41'''.strip() 42 43 44class MozconfigFindException(Exception): 45 """Raised when a mozconfig location is not defined properly.""" 46 47 48class MozconfigLoadException(Exception): 49 """Raised when a mozconfig could not be loaded properly. 50 51 This typically indicates a malformed or misbehaving mozconfig file. 52 """ 53 54 def __init__(self, path, message, output=None): 55 self.path = path 56 self.output = output 57 Exception.__init__(self, message) 58 59 60class MozconfigLoader(object): 61 """Handles loading and parsing of mozconfig files.""" 62 63 RE_MAKE_VARIABLE = re.compile(''' 64 ^\s* # Leading whitespace 65 (?P<var>[a-zA-Z_0-9]+) # Variable name 66 \s* [?:]?= \s* # Assignment operator surrounded by optional 67 # spaces 68 (?P<value>.*$)''', # Everything else (likely the value) 69 re.VERBOSE) 70 71 # Default mozconfig files in the topsrcdir. 72 DEFAULT_TOPSRCDIR_PATHS = ('.mozconfig', 'mozconfig') 73 74 DEPRECATED_TOPSRCDIR_PATHS = ('mozconfig.sh', 'myconfig.sh') 75 DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh') 76 77 IGNORE_SHELL_VARIABLES = {'_'} 78 79 ENVIRONMENT_VARIABLES = { 80 'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'MOZ_OBJDIR', 81 } 82 83 AUTODETECT = object() 84 85 def __init__(self, topsrcdir): 86 self.topsrcdir = topsrcdir 87 88 @property 89 def _loader_script(self): 90 our_dir = os.path.abspath(os.path.dirname(__file__)) 91 92 return os.path.join(our_dir, 'mozconfig_loader') 93 94 def find_mozconfig(self, env=os.environ): 95 """Find the active mozconfig file for the current environment. 96 97 This emulates the logic in mozconfig-find. 98 99 1) If ENV[MOZCONFIG] is set, use that 100 2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it. 101 3) If both exist or if there are legacy locations detected, error out. 102 103 The absolute path to the found mozconfig will be returned on success. 104 None will be returned if no mozconfig could be found. A 105 MozconfigFindException will be raised if there is a bad state, 106 including conditions from #3 above. 107 """ 108 # Check for legacy methods first. 109 110 if 'MOZ_MYCONFIG' in env: 111 raise MozconfigFindException(MOZ_MYCONFIG_ERROR) 112 113 env_path = env.get('MOZCONFIG', None) or None 114 if env_path is not None: 115 if not os.path.isabs(env_path): 116 potential_roots = [self.topsrcdir, os.getcwd()] 117 # Attempt to eliminate duplicates for e.g. 118 # self.topsrcdir == os.curdir. 119 potential_roots = set(os.path.abspath(p) for p in potential_roots) 120 existing = [root for root in potential_roots 121 if os.path.exists(os.path.join(root, env_path))] 122 if len(existing) > 1: 123 # There are multiple files, but we might have a setup like: 124 # 125 # somedirectory/ 126 # srcdir/ 127 # objdir/ 128 # 129 # MOZCONFIG=../srcdir/some/path/to/mozconfig 130 # 131 # and be configuring from the objdir. So even though we 132 # have multiple existing files, they are actually the same 133 # file. 134 mozconfigs = [os.path.join(root, env_path) 135 for root in existing] 136 if not all(map(lambda p1, p2: filecmp.cmp(p1, p2, shallow=False), 137 mozconfigs[:-1], mozconfigs[1:])): 138 raise MozconfigFindException( 139 'MOZCONFIG environment variable refers to a path that ' + 140 'exists in more than one of ' + ', '.join(potential_roots) + 141 '. Remove all but one.') 142 elif not existing: 143 raise MozconfigFindException( 144 'MOZCONFIG environment variable refers to a path that ' + 145 'does not exist in any of ' + ', '.join(potential_roots)) 146 147 env_path = os.path.join(existing[0], env_path) 148 elif not os.path.exists(env_path): # non-relative path 149 raise MozconfigFindException( 150 'MOZCONFIG environment variable refers to a path that ' 151 'does not exist: ' + env_path) 152 153 if not os.path.isfile(env_path): 154 raise MozconfigFindException( 155 'MOZCONFIG environment variable refers to a ' 156 'non-file: ' + env_path) 157 158 srcdir_paths = [os.path.join(self.topsrcdir, p) for p in 159 self.DEFAULT_TOPSRCDIR_PATHS] 160 existing = [p for p in srcdir_paths if os.path.isfile(p)] 161 162 if env_path is None and len(existing) > 1: 163 raise MozconfigFindException('Multiple default mozconfig files ' 164 'present. Remove all but one. ' + ', '.join(existing)) 165 166 path = None 167 168 if env_path is not None: 169 path = env_path 170 elif len(existing): 171 assert len(existing) == 1 172 path = existing[0] 173 174 if path is not None: 175 return os.path.abspath(path) 176 177 deprecated_paths = [os.path.join(self.topsrcdir, s) for s in 178 self.DEPRECATED_TOPSRCDIR_PATHS] 179 180 home = env.get('HOME', None) 181 if home is not None: 182 deprecated_paths.extend([os.path.join(home, s) for s in 183 self.DEPRECATED_HOME_PATHS]) 184 185 for path in deprecated_paths: 186 if os.path.exists(path): 187 raise MozconfigFindException( 188 MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir)) 189 190 return None 191 192 def read_mozconfig(self, path=None): 193 """Read the contents of a mozconfig into a data structure. 194 195 This takes the path to a mozconfig to load. If the given path is 196 AUTODETECT, will try to find a mozconfig from the environment using 197 find_mozconfig(). 198 199 mozconfig files are shell scripts. So, we can't just parse them. 200 Instead, we run the shell script in a wrapper which allows us to record 201 state from execution. Thus, the output from a mozconfig is a friendly 202 static data structure. 203 """ 204 if path is self.AUTODETECT: 205 path = self.find_mozconfig() 206 207 result = { 208 'path': path, 209 'topobjdir': None, 210 'configure_args': None, 211 'make_flags': None, 212 'make_extra': None, 213 'env': None, 214 'vars': None, 215 } 216 217 if path is None: 218 return result 219 220 path = mozpath.normsep(path) 221 222 result['configure_args'] = [] 223 result['make_extra'] = [] 224 result['make_flags'] = [] 225 226 env = dict(os.environ) 227 228 # Since mozconfig_loader is a shell script, running it "normally" 229 # actually leads to two shell executions on Windows. Avoid this by 230 # directly calling sh mozconfig_loader. 231 shell = 'sh' 232 if 'MOZILLABUILD' in os.environ: 233 shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh' 234 if sys.platform == 'win32': 235 shell = shell + '.exe' 236 237 command = [shell, mozpath.normsep(self._loader_script), 238 mozpath.normsep(self.topsrcdir), path, sys.executable, 239 mozpath.join(mozpath.dirname(self._loader_script), 240 'action', 'dump_env.py')] 241 242 try: 243 # We need to capture stderr because that's where the shell sends 244 # errors if execution fails. 245 output = subprocess.check_output(command, stderr=subprocess.STDOUT, 246 cwd=self.topsrcdir, env=env) 247 except subprocess.CalledProcessError as e: 248 lines = e.output.splitlines() 249 250 # Output before actual execution shouldn't be relevant. 251 try: 252 index = lines.index('------END_BEFORE_SOURCE') 253 lines = lines[index + 1:] 254 except ValueError: 255 pass 256 257 raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines) 258 259 try: 260 parsed = self._parse_loader_output(output) 261 except AssertionError: 262 # _parse_loader_output uses assertions to verify the 263 # well-formedness of the shell output; when these fail, it 264 # generally means there was a problem with the output, but we 265 # include the assertion traceback just to be sure. 266 print('Assertion failed in _parse_loader_output:') 267 traceback.print_exc() 268 raise MozconfigLoadException(path, MOZCONFIG_BAD_OUTPUT, 269 output.splitlines()) 270 271 def diff_vars(vars_before, vars_after): 272 set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES 273 set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES 274 added = set2 - set1 275 removed = set1 - set2 276 maybe_modified = set1 & set2 277 changed = { 278 'added': {}, 279 'removed': {}, 280 'modified': {}, 281 'unmodified': {}, 282 } 283 284 for key in added: 285 changed['added'][key] = vars_after[key] 286 287 for key in removed: 288 changed['removed'][key] = vars_before[key] 289 290 for key in maybe_modified: 291 if vars_before[key] != vars_after[key]: 292 changed['modified'][key] = ( 293 vars_before[key], vars_after[key]) 294 elif key in self.ENVIRONMENT_VARIABLES: 295 # In order for irrelevant environment variable changes not 296 # to incur in re-running configure, only a set of 297 # environment variables are stored when they are 298 # unmodified. Otherwise, changes such as using a different 299 # terminal window, or even rebooting, would trigger 300 # reconfigures. 301 changed['unmodified'][key] = vars_after[key] 302 303 return changed 304 305 result['env'] = diff_vars(parsed['env_before'], parsed['env_after']) 306 307 # Environment variables also appear as shell variables, but that's 308 # uninteresting duplication of information. Filter them out. 309 filt = lambda x, y: {k: v for k, v in x.items() if k not in y} 310 result['vars'] = diff_vars( 311 filt(parsed['vars_before'], parsed['env_before']), 312 filt(parsed['vars_after'], parsed['env_after']) 313 ) 314 315 result['configure_args'] = [self._expand(o) for o in parsed['ac']] 316 317 if 'MOZ_OBJDIR' in parsed['env_before']: 318 result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR'] 319 320 mk = [self._expand(o) for o in parsed['mk']] 321 322 for o in mk: 323 match = self.RE_MAKE_VARIABLE.match(o) 324 325 if match is None: 326 result['make_extra'].append(o) 327 continue 328 329 name, value = match.group('var'), match.group('value') 330 331 if name == 'MOZ_MAKE_FLAGS': 332 result['make_flags'] = value.split() 333 continue 334 335 if name == 'MOZ_OBJDIR': 336 result['topobjdir'] = value 337 continue 338 339 result['make_extra'].append(o) 340 341 return result 342 343 def _parse_loader_output(self, output): 344 mk_options = [] 345 ac_options = [] 346 before_source = {} 347 after_source = {} 348 env_before_source = {} 349 env_after_source = {} 350 351 current = None 352 current_type = None 353 in_variable = None 354 355 for line in output.splitlines(): 356 357 # XXX This is an ugly hack. Data may be lost from things 358 # like environment variable values. 359 # See https://bugzilla.mozilla.org/show_bug.cgi?id=831381 360 line = line.decode('mbcs' if sys.platform == 'win32' else 'utf-8', 361 'ignore') 362 363 if not line: 364 continue 365 366 if line.startswith('------BEGIN_'): 367 assert current_type is None 368 assert current is None 369 assert not in_variable 370 current_type = line[len('------BEGIN_'):] 371 current = [] 372 continue 373 374 if line.startswith('------END_'): 375 assert not in_variable 376 section = line[len('------END_'):] 377 assert current_type == section 378 379 if current_type == 'AC_OPTION': 380 ac_options.append('\n'.join(current)) 381 elif current_type == 'MK_OPTION': 382 mk_options.append('\n'.join(current)) 383 384 current = None 385 current_type = None 386 continue 387 388 assert current_type is not None 389 390 vars_mapping = { 391 'BEFORE_SOURCE': before_source, 392 'AFTER_SOURCE': after_source, 393 'ENV_BEFORE_SOURCE': env_before_source, 394 'ENV_AFTER_SOURCE': env_after_source, 395 } 396 397 if current_type in vars_mapping: 398 # mozconfigs are sourced using the Bourne shell (or at least 399 # in Bourne shell mode). This means |set| simply lists 400 # variables from the current shell (not functions). (Note that 401 # if Bash is installed in /bin/sh it acts like regular Bourne 402 # and doesn't print functions.) So, lines should have the 403 # form: 404 # 405 # key='value' 406 # key=value 407 # 408 # The only complication is multi-line variables. Those have the 409 # form: 410 # 411 # key='first 412 # second' 413 414 # TODO Bug 818377 Properly handle multi-line variables of form: 415 # $ foo="a='b' 416 # c='d'" 417 # $ set 418 # foo='a='"'"'b'"'"' 419 # c='"'"'d'"'" 420 421 name = in_variable 422 value = None 423 if in_variable: 424 # Reached the end of a multi-line variable. 425 if line.endswith("'") and not line.endswith("\\'"): 426 current.append(line[:-1]) 427 value = '\n'.join(current) 428 in_variable = None 429 else: 430 current.append(line) 431 continue 432 else: 433 equal_pos = line.find('=') 434 435 if equal_pos < 1: 436 # TODO log warning? 437 continue 438 439 name = line[0:equal_pos] 440 value = line[equal_pos + 1:] 441 442 if len(value): 443 has_quote = value[0] == "'" 444 445 if has_quote: 446 value = value[1:] 447 448 # Lines with a quote not ending in a quote are multi-line. 449 if has_quote and not value.endswith("'"): 450 in_variable = name 451 current.append(value) 452 continue 453 else: 454 value = value[:-1] if has_quote else value 455 456 assert name is not None 457 458 vars_mapping[current_type][name] = value 459 460 current = [] 461 462 continue 463 464 current.append(line) 465 466 return { 467 'mk': mk_options, 468 'ac': ac_options, 469 'vars_before': before_source, 470 'vars_after': after_source, 471 'env_before': env_before_source, 472 'env_after': env_after_source, 473 } 474 475 def _expand(self, s): 476 return s.replace('@TOPSRCDIR@', self.topsrcdir) 477