1# This Source Code Form is subject to the terms of the Mozilla Public 2# License, v. 2.0. If a copy of the MPL was not distributed with this 3# file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 5from __future__ import absolute_import, print_function 6 7import os 8import six 9import sys 10import json 11 12from collections.abc import Iterable 13from collections import OrderedDict 14from pathlib import Path 15from types import ModuleType 16 17import mozpack.path as mozpath 18 19from mozbuild.util import ( 20 FileAvoidWrite, 21 memoized_property, 22 ReadOnlyDict, 23 system_encoding, 24) 25from mozbuild.shellutil import quote as shell_quote 26 27 28class ConfigStatusFailure(Exception): 29 """Error loading config.status""" 30 31 32class BuildConfig(object): 33 """Represents the output of configure.""" 34 35 _CODE_CACHE = {} 36 37 def __init__(self): 38 self.topsrcdir = None 39 self.topobjdir = None 40 self.defines = {} 41 self.substs = {} 42 self.files = [] 43 self.mozconfig = None 44 45 @classmethod 46 def from_config_status(cls, path): 47 """Create an instance from a config.status file.""" 48 code_cache = cls._CODE_CACHE 49 mtime = os.path.getmtime(path) 50 51 # cache the compiled code as it can be reused 52 # we cache it the first time, or if the file changed 53 if path not in code_cache or code_cache[path][0] != mtime: 54 # Add config.status manually to sys.modules so it gets picked up by 55 # iter_modules_in_path() for automatic dependencies. 56 mod = ModuleType("config.status") 57 mod.__file__ = path 58 sys.modules["config.status"] = mod 59 60 with open(path, "rt") as fh: 61 source = fh.read() 62 code_cache[path] = ( 63 mtime, 64 compile(source, path, "exec", dont_inherit=1), 65 ) 66 67 g = {"__builtins__": __builtins__, "__file__": path} 68 l = {} 69 try: 70 exec(code_cache[path][1], g, l) 71 except Exception: 72 raise ConfigStatusFailure() 73 74 config = BuildConfig() 75 76 for name in l["__all__"]: 77 setattr(config, name, l[name]) 78 79 return config 80 81 82class ConfigEnvironment(object): 83 """Perform actions associated with a configured but bare objdir. 84 85 The purpose of this class is to preprocess files from the source directory 86 and output results in the object directory. 87 88 There are two types of files: config files and config headers, 89 each treated through a different member function. 90 91 Creating a ConfigEnvironment requires a few arguments: 92 - topsrcdir and topobjdir are, respectively, the top source and 93 the top object directory. 94 - defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in 95 autoconf. 96 - substs is a dict filled from AC_SUBST in autoconf. 97 98 ConfigEnvironment automatically defines one additional substs variable 99 from all the defines: 100 - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on 101 preprocessor command lines. The order in which defines were given 102 when creating the ConfigEnvironment is preserved. 103 and two other additional subst variables from all the other substs: 104 - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted 105 order, for use in autoconf.mk. It includes ACDEFINES 106 Only substs with a VALUE are included, such that the resulting file 107 doesn't change when new empty substs are added. 108 This results in less invalidation of build dependencies in the case 109 of autoconf.mk.. 110 - ALLEMPTYSUBSTS contains the substs with an empty value, in the form 111 NAME =. 112 113 ConfigEnvironment expects a "top_srcdir" subst to be set with the top 114 source directory, in msys format on windows. It is used to derive a 115 "srcdir" subst when treating config files. It can either be an absolute 116 path or a path relative to the topobjdir. 117 """ 118 119 def __init__( 120 self, 121 topsrcdir, 122 topobjdir, 123 defines=None, 124 substs=None, 125 source=None, 126 mozconfig=None, 127 ): 128 129 if not source: 130 source = mozpath.join(topobjdir, "config.status") 131 self.source = source 132 self.defines = ReadOnlyDict(defines or {}) 133 self.substs = dict(substs or {}) 134 self.topsrcdir = mozpath.abspath(topsrcdir) 135 self.topobjdir = mozpath.abspath(topobjdir) 136 self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None 137 self.lib_prefix = self.substs.get("LIB_PREFIX", "") 138 if "LIB_SUFFIX" in self.substs: 139 self.lib_suffix = ".%s" % self.substs["LIB_SUFFIX"] 140 self.dll_prefix = self.substs.get("DLL_PREFIX", "") 141 self.dll_suffix = self.substs.get("DLL_SUFFIX", "") 142 self.host_dll_prefix = self.substs.get("HOST_DLL_PREFIX", "") 143 self.host_dll_suffix = self.substs.get("HOST_DLL_SUFFIX", "") 144 if self.substs.get("IMPORT_LIB_SUFFIX"): 145 self.import_prefix = self.lib_prefix 146 self.import_suffix = ".%s" % self.substs["IMPORT_LIB_SUFFIX"] 147 else: 148 self.import_prefix = self.dll_prefix 149 self.import_suffix = self.dll_suffix 150 if self.substs.get("HOST_IMPORT_LIB_SUFFIX"): 151 self.host_import_prefix = self.substs.get("HOST_LIB_PREFIX", "") 152 self.host_import_suffix = ".%s" % self.substs["HOST_IMPORT_LIB_SUFFIX"] 153 else: 154 self.host_import_prefix = self.host_dll_prefix 155 self.host_import_suffix = self.host_dll_suffix 156 self.bin_suffix = self.substs.get("BIN_SUFFIX", "") 157 158 global_defines = [name for name in self.defines] 159 self.substs["ACDEFINES"] = " ".join( 160 [ 161 "-D%s=%s" % (name, shell_quote(self.defines[name]).replace("$", "$$")) 162 for name in sorted(global_defines) 163 ] 164 ) 165 166 def serialize(name, obj): 167 if isinstance(obj, six.string_types): 168 return obj 169 if isinstance(obj, Iterable): 170 return " ".join(obj) 171 raise Exception("Unhandled type %s for %s", type(obj), str(name)) 172 173 self.substs["ALLSUBSTS"] = "\n".join( 174 sorted( 175 [ 176 "%s = %s" % (name, serialize(name, self.substs[name])) 177 for name in self.substs 178 if self.substs[name] 179 ] 180 ) 181 ) 182 self.substs["ALLEMPTYSUBSTS"] = "\n".join( 183 sorted(["%s =" % name for name in self.substs if not self.substs[name]]) 184 ) 185 186 self.substs = ReadOnlyDict(self.substs) 187 188 @property 189 def is_artifact_build(self): 190 return self.substs.get("MOZ_ARTIFACT_BUILDS", False) 191 192 @memoized_property 193 def acdefines(self): 194 acdefines = dict((name, self.defines[name]) for name in self.defines) 195 return ReadOnlyDict(acdefines) 196 197 @staticmethod 198 def from_config_status(path): 199 config = BuildConfig.from_config_status(path) 200 201 return ConfigEnvironment( 202 config.topsrcdir, config.topobjdir, config.defines, config.substs, path 203 ) 204 205 206class PartialConfigDict(object): 207 """Facilitates mapping the config.statusd defines & substs with dict-like access. 208 209 This allows a buildconfig client to use buildconfig.defines['FOO'] (and 210 similar for substs), where the value of FOO is delay-loaded until it is 211 needed. 212 """ 213 214 def __init__(self, config_statusd, typ, environ_override=False): 215 self._dict = {} 216 self._datadir = mozpath.join(config_statusd, typ) 217 self._config_track = mozpath.join(self._datadir, "config.track") 218 self._files = set() 219 self._environ_override = environ_override 220 221 def _load_config_track(self): 222 existing_files = set() 223 try: 224 with open(self._config_track) as fh: 225 existing_files.update(fh.read().splitlines()) 226 except IOError: 227 pass 228 return existing_files 229 230 def _write_file(self, key, value): 231 filename = mozpath.join(self._datadir, key) 232 with FileAvoidWrite(filename) as fh: 233 to_write = json.dumps(value, indent=4) 234 fh.write(to_write.encode(system_encoding)) 235 return filename 236 237 def _fill_group(self, values): 238 # Clear out any cached values. This is mostly for tests that will check 239 # the environment, write out a new set of variables, and then check the 240 # environment again. Normally only configure ends up calling this 241 # function, and other consumers create their own 242 # PartialConfigEnvironments in new python processes. 243 self._dict = {} 244 245 existing_files = self._load_config_track() 246 existing_files = {Path(f) for f in existing_files} 247 248 new_files = set() 249 for k, v in six.iteritems(values): 250 new_files.add(Path(self._write_file(k, v))) 251 252 for filename in existing_files - new_files: 253 # We can't actually os.remove() here, since make would not see that the 254 # file has been removed and that the target needs to be updated. Instead 255 # we just overwrite the file with a value of None, which is equivalent 256 # to a non-existing file. 257 with FileAvoidWrite(filename) as fh: 258 json.dump(None, fh) 259 260 with FileAvoidWrite(self._config_track) as fh: 261 for f in sorted(new_files): 262 fh.write("%s\n" % f) 263 264 def __getitem__(self, key): 265 if self._environ_override: 266 if (key not in ("CPP", "CXXCPP", "SHELL")) and (key in os.environ): 267 return os.environ[key] 268 269 if key not in self._dict: 270 data = None 271 try: 272 filename = mozpath.join(self._datadir, key) 273 self._files.add(filename) 274 with open(filename) as f: 275 data = json.load(f) 276 except IOError: 277 pass 278 self._dict[key] = data 279 280 if self._dict[key] is None: 281 raise KeyError("'%s'" % key) 282 return self._dict[key] 283 284 def __setitem__(self, key, value): 285 self._dict[key] = value 286 287 def get(self, key, default=None): 288 return self[key] if key in self else default 289 290 def __contains__(self, key): 291 try: 292 return self[key] is not None 293 except KeyError: 294 return False 295 296 def iteritems(self): 297 existing_files = self._load_config_track() 298 for f in existing_files: 299 # The track file contains filenames, and the basename is the 300 # variable name. 301 var = mozpath.basename(f) 302 yield var, self[var] 303 304 305class PartialConfigEnvironment(object): 306 """Allows access to individual config.status items via config.statusd/* files. 307 308 This class is similar to the full ConfigEnvironment, which uses 309 config.status, except this allows access and tracks dependencies to 310 individual configure values. It is intended to be used during the build 311 process to handle things like GENERATED_FILES, CONFIGURE_DEFINE_FILES, and 312 anything else that may need to access specific substs or defines. 313 314 Creating a PartialConfigEnvironment requires only the topobjdir, which is 315 needed to distinguish between the top-level environment and the js/src 316 environment. 317 318 The PartialConfigEnvironment automatically defines one additional subst variable 319 from all the defines: 320 - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on 321 preprocessor command lines. The order in which defines were given 322 when creating the ConfigEnvironment is preserved. 323 324 and one additional define from all the defines as a dictionary: 325 - ALLDEFINES contains all of the global defines as a dictionary. This is 326 intended to be used instead of the defines structure from config.status so 327 that scripts can depend directly on its value. 328 """ 329 330 def __init__(self, topobjdir): 331 config_statusd = mozpath.join(topobjdir, "config.statusd") 332 self.substs = PartialConfigDict(config_statusd, "substs", environ_override=True) 333 self.defines = PartialConfigDict(config_statusd, "defines") 334 self.topobjdir = topobjdir 335 336 def write_vars(self, config): 337 substs = config["substs"].copy() 338 defines = config["defines"].copy() 339 340 global_defines = [name for name in config["defines"]] 341 acdefines = " ".join( 342 [ 343 "-D%s=%s" 344 % (name, shell_quote(config["defines"][name]).replace("$", "$$")) 345 for name in sorted(global_defines) 346 ] 347 ) 348 substs["ACDEFINES"] = acdefines 349 350 all_defines = OrderedDict() 351 for k in global_defines: 352 all_defines[k] = config["defines"][k] 353 defines["ALLDEFINES"] = all_defines 354 355 self.substs._fill_group(substs) 356 self.defines._fill_group(defines) 357 358 def get_dependencies(self): 359 return ["$(wildcard %s)" % f for f in self.substs._files | self.defines._files] 360