1"""Lazy and self destructive containers for speeding up module import.""" 2# Copyright 2015-2016, the xonsh developers. All rights reserved. 3import os 4import sys 5import time 6import types 7import builtins 8import threading 9import importlib 10import importlib.util 11import collections.abc as cabc 12 13__version__ = "0.1.3" 14 15 16class LazyObject(object): 17 def __init__(self, load, ctx, name): 18 """Lazily loads an object via the load function the first time an 19 attribute is accessed. Once loaded it will replace itself in the 20 provided context (typically the globals of the call site) with the 21 given name. 22 23 For example, you can prevent the compilation of a regular expression 24 until it is actually used:: 25 26 DOT = LazyObject((lambda: re.compile('.')), globals(), 'DOT') 27 28 Parameters 29 ---------- 30 load : function with no arguments 31 A loader function that performs the actual object construction. 32 ctx : Mapping 33 Context to replace the LazyObject instance in 34 with the object returned by load(). 35 name : str 36 Name in the context to give the loaded object. This *should* 37 be the name on the LHS of the assignment. 38 """ 39 self._lasdo = {"loaded": False, "load": load, "ctx": ctx, "name": name} 40 41 def _lazy_obj(self): 42 d = self._lasdo 43 if d["loaded"]: 44 obj = d["obj"] 45 else: 46 obj = d["load"]() 47 d["ctx"][d["name"]] = d["obj"] = obj 48 d["loaded"] = True 49 return obj 50 51 def __getattribute__(self, name): 52 if name == "_lasdo" or name == "_lazy_obj": 53 return super().__getattribute__(name) 54 obj = self._lazy_obj() 55 return getattr(obj, name) 56 57 def __bool__(self): 58 obj = self._lazy_obj() 59 return bool(obj) 60 61 def __iter__(self): 62 obj = self._lazy_obj() 63 yield from obj 64 65 def __getitem__(self, item): 66 obj = self._lazy_obj() 67 return obj[item] 68 69 def __setitem__(self, key, value): 70 obj = self._lazy_obj() 71 obj[key] = value 72 73 def __delitem__(self, item): 74 obj = self._lazy_obj() 75 del obj[item] 76 77 def __call__(self, *args, **kwargs): 78 obj = self._lazy_obj() 79 return obj(*args, **kwargs) 80 81 def __lt__(self, other): 82 obj = self._lazy_obj() 83 return obj < other 84 85 def __le__(self, other): 86 obj = self._lazy_obj() 87 return obj <= other 88 89 def __eq__(self, other): 90 obj = self._lazy_obj() 91 return obj == other 92 93 def __ne__(self, other): 94 obj = self._lazy_obj() 95 return obj != other 96 97 def __gt__(self, other): 98 obj = self._lazy_obj() 99 return obj > other 100 101 def __ge__(self, other): 102 obj = self._lazy_obj() 103 return obj >= other 104 105 def __hash__(self): 106 obj = self._lazy_obj() 107 return hash(obj) 108 109 def __or__(self, other): 110 obj = self._lazy_obj() 111 return obj | other 112 113 def __str__(self): 114 return str(self._lazy_obj()) 115 116 def __repr__(self): 117 return repr(self._lazy_obj()) 118 119 120def lazyobject(f): 121 """Decorator for constructing lazy objects from a function.""" 122 return LazyObject(f, f.__globals__, f.__name__) 123 124 125class LazyDict(cabc.MutableMapping): 126 def __init__(self, loaders, ctx, name): 127 """Dictionary like object that lazily loads its values from an initial 128 dict of key-loader function pairs. Each key is loaded when its value 129 is first accessed. Once fully loaded, this object will replace itself 130 in the provided context (typically the globals of the call site) with 131 the given name. 132 133 For example, you can prevent the compilation of a bunch of regular 134 expressions until they are actually used:: 135 136 RES = LazyDict({ 137 'dot': lambda: re.compile('.'), 138 'all': lambda: re.compile('.*'), 139 'two': lambda: re.compile('..'), 140 }, globals(), 'RES') 141 142 Parameters 143 ---------- 144 loaders : Mapping of keys to functions with no arguments 145 A mapping of loader function that performs the actual value 146 construction upon access. 147 ctx : Mapping 148 Context to replace the LazyDict instance in 149 with the the fully loaded mapping. 150 name : str 151 Name in the context to give the loaded mapping. This *should* 152 be the name on the LHS of the assignment. 153 """ 154 self._loaders = loaders 155 self._ctx = ctx 156 self._name = name 157 self._d = type(loaders)() # make sure to return the same type 158 159 def _destruct(self): 160 if len(self._loaders) == 0: 161 self._ctx[self._name] = self._d 162 163 def __getitem__(self, key): 164 d = self._d 165 if key in d: 166 val = d[key] 167 else: 168 # pop will raise a key error for us 169 loader = self._loaders.pop(key) 170 d[key] = val = loader() 171 self._destruct() 172 return val 173 174 def __setitem__(self, key, value): 175 self._d[key] = value 176 if key in self._loaders: 177 del self._loaders[key] 178 self._destruct() 179 180 def __delitem__(self, key): 181 if key in self._d: 182 del self._d[key] 183 else: 184 del self._loaders[key] 185 self._destruct() 186 187 def __iter__(self): 188 yield from (set(self._d.keys()) | set(self._loaders.keys())) 189 190 def __len__(self): 191 return len(self._d) + len(self._loaders) 192 193 194def lazydict(f): 195 """Decorator for constructing lazy dicts from a function.""" 196 return LazyDict(f, f.__globals__, f.__name__) 197 198 199class LazyBool(object): 200 def __init__(self, load, ctx, name): 201 """Boolean like object that lazily computes it boolean value when it is 202 first asked. Once loaded, this result will replace itself 203 in the provided context (typically the globals of the call site) with 204 the given name. 205 206 For example, you can prevent the complex boolean until it is actually 207 used:: 208 209 ALIVE = LazyDict(lambda: not DEAD, globals(), 'ALIVE') 210 211 Parameters 212 ---------- 213 load : function with no arguments 214 A loader function that performs the actual boolean evaluation. 215 ctx : Mapping 216 Context to replace the LazyBool instance in 217 with the the fully loaded mapping. 218 name : str 219 Name in the context to give the loaded mapping. This *should* 220 be the name on the LHS of the assignment. 221 """ 222 self._load = load 223 self._ctx = ctx 224 self._name = name 225 self._result = None 226 227 def __bool__(self): 228 if self._result is None: 229 res = self._ctx[self._name] = self._result = self._load() 230 else: 231 res = self._result 232 return res 233 234 235def lazybool(f): 236 """Decorator for constructing lazy booleans from a function.""" 237 return LazyBool(f, f.__globals__, f.__name__) 238 239 240# 241# Background module loaders 242# 243 244 245class BackgroundModuleProxy(types.ModuleType): 246 """Proxy object for modules loaded in the background that block attribute 247 access until the module is loaded.. 248 """ 249 250 def __init__(self, modname): 251 self.__dct__ = {"loaded": False, "modname": modname} 252 253 def __getattribute__(self, name): 254 passthrough = frozenset({"__dct__", "__class__", "__spec__"}) 255 if name in passthrough: 256 return super().__getattribute__(name) 257 dct = self.__dct__ 258 modname = dct["modname"] 259 if dct["loaded"]: 260 mod = sys.modules[modname] 261 else: 262 delay_types = (BackgroundModuleProxy, type(None)) 263 while isinstance(sys.modules.get(modname, None), delay_types): 264 time.sleep(0.001) 265 mod = sys.modules[modname] 266 dct["loaded"] = True 267 # some modules may do construction after import, give them a second 268 stall = 0 269 while not hasattr(mod, name) and stall < 1000: 270 stall += 1 271 time.sleep(0.001) 272 return getattr(mod, name) 273 274 275class BackgroundModuleLoader(threading.Thread): 276 """Thread to load modules in the background.""" 277 278 def __init__(self, name, package, replacements, *args, **kwargs): 279 super().__init__(*args, **kwargs) 280 self.daemon = True 281 self.name = name 282 self.package = package 283 self.replacements = replacements 284 self.start() 285 286 def run(self): 287 # wait for other modules to stop being imported 288 # We assume that module loading is finished when sys.modules doesn't 289 # get longer in 5 consecutive 1ms waiting steps 290 counter = 0 291 last = -1 292 while counter < 5: 293 new = len(sys.modules) 294 if new == last: 295 counter += 1 296 else: 297 last = new 298 counter = 0 299 time.sleep(0.001) 300 # now import module properly 301 modname = importlib.util.resolve_name(self.name, self.package) 302 if isinstance(sys.modules[modname], BackgroundModuleProxy): 303 del sys.modules[modname] 304 mod = importlib.import_module(self.name, package=self.package) 305 for targname, varname in self.replacements.items(): 306 if targname in sys.modules: 307 targmod = sys.modules[targname] 308 setattr(targmod, varname, mod) 309 310 311def load_module_in_background( 312 name, package=None, debug="DEBUG", env=None, replacements=None 313): 314 """Entry point for loading modules in background thread. 315 316 Parameters 317 ---------- 318 name : str 319 Module name to load in background thread. 320 package : str or None, optional 321 Package name, has the same meaning as in importlib.import_module(). 322 debug : str, optional 323 Debugging symbol name to look up in the environment. 324 env : Mapping or None, optional 325 Environment this will default to __xonsh_env__, if available, and 326 os.environ otherwise. 327 replacements : Mapping or None, optional 328 Dictionary mapping fully qualified module names (eg foo.bar.baz) that 329 import the lazily loaded module, with the variable name in that 330 module. For example, suppose that foo.bar imports module a as b, 331 this dict is then {'foo.bar': 'b'}. 332 333 Returns 334 ------- 335 module : ModuleType 336 This is either the original module that is found in sys.modules or 337 a proxy module that will block until delay attribute access until the 338 module is fully loaded. 339 """ 340 modname = importlib.util.resolve_name(name, package) 341 if modname in sys.modules: 342 return sys.modules[modname] 343 if env is None: 344 env = getattr(builtins, "__xonsh_env__", os.environ) 345 if env.get(debug, None): 346 mod = importlib.import_module(name, package=package) 347 return mod 348 proxy = sys.modules[modname] = BackgroundModuleProxy(modname) 349 BackgroundModuleLoader(name, package, replacements or {}) 350 return proxy 351