1"""Lazy and self destructive containers for speeding up module import."""
2# Copyright 2015-2016, the xonsh developers. All rights reserved.
3import os
4import sys
5import time
6import types
7import builtins
8import threading
9import importlib
10import importlib.util
11import collections.abc as cabc
12
13__version__ = "0.1.3"
14
15
16class LazyObject(object):
17    def __init__(self, load, ctx, name):
18        """Lazily loads an object via the load function the first time an
19        attribute is accessed. Once loaded it will replace itself in the
20        provided context (typically the globals of the call site) with the
21        given name.
22
23        For example, you can prevent the compilation of a regular expression
24        until it is actually used::
25
26            DOT = LazyObject((lambda: re.compile('.')), globals(), 'DOT')
27
28        Parameters
29        ----------
30        load : function with no arguments
31            A loader function that performs the actual object construction.
32        ctx : Mapping
33            Context to replace the LazyObject instance in
34            with the object returned by load().
35        name : str
36            Name in the context to give the loaded object. This *should*
37            be the name on the LHS of the assignment.
38        """
39        self._lasdo = {"loaded": False, "load": load, "ctx": ctx, "name": name}
40
41    def _lazy_obj(self):
42        d = self._lasdo
43        if d["loaded"]:
44            obj = d["obj"]
45        else:
46            obj = d["load"]()
47            d["ctx"][d["name"]] = d["obj"] = obj
48            d["loaded"] = True
49        return obj
50
51    def __getattribute__(self, name):
52        if name == "_lasdo" or name == "_lazy_obj":
53            return super().__getattribute__(name)
54        obj = self._lazy_obj()
55        return getattr(obj, name)
56
57    def __bool__(self):
58        obj = self._lazy_obj()
59        return bool(obj)
60
61    def __iter__(self):
62        obj = self._lazy_obj()
63        yield from obj
64
65    def __getitem__(self, item):
66        obj = self._lazy_obj()
67        return obj[item]
68
69    def __setitem__(self, key, value):
70        obj = self._lazy_obj()
71        obj[key] = value
72
73    def __delitem__(self, item):
74        obj = self._lazy_obj()
75        del obj[item]
76
77    def __call__(self, *args, **kwargs):
78        obj = self._lazy_obj()
79        return obj(*args, **kwargs)
80
81    def __lt__(self, other):
82        obj = self._lazy_obj()
83        return obj < other
84
85    def __le__(self, other):
86        obj = self._lazy_obj()
87        return obj <= other
88
89    def __eq__(self, other):
90        obj = self._lazy_obj()
91        return obj == other
92
93    def __ne__(self, other):
94        obj = self._lazy_obj()
95        return obj != other
96
97    def __gt__(self, other):
98        obj = self._lazy_obj()
99        return obj > other
100
101    def __ge__(self, other):
102        obj = self._lazy_obj()
103        return obj >= other
104
105    def __hash__(self):
106        obj = self._lazy_obj()
107        return hash(obj)
108
109    def __or__(self, other):
110        obj = self._lazy_obj()
111        return obj | other
112
113    def __str__(self):
114        return str(self._lazy_obj())
115
116    def __repr__(self):
117        return repr(self._lazy_obj())
118
119
120def lazyobject(f):
121    """Decorator for constructing lazy objects from a function."""
122    return LazyObject(f, f.__globals__, f.__name__)
123
124
125class LazyDict(cabc.MutableMapping):
126    def __init__(self, loaders, ctx, name):
127        """Dictionary like object that lazily loads its values from an initial
128        dict of key-loader function pairs.  Each key is loaded when its value
129        is first accessed. Once fully loaded, this object will replace itself
130        in the provided context (typically the globals of the call site) with
131        the given name.
132
133        For example, you can prevent the compilation of a bunch of regular
134        expressions until they are actually used::
135
136            RES = LazyDict({
137                    'dot': lambda: re.compile('.'),
138                    'all': lambda: re.compile('.*'),
139                    'two': lambda: re.compile('..'),
140                    }, globals(), 'RES')
141
142        Parameters
143        ----------
144        loaders : Mapping of keys to functions with no arguments
145            A mapping of loader function that performs the actual value
146            construction upon access.
147        ctx : Mapping
148            Context to replace the LazyDict instance in
149            with the the fully loaded mapping.
150        name : str
151            Name in the context to give the loaded mapping. This *should*
152            be the name on the LHS of the assignment.
153        """
154        self._loaders = loaders
155        self._ctx = ctx
156        self._name = name
157        self._d = type(loaders)()  # make sure to return the same type
158
159    def _destruct(self):
160        if len(self._loaders) == 0:
161            self._ctx[self._name] = self._d
162
163    def __getitem__(self, key):
164        d = self._d
165        if key in d:
166            val = d[key]
167        else:
168            # pop will raise a key error for us
169            loader = self._loaders.pop(key)
170            d[key] = val = loader()
171            self._destruct()
172        return val
173
174    def __setitem__(self, key, value):
175        self._d[key] = value
176        if key in self._loaders:
177            del self._loaders[key]
178            self._destruct()
179
180    def __delitem__(self, key):
181        if key in self._d:
182            del self._d[key]
183        else:
184            del self._loaders[key]
185            self._destruct()
186
187    def __iter__(self):
188        yield from (set(self._d.keys()) | set(self._loaders.keys()))
189
190    def __len__(self):
191        return len(self._d) + len(self._loaders)
192
193
194def lazydict(f):
195    """Decorator for constructing lazy dicts from a function."""
196    return LazyDict(f, f.__globals__, f.__name__)
197
198
199class LazyBool(object):
200    def __init__(self, load, ctx, name):
201        """Boolean like object that lazily computes it boolean value when it is
202        first asked. Once loaded, this result will replace itself
203        in the provided context (typically the globals of the call site) with
204        the given name.
205
206        For example, you can prevent the complex boolean until it is actually
207        used::
208
209            ALIVE = LazyDict(lambda: not DEAD, globals(), 'ALIVE')
210
211        Parameters
212        ----------
213        load : function with no arguments
214            A loader function that performs the actual boolean evaluation.
215        ctx : Mapping
216            Context to replace the LazyBool instance in
217            with the the fully loaded mapping.
218        name : str
219            Name in the context to give the loaded mapping. This *should*
220            be the name on the LHS of the assignment.
221        """
222        self._load = load
223        self._ctx = ctx
224        self._name = name
225        self._result = None
226
227    def __bool__(self):
228        if self._result is None:
229            res = self._ctx[self._name] = self._result = self._load()
230        else:
231            res = self._result
232        return res
233
234
235def lazybool(f):
236    """Decorator for constructing lazy booleans from a function."""
237    return LazyBool(f, f.__globals__, f.__name__)
238
239
240#
241# Background module loaders
242#
243
244
245class BackgroundModuleProxy(types.ModuleType):
246    """Proxy object for modules loaded in the background that block attribute
247    access until the module is loaded..
248    """
249
250    def __init__(self, modname):
251        self.__dct__ = {"loaded": False, "modname": modname}
252
253    def __getattribute__(self, name):
254        passthrough = frozenset({"__dct__", "__class__", "__spec__"})
255        if name in passthrough:
256            return super().__getattribute__(name)
257        dct = self.__dct__
258        modname = dct["modname"]
259        if dct["loaded"]:
260            mod = sys.modules[modname]
261        else:
262            delay_types = (BackgroundModuleProxy, type(None))
263            while isinstance(sys.modules.get(modname, None), delay_types):
264                time.sleep(0.001)
265            mod = sys.modules[modname]
266            dct["loaded"] = True
267        # some modules may do construction after import, give them a second
268        stall = 0
269        while not hasattr(mod, name) and stall < 1000:
270            stall += 1
271            time.sleep(0.001)
272        return getattr(mod, name)
273
274
275class BackgroundModuleLoader(threading.Thread):
276    """Thread to load modules in the background."""
277
278    def __init__(self, name, package, replacements, *args, **kwargs):
279        super().__init__(*args, **kwargs)
280        self.daemon = True
281        self.name = name
282        self.package = package
283        self.replacements = replacements
284        self.start()
285
286    def run(self):
287        # wait for other modules to stop being imported
288        # We assume that module loading is finished when sys.modules doesn't
289        # get longer in 5 consecutive 1ms waiting steps
290        counter = 0
291        last = -1
292        while counter < 5:
293            new = len(sys.modules)
294            if new == last:
295                counter += 1
296            else:
297                last = new
298                counter = 0
299            time.sleep(0.001)
300        # now import module properly
301        modname = importlib.util.resolve_name(self.name, self.package)
302        if isinstance(sys.modules[modname], BackgroundModuleProxy):
303            del sys.modules[modname]
304        mod = importlib.import_module(self.name, package=self.package)
305        for targname, varname in self.replacements.items():
306            if targname in sys.modules:
307                targmod = sys.modules[targname]
308                setattr(targmod, varname, mod)
309
310
311def load_module_in_background(
312    name, package=None, debug="DEBUG", env=None, replacements=None
313):
314    """Entry point for loading modules in background thread.
315
316    Parameters
317    ----------
318    name : str
319        Module name to load in background thread.
320    package : str or None, optional
321        Package name, has the same meaning as in importlib.import_module().
322    debug : str, optional
323        Debugging symbol name to look up in the environment.
324    env : Mapping or None, optional
325        Environment this will default to __xonsh_env__, if available, and
326        os.environ otherwise.
327    replacements : Mapping or None, optional
328        Dictionary mapping fully qualified module names (eg foo.bar.baz) that
329        import the lazily loaded module, with the variable name in that
330        module. For example, suppose that foo.bar imports module a as b,
331        this dict is then {'foo.bar': 'b'}.
332
333    Returns
334    -------
335    module : ModuleType
336        This is either the original module that is found in sys.modules or
337        a proxy module that will block until delay attribute access until the
338        module is fully loaded.
339    """
340    modname = importlib.util.resolve_name(name, package)
341    if modname in sys.modules:
342        return sys.modules[modname]
343    if env is None:
344        env = getattr(builtins, "__xonsh_env__", os.environ)
345    if env.get(debug, None):
346        mod = importlib.import_module(name, package=package)
347        return mod
348    proxy = sys.modules[modname] = BackgroundModuleProxy(modname)
349    BackgroundModuleLoader(name, package, replacements or {})
350    return proxy
351