1# MIT License
2#
3# Copyright The SCons Foundation
4#
5# Permission is hereby granted, free of charge, to any person obtaining
6# a copy of this software and associated documentation files (the
7# "Software"), to deal in the Software without restriction, including
8# without limitation the rights to use, copy, modify, merge, publish,
9# distribute, sublicense, and/or sell copies of the Software, and to
10# permit persons to whom the Software is furnished to do so, subject to
11# the following conditions:
12#
13# The above copyright notice and this permission notice shall be included
14# in all copies or substantial portions of the Software.
15#
16# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
17# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
18# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23
24"""CacheDir support
25"""
26
27import atexit
28import json
29import os
30import stat
31import sys
32import uuid
33
34import SCons.Action
35import SCons.Errors
36import SCons.Warnings
37import SCons
38
39cache_enabled = True
40cache_debug = False
41cache_force = False
42cache_show = False
43cache_readonly = False
44cache_tmp_uuid = uuid.uuid4().hex
45
46def CacheRetrieveFunc(target, source, env):
47    t = target[0]
48    fs = t.fs
49    cd = env.get_CacheDir()
50    cd.requests += 1
51    cachedir, cachefile = cd.cachepath(t)
52    if not fs.exists(cachefile):
53        cd.CacheDebug('CacheRetrieve(%s):  %s not in cache\n', t, cachefile)
54        return 1
55    cd.hits += 1
56    cd.CacheDebug('CacheRetrieve(%s):  retrieving from %s\n', t, cachefile)
57    if SCons.Action.execute_actions:
58        if fs.islink(cachefile):
59            fs.symlink(fs.readlink(cachefile), t.get_internal_path())
60        else:
61            cd.copy_from_cache(env, cachefile, t.get_internal_path())
62            try:
63                os.utime(cachefile, None)
64            except OSError:
65                pass
66        st = fs.stat(cachefile)
67        fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
68    return 0
69
70def CacheRetrieveString(target, source, env):
71    t = target[0]
72    fs = t.fs
73    cd = env.get_CacheDir()
74    cachedir, cachefile = cd.cachepath(t)
75    if t.fs.exists(cachefile):
76        return "Retrieved `%s' from cache" % t.get_internal_path()
77    return None
78
79CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
80
81CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
82
83def CachePushFunc(target, source, env):
84    if cache_readonly:
85        return
86
87    t = target[0]
88    if t.nocache:
89        return
90    fs = t.fs
91    cd = env.get_CacheDir()
92    cachedir, cachefile = cd.cachepath(t)
93    if fs.exists(cachefile):
94        # Don't bother copying it if it's already there.  Note that
95        # usually this "shouldn't happen" because if the file already
96        # existed in cache, we'd have retrieved the file from there,
97        # not built it.  This can happen, though, in a race, if some
98        # other person running the same build pushes their copy to
99        # the cache after we decide we need to build it but before our
100        # build completes.
101        cd.CacheDebug('CachePush(%s):  %s already exists in cache\n', t, cachefile)
102        return
103
104    cd.CacheDebug('CachePush(%s):  pushing to %s\n', t, cachefile)
105
106    tempfile = "%s.tmp%s"%(cachefile,cache_tmp_uuid)
107    errfmt = "Unable to copy %s to cache. Cache file is %s"
108
109    try:
110        fs.makedirs(cachedir, exist_ok=True)
111    except OSError:
112        msg = errfmt % (str(target), cachefile)
113        raise SCons.Errors.SConsEnvironmentError(msg)
114    try:
115        if fs.islink(t.get_internal_path()):
116            fs.symlink(fs.readlink(t.get_internal_path()), tempfile)
117        else:
118            cd.copy_to_cache(env, t.get_internal_path(), tempfile)
119        fs.rename(tempfile, cachefile)
120
121    except EnvironmentError:
122        # It's possible someone else tried writing the file at the
123        # same time we did, or else that there was some problem like
124        # the CacheDir being on a separate file system that's full.
125        # In any case, inability to push a file to cache doesn't affect
126        # the correctness of the build, so just print a warning.
127        msg = errfmt % (str(target), cachefile)
128        SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
129
130CachePush = SCons.Action.Action(CachePushFunc, None)
131
132
133class CacheDir:
134
135    def __init__(self, path):
136        """
137        Initialize a CacheDir object.
138
139        The cache configuration is stored in the object. It
140        is read from the config file in the supplied path if
141        one exists,  if not the config file is created and
142        the default config is written, as well as saved in the object.
143        """
144        self.requests = 0
145        self.hits = 0
146        self.path = path
147        self.current_cache_debug = None
148        self.debugFP = None
149        self.config = dict()
150        if path is None:
151            return
152
153        self._readconfig(path)
154
155
156    def _readconfig(self, path):
157        """
158        Read the cache config.
159
160        If directory or config file do not exist, create.  Take advantage
161        of Py3 capability in os.makedirs() and in file open(): just try
162        the operation and handle failure appropriately.
163
164        Omit the check for old cache format, assume that's old enough
165        there will be none of those left to worry about.
166
167        :param path: path to the cache directory
168        """
169        config_file = os.path.join(path, 'config')
170        try:
171            os.makedirs(path, exist_ok=True)
172        except FileExistsError:
173            pass
174        except OSError:
175            msg = "Failed to create cache directory " + path
176            raise SCons.Errors.SConsEnvironmentError(msg)
177
178        try:
179            with open(config_file, 'x') as config:
180                self.config['prefix_len'] = 2
181                try:
182                    json.dump(self.config, config)
183                except Exception:
184                    msg = "Failed to write cache configuration for " + path
185                    raise SCons.Errors.SConsEnvironmentError(msg)
186        except FileExistsError:
187            try:
188                with open(config_file) as config:
189                    self.config = json.load(config)
190            except ValueError:
191                msg = "Failed to read cache configuration for " + path
192                raise SCons.Errors.SConsEnvironmentError(msg)
193
194    def CacheDebug(self, fmt, target, cachefile):
195        if cache_debug != self.current_cache_debug:
196            if cache_debug == '-':
197                self.debugFP = sys.stdout
198            elif cache_debug:
199                def debug_cleanup(debugFP):
200                    debugFP.close()
201
202                self.debugFP = open(cache_debug, 'w')
203                atexit.register(debug_cleanup, self.debugFP)
204            else:
205                self.debugFP = None
206            self.current_cache_debug = cache_debug
207        if self.debugFP:
208            self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
209            self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" %
210                               (self.requests, self.hits, self.misses, self.hit_ratio))
211
212    @classmethod
213    def copy_from_cache(cls, env, src, dst):
214        if env.cache_timestamp_newer:
215            return env.fs.copy(src, dst)
216        else:
217            return env.fs.copy2(src, dst)
218
219    @classmethod
220    def copy_to_cache(cls, env, src, dst):
221        try:
222            result = env.fs.copy2(src, dst)
223            fs = env.File(src).fs
224            st = fs.stat(src)
225            fs.chmod(dst, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
226            return result
227        except AttributeError as ex:
228            raise EnvironmentError from ex
229
230    @property
231    def hit_ratio(self):
232        return (100.0 * self.hits / self.requests if self.requests > 0 else 100)
233
234    @property
235    def misses(self):
236        return self.requests - self.hits
237
238    def is_enabled(self):
239        return cache_enabled and self.path is not None
240
241    def is_readonly(self):
242        return cache_readonly
243
244    def get_cachedir_csig(self, node):
245        cachedir, cachefile = self.cachepath(node)
246        if cachefile and os.path.exists(cachefile):
247            return SCons.Util.hash_file_signature(cachefile, SCons.Node.FS.File.hash_chunksize)
248
249    def cachepath(self, node):
250        """
251        """
252        if not self.is_enabled():
253            return None, None
254
255        sig = node.get_cachedir_bsig()
256
257        subdir = sig[:self.config['prefix_len']].upper()
258
259        dir = os.path.join(self.path, subdir)
260        return dir, os.path.join(dir, sig)
261
262    def retrieve(self, node):
263        """
264        This method is called from multiple threads in a parallel build,
265        so only do thread safe stuff here. Do thread unsafe stuff in
266        built().
267
268        Note that there's a special trick here with the execute flag
269        (one that's not normally done for other actions).  Basically
270        if the user requested a no_exec (-n) build, then
271        SCons.Action.execute_actions is set to 0 and when any action
272        is called, it does its showing but then just returns zero
273        instead of actually calling the action execution operation.
274        The problem for caching is that if the file does NOT exist in
275        cache then the CacheRetrieveString won't return anything to
276        show for the task, but the Action.__call__ won't call
277        CacheRetrieveFunc; instead it just returns zero, which makes
278        the code below think that the file *was* successfully
279        retrieved from the cache, therefore it doesn't do any
280        subsequent building.  However, the CacheRetrieveString didn't
281        print anything because it didn't actually exist in the cache,
282        and no more build actions will be performed, so the user just
283        sees nothing.  The fix is to tell Action.__call__ to always
284        execute the CacheRetrieveFunc and then have the latter
285        explicitly check SCons.Action.execute_actions itself.
286        """
287        if not self.is_enabled():
288            return False
289
290        env = node.get_build_env()
291        if cache_show:
292            if CacheRetrieveSilent(node, [], env, execute=1) == 0:
293                node.build(presub=0, execute=0)
294                return True
295        else:
296            if CacheRetrieve(node, [], env, execute=1) == 0:
297                return True
298
299        return False
300
301    def push(self, node):
302        if self.is_readonly() or not self.is_enabled():
303            return
304        return CachePush(node, [], node.get_build_env())
305
306    def push_if_forced(self, node):
307        if cache_force:
308            return self.push(node)
309
310# Local Variables:
311# tab-width:4
312# indent-tabs-mode:nil
313# End:
314# vim: set expandtab tabstop=4 shiftwidth=4:
315