1# MIT License 2# 3# Copyright The SCons Foundation 4# 5# Permission is hereby granted, free of charge, to any person obtaining 6# a copy of this software and associated documentation files (the 7# "Software"), to deal in the Software without restriction, including 8# without limitation the rights to use, copy, modify, merge, publish, 9# distribute, sublicense, and/or sell copies of the Software, and to 10# permit persons to whom the Software is furnished to do so, subject to 11# the following conditions: 12# 13# The above copyright notice and this permission notice shall be included 14# in all copies or substantial portions of the Software. 15# 16# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 17# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 18# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 24"""CacheDir support 25""" 26 27import atexit 28import json 29import os 30import stat 31import sys 32import uuid 33 34import SCons.Action 35import SCons.Errors 36import SCons.Warnings 37import SCons 38 39cache_enabled = True 40cache_debug = False 41cache_force = False 42cache_show = False 43cache_readonly = False 44cache_tmp_uuid = uuid.uuid4().hex 45 46def CacheRetrieveFunc(target, source, env): 47 t = target[0] 48 fs = t.fs 49 cd = env.get_CacheDir() 50 cd.requests += 1 51 cachedir, cachefile = cd.cachepath(t) 52 if not fs.exists(cachefile): 53 cd.CacheDebug('CacheRetrieve(%s): %s not in cache\n', t, cachefile) 54 return 1 55 cd.hits += 1 56 cd.CacheDebug('CacheRetrieve(%s): retrieving from %s\n', t, cachefile) 57 if SCons.Action.execute_actions: 58 if fs.islink(cachefile): 59 fs.symlink(fs.readlink(cachefile), t.get_internal_path()) 60 else: 61 cd.copy_from_cache(env, cachefile, t.get_internal_path()) 62 try: 63 os.utime(cachefile, None) 64 except OSError: 65 pass 66 st = fs.stat(cachefile) 67 fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 68 return 0 69 70def CacheRetrieveString(target, source, env): 71 t = target[0] 72 fs = t.fs 73 cd = env.get_CacheDir() 74 cachedir, cachefile = cd.cachepath(t) 75 if t.fs.exists(cachefile): 76 return "Retrieved `%s' from cache" % t.get_internal_path() 77 return None 78 79CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) 80 81CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None) 82 83def CachePushFunc(target, source, env): 84 if cache_readonly: 85 return 86 87 t = target[0] 88 if t.nocache: 89 return 90 fs = t.fs 91 cd = env.get_CacheDir() 92 cachedir, cachefile = cd.cachepath(t) 93 if fs.exists(cachefile): 94 # Don't bother copying it if it's already there. Note that 95 # usually this "shouldn't happen" because if the file already 96 # existed in cache, we'd have retrieved the file from there, 97 # not built it. This can happen, though, in a race, if some 98 # other person running the same build pushes their copy to 99 # the cache after we decide we need to build it but before our 100 # build completes. 101 cd.CacheDebug('CachePush(%s): %s already exists in cache\n', t, cachefile) 102 return 103 104 cd.CacheDebug('CachePush(%s): pushing to %s\n', t, cachefile) 105 106 tempfile = "%s.tmp%s"%(cachefile,cache_tmp_uuid) 107 errfmt = "Unable to copy %s to cache. Cache file is %s" 108 109 try: 110 fs.makedirs(cachedir, exist_ok=True) 111 except OSError: 112 msg = errfmt % (str(target), cachefile) 113 raise SCons.Errors.SConsEnvironmentError(msg) 114 try: 115 if fs.islink(t.get_internal_path()): 116 fs.symlink(fs.readlink(t.get_internal_path()), tempfile) 117 else: 118 cd.copy_to_cache(env, t.get_internal_path(), tempfile) 119 fs.rename(tempfile, cachefile) 120 121 except EnvironmentError: 122 # It's possible someone else tried writing the file at the 123 # same time we did, or else that there was some problem like 124 # the CacheDir being on a separate file system that's full. 125 # In any case, inability to push a file to cache doesn't affect 126 # the correctness of the build, so just print a warning. 127 msg = errfmt % (str(target), cachefile) 128 SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg) 129 130CachePush = SCons.Action.Action(CachePushFunc, None) 131 132 133class CacheDir: 134 135 def __init__(self, path): 136 """ 137 Initialize a CacheDir object. 138 139 The cache configuration is stored in the object. It 140 is read from the config file in the supplied path if 141 one exists, if not the config file is created and 142 the default config is written, as well as saved in the object. 143 """ 144 self.requests = 0 145 self.hits = 0 146 self.path = path 147 self.current_cache_debug = None 148 self.debugFP = None 149 self.config = dict() 150 if path is None: 151 return 152 153 self._readconfig(path) 154 155 156 def _readconfig(self, path): 157 """ 158 Read the cache config. 159 160 If directory or config file do not exist, create. Take advantage 161 of Py3 capability in os.makedirs() and in file open(): just try 162 the operation and handle failure appropriately. 163 164 Omit the check for old cache format, assume that's old enough 165 there will be none of those left to worry about. 166 167 :param path: path to the cache directory 168 """ 169 config_file = os.path.join(path, 'config') 170 try: 171 os.makedirs(path, exist_ok=True) 172 except FileExistsError: 173 pass 174 except OSError: 175 msg = "Failed to create cache directory " + path 176 raise SCons.Errors.SConsEnvironmentError(msg) 177 178 try: 179 with open(config_file, 'x') as config: 180 self.config['prefix_len'] = 2 181 try: 182 json.dump(self.config, config) 183 except Exception: 184 msg = "Failed to write cache configuration for " + path 185 raise SCons.Errors.SConsEnvironmentError(msg) 186 except FileExistsError: 187 try: 188 with open(config_file) as config: 189 self.config = json.load(config) 190 except ValueError: 191 msg = "Failed to read cache configuration for " + path 192 raise SCons.Errors.SConsEnvironmentError(msg) 193 194 def CacheDebug(self, fmt, target, cachefile): 195 if cache_debug != self.current_cache_debug: 196 if cache_debug == '-': 197 self.debugFP = sys.stdout 198 elif cache_debug: 199 def debug_cleanup(debugFP): 200 debugFP.close() 201 202 self.debugFP = open(cache_debug, 'w') 203 atexit.register(debug_cleanup, self.debugFP) 204 else: 205 self.debugFP = None 206 self.current_cache_debug = cache_debug 207 if self.debugFP: 208 self.debugFP.write(fmt % (target, os.path.split(cachefile)[1])) 209 self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" % 210 (self.requests, self.hits, self.misses, self.hit_ratio)) 211 212 @classmethod 213 def copy_from_cache(cls, env, src, dst): 214 if env.cache_timestamp_newer: 215 return env.fs.copy(src, dst) 216 else: 217 return env.fs.copy2(src, dst) 218 219 @classmethod 220 def copy_to_cache(cls, env, src, dst): 221 try: 222 result = env.fs.copy2(src, dst) 223 fs = env.File(src).fs 224 st = fs.stat(src) 225 fs.chmod(dst, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 226 return result 227 except AttributeError as ex: 228 raise EnvironmentError from ex 229 230 @property 231 def hit_ratio(self): 232 return (100.0 * self.hits / self.requests if self.requests > 0 else 100) 233 234 @property 235 def misses(self): 236 return self.requests - self.hits 237 238 def is_enabled(self): 239 return cache_enabled and self.path is not None 240 241 def is_readonly(self): 242 return cache_readonly 243 244 def get_cachedir_csig(self, node): 245 cachedir, cachefile = self.cachepath(node) 246 if cachefile and os.path.exists(cachefile): 247 return SCons.Util.hash_file_signature(cachefile, SCons.Node.FS.File.hash_chunksize) 248 249 def cachepath(self, node): 250 """ 251 """ 252 if not self.is_enabled(): 253 return None, None 254 255 sig = node.get_cachedir_bsig() 256 257 subdir = sig[:self.config['prefix_len']].upper() 258 259 dir = os.path.join(self.path, subdir) 260 return dir, os.path.join(dir, sig) 261 262 def retrieve(self, node): 263 """ 264 This method is called from multiple threads in a parallel build, 265 so only do thread safe stuff here. Do thread unsafe stuff in 266 built(). 267 268 Note that there's a special trick here with the execute flag 269 (one that's not normally done for other actions). Basically 270 if the user requested a no_exec (-n) build, then 271 SCons.Action.execute_actions is set to 0 and when any action 272 is called, it does its showing but then just returns zero 273 instead of actually calling the action execution operation. 274 The problem for caching is that if the file does NOT exist in 275 cache then the CacheRetrieveString won't return anything to 276 show for the task, but the Action.__call__ won't call 277 CacheRetrieveFunc; instead it just returns zero, which makes 278 the code below think that the file *was* successfully 279 retrieved from the cache, therefore it doesn't do any 280 subsequent building. However, the CacheRetrieveString didn't 281 print anything because it didn't actually exist in the cache, 282 and no more build actions will be performed, so the user just 283 sees nothing. The fix is to tell Action.__call__ to always 284 execute the CacheRetrieveFunc and then have the latter 285 explicitly check SCons.Action.execute_actions itself. 286 """ 287 if not self.is_enabled(): 288 return False 289 290 env = node.get_build_env() 291 if cache_show: 292 if CacheRetrieveSilent(node, [], env, execute=1) == 0: 293 node.build(presub=0, execute=0) 294 return True 295 else: 296 if CacheRetrieve(node, [], env, execute=1) == 0: 297 return True 298 299 return False 300 301 def push(self, node): 302 if self.is_readonly() or not self.is_enabled(): 303 return 304 return CachePush(node, [], node.get_build_env()) 305 306 def push_if_forced(self, node): 307 if cache_force: 308 return self.push(node) 309 310# Local Variables: 311# tab-width:4 312# indent-tabs-mode:nil 313# End: 314# vim: set expandtab tabstop=4 shiftwidth=4: 315