1# MIT License 2# 3# Copyright The SCons Foundation 4# 5# Permission is hereby granted, free of charge, to any person obtaining 6# a copy of this software and associated documentation files (the 7# "Software"), to deal in the Software without restriction, including 8# without limitation the rights to use, copy, modify, merge, publish, 9# distribute, sublicense, and/or sell copies of the Software, and to 10# permit persons to whom the Software is furnished to do so, subject to 11# the following conditions: 12# 13# The above copyright notice and this permission notice shall be included 14# in all copies or substantial portions of the Software. 15# 16# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 17# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 18# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 24"""Operations on signature database files (.sconsign). """ 25 26import SCons.compat 27 28import os 29import pickle 30import time 31 32import SCons.dblite 33import SCons.Warnings 34from SCons.compat import PICKLE_PROTOCOL 35from SCons.Util import print_time 36 37 38def corrupt_dblite_warning(filename): 39 SCons.Warnings.warn( 40 SCons.Warnings.CorruptSConsignWarning, 41 "Ignoring corrupt .sconsign file: %s" % filename, 42 ) 43 44SCons.dblite.IGNORE_CORRUPT_DBFILES = True 45SCons.dblite.corruption_warning = corrupt_dblite_warning 46 47# XXX Get rid of the global array so this becomes re-entrant. 48sig_files = [] 49 50# Info for the database SConsign implementation (now the default): 51# "DataBase" is a dictionary that maps top-level SConstruct directories 52# to open database handles. 53# "DB_Module" is the Python database module to create the handles. 54# "DB_Name" is the base name of the database file (minus any 55# extension the underlying DB module will add). 56DataBase = {} 57DB_Module = SCons.dblite 58DB_Name = None 59DB_sync_list = [] 60 61 62def Get_DataBase(dir): 63 global DataBase, DB_Module, DB_Name 64 65 if DB_Name is None: 66 hash_format = SCons.Util.get_hash_format() 67 if hash_format is None: 68 DB_Name = ".sconsign" 69 else: 70 DB_Name = ".sconsign_%s" % hash_format 71 72 top = dir.fs.Top 73 if not os.path.isabs(DB_Name) and top.repositories: 74 mode = "c" 75 for d in [top] + top.repositories: 76 if dir.is_under(d): 77 try: 78 return DataBase[d], mode 79 except KeyError: 80 path = d.entry_abspath(DB_Name) 81 try: db = DataBase[d] = DB_Module.open(path, mode) 82 except (IOError, OSError): 83 pass 84 else: 85 if mode != "r": 86 DB_sync_list.append(db) 87 return db, mode 88 mode = "r" 89 try: 90 return DataBase[top], "c" 91 except KeyError: 92 db = DataBase[top] = DB_Module.open(DB_Name, "c") 93 DB_sync_list.append(db) 94 return db, "c" 95 except TypeError: 96 print("DataBase =", DataBase) 97 raise 98 99 100def Reset(): 101 """Reset global state. Used by unit tests that end up using 102 SConsign multiple times to get a clean slate for each test.""" 103 global sig_files, DB_sync_list 104 sig_files = [] 105 DB_sync_list = [] 106 107normcase = os.path.normcase 108 109 110def write(): 111 global sig_files 112 113 if print_time(): 114 start_time = time.perf_counter() 115 116 for sig_file in sig_files: 117 sig_file.write(sync=0) 118 for db in DB_sync_list: 119 try: 120 syncmethod = db.sync 121 except AttributeError: 122 pass # Not all dbm modules have sync() methods. 123 else: 124 syncmethod() 125 try: 126 closemethod = db.close 127 except AttributeError: 128 pass # Not all dbm modules have close() methods. 129 else: 130 closemethod() 131 132 if print_time(): 133 elapsed = time.perf_counter() - start_time 134 print('Total SConsign sync time: %f seconds' % elapsed) 135 136 137class SConsignEntry: 138 """ 139 Wrapper class for the generic entry in a .sconsign file. 140 The Node subclass populates it with attributes as it pleases. 141 142 XXX As coded below, we do expect a '.binfo' attribute to be added, 143 but we'll probably generalize this in the next refactorings. 144 """ 145 __slots__ = ("binfo", "ninfo", "__weakref__") 146 current_version_id = 2 147 148 def __init__(self): 149 # Create an object attribute from the class attribute so it ends up 150 # in the pickled data in the .sconsign file. 151 #_version_id = self.current_version_id 152 pass 153 154 def convert_to_sconsign(self): 155 self.binfo.convert_to_sconsign() 156 157 def convert_from_sconsign(self, dir, name): 158 self.binfo.convert_from_sconsign(dir, name) 159 160 def __getstate__(self): 161 state = getattr(self, '__dict__', {}).copy() 162 for obj in type(self).mro(): 163 for name in getattr(obj, '__slots__', ()): 164 if hasattr(self, name): 165 state[name] = getattr(self, name) 166 167 state['_version_id'] = self.current_version_id 168 try: 169 del state['__weakref__'] 170 except KeyError: 171 pass 172 return state 173 174 def __setstate__(self, state): 175 for key, value in state.items(): 176 if key not in ('_version_id', '__weakref__'): 177 setattr(self, key, value) 178 179 180class Base: 181 """ 182 This is the controlling class for the signatures for the collection of 183 entries associated with a specific directory. The actual directory 184 association will be maintained by a subclass that is specific to 185 the underlying storage method. This class provides a common set of 186 methods for fetching and storing the individual bits of information 187 that make up signature entry. 188 """ 189 def __init__(self): 190 self.entries = {} 191 self.dirty = False 192 self.to_be_merged = {} 193 194 def get_entry(self, filename): 195 """ 196 Fetch the specified entry attribute. 197 """ 198 return self.entries[filename] 199 200 def set_entry(self, filename, obj): 201 """ 202 Set the entry. 203 """ 204 self.entries[filename] = obj 205 self.dirty = True 206 207 def do_not_set_entry(self, filename, obj): 208 pass 209 210 def store_info(self, filename, node): 211 entry = node.get_stored_info() 212 entry.binfo.merge(node.get_binfo()) 213 self.to_be_merged[filename] = node 214 self.dirty = True 215 216 def do_not_store_info(self, filename, node): 217 pass 218 219 def merge(self): 220 for key, node in self.to_be_merged.items(): 221 entry = node.get_stored_info() 222 try: 223 ninfo = entry.ninfo 224 except AttributeError: 225 # This happens with SConf Nodes, because the configuration 226 # subsystem takes direct control over how the build decision 227 # is made and its information stored. 228 pass 229 else: 230 ninfo.merge(node.get_ninfo()) 231 self.entries[key] = entry 232 self.to_be_merged = {} 233 234 235class DB(Base): 236 """ 237 A Base subclass that reads and writes signature information 238 from a global .sconsign.db* file--the actual file suffix is 239 determined by the database module. 240 """ 241 def __init__(self, dir): 242 Base.__init__(self) 243 244 self.dir = dir 245 246 db, mode = Get_DataBase(dir) 247 248 # Read using the path relative to the top of the Repository 249 # (self.dir.tpath) from which we're fetching the signature 250 # information. 251 path = normcase(dir.get_tpath()) 252 try: 253 rawentries = db[path] 254 except KeyError: 255 pass 256 else: 257 try: 258 self.entries = pickle.loads(rawentries) 259 if not isinstance(self.entries, dict): 260 self.entries = {} 261 raise TypeError 262 except KeyboardInterrupt: 263 raise 264 except Exception as e: 265 SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning, 266 "Ignoring corrupt sconsign entry : %s (%s)\n"%(self.dir.get_tpath(), e)) 267 for key, entry in self.entries.items(): 268 entry.convert_from_sconsign(dir, key) 269 270 if mode == "r": 271 # This directory is actually under a repository, which means 272 # likely they're reaching in directly for a dependency on 273 # a file there. Don't actually set any entry info, so we 274 # won't try to write to that .sconsign.dblite file. 275 self.set_entry = self.do_not_set_entry 276 self.store_info = self.do_not_store_info 277 278 global sig_files 279 sig_files.append(self) 280 281 def write(self, sync=1): 282 if not self.dirty: 283 return 284 285 self.merge() 286 287 db, mode = Get_DataBase(self.dir) 288 289 # Write using the path relative to the top of the SConstruct 290 # directory (self.dir.path), not relative to the top of 291 # the Repository; we only write to our own .sconsign file, 292 # not to .sconsign files in Repositories. 293 path = normcase(self.dir.get_internal_path()) 294 for key, entry in self.entries.items(): 295 entry.convert_to_sconsign() 296 db[path] = pickle.dumps(self.entries, PICKLE_PROTOCOL) 297 298 if sync: 299 try: 300 syncmethod = db.sync 301 except AttributeError: 302 # Not all anydbm modules have sync() methods. 303 pass 304 else: 305 syncmethod() 306 307 308class Dir(Base): 309 def __init__(self, fp=None, dir=None): 310 """ 311 fp - file pointer to read entries from 312 """ 313 Base.__init__(self) 314 315 if not fp: 316 return 317 318 self.entries = pickle.load(fp) 319 if not isinstance(self.entries, dict): 320 self.entries = {} 321 raise TypeError 322 323 if dir: 324 for key, entry in self.entries.items(): 325 entry.convert_from_sconsign(dir, key) 326 327 328class DirFile(Dir): 329 """ 330 Encapsulates reading and writing a per-directory .sconsign file. 331 """ 332 def __init__(self, dir): 333 """ 334 dir - the directory for the file 335 """ 336 337 self.dir = dir 338 self.sconsign = os.path.join(dir.get_internal_path(), '.sconsign') 339 340 try: 341 fp = open(self.sconsign, 'rb') 342 except IOError: 343 fp = None 344 345 try: 346 Dir.__init__(self, fp, dir) 347 except KeyboardInterrupt: 348 raise 349 except Exception: 350 SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning, 351 "Ignoring corrupt .sconsign file: %s"%self.sconsign) 352 353 try: 354 fp.close() 355 except AttributeError: 356 pass 357 358 global sig_files 359 sig_files.append(self) 360 361 def write(self, sync=1): 362 """ 363 Write the .sconsign file to disk. 364 365 Try to write to a temporary file first, and rename it if we 366 succeed. If we can't write to the temporary file, it's 367 probably because the directory isn't writable (and if so, 368 how did we build anything in this directory, anyway?), so 369 try to write directly to the .sconsign file as a backup. 370 If we can't rename, try to copy the temporary contents back 371 to the .sconsign file. Either way, always try to remove 372 the temporary file at the end. 373 """ 374 if not self.dirty: 375 return 376 377 self.merge() 378 379 temp = os.path.join(self.dir.get_internal_path(), '.scons%d' % os.getpid()) 380 try: 381 file = open(temp, 'wb') 382 fname = temp 383 except IOError: 384 try: 385 file = open(self.sconsign, 'wb') 386 fname = self.sconsign 387 except IOError: 388 return 389 for key, entry in self.entries.items(): 390 entry.convert_to_sconsign() 391 pickle.dump(self.entries, file, PICKLE_PROTOCOL) 392 file.close() 393 if fname != self.sconsign: 394 try: 395 mode = os.stat(self.sconsign)[0] 396 os.chmod(self.sconsign, 0o666) 397 os.unlink(self.sconsign) 398 except (IOError, OSError): 399 # Try to carry on in the face of either OSError 400 # (things like permission issues) or IOError (disk 401 # or network issues). If there's a really dangerous 402 # issue, it should get re-raised by the calls below. 403 pass 404 try: 405 os.rename(fname, self.sconsign) 406 except OSError: 407 # An OSError failure to rename may indicate something 408 # like the directory has no write permission, but 409 # the .sconsign file itself might still be writable, 410 # so try writing on top of it directly. An IOError 411 # here, or in any of the following calls, would get 412 # raised, indicating something like a potentially 413 # serious disk or network issue. 414 with open(self.sconsign, 'wb') as f, open(fname, 'rb') as f2: 415 f.write(f2.read()) 416 os.chmod(self.sconsign, mode) 417 try: 418 os.unlink(temp) 419 except (IOError, OSError): 420 pass 421 422ForDirectory = DB 423 424 425def File(name, dbm_module=None): 426 """ 427 Arrange for all signatures to be stored in a global .sconsign.db* 428 file. 429 """ 430 global ForDirectory, DB_Name, DB_Module 431 if name is None: 432 ForDirectory = DirFile 433 DB_Module = None 434 else: 435 ForDirectory = DB 436 DB_Name = name 437 if dbm_module is not None: 438 DB_Module = dbm_module 439 440# Local Variables: 441# tab-width:4 442# indent-tabs-mode:nil 443# End: 444# vim: set expandtab tabstop=4 shiftwidth=4: 445