1# MIT License
2#
3# Copyright The SCons Foundation
4#
5# Permission is hereby granted, free of charge, to any person obtaining
6# a copy of this software and associated documentation files (the
7# "Software"), to deal in the Software without restriction, including
8# without limitation the rights to use, copy, modify, merge, publish,
9# distribute, sublicense, and/or sell copies of the Software, and to
10# permit persons to whom the Software is furnished to do so, subject to
11# the following conditions:
12#
13# The above copyright notice and this permission notice shall be included
14# in all copies or substantial portions of the Software.
15#
16# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
17# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
18# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
23
24"""Operations on signature database files (.sconsign). """
25
26import SCons.compat
27
28import os
29import pickle
30import time
31
32import SCons.dblite
33import SCons.Warnings
34from SCons.compat import PICKLE_PROTOCOL
35from SCons.Util import print_time
36
37
38def corrupt_dblite_warning(filename):
39    SCons.Warnings.warn(
40        SCons.Warnings.CorruptSConsignWarning,
41        "Ignoring corrupt .sconsign file: %s" % filename,
42    )
43
44SCons.dblite.IGNORE_CORRUPT_DBFILES = True
45SCons.dblite.corruption_warning = corrupt_dblite_warning
46
47# XXX Get rid of the global array so this becomes re-entrant.
48sig_files = []
49
50# Info for the database SConsign implementation (now the default):
51# "DataBase" is a dictionary that maps top-level SConstruct directories
52# to open database handles.
53# "DB_Module" is the Python database module to create the handles.
54# "DB_Name" is the base name of the database file (minus any
55# extension the underlying DB module will add).
56DataBase = {}
57DB_Module = SCons.dblite
58DB_Name = None
59DB_sync_list = []
60
61
62def Get_DataBase(dir):
63    global DataBase, DB_Module, DB_Name
64
65    if DB_Name is None:
66        hash_format = SCons.Util.get_hash_format()
67        if hash_format is None:
68            DB_Name = ".sconsign"
69        else:
70            DB_Name = ".sconsign_%s" % hash_format
71
72    top = dir.fs.Top
73    if not os.path.isabs(DB_Name) and top.repositories:
74        mode = "c"
75        for d in [top] + top.repositories:
76            if dir.is_under(d):
77                try:
78                    return DataBase[d], mode
79                except KeyError:
80                    path = d.entry_abspath(DB_Name)
81                    try: db = DataBase[d] = DB_Module.open(path, mode)
82                    except (IOError, OSError):
83                        pass
84                    else:
85                        if mode != "r":
86                            DB_sync_list.append(db)
87                        return db, mode
88            mode = "r"
89    try:
90        return DataBase[top], "c"
91    except KeyError:
92        db = DataBase[top] = DB_Module.open(DB_Name, "c")
93        DB_sync_list.append(db)
94        return db, "c"
95    except TypeError:
96        print("DataBase =", DataBase)
97        raise
98
99
100def Reset():
101    """Reset global state.  Used by unit tests that end up using
102    SConsign multiple times to get a clean slate for each test."""
103    global sig_files, DB_sync_list
104    sig_files = []
105    DB_sync_list = []
106
107normcase = os.path.normcase
108
109
110def write():
111    global sig_files
112
113    if print_time():
114        start_time = time.perf_counter()
115
116    for sig_file in sig_files:
117        sig_file.write(sync=0)
118    for db in DB_sync_list:
119        try:
120            syncmethod = db.sync
121        except AttributeError:
122            pass # Not all dbm modules have sync() methods.
123        else:
124            syncmethod()
125        try:
126            closemethod = db.close
127        except AttributeError:
128            pass # Not all dbm modules have close() methods.
129        else:
130            closemethod()
131
132    if print_time():
133        elapsed = time.perf_counter() - start_time
134        print('Total SConsign sync time: %f seconds' % elapsed)
135
136
137class SConsignEntry:
138    """
139    Wrapper class for the generic entry in a .sconsign file.
140    The Node subclass populates it with attributes as it pleases.
141
142    XXX As coded below, we do expect a '.binfo' attribute to be added,
143    but we'll probably generalize this in the next refactorings.
144    """
145    __slots__ = ("binfo", "ninfo", "__weakref__")
146    current_version_id = 2
147
148    def __init__(self):
149        # Create an object attribute from the class attribute so it ends up
150        # in the pickled data in the .sconsign file.
151        #_version_id = self.current_version_id
152        pass
153
154    def convert_to_sconsign(self):
155        self.binfo.convert_to_sconsign()
156
157    def convert_from_sconsign(self, dir, name):
158        self.binfo.convert_from_sconsign(dir, name)
159
160    def __getstate__(self):
161        state = getattr(self, '__dict__', {}).copy()
162        for obj in type(self).mro():
163            for name in getattr(obj, '__slots__', ()):
164                if hasattr(self, name):
165                    state[name] = getattr(self, name)
166
167        state['_version_id'] = self.current_version_id
168        try:
169            del state['__weakref__']
170        except KeyError:
171            pass
172        return state
173
174    def __setstate__(self, state):
175        for key, value in state.items():
176            if key not in ('_version_id', '__weakref__'):
177                setattr(self, key, value)
178
179
180class Base:
181    """
182    This is the controlling class for the signatures for the collection of
183    entries associated with a specific directory.  The actual directory
184    association will be maintained by a subclass that is specific to
185    the underlying storage method.  This class provides a common set of
186    methods for fetching and storing the individual bits of information
187    that make up signature entry.
188    """
189    def __init__(self):
190        self.entries = {}
191        self.dirty = False
192        self.to_be_merged = {}
193
194    def get_entry(self, filename):
195        """
196        Fetch the specified entry attribute.
197        """
198        return self.entries[filename]
199
200    def set_entry(self, filename, obj):
201        """
202        Set the entry.
203        """
204        self.entries[filename] = obj
205        self.dirty = True
206
207    def do_not_set_entry(self, filename, obj):
208        pass
209
210    def store_info(self, filename, node):
211        entry = node.get_stored_info()
212        entry.binfo.merge(node.get_binfo())
213        self.to_be_merged[filename] = node
214        self.dirty = True
215
216    def do_not_store_info(self, filename, node):
217        pass
218
219    def merge(self):
220        for key, node in self.to_be_merged.items():
221            entry = node.get_stored_info()
222            try:
223                ninfo = entry.ninfo
224            except AttributeError:
225                # This happens with SConf Nodes, because the configuration
226                # subsystem takes direct control over how the build decision
227                # is made and its information stored.
228                pass
229            else:
230                ninfo.merge(node.get_ninfo())
231            self.entries[key] = entry
232        self.to_be_merged = {}
233
234
235class DB(Base):
236    """
237    A Base subclass that reads and writes signature information
238    from a global .sconsign.db* file--the actual file suffix is
239    determined by the database module.
240    """
241    def __init__(self, dir):
242        Base.__init__(self)
243
244        self.dir = dir
245
246        db, mode = Get_DataBase(dir)
247
248        # Read using the path relative to the top of the Repository
249        # (self.dir.tpath) from which we're fetching the signature
250        # information.
251        path = normcase(dir.get_tpath())
252        try:
253            rawentries = db[path]
254        except KeyError:
255            pass
256        else:
257            try:
258                self.entries = pickle.loads(rawentries)
259                if not isinstance(self.entries, dict):
260                    self.entries = {}
261                    raise TypeError
262            except KeyboardInterrupt:
263                raise
264            except Exception as e:
265                SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
266                                    "Ignoring corrupt sconsign entry : %s (%s)\n"%(self.dir.get_tpath(), e))
267            for key, entry in self.entries.items():
268                entry.convert_from_sconsign(dir, key)
269
270        if mode == "r":
271            # This directory is actually under a repository, which means
272            # likely they're reaching in directly for a dependency on
273            # a file there.  Don't actually set any entry info, so we
274            # won't try to write to that .sconsign.dblite file.
275            self.set_entry = self.do_not_set_entry
276            self.store_info = self.do_not_store_info
277
278        global sig_files
279        sig_files.append(self)
280
281    def write(self, sync=1):
282        if not self.dirty:
283            return
284
285        self.merge()
286
287        db, mode = Get_DataBase(self.dir)
288
289        # Write using the path relative to the top of the SConstruct
290        # directory (self.dir.path), not relative to the top of
291        # the Repository; we only write to our own .sconsign file,
292        # not to .sconsign files in Repositories.
293        path = normcase(self.dir.get_internal_path())
294        for key, entry in self.entries.items():
295            entry.convert_to_sconsign()
296        db[path] = pickle.dumps(self.entries, PICKLE_PROTOCOL)
297
298        if sync:
299            try:
300                syncmethod = db.sync
301            except AttributeError:
302                # Not all anydbm modules have sync() methods.
303                pass
304            else:
305                syncmethod()
306
307
308class Dir(Base):
309    def __init__(self, fp=None, dir=None):
310        """
311        fp - file pointer to read entries from
312        """
313        Base.__init__(self)
314
315        if not fp:
316            return
317
318        self.entries = pickle.load(fp)
319        if not isinstance(self.entries, dict):
320            self.entries = {}
321            raise TypeError
322
323        if dir:
324            for key, entry in self.entries.items():
325                entry.convert_from_sconsign(dir, key)
326
327
328class DirFile(Dir):
329    """
330    Encapsulates reading and writing a per-directory .sconsign file.
331    """
332    def __init__(self, dir):
333        """
334        dir - the directory for the file
335        """
336
337        self.dir = dir
338        self.sconsign = os.path.join(dir.get_internal_path(), '.sconsign')
339
340        try:
341            fp = open(self.sconsign, 'rb')
342        except IOError:
343            fp = None
344
345        try:
346            Dir.__init__(self, fp, dir)
347        except KeyboardInterrupt:
348            raise
349        except Exception:
350            SCons.Warnings.warn(SCons.Warnings.CorruptSConsignWarning,
351                                "Ignoring corrupt .sconsign file: %s"%self.sconsign)
352
353        try:
354            fp.close()
355        except AttributeError:
356            pass
357
358        global sig_files
359        sig_files.append(self)
360
361    def write(self, sync=1):
362        """
363        Write the .sconsign file to disk.
364
365        Try to write to a temporary file first, and rename it if we
366        succeed.  If we can't write to the temporary file, it's
367        probably because the directory isn't writable (and if so,
368        how did we build anything in this directory, anyway?), so
369        try to write directly to the .sconsign file as a backup.
370        If we can't rename, try to copy the temporary contents back
371        to the .sconsign file.  Either way, always try to remove
372        the temporary file at the end.
373        """
374        if not self.dirty:
375            return
376
377        self.merge()
378
379        temp = os.path.join(self.dir.get_internal_path(), '.scons%d' % os.getpid())
380        try:
381            file = open(temp, 'wb')
382            fname = temp
383        except IOError:
384            try:
385                file = open(self.sconsign, 'wb')
386                fname = self.sconsign
387            except IOError:
388                return
389        for key, entry in self.entries.items():
390            entry.convert_to_sconsign()
391        pickle.dump(self.entries, file, PICKLE_PROTOCOL)
392        file.close()
393        if fname != self.sconsign:
394            try:
395                mode = os.stat(self.sconsign)[0]
396                os.chmod(self.sconsign, 0o666)
397                os.unlink(self.sconsign)
398            except (IOError, OSError):
399                # Try to carry on in the face of either OSError
400                # (things like permission issues) or IOError (disk
401                # or network issues).  If there's a really dangerous
402                # issue, it should get re-raised by the calls below.
403                pass
404            try:
405                os.rename(fname, self.sconsign)
406            except OSError:
407                # An OSError failure to rename may indicate something
408                # like the directory has no write permission, but
409                # the .sconsign file itself might still be writable,
410                # so try writing on top of it directly.  An IOError
411                # here, or in any of the following calls, would get
412                # raised, indicating something like a potentially
413                # serious disk or network issue.
414                with open(self.sconsign, 'wb') as f, open(fname, 'rb') as f2:
415                    f.write(f2.read())
416                os.chmod(self.sconsign, mode)
417        try:
418            os.unlink(temp)
419        except (IOError, OSError):
420            pass
421
422ForDirectory = DB
423
424
425def File(name, dbm_module=None):
426    """
427    Arrange for all signatures to be stored in a global .sconsign.db*
428    file.
429    """
430    global ForDirectory, DB_Name, DB_Module
431    if name is None:
432        ForDirectory = DirFile
433        DB_Module = None
434    else:
435        ForDirectory = DB
436        DB_Name = name
437        if dbm_module is not None:
438            DB_Module = dbm_module
439
440# Local Variables:
441# tab-width:4
442# indent-tabs-mode:nil
443# End:
444# vim: set expandtab tabstop=4 shiftwidth=4:
445