1#! /usr/bin/env python 2# encoding: utf-8 3 4""" 5Use extended attributes instead of database files 6 71. Input files will be made writable 82. This is only for systems providing extended filesystem attributes 93. By default, hashes are calculated only if timestamp/size change (HASH_CACHE below) 104. The module enables "deep_inputs" on all tasks by propagating task signatures 115. This module also skips task signature comparisons for task code changes due to point 4. 126. This module is for Python3/Linux only, but it could be extended to Python2/other systems 13 using the xattr library 147. For projects in which tasks always declare output files, it should be possible to 15 store the rest of build context attributes on output files (imp_sigs, raw_deps and node_deps) 16 but this is not done here 17 18On a simple C++ project benchmark, the variations before and after adding waf_xattr.py were observed: 19total build time: 20s -> 22s 20no-op build time: 2.4s -> 1.8s 21pickle file size: 2.9MB -> 2.6MB 22""" 23 24import os 25from waflib import Logs, Node, Task, Utils, Errors 26from waflib.Task import SKIP_ME, RUN_ME, CANCEL_ME, ASK_LATER, SKIPPED, MISSING 27 28HASH_CACHE = True 29SIG_VAR = 'user.waf.sig' 30SEP = ','.encode() 31TEMPLATE = '%b%d,%d'.encode() 32 33try: 34 PermissionError 35except NameError: 36 PermissionError = IOError 37 38def getxattr(self): 39 return os.getxattr(self.abspath(), SIG_VAR) 40 41def setxattr(self, val): 42 os.setxattr(self.abspath(), SIG_VAR, val) 43 44def h_file(self): 45 try: 46 ret = getxattr(self) 47 except OSError: 48 if HASH_CACHE: 49 st = os.stat(self.abspath()) 50 mtime = st.st_mtime 51 size = st.st_size 52 else: 53 if len(ret) == 16: 54 # for build directory files 55 return ret 56 57 if HASH_CACHE: 58 # check if timestamp and mtime match to avoid re-hashing 59 st = os.stat(self.abspath()) 60 mtime, size = ret[16:].split(SEP) 61 if int(1000 * st.st_mtime) == int(mtime) and st.st_size == int(size): 62 return ret[:16] 63 64 ret = Utils.h_file(self.abspath()) 65 if HASH_CACHE: 66 val = TEMPLATE % (ret, int(1000 * st.st_mtime), int(st.st_size)) 67 try: 68 setxattr(self, val) 69 except PermissionError: 70 os.chmod(self.abspath(), st.st_mode | 128) 71 setxattr(self, val) 72 return ret 73 74def runnable_status(self): 75 bld = self.generator.bld 76 if bld.is_install < 0: 77 return SKIP_ME 78 79 for t in self.run_after: 80 if not t.hasrun: 81 return ASK_LATER 82 elif t.hasrun < SKIPPED: 83 # a dependency has an error 84 return CANCEL_ME 85 86 # first compute the signature 87 try: 88 new_sig = self.signature() 89 except Errors.TaskNotReady: 90 return ASK_LATER 91 92 if not self.outputs: 93 # compare the signature to a signature computed previously 94 # this part is only for tasks with no output files 95 key = self.uid() 96 try: 97 prev_sig = bld.task_sigs[key] 98 except KeyError: 99 Logs.debug('task: task %r must run: it was never run before or the task code changed', self) 100 return RUN_ME 101 if new_sig != prev_sig: 102 Logs.debug('task: task %r must run: the task signature changed', self) 103 return RUN_ME 104 105 # compare the signatures of the outputs to make a decision 106 for node in self.outputs: 107 try: 108 sig = node.h_file() 109 except EnvironmentError: 110 Logs.debug('task: task %r must run: an output node does not exist', self) 111 return RUN_ME 112 if sig != new_sig: 113 Logs.debug('task: task %r must run: an output node is stale', self) 114 return RUN_ME 115 116 return (self.always_run and RUN_ME) or SKIP_ME 117 118def post_run(self): 119 bld = self.generator.bld 120 sig = self.signature() 121 for node in self.outputs: 122 if not node.exists(): 123 self.hasrun = MISSING 124 self.err_msg = '-> missing file: %r' % node.abspath() 125 raise Errors.WafError(self.err_msg) 126 os.setxattr(node.abspath(), 'user.waf.sig', sig) 127 if not self.outputs: 128 # only for task with no outputs 129 bld.task_sigs[self.uid()] = sig 130 if not self.keep_last_cmd: 131 try: 132 del self.last_cmd 133 except AttributeError: 134 pass 135 136try: 137 os.getxattr 138except AttributeError: 139 pass 140else: 141 h_file.__doc__ = Node.Node.h_file.__doc__ 142 143 # keep file hashes as file attributes 144 Node.Node.h_file = h_file 145 146 # enable "deep_inputs" on all tasks 147 Task.Task.runnable_status = runnable_status 148 Task.Task.post_run = post_run 149 Task.Task.sig_deep_inputs = Utils.nada 150 151