1#! /usr/bin/env python 2# encoding: utf-8 3 4""" 5Windows-specific optimizations 6 7This module can help reducing the overhead of listing files on windows 8(more than 10000 files). Python 3.5 already provides the listdir 9optimization though. 10""" 11 12import os 13from waflib import Utils, Build, Node, Logs 14 15try: 16 TP = '%s\\*'.decode('ascii') 17except AttributeError: 18 TP = '%s\\*' 19 20if Utils.is_win32: 21 from waflib.Tools import md5_tstamp 22 import ctypes, ctypes.wintypes 23 24 FindFirstFile = ctypes.windll.kernel32.FindFirstFileW 25 FindNextFile = ctypes.windll.kernel32.FindNextFileW 26 FindClose = ctypes.windll.kernel32.FindClose 27 FILE_ATTRIBUTE_DIRECTORY = 0x10 28 INVALID_HANDLE_VALUE = -1 29 UPPER_FOLDERS = ('.', '..') 30 try: 31 UPPER_FOLDERS = [unicode(x) for x in UPPER_FOLDERS] 32 except NameError: 33 pass 34 35 def cached_hash_file(self): 36 try: 37 cache = self.ctx.cache_listdir_cache_hash_file 38 except AttributeError: 39 cache = self.ctx.cache_listdir_cache_hash_file = {} 40 41 if id(self.parent) in cache: 42 try: 43 t = cache[id(self.parent)][self.name] 44 except KeyError: 45 raise IOError('Not a file') 46 else: 47 # an opportunity to list the files and the timestamps at once 48 findData = ctypes.wintypes.WIN32_FIND_DATAW() 49 find = FindFirstFile(TP % self.parent.abspath(), ctypes.byref(findData)) 50 51 if find == INVALID_HANDLE_VALUE: 52 cache[id(self.parent)] = {} 53 raise IOError('Not a file') 54 55 cache[id(self.parent)] = lst_files = {} 56 try: 57 while True: 58 if findData.cFileName not in UPPER_FOLDERS: 59 thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY 60 if not thatsadir: 61 ts = findData.ftLastWriteTime 62 d = (ts.dwLowDateTime << 32) | ts.dwHighDateTime 63 lst_files[str(findData.cFileName)] = d 64 if not FindNextFile(find, ctypes.byref(findData)): 65 break 66 except Exception: 67 cache[id(self.parent)] = {} 68 raise IOError('Not a file') 69 finally: 70 FindClose(find) 71 t = lst_files[self.name] 72 73 fname = self.abspath() 74 if fname in Build.hashes_md5_tstamp: 75 if Build.hashes_md5_tstamp[fname][0] == t: 76 return Build.hashes_md5_tstamp[fname][1] 77 78 try: 79 fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT) 80 except OSError: 81 raise IOError('Cannot read from %r' % fname) 82 f = os.fdopen(fd, 'rb') 83 m = Utils.md5() 84 rb = 1 85 try: 86 while rb: 87 rb = f.read(200000) 88 m.update(rb) 89 finally: 90 f.close() 91 92 # ensure that the cache is overwritten 93 Build.hashes_md5_tstamp[fname] = (t, m.digest()) 94 return m.digest() 95 Node.Node.cached_hash_file = cached_hash_file 96 97 def get_bld_sig_win32(self): 98 try: 99 return self.ctx.hash_cache[id(self)] 100 except KeyError: 101 pass 102 except AttributeError: 103 self.ctx.hash_cache = {} 104 self.ctx.hash_cache[id(self)] = ret = Utils.h_file(self.abspath()) 105 return ret 106 Node.Node.get_bld_sig = get_bld_sig_win32 107 108 def isfile_cached(self): 109 # optimize for nt.stat calls, assuming there are many files for few folders 110 try: 111 cache = self.__class__.cache_isfile_cache 112 except AttributeError: 113 cache = self.__class__.cache_isfile_cache = {} 114 115 try: 116 c1 = cache[id(self.parent)] 117 except KeyError: 118 c1 = cache[id(self.parent)] = [] 119 120 curpath = self.parent.abspath() 121 findData = ctypes.wintypes.WIN32_FIND_DATAW() 122 find = FindFirstFile(TP % curpath, ctypes.byref(findData)) 123 124 if find == INVALID_HANDLE_VALUE: 125 Logs.error("invalid win32 handle isfile_cached %r", self.abspath()) 126 return os.path.isfile(self.abspath()) 127 128 try: 129 while True: 130 if findData.cFileName not in UPPER_FOLDERS: 131 thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY 132 if not thatsadir: 133 c1.append(str(findData.cFileName)) 134 if not FindNextFile(find, ctypes.byref(findData)): 135 break 136 except Exception as e: 137 Logs.error('exception while listing a folder %r %r', self.abspath(), e) 138 return os.path.isfile(self.abspath()) 139 finally: 140 FindClose(find) 141 return self.name in c1 142 Node.Node.isfile_cached = isfile_cached 143 144 def find_or_declare_win32(self, lst): 145 # assuming that "find_or_declare" is called before the build starts, remove the calls to os.path.isfile 146 if isinstance(lst, str): 147 lst = [x for x in Utils.split_path(lst) if x and x != '.'] 148 149 node = self.get_bld().search_node(lst) 150 if node: 151 if not node.isfile_cached(): 152 try: 153 node.parent.mkdir() 154 except OSError: 155 pass 156 return node 157 self = self.get_src() 158 node = self.find_node(lst) 159 if node: 160 if not node.isfile_cached(): 161 try: 162 node.parent.mkdir() 163 except OSError: 164 pass 165 return node 166 node = self.get_bld().make_node(lst) 167 node.parent.mkdir() 168 return node 169 Node.Node.find_or_declare = find_or_declare_win32 170 171