1#!/usr/bin/env python3 2# 3# This Source Code Form is subject to the terms of the Mozilla Public 4# License, v. 2.0. If a copy of the MPL was not distributed with this 5# file, You can obtain one at http://mozilla.org/MPL/2.0/. 6# 7# This parses the output of 'include-what-you-use', focusing on just removing 8# not needed includes and providing a relatively conservative output by 9# filtering out a number of LibreOffice-specific false positives. 10# 11# It assumes you have a 'compile_commands.json' around (similar to clang-tidy), 12# you can generate one with 'make vim-ide-integration'. 13# 14# Design goals: 15# - excludelist mechanism, so a warning is either fixed or excluded 16# - works in a plugins-enabled clang build 17# - no custom configure options required 18# - no need to generate a dummy library to build a header 19 20import glob 21import json 22import multiprocessing 23import os 24import queue 25import re 26import subprocess 27import sys 28import threading 29import yaml 30 31 32def ignoreRemoval(include, toAdd, absFileName, moduleRules): 33 # global rules 34 35 # Avoid replacing .hpp with .hdl in the com::sun::star and ooo::vba namespaces. 36 if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"): 37 hdl = include.replace(".hpp", ".hdl") 38 if hdl in toAdd: 39 return True 40 41 # Avoid debug STL. 42 debugStl = { 43 "array": ("debug/array", ), 44 "bitset": ("debug/bitset", ), 45 "deque": ("debug/deque", ), 46 "forward_list": ("debug/forward_list", ), 47 "list": ("debug/list", ), 48 "map": ("debug/map.h", "debug/multimap.h"), 49 "set": ("debug/set.h", "debug/multiset.h"), 50 "unordered_map": ("debug/unordered_map", ), 51 "unordered_set": ("debug/unordered_set", ), 52 "vector": ("debug/vector", ), 53 } 54 for k, values in debugStl.items(): 55 if include == k: 56 for value in values: 57 if value in toAdd: 58 return True 59 60 # Avoid proposing to use libstdc++ internal headers. 61 bits = { 62 "exception": "bits/exception.h", 63 "memory": "bits/shared_ptr.h", 64 "functional": "bits/std_function.h", 65 "cmath": "bits/std_abs.h", 66 "ctime": "bits/types/clock_t.h", 67 "cstdint": "bits/stdint-uintn.h", 68 } 69 for k, v in bits.items(): 70 if include == k and v in toAdd: 71 return True 72 73 # Avoid proposing o3tl fw declaration 74 o3tl = { 75 "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }", 76 "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }", 77 "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }", 78 } 79 for k, v, in o3tl.items(): 80 if include == k and v in toAdd: 81 return True 82 83 # Follow boost documentation. 84 if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd: 85 return True 86 if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd: 87 return True 88 if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd: 89 return True 90 if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd: 91 return True 92 if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd: 93 return True 94 95 # Avoid .hxx to .h proposals in basic css/uno/* API 96 unoapi = { 97 "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h", 98 "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h", 99 "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h", 100 "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h" 101 } 102 for k, v in unoapi.items(): 103 if include == k and v in toAdd: 104 return True 105 106 # 3rd-party, non-self-contained headers. 107 if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd: 108 return True 109 if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd: 110 return True 111 if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd: 112 return True 113 114 noRemove = ( 115 # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not 116 # removing this. 117 "sal/config.h", 118 # Works around a build breakage specific to the broken Android 119 # toolchain. 120 "android/compatibility.hxx", 121 # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'. 122 "osl/endian.h", 123 ) 124 if include in noRemove: 125 return True 126 127 # Ignore when <foo> is to be replaced with "foo". 128 if include in toAdd: 129 return True 130 131 fileName = os.path.relpath(absFileName, os.getcwd()) 132 133 # Skip headers used only for compile test 134 if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx": 135 if include.endswith(".hpp"): 136 return True 137 138 # yaml rules 139 140 if "excludelist" in moduleRules.keys(): 141 excludelistRules = moduleRules["excludelist"] 142 if fileName in excludelistRules.keys(): 143 if include in excludelistRules[fileName]: 144 return True 145 146 return False 147 148 149def unwrapInclude(include): 150 # Drop <> or "" around the include. 151 return include[1:-1] 152 153 154def processIWYUOutput(iwyuOutput, moduleRules, fileName): 155 inAdd = False 156 toAdd = [] 157 inRemove = False 158 toRemove = [] 159 currentFileName = None 160 161 for line in iwyuOutput: 162 line = line.strip() 163 164 # Bail out if IWYU gave an error due to non self-containedness 165 if re.match ("(.*): error: (.*)", line): 166 return -1 167 168 if len(line) == 0: 169 if inRemove: 170 inRemove = False 171 continue 172 if inAdd: 173 inAdd = False 174 continue 175 176 shouldAdd = fileName + " should add these lines:" 177 match = re.match(shouldAdd, line) 178 if match: 179 currentFileName = match.group(0).split(' ')[0] 180 inAdd = True 181 continue 182 183 shouldRemove = fileName + " should remove these lines:" 184 match = re.match(shouldRemove, line) 185 if match: 186 currentFileName = match.group(0).split(' ')[0] 187 inRemove = True 188 continue 189 190 if inAdd: 191 match = re.match('#include ([^ ]+)', line) 192 if match: 193 include = unwrapInclude(match.group(1)) 194 toAdd.append(include) 195 else: 196 # Forward declaration. 197 toAdd.append(line) 198 199 if inRemove: 200 match = re.match("- #include (.*) // lines (.*)-.*", line) 201 if match: 202 # Only suggest removals for now. Removing fwd decls is more complex: they may be 203 # indeed unused or they may removed to be replaced with an include. And we want to 204 # avoid the later. 205 include = unwrapInclude(match.group(1)) 206 lineno = match.group(2) 207 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules): 208 toRemove.append("%s:%s: %s" % (currentFileName, lineno, include)) 209 210 for remove in sorted(toRemove): 211 print("ERROR: %s: remove not needed include" % remove) 212 return len(toRemove) 213 214 215def run_tool(task_queue, failed_files): 216 while True: 217 invocation, moduleRules = task_queue.get() 218 if not len(failed_files): 219 print("[IWYU] " + invocation.split(' ')[-1]) 220 p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 221 retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1]) 222 if retcode == -1: 223 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation) 224 elif retcode > 0: 225 print("ERROR: The following command found unused includes:\n" + invocation) 226 failed_files.append(invocation) 227 task_queue.task_done() 228 229 230def isInUnoIncludeFile(path): 231 return path.startswith("include/com/") \ 232 or path.startswith("include/cppu/") \ 233 or path.startswith("include/cppuhelper/") \ 234 or path.startswith("include/osl/") \ 235 or path.startswith("include/rtl/") \ 236 or path.startswith("include/sal/") \ 237 or path.startswith("include/salhelper/") \ 238 or path.startswith("include/systools/") \ 239 or path.startswith("include/typelib/") \ 240 or path.startswith("include/uno/") 241 242 243def tidy(compileCommands, paths): 244 return_code = 0 245 try: 246 max_task = multiprocessing.cpu_count() 247 task_queue = queue.Queue(max_task) 248 failed_files = [] 249 for _ in range(max_task): 250 t = threading.Thread(target=run_tool, args=(task_queue, failed_files)) 251 t.daemon = True 252 t.start() 253 254 for path in sorted(paths): 255 if isInUnoIncludeFile(path): 256 continue 257 258 moduleName = path.split("/")[0] 259 260 rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml") 261 moduleRules = {} 262 if os.path.exists(rulePath): 263 moduleRules = yaml.full_load(open(rulePath)) 264 assume = None 265 pathAbs = os.path.abspath(path) 266 compileFile = pathAbs 267 matches = [i for i in compileCommands if i["file"] == compileFile] 268 if not len(matches): 269 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only 270 # code on Linux. 271 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"): 272 assume = moduleRules["assumeFilename"] 273 if assume: 274 assumeAbs = os.path.abspath(assume) 275 compileFile = assumeAbs 276 matches = [i for i in compileCommands if i["file"] == compileFile] 277 if not len(matches): 278 print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'") 279 continue 280 else: 281 print("WARNING: no compile commands for '" + path + "'") 282 continue 283 284 _, _, args = matches[0]["command"].partition(" ") 285 if assume: 286 args = args.replace(assumeAbs, "-x c++ " + pathAbs) 287 288 invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args 289 task_queue.put((invocation, moduleRules)) 290 291 task_queue.join() 292 if len(failed_files): 293 return_code = 1 294 295 except KeyboardInterrupt: 296 print('\nCtrl-C detected, goodbye.') 297 os.kill(0, 9) 298 299 sys.exit(return_code) 300 301 302def main(argv): 303 if not len(argv): 304 print("usage: find-unneeded-includes [FILE]...") 305 return 306 307 try: 308 with open("compile_commands.json", 'r') as compileCommandsSock: 309 compileCommands = json.load(compileCommandsSock) 310 except FileNotFoundError: 311 print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration") 312 sys.exit(-1) 313 314 tidy(compileCommands, paths=argv) 315 316if __name__ == '__main__': 317 main(sys.argv[1:]) 318 319# vim:set shiftwidth=4 softtabstop=4 expandtab: 320