1#!/usr/bin/env python3
2#
3# This Source Code Form is subject to the terms of the Mozilla Public
4# License, v. 2.0. If a copy of the MPL was not distributed with this
5# file, You can obtain one at http://mozilla.org/MPL/2.0/.
6#
7# This parses the output of 'include-what-you-use', focusing on just removing
8# not needed includes and providing a relatively conservative output by
9# filtering out a number of LibreOffice-specific false positives.
10#
11# It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
12# you can generate one with 'make vim-ide-integration'.
13#
14# Design goals:
15# - excludelist mechanism, so a warning is either fixed or excluded
16# - works in a plugins-enabled clang build
17# - no custom configure options required
18# - no need to generate a dummy library to build a header
19
20import glob
21import json
22import multiprocessing
23import os
24import queue
25import re
26import subprocess
27import sys
28import threading
29import yaml
30
31
32def ignoreRemoval(include, toAdd, absFileName, moduleRules):
33    # global rules
34
35    # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
36    if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
37        hdl = include.replace(".hpp", ".hdl")
38        if hdl in toAdd:
39            return True
40
41    # Avoid debug STL.
42    debugStl = {
43        "array": ("debug/array", ),
44        "bitset": ("debug/bitset", ),
45        "deque": ("debug/deque", ),
46        "forward_list": ("debug/forward_list", ),
47        "list": ("debug/list", ),
48        "map": ("debug/map.h", "debug/multimap.h"),
49        "set": ("debug/set.h", "debug/multiset.h"),
50        "unordered_map": ("debug/unordered_map", ),
51        "unordered_set": ("debug/unordered_set", ),
52        "vector": ("debug/vector", ),
53    }
54    for k, values in debugStl.items():
55        if include == k:
56            for value in values:
57                if value in toAdd:
58                    return True
59
60    # Avoid proposing to use libstdc++ internal headers.
61    bits = {
62        "exception": "bits/exception.h",
63        "memory": "bits/shared_ptr.h",
64        "functional": "bits/std_function.h",
65        "cmath": "bits/std_abs.h",
66        "ctime": "bits/types/clock_t.h",
67        "cstdint": "bits/stdint-uintn.h",
68    }
69    for k, v in bits.items():
70        if include == k and v in toAdd:
71            return True
72
73    # Avoid proposing o3tl fw declaration
74    o3tl = {
75        "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
76        "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
77        "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }",
78    }
79    for k, v, in o3tl.items():
80        if include == k and v in toAdd:
81            return True
82
83    # Follow boost documentation.
84    if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
85        return True
86    if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
87        return True
88    if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
89        return True
90    if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
91        return True
92    if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
93        return True
94
95    # Avoid .hxx to .h proposals in basic css/uno/* API
96    unoapi = {
97        "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
98        "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
99        "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
100        "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
101    }
102    for k, v in unoapi.items():
103        if include == k and v in toAdd:
104            return True
105
106    # 3rd-party, non-self-contained headers.
107    if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
108        return True
109    if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
110        return True
111    if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
112        return True
113
114    noRemove = (
115        # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
116        # removing this.
117        "sal/config.h",
118        # Works around a build breakage specific to the broken Android
119        # toolchain.
120        "android/compatibility.hxx",
121        # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
122        "osl/endian.h",
123    )
124    if include in noRemove:
125        return True
126
127    # Ignore when <foo> is to be replaced with "foo".
128    if include in toAdd:
129        return True
130
131    fileName = os.path.relpath(absFileName, os.getcwd())
132
133    # Skip headers used only for compile test
134    if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
135        if include.endswith(".hpp"):
136            return True
137
138    # yaml rules
139
140    if "excludelist" in moduleRules.keys():
141        excludelistRules = moduleRules["excludelist"]
142        if fileName in excludelistRules.keys():
143            if include in excludelistRules[fileName]:
144                return True
145
146    return False
147
148
149def unwrapInclude(include):
150    # Drop <> or "" around the include.
151    return include[1:-1]
152
153
154def processIWYUOutput(iwyuOutput, moduleRules, fileName):
155    inAdd = False
156    toAdd = []
157    inRemove = False
158    toRemove = []
159    currentFileName = None
160
161    for line in iwyuOutput:
162        line = line.strip()
163
164        # Bail out if IWYU gave an error due to non self-containedness
165        if re.match ("(.*): error: (.*)", line):
166            return -1
167
168        if len(line) == 0:
169            if inRemove:
170                inRemove = False
171                continue
172            if inAdd:
173                inAdd = False
174                continue
175
176        shouldAdd = fileName + " should add these lines:"
177        match = re.match(shouldAdd, line)
178        if match:
179            currentFileName = match.group(0).split(' ')[0]
180            inAdd = True
181            continue
182
183        shouldRemove = fileName + " should remove these lines:"
184        match = re.match(shouldRemove, line)
185        if match:
186            currentFileName = match.group(0).split(' ')[0]
187            inRemove = True
188            continue
189
190        if inAdd:
191            match = re.match('#include ([^ ]+)', line)
192            if match:
193                include = unwrapInclude(match.group(1))
194                toAdd.append(include)
195            else:
196                # Forward declaration.
197                toAdd.append(line)
198
199        if inRemove:
200            match = re.match("- #include (.*)  // lines (.*)-.*", line)
201            if match:
202                # Only suggest removals for now. Removing fwd decls is more complex: they may be
203                # indeed unused or they may removed to be replaced with an include. And we want to
204                # avoid the later.
205                include = unwrapInclude(match.group(1))
206                lineno = match.group(2)
207                if not ignoreRemoval(include, toAdd, currentFileName, moduleRules):
208                    toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
209
210    for remove in sorted(toRemove):
211        print("ERROR: %s: remove not needed include" % remove)
212    return len(toRemove)
213
214
215def run_tool(task_queue, failed_files):
216    while True:
217        invocation, moduleRules = task_queue.get()
218        if not len(failed_files):
219            print("[IWYU] " + invocation.split(' ')[-1])
220            p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
221            retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1])
222            if retcode == -1:
223                print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
224            elif retcode > 0:
225                print("ERROR: The following command found unused includes:\n" + invocation)
226                failed_files.append(invocation)
227        task_queue.task_done()
228
229
230def isInUnoIncludeFile(path):
231    return path.startswith("include/com/") \
232            or path.startswith("include/cppu/") \
233            or path.startswith("include/cppuhelper/") \
234            or path.startswith("include/osl/") \
235            or path.startswith("include/rtl/") \
236            or path.startswith("include/sal/") \
237            or path.startswith("include/salhelper/") \
238            or path.startswith("include/systools/") \
239            or path.startswith("include/typelib/") \
240            or path.startswith("include/uno/")
241
242
243def tidy(compileCommands, paths):
244    return_code = 0
245    try:
246        max_task = multiprocessing.cpu_count()
247        task_queue = queue.Queue(max_task)
248        failed_files = []
249        for _ in range(max_task):
250            t = threading.Thread(target=run_tool, args=(task_queue, failed_files))
251            t.daemon = True
252            t.start()
253
254        for path in sorted(paths):
255            if isInUnoIncludeFile(path):
256                continue
257
258            moduleName = path.split("/")[0]
259
260            rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
261            moduleRules = {}
262            if os.path.exists(rulePath):
263                moduleRules = yaml.full_load(open(rulePath))
264            assume = None
265            pathAbs = os.path.abspath(path)
266            compileFile = pathAbs
267            matches = [i for i in compileCommands if i["file"] == compileFile]
268            if not len(matches):
269                # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
270                # code on Linux.
271                if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
272                    assume = moduleRules["assumeFilename"]
273                if assume:
274                    assumeAbs = os.path.abspath(assume)
275                    compileFile = assumeAbs
276                    matches = [i for i in compileCommands if i["file"] == compileFile]
277                    if not len(matches):
278                        print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
279                        continue
280                else:
281                    print("WARNING: no compile commands for '" + path + "'")
282                    continue
283
284            _, _, args = matches[0]["command"].partition(" ")
285            if assume:
286                args = args.replace(assumeAbs, "-x c++ " + pathAbs)
287
288            invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
289            task_queue.put((invocation, moduleRules))
290
291        task_queue.join()
292        if len(failed_files):
293            return_code = 1
294
295    except KeyboardInterrupt:
296        print('\nCtrl-C detected, goodbye.')
297        os.kill(0, 9)
298
299    sys.exit(return_code)
300
301
302def main(argv):
303    if not len(argv):
304        print("usage: find-unneeded-includes [FILE]...")
305        return
306
307    try:
308        with open("compile_commands.json", 'r') as compileCommandsSock:
309            compileCommands = json.load(compileCommandsSock)
310    except FileNotFoundError:
311        print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
312        sys.exit(-1)
313
314    tidy(compileCommands, paths=argv)
315
316if __name__ == '__main__':
317    main(sys.argv[1:])
318
319# vim:set shiftwidth=4 softtabstop=4 expandtab:
320