1 //===- lib/Tooling/AllTUsExecution.cpp - Execute actions on all TUs. ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/AllTUsExecution.h"
10 #include "clang/Tooling/ToolExecutorPluginRegistry.h"
11 #include "llvm/Support/Regex.h"
12 #include "llvm/Support/ThreadPool.h"
13 #include "llvm/Support/Threading.h"
14 #include "llvm/Support/VirtualFileSystem.h"
15 
16 namespace clang {
17 namespace tooling {
18 
19 const char *AllTUsToolExecutor::ExecutorName = "AllTUsToolExecutor";
20 
21 namespace {
22 llvm::Error make_string_error(const llvm::Twine &Message) {
23   return llvm::make_error<llvm::StringError>(Message,
24                                              llvm::inconvertibleErrorCode());
25 }
26 
27 ArgumentsAdjuster getDefaultArgumentsAdjusters() {
28   return combineAdjusters(
29       getClangStripOutputAdjuster(),
30       combineAdjusters(getClangSyntaxOnlyAdjuster(),
31                        getClangStripDependencyFileAdjuster()));
32 }
33 
34 class ThreadSafeToolResults : public ToolResults {
35 public:
36   void addResult(StringRef Key, StringRef Value) override {
37     std::unique_lock<std::mutex> LockGuard(Mutex);
38     Results.addResult(Key, Value);
39   }
40 
41   std::vector<std::pair<llvm::StringRef, llvm::StringRef>>
42   AllKVResults() override {
43     return Results.AllKVResults();
44   }
45 
46   void forEachResult(llvm::function_ref<void(StringRef Key, StringRef Value)>
47                          Callback) override {
48     Results.forEachResult(Callback);
49   }
50 
51 private:
52   InMemoryToolResults Results;
53   std::mutex Mutex;
54 };
55 
56 } // namespace
57 
58 llvm::cl::opt<std::string>
59     Filter("filter",
60            llvm::cl::desc("Only process files that match this filter. "
61                           "This flag only applies to all-TUs."),
62            llvm::cl::init(".*"));
63 
64 AllTUsToolExecutor::AllTUsToolExecutor(
65     const CompilationDatabase &Compilations, unsigned ThreadCount,
66     std::shared_ptr<PCHContainerOperations> PCHContainerOps)
67     : Compilations(Compilations), Results(new ThreadSafeToolResults),
68       Context(Results.get()), ThreadCount(ThreadCount) {}
69 
70 AllTUsToolExecutor::AllTUsToolExecutor(
71     CommonOptionsParser Options, unsigned ThreadCount,
72     std::shared_ptr<PCHContainerOperations> PCHContainerOps)
73     : OptionsParser(std::move(Options)),
74       Compilations(OptionsParser->getCompilations()),
75       Results(new ThreadSafeToolResults), Context(Results.get()),
76       ThreadCount(ThreadCount) {}
77 
78 llvm::Error AllTUsToolExecutor::execute(
79     llvm::ArrayRef<
80         std::pair<std::unique_ptr<FrontendActionFactory>, ArgumentsAdjuster>>
81         Actions) {
82   if (Actions.empty())
83     return make_string_error("No action to execute.");
84 
85   if (Actions.size() != 1)
86     return make_string_error(
87         "Only support executing exactly 1 action at this point.");
88 
89   std::string ErrorMsg;
90   std::mutex TUMutex;
91   auto AppendError = [&](llvm::Twine Err) {
92     std::unique_lock<std::mutex> LockGuard(TUMutex);
93     ErrorMsg += Err.str();
94   };
95 
96   auto Log = [&](llvm::Twine Msg) {
97     std::unique_lock<std::mutex> LockGuard(TUMutex);
98     llvm::errs() << Msg.str() << "\n";
99   };
100 
101   std::vector<std::string> Files;
102   llvm::Regex RegexFilter(Filter);
103   for (const auto& File : Compilations.getAllFiles()) {
104     if (RegexFilter.match(File))
105       Files.push_back(File);
106   }
107   // Add a counter to track the progress.
108   const std::string TotalNumStr = std::to_string(Files.size());
109   unsigned Counter = 0;
110   auto Count = [&]() {
111     std::unique_lock<std::mutex> LockGuard(TUMutex);
112     return ++Counter;
113   };
114 
115   auto &Action = Actions.front();
116 
117   {
118     llvm::ThreadPool Pool(llvm::hardware_concurrency(ThreadCount));
119     for (std::string File : Files) {
120       Pool.async(
121           [&](std::string Path) {
122             Log("[" + std::to_string(Count()) + "/" + TotalNumStr +
123                 "] Processing file " + Path);
124             // Each thread gets an independent copy of a VFS to allow different
125             // concurrent working directories.
126             IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS =
127                 llvm::vfs::createPhysicalFileSystem();
128             ClangTool Tool(Compilations, {Path},
129                            std::make_shared<PCHContainerOperations>(), FS);
130             Tool.appendArgumentsAdjuster(Action.second);
131             Tool.appendArgumentsAdjuster(getDefaultArgumentsAdjusters());
132             for (const auto &FileAndContent : OverlayFiles)
133               Tool.mapVirtualFile(FileAndContent.first(),
134                                   FileAndContent.second);
135             if (Tool.run(Action.first.get()))
136               AppendError(llvm::Twine("Failed to run action on ") + Path +
137                           "\n");
138           },
139           File);
140     }
141     // Make sure all tasks have finished before resetting the working directory.
142     Pool.wait();
143   }
144 
145   if (!ErrorMsg.empty())
146     return make_string_error(ErrorMsg);
147 
148   return llvm::Error::success();
149 }
150 
151 llvm::cl::opt<unsigned> ExecutorConcurrency(
152     "execute-concurrency",
153     llvm::cl::desc("The number of threads used to process all files in "
154                    "parallel. Set to 0 for hardware concurrency. "
155                    "This flag only applies to all-TUs."),
156     llvm::cl::init(0));
157 
158 class AllTUsToolExecutorPlugin : public ToolExecutorPlugin {
159 public:
160   llvm::Expected<std::unique_ptr<ToolExecutor>>
161   create(CommonOptionsParser &OptionsParser) override {
162     if (OptionsParser.getSourcePathList().empty())
163       return make_string_error(
164           "[AllTUsToolExecutorPlugin] Please provide a directory/file path in "
165           "the compilation database.");
166     return std::make_unique<AllTUsToolExecutor>(std::move(OptionsParser),
167                                                  ExecutorConcurrency);
168   }
169 };
170 
171 static ToolExecutorPluginRegistry::Add<AllTUsToolExecutorPlugin>
172     X("all-TUs", "Runs FrontendActions on all TUs in the compilation database. "
173                  "Tool results are stored in memory.");
174 
175 // This anchor is used to force the linker to link in the generated object file
176 // and thus register the plugin.
177 volatile int AllTUsToolExecutorAnchorSource = 0;
178 
179 } // end namespace tooling
180 } // end namespace clang
181