1 //===- ClangScanDeps.cpp - Implementation of clang-scan-deps --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Frontend/CompilerInstance.h"
10 #include "clang/Tooling/CommonOptionsParser.h"
11 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
12 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
13 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
14 #include "clang/Tooling/JSONCompilationDatabase.h"
15 #include "llvm/Support/CommandLine.h"
16 #include "llvm/Support/FileUtilities.h"
17 #include "llvm/Support/InitLLVM.h"
18 #include "llvm/Support/Program.h"
19 #include "llvm/Support/Signals.h"
20 #include "llvm/Support/Threading.h"
21 #include <mutex>
22 #include <thread>
23 
24 using namespace clang;
25 using namespace tooling::dependencies;
26 
27 namespace {
28 
29 class SharedStream {
30 public:
31   SharedStream(raw_ostream &OS) : OS(OS) {}
32   void applyLocked(llvm::function_ref<void(raw_ostream &OS)> Fn) {
33     std::unique_lock<std::mutex> LockGuard(Lock);
34     Fn(OS);
35     OS.flush();
36   }
37 
38 private:
39   std::mutex Lock;
40   raw_ostream &OS;
41 };
42 
43 class ResourceDirectoryCache {
44 public:
45   /// findResourceDir finds the resource directory relative to the clang
46   /// compiler being used in Args, by running it with "-print-resource-dir"
47   /// option and cache the results for reuse. \returns resource directory path
48   /// associated with the given invocation command or empty string if the
49   /// compiler path is NOT an absolute path.
50   StringRef findResourceDir(const tooling::CommandLineArguments &Args) {
51     if (Args.size() < 1)
52       return "";
53 
54     const std::string &ClangBinaryPath = Args[0];
55     if (!llvm::sys::path::is_absolute(ClangBinaryPath))
56       return "";
57 
58     const std::string &ClangBinaryName =
59         llvm::sys::path::filename(ClangBinaryPath);
60 
61     std::unique_lock<std::mutex> LockGuard(CacheLock);
62     const auto &CachedResourceDir = Cache.find(ClangBinaryPath);
63     if (CachedResourceDir != Cache.end())
64       return CachedResourceDir->second;
65 
66     std::vector<StringRef> PrintResourceDirArgs{ClangBinaryName,
67                                                 "-print-resource-dir"};
68     llvm::SmallString<64> OutputFile, ErrorFile;
69     llvm::sys::fs::createTemporaryFile("print-resource-dir-output",
70                                        "" /*no-suffix*/, OutputFile);
71     llvm::sys::fs::createTemporaryFile("print-resource-dir-error",
72                                        "" /*no-suffix*/, ErrorFile);
73     llvm::FileRemover OutputRemover(OutputFile.c_str());
74     llvm::FileRemover ErrorRemover(ErrorFile.c_str());
75     llvm::Optional<StringRef> Redirects[] = {
76         {""}, // Stdin
77         StringRef(OutputFile),
78         StringRef(ErrorFile),
79     };
80     if (const int RC = llvm::sys::ExecuteAndWait(
81             ClangBinaryPath, PrintResourceDirArgs, {}, Redirects)) {
82       auto ErrorBuf = llvm::MemoryBuffer::getFile(ErrorFile.c_str());
83       llvm::errs() << ErrorBuf.get()->getBuffer();
84       return "";
85     }
86 
87     auto OutputBuf = llvm::MemoryBuffer::getFile(OutputFile.c_str());
88     if (!OutputBuf)
89       return "";
90     StringRef Output = OutputBuf.get()->getBuffer().rtrim('\n');
91 
92     Cache[ClangBinaryPath] = Output.str();
93     return Cache[ClangBinaryPath];
94   }
95 
96 private:
97   std::map<std::string, std::string> Cache;
98   std::mutex CacheLock;
99 };
100 
101 llvm::cl::opt<bool> Help("h", llvm::cl::desc("Alias for -help"),
102                          llvm::cl::Hidden);
103 
104 llvm::cl::OptionCategory DependencyScannerCategory("Tool options");
105 
106 static llvm::cl::opt<ScanningMode> ScanMode(
107     "mode",
108     llvm::cl::desc("The preprocessing mode used to compute the dependencies"),
109     llvm::cl::values(
110         clEnumValN(ScanningMode::MinimizedSourcePreprocessing,
111                    "preprocess-minimized-sources",
112                    "The set of dependencies is computed by preprocessing the "
113                    "source files that were minimized to only include the "
114                    "contents that might affect the dependencies"),
115         clEnumValN(ScanningMode::CanonicalPreprocessing, "preprocess",
116                    "The set of dependencies is computed by preprocessing the "
117                    "unmodified source files")),
118     llvm::cl::init(ScanningMode::MinimizedSourcePreprocessing),
119     llvm::cl::cat(DependencyScannerCategory));
120 
121 static llvm::cl::opt<ScanningOutputFormat> Format(
122     "format", llvm::cl::desc("The output format for the dependencies"),
123     llvm::cl::values(clEnumValN(ScanningOutputFormat::Make, "make",
124                                 "Makefile compatible dep file"),
125                      clEnumValN(ScanningOutputFormat::Full, "experimental-full",
126                                 "Full dependency graph suitable"
127                                 " for explicitly building modules. This format "
128                                 "is experimental and will change.")),
129     llvm::cl::init(ScanningOutputFormat::Make),
130     llvm::cl::cat(DependencyScannerCategory));
131 
132 llvm::cl::opt<unsigned>
133     NumThreads("j", llvm::cl::Optional,
134                llvm::cl::desc("Number of worker threads to use (default: use "
135                               "all concurrent threads)"),
136                llvm::cl::init(0), llvm::cl::cat(DependencyScannerCategory));
137 
138 llvm::cl::opt<std::string>
139     CompilationDB("compilation-database",
140                   llvm::cl::desc("Compilation database"), llvm::cl::Required,
141                   llvm::cl::cat(DependencyScannerCategory));
142 
143 llvm::cl::opt<bool> ReuseFileManager(
144     "reuse-filemanager",
145     llvm::cl::desc("Reuse the file manager and its cache between invocations."),
146     llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
147 
148 llvm::cl::opt<bool> SkipExcludedPPRanges(
149     "skip-excluded-pp-ranges",
150     llvm::cl::desc(
151         "Use the preprocessor optimization that skips excluded conditionals by "
152         "bumping the buffer pointer in the lexer instead of lexing the tokens  "
153         "until reaching the end directive."),
154     llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
155 
156 llvm::cl::opt<bool> Verbose("v", llvm::cl::Optional,
157                             llvm::cl::desc("Use verbose output."),
158                             llvm::cl::init(false),
159                             llvm::cl::cat(DependencyScannerCategory));
160 
161 } // end anonymous namespace
162 
163 /// \returns object-file path derived from source-file path.
164 static std::string getObjFilePath(StringRef SrcFile) {
165   SmallString<128> ObjFileName(SrcFile);
166   llvm::sys::path::replace_extension(ObjFileName, "o");
167   return ObjFileName.str();
168 }
169 
170 class SingleCommandCompilationDatabase : public tooling::CompilationDatabase {
171 public:
172   SingleCommandCompilationDatabase(tooling::CompileCommand Cmd)
173       : Command(std::move(Cmd)) {}
174 
175   virtual std::vector<tooling::CompileCommand>
176   getCompileCommands(StringRef FilePath) const {
177     return {Command};
178   }
179 
180   virtual std::vector<tooling::CompileCommand> getAllCompileCommands() const {
181     return {Command};
182   }
183 
184 private:
185   tooling::CompileCommand Command;
186 };
187 
188 /// Takes the result of a dependency scan and prints error / dependency files
189 /// based on the result.
190 ///
191 /// \returns True on error.
192 static bool handleDependencyToolResult(const std::string &Input,
193                                        llvm::Expected<std::string> &MaybeFile,
194                                        SharedStream &OS, SharedStream &Errs) {
195   if (!MaybeFile) {
196     llvm::handleAllErrors(
197         MaybeFile.takeError(), [&Input, &Errs](llvm::StringError &Err) {
198           Errs.applyLocked([&](raw_ostream &OS) {
199             OS << "Error while scanning dependencies for " << Input << ":\n";
200             OS << Err.getMessage();
201           });
202         });
203     return true;
204   }
205   OS.applyLocked([&](raw_ostream &OS) { OS << *MaybeFile; });
206   return false;
207 }
208 
209 int main(int argc, const char **argv) {
210   llvm::InitLLVM X(argc, argv);
211   llvm::cl::HideUnrelatedOptions(DependencyScannerCategory);
212   if (!llvm::cl::ParseCommandLineOptions(argc, argv))
213     return 1;
214 
215   std::string ErrorMessage;
216   std::unique_ptr<tooling::JSONCompilationDatabase> Compilations =
217       tooling::JSONCompilationDatabase::loadFromFile(
218           CompilationDB, ErrorMessage,
219           tooling::JSONCommandLineSyntax::AutoDetect);
220   if (!Compilations) {
221     llvm::errs() << "error: " << ErrorMessage << "\n";
222     return 1;
223   }
224 
225   llvm::cl::PrintOptionValues();
226 
227   // The command options are rewritten to run Clang in preprocessor only mode.
228   auto AdjustingCompilations =
229       std::make_unique<tooling::ArgumentsAdjustingCompilations>(
230           std::move(Compilations));
231   ResourceDirectoryCache ResourceDirCache;
232   AdjustingCompilations->appendArgumentsAdjuster(
233       [&ResourceDirCache](const tooling::CommandLineArguments &Args,
234                           StringRef FileName) {
235         std::string LastO = "";
236         bool HasMT = false;
237         bool HasMQ = false;
238         bool HasMD = false;
239         bool HasResourceDir = false;
240         // We need to find the last -o value.
241         if (!Args.empty()) {
242           std::size_t Idx = Args.size() - 1;
243           for (auto It = Args.rbegin(); It != Args.rend(); ++It) {
244             if (It != Args.rbegin()) {
245               if (Args[Idx] == "-o")
246                 LastO = Args[Idx + 1];
247               if (Args[Idx] == "-MT")
248                 HasMT = true;
249               if (Args[Idx] == "-MQ")
250                 HasMQ = true;
251               if (Args[Idx] == "-MD")
252                 HasMD = true;
253               if (Args[Idx] == "-resource-dir")
254                 HasResourceDir = true;
255             }
256             --Idx;
257           }
258         }
259         // If there's no -MT/-MQ Driver would add -MT with the value of the last
260         // -o option.
261         tooling::CommandLineArguments AdjustedArgs = Args;
262         AdjustedArgs.push_back("-o");
263         AdjustedArgs.push_back("/dev/null");
264         if (!HasMT && !HasMQ) {
265           AdjustedArgs.push_back("-M");
266           AdjustedArgs.push_back("-MT");
267           // We're interested in source dependencies of an object file.
268           if (!HasMD) {
269             // FIXME: We are missing the directory unless the -o value is an
270             // absolute path.
271             AdjustedArgs.push_back(!LastO.empty() ? LastO
272                                                   : getObjFilePath(FileName));
273           } else {
274             AdjustedArgs.push_back(FileName);
275           }
276         }
277         AdjustedArgs.push_back("-Xclang");
278         AdjustedArgs.push_back("-Eonly");
279         AdjustedArgs.push_back("-Xclang");
280         AdjustedArgs.push_back("-sys-header-deps");
281         AdjustedArgs.push_back("-Wno-error");
282 
283         if (!HasResourceDir) {
284           StringRef ResourceDir =
285               ResourceDirCache.findResourceDir(Args);
286           if (!ResourceDir.empty()) {
287             AdjustedArgs.push_back("-resource-dir");
288             AdjustedArgs.push_back(ResourceDir);
289           }
290         }
291         return AdjustedArgs;
292       });
293   AdjustingCompilations->appendArgumentsAdjuster(
294       tooling::getClangStripSerializeDiagnosticAdjuster());
295 
296   SharedStream Errs(llvm::errs());
297   // Print out the dependency results to STDOUT by default.
298   SharedStream DependencyOS(llvm::outs());
299 
300   DependencyScanningService Service(ScanMode, Format, ReuseFileManager,
301                                     SkipExcludedPPRanges);
302 #if LLVM_ENABLE_THREADS
303   unsigned NumWorkers =
304       NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;
305 #else
306   unsigned NumWorkers = 1;
307 #endif
308   std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
309   for (unsigned I = 0; I < NumWorkers; ++I)
310     WorkerTools.push_back(std::make_unique<DependencyScanningTool>(Service));
311 
312   std::vector<SingleCommandCompilationDatabase> Inputs;
313   for (tooling::CompileCommand Cmd :
314        AdjustingCompilations->getAllCompileCommands())
315     Inputs.emplace_back(Cmd);
316 
317   std::vector<std::thread> WorkerThreads;
318   std::atomic<bool> HadErrors(false);
319   std::mutex Lock;
320   size_t Index = 0;
321 
322   if (Verbose) {
323     llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
324                  << " files using " << NumWorkers << " workers\n";
325   }
326   for (unsigned I = 0; I < NumWorkers; ++I) {
327     auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools,
328                    &DependencyOS, &Errs]() {
329       while (true) {
330         const SingleCommandCompilationDatabase *Input;
331         std::string Filename;
332         std::string CWD;
333         // Take the next input.
334         {
335           std::unique_lock<std::mutex> LockGuard(Lock);
336           if (Index >= Inputs.size())
337             return;
338           Input = &Inputs[Index++];
339           tooling::CompileCommand Cmd = Input->getAllCompileCommands()[0];
340           Filename = std::move(Cmd.Filename);
341           CWD = std::move(Cmd.Directory);
342         }
343         // Run the tool on it.
344         auto MaybeFile = WorkerTools[I]->getDependencyFile(*Input, CWD);
345         if (handleDependencyToolResult(Filename, MaybeFile, DependencyOS, Errs))
346           HadErrors = true;
347       }
348     };
349 #if LLVM_ENABLE_THREADS
350     WorkerThreads.emplace_back(std::move(Worker));
351 #else
352     // Run the worker without spawning a thread when threads are disabled.
353     Worker();
354 #endif
355   }
356   for (auto &W : WorkerThreads)
357     W.join();
358 
359   return HadErrors;
360 }
361