1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
50 
51 /// A listener that collects the imported modules and optionally the input
52 /// files.
53 class PrebuiltModuleListener : public ASTReaderListener {
54 public:
55   PrebuiltModuleListener(PrebuiltModuleFilesT &PrebuiltModuleFiles,
56                          llvm::StringSet<> &InputFiles, bool VisitInputFiles,
57                          llvm::SmallVector<std::string> &NewModuleFiles)
58       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
59         VisitInputFiles(VisitInputFiles), NewModuleFiles(NewModuleFiles) {}
60 
61   bool needsImportVisitation() const override { return true; }
62   bool needsInputFileVisitation() override { return VisitInputFiles; }
63   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
64 
65   void visitImport(StringRef ModuleName, StringRef Filename) override {
66     if (PrebuiltModuleFiles.insert({ModuleName.str(), Filename.str()}).second)
67       NewModuleFiles.push_back(Filename.str());
68   }
69 
70   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
71                       bool isExplicitModule) override {
72     InputFiles.insert(Filename);
73     return true;
74   }
75 
76 private:
77   PrebuiltModuleFilesT &PrebuiltModuleFiles;
78   llvm::StringSet<> &InputFiles;
79   bool VisitInputFiles;
80   llvm::SmallVector<std::string> &NewModuleFiles;
81 };
82 
83 /// Visit the given prebuilt module and collect all of the modules it
84 /// transitively imports and contributing input files.
85 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
86                                 CompilerInstance &CI,
87                                 PrebuiltModuleFilesT &ModuleFiles,
88                                 llvm::StringSet<> &InputFiles,
89                                 bool VisitInputFiles) {
90   // List of module files to be processed.
91   llvm::SmallVector<std::string> Worklist{PrebuiltModuleFilename.str()};
92   PrebuiltModuleListener Listener(ModuleFiles, InputFiles, VisitInputFiles,
93                                   Worklist);
94 
95   while (!Worklist.empty())
96     ASTReader::readASTFileControlBlock(
97         Worklist.pop_back_val(), CI.getFileManager(),
98         CI.getPCHContainerReader(),
99         /*FindModuleFileExtensions=*/false, Listener,
100         /*ValidateDiagnosticOptions=*/false);
101 }
102 
103 /// Transform arbitrary file name into an object-like file name.
104 static std::string makeObjFileName(StringRef FileName) {
105   SmallString<128> ObjFileName(FileName);
106   llvm::sys::path::replace_extension(ObjFileName, "o");
107   return std::string(ObjFileName.str());
108 }
109 
110 /// Deduce the dependency target based on the output file and input files.
111 static std::string
112 deduceDepTarget(const std::string &OutputFile,
113                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
114   if (OutputFile != "-")
115     return OutputFile;
116 
117   if (InputFiles.empty() || !InputFiles.front().isFile())
118     return "clang-scan-deps\\ dependency";
119 
120   return makeObjFileName(InputFiles.front().getFile());
121 }
122 
123 /// Sanitize diagnostic options for dependency scan.
124 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
125   // Don't print 'X warnings and Y errors generated'.
126   DiagOpts.ShowCarets = false;
127   // Don't write out diagnostic file.
128   DiagOpts.DiagnosticSerializationFile.clear();
129   // Don't treat warnings as errors.
130   DiagOpts.Warnings.push_back("no-error");
131 }
132 
133 /// A clang tool that runs the preprocessor in a mode that's optimized for
134 /// dependency scanning for the given compiler invocation.
135 class DependencyScanningAction : public tooling::ToolAction {
136 public:
137   DependencyScanningAction(
138       StringRef WorkingDirectory, DependencyConsumer &Consumer,
139       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
140       ScanningOutputFormat Format, bool OptimizeArgs, bool DisableFree,
141       llvm::Optional<StringRef> ModuleName = None)
142       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
143         DepFS(std::move(DepFS)), Format(Format), OptimizeArgs(OptimizeArgs),
144         DisableFree(DisableFree), ModuleName(ModuleName) {}
145 
146   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
147                      FileManager *FileMgr,
148                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
149                      DiagnosticConsumer *DiagConsumer) override {
150     // Make a deep copy of the original Clang invocation.
151     CompilerInvocation OriginalInvocation(*Invocation);
152     // Restore the value of DisableFree, which may be modified by Tooling.
153     OriginalInvocation.getFrontendOpts().DisableFree = DisableFree;
154 
155     // Create a compiler instance to handle the actual work.
156     CompilerInstance ScanInstance(std::move(PCHContainerOps));
157     ScanInstance.setInvocation(std::move(Invocation));
158 
159     // Create the compiler's actual diagnostics engine.
160     sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
161     ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
162     if (!ScanInstance.hasDiagnostics())
163       return false;
164 
165     ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
166         true;
167 
168     ScanInstance.getFrontendOpts().GenerateGlobalModuleIndex = false;
169     ScanInstance.getFrontendOpts().UseGlobalModuleIndex = false;
170 
171     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
172     ScanInstance.setFileManager(FileMgr);
173     ScanInstance.createSourceManager(*FileMgr);
174 
175     llvm::StringSet<> PrebuiltModulesInputFiles;
176     // Store the list of prebuilt module files into header search options. This
177     // will prevent the implicit build to create duplicate modules and will
178     // force reuse of the existing prebuilt module files instead.
179     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
180       visitPrebuiltModule(
181           ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
182           ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
183           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
184 
185     // Use the dependency scanning optimized file system if requested to do so.
186     if (DepFS) {
187       // Support for virtual file system overlays on top of the caching
188       // filesystem.
189       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
190           ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
191 
192       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> LocalDepFS =
193           DepFS;
194       ScanInstance.getPreprocessorOpts().DependencyDirectivesForFile =
195           [LocalDepFS = std::move(LocalDepFS)](FileEntryRef File)
196           -> Optional<ArrayRef<dependency_directives_scan::Directive>> {
197         if (llvm::ErrorOr<EntryRef> Entry =
198                 LocalDepFS->getOrCreateFileSystemEntry(File.getName()))
199           return Entry->getDirectiveTokens();
200         return None;
201       };
202     }
203 
204     // Create the dependency collector that will collect the produced
205     // dependencies.
206     //
207     // This also moves the existing dependency output options from the
208     // invocation to the collector. The options in the invocation are reset,
209     // which ensures that the compiler won't create new dependency collectors,
210     // and thus won't write out the extra '.d' files to disk.
211     auto Opts = std::make_unique<DependencyOutputOptions>();
212     std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
213     // We need at least one -MT equivalent for the generator of make dependency
214     // files to work.
215     if (Opts->Targets.empty())
216       Opts->Targets = {
217           deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
218                           ScanInstance.getFrontendOpts().Inputs)};
219     Opts->IncludeSystemHeaders = true;
220 
221     switch (Format) {
222     case ScanningOutputFormat::Make:
223       ScanInstance.addDependencyCollector(
224           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
225                                                         Consumer));
226       break;
227     case ScanningOutputFormat::Full:
228       ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
229           std::move(Opts), ScanInstance, Consumer,
230           std::move(OriginalInvocation), OptimizeArgs));
231       break;
232     }
233 
234     // Consider different header search and diagnostic options to create
235     // different modules. This avoids the unsound aliasing of module PCMs.
236     //
237     // TODO: Implement diagnostic bucketing to reduce the impact of strict
238     // context hashing.
239     ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
240 
241     std::unique_ptr<FrontendAction> Action;
242 
243     if (ModuleName)
244       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
245     else
246       Action = std::make_unique<ReadPCHAndPreprocessAction>();
247 
248     const bool Result = ScanInstance.ExecuteAction(*Action);
249     if (!DepFS)
250       FileMgr->clearStatCache();
251     return Result;
252   }
253 
254 private:
255   StringRef WorkingDirectory;
256   DependencyConsumer &Consumer;
257   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
258   ScanningOutputFormat Format;
259   bool OptimizeArgs;
260   bool DisableFree;
261   llvm::Optional<StringRef> ModuleName;
262 };
263 
264 } // end anonymous namespace
265 
266 DependencyScanningWorker::DependencyScanningWorker(
267     DependencyScanningService &Service,
268     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
269     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
270   PCHContainerOps = std::make_shared<PCHContainerOperations>();
271   PCHContainerOps->registerReader(
272       std::make_unique<ObjectFilePCHContainerReader>());
273   // We don't need to write object files, but the current PCH implementation
274   // requires the writer to be registered as well.
275   PCHContainerOps->registerWriter(
276       std::make_unique<ObjectFilePCHContainerWriter>());
277 
278   auto OverlayFS =
279       llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(std::move(FS));
280   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
281   OverlayFS->pushOverlay(InMemoryFS);
282   RealFS = OverlayFS;
283 
284   if (Service.getMode() == ScanningMode::DependencyDirectivesScan)
285     DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
286                                                    RealFS);
287   if (Service.canReuseFileManager())
288     Files = new FileManager(FileSystemOptions(), RealFS);
289 }
290 
291 static llvm::Error
292 runWithDiags(DiagnosticOptions *DiagOpts,
293              llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
294                  BodyShouldSucceed) {
295   sanitizeDiagOpts(*DiagOpts);
296 
297   // Capture the emitted diagnostics and report them to the client
298   // in the case of a failure.
299   std::string DiagnosticOutput;
300   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
301   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
302 
303   if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
304     return llvm::Error::success();
305   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
306                                              llvm::inconvertibleErrorCode());
307 }
308 
309 llvm::Error DependencyScanningWorker::computeDependencies(
310     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
311     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
312   // Reset what might have been modified in the previous worker invocation.
313   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
314   if (Files)
315     Files->setVirtualFileSystem(RealFS);
316 
317   llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
318       Files ? Files : new FileManager(FileSystemOptions(), RealFS);
319 
320   Optional<std::vector<std::string>> ModifiedCommandLine;
321   if (ModuleName) {
322     ModifiedCommandLine = CommandLine;
323     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
324     ModifiedCommandLine->emplace_back(*ModuleName);
325   }
326 
327   const std::vector<std::string> &FinalCommandLine =
328       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
329 
330   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
331   llvm::transform(CommandLine, FinalCCommandLine.begin(),
332                   [](const std::string &Str) { return Str.c_str(); });
333 
334   return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
335                       [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
336                         // DisableFree is modified by Tooling for running
337                         // in-process; preserve the original value, which is
338                         // always true for a driver invocation.
339                         bool DisableFree = true;
340                         DependencyScanningAction Action(
341                             WorkingDirectory, Consumer, DepFS, Format,
342                             OptimizeArgs, DisableFree, ModuleName);
343                         // Create an invocation that uses the underlying file
344                         // system to ensure that any file system requests that
345                         // are made by the driver do not go through the
346                         // dependency scanning filesystem.
347                         ToolInvocation Invocation(FinalCommandLine, &Action,
348                                                   CurrentFiles.get(),
349                                                   PCHContainerOps);
350                         Invocation.setDiagnosticConsumer(&DC);
351                         Invocation.setDiagnosticOptions(&DiagOpts);
352                         return Invocation.run();
353                       });
354 }
355