1 //===- DependencyScanningWorker.cpp - clang-scan-deps worker --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
10 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
11 #include "clang/Frontend/CompilerInstance.h"
12 #include "clang/Frontend/CompilerInvocation.h"
13 #include "clang/Frontend/FrontendActions.h"
14 #include "clang/Frontend/TextDiagnosticPrinter.h"
15 #include "clang/Frontend/Utils.h"
16 #include "clang/Lex/PreprocessorOptions.h"
17 #include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
18 #include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
19 #include "clang/Tooling/Tooling.h"
20 
21 using namespace clang;
22 using namespace tooling;
23 using namespace dependencies;
24 
25 namespace {
26 
27 /// Forwards the gatherered dependencies to the consumer.
28 class DependencyConsumerForwarder : public DependencyFileGenerator {
29 public:
30   DependencyConsumerForwarder(std::unique_ptr<DependencyOutputOptions> Opts,
31                               DependencyConsumer &C)
32       : DependencyFileGenerator(*Opts), Opts(std::move(Opts)), C(C) {}
33 
34   void finishedMainFile(DiagnosticsEngine &Diags) override {
35     C.handleDependencyOutputOpts(*Opts);
36     llvm::SmallString<256> CanonPath;
37     for (const auto &File : getDependencies()) {
38       CanonPath = File;
39       llvm::sys::path::remove_dots(CanonPath, /*remove_dot_dot=*/true);
40       C.handleFileDependency(CanonPath);
41     }
42   }
43 
44 private:
45   std::unique_ptr<DependencyOutputOptions> Opts;
46   DependencyConsumer &C;
47 };
48 
49 /// A listener that collects the imported modules and optionally the input
50 /// files.
51 class PrebuiltModuleListener : public ASTReaderListener {
52 public:
53   PrebuiltModuleListener(llvm::StringMap<std::string> &PrebuiltModuleFiles,
54                          llvm::StringSet<> &InputFiles, bool VisitInputFiles)
55       : PrebuiltModuleFiles(PrebuiltModuleFiles), InputFiles(InputFiles),
56         VisitInputFiles(VisitInputFiles) {}
57 
58   bool needsImportVisitation() const override { return true; }
59   bool needsInputFileVisitation() override { return VisitInputFiles; }
60   bool needsSystemInputFileVisitation() override { return VisitInputFiles; }
61 
62   void visitImport(StringRef ModuleName, StringRef Filename) override {
63     PrebuiltModuleFiles.insert({ModuleName, Filename.str()});
64   }
65 
66   bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden,
67                       bool isExplicitModule) override {
68     InputFiles.insert(Filename);
69     return true;
70   }
71 
72 private:
73   llvm::StringMap<std::string> &PrebuiltModuleFiles;
74   llvm::StringSet<> &InputFiles;
75   bool VisitInputFiles;
76 };
77 
78 using PrebuiltModuleFilesT = decltype(HeaderSearchOptions::PrebuiltModuleFiles);
79 
80 /// Visit the given prebuilt module and collect all of the modules it
81 /// transitively imports and contributing input files.
82 static void visitPrebuiltModule(StringRef PrebuiltModuleFilename,
83                                 CompilerInstance &CI,
84                                 PrebuiltModuleFilesT &ModuleFiles,
85                                 llvm::StringSet<> &InputFiles,
86                                 bool VisitInputFiles) {
87   // Maps the names of modules that weren't yet visited to their PCM path.
88   llvm::StringMap<std::string> ModuleFilesWorklist;
89   // Contains PCM paths of all visited modules.
90   llvm::StringSet<> VisitedModuleFiles;
91 
92   PrebuiltModuleListener Listener(ModuleFilesWorklist, InputFiles,
93                                   VisitInputFiles);
94 
95   auto GatherModuleFileInfo = [&](StringRef ASTFile) {
96     ASTReader::readASTFileControlBlock(
97         ASTFile, CI.getFileManager(), CI.getPCHContainerReader(),
98         /*FindModuleFileExtensions=*/false, Listener,
99         /*ValidateDiagnosticOptions=*/false);
100   };
101 
102   GatherModuleFileInfo(PrebuiltModuleFilename);
103   while (!ModuleFilesWorklist.empty()) {
104     auto WorklistItemIt = ModuleFilesWorklist.begin();
105 
106     if (!VisitedModuleFiles.contains(WorklistItemIt->getValue())) {
107       VisitedModuleFiles.insert(WorklistItemIt->getValue());
108       GatherModuleFileInfo(WorklistItemIt->getValue());
109       ModuleFiles[WorklistItemIt->getKey().str()] = WorklistItemIt->getValue();
110     }
111 
112     ModuleFilesWorklist.erase(WorklistItemIt);
113   }
114 }
115 
116 /// Transform arbitrary file name into an object-like file name.
117 static std::string makeObjFileName(StringRef FileName) {
118   SmallString<128> ObjFileName(FileName);
119   llvm::sys::path::replace_extension(ObjFileName, "o");
120   return std::string(ObjFileName.str());
121 }
122 
123 /// Deduce the dependency target based on the output file and input files.
124 static std::string
125 deduceDepTarget(const std::string &OutputFile,
126                 const SmallVectorImpl<FrontendInputFile> &InputFiles) {
127   if (OutputFile != "-")
128     return OutputFile;
129 
130   if (InputFiles.empty() || !InputFiles.front().isFile())
131     return "clang-scan-deps\\ dependency";
132 
133   return makeObjFileName(InputFiles.front().getFile());
134 }
135 
136 /// Sanitize diagnostic options for dependency scan.
137 static void sanitizeDiagOpts(DiagnosticOptions &DiagOpts) {
138   // Don't print 'X warnings and Y errors generated'.
139   DiagOpts.ShowCarets = false;
140   // Don't write out diagnostic file.
141   DiagOpts.DiagnosticSerializationFile.clear();
142   // Don't treat warnings as errors.
143   DiagOpts.Warnings.push_back("no-error");
144 }
145 
146 /// A clang tool that runs the preprocessor in a mode that's optimized for
147 /// dependency scanning for the given compiler invocation.
148 class DependencyScanningAction : public tooling::ToolAction {
149 public:
150   DependencyScanningAction(
151       StringRef WorkingDirectory, DependencyConsumer &Consumer,
152       llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
153       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
154       ScanningOutputFormat Format, bool OptimizeArgs,
155       llvm::Optional<StringRef> ModuleName = None)
156       : WorkingDirectory(WorkingDirectory), Consumer(Consumer),
157         DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings), Format(Format),
158         OptimizeArgs(OptimizeArgs), ModuleName(ModuleName) {}
159 
160   bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
161                      FileManager *FileMgr,
162                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
163                      DiagnosticConsumer *DiagConsumer) override {
164     // Make a deep copy of the original Clang invocation.
165     CompilerInvocation OriginalInvocation(*Invocation);
166 
167     // Create a compiler instance to handle the actual work.
168     CompilerInstance ScanInstance(std::move(PCHContainerOps));
169     ScanInstance.setInvocation(std::move(Invocation));
170 
171     // Create the compiler's actual diagnostics engine.
172     sanitizeDiagOpts(ScanInstance.getDiagnosticOpts());
173     ScanInstance.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
174     if (!ScanInstance.hasDiagnostics())
175       return false;
176 
177     ScanInstance.getPreprocessorOpts().AllowPCHWithDifferentModulesCachePath =
178         true;
179 
180     FileMgr->getFileSystemOpts().WorkingDir = std::string(WorkingDirectory);
181     ScanInstance.setFileManager(FileMgr);
182     ScanInstance.createSourceManager(*FileMgr);
183 
184     llvm::StringSet<> PrebuiltModulesInputFiles;
185     // Store the list of prebuilt module files into header search options. This
186     // will prevent the implicit build to create duplicate modules and will
187     // force reuse of the existing prebuilt module files instead.
188     if (!ScanInstance.getPreprocessorOpts().ImplicitPCHInclude.empty())
189       visitPrebuiltModule(
190           ScanInstance.getPreprocessorOpts().ImplicitPCHInclude, ScanInstance,
191           ScanInstance.getHeaderSearchOpts().PrebuiltModuleFiles,
192           PrebuiltModulesInputFiles, /*VisitInputFiles=*/DepFS != nullptr);
193 
194     // Use the dependency scanning optimized file system if requested to do so.
195     if (DepFS) {
196       DepFS->enableMinimizationOfAllFiles();
197       // Don't minimize any files that contributed to prebuilt modules. The
198       // implicit build validates the modules by comparing the reported sizes of
199       // their inputs to the current state of the filesystem. Minimization would
200       // throw this mechanism off.
201       for (const auto &File : PrebuiltModulesInputFiles)
202         DepFS->disableMinimization(File.getKey());
203       // Don't minimize any files that were explicitly passed in the build
204       // settings and that might be opened.
205       for (const auto &E : ScanInstance.getHeaderSearchOpts().UserEntries)
206         DepFS->disableMinimization(E.Path);
207       for (const auto &F : ScanInstance.getHeaderSearchOpts().VFSOverlayFiles)
208         DepFS->disableMinimization(F);
209 
210       // Support for virtual file system overlays on top of the caching
211       // filesystem.
212       FileMgr->setVirtualFileSystem(createVFSFromCompilerInvocation(
213           ScanInstance.getInvocation(), ScanInstance.getDiagnostics(), DepFS));
214 
215       // Pass the skip mappings which should speed up excluded conditional block
216       // skipping in the preprocessor.
217       if (PPSkipMappings)
218         ScanInstance.getPreprocessorOpts()
219             .ExcludedConditionalDirectiveSkipMappings = PPSkipMappings;
220     }
221 
222     // Create the dependency collector that will collect the produced
223     // dependencies.
224     //
225     // This also moves the existing dependency output options from the
226     // invocation to the collector. The options in the invocation are reset,
227     // which ensures that the compiler won't create new dependency collectors,
228     // and thus won't write out the extra '.d' files to disk.
229     auto Opts = std::make_unique<DependencyOutputOptions>();
230     std::swap(*Opts, ScanInstance.getInvocation().getDependencyOutputOpts());
231     // We need at least one -MT equivalent for the generator of make dependency
232     // files to work.
233     if (Opts->Targets.empty())
234       Opts->Targets = {
235           deduceDepTarget(ScanInstance.getFrontendOpts().OutputFile,
236                           ScanInstance.getFrontendOpts().Inputs)};
237     Opts->IncludeSystemHeaders = true;
238 
239     switch (Format) {
240     case ScanningOutputFormat::Make:
241       ScanInstance.addDependencyCollector(
242           std::make_shared<DependencyConsumerForwarder>(std::move(Opts),
243                                                         Consumer));
244       break;
245     case ScanningOutputFormat::Full:
246       ScanInstance.addDependencyCollector(std::make_shared<ModuleDepCollector>(
247           std::move(Opts), ScanInstance, Consumer,
248           std::move(OriginalInvocation), OptimizeArgs));
249       break;
250     }
251 
252     // Consider different header search and diagnostic options to create
253     // different modules. This avoids the unsound aliasing of module PCMs.
254     //
255     // TODO: Implement diagnostic bucketing to reduce the impact of strict
256     // context hashing.
257     ScanInstance.getHeaderSearchOpts().ModulesStrictContextHash = true;
258 
259     std::unique_ptr<FrontendAction> Action;
260 
261     if (ModuleName.hasValue())
262       Action = std::make_unique<GetDependenciesByModuleNameAction>(*ModuleName);
263     else
264       Action = std::make_unique<ReadPCHAndPreprocessAction>();
265 
266     const bool Result = ScanInstance.ExecuteAction(*Action);
267     if (!DepFS)
268       FileMgr->clearStatCache();
269     return Result;
270   }
271 
272 private:
273   StringRef WorkingDirectory;
274   DependencyConsumer &Consumer;
275   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
276   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
277   ScanningOutputFormat Format;
278   bool OptimizeArgs;
279   llvm::Optional<StringRef> ModuleName;
280 };
281 
282 } // end anonymous namespace
283 
284 DependencyScanningWorker::DependencyScanningWorker(
285     DependencyScanningService &Service)
286     : Format(Service.getFormat()), OptimizeArgs(Service.canOptimizeArgs()) {
287   PCHContainerOps = std::make_shared<PCHContainerOperations>();
288   PCHContainerOps->registerReader(
289       std::make_unique<ObjectFilePCHContainerReader>());
290   // We don't need to write object files, but the current PCH implementation
291   // requires the writer to be registered as well.
292   PCHContainerOps->registerWriter(
293       std::make_unique<ObjectFilePCHContainerWriter>());
294 
295   auto OverlayFS = llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(
296       llvm::vfs::createPhysicalFileSystem());
297   InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
298   OverlayFS->pushOverlay(InMemoryFS);
299   RealFS = OverlayFS;
300 
301   if (Service.canSkipExcludedPPRanges())
302     PPSkipMappings =
303         std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
304   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
305     DepFS = new DependencyScanningWorkerFilesystem(
306         Service.getSharedCache(), RealFS, PPSkipMappings.get());
307   if (Service.canReuseFileManager())
308     Files = new FileManager(FileSystemOptions(), RealFS);
309 }
310 
311 static llvm::Error
312 runWithDiags(DiagnosticOptions *DiagOpts,
313              llvm::function_ref<bool(DiagnosticConsumer &, DiagnosticOptions &)>
314                  BodyShouldSucceed) {
315   sanitizeDiagOpts(*DiagOpts);
316 
317   // Capture the emitted diagnostics and report them to the client
318   // in the case of a failure.
319   std::string DiagnosticOutput;
320   llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
321   TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts);
322 
323   if (BodyShouldSucceed(DiagPrinter, *DiagOpts))
324     return llvm::Error::success();
325   return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
326                                              llvm::inconvertibleErrorCode());
327 }
328 
329 llvm::Error DependencyScanningWorker::computeDependencies(
330     StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
331     DependencyConsumer &Consumer, llvm::Optional<StringRef> ModuleName) {
332   // Reset what might have been modified in the previous worker invocation.
333   RealFS->setCurrentWorkingDirectory(WorkingDirectory);
334   if (Files)
335     Files->setVirtualFileSystem(RealFS);
336 
337   llvm::IntrusiveRefCntPtr<FileManager> CurrentFiles =
338       Files ? Files : new FileManager(FileSystemOptions(), RealFS);
339 
340   Optional<std::vector<std::string>> ModifiedCommandLine;
341   if (ModuleName.hasValue()) {
342     ModifiedCommandLine = CommandLine;
343     InMemoryFS->addFile(*ModuleName, 0, llvm::MemoryBuffer::getMemBuffer(""));
344     ModifiedCommandLine->emplace_back(*ModuleName);
345   }
346 
347   const std::vector<std::string> &FinalCommandLine =
348       ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
349 
350   std::vector<const char *> FinalCCommandLine(CommandLine.size(), nullptr);
351   llvm::transform(CommandLine, FinalCCommandLine.begin(),
352                   [](const std::string &Str) { return Str.c_str(); });
353 
354   return runWithDiags(CreateAndPopulateDiagOpts(FinalCCommandLine).release(),
355                       [&](DiagnosticConsumer &DC, DiagnosticOptions &DiagOpts) {
356                         DependencyScanningAction Action(
357                             WorkingDirectory, Consumer, DepFS,
358                             PPSkipMappings.get(), Format, OptimizeArgs,
359                             ModuleName);
360                         // Create an invocation that uses the underlying file
361                         // system to ensure that any file system requests that
362                         // are made by the driver do not go through the
363                         // dependency scanning filesystem.
364                         ToolInvocation Invocation(FinalCommandLine, &Action,
365                                                   CurrentFiles.get(),
366                                                   PCHContainerOps);
367                         Invocation.setDiagnosticConsumer(&DC);
368                         Invocation.setDiagnosticOptions(&DiagOpts);
369                         return Invocation.run();
370                       });
371 }
372