1 //===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
10 #define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
11 
12 #include "clang/Basic/LLVM.h"
13 #include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
14 #include "llvm/ADT/StringMap.h"
15 #include "llvm/ADT/StringSet.h"
16 #include "llvm/Support/Allocator.h"
17 #include "llvm/Support/ErrorOr.h"
18 #include "llvm/Support/VirtualFileSystem.h"
19 #include <mutex>
20 
21 namespace clang {
22 namespace tooling {
23 namespace dependencies {
24 
25 /// An in-memory representation of a file system entity that is of interest to
26 /// the dependency scanning filesystem.
27 ///
28 /// It represents one of the following:
29 /// - an opened source file with minimized contents and a stat value.
30 /// - an opened source file with original contents and a stat value.
31 /// - a directory entry with its stat value.
32 /// - an error value to represent a file system error.
33 /// - a placeholder with an invalid stat indicating a not yet initialized entry.
34 class CachedFileSystemEntry {
35 public:
36   /// Default constructor creates an entry with an invalid stat.
37   CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {}
38 
39   CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {}
40 
41   /// Create an entry that represents an opened source file with minimized or
42   /// original contents.
43   ///
44   /// The filesystem opens the file even for `stat` calls open to avoid the
45   /// issues with stat + open of minimized files that might lead to a
46   /// mismatching size of the file. If file is not minimized, the full file is
47   /// read and copied into memory to ensure that it's not memory mapped to avoid
48   /// running out of file descriptors.
49   static CachedFileSystemEntry createFileEntry(StringRef Filename,
50                                                llvm::vfs::FileSystem &FS,
51                                                bool Minimize = true);
52 
53   /// Create an entry that represents a directory on the filesystem.
54   static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat);
55 
56   /// \returns True if the entry is valid.
57   bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); }
58 
59   /// \returns True if the current entry points to a directory.
60   bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); }
61 
62   /// \returns The error or the file's contents.
63   llvm::ErrorOr<StringRef> getContents() const {
64     if (!MaybeStat)
65       return MaybeStat.getError();
66     assert(!MaybeStat->isDirectory() && "not a file");
67     assert(isValid() && "not initialized");
68     return Contents.str();
69   }
70 
71   /// \returns The error or the status of the entry.
72   llvm::ErrorOr<llvm::vfs::Status> getStatus() const {
73     assert(isValid() && "not initialized");
74     return MaybeStat;
75   }
76 
77   /// \returns the name of the file.
78   StringRef getName() const {
79     assert(isValid() && "not initialized");
80     return MaybeStat->getName();
81   }
82 
83   /// Return the mapping between location -> distance that is used to speed up
84   /// the block skipping in the preprocessor.
85   const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const {
86     return PPSkippedRangeMapping;
87   }
88 
89   CachedFileSystemEntry(CachedFileSystemEntry &&) = default;
90   CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default;
91 
92   CachedFileSystemEntry(const CachedFileSystemEntry &) = delete;
93   CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete;
94 
95 private:
96   llvm::ErrorOr<llvm::vfs::Status> MaybeStat;
97   // Store the contents in a small string to allow a
98   // move from the small string for the minimized contents.
99   // Note: small size of 1 allows us to store an empty string with an implicit
100   // null terminator without any allocations.
101   llvm::SmallString<1> Contents;
102   PreprocessorSkippedRangeMapping PPSkippedRangeMapping;
103 };
104 
105 /// This class is a shared cache, that caches the 'stat' and 'open' calls to the
106 /// underlying real file system. It distinguishes between minimized and original
107 /// files.
108 ///
109 /// It is sharded based on the hash of the key to reduce the lock contention for
110 /// the worker threads.
111 class DependencyScanningFilesystemSharedCache {
112 public:
113   struct SharedFileSystemEntry {
114     std::mutex ValueLock;
115     CachedFileSystemEntry Value;
116   };
117 
118   /// Returns a cache entry for the corresponding key.
119   ///
120   /// A new cache entry is created if the key is not in the cache. This is a
121   /// thread safe call.
122   SharedFileSystemEntry &get(StringRef Key, bool Minimized);
123 
124 private:
125   class SingleCache {
126   public:
127     SingleCache();
128 
129     SharedFileSystemEntry &get(StringRef Key);
130 
131   private:
132     struct CacheShard {
133       std::mutex CacheLock;
134       llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache;
135     };
136     std::unique_ptr<CacheShard[]> CacheShards;
137     unsigned NumShards;
138   };
139 
140   SingleCache CacheMinimized;
141   SingleCache CacheOriginal;
142 };
143 
144 /// This class is a local cache, that caches the 'stat' and 'open' calls to the
145 /// underlying real file system. It distinguishes between minimized and original
146 /// files.
147 class DependencyScanningFilesystemLocalCache {
148 private:
149   using SingleCache =
150       llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>;
151 
152   SingleCache CacheMinimized;
153   SingleCache CacheOriginal;
154 
155   SingleCache &selectCache(bool Minimized) {
156     return Minimized ? CacheMinimized : CacheOriginal;
157   }
158 
159 public:
160   void setCachedEntry(StringRef Filename, bool Minimized,
161                       const CachedFileSystemEntry *Entry) {
162     SingleCache &Cache = selectCache(Minimized);
163     bool IsInserted = Cache.try_emplace(Filename, Entry).second;
164     (void)IsInserted;
165     assert(IsInserted && "local cache is updated more than once");
166   }
167 
168   const CachedFileSystemEntry *getCachedEntry(StringRef Filename,
169                                               bool Minimized) {
170     SingleCache &Cache = selectCache(Minimized);
171     auto It = Cache.find(Filename);
172     return It == Cache.end() ? nullptr : It->getValue();
173   }
174 };
175 
176 /// A virtual file system optimized for the dependency discovery.
177 ///
178 /// It is primarily designed to work with source files whose contents was was
179 /// preprocessed to remove any tokens that are unlikely to affect the dependency
180 /// computation.
181 ///
182 /// This is not a thread safe VFS. A single instance is meant to be used only in
183 /// one thread. Multiple instances are allowed to service multiple threads
184 /// running in parallel.
185 class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
186 public:
187   DependencyScanningWorkerFilesystem(
188       DependencyScanningFilesystemSharedCache &SharedCache,
189       IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
190       ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings)
191       : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
192         PPSkipMappings(PPSkipMappings) {}
193 
194   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
195   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
196   openFileForRead(const Twine &Path) override;
197 
198   void clearIgnoredFiles() { IgnoredFiles.clear(); }
199   void ignoreFile(StringRef Filename);
200 
201 private:
202   bool shouldIgnoreFile(StringRef Filename);
203 
204   llvm::ErrorOr<const CachedFileSystemEntry *>
205   getOrCreateFileSystemEntry(const StringRef Filename);
206 
207   /// The global cache shared between worker threads.
208   DependencyScanningFilesystemSharedCache &SharedCache;
209   /// The local cache is used by the worker thread to cache file system queries
210   /// locally instead of querying the global cache every time.
211   DependencyScanningFilesystemLocalCache Cache;
212   /// The optional mapping structure which records information about the
213   /// excluded conditional directive skip mappings that are used by the
214   /// currently active preprocessor.
215   ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
216   /// The set of files that should not be minimized.
217   llvm::StringSet<> IgnoredFiles;
218 };
219 
220 } // end namespace dependencies
221 } // end namespace tooling
222 } // end namespace clang
223 
224 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
225