1 //===- DependencyScanningFilesystem.h - clang-scan-deps fs ===---*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #ifndef LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H 10 #define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H 11 12 #include "clang/Basic/LLVM.h" 13 #include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h" 14 #include "llvm/ADT/StringMap.h" 15 #include "llvm/ADT/StringSet.h" 16 #include "llvm/Support/Allocator.h" 17 #include "llvm/Support/ErrorOr.h" 18 #include "llvm/Support/VirtualFileSystem.h" 19 #include <mutex> 20 21 namespace clang { 22 namespace tooling { 23 namespace dependencies { 24 25 /// An in-memory representation of a file system entity that is of interest to 26 /// the dependency scanning filesystem. 27 /// 28 /// It represents one of the following: 29 /// - an opened source file with minimized contents and a stat value. 30 /// - an opened source file with original contents and a stat value. 31 /// - a directory entry with its stat value. 32 /// - an error value to represent a file system error. 33 /// - a placeholder with an invalid stat indicating a not yet initialized entry. 34 class CachedFileSystemEntry { 35 public: 36 /// Default constructor creates an entry with an invalid stat. 37 CachedFileSystemEntry() : MaybeStat(llvm::vfs::Status()) {} 38 39 CachedFileSystemEntry(std::error_code Error) : MaybeStat(std::move(Error)) {} 40 41 /// Create an entry that represents an opened source file with minimized or 42 /// original contents. 43 /// 44 /// The filesystem opens the file even for `stat` calls open to avoid the 45 /// issues with stat + open of minimized files that might lead to a 46 /// mismatching size of the file. If file is not minimized, the full file is 47 /// read and copied into memory to ensure that it's not memory mapped to avoid 48 /// running out of file descriptors. 49 static CachedFileSystemEntry createFileEntry(StringRef Filename, 50 llvm::vfs::FileSystem &FS, 51 bool Minimize = true); 52 53 /// Create an entry that represents a directory on the filesystem. 54 static CachedFileSystemEntry createDirectoryEntry(llvm::vfs::Status &&Stat); 55 56 /// \returns True if the entry is valid. 57 bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); } 58 59 /// \returns True if the current entry points to a directory. 60 bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); } 61 62 /// \returns The error or the file's contents. 63 llvm::ErrorOr<StringRef> getContents() const { 64 if (!MaybeStat) 65 return MaybeStat.getError(); 66 assert(!MaybeStat->isDirectory() && "not a file"); 67 assert(isValid() && "not initialized"); 68 return Contents.str(); 69 } 70 71 /// \returns The error or the status of the entry. 72 llvm::ErrorOr<llvm::vfs::Status> getStatus() const { 73 assert(isValid() && "not initialized"); 74 return MaybeStat; 75 } 76 77 /// \returns the name of the file. 78 StringRef getName() const { 79 assert(isValid() && "not initialized"); 80 return MaybeStat->getName(); 81 } 82 83 /// Return the mapping between location -> distance that is used to speed up 84 /// the block skipping in the preprocessor. 85 const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const { 86 return PPSkippedRangeMapping; 87 } 88 89 CachedFileSystemEntry(CachedFileSystemEntry &&) = default; 90 CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default; 91 92 CachedFileSystemEntry(const CachedFileSystemEntry &) = delete; 93 CachedFileSystemEntry &operator=(const CachedFileSystemEntry &) = delete; 94 95 private: 96 llvm::ErrorOr<llvm::vfs::Status> MaybeStat; 97 // Store the contents in a small string to allow a 98 // move from the small string for the minimized contents. 99 // Note: small size of 1 allows us to store an empty string with an implicit 100 // null terminator without any allocations. 101 llvm::SmallString<1> Contents; 102 PreprocessorSkippedRangeMapping PPSkippedRangeMapping; 103 }; 104 105 /// This class is a shared cache, that caches the 'stat' and 'open' calls to the 106 /// underlying real file system. It distinguishes between minimized and original 107 /// files. 108 /// 109 /// It is sharded based on the hash of the key to reduce the lock contention for 110 /// the worker threads. 111 class DependencyScanningFilesystemSharedCache { 112 public: 113 struct SharedFileSystemEntry { 114 std::mutex ValueLock; 115 CachedFileSystemEntry Value; 116 }; 117 118 /// Returns a cache entry for the corresponding key. 119 /// 120 /// A new cache entry is created if the key is not in the cache. This is a 121 /// thread safe call. 122 SharedFileSystemEntry &get(StringRef Key, bool Minimized); 123 124 private: 125 class SingleCache { 126 public: 127 SingleCache(); 128 129 SharedFileSystemEntry &get(StringRef Key); 130 131 private: 132 struct CacheShard { 133 std::mutex CacheLock; 134 llvm::StringMap<SharedFileSystemEntry, llvm::BumpPtrAllocator> Cache; 135 }; 136 std::unique_ptr<CacheShard[]> CacheShards; 137 unsigned NumShards; 138 }; 139 140 SingleCache CacheMinimized; 141 SingleCache CacheOriginal; 142 }; 143 144 /// This class is a local cache, that caches the 'stat' and 'open' calls to the 145 /// underlying real file system. It distinguishes between minimized and original 146 /// files. 147 class DependencyScanningFilesystemLocalCache { 148 private: 149 using SingleCache = 150 llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>; 151 152 SingleCache CacheMinimized; 153 SingleCache CacheOriginal; 154 155 SingleCache &selectCache(bool Minimized) { 156 return Minimized ? CacheMinimized : CacheOriginal; 157 } 158 159 public: 160 void setCachedEntry(StringRef Filename, bool Minimized, 161 const CachedFileSystemEntry *Entry) { 162 SingleCache &Cache = selectCache(Minimized); 163 bool IsInserted = Cache.try_emplace(Filename, Entry).second; 164 (void)IsInserted; 165 assert(IsInserted && "local cache is updated more than once"); 166 } 167 168 const CachedFileSystemEntry *getCachedEntry(StringRef Filename, 169 bool Minimized) { 170 SingleCache &Cache = selectCache(Minimized); 171 auto It = Cache.find(Filename); 172 return It == Cache.end() ? nullptr : It->getValue(); 173 } 174 }; 175 176 /// A virtual file system optimized for the dependency discovery. 177 /// 178 /// It is primarily designed to work with source files whose contents was was 179 /// preprocessed to remove any tokens that are unlikely to affect the dependency 180 /// computation. 181 /// 182 /// This is not a thread safe VFS. A single instance is meant to be used only in 183 /// one thread. Multiple instances are allowed to service multiple threads 184 /// running in parallel. 185 class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem { 186 public: 187 DependencyScanningWorkerFilesystem( 188 DependencyScanningFilesystemSharedCache &SharedCache, 189 IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS, 190 ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) 191 : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache), 192 PPSkipMappings(PPSkipMappings) {} 193 194 llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override; 195 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>> 196 openFileForRead(const Twine &Path) override; 197 198 void clearIgnoredFiles() { IgnoredFiles.clear(); } 199 void ignoreFile(StringRef Filename); 200 201 private: 202 bool shouldIgnoreFile(StringRef Filename); 203 204 llvm::ErrorOr<const CachedFileSystemEntry *> 205 getOrCreateFileSystemEntry(const StringRef Filename); 206 207 /// The global cache shared between worker threads. 208 DependencyScanningFilesystemSharedCache &SharedCache; 209 /// The local cache is used by the worker thread to cache file system queries 210 /// locally instead of querying the global cache every time. 211 DependencyScanningFilesystemLocalCache Cache; 212 /// The optional mapping structure which records information about the 213 /// excluded conditional directive skip mappings that are used by the 214 /// currently active preprocessor. 215 ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings; 216 /// The set of files that should not be minimized. 217 llvm::StringSet<> IgnoredFiles; 218 }; 219 220 } // end namespace dependencies 221 } // end namespace tooling 222 } // end namespace clang 223 224 #endif // LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H 225