1 //===--- CrossTranslationUnit.h - -------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides an interface to load binary AST dumps on demand. This 10 // feature can be utilized for tools that require cross translation unit 11 // support. 12 // 13 //===----------------------------------------------------------------------===// 14 #ifndef LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H 15 #define LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H 16 17 #include "clang/AST/ASTImporterSharedState.h" 18 #include "clang/Analysis/MacroExpansionContext.h" 19 #include "clang/Basic/LLVM.h" 20 #include "llvm/ADT/DenseMap.h" 21 #include "llvm/ADT/Optional.h" 22 #include "llvm/ADT/SmallPtrSet.h" 23 #include "llvm/ADT/StringMap.h" 24 #include "llvm/Support/Error.h" 25 #include "llvm/Support/Path.h" 26 27 namespace clang { 28 class CompilerInstance; 29 class ASTContext; 30 class ASTImporter; 31 class ASTUnit; 32 class DeclContext; 33 class FunctionDecl; 34 class VarDecl; 35 class NamedDecl; 36 class TranslationUnitDecl; 37 38 namespace cross_tu { 39 40 enum class index_error_code { 41 success = 0, 42 unspecified = 1, 43 missing_index_file, 44 invalid_index_format, 45 multiple_definitions, 46 missing_definition, 47 failed_import, 48 failed_to_get_external_ast, 49 failed_to_generate_usr, 50 triple_mismatch, 51 lang_mismatch, 52 lang_dialect_mismatch, 53 load_threshold_reached, 54 invocation_list_ambiguous, 55 invocation_list_file_not_found, 56 invocation_list_empty, 57 invocation_list_wrong_format, 58 invocation_list_lookup_unsuccessful 59 }; 60 61 class IndexError : public llvm::ErrorInfo<IndexError> { 62 public: 63 static char ID; IndexError(index_error_code C)64 IndexError(index_error_code C) : Code(C), LineNo(0) {} 65 IndexError(index_error_code C, std::string FileName, int LineNo = 0) Code(C)66 : Code(C), FileName(std::move(FileName)), LineNo(LineNo) {} IndexError(index_error_code C,std::string FileName,std::string TripleToName,std::string TripleFromName)67 IndexError(index_error_code C, std::string FileName, std::string TripleToName, 68 std::string TripleFromName) 69 : Code(C), FileName(std::move(FileName)), 70 TripleToName(std::move(TripleToName)), 71 TripleFromName(std::move(TripleFromName)) {} 72 void log(raw_ostream &OS) const override; 73 std::error_code convertToErrorCode() const override; getCode()74 index_error_code getCode() const { return Code; } getLineNum()75 int getLineNum() const { return LineNo; } getFileName()76 std::string getFileName() const { return FileName; } getTripleToName()77 std::string getTripleToName() const { return TripleToName; } getTripleFromName()78 std::string getTripleFromName() const { return TripleFromName; } 79 80 private: 81 index_error_code Code; 82 std::string FileName; 83 int LineNo; 84 std::string TripleToName; 85 std::string TripleFromName; 86 }; 87 88 /// This function parses an index file that determines which 89 /// translation unit contains which definition. The IndexPath is not prefixed 90 /// with CTUDir, so an absolute path is expected for consistent results. 91 /// 92 /// The index file format is the following: 93 /// each line consists of an USR and a filepath separated by a space. 94 /// 95 /// \return Returns a map where the USR is the key and the filepath is the value 96 /// or an error. 97 llvm::Expected<llvm::StringMap<std::string>> 98 parseCrossTUIndex(StringRef IndexPath); 99 100 std::string createCrossTUIndexString(const llvm::StringMap<std::string> &Index); 101 102 using InvocationListTy = llvm::StringMap<llvm::SmallVector<std::string, 32>>; 103 /// Parse the YAML formatted invocation list file content \p FileContent. 104 /// The format is expected to be a mapping from from absolute source file 105 /// paths in the filesystem to a list of command-line parts, which 106 /// constitute the invocation needed to compile that file. That invocation 107 /// will be used to produce the AST of the TU. 108 llvm::Expected<InvocationListTy> parseInvocationList( 109 StringRef FileContent, 110 llvm::sys::path::Style PathStyle = llvm::sys::path::Style::posix); 111 112 // Returns true if the variable or any field of a record variable is const. 113 bool containsConst(const VarDecl *VD, const ASTContext &ACtx); 114 115 /// This class is used for tools that requires cross translation 116 /// unit capability. 117 /// 118 /// This class can load definitions from external AST sources. 119 /// The loaded definition will be merged back to the original AST using the 120 /// AST Importer. 121 /// In order to use this class, an index file is required that describes 122 /// the locations of the AST files for each definition. 123 /// 124 /// Note that this class also implements caching. 125 class CrossTranslationUnitContext { 126 public: 127 CrossTranslationUnitContext(CompilerInstance &CI); 128 ~CrossTranslationUnitContext(); 129 130 /// This function loads a function or variable definition from an 131 /// external AST file and merges it into the original AST. 132 /// 133 /// This method should only be used on functions that have no definitions or 134 /// variables that have no initializer in 135 /// the current translation unit. A function definition with the same 136 /// declaration will be looked up in the index file which should be in the 137 /// \p CrossTUDir directory, called \p IndexName. In case the declaration is 138 /// found in the index the corresponding AST will be loaded and the 139 /// definition will be merged into the original AST using the AST Importer. 140 /// 141 /// \return The declaration with the definition will be returned. 142 /// If no suitable definition is found in the index file or multiple 143 /// definitions found error will be returned. 144 /// 145 /// Note that the AST files should also be in the \p CrossTUDir. 146 llvm::Expected<const FunctionDecl *> 147 getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir, 148 StringRef IndexName, bool DisplayCTUProgress = false); 149 llvm::Expected<const VarDecl *> 150 getCrossTUDefinition(const VarDecl *VD, StringRef CrossTUDir, 151 StringRef IndexName, bool DisplayCTUProgress = false); 152 153 /// This function loads a definition from an external AST file. 154 /// 155 /// A definition with the same declaration will be looked up in the 156 /// index file which should be in the \p CrossTUDir directory, called 157 /// \p IndexName. In case the declaration is found in the index the 158 /// corresponding AST will be loaded. If the number of TUs imported 159 /// reaches \p CTULoadTreshold, no loading is performed. 160 /// 161 /// \return Returns a pointer to the ASTUnit that contains the definition of 162 /// the looked up name or an Error. 163 /// The returned pointer is never a nullptr. 164 /// 165 /// Note that the AST files should also be in the \p CrossTUDir. 166 llvm::Expected<ASTUnit *> loadExternalAST(StringRef LookupName, 167 StringRef CrossTUDir, 168 StringRef IndexName, 169 bool DisplayCTUProgress = false); 170 171 /// This function merges a definition from a separate AST Unit into 172 /// the current one which was created by the compiler instance that 173 /// was passed to the constructor. 174 /// 175 /// \return Returns the resulting definition or an error. 176 llvm::Expected<const FunctionDecl *> importDefinition(const FunctionDecl *FD, 177 ASTUnit *Unit); 178 llvm::Expected<const VarDecl *> importDefinition(const VarDecl *VD, 179 ASTUnit *Unit); 180 181 /// Get a name to identify a named decl. 182 static llvm::Optional<std::string> getLookupName(const NamedDecl *ND); 183 184 /// Emit diagnostics for the user for potential configuration errors. 185 void emitCrossTUDiagnostics(const IndexError &IE); 186 187 /// Returns the MacroExpansionContext for the imported TU to which the given 188 /// source-location corresponds. 189 /// \p ToLoc Source location in the imported-to AST. 190 /// \note If any error happens such as \p ToLoc is a non-imported 191 /// source-location, empty is returned. 192 /// \note Macro expansion tracking for imported TUs is not implemented yet. 193 /// It returns empty unconditionally. 194 llvm::Optional<clang::MacroExpansionContext> 195 getMacroExpansionContextForSourceLocation( 196 const clang::SourceLocation &ToLoc) const; 197 198 private: 199 void lazyInitImporterSharedSt(TranslationUnitDecl *ToTU); 200 ASTImporter &getOrCreateASTImporter(ASTUnit *Unit); 201 template <typename T> 202 llvm::Expected<const T *> getCrossTUDefinitionImpl(const T *D, 203 StringRef CrossTUDir, 204 StringRef IndexName, 205 bool DisplayCTUProgress); 206 template <typename T> 207 const T *findDefInDeclContext(const DeclContext *DC, 208 StringRef LookupName); 209 template <typename T> 210 llvm::Expected<const T *> importDefinitionImpl(const T *D, ASTUnit *Unit); 211 212 using ImporterMapTy = 213 llvm::DenseMap<TranslationUnitDecl *, std::unique_ptr<ASTImporter>>; 214 215 ImporterMapTy ASTUnitImporterMap; 216 217 ASTContext &Context; 218 std::shared_ptr<ASTImporterSharedState> ImporterSharedSt; 219 220 using LoadResultTy = llvm::Expected<std::unique_ptr<ASTUnit>>; 221 222 /// Loads ASTUnits from AST-dumps or source-files. 223 class ASTLoader { 224 public: 225 ASTLoader(CompilerInstance &CI, StringRef CTUDir, 226 StringRef InvocationListFilePath); 227 228 /// Load the ASTUnit by its identifier found in the index file. If the 229 /// indentifier is suffixed with '.ast' it is considered a dump. Otherwise 230 /// it is treated as source-file, and on-demand parsed. Relative paths are 231 /// prefixed with CTUDir. 232 LoadResultTy load(StringRef Identifier); 233 234 /// Lazily initialize the invocation list information, which is needed for 235 /// on-demand parsing. 236 llvm::Error lazyInitInvocationList(); 237 238 private: 239 /// The style used for storage and lookup of filesystem paths. 240 /// Defaults to posix. 241 const llvm::sys::path::Style PathStyle = llvm::sys::path::Style::posix; 242 243 /// Loads an AST from a pch-dump. 244 LoadResultTy loadFromDump(StringRef Identifier); 245 /// Loads an AST from a source-file. 246 LoadResultTy loadFromSource(StringRef Identifier); 247 248 CompilerInstance &CI; 249 StringRef CTUDir; 250 /// The path to the file containing the invocation list, which is in YAML 251 /// format, and contains a mapping from source files to compiler invocations 252 /// that produce the AST used for analysis. 253 StringRef InvocationListFilePath; 254 /// In case of on-demand parsing, the invocations for parsing the source 255 /// files is stored. 256 llvm::Optional<InvocationListTy> InvocationList; 257 index_error_code PreviousParsingResult = index_error_code::success; 258 }; 259 260 /// Maintain number of AST loads and check for reaching the load limit. 261 class ASTLoadGuard { 262 public: ASTLoadGuard(unsigned Limit)263 ASTLoadGuard(unsigned Limit) : Limit(Limit) {} 264 265 /// Indicates, whether a new load operation is permitted, it is within the 266 /// threshold. 267 operator bool() const { return Count < Limit; } 268 269 /// Tell that a new AST was loaded successfully. indicateLoadSuccess()270 void indicateLoadSuccess() { ++Count; } 271 272 private: 273 /// The number of ASTs actually imported. 274 unsigned Count{0u}; 275 /// The limit (threshold) value for number of loaded ASTs. 276 const unsigned Limit; 277 }; 278 279 /// Storage and load of ASTUnits, cached access, and providing searchability 280 /// are the concerns of ASTUnitStorage class. 281 class ASTUnitStorage { 282 public: 283 ASTUnitStorage(CompilerInstance &CI); 284 /// Loads an ASTUnit for a function. 285 /// 286 /// \param FunctionName USR name of the function. 287 /// \param CrossTUDir Path to the directory used to store CTU related files. 288 /// \param IndexName Name of the file inside \p CrossTUDir which maps 289 /// function USR names to file paths. These files contain the corresponding 290 /// AST-dumps. 291 /// \param DisplayCTUProgress Display a message about loading new ASTs. 292 /// 293 /// \return An Expected instance which contains the ASTUnit pointer or the 294 /// error occured during the load. 295 llvm::Expected<ASTUnit *> getASTUnitForFunction(StringRef FunctionName, 296 StringRef CrossTUDir, 297 StringRef IndexName, 298 bool DisplayCTUProgress); 299 /// Identifies the path of the file which can be used to load the ASTUnit 300 /// for a given function. 301 /// 302 /// \param FunctionName USR name of the function. 303 /// \param CrossTUDir Path to the directory used to store CTU related files. 304 /// \param IndexName Name of the file inside \p CrossTUDir which maps 305 /// function USR names to file paths. These files contain the corresponding 306 /// AST-dumps. 307 /// 308 /// \return An Expected instance containing the filepath. 309 llvm::Expected<std::string> getFileForFunction(StringRef FunctionName, 310 StringRef CrossTUDir, 311 StringRef IndexName); 312 313 private: 314 llvm::Error ensureCTUIndexLoaded(StringRef CrossTUDir, StringRef IndexName); 315 llvm::Expected<ASTUnit *> getASTUnitForFile(StringRef FileName, 316 bool DisplayCTUProgress); 317 318 template <typename... T> using BaseMapTy = llvm::StringMap<T...>; 319 using OwningMapTy = BaseMapTy<std::unique_ptr<clang::ASTUnit>>; 320 using NonOwningMapTy = BaseMapTy<clang::ASTUnit *>; 321 322 OwningMapTy FileASTUnitMap; 323 NonOwningMapTy NameASTUnitMap; 324 325 using IndexMapTy = BaseMapTy<std::string>; 326 IndexMapTy NameFileMap; 327 328 /// Loads the AST based on the identifier found in the index. 329 ASTLoader Loader; 330 331 /// Limit the number of loaded ASTs. It is used to limit the memory usage 332 /// of the CrossTranslationUnitContext. The ASTUnitStorage has the 333 /// information whether the AST to load is actually loaded or returned from 334 /// cache. This information is needed to maintain the counter. 335 ASTLoadGuard LoadGuard; 336 }; 337 338 ASTUnitStorage ASTStorage; 339 }; 340 341 } // namespace cross_tu 342 } // namespace clang 343 344 #endif // LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H 345