1 //===--- CrossTranslationUnit.h - -------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file provides an interface to load binary AST dumps on demand. This 10 // feature can be utilized for tools that require cross translation unit 11 // support. 12 // 13 //===----------------------------------------------------------------------===// 14 #ifndef LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H 15 #define LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H 16 17 #include "clang/AST/ASTImporterSharedState.h" 18 #include "clang/Analysis/MacroExpansionContext.h" 19 #include "clang/Basic/LLVM.h" 20 #include "llvm/ADT/DenseMap.h" 21 #include "llvm/ADT/Optional.h" 22 #include "llvm/ADT/SmallPtrSet.h" 23 #include "llvm/ADT/StringMap.h" 24 #include "llvm/Support/Error.h" 25 #include "llvm/Support/Path.h" 26 27 namespace clang { 28 class CompilerInstance; 29 class ASTContext; 30 class ASTImporter; 31 class ASTUnit; 32 class DeclContext; 33 class FunctionDecl; 34 class VarDecl; 35 class NamedDecl; 36 class TranslationUnitDecl; 37 38 namespace cross_tu { 39 40 enum class index_error_code { 41 unspecified = 1, 42 missing_index_file, 43 invalid_index_format, 44 multiple_definitions, 45 missing_definition, 46 failed_import, 47 failed_to_get_external_ast, 48 failed_to_generate_usr, 49 triple_mismatch, 50 lang_mismatch, 51 lang_dialect_mismatch, 52 load_threshold_reached, 53 invocation_list_ambiguous, 54 invocation_list_file_not_found, 55 invocation_list_empty, 56 invocation_list_wrong_format, 57 invocation_list_lookup_unsuccessful 58 }; 59 60 class IndexError : public llvm::ErrorInfo<IndexError> { 61 public: 62 static char ID; IndexError(index_error_code C)63 IndexError(index_error_code C) : Code(C), LineNo(0) {} 64 IndexError(index_error_code C, std::string FileName, int LineNo = 0) Code(C)65 : Code(C), FileName(std::move(FileName)), LineNo(LineNo) {} IndexError(index_error_code C,std::string FileName,std::string TripleToName,std::string TripleFromName)66 IndexError(index_error_code C, std::string FileName, std::string TripleToName, 67 std::string TripleFromName) 68 : Code(C), FileName(std::move(FileName)), 69 TripleToName(std::move(TripleToName)), 70 TripleFromName(std::move(TripleFromName)) {} 71 void log(raw_ostream &OS) const override; 72 std::error_code convertToErrorCode() const override; getCode()73 index_error_code getCode() const { return Code; } getLineNum()74 int getLineNum() const { return LineNo; } getFileName()75 std::string getFileName() const { return FileName; } getTripleToName()76 std::string getTripleToName() const { return TripleToName; } getTripleFromName()77 std::string getTripleFromName() const { return TripleFromName; } 78 79 private: 80 index_error_code Code; 81 std::string FileName; 82 int LineNo; 83 std::string TripleToName; 84 std::string TripleFromName; 85 }; 86 87 /// This function parses an index file that determines which 88 /// translation unit contains which definition. The IndexPath is not prefixed 89 /// with CTUDir, so an absolute path is expected for consistent results. 90 /// 91 /// The index file format is the following: 92 /// each line consists of an USR and a filepath separated by a space. 93 /// 94 /// \return Returns a map where the USR is the key and the filepath is the value 95 /// or an error. 96 llvm::Expected<llvm::StringMap<std::string>> 97 parseCrossTUIndex(StringRef IndexPath); 98 99 std::string createCrossTUIndexString(const llvm::StringMap<std::string> &Index); 100 101 using InvocationListTy = llvm::StringMap<llvm::SmallVector<std::string, 32>>; 102 /// Parse the YAML formatted invocation list file content \p FileContent. 103 /// The format is expected to be a mapping from from absolute source file 104 /// paths in the filesystem to a list of command-line parts, which 105 /// constitute the invocation needed to compile that file. That invocation 106 /// will be used to produce the AST of the TU. 107 llvm::Expected<InvocationListTy> parseInvocationList( 108 StringRef FileContent, 109 llvm::sys::path::Style PathStyle = llvm::sys::path::Style::posix); 110 111 // Returns true if the variable or any field of a record variable is const. 112 bool containsConst(const VarDecl *VD, const ASTContext &ACtx); 113 114 /// This class is used for tools that requires cross translation 115 /// unit capability. 116 /// 117 /// This class can load definitions from external AST sources. 118 /// The loaded definition will be merged back to the original AST using the 119 /// AST Importer. 120 /// In order to use this class, an index file is required that describes 121 /// the locations of the AST files for each definition. 122 /// 123 /// Note that this class also implements caching. 124 class CrossTranslationUnitContext { 125 public: 126 CrossTranslationUnitContext(CompilerInstance &CI); 127 ~CrossTranslationUnitContext(); 128 129 /// This function loads a function or variable definition from an 130 /// external AST file and merges it into the original AST. 131 /// 132 /// This method should only be used on functions that have no definitions or 133 /// variables that have no initializer in 134 /// the current translation unit. A function definition with the same 135 /// declaration will be looked up in the index file which should be in the 136 /// \p CrossTUDir directory, called \p IndexName. In case the declaration is 137 /// found in the index the corresponding AST will be loaded and the 138 /// definition will be merged into the original AST using the AST Importer. 139 /// 140 /// \return The declaration with the definition will be returned. 141 /// If no suitable definition is found in the index file or multiple 142 /// definitions found error will be returned. 143 /// 144 /// Note that the AST files should also be in the \p CrossTUDir. 145 llvm::Expected<const FunctionDecl *> 146 getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir, 147 StringRef IndexName, bool DisplayCTUProgress = false); 148 llvm::Expected<const VarDecl *> 149 getCrossTUDefinition(const VarDecl *VD, StringRef CrossTUDir, 150 StringRef IndexName, bool DisplayCTUProgress = false); 151 152 /// This function loads a definition from an external AST file. 153 /// 154 /// A definition with the same declaration will be looked up in the 155 /// index file which should be in the \p CrossTUDir directory, called 156 /// \p IndexName. In case the declaration is found in the index the 157 /// corresponding AST will be loaded. If the number of TUs imported 158 /// reaches \p CTULoadTreshold, no loading is performed. 159 /// 160 /// \return Returns a pointer to the ASTUnit that contains the definition of 161 /// the looked up name or an Error. 162 /// The returned pointer is never a nullptr. 163 /// 164 /// Note that the AST files should also be in the \p CrossTUDir. 165 llvm::Expected<ASTUnit *> loadExternalAST(StringRef LookupName, 166 StringRef CrossTUDir, 167 StringRef IndexName, 168 bool DisplayCTUProgress = false); 169 170 /// This function merges a definition from a separate AST Unit into 171 /// the current one which was created by the compiler instance that 172 /// was passed to the constructor. 173 /// 174 /// \return Returns the resulting definition or an error. 175 llvm::Expected<const FunctionDecl *> importDefinition(const FunctionDecl *FD, 176 ASTUnit *Unit); 177 llvm::Expected<const VarDecl *> importDefinition(const VarDecl *VD, 178 ASTUnit *Unit); 179 180 /// Get a name to identify a named decl. 181 static llvm::Optional<std::string> getLookupName(const NamedDecl *ND); 182 183 /// Emit diagnostics for the user for potential configuration errors. 184 void emitCrossTUDiagnostics(const IndexError &IE); 185 186 /// Returns the MacroExpansionContext for the imported TU to which the given 187 /// source-location corresponds. 188 /// \p ToLoc Source location in the imported-to AST. 189 /// \note If any error happens such as \p ToLoc is a non-imported 190 /// source-location, empty is returned. 191 /// \note Macro expansion tracking for imported TUs is not implemented yet. 192 /// It returns empty unconditionally. 193 llvm::Optional<clang::MacroExpansionContext> 194 getMacroExpansionContextForSourceLocation( 195 const clang::SourceLocation &ToLoc) const; 196 197 private: 198 void lazyInitImporterSharedSt(TranslationUnitDecl *ToTU); 199 ASTImporter &getOrCreateASTImporter(ASTUnit *Unit); 200 template <typename T> 201 llvm::Expected<const T *> getCrossTUDefinitionImpl(const T *D, 202 StringRef CrossTUDir, 203 StringRef IndexName, 204 bool DisplayCTUProgress); 205 template <typename T> 206 const T *findDefInDeclContext(const DeclContext *DC, 207 StringRef LookupName); 208 template <typename T> 209 llvm::Expected<const T *> importDefinitionImpl(const T *D, ASTUnit *Unit); 210 211 using ImporterMapTy = 212 llvm::DenseMap<TranslationUnitDecl *, std::unique_ptr<ASTImporter>>; 213 214 ImporterMapTy ASTUnitImporterMap; 215 216 ASTContext &Context; 217 std::shared_ptr<ASTImporterSharedState> ImporterSharedSt; 218 219 using LoadResultTy = llvm::Expected<std::unique_ptr<ASTUnit>>; 220 221 /// Loads ASTUnits from AST-dumps or source-files. 222 class ASTLoader { 223 public: 224 ASTLoader(CompilerInstance &CI, StringRef CTUDir, 225 StringRef InvocationListFilePath); 226 227 /// Load the ASTUnit by its identifier found in the index file. If the 228 /// indentifier is suffixed with '.ast' it is considered a dump. Otherwise 229 /// it is treated as source-file, and on-demand parsed. Relative paths are 230 /// prefixed with CTUDir. 231 LoadResultTy load(StringRef Identifier); 232 233 /// Lazily initialize the invocation list information, which is needed for 234 /// on-demand parsing. 235 llvm::Error lazyInitInvocationList(); 236 237 private: 238 /// The style used for storage and lookup of filesystem paths. 239 /// Defaults to posix. 240 const llvm::sys::path::Style PathStyle = llvm::sys::path::Style::posix; 241 242 /// Loads an AST from a pch-dump. 243 LoadResultTy loadFromDump(StringRef Identifier); 244 /// Loads an AST from a source-file. 245 LoadResultTy loadFromSource(StringRef Identifier); 246 247 CompilerInstance &CI; 248 StringRef CTUDir; 249 /// The path to the file containing the invocation list, which is in YAML 250 /// format, and contains a mapping from source files to compiler invocations 251 /// that produce the AST used for analysis. 252 StringRef InvocationListFilePath; 253 /// In case of on-demand parsing, the invocations for parsing the source 254 /// files is stored. 255 llvm::Optional<InvocationListTy> InvocationList; 256 }; 257 258 /// Maintain number of AST loads and check for reaching the load limit. 259 class ASTLoadGuard { 260 public: ASTLoadGuard(unsigned Limit)261 ASTLoadGuard(unsigned Limit) : Limit(Limit) {} 262 263 /// Indicates, whether a new load operation is permitted, it is within the 264 /// threshold. 265 operator bool() const { return Count < Limit; } 266 267 /// Tell that a new AST was loaded successfully. indicateLoadSuccess()268 void indicateLoadSuccess() { ++Count; } 269 270 private: 271 /// The number of ASTs actually imported. 272 unsigned Count{0u}; 273 /// The limit (threshold) value for number of loaded ASTs. 274 const unsigned Limit; 275 }; 276 277 /// Storage and load of ASTUnits, cached access, and providing searchability 278 /// are the concerns of ASTUnitStorage class. 279 class ASTUnitStorage { 280 public: 281 ASTUnitStorage(CompilerInstance &CI); 282 /// Loads an ASTUnit for a function. 283 /// 284 /// \param FunctionName USR name of the function. 285 /// \param CrossTUDir Path to the directory used to store CTU related files. 286 /// \param IndexName Name of the file inside \p CrossTUDir which maps 287 /// function USR names to file paths. These files contain the corresponding 288 /// AST-dumps. 289 /// \param DisplayCTUProgress Display a message about loading new ASTs. 290 /// 291 /// \return An Expected instance which contains the ASTUnit pointer or the 292 /// error occured during the load. 293 llvm::Expected<ASTUnit *> getASTUnitForFunction(StringRef FunctionName, 294 StringRef CrossTUDir, 295 StringRef IndexName, 296 bool DisplayCTUProgress); 297 /// Identifies the path of the file which can be used to load the ASTUnit 298 /// for a given function. 299 /// 300 /// \param FunctionName USR name of the function. 301 /// \param CrossTUDir Path to the directory used to store CTU related files. 302 /// \param IndexName Name of the file inside \p CrossTUDir which maps 303 /// function USR names to file paths. These files contain the corresponding 304 /// AST-dumps. 305 /// 306 /// \return An Expected instance containing the filepath. 307 llvm::Expected<std::string> getFileForFunction(StringRef FunctionName, 308 StringRef CrossTUDir, 309 StringRef IndexName); 310 311 private: 312 llvm::Error ensureCTUIndexLoaded(StringRef CrossTUDir, StringRef IndexName); 313 llvm::Expected<ASTUnit *> getASTUnitForFile(StringRef FileName, 314 bool DisplayCTUProgress); 315 316 template <typename... T> using BaseMapTy = llvm::StringMap<T...>; 317 using OwningMapTy = BaseMapTy<std::unique_ptr<clang::ASTUnit>>; 318 using NonOwningMapTy = BaseMapTy<clang::ASTUnit *>; 319 320 OwningMapTy FileASTUnitMap; 321 NonOwningMapTy NameASTUnitMap; 322 323 using IndexMapTy = BaseMapTy<std::string>; 324 IndexMapTy NameFileMap; 325 326 /// Loads the AST based on the identifier found in the index. 327 ASTLoader Loader; 328 329 /// Limit the number of loaded ASTs. It is used to limit the memory usage 330 /// of the CrossTranslationUnitContext. The ASTUnitStorage has the 331 /// information whether the AST to load is actually loaded or returned from 332 /// cache. This information is needed to maintain the counter. 333 ASTLoadGuard LoadGuard; 334 }; 335 336 ASTUnitStorage ASTStorage; 337 }; 338 339 } // namespace cross_tu 340 } // namespace clang 341 342 #endif // LLVM_CLANG_CROSSTU_CROSSTRANSLATIONUNIT_H 343