1 //===-- clang-import-test.cpp - ASTImporter/ExternalASTSource testbed -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/AST/ASTContext.h"
10 #include "clang/AST/ASTImporter.h"
11 #include "clang/AST/DeclObjC.h"
12 #include "clang/AST/ExternalASTMerger.h"
13 #include "clang/Basic/Builtins.h"
14 #include "clang/Basic/IdentifierTable.h"
15 #include "clang/Basic/SourceLocation.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/Basic/TargetOptions.h"
18 #include "clang/CodeGen/ModuleBuilder.h"
19 #include "clang/Driver/Types.h"
20 #include "clang/Frontend/ASTConsumers.h"
21 #include "clang/Frontend/CompilerInstance.h"
22 #include "clang/Frontend/MultiplexConsumer.h"
23 #include "clang/Frontend/TextDiagnosticBuffer.h"
24 #include "clang/Lex/Lexer.h"
25 #include "clang/Lex/Preprocessor.h"
26 #include "clang/Parse/ParseAST.h"
27 
28 #include "llvm/IR/LLVMContext.h"
29 #include "llvm/IR/Module.h"
30 #include "llvm/Support/CommandLine.h"
31 #include "llvm/Support/Error.h"
32 #include "llvm/Support/Host.h"
33 #include "llvm/Support/Signals.h"
34 
35 #include <memory>
36 #include <string>
37 
38 using namespace clang;
39 
40 static llvm::cl::opt<std::string> Expression(
41     "expression", llvm::cl::Required,
42     llvm::cl::desc("Path to a file containing the expression to parse"));
43 
44 static llvm::cl::list<std::string>
45     Imports("import", llvm::cl::ZeroOrMore,
46             llvm::cl::desc("Path to a file containing declarations to import"));
47 
48 static llvm::cl::opt<bool>
49     Direct("direct", llvm::cl::Optional,
50            llvm::cl::desc("Use the parsed declarations without indirection"));
51 
52 static llvm::cl::opt<bool> UseOrigins(
53     "use-origins", llvm::cl::Optional,
54     llvm::cl::desc(
55         "Use DeclContext origin information for more accurate lookups"));
56 
57 static llvm::cl::list<std::string>
58     ClangArgs("Xcc", llvm::cl::ZeroOrMore,
59               llvm::cl::desc("Argument to pass to the CompilerInvocation"),
60               llvm::cl::CommaSeparated);
61 
62 static llvm::cl::opt<std::string>
63     Input("x", llvm::cl::Optional,
64           llvm::cl::desc("The language to parse (default: c++)"),
65           llvm::cl::init("c++"));
66 
67 static llvm::cl::opt<bool> DumpAST("dump-ast", llvm::cl::init(false),
68                                    llvm::cl::desc("Dump combined AST"));
69 
70 static llvm::cl::opt<bool> DumpIR("dump-ir", llvm::cl::init(false),
71                                   llvm::cl::desc("Dump IR from final parse"));
72 
73 namespace init_convenience {
74 class TestDiagnosticConsumer : public DiagnosticConsumer {
75 private:
76   std::unique_ptr<TextDiagnosticBuffer> Passthrough;
77   const LangOptions *LangOpts = nullptr;
78 
79 public:
80   TestDiagnosticConsumer()
81       : Passthrough(std::make_unique<TextDiagnosticBuffer>()) {}
82 
83   virtual void BeginSourceFile(const LangOptions &LangOpts,
84                                const Preprocessor *PP = nullptr) override {
85     this->LangOpts = &LangOpts;
86     return Passthrough->BeginSourceFile(LangOpts, PP);
87   }
88 
89   virtual void EndSourceFile() override {
90     this->LangOpts = nullptr;
91     Passthrough->EndSourceFile();
92   }
93 
94   virtual bool IncludeInDiagnosticCounts() const override {
95     return Passthrough->IncludeInDiagnosticCounts();
96   }
97 
98 private:
99   static void PrintSourceForLocation(const SourceLocation &Loc,
100                                      SourceManager &SM) {
101     const char *LocData = SM.getCharacterData(Loc, /*Invalid=*/nullptr);
102     unsigned LocColumn =
103         SM.getSpellingColumnNumber(Loc, /*Invalid=*/nullptr) - 1;
104     FileID FID = SM.getFileID(Loc);
105     const llvm::MemoryBuffer *Buffer =
106         SM.getBuffer(FID, Loc, /*Invalid=*/nullptr);
107 
108     assert(LocData >= Buffer->getBufferStart() &&
109            LocData < Buffer->getBufferEnd());
110 
111     const char *LineBegin = LocData - LocColumn;
112 
113     assert(LineBegin >= Buffer->getBufferStart());
114 
115     const char *LineEnd = nullptr;
116 
117     for (LineEnd = LineBegin; *LineEnd != '\n' && *LineEnd != '\r' &&
118                               LineEnd < Buffer->getBufferEnd();
119          ++LineEnd)
120       ;
121 
122     llvm::StringRef LineString(LineBegin, LineEnd - LineBegin);
123 
124     llvm::errs() << LineString << '\n';
125     llvm::errs().indent(LocColumn);
126     llvm::errs() << '^';
127     llvm::errs() << '\n';
128   }
129 
130   virtual void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
131                                 const Diagnostic &Info) override {
132     if (Info.hasSourceManager() && LangOpts) {
133       SourceManager &SM = Info.getSourceManager();
134 
135       if (Info.getLocation().isValid()) {
136         Info.getLocation().print(llvm::errs(), SM);
137         llvm::errs() << ": ";
138       }
139 
140       SmallString<16> DiagText;
141       Info.FormatDiagnostic(DiagText);
142       llvm::errs() << DiagText << '\n';
143 
144       if (Info.getLocation().isValid()) {
145         PrintSourceForLocation(Info.getLocation(), SM);
146       }
147 
148       for (const CharSourceRange &Range : Info.getRanges()) {
149         bool Invalid = true;
150         StringRef Ref = Lexer::getSourceText(Range, SM, *LangOpts, &Invalid);
151         if (!Invalid) {
152           llvm::errs() << Ref << '\n';
153         }
154       }
155     }
156     DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info);
157   }
158 };
159 
160 std::unique_ptr<CompilerInstance> BuildCompilerInstance() {
161   auto Ins = std::make_unique<CompilerInstance>();
162   auto DC = std::make_unique<TestDiagnosticConsumer>();
163   const bool ShouldOwnClient = true;
164   Ins->createDiagnostics(DC.release(), ShouldOwnClient);
165 
166   auto Inv = std::make_unique<CompilerInvocation>();
167 
168   std::vector<const char *> ClangArgv(ClangArgs.size());
169   std::transform(ClangArgs.begin(), ClangArgs.end(), ClangArgv.begin(),
170                  [](const std::string &s) -> const char * { return s.data(); });
171   CompilerInvocation::CreateFromArgs(*Inv, ClangArgv, Ins->getDiagnostics());
172 
173   {
174     using namespace driver::types;
175     ID Id = lookupTypeForTypeSpecifier(Input.c_str());
176     assert(Id != TY_INVALID);
177     if (isCXX(Id)) {
178       Inv->getLangOpts()->CPlusPlus = true;
179       Inv->getLangOpts()->CPlusPlus11 = true;
180       Inv->getHeaderSearchOpts().UseLibcxx = true;
181     }
182     if (isObjC(Id)) {
183       Inv->getLangOpts()->ObjC = 1;
184     }
185   }
186   Inv->getLangOpts()->Bool = true;
187   Inv->getLangOpts()->WChar = true;
188   Inv->getLangOpts()->Blocks = true;
189   Inv->getLangOpts()->DebuggerSupport = true;
190   Inv->getLangOpts()->SpellChecking = false;
191   Inv->getLangOpts()->ThreadsafeStatics = false;
192   Inv->getLangOpts()->AccessControl = false;
193   Inv->getLangOpts()->DollarIdents = true;
194   Inv->getLangOpts()->Exceptions = true;
195   Inv->getLangOpts()->CXXExceptions = true;
196   // Needed for testing dynamic_cast.
197   Inv->getLangOpts()->RTTI = true;
198   Inv->getCodeGenOpts().setDebugInfo(codegenoptions::FullDebugInfo);
199   Inv->getTargetOpts().Triple = llvm::sys::getDefaultTargetTriple();
200 
201   Ins->setInvocation(std::move(Inv));
202 
203   TargetInfo *TI = TargetInfo::CreateTargetInfo(
204       Ins->getDiagnostics(), Ins->getInvocation().TargetOpts);
205   Ins->setTarget(TI);
206   Ins->getTarget().adjust(Ins->getLangOpts());
207   Ins->createFileManager();
208   Ins->createSourceManager(Ins->getFileManager());
209   Ins->createPreprocessor(TU_Complete);
210 
211   return Ins;
212 }
213 
214 std::unique_ptr<ASTContext>
215 BuildASTContext(CompilerInstance &CI, SelectorTable &ST, Builtin::Context &BC) {
216   auto AST = std::make_unique<ASTContext>(
217       CI.getLangOpts(), CI.getSourceManager(),
218       CI.getPreprocessor().getIdentifierTable(), ST, BC);
219   AST->InitBuiltinTypes(CI.getTarget());
220   return AST;
221 }
222 
223 std::unique_ptr<CodeGenerator> BuildCodeGen(CompilerInstance &CI,
224                                             llvm::LLVMContext &LLVMCtx) {
225   StringRef ModuleName("$__module");
226   return std::unique_ptr<CodeGenerator>(CreateLLVMCodeGen(
227       CI.getDiagnostics(), ModuleName, CI.getHeaderSearchOpts(),
228       CI.getPreprocessorOpts(), CI.getCodeGenOpts(), LLVMCtx));
229 }
230 } // namespace init_convenience
231 
232 namespace {
233 
234 /// A container for a CompilerInstance (possibly with an ExternalASTMerger
235 /// attached to its ASTContext).
236 ///
237 /// Provides an accessor for the DeclContext origins associated with the
238 /// ExternalASTMerger (or an empty list of origins if no ExternalASTMerger is
239 /// attached).
240 ///
241 /// This is the main unit of parsed source code maintained by clang-import-test.
242 struct CIAndOrigins {
243   using OriginMap = clang::ExternalASTMerger::OriginMap;
244   std::unique_ptr<CompilerInstance> CI;
245 
246   ASTContext &getASTContext() { return CI->getASTContext(); }
247   FileManager &getFileManager() { return CI->getFileManager(); }
248   const OriginMap &getOriginMap() {
249     static const OriginMap EmptyOriginMap{};
250     if (ExternalASTSource *Source = CI->getASTContext().getExternalSource())
251       return static_cast<ExternalASTMerger *>(Source)->GetOrigins();
252     return EmptyOriginMap;
253   }
254   DiagnosticConsumer &getDiagnosticClient() {
255     return CI->getDiagnosticClient();
256   }
257   CompilerInstance &getCompilerInstance() { return *CI; }
258 };
259 
260 void AddExternalSource(CIAndOrigins &CI,
261                        llvm::MutableArrayRef<CIAndOrigins> Imports) {
262   ExternalASTMerger::ImporterTarget Target(
263       {CI.getASTContext(), CI.getFileManager()});
264   llvm::SmallVector<ExternalASTMerger::ImporterSource, 3> Sources;
265   for (CIAndOrigins &Import : Imports)
266     Sources.emplace_back(Import.getASTContext(), Import.getFileManager(),
267                          Import.getOriginMap());
268   auto ES = std::make_unique<ExternalASTMerger>(Target, Sources);
269   CI.getASTContext().setExternalSource(ES.release());
270   CI.getASTContext().getTranslationUnitDecl()->setHasExternalVisibleStorage();
271 }
272 
273 CIAndOrigins BuildIndirect(CIAndOrigins &CI) {
274   CIAndOrigins IndirectCI{init_convenience::BuildCompilerInstance()};
275   auto ST = std::make_unique<SelectorTable>();
276   auto BC = std::make_unique<Builtin::Context>();
277   std::unique_ptr<ASTContext> AST = init_convenience::BuildASTContext(
278       IndirectCI.getCompilerInstance(), *ST, *BC);
279   IndirectCI.getCompilerInstance().setASTContext(AST.release());
280   AddExternalSource(IndirectCI, CI);
281   return IndirectCI;
282 }
283 
284 llvm::Error ParseSource(const std::string &Path, CompilerInstance &CI,
285                         ASTConsumer &Consumer) {
286   SourceManager &SM = CI.getSourceManager();
287   auto FE = CI.getFileManager().getFile(Path);
288   if (!FE) {
289     return llvm::make_error<llvm::StringError>(
290         llvm::Twine("Couldn't open ", Path), std::error_code());
291   }
292   SM.setMainFileID(SM.createFileID(*FE, SourceLocation(), SrcMgr::C_User));
293   ParseAST(CI.getPreprocessor(), &Consumer, CI.getASTContext());
294   return llvm::Error::success();
295 }
296 
297 llvm::Expected<CIAndOrigins> Parse(const std::string &Path,
298                                    llvm::MutableArrayRef<CIAndOrigins> Imports,
299                                    bool ShouldDumpAST, bool ShouldDumpIR) {
300   CIAndOrigins CI{init_convenience::BuildCompilerInstance()};
301   auto ST = std::make_unique<SelectorTable>();
302   auto BC = std::make_unique<Builtin::Context>();
303   std::unique_ptr<ASTContext> AST =
304       init_convenience::BuildASTContext(CI.getCompilerInstance(), *ST, *BC);
305   CI.getCompilerInstance().setASTContext(AST.release());
306   if (Imports.size())
307     AddExternalSource(CI, Imports);
308 
309   std::vector<std::unique_ptr<ASTConsumer>> ASTConsumers;
310 
311   auto LLVMCtx = std::make_unique<llvm::LLVMContext>();
312   ASTConsumers.push_back(
313       init_convenience::BuildCodeGen(CI.getCompilerInstance(), *LLVMCtx));
314   auto &CG = *static_cast<CodeGenerator *>(ASTConsumers.back().get());
315 
316   if (ShouldDumpAST)
317     ASTConsumers.push_back(
318         CreateASTDumper(nullptr /*Dump to stdout.*/, "", true, false, false,
319                         clang::ADOF_Default));
320 
321   CI.getDiagnosticClient().BeginSourceFile(
322       CI.getCompilerInstance().getLangOpts(),
323       &CI.getCompilerInstance().getPreprocessor());
324   MultiplexConsumer Consumers(std::move(ASTConsumers));
325   Consumers.Initialize(CI.getASTContext());
326 
327   if (llvm::Error PE = ParseSource(Path, CI.getCompilerInstance(), Consumers))
328     return std::move(PE);
329   CI.getDiagnosticClient().EndSourceFile();
330   if (ShouldDumpIR)
331     CG.GetModule()->print(llvm::outs(), nullptr);
332   if (CI.getDiagnosticClient().getNumErrors())
333     return llvm::make_error<llvm::StringError>(
334         "Errors occurred while parsing the expression.", std::error_code());
335   return std::move(CI);
336 }
337 
338 void Forget(CIAndOrigins &CI, llvm::MutableArrayRef<CIAndOrigins> Imports) {
339   llvm::SmallVector<ExternalASTMerger::ImporterSource, 3> Sources;
340   for (CIAndOrigins &Import : Imports)
341     Sources.push_back({Import.getASTContext(), Import.getFileManager(),
342                        Import.getOriginMap()});
343   ExternalASTSource *Source = CI.CI->getASTContext().getExternalSource();
344   auto *Merger = static_cast<ExternalASTMerger *>(Source);
345   Merger->RemoveSources(Sources);
346 }
347 
348 } // end namespace
349 
350 int main(int argc, const char **argv) {
351   const bool DisableCrashReporting = true;
352   llvm::sys::PrintStackTraceOnErrorSignal(argv[0], DisableCrashReporting);
353   llvm::cl::ParseCommandLineOptions(argc, argv);
354   std::vector<CIAndOrigins> ImportCIs;
355   for (auto I : Imports) {
356     llvm::Expected<CIAndOrigins> ImportCI = Parse(I, {}, false, false);
357     if (auto E = ImportCI.takeError()) {
358       llvm::errs() << llvm::toString(std::move(E));
359       exit(-1);
360     }
361     ImportCIs.push_back(std::move(*ImportCI));
362   }
363   std::vector<CIAndOrigins> IndirectCIs;
364   if (!Direct || UseOrigins) {
365     for (auto &ImportCI : ImportCIs) {
366       CIAndOrigins IndirectCI = BuildIndirect(ImportCI);
367       IndirectCIs.push_back(std::move(IndirectCI));
368     }
369   }
370   if (UseOrigins)
371     for (auto &ImportCI : ImportCIs)
372       IndirectCIs.push_back(std::move(ImportCI));
373   llvm::Expected<CIAndOrigins> ExpressionCI =
374       Parse(Expression, (Direct && !UseOrigins) ? ImportCIs : IndirectCIs,
375             DumpAST, DumpIR);
376   if (auto E = ExpressionCI.takeError()) {
377     llvm::errs() << llvm::toString(std::move(E));
378     exit(-1);
379   }
380   Forget(*ExpressionCI, (Direct && !UseOrigins) ? ImportCIs : IndirectCIs);
381   return 0;
382 }
383