1 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class which performs incremental code compilation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "IncrementalParser.h"
14 
15 #include "clang/AST/DeclContextInternals.h"
16 #include "clang/CodeGen/BackendUtil.h"
17 #include "clang/CodeGen/CodeGenAction.h"
18 #include "clang/CodeGen/ModuleBuilder.h"
19 #include "clang/Frontend/CompilerInstance.h"
20 #include "clang/Frontend/FrontendAction.h"
21 #include "clang/FrontendTool/Utils.h"
22 #include "clang/Parse/Parser.h"
23 #include "clang/Sema/Sema.h"
24 
25 #include "llvm/Option/ArgList.h"
26 #include "llvm/Support/CrashRecoveryContext.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/Support/Timer.h"
29 
30 #include <sstream>
31 
32 namespace clang {
33 
34 /// A custom action enabling the incremental processing functionality.
35 ///
36 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
37 /// sees a call to \p EndSourceFile it deletes some of the important objects
38 /// such as \p Preprocessor and \p Sema assuming no further input will come.
39 ///
40 /// \p IncrementalAction ensures it keep its underlying action's objects alive
41 /// as long as the \p IncrementalParser needs them.
42 ///
43 class IncrementalAction : public WrapperFrontendAction {
44 private:
45   bool IsTerminating = false;
46 
47 public:
48   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
49                     llvm::Error &Err)
50       : WrapperFrontendAction([&]() {
51           llvm::ErrorAsOutParameter EAO(&Err);
52           std::unique_ptr<FrontendAction> Act;
53           switch (CI.getFrontendOpts().ProgramAction) {
54           default:
55             Err = llvm::createStringError(
56                 std::errc::state_not_recoverable,
57                 "Driver initialization failed. "
58                 "Incremental mode for action %d is not supported",
59                 CI.getFrontendOpts().ProgramAction);
60             return Act;
61           case frontend::ASTDump:
62             LLVM_FALLTHROUGH;
63           case frontend::ASTPrint:
64             LLVM_FALLTHROUGH;
65           case frontend::ParseSyntaxOnly:
66             Act = CreateFrontendAction(CI);
67             break;
68           case frontend::PluginAction:
69             LLVM_FALLTHROUGH;
70           case frontend::EmitAssembly:
71             LLVM_FALLTHROUGH;
72           case frontend::EmitObj:
73             LLVM_FALLTHROUGH;
74           case frontend::EmitLLVMOnly:
75             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
76             break;
77           }
78           return Act;
79         }()) {}
80   FrontendAction *getWrapped() const { return WrappedAction.get(); }
81   TranslationUnitKind getTranslationUnitKind() override {
82     return TU_Incremental;
83   }
84   void ExecuteAction() override {
85     CompilerInstance &CI = getCompilerInstance();
86     assert(CI.hasPreprocessor() && "No PP!");
87 
88     // FIXME: Move the truncation aspect of this into Sema, we delayed this till
89     // here so the source manager would be initialized.
90     if (hasCodeCompletionSupport() &&
91         !CI.getFrontendOpts().CodeCompletionAt.FileName.empty())
92       CI.createCodeCompletionConsumer();
93 
94     // Use a code completion consumer?
95     CodeCompleteConsumer *CompletionConsumer = nullptr;
96     if (CI.hasCodeCompletionConsumer())
97       CompletionConsumer = &CI.getCodeCompletionConsumer();
98 
99     Preprocessor &PP = CI.getPreprocessor();
100     PP.enableIncrementalProcessing();
101     PP.EnterMainSourceFile();
102 
103     if (!CI.hasSema())
104       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
105   }
106 
107   // Do not terminate after processing the input. This allows us to keep various
108   // clang objects alive and to incrementally grow the current TU.
109   void EndSourceFile() override {
110     // The WrappedAction can be nullptr if we issued an error in the ctor.
111     if (IsTerminating && getWrapped())
112       WrapperFrontendAction::EndSourceFile();
113   }
114 
115   void FinalizeAction() {
116     assert(!IsTerminating && "Already finalized!");
117     IsTerminating = true;
118     EndSourceFile();
119   }
120 };
121 
122 IncrementalParser::IncrementalParser(std::unique_ptr<CompilerInstance> Instance,
123                                      llvm::LLVMContext &LLVMCtx,
124                                      llvm::Error &Err)
125     : CI(std::move(Instance)) {
126   llvm::ErrorAsOutParameter EAO(&Err);
127   Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
128   if (Err)
129     return;
130   CI->ExecuteAction(*Act);
131   Consumer = &CI->getASTConsumer();
132   P.reset(
133       new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
134   P->Initialize();
135 }
136 
137 IncrementalParser::~IncrementalParser() {
138   P.reset();
139   Act->FinalizeAction();
140 }
141 
142 llvm::Expected<PartialTranslationUnit &>
143 IncrementalParser::ParseOrWrapTopLevelDecl() {
144   // Recover resources if we crash before exiting this method.
145   Sema &S = CI->getSema();
146   llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
147   Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
148   Sema::LocalEagerInstantiationScope LocalInstantiations(S);
149 
150   PTUs.emplace_back(PartialTranslationUnit());
151   PartialTranslationUnit &LastPTU = PTUs.back();
152   // Add a new PTU.
153   ASTContext &C = S.getASTContext();
154   C.addTranslationUnitDecl();
155   LastPTU.TUPart = C.getTranslationUnitDecl();
156 
157   // Skip previous eof due to last incremental input.
158   if (P->getCurToken().is(tok::eof)) {
159     P->ConsumeToken();
160     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
161     // might want to do that around HandleEndOfTranslationUnit.
162     P->ExitScope();
163     S.CurContext = nullptr;
164     // Start a new PTU.
165     P->EnterScope(Scope::DeclScope);
166     S.ActOnTranslationUnitScope(P->getCurScope());
167   }
168 
169   Parser::DeclGroupPtrTy ADecl;
170   Sema::ModuleImportState ImportState;
171   for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl, ImportState); !AtEOF;
172        AtEOF = P->ParseTopLevelDecl(ADecl, ImportState)) {
173     // If we got a null return and something *was* parsed, ignore it.  This
174     // is due to a top-level semicolon, an action override, or a parse error
175     // skipping something.
176     if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
177       return llvm::make_error<llvm::StringError>("Parsing failed. "
178                                                  "The consumer rejected a decl",
179                                                  std::error_code());
180   }
181 
182   DiagnosticsEngine &Diags = getCI()->getDiagnostics();
183   if (Diags.hasErrorOccurred()) {
184     PartialTranslationUnit MostRecentPTU = {C.getTranslationUnitDecl(),
185                                             nullptr};
186     CleanUpPTU(MostRecentPTU);
187 
188     Diags.Reset(/*soft=*/true);
189     Diags.getClient()->clear();
190     return llvm::make_error<llvm::StringError>("Parsing failed.",
191                                                std::error_code());
192   }
193 
194   // Process any TopLevelDecls generated by #pragma weak.
195   for (Decl *D : S.WeakTopLevelDecls()) {
196     DeclGroupRef DGR(D);
197     Consumer->HandleTopLevelDecl(DGR);
198   }
199 
200   LocalInstantiations.perform();
201   GlobalInstantiations.perform();
202 
203   Consumer->HandleTranslationUnit(C);
204 
205   return LastPTU;
206 }
207 
208 static CodeGenerator *getCodeGen(FrontendAction *Act) {
209   IncrementalAction *IncrAct = static_cast<IncrementalAction *>(Act);
210   FrontendAction *WrappedAct = IncrAct->getWrapped();
211   if (!WrappedAct->hasIRSupport())
212     return nullptr;
213   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
214 }
215 
216 llvm::Expected<PartialTranslationUnit &>
217 IncrementalParser::Parse(llvm::StringRef input) {
218   Preprocessor &PP = CI->getPreprocessor();
219   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
220 
221   std::ostringstream SourceName;
222   SourceName << "input_line_" << InputCount++;
223 
224   // Create an uninitialized memory buffer, copy code in and append "\n"
225   size_t InputSize = input.size(); // don't include trailing 0
226   // MemBuffer size should *not* include terminating zero
227   std::unique_ptr<llvm::MemoryBuffer> MB(
228       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
229                                                         SourceName.str()));
230   char *MBStart = const_cast<char *>(MB->getBufferStart());
231   memcpy(MBStart, input.data(), InputSize);
232   MBStart[InputSize] = '\n';
233 
234   SourceManager &SM = CI->getSourceManager();
235 
236   // FIXME: Create SourceLocation, which will allow clang to order the overload
237   // candidates for example
238   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
239 
240   // Create FileID for the current buffer.
241   FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
242                                /*LoadedOffset=*/0, NewLoc);
243 
244   // NewLoc only used for diags.
245   if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
246     return llvm::make_error<llvm::StringError>("Parsing failed. "
247                                                "Cannot enter source file.",
248                                                std::error_code());
249 
250   auto PTU = ParseOrWrapTopLevelDecl();
251   if (!PTU)
252     return PTU.takeError();
253 
254   if (PP.getLangOpts().DelayedTemplateParsing) {
255     // Microsoft-specific:
256     // Late parsed templates can leave unswallowed "macro"-like tokens.
257     // They will seriously confuse the Parser when entering the next
258     // source file. So lex until we are EOF.
259     Token Tok;
260     do {
261       PP.Lex(Tok);
262     } while (Tok.isNot(tok::eof));
263   }
264 
265   Token AssertTok;
266   PP.Lex(AssertTok);
267   assert(AssertTok.is(tok::eof) &&
268          "Lexer must be EOF when starting incremental parse!");
269 
270   if (CodeGenerator *CG = getCodeGen(Act.get())) {
271     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
272     CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
273                     M->getContext());
274 
275     PTU->TheModule = std::move(M);
276   }
277 
278   return PTU;
279 }
280 
281 void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
282   TranslationUnitDecl *MostRecentTU = PTU.TUPart;
283   TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
284   if (StoredDeclsMap *Map = FirstTU->getPrimaryContext()->getLookupPtr()) {
285     for (auto I = Map->begin(); I != Map->end(); ++I) {
286       StoredDeclsList &List = I->second;
287       DeclContextLookupResult R = List.getLookupResult();
288       for (NamedDecl *D : R) {
289         if (D->getTranslationUnitDecl() == MostRecentTU) {
290           List.remove(D);
291         }
292       }
293       if (List.isNull())
294         Map->erase(I);
295     }
296   }
297 }
298 
299 llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
300   CodeGenerator *CG = getCodeGen(Act.get());
301   assert(CG);
302   return CG->GetMangledName(GD);
303 }
304 
305 } // end namespace clang
306