1 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the class which performs incremental code compilation.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "IncrementalParser.h"
14 
15 #include "clang/AST/DeclContextInternals.h"
16 #include "clang/CodeGen/BackendUtil.h"
17 #include "clang/CodeGen/CodeGenAction.h"
18 #include "clang/CodeGen/ModuleBuilder.h"
19 #include "clang/Frontend/CompilerInstance.h"
20 #include "clang/Frontend/FrontendAction.h"
21 #include "clang/FrontendTool/Utils.h"
22 #include "clang/Parse/Parser.h"
23 #include "clang/Sema/Sema.h"
24 
25 #include "llvm/Option/ArgList.h"
26 #include "llvm/Support/CrashRecoveryContext.h"
27 #include "llvm/Support/Error.h"
28 #include "llvm/Support/Timer.h"
29 
30 #include <sstream>
31 
32 namespace clang {
33 
34 /// A custom action enabling the incremental processing functionality.
35 ///
36 /// The usual \p FrontendAction expects one call to ExecuteAction and once it
37 /// sees a call to \p EndSourceFile it deletes some of the important objects
38 /// such as \p Preprocessor and \p Sema assuming no further input will come.
39 ///
40 /// \p IncrementalAction ensures it keep its underlying action's objects alive
41 /// as long as the \p IncrementalParser needs them.
42 ///
43 class IncrementalAction : public WrapperFrontendAction {
44 private:
45   bool IsTerminating = false;
46 
47 public:
48   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
49                     llvm::Error &Err)
50       : WrapperFrontendAction([&]() {
51           llvm::ErrorAsOutParameter EAO(&Err);
52           std::unique_ptr<FrontendAction> Act;
53           switch (CI.getFrontendOpts().ProgramAction) {
54           default:
55             Err = llvm::createStringError(
56                 std::errc::state_not_recoverable,
57                 "Driver initialization failed. "
58                 "Incremental mode for action %d is not supported",
59                 CI.getFrontendOpts().ProgramAction);
60             return Act;
61           case frontend::ASTDump:
62             LLVM_FALLTHROUGH;
63           case frontend::ASTPrint:
64             LLVM_FALLTHROUGH;
65           case frontend::ParseSyntaxOnly:
66             Act = CreateFrontendAction(CI);
67             break;
68           case frontend::EmitAssembly:
69             LLVM_FALLTHROUGH;
70           case frontend::EmitObj:
71             LLVM_FALLTHROUGH;
72           case frontend::EmitLLVMOnly:
73             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
74             break;
75           }
76           return Act;
77         }()) {}
78   FrontendAction *getWrapped() const { return WrappedAction.get(); }
79   TranslationUnitKind getTranslationUnitKind() override {
80     return TU_Incremental;
81   }
82   void ExecuteAction() override {
83     CompilerInstance &CI = getCompilerInstance();
84     assert(CI.hasPreprocessor() && "No PP!");
85 
86     // FIXME: Move the truncation aspect of this into Sema, we delayed this till
87     // here so the source manager would be initialized.
88     if (hasCodeCompletionSupport() &&
89         !CI.getFrontendOpts().CodeCompletionAt.FileName.empty())
90       CI.createCodeCompletionConsumer();
91 
92     // Use a code completion consumer?
93     CodeCompleteConsumer *CompletionConsumer = nullptr;
94     if (CI.hasCodeCompletionConsumer())
95       CompletionConsumer = &CI.getCodeCompletionConsumer();
96 
97     Preprocessor &PP = CI.getPreprocessor();
98     PP.enableIncrementalProcessing();
99     PP.EnterMainSourceFile();
100 
101     if (!CI.hasSema())
102       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
103   }
104 
105   // Do not terminate after processing the input. This allows us to keep various
106   // clang objects alive and to incrementally grow the current TU.
107   void EndSourceFile() override {
108     // The WrappedAction can be nullptr if we issued an error in the ctor.
109     if (IsTerminating && getWrapped())
110       WrapperFrontendAction::EndSourceFile();
111   }
112 
113   void FinalizeAction() {
114     assert(!IsTerminating && "Already finalized!");
115     IsTerminating = true;
116     EndSourceFile();
117   }
118 };
119 
120 IncrementalParser::IncrementalParser(std::unique_ptr<CompilerInstance> Instance,
121                                      llvm::LLVMContext &LLVMCtx,
122                                      llvm::Error &Err)
123     : CI(std::move(Instance)) {
124   llvm::ErrorAsOutParameter EAO(&Err);
125   Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
126   if (Err)
127     return;
128   CI->ExecuteAction(*Act);
129   Consumer = &CI->getASTConsumer();
130   P.reset(
131       new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
132   P->Initialize();
133 }
134 
135 IncrementalParser::~IncrementalParser() { Act->FinalizeAction(); }
136 
137 llvm::Expected<PartialTranslationUnit &>
138 IncrementalParser::ParseOrWrapTopLevelDecl() {
139   // Recover resources if we crash before exiting this method.
140   Sema &S = CI->getSema();
141   llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
142   Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
143   Sema::LocalEagerInstantiationScope LocalInstantiations(S);
144 
145   PTUs.emplace_back(PartialTranslationUnit());
146   PartialTranslationUnit &LastPTU = PTUs.back();
147   // Add a new PTU.
148   ASTContext &C = S.getASTContext();
149   C.addTranslationUnitDecl();
150   LastPTU.TUPart = C.getTranslationUnitDecl();
151 
152   // Skip previous eof due to last incremental input.
153   if (P->getCurToken().is(tok::eof)) {
154     P->ConsumeToken();
155     // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
156     // might want to do that around HandleEndOfTranslationUnit.
157     P->ExitScope();
158     S.CurContext = nullptr;
159     // Start a new PTU.
160     P->EnterScope(Scope::DeclScope);
161     S.ActOnTranslationUnitScope(P->getCurScope());
162   }
163 
164   Parser::DeclGroupPtrTy ADecl;
165   for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl); !AtEOF;
166        AtEOF = P->ParseTopLevelDecl(ADecl)) {
167     // If we got a null return and something *was* parsed, ignore it.  This
168     // is due to a top-level semicolon, an action override, or a parse error
169     // skipping something.
170     if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get()))
171       return llvm::make_error<llvm::StringError>("Parsing failed. "
172                                                  "The consumer rejected a decl",
173                                                  std::error_code());
174   }
175 
176   DiagnosticsEngine &Diags = getCI()->getDiagnostics();
177   if (Diags.hasErrorOccurred()) {
178     TranslationUnitDecl *MostRecentTU = C.getTranslationUnitDecl();
179     TranslationUnitDecl *PreviousTU = MostRecentTU->getPreviousDecl();
180     assert(PreviousTU && "Must have a TU from the ASTContext initialization!");
181     TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
182     assert(FirstTU);
183     FirstTU->RedeclLink.setLatest(PreviousTU);
184     C.TUDecl = PreviousTU;
185     S.TUScope->setEntity(PreviousTU);
186 
187     // Clean up the lookup table
188     if (StoredDeclsMap *Map = PreviousTU->getLookupPtr()) {
189       for (auto I = Map->begin(); I != Map->end(); ++I) {
190         StoredDeclsList &List = I->second;
191         DeclContextLookupResult R = List.getLookupResult();
192         for (NamedDecl *D : R)
193           if (D->getTranslationUnitDecl() == MostRecentTU)
194             List.remove(D);
195         if (List.isNull())
196           Map->erase(I);
197       }
198     }
199 
200     // FIXME: Do not reset the pragma handlers.
201     Diags.Reset();
202     return llvm::make_error<llvm::StringError>("Parsing failed.",
203                                                std::error_code());
204   }
205 
206   // Process any TopLevelDecls generated by #pragma weak.
207   for (Decl *D : S.WeakTopLevelDecls()) {
208     DeclGroupRef DGR(D);
209     Consumer->HandleTopLevelDecl(DGR);
210   }
211 
212   LocalInstantiations.perform();
213   GlobalInstantiations.perform();
214 
215   Consumer->HandleTranslationUnit(C);
216 
217   return LastPTU;
218 }
219 
220 static CodeGenerator *getCodeGen(FrontendAction *Act) {
221   IncrementalAction *IncrAct = static_cast<IncrementalAction *>(Act);
222   FrontendAction *WrappedAct = IncrAct->getWrapped();
223   if (!WrappedAct->hasIRSupport())
224     return nullptr;
225   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
226 }
227 
228 llvm::Expected<PartialTranslationUnit &>
229 IncrementalParser::Parse(llvm::StringRef input) {
230   Preprocessor &PP = CI->getPreprocessor();
231   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
232 
233   std::ostringstream SourceName;
234   SourceName << "input_line_" << InputCount++;
235 
236   // Create an uninitialized memory buffer, copy code in and append "\n"
237   size_t InputSize = input.size(); // don't include trailing 0
238   // MemBuffer size should *not* include terminating zero
239   std::unique_ptr<llvm::MemoryBuffer> MB(
240       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
241                                                         SourceName.str()));
242   char *MBStart = const_cast<char *>(MB->getBufferStart());
243   memcpy(MBStart, input.data(), InputSize);
244   MBStart[InputSize] = '\n';
245 
246   SourceManager &SM = CI->getSourceManager();
247 
248   // FIXME: Create SourceLocation, which will allow clang to order the overload
249   // candidates for example
250   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
251 
252   // Create FileID for the current buffer.
253   FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
254                                /*LoadedOffset=*/0, NewLoc);
255 
256   // NewLoc only used for diags.
257   if (PP.EnterSourceFile(FID, /*DirLookup=*/0, NewLoc))
258     return llvm::make_error<llvm::StringError>("Parsing failed. "
259                                                "Cannot enter source file.",
260                                                std::error_code());
261 
262   auto PTU = ParseOrWrapTopLevelDecl();
263   if (!PTU)
264     return PTU.takeError();
265 
266   if (PP.getLangOpts().DelayedTemplateParsing) {
267     // Microsoft-specific:
268     // Late parsed templates can leave unswallowed "macro"-like tokens.
269     // They will seriously confuse the Parser when entering the next
270     // source file. So lex until we are EOF.
271     Token Tok;
272     do {
273       PP.Lex(Tok);
274     } while (Tok.isNot(tok::eof));
275   }
276 
277   Token AssertTok;
278   PP.Lex(AssertTok);
279   assert(AssertTok.is(tok::eof) &&
280          "Lexer must be EOF when starting incremental parse!");
281 
282   if (CodeGenerator *CG = getCodeGen(Act.get())) {
283     std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
284     CG->StartModule("incr_module_" + std::to_string(PTUs.size()),
285                     M->getContext());
286 
287     PTU->TheModule = std::move(M);
288   }
289 
290   return PTU;
291 }
292 } // end namespace clang
293