1//===--------- IncrementalParser.cpp - Incremental Compilation -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class which performs incremental code compilation.
10//
11//===----------------------------------------------------------------------===//
12
13#include "IncrementalParser.h"
14
15#include "clang/AST/DeclContextInternals.h"
16#include "clang/CodeGen/BackendUtil.h"
17#include "clang/CodeGen/CodeGenAction.h"
18#include "clang/CodeGen/ModuleBuilder.h"
19#include "clang/Frontend/CompilerInstance.h"
20#include "clang/Frontend/FrontendAction.h"
21#include "clang/FrontendTool/Utils.h"
22#include "clang/Interpreter/Interpreter.h"
23#include "clang/Parse/Parser.h"
24#include "clang/Sema/Sema.h"
25#include "llvm/Option/ArgList.h"
26#include "llvm/Support/CrashRecoveryContext.h"
27#include "llvm/Support/Error.h"
28#include "llvm/Support/Timer.h"
29
30#include <sstream>
31
32namespace clang {
33
34class IncrementalASTConsumer final : public ASTConsumer {
35 Interpreter &Interp;
36 std::unique_ptr<ASTConsumer> Consumer;
37
38public:
39 IncrementalASTConsumer(Interpreter &InterpRef, std::unique_ptr<ASTConsumer> C)
40 : Interp(InterpRef), Consumer(std::move(C)) {}
41
42 bool HandleTopLevelDecl(DeclGroupRef DGR) override final {
43 if (DGR.isNull())
44 return true;
45 if (!Consumer)
46 return true;
47
48 for (Decl *D : DGR)
49 if (auto *TSD = llvm::dyn_cast<TopLevelStmtDecl>(Val: D);
50 TSD && TSD->isSemiMissing())
51 TSD->setStmt(Interp.SynthesizeExpr(E: cast<Expr>(Val: TSD->getStmt())));
52
53 return Consumer->HandleTopLevelDecl(D: DGR);
54 }
55 void HandleTranslationUnit(ASTContext &Ctx) override final {
56 Consumer->HandleTranslationUnit(Ctx);
57 }
58 void HandleInlineFunctionDefinition(FunctionDecl *D) override final {
59 Consumer->HandleInlineFunctionDefinition(D);
60 }
61 void HandleInterestingDecl(DeclGroupRef D) override final {
62 Consumer->HandleInterestingDecl(D);
63 }
64 void HandleTagDeclDefinition(TagDecl *D) override final {
65 Consumer->HandleTagDeclDefinition(D);
66 }
67 void HandleTagDeclRequiredDefinition(const TagDecl *D) override final {
68 Consumer->HandleTagDeclRequiredDefinition(D);
69 }
70 void HandleCXXImplicitFunctionInstantiation(FunctionDecl *D) override final {
71 Consumer->HandleCXXImplicitFunctionInstantiation(D);
72 }
73 void HandleTopLevelDeclInObjCContainer(DeclGroupRef D) override final {
74 Consumer->HandleTopLevelDeclInObjCContainer(D);
75 }
76 void HandleImplicitImportDecl(ImportDecl *D) override final {
77 Consumer->HandleImplicitImportDecl(D);
78 }
79 void CompleteTentativeDefinition(VarDecl *D) override final {
80 Consumer->CompleteTentativeDefinition(D);
81 }
82 void CompleteExternalDeclaration(VarDecl *D) override final {
83 Consumer->CompleteExternalDeclaration(D);
84 }
85 void AssignInheritanceModel(CXXRecordDecl *RD) override final {
86 Consumer->AssignInheritanceModel(RD);
87 }
88 void HandleCXXStaticMemberVarInstantiation(VarDecl *D) override final {
89 Consumer->HandleCXXStaticMemberVarInstantiation(D);
90 }
91 void HandleVTable(CXXRecordDecl *RD) override final {
92 Consumer->HandleVTable(RD);
93 }
94 ASTMutationListener *GetASTMutationListener() override final {
95 return Consumer->GetASTMutationListener();
96 }
97 ASTDeserializationListener *GetASTDeserializationListener() override final {
98 return Consumer->GetASTDeserializationListener();
99 }
100 void PrintStats() override final { Consumer->PrintStats(); }
101 bool shouldSkipFunctionBody(Decl *D) override final {
102 return Consumer->shouldSkipFunctionBody(D);
103 }
104 static bool classof(const clang::ASTConsumer *) { return true; }
105};
106
107/// A custom action enabling the incremental processing functionality.
108///
109/// The usual \p FrontendAction expects one call to ExecuteAction and once it
110/// sees a call to \p EndSourceFile it deletes some of the important objects
111/// such as \p Preprocessor and \p Sema assuming no further input will come.
112///
113/// \p IncrementalAction ensures it keep its underlying action's objects alive
114/// as long as the \p IncrementalParser needs them.
115///
116class IncrementalAction : public WrapperFrontendAction {
117private:
118 bool IsTerminating = false;
119
120public:
121 IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
122 llvm::Error &Err)
123 : WrapperFrontendAction([&]() {
124 llvm::ErrorAsOutParameter EAO(&Err);
125 std::unique_ptr<FrontendAction> Act;
126 switch (CI.getFrontendOpts().ProgramAction) {
127 default:
128 Err = llvm::createStringError(
129 EC: std::errc::state_not_recoverable,
130 Fmt: "Driver initialization failed. "
131 "Incremental mode for action %d is not supported",
132 Vals: CI.getFrontendOpts().ProgramAction);
133 return Act;
134 case frontend::ASTDump:
135 [[fallthrough]];
136 case frontend::ASTPrint:
137 [[fallthrough]];
138 case frontend::ParseSyntaxOnly:
139 Act = CreateFrontendAction(CI);
140 break;
141 case frontend::PluginAction:
142 [[fallthrough]];
143 case frontend::EmitAssembly:
144 [[fallthrough]];
145 case frontend::EmitBC:
146 [[fallthrough]];
147 case frontend::EmitObj:
148 [[fallthrough]];
149 case frontend::PrintPreprocessedInput:
150 [[fallthrough]];
151 case frontend::EmitLLVMOnly:
152 Act.reset(p: new EmitLLVMOnlyAction(&LLVMCtx));
153 break;
154 }
155 return Act;
156 }()) {}
157 FrontendAction *getWrapped() const { return WrappedAction.get(); }
158 TranslationUnitKind getTranslationUnitKind() override {
159 return TU_Incremental;
160 }
161
162 void ExecuteAction() override {
163 CompilerInstance &CI = getCompilerInstance();
164 assert(CI.hasPreprocessor() && "No PP!");
165
166 // Use a code completion consumer?
167 CodeCompleteConsumer *CompletionConsumer = nullptr;
168 if (CI.hasCodeCompletionConsumer())
169 CompletionConsumer = &CI.getCodeCompletionConsumer();
170
171 Preprocessor &PP = CI.getPreprocessor();
172 PP.EnterMainSourceFile();
173
174 if (!CI.hasSema())
175 CI.createSema(TUKind: getTranslationUnitKind(), CompletionConsumer);
176 }
177
178 // Do not terminate after processing the input. This allows us to keep various
179 // clang objects alive and to incrementally grow the current TU.
180 void EndSourceFile() override {
181 // The WrappedAction can be nullptr if we issued an error in the ctor.
182 if (IsTerminating && getWrapped())
183 WrapperFrontendAction::EndSourceFile();
184 }
185
186 void FinalizeAction() {
187 assert(!IsTerminating && "Already finalized!");
188 IsTerminating = true;
189 EndSourceFile();
190 }
191};
192
193CodeGenerator *IncrementalParser::getCodeGen() const {
194 FrontendAction *WrappedAct = Act->getWrapped();
195 if (!WrappedAct->hasIRSupport())
196 return nullptr;
197 return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
198}
199
200IncrementalParser::IncrementalParser() {}
201
202IncrementalParser::IncrementalParser(Interpreter &Interp,
203 std::unique_ptr<CompilerInstance> Instance,
204 llvm::LLVMContext &LLVMCtx,
205 llvm::Error &Err)
206 : CI(std::move(Instance)) {
207 llvm::ErrorAsOutParameter EAO(&Err);
208 Act = std::make_unique<IncrementalAction>(args&: *CI, args&: LLVMCtx, args&: Err);
209 if (Err)
210 return;
211 CI->ExecuteAction(Act&: *Act);
212 std::unique_ptr<ASTConsumer> IncrConsumer =
213 std::make_unique<IncrementalASTConsumer>(args&: Interp, args: CI->takeASTConsumer());
214 CI->setASTConsumer(std::move(IncrConsumer));
215 Consumer = &CI->getASTConsumer();
216 P.reset(
217 p: new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false));
218 P->Initialize();
219
220 // An initial PTU is needed as CUDA includes some headers automatically
221 auto PTU = ParseOrWrapTopLevelDecl();
222 if (auto E = PTU.takeError()) {
223 consumeError(Err: std::move(E)); // FIXME
224 return; // PTU.takeError();
225 }
226
227 if (CodeGenerator *CG = getCodeGen()) {
228 std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
229 CG->StartModule(ModuleName: "incr_module_" + std::to_string(val: PTUs.size()),
230 C&: M->getContext());
231 PTU->TheModule = std::move(M);
232 assert(PTU->TheModule && "Failed to create initial PTU");
233 }
234}
235
236IncrementalParser::~IncrementalParser() {
237 P.reset();
238 Act->FinalizeAction();
239}
240
241llvm::Expected<PartialTranslationUnit &>
242IncrementalParser::ParseOrWrapTopLevelDecl() {
243 // Recover resources if we crash before exiting this method.
244 Sema &S = CI->getSema();
245 llvm::CrashRecoveryContextCleanupRegistrar<Sema> CleanupSema(&S);
246 Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true);
247 Sema::LocalEagerInstantiationScope LocalInstantiations(S);
248
249 PTUs.emplace_back(args: PartialTranslationUnit());
250 PartialTranslationUnit &LastPTU = PTUs.back();
251 // Add a new PTU.
252 ASTContext &C = S.getASTContext();
253 C.addTranslationUnitDecl();
254 LastPTU.TUPart = C.getTranslationUnitDecl();
255
256 // Skip previous eof due to last incremental input.
257 if (P->getCurToken().is(K: tok::annot_repl_input_end)) {
258 P->ConsumeAnyToken();
259 // FIXME: Clang does not call ExitScope on finalizing the regular TU, we
260 // might want to do that around HandleEndOfTranslationUnit.
261 P->ExitScope();
262 S.CurContext = nullptr;
263 // Start a new PTU.
264 P->EnterScope(ScopeFlags: Scope::DeclScope);
265 S.ActOnTranslationUnitScope(S: P->getCurScope());
266 }
267
268 Parser::DeclGroupPtrTy ADecl;
269 Sema::ModuleImportState ImportState;
270 for (bool AtEOF = P->ParseFirstTopLevelDecl(Result&: ADecl, ImportState); !AtEOF;
271 AtEOF = P->ParseTopLevelDecl(Result&: ADecl, ImportState)) {
272 if (ADecl && !Consumer->HandleTopLevelDecl(D: ADecl.get()))
273 return llvm::make_error<llvm::StringError>(Args: "Parsing failed. "
274 "The consumer rejected a decl",
275 Args: std::error_code());
276 }
277
278 DiagnosticsEngine &Diags = getCI()->getDiagnostics();
279 if (Diags.hasErrorOccurred()) {
280 PartialTranslationUnit MostRecentPTU = {.TUPart: C.getTranslationUnitDecl(),
281 .TheModule: nullptr};
282 CleanUpPTU(PTU&: MostRecentPTU);
283
284 Diags.Reset(/*soft=*/true);
285 Diags.getClient()->clear();
286 return llvm::make_error<llvm::StringError>(Args: "Parsing failed.",
287 Args: std::error_code());
288 }
289
290 // Process any TopLevelDecls generated by #pragma weak.
291 for (Decl *D : S.WeakTopLevelDecls()) {
292 DeclGroupRef DGR(D);
293 Consumer->HandleTopLevelDecl(D: DGR);
294 }
295
296 LocalInstantiations.perform();
297 GlobalInstantiations.perform();
298
299 Consumer->HandleTranslationUnit(Ctx&: C);
300
301 return LastPTU;
302}
303
304llvm::Expected<PartialTranslationUnit &>
305IncrementalParser::Parse(llvm::StringRef input) {
306 Preprocessor &PP = CI->getPreprocessor();
307 assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
308
309 std::ostringstream SourceName;
310 SourceName << "input_line_" << InputCount++;
311
312 // Create an uninitialized memory buffer, copy code in and append "\n"
313 size_t InputSize = input.size(); // don't include trailing 0
314 // MemBuffer size should *not* include terminating zero
315 std::unique_ptr<llvm::MemoryBuffer> MB(
316 llvm::WritableMemoryBuffer::getNewUninitMemBuffer(Size: InputSize + 1,
317 BufferName: SourceName.str()));
318 char *MBStart = const_cast<char *>(MB->getBufferStart());
319 memcpy(dest: MBStart, src: input.data(), n: InputSize);
320 MBStart[InputSize] = '\n';
321
322 SourceManager &SM = CI->getSourceManager();
323
324 // FIXME: Create SourceLocation, which will allow clang to order the overload
325 // candidates for example
326 SourceLocation NewLoc = SM.getLocForStartOfFile(FID: SM.getMainFileID());
327
328 // Create FileID for the current buffer.
329 FileID FID = SM.createFileID(Buffer: std::move(MB), FileCharacter: SrcMgr::C_User, /*LoadedID=*/0,
330 /*LoadedOffset=*/0, IncludeLoc: NewLoc);
331
332 // NewLoc only used for diags.
333 if (PP.EnterSourceFile(FID, /*DirLookup=*/Dir: nullptr, Loc: NewLoc))
334 return llvm::make_error<llvm::StringError>(Args: "Parsing failed. "
335 "Cannot enter source file.",
336 Args: std::error_code());
337
338 auto PTU = ParseOrWrapTopLevelDecl();
339 if (!PTU)
340 return PTU.takeError();
341
342 if (PP.getLangOpts().DelayedTemplateParsing) {
343 // Microsoft-specific:
344 // Late parsed templates can leave unswallowed "macro"-like tokens.
345 // They will seriously confuse the Parser when entering the next
346 // source file. So lex until we are EOF.
347 Token Tok;
348 do {
349 PP.Lex(Result&: Tok);
350 } while (Tok.isNot(K: tok::annot_repl_input_end));
351 } else {
352 Token AssertTok;
353 PP.Lex(Result&: AssertTok);
354 assert(AssertTok.is(tok::annot_repl_input_end) &&
355 "Lexer must be EOF when starting incremental parse!");
356 }
357
358 if (std::unique_ptr<llvm::Module> M = GenModule())
359 PTU->TheModule = std::move(M);
360
361 return PTU;
362}
363
364std::unique_ptr<llvm::Module> IncrementalParser::GenModule() {
365 static unsigned ID = 0;
366 if (CodeGenerator *CG = getCodeGen()) {
367 std::unique_ptr<llvm::Module> M(CG->ReleaseModule());
368 CG->StartModule(ModuleName: "incr_module_" + std::to_string(val: ID++), C&: M->getContext());
369 return M;
370 }
371 return nullptr;
372}
373
374void IncrementalParser::CleanUpPTU(PartialTranslationUnit &PTU) {
375 TranslationUnitDecl *MostRecentTU = PTU.TUPart;
376 TranslationUnitDecl *FirstTU = MostRecentTU->getFirstDecl();
377 if (StoredDeclsMap *Map = FirstTU->getPrimaryContext()->getLookupPtr()) {
378 for (auto I = Map->begin(); I != Map->end(); ++I) {
379 StoredDeclsList &List = I->second;
380 DeclContextLookupResult R = List.getLookupResult();
381 for (NamedDecl *D : R) {
382 if (D->getTranslationUnitDecl() == MostRecentTU) {
383 List.remove(D);
384 }
385 }
386 if (List.isNull())
387 Map->erase(I);
388 }
389 }
390}
391
392llvm::StringRef IncrementalParser::GetMangledName(GlobalDecl GD) const {
393 CodeGenerator *CG = getCodeGen();
394 assert(CG);
395 return CG->GetMangledName(GD);
396}
397} // end namespace clang
398

source code of clang/lib/Interpreter/IncrementalParser.cpp