1//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Preprocessor interface.
10//
11//===----------------------------------------------------------------------===//
12//
13// Options to support:
14// -H - Print the name of each header file used.
15// -d[DNI] - Dump various things.
16// -fworking-directory - #line's with preprocessor's working dir.
17// -fpreprocessed
18// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19// -W*
20// -w
21//
22// Messages to emit:
23// "Multiple include guards may be useful for:\n"
24//
25//===----------------------------------------------------------------------===//
26
27#include "clang/Lex/Preprocessor.h"
28#include "clang/Basic/Builtins.h"
29#include "clang/Basic/FileManager.h"
30#include "clang/Basic/IdentifierTable.h"
31#include "clang/Basic/LLVM.h"
32#include "clang/Basic/LangOptions.h"
33#include "clang/Basic/Module.h"
34#include "clang/Basic/SourceLocation.h"
35#include "clang/Basic/SourceManager.h"
36#include "clang/Basic/TargetInfo.h"
37#include "clang/Lex/CodeCompletionHandler.h"
38#include "clang/Lex/ExternalPreprocessorSource.h"
39#include "clang/Lex/HeaderSearch.h"
40#include "clang/Lex/LexDiagnostic.h"
41#include "clang/Lex/Lexer.h"
42#include "clang/Lex/LiteralSupport.h"
43#include "clang/Lex/MacroArgs.h"
44#include "clang/Lex/MacroInfo.h"
45#include "clang/Lex/ModuleLoader.h"
46#include "clang/Lex/NoTrivialPPDirectiveTracer.h"
47#include "clang/Lex/Pragma.h"
48#include "clang/Lex/PreprocessingRecord.h"
49#include "clang/Lex/PreprocessorLexer.h"
50#include "clang/Lex/PreprocessorOptions.h"
51#include "clang/Lex/ScratchBuffer.h"
52#include "clang/Lex/Token.h"
53#include "clang/Lex/TokenLexer.h"
54#include "llvm/ADT/APInt.h"
55#include "llvm/ADT/ArrayRef.h"
56#include "llvm/ADT/DenseMap.h"
57#include "llvm/ADT/STLExtras.h"
58#include "llvm/ADT/SmallVector.h"
59#include "llvm/ADT/StringRef.h"
60#include "llvm/Support/Capacity.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/MemoryBuffer.h"
63#include "llvm/Support/raw_ostream.h"
64#include <algorithm>
65#include <cassert>
66#include <memory>
67#include <optional>
68#include <string>
69#include <utility>
70#include <vector>
71
72using namespace clang;
73
74/// Minimum distance between two check points, in tokens.
75static constexpr unsigned CheckPointStepSize = 1024;
76
77LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
78
79ExternalPreprocessorSource::~ExternalPreprocessorSource() = default;
80
81Preprocessor::Preprocessor(const PreprocessorOptions &PPOpts,
82 DiagnosticsEngine &diags, const LangOptions &opts,
83 SourceManager &SM, HeaderSearch &Headers,
84 ModuleLoader &TheModuleLoader,
85 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
86 TranslationUnitKind TUKind)
87 : PPOpts(PPOpts), Diags(&diags), LangOpts(opts),
88 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
89 ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
90 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
91 // As the language options may have not been loaded yet (when
92 // deserializing an ASTUnit), adding keywords to the identifier table is
93 // deferred to Preprocessor::Initialize().
94 Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
95 TUKind(TUKind), SkipMainFilePreamble(0, true),
96 CurSubmoduleState(&NullSubmoduleState) {
97 OwnsHeaderSearch = OwnsHeaders;
98
99 // Default to discarding comments.
100 KeepComments = false;
101 KeepMacroComments = false;
102 SuppressIncludeNotFoundError = false;
103
104 // Macro expansion is enabled.
105 DisableMacroExpansion = false;
106 MacroExpansionInDirectivesOverride = false;
107 InMacroArgs = false;
108 ArgMacro = nullptr;
109 InMacroArgPreExpansion = false;
110 NumCachedTokenLexers = 0;
111 PragmasEnabled = true;
112 ParsingIfOrElifDirective = false;
113 PreprocessedOutput = false;
114
115 // We haven't read anything from the external source.
116 ReadMacrosFromExternalSource = false;
117
118 BuiltinInfo = std::make_unique<Builtin::Context>();
119
120 // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
121 // a macro. They get unpoisoned where it is allowed.
122 (Ident__VA_ARGS__ = getIdentifierInfo(Name: "__VA_ARGS__"))->setIsPoisoned();
123 SetPoisonReason(II: Ident__VA_ARGS__,DiagID: diag::ext_pp_bad_vaargs_use);
124 (Ident__VA_OPT__ = getIdentifierInfo(Name: "__VA_OPT__"))->setIsPoisoned();
125 SetPoisonReason(II: Ident__VA_OPT__,DiagID: diag::ext_pp_bad_vaopt_use);
126
127 // Initialize the pragma handlers.
128 RegisterBuiltinPragmas();
129
130 // Initialize builtin macros like __LINE__ and friends.
131 RegisterBuiltinMacros();
132
133 if(LangOpts.Borland) {
134 Ident__exception_info = getIdentifierInfo(Name: "_exception_info");
135 Ident___exception_info = getIdentifierInfo(Name: "__exception_info");
136 Ident_GetExceptionInfo = getIdentifierInfo(Name: "GetExceptionInformation");
137 Ident__exception_code = getIdentifierInfo(Name: "_exception_code");
138 Ident___exception_code = getIdentifierInfo(Name: "__exception_code");
139 Ident_GetExceptionCode = getIdentifierInfo(Name: "GetExceptionCode");
140 Ident__abnormal_termination = getIdentifierInfo(Name: "_abnormal_termination");
141 Ident___abnormal_termination = getIdentifierInfo(Name: "__abnormal_termination");
142 Ident_AbnormalTermination = getIdentifierInfo(Name: "AbnormalTermination");
143 } else {
144 Ident__exception_info = Ident__exception_code = nullptr;
145 Ident__abnormal_termination = Ident___exception_info = nullptr;
146 Ident___exception_code = Ident___abnormal_termination = nullptr;
147 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
148 Ident_AbnormalTermination = nullptr;
149 }
150
151 // Default incremental processing to -fincremental-extensions, clients can
152 // override with `enableIncrementalProcessing` if desired.
153 IncrementalProcessing = LangOpts.IncrementalExtensions;
154
155 // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
156 if (usingPCHWithPragmaHdrStop())
157 SkippingUntilPragmaHdrStop = true;
158
159 // If using a PCH with a through header, start skipping tokens.
160 if (!this->PPOpts.PCHThroughHeader.empty() &&
161 !this->PPOpts.ImplicitPCHInclude.empty())
162 SkippingUntilPCHThroughHeader = true;
163
164 if (this->PPOpts.GeneratePreamble)
165 PreambleConditionalStack.startRecording();
166
167 MaxTokens = LangOpts.MaxTokens;
168}
169
170Preprocessor::~Preprocessor() {
171 assert(!isBacktrackEnabled() && "EnableBacktrack/Backtrack imbalance!");
172
173 IncludeMacroStack.clear();
174
175 // Free any cached macro expanders.
176 // This populates MacroArgCache, so all TokenLexers need to be destroyed
177 // before the code below that frees up the MacroArgCache list.
178 std::fill(first: TokenLexerCache, last: TokenLexerCache + NumCachedTokenLexers, value: nullptr);
179 CurTokenLexer.reset();
180
181 // Free any cached MacroArgs.
182 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
183 ArgList = ArgList->deallocate();
184
185 // Delete the header search info, if we own it.
186 if (OwnsHeaderSearch)
187 delete &HeaderInfo;
188}
189
190void Preprocessor::Initialize(const TargetInfo &Target,
191 const TargetInfo *AuxTarget) {
192 assert((!this->Target || this->Target == &Target) &&
193 "Invalid override of target information");
194 this->Target = &Target;
195
196 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
197 "Invalid override of aux target information.");
198 this->AuxTarget = AuxTarget;
199
200 // Initialize information about built-ins.
201 BuiltinInfo->InitializeTarget(Target, AuxTarget);
202 HeaderInfo.setTarget(Target);
203
204 // Populate the identifier table with info about keywords for the current language.
205 Identifiers.AddKeywords(LangOpts);
206
207 // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
208 setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
209
210 if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine)
211 // Use setting from TargetInfo.
212 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: Target.getFPEvalMethod());
213 else
214 // Set initial value of __FLT_EVAL_METHOD__ from the command line.
215 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: getLangOpts().getFPEvalMethod());
216}
217
218void Preprocessor::InitializeForModelFile() {
219 NumEnteredSourceFiles = 0;
220
221 // Reset pragmas
222 PragmaHandlersBackup = std::move(PragmaHandlers);
223 PragmaHandlers = std::make_unique<PragmaNamespace>(args: StringRef());
224 RegisterBuiltinPragmas();
225
226 // Reset PredefinesFileID
227 PredefinesFileID = FileID();
228}
229
230void Preprocessor::FinalizeForModelFile() {
231 NumEnteredSourceFiles = 1;
232
233 PragmaHandlers = std::move(PragmaHandlersBackup);
234}
235
236void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
237 llvm::errs() << tok::getTokenName(Kind: Tok.getKind());
238
239 if (!Tok.isAnnotation())
240 llvm::errs() << " '" << getSpelling(Tok) << "'";
241
242 if (!DumpFlags) return;
243
244 llvm::errs() << "\t";
245 if (Tok.isAtStartOfLine())
246 llvm::errs() << " [StartOfLine]";
247 if (Tok.hasLeadingSpace())
248 llvm::errs() << " [LeadingSpace]";
249 if (Tok.isExpandDisabled())
250 llvm::errs() << " [ExpandDisabled]";
251 if (Tok.needsCleaning()) {
252 const char *Start = SourceMgr.getCharacterData(SL: Tok.getLocation());
253 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
254 << "']";
255 }
256
257 llvm::errs() << "\tLoc=<";
258 DumpLocation(Loc: Tok.getLocation());
259 llvm::errs() << ">";
260}
261
262void Preprocessor::DumpLocation(SourceLocation Loc) const {
263 Loc.print(OS&: llvm::errs(), SM: SourceMgr);
264}
265
266void Preprocessor::DumpMacro(const MacroInfo &MI) const {
267 llvm::errs() << "MACRO: ";
268 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
269 DumpToken(Tok: MI.getReplacementToken(Tok: i));
270 llvm::errs() << " ";
271 }
272 llvm::errs() << "\n";
273}
274
275void Preprocessor::PrintStats() {
276 llvm::errs() << "\n*** Preprocessor Stats:\n";
277 llvm::errs() << NumDirectives << " directives found:\n";
278 llvm::errs() << " " << NumDefined << " #define.\n";
279 llvm::errs() << " " << NumUndefined << " #undef.\n";
280 llvm::errs() << " #include/#include_next/#import:\n";
281 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
282 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
283 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
284 llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
285 llvm::errs() << " " << NumEndif << " #endif.\n";
286 llvm::errs() << " " << NumPragma << " #pragma.\n";
287 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
288
289 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
290 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
291 << NumFastMacroExpanded << " on the fast path.\n";
292 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
293 << " token paste (##) operations performed, "
294 << NumFastTokenPaste << " on the fast path.\n";
295
296 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
297
298 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
299 llvm::errs() << "\n Macro Expanded Tokens: "
300 << llvm::capacity_in_bytes(X: MacroExpandedTokens);
301 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
302 // FIXME: List information for all submodules.
303 llvm::errs() << "\n Macros: "
304 << llvm::capacity_in_bytes(X: CurSubmoduleState->Macros);
305 llvm::errs() << "\n #pragma push_macro Info: "
306 << llvm::capacity_in_bytes(X: PragmaPushMacroInfo);
307 llvm::errs() << "\n Poison Reasons: "
308 << llvm::capacity_in_bytes(X: PoisonReasons);
309 llvm::errs() << "\n Comment Handlers: "
310 << llvm::capacity_in_bytes(x: CommentHandlers) << "\n";
311}
312
313Preprocessor::macro_iterator
314Preprocessor::macro_begin(bool IncludeExternalMacros) const {
315 if (IncludeExternalMacros && ExternalSource &&
316 !ReadMacrosFromExternalSource) {
317 ReadMacrosFromExternalSource = true;
318 ExternalSource->ReadDefinedMacros();
319 }
320
321 // Make sure we cover all macros in visible modules.
322 for (const ModuleMacro &Macro : ModuleMacros)
323 CurSubmoduleState->Macros.try_emplace(Key: Macro.II);
324
325 return CurSubmoduleState->Macros.begin();
326}
327
328size_t Preprocessor::getTotalMemory() const {
329 return BP.getTotalMemory()
330 + llvm::capacity_in_bytes(X: MacroExpandedTokens)
331 + Predefines.capacity() /* Predefines buffer. */
332 // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
333 // and ModuleMacros.
334 + llvm::capacity_in_bytes(X: CurSubmoduleState->Macros)
335 + llvm::capacity_in_bytes(X: PragmaPushMacroInfo)
336 + llvm::capacity_in_bytes(X: PoisonReasons)
337 + llvm::capacity_in_bytes(x: CommentHandlers);
338}
339
340Preprocessor::macro_iterator
341Preprocessor::macro_end(bool IncludeExternalMacros) const {
342 if (IncludeExternalMacros && ExternalSource &&
343 !ReadMacrosFromExternalSource) {
344 ReadMacrosFromExternalSource = true;
345 ExternalSource->ReadDefinedMacros();
346 }
347
348 return CurSubmoduleState->Macros.end();
349}
350
351/// Compares macro tokens with a specified token value sequence.
352static bool MacroDefinitionEquals(const MacroInfo *MI,
353 ArrayRef<TokenValue> Tokens) {
354 return Tokens.size() == MI->getNumTokens() &&
355 std::equal(first1: Tokens.begin(), last1: Tokens.end(), first2: MI->tokens_begin());
356}
357
358StringRef Preprocessor::getLastMacroWithSpelling(
359 SourceLocation Loc,
360 ArrayRef<TokenValue> Tokens) const {
361 SourceLocation BestLocation;
362 StringRef BestSpelling;
363 for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
364 I != E; ++I) {
365 const MacroDirective::DefInfo
366 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
367 if (!Def || !Def.getMacroInfo())
368 continue;
369 if (!Def.getMacroInfo()->isObjectLike())
370 continue;
371 if (!MacroDefinitionEquals(MI: Def.getMacroInfo(), Tokens))
372 continue;
373 SourceLocation Location = Def.getLocation();
374 // Choose the macro defined latest.
375 if (BestLocation.isInvalid() ||
376 (Location.isValid() &&
377 SourceMgr.isBeforeInTranslationUnit(LHS: BestLocation, RHS: Location))) {
378 BestLocation = Location;
379 BestSpelling = I->first->getName();
380 }
381 }
382 return BestSpelling;
383}
384
385void Preprocessor::recomputeCurLexerKind() {
386 if (CurLexer)
387 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
388 ? CLK_DependencyDirectivesLexer
389 : CLK_Lexer;
390 else if (CurTokenLexer)
391 CurLexerCallback = CLK_TokenLexer;
392 else
393 CurLexerCallback = CLK_CachingLexer;
394}
395
396bool Preprocessor::SetCodeCompletionPoint(FileEntryRef File,
397 unsigned CompleteLine,
398 unsigned CompleteColumn) {
399 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
400 assert(!CodeCompletionFile && "Already set");
401
402 // Load the actual file's contents.
403 std::optional<llvm::MemoryBufferRef> Buffer =
404 SourceMgr.getMemoryBufferForFileOrNone(File);
405 if (!Buffer)
406 return true;
407
408 // Find the byte position of the truncation point.
409 const char *Position = Buffer->getBufferStart();
410 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
411 for (; *Position; ++Position) {
412 if (*Position != '\r' && *Position != '\n')
413 continue;
414
415 // Eat \r\n or \n\r as a single line.
416 if ((Position[1] == '\r' || Position[1] == '\n') &&
417 Position[0] != Position[1])
418 ++Position;
419 ++Position;
420 break;
421 }
422 }
423
424 Position += CompleteColumn - 1;
425
426 // If pointing inside the preamble, adjust the position at the beginning of
427 // the file after the preamble.
428 if (SkipMainFilePreamble.first &&
429 SourceMgr.getFileEntryForID(FID: SourceMgr.getMainFileID()) == File) {
430 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
431 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
432 }
433
434 if (Position > Buffer->getBufferEnd())
435 Position = Buffer->getBufferEnd();
436
437 CodeCompletionFile = File;
438 CodeCompletionOffset = Position - Buffer->getBufferStart();
439
440 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
441 Size: Buffer->getBufferSize() + 1, BufferName: Buffer->getBufferIdentifier());
442 char *NewBuf = NewBuffer->getBufferStart();
443 char *NewPos = std::copy(first: Buffer->getBufferStart(), last: Position, result: NewBuf);
444 *NewPos = '\0';
445 std::copy(first: Position, last: Buffer->getBufferEnd(), result: NewPos+1);
446 SourceMgr.overrideFileContents(SourceFile: File, Buffer: std::move(NewBuffer));
447
448 return false;
449}
450
451void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
452 bool IsAngled) {
453 setCodeCompletionReached();
454 if (CodeComplete)
455 CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
456}
457
458void Preprocessor::CodeCompleteNaturalLanguage() {
459 setCodeCompletionReached();
460 if (CodeComplete)
461 CodeComplete->CodeCompleteNaturalLanguage();
462}
463
464/// getSpelling - This method is used to get the spelling of a token into a
465/// SmallVector. Note that the returned StringRef may not point to the
466/// supplied buffer if a copy can be avoided.
467StringRef Preprocessor::getSpelling(const Token &Tok,
468 SmallVectorImpl<char> &Buffer,
469 bool *Invalid) const {
470 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
471 if (Tok.isNot(K: tok::raw_identifier) && !Tok.hasUCN()) {
472 // Try the fast path.
473 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
474 return II->getName();
475 }
476
477 // Resize the buffer if we need to copy into it.
478 if (Tok.needsCleaning())
479 Buffer.resize(N: Tok.getLength());
480
481 const char *Ptr = Buffer.data();
482 unsigned Len = getSpelling(Tok, Buffer&: Ptr, Invalid);
483 return StringRef(Ptr, Len);
484}
485
486/// CreateString - Plop the specified string into a scratch buffer and return a
487/// location for it. If specified, the source location provides a source
488/// location for the token.
489void Preprocessor::CreateString(StringRef Str, Token &Tok,
490 SourceLocation ExpansionLocStart,
491 SourceLocation ExpansionLocEnd) {
492 Tok.setLength(Str.size());
493
494 const char *DestPtr;
495 SourceLocation Loc = ScratchBuf->getToken(Buf: Str.data(), Len: Str.size(), DestPtr);
496
497 if (ExpansionLocStart.isValid())
498 Loc = SourceMgr.createExpansionLoc(SpellingLoc: Loc, ExpansionLocStart,
499 ExpansionLocEnd, Length: Str.size());
500 Tok.setLocation(Loc);
501
502 // If this is a raw identifier or a literal token, set the pointer data.
503 if (Tok.is(K: tok::raw_identifier))
504 Tok.setRawIdentifierData(DestPtr);
505 else if (Tok.isLiteral())
506 Tok.setLiteralData(DestPtr);
507}
508
509SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
510 auto &SM = getSourceManager();
511 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
512 FileIDAndOffset LocInfo = SM.getDecomposedLoc(Loc: SpellingLoc);
513 bool Invalid = false;
514 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
515 if (Invalid)
516 return SourceLocation();
517
518 // FIXME: We could consider re-using spelling for tokens we see repeatedly.
519 const char *DestPtr;
520 SourceLocation Spelling =
521 ScratchBuf->getToken(Buf: Buffer.data() + LocInfo.second, Len: Length, DestPtr);
522 return SM.createTokenSplitLoc(SpellingLoc: Spelling, TokenStart: Loc, TokenEnd: Loc.getLocWithOffset(Offset: Length));
523}
524
525Module *Preprocessor::getCurrentModule() {
526 if (!getLangOpts().isCompilingModule())
527 return nullptr;
528
529 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().CurrentModule);
530}
531
532Module *Preprocessor::getCurrentModuleImplementation() {
533 if (!getLangOpts().isCompilingModuleImplementation())
534 return nullptr;
535
536 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().ModuleName);
537}
538
539//===----------------------------------------------------------------------===//
540// Preprocessor Initialization Methods
541//===----------------------------------------------------------------------===//
542
543/// EnterMainSourceFile - Enter the specified FileID as the main source file,
544/// which implicitly adds the builtin defines etc.
545void Preprocessor::EnterMainSourceFile() {
546 // We do not allow the preprocessor to reenter the main file. Doing so will
547 // cause FileID's to accumulate information from both runs (e.g. #line
548 // information) and predefined macros aren't guaranteed to be set properly.
549 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
550 FileID MainFileID = SourceMgr.getMainFileID();
551
552 // If MainFileID is loaded it means we loaded an AST file, no need to enter
553 // a main file.
554 if (!SourceMgr.isLoadedFileID(FID: MainFileID)) {
555 // Enter the main file source buffer.
556 EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
557
558 // If we've been asked to skip bytes in the main file (e.g., as part of a
559 // precompiled preamble), do so now.
560 if (SkipMainFilePreamble.first > 0)
561 CurLexer->SetByteOffset(Offset: SkipMainFilePreamble.first,
562 StartOfLine: SkipMainFilePreamble.second);
563
564 // Tell the header info that the main file was entered. If the file is later
565 // #imported, it won't be re-entered.
566 if (OptionalFileEntryRef FE = SourceMgr.getFileEntryRefForID(FID: MainFileID))
567 markIncluded(File: *FE);
568
569 // Record the first PP token in the main file. This is used to generate
570 // better diagnostics for C++ modules.
571 //
572 // // This is a comment.
573 // #define FOO int // note: add 'module;' to the start of the file
574 // ^ FirstPPToken // to introduce a global module fragment.
575 //
576 // export module M; // error: module declaration must occur
577 // // at the start of the translation unit.
578 if (getLangOpts().CPlusPlusModules) {
579 auto Tracer = std::make_unique<NoTrivialPPDirectiveTracer>(args&: *this);
580 DirTracer = Tracer.get();
581 addPPCallbacks(C: std::move(Tracer));
582 std::optional<Token> FirstPPTok = CurLexer->peekNextPPToken();
583 if (FirstPPTok)
584 FirstPPTokenLoc = FirstPPTok->getLocation();
585 }
586 }
587
588 // Preprocess Predefines to populate the initial preprocessor state.
589 std::unique_ptr<llvm::MemoryBuffer> SB =
590 llvm::MemoryBuffer::getMemBufferCopy(InputData: Predefines, BufferName: "<built-in>");
591 assert(SB && "Cannot create predefined source buffer");
592 FileID FID = SourceMgr.createFileID(Buffer: std::move(SB));
593 assert(FID.isValid() && "Could not create FileID for predefines?");
594 setPredefinesFileID(FID);
595
596 // Start parsing the predefines.
597 EnterSourceFile(FID, Dir: nullptr, Loc: SourceLocation());
598
599 if (!PPOpts.PCHThroughHeader.empty()) {
600 // Lookup and save the FileID for the through header. If it isn't found
601 // in the search path, it's a fatal error.
602 OptionalFileEntryRef File = LookupFile(
603 FilenameLoc: SourceLocation(), Filename: PPOpts.PCHThroughHeader,
604 /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
605 /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
606 /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
607 /*IsFrameworkFound=*/nullptr);
608 if (!File) {
609 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_through_header_not_found)
610 << PPOpts.PCHThroughHeader;
611 return;
612 }
613 setPCHThroughHeaderFileID(
614 SourceMgr.createFileID(SourceFile: *File, IncludePos: SourceLocation(), FileCharacter: SrcMgr::C_User));
615 }
616
617 // Skip tokens from the Predefines and if needed the main file.
618 if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
619 (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
620 SkipTokensWhileUsingPCH();
621}
622
623void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
624 assert(PCHThroughHeaderFileID.isInvalid() &&
625 "PCHThroughHeaderFileID already set!");
626 PCHThroughHeaderFileID = FID;
627}
628
629bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) {
630 assert(PCHThroughHeaderFileID.isValid() &&
631 "Invalid PCH through header FileID");
632 return FE == SourceMgr.getFileEntryForID(FID: PCHThroughHeaderFileID);
633}
634
635bool Preprocessor::creatingPCHWithThroughHeader() {
636 return TUKind == TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
637 PCHThroughHeaderFileID.isValid();
638}
639
640bool Preprocessor::usingPCHWithThroughHeader() {
641 return TUKind != TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
642 PCHThroughHeaderFileID.isValid();
643}
644
645bool Preprocessor::creatingPCHWithPragmaHdrStop() {
646 return TUKind == TU_Prefix && PPOpts.PCHWithHdrStop;
647}
648
649bool Preprocessor::usingPCHWithPragmaHdrStop() {
650 return TUKind != TU_Prefix && PPOpts.PCHWithHdrStop;
651}
652
653/// Skip tokens until after the #include of the through header or
654/// until after a #pragma hdrstop is seen. Tokens in the predefines file
655/// and the main file may be skipped. If the end of the predefines file
656/// is reached, skipping continues into the main file. If the end of the
657/// main file is reached, it's a fatal error.
658void Preprocessor::SkipTokensWhileUsingPCH() {
659 bool ReachedMainFileEOF = false;
660 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
661 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
662 Token Tok;
663 while (true) {
664 bool InPredefines =
665 (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
666 CurLexerCallback(*this, Tok);
667 if (Tok.is(K: tok::eof) && !InPredefines) {
668 ReachedMainFileEOF = true;
669 break;
670 }
671 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
672 break;
673 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
674 break;
675 }
676 if (ReachedMainFileEOF) {
677 if (UsingPCHThroughHeader)
678 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_through_header_not_seen)
679 << PPOpts.PCHThroughHeader << 1;
680 else if (!PPOpts.PCHWithHdrStopCreate)
681 Diag(Loc: SourceLocation(), DiagID: diag::err_pp_pragma_hdrstop_not_seen);
682 }
683}
684
685void Preprocessor::replayPreambleConditionalStack() {
686 // Restore the conditional stack from the preamble, if there is one.
687 if (PreambleConditionalStack.isReplaying()) {
688 assert(CurPPLexer &&
689 "CurPPLexer is null when calling replayPreambleConditionalStack.");
690 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
691 PreambleConditionalStack.doneReplaying();
692 if (PreambleConditionalStack.reachedEOFWhileSkipping())
693 SkipExcludedConditionalBlock(
694 HashTokenLoc: PreambleConditionalStack.SkipInfo->HashTokenLoc,
695 IfTokenLoc: PreambleConditionalStack.SkipInfo->IfTokenLoc,
696 FoundNonSkipPortion: PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
697 FoundElse: PreambleConditionalStack.SkipInfo->FoundElse,
698 ElseLoc: PreambleConditionalStack.SkipInfo->ElseLoc);
699 }
700}
701
702void Preprocessor::EndSourceFile() {
703 // Notify the client that we reached the end of the source file.
704 if (Callbacks)
705 Callbacks->EndOfMainFile();
706}
707
708//===----------------------------------------------------------------------===//
709// Lexer Event Handling.
710//===----------------------------------------------------------------------===//
711
712/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
713/// identifier information for the token and install it into the token,
714/// updating the token kind accordingly.
715IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
716 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
717
718 // Look up this token, see if it is a macro, or if it is a language keyword.
719 IdentifierInfo *II;
720 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
721 // No cleaning needed, just use the characters from the lexed buffer.
722 II = getIdentifierInfo(Name: Identifier.getRawIdentifier());
723 } else {
724 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
725 SmallString<64> IdentifierBuffer;
726 StringRef CleanedStr = getSpelling(Tok: Identifier, Buffer&: IdentifierBuffer);
727
728 if (Identifier.hasUCN()) {
729 SmallString<64> UCNIdentifierBuffer;
730 expandUCNs(Buf&: UCNIdentifierBuffer, Input: CleanedStr);
731 II = getIdentifierInfo(Name: UCNIdentifierBuffer);
732 } else {
733 II = getIdentifierInfo(Name: CleanedStr);
734 }
735 }
736
737 // Update the token info (identifier info and appropriate token kind).
738 // FIXME: the raw_identifier may contain leading whitespace which is removed
739 // from the cleaned identifier token. The SourceLocation should be updated to
740 // refer to the non-whitespace character. For instance, the text "\\\nB" (a
741 // line continuation before 'B') is parsed as a single tok::raw_identifier and
742 // is cleaned to tok::identifier "B". After cleaning the token's length is
743 // still 3 and the SourceLocation refers to the location of the backslash.
744 Identifier.setIdentifierInfo(II);
745 Identifier.setKind(II->getTokenID());
746
747 return II;
748}
749
750void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
751 PoisonReasons[II] = DiagID;
752}
753
754void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
755 assert(Ident__exception_code && Ident__exception_info);
756 assert(Ident___exception_code && Ident___exception_info);
757 Ident__exception_code->setIsPoisoned(Poison);
758 Ident___exception_code->setIsPoisoned(Poison);
759 Ident_GetExceptionCode->setIsPoisoned(Poison);
760 Ident__exception_info->setIsPoisoned(Poison);
761 Ident___exception_info->setIsPoisoned(Poison);
762 Ident_GetExceptionInfo->setIsPoisoned(Poison);
763 Ident__abnormal_termination->setIsPoisoned(Poison);
764 Ident___abnormal_termination->setIsPoisoned(Poison);
765 Ident_AbnormalTermination->setIsPoisoned(Poison);
766}
767
768void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
769 assert(Identifier.getIdentifierInfo() &&
770 "Can't handle identifiers without identifier info!");
771 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
772 PoisonReasons.find(Val: Identifier.getIdentifierInfo());
773 if(it == PoisonReasons.end())
774 Diag(Tok: Identifier, DiagID: diag::err_pp_used_poisoned_id);
775 else
776 Diag(Tok: Identifier,DiagID: it->second) << Identifier.getIdentifierInfo();
777}
778
779void Preprocessor::updateOutOfDateIdentifier(const IdentifierInfo &II) const {
780 assert(II.isOutOfDate() && "not out of date");
781 assert(getExternalSource() &&
782 "getExternalSource() should not return nullptr");
783 getExternalSource()->updateOutOfDateIdentifier(II);
784}
785
786/// HandleIdentifier - This callback is invoked when the lexer reads an
787/// identifier. This callback looks up the identifier in the map and/or
788/// potentially macro expands it or turns it into a named token (like 'for').
789///
790/// Note that callers of this method are guarded by checking the
791/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
792/// IdentifierInfo methods that compute these properties will need to change to
793/// match.
794bool Preprocessor::HandleIdentifier(Token &Identifier) {
795 assert(Identifier.getIdentifierInfo() &&
796 "Can't handle identifiers without identifier info!");
797
798 IdentifierInfo &II = *Identifier.getIdentifierInfo();
799
800 // If the information about this identifier is out of date, update it from
801 // the external source.
802 // We have to treat __VA_ARGS__ in a special way, since it gets
803 // serialized with isPoisoned = true, but our preprocessor may have
804 // unpoisoned it if we're defining a C99 macro.
805 if (II.isOutOfDate()) {
806 bool CurrentIsPoisoned = false;
807 const bool IsSpecialVariadicMacro =
808 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
809 if (IsSpecialVariadicMacro)
810 CurrentIsPoisoned = II.isPoisoned();
811
812 updateOutOfDateIdentifier(II);
813 Identifier.setKind(II.getTokenID());
814
815 if (IsSpecialVariadicMacro)
816 II.setIsPoisoned(CurrentIsPoisoned);
817 }
818
819 // If this identifier was poisoned, and if it was not produced from a macro
820 // expansion, emit an error.
821 if (II.isPoisoned() && CurPPLexer) {
822 HandlePoisonedIdentifier(Identifier);
823 }
824
825 // If this is a macro to be expanded, do it.
826 if (const MacroDefinition MD = getMacroDefinition(II: &II)) {
827 const auto *MI = MD.getMacroInfo();
828 assert(MI && "macro definition with no macro info?");
829 if (!DisableMacroExpansion) {
830 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
831 // C99 6.10.3p10: If the preprocessing token immediately after the
832 // macro name isn't a '(', this macro should not be expanded.
833 if (!MI->isFunctionLike() || isNextPPTokenOneOf(Ks: tok::l_paren))
834 return HandleMacroExpandedIdentifier(Identifier, MD);
835 } else {
836 // C99 6.10.3.4p2 says that a disabled macro may never again be
837 // expanded, even if it's in a context where it could be expanded in the
838 // future.
839 Identifier.setFlag(Token::DisableExpand);
840 if (MI->isObjectLike() || isNextPPTokenOneOf(Ks: tok::l_paren))
841 Diag(Tok: Identifier, DiagID: diag::pp_disabled_macro_expansion);
842 }
843 }
844 }
845
846 // If this identifier is a keyword in a newer Standard or proposed Standard,
847 // produce a warning. Don't warn if we're not considering macro expansion,
848 // since this identifier might be the name of a macro.
849 // FIXME: This warning is disabled in cases where it shouldn't be, like
850 // "#define constexpr constexpr", "int constexpr;"
851 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
852 Diag(Tok: Identifier, DiagID: getIdentifierTable().getFutureCompatDiagKind(II, LangOpts: getLangOpts()))
853 << II.getName();
854 // Don't diagnose this keyword again in this translation unit.
855 II.setIsFutureCompatKeyword(false);
856 }
857
858 // If this identifier would be a keyword in C++, diagnose as a compatibility
859 // issue.
860 if (II.IsKeywordInCPlusPlus() && !DisableMacroExpansion)
861 Diag(Tok: Identifier, DiagID: diag::warn_pp_identifier_is_cpp_keyword) << &II;
862
863 // If this is an extension token, diagnose its use.
864 // We avoid diagnosing tokens that originate from macro definitions.
865 // FIXME: This warning is disabled in cases where it shouldn't be,
866 // like "#define TY typeof", "TY(1) x".
867 if (II.isExtensionToken() && !DisableMacroExpansion)
868 Diag(Tok: Identifier, DiagID: diag::ext_token_used);
869
870 // If this is the 'import' contextual keyword following an '@', note
871 // that the next token indicates a module name.
872 //
873 // Note that we do not treat 'import' as a contextual
874 // keyword when we're in a caching lexer, because caching lexers only get
875 // used in contexts where import declarations are disallowed.
876 //
877 // Likewise if this is the standard C++ import keyword.
878 if (((LastTokenWasAt && II.isModulesImport()) ||
879 Identifier.is(K: tok::kw_import)) &&
880 !InMacroArgs && !DisableMacroExpansion &&
881 (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
882 CurLexerCallback != CLK_CachingLexer) {
883 ModuleImportLoc = Identifier.getLocation();
884 NamedModuleImportPath.clear();
885 IsAtImport = true;
886 ModuleImportExpectsIdentifier = true;
887 CurLexerCallback = CLK_LexAfterModuleImport;
888 }
889 return true;
890}
891
892void Preprocessor::Lex(Token &Result) {
893 ++LexLevel;
894
895 // We loop here until a lex function returns a token; this avoids recursion.
896 while (!CurLexerCallback(*this, Result))
897 ;
898
899 if (Result.is(K: tok::unknown) && TheModuleLoader.HadFatalFailure)
900 return;
901
902 if (Result.is(K: tok::code_completion) && Result.getIdentifierInfo()) {
903 // Remember the identifier before code completion token.
904 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
905 setCodeCompletionTokenRange(Start: Result.getLocation(), End: Result.getEndLoc());
906 // Set IdenfitierInfo to null to avoid confusing code that handles both
907 // identifiers and completion tokens.
908 Result.setIdentifierInfo(nullptr);
909 }
910
911 // Update StdCXXImportSeqState to track our position within a C++20 import-seq
912 // if this token is being produced as a result of phase 4 of translation.
913 // Update TrackGMFState to decide if we are currently in a Global Module
914 // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state
915 // depends on the prevailing StdCXXImportSeq state in two cases.
916 if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
917 !Result.getFlag(Flag: Token::IsReinjected)) {
918 switch (Result.getKind()) {
919 case tok::l_paren: case tok::l_square: case tok::l_brace:
920 StdCXXImportSeqState.handleOpenBracket();
921 break;
922 case tok::r_paren: case tok::r_square:
923 StdCXXImportSeqState.handleCloseBracket();
924 break;
925 case tok::r_brace:
926 StdCXXImportSeqState.handleCloseBrace();
927 break;
928#define PRAGMA_ANNOTATION(X) case tok::annot_##X:
929// For `#pragma ...` mimic ';'.
930#include "clang/Basic/TokenKinds.def"
931#undef PRAGMA_ANNOTATION
932 // This token is injected to represent the translation of '#include "a.h"'
933 // into "import a.h;". Mimic the notional ';'.
934 case tok::annot_module_include:
935 case tok::semi:
936 TrackGMFState.handleSemi();
937 StdCXXImportSeqState.handleSemi();
938 ModuleDeclState.handleSemi();
939 break;
940 case tok::header_name:
941 case tok::annot_header_unit:
942 StdCXXImportSeqState.handleHeaderName();
943 break;
944 case tok::kw_export:
945 if (hasSeenNoTrivialPPDirective())
946 Result.setFlag(Token::HasSeenNoTrivialPPDirective);
947 TrackGMFState.handleExport();
948 StdCXXImportSeqState.handleExport();
949 ModuleDeclState.handleExport();
950 break;
951 case tok::colon:
952 ModuleDeclState.handleColon();
953 break;
954 case tok::period:
955 ModuleDeclState.handlePeriod();
956 break;
957 case tok::identifier:
958 // Check "import" and "module" when there is no open bracket. The two
959 // identifiers are not meaningful with open brackets.
960 if (StdCXXImportSeqState.atTopLevel()) {
961 if (Result.getIdentifierInfo()->isModulesImport()) {
962 TrackGMFState.handleImport(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
963 StdCXXImportSeqState.handleImport();
964 if (StdCXXImportSeqState.afterImportSeq()) {
965 ModuleImportLoc = Result.getLocation();
966 NamedModuleImportPath.clear();
967 IsAtImport = false;
968 ModuleImportExpectsIdentifier = true;
969 CurLexerCallback = CLK_LexAfterModuleImport;
970 }
971 break;
972 } else if (Result.getIdentifierInfo() == getIdentifierInfo(Name: "module")) {
973 if (hasSeenNoTrivialPPDirective())
974 Result.setFlag(Token::HasSeenNoTrivialPPDirective);
975 TrackGMFState.handleModule(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
976 ModuleDeclState.handleModule();
977 break;
978 }
979 }
980 ModuleDeclState.handleIdentifier(Identifier: Result.getIdentifierInfo());
981 if (ModuleDeclState.isModuleCandidate())
982 break;
983 [[fallthrough]];
984 default:
985 TrackGMFState.handleMisc();
986 StdCXXImportSeqState.handleMisc();
987 ModuleDeclState.handleMisc();
988 break;
989 }
990 }
991
992 if (CurLexer && ++CheckPointCounter == CheckPointStepSize) {
993 CheckPoints[CurLexer->getFileID()].push_back(Elt: CurLexer->BufferPtr);
994 CheckPointCounter = 0;
995 }
996
997 LastTokenWasAt = Result.is(K: tok::at);
998 --LexLevel;
999
1000 if ((LexLevel == 0 || PreprocessToken) &&
1001 !Result.getFlag(Flag: Token::IsReinjected)) {
1002 if (LexLevel == 0)
1003 ++TokenCount;
1004 if (OnToken)
1005 OnToken(Result);
1006 }
1007}
1008
1009void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
1010 while (1) {
1011 Token Tok;
1012 Lex(Result&: Tok);
1013 if (Tok.isOneOf(Ks: tok::unknown, Ks: tok::eof, Ks: tok::eod,
1014 Ks: tok::annot_repl_input_end))
1015 break;
1016 if (Tokens != nullptr)
1017 Tokens->push_back(x: Tok);
1018 }
1019}
1020
1021/// Lex a header-name token (including one formed from header-name-tokens if
1022/// \p AllowMacroExpansion is \c true).
1023///
1024/// \param FilenameTok Filled in with the next token. On success, this will
1025/// be either a header_name token. On failure, it will be whatever other
1026/// token was found instead.
1027/// \param AllowMacroExpansion If \c true, allow the header name to be formed
1028/// by macro expansion (concatenating tokens as necessary if the first
1029/// token is a '<').
1030/// \return \c true if we reached EOD or EOF while looking for a > token in
1031/// a concatenated header name and diagnosed it. \c false otherwise.
1032bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
1033 // Lex using header-name tokenization rules if tokens are being lexed from
1034 // a file. Just grab a token normally if we're in a macro expansion.
1035 if (CurPPLexer)
1036 CurPPLexer->LexIncludeFilename(FilenameTok);
1037 else
1038 Lex(Result&: FilenameTok);
1039
1040 // This could be a <foo/bar.h> file coming from a macro expansion. In this
1041 // case, glue the tokens together into an angle_string_literal token.
1042 SmallString<128> FilenameBuffer;
1043 if (FilenameTok.is(K: tok::less) && AllowMacroExpansion) {
1044 bool StartOfLine = FilenameTok.isAtStartOfLine();
1045 bool LeadingSpace = FilenameTok.hasLeadingSpace();
1046 bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1047
1048 SourceLocation Start = FilenameTok.getLocation();
1049 SourceLocation End;
1050 FilenameBuffer.push_back(Elt: '<');
1051
1052 // Consume tokens until we find a '>'.
1053 // FIXME: A header-name could be formed starting or ending with an
1054 // alternative token. It's not clear whether that's ill-formed in all
1055 // cases.
1056 while (FilenameTok.isNot(K: tok::greater)) {
1057 Lex(Result&: FilenameTok);
1058 if (FilenameTok.isOneOf(Ks: tok::eod, Ks: tok::eof)) {
1059 Diag(Loc: FilenameTok.getLocation(), DiagID: diag::err_expected) << tok::greater;
1060 Diag(Loc: Start, DiagID: diag::note_matching) << tok::less;
1061 return true;
1062 }
1063
1064 End = FilenameTok.getLocation();
1065
1066 // FIXME: Provide code completion for #includes.
1067 if (FilenameTok.is(K: tok::code_completion)) {
1068 setCodeCompletionReached();
1069 Lex(Result&: FilenameTok);
1070 continue;
1071 }
1072
1073 // Append the spelling of this token to the buffer. If there was a space
1074 // before it, add it now.
1075 if (FilenameTok.hasLeadingSpace())
1076 FilenameBuffer.push_back(Elt: ' ');
1077
1078 // Get the spelling of the token, directly into FilenameBuffer if
1079 // possible.
1080 size_t PreAppendSize = FilenameBuffer.size();
1081 FilenameBuffer.resize(N: PreAppendSize + FilenameTok.getLength());
1082
1083 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1084 unsigned ActualLen = getSpelling(Tok: FilenameTok, Buffer&: BufPtr);
1085
1086 // If the token was spelled somewhere else, copy it into FilenameBuffer.
1087 if (BufPtr != &FilenameBuffer[PreAppendSize])
1088 memcpy(dest: &FilenameBuffer[PreAppendSize], src: BufPtr, n: ActualLen);
1089
1090 // Resize FilenameBuffer to the correct size.
1091 if (FilenameTok.getLength() != ActualLen)
1092 FilenameBuffer.resize(N: PreAppendSize + ActualLen);
1093 }
1094
1095 FilenameTok.startToken();
1096 FilenameTok.setKind(tok::header_name);
1097 FilenameTok.setFlagValue(Flag: Token::StartOfLine, Val: StartOfLine);
1098 FilenameTok.setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace);
1099 FilenameTok.setFlagValue(Flag: Token::LeadingEmptyMacro, Val: LeadingEmptyMacro);
1100 CreateString(Str: FilenameBuffer, Tok&: FilenameTok, ExpansionLocStart: Start, ExpansionLocEnd: End);
1101 } else if (FilenameTok.is(K: tok::string_literal) && AllowMacroExpansion) {
1102 // Convert a string-literal token of the form " h-char-sequence "
1103 // (produced by macro expansion) into a header-name token.
1104 //
1105 // The rules for header-names don't quite match the rules for
1106 // string-literals, but all the places where they differ result in
1107 // undefined behavior, so we can and do treat them the same.
1108 //
1109 // A string-literal with a prefix or suffix is not translated into a
1110 // header-name. This could theoretically be observable via the C++20
1111 // context-sensitive header-name formation rules.
1112 StringRef Str = getSpelling(Tok: FilenameTok, Buffer&: FilenameBuffer);
1113 if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1114 FilenameTok.setKind(tok::header_name);
1115 }
1116
1117 return false;
1118}
1119
1120/// Collect the tokens of a C++20 pp-import-suffix.
1121void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) {
1122 // FIXME: For error recovery, consider recognizing attribute syntax here
1123 // and terminating / diagnosing a missing semicolon if we find anything
1124 // else? (Can we leave that to the parser?)
1125 unsigned BracketDepth = 0;
1126 while (true) {
1127 Toks.emplace_back();
1128 Lex(Result&: Toks.back());
1129
1130 switch (Toks.back().getKind()) {
1131 case tok::l_paren: case tok::l_square: case tok::l_brace:
1132 ++BracketDepth;
1133 break;
1134
1135 case tok::r_paren: case tok::r_square: case tok::r_brace:
1136 if (BracketDepth == 0)
1137 return;
1138 --BracketDepth;
1139 break;
1140
1141 case tok::semi:
1142 if (BracketDepth == 0)
1143 return;
1144 break;
1145
1146 case tok::eof:
1147 return;
1148
1149 default:
1150 break;
1151 }
1152 }
1153}
1154
1155
1156/// Lex a token following the 'import' contextual keyword.
1157///
1158/// pp-import: [C++20]
1159/// import header-name pp-import-suffix[opt] ;
1160/// import header-name-tokens pp-import-suffix[opt] ;
1161/// [ObjC] @ import module-name ;
1162/// [Clang] import module-name ;
1163///
1164/// header-name-tokens:
1165/// string-literal
1166/// < [any sequence of preprocessing-tokens other than >] >
1167///
1168/// module-name:
1169/// module-name-qualifier[opt] identifier
1170///
1171/// module-name-qualifier
1172/// module-name-qualifier[opt] identifier .
1173///
1174/// We respond to a pp-import by importing macros from the named module.
1175bool Preprocessor::LexAfterModuleImport(Token &Result) {
1176 // Figure out what kind of lexer we actually have.
1177 recomputeCurLexerKind();
1178
1179 // Lex the next token. The header-name lexing rules are used at the start of
1180 // a pp-import.
1181 //
1182 // For now, we only support header-name imports in C++20 mode.
1183 // FIXME: Should we allow this in all language modes that support an import
1184 // declaration as an extension?
1185 if (NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1186 if (LexHeaderName(FilenameTok&: Result))
1187 return true;
1188
1189 if (Result.is(K: tok::colon) && ModuleDeclState.isNamedModule()) {
1190 std::string Name = ModuleDeclState.getPrimaryName().str();
1191 Name += ":";
1192 NamedModuleImportPath.emplace_back(Args: Result.getLocation(),
1193 Args: getIdentifierInfo(Name));
1194 CurLexerCallback = CLK_LexAfterModuleImport;
1195 return true;
1196 }
1197 } else {
1198 Lex(Result);
1199 }
1200
1201 // Allocate a holding buffer for a sequence of tokens and introduce it into
1202 // the token stream.
1203 auto EnterTokens = [this](ArrayRef<Token> Toks) {
1204 auto ToksCopy = std::make_unique<Token[]>(num: Toks.size());
1205 std::copy(first: Toks.begin(), last: Toks.end(), result: ToksCopy.get());
1206 EnterTokenStream(Toks: std::move(ToksCopy), NumToks: Toks.size(),
1207 /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1208 };
1209
1210 bool ImportingHeader = Result.is(K: tok::header_name);
1211 // Check for a header-name.
1212 SmallVector<Token, 32> Suffix;
1213 if (ImportingHeader) {
1214 // Enter the header-name token into the token stream; a Lex action cannot
1215 // both return a token and cache tokens (doing so would corrupt the token
1216 // cache if the call to Lex comes from CachingLex / PeekAhead).
1217 Suffix.push_back(Elt: Result);
1218
1219 // Consume the pp-import-suffix and expand any macros in it now. We'll add
1220 // it back into the token stream later.
1221 CollectPpImportSuffix(Toks&: Suffix);
1222 if (Suffix.back().isNot(K: tok::semi)) {
1223 // This is not a pp-import after all.
1224 EnterTokens(Suffix);
1225 return false;
1226 }
1227
1228 // C++2a [cpp.module]p1:
1229 // The ';' preprocessing-token terminating a pp-import shall not have
1230 // been produced by macro replacement.
1231 SourceLocation SemiLoc = Suffix.back().getLocation();
1232 if (SemiLoc.isMacroID())
1233 Diag(Loc: SemiLoc, DiagID: diag::err_header_import_semi_in_macro);
1234
1235 // Reconstitute the import token.
1236 Token ImportTok;
1237 ImportTok.startToken();
1238 ImportTok.setKind(tok::kw_import);
1239 ImportTok.setLocation(ModuleImportLoc);
1240 ImportTok.setIdentifierInfo(getIdentifierInfo(Name: "import"));
1241 ImportTok.setLength(6);
1242
1243 auto Action = HandleHeaderIncludeOrImport(
1244 /*HashLoc*/ SourceLocation(), IncludeTok&: ImportTok, FilenameTok&: Suffix.front(), EndLoc: SemiLoc);
1245 switch (Action.Kind) {
1246 case ImportAction::None:
1247 break;
1248
1249 case ImportAction::ModuleBegin:
1250 // Let the parser know we're textually entering the module.
1251 Suffix.emplace_back();
1252 Suffix.back().startToken();
1253 Suffix.back().setKind(tok::annot_module_begin);
1254 Suffix.back().setLocation(SemiLoc);
1255 Suffix.back().setAnnotationEndLoc(SemiLoc);
1256 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1257 [[fallthrough]];
1258
1259 case ImportAction::ModuleImport:
1260 case ImportAction::HeaderUnitImport:
1261 case ImportAction::SkippedModuleImport:
1262 // We chose to import (or textually enter) the file. Convert the
1263 // header-name token into a header unit annotation token.
1264 Suffix[0].setKind(tok::annot_header_unit);
1265 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1266 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1267 // FIXME: Call the moduleImport callback?
1268 break;
1269 case ImportAction::Failure:
1270 assert(TheModuleLoader.HadFatalFailure &&
1271 "This should be an early exit only to a fatal error");
1272 Result.setKind(tok::eof);
1273 CurLexer->cutOffLexing();
1274 EnterTokens(Suffix);
1275 return true;
1276 }
1277
1278 EnterTokens(Suffix);
1279 return false;
1280 }
1281
1282 // The token sequence
1283 //
1284 // import identifier (. identifier)*
1285 //
1286 // indicates a module import directive. We already saw the 'import'
1287 // contextual keyword, so now we're looking for the identifiers.
1288 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1289 // We expected to see an identifier here, and we did; continue handling
1290 // identifiers.
1291 NamedModuleImportPath.emplace_back(Args: Result.getLocation(),
1292 Args: Result.getIdentifierInfo());
1293 ModuleImportExpectsIdentifier = false;
1294 CurLexerCallback = CLK_LexAfterModuleImport;
1295 return true;
1296 }
1297
1298 // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1299 // see the next identifier. (We can also see a '[[' that begins an
1300 // attribute-specifier-seq here under the Standard C++ Modules.)
1301 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1302 ModuleImportExpectsIdentifier = true;
1303 CurLexerCallback = CLK_LexAfterModuleImport;
1304 return true;
1305 }
1306
1307 // If we didn't recognize a module name at all, this is not a (valid) import.
1308 if (NamedModuleImportPath.empty() || Result.is(K: tok::eof))
1309 return true;
1310
1311 // Consume the pp-import-suffix and expand any macros in it now, if we're not
1312 // at the semicolon already.
1313 SourceLocation SemiLoc = Result.getLocation();
1314 if (Result.isNot(K: tok::semi)) {
1315 Suffix.push_back(Elt: Result);
1316 CollectPpImportSuffix(Toks&: Suffix);
1317 if (Suffix.back().isNot(K: tok::semi)) {
1318 // This is not an import after all.
1319 EnterTokens(Suffix);
1320 return false;
1321 }
1322 SemiLoc = Suffix.back().getLocation();
1323 }
1324
1325 // Under the standard C++ Modules, the dot is just part of the module name,
1326 // and not a real hierarchy separator. Flatten such module names now.
1327 //
1328 // FIXME: Is this the right level to be performing this transformation?
1329 std::string FlatModuleName;
1330 if (getLangOpts().CPlusPlusModules) {
1331 for (auto &Piece : NamedModuleImportPath) {
1332 // If the FlatModuleName ends with colon, it implies it is a partition.
1333 if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
1334 FlatModuleName += ".";
1335 FlatModuleName += Piece.getIdentifierInfo()->getName();
1336 }
1337 SourceLocation FirstPathLoc = NamedModuleImportPath[0].getLoc();
1338 NamedModuleImportPath.clear();
1339 NamedModuleImportPath.emplace_back(Args&: FirstPathLoc,
1340 Args: getIdentifierInfo(Name: FlatModuleName));
1341 }
1342
1343 Module *Imported = nullptr;
1344 // We don't/shouldn't load the standard c++20 modules when preprocessing.
1345 if (getLangOpts().Modules && !isInImportingCXXNamedModules()) {
1346 Imported = TheModuleLoader.loadModule(ImportLoc: ModuleImportLoc,
1347 Path: NamedModuleImportPath,
1348 Visibility: Module::Hidden,
1349 /*IsInclusionDirective=*/false);
1350 if (Imported)
1351 makeModuleVisible(M: Imported, Loc: SemiLoc);
1352 }
1353
1354 if (Callbacks)
1355 Callbacks->moduleImport(ImportLoc: ModuleImportLoc, Path: NamedModuleImportPath, Imported);
1356
1357 if (!Suffix.empty()) {
1358 EnterTokens(Suffix);
1359 return false;
1360 }
1361 return true;
1362}
1363
1364void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc,
1365 bool IncludeExports) {
1366 CurSubmoduleState->VisibleModules.setVisible(
1367 M, Loc, IncludeExports, Vis: [](Module *) {},
1368 Cb: [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1369 // FIXME: Include the path in the diagnostic.
1370 // FIXME: Include the import location for the conflicting module.
1371 Diag(Loc: ModuleImportLoc, DiagID: diag::warn_module_conflict)
1372 << Path[0]->getFullModuleName()
1373 << Conflict->getFullModuleName()
1374 << Message;
1375 });
1376
1377 // Add this module to the imports list of the currently-built submodule.
1378 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1379 BuildingSubmoduleStack.back().M->Imports.insert(X: M);
1380}
1381
1382bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
1383 const char *DiagnosticTag,
1384 bool AllowMacroExpansion) {
1385 // We need at least one string literal.
1386 if (Result.isNot(K: tok::string_literal)) {
1387 Diag(Tok: Result, DiagID: diag::err_expected_string_literal)
1388 << /*Source='in...'*/0 << DiagnosticTag;
1389 return false;
1390 }
1391
1392 // Lex string literal tokens, optionally with macro expansion.
1393 SmallVector<Token, 4> StrToks;
1394 do {
1395 StrToks.push_back(Elt: Result);
1396
1397 if (Result.hasUDSuffix())
1398 Diag(Tok: Result, DiagID: diag::err_invalid_string_udl);
1399
1400 if (AllowMacroExpansion)
1401 Lex(Result);
1402 else
1403 LexUnexpandedToken(Result);
1404 } while (Result.is(K: tok::string_literal));
1405
1406 // Concatenate and parse the strings.
1407 StringLiteralParser Literal(StrToks, *this);
1408 assert(Literal.isOrdinary() && "Didn't allow wide strings in");
1409
1410 if (Literal.hadError)
1411 return false;
1412
1413 if (Literal.Pascal) {
1414 Diag(Loc: StrToks[0].getLocation(), DiagID: diag::err_expected_string_literal)
1415 << /*Source='in...'*/0 << DiagnosticTag;
1416 return false;
1417 }
1418
1419 String = std::string(Literal.GetString());
1420 return true;
1421}
1422
1423bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
1424 assert(Tok.is(tok::numeric_constant));
1425 SmallString<8> IntegerBuffer;
1426 bool NumberInvalid = false;
1427 StringRef Spelling = getSpelling(Tok, Buffer&: IntegerBuffer, Invalid: &NumberInvalid);
1428 if (NumberInvalid)
1429 return false;
1430 NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
1431 getLangOpts(), getTargetInfo(),
1432 getDiagnostics());
1433 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1434 return false;
1435 llvm::APInt APVal(64, 0);
1436 if (Literal.GetIntegerValue(Val&: APVal))
1437 return false;
1438 Lex(Result&: Tok);
1439 Value = APVal.getLimitedValue();
1440 return true;
1441}
1442
1443void Preprocessor::addCommentHandler(CommentHandler *Handler) {
1444 assert(Handler && "NULL comment handler");
1445 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1446 "Comment handler already registered");
1447 CommentHandlers.push_back(x: Handler);
1448}
1449
1450void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
1451 std::vector<CommentHandler *>::iterator Pos =
1452 llvm::find(Range&: CommentHandlers, Val: Handler);
1453 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1454 CommentHandlers.erase(position: Pos);
1455}
1456
1457bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
1458 bool AnyPendingTokens = false;
1459 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1460 HEnd = CommentHandlers.end();
1461 H != HEnd; ++H) {
1462 if ((*H)->HandleComment(PP&: *this, Comment))
1463 AnyPendingTokens = true;
1464 }
1465 if (!AnyPendingTokens || getCommentRetentionState())
1466 return false;
1467 Lex(Result&: result);
1468 return true;
1469}
1470
1471void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1472 const MacroAnnotations &A =
1473 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1474 assert(A.DeprecationInfo &&
1475 "Macro deprecation warning without recorded annotation!");
1476 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1477 if (Info.Message.empty())
1478 Diag(Tok: Identifier, DiagID: diag::warn_pragma_deprecated_macro_use)
1479 << Identifier.getIdentifierInfo() << 0;
1480 else
1481 Diag(Tok: Identifier, DiagID: diag::warn_pragma_deprecated_macro_use)
1482 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1483 Diag(Loc: Info.Location, DiagID: diag::note_pp_macro_annotation) << 0;
1484}
1485
1486void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1487 const MacroAnnotations &A =
1488 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1489 assert(A.RestrictExpansionInfo &&
1490 "Macro restricted expansion warning without recorded annotation!");
1491 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1492 if (Info.Message.empty())
1493 Diag(Tok: Identifier, DiagID: diag::warn_pragma_restrict_expansion_macro_use)
1494 << Identifier.getIdentifierInfo() << 0;
1495 else
1496 Diag(Tok: Identifier, DiagID: diag::warn_pragma_restrict_expansion_macro_use)
1497 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1498 Diag(Loc: Info.Location, DiagID: diag::note_pp_macro_annotation) << 1;
1499}
1500
1501void Preprocessor::emitRestrictInfNaNWarning(const Token &Identifier,
1502 unsigned DiagSelection) const {
1503 Diag(Tok: Identifier, DiagID: diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1;
1504}
1505
1506void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1507 bool IsUndef) const {
1508 const MacroAnnotations &A =
1509 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1510 assert(A.FinalAnnotationLoc &&
1511 "Final macro warning without recorded annotation!");
1512
1513 Diag(Tok: Identifier, DiagID: diag::warn_pragma_final_macro)
1514 << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1515 Diag(Loc: *A.FinalAnnotationLoc, DiagID: diag::note_pp_macro_annotation) << 2;
1516}
1517
1518bool Preprocessor::isSafeBufferOptOut(const SourceManager &SourceMgr,
1519 const SourceLocation &Loc) const {
1520 // The lambda that tests if a `Loc` is in an opt-out region given one opt-out
1521 // region map:
1522 auto TestInMap = [&SourceMgr](const SafeBufferOptOutRegionsTy &Map,
1523 const SourceLocation &Loc) -> bool {
1524 // Try to find a region in `SafeBufferOptOutMap` where `Loc` is in:
1525 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1526 Range: Map, P: [&SourceMgr,
1527 &Loc](const std::pair<SourceLocation, SourceLocation> &Region) {
1528 return SourceMgr.isBeforeInTranslationUnit(LHS: Region.second, RHS: Loc);
1529 });
1530
1531 if (FirstRegionEndingAfterLoc != Map.end()) {
1532 // To test if the start location of the found region precedes `Loc`:
1533 return SourceMgr.isBeforeInTranslationUnit(
1534 LHS: FirstRegionEndingAfterLoc->first, RHS: Loc);
1535 }
1536 // If we do not find a region whose end location passes `Loc`, we want to
1537 // check if the current region is still open:
1538 if (!Map.empty() && Map.back().first == Map.back().second)
1539 return SourceMgr.isBeforeInTranslationUnit(LHS: Map.back().first, RHS: Loc);
1540 return false;
1541 };
1542
1543 // What the following does:
1544 //
1545 // If `Loc` belongs to the local TU, we just look up `SafeBufferOptOutMap`.
1546 // Otherwise, `Loc` is from a loaded AST. We look up the
1547 // `LoadedSafeBufferOptOutMap` first to get the opt-out region map of the
1548 // loaded AST where `Loc` is at. Then we find if `Loc` is in an opt-out
1549 // region w.r.t. the region map. If the region map is absent, it means there
1550 // is no opt-out pragma in that loaded AST.
1551 //
1552 // Opt-out pragmas in the local TU or a loaded AST is not visible to another
1553 // one of them. That means if you put the pragmas around a `#include
1554 // "module.h"`, where module.h is a module, it is not actually suppressing
1555 // warnings in module.h. This is fine because warnings in module.h will be
1556 // reported when module.h is compiled in isolation and nothing in module.h
1557 // will be analyzed ever again. So you will not see warnings from the file
1558 // that imports module.h anyway. And you can't even do the same thing for PCHs
1559 // because they can only be included from the command line.
1560
1561 if (SourceMgr.isLocalSourceLocation(Loc))
1562 return TestInMap(SafeBufferOptOutMap, Loc);
1563
1564 const SafeBufferOptOutRegionsTy *LoadedRegions =
1565 LoadedSafeBufferOptOutMap.lookupLoadedOptOutMap(Loc, SrcMgr: SourceMgr);
1566
1567 if (LoadedRegions)
1568 return TestInMap(*LoadedRegions, Loc);
1569 return false;
1570}
1571
1572bool Preprocessor::enterOrExitSafeBufferOptOutRegion(
1573 bool isEnter, const SourceLocation &Loc) {
1574 if (isEnter) {
1575 if (isPPInSafeBufferOptOutRegion())
1576 return true; // invalid enter action
1577 InSafeBufferOptOutRegion = true;
1578 CurrentSafeBufferOptOutStart = Loc;
1579
1580 // To set the start location of a new region:
1581
1582 if (!SafeBufferOptOutMap.empty()) {
1583 [[maybe_unused]] auto *PrevRegion = &SafeBufferOptOutMap.back();
1584 assert(PrevRegion->first != PrevRegion->second &&
1585 "Shall not begin a safe buffer opt-out region before closing the "
1586 "previous one.");
1587 }
1588 // If the start location equals to the end location, we call the region a
1589 // open region or a unclosed region (i.e., end location has not been set
1590 // yet).
1591 SafeBufferOptOutMap.emplace_back(Args: Loc, Args: Loc);
1592 } else {
1593 if (!isPPInSafeBufferOptOutRegion())
1594 return true; // invalid enter action
1595 InSafeBufferOptOutRegion = false;
1596
1597 // To set the end location of the current open region:
1598
1599 assert(!SafeBufferOptOutMap.empty() &&
1600 "Misordered safe buffer opt-out regions");
1601 auto *CurrRegion = &SafeBufferOptOutMap.back();
1602 assert(CurrRegion->first == CurrRegion->second &&
1603 "Set end location to a closed safe buffer opt-out region");
1604 CurrRegion->second = Loc;
1605 }
1606 return false;
1607}
1608
1609bool Preprocessor::isPPInSafeBufferOptOutRegion() {
1610 return InSafeBufferOptOutRegion;
1611}
1612bool Preprocessor::isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc) {
1613 StartLoc = CurrentSafeBufferOptOutStart;
1614 return InSafeBufferOptOutRegion;
1615}
1616
1617SmallVector<SourceLocation, 64>
1618Preprocessor::serializeSafeBufferOptOutMap() const {
1619 assert(!InSafeBufferOptOutRegion &&
1620 "Attempt to serialize safe buffer opt-out regions before file being "
1621 "completely preprocessed");
1622
1623 SmallVector<SourceLocation, 64> SrcSeq;
1624
1625 for (const auto &[begin, end] : SafeBufferOptOutMap) {
1626 SrcSeq.push_back(Elt: begin);
1627 SrcSeq.push_back(Elt: end);
1628 }
1629 // Only `SafeBufferOptOutMap` gets serialized. No need to serialize
1630 // `LoadedSafeBufferOptOutMap` because if this TU loads a pch/module, every
1631 // pch/module in the pch-chain/module-DAG will be loaded one by one in order.
1632 // It means that for each loading pch/module m, it just needs to load m's own
1633 // `SafeBufferOptOutMap`.
1634 return SrcSeq;
1635}
1636
1637bool Preprocessor::setDeserializedSafeBufferOptOutMap(
1638 const SmallVectorImpl<SourceLocation> &SourceLocations) {
1639 if (SourceLocations.size() == 0)
1640 return false;
1641
1642 assert(SourceLocations.size() % 2 == 0 &&
1643 "ill-formed SourceLocation sequence");
1644
1645 auto It = SourceLocations.begin();
1646 SafeBufferOptOutRegionsTy &Regions =
1647 LoadedSafeBufferOptOutMap.findAndConsLoadedOptOutMap(Loc: *It, SrcMgr&: SourceMgr);
1648
1649 do {
1650 SourceLocation Begin = *It++;
1651 SourceLocation End = *It++;
1652
1653 Regions.emplace_back(Args&: Begin, Args&: End);
1654 } while (It != SourceLocations.end());
1655 return true;
1656}
1657
1658ModuleLoader::~ModuleLoader() = default;
1659
1660CommentHandler::~CommentHandler() = default;
1661
1662EmptylineHandler::~EmptylineHandler() = default;
1663
1664CodeCompletionHandler::~CodeCompletionHandler() = default;
1665
1666void Preprocessor::createPreprocessingRecord() {
1667 if (Record)
1668 return;
1669
1670 Record = new PreprocessingRecord(getSourceManager());
1671 addPPCallbacks(C: std::unique_ptr<PPCallbacks>(Record));
1672}
1673
1674const char *Preprocessor::getCheckPoint(FileID FID, const char *Start) const {
1675 if (auto It = CheckPoints.find(Val: FID); It != CheckPoints.end()) {
1676 const SmallVector<const char *> &FileCheckPoints = It->second;
1677 const char *Last = nullptr;
1678 // FIXME: Do better than a linear search.
1679 for (const char *P : FileCheckPoints) {
1680 if (P > Start)
1681 break;
1682 Last = P;
1683 }
1684 return Last;
1685 }
1686
1687 return nullptr;
1688}
1689
1690bool Preprocessor::hasSeenNoTrivialPPDirective() const {
1691 return DirTracer && DirTracer->hasSeenNoTrivialPPDirective();
1692}
1693
1694bool NoTrivialPPDirectiveTracer::hasSeenNoTrivialPPDirective() const {
1695 return SeenNoTrivialPPDirective;
1696}
1697
1698void NoTrivialPPDirectiveTracer::setSeenNoTrivialPPDirective() {
1699 if (InMainFile && !SeenNoTrivialPPDirective)
1700 SeenNoTrivialPPDirective = true;
1701}
1702
1703void NoTrivialPPDirectiveTracer::LexedFileChanged(
1704 FileID FID, LexedFileChangeReason Reason,
1705 SrcMgr::CharacteristicKind FileType, FileID PrevFID, SourceLocation Loc) {
1706 InMainFile = (FID == PP.getSourceManager().getMainFileID());
1707}
1708
1709void NoTrivialPPDirectiveTracer::MacroExpands(const Token &MacroNameTok,
1710 const MacroDefinition &MD,
1711 SourceRange Range,
1712 const MacroArgs *Args) {
1713 // FIXME: Does only enable builtin macro expansion make sense?
1714 if (!MD.getMacroInfo()->isBuiltinMacro())
1715 setSeenNoTrivialPPDirective();
1716}
1717

source code of clang/lib/Lex/Preprocessor.cpp