1//===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Preprocessor interface.
10//
11//===----------------------------------------------------------------------===//
12//
13// Options to support:
14// -H - Print the name of each header file used.
15// -d[DNI] - Dump various things.
16// -fworking-directory - #line's with preprocessor's working dir.
17// -fpreprocessed
18// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
19// -W*
20// -w
21//
22// Messages to emit:
23// "Multiple include guards may be useful for:\n"
24//
25//===----------------------------------------------------------------------===//
26
27#include "clang/Lex/Preprocessor.h"
28#include "clang/Basic/Builtins.h"
29#include "clang/Basic/FileManager.h"
30#include "clang/Basic/IdentifierTable.h"
31#include "clang/Basic/LLVM.h"
32#include "clang/Basic/LangOptions.h"
33#include "clang/Basic/Module.h"
34#include "clang/Basic/SourceLocation.h"
35#include "clang/Basic/SourceManager.h"
36#include "clang/Basic/TargetInfo.h"
37#include "clang/Lex/CodeCompletionHandler.h"
38#include "clang/Lex/ExternalPreprocessorSource.h"
39#include "clang/Lex/HeaderSearch.h"
40#include "clang/Lex/LexDiagnostic.h"
41#include "clang/Lex/Lexer.h"
42#include "clang/Lex/LiteralSupport.h"
43#include "clang/Lex/MacroArgs.h"
44#include "clang/Lex/MacroInfo.h"
45#include "clang/Lex/ModuleLoader.h"
46#include "clang/Lex/Pragma.h"
47#include "clang/Lex/PreprocessingRecord.h"
48#include "clang/Lex/PreprocessorLexer.h"
49#include "clang/Lex/PreprocessorOptions.h"
50#include "clang/Lex/ScratchBuffer.h"
51#include "clang/Lex/Token.h"
52#include "clang/Lex/TokenLexer.h"
53#include "llvm/ADT/APInt.h"
54#include "llvm/ADT/ArrayRef.h"
55#include "llvm/ADT/DenseMap.h"
56#include "llvm/ADT/STLExtras.h"
57#include "llvm/ADT/SmallVector.h"
58#include "llvm/ADT/StringRef.h"
59#include "llvm/Support/Capacity.h"
60#include "llvm/Support/ErrorHandling.h"
61#include "llvm/Support/MemoryBuffer.h"
62#include "llvm/Support/raw_ostream.h"
63#include <algorithm>
64#include <cassert>
65#include <memory>
66#include <optional>
67#include <string>
68#include <utility>
69#include <vector>
70
71using namespace clang;
72
73/// Minimum distance between two check points, in tokens.
74static constexpr unsigned CheckPointStepSize = 1024;
75
76LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
77
78ExternalPreprocessorSource::~ExternalPreprocessorSource() = default;
79
80Preprocessor::Preprocessor(const PreprocessorOptions &PPOpts,
81 DiagnosticsEngine &diags, const LangOptions &opts,
82 SourceManager &SM, HeaderSearch &Headers,
83 ModuleLoader &TheModuleLoader,
84 IdentifierInfoLookup *IILookup, bool OwnsHeaders,
85 TranslationUnitKind TUKind)
86 : PPOpts(PPOpts), Diags(&diags), LangOpts(opts),
87 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
88 ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
89 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
90 // As the language options may have not been loaded yet (when
91 // deserializing an ASTUnit), adding keywords to the identifier table is
92 // deferred to Preprocessor::Initialize().
93 Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())),
94 TUKind(TUKind), SkipMainFilePreamble(0, true),
95 CurSubmoduleState(&NullSubmoduleState) {
96 OwnsHeaderSearch = OwnsHeaders;
97
98 // Default to discarding comments.
99 KeepComments = false;
100 KeepMacroComments = false;
101 SuppressIncludeNotFoundError = false;
102
103 // Macro expansion is enabled.
104 DisableMacroExpansion = false;
105 MacroExpansionInDirectivesOverride = false;
106 InMacroArgs = false;
107 ArgMacro = nullptr;
108 InMacroArgPreExpansion = false;
109 NumCachedTokenLexers = 0;
110 PragmasEnabled = true;
111 ParsingIfOrElifDirective = false;
112 PreprocessedOutput = false;
113
114 // We haven't read anything from the external source.
115 ReadMacrosFromExternalSource = false;
116
117 BuiltinInfo = std::make_unique<Builtin::Context>();
118
119 // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
120 // a macro. They get unpoisoned where it is allowed.
121 (Ident__VA_ARGS__ = getIdentifierInfo(Name: "__VA_ARGS__"))->setIsPoisoned();
122 SetPoisonReason(II: Ident__VA_ARGS__,diag::DiagID: ext_pp_bad_vaargs_use);
123 (Ident__VA_OPT__ = getIdentifierInfo(Name: "__VA_OPT__"))->setIsPoisoned();
124 SetPoisonReason(II: Ident__VA_OPT__,diag::DiagID: ext_pp_bad_vaopt_use);
125
126 // Initialize the pragma handlers.
127 RegisterBuiltinPragmas();
128
129 // Initialize builtin macros like __LINE__ and friends.
130 RegisterBuiltinMacros();
131
132 if(LangOpts.Borland) {
133 Ident__exception_info = getIdentifierInfo(Name: "_exception_info");
134 Ident___exception_info = getIdentifierInfo(Name: "__exception_info");
135 Ident_GetExceptionInfo = getIdentifierInfo(Name: "GetExceptionInformation");
136 Ident__exception_code = getIdentifierInfo(Name: "_exception_code");
137 Ident___exception_code = getIdentifierInfo(Name: "__exception_code");
138 Ident_GetExceptionCode = getIdentifierInfo(Name: "GetExceptionCode");
139 Ident__abnormal_termination = getIdentifierInfo(Name: "_abnormal_termination");
140 Ident___abnormal_termination = getIdentifierInfo(Name: "__abnormal_termination");
141 Ident_AbnormalTermination = getIdentifierInfo(Name: "AbnormalTermination");
142 } else {
143 Ident__exception_info = Ident__exception_code = nullptr;
144 Ident__abnormal_termination = Ident___exception_info = nullptr;
145 Ident___exception_code = Ident___abnormal_termination = nullptr;
146 Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
147 Ident_AbnormalTermination = nullptr;
148 }
149
150 // Default incremental processing to -fincremental-extensions, clients can
151 // override with `enableIncrementalProcessing` if desired.
152 IncrementalProcessing = LangOpts.IncrementalExtensions;
153
154 // If using a PCH where a #pragma hdrstop is expected, start skipping tokens.
155 if (usingPCHWithPragmaHdrStop())
156 SkippingUntilPragmaHdrStop = true;
157
158 // If using a PCH with a through header, start skipping tokens.
159 if (!this->PPOpts.PCHThroughHeader.empty() &&
160 !this->PPOpts.ImplicitPCHInclude.empty())
161 SkippingUntilPCHThroughHeader = true;
162
163 if (this->PPOpts.GeneratePreamble)
164 PreambleConditionalStack.startRecording();
165
166 MaxTokens = LangOpts.MaxTokens;
167}
168
169Preprocessor::~Preprocessor() {
170 assert(!isBacktrackEnabled() && "EnableBacktrack/Backtrack imbalance!");
171
172 IncludeMacroStack.clear();
173
174 // Free any cached macro expanders.
175 // This populates MacroArgCache, so all TokenLexers need to be destroyed
176 // before the code below that frees up the MacroArgCache list.
177 std::fill(first: TokenLexerCache, last: TokenLexerCache + NumCachedTokenLexers, value: nullptr);
178 CurTokenLexer.reset();
179
180 // Free any cached MacroArgs.
181 for (MacroArgs *ArgList = MacroArgCache; ArgList;)
182 ArgList = ArgList->deallocate();
183
184 // Delete the header search info, if we own it.
185 if (OwnsHeaderSearch)
186 delete &HeaderInfo;
187}
188
189void Preprocessor::Initialize(const TargetInfo &Target,
190 const TargetInfo *AuxTarget) {
191 assert((!this->Target || this->Target == &Target) &&
192 "Invalid override of target information");
193 this->Target = &Target;
194
195 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
196 "Invalid override of aux target information.");
197 this->AuxTarget = AuxTarget;
198
199 // Initialize information about built-ins.
200 BuiltinInfo->InitializeTarget(Target, AuxTarget);
201 HeaderInfo.setTarget(Target);
202
203 // Populate the identifier table with info about keywords for the current language.
204 Identifiers.AddKeywords(LangOpts);
205
206 // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo.
207 setTUFPEvalMethod(getTargetInfo().getFPEvalMethod());
208
209 if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine)
210 // Use setting from TargetInfo.
211 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: Target.getFPEvalMethod());
212 else
213 // Set initial value of __FLT_EVAL_METHOD__ from the command line.
214 setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: getLangOpts().getFPEvalMethod());
215}
216
217void Preprocessor::InitializeForModelFile() {
218 NumEnteredSourceFiles = 0;
219
220 // Reset pragmas
221 PragmaHandlersBackup = std::move(PragmaHandlers);
222 PragmaHandlers = std::make_unique<PragmaNamespace>(args: StringRef());
223 RegisterBuiltinPragmas();
224
225 // Reset PredefinesFileID
226 PredefinesFileID = FileID();
227}
228
229void Preprocessor::FinalizeForModelFile() {
230 NumEnteredSourceFiles = 1;
231
232 PragmaHandlers = std::move(PragmaHandlersBackup);
233}
234
235void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
236 llvm::errs() << tok::getTokenName(Kind: Tok.getKind());
237
238 if (!Tok.isAnnotation())
239 llvm::errs() << " '" << getSpelling(Tok) << "'";
240
241 if (!DumpFlags) return;
242
243 llvm::errs() << "\t";
244 if (Tok.isAtStartOfLine())
245 llvm::errs() << " [StartOfLine]";
246 if (Tok.hasLeadingSpace())
247 llvm::errs() << " [LeadingSpace]";
248 if (Tok.isExpandDisabled())
249 llvm::errs() << " [ExpandDisabled]";
250 if (Tok.needsCleaning()) {
251 const char *Start = SourceMgr.getCharacterData(SL: Tok.getLocation());
252 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
253 << "']";
254 }
255
256 llvm::errs() << "\tLoc=<";
257 DumpLocation(Loc: Tok.getLocation());
258 llvm::errs() << ">";
259}
260
261void Preprocessor::DumpLocation(SourceLocation Loc) const {
262 Loc.print(OS&: llvm::errs(), SM: SourceMgr);
263}
264
265void Preprocessor::DumpMacro(const MacroInfo &MI) const {
266 llvm::errs() << "MACRO: ";
267 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
268 DumpToken(Tok: MI.getReplacementToken(Tok: i));
269 llvm::errs() << " ";
270 }
271 llvm::errs() << "\n";
272}
273
274void Preprocessor::PrintStats() {
275 llvm::errs() << "\n*** Preprocessor Stats:\n";
276 llvm::errs() << NumDirectives << " directives found:\n";
277 llvm::errs() << " " << NumDefined << " #define.\n";
278 llvm::errs() << " " << NumUndefined << " #undef.\n";
279 llvm::errs() << " #include/#include_next/#import:\n";
280 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
281 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
282 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
283 llvm::errs() << " " << NumElse << " #else/#elif/#elifdef/#elifndef.\n";
284 llvm::errs() << " " << NumEndif << " #endif.\n";
285 llvm::errs() << " " << NumPragma << " #pragma.\n";
286 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
287
288 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
289 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
290 << NumFastMacroExpanded << " on the fast path.\n";
291 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
292 << " token paste (##) operations performed, "
293 << NumFastTokenPaste << " on the fast path.\n";
294
295 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
296
297 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
298 llvm::errs() << "\n Macro Expanded Tokens: "
299 << llvm::capacity_in_bytes(X: MacroExpandedTokens);
300 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
301 // FIXME: List information for all submodules.
302 llvm::errs() << "\n Macros: "
303 << llvm::capacity_in_bytes(X: CurSubmoduleState->Macros);
304 llvm::errs() << "\n #pragma push_macro Info: "
305 << llvm::capacity_in_bytes(X: PragmaPushMacroInfo);
306 llvm::errs() << "\n Poison Reasons: "
307 << llvm::capacity_in_bytes(X: PoisonReasons);
308 llvm::errs() << "\n Comment Handlers: "
309 << llvm::capacity_in_bytes(x: CommentHandlers) << "\n";
310}
311
312Preprocessor::macro_iterator
313Preprocessor::macro_begin(bool IncludeExternalMacros) const {
314 if (IncludeExternalMacros && ExternalSource &&
315 !ReadMacrosFromExternalSource) {
316 ReadMacrosFromExternalSource = true;
317 ExternalSource->ReadDefinedMacros();
318 }
319
320 // Make sure we cover all macros in visible modules.
321 for (const ModuleMacro &Macro : ModuleMacros)
322 CurSubmoduleState->Macros.try_emplace(Key: Macro.II);
323
324 return CurSubmoduleState->Macros.begin();
325}
326
327size_t Preprocessor::getTotalMemory() const {
328 return BP.getTotalMemory()
329 + llvm::capacity_in_bytes(X: MacroExpandedTokens)
330 + Predefines.capacity() /* Predefines buffer. */
331 // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
332 // and ModuleMacros.
333 + llvm::capacity_in_bytes(X: CurSubmoduleState->Macros)
334 + llvm::capacity_in_bytes(X: PragmaPushMacroInfo)
335 + llvm::capacity_in_bytes(X: PoisonReasons)
336 + llvm::capacity_in_bytes(x: CommentHandlers);
337}
338
339Preprocessor::macro_iterator
340Preprocessor::macro_end(bool IncludeExternalMacros) const {
341 if (IncludeExternalMacros && ExternalSource &&
342 !ReadMacrosFromExternalSource) {
343 ReadMacrosFromExternalSource = true;
344 ExternalSource->ReadDefinedMacros();
345 }
346
347 return CurSubmoduleState->Macros.end();
348}
349
350/// Compares macro tokens with a specified token value sequence.
351static bool MacroDefinitionEquals(const MacroInfo *MI,
352 ArrayRef<TokenValue> Tokens) {
353 return Tokens.size() == MI->getNumTokens() &&
354 std::equal(first1: Tokens.begin(), last1: Tokens.end(), first2: MI->tokens_begin());
355}
356
357StringRef Preprocessor::getLastMacroWithSpelling(
358 SourceLocation Loc,
359 ArrayRef<TokenValue> Tokens) const {
360 SourceLocation BestLocation;
361 StringRef BestSpelling;
362 for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
363 I != E; ++I) {
364 const MacroDirective::DefInfo
365 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
366 if (!Def || !Def.getMacroInfo())
367 continue;
368 if (!Def.getMacroInfo()->isObjectLike())
369 continue;
370 if (!MacroDefinitionEquals(MI: Def.getMacroInfo(), Tokens))
371 continue;
372 SourceLocation Location = Def.getLocation();
373 // Choose the macro defined latest.
374 if (BestLocation.isInvalid() ||
375 (Location.isValid() &&
376 SourceMgr.isBeforeInTranslationUnit(LHS: BestLocation, RHS: Location))) {
377 BestLocation = Location;
378 BestSpelling = I->first->getName();
379 }
380 }
381 return BestSpelling;
382}
383
384void Preprocessor::recomputeCurLexerKind() {
385 if (CurLexer)
386 CurLexerCallback = CurLexer->isDependencyDirectivesLexer()
387 ? CLK_DependencyDirectivesLexer
388 : CLK_Lexer;
389 else if (CurTokenLexer)
390 CurLexerCallback = CLK_TokenLexer;
391 else
392 CurLexerCallback = CLK_CachingLexer;
393}
394
395bool Preprocessor::SetCodeCompletionPoint(FileEntryRef File,
396 unsigned CompleteLine,
397 unsigned CompleteColumn) {
398 assert(CompleteLine && CompleteColumn && "Starts from 1:1");
399 assert(!CodeCompletionFile && "Already set");
400
401 // Load the actual file's contents.
402 std::optional<llvm::MemoryBufferRef> Buffer =
403 SourceMgr.getMemoryBufferForFileOrNone(File);
404 if (!Buffer)
405 return true;
406
407 // Find the byte position of the truncation point.
408 const char *Position = Buffer->getBufferStart();
409 for (unsigned Line = 1; Line < CompleteLine; ++Line) {
410 for (; *Position; ++Position) {
411 if (*Position != '\r' && *Position != '\n')
412 continue;
413
414 // Eat \r\n or \n\r as a single line.
415 if ((Position[1] == '\r' || Position[1] == '\n') &&
416 Position[0] != Position[1])
417 ++Position;
418 ++Position;
419 break;
420 }
421 }
422
423 Position += CompleteColumn - 1;
424
425 // If pointing inside the preamble, adjust the position at the beginning of
426 // the file after the preamble.
427 if (SkipMainFilePreamble.first &&
428 SourceMgr.getFileEntryForID(FID: SourceMgr.getMainFileID()) == File) {
429 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
430 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
431 }
432
433 if (Position > Buffer->getBufferEnd())
434 Position = Buffer->getBufferEnd();
435
436 CodeCompletionFile = File;
437 CodeCompletionOffset = Position - Buffer->getBufferStart();
438
439 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
440 Size: Buffer->getBufferSize() + 1, BufferName: Buffer->getBufferIdentifier());
441 char *NewBuf = NewBuffer->getBufferStart();
442 char *NewPos = std::copy(first: Buffer->getBufferStart(), last: Position, result: NewBuf);
443 *NewPos = '\0';
444 std::copy(first: Position, last: Buffer->getBufferEnd(), result: NewPos+1);
445 SourceMgr.overrideFileContents(SourceFile: File, Buffer: std::move(NewBuffer));
446
447 return false;
448}
449
450void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir,
451 bool IsAngled) {
452 setCodeCompletionReached();
453 if (CodeComplete)
454 CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled);
455}
456
457void Preprocessor::CodeCompleteNaturalLanguage() {
458 setCodeCompletionReached();
459 if (CodeComplete)
460 CodeComplete->CodeCompleteNaturalLanguage();
461}
462
463/// getSpelling - This method is used to get the spelling of a token into a
464/// SmallVector. Note that the returned StringRef may not point to the
465/// supplied buffer if a copy can be avoided.
466StringRef Preprocessor::getSpelling(const Token &Tok,
467 SmallVectorImpl<char> &Buffer,
468 bool *Invalid) const {
469 // NOTE: this has to be checked *before* testing for an IdentifierInfo.
470 if (Tok.isNot(K: tok::raw_identifier) && !Tok.hasUCN()) {
471 // Try the fast path.
472 if (const IdentifierInfo *II = Tok.getIdentifierInfo())
473 return II->getName();
474 }
475
476 // Resize the buffer if we need to copy into it.
477 if (Tok.needsCleaning())
478 Buffer.resize(N: Tok.getLength());
479
480 const char *Ptr = Buffer.data();
481 unsigned Len = getSpelling(Tok, Buffer&: Ptr, Invalid);
482 return StringRef(Ptr, Len);
483}
484
485/// CreateString - Plop the specified string into a scratch buffer and return a
486/// location for it. If specified, the source location provides a source
487/// location for the token.
488void Preprocessor::CreateString(StringRef Str, Token &Tok,
489 SourceLocation ExpansionLocStart,
490 SourceLocation ExpansionLocEnd) {
491 Tok.setLength(Str.size());
492
493 const char *DestPtr;
494 SourceLocation Loc = ScratchBuf->getToken(Buf: Str.data(), Len: Str.size(), DestPtr);
495
496 if (ExpansionLocStart.isValid())
497 Loc = SourceMgr.createExpansionLoc(SpellingLoc: Loc, ExpansionLocStart,
498 ExpansionLocEnd, Length: Str.size());
499 Tok.setLocation(Loc);
500
501 // If this is a raw identifier or a literal token, set the pointer data.
502 if (Tok.is(K: tok::raw_identifier))
503 Tok.setRawIdentifierData(DestPtr);
504 else if (Tok.isLiteral())
505 Tok.setLiteralData(DestPtr);
506}
507
508SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) {
509 auto &SM = getSourceManager();
510 SourceLocation SpellingLoc = SM.getSpellingLoc(Loc);
511 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc: SpellingLoc);
512 bool Invalid = false;
513 StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid);
514 if (Invalid)
515 return SourceLocation();
516
517 // FIXME: We could consider re-using spelling for tokens we see repeatedly.
518 const char *DestPtr;
519 SourceLocation Spelling =
520 ScratchBuf->getToken(Buf: Buffer.data() + LocInfo.second, Len: Length, DestPtr);
521 return SM.createTokenSplitLoc(SpellingLoc: Spelling, TokenStart: Loc, TokenEnd: Loc.getLocWithOffset(Offset: Length));
522}
523
524Module *Preprocessor::getCurrentModule() {
525 if (!getLangOpts().isCompilingModule())
526 return nullptr;
527
528 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().CurrentModule);
529}
530
531Module *Preprocessor::getCurrentModuleImplementation() {
532 if (!getLangOpts().isCompilingModuleImplementation())
533 return nullptr;
534
535 return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().ModuleName);
536}
537
538//===----------------------------------------------------------------------===//
539// Preprocessor Initialization Methods
540//===----------------------------------------------------------------------===//
541
542/// EnterMainSourceFile - Enter the specified FileID as the main source file,
543/// which implicitly adds the builtin defines etc.
544void Preprocessor::EnterMainSourceFile() {
545 // We do not allow the preprocessor to reenter the main file. Doing so will
546 // cause FileID's to accumulate information from both runs (e.g. #line
547 // information) and predefined macros aren't guaranteed to be set properly.
548 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
549 FileID MainFileID = SourceMgr.getMainFileID();
550
551 // If MainFileID is loaded it means we loaded an AST file, no need to enter
552 // a main file.
553 if (!SourceMgr.isLoadedFileID(FID: MainFileID)) {
554 // Enter the main file source buffer.
555 EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation());
556
557 // If we've been asked to skip bytes in the main file (e.g., as part of a
558 // precompiled preamble), do so now.
559 if (SkipMainFilePreamble.first > 0)
560 CurLexer->SetByteOffset(Offset: SkipMainFilePreamble.first,
561 StartOfLine: SkipMainFilePreamble.second);
562
563 // Tell the header info that the main file was entered. If the file is later
564 // #imported, it won't be re-entered.
565 if (OptionalFileEntryRef FE = SourceMgr.getFileEntryRefForID(FID: MainFileID))
566 markIncluded(File: *FE);
567 }
568
569 // Preprocess Predefines to populate the initial preprocessor state.
570 std::unique_ptr<llvm::MemoryBuffer> SB =
571 llvm::MemoryBuffer::getMemBufferCopy(InputData: Predefines, BufferName: "<built-in>");
572 assert(SB && "Cannot create predefined source buffer");
573 FileID FID = SourceMgr.createFileID(Buffer: std::move(SB));
574 assert(FID.isValid() && "Could not create FileID for predefines?");
575 setPredefinesFileID(FID);
576
577 // Start parsing the predefines.
578 EnterSourceFile(FID, Dir: nullptr, Loc: SourceLocation());
579
580 if (!PPOpts.PCHThroughHeader.empty()) {
581 // Lookup and save the FileID for the through header. If it isn't found
582 // in the search path, it's a fatal error.
583 OptionalFileEntryRef File = LookupFile(
584 FilenameLoc: SourceLocation(), Filename: PPOpts.PCHThroughHeader,
585 /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr,
586 /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr,
587 /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr,
588 /*IsFrameworkFound=*/nullptr);
589 if (!File) {
590 Diag(SourceLocation(), diag::err_pp_through_header_not_found)
591 << PPOpts.PCHThroughHeader;
592 return;
593 }
594 setPCHThroughHeaderFileID(
595 SourceMgr.createFileID(SourceFile: *File, IncludePos: SourceLocation(), FileCharacter: SrcMgr::C_User));
596 }
597
598 // Skip tokens from the Predefines and if needed the main file.
599 if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) ||
600 (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop))
601 SkipTokensWhileUsingPCH();
602}
603
604void Preprocessor::setPCHThroughHeaderFileID(FileID FID) {
605 assert(PCHThroughHeaderFileID.isInvalid() &&
606 "PCHThroughHeaderFileID already set!");
607 PCHThroughHeaderFileID = FID;
608}
609
610bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) {
611 assert(PCHThroughHeaderFileID.isValid() &&
612 "Invalid PCH through header FileID");
613 return FE == SourceMgr.getFileEntryForID(FID: PCHThroughHeaderFileID);
614}
615
616bool Preprocessor::creatingPCHWithThroughHeader() {
617 return TUKind == TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
618 PCHThroughHeaderFileID.isValid();
619}
620
621bool Preprocessor::usingPCHWithThroughHeader() {
622 return TUKind != TU_Prefix && !PPOpts.PCHThroughHeader.empty() &&
623 PCHThroughHeaderFileID.isValid();
624}
625
626bool Preprocessor::creatingPCHWithPragmaHdrStop() {
627 return TUKind == TU_Prefix && PPOpts.PCHWithHdrStop;
628}
629
630bool Preprocessor::usingPCHWithPragmaHdrStop() {
631 return TUKind != TU_Prefix && PPOpts.PCHWithHdrStop;
632}
633
634/// Skip tokens until after the #include of the through header or
635/// until after a #pragma hdrstop is seen. Tokens in the predefines file
636/// and the main file may be skipped. If the end of the predefines file
637/// is reached, skipping continues into the main file. If the end of the
638/// main file is reached, it's a fatal error.
639void Preprocessor::SkipTokensWhileUsingPCH() {
640 bool ReachedMainFileEOF = false;
641 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
642 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
643 Token Tok;
644 while (true) {
645 bool InPredefines =
646 (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
647 CurLexerCallback(*this, Tok);
648 if (Tok.is(K: tok::eof) && !InPredefines) {
649 ReachedMainFileEOF = true;
650 break;
651 }
652 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
653 break;
654 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
655 break;
656 }
657 if (ReachedMainFileEOF) {
658 if (UsingPCHThroughHeader)
659 Diag(SourceLocation(), diag::err_pp_through_header_not_seen)
660 << PPOpts.PCHThroughHeader << 1;
661 else if (!PPOpts.PCHWithHdrStopCreate)
662 Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen);
663 }
664}
665
666void Preprocessor::replayPreambleConditionalStack() {
667 // Restore the conditional stack from the preamble, if there is one.
668 if (PreambleConditionalStack.isReplaying()) {
669 assert(CurPPLexer &&
670 "CurPPLexer is null when calling replayPreambleConditionalStack.");
671 CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
672 PreambleConditionalStack.doneReplaying();
673 if (PreambleConditionalStack.reachedEOFWhileSkipping())
674 SkipExcludedConditionalBlock(
675 HashTokenLoc: PreambleConditionalStack.SkipInfo->HashTokenLoc,
676 IfTokenLoc: PreambleConditionalStack.SkipInfo->IfTokenLoc,
677 FoundNonSkipPortion: PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
678 FoundElse: PreambleConditionalStack.SkipInfo->FoundElse,
679 ElseLoc: PreambleConditionalStack.SkipInfo->ElseLoc);
680 }
681}
682
683void Preprocessor::EndSourceFile() {
684 // Notify the client that we reached the end of the source file.
685 if (Callbacks)
686 Callbacks->EndOfMainFile();
687}
688
689//===----------------------------------------------------------------------===//
690// Lexer Event Handling.
691//===----------------------------------------------------------------------===//
692
693/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
694/// identifier information for the token and install it into the token,
695/// updating the token kind accordingly.
696IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const {
697 assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
698
699 // Look up this token, see if it is a macro, or if it is a language keyword.
700 IdentifierInfo *II;
701 if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
702 // No cleaning needed, just use the characters from the lexed buffer.
703 II = getIdentifierInfo(Name: Identifier.getRawIdentifier());
704 } else {
705 // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
706 SmallString<64> IdentifierBuffer;
707 StringRef CleanedStr = getSpelling(Tok: Identifier, Buffer&: IdentifierBuffer);
708
709 if (Identifier.hasUCN()) {
710 SmallString<64> UCNIdentifierBuffer;
711 expandUCNs(Buf&: UCNIdentifierBuffer, Input: CleanedStr);
712 II = getIdentifierInfo(Name: UCNIdentifierBuffer);
713 } else {
714 II = getIdentifierInfo(Name: CleanedStr);
715 }
716 }
717
718 // Update the token info (identifier info and appropriate token kind).
719 // FIXME: the raw_identifier may contain leading whitespace which is removed
720 // from the cleaned identifier token. The SourceLocation should be updated to
721 // refer to the non-whitespace character. For instance, the text "\\\nB" (a
722 // line continuation before 'B') is parsed as a single tok::raw_identifier and
723 // is cleaned to tok::identifier "B". After cleaning the token's length is
724 // still 3 and the SourceLocation refers to the location of the backslash.
725 Identifier.setIdentifierInfo(II);
726 Identifier.setKind(II->getTokenID());
727
728 return II;
729}
730
731void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
732 PoisonReasons[II] = DiagID;
733}
734
735void Preprocessor::PoisonSEHIdentifiers(bool Poison) {
736 assert(Ident__exception_code && Ident__exception_info);
737 assert(Ident___exception_code && Ident___exception_info);
738 Ident__exception_code->setIsPoisoned(Poison);
739 Ident___exception_code->setIsPoisoned(Poison);
740 Ident_GetExceptionCode->setIsPoisoned(Poison);
741 Ident__exception_info->setIsPoisoned(Poison);
742 Ident___exception_info->setIsPoisoned(Poison);
743 Ident_GetExceptionInfo->setIsPoisoned(Poison);
744 Ident__abnormal_termination->setIsPoisoned(Poison);
745 Ident___abnormal_termination->setIsPoisoned(Poison);
746 Ident_AbnormalTermination->setIsPoisoned(Poison);
747}
748
749void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) {
750 assert(Identifier.getIdentifierInfo() &&
751 "Can't handle identifiers without identifier info!");
752 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
753 PoisonReasons.find(Val: Identifier.getIdentifierInfo());
754 if(it == PoisonReasons.end())
755 Diag(Identifier, diag::err_pp_used_poisoned_id);
756 else
757 Diag(Tok: Identifier,DiagID: it->second) << Identifier.getIdentifierInfo();
758}
759
760void Preprocessor::updateOutOfDateIdentifier(const IdentifierInfo &II) const {
761 assert(II.isOutOfDate() && "not out of date");
762 assert(getExternalSource() &&
763 "getExternalSource() should not return nullptr");
764 getExternalSource()->updateOutOfDateIdentifier(II);
765}
766
767/// HandleIdentifier - This callback is invoked when the lexer reads an
768/// identifier. This callback looks up the identifier in the map and/or
769/// potentially macro expands it or turns it into a named token (like 'for').
770///
771/// Note that callers of this method are guarded by checking the
772/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
773/// IdentifierInfo methods that compute these properties will need to change to
774/// match.
775bool Preprocessor::HandleIdentifier(Token &Identifier) {
776 assert(Identifier.getIdentifierInfo() &&
777 "Can't handle identifiers without identifier info!");
778
779 IdentifierInfo &II = *Identifier.getIdentifierInfo();
780
781 // If the information about this identifier is out of date, update it from
782 // the external source.
783 // We have to treat __VA_ARGS__ in a special way, since it gets
784 // serialized with isPoisoned = true, but our preprocessor may have
785 // unpoisoned it if we're defining a C99 macro.
786 if (II.isOutOfDate()) {
787 bool CurrentIsPoisoned = false;
788 const bool IsSpecialVariadicMacro =
789 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
790 if (IsSpecialVariadicMacro)
791 CurrentIsPoisoned = II.isPoisoned();
792
793 updateOutOfDateIdentifier(II);
794 Identifier.setKind(II.getTokenID());
795
796 if (IsSpecialVariadicMacro)
797 II.setIsPoisoned(CurrentIsPoisoned);
798 }
799
800 // If this identifier was poisoned, and if it was not produced from a macro
801 // expansion, emit an error.
802 if (II.isPoisoned() && CurPPLexer) {
803 HandlePoisonedIdentifier(Identifier);
804 }
805
806 // If this is a macro to be expanded, do it.
807 if (const MacroDefinition MD = getMacroDefinition(II: &II)) {
808 const auto *MI = MD.getMacroInfo();
809 assert(MI && "macro definition with no macro info?");
810 if (!DisableMacroExpansion) {
811 if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
812 // C99 6.10.3p10: If the preprocessing token immediately after the
813 // macro name isn't a '(', this macro should not be expanded.
814 if (!MI->isFunctionLike() || isNextPPTokenLParen())
815 return HandleMacroExpandedIdentifier(Identifier, MD);
816 } else {
817 // C99 6.10.3.4p2 says that a disabled macro may never again be
818 // expanded, even if it's in a context where it could be expanded in the
819 // future.
820 Identifier.setFlag(Token::DisableExpand);
821 if (MI->isObjectLike() || isNextPPTokenLParen())
822 Diag(Tok: Identifier, diag::DiagID: pp_disabled_macro_expansion);
823 }
824 }
825 }
826
827 // If this identifier is a keyword in a newer Standard or proposed Standard,
828 // produce a warning. Don't warn if we're not considering macro expansion,
829 // since this identifier might be the name of a macro.
830 // FIXME: This warning is disabled in cases where it shouldn't be, like
831 // "#define constexpr constexpr", "int constexpr;"
832 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
833 Diag(Tok: Identifier, DiagID: getIdentifierTable().getFutureCompatDiagKind(II, LangOpts: getLangOpts()))
834 << II.getName();
835 // Don't diagnose this keyword again in this translation unit.
836 II.setIsFutureCompatKeyword(false);
837 }
838
839 // If this identifier would be a keyword in C++, diagnose as a compatibility
840 // issue.
841 if (II.IsKeywordInCPlusPlus() && !DisableMacroExpansion)
842 Diag(Identifier, diag::warn_pp_identifier_is_cpp_keyword) << &II;
843
844 // If this is an extension token, diagnose its use.
845 // We avoid diagnosing tokens that originate from macro definitions.
846 // FIXME: This warning is disabled in cases where it shouldn't be,
847 // like "#define TY typeof", "TY(1) x".
848 if (II.isExtensionToken() && !DisableMacroExpansion)
849 Diag(Identifier, diag::ext_token_used);
850
851 // If this is the 'import' contextual keyword following an '@', note
852 // that the next token indicates a module name.
853 //
854 // Note that we do not treat 'import' as a contextual
855 // keyword when we're in a caching lexer, because caching lexers only get
856 // used in contexts where import declarations are disallowed.
857 //
858 // Likewise if this is the standard C++ import keyword.
859 if (((LastTokenWasAt && II.isModulesImport()) ||
860 Identifier.is(K: tok::kw_import)) &&
861 !InMacroArgs && !DisableMacroExpansion &&
862 (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
863 CurLexerCallback != CLK_CachingLexer) {
864 ModuleImportLoc = Identifier.getLocation();
865 NamedModuleImportPath.clear();
866 IsAtImport = true;
867 ModuleImportExpectsIdentifier = true;
868 CurLexerCallback = CLK_LexAfterModuleImport;
869 }
870 return true;
871}
872
873void Preprocessor::Lex(Token &Result) {
874 ++LexLevel;
875
876 // We loop here until a lex function returns a token; this avoids recursion.
877 while (!CurLexerCallback(*this, Result))
878 ;
879
880 if (Result.is(K: tok::unknown) && TheModuleLoader.HadFatalFailure)
881 return;
882
883 if (Result.is(K: tok::code_completion) && Result.getIdentifierInfo()) {
884 // Remember the identifier before code completion token.
885 setCodeCompletionIdentifierInfo(Result.getIdentifierInfo());
886 setCodeCompletionTokenRange(Start: Result.getLocation(), End: Result.getEndLoc());
887 // Set IdenfitierInfo to null to avoid confusing code that handles both
888 // identifiers and completion tokens.
889 Result.setIdentifierInfo(nullptr);
890 }
891
892 // Update StdCXXImportSeqState to track our position within a C++20 import-seq
893 // if this token is being produced as a result of phase 4 of translation.
894 // Update TrackGMFState to decide if we are currently in a Global Module
895 // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state
896 // depends on the prevailing StdCXXImportSeq state in two cases.
897 if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
898 !Result.getFlag(Flag: Token::IsReinjected)) {
899 switch (Result.getKind()) {
900 case tok::l_paren: case tok::l_square: case tok::l_brace:
901 StdCXXImportSeqState.handleOpenBracket();
902 break;
903 case tok::r_paren: case tok::r_square:
904 StdCXXImportSeqState.handleCloseBracket();
905 break;
906 case tok::r_brace:
907 StdCXXImportSeqState.handleCloseBrace();
908 break;
909#define PRAGMA_ANNOTATION(X) case tok::annot_##X:
910// For `#pragma ...` mimic ';'.
911#include "clang/Basic/TokenKinds.def"
912#undef PRAGMA_ANNOTATION
913 // This token is injected to represent the translation of '#include "a.h"'
914 // into "import a.h;". Mimic the notional ';'.
915 case tok::annot_module_include:
916 case tok::semi:
917 TrackGMFState.handleSemi();
918 StdCXXImportSeqState.handleSemi();
919 ModuleDeclState.handleSemi();
920 break;
921 case tok::header_name:
922 case tok::annot_header_unit:
923 StdCXXImportSeqState.handleHeaderName();
924 break;
925 case tok::kw_export:
926 TrackGMFState.handleExport();
927 StdCXXImportSeqState.handleExport();
928 ModuleDeclState.handleExport();
929 break;
930 case tok::colon:
931 ModuleDeclState.handleColon();
932 break;
933 case tok::period:
934 ModuleDeclState.handlePeriod();
935 break;
936 case tok::identifier:
937 // Check "import" and "module" when there is no open bracket. The two
938 // identifiers are not meaningful with open brackets.
939 if (StdCXXImportSeqState.atTopLevel()) {
940 if (Result.getIdentifierInfo()->isModulesImport()) {
941 TrackGMFState.handleImport(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
942 StdCXXImportSeqState.handleImport();
943 if (StdCXXImportSeqState.afterImportSeq()) {
944 ModuleImportLoc = Result.getLocation();
945 NamedModuleImportPath.clear();
946 IsAtImport = false;
947 ModuleImportExpectsIdentifier = true;
948 CurLexerCallback = CLK_LexAfterModuleImport;
949 }
950 break;
951 } else if (Result.getIdentifierInfo() == getIdentifierInfo(Name: "module")) {
952 TrackGMFState.handleModule(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq());
953 ModuleDeclState.handleModule();
954 break;
955 }
956 }
957 ModuleDeclState.handleIdentifier(Identifier: Result.getIdentifierInfo());
958 if (ModuleDeclState.isModuleCandidate())
959 break;
960 [[fallthrough]];
961 default:
962 TrackGMFState.handleMisc();
963 StdCXXImportSeqState.handleMisc();
964 ModuleDeclState.handleMisc();
965 break;
966 }
967 }
968
969 if (CurLexer && ++CheckPointCounter == CheckPointStepSize) {
970 CheckPoints[CurLexer->getFileID()].push_back(CurLexer->BufferPtr);
971 CheckPointCounter = 0;
972 }
973
974 LastTokenWasAt = Result.is(K: tok::at);
975 --LexLevel;
976
977 if ((LexLevel == 0 || PreprocessToken) &&
978 !Result.getFlag(Flag: Token::IsReinjected)) {
979 if (LexLevel == 0)
980 ++TokenCount;
981 if (OnToken)
982 OnToken(Result);
983 }
984}
985
986void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) {
987 while (1) {
988 Token Tok;
989 Lex(Result&: Tok);
990 if (Tok.isOneOf(K1: tok::unknown, Ks: tok::eof, Ks: tok::eod,
991 Ks: tok::annot_repl_input_end))
992 break;
993 if (Tokens != nullptr)
994 Tokens->push_back(x: Tok);
995 }
996}
997
998/// Lex a header-name token (including one formed from header-name-tokens if
999/// \p AllowMacroExpansion is \c true).
1000///
1001/// \param FilenameTok Filled in with the next token. On success, this will
1002/// be either a header_name token. On failure, it will be whatever other
1003/// token was found instead.
1004/// \param AllowMacroExpansion If \c true, allow the header name to be formed
1005/// by macro expansion (concatenating tokens as necessary if the first
1006/// token is a '<').
1007/// \return \c true if we reached EOD or EOF while looking for a > token in
1008/// a concatenated header name and diagnosed it. \c false otherwise.
1009bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) {
1010 // Lex using header-name tokenization rules if tokens are being lexed from
1011 // a file. Just grab a token normally if we're in a macro expansion.
1012 if (CurPPLexer)
1013 CurPPLexer->LexIncludeFilename(FilenameTok);
1014 else
1015 Lex(Result&: FilenameTok);
1016
1017 // This could be a <foo/bar.h> file coming from a macro expansion. In this
1018 // case, glue the tokens together into an angle_string_literal token.
1019 SmallString<128> FilenameBuffer;
1020 if (FilenameTok.is(K: tok::less) && AllowMacroExpansion) {
1021 bool StartOfLine = FilenameTok.isAtStartOfLine();
1022 bool LeadingSpace = FilenameTok.hasLeadingSpace();
1023 bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro();
1024
1025 SourceLocation Start = FilenameTok.getLocation();
1026 SourceLocation End;
1027 FilenameBuffer.push_back(Elt: '<');
1028
1029 // Consume tokens until we find a '>'.
1030 // FIXME: A header-name could be formed starting or ending with an
1031 // alternative token. It's not clear whether that's ill-formed in all
1032 // cases.
1033 while (FilenameTok.isNot(K: tok::greater)) {
1034 Lex(Result&: FilenameTok);
1035 if (FilenameTok.isOneOf(K1: tok::eod, K2: tok::eof)) {
1036 Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
1037 Diag(Start, diag::note_matching) << tok::less;
1038 return true;
1039 }
1040
1041 End = FilenameTok.getLocation();
1042
1043 // FIXME: Provide code completion for #includes.
1044 if (FilenameTok.is(K: tok::code_completion)) {
1045 setCodeCompletionReached();
1046 Lex(Result&: FilenameTok);
1047 continue;
1048 }
1049
1050 // Append the spelling of this token to the buffer. If there was a space
1051 // before it, add it now.
1052 if (FilenameTok.hasLeadingSpace())
1053 FilenameBuffer.push_back(Elt: ' ');
1054
1055 // Get the spelling of the token, directly into FilenameBuffer if
1056 // possible.
1057 size_t PreAppendSize = FilenameBuffer.size();
1058 FilenameBuffer.resize(N: PreAppendSize + FilenameTok.getLength());
1059
1060 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1061 unsigned ActualLen = getSpelling(Tok: FilenameTok, Buffer&: BufPtr);
1062
1063 // If the token was spelled somewhere else, copy it into FilenameBuffer.
1064 if (BufPtr != &FilenameBuffer[PreAppendSize])
1065 memcpy(dest: &FilenameBuffer[PreAppendSize], src: BufPtr, n: ActualLen);
1066
1067 // Resize FilenameBuffer to the correct size.
1068 if (FilenameTok.getLength() != ActualLen)
1069 FilenameBuffer.resize(N: PreAppendSize + ActualLen);
1070 }
1071
1072 FilenameTok.startToken();
1073 FilenameTok.setKind(tok::header_name);
1074 FilenameTok.setFlagValue(Flag: Token::StartOfLine, Val: StartOfLine);
1075 FilenameTok.setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace);
1076 FilenameTok.setFlagValue(Flag: Token::LeadingEmptyMacro, Val: LeadingEmptyMacro);
1077 CreateString(Str: FilenameBuffer, Tok&: FilenameTok, ExpansionLocStart: Start, ExpansionLocEnd: End);
1078 } else if (FilenameTok.is(K: tok::string_literal) && AllowMacroExpansion) {
1079 // Convert a string-literal token of the form " h-char-sequence "
1080 // (produced by macro expansion) into a header-name token.
1081 //
1082 // The rules for header-names don't quite match the rules for
1083 // string-literals, but all the places where they differ result in
1084 // undefined behavior, so we can and do treat them the same.
1085 //
1086 // A string-literal with a prefix or suffix is not translated into a
1087 // header-name. This could theoretically be observable via the C++20
1088 // context-sensitive header-name formation rules.
1089 StringRef Str = getSpelling(Tok: FilenameTok, Buffer&: FilenameBuffer);
1090 if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"')
1091 FilenameTok.setKind(tok::header_name);
1092 }
1093
1094 return false;
1095}
1096
1097/// Collect the tokens of a C++20 pp-import-suffix.
1098void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) {
1099 // FIXME: For error recovery, consider recognizing attribute syntax here
1100 // and terminating / diagnosing a missing semicolon if we find anything
1101 // else? (Can we leave that to the parser?)
1102 unsigned BracketDepth = 0;
1103 while (true) {
1104 Toks.emplace_back();
1105 Lex(Result&: Toks.back());
1106
1107 switch (Toks.back().getKind()) {
1108 case tok::l_paren: case tok::l_square: case tok::l_brace:
1109 ++BracketDepth;
1110 break;
1111
1112 case tok::r_paren: case tok::r_square: case tok::r_brace:
1113 if (BracketDepth == 0)
1114 return;
1115 --BracketDepth;
1116 break;
1117
1118 case tok::semi:
1119 if (BracketDepth == 0)
1120 return;
1121 break;
1122
1123 case tok::eof:
1124 return;
1125
1126 default:
1127 break;
1128 }
1129 }
1130}
1131
1132
1133/// Lex a token following the 'import' contextual keyword.
1134///
1135/// pp-import: [C++20]
1136/// import header-name pp-import-suffix[opt] ;
1137/// import header-name-tokens pp-import-suffix[opt] ;
1138/// [ObjC] @ import module-name ;
1139/// [Clang] import module-name ;
1140///
1141/// header-name-tokens:
1142/// string-literal
1143/// < [any sequence of preprocessing-tokens other than >] >
1144///
1145/// module-name:
1146/// module-name-qualifier[opt] identifier
1147///
1148/// module-name-qualifier
1149/// module-name-qualifier[opt] identifier .
1150///
1151/// We respond to a pp-import by importing macros from the named module.
1152bool Preprocessor::LexAfterModuleImport(Token &Result) {
1153 // Figure out what kind of lexer we actually have.
1154 recomputeCurLexerKind();
1155
1156 // Lex the next token. The header-name lexing rules are used at the start of
1157 // a pp-import.
1158 //
1159 // For now, we only support header-name imports in C++20 mode.
1160 // FIXME: Should we allow this in all language modes that support an import
1161 // declaration as an extension?
1162 if (NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules) {
1163 if (LexHeaderName(FilenameTok&: Result))
1164 return true;
1165
1166 if (Result.is(K: tok::colon) && ModuleDeclState.isNamedModule()) {
1167 std::string Name = ModuleDeclState.getPrimaryName().str();
1168 Name += ":";
1169 NamedModuleImportPath.emplace_back(Args: Result.getLocation(),
1170 Args: getIdentifierInfo(Name));
1171 CurLexerCallback = CLK_LexAfterModuleImport;
1172 return true;
1173 }
1174 } else {
1175 Lex(Result);
1176 }
1177
1178 // Allocate a holding buffer for a sequence of tokens and introduce it into
1179 // the token stream.
1180 auto EnterTokens = [this](ArrayRef<Token> Toks) {
1181 auto ToksCopy = std::make_unique<Token[]>(num: Toks.size());
1182 std::copy(first: Toks.begin(), last: Toks.end(), result: ToksCopy.get());
1183 EnterTokenStream(Toks: std::move(ToksCopy), NumToks: Toks.size(),
1184 /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
1185 };
1186
1187 bool ImportingHeader = Result.is(K: tok::header_name);
1188 // Check for a header-name.
1189 SmallVector<Token, 32> Suffix;
1190 if (ImportingHeader) {
1191 // Enter the header-name token into the token stream; a Lex action cannot
1192 // both return a token and cache tokens (doing so would corrupt the token
1193 // cache if the call to Lex comes from CachingLex / PeekAhead).
1194 Suffix.push_back(Elt: Result);
1195
1196 // Consume the pp-import-suffix and expand any macros in it now. We'll add
1197 // it back into the token stream later.
1198 CollectPpImportSuffix(Toks&: Suffix);
1199 if (Suffix.back().isNot(K: tok::semi)) {
1200 // This is not a pp-import after all.
1201 EnterTokens(Suffix);
1202 return false;
1203 }
1204
1205 // C++2a [cpp.module]p1:
1206 // The ';' preprocessing-token terminating a pp-import shall not have
1207 // been produced by macro replacement.
1208 SourceLocation SemiLoc = Suffix.back().getLocation();
1209 if (SemiLoc.isMacroID())
1210 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1211
1212 // Reconstitute the import token.
1213 Token ImportTok;
1214 ImportTok.startToken();
1215 ImportTok.setKind(tok::kw_import);
1216 ImportTok.setLocation(ModuleImportLoc);
1217 ImportTok.setIdentifierInfo(getIdentifierInfo(Name: "import"));
1218 ImportTok.setLength(6);
1219
1220 auto Action = HandleHeaderIncludeOrImport(
1221 /*HashLoc*/ SourceLocation(), IncludeTok&: ImportTok, FilenameTok&: Suffix.front(), EndLoc: SemiLoc);
1222 switch (Action.Kind) {
1223 case ImportAction::None:
1224 break;
1225
1226 case ImportAction::ModuleBegin:
1227 // Let the parser know we're textually entering the module.
1228 Suffix.emplace_back();
1229 Suffix.back().startToken();
1230 Suffix.back().setKind(tok::annot_module_begin);
1231 Suffix.back().setLocation(SemiLoc);
1232 Suffix.back().setAnnotationEndLoc(SemiLoc);
1233 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1234 [[fallthrough]];
1235
1236 case ImportAction::ModuleImport:
1237 case ImportAction::HeaderUnitImport:
1238 case ImportAction::SkippedModuleImport:
1239 // We chose to import (or textually enter) the file. Convert the
1240 // header-name token into a header unit annotation token.
1241 Suffix[0].setKind(tok::annot_header_unit);
1242 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1243 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1244 // FIXME: Call the moduleImport callback?
1245 break;
1246 case ImportAction::Failure:
1247 assert(TheModuleLoader.HadFatalFailure &&
1248 "This should be an early exit only to a fatal error");
1249 Result.setKind(tok::eof);
1250 CurLexer->cutOffLexing();
1251 EnterTokens(Suffix);
1252 return true;
1253 }
1254
1255 EnterTokens(Suffix);
1256 return false;
1257 }
1258
1259 // The token sequence
1260 //
1261 // import identifier (. identifier)*
1262 //
1263 // indicates a module import directive. We already saw the 'import'
1264 // contextual keyword, so now we're looking for the identifiers.
1265 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
1266 // We expected to see an identifier here, and we did; continue handling
1267 // identifiers.
1268 NamedModuleImportPath.emplace_back(Args: Result.getLocation(),
1269 Args: Result.getIdentifierInfo());
1270 ModuleImportExpectsIdentifier = false;
1271 CurLexerCallback = CLK_LexAfterModuleImport;
1272 return true;
1273 }
1274
1275 // If we're expecting a '.' or a ';', and we got a '.', then wait until we
1276 // see the next identifier. (We can also see a '[[' that begins an
1277 // attribute-specifier-seq here under the Standard C++ Modules.)
1278 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
1279 ModuleImportExpectsIdentifier = true;
1280 CurLexerCallback = CLK_LexAfterModuleImport;
1281 return true;
1282 }
1283
1284 // If we didn't recognize a module name at all, this is not a (valid) import.
1285 if (NamedModuleImportPath.empty() || Result.is(K: tok::eof))
1286 return true;
1287
1288 // Consume the pp-import-suffix and expand any macros in it now, if we're not
1289 // at the semicolon already.
1290 SourceLocation SemiLoc = Result.getLocation();
1291 if (Result.isNot(K: tok::semi)) {
1292 Suffix.push_back(Elt: Result);
1293 CollectPpImportSuffix(Toks&: Suffix);
1294 if (Suffix.back().isNot(K: tok::semi)) {
1295 // This is not an import after all.
1296 EnterTokens(Suffix);
1297 return false;
1298 }
1299 SemiLoc = Suffix.back().getLocation();
1300 }
1301
1302 // Under the standard C++ Modules, the dot is just part of the module name,
1303 // and not a real hierarchy separator. Flatten such module names now.
1304 //
1305 // FIXME: Is this the right level to be performing this transformation?
1306 std::string FlatModuleName;
1307 if (getLangOpts().CPlusPlusModules) {
1308 for (auto &Piece : NamedModuleImportPath) {
1309 // If the FlatModuleName ends with colon, it implies it is a partition.
1310 if (!FlatModuleName.empty() && FlatModuleName.back() != ':')
1311 FlatModuleName += ".";
1312 FlatModuleName += Piece.getIdentifierInfo()->getName();
1313 }
1314 SourceLocation FirstPathLoc = NamedModuleImportPath[0].getLoc();
1315 NamedModuleImportPath.clear();
1316 NamedModuleImportPath.emplace_back(Args&: FirstPathLoc,
1317 Args: getIdentifierInfo(Name: FlatModuleName));
1318 }
1319
1320 Module *Imported = nullptr;
1321 // We don't/shouldn't load the standard c++20 modules when preprocessing.
1322 if (getLangOpts().Modules && !isInImportingCXXNamedModules()) {
1323 Imported = TheModuleLoader.loadModule(ImportLoc: ModuleImportLoc,
1324 Path: NamedModuleImportPath,
1325 Visibility: Module::Hidden,
1326 /*IsInclusionDirective=*/false);
1327 if (Imported)
1328 makeModuleVisible(M: Imported, Loc: SemiLoc);
1329 }
1330
1331 if (Callbacks)
1332 Callbacks->moduleImport(ImportLoc: ModuleImportLoc, Path: NamedModuleImportPath, Imported);
1333
1334 if (!Suffix.empty()) {
1335 EnterTokens(Suffix);
1336 return false;
1337 }
1338 return true;
1339}
1340
1341void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc,
1342 bool IncludeExports) {
1343 CurSubmoduleState->VisibleModules.setVisible(
1344 M, Loc, IncludeExports, Vis: [](Module *) {},
1345 Cb: [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
1346 // FIXME: Include the path in the diagnostic.
1347 // FIXME: Include the import location for the conflicting module.
1348 Diag(ModuleImportLoc, diag::warn_module_conflict)
1349 << Path[0]->getFullModuleName()
1350 << Conflict->getFullModuleName()
1351 << Message;
1352 });
1353
1354 // Add this module to the imports list of the currently-built submodule.
1355 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1356 BuildingSubmoduleStack.back().M->Imports.insert(X: M);
1357}
1358
1359bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
1360 const char *DiagnosticTag,
1361 bool AllowMacroExpansion) {
1362 // We need at least one string literal.
1363 if (Result.isNot(K: tok::string_literal)) {
1364 Diag(Result, diag::err_expected_string_literal)
1365 << /*Source='in...'*/0 << DiagnosticTag;
1366 return false;
1367 }
1368
1369 // Lex string literal tokens, optionally with macro expansion.
1370 SmallVector<Token, 4> StrToks;
1371 do {
1372 StrToks.push_back(Elt: Result);
1373
1374 if (Result.hasUDSuffix())
1375 Diag(Result, diag::err_invalid_string_udl);
1376
1377 if (AllowMacroExpansion)
1378 Lex(Result);
1379 else
1380 LexUnexpandedToken(Result);
1381 } while (Result.is(K: tok::string_literal));
1382
1383 // Concatenate and parse the strings.
1384 StringLiteralParser Literal(StrToks, *this);
1385 assert(Literal.isOrdinary() && "Didn't allow wide strings in");
1386
1387 if (Literal.hadError)
1388 return false;
1389
1390 if (Literal.Pascal) {
1391 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1392 << /*Source='in...'*/0 << DiagnosticTag;
1393 return false;
1394 }
1395
1396 String = std::string(Literal.GetString());
1397 return true;
1398}
1399
1400bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) {
1401 assert(Tok.is(tok::numeric_constant));
1402 SmallString<8> IntegerBuffer;
1403 bool NumberInvalid = false;
1404 StringRef Spelling = getSpelling(Tok, Buffer&: IntegerBuffer, Invalid: &NumberInvalid);
1405 if (NumberInvalid)
1406 return false;
1407 NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(),
1408 getLangOpts(), getTargetInfo(),
1409 getDiagnostics());
1410 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1411 return false;
1412 llvm::APInt APVal(64, 0);
1413 if (Literal.GetIntegerValue(Val&: APVal))
1414 return false;
1415 Lex(Result&: Tok);
1416 Value = APVal.getLimitedValue();
1417 return true;
1418}
1419
1420void Preprocessor::addCommentHandler(CommentHandler *Handler) {
1421 assert(Handler && "NULL comment handler");
1422 assert(!llvm::is_contained(CommentHandlers, Handler) &&
1423 "Comment handler already registered");
1424 CommentHandlers.push_back(x: Handler);
1425}
1426
1427void Preprocessor::removeCommentHandler(CommentHandler *Handler) {
1428 std::vector<CommentHandler *>::iterator Pos =
1429 llvm::find(Range&: CommentHandlers, Val: Handler);
1430 assert(Pos != CommentHandlers.end() && "Comment handler not registered");
1431 CommentHandlers.erase(position: Pos);
1432}
1433
1434bool Preprocessor::HandleComment(Token &result, SourceRange Comment) {
1435 bool AnyPendingTokens = false;
1436 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1437 HEnd = CommentHandlers.end();
1438 H != HEnd; ++H) {
1439 if ((*H)->HandleComment(PP&: *this, Comment))
1440 AnyPendingTokens = true;
1441 }
1442 if (!AnyPendingTokens || getCommentRetentionState())
1443 return false;
1444 Lex(Result&: result);
1445 return true;
1446}
1447
1448void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const {
1449 const MacroAnnotations &A =
1450 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1451 assert(A.DeprecationInfo &&
1452 "Macro deprecation warning without recorded annotation!");
1453 const MacroAnnotationInfo &Info = *A.DeprecationInfo;
1454 if (Info.Message.empty())
1455 Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1456 << Identifier.getIdentifierInfo() << 0;
1457 else
1458 Diag(Identifier, diag::warn_pragma_deprecated_macro_use)
1459 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1460 Diag(Info.Location, diag::note_pp_macro_annotation) << 0;
1461}
1462
1463void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const {
1464 const MacroAnnotations &A =
1465 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1466 assert(A.RestrictExpansionInfo &&
1467 "Macro restricted expansion warning without recorded annotation!");
1468 const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo;
1469 if (Info.Message.empty())
1470 Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1471 << Identifier.getIdentifierInfo() << 0;
1472 else
1473 Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use)
1474 << Identifier.getIdentifierInfo() << 1 << Info.Message;
1475 Diag(Info.Location, diag::note_pp_macro_annotation) << 1;
1476}
1477
1478void Preprocessor::emitRestrictInfNaNWarning(const Token &Identifier,
1479 unsigned DiagSelection) const {
1480 Diag(Identifier, diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1;
1481}
1482
1483void Preprocessor::emitFinalMacroWarning(const Token &Identifier,
1484 bool IsUndef) const {
1485 const MacroAnnotations &A =
1486 getMacroAnnotations(II: Identifier.getIdentifierInfo());
1487 assert(A.FinalAnnotationLoc &&
1488 "Final macro warning without recorded annotation!");
1489
1490 Diag(Identifier, diag::warn_pragma_final_macro)
1491 << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1);
1492 Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2;
1493}
1494
1495bool Preprocessor::isSafeBufferOptOut(const SourceManager &SourceMgr,
1496 const SourceLocation &Loc) const {
1497 // The lambda that tests if a `Loc` is in an opt-out region given one opt-out
1498 // region map:
1499 auto TestInMap = [&SourceMgr](const SafeBufferOptOutRegionsTy &Map,
1500 const SourceLocation &Loc) -> bool {
1501 // Try to find a region in `SafeBufferOptOutMap` where `Loc` is in:
1502 auto FirstRegionEndingAfterLoc = llvm::partition_point(
1503 Range: Map, P: [&SourceMgr,
1504 &Loc](const std::pair<SourceLocation, SourceLocation> &Region) {
1505 return SourceMgr.isBeforeInTranslationUnit(LHS: Region.second, RHS: Loc);
1506 });
1507
1508 if (FirstRegionEndingAfterLoc != Map.end()) {
1509 // To test if the start location of the found region precedes `Loc`:
1510 return SourceMgr.isBeforeInTranslationUnit(
1511 LHS: FirstRegionEndingAfterLoc->first, RHS: Loc);
1512 }
1513 // If we do not find a region whose end location passes `Loc`, we want to
1514 // check if the current region is still open:
1515 if (!Map.empty() && Map.back().first == Map.back().second)
1516 return SourceMgr.isBeforeInTranslationUnit(LHS: Map.back().first, RHS: Loc);
1517 return false;
1518 };
1519
1520 // What the following does:
1521 //
1522 // If `Loc` belongs to the local TU, we just look up `SafeBufferOptOutMap`.
1523 // Otherwise, `Loc` is from a loaded AST. We look up the
1524 // `LoadedSafeBufferOptOutMap` first to get the opt-out region map of the
1525 // loaded AST where `Loc` is at. Then we find if `Loc` is in an opt-out
1526 // region w.r.t. the region map. If the region map is absent, it means there
1527 // is no opt-out pragma in that loaded AST.
1528 //
1529 // Opt-out pragmas in the local TU or a loaded AST is not visible to another
1530 // one of them. That means if you put the pragmas around a `#include
1531 // "module.h"`, where module.h is a module, it is not actually suppressing
1532 // warnings in module.h. This is fine because warnings in module.h will be
1533 // reported when module.h is compiled in isolation and nothing in module.h
1534 // will be analyzed ever again. So you will not see warnings from the file
1535 // that imports module.h anyway. And you can't even do the same thing for PCHs
1536 // because they can only be included from the command line.
1537
1538 if (SourceMgr.isLocalSourceLocation(Loc))
1539 return TestInMap(SafeBufferOptOutMap, Loc);
1540
1541 const SafeBufferOptOutRegionsTy *LoadedRegions =
1542 LoadedSafeBufferOptOutMap.lookupLoadedOptOutMap(Loc, SrcMgr: SourceMgr);
1543
1544 if (LoadedRegions)
1545 return TestInMap(*LoadedRegions, Loc);
1546 return false;
1547}
1548
1549bool Preprocessor::enterOrExitSafeBufferOptOutRegion(
1550 bool isEnter, const SourceLocation &Loc) {
1551 if (isEnter) {
1552 if (isPPInSafeBufferOptOutRegion())
1553 return true; // invalid enter action
1554 InSafeBufferOptOutRegion = true;
1555 CurrentSafeBufferOptOutStart = Loc;
1556
1557 // To set the start location of a new region:
1558
1559 if (!SafeBufferOptOutMap.empty()) {
1560 [[maybe_unused]] auto *PrevRegion = &SafeBufferOptOutMap.back();
1561 assert(PrevRegion->first != PrevRegion->second &&
1562 "Shall not begin a safe buffer opt-out region before closing the "
1563 "previous one.");
1564 }
1565 // If the start location equals to the end location, we call the region a
1566 // open region or a unclosed region (i.e., end location has not been set
1567 // yet).
1568 SafeBufferOptOutMap.emplace_back(Args: Loc, Args: Loc);
1569 } else {
1570 if (!isPPInSafeBufferOptOutRegion())
1571 return true; // invalid enter action
1572 InSafeBufferOptOutRegion = false;
1573
1574 // To set the end location of the current open region:
1575
1576 assert(!SafeBufferOptOutMap.empty() &&
1577 "Misordered safe buffer opt-out regions");
1578 auto *CurrRegion = &SafeBufferOptOutMap.back();
1579 assert(CurrRegion->first == CurrRegion->second &&
1580 "Set end location to a closed safe buffer opt-out region");
1581 CurrRegion->second = Loc;
1582 }
1583 return false;
1584}
1585
1586bool Preprocessor::isPPInSafeBufferOptOutRegion() {
1587 return InSafeBufferOptOutRegion;
1588}
1589bool Preprocessor::isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc) {
1590 StartLoc = CurrentSafeBufferOptOutStart;
1591 return InSafeBufferOptOutRegion;
1592}
1593
1594SmallVector<SourceLocation, 64>
1595Preprocessor::serializeSafeBufferOptOutMap() const {
1596 assert(!InSafeBufferOptOutRegion &&
1597 "Attempt to serialize safe buffer opt-out regions before file being "
1598 "completely preprocessed");
1599
1600 SmallVector<SourceLocation, 64> SrcSeq;
1601
1602 for (const auto &[begin, end] : SafeBufferOptOutMap) {
1603 SrcSeq.push_back(Elt: begin);
1604 SrcSeq.push_back(Elt: end);
1605 }
1606 // Only `SafeBufferOptOutMap` gets serialized. No need to serialize
1607 // `LoadedSafeBufferOptOutMap` because if this TU loads a pch/module, every
1608 // pch/module in the pch-chain/module-DAG will be loaded one by one in order.
1609 // It means that for each loading pch/module m, it just needs to load m's own
1610 // `SafeBufferOptOutMap`.
1611 return SrcSeq;
1612}
1613
1614bool Preprocessor::setDeserializedSafeBufferOptOutMap(
1615 const SmallVectorImpl<SourceLocation> &SourceLocations) {
1616 if (SourceLocations.size() == 0)
1617 return false;
1618
1619 assert(SourceLocations.size() % 2 == 0 &&
1620 "ill-formed SourceLocation sequence");
1621
1622 auto It = SourceLocations.begin();
1623 SafeBufferOptOutRegionsTy &Regions =
1624 LoadedSafeBufferOptOutMap.findAndConsLoadedOptOutMap(Loc: *It, SrcMgr&: SourceMgr);
1625
1626 do {
1627 SourceLocation Begin = *It++;
1628 SourceLocation End = *It++;
1629
1630 Regions.emplace_back(Args&: Begin, Args&: End);
1631 } while (It != SourceLocations.end());
1632 return true;
1633}
1634
1635ModuleLoader::~ModuleLoader() = default;
1636
1637CommentHandler::~CommentHandler() = default;
1638
1639EmptylineHandler::~EmptylineHandler() = default;
1640
1641CodeCompletionHandler::~CodeCompletionHandler() = default;
1642
1643void Preprocessor::createPreprocessingRecord() {
1644 if (Record)
1645 return;
1646
1647 Record = new PreprocessingRecord(getSourceManager());
1648 addPPCallbacks(C: std::unique_ptr<PPCallbacks>(Record));
1649}
1650
1651const char *Preprocessor::getCheckPoint(FileID FID, const char *Start) const {
1652 if (auto It = CheckPoints.find(Val: FID); It != CheckPoints.end()) {
1653 const SmallVector<const char *> &FileCheckPoints = It->second;
1654 const char *Last = nullptr;
1655 // FIXME: Do better than a linear search.
1656 for (const char *P : FileCheckPoints) {
1657 if (P > Start)
1658 break;
1659 Last = P;
1660 }
1661 return Last;
1662 }
1663
1664 return nullptr;
1665}
1666

source code of clang/lib/Lex/Preprocessor.cpp