1 | //===- Preprocessor.cpp - C Language Family Preprocessor Implementation ---===// |
---|---|
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the Preprocessor interface. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | // |
13 | // Options to support: |
14 | // -H - Print the name of each header file used. |
15 | // -d[DNI] - Dump various things. |
16 | // -fworking-directory - #line's with preprocessor's working dir. |
17 | // -fpreprocessed |
18 | // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD |
19 | // -W* |
20 | // -w |
21 | // |
22 | // Messages to emit: |
23 | // "Multiple include guards may be useful for:\n" |
24 | // |
25 | //===----------------------------------------------------------------------===// |
26 | |
27 | #include "clang/Lex/Preprocessor.h" |
28 | #include "clang/Basic/Builtins.h" |
29 | #include "clang/Basic/FileManager.h" |
30 | #include "clang/Basic/IdentifierTable.h" |
31 | #include "clang/Basic/LLVM.h" |
32 | #include "clang/Basic/LangOptions.h" |
33 | #include "clang/Basic/Module.h" |
34 | #include "clang/Basic/SourceLocation.h" |
35 | #include "clang/Basic/SourceManager.h" |
36 | #include "clang/Basic/TargetInfo.h" |
37 | #include "clang/Lex/CodeCompletionHandler.h" |
38 | #include "clang/Lex/ExternalPreprocessorSource.h" |
39 | #include "clang/Lex/HeaderSearch.h" |
40 | #include "clang/Lex/LexDiagnostic.h" |
41 | #include "clang/Lex/Lexer.h" |
42 | #include "clang/Lex/LiteralSupport.h" |
43 | #include "clang/Lex/MacroArgs.h" |
44 | #include "clang/Lex/MacroInfo.h" |
45 | #include "clang/Lex/ModuleLoader.h" |
46 | #include "clang/Lex/Pragma.h" |
47 | #include "clang/Lex/PreprocessingRecord.h" |
48 | #include "clang/Lex/PreprocessorLexer.h" |
49 | #include "clang/Lex/PreprocessorOptions.h" |
50 | #include "clang/Lex/ScratchBuffer.h" |
51 | #include "clang/Lex/Token.h" |
52 | #include "clang/Lex/TokenLexer.h" |
53 | #include "llvm/ADT/APInt.h" |
54 | #include "llvm/ADT/ArrayRef.h" |
55 | #include "llvm/ADT/DenseMap.h" |
56 | #include "llvm/ADT/STLExtras.h" |
57 | #include "llvm/ADT/SmallVector.h" |
58 | #include "llvm/ADT/StringRef.h" |
59 | #include "llvm/Support/Capacity.h" |
60 | #include "llvm/Support/ErrorHandling.h" |
61 | #include "llvm/Support/MemoryBuffer.h" |
62 | #include "llvm/Support/raw_ostream.h" |
63 | #include <algorithm> |
64 | #include <cassert> |
65 | #include <memory> |
66 | #include <optional> |
67 | #include <string> |
68 | #include <utility> |
69 | #include <vector> |
70 | |
71 | using namespace clang; |
72 | |
73 | /// Minimum distance between two check points, in tokens. |
74 | static constexpr unsigned CheckPointStepSize = 1024; |
75 | |
76 | LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry) |
77 | |
78 | ExternalPreprocessorSource::~ExternalPreprocessorSource() = default; |
79 | |
80 | Preprocessor::Preprocessor(const PreprocessorOptions &PPOpts, |
81 | DiagnosticsEngine &diags, const LangOptions &opts, |
82 | SourceManager &SM, HeaderSearch &Headers, |
83 | ModuleLoader &TheModuleLoader, |
84 | IdentifierInfoLookup *IILookup, bool OwnsHeaders, |
85 | TranslationUnitKind TUKind) |
86 | : PPOpts(PPOpts), Diags(&diags), LangOpts(opts), |
87 | FileMgr(Headers.getFileMgr()), SourceMgr(SM), |
88 | ScratchBuf(new ScratchBuffer(SourceMgr)), HeaderInfo(Headers), |
89 | TheModuleLoader(TheModuleLoader), ExternalSource(nullptr), |
90 | // As the language options may have not been loaded yet (when |
91 | // deserializing an ASTUnit), adding keywords to the identifier table is |
92 | // deferred to Preprocessor::Initialize(). |
93 | Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), |
94 | TUKind(TUKind), SkipMainFilePreamble(0, true), |
95 | CurSubmoduleState(&NullSubmoduleState) { |
96 | OwnsHeaderSearch = OwnsHeaders; |
97 | |
98 | // Default to discarding comments. |
99 | KeepComments = false; |
100 | KeepMacroComments = false; |
101 | SuppressIncludeNotFoundError = false; |
102 | |
103 | // Macro expansion is enabled. |
104 | DisableMacroExpansion = false; |
105 | MacroExpansionInDirectivesOverride = false; |
106 | InMacroArgs = false; |
107 | ArgMacro = nullptr; |
108 | InMacroArgPreExpansion = false; |
109 | NumCachedTokenLexers = 0; |
110 | PragmasEnabled = true; |
111 | ParsingIfOrElifDirective = false; |
112 | PreprocessedOutput = false; |
113 | |
114 | // We haven't read anything from the external source. |
115 | ReadMacrosFromExternalSource = false; |
116 | |
117 | BuiltinInfo = std::make_unique<Builtin::Context>(); |
118 | |
119 | // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of |
120 | // a macro. They get unpoisoned where it is allowed. |
121 | (Ident__VA_ARGS__ = getIdentifierInfo(Name: "__VA_ARGS__"))->setIsPoisoned(); |
122 | SetPoisonReason(II: Ident__VA_ARGS__,diag::DiagID: ext_pp_bad_vaargs_use); |
123 | (Ident__VA_OPT__ = getIdentifierInfo(Name: "__VA_OPT__"))->setIsPoisoned(); |
124 | SetPoisonReason(II: Ident__VA_OPT__,diag::DiagID: ext_pp_bad_vaopt_use); |
125 | |
126 | // Initialize the pragma handlers. |
127 | RegisterBuiltinPragmas(); |
128 | |
129 | // Initialize builtin macros like __LINE__ and friends. |
130 | RegisterBuiltinMacros(); |
131 | |
132 | if(LangOpts.Borland) { |
133 | Ident__exception_info = getIdentifierInfo(Name: "_exception_info"); |
134 | Ident___exception_info = getIdentifierInfo(Name: "__exception_info"); |
135 | Ident_GetExceptionInfo = getIdentifierInfo(Name: "GetExceptionInformation"); |
136 | Ident__exception_code = getIdentifierInfo(Name: "_exception_code"); |
137 | Ident___exception_code = getIdentifierInfo(Name: "__exception_code"); |
138 | Ident_GetExceptionCode = getIdentifierInfo(Name: "GetExceptionCode"); |
139 | Ident__abnormal_termination = getIdentifierInfo(Name: "_abnormal_termination"); |
140 | Ident___abnormal_termination = getIdentifierInfo(Name: "__abnormal_termination"); |
141 | Ident_AbnormalTermination = getIdentifierInfo(Name: "AbnormalTermination"); |
142 | } else { |
143 | Ident__exception_info = Ident__exception_code = nullptr; |
144 | Ident__abnormal_termination = Ident___exception_info = nullptr; |
145 | Ident___exception_code = Ident___abnormal_termination = nullptr; |
146 | Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr; |
147 | Ident_AbnormalTermination = nullptr; |
148 | } |
149 | |
150 | // Default incremental processing to -fincremental-extensions, clients can |
151 | // override with `enableIncrementalProcessing` if desired. |
152 | IncrementalProcessing = LangOpts.IncrementalExtensions; |
153 | |
154 | // If using a PCH where a #pragma hdrstop is expected, start skipping tokens. |
155 | if (usingPCHWithPragmaHdrStop()) |
156 | SkippingUntilPragmaHdrStop = true; |
157 | |
158 | // If using a PCH with a through header, start skipping tokens. |
159 | if (!this->PPOpts.PCHThroughHeader.empty() && |
160 | !this->PPOpts.ImplicitPCHInclude.empty()) |
161 | SkippingUntilPCHThroughHeader = true; |
162 | |
163 | if (this->PPOpts.GeneratePreamble) |
164 | PreambleConditionalStack.startRecording(); |
165 | |
166 | MaxTokens = LangOpts.MaxTokens; |
167 | } |
168 | |
169 | Preprocessor::~Preprocessor() { |
170 | assert(!isBacktrackEnabled() && "EnableBacktrack/Backtrack imbalance!"); |
171 | |
172 | IncludeMacroStack.clear(); |
173 | |
174 | // Free any cached macro expanders. |
175 | // This populates MacroArgCache, so all TokenLexers need to be destroyed |
176 | // before the code below that frees up the MacroArgCache list. |
177 | std::fill(first: TokenLexerCache, last: TokenLexerCache + NumCachedTokenLexers, value: nullptr); |
178 | CurTokenLexer.reset(); |
179 | |
180 | // Free any cached MacroArgs. |
181 | for (MacroArgs *ArgList = MacroArgCache; ArgList;) |
182 | ArgList = ArgList->deallocate(); |
183 | |
184 | // Delete the header search info, if we own it. |
185 | if (OwnsHeaderSearch) |
186 | delete &HeaderInfo; |
187 | } |
188 | |
189 | void Preprocessor::Initialize(const TargetInfo &Target, |
190 | const TargetInfo *AuxTarget) { |
191 | assert((!this->Target || this->Target == &Target) && |
192 | "Invalid override of target information"); |
193 | this->Target = &Target; |
194 | |
195 | assert((!this->AuxTarget || this->AuxTarget == AuxTarget) && |
196 | "Invalid override of aux target information."); |
197 | this->AuxTarget = AuxTarget; |
198 | |
199 | // Initialize information about built-ins. |
200 | BuiltinInfo->InitializeTarget(Target, AuxTarget); |
201 | HeaderInfo.setTarget(Target); |
202 | |
203 | // Populate the identifier table with info about keywords for the current language. |
204 | Identifiers.AddKeywords(LangOpts); |
205 | |
206 | // Initialize the __FTL_EVAL_METHOD__ macro to the TargetInfo. |
207 | setTUFPEvalMethod(getTargetInfo().getFPEvalMethod()); |
208 | |
209 | if (getLangOpts().getFPEvalMethod() == LangOptions::FEM_UnsetOnCommandLine) |
210 | // Use setting from TargetInfo. |
211 | setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: Target.getFPEvalMethod()); |
212 | else |
213 | // Set initial value of __FLT_EVAL_METHOD__ from the command line. |
214 | setCurrentFPEvalMethod(PragmaLoc: SourceLocation(), Val: getLangOpts().getFPEvalMethod()); |
215 | } |
216 | |
217 | void Preprocessor::InitializeForModelFile() { |
218 | NumEnteredSourceFiles = 0; |
219 | |
220 | // Reset pragmas |
221 | PragmaHandlersBackup = std::move(PragmaHandlers); |
222 | PragmaHandlers = std::make_unique<PragmaNamespace>(args: StringRef()); |
223 | RegisterBuiltinPragmas(); |
224 | |
225 | // Reset PredefinesFileID |
226 | PredefinesFileID = FileID(); |
227 | } |
228 | |
229 | void Preprocessor::FinalizeForModelFile() { |
230 | NumEnteredSourceFiles = 1; |
231 | |
232 | PragmaHandlers = std::move(PragmaHandlersBackup); |
233 | } |
234 | |
235 | void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { |
236 | llvm::errs() << tok::getTokenName(Kind: Tok.getKind()); |
237 | |
238 | if (!Tok.isAnnotation()) |
239 | llvm::errs() << " '"<< getSpelling(Tok) << "'"; |
240 | |
241 | if (!DumpFlags) return; |
242 | |
243 | llvm::errs() << "\t"; |
244 | if (Tok.isAtStartOfLine()) |
245 | llvm::errs() << " [StartOfLine]"; |
246 | if (Tok.hasLeadingSpace()) |
247 | llvm::errs() << " [LeadingSpace]"; |
248 | if (Tok.isExpandDisabled()) |
249 | llvm::errs() << " [ExpandDisabled]"; |
250 | if (Tok.needsCleaning()) { |
251 | const char *Start = SourceMgr.getCharacterData(SL: Tok.getLocation()); |
252 | llvm::errs() << " [UnClean='"<< StringRef(Start, Tok.getLength()) |
253 | << "']"; |
254 | } |
255 | |
256 | llvm::errs() << "\tLoc=<"; |
257 | DumpLocation(Loc: Tok.getLocation()); |
258 | llvm::errs() << ">"; |
259 | } |
260 | |
261 | void Preprocessor::DumpLocation(SourceLocation Loc) const { |
262 | Loc.print(OS&: llvm::errs(), SM: SourceMgr); |
263 | } |
264 | |
265 | void Preprocessor::DumpMacro(const MacroInfo &MI) const { |
266 | llvm::errs() << "MACRO: "; |
267 | for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) { |
268 | DumpToken(Tok: MI.getReplacementToken(Tok: i)); |
269 | llvm::errs() << " "; |
270 | } |
271 | llvm::errs() << "\n"; |
272 | } |
273 | |
274 | void Preprocessor::PrintStats() { |
275 | llvm::errs() << "\n*** Preprocessor Stats:\n"; |
276 | llvm::errs() << NumDirectives << " directives found:\n"; |
277 | llvm::errs() << " "<< NumDefined << " #define.\n"; |
278 | llvm::errs() << " "<< NumUndefined << " #undef.\n"; |
279 | llvm::errs() << " #include/#include_next/#import:\n"; |
280 | llvm::errs() << " "<< NumEnteredSourceFiles << " source files entered.\n"; |
281 | llvm::errs() << " "<< MaxIncludeStackDepth << " max include stack depth\n"; |
282 | llvm::errs() << " "<< NumIf << " #if/#ifndef/#ifdef.\n"; |
283 | llvm::errs() << " "<< NumElse << " #else/#elif/#elifdef/#elifndef.\n"; |
284 | llvm::errs() << " "<< NumEndif << " #endif.\n"; |
285 | llvm::errs() << " "<< NumPragma << " #pragma.\n"; |
286 | llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n"; |
287 | |
288 | llvm::errs() << NumMacroExpanded << "/"<< NumFnMacroExpanded << "/" |
289 | << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, " |
290 | << NumFastMacroExpanded << " on the fast path.\n"; |
291 | llvm::errs() << (NumFastTokenPaste+NumTokenPaste) |
292 | << " token paste (##) operations performed, " |
293 | << NumFastTokenPaste << " on the fast path.\n"; |
294 | |
295 | llvm::errs() << "\nPreprocessor Memory: "<< getTotalMemory() << "B total"; |
296 | |
297 | llvm::errs() << "\n BumpPtr: "<< BP.getTotalMemory(); |
298 | llvm::errs() << "\n Macro Expanded Tokens: " |
299 | << llvm::capacity_in_bytes(X: MacroExpandedTokens); |
300 | llvm::errs() << "\n Predefines Buffer: "<< Predefines.capacity(); |
301 | // FIXME: List information for all submodules. |
302 | llvm::errs() << "\n Macros: " |
303 | << llvm::capacity_in_bytes(X: CurSubmoduleState->Macros); |
304 | llvm::errs() << "\n #pragma push_macro Info: " |
305 | << llvm::capacity_in_bytes(X: PragmaPushMacroInfo); |
306 | llvm::errs() << "\n Poison Reasons: " |
307 | << llvm::capacity_in_bytes(X: PoisonReasons); |
308 | llvm::errs() << "\n Comment Handlers: " |
309 | << llvm::capacity_in_bytes(x: CommentHandlers) << "\n"; |
310 | } |
311 | |
312 | Preprocessor::macro_iterator |
313 | Preprocessor::macro_begin(bool IncludeExternalMacros) const { |
314 | if (IncludeExternalMacros && ExternalSource && |
315 | !ReadMacrosFromExternalSource) { |
316 | ReadMacrosFromExternalSource = true; |
317 | ExternalSource->ReadDefinedMacros(); |
318 | } |
319 | |
320 | // Make sure we cover all macros in visible modules. |
321 | for (const ModuleMacro &Macro : ModuleMacros) |
322 | CurSubmoduleState->Macros.try_emplace(Key: Macro.II); |
323 | |
324 | return CurSubmoduleState->Macros.begin(); |
325 | } |
326 | |
327 | size_t Preprocessor::getTotalMemory() const { |
328 | return BP.getTotalMemory() |
329 | + llvm::capacity_in_bytes(X: MacroExpandedTokens) |
330 | + Predefines.capacity() /* Predefines buffer. */ |
331 | // FIXME: Include sizes from all submodules, and include MacroInfo sizes, |
332 | // and ModuleMacros. |
333 | + llvm::capacity_in_bytes(X: CurSubmoduleState->Macros) |
334 | + llvm::capacity_in_bytes(X: PragmaPushMacroInfo) |
335 | + llvm::capacity_in_bytes(X: PoisonReasons) |
336 | + llvm::capacity_in_bytes(x: CommentHandlers); |
337 | } |
338 | |
339 | Preprocessor::macro_iterator |
340 | Preprocessor::macro_end(bool IncludeExternalMacros) const { |
341 | if (IncludeExternalMacros && ExternalSource && |
342 | !ReadMacrosFromExternalSource) { |
343 | ReadMacrosFromExternalSource = true; |
344 | ExternalSource->ReadDefinedMacros(); |
345 | } |
346 | |
347 | return CurSubmoduleState->Macros.end(); |
348 | } |
349 | |
350 | /// Compares macro tokens with a specified token value sequence. |
351 | static bool MacroDefinitionEquals(const MacroInfo *MI, |
352 | ArrayRef<TokenValue> Tokens) { |
353 | return Tokens.size() == MI->getNumTokens() && |
354 | std::equal(first1: Tokens.begin(), last1: Tokens.end(), first2: MI->tokens_begin()); |
355 | } |
356 | |
357 | StringRef Preprocessor::getLastMacroWithSpelling( |
358 | SourceLocation Loc, |
359 | ArrayRef<TokenValue> Tokens) const { |
360 | SourceLocation BestLocation; |
361 | StringRef BestSpelling; |
362 | for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end(); |
363 | I != E; ++I) { |
364 | const MacroDirective::DefInfo |
365 | Def = I->second.findDirectiveAtLoc(Loc, SourceMgr); |
366 | if (!Def || !Def.getMacroInfo()) |
367 | continue; |
368 | if (!Def.getMacroInfo()->isObjectLike()) |
369 | continue; |
370 | if (!MacroDefinitionEquals(MI: Def.getMacroInfo(), Tokens)) |
371 | continue; |
372 | SourceLocation Location = Def.getLocation(); |
373 | // Choose the macro defined latest. |
374 | if (BestLocation.isInvalid() || |
375 | (Location.isValid() && |
376 | SourceMgr.isBeforeInTranslationUnit(LHS: BestLocation, RHS: Location))) { |
377 | BestLocation = Location; |
378 | BestSpelling = I->first->getName(); |
379 | } |
380 | } |
381 | return BestSpelling; |
382 | } |
383 | |
384 | void Preprocessor::recomputeCurLexerKind() { |
385 | if (CurLexer) |
386 | CurLexerCallback = CurLexer->isDependencyDirectivesLexer() |
387 | ? CLK_DependencyDirectivesLexer |
388 | : CLK_Lexer; |
389 | else if (CurTokenLexer) |
390 | CurLexerCallback = CLK_TokenLexer; |
391 | else |
392 | CurLexerCallback = CLK_CachingLexer; |
393 | } |
394 | |
395 | bool Preprocessor::SetCodeCompletionPoint(FileEntryRef File, |
396 | unsigned CompleteLine, |
397 | unsigned CompleteColumn) { |
398 | assert(CompleteLine && CompleteColumn && "Starts from 1:1"); |
399 | assert(!CodeCompletionFile && "Already set"); |
400 | |
401 | // Load the actual file's contents. |
402 | std::optional<llvm::MemoryBufferRef> Buffer = |
403 | SourceMgr.getMemoryBufferForFileOrNone(File); |
404 | if (!Buffer) |
405 | return true; |
406 | |
407 | // Find the byte position of the truncation point. |
408 | const char *Position = Buffer->getBufferStart(); |
409 | for (unsigned Line = 1; Line < CompleteLine; ++Line) { |
410 | for (; *Position; ++Position) { |
411 | if (*Position != '\r' && *Position != '\n') |
412 | continue; |
413 | |
414 | // Eat \r\n or \n\r as a single line. |
415 | if ((Position[1] == '\r' || Position[1] == '\n') && |
416 | Position[0] != Position[1]) |
417 | ++Position; |
418 | ++Position; |
419 | break; |
420 | } |
421 | } |
422 | |
423 | Position += CompleteColumn - 1; |
424 | |
425 | // If pointing inside the preamble, adjust the position at the beginning of |
426 | // the file after the preamble. |
427 | if (SkipMainFilePreamble.first && |
428 | SourceMgr.getFileEntryForID(FID: SourceMgr.getMainFileID()) == File) { |
429 | if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first) |
430 | Position = Buffer->getBufferStart() + SkipMainFilePreamble.first; |
431 | } |
432 | |
433 | if (Position > Buffer->getBufferEnd()) |
434 | Position = Buffer->getBufferEnd(); |
435 | |
436 | CodeCompletionFile = File; |
437 | CodeCompletionOffset = Position - Buffer->getBufferStart(); |
438 | |
439 | auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer( |
440 | Size: Buffer->getBufferSize() + 1, BufferName: Buffer->getBufferIdentifier()); |
441 | char *NewBuf = NewBuffer->getBufferStart(); |
442 | char *NewPos = std::copy(first: Buffer->getBufferStart(), last: Position, result: NewBuf); |
443 | *NewPos = '\0'; |
444 | std::copy(first: Position, last: Buffer->getBufferEnd(), result: NewPos+1); |
445 | SourceMgr.overrideFileContents(SourceFile: File, Buffer: std::move(NewBuffer)); |
446 | |
447 | return false; |
448 | } |
449 | |
450 | void Preprocessor::CodeCompleteIncludedFile(llvm::StringRef Dir, |
451 | bool IsAngled) { |
452 | setCodeCompletionReached(); |
453 | if (CodeComplete) |
454 | CodeComplete->CodeCompleteIncludedFile(Dir, IsAngled); |
455 | } |
456 | |
457 | void Preprocessor::CodeCompleteNaturalLanguage() { |
458 | setCodeCompletionReached(); |
459 | if (CodeComplete) |
460 | CodeComplete->CodeCompleteNaturalLanguage(); |
461 | } |
462 | |
463 | /// getSpelling - This method is used to get the spelling of a token into a |
464 | /// SmallVector. Note that the returned StringRef may not point to the |
465 | /// supplied buffer if a copy can be avoided. |
466 | StringRef Preprocessor::getSpelling(const Token &Tok, |
467 | SmallVectorImpl<char> &Buffer, |
468 | bool *Invalid) const { |
469 | // NOTE: this has to be checked *before* testing for an IdentifierInfo. |
470 | if (Tok.isNot(K: tok::raw_identifier) && !Tok.hasUCN()) { |
471 | // Try the fast path. |
472 | if (const IdentifierInfo *II = Tok.getIdentifierInfo()) |
473 | return II->getName(); |
474 | } |
475 | |
476 | // Resize the buffer if we need to copy into it. |
477 | if (Tok.needsCleaning()) |
478 | Buffer.resize(N: Tok.getLength()); |
479 | |
480 | const char *Ptr = Buffer.data(); |
481 | unsigned Len = getSpelling(Tok, Buffer&: Ptr, Invalid); |
482 | return StringRef(Ptr, Len); |
483 | } |
484 | |
485 | /// CreateString - Plop the specified string into a scratch buffer and return a |
486 | /// location for it. If specified, the source location provides a source |
487 | /// location for the token. |
488 | void Preprocessor::CreateString(StringRef Str, Token &Tok, |
489 | SourceLocation ExpansionLocStart, |
490 | SourceLocation ExpansionLocEnd) { |
491 | Tok.setLength(Str.size()); |
492 | |
493 | const char *DestPtr; |
494 | SourceLocation Loc = ScratchBuf->getToken(Buf: Str.data(), Len: Str.size(), DestPtr); |
495 | |
496 | if (ExpansionLocStart.isValid()) |
497 | Loc = SourceMgr.createExpansionLoc(SpellingLoc: Loc, ExpansionLocStart, |
498 | ExpansionLocEnd, Length: Str.size()); |
499 | Tok.setLocation(Loc); |
500 | |
501 | // If this is a raw identifier or a literal token, set the pointer data. |
502 | if (Tok.is(K: tok::raw_identifier)) |
503 | Tok.setRawIdentifierData(DestPtr); |
504 | else if (Tok.isLiteral()) |
505 | Tok.setLiteralData(DestPtr); |
506 | } |
507 | |
508 | SourceLocation Preprocessor::SplitToken(SourceLocation Loc, unsigned Length) { |
509 | auto &SM = getSourceManager(); |
510 | SourceLocation SpellingLoc = SM.getSpellingLoc(Loc); |
511 | std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc: SpellingLoc); |
512 | bool Invalid = false; |
513 | StringRef Buffer = SM.getBufferData(FID: LocInfo.first, Invalid: &Invalid); |
514 | if (Invalid) |
515 | return SourceLocation(); |
516 | |
517 | // FIXME: We could consider re-using spelling for tokens we see repeatedly. |
518 | const char *DestPtr; |
519 | SourceLocation Spelling = |
520 | ScratchBuf->getToken(Buf: Buffer.data() + LocInfo.second, Len: Length, DestPtr); |
521 | return SM.createTokenSplitLoc(SpellingLoc: Spelling, TokenStart: Loc, TokenEnd: Loc.getLocWithOffset(Offset: Length)); |
522 | } |
523 | |
524 | Module *Preprocessor::getCurrentModule() { |
525 | if (!getLangOpts().isCompilingModule()) |
526 | return nullptr; |
527 | |
528 | return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().CurrentModule); |
529 | } |
530 | |
531 | Module *Preprocessor::getCurrentModuleImplementation() { |
532 | if (!getLangOpts().isCompilingModuleImplementation()) |
533 | return nullptr; |
534 | |
535 | return getHeaderSearchInfo().lookupModule(ModuleName: getLangOpts().ModuleName); |
536 | } |
537 | |
538 | //===----------------------------------------------------------------------===// |
539 | // Preprocessor Initialization Methods |
540 | //===----------------------------------------------------------------------===// |
541 | |
542 | /// EnterMainSourceFile - Enter the specified FileID as the main source file, |
543 | /// which implicitly adds the builtin defines etc. |
544 | void Preprocessor::EnterMainSourceFile() { |
545 | // We do not allow the preprocessor to reenter the main file. Doing so will |
546 | // cause FileID's to accumulate information from both runs (e.g. #line |
547 | // information) and predefined macros aren't guaranteed to be set properly. |
548 | assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!"); |
549 | FileID MainFileID = SourceMgr.getMainFileID(); |
550 | |
551 | // If MainFileID is loaded it means we loaded an AST file, no need to enter |
552 | // a main file. |
553 | if (!SourceMgr.isLoadedFileID(FID: MainFileID)) { |
554 | // Enter the main file source buffer. |
555 | EnterSourceFile(FID: MainFileID, Dir: nullptr, Loc: SourceLocation()); |
556 | |
557 | // If we've been asked to skip bytes in the main file (e.g., as part of a |
558 | // precompiled preamble), do so now. |
559 | if (SkipMainFilePreamble.first > 0) |
560 | CurLexer->SetByteOffset(Offset: SkipMainFilePreamble.first, |
561 | StartOfLine: SkipMainFilePreamble.second); |
562 | |
563 | // Tell the header info that the main file was entered. If the file is later |
564 | // #imported, it won't be re-entered. |
565 | if (OptionalFileEntryRef FE = SourceMgr.getFileEntryRefForID(FID: MainFileID)) |
566 | markIncluded(File: *FE); |
567 | } |
568 | |
569 | // Preprocess Predefines to populate the initial preprocessor state. |
570 | std::unique_ptr<llvm::MemoryBuffer> SB = |
571 | llvm::MemoryBuffer::getMemBufferCopy(InputData: Predefines, BufferName: "<built-in>"); |
572 | assert(SB && "Cannot create predefined source buffer"); |
573 | FileID FID = SourceMgr.createFileID(Buffer: std::move(SB)); |
574 | assert(FID.isValid() && "Could not create FileID for predefines?"); |
575 | setPredefinesFileID(FID); |
576 | |
577 | // Start parsing the predefines. |
578 | EnterSourceFile(FID, Dir: nullptr, Loc: SourceLocation()); |
579 | |
580 | if (!PPOpts.PCHThroughHeader.empty()) { |
581 | // Lookup and save the FileID for the through header. If it isn't found |
582 | // in the search path, it's a fatal error. |
583 | OptionalFileEntryRef File = LookupFile( |
584 | FilenameLoc: SourceLocation(), Filename: PPOpts.PCHThroughHeader, |
585 | /*isAngled=*/false, /*FromDir=*/nullptr, /*FromFile=*/nullptr, |
586 | /*CurDir=*/nullptr, /*SearchPath=*/nullptr, /*RelativePath=*/nullptr, |
587 | /*SuggestedModule=*/nullptr, /*IsMapped=*/nullptr, |
588 | /*IsFrameworkFound=*/nullptr); |
589 | if (!File) { |
590 | Diag(SourceLocation(), diag::err_pp_through_header_not_found) |
591 | << PPOpts.PCHThroughHeader; |
592 | return; |
593 | } |
594 | setPCHThroughHeaderFileID( |
595 | SourceMgr.createFileID(SourceFile: *File, IncludePos: SourceLocation(), FileCharacter: SrcMgr::C_User)); |
596 | } |
597 | |
598 | // Skip tokens from the Predefines and if needed the main file. |
599 | if ((usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) || |
600 | (usingPCHWithPragmaHdrStop() && SkippingUntilPragmaHdrStop)) |
601 | SkipTokensWhileUsingPCH(); |
602 | } |
603 | |
604 | void Preprocessor::setPCHThroughHeaderFileID(FileID FID) { |
605 | assert(PCHThroughHeaderFileID.isInvalid() && |
606 | "PCHThroughHeaderFileID already set!"); |
607 | PCHThroughHeaderFileID = FID; |
608 | } |
609 | |
610 | bool Preprocessor::isPCHThroughHeader(const FileEntry *FE) { |
611 | assert(PCHThroughHeaderFileID.isValid() && |
612 | "Invalid PCH through header FileID"); |
613 | return FE == SourceMgr.getFileEntryForID(FID: PCHThroughHeaderFileID); |
614 | } |
615 | |
616 | bool Preprocessor::creatingPCHWithThroughHeader() { |
617 | return TUKind == TU_Prefix && !PPOpts.PCHThroughHeader.empty() && |
618 | PCHThroughHeaderFileID.isValid(); |
619 | } |
620 | |
621 | bool Preprocessor::usingPCHWithThroughHeader() { |
622 | return TUKind != TU_Prefix && !PPOpts.PCHThroughHeader.empty() && |
623 | PCHThroughHeaderFileID.isValid(); |
624 | } |
625 | |
626 | bool Preprocessor::creatingPCHWithPragmaHdrStop() { |
627 | return TUKind == TU_Prefix && PPOpts.PCHWithHdrStop; |
628 | } |
629 | |
630 | bool Preprocessor::usingPCHWithPragmaHdrStop() { |
631 | return TUKind != TU_Prefix && PPOpts.PCHWithHdrStop; |
632 | } |
633 | |
634 | /// Skip tokens until after the #include of the through header or |
635 | /// until after a #pragma hdrstop is seen. Tokens in the predefines file |
636 | /// and the main file may be skipped. If the end of the predefines file |
637 | /// is reached, skipping continues into the main file. If the end of the |
638 | /// main file is reached, it's a fatal error. |
639 | void Preprocessor::SkipTokensWhileUsingPCH() { |
640 | bool ReachedMainFileEOF = false; |
641 | bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader; |
642 | bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop; |
643 | Token Tok; |
644 | while (true) { |
645 | bool InPredefines = |
646 | (CurLexer && CurLexer->getFileID() == getPredefinesFileID()); |
647 | CurLexerCallback(*this, Tok); |
648 | if (Tok.is(K: tok::eof) && !InPredefines) { |
649 | ReachedMainFileEOF = true; |
650 | break; |
651 | } |
652 | if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader) |
653 | break; |
654 | if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop) |
655 | break; |
656 | } |
657 | if (ReachedMainFileEOF) { |
658 | if (UsingPCHThroughHeader) |
659 | Diag(SourceLocation(), diag::err_pp_through_header_not_seen) |
660 | << PPOpts.PCHThroughHeader << 1; |
661 | else if (!PPOpts.PCHWithHdrStopCreate) |
662 | Diag(SourceLocation(), diag::err_pp_pragma_hdrstop_not_seen); |
663 | } |
664 | } |
665 | |
666 | void Preprocessor::replayPreambleConditionalStack() { |
667 | // Restore the conditional stack from the preamble, if there is one. |
668 | if (PreambleConditionalStack.isReplaying()) { |
669 | assert(CurPPLexer && |
670 | "CurPPLexer is null when calling replayPreambleConditionalStack."); |
671 | CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); |
672 | PreambleConditionalStack.doneReplaying(); |
673 | if (PreambleConditionalStack.reachedEOFWhileSkipping()) |
674 | SkipExcludedConditionalBlock( |
675 | HashTokenLoc: PreambleConditionalStack.SkipInfo->HashTokenLoc, |
676 | IfTokenLoc: PreambleConditionalStack.SkipInfo->IfTokenLoc, |
677 | FoundNonSkipPortion: PreambleConditionalStack.SkipInfo->FoundNonSkipPortion, |
678 | FoundElse: PreambleConditionalStack.SkipInfo->FoundElse, |
679 | ElseLoc: PreambleConditionalStack.SkipInfo->ElseLoc); |
680 | } |
681 | } |
682 | |
683 | void Preprocessor::EndSourceFile() { |
684 | // Notify the client that we reached the end of the source file. |
685 | if (Callbacks) |
686 | Callbacks->EndOfMainFile(); |
687 | } |
688 | |
689 | //===----------------------------------------------------------------------===// |
690 | // Lexer Event Handling. |
691 | //===----------------------------------------------------------------------===// |
692 | |
693 | /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the |
694 | /// identifier information for the token and install it into the token, |
695 | /// updating the token kind accordingly. |
696 | IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const { |
697 | assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!"); |
698 | |
699 | // Look up this token, see if it is a macro, or if it is a language keyword. |
700 | IdentifierInfo *II; |
701 | if (!Identifier.needsCleaning() && !Identifier.hasUCN()) { |
702 | // No cleaning needed, just use the characters from the lexed buffer. |
703 | II = getIdentifierInfo(Name: Identifier.getRawIdentifier()); |
704 | } else { |
705 | // Cleaning needed, alloca a buffer, clean into it, then use the buffer. |
706 | SmallString<64> IdentifierBuffer; |
707 | StringRef CleanedStr = getSpelling(Tok: Identifier, Buffer&: IdentifierBuffer); |
708 | |
709 | if (Identifier.hasUCN()) { |
710 | SmallString<64> UCNIdentifierBuffer; |
711 | expandUCNs(Buf&: UCNIdentifierBuffer, Input: CleanedStr); |
712 | II = getIdentifierInfo(Name: UCNIdentifierBuffer); |
713 | } else { |
714 | II = getIdentifierInfo(Name: CleanedStr); |
715 | } |
716 | } |
717 | |
718 | // Update the token info (identifier info and appropriate token kind). |
719 | // FIXME: the raw_identifier may contain leading whitespace which is removed |
720 | // from the cleaned identifier token. The SourceLocation should be updated to |
721 | // refer to the non-whitespace character. For instance, the text "\\\nB" (a |
722 | // line continuation before 'B') is parsed as a single tok::raw_identifier and |
723 | // is cleaned to tok::identifier "B". After cleaning the token's length is |
724 | // still 3 and the SourceLocation refers to the location of the backslash. |
725 | Identifier.setIdentifierInfo(II); |
726 | Identifier.setKind(II->getTokenID()); |
727 | |
728 | return II; |
729 | } |
730 | |
731 | void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) { |
732 | PoisonReasons[II] = DiagID; |
733 | } |
734 | |
735 | void Preprocessor::PoisonSEHIdentifiers(bool Poison) { |
736 | assert(Ident__exception_code && Ident__exception_info); |
737 | assert(Ident___exception_code && Ident___exception_info); |
738 | Ident__exception_code->setIsPoisoned(Poison); |
739 | Ident___exception_code->setIsPoisoned(Poison); |
740 | Ident_GetExceptionCode->setIsPoisoned(Poison); |
741 | Ident__exception_info->setIsPoisoned(Poison); |
742 | Ident___exception_info->setIsPoisoned(Poison); |
743 | Ident_GetExceptionInfo->setIsPoisoned(Poison); |
744 | Ident__abnormal_termination->setIsPoisoned(Poison); |
745 | Ident___abnormal_termination->setIsPoisoned(Poison); |
746 | Ident_AbnormalTermination->setIsPoisoned(Poison); |
747 | } |
748 | |
749 | void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) { |
750 | assert(Identifier.getIdentifierInfo() && |
751 | "Can't handle identifiers without identifier info!"); |
752 | llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it = |
753 | PoisonReasons.find(Val: Identifier.getIdentifierInfo()); |
754 | if(it == PoisonReasons.end()) |
755 | Diag(Identifier, diag::err_pp_used_poisoned_id); |
756 | else |
757 | Diag(Tok: Identifier,DiagID: it->second) << Identifier.getIdentifierInfo(); |
758 | } |
759 | |
760 | void Preprocessor::updateOutOfDateIdentifier(const IdentifierInfo &II) const { |
761 | assert(II.isOutOfDate() && "not out of date"); |
762 | assert(getExternalSource() && |
763 | "getExternalSource() should not return nullptr"); |
764 | getExternalSource()->updateOutOfDateIdentifier(II); |
765 | } |
766 | |
767 | /// HandleIdentifier - This callback is invoked when the lexer reads an |
768 | /// identifier. This callback looks up the identifier in the map and/or |
769 | /// potentially macro expands it or turns it into a named token (like 'for'). |
770 | /// |
771 | /// Note that callers of this method are guarded by checking the |
772 | /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the |
773 | /// IdentifierInfo methods that compute these properties will need to change to |
774 | /// match. |
775 | bool Preprocessor::HandleIdentifier(Token &Identifier) { |
776 | assert(Identifier.getIdentifierInfo() && |
777 | "Can't handle identifiers without identifier info!"); |
778 | |
779 | IdentifierInfo &II = *Identifier.getIdentifierInfo(); |
780 | |
781 | // If the information about this identifier is out of date, update it from |
782 | // the external source. |
783 | // We have to treat __VA_ARGS__ in a special way, since it gets |
784 | // serialized with isPoisoned = true, but our preprocessor may have |
785 | // unpoisoned it if we're defining a C99 macro. |
786 | if (II.isOutOfDate()) { |
787 | bool CurrentIsPoisoned = false; |
788 | const bool IsSpecialVariadicMacro = |
789 | &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__; |
790 | if (IsSpecialVariadicMacro) |
791 | CurrentIsPoisoned = II.isPoisoned(); |
792 | |
793 | updateOutOfDateIdentifier(II); |
794 | Identifier.setKind(II.getTokenID()); |
795 | |
796 | if (IsSpecialVariadicMacro) |
797 | II.setIsPoisoned(CurrentIsPoisoned); |
798 | } |
799 | |
800 | // If this identifier was poisoned, and if it was not produced from a macro |
801 | // expansion, emit an error. |
802 | if (II.isPoisoned() && CurPPLexer) { |
803 | HandlePoisonedIdentifier(Identifier); |
804 | } |
805 | |
806 | // If this is a macro to be expanded, do it. |
807 | if (const MacroDefinition MD = getMacroDefinition(II: &II)) { |
808 | const auto *MI = MD.getMacroInfo(); |
809 | assert(MI && "macro definition with no macro info?"); |
810 | if (!DisableMacroExpansion) { |
811 | if (!Identifier.isExpandDisabled() && MI->isEnabled()) { |
812 | // C99 6.10.3p10: If the preprocessing token immediately after the |
813 | // macro name isn't a '(', this macro should not be expanded. |
814 | if (!MI->isFunctionLike() || isNextPPTokenLParen()) |
815 | return HandleMacroExpandedIdentifier(Identifier, MD); |
816 | } else { |
817 | // C99 6.10.3.4p2 says that a disabled macro may never again be |
818 | // expanded, even if it's in a context where it could be expanded in the |
819 | // future. |
820 | Identifier.setFlag(Token::DisableExpand); |
821 | if (MI->isObjectLike() || isNextPPTokenLParen()) |
822 | Diag(Tok: Identifier, diag::DiagID: pp_disabled_macro_expansion); |
823 | } |
824 | } |
825 | } |
826 | |
827 | // If this identifier is a keyword in a newer Standard or proposed Standard, |
828 | // produce a warning. Don't warn if we're not considering macro expansion, |
829 | // since this identifier might be the name of a macro. |
830 | // FIXME: This warning is disabled in cases where it shouldn't be, like |
831 | // "#define constexpr constexpr", "int constexpr;" |
832 | if (II.isFutureCompatKeyword() && !DisableMacroExpansion) { |
833 | Diag(Tok: Identifier, DiagID: getIdentifierTable().getFutureCompatDiagKind(II, LangOpts: getLangOpts())) |
834 | << II.getName(); |
835 | // Don't diagnose this keyword again in this translation unit. |
836 | II.setIsFutureCompatKeyword(false); |
837 | } |
838 | |
839 | // If this identifier would be a keyword in C++, diagnose as a compatibility |
840 | // issue. |
841 | if (II.IsKeywordInCPlusPlus() && !DisableMacroExpansion) |
842 | Diag(Identifier, diag::warn_pp_identifier_is_cpp_keyword) << &II; |
843 | |
844 | // If this is an extension token, diagnose its use. |
845 | // We avoid diagnosing tokens that originate from macro definitions. |
846 | // FIXME: This warning is disabled in cases where it shouldn't be, |
847 | // like "#define TY typeof", "TY(1) x". |
848 | if (II.isExtensionToken() && !DisableMacroExpansion) |
849 | Diag(Identifier, diag::ext_token_used); |
850 | |
851 | // If this is the 'import' contextual keyword following an '@', note |
852 | // that the next token indicates a module name. |
853 | // |
854 | // Note that we do not treat 'import' as a contextual |
855 | // keyword when we're in a caching lexer, because caching lexers only get |
856 | // used in contexts where import declarations are disallowed. |
857 | // |
858 | // Likewise if this is the standard C++ import keyword. |
859 | if (((LastTokenWasAt && II.isModulesImport()) || |
860 | Identifier.is(K: tok::kw_import)) && |
861 | !InMacroArgs && !DisableMacroExpansion && |
862 | (getLangOpts().Modules || getLangOpts().DebuggerSupport) && |
863 | CurLexerCallback != CLK_CachingLexer) { |
864 | ModuleImportLoc = Identifier.getLocation(); |
865 | NamedModuleImportPath.clear(); |
866 | IsAtImport = true; |
867 | ModuleImportExpectsIdentifier = true; |
868 | CurLexerCallback = CLK_LexAfterModuleImport; |
869 | } |
870 | return true; |
871 | } |
872 | |
873 | void Preprocessor::Lex(Token &Result) { |
874 | ++LexLevel; |
875 | |
876 | // We loop here until a lex function returns a token; this avoids recursion. |
877 | while (!CurLexerCallback(*this, Result)) |
878 | ; |
879 | |
880 | if (Result.is(K: tok::unknown) && TheModuleLoader.HadFatalFailure) |
881 | return; |
882 | |
883 | if (Result.is(K: tok::code_completion) && Result.getIdentifierInfo()) { |
884 | // Remember the identifier before code completion token. |
885 | setCodeCompletionIdentifierInfo(Result.getIdentifierInfo()); |
886 | setCodeCompletionTokenRange(Start: Result.getLocation(), End: Result.getEndLoc()); |
887 | // Set IdenfitierInfo to null to avoid confusing code that handles both |
888 | // identifiers and completion tokens. |
889 | Result.setIdentifierInfo(nullptr); |
890 | } |
891 | |
892 | // Update StdCXXImportSeqState to track our position within a C++20 import-seq |
893 | // if this token is being produced as a result of phase 4 of translation. |
894 | // Update TrackGMFState to decide if we are currently in a Global Module |
895 | // Fragment. GMF state updates should precede StdCXXImportSeq ones, since GMF state |
896 | // depends on the prevailing StdCXXImportSeq state in two cases. |
897 | if (getLangOpts().CPlusPlusModules && LexLevel == 1 && |
898 | !Result.getFlag(Flag: Token::IsReinjected)) { |
899 | switch (Result.getKind()) { |
900 | case tok::l_paren: case tok::l_square: case tok::l_brace: |
901 | StdCXXImportSeqState.handleOpenBracket(); |
902 | break; |
903 | case tok::r_paren: case tok::r_square: |
904 | StdCXXImportSeqState.handleCloseBracket(); |
905 | break; |
906 | case tok::r_brace: |
907 | StdCXXImportSeqState.handleCloseBrace(); |
908 | break; |
909 | #define PRAGMA_ANNOTATION(X) case tok::annot_##X: |
910 | // For `#pragma ...` mimic ';'. |
911 | #include "clang/Basic/TokenKinds.def" |
912 | #undef PRAGMA_ANNOTATION |
913 | // This token is injected to represent the translation of '#include "a.h"' |
914 | // into "import a.h;". Mimic the notional ';'. |
915 | case tok::annot_module_include: |
916 | case tok::semi: |
917 | TrackGMFState.handleSemi(); |
918 | StdCXXImportSeqState.handleSemi(); |
919 | ModuleDeclState.handleSemi(); |
920 | break; |
921 | case tok::header_name: |
922 | case tok::annot_header_unit: |
923 | StdCXXImportSeqState.handleHeaderName(); |
924 | break; |
925 | case tok::kw_export: |
926 | TrackGMFState.handleExport(); |
927 | StdCXXImportSeqState.handleExport(); |
928 | ModuleDeclState.handleExport(); |
929 | break; |
930 | case tok::colon: |
931 | ModuleDeclState.handleColon(); |
932 | break; |
933 | case tok::period: |
934 | ModuleDeclState.handlePeriod(); |
935 | break; |
936 | case tok::identifier: |
937 | // Check "import" and "module" when there is no open bracket. The two |
938 | // identifiers are not meaningful with open brackets. |
939 | if (StdCXXImportSeqState.atTopLevel()) { |
940 | if (Result.getIdentifierInfo()->isModulesImport()) { |
941 | TrackGMFState.handleImport(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq()); |
942 | StdCXXImportSeqState.handleImport(); |
943 | if (StdCXXImportSeqState.afterImportSeq()) { |
944 | ModuleImportLoc = Result.getLocation(); |
945 | NamedModuleImportPath.clear(); |
946 | IsAtImport = false; |
947 | ModuleImportExpectsIdentifier = true; |
948 | CurLexerCallback = CLK_LexAfterModuleImport; |
949 | } |
950 | break; |
951 | } else if (Result.getIdentifierInfo() == getIdentifierInfo(Name: "module")) { |
952 | TrackGMFState.handleModule(AfterTopLevelTokenSeq: StdCXXImportSeqState.afterTopLevelSeq()); |
953 | ModuleDeclState.handleModule(); |
954 | break; |
955 | } |
956 | } |
957 | ModuleDeclState.handleIdentifier(Identifier: Result.getIdentifierInfo()); |
958 | if (ModuleDeclState.isModuleCandidate()) |
959 | break; |
960 | [[fallthrough]]; |
961 | default: |
962 | TrackGMFState.handleMisc(); |
963 | StdCXXImportSeqState.handleMisc(); |
964 | ModuleDeclState.handleMisc(); |
965 | break; |
966 | } |
967 | } |
968 | |
969 | if (CurLexer && ++CheckPointCounter == CheckPointStepSize) { |
970 | CheckPoints[CurLexer->getFileID()].push_back(CurLexer->BufferPtr); |
971 | CheckPointCounter = 0; |
972 | } |
973 | |
974 | LastTokenWasAt = Result.is(K: tok::at); |
975 | --LexLevel; |
976 | |
977 | if ((LexLevel == 0 || PreprocessToken) && |
978 | !Result.getFlag(Flag: Token::IsReinjected)) { |
979 | if (LexLevel == 0) |
980 | ++TokenCount; |
981 | if (OnToken) |
982 | OnToken(Result); |
983 | } |
984 | } |
985 | |
986 | void Preprocessor::LexTokensUntilEOF(std::vector<Token> *Tokens) { |
987 | while (1) { |
988 | Token Tok; |
989 | Lex(Result&: Tok); |
990 | if (Tok.isOneOf(K1: tok::unknown, Ks: tok::eof, Ks: tok::eod, |
991 | Ks: tok::annot_repl_input_end)) |
992 | break; |
993 | if (Tokens != nullptr) |
994 | Tokens->push_back(x: Tok); |
995 | } |
996 | } |
997 | |
998 | /// Lex a header-name token (including one formed from header-name-tokens if |
999 | /// \p AllowMacroExpansion is \c true). |
1000 | /// |
1001 | /// \param FilenameTok Filled in with the next token. On success, this will |
1002 | /// be either a header_name token. On failure, it will be whatever other |
1003 | /// token was found instead. |
1004 | /// \param AllowMacroExpansion If \c true, allow the header name to be formed |
1005 | /// by macro expansion (concatenating tokens as necessary if the first |
1006 | /// token is a '<'). |
1007 | /// \return \c true if we reached EOD or EOF while looking for a > token in |
1008 | /// a concatenated header name and diagnosed it. \c false otherwise. |
1009 | bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowMacroExpansion) { |
1010 | // Lex using header-name tokenization rules if tokens are being lexed from |
1011 | // a file. Just grab a token normally if we're in a macro expansion. |
1012 | if (CurPPLexer) |
1013 | CurPPLexer->LexIncludeFilename(FilenameTok); |
1014 | else |
1015 | Lex(Result&: FilenameTok); |
1016 | |
1017 | // This could be a <foo/bar.h> file coming from a macro expansion. In this |
1018 | // case, glue the tokens together into an angle_string_literal token. |
1019 | SmallString<128> FilenameBuffer; |
1020 | if (FilenameTok.is(K: tok::less) && AllowMacroExpansion) { |
1021 | bool StartOfLine = FilenameTok.isAtStartOfLine(); |
1022 | bool LeadingSpace = FilenameTok.hasLeadingSpace(); |
1023 | bool LeadingEmptyMacro = FilenameTok.hasLeadingEmptyMacro(); |
1024 | |
1025 | SourceLocation Start = FilenameTok.getLocation(); |
1026 | SourceLocation End; |
1027 | FilenameBuffer.push_back(Elt: '<'); |
1028 | |
1029 | // Consume tokens until we find a '>'. |
1030 | // FIXME: A header-name could be formed starting or ending with an |
1031 | // alternative token. It's not clear whether that's ill-formed in all |
1032 | // cases. |
1033 | while (FilenameTok.isNot(K: tok::greater)) { |
1034 | Lex(Result&: FilenameTok); |
1035 | if (FilenameTok.isOneOf(K1: tok::eod, K2: tok::eof)) { |
1036 | Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater; |
1037 | Diag(Start, diag::note_matching) << tok::less; |
1038 | return true; |
1039 | } |
1040 | |
1041 | End = FilenameTok.getLocation(); |
1042 | |
1043 | // FIXME: Provide code completion for #includes. |
1044 | if (FilenameTok.is(K: tok::code_completion)) { |
1045 | setCodeCompletionReached(); |
1046 | Lex(Result&: FilenameTok); |
1047 | continue; |
1048 | } |
1049 | |
1050 | // Append the spelling of this token to the buffer. If there was a space |
1051 | // before it, add it now. |
1052 | if (FilenameTok.hasLeadingSpace()) |
1053 | FilenameBuffer.push_back(Elt: ' '); |
1054 | |
1055 | // Get the spelling of the token, directly into FilenameBuffer if |
1056 | // possible. |
1057 | size_t PreAppendSize = FilenameBuffer.size(); |
1058 | FilenameBuffer.resize(N: PreAppendSize + FilenameTok.getLength()); |
1059 | |
1060 | const char *BufPtr = &FilenameBuffer[PreAppendSize]; |
1061 | unsigned ActualLen = getSpelling(Tok: FilenameTok, Buffer&: BufPtr); |
1062 | |
1063 | // If the token was spelled somewhere else, copy it into FilenameBuffer. |
1064 | if (BufPtr != &FilenameBuffer[PreAppendSize]) |
1065 | memcpy(dest: &FilenameBuffer[PreAppendSize], src: BufPtr, n: ActualLen); |
1066 | |
1067 | // Resize FilenameBuffer to the correct size. |
1068 | if (FilenameTok.getLength() != ActualLen) |
1069 | FilenameBuffer.resize(N: PreAppendSize + ActualLen); |
1070 | } |
1071 | |
1072 | FilenameTok.startToken(); |
1073 | FilenameTok.setKind(tok::header_name); |
1074 | FilenameTok.setFlagValue(Flag: Token::StartOfLine, Val: StartOfLine); |
1075 | FilenameTok.setFlagValue(Flag: Token::LeadingSpace, Val: LeadingSpace); |
1076 | FilenameTok.setFlagValue(Flag: Token::LeadingEmptyMacro, Val: LeadingEmptyMacro); |
1077 | CreateString(Str: FilenameBuffer, Tok&: FilenameTok, ExpansionLocStart: Start, ExpansionLocEnd: End); |
1078 | } else if (FilenameTok.is(K: tok::string_literal) && AllowMacroExpansion) { |
1079 | // Convert a string-literal token of the form " h-char-sequence " |
1080 | // (produced by macro expansion) into a header-name token. |
1081 | // |
1082 | // The rules for header-names don't quite match the rules for |
1083 | // string-literals, but all the places where they differ result in |
1084 | // undefined behavior, so we can and do treat them the same. |
1085 | // |
1086 | // A string-literal with a prefix or suffix is not translated into a |
1087 | // header-name. This could theoretically be observable via the C++20 |
1088 | // context-sensitive header-name formation rules. |
1089 | StringRef Str = getSpelling(Tok: FilenameTok, Buffer&: FilenameBuffer); |
1090 | if (Str.size() >= 2 && Str.front() == '"' && Str.back() == '"') |
1091 | FilenameTok.setKind(tok::header_name); |
1092 | } |
1093 | |
1094 | return false; |
1095 | } |
1096 | |
1097 | /// Collect the tokens of a C++20 pp-import-suffix. |
1098 | void Preprocessor::CollectPpImportSuffix(SmallVectorImpl<Token> &Toks) { |
1099 | // FIXME: For error recovery, consider recognizing attribute syntax here |
1100 | // and terminating / diagnosing a missing semicolon if we find anything |
1101 | // else? (Can we leave that to the parser?) |
1102 | unsigned BracketDepth = 0; |
1103 | while (true) { |
1104 | Toks.emplace_back(); |
1105 | Lex(Result&: Toks.back()); |
1106 | |
1107 | switch (Toks.back().getKind()) { |
1108 | case tok::l_paren: case tok::l_square: case tok::l_brace: |
1109 | ++BracketDepth; |
1110 | break; |
1111 | |
1112 | case tok::r_paren: case tok::r_square: case tok::r_brace: |
1113 | if (BracketDepth == 0) |
1114 | return; |
1115 | --BracketDepth; |
1116 | break; |
1117 | |
1118 | case tok::semi: |
1119 | if (BracketDepth == 0) |
1120 | return; |
1121 | break; |
1122 | |
1123 | case tok::eof: |
1124 | return; |
1125 | |
1126 | default: |
1127 | break; |
1128 | } |
1129 | } |
1130 | } |
1131 | |
1132 | |
1133 | /// Lex a token following the 'import' contextual keyword. |
1134 | /// |
1135 | /// pp-import: [C++20] |
1136 | /// import header-name pp-import-suffix[opt] ; |
1137 | /// import header-name-tokens pp-import-suffix[opt] ; |
1138 | /// [ObjC] @ import module-name ; |
1139 | /// [Clang] import module-name ; |
1140 | /// |
1141 | /// header-name-tokens: |
1142 | /// string-literal |
1143 | /// < [any sequence of preprocessing-tokens other than >] > |
1144 | /// |
1145 | /// module-name: |
1146 | /// module-name-qualifier[opt] identifier |
1147 | /// |
1148 | /// module-name-qualifier |
1149 | /// module-name-qualifier[opt] identifier . |
1150 | /// |
1151 | /// We respond to a pp-import by importing macros from the named module. |
1152 | bool Preprocessor::LexAfterModuleImport(Token &Result) { |
1153 | // Figure out what kind of lexer we actually have. |
1154 | recomputeCurLexerKind(); |
1155 | |
1156 | // Lex the next token. The header-name lexing rules are used at the start of |
1157 | // a pp-import. |
1158 | // |
1159 | // For now, we only support header-name imports in C++20 mode. |
1160 | // FIXME: Should we allow this in all language modes that support an import |
1161 | // declaration as an extension? |
1162 | if (NamedModuleImportPath.empty() && getLangOpts().CPlusPlusModules) { |
1163 | if (LexHeaderName(FilenameTok&: Result)) |
1164 | return true; |
1165 | |
1166 | if (Result.is(K: tok::colon) && ModuleDeclState.isNamedModule()) { |
1167 | std::string Name = ModuleDeclState.getPrimaryName().str(); |
1168 | Name += ":"; |
1169 | NamedModuleImportPath.emplace_back(Args: Result.getLocation(), |
1170 | Args: getIdentifierInfo(Name)); |
1171 | CurLexerCallback = CLK_LexAfterModuleImport; |
1172 | return true; |
1173 | } |
1174 | } else { |
1175 | Lex(Result); |
1176 | } |
1177 | |
1178 | // Allocate a holding buffer for a sequence of tokens and introduce it into |
1179 | // the token stream. |
1180 | auto EnterTokens = [this](ArrayRef<Token> Toks) { |
1181 | auto ToksCopy = std::make_unique<Token[]>(num: Toks.size()); |
1182 | std::copy(first: Toks.begin(), last: Toks.end(), result: ToksCopy.get()); |
1183 | EnterTokenStream(Toks: std::move(ToksCopy), NumToks: Toks.size(), |
1184 | /*DisableMacroExpansion*/ true, /*IsReinject*/ false); |
1185 | }; |
1186 | |
1187 | bool ImportingHeader = Result.is(K: tok::header_name); |
1188 | // Check for a header-name. |
1189 | SmallVector<Token, 32> Suffix; |
1190 | if (ImportingHeader) { |
1191 | // Enter the header-name token into the token stream; a Lex action cannot |
1192 | // both return a token and cache tokens (doing so would corrupt the token |
1193 | // cache if the call to Lex comes from CachingLex / PeekAhead). |
1194 | Suffix.push_back(Elt: Result); |
1195 | |
1196 | // Consume the pp-import-suffix and expand any macros in it now. We'll add |
1197 | // it back into the token stream later. |
1198 | CollectPpImportSuffix(Toks&: Suffix); |
1199 | if (Suffix.back().isNot(K: tok::semi)) { |
1200 | // This is not a pp-import after all. |
1201 | EnterTokens(Suffix); |
1202 | return false; |
1203 | } |
1204 | |
1205 | // C++2a [cpp.module]p1: |
1206 | // The ';' preprocessing-token terminating a pp-import shall not have |
1207 | // been produced by macro replacement. |
1208 | SourceLocation SemiLoc = Suffix.back().getLocation(); |
1209 | if (SemiLoc.isMacroID()) |
1210 | Diag(SemiLoc, diag::err_header_import_semi_in_macro); |
1211 | |
1212 | // Reconstitute the import token. |
1213 | Token ImportTok; |
1214 | ImportTok.startToken(); |
1215 | ImportTok.setKind(tok::kw_import); |
1216 | ImportTok.setLocation(ModuleImportLoc); |
1217 | ImportTok.setIdentifierInfo(getIdentifierInfo(Name: "import")); |
1218 | ImportTok.setLength(6); |
1219 | |
1220 | auto Action = HandleHeaderIncludeOrImport( |
1221 | /*HashLoc*/ SourceLocation(), IncludeTok&: ImportTok, FilenameTok&: Suffix.front(), EndLoc: SemiLoc); |
1222 | switch (Action.Kind) { |
1223 | case ImportAction::None: |
1224 | break; |
1225 | |
1226 | case ImportAction::ModuleBegin: |
1227 | // Let the parser know we're textually entering the module. |
1228 | Suffix.emplace_back(); |
1229 | Suffix.back().startToken(); |
1230 | Suffix.back().setKind(tok::annot_module_begin); |
1231 | Suffix.back().setLocation(SemiLoc); |
1232 | Suffix.back().setAnnotationEndLoc(SemiLoc); |
1233 | Suffix.back().setAnnotationValue(Action.ModuleForHeader); |
1234 | [[fallthrough]]; |
1235 | |
1236 | case ImportAction::ModuleImport: |
1237 | case ImportAction::HeaderUnitImport: |
1238 | case ImportAction::SkippedModuleImport: |
1239 | // We chose to import (or textually enter) the file. Convert the |
1240 | // header-name token into a header unit annotation token. |
1241 | Suffix[0].setKind(tok::annot_header_unit); |
1242 | Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation()); |
1243 | Suffix[0].setAnnotationValue(Action.ModuleForHeader); |
1244 | // FIXME: Call the moduleImport callback? |
1245 | break; |
1246 | case ImportAction::Failure: |
1247 | assert(TheModuleLoader.HadFatalFailure && |
1248 | "This should be an early exit only to a fatal error"); |
1249 | Result.setKind(tok::eof); |
1250 | CurLexer->cutOffLexing(); |
1251 | EnterTokens(Suffix); |
1252 | return true; |
1253 | } |
1254 | |
1255 | EnterTokens(Suffix); |
1256 | return false; |
1257 | } |
1258 | |
1259 | // The token sequence |
1260 | // |
1261 | // import identifier (. identifier)* |
1262 | // |
1263 | // indicates a module import directive. We already saw the 'import' |
1264 | // contextual keyword, so now we're looking for the identifiers. |
1265 | if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) { |
1266 | // We expected to see an identifier here, and we did; continue handling |
1267 | // identifiers. |
1268 | NamedModuleImportPath.emplace_back(Args: Result.getLocation(), |
1269 | Args: Result.getIdentifierInfo()); |
1270 | ModuleImportExpectsIdentifier = false; |
1271 | CurLexerCallback = CLK_LexAfterModuleImport; |
1272 | return true; |
1273 | } |
1274 | |
1275 | // If we're expecting a '.' or a ';', and we got a '.', then wait until we |
1276 | // see the next identifier. (We can also see a '[[' that begins an |
1277 | // attribute-specifier-seq here under the Standard C++ Modules.) |
1278 | if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) { |
1279 | ModuleImportExpectsIdentifier = true; |
1280 | CurLexerCallback = CLK_LexAfterModuleImport; |
1281 | return true; |
1282 | } |
1283 | |
1284 | // If we didn't recognize a module name at all, this is not a (valid) import. |
1285 | if (NamedModuleImportPath.empty() || Result.is(K: tok::eof)) |
1286 | return true; |
1287 | |
1288 | // Consume the pp-import-suffix and expand any macros in it now, if we're not |
1289 | // at the semicolon already. |
1290 | SourceLocation SemiLoc = Result.getLocation(); |
1291 | if (Result.isNot(K: tok::semi)) { |
1292 | Suffix.push_back(Elt: Result); |
1293 | CollectPpImportSuffix(Toks&: Suffix); |
1294 | if (Suffix.back().isNot(K: tok::semi)) { |
1295 | // This is not an import after all. |
1296 | EnterTokens(Suffix); |
1297 | return false; |
1298 | } |
1299 | SemiLoc = Suffix.back().getLocation(); |
1300 | } |
1301 | |
1302 | // Under the standard C++ Modules, the dot is just part of the module name, |
1303 | // and not a real hierarchy separator. Flatten such module names now. |
1304 | // |
1305 | // FIXME: Is this the right level to be performing this transformation? |
1306 | std::string FlatModuleName; |
1307 | if (getLangOpts().CPlusPlusModules) { |
1308 | for (auto &Piece : NamedModuleImportPath) { |
1309 | // If the FlatModuleName ends with colon, it implies it is a partition. |
1310 | if (!FlatModuleName.empty() && FlatModuleName.back() != ':') |
1311 | FlatModuleName += "."; |
1312 | FlatModuleName += Piece.getIdentifierInfo()->getName(); |
1313 | } |
1314 | SourceLocation FirstPathLoc = NamedModuleImportPath[0].getLoc(); |
1315 | NamedModuleImportPath.clear(); |
1316 | NamedModuleImportPath.emplace_back(Args&: FirstPathLoc, |
1317 | Args: getIdentifierInfo(Name: FlatModuleName)); |
1318 | } |
1319 | |
1320 | Module *Imported = nullptr; |
1321 | // We don't/shouldn't load the standard c++20 modules when preprocessing. |
1322 | if (getLangOpts().Modules && !isInImportingCXXNamedModules()) { |
1323 | Imported = TheModuleLoader.loadModule(ImportLoc: ModuleImportLoc, |
1324 | Path: NamedModuleImportPath, |
1325 | Visibility: Module::Hidden, |
1326 | /*IsInclusionDirective=*/false); |
1327 | if (Imported) |
1328 | makeModuleVisible(M: Imported, Loc: SemiLoc); |
1329 | } |
1330 | |
1331 | if (Callbacks) |
1332 | Callbacks->moduleImport(ImportLoc: ModuleImportLoc, Path: NamedModuleImportPath, Imported); |
1333 | |
1334 | if (!Suffix.empty()) { |
1335 | EnterTokens(Suffix); |
1336 | return false; |
1337 | } |
1338 | return true; |
1339 | } |
1340 | |
1341 | void Preprocessor::makeModuleVisible(Module *M, SourceLocation Loc, |
1342 | bool IncludeExports) { |
1343 | CurSubmoduleState->VisibleModules.setVisible( |
1344 | M, Loc, IncludeExports, Vis: [](Module *) {}, |
1345 | Cb: [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) { |
1346 | // FIXME: Include the path in the diagnostic. |
1347 | // FIXME: Include the import location for the conflicting module. |
1348 | Diag(ModuleImportLoc, diag::warn_module_conflict) |
1349 | << Path[0]->getFullModuleName() |
1350 | << Conflict->getFullModuleName() |
1351 | << Message; |
1352 | }); |
1353 | |
1354 | // Add this module to the imports list of the currently-built submodule. |
1355 | if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M) |
1356 | BuildingSubmoduleStack.back().M->Imports.insert(X: M); |
1357 | } |
1358 | |
1359 | bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String, |
1360 | const char *DiagnosticTag, |
1361 | bool AllowMacroExpansion) { |
1362 | // We need at least one string literal. |
1363 | if (Result.isNot(K: tok::string_literal)) { |
1364 | Diag(Result, diag::err_expected_string_literal) |
1365 | << /*Source='in...'*/0 << DiagnosticTag; |
1366 | return false; |
1367 | } |
1368 | |
1369 | // Lex string literal tokens, optionally with macro expansion. |
1370 | SmallVector<Token, 4> StrToks; |
1371 | do { |
1372 | StrToks.push_back(Elt: Result); |
1373 | |
1374 | if (Result.hasUDSuffix()) |
1375 | Diag(Result, diag::err_invalid_string_udl); |
1376 | |
1377 | if (AllowMacroExpansion) |
1378 | Lex(Result); |
1379 | else |
1380 | LexUnexpandedToken(Result); |
1381 | } while (Result.is(K: tok::string_literal)); |
1382 | |
1383 | // Concatenate and parse the strings. |
1384 | StringLiteralParser Literal(StrToks, *this); |
1385 | assert(Literal.isOrdinary() && "Didn't allow wide strings in"); |
1386 | |
1387 | if (Literal.hadError) |
1388 | return false; |
1389 | |
1390 | if (Literal.Pascal) { |
1391 | Diag(StrToks[0].getLocation(), diag::err_expected_string_literal) |
1392 | << /*Source='in...'*/0 << DiagnosticTag; |
1393 | return false; |
1394 | } |
1395 | |
1396 | String = std::string(Literal.GetString()); |
1397 | return true; |
1398 | } |
1399 | |
1400 | bool Preprocessor::parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value) { |
1401 | assert(Tok.is(tok::numeric_constant)); |
1402 | SmallString<8> IntegerBuffer; |
1403 | bool NumberInvalid = false; |
1404 | StringRef Spelling = getSpelling(Tok, Buffer&: IntegerBuffer, Invalid: &NumberInvalid); |
1405 | if (NumberInvalid) |
1406 | return false; |
1407 | NumericLiteralParser Literal(Spelling, Tok.getLocation(), getSourceManager(), |
1408 | getLangOpts(), getTargetInfo(), |
1409 | getDiagnostics()); |
1410 | if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix()) |
1411 | return false; |
1412 | llvm::APInt APVal(64, 0); |
1413 | if (Literal.GetIntegerValue(Val&: APVal)) |
1414 | return false; |
1415 | Lex(Result&: Tok); |
1416 | Value = APVal.getLimitedValue(); |
1417 | return true; |
1418 | } |
1419 | |
1420 | void Preprocessor::addCommentHandler(CommentHandler *Handler) { |
1421 | assert(Handler && "NULL comment handler"); |
1422 | assert(!llvm::is_contained(CommentHandlers, Handler) && |
1423 | "Comment handler already registered"); |
1424 | CommentHandlers.push_back(x: Handler); |
1425 | } |
1426 | |
1427 | void Preprocessor::removeCommentHandler(CommentHandler *Handler) { |
1428 | std::vector<CommentHandler *>::iterator Pos = |
1429 | llvm::find(Range&: CommentHandlers, Val: Handler); |
1430 | assert(Pos != CommentHandlers.end() && "Comment handler not registered"); |
1431 | CommentHandlers.erase(position: Pos); |
1432 | } |
1433 | |
1434 | bool Preprocessor::HandleComment(Token &result, SourceRange Comment) { |
1435 | bool AnyPendingTokens = false; |
1436 | for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(), |
1437 | HEnd = CommentHandlers.end(); |
1438 | H != HEnd; ++H) { |
1439 | if ((*H)->HandleComment(PP&: *this, Comment)) |
1440 | AnyPendingTokens = true; |
1441 | } |
1442 | if (!AnyPendingTokens || getCommentRetentionState()) |
1443 | return false; |
1444 | Lex(Result&: result); |
1445 | return true; |
1446 | } |
1447 | |
1448 | void Preprocessor::emitMacroDeprecationWarning(const Token &Identifier) const { |
1449 | const MacroAnnotations &A = |
1450 | getMacroAnnotations(II: Identifier.getIdentifierInfo()); |
1451 | assert(A.DeprecationInfo && |
1452 | "Macro deprecation warning without recorded annotation!"); |
1453 | const MacroAnnotationInfo &Info = *A.DeprecationInfo; |
1454 | if (Info.Message.empty()) |
1455 | Diag(Identifier, diag::warn_pragma_deprecated_macro_use) |
1456 | << Identifier.getIdentifierInfo() << 0; |
1457 | else |
1458 | Diag(Identifier, diag::warn_pragma_deprecated_macro_use) |
1459 | << Identifier.getIdentifierInfo() << 1 << Info.Message; |
1460 | Diag(Info.Location, diag::note_pp_macro_annotation) << 0; |
1461 | } |
1462 | |
1463 | void Preprocessor::emitRestrictExpansionWarning(const Token &Identifier) const { |
1464 | const MacroAnnotations &A = |
1465 | getMacroAnnotations(II: Identifier.getIdentifierInfo()); |
1466 | assert(A.RestrictExpansionInfo && |
1467 | "Macro restricted expansion warning without recorded annotation!"); |
1468 | const MacroAnnotationInfo &Info = *A.RestrictExpansionInfo; |
1469 | if (Info.Message.empty()) |
1470 | Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use) |
1471 | << Identifier.getIdentifierInfo() << 0; |
1472 | else |
1473 | Diag(Identifier, diag::warn_pragma_restrict_expansion_macro_use) |
1474 | << Identifier.getIdentifierInfo() << 1 << Info.Message; |
1475 | Diag(Info.Location, diag::note_pp_macro_annotation) << 1; |
1476 | } |
1477 | |
1478 | void Preprocessor::emitRestrictInfNaNWarning(const Token &Identifier, |
1479 | unsigned DiagSelection) const { |
1480 | Diag(Identifier, diag::warn_fp_nan_inf_when_disabled) << DiagSelection << 1; |
1481 | } |
1482 | |
1483 | void Preprocessor::emitFinalMacroWarning(const Token &Identifier, |
1484 | bool IsUndef) const { |
1485 | const MacroAnnotations &A = |
1486 | getMacroAnnotations(II: Identifier.getIdentifierInfo()); |
1487 | assert(A.FinalAnnotationLoc && |
1488 | "Final macro warning without recorded annotation!"); |
1489 | |
1490 | Diag(Identifier, diag::warn_pragma_final_macro) |
1491 | << Identifier.getIdentifierInfo() << (IsUndef ? 0 : 1); |
1492 | Diag(*A.FinalAnnotationLoc, diag::note_pp_macro_annotation) << 2; |
1493 | } |
1494 | |
1495 | bool Preprocessor::isSafeBufferOptOut(const SourceManager &SourceMgr, |
1496 | const SourceLocation &Loc) const { |
1497 | // The lambda that tests if a `Loc` is in an opt-out region given one opt-out |
1498 | // region map: |
1499 | auto TestInMap = [&SourceMgr](const SafeBufferOptOutRegionsTy &Map, |
1500 | const SourceLocation &Loc) -> bool { |
1501 | // Try to find a region in `SafeBufferOptOutMap` where `Loc` is in: |
1502 | auto FirstRegionEndingAfterLoc = llvm::partition_point( |
1503 | Range: Map, P: [&SourceMgr, |
1504 | &Loc](const std::pair<SourceLocation, SourceLocation> &Region) { |
1505 | return SourceMgr.isBeforeInTranslationUnit(LHS: Region.second, RHS: Loc); |
1506 | }); |
1507 | |
1508 | if (FirstRegionEndingAfterLoc != Map.end()) { |
1509 | // To test if the start location of the found region precedes `Loc`: |
1510 | return SourceMgr.isBeforeInTranslationUnit( |
1511 | LHS: FirstRegionEndingAfterLoc->first, RHS: Loc); |
1512 | } |
1513 | // If we do not find a region whose end location passes `Loc`, we want to |
1514 | // check if the current region is still open: |
1515 | if (!Map.empty() && Map.back().first == Map.back().second) |
1516 | return SourceMgr.isBeforeInTranslationUnit(LHS: Map.back().first, RHS: Loc); |
1517 | return false; |
1518 | }; |
1519 | |
1520 | // What the following does: |
1521 | // |
1522 | // If `Loc` belongs to the local TU, we just look up `SafeBufferOptOutMap`. |
1523 | // Otherwise, `Loc` is from a loaded AST. We look up the |
1524 | // `LoadedSafeBufferOptOutMap` first to get the opt-out region map of the |
1525 | // loaded AST where `Loc` is at. Then we find if `Loc` is in an opt-out |
1526 | // region w.r.t. the region map. If the region map is absent, it means there |
1527 | // is no opt-out pragma in that loaded AST. |
1528 | // |
1529 | // Opt-out pragmas in the local TU or a loaded AST is not visible to another |
1530 | // one of them. That means if you put the pragmas around a `#include |
1531 | // "module.h"`, where module.h is a module, it is not actually suppressing |
1532 | // warnings in module.h. This is fine because warnings in module.h will be |
1533 | // reported when module.h is compiled in isolation and nothing in module.h |
1534 | // will be analyzed ever again. So you will not see warnings from the file |
1535 | // that imports module.h anyway. And you can't even do the same thing for PCHs |
1536 | // because they can only be included from the command line. |
1537 | |
1538 | if (SourceMgr.isLocalSourceLocation(Loc)) |
1539 | return TestInMap(SafeBufferOptOutMap, Loc); |
1540 | |
1541 | const SafeBufferOptOutRegionsTy *LoadedRegions = |
1542 | LoadedSafeBufferOptOutMap.lookupLoadedOptOutMap(Loc, SrcMgr: SourceMgr); |
1543 | |
1544 | if (LoadedRegions) |
1545 | return TestInMap(*LoadedRegions, Loc); |
1546 | return false; |
1547 | } |
1548 | |
1549 | bool Preprocessor::enterOrExitSafeBufferOptOutRegion( |
1550 | bool isEnter, const SourceLocation &Loc) { |
1551 | if (isEnter) { |
1552 | if (isPPInSafeBufferOptOutRegion()) |
1553 | return true; // invalid enter action |
1554 | InSafeBufferOptOutRegion = true; |
1555 | CurrentSafeBufferOptOutStart = Loc; |
1556 | |
1557 | // To set the start location of a new region: |
1558 | |
1559 | if (!SafeBufferOptOutMap.empty()) { |
1560 | [[maybe_unused]] auto *PrevRegion = &SafeBufferOptOutMap.back(); |
1561 | assert(PrevRegion->first != PrevRegion->second && |
1562 | "Shall not begin a safe buffer opt-out region before closing the " |
1563 | "previous one."); |
1564 | } |
1565 | // If the start location equals to the end location, we call the region a |
1566 | // open region or a unclosed region (i.e., end location has not been set |
1567 | // yet). |
1568 | SafeBufferOptOutMap.emplace_back(Args: Loc, Args: Loc); |
1569 | } else { |
1570 | if (!isPPInSafeBufferOptOutRegion()) |
1571 | return true; // invalid enter action |
1572 | InSafeBufferOptOutRegion = false; |
1573 | |
1574 | // To set the end location of the current open region: |
1575 | |
1576 | assert(!SafeBufferOptOutMap.empty() && |
1577 | "Misordered safe buffer opt-out regions"); |
1578 | auto *CurrRegion = &SafeBufferOptOutMap.back(); |
1579 | assert(CurrRegion->first == CurrRegion->second && |
1580 | "Set end location to a closed safe buffer opt-out region"); |
1581 | CurrRegion->second = Loc; |
1582 | } |
1583 | return false; |
1584 | } |
1585 | |
1586 | bool Preprocessor::isPPInSafeBufferOptOutRegion() { |
1587 | return InSafeBufferOptOutRegion; |
1588 | } |
1589 | bool Preprocessor::isPPInSafeBufferOptOutRegion(SourceLocation &StartLoc) { |
1590 | StartLoc = CurrentSafeBufferOptOutStart; |
1591 | return InSafeBufferOptOutRegion; |
1592 | } |
1593 | |
1594 | SmallVector<SourceLocation, 64> |
1595 | Preprocessor::serializeSafeBufferOptOutMap() const { |
1596 | assert(!InSafeBufferOptOutRegion && |
1597 | "Attempt to serialize safe buffer opt-out regions before file being " |
1598 | "completely preprocessed"); |
1599 | |
1600 | SmallVector<SourceLocation, 64> SrcSeq; |
1601 | |
1602 | for (const auto &[begin, end] : SafeBufferOptOutMap) { |
1603 | SrcSeq.push_back(Elt: begin); |
1604 | SrcSeq.push_back(Elt: end); |
1605 | } |
1606 | // Only `SafeBufferOptOutMap` gets serialized. No need to serialize |
1607 | // `LoadedSafeBufferOptOutMap` because if this TU loads a pch/module, every |
1608 | // pch/module in the pch-chain/module-DAG will be loaded one by one in order. |
1609 | // It means that for each loading pch/module m, it just needs to load m's own |
1610 | // `SafeBufferOptOutMap`. |
1611 | return SrcSeq; |
1612 | } |
1613 | |
1614 | bool Preprocessor::setDeserializedSafeBufferOptOutMap( |
1615 | const SmallVectorImpl<SourceLocation> &SourceLocations) { |
1616 | if (SourceLocations.size() == 0) |
1617 | return false; |
1618 | |
1619 | assert(SourceLocations.size() % 2 == 0 && |
1620 | "ill-formed SourceLocation sequence"); |
1621 | |
1622 | auto It = SourceLocations.begin(); |
1623 | SafeBufferOptOutRegionsTy &Regions = |
1624 | LoadedSafeBufferOptOutMap.findAndConsLoadedOptOutMap(Loc: *It, SrcMgr&: SourceMgr); |
1625 | |
1626 | do { |
1627 | SourceLocation Begin = *It++; |
1628 | SourceLocation End = *It++; |
1629 | |
1630 | Regions.emplace_back(Args&: Begin, Args&: End); |
1631 | } while (It != SourceLocations.end()); |
1632 | return true; |
1633 | } |
1634 | |
1635 | ModuleLoader::~ModuleLoader() = default; |
1636 | |
1637 | CommentHandler::~CommentHandler() = default; |
1638 | |
1639 | EmptylineHandler::~EmptylineHandler() = default; |
1640 | |
1641 | CodeCompletionHandler::~CodeCompletionHandler() = default; |
1642 | |
1643 | void Preprocessor::createPreprocessingRecord() { |
1644 | if (Record) |
1645 | return; |
1646 | |
1647 | Record = new PreprocessingRecord(getSourceManager()); |
1648 | addPPCallbacks(C: std::unique_ptr<PPCallbacks>(Record)); |
1649 | } |
1650 | |
1651 | const char *Preprocessor::getCheckPoint(FileID FID, const char *Start) const { |
1652 | if (auto It = CheckPoints.find(Val: FID); It != CheckPoints.end()) { |
1653 | const SmallVector<const char *> &FileCheckPoints = It->second; |
1654 | const char *Last = nullptr; |
1655 | // FIXME: Do better than a linear search. |
1656 | for (const char *P : FileCheckPoints) { |
1657 | if (P > Start) |
1658 | break; |
1659 | Last = P; |
1660 | } |
1661 | return Last; |
1662 | } |
1663 | |
1664 | return nullptr; |
1665 | } |
1666 |
Definitions
- CheckPointStepSize
- ~ExternalPreprocessorSource
- Preprocessor
- ~Preprocessor
- Initialize
- InitializeForModelFile
- FinalizeForModelFile
- DumpToken
- DumpLocation
- DumpMacro
- PrintStats
- macro_begin
- getTotalMemory
- macro_end
- MacroDefinitionEquals
- getLastMacroWithSpelling
- recomputeCurLexerKind
- SetCodeCompletionPoint
- CodeCompleteIncludedFile
- CodeCompleteNaturalLanguage
- getSpelling
- CreateString
- SplitToken
- getCurrentModule
- getCurrentModuleImplementation
- EnterMainSourceFile
- setPCHThroughHeaderFileID
- isPCHThroughHeader
- creatingPCHWithThroughHeader
- usingPCHWithThroughHeader
- creatingPCHWithPragmaHdrStop
- usingPCHWithPragmaHdrStop
- SkipTokensWhileUsingPCH
- replayPreambleConditionalStack
- EndSourceFile
- LookUpIdentifierInfo
- SetPoisonReason
- PoisonSEHIdentifiers
- HandlePoisonedIdentifier
- updateOutOfDateIdentifier
- HandleIdentifier
- Lex
- LexTokensUntilEOF
- LexHeaderName
- CollectPpImportSuffix
- LexAfterModuleImport
- makeModuleVisible
- FinishLexStringLiteral
- parseSimpleIntegerLiteral
- addCommentHandler
- removeCommentHandler
- HandleComment
- emitMacroDeprecationWarning
- emitRestrictExpansionWarning
- emitRestrictInfNaNWarning
- emitFinalMacroWarning
- isSafeBufferOptOut
- enterOrExitSafeBufferOptOutRegion
- isPPInSafeBufferOptOutRegion
- isPPInSafeBufferOptOutRegion
- serializeSafeBufferOptOutMap
- setDeserializedSafeBufferOptOutMap
- ~ModuleLoader
- ~CommentHandler
- ~EmptylineHandler
- ~CodeCompletionHandler
- createPreprocessingRecord
Learn to use CMake with our Intro Training
Find out more