1 | //===- TokenLexer.cpp - Lex from a token stream ---------------------------===// |
---|---|
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file implements the TokenLexer interface. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "clang/Lex/TokenLexer.h" |
14 | #include "clang/Basic/Diagnostic.h" |
15 | #include "clang/Basic/IdentifierTable.h" |
16 | #include "clang/Basic/LangOptions.h" |
17 | #include "clang/Basic/SourceLocation.h" |
18 | #include "clang/Basic/SourceManager.h" |
19 | #include "clang/Basic/TokenKinds.h" |
20 | #include "clang/Lex/LexDiagnostic.h" |
21 | #include "clang/Lex/Lexer.h" |
22 | #include "clang/Lex/MacroArgs.h" |
23 | #include "clang/Lex/MacroInfo.h" |
24 | #include "clang/Lex/Preprocessor.h" |
25 | #include "clang/Lex/Token.h" |
26 | #include "clang/Lex/VariadicMacroSupport.h" |
27 | #include "llvm/ADT/ArrayRef.h" |
28 | #include "llvm/ADT/STLExtras.h" |
29 | #include "llvm/ADT/SmallVector.h" |
30 | #include "llvm/ADT/iterator_range.h" |
31 | #include <cassert> |
32 | #include <cstring> |
33 | #include <optional> |
34 | |
35 | using namespace clang; |
36 | |
37 | /// Create a TokenLexer for the specified macro with the specified actual |
38 | /// arguments. Note that this ctor takes ownership of the ActualArgs pointer. |
39 | void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, |
40 | MacroArgs *Actuals) { |
41 | // If the client is reusing a TokenLexer, make sure to free any memory |
42 | // associated with it. |
43 | destroy(); |
44 | |
45 | Macro = MI; |
46 | ActualArgs = Actuals; |
47 | CurTokenIdx = 0; |
48 | |
49 | ExpandLocStart = Tok.getLocation(); |
50 | ExpandLocEnd = ELEnd; |
51 | AtStartOfLine = Tok.isAtStartOfLine(); |
52 | HasLeadingSpace = Tok.hasLeadingSpace(); |
53 | NextTokGetsSpace = false; |
54 | Tokens = &*Macro->tokens_begin(); |
55 | OwnsTokens = false; |
56 | DisableMacroExpansion = false; |
57 | IsReinject = false; |
58 | NumTokens = Macro->tokens_end()-Macro->tokens_begin(); |
59 | MacroExpansionStart = SourceLocation(); |
60 | |
61 | SourceManager &SM = PP.getSourceManager(); |
62 | MacroStartSLocOffset = SM.getNextLocalOffset(); |
63 | |
64 | if (NumTokens > 0) { |
65 | assert(Tokens[0].getLocation().isValid()); |
66 | assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) && |
67 | "Macro defined in macro?"); |
68 | assert(ExpandLocStart.isValid()); |
69 | |
70 | // Reserve a source location entry chunk for the length of the macro |
71 | // definition. Tokens that get lexed directly from the definition will |
72 | // have their locations pointing inside this chunk. This is to avoid |
73 | // creating separate source location entries for each token. |
74 | MacroDefStart = SM.getExpansionLoc(Loc: Tokens[0].getLocation()); |
75 | MacroDefLength = Macro->getDefinitionLength(SM); |
76 | MacroExpansionStart = SM.createExpansionLoc(SpellingLoc: MacroDefStart, |
77 | ExpansionLocStart: ExpandLocStart, |
78 | ExpansionLocEnd: ExpandLocEnd, |
79 | Length: MacroDefLength); |
80 | } |
81 | |
82 | // If this is a function-like macro, expand the arguments and change |
83 | // Tokens to point to the expanded tokens. |
84 | if (Macro->isFunctionLike() && Macro->getNumParams()) |
85 | ExpandFunctionArguments(); |
86 | |
87 | // Mark the macro as currently disabled, so that it is not recursively |
88 | // expanded. The macro must be disabled only after argument pre-expansion of |
89 | // function-like macro arguments occurs. |
90 | Macro->DisableMacro(); |
91 | } |
92 | |
93 | /// Create a TokenLexer for the specified token stream. This does not |
94 | /// take ownership of the specified token vector. |
95 | void TokenLexer::Init(const Token *TokArray, unsigned NumToks, |
96 | bool disableMacroExpansion, bool ownsTokens, |
97 | bool isReinject) { |
98 | assert(!isReinject || disableMacroExpansion); |
99 | // If the client is reusing a TokenLexer, make sure to free any memory |
100 | // associated with it. |
101 | destroy(); |
102 | |
103 | Macro = nullptr; |
104 | ActualArgs = nullptr; |
105 | Tokens = TokArray; |
106 | OwnsTokens = ownsTokens; |
107 | DisableMacroExpansion = disableMacroExpansion; |
108 | IsReinject = isReinject; |
109 | NumTokens = NumToks; |
110 | CurTokenIdx = 0; |
111 | ExpandLocStart = ExpandLocEnd = SourceLocation(); |
112 | AtStartOfLine = false; |
113 | HasLeadingSpace = false; |
114 | NextTokGetsSpace = false; |
115 | MacroExpansionStart = SourceLocation(); |
116 | |
117 | // Set HasLeadingSpace/AtStartOfLine so that the first token will be |
118 | // returned unmodified. |
119 | if (NumToks != 0) { |
120 | AtStartOfLine = TokArray[0].isAtStartOfLine(); |
121 | HasLeadingSpace = TokArray[0].hasLeadingSpace(); |
122 | } |
123 | } |
124 | |
125 | void TokenLexer::destroy() { |
126 | // If this was a function-like macro that actually uses its arguments, delete |
127 | // the expanded tokens. |
128 | if (OwnsTokens) { |
129 | delete [] Tokens; |
130 | Tokens = nullptr; |
131 | OwnsTokens = false; |
132 | } |
133 | |
134 | // TokenLexer owns its formal arguments. |
135 | if (ActualArgs) ActualArgs->destroy(PP); |
136 | } |
137 | |
138 | bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( |
139 | SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro, |
140 | unsigned MacroArgNo, Preprocessor &PP) { |
141 | // Is the macro argument __VA_ARGS__? |
142 | if (!Macro->isVariadic() || MacroArgNo != Macro->getNumParams()-1) |
143 | return false; |
144 | |
145 | // In Microsoft-compatibility mode, a comma is removed in the expansion |
146 | // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is |
147 | // not supported by gcc. |
148 | if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat) |
149 | return false; |
150 | |
151 | // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if |
152 | // __VA_ARGS__ is empty, but not in strict C99 mode where there are no |
153 | // named arguments, where it remains. In all other modes, including C99 |
154 | // with GNU extensions, it is removed regardless of named arguments. |
155 | // Microsoft also appears to support this extension, unofficially. |
156 | if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode |
157 | && Macro->getNumParams() < 2) |
158 | return false; |
159 | |
160 | // Is a comma available to be removed? |
161 | if (ResultToks.empty() || !ResultToks.back().is(K: tok::comma)) |
162 | return false; |
163 | |
164 | // Issue an extension diagnostic for the paste operator. |
165 | if (HasPasteOperator) |
166 | PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); |
167 | |
168 | // Remove the comma. |
169 | ResultToks.pop_back(); |
170 | |
171 | if (!ResultToks.empty()) { |
172 | // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"), |
173 | // then removal of the comma should produce a placemarker token (in C99 |
174 | // terms) which we model by popping off the previous ##, giving us a plain |
175 | // "X" when __VA_ARGS__ is empty. |
176 | if (ResultToks.back().is(K: tok::hashhash)) |
177 | ResultToks.pop_back(); |
178 | |
179 | // Remember that this comma was elided. |
180 | ResultToks.back().setFlag(Token::CommaAfterElided); |
181 | } |
182 | |
183 | // Never add a space, even if the comma, ##, or arg had a space. |
184 | NextTokGetsSpace = false; |
185 | return true; |
186 | } |
187 | |
188 | void TokenLexer::stringifyVAOPTContents( |
189 | SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx, |
190 | const SourceLocation VAOPTClosingParenLoc) { |
191 | const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt(); |
192 | const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt; |
193 | Token *const VAOPTTokens = |
194 | NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr; |
195 | |
196 | SmallVector<Token, 64> ConcatenatedVAOPTResultToks; |
197 | // FIXME: Should we keep track within VCtx that we did or didnot |
198 | // encounter pasting - and only then perform this loop. |
199 | |
200 | // Perform token pasting (concatenation) prior to stringization. |
201 | for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens; |
202 | ++CurTokenIdx) { |
203 | if (VAOPTTokens[CurTokenIdx].is(K: tok::hashhash)) { |
204 | assert(CurTokenIdx != 0 && |
205 | "Can not have __VAOPT__ contents begin with a ##"); |
206 | Token &LHS = VAOPTTokens[CurTokenIdx - 1]; |
207 | pasteTokens(LHSTok&: LHS, TokenStream: llvm::ArrayRef(VAOPTTokens, NumVAOptTokens), |
208 | CurIdx&: CurTokenIdx); |
209 | // Replace the token prior to the first ## in this iteration. |
210 | ConcatenatedVAOPTResultToks.back() = LHS; |
211 | if (CurTokenIdx == NumVAOptTokens) |
212 | break; |
213 | } |
214 | ConcatenatedVAOPTResultToks.push_back(Elt: VAOPTTokens[CurTokenIdx]); |
215 | } |
216 | |
217 | ConcatenatedVAOPTResultToks.push_back(Elt: VCtx.getEOFTok()); |
218 | // Get the SourceLocation that represents the start location within |
219 | // the macro definition that marks where this string is substituted |
220 | // into: i.e. the __VA_OPT__ and the ')' within the spelling of the |
221 | // macro definition, and use it to indicate that the stringified token |
222 | // was generated from that location. |
223 | const SourceLocation ExpansionLocStartWithinMacro = |
224 | getExpansionLocForMacroDefLoc(loc: VCtx.getVAOptLoc()); |
225 | const SourceLocation ExpansionLocEndWithinMacro = |
226 | getExpansionLocForMacroDefLoc(loc: VAOPTClosingParenLoc); |
227 | |
228 | Token StringifiedVAOPT = MacroArgs::StringifyArgument( |
229 | ArgToks: &ConcatenatedVAOPTResultToks[0], PP, Charify: VCtx.hasCharifyBefore() /*Charify*/, |
230 | ExpansionLocStart: ExpansionLocStartWithinMacro, ExpansionLocEnd: ExpansionLocEndWithinMacro); |
231 | |
232 | if (VCtx.getLeadingSpaceForStringifiedToken()) |
233 | StringifiedVAOPT.setFlag(Token::LeadingSpace); |
234 | |
235 | StringifiedVAOPT.setFlag(Token::StringifiedInMacro); |
236 | // Resize (shrink) the token stream to just capture this stringified token. |
237 | ResultToks.resize(N: NumToksPriorToVAOpt + 1); |
238 | ResultToks.back() = StringifiedVAOPT; |
239 | } |
240 | |
241 | /// Expand the arguments of a function-like macro so that we can quickly |
242 | /// return preexpanded tokens from Tokens. |
243 | void TokenLexer::ExpandFunctionArguments() { |
244 | SmallVector<Token, 128> ResultToks; |
245 | |
246 | // Loop through 'Tokens', expanding them into ResultToks. Keep |
247 | // track of whether we change anything. If not, no need to keep them. If so, |
248 | // we install the newly expanded sequence as the new 'Tokens' list. |
249 | bool MadeChange = false; |
250 | |
251 | std::optional<bool> CalledWithVariadicArguments; |
252 | |
253 | VAOptExpansionContext VCtx(PP); |
254 | |
255 | for (unsigned I = 0, E = NumTokens; I != E; ++I) { |
256 | const Token &CurTok = Tokens[I]; |
257 | // We don't want a space for the next token after a paste |
258 | // operator. In valid code, the token will get smooshed onto the |
259 | // preceding one anyway. In assembler-with-cpp mode, invalid |
260 | // pastes are allowed through: in this case, we do not want the |
261 | // extra whitespace to be added. For example, we want ". ## foo" |
262 | // -> ".foo" not ". foo". |
263 | if (I != 0 && !Tokens[I-1].is(K: tok::hashhash) && CurTok.hasLeadingSpace()) |
264 | NextTokGetsSpace = true; |
265 | |
266 | if (VCtx.isVAOptToken(T: CurTok)) { |
267 | MadeChange = true; |
268 | assert(Tokens[I + 1].is(tok::l_paren) && |
269 | "__VA_OPT__ must be followed by '('"); |
270 | |
271 | ++I; // Skip the l_paren |
272 | VCtx.sawVAOptFollowedByOpeningParens(VAOptLoc: CurTok.getLocation(), |
273 | NumPriorTokens: ResultToks.size()); |
274 | |
275 | continue; |
276 | } |
277 | |
278 | // We have entered into the __VA_OPT__ context, so handle tokens |
279 | // appropriately. |
280 | if (VCtx.isInVAOpt()) { |
281 | // If we are about to process a token that is either an argument to |
282 | // __VA_OPT__ or its closing rparen, then: |
283 | // 1) If the token is the closing rparen that exits us out of __VA_OPT__, |
284 | // perform any necessary stringification or placemarker processing, |
285 | // and/or skip to the next token. |
286 | // 2) else if macro was invoked without variadic arguments skip this |
287 | // token. |
288 | // 3) else (macro was invoked with variadic arguments) process the token |
289 | // normally. |
290 | |
291 | if (Tokens[I].is(K: tok::l_paren)) |
292 | VCtx.sawOpeningParen(LParenLoc: Tokens[I].getLocation()); |
293 | // Continue skipping tokens within __VA_OPT__ if the macro was not |
294 | // called with variadic arguments, else let the rest of the loop handle |
295 | // this token. Note sawClosingParen() returns true only if the r_paren matches |
296 | // the closing r_paren of the __VA_OPT__. |
297 | if (!Tokens[I].is(K: tok::r_paren) || !VCtx.sawClosingParen()) { |
298 | // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__. |
299 | if (!CalledWithVariadicArguments) { |
300 | CalledWithVariadicArguments = |
301 | ActualArgs->invokedWithVariadicArgument(MI: Macro, PP); |
302 | } |
303 | if (!*CalledWithVariadicArguments) { |
304 | // Skip this token. |
305 | continue; |
306 | } |
307 | // ... else the macro was called with variadic arguments, and we do not |
308 | // have a closing rparen - so process this token normally. |
309 | } else { |
310 | // Current token is the closing r_paren which marks the end of the |
311 | // __VA_OPT__ invocation, so handle any place-marker pasting (if |
312 | // empty) by removing hashhash either before (if exists) or after. And |
313 | // also stringify the entire contents if VAOPT was preceded by a hash, |
314 | // but do so only after any token concatenation that needs to occur |
315 | // within the contents of VAOPT. |
316 | |
317 | if (VCtx.hasStringifyOrCharifyBefore()) { |
318 | // Replace all the tokens just added from within VAOPT into a single |
319 | // stringified token. This requires token-pasting to eagerly occur |
320 | // within these tokens. If either the contents of VAOPT were empty |
321 | // or the macro wasn't called with any variadic arguments, the result |
322 | // is a token that represents an empty string. |
323 | stringifyVAOPTContents(ResultToks, VCtx, |
324 | /*ClosingParenLoc*/ VAOPTClosingParenLoc: Tokens[I].getLocation()); |
325 | |
326 | } else if (/*No tokens within VAOPT*/ |
327 | ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) { |
328 | // Treat VAOPT as a placemarker token. Eat either the '##' before the |
329 | // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that |
330 | // hashhash was not a placemarker) or the '##' |
331 | // after VAOPT, but not both. |
332 | |
333 | if (ResultToks.size() && ResultToks.back().is(K: tok::hashhash)) { |
334 | ResultToks.pop_back(); |
335 | } else if ((I + 1 != E) && Tokens[I + 1].is(K: tok::hashhash)) { |
336 | ++I; // Skip the following hashhash. |
337 | } |
338 | } else { |
339 | // If there's a ## before the __VA_OPT__, we might have discovered |
340 | // that the __VA_OPT__ begins with a placeholder. We delay action on |
341 | // that to now to avoid messing up our stashed count of tokens before |
342 | // __VA_OPT__. |
343 | if (VCtx.beginsWithPlaceholder()) { |
344 | assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 && |
345 | ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() && |
346 | ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is( |
347 | tok::hashhash) && |
348 | "no token paste before __VA_OPT__"); |
349 | ResultToks.erase(CI: ResultToks.begin() + |
350 | VCtx.getNumberOfTokensPriorToVAOpt() - 1); |
351 | } |
352 | // If the expansion of __VA_OPT__ ends with a placeholder, eat any |
353 | // following '##' token. |
354 | if (VCtx.endsWithPlaceholder() && I + 1 != E && |
355 | Tokens[I + 1].is(K: tok::hashhash)) { |
356 | ++I; |
357 | } |
358 | } |
359 | VCtx.reset(); |
360 | // We processed __VA_OPT__'s closing paren (and the exit out of |
361 | // __VA_OPT__), so skip to the next token. |
362 | continue; |
363 | } |
364 | } |
365 | |
366 | // If we found the stringify operator, get the argument stringified. The |
367 | // preprocessor already verified that the following token is a macro |
368 | // parameter or __VA_OPT__ when the #define was lexed. |
369 | |
370 | if (CurTok.isOneOf(K1: tok::hash, K2: tok::hashat)) { |
371 | int ArgNo = Macro->getParameterNum(Arg: Tokens[I+1].getIdentifierInfo()); |
372 | assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) && |
373 | "Token following # is not an argument or __VA_OPT__!"); |
374 | |
375 | if (ArgNo == -1) { |
376 | // Handle the __VA_OPT__ case. |
377 | VCtx.sawHashOrHashAtBefore(HasLeadingSpace: NextTokGetsSpace, |
378 | IsHashAt: CurTok.is(K: tok::hashat)); |
379 | continue; |
380 | } |
381 | // Else handle the simple argument case. |
382 | SourceLocation ExpansionLocStart = |
383 | getExpansionLocForMacroDefLoc(loc: CurTok.getLocation()); |
384 | SourceLocation ExpansionLocEnd = |
385 | getExpansionLocForMacroDefLoc(loc: Tokens[I+1].getLocation()); |
386 | |
387 | bool Charify = CurTok.is(K: tok::hashat); |
388 | const Token *UnexpArg = ActualArgs->getUnexpArgument(Arg: ArgNo); |
389 | Token Res = MacroArgs::StringifyArgument( |
390 | ArgToks: UnexpArg, PP, Charify, ExpansionLocStart, ExpansionLocEnd); |
391 | Res.setFlag(Token::StringifiedInMacro); |
392 | |
393 | // The stringified/charified string leading space flag gets set to match |
394 | // the #/#@ operator. |
395 | if (NextTokGetsSpace) |
396 | Res.setFlag(Token::LeadingSpace); |
397 | |
398 | ResultToks.push_back(Elt: Res); |
399 | MadeChange = true; |
400 | ++I; // Skip arg name. |
401 | NextTokGetsSpace = false; |
402 | continue; |
403 | } |
404 | |
405 | // Find out if there is a paste (##) operator before or after the token. |
406 | bool NonEmptyPasteBefore = |
407 | !ResultToks.empty() && ResultToks.back().is(K: tok::hashhash); |
408 | bool PasteBefore = I != 0 && Tokens[I-1].is(K: tok::hashhash); |
409 | bool PasteAfter = I+1 != E && Tokens[I+1].is(K: tok::hashhash); |
410 | bool RParenAfter = I+1 != E && Tokens[I+1].is(K: tok::r_paren); |
411 | |
412 | assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) && |
413 | "unexpected ## in ResultToks"); |
414 | |
415 | // Otherwise, if this is not an argument token, just add the token to the |
416 | // output buffer. |
417 | IdentifierInfo *II = CurTok.getIdentifierInfo(); |
418 | int ArgNo = II ? Macro->getParameterNum(Arg: II) : -1; |
419 | if (ArgNo == -1) { |
420 | // This isn't an argument, just add it. |
421 | ResultToks.push_back(Elt: CurTok); |
422 | |
423 | if (NextTokGetsSpace) { |
424 | ResultToks.back().setFlag(Token::LeadingSpace); |
425 | NextTokGetsSpace = false; |
426 | } else if (PasteBefore && !NonEmptyPasteBefore) |
427 | ResultToks.back().clearFlag(Flag: Token::LeadingSpace); |
428 | |
429 | continue; |
430 | } |
431 | |
432 | // An argument is expanded somehow, the result is different than the |
433 | // input. |
434 | MadeChange = true; |
435 | |
436 | // Otherwise, this is a use of the argument. |
437 | |
438 | // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there |
439 | // are no trailing commas if __VA_ARGS__ is empty. |
440 | if (!PasteBefore && ActualArgs->isVarargsElidedUse() && |
441 | MaybeRemoveCommaBeforeVaArgs(ResultToks, |
442 | /*HasPasteOperator=*/false, |
443 | Macro, MacroArgNo: ArgNo, PP)) |
444 | continue; |
445 | |
446 | // If it is not the LHS/RHS of a ## operator, we must pre-expand the |
447 | // argument and substitute the expanded tokens into the result. This is |
448 | // C99 6.10.3.1p1. |
449 | if (!PasteBefore && !PasteAfter) { |
450 | const Token *ResultArgToks; |
451 | |
452 | // Only preexpand the argument if it could possibly need it. This |
453 | // avoids some work in common cases. |
454 | const Token *ArgTok = ActualArgs->getUnexpArgument(Arg: ArgNo); |
455 | if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) |
456 | ResultArgToks = &ActualArgs->getPreExpArgument(Arg: ArgNo, PP)[0]; |
457 | else |
458 | ResultArgToks = ArgTok; // Use non-preexpanded tokens. |
459 | |
460 | // If the arg token expanded into anything, append it. |
461 | if (ResultArgToks->isNot(K: tok::eof)) { |
462 | size_t FirstResult = ResultToks.size(); |
463 | unsigned NumToks = MacroArgs::getArgLength(ArgPtr: ResultArgToks); |
464 | ResultToks.append(in_start: ResultArgToks, in_end: ResultArgToks+NumToks); |
465 | |
466 | // In Microsoft-compatibility mode, we follow MSVC's preprocessing |
467 | // behavior by not considering single commas from nested macro |
468 | // expansions as argument separators. Set a flag on the token so we can |
469 | // test for this later when the macro expansion is processed. |
470 | if (PP.getLangOpts().MSVCCompat && NumToks == 1 && |
471 | ResultToks.back().is(K: tok::comma)) |
472 | ResultToks.back().setFlag(Token::IgnoredComma); |
473 | |
474 | // If the '##' came from expanding an argument, turn it into 'unknown' |
475 | // to avoid pasting. |
476 | for (Token &Tok : llvm::drop_begin(RangeOrContainer&: ResultToks, N: FirstResult)) |
477 | if (Tok.is(K: tok::hashhash)) |
478 | Tok.setKind(tok::unknown); |
479 | |
480 | if(ExpandLocStart.isValid()) { |
481 | updateLocForMacroArgTokens(ArgIdSpellLoc: CurTok.getLocation(), |
482 | begin_tokens: ResultToks.begin()+FirstResult, |
483 | end_tokens: ResultToks.end()); |
484 | } |
485 | |
486 | // If any tokens were substituted from the argument, the whitespace |
487 | // before the first token should match the whitespace of the arg |
488 | // identifier. |
489 | ResultToks[FirstResult].setFlagValue(Flag: Token::LeadingSpace, |
490 | Val: NextTokGetsSpace); |
491 | ResultToks[FirstResult].setFlagValue(Flag: Token::StartOfLine, Val: false); |
492 | NextTokGetsSpace = false; |
493 | } else { |
494 | // We're creating a placeholder token. Usually this doesn't matter, |
495 | // but it can affect paste behavior when at the start or end of a |
496 | // __VA_OPT__. |
497 | if (NonEmptyPasteBefore) { |
498 | // We're imagining a placeholder token is inserted here. If this is |
499 | // the first token in a __VA_OPT__ after a ##, delete the ##. |
500 | assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__"); |
501 | VCtx.hasPlaceholderAfterHashhashAtStart(); |
502 | } else if (RParenAfter) |
503 | VCtx.hasPlaceholderBeforeRParen(); |
504 | } |
505 | continue; |
506 | } |
507 | |
508 | // Okay, we have a token that is either the LHS or RHS of a paste (##) |
509 | // argument. It gets substituted as its non-pre-expanded tokens. |
510 | const Token *ArgToks = ActualArgs->getUnexpArgument(Arg: ArgNo); |
511 | unsigned NumToks = MacroArgs::getArgLength(ArgPtr: ArgToks); |
512 | if (NumToks) { // Not an empty argument? |
513 | bool VaArgsPseudoPaste = false; |
514 | // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned |
515 | // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when |
516 | // the expander tries to paste ',' with the first token of the __VA_ARGS__ |
517 | // expansion. |
518 | if (NonEmptyPasteBefore && ResultToks.size() >= 2 && |
519 | ResultToks[ResultToks.size()-2].is(K: tok::comma) && |
520 | (unsigned)ArgNo == Macro->getNumParams()-1 && |
521 | Macro->isVariadic()) { |
522 | VaArgsPseudoPaste = true; |
523 | // Remove the paste operator, report use of the extension. |
524 | PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma); |
525 | } |
526 | |
527 | ResultToks.append(in_start: ArgToks, in_end: ArgToks+NumToks); |
528 | |
529 | // If the '##' came from expanding an argument, turn it into 'unknown' |
530 | // to avoid pasting. |
531 | for (Token &Tok : llvm::make_range(x: ResultToks.end() - NumToks, |
532 | y: ResultToks.end())) { |
533 | if (Tok.is(K: tok::hashhash)) |
534 | Tok.setKind(tok::unknown); |
535 | } |
536 | |
537 | if (ExpandLocStart.isValid()) { |
538 | updateLocForMacroArgTokens(ArgIdSpellLoc: CurTok.getLocation(), |
539 | begin_tokens: ResultToks.end()-NumToks, end_tokens: ResultToks.end()); |
540 | } |
541 | |
542 | // Transfer the leading whitespace information from the token |
543 | // (the macro argument) onto the first token of the |
544 | // expansion. Note that we don't do this for the GNU |
545 | // pseudo-paste extension ", ## __VA_ARGS__". |
546 | if (!VaArgsPseudoPaste) { |
547 | ResultToks[ResultToks.size() - NumToks].setFlagValue(Flag: Token::StartOfLine, |
548 | Val: false); |
549 | ResultToks[ResultToks.size() - NumToks].setFlagValue( |
550 | Flag: Token::LeadingSpace, Val: NextTokGetsSpace); |
551 | } |
552 | |
553 | NextTokGetsSpace = false; |
554 | continue; |
555 | } |
556 | |
557 | // If an empty argument is on the LHS or RHS of a paste, the standard (C99 |
558 | // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We |
559 | // implement this by eating ## operators when a LHS or RHS expands to |
560 | // empty. |
561 | if (PasteAfter) { |
562 | // Discard the argument token and skip (don't copy to the expansion |
563 | // buffer) the paste operator after it. |
564 | ++I; |
565 | continue; |
566 | } |
567 | |
568 | if (RParenAfter && !NonEmptyPasteBefore) |
569 | VCtx.hasPlaceholderBeforeRParen(); |
570 | |
571 | // If this is on the RHS of a paste operator, we've already copied the |
572 | // paste operator to the ResultToks list, unless the LHS was empty too. |
573 | // Remove it. |
574 | assert(PasteBefore); |
575 | if (NonEmptyPasteBefore) { |
576 | assert(ResultToks.back().is(tok::hashhash)); |
577 | // Do not remove the paste operator if it is the one before __VA_OPT__ |
578 | // (and we are still processing tokens within VA_OPT). We handle the case |
579 | // of removing the paste operator if __VA_OPT__ reduces to the notional |
580 | // placemarker above when we encounter the closing paren of VA_OPT. |
581 | if (!VCtx.isInVAOpt() || |
582 | ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt()) |
583 | ResultToks.pop_back(); |
584 | else |
585 | VCtx.hasPlaceholderAfterHashhashAtStart(); |
586 | } |
587 | |
588 | // If this is the __VA_ARGS__ token, and if the argument wasn't provided, |
589 | // and if the macro had at least one real argument, and if the token before |
590 | // the ## was a comma, remove the comma. This is a GCC extension which is |
591 | // disabled when using -std=c99. |
592 | if (ActualArgs->isVarargsElidedUse()) |
593 | MaybeRemoveCommaBeforeVaArgs(ResultToks, |
594 | /*HasPasteOperator=*/true, |
595 | Macro, MacroArgNo: ArgNo, PP); |
596 | } |
597 | |
598 | // If anything changed, install this as the new Tokens list. |
599 | if (MadeChange) { |
600 | assert(!OwnsTokens && "This would leak if we already own the token list"); |
601 | // This is deleted in the dtor. |
602 | NumTokens = ResultToks.size(); |
603 | // The tokens will be added to Preprocessor's cache and will be removed |
604 | // when this TokenLexer finishes lexing them. |
605 | Tokens = PP.cacheMacroExpandedTokens(tokLexer: this, tokens: ResultToks); |
606 | |
607 | // The preprocessor cache of macro expanded tokens owns these tokens,not us. |
608 | OwnsTokens = false; |
609 | } |
610 | } |
611 | |
612 | /// Checks if two tokens form wide string literal. |
613 | static bool isWideStringLiteralFromMacro(const Token &FirstTok, |
614 | const Token &SecondTok) { |
615 | return FirstTok.is(K: tok::identifier) && |
616 | FirstTok.getIdentifierInfo()->isStr(Str: "L") && SecondTok.isLiteral() && |
617 | SecondTok.stringifiedInMacro(); |
618 | } |
619 | |
620 | /// Lex - Lex and return a token from this macro stream. |
621 | bool TokenLexer::Lex(Token &Tok) { |
622 | // Lexing off the end of the macro, pop this macro off the expansion stack. |
623 | if (isAtEnd()) { |
624 | // If this is a macro (not a token stream), mark the macro enabled now |
625 | // that it is no longer being expanded. |
626 | if (Macro) Macro->EnableMacro(); |
627 | |
628 | Tok.startToken(); |
629 | Tok.setFlagValue(Flag: Token::StartOfLine , Val: AtStartOfLine); |
630 | Tok.setFlagValue(Flag: Token::LeadingSpace, Val: HasLeadingSpace || NextTokGetsSpace); |
631 | if (CurTokenIdx == 0) |
632 | Tok.setFlag(Token::LeadingEmptyMacro); |
633 | return PP.HandleEndOfTokenLexer(Result&: Tok); |
634 | } |
635 | |
636 | SourceManager &SM = PP.getSourceManager(); |
637 | |
638 | // If this is the first token of the expanded result, we inherit spacing |
639 | // properties later. |
640 | bool isFirstToken = CurTokenIdx == 0; |
641 | |
642 | // Get the next token to return. |
643 | Tok = Tokens[CurTokenIdx++]; |
644 | if (IsReinject) |
645 | Tok.setFlag(Token::IsReinjected); |
646 | |
647 | bool TokenIsFromPaste = false; |
648 | |
649 | // If this token is followed by a token paste (##) operator, paste the tokens! |
650 | // Note that ## is a normal token when not expanding a macro. |
651 | if (!isAtEnd() && Macro && |
652 | (Tokens[CurTokenIdx].is(K: tok::hashhash) || |
653 | // Special processing of L#x macros in -fms-compatibility mode. |
654 | // Microsoft compiler is able to form a wide string literal from |
655 | // 'L#macro_arg' construct in a function-like macro. |
656 | (PP.getLangOpts().MSVCCompat && |
657 | isWideStringLiteralFromMacro(FirstTok: Tok, SecondTok: Tokens[CurTokenIdx])))) { |
658 | // When handling the microsoft /##/ extension, the final token is |
659 | // returned by pasteTokens, not the pasted token. |
660 | if (pasteTokens(Tok)) |
661 | return true; |
662 | |
663 | TokenIsFromPaste = true; |
664 | } |
665 | |
666 | // The token's current location indicate where the token was lexed from. We |
667 | // need this information to compute the spelling of the token, but any |
668 | // diagnostics for the expanded token should appear as if they came from |
669 | // ExpansionLoc. Pull this information together into a new SourceLocation |
670 | // that captures all of this. |
671 | if (ExpandLocStart.isValid() && // Don't do this for token streams. |
672 | // Check that the token's location was not already set properly. |
673 | SM.isBeforeInSLocAddrSpace(LHS: Tok.getLocation(), RHS: MacroStartSLocOffset)) { |
674 | SourceLocation instLoc; |
675 | if (Tok.is(K: tok::comment)) { |
676 | instLoc = SM.createExpansionLoc(SpellingLoc: Tok.getLocation(), |
677 | ExpansionLocStart: ExpandLocStart, |
678 | ExpansionLocEnd: ExpandLocEnd, |
679 | Length: Tok.getLength()); |
680 | } else { |
681 | instLoc = getExpansionLocForMacroDefLoc(loc: Tok.getLocation()); |
682 | } |
683 | |
684 | Tok.setLocation(instLoc); |
685 | } |
686 | |
687 | // If this is the first token, set the lexical properties of the token to |
688 | // match the lexical properties of the macro identifier. |
689 | if (isFirstToken) { |
690 | Tok.setFlagValue(Flag: Token::StartOfLine , Val: AtStartOfLine); |
691 | Tok.setFlagValue(Flag: Token::LeadingSpace, Val: HasLeadingSpace); |
692 | } else { |
693 | // If this is not the first token, we may still need to pass through |
694 | // leading whitespace if we've expanded a macro. |
695 | if (AtStartOfLine) Tok.setFlag(Token::StartOfLine); |
696 | if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace); |
697 | } |
698 | AtStartOfLine = false; |
699 | HasLeadingSpace = false; |
700 | |
701 | // Handle recursive expansion! |
702 | if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) { |
703 | // Change the kind of this identifier to the appropriate token kind, e.g. |
704 | // turning "for" into a keyword. |
705 | IdentifierInfo *II = Tok.getIdentifierInfo(); |
706 | Tok.setKind(II->getTokenID()); |
707 | |
708 | // If this identifier was poisoned and from a paste, emit an error. This |
709 | // won't be handled by Preprocessor::HandleIdentifier because this is coming |
710 | // from a macro expansion. |
711 | if (II->isPoisoned() && TokenIsFromPaste) { |
712 | PP.HandlePoisonedIdentifier(Identifier&: Tok); |
713 | } |
714 | |
715 | if (!DisableMacroExpansion && II->isHandleIdentifierCase()) |
716 | return PP.HandleIdentifier(Identifier&: Tok); |
717 | } |
718 | |
719 | // Otherwise, return a normal token. |
720 | return true; |
721 | } |
722 | |
723 | bool TokenLexer::pasteTokens(Token &Tok) { |
724 | return pasteTokens(LHSTok&: Tok, TokenStream: llvm::ArrayRef(Tokens, NumTokens), CurIdx&: CurTokenIdx); |
725 | } |
726 | |
727 | /// LHSTok is the LHS of a ## operator, and CurTokenIdx is the ## |
728 | /// operator. Read the ## and RHS, and paste the LHS/RHS together. If there |
729 | /// are more ## after it, chomp them iteratively. Return the result as LHSTok. |
730 | /// If this returns true, the caller should immediately return the token. |
731 | bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef<Token> TokenStream, |
732 | unsigned int &CurIdx) { |
733 | assert(CurIdx > 0 && "## can not be the first token within tokens"); |
734 | assert((TokenStream[CurIdx].is(tok::hashhash) || |
735 | (PP.getLangOpts().MSVCCompat && |
736 | isWideStringLiteralFromMacro(LHSTok, TokenStream[CurIdx]))) && |
737 | "Token at this Index must be ## or part of the MSVC 'L " |
738 | "#macro-arg' pasting pair"); |
739 | |
740 | // MSVC: If previous token was pasted, this must be a recovery from an invalid |
741 | // paste operation. Ignore spaces before this token to mimic MSVC output. |
742 | // Required for generating valid UUID strings in some MS headers. |
743 | if (PP.getLangOpts().MicrosoftExt && (CurIdx >= 2) && |
744 | TokenStream[CurIdx - 2].is(K: tok::hashhash)) |
745 | LHSTok.clearFlag(Flag: Token::LeadingSpace); |
746 | |
747 | SmallString<128> Buffer; |
748 | const char *ResultTokStrPtr = nullptr; |
749 | SourceLocation StartLoc = LHSTok.getLocation(); |
750 | SourceLocation PasteOpLoc; |
751 | |
752 | auto IsAtEnd = [&TokenStream, &CurIdx] { |
753 | return TokenStream.size() == CurIdx; |
754 | }; |
755 | |
756 | do { |
757 | // Consume the ## operator if any. |
758 | PasteOpLoc = TokenStream[CurIdx].getLocation(); |
759 | if (TokenStream[CurIdx].is(K: tok::hashhash)) |
760 | ++CurIdx; |
761 | assert(!IsAtEnd() && "No token on the RHS of a paste operator!"); |
762 | |
763 | // Get the RHS token. |
764 | const Token &RHS = TokenStream[CurIdx]; |
765 | |
766 | // Allocate space for the result token. This is guaranteed to be enough for |
767 | // the two tokens. |
768 | Buffer.resize(N: LHSTok.getLength() + RHS.getLength()); |
769 | |
770 | // Get the spelling of the LHS token in Buffer. |
771 | const char *BufPtr = &Buffer[0]; |
772 | bool Invalid = false; |
773 | unsigned LHSLen = PP.getSpelling(Tok: LHSTok, Buffer&: BufPtr, Invalid: &Invalid); |
774 | if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! |
775 | memcpy(dest: &Buffer[0], src: BufPtr, n: LHSLen); |
776 | if (Invalid) |
777 | return true; |
778 | |
779 | BufPtr = Buffer.data() + LHSLen; |
780 | unsigned RHSLen = PP.getSpelling(Tok: RHS, Buffer&: BufPtr, Invalid: &Invalid); |
781 | if (Invalid) |
782 | return true; |
783 | if (RHSLen && BufPtr != &Buffer[LHSLen]) |
784 | // Really, we want the chars in Buffer! |
785 | memcpy(dest: &Buffer[LHSLen], src: BufPtr, n: RHSLen); |
786 | |
787 | // Trim excess space. |
788 | Buffer.resize(N: LHSLen+RHSLen); |
789 | |
790 | // Plop the pasted result (including the trailing newline and null) into a |
791 | // scratch buffer where we can lex it. |
792 | Token ResultTokTmp; |
793 | ResultTokTmp.startToken(); |
794 | |
795 | // Claim that the tmp token is a string_literal so that we can get the |
796 | // character pointer back from CreateString in getLiteralData(). |
797 | ResultTokTmp.setKind(tok::string_literal); |
798 | PP.CreateString(Str: Buffer, Tok&: ResultTokTmp); |
799 | SourceLocation ResultTokLoc = ResultTokTmp.getLocation(); |
800 | ResultTokStrPtr = ResultTokTmp.getLiteralData(); |
801 | |
802 | // Lex the resultant pasted token into Result. |
803 | Token Result; |
804 | |
805 | if (LHSTok.isAnyIdentifier() && RHS.isAnyIdentifier()) { |
806 | // Common paste case: identifier+identifier = identifier. Avoid creating |
807 | // a lexer and other overhead. |
808 | PP.IncrementPasteCounter(isFast: true); |
809 | Result.startToken(); |
810 | Result.setKind(tok::raw_identifier); |
811 | Result.setRawIdentifierData(ResultTokStrPtr); |
812 | Result.setLocation(ResultTokLoc); |
813 | Result.setLength(LHSLen+RHSLen); |
814 | } else { |
815 | PP.IncrementPasteCounter(isFast: false); |
816 | |
817 | assert(ResultTokLoc.isFileID() && |
818 | "Should be a raw location into scratch buffer"); |
819 | SourceManager &SourceMgr = PP.getSourceManager(); |
820 | FileID LocFileID = SourceMgr.getFileID(SpellingLoc: ResultTokLoc); |
821 | |
822 | bool Invalid = false; |
823 | const char *ScratchBufStart |
824 | = SourceMgr.getBufferData(FID: LocFileID, Invalid: &Invalid).data(); |
825 | if (Invalid) |
826 | return false; |
827 | |
828 | // Make a lexer to lex this string from. Lex just this one token. |
829 | // Make a lexer object so that we lex and expand the paste result. |
830 | Lexer TL(SourceMgr.getLocForStartOfFile(FID: LocFileID), |
831 | PP.getLangOpts(), ScratchBufStart, |
832 | ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen); |
833 | |
834 | // Lex a token in raw mode. This way it won't look up identifiers |
835 | // automatically, lexing off the end will return an eof token, and |
836 | // warnings are disabled. This returns true if the result token is the |
837 | // entire buffer. |
838 | bool isInvalid = !TL.LexFromRawLexer(Result); |
839 | |
840 | // If we got an EOF token, we didn't form even ONE token. For example, we |
841 | // did "/ ## /" to get "//". |
842 | isInvalid |= Result.is(K: tok::eof); |
843 | |
844 | // If pasting the two tokens didn't form a full new token, this is an |
845 | // error. This occurs with "x ## +" and other stuff. Return with LHSTok |
846 | // unmodified and with RHS as the next token to lex. |
847 | if (isInvalid) { |
848 | // Explicitly convert the token location to have proper expansion |
849 | // information so that the user knows where it came from. |
850 | SourceManager &SM = PP.getSourceManager(); |
851 | SourceLocation Loc = |
852 | SM.createExpansionLoc(SpellingLoc: PasteOpLoc, ExpansionLocStart: ExpandLocStart, ExpansionLocEnd: ExpandLocEnd, Length: 2); |
853 | |
854 | // Test for the Microsoft extension of /##/ turning into // here on the |
855 | // error path. |
856 | if (PP.getLangOpts().MicrosoftExt && LHSTok.is(K: tok::slash) && |
857 | RHS.is(K: tok::slash)) { |
858 | HandleMicrosoftCommentPaste(Tok&: LHSTok, OpLoc: Loc); |
859 | return true; |
860 | } |
861 | |
862 | // Do not emit the error when preprocessing assembler code. |
863 | if (!PP.getLangOpts().AsmPreprocessor) { |
864 | // If we're in microsoft extensions mode, downgrade this from a hard |
865 | // error to an extension that defaults to an error. This allows |
866 | // disabling it. |
867 | PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms |
868 | : diag::err_pp_bad_paste) |
869 | << Buffer; |
870 | } |
871 | |
872 | // An error has occurred so exit loop. |
873 | break; |
874 | } |
875 | |
876 | // Turn ## into 'unknown' to avoid # ## # from looking like a paste |
877 | // operator. |
878 | if (Result.is(K: tok::hashhash)) |
879 | Result.setKind(tok::unknown); |
880 | } |
881 | |
882 | // Transfer properties of the LHS over the Result. |
883 | Result.setFlagValue(Flag: Token::StartOfLine , Val: LHSTok.isAtStartOfLine()); |
884 | Result.setFlagValue(Flag: Token::LeadingSpace, Val: LHSTok.hasLeadingSpace()); |
885 | |
886 | // Finally, replace LHS with the result, consume the RHS, and iterate. |
887 | ++CurIdx; |
888 | LHSTok = Result; |
889 | } while (!IsAtEnd() && TokenStream[CurIdx].is(K: tok::hashhash)); |
890 | |
891 | SourceLocation EndLoc = TokenStream[CurIdx - 1].getLocation(); |
892 | |
893 | // The token's current location indicate where the token was lexed from. We |
894 | // need this information to compute the spelling of the token, but any |
895 | // diagnostics for the expanded token should appear as if the token was |
896 | // expanded from the full ## expression. Pull this information together into |
897 | // a new SourceLocation that captures all of this. |
898 | SourceManager &SM = PP.getSourceManager(); |
899 | if (StartLoc.isFileID()) |
900 | StartLoc = getExpansionLocForMacroDefLoc(loc: StartLoc); |
901 | if (EndLoc.isFileID()) |
902 | EndLoc = getExpansionLocForMacroDefLoc(loc: EndLoc); |
903 | FileID MacroFID = SM.getFileID(SpellingLoc: MacroExpansionStart); |
904 | while (SM.getFileID(SpellingLoc: StartLoc) != MacroFID) |
905 | StartLoc = SM.getImmediateExpansionRange(Loc: StartLoc).getBegin(); |
906 | while (SM.getFileID(SpellingLoc: EndLoc) != MacroFID) |
907 | EndLoc = SM.getImmediateExpansionRange(Loc: EndLoc).getEnd(); |
908 | |
909 | LHSTok.setLocation(SM.createExpansionLoc(SpellingLoc: LHSTok.getLocation(), ExpansionLocStart: StartLoc, ExpansionLocEnd: EndLoc, |
910 | Length: LHSTok.getLength())); |
911 | |
912 | // Now that we got the result token, it will be subject to expansion. Since |
913 | // token pasting re-lexes the result token in raw mode, identifier information |
914 | // isn't looked up. As such, if the result is an identifier, look up id info. |
915 | if (LHSTok.is(K: tok::raw_identifier)) { |
916 | // Look up the identifier info for the token. We disabled identifier lookup |
917 | // by saying we're skipping contents, so we need to do this manually. |
918 | PP.LookUpIdentifierInfo(Identifier&: LHSTok); |
919 | } |
920 | return false; |
921 | } |
922 | |
923 | /// isNextTokenLParen - If the next token lexed will pop this macro off the |
924 | /// expansion stack, return 2. If the next unexpanded token is a '(', return |
925 | /// 1, otherwise return 0. |
926 | unsigned TokenLexer::isNextTokenLParen() const { |
927 | // Out of tokens? |
928 | if (isAtEnd()) |
929 | return 2; |
930 | return Tokens[CurTokenIdx].is(K: tok::l_paren); |
931 | } |
932 | |
933 | /// isParsingPreprocessorDirective - Return true if we are in the middle of a |
934 | /// preprocessor directive. |
935 | bool TokenLexer::isParsingPreprocessorDirective() const { |
936 | return Tokens[NumTokens-1].is(K: tok::eod) && !isAtEnd(); |
937 | } |
938 | |
939 | /// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes |
940 | /// together to form a comment that comments out everything in the current |
941 | /// macro, other active macros, and anything left on the current physical |
942 | /// source line of the expanded buffer. Handle this by returning the |
943 | /// first token on the next line. |
944 | void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok, SourceLocation OpLoc) { |
945 | PP.Diag(OpLoc, diag::ext_comment_paste_microsoft); |
946 | |
947 | // We 'comment out' the rest of this macro by just ignoring the rest of the |
948 | // tokens that have not been lexed yet, if any. |
949 | |
950 | // Since this must be a macro, mark the macro enabled now that it is no longer |
951 | // being expanded. |
952 | assert(Macro && "Token streams can't paste comments"); |
953 | Macro->EnableMacro(); |
954 | |
955 | PP.HandleMicrosoftCommentPaste(Tok); |
956 | } |
957 | |
958 | /// If \arg loc is a file ID and points inside the current macro |
959 | /// definition, returns the appropriate source location pointing at the |
960 | /// macro expansion source location entry, otherwise it returns an invalid |
961 | /// SourceLocation. |
962 | SourceLocation |
963 | TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const { |
964 | assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() && |
965 | "Not appropriate for token streams"); |
966 | assert(loc.isValid() && loc.isFileID()); |
967 | |
968 | SourceManager &SM = PP.getSourceManager(); |
969 | assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) && |
970 | "Expected loc to come from the macro definition"); |
971 | |
972 | SourceLocation::UIntTy relativeOffset = 0; |
973 | SM.isInSLocAddrSpace(Loc: loc, Start: MacroDefStart, Length: MacroDefLength, RelativeOffset: &relativeOffset); |
974 | return MacroExpansionStart.getLocWithOffset(Offset: relativeOffset); |
975 | } |
976 | |
977 | /// Finds the tokens that are consecutive (from the same FileID) |
978 | /// creates a single SLocEntry, and assigns SourceLocations to each token that |
979 | /// point to that SLocEntry. e.g for |
980 | /// assert(foo == bar); |
981 | /// There will be a single SLocEntry for the "foo == bar" chunk and locations |
982 | /// for the 'foo', '==', 'bar' tokens will point inside that chunk. |
983 | /// |
984 | /// \arg begin_tokens will be updated to a position past all the found |
985 | /// consecutive tokens. |
986 | static void updateConsecutiveMacroArgTokens(SourceManager &SM, |
987 | SourceLocation ExpandLoc, |
988 | Token *&begin_tokens, |
989 | Token * end_tokens) { |
990 | assert(begin_tokens + 1 < end_tokens); |
991 | SourceLocation BeginLoc = begin_tokens->getLocation(); |
992 | llvm::MutableArrayRef<Token> All(begin_tokens, end_tokens); |
993 | llvm::MutableArrayRef<Token> Partition; |
994 | |
995 | auto NearLast = [&, Last = BeginLoc](SourceLocation Loc) mutable { |
996 | // The maximum distance between two consecutive tokens in a partition. |
997 | // This is an important trick to avoid using too much SourceLocation address |
998 | // space! |
999 | static constexpr SourceLocation::IntTy MaxDistance = 50; |
1000 | auto Distance = Loc.getRawEncoding() - Last.getRawEncoding(); |
1001 | Last = Loc; |
1002 | return Distance <= MaxDistance; |
1003 | }; |
1004 | |
1005 | // Partition the tokens by their FileID. |
1006 | // This is a hot function, and calling getFileID can be expensive, the |
1007 | // implementation is optimized by reducing the number of getFileID. |
1008 | if (BeginLoc.isFileID()) { |
1009 | // Consecutive tokens not written in macros must be from the same file. |
1010 | // (Neither #include nor eof can occur inside a macro argument.) |
1011 | Partition = All.take_while(Pred: [&](const Token &T) { |
1012 | return T.getLocation().isFileID() && NearLast(T.getLocation()); |
1013 | }); |
1014 | } else { |
1015 | // Call getFileID once to calculate the bounds, and use the cheaper |
1016 | // sourcelocation-against-bounds comparison. |
1017 | FileID BeginFID = SM.getFileID(SpellingLoc: BeginLoc); |
1018 | SourceLocation Limit = |
1019 | SM.getComposedLoc(FID: BeginFID, Offset: SM.getFileIDSize(FID: BeginFID)); |
1020 | Partition = All.take_while(Pred: [&](const Token &T) { |
1021 | // NOTE: the Limit is included! The lexer recovery only ever inserts a |
1022 | // single token past the end of the FileID, specifically the ) when a |
1023 | // macro-arg containing a comma should be guarded by parentheses. |
1024 | // |
1025 | // It is safe to include the Limit here because SourceManager allocates |
1026 | // FileSize + 1 for each SLocEntry. |
1027 | // |
1028 | // See https://github.com/llvm/llvm-project/issues/60722. |
1029 | return T.getLocation() >= BeginLoc && T.getLocation() <= Limit |
1030 | && NearLast(T.getLocation()); |
1031 | }); |
1032 | } |
1033 | assert(!Partition.empty()); |
1034 | |
1035 | // For the consecutive tokens, find the length of the SLocEntry to contain |
1036 | // all of them. |
1037 | SourceLocation::UIntTy FullLength = |
1038 | Partition.back().getEndLoc().getRawEncoding() - |
1039 | Partition.front().getLocation().getRawEncoding(); |
1040 | // Create a macro expansion SLocEntry that will "contain" all of the tokens. |
1041 | SourceLocation Expansion = |
1042 | SM.createMacroArgExpansionLoc(SpellingLoc: BeginLoc, ExpansionLoc: ExpandLoc, Length: FullLength); |
1043 | |
1044 | #ifdef EXPENSIVE_CHECKS |
1045 | assert(llvm::all_of(Partition.drop_front(), |
1046 | [&SM, ID = SM.getFileID(Partition.front().getLocation())]( |
1047 | const Token &T) { |
1048 | return ID == SM.getFileID(T.getLocation()); |
1049 | }) && |
1050 | "Must have the same FIleID!"); |
1051 | #endif |
1052 | // Change the location of the tokens from the spelling location to the new |
1053 | // expanded location. |
1054 | for (Token& T : Partition) { |
1055 | SourceLocation::IntTy RelativeOffset = |
1056 | T.getLocation().getRawEncoding() - BeginLoc.getRawEncoding(); |
1057 | T.setLocation(Expansion.getLocWithOffset(Offset: RelativeOffset)); |
1058 | } |
1059 | begin_tokens = &Partition.back() + 1; |
1060 | } |
1061 | |
1062 | /// Creates SLocEntries and updates the locations of macro argument |
1063 | /// tokens to their new expanded locations. |
1064 | /// |
1065 | /// \param ArgIdSpellLoc the location of the macro argument id inside the macro |
1066 | /// definition. |
1067 | void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, |
1068 | Token *begin_tokens, |
1069 | Token *end_tokens) { |
1070 | SourceManager &SM = PP.getSourceManager(); |
1071 | |
1072 | SourceLocation ExpandLoc = |
1073 | getExpansionLocForMacroDefLoc(loc: ArgIdSpellLoc); |
1074 | |
1075 | while (begin_tokens < end_tokens) { |
1076 | // If there's only one token just create a SLocEntry for it. |
1077 | if (end_tokens - begin_tokens == 1) { |
1078 | Token &Tok = *begin_tokens; |
1079 | Tok.setLocation(SM.createMacroArgExpansionLoc(SpellingLoc: Tok.getLocation(), |
1080 | ExpansionLoc: ExpandLoc, |
1081 | Length: Tok.getLength())); |
1082 | return; |
1083 | } |
1084 | |
1085 | updateConsecutiveMacroArgTokens(SM, ExpandLoc, begin_tokens, end_tokens); |
1086 | } |
1087 | } |
1088 | |
1089 | void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { |
1090 | AtStartOfLine = Result.isAtStartOfLine(); |
1091 | HasLeadingSpace = Result.hasLeadingSpace(); |
1092 | } |
1093 |
Definitions
- Init
- Init
- destroy
- MaybeRemoveCommaBeforeVaArgs
- stringifyVAOPTContents
- ExpandFunctionArguments
- isWideStringLiteralFromMacro
- Lex
- pasteTokens
- pasteTokens
- isNextTokenLParen
- isParsingPreprocessorDirective
- HandleMicrosoftCommentPaste
- getExpansionLocForMacroDefLoc
- updateConsecutiveMacroArgTokens
- updateLocForMacroArgTokens
Update your C++ knowledge – Modern C++11/14/17 Training
Find out more