1//===- DependencyDirectivesScanner.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This is the interface for scanning header and source files to get the
11/// minimum necessary preprocessor directives for evaluating includes. It
12/// reduces the source down to #define, #include, #import, @import, and any
13/// conditional preprocessor logic that contains one of those.
14///
15//===----------------------------------------------------------------------===//
16
17#include "clang/Lex/DependencyDirectivesScanner.h"
18#include "clang/Basic/CharInfo.h"
19#include "clang/Basic/Diagnostic.h"
20#include "clang/Lex/LexDiagnostic.h"
21#include "clang/Lex/Lexer.h"
22#include "clang/Lex/Pragma.h"
23#include "llvm/ADT/ScopeExit.h"
24#include "llvm/ADT/SmallString.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/StringSwitch.h"
27#include <optional>
28
29using namespace clang;
30using namespace clang::dependency_directives_scan;
31using namespace llvm;
32
33namespace {
34
35struct DirectiveWithTokens {
36 DirectiveKind Kind;
37 unsigned NumTokens;
38
39 DirectiveWithTokens(DirectiveKind Kind, unsigned NumTokens)
40 : Kind(Kind), NumTokens(NumTokens) {}
41};
42
43/// Does an efficient "scan" of the sources to detect the presence of
44/// preprocessor (or module import) directives and collects the raw lexed tokens
45/// for those directives so that the \p Lexer can "replay" them when the file is
46/// included.
47///
48/// Note that the behavior of the raw lexer is affected by the language mode,
49/// while at this point we want to do a scan and collect tokens once,
50/// irrespective of the language mode that the file will get included in. To
51/// compensate for that the \p Lexer, while "replaying", will adjust a token
52/// where appropriate, when it could affect the preprocessor's state.
53/// For example in a directive like
54///
55/// \code
56/// #if __has_cpp_attribute(clang::fallthrough)
57/// \endcode
58///
59/// The preprocessor needs to see '::' as 'tok::coloncolon' instead of 2
60/// 'tok::colon'. The \p Lexer will adjust if it sees consecutive 'tok::colon'
61/// while in C++ mode.
62struct Scanner {
63 Scanner(StringRef Input,
64 SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
65 DiagnosticsEngine *Diags, SourceLocation InputSourceLoc)
66 : Input(Input), Tokens(Tokens), Diags(Diags),
67 InputSourceLoc(InputSourceLoc), LangOpts(getLangOptsForDepScanning()),
68 TheLexer(InputSourceLoc, LangOpts, Input.begin(), Input.begin(),
69 Input.end()) {}
70
71 static LangOptions getLangOptsForDepScanning() {
72 LangOptions LangOpts;
73 // Set the lexer to use 'tok::at' for '@', instead of 'tok::unknown'.
74 LangOpts.ObjC = true;
75 LangOpts.LineComment = true;
76 LangOpts.RawStringLiterals = true;
77 // FIXME: we do not enable C11 or C++11, so we are missing u/u8/U"".
78 return LangOpts;
79 }
80
81 /// Lex the provided source and emit the directive tokens.
82 ///
83 /// \returns True on error.
84 bool scan(SmallVectorImpl<Directive> &Directives);
85
86private:
87 /// Lexes next token and advances \p First and the \p Lexer.
88 [[nodiscard]] dependency_directives_scan::Token &
89 lexToken(const char *&First, const char *const End);
90
91 [[nodiscard]] dependency_directives_scan::Token &
92 lexIncludeFilename(const char *&First, const char *const End);
93
94 void skipLine(const char *&First, const char *const End);
95 void skipDirective(StringRef Name, const char *&First, const char *const End);
96
97 /// Returns the spelling of a string literal or identifier after performing
98 /// any processing needed to handle \c clang::Token::NeedsCleaning.
99 StringRef cleanStringIfNeeded(const dependency_directives_scan::Token &Tok);
100
101 /// Lexes next token and if it is identifier returns its string, otherwise
102 /// it skips the current line and returns \p std::nullopt.
103 ///
104 /// In any case (whatever the token kind) \p First and the \p Lexer will
105 /// advance beyond the token.
106 [[nodiscard]] std::optional<StringRef>
107 tryLexIdentifierOrSkipLine(const char *&First, const char *const End);
108
109 /// Used when it is certain that next token is an identifier.
110 [[nodiscard]] StringRef lexIdentifier(const char *&First,
111 const char *const End);
112
113 /// Lexes next token and returns true iff it is an identifier that matches \p
114 /// Id, otherwise it skips the current line and returns false.
115 ///
116 /// In any case (whatever the token kind) \p First and the \p Lexer will
117 /// advance beyond the token.
118 [[nodiscard]] bool isNextIdentifierOrSkipLine(StringRef Id,
119 const char *&First,
120 const char *const End);
121
122 /// Lexes next token and returns true iff it matches the kind \p K.
123 /// Otherwise it skips the current line and returns false.
124 ///
125 /// In any case (whatever the token kind) \p First and the \p Lexer will
126 /// advance beyond the token.
127 [[nodiscard]] bool isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
128 const char *const End);
129
130 /// Lexes next token and if it is string literal, returns its string.
131 /// Otherwise, it skips the current line and returns \p std::nullopt.
132 ///
133 /// In any case (whatever the token kind) \p First and the \p Lexer will
134 /// advance beyond the token.
135 [[nodiscard]] std::optional<StringRef>
136 tryLexStringLiteralOrSkipLine(const char *&First, const char *const End);
137
138 [[nodiscard]] bool scanImpl(const char *First, const char *const End);
139 [[nodiscard]] bool lexPPLine(const char *&First, const char *const End);
140 [[nodiscard]] bool lexAt(const char *&First, const char *const End);
141 [[nodiscard]] bool lexModule(const char *&First, const char *const End);
142 [[nodiscard]] bool lexDefine(const char *HashLoc, const char *&First,
143 const char *const End);
144 [[nodiscard]] bool lexPragma(const char *&First, const char *const End);
145 [[nodiscard]] bool lex_Pragma(const char *&First, const char *const End);
146 [[nodiscard]] bool lexEndif(const char *&First, const char *const End);
147 [[nodiscard]] bool lexDefault(DirectiveKind Kind, const char *&First,
148 const char *const End);
149 [[nodiscard]] bool lexModuleDirectiveBody(DirectiveKind Kind,
150 const char *&First,
151 const char *const End);
152 void lexPPDirectiveBody(const char *&First, const char *const End);
153
154 DirectiveWithTokens &pushDirective(DirectiveKind Kind) {
155 Tokens.append(RHS: CurDirToks);
156 DirsWithToks.emplace_back(Args&: Kind, Args: CurDirToks.size());
157 CurDirToks.clear();
158 return DirsWithToks.back();
159 }
160 void popDirective() {
161 Tokens.pop_back_n(NumItems: DirsWithToks.pop_back_val().NumTokens);
162 }
163 DirectiveKind topDirective() const {
164 return DirsWithToks.empty() ? pp_none : DirsWithToks.back().Kind;
165 }
166
167 unsigned getOffsetAt(const char *CurPtr) const {
168 return CurPtr - Input.data();
169 }
170
171 /// Reports a diagnostic if the diagnostic engine is provided. Always returns
172 /// true at the end.
173 bool reportError(const char *CurPtr, unsigned Err);
174
175 StringMap<char> SplitIds;
176 StringRef Input;
177 SmallVectorImpl<dependency_directives_scan::Token> &Tokens;
178 DiagnosticsEngine *Diags;
179 SourceLocation InputSourceLoc;
180
181 const char *LastTokenPtr = nullptr;
182 /// Keeps track of the tokens for the currently lexed directive. Once a
183 /// directive is fully lexed and "committed" then the tokens get appended to
184 /// \p Tokens and \p CurDirToks is cleared for the next directive.
185 SmallVector<dependency_directives_scan::Token, 32> CurDirToks;
186 /// The directives that were lexed along with the number of tokens that each
187 /// directive contains. The tokens of all the directives are kept in \p Tokens
188 /// vector, in the same order as the directives order in \p DirsWithToks.
189 SmallVector<DirectiveWithTokens, 64> DirsWithToks;
190 LangOptions LangOpts;
191 Lexer TheLexer;
192};
193
194} // end anonymous namespace
195
196bool Scanner::reportError(const char *CurPtr, unsigned Err) {
197 if (!Diags)
198 return true;
199 assert(CurPtr >= Input.data() && "invalid buffer ptr");
200 Diags->Report(Loc: InputSourceLoc.getLocWithOffset(Offset: getOffsetAt(CurPtr)), DiagID: Err);
201 return true;
202}
203
204static void skipOverSpaces(const char *&First, const char *const End) {
205 while (First != End && isHorizontalWhitespace(c: *First))
206 ++First;
207}
208
209[[nodiscard]] static bool isRawStringLiteral(const char *First,
210 const char *Current) {
211 assert(First <= Current);
212
213 // Check if we can even back up.
214 if (*Current != '"' || First == Current)
215 return false;
216
217 // Check for an "R".
218 --Current;
219 if (*Current != 'R')
220 return false;
221 if (First == Current || !isAsciiIdentifierContinue(c: *--Current))
222 return true;
223
224 // Check for a prefix of "u", "U", or "L".
225 if (*Current == 'u' || *Current == 'U' || *Current == 'L')
226 return First == Current || !isAsciiIdentifierContinue(c: *--Current);
227
228 // Check for a prefix of "u8".
229 if (*Current != '8' || First == Current || *Current-- != 'u')
230 return false;
231 return First == Current || !isAsciiIdentifierContinue(c: *--Current);
232}
233
234static void skipRawString(const char *&First, const char *const End) {
235 assert(First[0] == '"');
236 assert(First[-1] == 'R');
237
238 const char *Last = ++First;
239 while (Last != End && *Last != '(')
240 ++Last;
241 if (Last == End) {
242 First = Last; // Hit the end... just give up.
243 return;
244 }
245
246 StringRef Terminator(First, Last - First);
247 for (;;) {
248 // Move First to just past the next ")".
249 First = Last;
250 while (First != End && *First != ')')
251 ++First;
252 if (First == End)
253 return;
254 ++First;
255
256 // Look ahead for the terminator sequence.
257 Last = First;
258 while (Last != End && size_t(Last - First) < Terminator.size() &&
259 Terminator[Last - First] == *Last)
260 ++Last;
261
262 // Check if we hit it (or the end of the file).
263 if (Last == End) {
264 First = Last;
265 return;
266 }
267 if (size_t(Last - First) < Terminator.size())
268 continue;
269 if (*Last != '"')
270 continue;
271 First = Last + 1;
272 return;
273 }
274}
275
276// Returns the length of EOL, either 0 (no end-of-line), 1 (\n) or 2 (\r\n)
277static unsigned isEOL(const char *First, const char *const End) {
278 if (First == End)
279 return 0;
280 if (End - First > 1 && isVerticalWhitespace(c: First[0]) &&
281 isVerticalWhitespace(c: First[1]) && First[0] != First[1])
282 return 2;
283 return !!isVerticalWhitespace(c: First[0]);
284}
285
286static void skipString(const char *&First, const char *const End) {
287 assert(*First == '\'' || *First == '"' || *First == '<');
288 const char Terminator = *First == '<' ? '>' : *First;
289 for (++First; First != End && *First != Terminator; ++First) {
290 // String and character literals don't extend past the end of the line.
291 if (isVerticalWhitespace(c: *First))
292 return;
293 if (*First != '\\')
294 continue;
295 // Skip past backslash to the next character. This ensures that the
296 // character right after it is skipped as well, which matters if it's
297 // the terminator.
298 if (++First == End)
299 return;
300 if (!isWhitespace(c: *First))
301 continue;
302 // Whitespace after the backslash might indicate a line continuation.
303 const char *FirstAfterBackslashPastSpace = First;
304 skipOverSpaces(First&: FirstAfterBackslashPastSpace, End);
305 if (unsigned NLSize = isEOL(First: FirstAfterBackslashPastSpace, End)) {
306 // Advance the character pointer to the next line for the next
307 // iteration.
308 First = FirstAfterBackslashPastSpace + NLSize - 1;
309 }
310 }
311 if (First != End)
312 ++First; // Finish off the string.
313}
314
315// Returns the length of the skipped newline
316static unsigned skipNewline(const char *&First, const char *End) {
317 if (First == End)
318 return 0;
319 assert(isVerticalWhitespace(*First));
320 unsigned Len = isEOL(First, End);
321 assert(Len && "expected newline");
322 First += Len;
323 return Len;
324}
325
326static bool wasLineContinuation(const char *First, unsigned EOLLen) {
327 return *(First - (int)EOLLen - 1) == '\\';
328}
329
330static void skipToNewlineRaw(const char *&First, const char *const End) {
331 for (;;) {
332 if (First == End)
333 return;
334
335 unsigned Len = isEOL(First, End);
336 if (Len)
337 return;
338
339 do {
340 if (++First == End)
341 return;
342 Len = isEOL(First, End);
343 } while (!Len);
344
345 if (First[-1] != '\\')
346 return;
347
348 First += Len;
349 // Keep skipping lines...
350 }
351}
352
353static void skipLineComment(const char *&First, const char *const End) {
354 assert(First[0] == '/' && First[1] == '/');
355 First += 2;
356 skipToNewlineRaw(First, End);
357}
358
359static void skipBlockComment(const char *&First, const char *const End) {
360 assert(First[0] == '/' && First[1] == '*');
361 if (End - First < 4) {
362 First = End;
363 return;
364 }
365 for (First += 3; First != End; ++First)
366 if (First[-1] == '*' && First[0] == '/') {
367 ++First;
368 return;
369 }
370}
371
372/// \returns True if the current single quotation mark character is a C++14
373/// digit separator.
374static bool isQuoteCppDigitSeparator(const char *const Start,
375 const char *const Cur,
376 const char *const End) {
377 assert(*Cur == '\'' && "expected quotation character");
378 // skipLine called in places where we don't expect a valid number
379 // body before `start` on the same line, so always return false at the start.
380 if (Start == Cur)
381 return false;
382 // The previous character must be a valid PP number character.
383 // Make sure that the L, u, U, u8 prefixes don't get marked as a
384 // separator though.
385 char Prev = *(Cur - 1);
386 if (Prev == 'L' || Prev == 'U' || Prev == 'u')
387 return false;
388 if (Prev == '8' && (Cur - 1 != Start) && *(Cur - 2) == 'u')
389 return false;
390 if (!isPreprocessingNumberBody(c: Prev))
391 return false;
392 // The next character should be a valid identifier body character.
393 return (Cur + 1) < End && isAsciiIdentifierContinue(c: *(Cur + 1));
394}
395
396void Scanner::skipLine(const char *&First, const char *const End) {
397 for (;;) {
398 assert(First <= End);
399 if (First == End)
400 return;
401
402 if (isVerticalWhitespace(c: *First)) {
403 skipNewline(First, End);
404 return;
405 }
406 const char *Start = First;
407 while (First != End && !isVerticalWhitespace(c: *First)) {
408 // Iterate over strings correctly to avoid comments and newlines.
409 if (*First == '"' ||
410 (*First == '\'' && !isQuoteCppDigitSeparator(Start, Cur: First, End))) {
411 LastTokenPtr = First;
412 if (isRawStringLiteral(First: Start, Current: First))
413 skipRawString(First, End);
414 else
415 skipString(First, End);
416 continue;
417 }
418
419 // Iterate over comments correctly.
420 if (*First != '/' || End - First < 2) {
421 LastTokenPtr = First;
422 ++First;
423 continue;
424 }
425
426 if (First[1] == '/') {
427 // "//...".
428 skipLineComment(First, End);
429 continue;
430 }
431
432 if (First[1] != '*') {
433 LastTokenPtr = First;
434 ++First;
435 continue;
436 }
437
438 // "/*...*/".
439 skipBlockComment(First, End);
440 }
441 if (First == End)
442 return;
443
444 // Skip over the newline.
445 unsigned Len = skipNewline(First, End);
446 if (!wasLineContinuation(First, EOLLen: Len)) // Continue past line-continuations.
447 break;
448 }
449}
450
451void Scanner::skipDirective(StringRef Name, const char *&First,
452 const char *const End) {
453 if (llvm::StringSwitch<bool>(Name)
454 .Case(S: "warning", Value: true)
455 .Case(S: "error", Value: true)
456 .Default(Value: false))
457 // Do not process quotes or comments.
458 skipToNewlineRaw(First, End);
459 else
460 skipLine(First, End);
461}
462
463static void skipWhitespace(const char *&First, const char *const End) {
464 for (;;) {
465 assert(First <= End);
466 skipOverSpaces(First, End);
467
468 if (End - First < 2)
469 return;
470
471 if (First[0] == '\\' && isVerticalWhitespace(c: First[1])) {
472 skipNewline(First&: ++First, End);
473 continue;
474 }
475
476 // Check for a non-comment character.
477 if (First[0] != '/')
478 return;
479
480 // "// ...".
481 if (First[1] == '/') {
482 skipLineComment(First, End);
483 return;
484 }
485
486 // Cannot be a comment.
487 if (First[1] != '*')
488 return;
489
490 // "/*...*/".
491 skipBlockComment(First, End);
492 }
493}
494
495bool Scanner::lexModuleDirectiveBody(DirectiveKind Kind, const char *&First,
496 const char *const End) {
497 const char *DirectiveLoc = Input.data() + CurDirToks.front().Offset;
498 for (;;) {
499 // Keep a copy of the First char incase it needs to be reset.
500 const char *Previous = First;
501 const dependency_directives_scan::Token &Tok = lexToken(First, End);
502 if ((Tok.is(K: tok::hash) || Tok.is(K: tok::at)) &&
503 (Tok.Flags & clang::Token::StartOfLine)) {
504 CurDirToks.pop_back();
505 First = Previous;
506 return false;
507 }
508 if (Tok.is(K: tok::eof))
509 return reportError(
510 CurPtr: DirectiveLoc,
511 diag::Err: err_dep_source_scanner_missing_semi_after_at_import);
512 if (Tok.is(K: tok::semi))
513 break;
514 }
515 pushDirective(Kind);
516 skipWhitespace(First, End);
517 if (First == End)
518 return false;
519 if (!isVerticalWhitespace(c: *First))
520 return reportError(
521 CurPtr: DirectiveLoc, diag::Err: err_dep_source_scanner_unexpected_tokens_at_import);
522 skipNewline(First, End);
523 return false;
524}
525
526dependency_directives_scan::Token &Scanner::lexToken(const char *&First,
527 const char *const End) {
528 clang::Token Tok;
529 TheLexer.LexFromRawLexer(Result&: Tok);
530 First = Input.data() + TheLexer.getCurrentBufferOffset();
531 assert(First <= End);
532
533 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
534 CurDirToks.emplace_back(Args&: Offset, Args: Tok.getLength(), Args: Tok.getKind(),
535 Args: Tok.getFlags());
536 return CurDirToks.back();
537}
538
539dependency_directives_scan::Token &
540Scanner::lexIncludeFilename(const char *&First, const char *const End) {
541 clang::Token Tok;
542 TheLexer.LexIncludeFilename(FilenameTok&: Tok);
543 First = Input.data() + TheLexer.getCurrentBufferOffset();
544 assert(First <= End);
545
546 unsigned Offset = TheLexer.getCurrentBufferOffset() - Tok.getLength();
547 CurDirToks.emplace_back(Args&: Offset, Args: Tok.getLength(), Args: Tok.getKind(),
548 Args: Tok.getFlags());
549 return CurDirToks.back();
550}
551
552void Scanner::lexPPDirectiveBody(const char *&First, const char *const End) {
553 while (true) {
554 const dependency_directives_scan::Token &Tok = lexToken(First, End);
555 if (Tok.is(K: tok::eod) || Tok.is(K: tok::eof))
556 break;
557 }
558}
559
560StringRef
561Scanner::cleanStringIfNeeded(const dependency_directives_scan::Token &Tok) {
562 bool NeedsCleaning = Tok.Flags & clang::Token::NeedsCleaning;
563 if (LLVM_LIKELY(!NeedsCleaning))
564 return Input.slice(Start: Tok.Offset, End: Tok.getEnd());
565
566 SmallString<64> Spelling;
567 Spelling.resize(N: Tok.Length);
568
569 // FIXME: C++11 raw string literals need special handling (see getSpellingSlow
570 // in the Lexer). Currently we cannot see them due to our LangOpts.
571
572 unsigned SpellingLength = 0;
573 const char *BufPtr = Input.begin() + Tok.Offset;
574 const char *AfterIdent = Input.begin() + Tok.getEnd();
575 while (BufPtr < AfterIdent) {
576 auto [Char, Size] = Lexer::getCharAndSizeNoWarn(Ptr: BufPtr, LangOpts);
577 Spelling[SpellingLength++] = Char;
578 BufPtr += Size;
579 }
580
581 return SplitIds.try_emplace(Key: StringRef(Spelling.begin(), SpellingLength), Args: 0)
582 .first->first();
583}
584
585std::optional<StringRef>
586Scanner::tryLexIdentifierOrSkipLine(const char *&First, const char *const End) {
587 const dependency_directives_scan::Token &Tok = lexToken(First, End);
588 if (Tok.isNot(K: tok::raw_identifier)) {
589 if (!Tok.is(K: tok::eod))
590 skipLine(First, End);
591 return std::nullopt;
592 }
593
594 return cleanStringIfNeeded(Tok);
595}
596
597StringRef Scanner::lexIdentifier(const char *&First, const char *const End) {
598 std::optional<StringRef> Id = tryLexIdentifierOrSkipLine(First, End);
599 assert(Id && "expected identifier token");
600 return *Id;
601}
602
603bool Scanner::isNextIdentifierOrSkipLine(StringRef Id, const char *&First,
604 const char *const End) {
605 if (std::optional<StringRef> FoundId =
606 tryLexIdentifierOrSkipLine(First, End)) {
607 if (*FoundId == Id)
608 return true;
609 skipLine(First, End);
610 }
611 return false;
612}
613
614bool Scanner::isNextTokenOrSkipLine(tok::TokenKind K, const char *&First,
615 const char *const End) {
616 const dependency_directives_scan::Token &Tok = lexToken(First, End);
617 if (Tok.is(K))
618 return true;
619 skipLine(First, End);
620 return false;
621}
622
623std::optional<StringRef>
624Scanner::tryLexStringLiteralOrSkipLine(const char *&First,
625 const char *const End) {
626 const dependency_directives_scan::Token &Tok = lexToken(First, End);
627 if (!tok::isStringLiteral(K: Tok.Kind)) {
628 if (!Tok.is(K: tok::eod))
629 skipLine(First, End);
630 return std::nullopt;
631 }
632
633 return cleanStringIfNeeded(Tok);
634}
635
636bool Scanner::lexAt(const char *&First, const char *const End) {
637 // Handle "@import".
638
639 // Lex '@'.
640 const dependency_directives_scan::Token &AtTok = lexToken(First, End);
641 assert(AtTok.is(tok::at));
642 (void)AtTok;
643
644 if (!isNextIdentifierOrSkipLine(Id: "import", First, End))
645 return false;
646 return lexModuleDirectiveBody(Kind: decl_at_import, First, End);
647}
648
649bool Scanner::lexModule(const char *&First, const char *const End) {
650 StringRef Id = lexIdentifier(First, End);
651 bool Export = false;
652 if (Id == "export") {
653 Export = true;
654 std::optional<StringRef> NextId = tryLexIdentifierOrSkipLine(First, End);
655 if (!NextId)
656 return false;
657 Id = *NextId;
658 }
659
660 if (Id != "module" && Id != "import") {
661 skipLine(First, End);
662 return false;
663 }
664
665 skipWhitespace(First, End);
666
667 // Ignore this as a module directive if the next character can't be part of
668 // an import.
669
670 switch (*First) {
671 case ':': {
672 // `module :` is never the start of a valid module declaration.
673 if (Id == "module") {
674 skipLine(First, End);
675 return false;
676 }
677 // `import:(type)name` is a valid ObjC method decl, so check one more token.
678 (void)lexToken(First, End);
679 if (!tryLexIdentifierOrSkipLine(First, End))
680 return false;
681 break;
682 }
683 case '<':
684 case '"':
685 break;
686 default:
687 if (!isAsciiIdentifierContinue(c: *First)) {
688 skipLine(First, End);
689 return false;
690 }
691 }
692
693 TheLexer.seek(Offset: getOffsetAt(CurPtr: First), /*IsAtStartOfLine*/ false);
694
695 DirectiveKind Kind;
696 if (Id == "module")
697 Kind = Export ? cxx_export_module_decl : cxx_module_decl;
698 else
699 Kind = Export ? cxx_export_import_decl : cxx_import_decl;
700
701 return lexModuleDirectiveBody(Kind, First, End);
702}
703
704bool Scanner::lex_Pragma(const char *&First, const char *const End) {
705 if (!isNextTokenOrSkipLine(K: tok::l_paren, First, End))
706 return false;
707
708 std::optional<StringRef> Str = tryLexStringLiteralOrSkipLine(First, End);
709
710 if (!Str || !isNextTokenOrSkipLine(K: tok::r_paren, First, End))
711 return false;
712
713 SmallString<64> Buffer(*Str);
714 prepare_PragmaString(StrVal&: Buffer);
715
716 // Use a new scanner instance since the tokens will be inside the allocated
717 // string. We should already have captured all the relevant tokens in the
718 // current scanner.
719 SmallVector<dependency_directives_scan::Token> DiscardTokens;
720 const char *Begin = Buffer.c_str();
721 Scanner PragmaScanner{StringRef(Begin, Buffer.size()), DiscardTokens, Diags,
722 InputSourceLoc};
723
724 PragmaScanner.TheLexer.setParsingPreprocessorDirective(true);
725 if (PragmaScanner.lexPragma(First&: Begin, End: Buffer.end()))
726 return true;
727
728 DirectiveKind K = PragmaScanner.topDirective();
729 if (K == pp_none) {
730 skipLine(First, End);
731 return false;
732 }
733
734 assert(Begin == Buffer.end());
735 pushDirective(Kind: K);
736 return false;
737}
738
739bool Scanner::lexPragma(const char *&First, const char *const End) {
740 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
741 if (!FoundId)
742 return false;
743
744 StringRef Id = *FoundId;
745 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
746 .Case(S: "once", Value: pp_pragma_once)
747 .Case(S: "push_macro", Value: pp_pragma_push_macro)
748 .Case(S: "pop_macro", Value: pp_pragma_pop_macro)
749 .Case(S: "include_alias", Value: pp_pragma_include_alias)
750 .Default(Value: pp_none);
751 if (Kind != pp_none) {
752 lexPPDirectiveBody(First, End);
753 pushDirective(Kind);
754 return false;
755 }
756
757 if (Id != "clang") {
758 skipLine(First, End);
759 return false;
760 }
761
762 FoundId = tryLexIdentifierOrSkipLine(First, End);
763 if (!FoundId)
764 return false;
765 Id = *FoundId;
766
767 // #pragma clang system_header
768 if (Id == "system_header") {
769 lexPPDirectiveBody(First, End);
770 pushDirective(Kind: pp_pragma_system_header);
771 return false;
772 }
773
774 if (Id != "module") {
775 skipLine(First, End);
776 return false;
777 }
778
779 // #pragma clang module.
780 if (!isNextIdentifierOrSkipLine(Id: "import", First, End))
781 return false;
782
783 // #pragma clang module import.
784 lexPPDirectiveBody(First, End);
785 pushDirective(Kind: pp_pragma_import);
786 return false;
787}
788
789bool Scanner::lexEndif(const char *&First, const char *const End) {
790 // Strip out "#else" if it's empty.
791 if (topDirective() == pp_else)
792 popDirective();
793
794 // If "#ifdef" is empty, strip it and skip the "#endif".
795 //
796 // FIXME: Once/if Clang starts disallowing __has_include in macro expansions,
797 // we can skip empty `#if` and `#elif` blocks as well after scanning for a
798 // literal __has_include in the condition. Even without that rule we could
799 // drop the tokens if we scan for identifiers in the condition and find none.
800 if (topDirective() == pp_ifdef || topDirective() == pp_ifndef) {
801 popDirective();
802 skipLine(First, End);
803 return false;
804 }
805
806 return lexDefault(Kind: pp_endif, First, End);
807}
808
809bool Scanner::lexDefault(DirectiveKind Kind, const char *&First,
810 const char *const End) {
811 lexPPDirectiveBody(First, End);
812 pushDirective(Kind);
813 return false;
814}
815
816static bool isStartOfRelevantLine(char First) {
817 switch (First) {
818 case '#':
819 case '@':
820 case 'i':
821 case 'e':
822 case 'm':
823 case '_':
824 return true;
825 }
826 return false;
827}
828
829bool Scanner::lexPPLine(const char *&First, const char *const End) {
830 assert(First != End);
831
832 skipWhitespace(First, End);
833 assert(First <= End);
834 if (First == End)
835 return false;
836
837 if (!isStartOfRelevantLine(First: *First)) {
838 skipLine(First, End);
839 assert(First <= End);
840 return false;
841 }
842
843 LastTokenPtr = First;
844
845 TheLexer.seek(Offset: getOffsetAt(CurPtr: First), /*IsAtStartOfLine*/ true);
846
847 auto ScEx1 = make_scope_exit(F: [&]() {
848 /// Clear Scanner's CurDirToks before returning, in case we didn't push a
849 /// new directive.
850 CurDirToks.clear();
851 });
852
853 // Handle "@import".
854 if (*First == '@')
855 return lexAt(First, End);
856
857 // Handle module directives for C++20 modules.
858 if (*First == 'i' || *First == 'e' || *First == 'm')
859 return lexModule(First, End);
860
861 if (*First == '_') {
862 if (isNextIdentifierOrSkipLine(Id: "_Pragma", First, End))
863 return lex_Pragma(First, End);
864 return false;
865 }
866
867 // Handle preprocessing directives.
868
869 TheLexer.setParsingPreprocessorDirective(true);
870 auto ScEx2 = make_scope_exit(
871 F: [&]() { TheLexer.setParsingPreprocessorDirective(false); });
872
873 // Lex '#'.
874 const dependency_directives_scan::Token &HashTok = lexToken(First, End);
875 if (HashTok.is(K: tok::hashhash)) {
876 // A \p tok::hashhash at this location is passed by the preprocessor to the
877 // parser to interpret, like any other token. So for dependency scanning
878 // skip it like a normal token not affecting the preprocessor.
879 skipLine(First, End);
880 assert(First <= End);
881 return false;
882 }
883 assert(HashTok.is(tok::hash));
884 (void)HashTok;
885
886 std::optional<StringRef> FoundId = tryLexIdentifierOrSkipLine(First, End);
887 if (!FoundId)
888 return false;
889
890 StringRef Id = *FoundId;
891
892 if (Id == "pragma")
893 return lexPragma(First, End);
894
895 auto Kind = llvm::StringSwitch<DirectiveKind>(Id)
896 .Case(S: "include", Value: pp_include)
897 .Case(S: "__include_macros", Value: pp___include_macros)
898 .Case(S: "define", Value: pp_define)
899 .Case(S: "undef", Value: pp_undef)
900 .Case(S: "import", Value: pp_import)
901 .Case(S: "include_next", Value: pp_include_next)
902 .Case(S: "if", Value: pp_if)
903 .Case(S: "ifdef", Value: pp_ifdef)
904 .Case(S: "ifndef", Value: pp_ifndef)
905 .Case(S: "elif", Value: pp_elif)
906 .Case(S: "elifdef", Value: pp_elifdef)
907 .Case(S: "elifndef", Value: pp_elifndef)
908 .Case(S: "else", Value: pp_else)
909 .Case(S: "endif", Value: pp_endif)
910 .Default(Value: pp_none);
911 if (Kind == pp_none) {
912 skipDirective(Name: Id, First, End);
913 return false;
914 }
915
916 if (Kind == pp_endif)
917 return lexEndif(First, End);
918
919 switch (Kind) {
920 case pp_include:
921 case pp___include_macros:
922 case pp_include_next:
923 case pp_import:
924 // Ignore missing filenames in include or import directives.
925 if (lexIncludeFilename(First, End).is(K: tok::eod)) {
926 return false;
927 }
928 break;
929 default:
930 break;
931 }
932
933 // Everything else.
934 return lexDefault(Kind, First, End);
935}
936
937static void skipUTF8ByteOrderMark(const char *&First, const char *const End) {
938 if ((End - First) >= 3 && First[0] == '\xef' && First[1] == '\xbb' &&
939 First[2] == '\xbf')
940 First += 3;
941}
942
943bool Scanner::scanImpl(const char *First, const char *const End) {
944 skipUTF8ByteOrderMark(First, End);
945 while (First != End)
946 if (lexPPLine(First, End))
947 return true;
948 return false;
949}
950
951bool Scanner::scan(SmallVectorImpl<Directive> &Directives) {
952 bool Error = scanImpl(First: Input.begin(), End: Input.end());
953
954 if (!Error) {
955 // Add an EOF on success.
956 if (LastTokenPtr &&
957 (Tokens.empty() || LastTokenPtr > Input.begin() + Tokens.back().Offset))
958 pushDirective(Kind: tokens_present_before_eof);
959 pushDirective(Kind: pp_eof);
960 }
961
962 ArrayRef<dependency_directives_scan::Token> RemainingTokens = Tokens;
963 for (const DirectiveWithTokens &DirWithToks : DirsWithToks) {
964 assert(RemainingTokens.size() >= DirWithToks.NumTokens);
965 Directives.emplace_back(Args: DirWithToks.Kind,
966 Args: RemainingTokens.take_front(N: DirWithToks.NumTokens));
967 RemainingTokens = RemainingTokens.drop_front(N: DirWithToks.NumTokens);
968 }
969 assert(RemainingTokens.empty());
970
971 return Error;
972}
973
974bool clang::scanSourceForDependencyDirectives(
975 StringRef Input, SmallVectorImpl<dependency_directives_scan::Token> &Tokens,
976 SmallVectorImpl<Directive> &Directives, DiagnosticsEngine *Diags,
977 SourceLocation InputSourceLoc) {
978 return Scanner(Input, Tokens, Diags, InputSourceLoc).scan(Directives);
979}
980
981void clang::printDependencyDirectivesAsSource(
982 StringRef Source,
983 ArrayRef<dependency_directives_scan::Directive> Directives,
984 llvm::raw_ostream &OS) {
985 // Add a space separator where it is convenient for testing purposes.
986 auto needsSpaceSeparator =
987 [](tok::TokenKind Prev,
988 const dependency_directives_scan::Token &Tok) -> bool {
989 if (Prev == Tok.Kind)
990 return !Tok.isOneOf(K1: tok::l_paren, Ks: tok::r_paren, Ks: tok::l_square,
991 Ks: tok::r_square);
992 if (Prev == tok::raw_identifier &&
993 Tok.isOneOf(K1: tok::hash, Ks: tok::numeric_constant, Ks: tok::string_literal,
994 Ks: tok::char_constant, Ks: tok::header_name))
995 return true;
996 if (Prev == tok::r_paren &&
997 Tok.isOneOf(K1: tok::raw_identifier, Ks: tok::hash, Ks: tok::string_literal,
998 Ks: tok::char_constant, Ks: tok::unknown))
999 return true;
1000 if (Prev == tok::comma &&
1001 Tok.isOneOf(K1: tok::l_paren, Ks: tok::string_literal, Ks: tok::less))
1002 return true;
1003 return false;
1004 };
1005
1006 for (const dependency_directives_scan::Directive &Directive : Directives) {
1007 if (Directive.Kind == tokens_present_before_eof)
1008 OS << "<TokBeforeEOF>";
1009 std::optional<tok::TokenKind> PrevTokenKind;
1010 for (const dependency_directives_scan::Token &Tok : Directive.Tokens) {
1011 if (PrevTokenKind && needsSpaceSeparator(*PrevTokenKind, Tok))
1012 OS << ' ';
1013 PrevTokenKind = Tok.Kind;
1014 OS << Source.slice(Start: Tok.Offset, End: Tok.getEnd());
1015 }
1016 }
1017}
1018

source code of clang/lib/Lex/DependencyDirectivesScanner.cpp