1 | //===--- RawCommentList.cpp - Processing raw comments -----------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #include "clang/AST/RawCommentList.h" |
10 | #include "clang/AST/ASTContext.h" |
11 | #include "clang/AST/Comment.h" |
12 | #include "clang/AST/CommentBriefParser.h" |
13 | #include "clang/AST/CommentCommandTraits.h" |
14 | #include "clang/AST/CommentLexer.h" |
15 | #include "clang/AST/CommentParser.h" |
16 | #include "clang/AST/CommentSema.h" |
17 | #include "clang/Basic/CharInfo.h" |
18 | #include "llvm/Support/Allocator.h" |
19 | |
20 | using namespace clang; |
21 | |
22 | namespace { |
23 | /// Get comment kind and bool describing if it is a trailing comment. |
24 | std::pair<RawComment::CommentKind, bool> (StringRef , |
25 | bool ) { |
26 | const size_t = ParseAllComments ? 2 : 3; |
27 | if ((Comment.size() < MinCommentLength) || Comment[0] != '/') |
28 | return std::make_pair(x: RawComment::RCK_Invalid, y: false); |
29 | |
30 | RawComment::CommentKind K; |
31 | if (Comment[1] == '/') { |
32 | if (Comment.size() < 3) |
33 | return std::make_pair(x: RawComment::RCK_OrdinaryBCPL, y: false); |
34 | |
35 | if (Comment[2] == '/') |
36 | K = RawComment::RCK_BCPLSlash; |
37 | else if (Comment[2] == '!') |
38 | K = RawComment::RCK_BCPLExcl; |
39 | else |
40 | return std::make_pair(x: RawComment::RCK_OrdinaryBCPL, y: false); |
41 | } else { |
42 | assert(Comment.size() >= 4); |
43 | |
44 | // Comment lexer does not understand escapes in comment markers, so pretend |
45 | // that this is not a comment. |
46 | if (Comment[1] != '*' || |
47 | Comment[Comment.size() - 2] != '*' || |
48 | Comment[Comment.size() - 1] != '/') |
49 | return std::make_pair(x: RawComment::RCK_Invalid, y: false); |
50 | |
51 | if (Comment[2] == '*') |
52 | K = RawComment::RCK_JavaDoc; |
53 | else if (Comment[2] == '!') |
54 | K = RawComment::RCK_Qt; |
55 | else |
56 | return std::make_pair(x: RawComment::RCK_OrdinaryC, y: false); |
57 | } |
58 | const bool = (Comment.size() > 3) && (Comment[3] == '<'); |
59 | return std::make_pair(x&: K, y: TrailingComment); |
60 | } |
61 | |
62 | bool (StringRef ) { |
63 | return (Comment.size() > 3) && (Comment[3] == '<'); |
64 | } |
65 | |
66 | /// Returns true if R1 and R2 both have valid locations that start on the same |
67 | /// column. |
68 | bool commentsStartOnSameColumn(const SourceManager &SM, const RawComment &R1, |
69 | const RawComment &R2) { |
70 | SourceLocation L1 = R1.getBeginLoc(); |
71 | SourceLocation L2 = R2.getBeginLoc(); |
72 | bool Invalid = false; |
73 | unsigned C1 = SM.getPresumedColumnNumber(Loc: L1, Invalid: &Invalid); |
74 | if (!Invalid) { |
75 | unsigned C2 = SM.getPresumedColumnNumber(Loc: L2, Invalid: &Invalid); |
76 | return !Invalid && (C1 == C2); |
77 | } |
78 | return false; |
79 | } |
80 | } // unnamed namespace |
81 | |
82 | /// Determines whether there is only whitespace in `Buffer` between `P` |
83 | /// and the previous line. |
84 | /// \param Buffer The buffer to search in. |
85 | /// \param P The offset from the beginning of `Buffer` to start from. |
86 | /// \return true if all of the characters in `Buffer` ranging from the closest |
87 | /// line-ending character before `P` (or the beginning of `Buffer`) to `P - 1` |
88 | /// are whitespace. |
89 | static bool onlyWhitespaceOnLineBefore(const char *Buffer, unsigned P) { |
90 | // Search backwards until we see linefeed or carriage return. |
91 | for (unsigned I = P; I != 0; --I) { |
92 | char C = Buffer[I - 1]; |
93 | if (isVerticalWhitespace(c: C)) |
94 | return true; |
95 | if (!isHorizontalWhitespace(c: C)) |
96 | return false; |
97 | } |
98 | // We hit the beginning of the buffer. |
99 | return true; |
100 | } |
101 | |
102 | /// Returns whether `K` is an ordinary comment kind. |
103 | static bool (RawComment::CommentKind K) { |
104 | return (K == RawComment::RCK_OrdinaryBCPL) || |
105 | (K == RawComment::RCK_OrdinaryC); |
106 | } |
107 | |
108 | RawComment::(const SourceManager &SourceMgr, SourceRange SR, |
109 | const CommentOptions &, bool Merged) : |
110 | Range(SR), RawTextValid(false), BriefTextValid(false), |
111 | IsAttached(false), IsTrailingComment(false), |
112 | IsAlmostTrailingComment(false) { |
113 | // Extract raw comment text, if possible. |
114 | if (SR.getBegin() == SR.getEnd() || getRawText(SourceMgr).empty()) { |
115 | Kind = RCK_Invalid; |
116 | return; |
117 | } |
118 | |
119 | // Guess comment kind. |
120 | std::pair<CommentKind, bool> K = |
121 | getCommentKind(Comment: RawText, ParseAllComments: CommentOpts.ParseAllComments); |
122 | |
123 | // Guess whether an ordinary comment is trailing. |
124 | if (CommentOpts.ParseAllComments && isOrdinaryKind(K: K.first)) { |
125 | FileID BeginFileID; |
126 | unsigned BeginOffset; |
127 | std::tie(args&: BeginFileID, args&: BeginOffset) = |
128 | SourceMgr.getDecomposedLoc(Loc: Range.getBegin()); |
129 | if (BeginOffset != 0) { |
130 | bool Invalid = false; |
131 | const char *Buffer = |
132 | SourceMgr.getBufferData(FID: BeginFileID, Invalid: &Invalid).data(); |
133 | IsTrailingComment |= |
134 | (!Invalid && !onlyWhitespaceOnLineBefore(Buffer, P: BeginOffset)); |
135 | } |
136 | } |
137 | |
138 | if (!Merged) { |
139 | Kind = K.first; |
140 | IsTrailingComment |= K.second; |
141 | |
142 | IsAlmostTrailingComment = |
143 | RawText.starts_with(Prefix: "//<" ) || RawText.starts_with(Prefix: "/*<" ); |
144 | } else { |
145 | Kind = RCK_Merged; |
146 | IsTrailingComment = |
147 | IsTrailingComment || mergedCommentIsTrailingComment(Comment: RawText); |
148 | } |
149 | } |
150 | |
151 | StringRef RawComment::(const SourceManager &SourceMgr) const { |
152 | FileID BeginFileID; |
153 | FileID EndFileID; |
154 | unsigned BeginOffset; |
155 | unsigned EndOffset; |
156 | |
157 | std::tie(args&: BeginFileID, args&: BeginOffset) = |
158 | SourceMgr.getDecomposedLoc(Loc: Range.getBegin()); |
159 | std::tie(args&: EndFileID, args&: EndOffset) = SourceMgr.getDecomposedLoc(Loc: Range.getEnd()); |
160 | |
161 | const unsigned Length = EndOffset - BeginOffset; |
162 | if (Length < 2) |
163 | return StringRef(); |
164 | |
165 | // The comment can't begin in one file and end in another. |
166 | assert(BeginFileID == EndFileID); |
167 | |
168 | bool Invalid = false; |
169 | const char *BufferStart = SourceMgr.getBufferData(FID: BeginFileID, |
170 | Invalid: &Invalid).data(); |
171 | if (Invalid) |
172 | return StringRef(); |
173 | |
174 | return StringRef(BufferStart + BeginOffset, Length); |
175 | } |
176 | |
177 | const char *RawComment::(const ASTContext &Context) const { |
178 | // Lazily initialize RawText using the accessor before using it. |
179 | (void)getRawText(SourceMgr: Context.getSourceManager()); |
180 | |
181 | // Since we will be copying the resulting text, all allocations made during |
182 | // parsing are garbage after resulting string is formed. Thus we can use |
183 | // a separate allocator for all temporary stuff. |
184 | llvm::BumpPtrAllocator Allocator; |
185 | |
186 | comments::Lexer L(Allocator, Context.getDiagnostics(), |
187 | Context.getCommentCommandTraits(), |
188 | Range.getBegin(), |
189 | RawText.begin(), RawText.end()); |
190 | comments::BriefParser P(L, Context.getCommentCommandTraits()); |
191 | |
192 | const std::string Result = P.Parse(); |
193 | const unsigned BriefTextLength = Result.size(); |
194 | char *BriefTextPtr = new (Context) char[BriefTextLength + 1]; |
195 | memcpy(dest: BriefTextPtr, src: Result.c_str(), n: BriefTextLength + 1); |
196 | BriefText = BriefTextPtr; |
197 | BriefTextValid = true; |
198 | |
199 | return BriefTextPtr; |
200 | } |
201 | |
202 | comments::FullComment *RawComment::(const ASTContext &Context, |
203 | const Preprocessor *PP, |
204 | const Decl *D) const { |
205 | // Lazily initialize RawText using the accessor before using it. |
206 | (void)getRawText(SourceMgr: Context.getSourceManager()); |
207 | |
208 | comments::Lexer L(Context.getAllocator(), Context.getDiagnostics(), |
209 | Context.getCommentCommandTraits(), |
210 | getSourceRange().getBegin(), |
211 | RawText.begin(), RawText.end()); |
212 | comments::Sema S(Context.getAllocator(), Context.getSourceManager(), |
213 | Context.getDiagnostics(), |
214 | Context.getCommentCommandTraits(), |
215 | PP); |
216 | S.setDecl(D); |
217 | comments::Parser P(L, S, Context.getAllocator(), Context.getSourceManager(), |
218 | Context.getDiagnostics(), |
219 | Context.getCommentCommandTraits()); |
220 | |
221 | return P.parseFullComment(); |
222 | } |
223 | |
224 | static bool onlyWhitespaceBetween(SourceManager &SM, |
225 | SourceLocation Loc1, SourceLocation Loc2, |
226 | unsigned MaxNewlinesAllowed) { |
227 | std::pair<FileID, unsigned> Loc1Info = SM.getDecomposedLoc(Loc: Loc1); |
228 | std::pair<FileID, unsigned> Loc2Info = SM.getDecomposedLoc(Loc: Loc2); |
229 | |
230 | // Question does not make sense if locations are in different files. |
231 | if (Loc1Info.first != Loc2Info.first) |
232 | return false; |
233 | |
234 | bool Invalid = false; |
235 | const char *Buffer = SM.getBufferData(FID: Loc1Info.first, Invalid: &Invalid).data(); |
236 | if (Invalid) |
237 | return false; |
238 | |
239 | unsigned NumNewlines = 0; |
240 | assert(Loc1Info.second <= Loc2Info.second && "Loc1 after Loc2!" ); |
241 | // Look for non-whitespace characters and remember any newlines seen. |
242 | for (unsigned I = Loc1Info.second; I != Loc2Info.second; ++I) { |
243 | switch (Buffer[I]) { |
244 | default: |
245 | return false; |
246 | case ' ': |
247 | case '\t': |
248 | case '\f': |
249 | case '\v': |
250 | break; |
251 | case '\r': |
252 | case '\n': |
253 | ++NumNewlines; |
254 | |
255 | // Check if we have found more than the maximum allowed number of |
256 | // newlines. |
257 | if (NumNewlines > MaxNewlinesAllowed) |
258 | return false; |
259 | |
260 | // Collapse \r\n and \n\r into a single newline. |
261 | if (I + 1 != Loc2Info.second && |
262 | (Buffer[I + 1] == '\n' || Buffer[I + 1] == '\r') && |
263 | Buffer[I] != Buffer[I + 1]) |
264 | ++I; |
265 | break; |
266 | } |
267 | } |
268 | |
269 | return true; |
270 | } |
271 | |
272 | void RawCommentList::(const RawComment &RC, |
273 | const CommentOptions &, |
274 | llvm::BumpPtrAllocator &Allocator) { |
275 | if (RC.isInvalid()) |
276 | return; |
277 | |
278 | // Ordinary comments are not interesting for us. |
279 | if (RC.isOrdinary() && !CommentOpts.ParseAllComments) |
280 | return; |
281 | |
282 | std::pair<FileID, unsigned> Loc = |
283 | SourceMgr.getDecomposedLoc(Loc: RC.getBeginLoc()); |
284 | |
285 | const FileID = Loc.first; |
286 | const unsigned = Loc.second; |
287 | |
288 | // If this is the first Doxygen comment, save it (because there isn't |
289 | // anything to merge it with). |
290 | auto &OC = OrderedComments[CommentFile]; |
291 | if (OC.empty()) { |
292 | OC[CommentOffset] = new (Allocator) RawComment(RC); |
293 | return; |
294 | } |
295 | |
296 | const RawComment &C1 = *OC.rbegin()->second; |
297 | const RawComment &C2 = RC; |
298 | |
299 | // Merge comments only if there is only whitespace between them. |
300 | // Can't merge trailing and non-trailing comments unless the second is |
301 | // non-trailing ordinary in the same column, as in the case: |
302 | // int x; // documents x |
303 | // // more text |
304 | // versus: |
305 | // int x; // documents x |
306 | // int y; // documents y |
307 | // or: |
308 | // int x; // documents x |
309 | // // documents y |
310 | // int y; |
311 | // Merge comments if they are on same or consecutive lines. |
312 | if ((C1.isTrailingComment() == C2.isTrailingComment() || |
313 | (C1.isTrailingComment() && !C2.isTrailingComment() && |
314 | isOrdinaryKind(K: C2.getKind()) && |
315 | commentsStartOnSameColumn(SM: SourceMgr, R1: C1, R2: C2))) && |
316 | onlyWhitespaceBetween(SM&: SourceMgr, Loc1: C1.getEndLoc(), Loc2: C2.getBeginLoc(), |
317 | /*MaxNewlinesAllowed=*/1)) { |
318 | SourceRange MergedRange(C1.getBeginLoc(), C2.getEndLoc()); |
319 | *OrderedComments[CommentFile].rbegin()->second = |
320 | RawComment(SourceMgr, MergedRange, CommentOpts, true); |
321 | } else { |
322 | OrderedComments[CommentFile][CommentOffset] = |
323 | new (Allocator) RawComment(RC); |
324 | } |
325 | } |
326 | |
327 | const std::map<unsigned, RawComment *> * |
328 | RawCommentList::(FileID File) const { |
329 | auto = OrderedComments.find(Val: File); |
330 | if (CommentsInFile == OrderedComments.end()) |
331 | return nullptr; |
332 | |
333 | return &CommentsInFile->second; |
334 | } |
335 | |
336 | bool RawCommentList::() const { return OrderedComments.empty(); } |
337 | |
338 | unsigned RawCommentList::(RawComment *C, FileID File, |
339 | unsigned Offset) const { |
340 | auto Cached = CommentBeginLine.find(Val: C); |
341 | if (Cached != CommentBeginLine.end()) |
342 | return Cached->second; |
343 | const unsigned Line = SourceMgr.getLineNumber(FID: File, FilePos: Offset); |
344 | CommentBeginLine[C] = Line; |
345 | return Line; |
346 | } |
347 | |
348 | unsigned RawCommentList::(RawComment *C) const { |
349 | auto Cached = CommentEndOffset.find(Val: C); |
350 | if (Cached != CommentEndOffset.end()) |
351 | return Cached->second; |
352 | const unsigned Offset = |
353 | SourceMgr.getDecomposedLoc(Loc: C->getSourceRange().getEnd()).second; |
354 | CommentEndOffset[C] = Offset; |
355 | return Offset; |
356 | } |
357 | |
358 | std::string RawComment::(const SourceManager &SourceMgr, |
359 | DiagnosticsEngine &Diags) const { |
360 | llvm::StringRef = getRawText(SourceMgr); |
361 | if (CommentText.empty()) |
362 | return "" ; |
363 | |
364 | std::string Result; |
365 | for (const RawComment::CommentLine &Line : |
366 | getFormattedLines(SourceMgr, Diags)) |
367 | Result += Line.Text + "\n" ; |
368 | |
369 | auto LastChar = Result.find_last_not_of(c: '\n'); |
370 | Result.erase(pos: LastChar + 1, n: Result.size()); |
371 | |
372 | return Result; |
373 | } |
374 | |
375 | std::vector<RawComment::CommentLine> |
376 | RawComment::(const SourceManager &SourceMgr, |
377 | DiagnosticsEngine &Diags) const { |
378 | llvm::StringRef = getRawText(SourceMgr); |
379 | if (CommentText.empty()) |
380 | return {}; |
381 | |
382 | llvm::BumpPtrAllocator Allocator; |
383 | // We do not parse any commands, so CommentOptions are ignored by |
384 | // comments::Lexer. Therefore, we just use default-constructed options. |
385 | CommentOptions DefOpts; |
386 | comments::CommandTraits EmptyTraits(Allocator, DefOpts); |
387 | comments::Lexer L(Allocator, Diags, EmptyTraits, getSourceRange().getBegin(), |
388 | CommentText.begin(), CommentText.end(), |
389 | /*ParseCommands=*/false); |
390 | |
391 | std::vector<RawComment::CommentLine> Result; |
392 | // A column number of the first non-whitespace token in the comment text. |
393 | // We skip whitespace up to this column, but keep the whitespace after this |
394 | // column. IndentColumn is calculated when lexing the first line and reused |
395 | // for the rest of lines. |
396 | unsigned IndentColumn = 0; |
397 | |
398 | // Record the line number of the last processed comment line. |
399 | // For block-style comments, an extra newline token will be produced after |
400 | // the end-comment marker, e.g.: |
401 | // /** This is a multi-line comment block. |
402 | // The lexer will produce two newline tokens here > */ |
403 | // previousLine will record the line number when we previously saw a newline |
404 | // token and recorded a comment line. If we see another newline token on the |
405 | // same line, don't record anything in between. |
406 | unsigned PreviousLine = 0; |
407 | |
408 | // Processes one line of the comment and adds it to the result. |
409 | // Handles skipping the indent at the start of the line. |
410 | // Returns false when eof is reached and true otherwise. |
411 | auto LexLine = [&](bool IsFirstLine) -> bool { |
412 | comments::Token Tok; |
413 | // Lex the first token on the line. We handle it separately, because we to |
414 | // fix up its indentation. |
415 | L.lex(T&: Tok); |
416 | if (Tok.is(K: comments::tok::eof)) |
417 | return false; |
418 | if (Tok.is(K: comments::tok::newline)) { |
419 | PresumedLoc Loc = SourceMgr.getPresumedLoc(Loc: Tok.getLocation()); |
420 | if (Loc.getLine() != PreviousLine) { |
421 | Result.emplace_back(args: "" , args&: Loc, args&: Loc); |
422 | PreviousLine = Loc.getLine(); |
423 | } |
424 | return true; |
425 | } |
426 | SmallString<124> Line; |
427 | llvm::StringRef TokText = L.getSpelling(Tok, SourceMgr); |
428 | bool LocInvalid = false; |
429 | unsigned TokColumn = |
430 | SourceMgr.getSpellingColumnNumber(Loc: Tok.getLocation(), Invalid: &LocInvalid); |
431 | assert(!LocInvalid && "getFormattedText for invalid location" ); |
432 | |
433 | // Amount of leading whitespace in TokText. |
434 | size_t WhitespaceLen = TokText.find_first_not_of(Chars: " \t" ); |
435 | if (WhitespaceLen == StringRef::npos) |
436 | WhitespaceLen = TokText.size(); |
437 | // Remember the amount of whitespace we skipped in the first line to remove |
438 | // indent up to that column in the following lines. |
439 | if (IsFirstLine) |
440 | IndentColumn = TokColumn + WhitespaceLen; |
441 | |
442 | // Amount of leading whitespace we actually want to skip. |
443 | // For the first line we skip all the whitespace. |
444 | // For the rest of the lines, we skip whitespace up to IndentColumn. |
445 | unsigned SkipLen = |
446 | IsFirstLine |
447 | ? WhitespaceLen |
448 | : std::min<size_t>( |
449 | a: WhitespaceLen, |
450 | b: std::max<int>(a: static_cast<int>(IndentColumn) - TokColumn, b: 0)); |
451 | llvm::StringRef Trimmed = TokText.drop_front(N: SkipLen); |
452 | Line += Trimmed; |
453 | // Get the beginning location of the adjusted comment line. |
454 | PresumedLoc Begin = |
455 | SourceMgr.getPresumedLoc(Loc: Tok.getLocation().getLocWithOffset(Offset: SkipLen)); |
456 | |
457 | // Lex all tokens in the rest of the line. |
458 | for (L.lex(T&: Tok); Tok.isNot(K: comments::tok::eof); L.lex(T&: Tok)) { |
459 | if (Tok.is(K: comments::tok::newline)) { |
460 | // Get the ending location of the comment line. |
461 | PresumedLoc End = SourceMgr.getPresumedLoc(Loc: Tok.getLocation()); |
462 | if (End.getLine() != PreviousLine) { |
463 | Result.emplace_back(args&: Line, args&: Begin, args&: End); |
464 | PreviousLine = End.getLine(); |
465 | } |
466 | return true; |
467 | } |
468 | Line += L.getSpelling(Tok, SourceMgr); |
469 | } |
470 | PresumedLoc End = SourceMgr.getPresumedLoc(Loc: Tok.getLocation()); |
471 | Result.emplace_back(args&: Line, args&: Begin, args&: End); |
472 | // We've reached the end of file token. |
473 | return false; |
474 | }; |
475 | |
476 | // Process first line separately to remember indent for the following lines. |
477 | if (!LexLine(/*IsFirstLine=*/true)) |
478 | return Result; |
479 | // Process the rest of the lines. |
480 | while (LexLine(/*IsFirstLine=*/false)) |
481 | ; |
482 | return Result; |
483 | } |
484 | |