1//===--- ParsedAST.cpp -------------------------------------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ParsedAST.h"
10#include "../clang-tidy/ClangTidyCheck.h"
11#include "../clang-tidy/ClangTidyDiagnosticConsumer.h"
12#include "../clang-tidy/ClangTidyModule.h"
13#include "../clang-tidy/ClangTidyModuleRegistry.h"
14#include "../clang-tidy/ClangTidyOptions.h"
15#include "AST.h"
16#include "CollectMacros.h"
17#include "Compiler.h"
18#include "Config.h"
19#include "Diagnostics.h"
20#include "Feature.h"
21#include "FeatureModule.h"
22#include "Headers.h"
23#include "IncludeCleaner.h"
24#include "IncludeFixer.h"
25#include "Preamble.h"
26#include "SourceCode.h"
27#include "TidyProvider.h"
28#include "clang-include-cleaner/Record.h"
29#include "index/Symbol.h"
30#include "support/Logger.h"
31#include "support/Path.h"
32#include "support/Trace.h"
33#include "clang/AST/ASTContext.h"
34#include "clang/AST/Decl.h"
35#include "clang/AST/DeclGroup.h"
36#include "clang/AST/ExternalASTSource.h"
37#include "clang/ASTMatchers/ASTMatchFinder.h"
38#include "clang/Basic/Diagnostic.h"
39#include "clang/Basic/DiagnosticIDs.h"
40#include "clang/Basic/DiagnosticSema.h"
41#include "clang/Basic/FileEntry.h"
42#include "clang/Basic/LLVM.h"
43#include "clang/Basic/LangOptions.h"
44#include "clang/Basic/SourceLocation.h"
45#include "clang/Basic/SourceManager.h"
46#include "clang/Basic/TokenKinds.h"
47#include "clang/Frontend/CompilerInstance.h"
48#include "clang/Frontend/CompilerInvocation.h"
49#include "clang/Frontend/FrontendActions.h"
50#include "clang/Frontend/FrontendOptions.h"
51#include "clang/Frontend/PrecompiledPreamble.h"
52#include "clang/Lex/Lexer.h"
53#include "clang/Lex/PPCallbacks.h"
54#include "clang/Lex/Preprocessor.h"
55#include "clang/Sema/HeuristicResolver.h"
56#include "clang/Serialization/ASTWriter.h"
57#include "clang/Tooling/CompilationDatabase.h"
58#include "clang/Tooling/Core/Diagnostic.h"
59#include "clang/Tooling/Syntax/Tokens.h"
60#include "llvm/ADT/ArrayRef.h"
61#include "llvm/ADT/DenseMap.h"
62#include "llvm/ADT/DenseSet.h"
63#include "llvm/ADT/STLExtras.h"
64#include "llvm/ADT/STLFunctionalExtras.h"
65#include "llvm/ADT/SmallVector.h"
66#include "llvm/ADT/StringRef.h"
67#include "llvm/Support/Error.h"
68#include "llvm/Support/MemoryBuffer.h"
69#include <cassert>
70#include <cstddef>
71#include <iterator>
72#include <memory>
73#include <optional>
74#include <string>
75#include <tuple>
76#include <utility>
77#include <vector>
78
79// Force the linker to link in Clang-tidy modules.
80// clangd doesn't support the static analyzer.
81#if CLANGD_TIDY_CHECKS
82#define CLANG_TIDY_DISABLE_STATIC_ANALYZER_CHECKS
83#include "../clang-tidy/ClangTidyForceLinker.h"
84#endif
85
86namespace clang {
87namespace clangd {
88namespace {
89
90template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
91 return Vec.capacity() * sizeof(T);
92}
93
94class DeclTrackingASTConsumer : public ASTConsumer {
95public:
96 DeclTrackingASTConsumer(std::vector<Decl *> &TopLevelDecls)
97 : TopLevelDecls(TopLevelDecls) {}
98
99 bool HandleTopLevelDecl(DeclGroupRef DG) override {
100 for (Decl *D : DG) {
101 auto &SM = D->getASTContext().getSourceManager();
102 if (!isInsideMainFile(Loc: D->getLocation(), SM))
103 continue;
104 if (const NamedDecl *ND = dyn_cast<NamedDecl>(Val: D))
105 if (isImplicitTemplateInstantiation(D: ND))
106 continue;
107
108 // ObjCMethodDecl are not actually top-level decls.
109 if (isa<ObjCMethodDecl>(Val: D))
110 continue;
111
112 TopLevelDecls.push_back(x: D);
113 }
114 return true;
115 }
116
117private:
118 std::vector<Decl *> &TopLevelDecls;
119};
120
121class ClangdFrontendAction : public SyntaxOnlyAction {
122public:
123 std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); }
124
125protected:
126 std::unique_ptr<ASTConsumer>
127 CreateASTConsumer(CompilerInstance &CI, llvm::StringRef InFile) override {
128 return std::make_unique<DeclTrackingASTConsumer>(/*ref*/ args&: TopLevelDecls);
129 }
130
131private:
132 std::vector<Decl *> TopLevelDecls;
133};
134
135// When using a preamble, only preprocessor events outside its bounds are seen.
136// This is almost what we want: replaying transitive preprocessing wastes time.
137// However this confuses clang-tidy checks: they don't see any #includes!
138// So we replay the *non-transitive* #includes that appear in the main-file.
139// It would be nice to replay other events (macro definitions, ifdefs etc) but
140// this addresses the most common cases fairly cheaply.
141class ReplayPreamble : private PPCallbacks {
142public:
143 // Attach preprocessor hooks such that preamble events will be injected at
144 // the appropriate time.
145 // Events will be delivered to the *currently registered* PP callbacks.
146 static void attach(std::vector<Inclusion> Includes, CompilerInstance &Clang,
147 const PreambleBounds &PB) {
148 auto &PP = Clang.getPreprocessor();
149 auto *ExistingCallbacks = PP.getPPCallbacks();
150 // No need to replay events if nobody is listening.
151 if (!ExistingCallbacks)
152 return;
153 PP.addPPCallbacks(C: std::unique_ptr<PPCallbacks>(new ReplayPreamble(
154 std::move(Includes), ExistingCallbacks, Clang.getSourceManager(), PP,
155 Clang.getLangOpts(), PB)));
156 // We're relying on the fact that addPPCallbacks keeps the old PPCallbacks
157 // around, creating a chaining wrapper. Guard against other implementations.
158 assert(PP.getPPCallbacks() != ExistingCallbacks &&
159 "Expected chaining implementation");
160 }
161
162private:
163 ReplayPreamble(std::vector<Inclusion> Includes, PPCallbacks *Delegate,
164 const SourceManager &SM, Preprocessor &PP,
165 const LangOptions &LangOpts, const PreambleBounds &PB)
166 : Includes(std::move(Includes)), Delegate(Delegate), SM(SM), PP(PP) {
167 // Only tokenize the preamble section of the main file, as we are not
168 // interested in the rest of the tokens.
169 MainFileTokens = syntax::tokenize(
170 FR: syntax::FileRange(SM.getMainFileID(), 0, PB.Size), SM, LO: LangOpts);
171 }
172
173 // In a normal compile, the preamble traverses the following structure:
174 //
175 // mainfile.cpp
176 // <built-in>
177 // ... macro definitions like __cplusplus ...
178 // <command-line>
179 // ... macro definitions for args like -Dfoo=bar ...
180 // "header1.h"
181 // ... header file contents ...
182 // "header2.h"
183 // ... header file contents ...
184 // ... main file contents ...
185 //
186 // When using a preamble, the "header1" and "header2" subtrees get skipped.
187 // We insert them right after the built-in header, which still appears.
188 void FileChanged(SourceLocation Loc, FileChangeReason Reason,
189 SrcMgr::CharacteristicKind Kind, FileID PrevFID) override {
190 // It'd be nice if there was a better way to identify built-in headers...
191 if (Reason == FileChangeReason::ExitFile &&
192 SM.getBufferOrFake(FID: PrevFID).getBufferIdentifier() == "<built-in>")
193 replay();
194 }
195
196 void replay() {
197 for (const auto &Inc : Includes) {
198 OptionalFileEntryRef File;
199 if (Inc.Resolved != "")
200 File = expectedToOptional(E: SM.getFileManager().getFileRef(Filename: Inc.Resolved));
201
202 // Re-lex the #include directive to find its interesting parts.
203 auto HashLoc = SM.getComposedLoc(FID: SM.getMainFileID(), Offset: Inc.HashOffset);
204 auto HashTok = llvm::partition_point(Range&: MainFileTokens,
205 P: [&HashLoc](const syntax::Token &T) {
206 return T.location() < HashLoc;
207 });
208 assert(HashTok != MainFileTokens.end() && HashTok->kind() == tok::hash);
209
210 auto IncludeTok = std::next(x: HashTok);
211 assert(IncludeTok != MainFileTokens.end());
212
213 auto FileTok = std::next(x: IncludeTok);
214 assert(FileTok != MainFileTokens.end());
215
216 // Create a fake import/include token, none of the callers seem to care
217 // about clang::Token::Flags.
218 Token SynthesizedIncludeTok;
219 SynthesizedIncludeTok.startToken();
220 SynthesizedIncludeTok.setLocation(IncludeTok->location());
221 SynthesizedIncludeTok.setLength(IncludeTok->length());
222 SynthesizedIncludeTok.setKind(tok::raw_identifier);
223 SynthesizedIncludeTok.setRawIdentifierData(IncludeTok->text(SM).data());
224 PP.LookUpIdentifierInfo(Identifier&: SynthesizedIncludeTok);
225
226 // Same here, create a fake one for Filename, including angles or quotes.
227 Token SynthesizedFilenameTok;
228 SynthesizedFilenameTok.startToken();
229 SynthesizedFilenameTok.setLocation(FileTok->location());
230 // Note that we can't make use of FileTok->length/text in here as in the
231 // case of angled includes this will contain tok::less instead of
232 // filename. Whereas Inc.Written contains the full header name including
233 // quotes/angles.
234 SynthesizedFilenameTok.setLength(Inc.Written.length());
235 SynthesizedFilenameTok.setKind(tok::header_name);
236 SynthesizedFilenameTok.setLiteralData(Inc.Written.data());
237
238 llvm::StringRef WrittenFilename =
239 llvm::StringRef(Inc.Written).drop_front().drop_back();
240 Delegate->InclusionDirective(
241 HashLoc: HashTok->location(), IncludeTok: SynthesizedIncludeTok, FileName: WrittenFilename,
242 IsAngled: Inc.Written.front() == '<',
243 FilenameRange: syntax::FileRange(SM, SynthesizedFilenameTok.getLocation(),
244 SynthesizedFilenameTok.getEndLoc())
245 .toCharRange(SM),
246 File, SearchPath: "SearchPath", RelativePath: "RelPath",
247 /*SuggestedModule=*/nullptr, /*ModuleImported=*/false, FileType: Inc.FileKind);
248 if (File)
249 Delegate->FileSkipped(SkippedFile: *File, FilenameTok: SynthesizedFilenameTok, FileType: Inc.FileKind);
250 }
251 }
252
253 const std::vector<Inclusion> Includes;
254 PPCallbacks *Delegate;
255 const SourceManager &SM;
256 Preprocessor &PP;
257 std::vector<syntax::Token> MainFileTokens;
258};
259
260// Filter for clang diagnostics groups enabled by CTOptions.Checks.
261//
262// These are check names like clang-diagnostics-unused.
263// Note that unlike -Wunused, clang-diagnostics-unused does not imply
264// subcategories like clang-diagnostics-unused-function.
265//
266// This is used to determine which diagnostics can be enabled by ExtraArgs in
267// the clang-tidy configuration.
268class TidyDiagnosticGroups {
269 // Whether all diagnostic groups are enabled by default.
270 // True if we've seen clang-diagnostic-*.
271 bool Default = false;
272 // Set of diag::Group whose enablement != Default.
273 // If Default is false, this is foo where we've seen clang-diagnostic-foo.
274 llvm::DenseSet<unsigned> Exceptions;
275
276public:
277 TidyDiagnosticGroups(llvm::StringRef Checks) {
278 constexpr llvm::StringLiteral CDPrefix = "clang-diagnostic-";
279
280 llvm::StringRef Check;
281 while (!Checks.empty()) {
282 std::tie(args&: Check, args&: Checks) = Checks.split(Separator: ',');
283 Check = Check.trim();
284
285 if (Check.empty())
286 continue;
287
288 bool Enable = !Check.consume_front(Prefix: "-");
289 bool Glob = Check.consume_back(Suffix: "*");
290 if (Glob) {
291 // Is this clang-diagnostic-*, or *, or so?
292 // (We ignore all other types of globs).
293 if (CDPrefix.starts_with(Prefix: Check)) {
294 Default = Enable;
295 Exceptions.clear();
296 }
297 continue;
298 }
299
300 // In "*,clang-diagnostic-foo", the latter is a no-op.
301 if (Default == Enable)
302 continue;
303 // The only non-glob entries we care about are clang-diagnostic-foo.
304 if (!Check.consume_front(Prefix: CDPrefix))
305 continue;
306
307 if (auto Group = DiagnosticIDs::getGroupForWarningOption(Check))
308 Exceptions.insert(V: static_cast<unsigned>(*Group));
309 }
310 }
311
312 bool operator()(diag::Group GroupID) const {
313 return Exceptions.contains(V: static_cast<unsigned>(GroupID)) ? !Default
314 : Default;
315 }
316};
317
318// Find -W<group> and -Wno-<group> options in ExtraArgs and apply them to Diags.
319//
320// This is used to handle ExtraArgs in clang-tidy configuration.
321// We don't use clang's standard handling of this as we want slightly different
322// behavior (e.g. we want to exclude these from -Wno-error).
323void applyWarningOptions(llvm::ArrayRef<std::string> ExtraArgs,
324 llvm::function_ref<bool(diag::Group)> EnabledGroups,
325 DiagnosticsEngine &Diags) {
326 for (llvm::StringRef Group : ExtraArgs) {
327 // Only handle args that are of the form -W[no-]<group>.
328 // Other flags are possible but rare and deliberately out of scope.
329 llvm::SmallVector<diag::kind> Members;
330 if (!Group.consume_front(Prefix: "-W") || Group.empty())
331 continue;
332 bool Enable = !Group.consume_front(Prefix: "no-");
333 if (Diags.getDiagnosticIDs()->getDiagnosticsInGroup(
334 Flavor: diag::Flavor::WarningOrError, Group, Diags&: Members))
335 continue;
336
337 // Upgrade (or downgrade) the severity of each diagnostic in the group.
338 // If -Werror is on, newly added warnings will be treated as errors.
339 // We don't want this, so keep track of them to fix afterwards.
340 bool NeedsWerrorExclusion = false;
341 for (diag::kind ID : Members) {
342 if (Enable) {
343 if (Diags.getDiagnosticLevel(DiagID: ID, Loc: SourceLocation()) <
344 DiagnosticsEngine::Warning) {
345 auto Group = Diags.getDiagnosticIDs()->getGroupForDiag(DiagID: ID);
346 if (!Group || !EnabledGroups(*Group))
347 continue;
348 Diags.setSeverity(Diag: ID, Map: diag::Severity::Warning, Loc: SourceLocation());
349 if (Diags.getWarningsAsErrors())
350 NeedsWerrorExclusion = true;
351 }
352 } else {
353 Diags.setSeverity(Diag: ID, Map: diag::Severity::Ignored, Loc: SourceLocation());
354 }
355 }
356 if (NeedsWerrorExclusion) {
357 // FIXME: there's no API to suppress -Werror for single diagnostics.
358 // In some cases with sub-groups, we may end up erroneously
359 // downgrading diagnostics that were -Werror in the compile command.
360 Diags.setDiagnosticGroupWarningAsError(Group, Enabled: false);
361 }
362 }
363}
364
365std::vector<Diag> getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code,
366 const ThreadsafeFS &TFS) {
367 auto &Cfg = Config::current();
368 if (Cfg.Diagnostics.SuppressAll)
369 return {};
370 bool SuppressMissing =
371 Cfg.Diagnostics.Suppress.contains(key: "missing-includes") ||
372 Cfg.Diagnostics.MissingIncludes == Config::IncludesPolicy::None;
373 bool SuppressUnused =
374 Cfg.Diagnostics.Suppress.contains(key: "unused-includes") ||
375 Cfg.Diagnostics.UnusedIncludes == Config::IncludesPolicy::None;
376 if (SuppressMissing && SuppressUnused)
377 return {};
378 auto Findings = computeIncludeCleanerFindings(
379 AST, AnalyzeAngledIncludes: Cfg.Diagnostics.Includes.AnalyzeAngledIncludes);
380 if (SuppressMissing)
381 Findings.MissingIncludes.clear();
382 if (SuppressUnused)
383 Findings.UnusedIncludes.clear();
384 return issueIncludeCleanerDiagnostics(
385 AST, Code, Findings, TFS, IgnoreHeader: Cfg.Diagnostics.Includes.IgnoreHeader,
386 AngledHeaders: Cfg.Style.AngledHeaders, QuotedHeaders: Cfg.Style.QuotedHeaders);
387}
388
389tidy::ClangTidyCheckFactories
390filterFastTidyChecks(const tidy::ClangTidyCheckFactories &All,
391 Config::FastCheckPolicy Policy) {
392 if (Policy == Config::FastCheckPolicy::None)
393 return All;
394 bool AllowUnknown = Policy == Config::FastCheckPolicy::Loose;
395 tidy::ClangTidyCheckFactories Fast;
396 for (const auto &Factory : All) {
397 if (isFastTidyCheck(Check: Factory.getKey()).value_or(u&: AllowUnknown))
398 Fast.registerCheckFactory(Name: Factory.first(), Factory: Factory.second);
399 }
400 return Fast;
401}
402
403} // namespace
404
405std::optional<ParsedAST>
406ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
407 std::unique_ptr<clang::CompilerInvocation> CI,
408 llvm::ArrayRef<Diag> CompilerInvocationDiags,
409 std::shared_ptr<const PreambleData> Preamble) {
410 trace::Span Tracer("BuildAST");
411 SPAN_ATTACH(Tracer, "File", Filename);
412 const Config &Cfg = Config::current();
413
414 auto VFS = Inputs.TFS->view(CWD: Inputs.CompileCommand.Directory);
415 if (Preamble && Preamble->StatCache)
416 VFS = Preamble->StatCache->getConsumingFS(FS: std::move(VFS));
417
418 assert(CI);
419
420 if (CI->getFrontendOpts().Inputs.size() > 0) {
421 auto Lang = CI->getFrontendOpts().Inputs[0].getKind().getLanguage();
422 if (Lang == Language::Asm || Lang == Language::LLVM_IR) {
423 elog(Fmt: "Clangd does not support assembly or IR source files");
424 return std::nullopt;
425 }
426 }
427
428 // Command-line parsing sets DisableFree to true by default, but we don't want
429 // to leak memory in clangd.
430 CI->getFrontendOpts().DisableFree = false;
431 const PrecompiledPreamble *PreamblePCH =
432 Preamble ? &Preamble->Preamble : nullptr;
433
434 // This is on-by-default in windows to allow parsing SDK headers, but it
435 // breaks many features. Disable it for the main-file (not preamble).
436 CI->getLangOpts().DelayedTemplateParsing = false;
437
438 std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
439 if (Inputs.FeatureModules) {
440 for (auto &M : *Inputs.FeatureModules) {
441 if (auto Listener = M.astListeners())
442 ASTListeners.emplace_back(args: std::move(Listener));
443 }
444 }
445 StoreDiags ASTDiags;
446 ASTDiags.setDiagCallback(
447 [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
448 for (const auto &L : ASTListeners)
449 L->sawDiagnostic(D, Diag);
450 });
451
452 // Adjust header search options to load the built module files recorded
453 // in RequiredModules.
454 if (Preamble && Preamble->RequiredModules)
455 Preamble->RequiredModules->adjustHeaderSearchOptions(
456 Options&: CI->getHeaderSearchOpts());
457
458 std::optional<PreamblePatch> Patch;
459 // We might use an ignoring diagnostic consumer if they are going to be
460 // dropped later on to not pay for extra latency by processing them.
461 DiagnosticConsumer *DiagConsumer = &ASTDiags;
462 IgnoreDiagnostics DropDiags;
463 if (Preamble) {
464 Patch = PreamblePatch::createFullPatch(FileName: Filename, Modified: Inputs, Baseline: *Preamble);
465 Patch->apply(CI&: *CI);
466 }
467 auto Clang = prepareCompilerInstance(
468 std::move(CI), PreamblePCH,
469 MainFile: llvm::MemoryBuffer::getMemBufferCopy(InputData: Inputs.Contents, BufferName: Filename), VFS,
470 *DiagConsumer);
471
472 if (!Clang) {
473 // The last diagnostic contains information about the reason of this
474 // failure.
475 std::vector<Diag> Diags(ASTDiags.take());
476 elog(Fmt: "Failed to prepare a compiler instance: {0}",
477 Vals: !Diags.empty() ? static_cast<DiagBase &>(Diags.back()).Message
478 : "unknown error");
479 return std::nullopt;
480 }
481 tidy::ClangTidyOptions ClangTidyOpts;
482 {
483 trace::Span Tracer("ClangTidyOpts");
484 ClangTidyOpts = getTidyOptionsForFile(Provider: Inputs.ClangTidyProvider, Filename);
485 dlog("ClangTidy configuration for file {0}: {1}", Filename,
486 tidy::configurationAsText(ClangTidyOpts));
487
488 // If clang-tidy is configured to emit clang warnings, we should too.
489 //
490 // Such clang-tidy configuration consists of two parts:
491 // - ExtraArgs: ["-Wfoo"] causes clang to produce the warnings
492 // - Checks: "clang-diagnostic-foo" prevents clang-tidy filtering them out
493 //
494 // In clang-tidy, diagnostics are emitted if they pass both checks.
495 // When groups contain subgroups, -Wparent includes the child, but
496 // clang-diagnostic-parent does not.
497 //
498 // We *don't* want to change the compile command directly. This can have
499 // too many unexpected effects: breaking the command, interactions with
500 // -- and -Werror, etc. Besides, we've already parsed the command.
501 // Instead we parse the -W<group> flags and handle them directly.
502 //
503 // Similarly, we don't want to use Checks to filter clang diagnostics after
504 // they are generated, as this spreads clang-tidy emulation everywhere.
505 // Instead, we just use these to filter which extra diagnostics we enable.
506 auto &Diags = Clang->getDiagnostics();
507 TidyDiagnosticGroups TidyGroups(ClangTidyOpts.Checks ? *ClangTidyOpts.Checks
508 : llvm::StringRef());
509 if (ClangTidyOpts.ExtraArgsBefore)
510 applyWarningOptions(ExtraArgs: *ClangTidyOpts.ExtraArgsBefore, EnabledGroups: TidyGroups, Diags);
511 if (ClangTidyOpts.ExtraArgs)
512 applyWarningOptions(ExtraArgs: *ClangTidyOpts.ExtraArgs, EnabledGroups: TidyGroups, Diags);
513 }
514
515 auto Action = std::make_unique<ClangdFrontendAction>();
516 const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
517 if (!Action->BeginSourceFile(CI&: *Clang, Input: MainInput)) {
518 elog(Fmt: "BeginSourceFile() failed when building AST for {0}",
519 Vals: MainInput.getFile());
520 return std::nullopt;
521 }
522 // If we saw an include guard in the preamble section of the main file,
523 // mark the main-file as include-guarded.
524 // This information is part of the HeaderFileInfo but is not loaded from the
525 // preamble as the file's size is part of its identity and may have changed.
526 // (The rest of HeaderFileInfo is not relevant for our purposes).
527 if (Preamble && Preamble->MainIsIncludeGuarded) {
528 const SourceManager &SM = Clang->getSourceManager();
529 OptionalFileEntryRef MainFE = SM.getFileEntryRefForID(FID: SM.getMainFileID());
530 Clang->getPreprocessor().getHeaderSearchInfo().MarkFileIncludeOnce(File: *MainFE);
531 }
532
533 // Set up ClangTidy. Must happen after BeginSourceFile() so ASTContext exists.
534 // Clang-tidy has some limitations to ensure reasonable performance:
535 // - checks don't see all preprocessor events in the preamble
536 // - matchers run only over the main-file top-level decls (and can't see
537 // ancestors outside this scope).
538 // In practice almost all checks work well without modifications.
539 std::vector<std::unique_ptr<tidy::ClangTidyCheck>> CTChecks;
540 ast_matchers::MatchFinder CTFinder;
541 std::optional<tidy::ClangTidyContext> CTContext;
542 // Must outlive FixIncludes.
543 auto BuildDir = VFS->getCurrentWorkingDirectory();
544 std::optional<IncludeFixer> FixIncludes;
545 llvm::DenseMap<diag::kind, DiagnosticsEngine::Level> OverriddenSeverity;
546 // No need to run clang-tidy or IncludeFixerif we are not going to surface
547 // diagnostics.
548 {
549 trace::Span Tracer("ClangTidyInit");
550 static const auto *AllCTFactories = [] {
551 auto *CTFactories = new tidy::ClangTidyCheckFactories;
552 for (const auto &E : tidy::ClangTidyModuleRegistry::entries())
553 E.instantiate()->addCheckFactories(CheckFactories&: *CTFactories);
554 return CTFactories;
555 }();
556 tidy::ClangTidyCheckFactories FastFactories = filterFastTidyChecks(
557 All: *AllCTFactories, Policy: Cfg.Diagnostics.ClangTidy.FastCheckFilter);
558 CTContext.emplace(args: std::make_unique<tidy::DefaultOptionsProvider>(
559 args: tidy::ClangTidyGlobalOptions(), args&: ClangTidyOpts));
560 // The lifetime of DiagnosticOptions is managed by \c Clang.
561 CTContext->setDiagnosticsEngine(DiagOpts: nullptr, DiagEngine: &Clang->getDiagnostics());
562 CTContext->setASTContext(&Clang->getASTContext());
563 CTContext->setCurrentFile(Filename);
564 CTContext->setSelfContainedDiags(true);
565 CTChecks = FastFactories.createChecksForLanguage(Context: &*CTContext);
566 Preprocessor *PP = &Clang->getPreprocessor();
567 for (const auto &Check : CTChecks) {
568 Check->registerPPCallbacks(SM: Clang->getSourceManager(), PP, ModuleExpanderPP: PP);
569 Check->registerMatchers(Finder: &CTFinder);
570 }
571
572 // Clang only corrects typos for use of undeclared functions in C if that
573 // use is an error. Include fixer relies on typo correction, so pretend
574 // this is an error. (The actual typo correction is nice too).
575 // We restore the original severity in the level adjuster.
576 // FIXME: It would be better to have a real API for this, but what?
577 for (auto ID : {diag::ext_implicit_function_decl_c99,
578 diag::ext_implicit_lib_function_decl,
579 diag::ext_implicit_lib_function_decl_c99,
580 diag::warn_implicit_function_decl}) {
581 OverriddenSeverity.try_emplace(
582 ID, Clang->getDiagnostics().getDiagnosticLevel(ID, SourceLocation()));
583 Clang->getDiagnostics().setSeverity(ID, diag::Severity::Error,
584 SourceLocation());
585 }
586
587 ASTDiags.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
588 const clang::Diagnostic &Info) {
589 if (Cfg.Diagnostics.SuppressAll ||
590 isDiagnosticSuppressed(Diag: Info, Suppressed: Cfg.Diagnostics.Suppress,
591 Clang->getLangOpts()))
592 return DiagnosticsEngine::Ignored;
593
594 auto It = OverriddenSeverity.find(Val: Info.getID());
595 if (It != OverriddenSeverity.end())
596 DiagLevel = It->second;
597
598 if (!CTChecks.empty()) {
599 std::string CheckName = CTContext->getCheckName(DiagnosticID: Info.getID());
600 bool IsClangTidyDiag = !CheckName.empty();
601 if (IsClangTidyDiag) {
602 if (Cfg.Diagnostics.Suppress.contains(key: CheckName))
603 return DiagnosticsEngine::Ignored;
604 // Check for suppression comment. Skip the check for diagnostics not
605 // in the main file, because we don't want that function to query the
606 // source buffer for preamble files. For the same reason, we ask
607 // shouldSuppressDiagnostic to avoid I/O.
608 // We let suppression comments take precedence over warning-as-error
609 // to match clang-tidy's behaviour.
610 bool IsInsideMainFile =
611 Info.hasSourceManager() &&
612 isInsideMainFile(Loc: Info.getLocation(), SM: Info.getSourceManager());
613 SmallVector<tooling::Diagnostic, 1> TidySuppressedErrors;
614 if (IsInsideMainFile && CTContext->shouldSuppressDiagnostic(
615 DiagLevel, Info, NoLintErrors&: TidySuppressedErrors,
616 /*AllowIO=*/false,
617 /*EnableNolintBlocks=*/EnableNoLintBlocks: true)) {
618 // FIXME: should we expose the suppression error (invalid use of
619 // NOLINT comments)?
620 return DiagnosticsEngine::Ignored;
621 }
622 if (!CTContext->getOptions().SystemHeaders.value_or(u: false) &&
623 Info.hasSourceManager() &&
624 Info.getSourceManager().isInSystemMacro(loc: Info.getLocation()))
625 return DiagnosticsEngine::Ignored;
626
627 // Check for warning-as-error.
628 if (DiagLevel == DiagnosticsEngine::Warning &&
629 CTContext->treatAsError(CheckName)) {
630 return DiagnosticsEngine::Error;
631 }
632 }
633 }
634 return DiagLevel;
635 });
636
637 // Add IncludeFixer which can recover diagnostics caused by missing includes
638 // (e.g. incomplete type) and attach include insertion fixes to diagnostics.
639 if (Inputs.Index && !BuildDir.getError()) {
640 auto Style =
641 getFormatStyleForFile(File: Filename, Content: Inputs.Contents, TFS: *Inputs.TFS, FormatFile: false);
642 auto Inserter = std::make_shared<IncludeInserter>(
643 args&: Filename, args: Inputs.Contents, args&: Style, args&: BuildDir.get(),
644 args: &Clang->getPreprocessor().getHeaderSearchInfo(),
645 args: Cfg.Style.QuotedHeaders, args: Cfg.Style.AngledHeaders);
646 ArrayRef<Inclusion> MainFileIncludes;
647 if (Preamble) {
648 MainFileIncludes = Preamble->Includes.MainFileIncludes;
649 for (const auto &Inc : Preamble->Includes.MainFileIncludes)
650 Inserter->addExisting(Inc);
651 }
652 // FIXME: Consider piping through ASTSignals to fetch this to handle the
653 // case where a header file contains ObjC decls but no #imports.
654 Symbol::IncludeDirective Directive =
655 Inputs.Opts.ImportInsertions
656 ? preferredIncludeDirective(FileName: Filename, LangOpts: Clang->getLangOpts(),
657 MainFileIncludes, TopLevelDecls: {})
658 : Symbol::Include;
659 FixIncludes.emplace(args&: Filename, args&: Inserter, args: *Inputs.Index,
660 /*IndexRequestLimit=*/args: 5, args&: Directive);
661 ASTDiags.contributeFixes(Fixer: [&FixIncludes](DiagnosticsEngine::Level DiagLevl,
662 const clang::Diagnostic &Info) {
663 return FixIncludes->fix(DiagLevel: DiagLevl, Info);
664 });
665 Clang->setExternalSemaSource(FixIncludes->unresolvedNameRecorder());
666 }
667 }
668
669 IncludeStructure Includes;
670 include_cleaner::PragmaIncludes PI;
671 // If we are using a preamble, copy existing includes.
672 if (Preamble) {
673 Includes = Preamble->Includes;
674 Includes.MainFileIncludes = Patch->preambleIncludes();
675 // Replay the preamble includes so that clang-tidy checks can see them.
676 ReplayPreamble::attach(Includes: Patch->preambleIncludes(), Clang&: *Clang,
677 PB: Patch->modifiedBounds());
678 PI = *Preamble->Pragmas;
679 }
680 // Important: collectIncludeStructure is registered *after* ReplayPreamble!
681 // Otherwise we would collect the replayed includes again...
682 // (We can't *just* use the replayed includes, they don't have Resolved path).
683 Includes.collect(CI: *Clang);
684 // Same for pragma-includes, we're already inheriting preamble includes, so we
685 // should only receive callbacks for non-preamble mainfile includes.
686 PI.record(CI: *Clang);
687 // Copy over the macros in the preamble region of the main file, and combine
688 // with non-preamble macros below.
689 MainFileMacros Macros;
690 std::vector<PragmaMark> Marks;
691 if (Preamble) {
692 Macros = Patch->mainFileMacros();
693 Marks = Patch->marks();
694 }
695 auto &PP = Clang->getPreprocessor();
696 auto MacroCollector = std::make_unique<CollectMainFileMacros>(args&: PP, args&: Macros);
697 auto *MacroCollectorPtr = MacroCollector.get(); // so we can call doneParse()
698 PP.addPPCallbacks(C: std::move(MacroCollector));
699
700 PP.addPPCallbacks(
701 C: collectPragmaMarksCallback(Clang->getSourceManager(), Out&: Marks));
702
703 // FIXME: Attach a comment handler to take care of
704 // keep/export/no_include etc. IWYU pragmas.
705
706 // Collect tokens of the main file.
707 syntax::TokenCollector CollectTokens(PP);
708
709 // To remain consistent with preamble builds, these callbacks must be called
710 // exactly here, after preprocessor is initialized and BeginSourceFile() was
711 // called already.
712 for (const auto &L : ASTListeners)
713 L->beforeExecute(CI&: *Clang);
714
715 if (llvm::Error Err = Action->Execute())
716 log(Fmt: "Execute() failed when building AST for {0}: {1}", Vals: MainInput.getFile(),
717 Vals: toString(E: std::move(Err)));
718
719 // Disable the macro collector for the remainder of this function, e.g.
720 // clang-tidy checkers.
721 MacroCollectorPtr->doneParse();
722
723 // We have to consume the tokens before running clang-tidy to avoid collecting
724 // tokens from running the preprocessor inside the checks (only
725 // modernize-use-trailing-return-type does that today).
726 syntax::TokenBuffer Tokens = std::move(CollectTokens).consume();
727 // Makes SelectionTree build much faster.
728 Tokens.indexExpandedTokens();
729 std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls();
730 // AST traversals should exclude the preamble, to avoid performance cliffs.
731 Clang->getASTContext().setTraversalScope(ParsedDecls);
732 if (!CTChecks.empty()) {
733 // Run the AST-dependent part of the clang-tidy checks.
734 // (The preprocessor part ran already, via PPCallbacks).
735 trace::Span Tracer("ClangTidyMatch");
736 CTFinder.matchAST(Context&: Clang->getASTContext());
737 }
738
739 // XXX: This is messy: clang-tidy checks flush some diagnostics at EOF.
740 // However Action->EndSourceFile() would destroy the ASTContext!
741 // So just inform the preprocessor of EOF, while keeping everything alive.
742 PP.EndSourceFile();
743 // UnitDiagsConsumer is local, we can not store it in CompilerInstance that
744 // has a longer lifetime.
745 Clang->getDiagnostics().setClient(client: new IgnoreDiagnostics);
746 // CompilerInstance won't run this callback, do it directly.
747 ASTDiags.EndSourceFile();
748
749 std::vector<Diag> Diags = CompilerInvocationDiags;
750 // FIXME: Also skip generation of diagnostics altogether to speed up ast
751 // builds when we are patching a stale preamble.
752 // Add diagnostics from the preamble, if any.
753 if (Preamble)
754 llvm::append_range(C&: Diags, R: Patch->patchedDiags());
755 // Finally, add diagnostics coming from the AST.
756 {
757 std::vector<Diag> D = ASTDiags.take(Tidy: &*CTContext);
758 Diags.insert(position: Diags.end(), first: D.begin(), last: D.end());
759 }
760 ParsedAST Result(Filename, Inputs.Version, std::move(Preamble),
761 std::move(Clang), std::move(Action), std::move(Tokens),
762 std::move(Macros), std::move(Marks), std::move(ParsedDecls),
763 std::move(Diags), std::move(Includes), std::move(PI));
764 llvm::move(Range: getIncludeCleanerDiags(AST&: Result, Code: Inputs.Contents, TFS: *Inputs.TFS),
765 Out: std::back_inserter(x&: Result.Diags));
766 return std::move(Result);
767}
768
769ParsedAST::ParsedAST(ParsedAST &&Other) = default;
770
771ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default;
772
773ParsedAST::~ParsedAST() {
774 if (Action) {
775 // We already notified the PP of end-of-file earlier, so detach it first.
776 // We must keep it alive until after EndSourceFile(), Sema relies on this.
777 auto PP = Clang->getPreprocessorPtr(); // Keep PP alive for now.
778 Clang->setPreprocessor(nullptr); // Detach so we don't send EOF again.
779 Action->EndSourceFile(); // Destroy ASTContext and Sema.
780 // Now Sema is gone, it's safe for PP to go out of scope.
781 }
782}
783
784ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); }
785
786const ASTContext &ParsedAST::getASTContext() const {
787 return Clang->getASTContext();
788}
789
790Sema &ParsedAST::getSema() { return Clang->getSema(); }
791
792Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); }
793
794std::shared_ptr<Preprocessor> ParsedAST::getPreprocessorPtr() {
795 return Clang->getPreprocessorPtr();
796}
797
798const Preprocessor &ParsedAST::getPreprocessor() const {
799 return Clang->getPreprocessor();
800}
801
802llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() {
803 return LocalTopLevelDecls;
804}
805
806llvm::ArrayRef<const Decl *> ParsedAST::getLocalTopLevelDecls() const {
807 return LocalTopLevelDecls;
808}
809
810const MainFileMacros &ParsedAST::getMacros() const { return Macros; }
811const std::vector<PragmaMark> &ParsedAST::getMarks() const { return Marks; }
812
813std::size_t ParsedAST::getUsedBytes() const {
814 auto &AST = getASTContext();
815 // FIXME(ibiryukov): we do not account for the dynamically allocated part of
816 // Message and Fixes inside each diagnostic.
817 std::size_t Total =
818 clangd::getUsedBytes(Vec: LocalTopLevelDecls) + clangd::getUsedBytes(Vec: Diags);
819
820 // FIXME: the rest of the function is almost a direct copy-paste from
821 // libclang's clang_getCXTUResourceUsage. We could share the implementation.
822
823 // Sum up various allocators inside the ast context and the preprocessor.
824 Total += AST.getASTAllocatedMemory();
825 Total += AST.getSideTableAllocatedMemory();
826 Total += AST.Idents.getAllocator().getTotalMemory();
827 Total += AST.Selectors.getTotalMemory();
828
829 Total += AST.getSourceManager().getContentCacheSize();
830 Total += AST.getSourceManager().getDataStructureSizes();
831 Total += AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
832
833 if (ExternalASTSource *Ext = AST.getExternalSource())
834 Total += Ext->getMemoryBufferSizes().malloc_bytes;
835
836 const Preprocessor &PP = getPreprocessor();
837 Total += PP.getTotalMemory();
838 if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
839 Total += PRec->getTotalMemory();
840 Total += PP.getHeaderSearchInfo().getTotalMemory();
841
842 return Total;
843}
844
845const IncludeStructure &ParsedAST::getIncludeStructure() const {
846 return Includes;
847}
848
849ParsedAST::ParsedAST(PathRef TUPath, llvm::StringRef Version,
850 std::shared_ptr<const PreambleData> Preamble,
851 std::unique_ptr<CompilerInstance> Clang,
852 std::unique_ptr<FrontendAction> Action,
853 syntax::TokenBuffer Tokens, MainFileMacros Macros,
854 std::vector<PragmaMark> Marks,
855 std::vector<Decl *> LocalTopLevelDecls,
856 std::vector<Diag> Diags, IncludeStructure Includes,
857 include_cleaner::PragmaIncludes PI)
858 : TUPath(TUPath), Version(Version), Preamble(std::move(Preamble)),
859 Clang(std::move(Clang)), Action(std::move(Action)),
860 Tokens(std::move(Tokens)), Macros(std::move(Macros)),
861 Marks(std::move(Marks)), Diags(std::move(Diags)),
862 LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
863 Includes(std::move(Includes)), PI(std::move(PI)),
864 Resolver(std::make_unique<HeuristicResolver>(args&: getASTContext())) {
865 assert(this->Clang);
866 assert(this->Action);
867}
868
869const include_cleaner::PragmaIncludes &ParsedAST::getPragmaIncludes() const {
870 return PI;
871}
872
873std::optional<llvm::StringRef> ParsedAST::preambleVersion() const {
874 if (!Preamble)
875 return std::nullopt;
876 return llvm::StringRef(Preamble->Version);
877}
878
879llvm::ArrayRef<Diag> ParsedAST::getDiagnostics() const { return Diags; }
880} // namespace clangd
881} // namespace clang
882

source code of clang-tools-extra/clangd/ParsedAST.cpp