1//===- ExtractAPI/ExtractAPIConsumer.cpp ------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the ExtractAPIAction, and ASTConsumer to collect API
11/// information.
12///
13//===----------------------------------------------------------------------===//
14
15#include "clang/AST/ASTConcept.h"
16#include "clang/AST/ASTConsumer.h"
17#include "clang/AST/ASTContext.h"
18#include "clang/AST/DeclObjC.h"
19#include "clang/Basic/DiagnosticFrontend.h"
20#include "clang/Basic/FileEntry.h"
21#include "clang/Basic/SourceLocation.h"
22#include "clang/Basic/SourceManager.h"
23#include "clang/Basic/TargetInfo.h"
24#include "clang/ExtractAPI/API.h"
25#include "clang/ExtractAPI/APIIgnoresList.h"
26#include "clang/ExtractAPI/ExtractAPIVisitor.h"
27#include "clang/ExtractAPI/FrontendActions.h"
28#include "clang/ExtractAPI/Serialization/SymbolGraphSerializer.h"
29#include "clang/Frontend/ASTConsumers.h"
30#include "clang/Frontend/CompilerInstance.h"
31#include "clang/Frontend/FrontendOptions.h"
32#include "clang/Frontend/MultiplexConsumer.h"
33#include "clang/Index/USRGeneration.h"
34#include "clang/InstallAPI/HeaderFile.h"
35#include "clang/Lex/MacroInfo.h"
36#include "clang/Lex/PPCallbacks.h"
37#include "clang/Lex/Preprocessor.h"
38#include "clang/Lex/PreprocessorOptions.h"
39#include "llvm/ADT/DenseSet.h"
40#include "llvm/ADT/STLExtras.h"
41#include "llvm/ADT/SmallString.h"
42#include "llvm/ADT/SmallVector.h"
43#include "llvm/ADT/StringRef.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/Error.h"
46#include "llvm/Support/FileSystem.h"
47#include "llvm/Support/MemoryBuffer.h"
48#include "llvm/Support/Path.h"
49#include "llvm/Support/Regex.h"
50#include "llvm/Support/raw_ostream.h"
51#include <memory>
52#include <optional>
53#include <utility>
54
55using namespace clang;
56using namespace extractapi;
57
58namespace {
59
60std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
61 StringRef File,
62 bool *IsQuoted = nullptr) {
63 assert(CI.hasFileManager() &&
64 "CompilerInstance does not have a FileNamager!");
65
66 using namespace llvm::sys;
67 const auto &FS = CI.getVirtualFileSystem();
68
69 SmallString<128> FilePath(File.begin(), File.end());
70 FS.makeAbsolute(Path&: FilePath);
71 path::remove_dots(path&: FilePath, remove_dot_dot: true);
72 FilePath = path::convert_to_slash(path: FilePath);
73 File = FilePath;
74
75 // Checks whether `Dir` is a strict path prefix of `File`. If so returns
76 // the prefix length. Otherwise return 0.
77 auto CheckDir = [&](llvm::StringRef Dir) -> unsigned {
78 llvm::SmallString<32> DirPath(Dir.begin(), Dir.end());
79 FS.makeAbsolute(Path&: DirPath);
80 path::remove_dots(path&: DirPath, remove_dot_dot: true);
81 Dir = DirPath;
82 for (auto NI = path::begin(path: File), NE = path::end(path: File),
83 DI = path::begin(path: Dir), DE = path::end(path: Dir);
84 /*termination condition in loop*/; ++NI, ++DI) {
85 // '.' components in File are ignored.
86 while (NI != NE && *NI == ".")
87 ++NI;
88 if (NI == NE)
89 break;
90
91 // '.' components in Dir are ignored.
92 while (DI != DE && *DI == ".")
93 ++DI;
94
95 // Dir is a prefix of File, up to '.' components and choice of path
96 // separators.
97 if (DI == DE)
98 return NI - path::begin(path: File);
99
100 // Consider all path separators equal.
101 if (NI->size() == 1 && DI->size() == 1 &&
102 path::is_separator(value: NI->front()) && path::is_separator(value: DI->front()))
103 continue;
104
105 // Special case Apple .sdk folders since the search path is typically a
106 // symlink like `iPhoneSimulator14.5.sdk` while the file is instead
107 // located in `iPhoneSimulator.sdk` (the real folder).
108 if (NI->ends_with(Suffix: ".sdk") && DI->ends_with(Suffix: ".sdk")) {
109 StringRef NBasename = path::stem(path: *NI);
110 StringRef DBasename = path::stem(path: *DI);
111 if (DBasename.starts_with(Prefix: NBasename))
112 continue;
113 }
114
115 if (*NI != *DI)
116 break;
117 }
118 return 0;
119 };
120
121 unsigned PrefixLength = 0;
122
123 // Go through the search paths and find the first one that is a prefix of
124 // the header.
125 for (const auto &Entry : CI.getHeaderSearchOpts().UserEntries) {
126 // Note whether the match is found in a quoted entry.
127 if (IsQuoted)
128 *IsQuoted = Entry.Group == frontend::Quoted;
129
130 if (auto EntryFile = CI.getFileManager().getOptionalFileRef(Filename: Entry.Path)) {
131 if (auto HMap = HeaderMap::Create(FE: *EntryFile, FM&: CI.getFileManager())) {
132 // If this is a headermap entry, try to reverse lookup the full path
133 // for a spelled name before mapping.
134 StringRef SpelledFilename = HMap->reverseLookupFilename(DestPath: File);
135 if (!SpelledFilename.empty())
136 return SpelledFilename.str();
137
138 // No matching mapping in this headermap, try next search entry.
139 continue;
140 }
141 }
142
143 // Entry is a directory search entry, try to check if it's a prefix of File.
144 PrefixLength = CheckDir(Entry.Path);
145 if (PrefixLength > 0) {
146 // The header is found in a framework path, construct the framework-style
147 // include name `<Framework/Header.h>`
148 if (Entry.IsFramework) {
149 SmallVector<StringRef, 4> Matches;
150 clang::installapi::HeaderFile::getFrameworkIncludeRule().match(
151 String: File, Matches: &Matches);
152 // Returned matches are always in stable order.
153 if (Matches.size() != 4)
154 return std::nullopt;
155
156 return path::convert_to_slash(
157 path: (Matches[1].drop_front(N: Matches[1].rfind(C: '/') + 1) + "/" +
158 Matches[3])
159 .str());
160 }
161
162 // The header is found in a normal search path, strip the search path
163 // prefix to get an include name.
164 return path::convert_to_slash(path: File.drop_front(N: PrefixLength));
165 }
166 }
167
168 // Couldn't determine a include name, use full path instead.
169 return std::nullopt;
170}
171
172std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
173 FileEntryRef FE,
174 bool *IsQuoted = nullptr) {
175 return getRelativeIncludeName(CI, File: FE.getNameAsRequested(), IsQuoted);
176}
177
178struct LocationFileChecker {
179 bool operator()(SourceLocation Loc) {
180 // If the loc refers to a macro expansion we need to first get the file
181 // location of the expansion.
182 auto &SM = CI.getSourceManager();
183 auto FileLoc = SM.getFileLoc(Loc);
184 FileID FID = SM.getFileID(SpellingLoc: FileLoc);
185 if (FID.isInvalid())
186 return false;
187
188 OptionalFileEntryRef File = SM.getFileEntryRefForID(FID);
189 if (!File)
190 return false;
191
192 if (KnownFileEntries.count(V: *File))
193 return true;
194
195 if (ExternalFileEntries.count(V: *File))
196 return false;
197
198 // Try to reduce the include name the same way we tried to include it.
199 bool IsQuoted = false;
200 if (auto IncludeName = getRelativeIncludeName(CI, FE: *File, IsQuoted: &IsQuoted))
201 if (llvm::any_of(Range&: KnownFiles,
202 P: [&IsQuoted, &IncludeName](const auto &KnownFile) {
203 return KnownFile.first.equals(*IncludeName) &&
204 KnownFile.second == IsQuoted;
205 })) {
206 KnownFileEntries.insert(V: *File);
207 return true;
208 }
209
210 // Record that the file was not found to avoid future reverse lookup for
211 // the same file.
212 ExternalFileEntries.insert(V: *File);
213 return false;
214 }
215
216 LocationFileChecker(const CompilerInstance &CI,
217 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles)
218 : CI(CI), KnownFiles(KnownFiles), ExternalFileEntries() {
219 for (const auto &KnownFile : KnownFiles)
220 if (auto FE = CI.getFileManager().getOptionalFileRef(Filename: KnownFile.first))
221 KnownFileEntries.insert(V: *FE);
222 }
223
224private:
225 const CompilerInstance &CI;
226 SmallVector<std::pair<SmallString<32>, bool>> &KnownFiles;
227 llvm::DenseSet<const FileEntry *> KnownFileEntries;
228 llvm::DenseSet<const FileEntry *> ExternalFileEntries;
229};
230
231struct BatchExtractAPIVisitor : ExtractAPIVisitor<BatchExtractAPIVisitor> {
232 bool shouldDeclBeIncluded(const Decl *D) const {
233 bool ShouldBeIncluded = true;
234 // Check that we have the definition for redeclarable types.
235 if (auto *TD = llvm::dyn_cast<TagDecl>(Val: D))
236 ShouldBeIncluded = TD->isThisDeclarationADefinition();
237 else if (auto *Interface = llvm::dyn_cast<ObjCInterfaceDecl>(Val: D))
238 ShouldBeIncluded = Interface->isThisDeclarationADefinition();
239 else if (auto *Protocol = llvm::dyn_cast<ObjCProtocolDecl>(Val: D))
240 ShouldBeIncluded = Protocol->isThisDeclarationADefinition();
241
242 ShouldBeIncluded = ShouldBeIncluded && LCF(D->getLocation());
243 return ShouldBeIncluded;
244 }
245
246 BatchExtractAPIVisitor(LocationFileChecker &LCF, ASTContext &Context,
247 APISet &API)
248 : ExtractAPIVisitor<BatchExtractAPIVisitor>(Context, API), LCF(LCF) {}
249
250private:
251 LocationFileChecker &LCF;
252};
253
254class WrappingExtractAPIConsumer : public ASTConsumer {
255public:
256 WrappingExtractAPIConsumer(ASTContext &Context, APISet &API)
257 : Visitor(Context, API) {}
258
259 void HandleTranslationUnit(ASTContext &Context) override {
260 // Use ExtractAPIVisitor to traverse symbol declarations in the context.
261 Visitor.TraverseDecl(Context.getTranslationUnitDecl());
262 }
263
264private:
265 ExtractAPIVisitor<> Visitor;
266};
267
268class ExtractAPIConsumer : public ASTConsumer {
269public:
270 ExtractAPIConsumer(ASTContext &Context,
271 std::unique_ptr<LocationFileChecker> LCF, APISet &API)
272 : Visitor(*LCF, Context, API), LCF(std::move(LCF)) {}
273
274 void HandleTranslationUnit(ASTContext &Context) override {
275 // Use ExtractAPIVisitor to traverse symbol declarations in the context.
276 Visitor.TraverseDecl(Context.getTranslationUnitDecl());
277 }
278
279private:
280 BatchExtractAPIVisitor Visitor;
281 std::unique_ptr<LocationFileChecker> LCF;
282};
283
284class MacroCallback : public PPCallbacks {
285public:
286 MacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP)
287 : SM(SM), API(API), PP(PP) {}
288
289 void EndOfMainFile() override {
290 for (const auto &M : PP.macros()) {
291 auto *II = M.getFirst();
292 auto MD = PP.getMacroDefinition(II);
293 auto *MI = MD.getMacroInfo();
294
295 if (!MI)
296 continue;
297
298 // Ignore header guard macros
299 if (MI->isUsedForHeaderGuard())
300 continue;
301
302 // Ignore builtin macros and ones defined via the command line.
303 if (MI->isBuiltinMacro())
304 continue;
305
306 auto DefLoc = MI->getDefinitionLoc();
307
308 if (SM.isInPredefinedFile(Loc: DefLoc))
309 continue;
310
311 auto AssociatedModuleMacros = MD.getModuleMacros();
312 StringRef OwningModuleName;
313 if (!AssociatedModuleMacros.empty())
314 OwningModuleName = AssociatedModuleMacros.back()
315 ->getOwningModule()
316 ->getTopLevelModuleName();
317
318 if (!shouldMacroBeIncluded(MacroLoc: DefLoc, ModuleName: OwningModuleName))
319 continue;
320
321 StringRef Name = II->getName();
322 PresumedLoc Loc = SM.getPresumedLoc(Loc: DefLoc);
323 SmallString<128> USR;
324 index::generateUSRForMacro(MacroName: Name, Loc: DefLoc, SM, Buf&: USR);
325 API.createRecord<extractapi::MacroDefinitionRecord>(
326 USR, Name, CtorArgs: SymbolReference(), CtorArgs&: Loc,
327 CtorArgs: DeclarationFragmentsBuilder::getFragmentsForMacro(Name, MI),
328 CtorArgs: DeclarationFragmentsBuilder::getSubHeadingForMacro(Name),
329 CtorArgs: SM.isInSystemHeader(Loc: DefLoc));
330 }
331 }
332
333 virtual bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,
334 StringRef ModuleName) {
335 return true;
336 }
337
338 const SourceManager &SM;
339 APISet &API;
340 Preprocessor &PP;
341};
342
343class APIMacroCallback : public MacroCallback {
344public:
345 APIMacroCallback(const SourceManager &SM, APISet &API, Preprocessor &PP,
346 LocationFileChecker &LCF)
347 : MacroCallback(SM, API, PP), LCF(LCF) {}
348
349 bool shouldMacroBeIncluded(const SourceLocation &MacroLoc,
350 StringRef ModuleName) override {
351 // Do not include macros from external files
352 return LCF(MacroLoc);
353 }
354
355private:
356 LocationFileChecker &LCF;
357};
358
359std::unique_ptr<llvm::raw_pwrite_stream>
360createAdditionalSymbolGraphFile(CompilerInstance &CI, Twine BaseName) {
361 auto OutputDirectory = CI.getFrontendOpts().SymbolGraphOutputDir;
362
363 SmallString<256> FileName;
364 llvm::sys::path::append(path&: FileName, a: OutputDirectory,
365 b: BaseName + ".symbols.json");
366 return CI.createOutputFile(
367 OutputPath: FileName, /*Binary*/ false, /*RemoveFileOnSignal*/ false,
368 /*UseTemporary*/ true, /*CreateMissingDirectories*/ true);
369}
370
371} // namespace
372
373void ExtractAPIActionBase::ImplEndSourceFileAction(CompilerInstance &CI) {
374 SymbolGraphSerializerOption SerializationOptions;
375 SerializationOptions.Compact = !CI.getFrontendOpts().EmitPrettySymbolGraphs;
376 SerializationOptions.EmitSymbolLabelsForTesting =
377 CI.getFrontendOpts().EmitSymbolGraphSymbolLabelsForTesting;
378
379 if (CI.getFrontendOpts().EmitExtensionSymbolGraphs) {
380 auto ConstructOutputFile = [&CI](Twine BaseName) {
381 return createAdditionalSymbolGraphFile(CI, BaseName);
382 };
383
384 SymbolGraphSerializer::serializeWithExtensionGraphs(
385 MainOutput&: *OS, API: *API, IgnoresList, CreateOutputStream: ConstructOutputFile, Options: SerializationOptions);
386 } else {
387 SymbolGraphSerializer::serializeMainSymbolGraph(OS&: *OS, API: *API, IgnoresList,
388 Options: SerializationOptions);
389 }
390
391 // Flush the stream and close the main output stream.
392 OS.reset();
393}
394
395std::unique_ptr<ASTConsumer>
396ExtractAPIAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
397 auto ProductName = CI.getFrontendOpts().ProductName;
398
399 if (CI.getFrontendOpts().SymbolGraphOutputDir.empty())
400 OS = CI.createDefaultOutputFile(/*Binary*/ false, BaseInput: InFile,
401 /*Extension*/ "symbols.json",
402 /*RemoveFileOnSignal*/ false,
403 /*CreateMissingDirectories*/ true);
404 else
405 OS = createAdditionalSymbolGraphFile(CI, BaseName: ProductName);
406
407 if (!OS)
408 return nullptr;
409
410 // Now that we have enough information about the language options and the
411 // target triple, let's create the APISet before anyone uses it.
412 API = std::make_unique<APISet>(
413 args: CI.getTarget().getTriple(),
414 args: CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), args&: ProductName);
415
416 auto LCF = std::make_unique<LocationFileChecker>(args&: CI, args&: KnownInputFiles);
417
418 CI.getPreprocessor().addPPCallbacks(C: std::make_unique<APIMacroCallback>(
419 args&: CI.getSourceManager(), args&: *API, args&: CI.getPreprocessor(), args&: *LCF));
420
421 // Do not include location in anonymous decls.
422 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
423 Policy.AnonymousTagLocations = false;
424 CI.getASTContext().setPrintingPolicy(Policy);
425
426 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
427 llvm::handleAllErrors(
428 E: APIIgnoresList::create(IgnoresFilePathList: CI.getFrontendOpts().ExtractAPIIgnoresFileList,
429 FM&: CI.getFileManager())
430 .moveInto(Value&: IgnoresList),
431 Handlers: [&CI](const IgnoresFileNotFound &Err) {
432 CI.getDiagnostics().Report(
433 diag::err_extract_api_ignores_file_not_found)
434 << Err.Path;
435 });
436 }
437
438 return std::make_unique<ExtractAPIConsumer>(args&: CI.getASTContext(),
439 args: std::move(LCF), args&: *API);
440}
441
442bool ExtractAPIAction::PrepareToExecuteAction(CompilerInstance &CI) {
443 auto &Inputs = CI.getFrontendOpts().Inputs;
444 if (Inputs.empty())
445 return true;
446
447 if (!CI.hasFileManager())
448 if (!CI.createFileManager())
449 return false;
450
451 auto Kind = Inputs[0].getKind();
452
453 // Convert the header file inputs into a single input buffer.
454 SmallString<256> HeaderContents;
455 bool IsQuoted = false;
456 for (const FrontendInputFile &FIF : Inputs) {
457 if (Kind.isObjectiveC())
458 HeaderContents += "#import";
459 else
460 HeaderContents += "#include";
461
462 StringRef FilePath = FIF.getFile();
463 if (auto RelativeName = getRelativeIncludeName(CI, File: FilePath, IsQuoted: &IsQuoted)) {
464 if (IsQuoted)
465 HeaderContents += " \"";
466 else
467 HeaderContents += " <";
468
469 HeaderContents += *RelativeName;
470
471 if (IsQuoted)
472 HeaderContents += "\"\n";
473 else
474 HeaderContents += ">\n";
475 KnownInputFiles.emplace_back(Args: static_cast<SmallString<32>>(*RelativeName),
476 Args&: IsQuoted);
477 } else {
478 HeaderContents += " \"";
479 HeaderContents += FilePath;
480 HeaderContents += "\"\n";
481 KnownInputFiles.emplace_back(Args&: FilePath, Args: true);
482 }
483 }
484
485 if (CI.getHeaderSearchOpts().Verbose)
486 CI.getVerboseOutputStream() << getInputBufferName() << ":\n"
487 << HeaderContents << "\n";
488
489 Buffer = llvm::MemoryBuffer::getMemBufferCopy(InputData: HeaderContents,
490 BufferName: getInputBufferName());
491
492 // Set that buffer up as our "real" input in the CompilerInstance.
493 Inputs.clear();
494 Inputs.emplace_back(Args: Buffer->getMemBufferRef(), Args&: Kind, /*IsSystem*/ Args: false);
495
496 return true;
497}
498
499void ExtractAPIAction::EndSourceFileAction() {
500 ImplEndSourceFileAction(CI&: getCompilerInstance());
501}
502
503std::unique_ptr<ASTConsumer>
504WrappingExtractAPIAction::CreateASTConsumer(CompilerInstance &CI,
505 StringRef InFile) {
506 auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
507 if (!OtherConsumer)
508 return nullptr;
509
510 CreatedASTConsumer = true;
511
512 ProductName = CI.getFrontendOpts().ProductName;
513 auto InputFilename = llvm::sys::path::filename(path: InFile);
514 OS = createAdditionalSymbolGraphFile(CI, BaseName: InputFilename);
515
516 // Now that we have enough information about the language options and the
517 // target triple, let's create the APISet before anyone uses it.
518 API = std::make_unique<APISet>(
519 args: CI.getTarget().getTriple(),
520 args: CI.getFrontendOpts().Inputs.back().getKind().getLanguage(), args&: ProductName);
521
522 CI.getPreprocessor().addPPCallbacks(C: std::make_unique<MacroCallback>(
523 args&: CI.getSourceManager(), args&: *API, args&: CI.getPreprocessor()));
524
525 // Do not include location in anonymous decls.
526 PrintingPolicy Policy = CI.getASTContext().getPrintingPolicy();
527 Policy.AnonymousTagLocations = false;
528 CI.getASTContext().setPrintingPolicy(Policy);
529
530 if (!CI.getFrontendOpts().ExtractAPIIgnoresFileList.empty()) {
531 llvm::handleAllErrors(
532 E: APIIgnoresList::create(IgnoresFilePathList: CI.getFrontendOpts().ExtractAPIIgnoresFileList,
533 FM&: CI.getFileManager())
534 .moveInto(Value&: IgnoresList),
535 Handlers: [&CI](const IgnoresFileNotFound &Err) {
536 CI.getDiagnostics().Report(
537 diag::err_extract_api_ignores_file_not_found)
538 << Err.Path;
539 });
540 }
541
542 auto WrappingConsumer =
543 std::make_unique<WrappingExtractAPIConsumer>(args&: CI.getASTContext(), args&: *API);
544 std::vector<std::unique_ptr<ASTConsumer>> Consumers;
545 Consumers.push_back(x: std::move(OtherConsumer));
546 Consumers.push_back(x: std::move(WrappingConsumer));
547
548 return std::make_unique<MultiplexConsumer>(args: std::move(Consumers));
549}
550
551void WrappingExtractAPIAction::EndSourceFileAction() {
552 // Invoke wrapped action's method.
553 WrapperFrontendAction::EndSourceFileAction();
554
555 if (CreatedASTConsumer) {
556 ImplEndSourceFileAction(CI&: getCompilerInstance());
557 }
558}
559

source code of clang/lib/ExtractAPI/ExtractAPIConsumer.cpp