1//===- ClangScanDeps.cpp - Implementation of clang-scan-deps --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Driver/Compilation.h"
10#include "clang/Driver/Driver.h"
11#include "clang/Frontend/CompilerInstance.h"
12#include "clang/Frontend/TextDiagnosticPrinter.h"
13#include "clang/Tooling/CommonOptionsParser.h"
14#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
15#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
16#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
17#include "clang/Tooling/JSONCompilationDatabase.h"
18#include "clang/Tooling/Tooling.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/Support/CommandLine.h"
22#include "llvm/Support/FileUtilities.h"
23#include "llvm/Support/Format.h"
24#include "llvm/Support/JSON.h"
25#include "llvm/Support/LLVMDriver.h"
26#include "llvm/Support/MemoryBuffer.h"
27#include "llvm/Support/Program.h"
28#include "llvm/Support/Signals.h"
29#include "llvm/Support/TargetSelect.h"
30#include "llvm/Support/ThreadPool.h"
31#include "llvm/Support/Threading.h"
32#include "llvm/Support/Timer.h"
33#include "llvm/Support/VirtualFileSystem.h"
34#include "llvm/TargetParser/Host.h"
35#include <memory>
36#include <mutex>
37#include <optional>
38#include <thread>
39
40#include "Opts.inc"
41
42using namespace clang;
43using namespace tooling::dependencies;
44
45namespace {
46
47using namespace llvm::opt;
48enum ID {
49 OPT_INVALID = 0, // This is not an option ID.
50#define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),
51#include "Opts.inc"
52#undef OPTION
53};
54
55#define OPTTABLE_STR_TABLE_CODE
56#include "Opts.inc"
57#undef OPTTABLE_STR_TABLE_CODE
58
59#define OPTTABLE_PREFIXES_TABLE_CODE
60#include "Opts.inc"
61#undef OPTTABLE_PREFIXES_TABLE_CODE
62
63const llvm::opt::OptTable::Info InfoTable[] = {
64#define OPTION(...) LLVM_CONSTRUCT_OPT_INFO(__VA_ARGS__),
65#include "Opts.inc"
66#undef OPTION
67};
68
69class ScanDepsOptTable : public llvm::opt::GenericOptTable {
70public:
71 ScanDepsOptTable()
72 : GenericOptTable(OptionStrTable, OptionPrefixesTable, InfoTable) {
73 setGroupedShortOptions(true);
74 }
75};
76
77enum ResourceDirRecipeKind {
78 RDRK_ModifyCompilerPath,
79 RDRK_InvokeCompiler,
80};
81
82static std::string OutputFileName = "-";
83static ScanningMode ScanMode = ScanningMode::DependencyDirectivesScan;
84static ScanningOutputFormat Format = ScanningOutputFormat::Make;
85static ScanningOptimizations OptimizeArgs;
86static std::string ModuleFilesDir;
87static bool EagerLoadModules;
88static unsigned NumThreads = 0;
89static std::string CompilationDB;
90static std::optional<std::string> ModuleName;
91static std::vector<std::string> ModuleDepTargets;
92static std::string TranslationUnitFile;
93static bool DeprecatedDriverCommand;
94static ResourceDirRecipeKind ResourceDirRecipe;
95static bool Verbose;
96static bool PrintTiming;
97static llvm::BumpPtrAllocator Alloc;
98static llvm::StringSaver Saver{Alloc};
99static std::vector<const char *> CommandLine;
100
101#ifndef NDEBUG
102static constexpr bool DoRoundTripDefault = true;
103#else
104static constexpr bool DoRoundTripDefault = false;
105#endif
106
107static bool RoundTripArgs = DoRoundTripDefault;
108
109static void ParseArgs(int argc, char **argv) {
110 ScanDepsOptTable Tbl;
111 llvm::StringRef ToolName = argv[0];
112 llvm::opt::InputArgList Args =
113 Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {
114 llvm::errs() << Msg << '\n';
115 std::exit(1);
116 });
117
118 if (Args.hasArg(OPT_help)) {
119 Tbl.printHelp(OS&: llvm::outs(), Usage: "clang-scan-deps [options]", Title: "clang-scan-deps");
120 std::exit(status: 0);
121 }
122 if (Args.hasArg(OPT_version)) {
123 llvm::outs() << ToolName << '\n';
124 llvm::cl::PrintVersionMessage();
125 std::exit(status: 0);
126 }
127 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_mode_EQ)) {
128 auto ModeType =
129 llvm::StringSwitch<std::optional<ScanningMode>>(A->getValue())
130 .Case("preprocess-dependency-directives",
131 ScanningMode::DependencyDirectivesScan)
132 .Case("preprocess", ScanningMode::CanonicalPreprocessing)
133 .Default(std::nullopt);
134 if (!ModeType) {
135 llvm::errs() << ToolName
136 << ": for the --mode option: Cannot find option named '"
137 << A->getValue() << "'\n";
138 std::exit(status: 1);
139 }
140 ScanMode = *ModeType;
141 }
142
143 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_format_EQ)) {
144 auto FormatType =
145 llvm::StringSwitch<std::optional<ScanningOutputFormat>>(A->getValue())
146 .Case("make", ScanningOutputFormat::Make)
147 .Case("p1689", ScanningOutputFormat::P1689)
148 .Case("experimental-full", ScanningOutputFormat::Full)
149 .Default(std::nullopt);
150 if (!FormatType) {
151 llvm::errs() << ToolName
152 << ": for the --format option: Cannot find option named '"
153 << A->getValue() << "'\n";
154 std::exit(status: 1);
155 }
156 Format = *FormatType;
157 }
158
159 std::vector<std::string> OptimizationFlags =
160 Args.getAllArgValues(OPT_optimize_args_EQ);
161 OptimizeArgs = ScanningOptimizations::None;
162 for (const auto &Arg : OptimizationFlags) {
163 auto Optimization =
164 llvm::StringSwitch<std::optional<ScanningOptimizations>>(Arg)
165 .Case("none", ScanningOptimizations::None)
166 .Case("header-search", ScanningOptimizations::HeaderSearch)
167 .Case("system-warnings", ScanningOptimizations::SystemWarnings)
168 .Case("vfs", ScanningOptimizations::VFS)
169 .Case("canonicalize-macros", ScanningOptimizations::Macros)
170 .Case("ignore-current-working-dir",
171 ScanningOptimizations::IgnoreCWD)
172 .Case("all", ScanningOptimizations::All)
173 .Default(std::nullopt);
174 if (!Optimization) {
175 llvm::errs()
176 << ToolName
177 << ": for the --optimize-args option: Cannot find option named '"
178 << Arg << "'\n";
179 std::exit(1);
180 }
181 OptimizeArgs |= *Optimization;
182 }
183 if (OptimizationFlags.empty())
184 OptimizeArgs = ScanningOptimizations::Default;
185
186 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_files_dir_EQ))
187 ModuleFilesDir = A->getValue();
188
189 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_o))
190 OutputFileName = A->getValue();
191
192 EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
193
194 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
195 StringRef S{A->getValue()};
196 if (!llvm::to_integer(S, NumThreads, 0)) {
197 llvm::errs() << ToolName << ": for the -j option: '" << S
198 << "' value invalid for uint argument!\n";
199 std::exit(status: 1);
200 }
201 }
202
203 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_compilation_database_EQ))
204 CompilationDB = A->getValue();
205
206 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_name_EQ))
207 ModuleName = A->getValue();
208
209 for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ))
210 ModuleDepTargets.emplace_back(A->getValue());
211
212 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_tu_buffer_path_EQ))
213 TranslationUnitFile = A->getValue();
214
215 DeprecatedDriverCommand = Args.hasArg(OPT_deprecated_driver_command);
216
217 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_resource_dir_recipe_EQ)) {
218 auto Kind =
219 llvm::StringSwitch<std::optional<ResourceDirRecipeKind>>(A->getValue())
220 .Case("modify-compiler-path", RDRK_ModifyCompilerPath)
221 .Case("invoke-compiler", RDRK_InvokeCompiler)
222 .Default(std::nullopt);
223 if (!Kind) {
224 llvm::errs() << ToolName
225 << ": for the --resource-dir-recipe option: Cannot find "
226 "option named '"
227 << A->getValue() << "'\n";
228 std::exit(status: 1);
229 }
230 ResourceDirRecipe = *Kind;
231 }
232
233 PrintTiming = Args.hasArg(OPT_print_timing);
234
235 Verbose = Args.hasArg(OPT_verbose);
236
237 RoundTripArgs = Args.hasArg(OPT_round_trip_args);
238
239 if (const llvm::opt::Arg *A = Args.getLastArgNoClaim(OPT_DASH_DASH))
240 CommandLine.assign(A->getValues().begin(), A->getValues().end());
241}
242
243class SharedStream {
244public:
245 SharedStream(raw_ostream &OS) : OS(OS) {}
246 void applyLocked(llvm::function_ref<void(raw_ostream &OS)> Fn) {
247 std::unique_lock<std::mutex> LockGuard(Lock);
248 Fn(OS);
249 OS.flush();
250 }
251
252private:
253 std::mutex Lock;
254 raw_ostream &OS;
255};
256
257class ResourceDirectoryCache {
258public:
259 /// findResourceDir finds the resource directory relative to the clang
260 /// compiler being used in Args, by running it with "-print-resource-dir"
261 /// option and cache the results for reuse. \returns resource directory path
262 /// associated with the given invocation command or empty string if the
263 /// compiler path is NOT an absolute path.
264 StringRef findResourceDir(const tooling::CommandLineArguments &Args,
265 bool ClangCLMode) {
266 if (Args.size() < 1)
267 return "";
268
269 const std::string &ClangBinaryPath = Args[0];
270 if (!llvm::sys::path::is_absolute(path: ClangBinaryPath))
271 return "";
272
273 const std::string &ClangBinaryName =
274 std::string(llvm::sys::path::filename(path: ClangBinaryPath));
275
276 std::unique_lock<std::mutex> LockGuard(CacheLock);
277 const auto &CachedResourceDir = Cache.find(ClangBinaryPath);
278 if (CachedResourceDir != Cache.end())
279 return CachedResourceDir->second;
280
281 std::vector<StringRef> PrintResourceDirArgs{ClangBinaryName};
282 if (ClangCLMode)
283 PrintResourceDirArgs.push_back("/clang:-print-resource-dir");
284 else
285 PrintResourceDirArgs.push_back("-print-resource-dir");
286
287 llvm::SmallString<64> OutputFile, ErrorFile;
288 llvm::sys::fs::createTemporaryFile("print-resource-dir-output",
289 "" /*no-suffix*/, OutputFile);
290 llvm::sys::fs::createTemporaryFile("print-resource-dir-error",
291 "" /*no-suffix*/, ErrorFile);
292 llvm::FileRemover OutputRemover(OutputFile.c_str());
293 llvm::FileRemover ErrorRemover(ErrorFile.c_str());
294 std::optional<StringRef> Redirects[] = {
295 {""}, // Stdin
296 OutputFile.str(),
297 ErrorFile.str(),
298 };
299 if (llvm::sys::ExecuteAndWait(Program: ClangBinaryPath, Args: PrintResourceDirArgs, Env: {},
300 Redirects)) {
301 auto ErrorBuf =
302 llvm::MemoryBuffer::getFile(Filename: ErrorFile.c_str(), /*IsText=*/true);
303 llvm::errs() << ErrorBuf.get()->getBuffer();
304 return "";
305 }
306
307 auto OutputBuf =
308 llvm::MemoryBuffer::getFile(Filename: OutputFile.c_str(), /*IsText=*/true);
309 if (!OutputBuf)
310 return "";
311 StringRef Output = OutputBuf.get()->getBuffer().rtrim('\n');
312
313 return Cache[ClangBinaryPath] = Output.str();
314 }
315
316private:
317 std::map<std::string, std::string> Cache;
318 std::mutex CacheLock;
319};
320
321} // end anonymous namespace
322
323/// Takes the result of a dependency scan and prints error / dependency files
324/// based on the result.
325///
326/// \returns True on error.
327static bool
328handleMakeDependencyToolResult(const std::string &Input,
329 llvm::Expected<std::string> &MaybeFile,
330 SharedStream &OS, SharedStream &Errs) {
331 if (!MaybeFile) {
332 llvm::handleAllErrors(
333 E: MaybeFile.takeError(), Handlers: [&Input, &Errs](llvm::StringError &Err) {
334 Errs.applyLocked(Fn: [&](raw_ostream &OS) {
335 OS << "Error while scanning dependencies for " << Input << ":\n";
336 OS << Err.getMessage();
337 });
338 });
339 return true;
340 }
341 OS.applyLocked(Fn: [&](raw_ostream &OS) { OS << *MaybeFile; });
342 return false;
343}
344
345template <typename Container>
346static auto toJSONStrings(llvm::json::OStream &JOS, Container &&Strings) {
347 return [&JOS, Strings = std::forward<Container>(Strings)] {
348 for (StringRef Str : Strings)
349 // Not reporting SDKSettings.json so that test checks can remain (mostly)
350 // platform-agnostic.
351 if (!Str.ends_with(Suffix: "SDKSettings.json"))
352 JOS.value(V: Str);
353 };
354}
355
356// Technically, we don't need to sort the dependency list to get determinism.
357// Leaving these be will simply preserve the import order.
358static auto toJSONSorted(llvm::json::OStream &JOS, std::vector<ModuleID> V) {
359 llvm::sort(C&: V);
360 return [&JOS, V = std::move(V)] {
361 for (const ModuleID &MID : V)
362 JOS.object(Contents: [&] {
363 JOS.attribute(Key: "context-hash", Contents: StringRef(MID.ContextHash));
364 JOS.attribute(Key: "module-name", Contents: StringRef(MID.ModuleName));
365 });
366 };
367}
368
369static auto toJSONSorted(llvm::json::OStream &JOS,
370 SmallVector<Module::LinkLibrary, 2> LinkLibs) {
371 llvm::sort(C&: LinkLibs, Comp: [](const auto &LHS, const auto &RHS) {
372 return LHS.Library < RHS.Library;
373 });
374 return [&JOS, LinkLibs = std::move(LinkLibs)] {
375 for (const auto &LL : LinkLibs)
376 JOS.object(Contents: [&] {
377 JOS.attribute(Key: "isFramework", Contents: LL.IsFramework);
378 JOS.attribute(Key: "link-name", Contents: StringRef(LL.Library));
379 });
380 };
381}
382
383// Thread safe.
384class FullDeps {
385public:
386 FullDeps(size_t NumInputs) : Inputs(NumInputs) {}
387
388 void mergeDeps(StringRef Input, TranslationUnitDeps TUDeps,
389 size_t InputIndex) {
390 mergeDeps(Graph: std::move(TUDeps.ModuleGraph), InputIndex);
391
392 InputDeps ID;
393 ID.FileName = std::string(Input);
394 ID.ContextHash = std::move(TUDeps.ID.ContextHash);
395 ID.FileDeps = std::move(TUDeps.FileDeps);
396 ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps);
397 ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine);
398 ID.Commands = std::move(TUDeps.Commands);
399
400 assert(InputIndex < Inputs.size() && "Input index out of bounds");
401 assert(Inputs[InputIndex].FileName.empty() && "Result already populated");
402 Inputs[InputIndex] = std::move(ID);
403 }
404
405 void mergeDeps(ModuleDepsGraph Graph, size_t InputIndex) {
406 std::vector<ModuleDeps *> NewMDs;
407 {
408 std::unique_lock<std::mutex> ul(Lock);
409 for (const ModuleDeps &MD : Graph) {
410 auto I = Modules.find(x: {.ID: MD.ID, .InputIndex: 0});
411 if (I != Modules.end()) {
412 I->first.InputIndex = std::min(a: I->first.InputIndex, b: InputIndex);
413 continue;
414 }
415 auto Res = Modules.insert(hint: I, x: {{.ID: MD.ID, .InputIndex: InputIndex}, std::move(MD)});
416 NewMDs.push_back(x: &Res->second);
417 }
418 // First call to \c getBuildArguments is somewhat expensive. Let's call it
419 // on the current thread (instead of the main one), and outside the
420 // critical section.
421 for (ModuleDeps *MD : NewMDs)
422 (void)MD->getBuildArguments();
423 }
424 }
425
426 bool roundTripCommand(ArrayRef<std::string> ArgStrs,
427 DiagnosticsEngine &Diags) {
428 if (ArgStrs.empty() || ArgStrs[0] != "-cc1")
429 return false;
430 SmallVector<const char *> Args;
431 for (const std::string &Arg : ArgStrs)
432 Args.push_back(Elt: Arg.c_str());
433 return !CompilerInvocation::checkCC1RoundTrip(Args, Diags);
434 }
435
436 // Returns \c true if any command lines fail to round-trip. We expect
437 // commands already be canonical when output by the scanner.
438 bool roundTripCommands(raw_ostream &ErrOS) {
439 DiagnosticOptions DiagOpts;
440 TextDiagnosticPrinter DiagConsumer(ErrOS, DiagOpts);
441 IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
442 CompilerInstance::createDiagnostics(VFS&: *llvm::vfs::getRealFileSystem(),
443 Opts&: DiagOpts, Client: &DiagConsumer,
444 /*ShouldOwnClient=*/false);
445
446 for (auto &&M : Modules)
447 if (roundTripCommand(ArgStrs: M.second.getBuildArguments(), Diags&: *Diags))
448 return true;
449
450 for (auto &&I : Inputs)
451 for (const auto &Cmd : I.Commands)
452 if (roundTripCommand(ArgStrs: Cmd.Arguments, Diags&: *Diags))
453 return true;
454
455 return false;
456 }
457
458 void printFullOutput(raw_ostream &OS) {
459 // Skip sorting modules and constructing the JSON object if the output
460 // cannot be observed anyway. This makes timings less noisy.
461 if (&OS == &llvm::nulls())
462 return;
463
464 // Sort the modules by name to get a deterministic order.
465 std::vector<IndexedModuleID> ModuleIDs;
466 for (auto &&M : Modules)
467 ModuleIDs.push_back(x: M.first);
468 llvm::sort(C&: ModuleIDs);
469
470 llvm::json::OStream JOS(OS, /*IndentSize=*/2);
471
472 JOS.object(Contents: [&] {
473 JOS.attributeArray(Key: "modules", Contents: [&] {
474 for (auto &&ModID : ModuleIDs) {
475 auto &MD = Modules[ModID];
476 JOS.object(Contents: [&] {
477 if (MD.IsInStableDirectories)
478 JOS.attribute(Key: "is-in-stable-directories",
479 Contents: MD.IsInStableDirectories);
480 JOS.attributeArray(Key: "clang-module-deps",
481 Contents: toJSONSorted(JOS, V: MD.ClangModuleDeps));
482 JOS.attribute(Key: "clang-modulemap-file",
483 Contents: StringRef(MD.ClangModuleMapFile));
484 JOS.attributeArray(Key: "command-line",
485 Contents: toJSONStrings(JOS, Strings: MD.getBuildArguments()));
486 JOS.attribute(Key: "context-hash", Contents: StringRef(MD.ID.ContextHash));
487 JOS.attributeArray(Key: "file-deps", Contents: [&] {
488 MD.forEachFileDep(Cb: [&](StringRef FileDep) {
489 // Not reporting SDKSettings.json so that test checks can remain
490 // (mostly) platform-agnostic.
491 if (!FileDep.ends_with(Suffix: "SDKSettings.json"))
492 JOS.value(V: FileDep);
493 });
494 });
495 JOS.attributeArray(Key: "link-libraries",
496 Contents: toJSONSorted(JOS, LinkLibs: MD.LinkLibraries));
497 JOS.attribute(Key: "name", Contents: StringRef(MD.ID.ModuleName));
498 });
499 }
500 });
501
502 JOS.attributeArray(Key: "translation-units", Contents: [&] {
503 for (auto &&I : Inputs) {
504 JOS.object(Contents: [&] {
505 JOS.attributeArray(Key: "commands", Contents: [&] {
506 if (I.DriverCommandLine.empty()) {
507 for (const auto &Cmd : I.Commands) {
508 JOS.object(Contents: [&] {
509 JOS.attribute(Key: "clang-context-hash",
510 Contents: StringRef(I.ContextHash));
511 JOS.attributeArray(Key: "clang-module-deps",
512 Contents: toJSONSorted(JOS, V: I.ModuleDeps));
513 JOS.attributeArray(Key: "command-line",
514 Contents: toJSONStrings(JOS, Strings: Cmd.Arguments));
515 JOS.attribute(Key: "executable", Contents: StringRef(Cmd.Executable));
516 JOS.attributeArray(Key: "file-deps",
517 Contents: toJSONStrings(JOS, Strings&: I.FileDeps));
518 JOS.attribute(Key: "input-file", Contents: StringRef(I.FileName));
519 });
520 }
521 } else {
522 JOS.object(Contents: [&] {
523 JOS.attribute(Key: "clang-context-hash", Contents: StringRef(I.ContextHash));
524 JOS.attributeArray(Key: "clang-module-deps",
525 Contents: toJSONSorted(JOS, V: I.ModuleDeps));
526 JOS.attributeArray(Key: "command-line",
527 Contents: toJSONStrings(JOS, Strings&: I.DriverCommandLine));
528 JOS.attribute(Key: "executable", Contents: "clang");
529 JOS.attributeArray(Key: "file-deps",
530 Contents: toJSONStrings(JOS, Strings&: I.FileDeps));
531 JOS.attribute(Key: "input-file", Contents: StringRef(I.FileName));
532 });
533 }
534 });
535 });
536 }
537 });
538 });
539 }
540
541private:
542 struct IndexedModuleID {
543 ModuleID ID;
544
545 // FIXME: This is mutable so that it can still be updated after insertion
546 // into an unordered associative container. This is "fine", since this
547 // field doesn't contribute to the hash, but it's a brittle hack.
548 mutable size_t InputIndex;
549
550 bool operator==(const IndexedModuleID &Other) const {
551 return ID == Other.ID;
552 }
553
554 bool operator<(const IndexedModuleID &Other) const {
555 /// We need the output of clang-scan-deps to be deterministic. However,
556 /// the dependency graph may contain two modules with the same name. How
557 /// do we decide which one to print first? If we made that decision based
558 /// on the context hash, the ordering would be deterministic, but
559 /// different across machines. This can happen for example when the inputs
560 /// or the SDKs (which both contribute to the "context" hash) live in
561 /// different absolute locations. We solve that by tracking the index of
562 /// the first input TU that (transitively) imports the dependency, which
563 /// is always the same for the same input, resulting in deterministic
564 /// sorting that's also reproducible across machines.
565 return std::tie(args: ID.ModuleName, args&: InputIndex) <
566 std::tie(args: Other.ID.ModuleName, args&: Other.InputIndex);
567 }
568
569 struct Hasher {
570 std::size_t operator()(const IndexedModuleID &IMID) const {
571 return llvm::hash_value(ID: IMID.ID);
572 }
573 };
574 };
575
576 struct InputDeps {
577 std::string FileName;
578 std::string ContextHash;
579 std::vector<std::string> FileDeps;
580 std::vector<ModuleID> ModuleDeps;
581 std::vector<std::string> DriverCommandLine;
582 std::vector<Command> Commands;
583 };
584
585 std::mutex Lock;
586 std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleID::Hasher>
587 Modules;
588 std::vector<InputDeps> Inputs;
589};
590
591static bool handleTranslationUnitResult(
592 StringRef Input, llvm::Expected<TranslationUnitDeps> &MaybeTUDeps,
593 FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
594 if (!MaybeTUDeps) {
595 llvm::handleAllErrors(
596 E: MaybeTUDeps.takeError(), Handlers: [&Input, &Errs](llvm::StringError &Err) {
597 Errs.applyLocked(Fn: [&](raw_ostream &OS) {
598 OS << "Error while scanning dependencies for " << Input << ":\n";
599 OS << Err.getMessage();
600 });
601 });
602 return true;
603 }
604 FD.mergeDeps(Input, TUDeps: std::move(*MaybeTUDeps), InputIndex);
605 return false;
606}
607
608static bool handleModuleResult(
609 StringRef ModuleName, llvm::Expected<ModuleDepsGraph> &MaybeModuleGraph,
610 FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
611 if (!MaybeModuleGraph) {
612 llvm::handleAllErrors(E: MaybeModuleGraph.takeError(),
613 Handlers: [&ModuleName, &Errs](llvm::StringError &Err) {
614 Errs.applyLocked(Fn: [&](raw_ostream &OS) {
615 OS << "Error while scanning dependencies for "
616 << ModuleName << ":\n";
617 OS << Err.getMessage();
618 });
619 });
620 return true;
621 }
622 FD.mergeDeps(Graph: std::move(*MaybeModuleGraph), InputIndex);
623 return false;
624}
625
626class P1689Deps {
627public:
628 void printDependencies(raw_ostream &OS) {
629 addSourcePathsToRequires();
630 // Sort the modules by name to get a deterministic order.
631 llvm::sort(C&: Rules, Comp: [](const P1689Rule &A, const P1689Rule &B) {
632 return A.PrimaryOutput < B.PrimaryOutput;
633 });
634
635 using namespace llvm::json;
636 Array OutputRules;
637 for (const P1689Rule &R : Rules) {
638 Object O{{.K: "primary-output", .V: R.PrimaryOutput}};
639
640 if (R.Provides) {
641 Array Provides;
642 Object Provided{{.K: "logical-name", .V: R.Provides->ModuleName},
643 {.K: "source-path", .V: R.Provides->SourcePath},
644 {.K: "is-interface", .V: R.Provides->IsStdCXXModuleInterface}};
645 Provides.push_back(E: std::move(Provided));
646 O.insert(E: {.K: "provides", .V: std::move(Provides)});
647 }
648
649 Array Requires;
650 for (const P1689ModuleInfo &Info : R.Requires) {
651 Object RequiredInfo{{.K: "logical-name", .V: Info.ModuleName}};
652 if (!Info.SourcePath.empty())
653 RequiredInfo.insert(E: {.K: "source-path", .V: Info.SourcePath});
654 Requires.push_back(E: std::move(RequiredInfo));
655 }
656
657 if (!Requires.empty())
658 O.insert(E: {.K: "requires", .V: std::move(Requires)});
659
660 OutputRules.push_back(E: std::move(O));
661 }
662
663 Object Output{
664 {.K: "version", .V: 1}, {.K: "revision", .V: 0}, {.K: "rules", .V: std::move(OutputRules)}};
665
666 OS << llvm::formatv(Fmt: "{0:2}\n", Vals: Value(std::move(Output)));
667 }
668
669 void addRules(P1689Rule &Rule) {
670 std::unique_lock<std::mutex> LockGuard(Lock);
671 Rules.push_back(x: Rule);
672 }
673
674private:
675 void addSourcePathsToRequires() {
676 llvm::DenseMap<StringRef, StringRef> ModuleSourceMapper;
677 for (const P1689Rule &R : Rules)
678 if (R.Provides && !R.Provides->SourcePath.empty())
679 ModuleSourceMapper[R.Provides->ModuleName] = R.Provides->SourcePath;
680
681 for (P1689Rule &R : Rules) {
682 for (P1689ModuleInfo &Info : R.Requires) {
683 auto Iter = ModuleSourceMapper.find(Val: Info.ModuleName);
684 if (Iter != ModuleSourceMapper.end())
685 Info.SourcePath = Iter->second;
686 }
687 }
688 }
689
690 std::mutex Lock;
691 std::vector<P1689Rule> Rules;
692};
693
694static bool
695handleP1689DependencyToolResult(const std::string &Input,
696 llvm::Expected<P1689Rule> &MaybeRule,
697 P1689Deps &PD, SharedStream &Errs) {
698 if (!MaybeRule) {
699 llvm::handleAllErrors(
700 E: MaybeRule.takeError(), Handlers: [&Input, &Errs](llvm::StringError &Err) {
701 Errs.applyLocked(Fn: [&](raw_ostream &OS) {
702 OS << "Error while scanning dependencies for " << Input << ":\n";
703 OS << Err.getMessage();
704 });
705 });
706 return true;
707 }
708 PD.addRules(Rule&: *MaybeRule);
709 return false;
710}
711
712/// Construct a path for the explicitly built PCM.
713static std::string constructPCMPath(ModuleID MID, StringRef OutputDir) {
714 SmallString<256> ExplicitPCMPath(OutputDir);
715 llvm::sys::path::append(path&: ExplicitPCMPath, a: MID.ContextHash,
716 b: MID.ModuleName + "-" + MID.ContextHash + ".pcm");
717 return std::string(ExplicitPCMPath);
718}
719
720static std::string lookupModuleOutput(const ModuleDeps &MD,
721 ModuleOutputKind MOK,
722 StringRef OutputDir) {
723 std::string PCMPath = constructPCMPath(MID: MD.ID, OutputDir);
724 switch (MOK) {
725 case ModuleOutputKind::ModuleFile:
726 return PCMPath;
727 case ModuleOutputKind::DependencyFile:
728 return PCMPath + ".d";
729 case ModuleOutputKind::DependencyTargets:
730 // Null-separate the list of targets.
731 return join(ModuleDepTargets, StringRef("\0", 1));
732 case ModuleOutputKind::DiagnosticSerializationFile:
733 return PCMPath + ".diag";
734 }
735 llvm_unreachable("Fully covered switch above!");
736}
737
738static std::string getModuleCachePath(ArrayRef<std::string> Args) {
739 for (StringRef Arg : llvm::reverse(C&: Args)) {
740 Arg.consume_front(Prefix: "/clang:");
741 if (Arg.consume_front(Prefix: "-fmodules-cache-path="))
742 return std::string(Arg);
743 }
744 SmallString<128> Path;
745 driver::Driver::getDefaultModuleCachePath(Result&: Path);
746 return std::string(Path);
747}
748
749/// Attempts to construct the compilation database from '-compilation-database'
750/// or from the arguments following the positional '--'.
751static std::unique_ptr<tooling::CompilationDatabase>
752getCompilationDatabase(int argc, char **argv, std::string &ErrorMessage) {
753 ParseArgs(argc, argv);
754
755 if (!(CommandLine.empty() ^ CompilationDB.empty())) {
756 llvm::errs() << "The compilation command line must be provided either via "
757 "'-compilation-database' or after '--'.";
758 return nullptr;
759 }
760
761 if (!CompilationDB.empty())
762 return tooling::JSONCompilationDatabase::loadFromFile(
763 FilePath: CompilationDB, ErrorMessage,
764 Syntax: tooling::JSONCommandLineSyntax::AutoDetect);
765
766 DiagnosticOptions DiagOpts;
767 llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
768 CompilerInstance::createDiagnostics(VFS&: *llvm::vfs::getRealFileSystem(),
769 Opts&: DiagOpts);
770 driver::Driver TheDriver(CommandLine[0], llvm::sys::getDefaultTargetTriple(),
771 *Diags);
772 TheDriver.setCheckInputsExist(false);
773 std::unique_ptr<driver::Compilation> C(
774 TheDriver.BuildCompilation(Args: CommandLine));
775 if (!C || C->getJobs().empty())
776 return nullptr;
777
778 auto Cmd = C->getJobs().begin();
779 auto CI = std::make_unique<CompilerInvocation>();
780 CompilerInvocation::CreateFromArgs(Res&: *CI, CommandLineArgs: Cmd->getArguments(), Diags&: *Diags,
781 Argv0: CommandLine[0]);
782 if (!CI)
783 return nullptr;
784
785 FrontendOptions &FEOpts = CI->getFrontendOpts();
786 if (FEOpts.Inputs.size() != 1) {
787 llvm::errs()
788 << "Exactly one input file is required in the per-file mode ('--').\n";
789 return nullptr;
790 }
791
792 // There might be multiple jobs for a compilation. Extract the specified
793 // output filename from the last job.
794 auto LastCmd = C->getJobs().end();
795 LastCmd--;
796 if (LastCmd->getOutputFilenames().size() != 1) {
797 llvm::errs()
798 << "Exactly one output file is required in the per-file mode ('--').\n";
799 return nullptr;
800 }
801 StringRef OutputFile = LastCmd->getOutputFilenames().front();
802
803 class InplaceCompilationDatabase : public tooling::CompilationDatabase {
804 public:
805 InplaceCompilationDatabase(StringRef InputFile, StringRef OutputFile,
806 ArrayRef<const char *> CommandLine)
807 : Command(".", InputFile, {}, OutputFile) {
808 for (auto *C : CommandLine)
809 Command.CommandLine.push_back(x: C);
810 }
811
812 std::vector<tooling::CompileCommand>
813 getCompileCommands(StringRef FilePath) const override {
814 if (FilePath != Command.Filename)
815 return {};
816 return {Command};
817 }
818
819 std::vector<std::string> getAllFiles() const override {
820 return {Command.Filename};
821 }
822
823 std::vector<tooling::CompileCommand>
824 getAllCompileCommands() const override {
825 return {Command};
826 }
827
828 private:
829 tooling::CompileCommand Command;
830 };
831
832 return std::make_unique<InplaceCompilationDatabase>(
833 FEOpts.Inputs[0].getFile(), OutputFile, CommandLine);
834}
835
836int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
837 llvm::InitializeAllTargetInfos();
838 std::string ErrorMessage;
839 std::unique_ptr<tooling::CompilationDatabase> Compilations =
840 getCompilationDatabase(argc, argv, ErrorMessage);
841 if (!Compilations) {
842 llvm::errs() << ErrorMessage << "\n";
843 return 1;
844 }
845
846 llvm::cl::PrintOptionValues();
847
848 // Expand response files in advance, so that we can "see" all the arguments
849 // when adjusting below.
850 Compilations = expandResponseFiles(Base: std::move(Compilations),
851 FS: llvm::vfs::getRealFileSystem());
852
853 Compilations = inferTargetAndDriverMode(Base: std::move(Compilations));
854
855 Compilations = inferToolLocation(Base: std::move(Compilations));
856
857 // The command options are rewritten to run Clang in preprocessor only mode.
858 auto AdjustingCompilations =
859 std::make_unique<tooling::ArgumentsAdjustingCompilations>(
860 args: std::move(Compilations));
861 ResourceDirectoryCache ResourceDirCache;
862
863 AdjustingCompilations->appendArgumentsAdjuster(
864 Adjuster: [&ResourceDirCache](const tooling::CommandLineArguments &Args,
865 StringRef FileName) {
866 std::string LastO;
867 bool HasResourceDir = false;
868 bool ClangCLMode = false;
869 auto FlagsEnd = llvm::find(Range: Args, Val: "--");
870 if (FlagsEnd != Args.begin()) {
871 ClangCLMode =
872 llvm::sys::path::stem(path: Args[0]).contains_insensitive(Other: "clang-cl") ||
873 llvm::is_contained(Range: Args, Element: "--driver-mode=cl");
874
875 // Reverse scan, starting at the end or at the element before "--".
876 auto R = std::make_reverse_iterator(i: FlagsEnd);
877 auto E = Args.rend();
878 // Don't include Args[0] in the iteration; that's the executable, not
879 // an option.
880 if (E != R)
881 E--;
882 for (auto I = R; I != E; ++I) {
883 StringRef Arg = *I;
884 if (ClangCLMode) {
885 // Ignore arguments that are preceded by "-Xclang".
886 if ((I + 1) != E && I[1] == "-Xclang")
887 continue;
888 if (LastO.empty()) {
889 // With clang-cl, the output obj file can be specified with
890 // "/opath", "/o path", "/Fopath", and the dash counterparts.
891 // Also, clang-cl adds ".obj" extension if none is found.
892 if ((Arg == "-o" || Arg == "/o") && I != R)
893 LastO = I[-1]; // Next argument (reverse iterator)
894 else if (Arg.starts_with(Prefix: "/Fo") || Arg.starts_with(Prefix: "-Fo"))
895 LastO = Arg.drop_front(N: 3).str();
896 else if (Arg.starts_with(Prefix: "/o") || Arg.starts_with(Prefix: "-o"))
897 LastO = Arg.drop_front(N: 2).str();
898
899 if (!LastO.empty() && !llvm::sys::path::has_extension(path: LastO))
900 LastO.append(s: ".obj");
901 }
902 }
903 if (Arg == "-resource-dir")
904 HasResourceDir = true;
905 }
906 }
907 tooling::CommandLineArguments AdjustedArgs(Args.begin(), FlagsEnd);
908 // The clang-cl driver passes "-o -" to the frontend. Inject the real
909 // file here to ensure "-MT" can be deduced if need be.
910 if (ClangCLMode && !LastO.empty()) {
911 AdjustedArgs.push_back(x: "/clang:-o");
912 AdjustedArgs.push_back(x: "/clang:" + LastO);
913 }
914
915 if (!HasResourceDir && ResourceDirRecipe == RDRK_InvokeCompiler) {
916 StringRef ResourceDir =
917 ResourceDirCache.findResourceDir(Args, ClangCLMode);
918 if (!ResourceDir.empty()) {
919 AdjustedArgs.push_back(x: "-resource-dir");
920 AdjustedArgs.push_back(x: std::string(ResourceDir));
921 }
922 }
923 AdjustedArgs.insert(position: AdjustedArgs.end(), first: FlagsEnd, last: Args.end());
924 return AdjustedArgs;
925 });
926
927 SharedStream Errs(llvm::errs());
928
929 std::optional<llvm::raw_fd_ostream> FileOS;
930 llvm::raw_ostream &ThreadUnsafeDependencyOS = [&]() -> llvm::raw_ostream & {
931 if (OutputFileName == "-")
932 return llvm::outs();
933
934 if (OutputFileName == "/dev/null")
935 return llvm::nulls();
936
937 std::error_code EC;
938 FileOS.emplace(args&: OutputFileName, args&: EC, args: llvm::sys::fs::OF_Text);
939 if (EC) {
940 llvm::errs() << "Failed to open output file '" << OutputFileName
941 << "': " << EC.message() << '\n';
942 std::exit(status: 1);
943 }
944 return *FileOS;
945 }();
946 SharedStream DependencyOS(ThreadUnsafeDependencyOS);
947
948 std::vector<tooling::CompileCommand> Inputs =
949 AdjustingCompilations->getAllCompileCommands();
950
951 std::atomic<bool> HadErrors(false);
952 std::optional<FullDeps> FD;
953 P1689Deps PD;
954
955 std::mutex Lock;
956 size_t Index = 0;
957 auto GetNextInputIndex = [&]() -> std::optional<size_t> {
958 std::unique_lock<std::mutex> LockGuard(Lock);
959 if (Index < Inputs.size())
960 return Index++;
961 return {};
962 };
963
964 if (Format == ScanningOutputFormat::Full)
965 FD.emplace(args: !ModuleName ? Inputs.size() : 0);
966
967 std::atomic<size_t> NumStatusCalls = 0;
968 std::atomic<size_t> NumOpenFileForReadCalls = 0;
969 std::atomic<size_t> NumDirBeginCalls = 0;
970 std::atomic<size_t> NumGetRealPathCalls = 0;
971 std::atomic<size_t> NumExistsCalls = 0;
972 std::atomic<size_t> NumIsLocalCalls = 0;
973
974 auto ScanningTask = [&](DependencyScanningService &Service) {
975 DependencyScanningTool WorkerTool(Service);
976
977 llvm::DenseSet<ModuleID> AlreadySeenModules;
978 while (auto MaybeInputIndex = GetNextInputIndex()) {
979 size_t LocalIndex = *MaybeInputIndex;
980 const tooling::CompileCommand *Input = &Inputs[LocalIndex];
981 std::string Filename = std::move(Input->Filename);
982 std::string CWD = std::move(Input->Directory);
983
984 std::string OutputDir(ModuleFilesDir);
985 if (OutputDir.empty())
986 OutputDir = getModuleCachePath(Args: Input->CommandLine);
987 auto LookupOutput = [&](const ModuleDeps &MD, ModuleOutputKind MOK) {
988 return ::lookupModuleOutput(MD, MOK, OutputDir);
989 };
990
991 // Run the tool on it.
992 if (Format == ScanningOutputFormat::Make) {
993 auto MaybeFile = WorkerTool.getDependencyFile(CommandLine: Input->CommandLine, CWD);
994 if (handleMakeDependencyToolResult(Input: Filename, MaybeFile, OS&: DependencyOS,
995 Errs))
996 HadErrors = true;
997 } else if (Format == ScanningOutputFormat::P1689) {
998 // It is useful to generate the make-format dependency output during
999 // the scanning for P1689. Otherwise the users need to scan again for
1000 // it. We will generate the make-format dependency output if we find
1001 // `-MF` in the command lines.
1002 std::string MakeformatOutputPath;
1003 std::string MakeformatOutput;
1004
1005 auto MaybeRule = WorkerTool.getP1689ModuleDependencyFile(
1006 Command: *Input, CWD, MakeformatOutput, MakeformatOutputPath);
1007
1008 if (handleP1689DependencyToolResult(Input: Filename, MaybeRule, PD, Errs))
1009 HadErrors = true;
1010
1011 if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
1012 !HadErrors) {
1013 static std::mutex Lock;
1014 // With compilation database, we may open different files
1015 // concurrently or we may write the same file concurrently. So we
1016 // use a map here to allow multiple compile commands to write to the
1017 // same file. Also we need a lock here to avoid data race.
1018 static llvm::StringMap<llvm::raw_fd_ostream> OSs;
1019 std::unique_lock<std::mutex> LockGuard(Lock);
1020
1021 auto OSIter = OSs.find(Key: MakeformatOutputPath);
1022 if (OSIter == OSs.end()) {
1023 std::error_code EC;
1024 OSIter = OSs.try_emplace(Key: MakeformatOutputPath, Args&: MakeformatOutputPath,
1025 Args&: EC, Args: llvm::sys::fs::OF_Text)
1026 .first;
1027 if (EC)
1028 llvm::errs() << "Failed to open P1689 make format output file \""
1029 << MakeformatOutputPath << "\" for " << EC.message()
1030 << "\n";
1031 }
1032
1033 SharedStream MakeformatOS(OSIter->second);
1034 llvm::Expected<std::string> MaybeOutput(MakeformatOutput);
1035 if (handleMakeDependencyToolResult(Input: Filename, MaybeFile&: MaybeOutput,
1036 OS&: MakeformatOS, Errs))
1037 HadErrors = true;
1038 }
1039 } else if (ModuleName) {
1040 auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies(
1041 ModuleName: *ModuleName, CommandLine: Input->CommandLine, CWD, AlreadySeen: AlreadySeenModules,
1042 LookupModuleOutput: LookupOutput);
1043 if (handleModuleResult(ModuleName: *ModuleName, MaybeModuleGraph&: MaybeModuleDepsGraph, FD&: *FD,
1044 InputIndex: LocalIndex, OS&: DependencyOS, Errs))
1045 HadErrors = true;
1046 } else {
1047 std::unique_ptr<llvm::MemoryBuffer> TU;
1048 std::optional<llvm::MemoryBufferRef> TUBuffer;
1049 if (!TranslationUnitFile.empty()) {
1050 auto MaybeTU =
1051 llvm::MemoryBuffer::getFile(Filename: TranslationUnitFile, /*IsText=*/true);
1052 if (!MaybeTU) {
1053 llvm::errs() << "cannot open input translation unit: "
1054 << MaybeTU.getError().message() << "\n";
1055 HadErrors = true;
1056 continue;
1057 }
1058 TU = std::move(*MaybeTU);
1059 TUBuffer = TU->getMemBufferRef();
1060 Filename = TU->getBufferIdentifier();
1061 }
1062 auto MaybeTUDeps = WorkerTool.getTranslationUnitDependencies(
1063 CommandLine: Input->CommandLine, CWD, AlreadySeen: AlreadySeenModules, LookupModuleOutput: LookupOutput,
1064 TUBuffer);
1065 if (handleTranslationUnitResult(Input: Filename, MaybeTUDeps, FD&: *FD, InputIndex: LocalIndex,
1066 OS&: DependencyOS, Errs))
1067 HadErrors = true;
1068 }
1069 }
1070
1071 WorkerTool.getWorkerVFS().visit(Callback: [&](llvm::vfs::FileSystem &VFS) {
1072 if (auto *T = dyn_cast_or_null<llvm::vfs::TracingFileSystem>(Val: &VFS)) {
1073 NumStatusCalls += T->NumStatusCalls;
1074 NumOpenFileForReadCalls += T->NumOpenFileForReadCalls;
1075 NumDirBeginCalls += T->NumDirBeginCalls;
1076 NumGetRealPathCalls += T->NumGetRealPathCalls;
1077 NumExistsCalls += T->NumExistsCalls;
1078 NumIsLocalCalls += T->NumIsLocalCalls;
1079 }
1080 });
1081 };
1082
1083 DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
1084 EagerLoadModules, /*TraceVFS=*/Verbose);
1085
1086 llvm::Timer T;
1087 T.startTimer();
1088
1089 if (Inputs.size() == 1) {
1090 ScanningTask(Service);
1091 } else {
1092 llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(ThreadCount: NumThreads));
1093
1094 if (Verbose) {
1095 llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
1096 << " files using " << Pool.getMaxConcurrency()
1097 << " workers\n";
1098 }
1099
1100 for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)
1101 Pool.async(F: [ScanningTask, &Service]() { ScanningTask(Service); });
1102
1103 Pool.wait();
1104 }
1105
1106 T.stopTimer();
1107
1108 if (Verbose)
1109 llvm::errs() << "\n*** Virtual File System Stats:\n"
1110 << NumStatusCalls << " status() calls\n"
1111 << NumOpenFileForReadCalls << " openFileForRead() calls\n"
1112 << NumDirBeginCalls << " dir_begin() calls\n"
1113 << NumGetRealPathCalls << " getRealPath() calls\n"
1114 << NumExistsCalls << " exists() calls\n"
1115 << NumIsLocalCalls << " isLocal() calls\n";
1116
1117 if (PrintTiming) {
1118 llvm::errs() << "wall time [s]\t"
1119 << "process time [s]\t"
1120 << "instruction count\n";
1121 const llvm::TimeRecord &R = T.getTotalTime();
1122 llvm::errs() << llvm::format(Fmt: "%0.4f", Vals: R.getWallTime()) << "\t"
1123 << llvm::format(Fmt: "%0.4f", Vals: R.getProcessTime()) << "\t"
1124 << llvm::format(Fmt: "%llu", Vals: R.getInstructionsExecuted()) << "\n";
1125 }
1126
1127 if (RoundTripArgs)
1128 if (FD && FD->roundTripCommands(ErrOS&: llvm::errs()))
1129 HadErrors = true;
1130
1131 if (Format == ScanningOutputFormat::Full)
1132 FD->printFullOutput(OS&: ThreadUnsafeDependencyOS);
1133 else if (Format == ScanningOutputFormat::P1689)
1134 PD.printDependencies(OS&: ThreadUnsafeDependencyOS);
1135
1136 return HadErrors;
1137}
1138

Provided by KDAB

Privacy Policy
Learn to use CMake with our Intro Training
Find out more

source code of clang/tools/clang-scan-deps/ClangScanDeps.cpp