1//===- ClangScanDeps.cpp - Implementation of clang-scan-deps --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "clang/Driver/Compilation.h"
10#include "clang/Driver/Driver.h"
11#include "clang/Frontend/CompilerInstance.h"
12#include "clang/Frontend/TextDiagnosticPrinter.h"
13#include "clang/Tooling/CommonOptionsParser.h"
14#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
15#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
16#include "clang/Tooling/DependencyScanning/DependencyScanningWorker.h"
17#include "clang/Tooling/JSONCompilationDatabase.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/Twine.h"
20#include "llvm/Support/CommandLine.h"
21#include "llvm/Support/FileUtilities.h"
22#include "llvm/Support/Format.h"
23#include "llvm/Support/JSON.h"
24#include "llvm/Support/LLVMDriver.h"
25#include "llvm/Support/Program.h"
26#include "llvm/Support/Signals.h"
27#include "llvm/Support/ThreadPool.h"
28#include "llvm/Support/Threading.h"
29#include "llvm/Support/Timer.h"
30#include "llvm/TargetParser/Host.h"
31#include <mutex>
32#include <optional>
33#include <thread>
34
35#include "Opts.inc"
36
37using namespace clang;
38using namespace tooling::dependencies;
39
40namespace {
41
42using namespace llvm::opt;
43enum ID {
44 OPT_INVALID = 0, // This is not an option ID.
45#define OPTION(...) LLVM_MAKE_OPT_ID(__VA_ARGS__),
46#include "Opts.inc"
47#undef OPTION
48};
49
50#define PREFIX(NAME, VALUE) \
51 constexpr llvm::StringLiteral NAME##_init[] = VALUE; \
52 constexpr llvm::ArrayRef<llvm::StringLiteral> NAME( \
53 NAME##_init, std::size(NAME##_init) - 1);
54#include "Opts.inc"
55#undef PREFIX
56
57const llvm::opt::OptTable::Info InfoTable[] = {
58#define OPTION(...) LLVM_CONSTRUCT_OPT_INFO(__VA_ARGS__),
59#include "Opts.inc"
60#undef OPTION
61};
62
63class ScanDepsOptTable : public llvm::opt::GenericOptTable {
64public:
65 ScanDepsOptTable() : GenericOptTable(InfoTable) {
66 setGroupedShortOptions(true);
67 }
68};
69
70enum ResourceDirRecipeKind {
71 RDRK_ModifyCompilerPath,
72 RDRK_InvokeCompiler,
73};
74
75static std::string OutputFileName = "-";
76static ScanningMode ScanMode = ScanningMode::DependencyDirectivesScan;
77static ScanningOutputFormat Format = ScanningOutputFormat::Make;
78static ScanningOptimizations OptimizeArgs;
79static std::string ModuleFilesDir;
80static bool EagerLoadModules;
81static unsigned NumThreads = 0;
82static std::string CompilationDB;
83static std::string ModuleName;
84static std::vector<std::string> ModuleDepTargets;
85static bool DeprecatedDriverCommand;
86static ResourceDirRecipeKind ResourceDirRecipe;
87static bool Verbose;
88static bool PrintTiming;
89static std::vector<const char *> CommandLine;
90
91#ifndef NDEBUG
92static constexpr bool DoRoundTripDefault = true;
93#else
94static constexpr bool DoRoundTripDefault = false;
95#endif
96
97static bool RoundTripArgs = DoRoundTripDefault;
98
99static void ParseArgs(int argc, char **argv) {
100 ScanDepsOptTable Tbl;
101 llvm::StringRef ToolName = argv[0];
102 llvm::BumpPtrAllocator Alloc;
103 llvm::StringSaver Saver{Alloc};
104 llvm::opt::InputArgList Args =
105 Tbl.parseArgs(argc, argv, OPT_UNKNOWN, Saver, [&](StringRef Msg) {
106 llvm::errs() << Msg << '\n';
107 std::exit(1);
108 });
109
110 if (Args.hasArg(OPT_help)) {
111 Tbl.printHelp(OS&: llvm::outs(), Usage: "clang-scan-deps [options]", Title: "clang-scan-deps");
112 std::exit(status: 0);
113 }
114 if (Args.hasArg(OPT_version)) {
115 llvm::outs() << ToolName << '\n';
116 llvm::cl::PrintVersionMessage();
117 std::exit(status: 0);
118 }
119 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_mode_EQ)) {
120 auto ModeType =
121 llvm::StringSwitch<std::optional<ScanningMode>>(A->getValue())
122 .Case("preprocess-dependency-directives",
123 ScanningMode::DependencyDirectivesScan)
124 .Case("preprocess", ScanningMode::CanonicalPreprocessing)
125 .Default(std::nullopt);
126 if (!ModeType) {
127 llvm::errs() << ToolName
128 << ": for the --mode option: Cannot find option named '"
129 << A->getValue() << "'\n";
130 std::exit(status: 1);
131 }
132 ScanMode = *ModeType;
133 }
134
135 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_format_EQ)) {
136 auto FormatType =
137 llvm::StringSwitch<std::optional<ScanningOutputFormat>>(A->getValue())
138 .Case("make", ScanningOutputFormat::Make)
139 .Case("p1689", ScanningOutputFormat::P1689)
140 .Case("experimental-full", ScanningOutputFormat::Full)
141 .Default(std::nullopt);
142 if (!FormatType) {
143 llvm::errs() << ToolName
144 << ": for the --format option: Cannot find option named '"
145 << A->getValue() << "'\n";
146 std::exit(status: 1);
147 }
148 Format = *FormatType;
149 }
150
151 std::vector<std::string> OptimizationFlags =
152 Args.getAllArgValues(OPT_optimize_args_EQ);
153 OptimizeArgs = ScanningOptimizations::None;
154 for (const auto &Arg : OptimizationFlags) {
155 auto Optimization =
156 llvm::StringSwitch<std::optional<ScanningOptimizations>>(Arg)
157 .Case("none", ScanningOptimizations::None)
158 .Case("header-search", ScanningOptimizations::HeaderSearch)
159 .Case("system-warnings", ScanningOptimizations::SystemWarnings)
160 .Case("vfs", ScanningOptimizations::VFS)
161 .Case("canonicalize-macros", ScanningOptimizations::Macros)
162 .Case("all", ScanningOptimizations::All)
163 .Default(std::nullopt);
164 if (!Optimization) {
165 llvm::errs()
166 << ToolName
167 << ": for the --optimize-args option: Cannot find option named '"
168 << Arg << "'\n";
169 std::exit(1);
170 }
171 OptimizeArgs |= *Optimization;
172 }
173 if (OptimizationFlags.empty())
174 OptimizeArgs = ScanningOptimizations::Default;
175
176 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_files_dir_EQ))
177 ModuleFilesDir = A->getValue();
178
179 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_o))
180 OutputFileName = A->getValue();
181
182 EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
183
184 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
185 StringRef S{A->getValue()};
186 if (!llvm::to_integer(S, NumThreads, 0)) {
187 llvm::errs() << ToolName << ": for the -j option: '" << S
188 << "' value invalid for uint argument!\n";
189 std::exit(status: 1);
190 }
191 }
192
193 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_compilation_database_EQ))
194 CompilationDB = A->getValue();
195
196 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_module_name_EQ))
197 ModuleName = A->getValue();
198
199 for (const llvm::opt::Arg *A : Args.filtered(OPT_dependency_target_EQ))
200 ModuleDepTargets.emplace_back(A->getValue());
201
202 DeprecatedDriverCommand = Args.hasArg(OPT_deprecated_driver_command);
203
204 if (const llvm::opt::Arg *A = Args.getLastArg(OPT_resource_dir_recipe_EQ)) {
205 auto Kind =
206 llvm::StringSwitch<std::optional<ResourceDirRecipeKind>>(A->getValue())
207 .Case("modify-compiler-path", RDRK_ModifyCompilerPath)
208 .Case("invoke-compiler", RDRK_InvokeCompiler)
209 .Default(std::nullopt);
210 if (!Kind) {
211 llvm::errs() << ToolName
212 << ": for the --resource-dir-recipe option: Cannot find "
213 "option named '"
214 << A->getValue() << "'\n";
215 std::exit(status: 1);
216 }
217 ResourceDirRecipe = *Kind;
218 }
219
220 PrintTiming = Args.hasArg(OPT_print_timing);
221
222 Verbose = Args.hasArg(OPT_verbose);
223
224 RoundTripArgs = Args.hasArg(OPT_round_trip_args);
225
226 if (const llvm::opt::Arg *A = Args.getLastArgNoClaim(OPT_DASH_DASH))
227 CommandLine.assign(A->getValues().begin(), A->getValues().end());
228}
229
230class SharedStream {
231public:
232 SharedStream(raw_ostream &OS) : OS(OS) {}
233 void applyLocked(llvm::function_ref<void(raw_ostream &OS)> Fn) {
234 std::unique_lock<std::mutex> LockGuard(Lock);
235 Fn(OS);
236 OS.flush();
237 }
238
239private:
240 std::mutex Lock;
241 raw_ostream &OS;
242};
243
244class ResourceDirectoryCache {
245public:
246 /// findResourceDir finds the resource directory relative to the clang
247 /// compiler being used in Args, by running it with "-print-resource-dir"
248 /// option and cache the results for reuse. \returns resource directory path
249 /// associated with the given invocation command or empty string if the
250 /// compiler path is NOT an absolute path.
251 StringRef findResourceDir(const tooling::CommandLineArguments &Args,
252 bool ClangCLMode) {
253 if (Args.size() < 1)
254 return "";
255
256 const std::string &ClangBinaryPath = Args[0];
257 if (!llvm::sys::path::is_absolute(path: ClangBinaryPath))
258 return "";
259
260 const std::string &ClangBinaryName =
261 std::string(llvm::sys::path::filename(path: ClangBinaryPath));
262
263 std::unique_lock<std::mutex> LockGuard(CacheLock);
264 const auto &CachedResourceDir = Cache.find(ClangBinaryPath);
265 if (CachedResourceDir != Cache.end())
266 return CachedResourceDir->second;
267
268 std::vector<StringRef> PrintResourceDirArgs{ClangBinaryName};
269 if (ClangCLMode)
270 PrintResourceDirArgs.push_back("/clang:-print-resource-dir");
271 else
272 PrintResourceDirArgs.push_back("-print-resource-dir");
273
274 llvm::SmallString<64> OutputFile, ErrorFile;
275 llvm::sys::fs::createTemporaryFile("print-resource-dir-output",
276 "" /*no-suffix*/, OutputFile);
277 llvm::sys::fs::createTemporaryFile("print-resource-dir-error",
278 "" /*no-suffix*/, ErrorFile);
279 llvm::FileRemover OutputRemover(OutputFile.c_str());
280 llvm::FileRemover ErrorRemover(ErrorFile.c_str());
281 std::optional<StringRef> Redirects[] = {
282 {""}, // Stdin
283 OutputFile.str(),
284 ErrorFile.str(),
285 };
286 if (llvm::sys::ExecuteAndWait(Program: ClangBinaryPath, Args: PrintResourceDirArgs, Env: {},
287 Redirects)) {
288 auto ErrorBuf = llvm::MemoryBuffer::getFile(Filename: ErrorFile.c_str());
289 llvm::errs() << ErrorBuf.get()->getBuffer();
290 return "";
291 }
292
293 auto OutputBuf = llvm::MemoryBuffer::getFile(Filename: OutputFile.c_str());
294 if (!OutputBuf)
295 return "";
296 StringRef Output = OutputBuf.get()->getBuffer().rtrim('\n');
297
298 Cache[ClangBinaryPath] = Output.str();
299 return Cache[ClangBinaryPath];
300 }
301
302private:
303 std::map<std::string, std::string> Cache;
304 std::mutex CacheLock;
305};
306
307} // end anonymous namespace
308
309/// Takes the result of a dependency scan and prints error / dependency files
310/// based on the result.
311///
312/// \returns True on error.
313static bool
314handleMakeDependencyToolResult(const std::string &Input,
315 llvm::Expected<std::string> &MaybeFile,
316 SharedStream &OS, SharedStream &Errs) {
317 if (!MaybeFile) {
318 llvm::handleAllErrors(
319 E: MaybeFile.takeError(), Handlers: [&Input, &Errs](llvm::StringError &Err) {
320 Errs.applyLocked(Fn: [&](raw_ostream &OS) {
321 OS << "Error while scanning dependencies for " << Input << ":\n";
322 OS << Err.getMessage();
323 });
324 });
325 return true;
326 }
327 OS.applyLocked(Fn: [&](raw_ostream &OS) { OS << *MaybeFile; });
328 return false;
329}
330
331static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
332 std::vector<llvm::StringRef> Strings;
333 for (auto &&I : Set)
334 Strings.push_back(x: I.getKey());
335 llvm::sort(C&: Strings);
336 return llvm::json::Array(Strings);
337}
338
339// Technically, we don't need to sort the dependency list to get determinism.
340// Leaving these be will simply preserve the import order.
341static llvm::json::Array toJSONSorted(std::vector<ModuleID> V) {
342 llvm::sort(C&: V);
343
344 llvm::json::Array Ret;
345 for (const ModuleID &MID : V)
346 Ret.push_back(E: llvm::json::Object(
347 {{.K: "module-name", .V: MID.ModuleName}, {.K: "context-hash", .V: MID.ContextHash}}));
348 return Ret;
349}
350
351// Thread safe.
352class FullDeps {
353public:
354 FullDeps(size_t NumInputs) : Inputs(NumInputs) {}
355
356 void mergeDeps(StringRef Input, TranslationUnitDeps TUDeps,
357 size_t InputIndex) {
358 mergeDeps(Graph: std::move(TUDeps.ModuleGraph), InputIndex);
359
360 InputDeps ID;
361 ID.FileName = std::string(Input);
362 ID.ContextHash = std::move(TUDeps.ID.ContextHash);
363 ID.FileDeps = std::move(TUDeps.FileDeps);
364 ID.ModuleDeps = std::move(TUDeps.ClangModuleDeps);
365 ID.DriverCommandLine = std::move(TUDeps.DriverCommandLine);
366 ID.Commands = std::move(TUDeps.Commands);
367
368 assert(InputIndex < Inputs.size() && "Input index out of bounds");
369 assert(Inputs[InputIndex].FileName.empty() && "Result already populated");
370 Inputs[InputIndex] = std::move(ID);
371 }
372
373 void mergeDeps(ModuleDepsGraph Graph, size_t InputIndex) {
374 std::vector<ModuleDeps *> NewMDs;
375 {
376 std::unique_lock<std::mutex> ul(Lock);
377 for (const ModuleDeps &MD : Graph) {
378 auto I = Modules.find(x: {.ID: MD.ID, .InputIndex: 0});
379 if (I != Modules.end()) {
380 I->first.InputIndex = std::min(a: I->first.InputIndex, b: InputIndex);
381 continue;
382 }
383 auto Res = Modules.insert(hint: I, x: {{.ID: MD.ID, .InputIndex: InputIndex}, std::move(MD)});
384 NewMDs.push_back(x: &Res->second);
385 }
386 // First call to \c getBuildArguments is somewhat expensive. Let's call it
387 // on the current thread (instead of the main one), and outside the
388 // critical section.
389 for (ModuleDeps *MD : NewMDs)
390 (void)MD->getBuildArguments();
391 }
392 }
393
394 bool roundTripCommand(ArrayRef<std::string> ArgStrs,
395 DiagnosticsEngine &Diags) {
396 if (ArgStrs.empty() || ArgStrs[0] != "-cc1")
397 return false;
398 SmallVector<const char *> Args;
399 for (const std::string &Arg : ArgStrs)
400 Args.push_back(Elt: Arg.c_str());
401 return !CompilerInvocation::checkCC1RoundTrip(Args, Diags);
402 }
403
404 // Returns \c true if any command lines fail to round-trip. We expect
405 // commands already be canonical when output by the scanner.
406 bool roundTripCommands(raw_ostream &ErrOS) {
407 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions{};
408 TextDiagnosticPrinter DiagConsumer(ErrOS, &*DiagOpts);
409 IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
410 CompilerInstance::createDiagnostics(Opts: &*DiagOpts, Client: &DiagConsumer,
411 /*ShouldOwnClient=*/false);
412
413 for (auto &&M : Modules)
414 if (roundTripCommand(ArgStrs: M.second.getBuildArguments(), Diags&: *Diags))
415 return true;
416
417 for (auto &&I : Inputs)
418 for (const auto &Cmd : I.Commands)
419 if (roundTripCommand(ArgStrs: Cmd.Arguments, Diags&: *Diags))
420 return true;
421
422 return false;
423 }
424
425 void printFullOutput(raw_ostream &OS) {
426 // Skip sorting modules and constructing the JSON object if the output
427 // cannot be observed anyway. This makes timings less noisy.
428 if (&OS == &llvm::nulls())
429 return;
430
431 // Sort the modules by name to get a deterministic order.
432 std::vector<IndexedModuleID> ModuleIDs;
433 for (auto &&M : Modules)
434 ModuleIDs.push_back(x: M.first);
435 llvm::sort(C&: ModuleIDs);
436
437 using namespace llvm::json;
438
439 Array OutModules;
440 for (auto &&ModID : ModuleIDs) {
441 auto &MD = Modules[ModID];
442 Object O{
443 {.K: "name", .V: MD.ID.ModuleName},
444 {.K: "context-hash", .V: MD.ID.ContextHash},
445 {.K: "file-deps", .V: toJSONSorted(Set: MD.FileDeps)},
446 {.K: "clang-module-deps", .V: toJSONSorted(V: MD.ClangModuleDeps)},
447 {.K: "clang-modulemap-file", .V: MD.ClangModuleMapFile},
448 {.K: "command-line", .V: MD.getBuildArguments()},
449 };
450 OutModules.push_back(E: std::move(O));
451 }
452
453 Array TUs;
454 for (auto &&I : Inputs) {
455 Array Commands;
456 if (I.DriverCommandLine.empty()) {
457 for (const auto &Cmd : I.Commands) {
458 Object O{
459 {.K: "input-file", .V: I.FileName},
460 {.K: "clang-context-hash", .V: I.ContextHash},
461 {.K: "file-deps", .V: I.FileDeps},
462 {.K: "clang-module-deps", .V: toJSONSorted(V: I.ModuleDeps)},
463 {.K: "executable", .V: Cmd.Executable},
464 {.K: "command-line", .V: Cmd.Arguments},
465 };
466 Commands.push_back(E: std::move(O));
467 }
468 } else {
469 Object O{
470 {.K: "input-file", .V: I.FileName},
471 {.K: "clang-context-hash", .V: I.ContextHash},
472 {.K: "file-deps", .V: I.FileDeps},
473 {.K: "clang-module-deps", .V: toJSONSorted(V: I.ModuleDeps)},
474 {.K: "executable", .V: "clang"},
475 {.K: "command-line", .V: I.DriverCommandLine},
476 };
477 Commands.push_back(E: std::move(O));
478 }
479 TUs.push_back(E: Object{
480 {.K: "commands", .V: std::move(Commands)},
481 });
482 }
483
484 Object Output{
485 {.K: "modules", .V: std::move(OutModules)},
486 {.K: "translation-units", .V: std::move(TUs)},
487 };
488
489 OS << llvm::formatv(Fmt: "{0:2}\n", Vals: Value(std::move(Output)));
490 }
491
492private:
493 struct IndexedModuleID {
494 ModuleID ID;
495
496 // FIXME: This is mutable so that it can still be updated after insertion
497 // into an unordered associative container. This is "fine", since this
498 // field doesn't contribute to the hash, but it's a brittle hack.
499 mutable size_t InputIndex;
500
501 bool operator==(const IndexedModuleID &Other) const {
502 return ID == Other.ID;
503 }
504
505 bool operator<(const IndexedModuleID &Other) const {
506 /// We need the output of clang-scan-deps to be deterministic. However,
507 /// the dependency graph may contain two modules with the same name. How
508 /// do we decide which one to print first? If we made that decision based
509 /// on the context hash, the ordering would be deterministic, but
510 /// different across machines. This can happen for example when the inputs
511 /// or the SDKs (which both contribute to the "context" hash) live in
512 /// different absolute locations. We solve that by tracking the index of
513 /// the first input TU that (transitively) imports the dependency, which
514 /// is always the same for the same input, resulting in deterministic
515 /// sorting that's also reproducible across machines.
516 return std::tie(args: ID.ModuleName, args&: InputIndex) <
517 std::tie(args: Other.ID.ModuleName, args&: Other.InputIndex);
518 }
519
520 struct Hasher {
521 std::size_t operator()(const IndexedModuleID &IMID) const {
522 return llvm::hash_value(ID: IMID.ID);
523 }
524 };
525 };
526
527 struct InputDeps {
528 std::string FileName;
529 std::string ContextHash;
530 std::vector<std::string> FileDeps;
531 std::vector<ModuleID> ModuleDeps;
532 std::vector<std::string> DriverCommandLine;
533 std::vector<Command> Commands;
534 };
535
536 std::mutex Lock;
537 std::unordered_map<IndexedModuleID, ModuleDeps, IndexedModuleID::Hasher>
538 Modules;
539 std::vector<InputDeps> Inputs;
540};
541
542static bool handleTranslationUnitResult(
543 StringRef Input, llvm::Expected<TranslationUnitDeps> &MaybeTUDeps,
544 FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
545 if (!MaybeTUDeps) {
546 llvm::handleAllErrors(
547 E: MaybeTUDeps.takeError(), Handlers: [&Input, &Errs](llvm::StringError &Err) {
548 Errs.applyLocked(Fn: [&](raw_ostream &OS) {
549 OS << "Error while scanning dependencies for " << Input << ":\n";
550 OS << Err.getMessage();
551 });
552 });
553 return true;
554 }
555 FD.mergeDeps(Input, TUDeps: std::move(*MaybeTUDeps), InputIndex);
556 return false;
557}
558
559static bool handleModuleResult(
560 StringRef ModuleName, llvm::Expected<ModuleDepsGraph> &MaybeModuleGraph,
561 FullDeps &FD, size_t InputIndex, SharedStream &OS, SharedStream &Errs) {
562 if (!MaybeModuleGraph) {
563 llvm::handleAllErrors(E: MaybeModuleGraph.takeError(),
564 Handlers: [&ModuleName, &Errs](llvm::StringError &Err) {
565 Errs.applyLocked(Fn: [&](raw_ostream &OS) {
566 OS << "Error while scanning dependencies for "
567 << ModuleName << ":\n";
568 OS << Err.getMessage();
569 });
570 });
571 return true;
572 }
573 FD.mergeDeps(Graph: std::move(*MaybeModuleGraph), InputIndex);
574 return false;
575}
576
577class P1689Deps {
578public:
579 void printDependencies(raw_ostream &OS) {
580 addSourcePathsToRequires();
581 // Sort the modules by name to get a deterministic order.
582 llvm::sort(C&: Rules, Comp: [](const P1689Rule &A, const P1689Rule &B) {
583 return A.PrimaryOutput < B.PrimaryOutput;
584 });
585
586 using namespace llvm::json;
587 Array OutputRules;
588 for (const P1689Rule &R : Rules) {
589 Object O{{.K: "primary-output", .V: R.PrimaryOutput}};
590
591 if (R.Provides) {
592 Array Provides;
593 Object Provided{{.K: "logical-name", .V: R.Provides->ModuleName},
594 {.K: "source-path", .V: R.Provides->SourcePath},
595 {.K: "is-interface", .V: R.Provides->IsStdCXXModuleInterface}};
596 Provides.push_back(E: std::move(Provided));
597 O.insert(E: {.K: "provides", .V: std::move(Provides)});
598 }
599
600 Array Requires;
601 for (const P1689ModuleInfo &Info : R.Requires) {
602 Object RequiredInfo{{.K: "logical-name", .V: Info.ModuleName}};
603 if (!Info.SourcePath.empty())
604 RequiredInfo.insert(E: {.K: "source-path", .V: Info.SourcePath});
605 Requires.push_back(E: std::move(RequiredInfo));
606 }
607
608 if (!Requires.empty())
609 O.insert(E: {.K: "requires", .V: std::move(Requires)});
610
611 OutputRules.push_back(E: std::move(O));
612 }
613
614 Object Output{
615 {.K: "version", .V: 1}, {.K: "revision", .V: 0}, {.K: "rules", .V: std::move(OutputRules)}};
616
617 OS << llvm::formatv(Fmt: "{0:2}\n", Vals: Value(std::move(Output)));
618 }
619
620 void addRules(P1689Rule &Rule) {
621 std::unique_lock<std::mutex> LockGuard(Lock);
622 Rules.push_back(x: Rule);
623 }
624
625private:
626 void addSourcePathsToRequires() {
627 llvm::DenseMap<StringRef, StringRef> ModuleSourceMapper;
628 for (const P1689Rule &R : Rules)
629 if (R.Provides && !R.Provides->SourcePath.empty())
630 ModuleSourceMapper[R.Provides->ModuleName] = R.Provides->SourcePath;
631
632 for (P1689Rule &R : Rules) {
633 for (P1689ModuleInfo &Info : R.Requires) {
634 auto Iter = ModuleSourceMapper.find(Val: Info.ModuleName);
635 if (Iter != ModuleSourceMapper.end())
636 Info.SourcePath = Iter->second;
637 }
638 }
639 }
640
641 std::mutex Lock;
642 std::vector<P1689Rule> Rules;
643};
644
645static bool
646handleP1689DependencyToolResult(const std::string &Input,
647 llvm::Expected<P1689Rule> &MaybeRule,
648 P1689Deps &PD, SharedStream &Errs) {
649 if (!MaybeRule) {
650 llvm::handleAllErrors(
651 E: MaybeRule.takeError(), Handlers: [&Input, &Errs](llvm::StringError &Err) {
652 Errs.applyLocked(Fn: [&](raw_ostream &OS) {
653 OS << "Error while scanning dependencies for " << Input << ":\n";
654 OS << Err.getMessage();
655 });
656 });
657 return true;
658 }
659 PD.addRules(Rule&: *MaybeRule);
660 return false;
661}
662
663/// Construct a path for the explicitly built PCM.
664static std::string constructPCMPath(ModuleID MID, StringRef OutputDir) {
665 SmallString<256> ExplicitPCMPath(OutputDir);
666 llvm::sys::path::append(path&: ExplicitPCMPath, a: MID.ContextHash,
667 b: MID.ModuleName + "-" + MID.ContextHash + ".pcm");
668 return std::string(ExplicitPCMPath);
669}
670
671static std::string lookupModuleOutput(const ModuleID &MID, ModuleOutputKind MOK,
672 StringRef OutputDir) {
673 std::string PCMPath = constructPCMPath(MID, OutputDir);
674 switch (MOK) {
675 case ModuleOutputKind::ModuleFile:
676 return PCMPath;
677 case ModuleOutputKind::DependencyFile:
678 return PCMPath + ".d";
679 case ModuleOutputKind::DependencyTargets:
680 // Null-separate the list of targets.
681 return join(ModuleDepTargets, StringRef("\0", 1));
682 case ModuleOutputKind::DiagnosticSerializationFile:
683 return PCMPath + ".diag";
684 }
685 llvm_unreachable("Fully covered switch above!");
686}
687
688static std::string getModuleCachePath(ArrayRef<std::string> Args) {
689 for (StringRef Arg : llvm::reverse(C&: Args)) {
690 Arg.consume_front(Prefix: "/clang:");
691 if (Arg.consume_front(Prefix: "-fmodules-cache-path="))
692 return std::string(Arg);
693 }
694 SmallString<128> Path;
695 driver::Driver::getDefaultModuleCachePath(Result&: Path);
696 return std::string(Path);
697}
698
699/// Attempts to construct the compilation database from '-compilation-database'
700/// or from the arguments following the positional '--'.
701static std::unique_ptr<tooling::CompilationDatabase>
702getCompilationDatabase(int argc, char **argv, std::string &ErrorMessage) {
703 ParseArgs(argc, argv);
704
705 if (!(CommandLine.empty() ^ CompilationDB.empty())) {
706 llvm::errs() << "The compilation command line must be provided either via "
707 "'-compilation-database' or after '--'.";
708 return nullptr;
709 }
710
711 if (!CompilationDB.empty())
712 return tooling::JSONCompilationDatabase::loadFromFile(
713 FilePath: CompilationDB, ErrorMessage,
714 Syntax: tooling::JSONCommandLineSyntax::AutoDetect);
715
716 llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
717 CompilerInstance::createDiagnostics(Opts: new DiagnosticOptions);
718 driver::Driver TheDriver(CommandLine[0], llvm::sys::getDefaultTargetTriple(),
719 *Diags);
720 TheDriver.setCheckInputsExist(false);
721 std::unique_ptr<driver::Compilation> C(
722 TheDriver.BuildCompilation(Args: CommandLine));
723 if (!C || C->getJobs().empty())
724 return nullptr;
725
726 auto Cmd = C->getJobs().begin();
727 auto CI = std::make_unique<CompilerInvocation>();
728 CompilerInvocation::CreateFromArgs(Res&: *CI, CommandLineArgs: Cmd->getArguments(), Diags&: *Diags,
729 Argv0: CommandLine[0]);
730 if (!CI)
731 return nullptr;
732
733 FrontendOptions &FEOpts = CI->getFrontendOpts();
734 if (FEOpts.Inputs.size() != 1) {
735 llvm::errs()
736 << "Exactly one input file is required in the per-file mode ('--').\n";
737 return nullptr;
738 }
739
740 // There might be multiple jobs for a compilation. Extract the specified
741 // output filename from the last job.
742 auto LastCmd = C->getJobs().end();
743 LastCmd--;
744 if (LastCmd->getOutputFilenames().size() != 1) {
745 llvm::errs()
746 << "Exactly one output file is required in the per-file mode ('--').\n";
747 return nullptr;
748 }
749 StringRef OutputFile = LastCmd->getOutputFilenames().front();
750
751 class InplaceCompilationDatabase : public tooling::CompilationDatabase {
752 public:
753 InplaceCompilationDatabase(StringRef InputFile, StringRef OutputFile,
754 ArrayRef<const char *> CommandLine)
755 : Command(".", InputFile, {}, OutputFile) {
756 for (auto *C : CommandLine)
757 Command.CommandLine.push_back(x: C);
758 }
759
760 std::vector<tooling::CompileCommand>
761 getCompileCommands(StringRef FilePath) const override {
762 if (FilePath != Command.Filename)
763 return {};
764 return {Command};
765 }
766
767 std::vector<std::string> getAllFiles() const override {
768 return {Command.Filename};
769 }
770
771 std::vector<tooling::CompileCommand>
772 getAllCompileCommands() const override {
773 return {Command};
774 }
775
776 private:
777 tooling::CompileCommand Command;
778 };
779
780 return std::make_unique<InplaceCompilationDatabase>(
781 FEOpts.Inputs[0].getFile(), OutputFile, CommandLine);
782}
783
784int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
785 std::string ErrorMessage;
786 std::unique_ptr<tooling::CompilationDatabase> Compilations =
787 getCompilationDatabase(argc, argv, ErrorMessage);
788 if (!Compilations) {
789 llvm::errs() << ErrorMessage << "\n";
790 return 1;
791 }
792
793 llvm::cl::PrintOptionValues();
794
795 // The command options are rewritten to run Clang in preprocessor only mode.
796 auto AdjustingCompilations =
797 std::make_unique<tooling::ArgumentsAdjustingCompilations>(
798 args: std::move(Compilations));
799 ResourceDirectoryCache ResourceDirCache;
800
801 AdjustingCompilations->appendArgumentsAdjuster(
802 Adjuster: [&ResourceDirCache](const tooling::CommandLineArguments &Args,
803 StringRef FileName) {
804 std::string LastO;
805 bool HasResourceDir = false;
806 bool ClangCLMode = false;
807 auto FlagsEnd = llvm::find(Range: Args, Val: "--");
808 if (FlagsEnd != Args.begin()) {
809 ClangCLMode =
810 llvm::sys::path::stem(path: Args[0]).contains_insensitive(Other: "clang-cl") ||
811 llvm::is_contained(Range: Args, Element: "--driver-mode=cl");
812
813 // Reverse scan, starting at the end or at the element before "--".
814 auto R = std::make_reverse_iterator(i: FlagsEnd);
815 for (auto I = R, E = Args.rend(); I != E; ++I) {
816 StringRef Arg = *I;
817 if (ClangCLMode) {
818 // Ignore arguments that are preceded by "-Xclang".
819 if ((I + 1) != E && I[1] == "-Xclang")
820 continue;
821 if (LastO.empty()) {
822 // With clang-cl, the output obj file can be specified with
823 // "/opath", "/o path", "/Fopath", and the dash counterparts.
824 // Also, clang-cl adds ".obj" extension if none is found.
825 if ((Arg == "-o" || Arg == "/o") && I != R)
826 LastO = I[-1]; // Next argument (reverse iterator)
827 else if (Arg.starts_with(Prefix: "/Fo") || Arg.starts_with(Prefix: "-Fo"))
828 LastO = Arg.drop_front(N: 3).str();
829 else if (Arg.starts_with(Prefix: "/o") || Arg.starts_with(Prefix: "-o"))
830 LastO = Arg.drop_front(N: 2).str();
831
832 if (!LastO.empty() && !llvm::sys::path::has_extension(path: LastO))
833 LastO.append(s: ".obj");
834 }
835 }
836 if (Arg == "-resource-dir")
837 HasResourceDir = true;
838 }
839 }
840 tooling::CommandLineArguments AdjustedArgs(Args.begin(), FlagsEnd);
841 // The clang-cl driver passes "-o -" to the frontend. Inject the real
842 // file here to ensure "-MT" can be deduced if need be.
843 if (ClangCLMode && !LastO.empty()) {
844 AdjustedArgs.push_back(x: "/clang:-o");
845 AdjustedArgs.push_back(x: "/clang:" + LastO);
846 }
847
848 if (!HasResourceDir && ResourceDirRecipe == RDRK_InvokeCompiler) {
849 StringRef ResourceDir =
850 ResourceDirCache.findResourceDir(Args, ClangCLMode);
851 if (!ResourceDir.empty()) {
852 AdjustedArgs.push_back(x: "-resource-dir");
853 AdjustedArgs.push_back(x: std::string(ResourceDir));
854 }
855 }
856 AdjustedArgs.insert(position: AdjustedArgs.end(), first: FlagsEnd, last: Args.end());
857 return AdjustedArgs;
858 });
859
860 SharedStream Errs(llvm::errs());
861
862 std::optional<llvm::raw_fd_ostream> FileOS;
863 llvm::raw_ostream &ThreadUnsafeDependencyOS = [&]() -> llvm::raw_ostream & {
864 if (OutputFileName == "-")
865 return llvm::outs();
866
867 if (OutputFileName == "/dev/null")
868 return llvm::nulls();
869
870 std::error_code EC;
871 FileOS.emplace(args&: OutputFileName, args&: EC);
872 if (EC) {
873 llvm::errs() << "Failed to open output file '" << OutputFileName
874 << "': " << llvm::errorCodeToError(EC) << '\n';
875 std::exit(status: 1);
876 }
877 return *FileOS;
878 }();
879 SharedStream DependencyOS(ThreadUnsafeDependencyOS);
880
881 std::vector<tooling::CompileCommand> Inputs =
882 AdjustingCompilations->getAllCompileCommands();
883
884 std::atomic<bool> HadErrors(false);
885 std::optional<FullDeps> FD;
886 P1689Deps PD;
887
888 std::mutex Lock;
889 size_t Index = 0;
890 auto GetNextInputIndex = [&]() -> std::optional<size_t> {
891 std::unique_lock<std::mutex> LockGuard(Lock);
892 if (Index < Inputs.size())
893 return Index++;
894 return {};
895 };
896
897 if (Format == ScanningOutputFormat::Full)
898 FD.emplace(args: ModuleName.empty() ? Inputs.size() : 0);
899
900 auto ScanningTask = [&](DependencyScanningService &Service) {
901 DependencyScanningTool WorkerTool(Service);
902
903 llvm::DenseSet<ModuleID> AlreadySeenModules;
904 while (auto MaybeInputIndex = GetNextInputIndex()) {
905 size_t LocalIndex = *MaybeInputIndex;
906 const tooling::CompileCommand *Input = &Inputs[LocalIndex];
907 std::string Filename = std::move(Input->Filename);
908 std::string CWD = std::move(Input->Directory);
909
910 std::optional<StringRef> MaybeModuleName;
911 if (!ModuleName.empty())
912 MaybeModuleName = ModuleName;
913
914 std::string OutputDir(ModuleFilesDir);
915 if (OutputDir.empty())
916 OutputDir = getModuleCachePath(Args: Input->CommandLine);
917 auto LookupOutput = [&](const ModuleID &MID, ModuleOutputKind MOK) {
918 return ::lookupModuleOutput(MID, MOK, OutputDir);
919 };
920
921 // Run the tool on it.
922 if (Format == ScanningOutputFormat::Make) {
923 auto MaybeFile = WorkerTool.getDependencyFile(CommandLine: Input->CommandLine, CWD);
924 if (handleMakeDependencyToolResult(Input: Filename, MaybeFile, OS&: DependencyOS,
925 Errs))
926 HadErrors = true;
927 } else if (Format == ScanningOutputFormat::P1689) {
928 // It is useful to generate the make-format dependency output during
929 // the scanning for P1689. Otherwise the users need to scan again for
930 // it. We will generate the make-format dependency output if we find
931 // `-MF` in the command lines.
932 std::string MakeformatOutputPath;
933 std::string MakeformatOutput;
934
935 auto MaybeRule = WorkerTool.getP1689ModuleDependencyFile(
936 Command: *Input, CWD, MakeformatOutput, MakeformatOutputPath);
937
938 if (handleP1689DependencyToolResult(Input: Filename, MaybeRule, PD, Errs))
939 HadErrors = true;
940
941 if (!MakeformatOutputPath.empty() && !MakeformatOutput.empty() &&
942 !HadErrors) {
943 static std::mutex Lock;
944 // With compilation database, we may open different files
945 // concurrently or we may write the same file concurrently. So we
946 // use a map here to allow multiple compile commands to write to the
947 // same file. Also we need a lock here to avoid data race.
948 static llvm::StringMap<llvm::raw_fd_ostream> OSs;
949 std::unique_lock<std::mutex> LockGuard(Lock);
950
951 auto OSIter = OSs.find(Key: MakeformatOutputPath);
952 if (OSIter == OSs.end()) {
953 std::error_code EC;
954 OSIter =
955 OSs.try_emplace(Key: MakeformatOutputPath, Args&: MakeformatOutputPath, Args&: EC)
956 .first;
957 if (EC)
958 llvm::errs() << "Failed to open P1689 make format output file \""
959 << MakeformatOutputPath << "\" for " << EC.message()
960 << "\n";
961 }
962
963 SharedStream MakeformatOS(OSIter->second);
964 llvm::Expected<std::string> MaybeOutput(MakeformatOutput);
965 if (handleMakeDependencyToolResult(Input: Filename, MaybeFile&: MaybeOutput,
966 OS&: MakeformatOS, Errs))
967 HadErrors = true;
968 }
969 } else if (MaybeModuleName) {
970 auto MaybeModuleDepsGraph = WorkerTool.getModuleDependencies(
971 ModuleName: *MaybeModuleName, CommandLine: Input->CommandLine, CWD, AlreadySeen: AlreadySeenModules,
972 LookupModuleOutput: LookupOutput);
973 if (handleModuleResult(ModuleName: *MaybeModuleName, MaybeModuleGraph&: MaybeModuleDepsGraph, FD&: *FD,
974 InputIndex: LocalIndex, OS&: DependencyOS, Errs))
975 HadErrors = true;
976 } else {
977 auto MaybeTUDeps = WorkerTool.getTranslationUnitDependencies(
978 CommandLine: Input->CommandLine, CWD, AlreadySeen: AlreadySeenModules, LookupModuleOutput: LookupOutput);
979 if (handleTranslationUnitResult(Input: Filename, MaybeTUDeps, FD&: *FD, InputIndex: LocalIndex,
980 OS&: DependencyOS, Errs))
981 HadErrors = true;
982 }
983 }
984 };
985
986 DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
987 EagerLoadModules);
988
989 llvm::Timer T;
990 T.startTimer();
991
992 if (Inputs.size() == 1) {
993 ScanningTask(Service);
994 } else {
995 llvm::DefaultThreadPool Pool(llvm::hardware_concurrency(ThreadCount: NumThreads));
996
997 if (Verbose) {
998 llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
999 << " files using " << Pool.getMaxConcurrency()
1000 << " workers\n";
1001 }
1002
1003 for (unsigned I = 0; I < Pool.getMaxConcurrency(); ++I)
1004 Pool.async(F: [ScanningTask, &Service]() { ScanningTask(Service); });
1005
1006 Pool.wait();
1007 }
1008
1009 T.stopTimer();
1010 if (PrintTiming)
1011 llvm::errs() << llvm::format(
1012 Fmt: "clang-scan-deps timing: %0.2fs wall, %0.2fs process\n",
1013 Vals: T.getTotalTime().getWallTime(), Vals: T.getTotalTime().getProcessTime());
1014
1015 if (RoundTripArgs)
1016 if (FD && FD->roundTripCommands(ErrOS&: llvm::errs()))
1017 HadErrors = true;
1018
1019 if (Format == ScanningOutputFormat::Full)
1020 FD->printFullOutput(OS&: ThreadUnsafeDependencyOS);
1021 else if (Format == ScanningOutputFormat::P1689)
1022 PD.printDependencies(OS&: ThreadUnsafeDependencyOS);
1023
1024 return HadErrors;
1025}
1026

source code of clang/tools/clang-scan-deps/ClangScanDeps.cpp