1//===--- SystemIncludeExtractor.cpp ------------------------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// Some compiler drivers have implicit search mechanism for system headers.
9// This compilation database implementation tries to extract that information by
10// executing the driver in verbose mode. gcc-compatible drivers print something
11// like:
12// ....
13// ....
14// #include <...> search starts here:
15// /usr/lib/gcc/x86_64-linux-gnu/7/include
16// /usr/local/include
17// /usr/lib/gcc/x86_64-linux-gnu/7/include-fixed
18// /usr/include/x86_64-linux-gnu
19// /usr/include
20// End of search list.
21// ....
22// ....
23// This component parses that output and adds each path to command line args
24// provided by Base, after prepending them with -isystem. Therefore current
25// implementation would not work with a driver that is not gcc-compatible.
26//
27// First argument of the command line received from underlying compilation
28// database is used as compiler driver path. Due to this arbitrary binary
29// execution, this mechanism is not used by default and only executes binaries
30// in the paths that are explicitly included by the user.
31
32#include "CompileCommands.h"
33#include "Config.h"
34#include "GlobalCompilationDatabase.h"
35#include "support/Logger.h"
36#include "support/Threading.h"
37#include "support/Trace.h"
38#include "clang/Basic/Diagnostic.h"
39#include "clang/Basic/DiagnosticIDs.h"
40#include "clang/Basic/DiagnosticOptions.h"
41#include "clang/Basic/TargetInfo.h"
42#include "clang/Basic/TargetOptions.h"
43#include "clang/Driver/Types.h"
44#include "clang/Tooling/CompilationDatabase.h"
45#include "llvm/ADT/ArrayRef.h"
46#include "llvm/ADT/DenseMap.h"
47#include "llvm/ADT/Hashing.h"
48#include "llvm/ADT/IntrusiveRefCntPtr.h"
49#include "llvm/ADT/STLExtras.h"
50#include "llvm/ADT/ScopeExit.h"
51#include "llvm/ADT/SmallString.h"
52#include "llvm/ADT/SmallVector.h"
53#include "llvm/ADT/StringExtras.h"
54#include "llvm/ADT/StringRef.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Support/FileSystem.h"
57#include "llvm/Support/MemoryBuffer.h"
58#include "llvm/Support/Path.h"
59#include "llvm/Support/Program.h"
60#include "llvm/Support/Regex.h"
61#include "llvm/Support/ScopedPrinter.h"
62#include "llvm/Support/raw_ostream.h"
63#include <cassert>
64#include <cstddef>
65#include <iterator>
66#include <memory>
67#include <optional>
68#include <string>
69#include <tuple>
70#include <utility>
71#include <vector>
72
73namespace clang::clangd {
74namespace {
75
76struct DriverInfo {
77 std::vector<std::string> SystemIncludes;
78 std::string Target;
79};
80
81struct DriverArgs {
82 // Name of the driver program to execute or absolute path to it.
83 std::string Driver;
84 // Whether certain includes should be part of query.
85 bool StandardIncludes = true;
86 bool StandardCXXIncludes = true;
87 // Language to use while querying.
88 std::string Lang;
89 std::string Sysroot;
90 std::string ISysroot;
91 std::string Target;
92 std::string Stdlib;
93 llvm::SmallVector<std::string> Specs;
94
95 bool operator==(const DriverArgs &RHS) const {
96 return std::tie(args: Driver, args: StandardIncludes, args: StandardCXXIncludes, args: Lang,
97 args: Sysroot, args: ISysroot, args: Target, args: Stdlib, args: Specs) ==
98 std::tie(args: RHS.Driver, args: RHS.StandardIncludes, args: RHS.StandardCXXIncludes,
99 args: RHS.Lang, args: RHS.Sysroot, args: RHS.ISysroot, args: RHS.Target, args: RHS.Stdlib,
100 args: RHS.Specs);
101 }
102
103 DriverArgs(const tooling::CompileCommand &Cmd, llvm::StringRef File) {
104 llvm::SmallString<128> Driver(Cmd.CommandLine.front());
105 // Driver is a not a single executable name but instead a path (either
106 // relative or absolute).
107 if (llvm::any_of(Range&: Driver,
108 P: [](char C) { return llvm::sys::path::is_separator(value: C); })) {
109 llvm::sys::fs::make_absolute(current_directory: Cmd.Directory, path&: Driver);
110 }
111 this->Driver = Driver.str().str();
112 for (size_t I = 0, E = Cmd.CommandLine.size(); I < E; ++I) {
113 llvm::StringRef Arg = Cmd.CommandLine[I];
114
115 // Look for Language related flags.
116 if (Arg.consume_front(Prefix: "-x")) {
117 if (Arg.empty() && I + 1 < E)
118 Lang = Cmd.CommandLine[I + 1];
119 else
120 Lang = Arg.str();
121 }
122 // Look for standard/builtin includes.
123 else if (Arg == "-nostdinc" || Arg == "--no-standard-includes")
124 StandardIncludes = false;
125 else if (Arg == "-nostdinc++")
126 StandardCXXIncludes = false;
127 // Figure out sysroot
128 else if (Arg.consume_front(Prefix: "--sysroot")) {
129 if (Arg.consume_front(Prefix: "="))
130 Sysroot = Arg.str();
131 else if (Arg.empty() && I + 1 < E)
132 Sysroot = Cmd.CommandLine[I + 1];
133 } else if (Arg.consume_front(Prefix: "-isysroot")) {
134 if (Arg.empty() && I + 1 < E)
135 ISysroot = Cmd.CommandLine[I + 1];
136 else
137 ISysroot = Arg.str();
138 } else if (Arg.consume_front(Prefix: "--target=")) {
139 Target = Arg.str();
140 } else if (Arg.consume_front(Prefix: "-target")) {
141 if (Arg.empty() && I + 1 < E)
142 Target = Cmd.CommandLine[I + 1];
143 } else if (Arg.consume_front(Prefix: "--stdlib")) {
144 if (Arg.consume_front(Prefix: "="))
145 Stdlib = Arg.str();
146 else if (Arg.empty() && I + 1 < E)
147 Stdlib = Cmd.CommandLine[I + 1];
148 } else if (Arg.consume_front(Prefix: "-stdlib=")) {
149 Stdlib = Arg.str();
150 } else if (Arg.starts_with(Prefix: "-specs=")) {
151 // clang requires a single token like `-specs=file` or `--specs=file`,
152 // but gcc will accept two tokens like `--specs file`. Since the
153 // compilation database is presumably correct, we just forward the flags
154 // as-is.
155 Specs.push_back(Elt: Arg.str());
156 } else if (Arg.starts_with(Prefix: "--specs=")) {
157 Specs.push_back(Elt: Arg.str());
158 } else if (Arg == "--specs" && I + 1 < E) {
159 Specs.push_back(Elt: Arg.str());
160 Specs.push_back(Elt: Cmd.CommandLine[I + 1]);
161 }
162 }
163
164 // Downgrade objective-c++-header (used in clangd's fallback flags for .h
165 // files) to c++-header, as some drivers may fail to run the extraction
166 // command if it contains `-xobjective-c++-header` and objective-c++ support
167 // is not installed.
168 // In practice, we don't see different include paths for the two on
169 // clang+mac, which is the most common objectve-c compiler.
170 if (Lang == "objective-c++-header") {
171 Lang = "c++-header";
172 }
173
174 // If language is not explicit in the flags, infer from the file.
175 // This is important as we want to cache each language separately.
176 if (Lang.empty()) {
177 llvm::StringRef Ext = llvm::sys::path::extension(path: File).trim(Char: '.');
178 auto Type = driver::types::lookupTypeForExtension(Ext);
179 if (Type == driver::types::TY_INVALID) {
180 elog(Fmt: "System include extraction: invalid file type for {0}", Vals&: Ext);
181 } else {
182 Lang = driver::types::getTypeName(Id: Type);
183 }
184 }
185 }
186 llvm::SmallVector<llvm::StringRef> render() const {
187 // FIXME: Don't treat lang specially?
188 assert(!Lang.empty());
189 llvm::SmallVector<llvm::StringRef> Args = {"-x", Lang};
190 if (!StandardIncludes)
191 Args.push_back(Elt: "-nostdinc");
192 if (!StandardCXXIncludes)
193 Args.push_back(Elt: "-nostdinc++");
194 if (!Sysroot.empty())
195 Args.append(IL: {"--sysroot", Sysroot});
196 if (!ISysroot.empty())
197 Args.append(IL: {"-isysroot", ISysroot});
198 if (!Target.empty())
199 Args.append(IL: {"-target", Target});
200 if (!Stdlib.empty())
201 Args.append(IL: {"--stdlib", Stdlib});
202
203 for (llvm::StringRef Spec : Specs) {
204 Args.push_back(Elt: Spec);
205 }
206
207 return Args;
208 }
209
210 static DriverArgs getEmpty() { return {}; }
211
212private:
213 DriverArgs() = default;
214};
215} // namespace
216} // namespace clang::clangd
217namespace llvm {
218using DriverArgs = clang::clangd::DriverArgs;
219template <> struct DenseMapInfo<DriverArgs> {
220 static DriverArgs getEmptyKey() {
221 auto Driver = DriverArgs::getEmpty();
222 Driver.Driver = "EMPTY_KEY";
223 return Driver;
224 }
225 static DriverArgs getTombstoneKey() {
226 auto Driver = DriverArgs::getEmpty();
227 Driver.Driver = "TOMBSTONE_KEY";
228 return Driver;
229 }
230 static unsigned getHashValue(const DriverArgs &Val) {
231 unsigned FixedFieldsHash = llvm::hash_value(arg: std::tuple{
232 Val.Driver,
233 Val.StandardIncludes,
234 Val.StandardCXXIncludes,
235 Val.Lang,
236 Val.Sysroot,
237 Val.ISysroot,
238 Val.Target,
239 Val.Stdlib,
240 });
241
242 unsigned SpecsHash = llvm::hash_combine_range(R: Val.Specs);
243
244 return llvm::hash_combine(args: FixedFieldsHash, args: SpecsHash);
245 }
246 static bool isEqual(const DriverArgs &LHS, const DriverArgs &RHS) {
247 return LHS == RHS;
248 }
249};
250} // namespace llvm
251namespace clang::clangd {
252namespace {
253bool isValidTarget(llvm::StringRef Triple) {
254 std::shared_ptr<TargetOptions> TargetOpts(new TargetOptions);
255 TargetOpts->Triple = Triple.str();
256 DiagnosticOptions DiagOpts;
257 DiagnosticsEngine Diags(new DiagnosticIDs, DiagOpts,
258 new IgnoringDiagConsumer);
259 llvm::IntrusiveRefCntPtr<TargetInfo> Target =
260 TargetInfo::CreateTargetInfo(Diags, Opts&: *TargetOpts);
261 return bool(Target);
262}
263
264std::optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
265 DriverInfo Info;
266 const char SIS[] = "#include <...> search starts here:";
267 const char SIE[] = "End of search list.";
268 const char TS[] = "Target: ";
269 llvm::SmallVector<llvm::StringRef> Lines;
270 Output.split(A&: Lines, Separator: '\n', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
271
272 enum {
273 Initial, // Initial state: searching for target or includes list.
274 IncludesExtracting, // Includes extracting.
275 Done // Includes and target extraction done.
276 } State = Initial;
277 bool SeenIncludes = false;
278 bool SeenTarget = false;
279 for (auto *It = Lines.begin(); State != Done && It != Lines.end(); ++It) {
280 auto Line = *It;
281 switch (State) {
282 case Initial:
283 if (!SeenIncludes && Line.trim() == SIS) {
284 SeenIncludes = true;
285 State = IncludesExtracting;
286 } else if (!SeenTarget && Line.trim().starts_with(Prefix: TS)) {
287 SeenTarget = true;
288 llvm::StringRef TargetLine = Line.trim();
289 TargetLine.consume_front(Prefix: TS);
290 // Only detect targets that clang understands
291 if (!isValidTarget(Triple: TargetLine)) {
292 elog(Fmt: "System include extraction: invalid target \"{0}\", ignoring",
293 Vals&: TargetLine);
294 } else {
295 Info.Target = TargetLine.str();
296 vlog(Fmt: "System include extraction: target extracted: \"{0}\"",
297 Vals&: TargetLine);
298 }
299 }
300 break;
301 case IncludesExtracting:
302 if (Line.trim() == SIE) {
303 State = SeenTarget ? Done : Initial;
304 } else {
305 Info.SystemIncludes.push_back(x: Line.trim().str());
306 vlog(Fmt: "System include extraction: adding {0}", Vals&: Line);
307 }
308 break;
309 default:
310 llvm_unreachable("Impossible state of the driver output parser");
311 break;
312 }
313 }
314 if (!SeenIncludes) {
315 elog(Fmt: "System include extraction: start marker not found: {0}", Vals&: Output);
316 return std::nullopt;
317 }
318 if (State == IncludesExtracting) {
319 elog(Fmt: "System include extraction: end marker missing: {0}", Vals&: Output);
320 return std::nullopt;
321 }
322 return std::move(Info);
323}
324
325std::optional<std::string> run(llvm::ArrayRef<llvm::StringRef> Argv,
326 bool OutputIsStderr) {
327 llvm::SmallString<128> OutputPath;
328 if (auto EC = llvm::sys::fs::createTemporaryFile(Prefix: "system-includes", Suffix: "clangd",
329 ResultPath&: OutputPath)) {
330 elog(Fmt: "System include extraction: failed to create temporary file with "
331 "error {0}",
332 Vals: EC.message());
333 return std::nullopt;
334 }
335 auto CleanUp = llvm::make_scope_exit(
336 F: [&OutputPath]() { llvm::sys::fs::remove(path: OutputPath); });
337
338 std::optional<llvm::StringRef> Redirects[] = {{""}, {""}, {""}};
339 Redirects[OutputIsStderr ? 2 : 1] = OutputPath.str();
340
341 std::string ErrMsg;
342 if (int RC =
343 llvm::sys::ExecuteAndWait(Program: Argv.front(), Args: Argv, /*Env=*/std::nullopt,
344 Redirects, /*SecondsToWait=*/0,
345 /*MemoryLimit=*/0, ErrMsg: &ErrMsg)) {
346 elog(Fmt: "System include extraction: driver execution failed with return code: "
347 "{0} - '{1}'. Args: [{2}]",
348 Vals: llvm::to_string(Value: RC), Vals&: ErrMsg, Vals: printArgv(Args: Argv));
349 return std::nullopt;
350 }
351
352 auto BufOrError = llvm::MemoryBuffer::getFile(Filename: OutputPath);
353 if (!BufOrError) {
354 elog(Fmt: "System include extraction: failed to read {0} with error {1}",
355 Vals&: OutputPath, Vals: BufOrError.getError().message());
356 return std::nullopt;
357 }
358 return BufOrError.get().get()->getBuffer().str();
359}
360
361std::optional<DriverInfo>
362extractSystemIncludesAndTarget(const DriverArgs &InputArgs,
363 const llvm::Regex &QueryDriverRegex) {
364 trace::Span Tracer("Extract system includes and target");
365
366 std::string Driver = InputArgs.Driver;
367 if (!llvm::sys::path::is_absolute(path: Driver)) {
368 auto DriverProgram = llvm::sys::findProgramByName(Name: Driver);
369 if (DriverProgram) {
370 vlog(Fmt: "System include extraction: driver {0} expanded to {1}", Vals&: Driver,
371 Vals&: *DriverProgram);
372 Driver = *DriverProgram;
373 } else {
374 elog(Fmt: "System include extraction: driver {0} not found in PATH", Vals&: Driver);
375 return std::nullopt;
376 }
377 }
378
379 SPAN_ATTACH(Tracer, "driver", Driver);
380 SPAN_ATTACH(Tracer, "lang", InputArgs.Lang);
381
382 // If driver was "../foo" then having to allowlist "/path/a/../foo" rather
383 // than "/path/foo" is absurd.
384 // Allow either to match the allowlist, then proceed with "/path/a/../foo".
385 // This was our historical behavior, and it *could* resolve to something else.
386 llvm::SmallString<256> NoDots(Driver);
387 llvm::sys::path::remove_dots(path&: NoDots, /*remove_dot_dot=*/true);
388 if (!QueryDriverRegex.match(String: Driver) && !QueryDriverRegex.match(String: NoDots)) {
389 vlog(Fmt: "System include extraction: not allowed driver {0}", Vals&: Driver);
390 return std::nullopt;
391 }
392
393 llvm::SmallVector<llvm::StringRef> Args = {Driver, "-E", "-v"};
394 Args.append(RHS: InputArgs.render());
395 // Input needs to go after Lang flags.
396 Args.push_back(Elt: "-");
397 auto Output = run(Argv: Args, /*OutputIsStderr=*/true);
398 if (!Output)
399 return std::nullopt;
400
401 std::optional<DriverInfo> Info = parseDriverOutput(Output: *Output);
402 if (!Info)
403 return std::nullopt;
404
405 switch (Config::current().CompileFlags.BuiltinHeaders) {
406 case Config::BuiltinHeaderPolicy::Clangd: {
407 // The built-in headers are tightly coupled to parser builtins.
408 // (These are clang's "resource dir", GCC's GCC_INCLUDE_DIR.)
409 // We should keep using clangd's versions, so exclude the queried
410 // builtins. They're not specially marked in the -v output, but we can
411 // get the path with `$DRIVER -print-file-name=include`.
412 if (auto BuiltinHeaders = run(Argv: {Driver, "-print-file-name=include"},
413 /*OutputIsStderr=*/false)) {
414 auto Path = llvm::StringRef(*BuiltinHeaders).trim();
415 if (!Path.empty() && llvm::sys::path::is_absolute(path: Path)) {
416 auto Size = Info->SystemIncludes.size();
417 llvm::erase(C&: Info->SystemIncludes, V: Path);
418 vlog(Fmt: "System includes extractor: builtin headers {0} {1}", Vals&: Path,
419 Vals: (Info->SystemIncludes.size() != Size)
420 ? "excluded"
421 : "not found in driver's response");
422 }
423 }
424 break;
425 }
426 case Config::BuiltinHeaderPolicy::QueryDriver:
427 vlog(Fmt: "System includes extractor: Using builtin headers from query driver.");
428 break;
429 }
430
431 log(Fmt: "System includes extractor: successfully executed {0}\n\tgot includes: "
432 "\"{1}\"\n\tgot target: \"{2}\"",
433 Vals&: Driver, Vals: llvm::join(R&: Info->SystemIncludes, Separator: ", "), Vals&: Info->Target);
434 return Info;
435}
436
437tooling::CompileCommand &
438addSystemIncludes(tooling::CompileCommand &Cmd,
439 llvm::ArrayRef<std::string> SystemIncludes) {
440 std::vector<std::string> ToAppend;
441 for (llvm::StringRef Include : SystemIncludes) {
442 // FIXME(kadircet): This doesn't work when we have "--driver-mode=cl"
443 ToAppend.push_back(x: "-isystem");
444 ToAppend.push_back(x: Include.str());
445 }
446 if (!ToAppend.empty()) {
447 // Just append when `--` isn't present.
448 auto InsertAt = llvm::find(Range&: Cmd.CommandLine, Val: "--");
449 Cmd.CommandLine.insert(position: InsertAt, first: std::make_move_iterator(i: ToAppend.begin()),
450 last: std::make_move_iterator(i: ToAppend.end()));
451 }
452 return Cmd;
453}
454
455tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
456 const std::string &Target) {
457 if (!Target.empty()) {
458 // We do not want to override existing target with extracted one.
459 for (llvm::StringRef Arg : Cmd.CommandLine) {
460 if (Arg == "-target" || Arg.starts_with(Prefix: "--target="))
461 return Cmd;
462 }
463 // Just append when `--` isn't present.
464 auto InsertAt = llvm::find(Range&: Cmd.CommandLine, Val: "--");
465 Cmd.CommandLine.insert(position: InsertAt, x: "--target=" + Target);
466 }
467 return Cmd;
468}
469
470/// Converts a glob containing only ** or * into a regex.
471std::string convertGlobToRegex(llvm::StringRef Glob) {
472 std::string RegText;
473 llvm::raw_string_ostream RegStream(RegText);
474 RegStream << '^';
475 for (size_t I = 0, E = Glob.size(); I < E; ++I) {
476 if (Glob[I] == '*') {
477 if (I + 1 < E && Glob[I + 1] == '*') {
478 // Double star, accept any sequence.
479 RegStream << ".*";
480 // Also skip the second star.
481 ++I;
482 } else {
483 // Single star, accept any sequence without a slash.
484 RegStream << "[^/]*";
485 }
486 } else if (llvm::sys::path::is_separator(value: Glob[I]) &&
487 llvm::sys::path::is_separator(value: '/') &&
488 llvm::sys::path::is_separator(value: '\\')) {
489 RegStream << R"([/\\])"; // Accept either slash on windows.
490 } else {
491 RegStream << llvm::Regex::escape(String: Glob.substr(Start: I, N: 1));
492 }
493 }
494 RegStream << '$';
495 return RegText;
496}
497
498/// Converts a glob containing only ** or * into a regex.
499llvm::Regex convertGlobsToRegex(llvm::ArrayRef<std::string> Globs) {
500 assert(!Globs.empty() && "Globs cannot be empty!");
501 std::vector<std::string> RegTexts;
502 RegTexts.reserve(n: Globs.size());
503 for (llvm::StringRef Glob : Globs)
504 RegTexts.push_back(x: convertGlobToRegex(Glob));
505
506 // Tempting to pass IgnoreCase, but we don't know the FS sensitivity.
507 llvm::Regex Reg(llvm::join(R&: RegTexts, Separator: "|"));
508 assert(Reg.isValid(RegTexts.front()) &&
509 "Created an invalid regex from globs");
510 return Reg;
511}
512
513/// Extracts system includes from a trusted driver by parsing the output of
514/// include search path and appends them to the commands coming from underlying
515/// compilation database.
516class SystemIncludeExtractor {
517public:
518 SystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs)
519 : QueryDriverRegex(convertGlobsToRegex(Globs: QueryDriverGlobs)) {}
520
521 void operator()(tooling::CompileCommand &Cmd, llvm::StringRef File) const {
522 if (Cmd.CommandLine.empty())
523 return;
524
525 DriverArgs Args(Cmd, File);
526 if (Args.Lang.empty())
527 return;
528 if (auto Info = QueriedDrivers.get(Key&: Args, Compute: [&] {
529 return extractSystemIncludesAndTarget(InputArgs: Args, QueryDriverRegex);
530 })) {
531 setTarget(Cmd&: addSystemIncludes(Cmd, SystemIncludes: Info->SystemIncludes), Target: Info->Target);
532 }
533 }
534
535private:
536 // Caches includes extracted from a driver. Key is driver:lang.
537 Memoize<llvm::DenseMap<DriverArgs, std::optional<DriverInfo>>> QueriedDrivers;
538 llvm::Regex QueryDriverRegex;
539};
540} // namespace
541
542SystemIncludeExtractorFn
543getSystemIncludeExtractor(llvm::ArrayRef<std::string> QueryDriverGlobs) {
544 if (QueryDriverGlobs.empty())
545 return nullptr;
546 return SystemIncludeExtractor(QueryDriverGlobs);
547}
548
549} // namespace clang::clangd
550

source code of clang-tools-extra/clangd/SystemIncludeExtractor.cpp