1//===--- extra/modularize/ModularizeUtilities.cpp -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a class for loading and validating a module map or
10// header list by checking that all headers in the corresponding directories
11// are accounted for.
12//
13//===----------------------------------------------------------------------===//
14
15#include "clang/Basic/SourceManager.h"
16#include "clang/Driver/Options.h"
17#include "clang/Frontend/CompilerInstance.h"
18#include "clang/Frontend/FrontendActions.h"
19#include "CoverageChecker.h"
20#include "llvm/ADT/SmallString.h"
21#include "llvm/Support/FileUtilities.h"
22#include "llvm/Support/MemoryBuffer.h"
23#include "llvm/Support/Path.h"
24#include "llvm/Support/raw_ostream.h"
25#include "ModularizeUtilities.h"
26
27using namespace clang;
28using namespace llvm;
29using namespace Modularize;
30
31namespace {
32// Subclass TargetOptions so we can construct it inline with
33// the minimal option, the triple.
34class ModuleMapTargetOptions : public clang::TargetOptions {
35public:
36 ModuleMapTargetOptions() { Triple = llvm::sys::getDefaultTargetTriple(); }
37};
38} // namespace
39
40// ModularizeUtilities class implementation.
41
42// Constructor.
43ModularizeUtilities::ModularizeUtilities(std::vector<std::string> &InputPaths,
44 llvm::StringRef Prefix,
45 llvm::StringRef ProblemFilesListPath)
46 : InputFilePaths(InputPaths), HeaderPrefix(Prefix),
47 ProblemFilesPath(ProblemFilesListPath), HasModuleMap(false),
48 MissingHeaderCount(0),
49 // Init clang stuff needed for loading the module map and preprocessing.
50 LangOpts(new LangOptions()), DiagIDs(new DiagnosticIDs()),
51 DC(llvm::errs(), DiagnosticOpts),
52 Diagnostics(new DiagnosticsEngine(DiagIDs, DiagnosticOpts, &DC, false)),
53 TargetOpts(new ModuleMapTargetOptions()),
54 Target(TargetInfo::CreateTargetInfo(Diags&: *Diagnostics, Opts&: *TargetOpts)),
55 FileMgr(new FileManager(FileSystemOpts)),
56 SourceMgr(new SourceManager(*Diagnostics, *FileMgr, false)), HSOpts(),
57 HeaderInfo(new HeaderSearch(HSOpts, *SourceMgr, *Diagnostics, *LangOpts,
58 Target.get())) {}
59
60// Create instance of ModularizeUtilities, to simplify setting up
61// subordinate objects.
62ModularizeUtilities *ModularizeUtilities::createModularizeUtilities(
63 std::vector<std::string> &InputPaths, llvm::StringRef Prefix,
64 llvm::StringRef ProblemFilesListPath) {
65
66 return new ModularizeUtilities(InputPaths, Prefix, ProblemFilesListPath);
67}
68
69// Load all header lists and dependencies.
70std::error_code ModularizeUtilities::loadAllHeaderListsAndDependencies() {
71 // For each input file.
72 for (auto I = InputFilePaths.begin(), E = InputFilePaths.end(); I != E; ++I) {
73 llvm::StringRef InputPath = *I;
74 // If it's a module map.
75 if (InputPath.ends_with(Suffix: ".modulemap")) {
76 // Load the module map.
77 if (std::error_code EC = loadModuleMap(InputPath))
78 return EC;
79 } else {
80 // Else we assume it's a header list and load it.
81 if (std::error_code EC = loadSingleHeaderListsAndDependencies(InputPath)) {
82 errs() << "modularize: error: Unable to get header list '" << InputPath
83 << "': " << EC.message() << '\n';
84 return EC;
85 }
86 }
87 }
88 // If we have a problem files list.
89 if (ProblemFilesPath.size() != 0) {
90 // Load problem files list.
91 if (std::error_code EC = loadProblemHeaderList(InputPath: ProblemFilesPath)) {
92 errs() << "modularize: error: Unable to get problem header list '" << ProblemFilesPath
93 << "': " << EC.message() << '\n';
94 return EC;
95 }
96 }
97 return std::error_code();
98}
99
100// Do coverage checks.
101// For each loaded module map, do header coverage check.
102// Starting from the directory of the module.modulemap file,
103// Find all header files, optionally looking only at files
104// covered by the include path options, and compare against
105// the headers referenced by the module.modulemap file.
106// Display warnings for unaccounted-for header files.
107// Returns 0 if there were no errors or warnings, 1 if there
108// were warnings, 2 if any other problem, such as a bad
109// module map path argument was specified.
110std::error_code ModularizeUtilities::doCoverageCheck(
111 std::vector<std::string> &IncludePaths,
112 llvm::ArrayRef<std::string> CommandLine) {
113 int ModuleMapCount = ModuleMaps.size();
114 int ModuleMapIndex;
115 std::error_code EC;
116 for (ModuleMapIndex = 0; ModuleMapIndex < ModuleMapCount; ++ModuleMapIndex) {
117 std::unique_ptr<clang::ModuleMap> &ModMap = ModuleMaps[ModuleMapIndex];
118 auto Checker = CoverageChecker::createCoverageChecker(
119 ModuleMapPath: InputFilePaths[ModuleMapIndex], IncludePaths, CommandLine,
120 ModuleMap: ModMap.get());
121 std::error_code LocalEC = Checker->doChecks();
122 if (LocalEC.value() > 0)
123 EC = LocalEC;
124 }
125 return EC;
126}
127
128// Load single header list and dependencies.
129std::error_code ModularizeUtilities::loadSingleHeaderListsAndDependencies(
130 llvm::StringRef InputPath) {
131
132 // By default, use the path component of the list file name.
133 SmallString<256> HeaderDirectory(InputPath);
134 llvm::sys::path::remove_filename(path&: HeaderDirectory);
135 SmallString<256> CurrentDirectory;
136 llvm::sys::fs::current_path(result&: CurrentDirectory);
137
138 // Get the prefix if we have one.
139 if (HeaderPrefix.size() != 0)
140 HeaderDirectory = HeaderPrefix;
141
142 // Read the header list file into a buffer.
143 ErrorOr<std::unique_ptr<MemoryBuffer>> listBuffer =
144 MemoryBuffer::getFile(Filename: InputPath);
145 if (std::error_code EC = listBuffer.getError())
146 return EC;
147
148 // Parse the header list into strings.
149 SmallVector<StringRef, 32> Strings;
150 listBuffer.get()->getBuffer().split(A&: Strings, Separator: "\n", MaxSplit: -1, KeepEmpty: false);
151
152 // Collect the header file names from the string list.
153 for (SmallVectorImpl<StringRef>::iterator I = Strings.begin(),
154 E = Strings.end();
155 I != E; ++I) {
156 StringRef Line = I->trim();
157 // Ignore comments and empty lines.
158 if (Line.empty() || (Line[0] == '#'))
159 continue;
160 std::pair<StringRef, StringRef> TargetAndDependents = Line.split(Separator: ':');
161 SmallString<256> HeaderFileName;
162 // Prepend header file name prefix if it's not absolute.
163 if (llvm::sys::path::is_absolute(path: TargetAndDependents.first))
164 llvm::sys::path::native(path: TargetAndDependents.first, result&: HeaderFileName);
165 else {
166 if (HeaderDirectory.size() != 0)
167 HeaderFileName = HeaderDirectory;
168 else
169 HeaderFileName = CurrentDirectory;
170 llvm::sys::path::append(path&: HeaderFileName, a: TargetAndDependents.first);
171 llvm::sys::path::native(path&: HeaderFileName);
172 }
173 // Handle optional dependencies.
174 DependentsVector Dependents;
175 SmallVector<StringRef, 4> DependentsList;
176 TargetAndDependents.second.split(A&: DependentsList, Separator: " ", MaxSplit: -1, KeepEmpty: false);
177 int Count = DependentsList.size();
178 for (int Index = 0; Index < Count; ++Index) {
179 SmallString<256> Dependent;
180 if (llvm::sys::path::is_absolute(path: DependentsList[Index]))
181 Dependent = DependentsList[Index];
182 else {
183 if (HeaderDirectory.size() != 0)
184 Dependent = HeaderDirectory;
185 else
186 Dependent = CurrentDirectory;
187 llvm::sys::path::append(path&: Dependent, a: DependentsList[Index]);
188 }
189 llvm::sys::path::native(path&: Dependent);
190 Dependents.push_back(Elt: getCanonicalPath(FilePath: Dependent.str()));
191 }
192 // Get canonical form.
193 HeaderFileName = getCanonicalPath(FilePath: HeaderFileName);
194 // Save the resulting header file path and dependencies.
195 HeaderFileNames.push_back(Elt: std::string(HeaderFileName));
196 Dependencies[HeaderFileName.str()] = Dependents;
197 }
198 return std::error_code();
199}
200
201// Load problem header list.
202std::error_code ModularizeUtilities::loadProblemHeaderList(
203 llvm::StringRef InputPath) {
204
205 // By default, use the path component of the list file name.
206 SmallString<256> HeaderDirectory(InputPath);
207 llvm::sys::path::remove_filename(path&: HeaderDirectory);
208 SmallString<256> CurrentDirectory;
209 llvm::sys::fs::current_path(result&: CurrentDirectory);
210
211 // Get the prefix if we have one.
212 if (HeaderPrefix.size() != 0)
213 HeaderDirectory = HeaderPrefix;
214
215 // Read the header list file into a buffer.
216 ErrorOr<std::unique_ptr<MemoryBuffer>> listBuffer =
217 MemoryBuffer::getFile(Filename: InputPath);
218 if (std::error_code EC = listBuffer.getError())
219 return EC;
220
221 // Parse the header list into strings.
222 SmallVector<StringRef, 32> Strings;
223 listBuffer.get()->getBuffer().split(A&: Strings, Separator: "\n", MaxSplit: -1, KeepEmpty: false);
224
225 // Collect the header file names from the string list.
226 for (SmallVectorImpl<StringRef>::iterator I = Strings.begin(),
227 E = Strings.end();
228 I != E; ++I) {
229 StringRef Line = I->trim();
230 // Ignore comments and empty lines.
231 if (Line.empty() || (Line[0] == '#'))
232 continue;
233 SmallString<256> HeaderFileName;
234 // Prepend header file name prefix if it's not absolute.
235 if (llvm::sys::path::is_absolute(path: Line))
236 llvm::sys::path::native(path: Line, result&: HeaderFileName);
237 else {
238 if (HeaderDirectory.size() != 0)
239 HeaderFileName = HeaderDirectory;
240 else
241 HeaderFileName = CurrentDirectory;
242 llvm::sys::path::append(path&: HeaderFileName, a: Line);
243 llvm::sys::path::native(path&: HeaderFileName);
244 }
245 // Get canonical form.
246 HeaderFileName = getCanonicalPath(FilePath: HeaderFileName);
247 // Save the resulting header file path.
248 ProblemFileNames.push_back(Elt: std::string(HeaderFileName));
249 }
250 return std::error_code();
251}
252
253// Load single module map and extract header file list.
254std::error_code ModularizeUtilities::loadModuleMap(
255 llvm::StringRef InputPath) {
256 // Get file entry for module.modulemap file.
257 auto ModuleMapEntryOrErr = SourceMgr->getFileManager().getFileRef(Filename: InputPath);
258
259 // return error if not found.
260 if (!ModuleMapEntryOrErr) {
261 llvm::errs() << "error: File \"" << InputPath << "\" not found.\n";
262 return errorToErrorCode(Err: ModuleMapEntryOrErr.takeError());
263 }
264 FileEntryRef ModuleMapEntry = *ModuleMapEntryOrErr;
265
266 // Because the module map parser uses a ForwardingDiagnosticConsumer,
267 // which doesn't forward the BeginSourceFile call, we do it explicitly here.
268 DC.BeginSourceFile(LO: *LangOpts, PP: nullptr);
269
270 // Figure out the home directory for the module map file.
271 DirectoryEntryRef Dir = ModuleMapEntry.getDir();
272 StringRef DirName(Dir.getName());
273 if (llvm::sys::path::filename(path: DirName) == "Modules") {
274 DirName = llvm::sys::path::parent_path(path: DirName);
275 if (DirName.ends_with(Suffix: ".framework")) {
276 auto FrameworkDirOrErr = FileMgr->getDirectoryRef(DirName);
277 if (!FrameworkDirOrErr) {
278 // This can happen if there's a race between the above check and the
279 // removal of the directory.
280 return errorToErrorCode(Err: FrameworkDirOrErr.takeError());
281 }
282 Dir = *FrameworkDirOrErr;
283 }
284 }
285
286 std::unique_ptr<ModuleMap> ModMap;
287 ModMap.reset(p: new ModuleMap(*SourceMgr, *Diagnostics, *LangOpts,
288 Target.get(), *HeaderInfo));
289
290 // Parse module.modulemap file into module map.
291 if (ModMap->parseAndLoadModuleMapFile(File: ModuleMapEntry, IsSystem: false, HomeDir: Dir)) {
292 return std::error_code(1, std::generic_category());
293 }
294
295 // Do matching end call.
296 DC.EndSourceFile();
297
298 // Reset missing header count.
299 MissingHeaderCount = 0;
300
301 if (!collectModuleMapHeaders(ModMap: ModMap.get()))
302 return std::error_code(1, std::generic_category());
303
304 // Save module map.
305 ModuleMaps.push_back(x: std::move(ModMap));
306
307 // Indicate we are using module maps.
308 HasModuleMap = true;
309
310 // Return code of 1 for missing headers.
311 if (MissingHeaderCount)
312 return std::error_code(1, std::generic_category());
313
314 return std::error_code();
315}
316
317// Collect module map headers.
318// Walks the modules and collects referenced headers into
319// HeaderFileNames.
320bool ModularizeUtilities::collectModuleMapHeaders(clang::ModuleMap *ModMap) {
321 SmallVector<std::pair<StringRef, const clang::Module *>, 0> Vec;
322 for (auto &M : ModMap->modules())
323 Vec.emplace_back(Args: M.first(), Args: M.second);
324 llvm::sort(C&: Vec, Comp: llvm::less_first());
325 for (auto &I : Vec)
326 if (!collectModuleHeaders(Mod: *I.second))
327 return false;
328 return true;
329}
330
331// Collect referenced headers from one module.
332// Collects the headers referenced in the given module into
333// HeaderFileNames.
334bool ModularizeUtilities::collectModuleHeaders(const clang::Module &Mod) {
335
336 // Ignore explicit modules because they often have dependencies
337 // we can't know.
338 if (Mod.IsExplicit)
339 return true;
340
341 // Treat headers in umbrella directory as dependencies.
342 DependentsVector UmbrellaDependents;
343
344 // Recursively do submodules.
345 for (auto *Submodule : Mod.submodules())
346 collectModuleHeaders(Mod: *Submodule);
347
348 if (std::optional<clang::Module::Header> UmbrellaHeader =
349 Mod.getUmbrellaHeaderAsWritten()) {
350 std::string HeaderPath = getCanonicalPath(FilePath: UmbrellaHeader->Entry.getName());
351 // Collect umbrella header.
352 HeaderFileNames.push_back(Elt: HeaderPath);
353
354 // FUTURE: When needed, umbrella header header collection goes here.
355 } else if (std::optional<clang::Module::DirectoryName> UmbrellaDir =
356 Mod.getUmbrellaDirAsWritten()) {
357 // If there normal headers, assume these are umbrellas and skip collection.
358 if (Mod.getHeaders(HK: Module::HK_Normal).empty()) {
359 // Collect headers in umbrella directory.
360 if (!collectUmbrellaHeaders(UmbrellaDirName: UmbrellaDir->Entry.getName(),
361 Dependents&: UmbrellaDependents))
362 return false;
363 }
364 }
365
366 // We ignore HK_Private, HK_Textual, HK_PrivateTextual, and HK_Excluded,
367 // assuming they are marked as such either because of unsuitability for
368 // modules or because they are meant to be included by another header,
369 // and thus should be ignored by modularize.
370
371 for (const auto &Header : Mod.getHeaders(HK: clang::Module::HK_Normal))
372 HeaderFileNames.push_back(Elt: getCanonicalPath(FilePath: Header.Entry.getName()));
373
374 int MissingCountThisModule = Mod.MissingHeaders.size();
375
376 for (int Index = 0; Index < MissingCountThisModule; ++Index) {
377 std::string MissingFile = Mod.MissingHeaders[Index].FileName;
378 SourceLocation Loc = Mod.MissingHeaders[Index].FileNameLoc;
379 errs() << Loc.printToString(SM: *SourceMgr)
380 << ": error : Header not found: " << MissingFile << "\n";
381 }
382
383 MissingHeaderCount += MissingCountThisModule;
384
385 return true;
386}
387
388// Collect headers from an umbrella directory.
389bool ModularizeUtilities::collectUmbrellaHeaders(StringRef UmbrellaDirName,
390 DependentsVector &Dependents) {
391 // Initialize directory name.
392 SmallString<256> Directory(UmbrellaDirName);
393 // Walk the directory.
394 std::error_code EC;
395 for (llvm::sys::fs::directory_iterator I(Directory.str(), EC), E; I != E;
396 I.increment(ec&: EC)) {
397 if (EC)
398 return false;
399 std::string File(I->path());
400 llvm::ErrorOr<llvm::sys::fs::basic_file_status> Status = I->status();
401 if (!Status)
402 return false;
403 llvm::sys::fs::file_type Type = Status->type();
404 // If the file is a directory, ignore the name and recurse.
405 if (Type == llvm::sys::fs::file_type::directory_file) {
406 if (!collectUmbrellaHeaders(UmbrellaDirName: File, Dependents))
407 return false;
408 continue;
409 }
410 // If the file does not have a common header extension, ignore it.
411 if (!isHeader(FileName: File))
412 continue;
413 // Save header name.
414 std::string HeaderPath = getCanonicalPath(FilePath: File);
415 Dependents.push_back(Elt: HeaderPath);
416 }
417 return true;
418}
419
420// Replace .. embedded in path for purposes of having
421// a canonical path.
422static std::string replaceDotDot(StringRef Path) {
423 SmallString<128> Buffer;
424 llvm::sys::path::const_iterator B = llvm::sys::path::begin(path: Path),
425 E = llvm::sys::path::end(path: Path);
426 while (B != E) {
427 if (*B == "..")
428 llvm::sys::path::remove_filename(path&: Buffer);
429 else if (*B != ".")
430 llvm::sys::path::append(path&: Buffer, a: *B);
431 ++B;
432 }
433 if (Path.ends_with(Suffix: "/") || Path.ends_with(Suffix: "\\"))
434 Buffer.append(NumInputs: 1, Elt: Path.back());
435 return Buffer.c_str();
436}
437
438// Convert header path to canonical form.
439// The canonical form is basically just use forward slashes, and remove "./".
440// \param FilePath The file path, relative to the module map directory.
441// \returns The file path in canonical form.
442std::string ModularizeUtilities::getCanonicalPath(StringRef FilePath) {
443 std::string Tmp(replaceDotDot(Path: FilePath));
444 llvm::replace(Range&: Tmp, OldValue: '\\', NewValue: '/');
445 StringRef Tmp2(Tmp);
446 if (Tmp2.starts_with(Prefix: "./"))
447 Tmp = std::string(Tmp2.substr(Start: 2));
448 return Tmp;
449}
450
451// Check for header file extension.
452// If the file extension is .h, .inc, or missing, it's
453// assumed to be a header.
454// \param FileName The file name. Must not be a directory.
455// \returns true if it has a header extension or no extension.
456bool ModularizeUtilities::isHeader(StringRef FileName) {
457 StringRef Extension = llvm::sys::path::extension(path: FileName);
458 if (Extension.size() == 0)
459 return true;
460 if (Extension.equals_insensitive(RHS: ".h"))
461 return true;
462 if (Extension.equals_insensitive(RHS: ".inc"))
463 return true;
464 return false;
465}
466
467// Get directory path component from file path.
468// \returns the component of the given path, which will be
469// relative if the given path is relative, absolute if the
470// given path is absolute, or "." if the path has no leading
471// path component.
472std::string ModularizeUtilities::getDirectoryFromPath(StringRef Path) {
473 SmallString<256> Directory(Path);
474 sys::path::remove_filename(path&: Directory);
475 if (Directory.size() == 0)
476 return ".";
477 return std::string(Directory);
478}
479
480// Add unique problem file.
481// Also standardizes the path.
482void ModularizeUtilities::addUniqueProblemFile(std::string FilePath) {
483 FilePath = getCanonicalPath(FilePath);
484 // Don't add if already present.
485 for(auto &TestFilePath : ProblemFileNames) {
486 if (TestFilePath == FilePath)
487 return;
488 }
489 ProblemFileNames.push_back(Elt: FilePath);
490}
491
492// Add file with no compile errors.
493// Also standardizes the path.
494void ModularizeUtilities::addNoCompileErrorsFile(std::string FilePath) {
495 FilePath = getCanonicalPath(FilePath);
496 GoodFileNames.push_back(Elt: FilePath);
497}
498
499// List problem files.
500void ModularizeUtilities::displayProblemFiles() {
501 errs() << "\nThese are the files with possible errors:\n\n";
502 for (auto &ProblemFile : ProblemFileNames) {
503 errs() << ProblemFile << "\n";
504 }
505}
506
507// List files with no problems.
508void ModularizeUtilities::displayGoodFiles() {
509 errs() << "\nThese are the files with no detected errors:\n\n";
510 for (auto &GoodFile : HeaderFileNames) {
511 bool Good = true;
512 for (auto &ProblemFile : ProblemFileNames) {
513 if (ProblemFile == GoodFile) {
514 Good = false;
515 break;
516 }
517 }
518 if (Good)
519 errs() << GoodFile << "\n";
520 }
521}
522
523// List files with problem files commented out.
524void ModularizeUtilities::displayCombinedFiles() {
525 errs() <<
526 "\nThese are the combined files, with problem files preceded by #:\n\n";
527 for (auto &File : HeaderFileNames) {
528 bool Good = true;
529 for (auto &ProblemFile : ProblemFileNames) {
530 if (ProblemFile == File) {
531 Good = false;
532 break;
533 }
534 }
535 errs() << (Good ? "" : "#") << File << "\n";
536 }
537}
538

source code of clang-tools-extra/modularize/ModularizeUtilities.cpp