1//===----------------- ModulesBuilder.cpp ------------------------*- C++-*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ModulesBuilder.h"
10#include "Compiler.h"
11#include "support/Logger.h"
12#include "clang/Frontend/FrontendAction.h"
13#include "clang/Frontend/FrontendActions.h"
14#include "clang/Serialization/ASTReader.h"
15#include "clang/Serialization/ModuleCache.h"
16#include "llvm/ADT/ScopeExit.h"
17#include <queue>
18
19namespace clang {
20namespace clangd {
21
22namespace {
23
24// Create a path to store module files. Generally it should be:
25//
26// {TEMP_DIRS}/clangd/module_files/{hashed-file-name}-%%-%%-%%-%%-%%-%%/.
27//
28// {TEMP_DIRS} is the temporary directory for the system, e.g., "/var/tmp"
29// or "C:/TEMP".
30//
31// '%%' means random value to make the generated path unique.
32//
33// \param MainFile is used to get the root of the project from global
34// compilation database.
35//
36// TODO: Move these module fils out of the temporary directory if the module
37// files are persistent.
38llvm::SmallString<256> getUniqueModuleFilesPath(PathRef MainFile) {
39 llvm::SmallString<128> HashedPrefix = llvm::sys::path::filename(path: MainFile);
40 // There might be multiple files with the same name in a project. So appending
41 // the hash value of the full path to make sure they won't conflict.
42 HashedPrefix += std::to_string(val: llvm::hash_value(S: MainFile));
43
44 llvm::SmallString<256> ResultPattern;
45
46 llvm::sys::path::system_temp_directory(/*erasedOnReboot=*/true,
47 result&: ResultPattern);
48
49 llvm::sys::path::append(path&: ResultPattern, a: "clangd");
50 llvm::sys::path::append(path&: ResultPattern, a: "module_files");
51
52 llvm::sys::path::append(path&: ResultPattern, a: HashedPrefix);
53
54 ResultPattern.append(RHS: "-%%-%%-%%-%%-%%-%%");
55
56 llvm::SmallString<256> Result;
57 llvm::sys::fs::createUniquePath(Model: ResultPattern, ResultPath&: Result,
58 /*MakeAbsolute=*/false);
59
60 llvm::sys::fs::create_directories(path: Result);
61 return Result;
62}
63
64// Get a unique module file path under \param ModuleFilesPrefix.
65std::string getModuleFilePath(llvm::StringRef ModuleName,
66 PathRef ModuleFilesPrefix) {
67 llvm::SmallString<256> ModuleFilePath(ModuleFilesPrefix);
68 auto [PrimaryModuleName, PartitionName] = ModuleName.split(Separator: ':');
69 llvm::sys::path::append(path&: ModuleFilePath, a: PrimaryModuleName);
70 if (!PartitionName.empty()) {
71 ModuleFilePath.append(RHS: "-");
72 ModuleFilePath.append(RHS: PartitionName);
73 }
74
75 ModuleFilePath.append(RHS: ".pcm");
76 return std::string(ModuleFilePath);
77}
78
79// FailedPrerequisiteModules - stands for the PrerequisiteModules which has
80// errors happened during the building process.
81class FailedPrerequisiteModules : public PrerequisiteModules {
82public:
83 ~FailedPrerequisiteModules() override = default;
84
85 // We shouldn't adjust the compilation commands based on
86 // FailedPrerequisiteModules.
87 void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
88 }
89
90 // FailedPrerequisiteModules can never be reused.
91 bool
92 canReuse(const CompilerInvocation &CI,
93 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override {
94 return false;
95 }
96};
97
98struct ModuleFile {
99 ModuleFile(StringRef ModuleName, PathRef ModuleFilePath)
100 : ModuleName(ModuleName.str()), ModuleFilePath(ModuleFilePath.str()) {}
101
102 ModuleFile() = delete;
103
104 ModuleFile(const ModuleFile &) = delete;
105 ModuleFile operator=(const ModuleFile &) = delete;
106
107 // The move constructor is needed for llvm::SmallVector.
108 ModuleFile(ModuleFile &&Other)
109 : ModuleName(std::move(Other.ModuleName)),
110 ModuleFilePath(std::move(Other.ModuleFilePath)) {
111 Other.ModuleName.clear();
112 Other.ModuleFilePath.clear();
113 }
114
115 ModuleFile &operator=(ModuleFile &&Other) {
116 if (this == &Other)
117 return *this;
118
119 this->~ModuleFile();
120 new (this) ModuleFile(std::move(Other));
121 return *this;
122 }
123
124 ~ModuleFile() {
125 if (!ModuleFilePath.empty())
126 llvm::sys::fs::remove(path: ModuleFilePath);
127 }
128
129 StringRef getModuleName() const { return ModuleName; }
130
131 StringRef getModuleFilePath() const { return ModuleFilePath; }
132
133private:
134 std::string ModuleName;
135 std::string ModuleFilePath;
136};
137
138// ReusablePrerequisiteModules - stands for PrerequisiteModules for which all
139// the required modules are built successfully. All the module files
140// are owned by the modules builder.
141class ReusablePrerequisiteModules : public PrerequisiteModules {
142public:
143 ReusablePrerequisiteModules() = default;
144
145 ReusablePrerequisiteModules(const ReusablePrerequisiteModules &Other) =
146 default;
147 ReusablePrerequisiteModules &
148 operator=(const ReusablePrerequisiteModules &) = default;
149 ReusablePrerequisiteModules(ReusablePrerequisiteModules &&) = delete;
150 ReusablePrerequisiteModules
151 operator=(ReusablePrerequisiteModules &&) = delete;
152
153 ~ReusablePrerequisiteModules() override = default;
154
155 void adjustHeaderSearchOptions(HeaderSearchOptions &Options) const override {
156 // Appending all built module files.
157 for (const auto &RequiredModule : RequiredModules)
158 Options.PrebuiltModuleFiles.insert_or_assign(
159 k: RequiredModule->getModuleName().str(),
160 obj: RequiredModule->getModuleFilePath().str());
161 }
162
163 bool canReuse(const CompilerInvocation &CI,
164 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>) const override;
165
166 bool isModuleUnitBuilt(llvm::StringRef ModuleName) const {
167 return BuiltModuleNames.contains(key: ModuleName);
168 }
169
170 void addModuleFile(std::shared_ptr<const ModuleFile> ModuleFile) {
171 BuiltModuleNames.insert(key: ModuleFile->getModuleName());
172 RequiredModules.emplace_back(Args: std::move(ModuleFile));
173 }
174
175private:
176 llvm::SmallVector<std::shared_ptr<const ModuleFile>, 8> RequiredModules;
177 // A helper class to speedup the query if a module is built.
178 llvm::StringSet<> BuiltModuleNames;
179};
180
181bool IsModuleFileUpToDate(PathRef ModuleFilePath,
182 const PrerequisiteModules &RequisiteModules,
183 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
184 HeaderSearchOptions HSOpts;
185 RequisiteModules.adjustHeaderSearchOptions(Options&: HSOpts);
186 HSOpts.ForceCheckCXX20ModulesInputFiles = true;
187 HSOpts.ValidateASTInputFilesContent = true;
188
189 clang::clangd::IgnoreDiagnostics IgnoreDiags;
190 DiagnosticOptions DiagOpts;
191 IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
192 CompilerInstance::createDiagnostics(VFS&: *VFS, Opts&: DiagOpts, Client: &IgnoreDiags,
193 /*ShouldOwnClient=*/false);
194
195 LangOptions LangOpts;
196 LangOpts.SkipODRCheckInGMF = true;
197
198 FileManager FileMgr(FileSystemOptions(), VFS);
199
200 SourceManager SourceMgr(*Diags, FileMgr);
201
202 HeaderSearch HeaderInfo(HSOpts, SourceMgr, *Diags, LangOpts,
203 /*Target=*/nullptr);
204
205 PreprocessorOptions PPOpts;
206 TrivialModuleLoader ModuleLoader;
207 Preprocessor PP(PPOpts, *Diags, LangOpts, SourceMgr, HeaderInfo,
208 ModuleLoader);
209
210 IntrusiveRefCntPtr<ModuleCache> ModCache = createCrossProcessModuleCache();
211 PCHContainerOperations PCHOperations;
212 ASTReader Reader(PP, *ModCache, /*ASTContext=*/nullptr,
213 PCHOperations.getRawReader(), {});
214
215 // We don't need any listener here. By default it will use a validator
216 // listener.
217 Reader.setListener(nullptr);
218
219 if (Reader.ReadAST(FileName: ModuleFilePath, Type: serialization::MK_MainFile,
220 ImportLoc: SourceLocation(),
221 ClientLoadCapabilities: ASTReader::ARR_None) != ASTReader::Success)
222 return false;
223
224 bool UpToDate = true;
225 Reader.getModuleManager().visit(Visitor: [&](serialization::ModuleFile &MF) -> bool {
226 Reader.visitInputFiles(
227 MF, /*IncludeSystem=*/false, /*Complain=*/false,
228 Visitor: [&](const serialization::InputFile &IF, bool isSystem) {
229 if (!IF.getFile() || IF.isOutOfDate())
230 UpToDate = false;
231 });
232 return !UpToDate;
233 });
234 return UpToDate;
235}
236
237bool IsModuleFilesUpToDate(
238 llvm::SmallVector<PathRef> ModuleFilePaths,
239 const PrerequisiteModules &RequisiteModules,
240 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
241 return llvm::all_of(
242 Range&: ModuleFilePaths, P: [&RequisiteModules, VFS](auto ModuleFilePath) {
243 return IsModuleFileUpToDate(ModuleFilePath, RequisiteModules, VFS);
244 });
245}
246
247/// Build a module file for module with `ModuleName`. The information of built
248/// module file are stored in \param BuiltModuleFiles.
249llvm::Expected<ModuleFile>
250buildModuleFile(llvm::StringRef ModuleName, PathRef ModuleUnitFileName,
251 const GlobalCompilationDatabase &CDB, const ThreadsafeFS &TFS,
252 const ReusablePrerequisiteModules &BuiltModuleFiles) {
253 // Try cheap operation earlier to boil-out cheaply if there are problems.
254 auto Cmd = CDB.getCompileCommand(File: ModuleUnitFileName);
255 if (!Cmd)
256 return llvm::createStringError(
257 S: llvm::formatv(Fmt: "No compile command for {0}", Vals&: ModuleUnitFileName));
258
259 llvm::SmallString<256> ModuleFilesPrefix =
260 getUniqueModuleFilesPath(MainFile: ModuleUnitFileName);
261
262 Cmd->Output = getModuleFilePath(ModuleName, ModuleFilesPrefix);
263
264 ParseInputs Inputs;
265 Inputs.TFS = &TFS;
266 Inputs.CompileCommand = std::move(*Cmd);
267
268 IgnoreDiagnostics IgnoreDiags;
269 auto CI = buildCompilerInvocation(Inputs, D&: IgnoreDiags);
270 if (!CI)
271 return llvm::createStringError(Fmt: "Failed to build compiler invocation");
272
273 auto FS = Inputs.TFS->view(CWD: Inputs.CompileCommand.Directory);
274 auto Buf = FS->getBufferForFile(Name: Inputs.CompileCommand.Filename);
275 if (!Buf)
276 return llvm::createStringError(Fmt: "Failed to create buffer");
277
278 // In clang's driver, we will suppress the check for ODR violation in GMF.
279 // See the implementation of RenderModulesOptions in Clang.cpp.
280 CI->getLangOpts().SkipODRCheckInGMF = true;
281
282 // Hash the contents of input files and store the hash value to the BMI files.
283 // So that we can check if the files are still valid when we want to reuse the
284 // BMI files.
285 CI->getHeaderSearchOpts().ValidateASTInputFilesContent = true;
286
287 BuiltModuleFiles.adjustHeaderSearchOptions(Options&: CI->getHeaderSearchOpts());
288
289 CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output;
290 auto Clang =
291 prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr,
292 MainFile: std::move(*Buf), std::move(FS), IgnoreDiags);
293 if (!Clang)
294 return llvm::createStringError(Fmt: "Failed to prepare compiler instance");
295
296 GenerateReducedModuleInterfaceAction Action;
297 Clang->ExecuteAction(Act&: Action);
298
299 if (Clang->getDiagnostics().hasErrorOccurred())
300 return llvm::createStringError(Fmt: "Compilation failed");
301
302 return ModuleFile{ModuleName, Inputs.CompileCommand.Output};
303}
304
305bool ReusablePrerequisiteModules::canReuse(
306 const CompilerInvocation &CI,
307 llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) const {
308 if (RequiredModules.empty())
309 return true;
310
311 llvm::SmallVector<llvm::StringRef> BMIPaths;
312 for (auto &MF : RequiredModules)
313 BMIPaths.push_back(Elt: MF->getModuleFilePath());
314 return IsModuleFilesUpToDate(ModuleFilePaths: BMIPaths, RequisiteModules: *this, VFS);
315}
316
317class ModuleFileCache {
318public:
319 ModuleFileCache(const GlobalCompilationDatabase &CDB) : CDB(CDB) {}
320 const GlobalCompilationDatabase &getCDB() const { return CDB; }
321
322 std::shared_ptr<const ModuleFile> getModule(StringRef ModuleName);
323
324 void add(StringRef ModuleName, std::shared_ptr<const ModuleFile> ModuleFile) {
325 std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
326
327 ModuleFiles[ModuleName] = ModuleFile;
328 }
329
330 void remove(StringRef ModuleName);
331
332private:
333 const GlobalCompilationDatabase &CDB;
334
335 llvm::StringMap<std::weak_ptr<const ModuleFile>> ModuleFiles;
336 // Mutex to guard accesses to ModuleFiles.
337 std::mutex ModuleFilesMutex;
338};
339
340std::shared_ptr<const ModuleFile>
341ModuleFileCache::getModule(StringRef ModuleName) {
342 std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
343
344 auto Iter = ModuleFiles.find(Key: ModuleName);
345 if (Iter == ModuleFiles.end())
346 return nullptr;
347
348 if (auto Res = Iter->second.lock())
349 return Res;
350
351 ModuleFiles.erase(I: Iter);
352 return nullptr;
353}
354
355void ModuleFileCache::remove(StringRef ModuleName) {
356 std::lock_guard<std::mutex> Lock(ModuleFilesMutex);
357
358 ModuleFiles.erase(Key: ModuleName);
359}
360
361class ModuleNameToSourceCache {
362public:
363 std::string getSourceForModuleName(llvm::StringRef ModuleName) {
364 std::lock_guard<std::mutex> Lock(CacheMutex);
365 auto Iter = ModuleNameToSourceCache.find(Key: ModuleName);
366 if (Iter != ModuleNameToSourceCache.end())
367 return Iter->second;
368 return "";
369 }
370
371 void addEntry(llvm::StringRef ModuleName, PathRef Source) {
372 std::lock_guard<std::mutex> Lock(CacheMutex);
373 ModuleNameToSourceCache[ModuleName] = Source.str();
374 }
375
376 void eraseEntry(llvm::StringRef ModuleName) {
377 std::lock_guard<std::mutex> Lock(CacheMutex);
378 ModuleNameToSourceCache.erase(Key: ModuleName);
379 }
380
381private:
382 std::mutex CacheMutex;
383 llvm::StringMap<std::string> ModuleNameToSourceCache;
384};
385
386class CachingProjectModules : public ProjectModules {
387public:
388 CachingProjectModules(std::unique_ptr<ProjectModules> MDB,
389 ModuleNameToSourceCache &Cache)
390 : MDB(std::move(MDB)), Cache(Cache) {
391 assert(this->MDB && "CachingProjectModules should only be created with a "
392 "valid underlying ProjectModules");
393 }
394
395 std::vector<std::string> getRequiredModules(PathRef File) override {
396 return MDB->getRequiredModules(File);
397 }
398
399 std::string getModuleNameForSource(PathRef File) override {
400 return MDB->getModuleNameForSource(File);
401 }
402
403 std::string getSourceForModuleName(llvm::StringRef ModuleName,
404 PathRef RequiredSrcFile) override {
405 std::string CachedResult = Cache.getSourceForModuleName(ModuleName);
406
407 // Verify Cached Result by seeing if the source declaring the same module
408 // as we query.
409 if (!CachedResult.empty()) {
410 std::string ModuleNameOfCachedSource =
411 MDB->getModuleNameForSource(File: CachedResult);
412 if (ModuleNameOfCachedSource == ModuleName)
413 return CachedResult;
414
415 // Cached Result is invalid. Clear it.
416 Cache.eraseEntry(ModuleName);
417 }
418
419 auto Result = MDB->getSourceForModuleName(ModuleName, RequiredSrcFile);
420 Cache.addEntry(ModuleName, Source: Result);
421
422 return Result;
423 }
424
425private:
426 std::unique_ptr<ProjectModules> MDB;
427 ModuleNameToSourceCache &Cache;
428};
429
430/// Collect the directly and indirectly required module names for \param
431/// ModuleName in topological order. The \param ModuleName is guaranteed to
432/// be the last element in \param ModuleNames.
433llvm::SmallVector<std::string> getAllRequiredModules(PathRef RequiredSource,
434 CachingProjectModules &MDB,
435 StringRef ModuleName) {
436 llvm::SmallVector<std::string> ModuleNames;
437 llvm::StringSet<> ModuleNamesSet;
438
439 auto VisitDeps = [&](StringRef ModuleName, auto Visitor) -> void {
440 ModuleNamesSet.insert(key: ModuleName);
441
442 for (StringRef RequiredModuleName : MDB.getRequiredModules(
443 File: MDB.getSourceForModuleName(ModuleName, RequiredSrcFile: RequiredSource)))
444 if (ModuleNamesSet.insert(key: RequiredModuleName).second)
445 Visitor(RequiredModuleName, Visitor);
446
447 ModuleNames.push_back(Elt: ModuleName.str());
448 };
449 VisitDeps(ModuleName, VisitDeps);
450
451 return ModuleNames;
452}
453
454} // namespace
455
456class ModulesBuilder::ModulesBuilderImpl {
457public:
458 ModulesBuilderImpl(const GlobalCompilationDatabase &CDB) : Cache(CDB) {}
459
460 ModuleNameToSourceCache &getProjectModulesCache() {
461 return ProjectModulesCache;
462 }
463 const GlobalCompilationDatabase &getCDB() const { return Cache.getCDB(); }
464
465 llvm::Error
466 getOrBuildModuleFile(PathRef RequiredSource, StringRef ModuleName,
467 const ThreadsafeFS &TFS, CachingProjectModules &MDB,
468 ReusablePrerequisiteModules &BuiltModuleFiles);
469
470private:
471 ModuleFileCache Cache;
472 ModuleNameToSourceCache ProjectModulesCache;
473};
474
475llvm::Error ModulesBuilder::ModulesBuilderImpl::getOrBuildModuleFile(
476 PathRef RequiredSource, StringRef ModuleName, const ThreadsafeFS &TFS,
477 CachingProjectModules &MDB, ReusablePrerequisiteModules &BuiltModuleFiles) {
478 if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName))
479 return llvm::Error::success();
480
481 std::string ModuleUnitFileName =
482 MDB.getSourceForModuleName(ModuleName, RequiredSrcFile: RequiredSource);
483 /// It is possible that we're meeting third party modules (modules whose
484 /// source are not in the project. e.g, the std module may be a third-party
485 /// module for most project) or something wrong with the implementation of
486 /// ProjectModules.
487 /// FIXME: How should we treat third party modules here? If we want to ignore
488 /// third party modules, we should return true instead of false here.
489 /// Currently we simply bail out.
490 if (ModuleUnitFileName.empty())
491 return llvm::createStringError(
492 S: llvm::formatv(Fmt: "Don't get the module unit for module {0}", Vals&: ModuleName));
493
494 // Get Required modules in topological order.
495 auto ReqModuleNames = getAllRequiredModules(RequiredSource, MDB, ModuleName);
496 for (llvm::StringRef ReqModuleName : ReqModuleNames) {
497 if (BuiltModuleFiles.isModuleUnitBuilt(ModuleName: ReqModuleName))
498 continue;
499
500 if (auto Cached = Cache.getModule(ModuleName: ReqModuleName)) {
501 if (IsModuleFileUpToDate(ModuleFilePath: Cached->getModuleFilePath(), RequisiteModules: BuiltModuleFiles,
502 VFS: TFS.view(CWD: std::nullopt))) {
503 log(Fmt: "Reusing module {0} from {1}", Vals&: ReqModuleName,
504 Vals: Cached->getModuleFilePath());
505 BuiltModuleFiles.addModuleFile(ModuleFile: std::move(Cached));
506 continue;
507 }
508 Cache.remove(ModuleName: ReqModuleName);
509 }
510
511 std::string ReqFileName =
512 MDB.getSourceForModuleName(ModuleName: ReqModuleName, RequiredSrcFile: RequiredSource);
513 llvm::Expected<ModuleFile> MF = buildModuleFile(
514 ModuleName: ReqModuleName, ModuleUnitFileName: ReqFileName, CDB: getCDB(), TFS, BuiltModuleFiles);
515 if (llvm::Error Err = MF.takeError())
516 return Err;
517
518 log(Fmt: "Built module {0} to {1}", Vals&: ReqModuleName, Vals: MF->getModuleFilePath());
519 auto BuiltModuleFile = std::make_shared<const ModuleFile>(args: std::move(*MF));
520 Cache.add(ModuleName: ReqModuleName, ModuleFile: BuiltModuleFile);
521 BuiltModuleFiles.addModuleFile(ModuleFile: std::move(BuiltModuleFile));
522 }
523
524 return llvm::Error::success();
525}
526
527std::unique_ptr<PrerequisiteModules>
528ModulesBuilder::buildPrerequisiteModulesFor(PathRef File,
529 const ThreadsafeFS &TFS) {
530 std::unique_ptr<ProjectModules> MDB = Impl->getCDB().getProjectModules(File);
531 if (!MDB) {
532 elog(Fmt: "Failed to get Project Modules information for {0}", Vals&: File);
533 return std::make_unique<FailedPrerequisiteModules>();
534 }
535 CachingProjectModules CachedMDB(std::move(MDB),
536 Impl->getProjectModulesCache());
537
538 std::vector<std::string> RequiredModuleNames =
539 CachedMDB.getRequiredModules(File);
540 if (RequiredModuleNames.empty())
541 return std::make_unique<ReusablePrerequisiteModules>();
542
543 auto RequiredModules = std::make_unique<ReusablePrerequisiteModules>();
544 for (llvm::StringRef RequiredModuleName : RequiredModuleNames) {
545 // Return early if there is any error.
546 if (llvm::Error Err = Impl->getOrBuildModuleFile(
547 RequiredSource: File, ModuleName: RequiredModuleName, TFS, MDB&: CachedMDB, BuiltModuleFiles&: *RequiredModules.get())) {
548 elog(Fmt: "Failed to build module {0}; due to {1}", Vals&: RequiredModuleName,
549 Vals: toString(E: std::move(Err)));
550 return std::make_unique<FailedPrerequisiteModules>();
551 }
552 }
553
554 return std::move(RequiredModules);
555}
556
557ModulesBuilder::ModulesBuilder(const GlobalCompilationDatabase &CDB) {
558 Impl = std::make_unique<ModulesBuilderImpl>(args: CDB);
559}
560
561ModulesBuilder::~ModulesBuilder() {}
562
563} // namespace clangd
564} // namespace clang
565

source code of clang-tools-extra/clangd/ModulesBuilder.cpp