1//===- ModuleManager.cpp - Module Manager ---------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the ModuleManager class, which manages a set of loaded
10// modules for the ASTReader.
11//
12//===----------------------------------------------------------------------===//
13
14#include "clang/Serialization/ModuleManager.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/LLVM.h"
17#include "clang/Lex/HeaderSearch.h"
18#include "clang/Lex/ModuleMap.h"
19#include "clang/Serialization/GlobalModuleIndex.h"
20#include "clang/Serialization/InMemoryModuleCache.h"
21#include "clang/Serialization/ModuleFile.h"
22#include "clang/Serialization/PCHContainerOperations.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallPtrSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/iterator.h"
29#include "llvm/Support/Chrono.h"
30#include "llvm/Support/DOTGraphTraits.h"
31#include "llvm/Support/ErrorOr.h"
32#include "llvm/Support/GraphWriter.h"
33#include "llvm/Support/MemoryBuffer.h"
34#include "llvm/Support/VirtualFileSystem.h"
35#include <algorithm>
36#include <cassert>
37#include <memory>
38#include <string>
39#include <system_error>
40
41using namespace clang;
42using namespace serialization;
43
44ModuleFile *ModuleManager::lookupByFileName(StringRef Name) const {
45 auto Entry = FileMgr.getFile(Filename: Name, /*OpenFile=*/false,
46 /*CacheFailure=*/false);
47 if (Entry)
48 return lookup(File: *Entry);
49
50 return nullptr;
51}
52
53ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const {
54 if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name))
55 if (OptionalFileEntryRef File = Mod->getASTFile())
56 return lookup(File: *File);
57
58 return nullptr;
59}
60
61ModuleFile *ModuleManager::lookup(const FileEntry *File) const {
62 return Modules.lookup(Val: File);
63}
64
65std::unique_ptr<llvm::MemoryBuffer>
66ModuleManager::lookupBuffer(StringRef Name) {
67 auto Entry = FileMgr.getFile(Filename: Name, /*OpenFile=*/false,
68 /*CacheFailure=*/false);
69 if (!Entry)
70 return nullptr;
71 return std::move(InMemoryBuffers[*Entry]);
72}
73
74static bool checkSignature(ASTFileSignature Signature,
75 ASTFileSignature ExpectedSignature,
76 std::string &ErrorStr) {
77 if (!ExpectedSignature || Signature == ExpectedSignature)
78 return false;
79
80 ErrorStr =
81 Signature ? "signature mismatch" : "could not read module signature";
82 return true;
83}
84
85static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,
86 SourceLocation ImportLoc) {
87 if (ImportedBy) {
88 MF.ImportedBy.insert(X: ImportedBy);
89 ImportedBy->Imports.insert(X: &MF);
90 } else {
91 if (!MF.DirectlyImported)
92 MF.ImportLoc = ImportLoc;
93
94 MF.DirectlyImported = true;
95 }
96}
97
98ModuleManager::AddModuleResult
99ModuleManager::addModule(StringRef FileName, ModuleKind Type,
100 SourceLocation ImportLoc, ModuleFile *ImportedBy,
101 unsigned Generation,
102 off_t ExpectedSize, time_t ExpectedModTime,
103 ASTFileSignature ExpectedSignature,
104 ASTFileSignatureReader ReadSignature,
105 ModuleFile *&Module,
106 std::string &ErrorStr) {
107 Module = nullptr;
108
109 // Look for the file entry. This only fails if the expected size or
110 // modification time differ.
111 OptionalFileEntryRef Entry;
112 if (Type == MK_ExplicitModule || Type == MK_PrebuiltModule) {
113 // If we're not expecting to pull this file out of the module cache, it
114 // might have a different mtime due to being moved across filesystems in
115 // a distributed build. The size must still match, though. (As must the
116 // contents, but we can't check that.)
117 ExpectedModTime = 0;
118 }
119 // Note: ExpectedSize and ExpectedModTime will be 0 for MK_ImplicitModule
120 // when using an ASTFileSignature.
121 if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, File&: Entry)) {
122 ErrorStr = "module file out of date";
123 return OutOfDate;
124 }
125
126 if (!Entry) {
127 ErrorStr = "module file not found";
128 return Missing;
129 }
130
131 // The ModuleManager's use of FileEntry nodes as the keys for its map of
132 // loaded modules is less than ideal. Uniqueness for FileEntry nodes is
133 // maintained by FileManager, which in turn uses inode numbers on hosts
134 // that support that. When coupled with the module cache's proclivity for
135 // turning over and deleting stale PCMs, this means entries for different
136 // module files can wind up reusing the same underlying inode. When this
137 // happens, subsequent accesses to the Modules map will disagree on the
138 // ModuleFile associated with a given file. In general, it is not sufficient
139 // to resolve this conundrum with a type like FileEntryRef that stores the
140 // name of the FileEntry node on first access because of path canonicalization
141 // issues. However, the paths constructed for implicit module builds are
142 // fully under Clang's control. We *can*, therefore, rely on their structure
143 // being consistent across operating systems and across subsequent accesses
144 // to the Modules map.
145 auto implicitModuleNamesMatch = [](ModuleKind Kind, const ModuleFile *MF,
146 FileEntryRef Entry) -> bool {
147 if (Kind != MK_ImplicitModule)
148 return true;
149 return Entry.getName() == MF->FileName;
150 };
151
152 // Check whether we already loaded this module, before
153 if (ModuleFile *ModuleEntry = Modules.lookup(Val: *Entry)) {
154 if (implicitModuleNamesMatch(Type, ModuleEntry, *Entry)) {
155 // Check the stored signature.
156 if (checkSignature(Signature: ModuleEntry->Signature, ExpectedSignature, ErrorStr))
157 return OutOfDate;
158
159 Module = ModuleEntry;
160 updateModuleImports(MF&: *ModuleEntry, ImportedBy, ImportLoc);
161 return AlreadyLoaded;
162 }
163 }
164
165 // Allocate a new module.
166 auto NewModule = std::make_unique<ModuleFile>(args&: Type, args&: *Entry, args&: Generation);
167 NewModule->Index = Chain.size();
168 NewModule->FileName = FileName.str();
169 NewModule->ImportLoc = ImportLoc;
170 NewModule->InputFilesValidationTimestamp = 0;
171
172 if (NewModule->Kind == MK_ImplicitModule) {
173 std::string TimestampFilename = NewModule->getTimestampFilename();
174 llvm::vfs::Status Status;
175 // A cached stat value would be fine as well.
176 if (!FileMgr.getNoncachedStatValue(Path: TimestampFilename, Result&: Status))
177 NewModule->InputFilesValidationTimestamp =
178 llvm::sys::toTimeT(TP: Status.getLastModificationTime());
179 }
180
181 // Load the contents of the module
182 if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(Name: FileName)) {
183 // The buffer was already provided for us.
184 NewModule->Buffer = &ModuleCache->addBuiltPCM(Filename: FileName, Buffer: std::move(Buffer));
185 // Since the cached buffer is reused, it is safe to close the file
186 // descriptor that was opened while stat()ing the PCM in
187 // lookupModuleFile() above, it won't be needed any longer.
188 Entry->closeFile();
189 } else if (llvm::MemoryBuffer *Buffer =
190 getModuleCache().lookupPCM(Filename: FileName)) {
191 NewModule->Buffer = Buffer;
192 // As above, the file descriptor is no longer needed.
193 Entry->closeFile();
194 } else if (getModuleCache().shouldBuildPCM(Filename: FileName)) {
195 // Report that the module is out of date, since we tried (and failed) to
196 // import it earlier.
197 Entry->closeFile();
198 return OutOfDate;
199 } else {
200 // Get a buffer of the file and close the file descriptor when done.
201 // The file is volatile because in a parallel build we expect multiple
202 // compiler processes to use the same module file rebuilding it if needed.
203 //
204 // RequiresNullTerminator is false because module files don't need it, and
205 // this allows the file to still be mmapped.
206 auto Buf = FileMgr.getBufferForFile(Entry: NewModule->File,
207 /*IsVolatile=*/isVolatile: true,
208 /*RequiresNullTerminator=*/false);
209
210 if (!Buf) {
211 ErrorStr = Buf.getError().message();
212 return Missing;
213 }
214
215 NewModule->Buffer = &getModuleCache().addPCM(Filename: FileName, Buffer: std::move(*Buf));
216 }
217
218 // Initialize the stream.
219 NewModule->Data = PCHContainerRdr.ExtractPCH(Buffer: *NewModule->Buffer);
220
221 // Read the signature eagerly now so that we can check it. Avoid calling
222 // ReadSignature unless there's something to check though.
223 if (ExpectedSignature && checkSignature(Signature: ReadSignature(NewModule->Data),
224 ExpectedSignature, ErrorStr))
225 return OutOfDate;
226
227 // We're keeping this module. Store it everywhere.
228 Module = Modules[*Entry] = NewModule.get();
229
230 updateModuleImports(MF&: *NewModule, ImportedBy, ImportLoc);
231
232 if (!NewModule->isModule())
233 PCHChain.push_back(Elt: NewModule.get());
234 if (!ImportedBy)
235 Roots.push_back(Elt: NewModule.get());
236
237 Chain.push_back(Elt: std::move(NewModule));
238 return NewlyLoaded;
239}
240
241void ModuleManager::removeModules(ModuleIterator First) {
242 auto Last = end();
243 if (First == Last)
244 return;
245
246 // Explicitly clear VisitOrder since we might not notice it is stale.
247 VisitOrder.clear();
248
249 // Collect the set of module file pointers that we'll be removing.
250 llvm::SmallPtrSet<ModuleFile *, 4> victimSet(
251 (llvm::pointer_iterator<ModuleIterator>(First)),
252 (llvm::pointer_iterator<ModuleIterator>(Last)));
253
254 auto IsVictim = [&](ModuleFile *MF) {
255 return victimSet.count(Ptr: MF);
256 };
257 // Remove any references to the now-destroyed modules.
258 for (auto I = begin(); I != First; ++I) {
259 I->Imports.remove_if(P: IsVictim);
260 I->ImportedBy.remove_if(P: IsVictim);
261 }
262 llvm::erase_if(C&: Roots, P: IsVictim);
263
264 // Remove the modules from the PCH chain.
265 for (auto I = First; I != Last; ++I) {
266 if (!I->isModule()) {
267 PCHChain.erase(CS: llvm::find(Range&: PCHChain, Val: &*I), CE: PCHChain.end());
268 break;
269 }
270 }
271
272 // Delete the modules.
273 for (ModuleIterator victim = First; victim != Last; ++victim)
274 Modules.erase(Val: victim->File);
275
276 Chain.erase(CS: Chain.begin() + (First - begin()), CE: Chain.end());
277}
278
279void
280ModuleManager::addInMemoryBuffer(StringRef FileName,
281 std::unique_ptr<llvm::MemoryBuffer> Buffer) {
282 const FileEntry *Entry =
283 FileMgr.getVirtualFile(Filename: FileName, Size: Buffer->getBufferSize(), ModificationTime: 0);
284 InMemoryBuffers[Entry] = std::move(Buffer);
285}
286
287std::unique_ptr<ModuleManager::VisitState> ModuleManager::allocateVisitState() {
288 // Fast path: if we have a cached state, use it.
289 if (FirstVisitState) {
290 auto Result = std::move(FirstVisitState);
291 FirstVisitState = std::move(Result->NextState);
292 return Result;
293 }
294
295 // Allocate and return a new state.
296 return std::make_unique<VisitState>(args: size());
297}
298
299void ModuleManager::returnVisitState(std::unique_ptr<VisitState> State) {
300 assert(State->NextState == nullptr && "Visited state is in list?");
301 State->NextState = std::move(FirstVisitState);
302 FirstVisitState = std::move(State);
303}
304
305void ModuleManager::setGlobalIndex(GlobalModuleIndex *Index) {
306 GlobalIndex = Index;
307 if (!GlobalIndex) {
308 ModulesInCommonWithGlobalIndex.clear();
309 return;
310 }
311
312 // Notify the global module index about all of the modules we've already
313 // loaded.
314 for (ModuleFile &M : *this)
315 if (!GlobalIndex->loadedModuleFile(File: &M))
316 ModulesInCommonWithGlobalIndex.push_back(Elt: &M);
317}
318
319void ModuleManager::moduleFileAccepted(ModuleFile *MF) {
320 if (!GlobalIndex || GlobalIndex->loadedModuleFile(File: MF))
321 return;
322
323 ModulesInCommonWithGlobalIndex.push_back(Elt: MF);
324}
325
326ModuleManager::ModuleManager(FileManager &FileMgr,
327 InMemoryModuleCache &ModuleCache,
328 const PCHContainerReader &PCHContainerRdr,
329 const HeaderSearch &HeaderSearchInfo)
330 : FileMgr(FileMgr), ModuleCache(&ModuleCache),
331 PCHContainerRdr(PCHContainerRdr), HeaderSearchInfo(HeaderSearchInfo) {}
332
333void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
334 llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {
335 // If the visitation order vector is the wrong size, recompute the order.
336 if (VisitOrder.size() != Chain.size()) {
337 unsigned N = size();
338 VisitOrder.clear();
339 VisitOrder.reserve(N);
340
341 // Record the number of incoming edges for each module. When we
342 // encounter a module with no incoming edges, push it into the queue
343 // to seed the queue.
344 SmallVector<ModuleFile *, 4> Queue;
345 Queue.reserve(N);
346 llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;
347 UnusedIncomingEdges.resize(N: size());
348 for (ModuleFile &M : llvm::reverse(C&: *this)) {
349 unsigned Size = M.ImportedBy.size();
350 UnusedIncomingEdges[M.Index] = Size;
351 if (!Size)
352 Queue.push_back(Elt: &M);
353 }
354
355 // Traverse the graph, making sure to visit a module before visiting any
356 // of its dependencies.
357 while (!Queue.empty()) {
358 ModuleFile *CurrentModule = Queue.pop_back_val();
359 VisitOrder.push_back(Elt: CurrentModule);
360
361 // For any module that this module depends on, push it on the
362 // stack (if it hasn't already been marked as visited).
363 for (ModuleFile *M : llvm::reverse(C&: CurrentModule->Imports)) {
364 // Remove our current module as an impediment to visiting the
365 // module we depend on. If we were the last unvisited module
366 // that depends on this particular module, push it into the
367 // queue to be visited.
368 unsigned &NumUnusedEdges = UnusedIncomingEdges[M->Index];
369 if (NumUnusedEdges && (--NumUnusedEdges == 0))
370 Queue.push_back(Elt: M);
371 }
372 }
373
374 assert(VisitOrder.size() == N && "Visitation order is wrong?");
375
376 FirstVisitState = nullptr;
377 }
378
379 auto State = allocateVisitState();
380 unsigned VisitNumber = State->NextVisitNumber++;
381
382 // If the caller has provided us with a hit-set that came from the global
383 // module index, mark every module file in common with the global module
384 // index that is *not* in that set as 'visited'.
385 if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {
386 for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
387 {
388 ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
389 if (!ModuleFilesHit->count(Ptr: M))
390 State->VisitNumber[M->Index] = VisitNumber;
391 }
392 }
393
394 for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {
395 ModuleFile *CurrentModule = VisitOrder[I];
396 // Should we skip this module file?
397 if (State->VisitNumber[CurrentModule->Index] == VisitNumber)
398 continue;
399
400 // Visit the module.
401 assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);
402 State->VisitNumber[CurrentModule->Index] = VisitNumber;
403 if (!Visitor(*CurrentModule))
404 continue;
405
406 // The visitor has requested that cut off visitation of any
407 // module that the current module depends on. To indicate this
408 // behavior, we mark all of the reachable modules as having been visited.
409 ModuleFile *NextModule = CurrentModule;
410 do {
411 // For any module that this module depends on, push it on the
412 // stack (if it hasn't already been marked as visited).
413 for (llvm::SetVector<ModuleFile *>::iterator
414 M = NextModule->Imports.begin(),
415 MEnd = NextModule->Imports.end();
416 M != MEnd; ++M) {
417 if (State->VisitNumber[(*M)->Index] != VisitNumber) {
418 State->Stack.push_back(Elt: *M);
419 State->VisitNumber[(*M)->Index] = VisitNumber;
420 }
421 }
422
423 if (State->Stack.empty())
424 break;
425
426 // Pop the next module off the stack.
427 NextModule = State->Stack.pop_back_val();
428 } while (true);
429 }
430
431 returnVisitState(State: std::move(State));
432}
433
434bool ModuleManager::lookupModuleFile(StringRef FileName, off_t ExpectedSize,
435 time_t ExpectedModTime,
436 OptionalFileEntryRef &File) {
437 if (FileName == "-") {
438 File = expectedToOptional(E: FileMgr.getSTDIN());
439 return false;
440 }
441
442 // Open the file immediately to ensure there is no race between stat'ing and
443 // opening the file.
444 File = FileMgr.getOptionalFileRef(Filename: FileName, /*OpenFile=*/true,
445 /*CacheFailure=*/false);
446
447 if (File &&
448 ((ExpectedSize && ExpectedSize != File->getSize()) ||
449 (ExpectedModTime && ExpectedModTime != File->getModificationTime())))
450 // Do not destroy File, as it may be referenced. If we need to rebuild it,
451 // it will be destroyed by removeModules.
452 return true;
453
454 return false;
455}
456
457#ifndef NDEBUG
458namespace llvm {
459
460 template<>
461 struct GraphTraits<ModuleManager> {
462 using NodeRef = ModuleFile *;
463 using ChildIteratorType = llvm::SetVector<ModuleFile *>::const_iterator;
464 using nodes_iterator = pointer_iterator<ModuleManager::ModuleConstIterator>;
465
466 static ChildIteratorType child_begin(NodeRef Node) {
467 return Node->Imports.begin();
468 }
469
470 static ChildIteratorType child_end(NodeRef Node) {
471 return Node->Imports.end();
472 }
473
474 static nodes_iterator nodes_begin(const ModuleManager &Manager) {
475 return nodes_iterator(Manager.begin());
476 }
477
478 static nodes_iterator nodes_end(const ModuleManager &Manager) {
479 return nodes_iterator(Manager.end());
480 }
481 };
482
483 template<>
484 struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
485 explicit DOTGraphTraits(bool IsSimple = false)
486 : DefaultDOTGraphTraits(IsSimple) {}
487
488 static bool renderGraphFromBottomUp() { return true; }
489
490 std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {
491 return M->ModuleName;
492 }
493 };
494
495} // namespace llvm
496
497void ModuleManager::viewGraph() {
498 llvm::ViewGraph(G: *this, Name: "Modules");
499}
500#endif
501

source code of clang/lib/Serialization/ModuleManager.cpp