1	//===- MetadataLoader.cpp - Internal BitcodeReader implementation ---------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "MetadataLoader.h"
10	#include "ValueList.h"
11
12	#include "llvm/ADT/APInt.h"
13	#include "llvm/ADT/ArrayRef.h"
14	#include "llvm/ADT/BitmaskEnum.h"
15	#include "llvm/ADT/DenseMap.h"
16	#include "llvm/ADT/DenseSet.h"
17	#include "llvm/ADT/STLFunctionalExtras.h"
18	#include "llvm/ADT/SetVector.h"
19	#include "llvm/ADT/SmallString.h"
20	#include "llvm/ADT/SmallVector.h"
21	#include "llvm/ADT/Statistic.h"
22	#include "llvm/ADT/StringRef.h"
23	#include "llvm/ADT/Twine.h"
24	#include "llvm/ADT/ilist_iterator.h"
25	#include "llvm/BinaryFormat/Dwarf.h"
26	#include "llvm/Bitcode/BitcodeReader.h"
27	#include "llvm/Bitcode/LLVMBitCodes.h"
28	#include "llvm/Bitstream/BitstreamReader.h"
29	#include "llvm/IR/AutoUpgrade.h"
30	#include "llvm/IR/BasicBlock.h"
31	#include "llvm/IR/Constants.h"
32	#include "llvm/IR/DebugInfoMetadata.h"
33	#include "llvm/IR/Function.h"
34	#include "llvm/IR/GlobalObject.h"
35	#include "llvm/IR/GlobalVariable.h"
36	#include "llvm/IR/Instruction.h"
37	#include "llvm/IR/IntrinsicInst.h"
38	#include "llvm/IR/LLVMContext.h"
39	#include "llvm/IR/Metadata.h"
40	#include "llvm/IR/Module.h"
41	#include "llvm/IR/TrackingMDRef.h"
42	#include "llvm/IR/Type.h"
43	#include "llvm/Support/Casting.h"
44	#include "llvm/Support/CommandLine.h"
45	#include "llvm/Support/Compiler.h"
46	#include "llvm/Support/ErrorHandling.h"
47	#include "llvm/Support/type_traits.h"
48
49	#include <algorithm>
50	#include <cassert>
51	#include <cstddef>
52	#include <cstdint>
53	#include <deque>
54	#include <iterator>
55	#include <limits>
56	#include <map>
57	#include <optional>
58	#include <string>
59	#include <tuple>
60	#include <type_traits>
61	#include <utility>
62	#include <vector>
63	namespace llvm {
64	class Argument;
65	}
66
67	using namespace llvm;
68
69	#define DEBUG_TYPE "bitcode-reader"
70
71	STATISTIC(NumMDStringLoaded, "Number of MDStrings loaded");
72	STATISTIC(NumMDNodeTemporary, "Number of MDNode::Temporary created");
73	STATISTIC(NumMDRecordLoaded, "Number of Metadata records loaded");
74
75	/// Flag whether we need to import full type definitions for ThinLTO.
76	/// Currently needed for Darwin and LLDB.
77	static cl::opt<bool> ImportFullTypeDefinitions(
78	"import-full-type-definitions", cl::init(Val: false), cl::Hidden,
79	cl::desc("Import full type definitions for ThinLTO."));
80
81	static cl::opt<bool> DisableLazyLoading(
82	"disable-ondemand-mds-loading", cl::init(Val: false), cl::Hidden,
83	cl::desc("Force disable the lazy-loading on-demand of metadata when "
84	"loading bitcode for importing."));
85
86	namespace {
87
88	static int64_t unrotateSign(uint64_t U) { return (U & 1) ? ~(U >> 1) : U >> 1; }
89
90	class BitcodeReaderMetadataList {
91	/// Array of metadata references.
92	///
93	/// Don't use std::vector here. Some versions of libc++ copy (instead of
94	/// move) on resize, and TrackingMDRef is very expensive to copy.
95	SmallVector<TrackingMDRef, 1> MetadataPtrs;
96
97	/// The set of indices in MetadataPtrs above of forward references that were
98	/// generated.
99	SmallDenseSet<unsigned, 1> ForwardReference;
100
101	/// The set of indices in MetadataPtrs above of Metadata that need to be
102	/// resolved.
103	SmallDenseSet<unsigned, 1> UnresolvedNodes;
104
105	/// Structures for resolving old type refs.
106	struct {
107	SmallDenseMap<MDString *, TempMDTuple, 1> Unknown;
108	SmallDenseMap<MDString *, DICompositeType *, 1> Final;
109	SmallDenseMap<MDString *, DICompositeType *, 1> FwdDecls;
110	SmallVector<std::pair<TrackingMDRef, TempMDTuple>, 1> Arrays;
111	} OldTypeRefs;
112
113	LLVMContext &Context;
114
115	/// Maximum number of valid references. Forward references exceeding the
116	/// maximum must be invalid.
117	unsigned RefsUpperBound;
118
119	public:
120	BitcodeReaderMetadataList(LLVMContext &C, size_t RefsUpperBound)
121	: Context(C),
122	RefsUpperBound(std::min(a: (size_t)std::numeric_limits<unsigned>::max(),
123	b: RefsUpperBound)) {}
124
125	// vector compatibility methods
126	unsigned size() const { return MetadataPtrs.size(); }
127	void resize(unsigned N) { MetadataPtrs.resize(N); }
128	void push_back(Metadata *MD) { MetadataPtrs.emplace_back(Args&: MD); }
129	void clear() { MetadataPtrs.clear(); }
130	Metadata *back() const { return MetadataPtrs.back(); }
131	void pop_back() { MetadataPtrs.pop_back(); }
132	bool empty() const { return MetadataPtrs.empty(); }
133
134	Metadata *operator[](unsigned i) const {
135	assert(i < MetadataPtrs.size());
136	return MetadataPtrs[i];
137	}
138
139	Metadata *lookup(unsigned I) const {
140	if (I < MetadataPtrs.size())
141	return MetadataPtrs[I];
142	return nullptr;
143	}
144
145	void shrinkTo(unsigned N) {
146	assert(N <= size() && "Invalid shrinkTo request!");
147	assert(ForwardReference.empty() && "Unexpected forward refs");
148	assert(UnresolvedNodes.empty() && "Unexpected unresolved node");
149	MetadataPtrs.resize(N);
150	}
151
152	/// Return the given metadata, creating a replaceable forward reference if
153	/// necessary.
154	Metadata *getMetadataFwdRef(unsigned Idx);
155
156	/// Return the given metadata only if it is fully resolved.
157	///
158	/// Gives the same result as \a lookup(), unless \a MDNode::isResolved()
159	/// would give \c false.
160	Metadata *getMetadataIfResolved(unsigned Idx);
161
162	MDNode *getMDNodeFwdRefOrNull(unsigned Idx);
163	void assignValue(Metadata *MD, unsigned Idx);
164	void tryToResolveCycles();
165	bool hasFwdRefs() const { return !ForwardReference.empty(); }
166	int getNextFwdRef() {
167	assert(hasFwdRefs());
168	return *ForwardReference.begin();
169	}
170
171	/// Upgrade a type that had an MDString reference.
172	void addTypeRef(MDString &UUID, DICompositeType &CT);
173
174	/// Upgrade a type that had an MDString reference.
175	Metadata *upgradeTypeRef(Metadata *MaybeUUID);
176
177	/// Upgrade a type ref array that may have MDString references.
178	Metadata *upgradeTypeRefArray(Metadata *MaybeTuple);
179
180	private:
181	Metadata *resolveTypeRefArray(Metadata *MaybeTuple);
182	};
183
184	void BitcodeReaderMetadataList::assignValue(Metadata *MD, unsigned Idx) {
185	if (auto *MDN = dyn_cast<MDNode>(Val: MD))
186	if (!MDN->isResolved())
187	UnresolvedNodes.insert(V: Idx);
188
189	if (Idx == size()) {
190	push_back(MD);
191	return;
192	}
193
194	if (Idx >= size())
195	resize(N: Idx + 1);
196
197	TrackingMDRef &OldMD = MetadataPtrs[Idx];
198	if (!OldMD) {
199	OldMD.reset(MD);
200	return;
201	}
202
203	// If there was a forward reference to this value, replace it.
204	TempMDTuple PrevMD(cast<MDTuple>(Val: OldMD.get()));
205	PrevMD->replaceAllUsesWith(MD);
206	ForwardReference.erase(V: Idx);
207	}
208
209	Metadata *BitcodeReaderMetadataList::getMetadataFwdRef(unsigned Idx) {
210	// Bail out for a clearly invalid value.
211	if (Idx >= RefsUpperBound)
212	return nullptr;
213
214	if (Idx >= size())
215	resize(N: Idx + 1);
216
217	if (Metadata *MD = MetadataPtrs[Idx])
218	return MD;
219
220	// Track forward refs to be resolved later.
221	ForwardReference.insert(V: Idx);
222
223	// Create and return a placeholder, which will later be RAUW'd.
224	++NumMDNodeTemporary;
225	Metadata *MD = MDNode::getTemporary(Context, MDs: std::nullopt).release();
226	MetadataPtrs[Idx].reset(MD);
227	return MD;
228	}
229
230	Metadata *BitcodeReaderMetadataList::getMetadataIfResolved(unsigned Idx) {
231	Metadata *MD = lookup(I: Idx);
232	if (auto *N = dyn_cast_or_null<MDNode>(Val: MD))
233	if (!N->isResolved())
234	return nullptr;
235	return MD;
236	}
237
238	MDNode *BitcodeReaderMetadataList::getMDNodeFwdRefOrNull(unsigned Idx) {
239	return dyn_cast_or_null<MDNode>(Val: getMetadataFwdRef(Idx));
240	}
241
242	void BitcodeReaderMetadataList::tryToResolveCycles() {
243	if (!ForwardReference.empty())
244	// Still forward references... can't resolve cycles.
245	return;
246
247	// Give up on finding a full definition for any forward decls that remain.
248	for (const auto &Ref : OldTypeRefs.FwdDecls)
249	OldTypeRefs.Final.insert(KV: Ref);
250	OldTypeRefs.FwdDecls.clear();
251
252	// Upgrade from old type ref arrays. In strange cases, this could add to
253	// OldTypeRefs.Unknown.
254	for (const auto &Array : OldTypeRefs.Arrays)
255	Array.second->replaceAllUsesWith(MD: resolveTypeRefArray(MaybeTuple: Array.first.get()));
256	OldTypeRefs.Arrays.clear();
257
258	// Replace old string-based type refs with the resolved node, if possible.
259	// If we haven't seen the node, leave it to the verifier to complain about
260	// the invalid string reference.
261	for (const auto &Ref : OldTypeRefs.Unknown) {
262	if (DICompositeType *CT = OldTypeRefs.Final.lookup(Val: Ref.first))
263	Ref.second->replaceAllUsesWith(MD: CT);
264	else
265	Ref.second->replaceAllUsesWith(MD: Ref.first);
266	}
267	OldTypeRefs.Unknown.clear();
268
269	if (UnresolvedNodes.empty())
270	// Nothing to do.
271	return;
272
273	// Resolve any cycles.
274	for (unsigned I : UnresolvedNodes) {
275	auto &MD = MetadataPtrs[I];
276	auto *N = dyn_cast_or_null<MDNode>(Val&: MD);
277	if (!N)
278	continue;
279
280	assert(!N->isTemporary() && "Unexpected forward reference");
281	N->resolveCycles();
282	}
283
284	// Make sure we return early again until there's another unresolved ref.
285	UnresolvedNodes.clear();
286	}
287
288	void BitcodeReaderMetadataList::addTypeRef(MDString &UUID,
289	DICompositeType &CT) {
290	assert(CT.getRawIdentifier() == &UUID && "Mismatched UUID");
291	if (CT.isForwardDecl())
292	OldTypeRefs.FwdDecls.insert(KV: std::make_pair(x: &UUID, y: &CT));
293	else
294	OldTypeRefs.Final.insert(KV: std::make_pair(x: &UUID, y: &CT));
295	}
296
297	Metadata *BitcodeReaderMetadataList::upgradeTypeRef(Metadata *MaybeUUID) {
298	auto *UUID = dyn_cast_or_null<MDString>(Val: MaybeUUID);
299	if (LLVM_LIKELY(!UUID))
300	return MaybeUUID;
301
302	if (auto *CT = OldTypeRefs.Final.lookup(Val: UUID))
303	return CT;
304
305	auto &Ref = OldTypeRefs.Unknown[UUID];
306	if (!Ref)
307	Ref = MDNode::getTemporary(Context, MDs: std::nullopt);
308	return Ref.get();
309	}
310
311	Metadata *BitcodeReaderMetadataList::upgradeTypeRefArray(Metadata *MaybeTuple) {
312	auto *Tuple = dyn_cast_or_null<MDTuple>(Val: MaybeTuple);
313	if (!Tuple || Tuple->isDistinct())
314	return MaybeTuple;
315
316	// Look through the array immediately if possible.
317	if (!Tuple->isTemporary())
318	return resolveTypeRefArray(MaybeTuple: Tuple);
319
320	// Create and return a placeholder to use for now. Eventually
321	// resolveTypeRefArrays() will be resolve this forward reference.
322	OldTypeRefs.Arrays.emplace_back(
323	Args: std::piecewise_construct, Args: std::forward_as_tuple(args&: Tuple),
324	Args: std::forward_as_tuple(args: MDTuple::getTemporary(Context, MDs: std::nullopt)));
325	return OldTypeRefs.Arrays.back().second.get();
326	}
327
328	Metadata *BitcodeReaderMetadataList::resolveTypeRefArray(Metadata *MaybeTuple) {
329	auto *Tuple = dyn_cast_or_null<MDTuple>(Val: MaybeTuple);
330	if (!Tuple || Tuple->isDistinct())
331	return MaybeTuple;
332
333	// Look through the DITypeRefArray, upgrading each DIType *.
334	SmallVector<Metadata *, 32> Ops;
335	Ops.reserve(N: Tuple->getNumOperands());
336	for (Metadata *MD : Tuple->operands())
337	Ops.push_back(Elt: upgradeTypeRef(MaybeUUID: MD));
338
339	return MDTuple::get(Context, MDs: Ops);
340	}
341
342	namespace {
343
344	class PlaceholderQueue {
345	// Placeholders would thrash around when moved, so store in a std::deque
346	// instead of some sort of vector.
347	std::deque<DistinctMDOperandPlaceholder> PHs;
348
349	public:
350	~PlaceholderQueue() {
351	assert(empty() &&
352	"PlaceholderQueue hasn't been flushed before being destroyed");
353	}
354	bool empty() const { return PHs.empty(); }
355	DistinctMDOperandPlaceholder &getPlaceholderOp(unsigned ID);
356	void flush(BitcodeReaderMetadataList &MetadataList);
357
358	/// Return the list of temporaries nodes in the queue, these need to be
359	/// loaded before we can flush the queue.
360	void getTemporaries(BitcodeReaderMetadataList &MetadataList,
361	DenseSet<unsigned> &Temporaries) {
362	for (auto &PH : PHs) {
363	auto ID = PH.getID();
364	auto *MD = MetadataList.lookup(I: ID);
365	if (!MD) {
366	Temporaries.insert(V: ID);
367	continue;
368	}
369	auto *N = dyn_cast_or_null<MDNode>(Val: MD);
370	if (N && N->isTemporary())
371	Temporaries.insert(V: ID);
372	}
373	}
374	};
375
376	} // end anonymous namespace
377
378	DistinctMDOperandPlaceholder &PlaceholderQueue::getPlaceholderOp(unsigned ID) {
379	PHs.emplace_back(args&: ID);
380	return PHs.back();
381	}
382
383	void PlaceholderQueue::flush(BitcodeReaderMetadataList &MetadataList) {
384	while (!PHs.empty()) {
385	auto *MD = MetadataList.lookup(I: PHs.front().getID());
386	assert(MD && "Flushing placeholder on unassigned MD");
387	#ifndef NDEBUG
388	if (auto *MDN = dyn_cast<MDNode>(Val: MD))
389	assert(MDN->isResolved() &&
390	"Flushing Placeholder while cycles aren't resolved");
391	#endif
392	PHs.front().replaceUseWith(MD);
393	PHs.pop_front();
394	}
395	}
396
397	} // anonymous namespace
398
399	static Error error(const Twine &Message) {
400	return make_error<StringError>(
401	Args: Message, Args: make_error_code(E: BitcodeError::CorruptedBitcode));
402	}
403
404	class MetadataLoader::MetadataLoaderImpl {
405	BitcodeReaderMetadataList MetadataList;
406	BitcodeReaderValueList &ValueList;
407	BitstreamCursor &Stream;
408	LLVMContext &Context;
409	Module &TheModule;
410	MetadataLoaderCallbacks Callbacks;
411
412	/// Cursor associated with the lazy-loading of Metadata. This is the easy way
413	/// to keep around the right "context" (Abbrev list) to be able to jump in
414	/// the middle of the metadata block and load any record.
415	BitstreamCursor IndexCursor;
416
417	/// Index that keeps track of MDString values.
418	std::vector<StringRef> MDStringRef;
419
420	/// On-demand loading of a single MDString. Requires the index above to be
421	/// populated.
422	MDString *lazyLoadOneMDString(unsigned Idx);
423
424	/// Index that keeps track of where to find a metadata record in the stream.
425	std::vector<uint64_t> GlobalMetadataBitPosIndex;
426
427	/// Cursor position of the start of the global decl attachments, to enable
428	/// loading using the index built for lazy loading, instead of forward
429	/// references.
430	uint64_t GlobalDeclAttachmentPos = 0;
431
432	#ifndef NDEBUG
433	/// Baisic correctness check that we end up parsing all of the global decl
434	/// attachments.
435	unsigned NumGlobalDeclAttachSkipped = 0;
436	unsigned NumGlobalDeclAttachParsed = 0;
437	#endif
438
439	/// Load the global decl attachments, using the index built for lazy loading.
440	Expected<bool> loadGlobalDeclAttachments();
441
442	/// Populate the index above to enable lazily loading of metadata, and load
443	/// the named metadata as well as the transitively referenced global
444	/// Metadata.
445	Expected<bool> lazyLoadModuleMetadataBlock();
446
447	/// On-demand loading of a single metadata. Requires the index above to be
448	/// populated.
449	void lazyLoadOneMetadata(unsigned Idx, PlaceholderQueue &Placeholders);
450
451	// Keep mapping of seens pair of old-style CU <-> SP, and update pointers to
452	// point from SP to CU after a block is completly parsed.
453	std::vector<std::pair<DICompileUnit *, Metadata *>> CUSubprograms;
454
455	/// Functions that need to be matched with subprograms when upgrading old
456	/// metadata.
457	SmallDenseMap<Function *, DISubprogram *, 16> FunctionsWithSPs;
458
459	// Map the bitcode's custom MDKind ID to the Module's MDKind ID.
460	DenseMap<unsigned, unsigned> MDKindMap;
461
462	bool StripTBAA = false;
463	bool HasSeenOldLoopTags = false;
464	bool NeedUpgradeToDIGlobalVariableExpression = false;
465	bool NeedDeclareExpressionUpgrade = false;
466
467	/// Map DILocalScope to the enclosing DISubprogram, if any.
468	DenseMap<DILocalScope *, DISubprogram *> ParentSubprogram;
469
470	/// True if metadata is being parsed for a module being ThinLTO imported.
471	bool IsImporting = false;
472
473	Error parseOneMetadata(SmallVectorImpl<uint64_t> &Record, unsigned Code,
474	PlaceholderQueue &Placeholders, StringRef Blob,
475	unsigned &NextMetadataNo);
476	Error parseMetadataStrings(ArrayRef<uint64_t> Record, StringRef Blob,
477	function_ref<void(StringRef)> CallBack);
478	Error parseGlobalObjectAttachment(GlobalObject &GO,
479	ArrayRef<uint64_t> Record);
480	Error parseMetadataKindRecord(SmallVectorImpl<uint64_t> &Record);
481
482	void resolveForwardRefsAndPlaceholders(PlaceholderQueue &Placeholders);
483
484	/// Upgrade old-style CU <-> SP pointers to point from SP to CU.
485	void upgradeCUSubprograms() {
486	for (auto CU_SP : CUSubprograms)
487	if (auto *SPs = dyn_cast_or_null<MDTuple>(Val: CU_SP.second))
488	for (auto &Op : SPs->operands())
489	if (auto *SP = dyn_cast_or_null<DISubprogram>(Val: Op))
490	SP->replaceUnit(CU: CU_SP.first);
491	CUSubprograms.clear();
492	}
493
494	/// Upgrade old-style bare DIGlobalVariables to DIGlobalVariableExpressions.
495	void upgradeCUVariables() {
496	if (!NeedUpgradeToDIGlobalVariableExpression)
497	return;
498
499	// Upgrade list of variables attached to the CUs.
500	if (NamedMDNode *CUNodes = TheModule.getNamedMetadata(Name: "llvm.dbg.cu"))
501	for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {
502	auto *CU = cast<DICompileUnit>(Val: CUNodes->getOperand(i: I));
503	if (auto *GVs = dyn_cast_or_null<MDTuple>(Val: CU->getRawGlobalVariables()))
504	for (unsigned I = 0; I < GVs->getNumOperands(); I++)
505	if (auto *GV =
506	dyn_cast_or_null<DIGlobalVariable>(Val: GVs->getOperand(I))) {
507	auto *DGVE = DIGlobalVariableExpression::getDistinct(
508	Context, Variable: GV, Expression: DIExpression::get(Context, Elements: {}));
509	GVs->replaceOperandWith(I, New: DGVE);
510	}
511	}
512
513	// Upgrade variables attached to globals.
514	for (auto &GV : TheModule.globals()) {
515	SmallVector<MDNode *, 1> MDs;
516	GV.getMetadata(KindID: LLVMContext::MD_dbg, MDs);
517	GV.eraseMetadata(KindID: LLVMContext::MD_dbg);
518	for (auto *MD : MDs)
519	if (auto *DGV = dyn_cast<DIGlobalVariable>(Val: MD)) {
520	auto *DGVE = DIGlobalVariableExpression::getDistinct(
521	Context, Variable: DGV, Expression: DIExpression::get(Context, Elements: {}));
522	GV.addMetadata(KindID: LLVMContext::MD_dbg, MD&: *DGVE);
523	} else
524	GV.addMetadata(KindID: LLVMContext::MD_dbg, MD&: *MD);
525	}
526	}
527
528	DISubprogram *findEnclosingSubprogram(DILocalScope *S) {
529	if (!S)
530	return nullptr;
531	if (auto *SP = ParentSubprogram[S]) {
532	return SP;
533	}
534
535	DILocalScope *InitialScope = S;
536	DenseSet<DILocalScope *> Visited;
537	while (S && !isa<DISubprogram>(Val: S)) {
538	S = dyn_cast_or_null<DILocalScope>(Val: S->getScope());
539	if (Visited.contains(V: S))
540	break;
541	Visited.insert(V: S);
542	}
543	ParentSubprogram[InitialScope] = llvm::dyn_cast_or_null<DISubprogram>(Val: S);
544
545	return ParentSubprogram[InitialScope];
546	}
547
548	/// Move local imports from DICompileUnit's 'imports' field to
549	/// DISubprogram's retainedNodes.
550	void upgradeCULocals() {
551	if (NamedMDNode *CUNodes = TheModule.getNamedMetadata(Name: "llvm.dbg.cu")) {
552	for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {
553	auto *CU = dyn_cast<DICompileUnit>(Val: CUNodes->getOperand(i: I));
554	if (!CU)
555	continue;
556
557	if (CU->getRawImportedEntities()) {
558	// Collect a set of imported entities to be moved.
559	SetVector<Metadata *> EntitiesToRemove;
560	for (Metadata *Op : CU->getImportedEntities()->operands()) {
561	auto *IE = cast<DIImportedEntity>(Val: Op);
562	if (dyn_cast_or_null<DILocalScope>(Val: IE->getScope())) {
563	EntitiesToRemove.insert(X: IE);
564	}
565	}
566
567	if (!EntitiesToRemove.empty()) {
568	// Make a new list of CU's 'imports'.
569	SmallVector<Metadata *> NewImports;
570	for (Metadata *Op : CU->getImportedEntities()->operands()) {
571	if (!EntitiesToRemove.contains(key: cast<DIImportedEntity>(Val: Op))) {
572	NewImports.push_back(Elt: Op);
573	}
574	}
575
576	// Find DISubprogram corresponding to each entity.
577	std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
578	for (auto *I : EntitiesToRemove) {
579	auto *Entity = cast<DIImportedEntity>(Val: I);
580	if (auto *SP = findEnclosingSubprogram(
581	S: cast<DILocalScope>(Val: Entity->getScope()))) {
582	SPToEntities[SP].push_back(Elt: Entity);
583	}
584	}
585
586	// Update DISubprograms' retainedNodes.
587	for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
588	auto *SP = I->first;
589	auto RetainedNodes = SP->getRetainedNodes();
590	SmallVector<Metadata *> MDs(RetainedNodes.begin(),
591	RetainedNodes.end());
592	MDs.append(RHS: I->second);
593	SP->replaceRetainedNodes(N: MDNode::get(Context, MDs));
594	}
595
596	// Remove entities with local scope from CU.
597	CU->replaceImportedEntities(N: MDTuple::get(Context, MDs: NewImports));
598	}
599	}
600	}
601	}
602
603	ParentSubprogram.clear();
604	}
605
606	/// Remove a leading DW_OP_deref from DIExpressions in a dbg.declare that
607	/// describes a function argument.
608	void upgradeDeclareExpressions(Function &F) {
609	if (!NeedDeclareExpressionUpgrade)
610	return;
611
612	auto UpdateDeclareIfNeeded = [&](auto *Declare) {
613	auto *DIExpr = Declare->getExpression();
614	if (!DIExpr || !DIExpr->startsWithDeref() ||
615	!isa_and_nonnull<Argument>(Declare->getAddress()))
616	return;
617	SmallVector<uint64_t, 8> Ops;
618	Ops.append(std::next(DIExpr->elements_begin()), DIExpr->elements_end());
619	Declare->setExpression(DIExpression::get(Context, Elements: Ops));
620	};
621
622	for (auto &BB : F)
623	for (auto &I : BB) {
624	for (DbgVariableRecord &DVR : filterDbgVars(R: I.getDbgRecordRange())) {
625	if (DVR.isDbgDeclare())
626	UpdateDeclareIfNeeded(&DVR);
627	}
628	if (auto *DDI = dyn_cast<DbgDeclareInst>(Val: &I))
629	UpdateDeclareIfNeeded(DDI);
630	}
631	}
632
633	/// Upgrade the expression from previous versions.
634	Error upgradeDIExpression(uint64_t FromVersion,
635	MutableArrayRef<uint64_t> &Expr,
636	SmallVectorImpl<uint64_t> &Buffer) {
637	auto N = Expr.size();
638	switch (FromVersion) {
639	default:
640	return error(Message: "Invalid record");
641	case 0:
642	if (N >= 3 && Expr[N - 3] == dwarf::DW_OP_bit_piece)
643	Expr[N - 3] = dwarf::DW_OP_LLVM_fragment;
644	[[fallthrough]];
645	case 1:
646	// Move DW_OP_deref to the end.
647	if (N && Expr[0] == dwarf::DW_OP_deref) {
648	auto End = Expr.end();
649	if (Expr.size() >= 3 &&
650	*std::prev(x: End, n: 3) == dwarf::DW_OP_LLVM_fragment)
651	End = std::prev(x: End, n: 3);
652	std::move(first: std::next(x: Expr.begin()), last: End, result: Expr.begin());
653	*std::prev(x: End) = dwarf::DW_OP_deref;
654	}
655	NeedDeclareExpressionUpgrade = true;
656	[[fallthrough]];
657	case 2: {
658	// Change DW_OP_plus to DW_OP_plus_uconst.
659	// Change DW_OP_minus to DW_OP_uconst, DW_OP_minus
660	auto SubExpr = ArrayRef<uint64_t>(Expr);
661	while (!SubExpr.empty()) {
662	// Skip past other operators with their operands
663	// for this version of the IR, obtained from
664	// from historic DIExpression::ExprOperand::getSize().
665	size_t HistoricSize;
666	switch (SubExpr.front()) {
667	default:
668	HistoricSize = 1;
669	break;
670	case dwarf::DW_OP_constu:
671	case dwarf::DW_OP_minus:
672	case dwarf::DW_OP_plus:
673	HistoricSize = 2;
674	break;
675	case dwarf::DW_OP_LLVM_fragment:
676	HistoricSize = 3;
677	break;
678	}
679
680	// If the expression is malformed, make sure we don't
681	// copy more elements than we should.
682	HistoricSize = std::min(a: SubExpr.size(), b: HistoricSize);
683	ArrayRef<uint64_t> Args = SubExpr.slice(N: 1, M: HistoricSize - 1);
684
685	switch (SubExpr.front()) {
686	case dwarf::DW_OP_plus:
687	Buffer.push_back(Elt: dwarf::DW_OP_plus_uconst);
688	Buffer.append(in_start: Args.begin(), in_end: Args.end());
689	break;
690	case dwarf::DW_OP_minus:
691	Buffer.push_back(Elt: dwarf::DW_OP_constu);
692	Buffer.append(in_start: Args.begin(), in_end: Args.end());
693	Buffer.push_back(Elt: dwarf::DW_OP_minus);
694	break;
695	default:
696	Buffer.push_back(Elt: *SubExpr.begin());
697	Buffer.append(in_start: Args.begin(), in_end: Args.end());
698	break;
699	}
700
701	// Continue with remaining elements.
702	SubExpr = SubExpr.slice(N: HistoricSize);
703	}
704	Expr = MutableArrayRef<uint64_t>(Buffer);
705	[[fallthrough]];
706	}
707	case 3:
708	// Up-to-date!
709	break;
710	}
711
712	return Error::success();
713	}
714
715	void upgradeDebugInfo(bool ModuleLevel) {
716	upgradeCUSubprograms();
717	upgradeCUVariables();
718	if (ModuleLevel)
719	upgradeCULocals();
720	}
721
722	void callMDTypeCallback(Metadata **Val, unsigned TypeID);
723
724	public:
725	MetadataLoaderImpl(BitstreamCursor &Stream, Module &TheModule,
726	BitcodeReaderValueList &ValueList,
727	MetadataLoaderCallbacks Callbacks, bool IsImporting)
728	: MetadataList(TheModule.getContext(), Stream.SizeInBytes()),
729	ValueList(ValueList), Stream(Stream), Context(TheModule.getContext()),
730	TheModule(TheModule), Callbacks(std::move(Callbacks)),
731	IsImporting(IsImporting) {}
732
733	Error parseMetadata(bool ModuleLevel);
734
735	bool hasFwdRefs() const { return MetadataList.hasFwdRefs(); }
736
737	Metadata *getMetadataFwdRefOrLoad(unsigned ID) {
738	if (ID < MDStringRef.size())
739	return lazyLoadOneMDString(Idx: ID);
740	if (auto *MD = MetadataList.lookup(I: ID))
741	return MD;
742	// If lazy-loading is enabled, we try recursively to load the operand
743	// instead of creating a temporary.
744	if (ID < (MDStringRef.size() + GlobalMetadataBitPosIndex.size())) {
745	PlaceholderQueue Placeholders;
746	lazyLoadOneMetadata(Idx: ID, Placeholders);
747	resolveForwardRefsAndPlaceholders(Placeholders);
748	return MetadataList.lookup(I: ID);
749	}
750	return MetadataList.getMetadataFwdRef(Idx: ID);
751	}
752
753	DISubprogram *lookupSubprogramForFunction(Function *F) {
754	return FunctionsWithSPs.lookup(Val: F);
755	}
756
757	bool hasSeenOldLoopTags() const { return HasSeenOldLoopTags; }
758
759	Error parseMetadataAttachment(Function &F,
760	ArrayRef<Instruction *> InstructionList);
761
762	Error parseMetadataKinds();
763
764	void setStripTBAA(bool Value) { StripTBAA = Value; }
765	bool isStrippingTBAA() const { return StripTBAA; }
766
767	unsigned size() const { return MetadataList.size(); }
768	void shrinkTo(unsigned N) { MetadataList.shrinkTo(N); }
769	void upgradeDebugIntrinsics(Function &F) { upgradeDeclareExpressions(F); }
770	};
771
772	Expected<bool>
773	MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() {
774	IndexCursor = Stream;
775	SmallVector<uint64_t, 64> Record;
776	GlobalDeclAttachmentPos = 0;
777	// Get the abbrevs, and preload record positions to make them lazy-loadable.
778	while (true) {
779	uint64_t SavedPos = IndexCursor.GetCurrentBitNo();
780	BitstreamEntry Entry;
781	if (Error E =
782	IndexCursor
783	.advanceSkippingSubblocks(Flags: BitstreamCursor::AF_DontPopBlockAtEnd)
784	.moveInto(Value&: Entry))
785	return std::move(E);
786
787	switch (Entry.Kind) {
788	case BitstreamEntry::SubBlock: // Handled for us already.
789	case BitstreamEntry::Error:
790	return error(Message: "Malformed block");
791	case BitstreamEntry::EndBlock: {
792	return true;
793	}
794	case BitstreamEntry::Record: {
795	// The interesting case.
796	++NumMDRecordLoaded;
797	uint64_t CurrentPos = IndexCursor.GetCurrentBitNo();
798	unsigned Code;
799	if (Error E = IndexCursor.skipRecord(AbbrevID: Entry.ID).moveInto(Value&: Code))
800	return std::move(E);
801	switch (Code) {
802	case bitc::METADATA_STRINGS: {
803	// Rewind and parse the strings.
804	if (Error Err = IndexCursor.JumpToBit(BitNo: CurrentPos))
805	return std::move(Err);
806	StringRef Blob;
807	Record.clear();
808	if (Expected<unsigned> MaybeRecord =
809	IndexCursor.readRecord(AbbrevID: Entry.ID, Vals&: Record, Blob: &Blob))
810	;
811	else
812	return MaybeRecord.takeError();
813	unsigned NumStrings = Record[0];
814	MDStringRef.reserve(n: NumStrings);
815	auto IndexNextMDString = [&](StringRef Str) {
816	MDStringRef.push_back(x: Str);
817	};
818	if (auto Err = parseMetadataStrings(Record, Blob, CallBack: IndexNextMDString))
819	return std::move(Err);
820	break;
821	}
822	case bitc::METADATA_INDEX_OFFSET: {
823	// This is the offset to the index, when we see this we skip all the
824	// records and load only an index to these.
825	if (Error Err = IndexCursor.JumpToBit(BitNo: CurrentPos))
826	return std::move(Err);
827	Record.clear();
828	if (Expected<unsigned> MaybeRecord =
829	IndexCursor.readRecord(AbbrevID: Entry.ID, Vals&: Record))
830	;
831	else
832	return MaybeRecord.takeError();
833	if (Record.size() != 2)
834	return error(Message: "Invalid record");
835	auto Offset = Record[0] + (Record[1] << 32);
836	auto BeginPos = IndexCursor.GetCurrentBitNo();
837	if (Error Err = IndexCursor.JumpToBit(BitNo: BeginPos + Offset))
838	return std::move(Err);
839	Expected<BitstreamEntry> MaybeEntry =
840	IndexCursor.advanceSkippingSubblocks(
841	Flags: BitstreamCursor::AF_DontPopBlockAtEnd);
842	if (!MaybeEntry)
843	return MaybeEntry.takeError();
844	Entry = MaybeEntry.get();
845	assert(Entry.Kind == BitstreamEntry::Record &&
846	"Corrupted bitcode: Expected `Record` when trying to find the "
847	"Metadata index");
848	Record.clear();
849	if (Expected<unsigned> MaybeCode =
850	IndexCursor.readRecord(AbbrevID: Entry.ID, Vals&: Record))
851	assert(MaybeCode.get() == bitc::METADATA_INDEX &&
852	"Corrupted bitcode: Expected `METADATA_INDEX` when trying to "
853	"find the Metadata index");
854	else
855	return MaybeCode.takeError();
856	// Delta unpack
857	auto CurrentValue = BeginPos;
858	GlobalMetadataBitPosIndex.reserve(n: Record.size());
859	for (auto &Elt : Record) {
860	CurrentValue += Elt;
861	GlobalMetadataBitPosIndex.push_back(x: CurrentValue);
862	}
863	break;
864	}
865	case bitc::METADATA_INDEX:
866	// We don't expect to get there, the Index is loaded when we encounter
867	// the offset.
868	return error(Message: "Corrupted Metadata block");
869	case bitc::METADATA_NAME: {
870	// Named metadata need to be materialized now and aren't deferred.
871	if (Error Err = IndexCursor.JumpToBit(BitNo: CurrentPos))
872	return std::move(Err);
873	Record.clear();
874
875	unsigned Code;
876	if (Expected<unsigned> MaybeCode =
877	IndexCursor.readRecord(AbbrevID: Entry.ID, Vals&: Record)) {
878	Code = MaybeCode.get();
879	assert(Code == bitc::METADATA_NAME);
880	} else
881	return MaybeCode.takeError();
882
883	// Read name of the named metadata.
884	SmallString<8> Name(Record.begin(), Record.end());
885	if (Expected<unsigned> MaybeCode = IndexCursor.ReadCode())
886	Code = MaybeCode.get();
887	else
888	return MaybeCode.takeError();
889
890	// Named Metadata comes in two parts, we expect the name to be followed
891	// by the node
892	Record.clear();
893	if (Expected<unsigned> MaybeNextBitCode =
894	IndexCursor.readRecord(AbbrevID: Code, Vals&: Record))
895	assert(MaybeNextBitCode.get() == bitc::METADATA_NAMED_NODE);
896	else
897	return MaybeNextBitCode.takeError();
898
899	// Read named metadata elements.
900	unsigned Size = Record.size();
901	NamedMDNode *NMD = TheModule.getOrInsertNamedMetadata(Name);
902	for (unsigned i = 0; i != Size; ++i) {
903	// FIXME: We could use a placeholder here, however NamedMDNode are
904	// taking MDNode as operand and not using the Metadata infrastructure.
905	// It is acknowledged by 'TODO: Inherit from Metadata' in the
906	// NamedMDNode class definition.
907	MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Idx: Record[i]);
908	assert(MD && "Invalid metadata: expect fwd ref to MDNode");
909	NMD->addOperand(M: MD);
910	}
911	break;
912	}
913	case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: {
914	if (!GlobalDeclAttachmentPos)
915	GlobalDeclAttachmentPos = SavedPos;
916	#ifndef NDEBUG
917	NumGlobalDeclAttachSkipped++;
918	#endif
919	break;
920	}
921	case bitc::METADATA_KIND:
922	case bitc::METADATA_STRING_OLD:
923	case bitc::METADATA_OLD_FN_NODE:
924	case bitc::METADATA_OLD_NODE:
925	case bitc::METADATA_VALUE:
926	case bitc::METADATA_DISTINCT_NODE:
927	case bitc::METADATA_NODE:
928	case bitc::METADATA_LOCATION:
929	case bitc::METADATA_GENERIC_DEBUG:
930	case bitc::METADATA_SUBRANGE:
931	case bitc::METADATA_ENUMERATOR:
932	case bitc::METADATA_BASIC_TYPE:
933	case bitc::METADATA_STRING_TYPE:
934	case bitc::METADATA_DERIVED_TYPE:
935	case bitc::METADATA_COMPOSITE_TYPE:
936	case bitc::METADATA_SUBROUTINE_TYPE:
937	case bitc::METADATA_MODULE:
938	case bitc::METADATA_FILE:
939	case bitc::METADATA_COMPILE_UNIT:
940	case bitc::METADATA_SUBPROGRAM:
941	case bitc::METADATA_LEXICAL_BLOCK:
942	case bitc::METADATA_LEXICAL_BLOCK_FILE:
943	case bitc::METADATA_NAMESPACE:
944	case bitc::METADATA_COMMON_BLOCK:
945	case bitc::METADATA_MACRO:
946	case bitc::METADATA_MACRO_FILE:
947	case bitc::METADATA_TEMPLATE_TYPE:
948	case bitc::METADATA_TEMPLATE_VALUE:
949	case bitc::METADATA_GLOBAL_VAR:
950	case bitc::METADATA_LOCAL_VAR:
951	case bitc::METADATA_ASSIGN_ID:
952	case bitc::METADATA_LABEL:
953	case bitc::METADATA_EXPRESSION:
954	case bitc::METADATA_OBJC_PROPERTY:
955	case bitc::METADATA_IMPORTED_ENTITY:
956	case bitc::METADATA_GLOBAL_VAR_EXPR:
957	case bitc::METADATA_GENERIC_SUBRANGE:
958	// We don't expect to see any of these, if we see one, give up on
959	// lazy-loading and fallback.
960	MDStringRef.clear();
961	GlobalMetadataBitPosIndex.clear();
962	return false;
963	}
964	break;
965	}
966	}
967	}
968	}
969
970	// Load the global decl attachments after building the lazy loading index.
971	// We don't load them "lazily" - all global decl attachments must be
972	// parsed since they aren't materialized on demand. However, by delaying
973	// their parsing until after the index is created, we can use the index
974	// instead of creating temporaries.
975	Expected<bool> MetadataLoader::MetadataLoaderImpl::loadGlobalDeclAttachments() {
976	// Nothing to do if we didn't find any of these metadata records.
977	if (!GlobalDeclAttachmentPos)
978	return true;
979	// Use a temporary cursor so that we don't mess up the main Stream cursor or
980	// the lazy loading IndexCursor (which holds the necessary abbrev ids).
981	BitstreamCursor TempCursor = Stream;
982	SmallVector<uint64_t, 64> Record;
983	// Jump to the position before the first global decl attachment, so we can
984	// scan for the first BitstreamEntry record.
985	if (Error Err = TempCursor.JumpToBit(BitNo: GlobalDeclAttachmentPos))
986	return std::move(Err);
987	while (true) {
988	BitstreamEntry Entry;
989	if (Error E =
990	TempCursor
991	.advanceSkippingSubblocks(Flags: BitstreamCursor::AF_DontPopBlockAtEnd)
992	.moveInto(Value&: Entry))
993	return std::move(E);
994
995	switch (Entry.Kind) {
996	case BitstreamEntry::SubBlock: // Handled for us already.
997	case BitstreamEntry::Error:
998	return error(Message: "Malformed block");
999	case BitstreamEntry::EndBlock:
1000	// Check that we parsed them all.
1001	assert(NumGlobalDeclAttachSkipped == NumGlobalDeclAttachParsed);
1002	return true;
1003	case BitstreamEntry::Record:
1004	break;
1005	}
1006	uint64_t CurrentPos = TempCursor.GetCurrentBitNo();
1007	Expected<unsigned> MaybeCode = TempCursor.skipRecord(AbbrevID: Entry.ID);
1008	if (!MaybeCode)
1009	return MaybeCode.takeError();
1010	if (MaybeCode.get() != bitc::METADATA_GLOBAL_DECL_ATTACHMENT) {
1011	// Anything other than a global decl attachment signals the end of
1012	// these records. Check that we parsed them all.
1013	assert(NumGlobalDeclAttachSkipped == NumGlobalDeclAttachParsed);
1014	return true;
1015	}
1016	#ifndef NDEBUG
1017	NumGlobalDeclAttachParsed++;
1018	#endif
1019	// FIXME: we need to do this early because we don't materialize global
1020	// value explicitly.
1021	if (Error Err = TempCursor.JumpToBit(BitNo: CurrentPos))
1022	return std::move(Err);
1023	Record.clear();
1024	if (Expected<unsigned> MaybeRecord =
1025	TempCursor.readRecord(AbbrevID: Entry.ID, Vals&: Record))
1026	;
1027	else
1028	return MaybeRecord.takeError();
1029	if (Record.size() % 2 == 0)
1030	return error(Message: "Invalid record");
1031	unsigned ValueID = Record[0];
1032	if (ValueID >= ValueList.size())
1033	return error(Message: "Invalid record");
1034	if (auto *GO = dyn_cast<GlobalObject>(Val: ValueList[ValueID])) {
1035	// Need to save and restore the current position since
1036	// parseGlobalObjectAttachment will resolve all forward references which
1037	// would require parsing from locations stored in the index.
1038	CurrentPos = TempCursor.GetCurrentBitNo();
1039	if (Error Err = parseGlobalObjectAttachment(
1040	GO&: *GO, Record: ArrayRef<uint64_t>(Record).slice(N: 1)))
1041	return std::move(Err);
1042	if (Error Err = TempCursor.JumpToBit(BitNo: CurrentPos))
1043	return std::move(Err);
1044	}
1045	}
1046	}
1047
1048	void MetadataLoader::MetadataLoaderImpl::callMDTypeCallback(Metadata **Val,
1049	unsigned TypeID) {
1050	if (Callbacks.MDType) {
1051	(*Callbacks.MDType)(Val, TypeID, Callbacks.GetTypeByID,
1052	Callbacks.GetContainedTypeID);
1053	}
1054	}
1055
1056	/// Parse a METADATA_BLOCK. If ModuleLevel is true then we are parsing
1057	/// module level metadata.
1058	Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) {
1059	if (!ModuleLevel && MetadataList.hasFwdRefs())
1060	return error(Message: "Invalid metadata: fwd refs into function blocks");
1061
1062	// Record the entry position so that we can jump back here and efficiently
1063	// skip the whole block in case we lazy-load.
1064	auto EntryPos = Stream.GetCurrentBitNo();
1065
1066	if (Error Err = Stream.EnterSubBlock(BlockID: bitc::METADATA_BLOCK_ID))
1067	return Err;
1068
1069	SmallVector<uint64_t, 64> Record;
1070	PlaceholderQueue Placeholders;
1071
1072	// We lazy-load module-level metadata: we build an index for each record, and
1073	// then load individual record as needed, starting with the named metadata.
1074	if (ModuleLevel && IsImporting && MetadataList.empty() &&
1075	!DisableLazyLoading) {
1076	auto SuccessOrErr = lazyLoadModuleMetadataBlock();
1077	if (!SuccessOrErr)
1078	return SuccessOrErr.takeError();
1079	if (SuccessOrErr.get()) {
1080	// An index was successfully created and we will be able to load metadata
1081	// on-demand.
1082	MetadataList.resize(N: MDStringRef.size() +
1083	GlobalMetadataBitPosIndex.size());
1084
1085	// Now that we have built the index, load the global decl attachments
1086	// that were deferred during that process. This avoids creating
1087	// temporaries.
1088	SuccessOrErr = loadGlobalDeclAttachments();
1089	if (!SuccessOrErr)
1090	return SuccessOrErr.takeError();
1091	assert(SuccessOrErr.get());
1092
1093	// Reading the named metadata created forward references and/or
1094	// placeholders, that we flush here.
1095	resolveForwardRefsAndPlaceholders(Placeholders);
1096	upgradeDebugInfo(ModuleLevel);
1097	// Return at the beginning of the block, since it is easy to skip it
1098	// entirely from there.
1099	Stream.ReadBlockEnd(); // Pop the abbrev block context.
1100	if (Error Err = IndexCursor.JumpToBit(BitNo: EntryPos))
1101	return Err;
1102	if (Error Err = Stream.SkipBlock()) {
1103	// FIXME this drops the error on the floor, which
1104	// ThinLTO/X86/debuginfo-cu-import.ll relies on.
1105	consumeError(Err: std::move(Err));
1106	return Error::success();
1107	}
1108	return Error::success();
1109	}
1110	// Couldn't load an index, fallback to loading all the block "old-style".
1111	}
1112
1113	unsigned NextMetadataNo = MetadataList.size();
1114
1115	// Read all the records.
1116	while (true) {
1117	BitstreamEntry Entry;
1118	if (Error E = Stream.advanceSkippingSubblocks().moveInto(Value&: Entry))
1119	return E;
1120
1121	switch (Entry.Kind) {
1122	case BitstreamEntry::SubBlock: // Handled for us already.
1123	case BitstreamEntry::Error:
1124	return error(Message: "Malformed block");
1125	case BitstreamEntry::EndBlock:
1126	resolveForwardRefsAndPlaceholders(Placeholders);
1127	upgradeDebugInfo(ModuleLevel);
1128	return Error::success();
1129	case BitstreamEntry::Record:
1130	// The interesting case.
1131	break;
1132	}
1133
1134	// Read a record.
1135	Record.clear();
1136	StringRef Blob;
1137	++NumMDRecordLoaded;
1138	if (Expected<unsigned> MaybeCode =
1139	Stream.readRecord(AbbrevID: Entry.ID, Vals&: Record, Blob: &Blob)) {
1140	if (Error Err = parseOneMetadata(Record, Code: MaybeCode.get(), Placeholders,
1141	Blob, NextMetadataNo))
1142	return Err;
1143	} else
1144	return MaybeCode.takeError();
1145	}
1146	}
1147
1148	MDString *MetadataLoader::MetadataLoaderImpl::lazyLoadOneMDString(unsigned ID) {
1149	++NumMDStringLoaded;
1150	if (Metadata *MD = MetadataList.lookup(I: ID))
1151	return cast<MDString>(Val: MD);
1152	auto MDS = MDString::get(Context, Str: MDStringRef[ID]);
1153	MetadataList.assignValue(MD: MDS, Idx: ID);
1154	return MDS;
1155	}
1156
1157	void MetadataLoader::MetadataLoaderImpl::lazyLoadOneMetadata(
1158	unsigned ID, PlaceholderQueue &Placeholders) {
1159	assert(ID < (MDStringRef.size()) + GlobalMetadataBitPosIndex.size());
1160	assert(ID >= MDStringRef.size() && "Unexpected lazy-loading of MDString");
1161	// Lookup first if the metadata hasn't already been loaded.
1162	if (auto *MD = MetadataList.lookup(I: ID)) {
1163	auto *N = cast<MDNode>(Val: MD);
1164	if (!N->isTemporary())
1165	return;
1166	}
1167	SmallVector<uint64_t, 64> Record;
1168	StringRef Blob;
1169	if (Error Err = IndexCursor.JumpToBit(
1170	BitNo: GlobalMetadataBitPosIndex[ID - MDStringRef.size()]))
1171	report_fatal_error(reason: "lazyLoadOneMetadata failed jumping: " +
1172	Twine(toString(E: std::move(Err))));
1173	BitstreamEntry Entry;
1174	if (Error E = IndexCursor.advanceSkippingSubblocks().moveInto(Value&: Entry))
1175	// FIXME this drops the error on the floor.
1176	report_fatal_error(reason: "lazyLoadOneMetadata failed advanceSkippingSubblocks: " +
1177	Twine(toString(E: std::move(E))));
1178	++NumMDRecordLoaded;
1179	if (Expected<unsigned> MaybeCode =
1180	IndexCursor.readRecord(AbbrevID: Entry.ID, Vals&: Record, Blob: &Blob)) {
1181	if (Error Err =
1182	parseOneMetadata(Record, Code: MaybeCode.get(), Placeholders, Blob, NextMetadataNo&: ID))
1183	report_fatal_error(reason: "Can't lazyload MD, parseOneMetadata: " +
1184	Twine(toString(E: std::move(Err))));
1185	} else
1186	report_fatal_error(reason: "Can't lazyload MD: " +
1187	Twine(toString(E: MaybeCode.takeError())));
1188	}
1189
1190	/// Ensure that all forward-references and placeholders are resolved.
1191	/// Iteratively lazy-loading metadata on-demand if needed.
1192	void MetadataLoader::MetadataLoaderImpl::resolveForwardRefsAndPlaceholders(
1193	PlaceholderQueue &Placeholders) {
1194	DenseSet<unsigned> Temporaries;
1195	while (true) {
1196	// Populate Temporaries with the placeholders that haven't been loaded yet.
1197	Placeholders.getTemporaries(MetadataList, Temporaries);
1198
1199	// If we don't have any temporary, or FwdReference, we're done!
1200	if (Temporaries.empty() && !MetadataList.hasFwdRefs())
1201	break;
1202
1203	// First, load all the temporaries. This can add new placeholders or
1204	// forward references.
1205	for (auto ID : Temporaries)
1206	lazyLoadOneMetadata(ID, Placeholders);
1207	Temporaries.clear();
1208
1209	// Second, load the forward-references. This can also add new placeholders
1210	// or forward references.
1211	while (MetadataList.hasFwdRefs())
1212	lazyLoadOneMetadata(ID: MetadataList.getNextFwdRef(), Placeholders);
1213	}
1214	// At this point we don't have any forward reference remaining, or temporary
1215	// that haven't been loaded. We can safely drop RAUW support and mark cycles
1216	// as resolved.
1217	MetadataList.tryToResolveCycles();
1218
1219	// Finally, everything is in place, we can replace the placeholders operands
1220	// with the final node they refer to.
1221	Placeholders.flush(MetadataList);
1222	}
1223
1224	static Value *getValueFwdRef(BitcodeReaderValueList &ValueList, unsigned Idx,
1225	Type *Ty, unsigned TyID) {
1226	Value *V = ValueList.getValueFwdRef(Idx, Ty, TyID,
1227	/*ConstExprInsertBB*/ nullptr);
1228	if (V)
1229	return V;
1230
1231	// This is a reference to a no longer supported constant expression.
1232	// Pretend that the constant was deleted, which will replace metadata
1233	// references with undef.
1234	// TODO: This is a rather indirect check. It would be more elegant to use
1235	// a separate ErrorInfo for constant materialization failure and thread
1236	// the error reporting through getValueFwdRef().
1237	if (Idx < ValueList.size() && ValueList[Idx] &&
1238	ValueList[Idx]->getType() == Ty)
1239	return UndefValue::get(T: Ty);
1240
1241	return nullptr;
1242	}
1243
1244	Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata(
1245	SmallVectorImpl<uint64_t> &Record, unsigned Code,
1246	PlaceholderQueue &Placeholders, StringRef Blob, unsigned &NextMetadataNo) {
1247
1248	bool IsDistinct = false;
1249	auto getMD = [&](unsigned ID) -> Metadata * {
1250	if (ID < MDStringRef.size())
1251	return lazyLoadOneMDString(ID);
1252	if (!IsDistinct) {
1253	if (auto *MD = MetadataList.lookup(I: ID))
1254	return MD;
1255	// If lazy-loading is enabled, we try recursively to load the operand
1256	// instead of creating a temporary.
1257	if (ID < (MDStringRef.size() + GlobalMetadataBitPosIndex.size())) {
1258	// Create a temporary for the node that is referencing the operand we
1259	// will lazy-load. It is needed before recursing in case there are
1260	// uniquing cycles.
1261	MetadataList.getMetadataFwdRef(Idx: NextMetadataNo);
1262	lazyLoadOneMetadata(ID, Placeholders);
1263	return MetadataList.lookup(I: ID);
1264	}
1265	// Return a temporary.
1266	return MetadataList.getMetadataFwdRef(Idx: ID);
1267	}
1268	if (auto *MD = MetadataList.getMetadataIfResolved(Idx: ID))
1269	return MD;
1270	return &Placeholders.getPlaceholderOp(ID);
1271	};
1272	auto getMDOrNull = [&](unsigned ID) -> Metadata * {
1273	if (ID)
1274	return getMD(ID - 1);
1275	return nullptr;
1276	};
1277	auto getMDOrNullWithoutPlaceholders = [&](unsigned ID) -> Metadata * {
1278	if (ID)
1279	return MetadataList.getMetadataFwdRef(Idx: ID - 1);
1280	return nullptr;
1281	};
1282	auto getMDString = [&](unsigned ID) -> MDString * {
1283	// This requires that the ID is not really a forward reference. In
1284	// particular, the MDString must already have been resolved.
1285	auto MDS = getMDOrNull(ID);
1286	return cast_or_null<MDString>(Val: MDS);
1287	};
1288
1289	// Support for old type refs.
1290	auto getDITypeRefOrNull = [&](unsigned ID) {
1291	return MetadataList.upgradeTypeRef(MaybeUUID: getMDOrNull(ID));
1292	};
1293
1294	#define GET_OR_DISTINCT(CLASS, ARGS) \
1295	(IsDistinct ? CLASS::getDistinct ARGS : CLASS::get ARGS)
1296
1297	switch (Code) {
1298	default: // Default behavior: ignore.
1299	break;
1300	case bitc::METADATA_NAME: {
1301	// Read name of the named metadata.
1302	SmallString<8> Name(Record.begin(), Record.end());
1303	Record.clear();
1304	if (Error E = Stream.ReadCode().moveInto(Value&: Code))
1305	return E;
1306
1307	++NumMDRecordLoaded;
1308	if (Expected<unsigned> MaybeNextBitCode = Stream.readRecord(AbbrevID: Code, Vals&: Record)) {
1309	if (MaybeNextBitCode.get() != bitc::METADATA_NAMED_NODE)
1310	return error(Message: "METADATA_NAME not followed by METADATA_NAMED_NODE");
1311	} else
1312	return MaybeNextBitCode.takeError();
1313
1314	// Read named metadata elements.
1315	unsigned Size = Record.size();
1316	NamedMDNode *NMD = TheModule.getOrInsertNamedMetadata(Name);
1317	for (unsigned i = 0; i != Size; ++i) {
1318	MDNode *MD = MetadataList.getMDNodeFwdRefOrNull(Idx: Record[i]);
1319	if (!MD)
1320	return error(Message: "Invalid named metadata: expect fwd ref to MDNode");
1321	NMD->addOperand(M: MD);
1322	}
1323	break;
1324	}
1325	case bitc::METADATA_OLD_FN_NODE: {
1326	// Deprecated, but still needed to read old bitcode files.
1327	// This is a LocalAsMetadata record, the only type of function-local
1328	// metadata.
1329	if (Record.size() % 2 == 1)
1330	return error(Message: "Invalid record");
1331
1332	// If this isn't a LocalAsMetadata record, we're dropping it. This used
1333	// to be legal, but there's no upgrade path.
1334	auto dropRecord = [&] {
1335	MetadataList.assignValue(MD: MDNode::get(Context, MDs: std::nullopt),
1336	Idx: NextMetadataNo);
1337	NextMetadataNo++;
1338	};
1339	if (Record.size() != 2) {
1340	dropRecord();
1341	break;
1342	}
1343
1344	unsigned TyID = Record[0];
1345	Type *Ty = Callbacks.GetTypeByID(TyID);
1346	if (!Ty || Ty->isMetadataTy() || Ty->isVoidTy()) {
1347	dropRecord();
1348	break;
1349	}
1350
1351	Value *V = ValueList.getValueFwdRef(Idx: Record[1], Ty, TyID,
1352	/*ConstExprInsertBB*/ nullptr);
1353	if (!V)
1354	return error(Message: "Invalid value reference from old fn metadata");
1355
1356	MetadataList.assignValue(MD: LocalAsMetadata::get(Local: V), Idx: NextMetadataNo);
1357	NextMetadataNo++;
1358	break;
1359	}
1360	case bitc::METADATA_OLD_NODE: {
1361	// Deprecated, but still needed to read old bitcode files.
1362	if (Record.size() % 2 == 1)
1363	return error(Message: "Invalid record");
1364
1365	unsigned Size = Record.size();
1366	SmallVector<Metadata *, 8> Elts;
1367	for (unsigned i = 0; i != Size; i += 2) {
1368	unsigned TyID = Record[i];
1369	Type *Ty = Callbacks.GetTypeByID(TyID);
1370	if (!Ty)
1371	return error(Message: "Invalid record");
1372	if (Ty->isMetadataTy())
1373	Elts.push_back(Elt: getMD(Record[i + 1]));
1374	else if (!Ty->isVoidTy()) {
1375	Value *V = getValueFwdRef(ValueList, Idx: Record[i + 1], Ty, TyID);
1376	if (!V)
1377	return error(Message: "Invalid value reference from old metadata");
1378	Metadata *MD = ValueAsMetadata::get(V);
1379	assert(isa<ConstantAsMetadata>(MD) &&
1380	"Expected non-function-local metadata");
1381	callMDTypeCallback(Val: &MD, TypeID: TyID);
1382	Elts.push_back(Elt: MD);
1383	} else
1384	Elts.push_back(Elt: nullptr);
1385	}
1386	MetadataList.assignValue(MD: MDNode::get(Context, MDs: Elts), Idx: NextMetadataNo);
1387	NextMetadataNo++;
1388	break;
1389	}
1390	case bitc::METADATA_VALUE: {
1391	if (Record.size() != 2)
1392	return error(Message: "Invalid record");
1393
1394	unsigned TyID = Record[0];
1395	Type *Ty = Callbacks.GetTypeByID(TyID);
1396	if (!Ty || Ty->isMetadataTy() || Ty->isVoidTy())
1397	return error(Message: "Invalid record");
1398
1399	Value *V = getValueFwdRef(ValueList, Idx: Record[1], Ty, TyID);
1400	if (!V)
1401	return error(Message: "Invalid value reference from metadata");
1402
1403	Metadata *MD = ValueAsMetadata::get(V);
1404	callMDTypeCallback(Val: &MD, TypeID: TyID);
1405	MetadataList.assignValue(MD, Idx: NextMetadataNo);
1406	NextMetadataNo++;
1407	break;
1408	}
1409	case bitc::METADATA_DISTINCT_NODE:
1410	IsDistinct = true;
1411	[[fallthrough]];
1412	case bitc::METADATA_NODE: {
1413	SmallVector<Metadata *, 8> Elts;
1414	Elts.reserve(N: Record.size());
1415	for (unsigned ID : Record)
1416	Elts.push_back(Elt: getMDOrNull(ID));
1417	MetadataList.assignValue(MD: IsDistinct ? MDNode::getDistinct(Context, MDs: Elts)
1418	: MDNode::get(Context, MDs: Elts),
1419	Idx: NextMetadataNo);
1420	NextMetadataNo++;
1421	break;
1422	}
1423	case bitc::METADATA_LOCATION: {
1424	if (Record.size() != 5 && Record.size() != 6)
1425	return error(Message: "Invalid record");
1426
1427	IsDistinct = Record[0];
1428	unsigned Line = Record[1];
1429	unsigned Column = Record[2];
1430	Metadata *Scope = getMD(Record[3]);
1431	Metadata *InlinedAt = getMDOrNull(Record[4]);
1432	bool ImplicitCode = Record.size() == 6 && Record[5];
1433	MetadataList.assignValue(
1434	GET_OR_DISTINCT(DILocation, (Context, Line, Column, Scope, InlinedAt,
1435	ImplicitCode)),
1436	Idx: NextMetadataNo);
1437	NextMetadataNo++;
1438	break;
1439	}
1440	case bitc::METADATA_GENERIC_DEBUG: {
1441	if (Record.size() < 4)
1442	return error(Message: "Invalid record");
1443
1444	IsDistinct = Record[0];
1445	unsigned Tag = Record[1];
1446	unsigned Version = Record[2];
1447
1448	if (Tag >= 1u << 16 || Version != 0)
1449	return error(Message: "Invalid record");
1450
1451	auto *Header = getMDString(Record[3]);
1452	SmallVector<Metadata *, 8> DwarfOps;
1453	for (unsigned I = 4, E = Record.size(); I != E; ++I)
1454	DwarfOps.push_back(Elt: getMDOrNull(Record[I]));
1455	MetadataList.assignValue(
1456	GET_OR_DISTINCT(GenericDINode, (Context, Tag, Header, DwarfOps)),
1457	Idx: NextMetadataNo);
1458	NextMetadataNo++;
1459	break;
1460	}
1461	case bitc::METADATA_SUBRANGE: {
1462	Metadata *Val = nullptr;
1463	// Operand 'count' is interpreted as:
1464	// - Signed integer (version 0)
1465	// - Metadata node (version 1)
1466	// Operand 'lowerBound' is interpreted as:
1467	// - Signed integer (version 0 and 1)
1468	// - Metadata node (version 2)
1469	// Operands 'upperBound' and 'stride' are interpreted as:
1470	// - Metadata node (version 2)
1471	switch (Record[0] >> 1) {
1472	case 0:
1473	Val = GET_OR_DISTINCT(DISubrange,
1474	(Context, Record[1], unrotateSign(Record[2])));
1475	break;
1476	case 1:
1477	Val = GET_OR_DISTINCT(DISubrange, (Context, getMDOrNull(Record[1]),
1478	unrotateSign(Record[2])));
1479	break;
1480	case 2:
1481	Val = GET_OR_DISTINCT(
1482	DISubrange, (Context, getMDOrNull(Record[1]), getMDOrNull(Record[2]),
1483	getMDOrNull(Record[3]), getMDOrNull(Record[4])));
1484	break;
1485	default:
1486	return error(Message: "Invalid record: Unsupported version of DISubrange");
1487	}
1488
1489	MetadataList.assignValue(MD: Val, Idx: NextMetadataNo);
1490	IsDistinct = Record[0] & 1;
1491	NextMetadataNo++;
1492	break;
1493	}
1494	case bitc::METADATA_GENERIC_SUBRANGE: {
1495	Metadata *Val = nullptr;
1496	Val = GET_OR_DISTINCT(DIGenericSubrange,
1497	(Context, getMDOrNull(Record[1]),
1498	getMDOrNull(Record[2]), getMDOrNull(Record[3]),
1499	getMDOrNull(Record[4])));
1500
1501	MetadataList.assignValue(MD: Val, Idx: NextMetadataNo);
1502	IsDistinct = Record[0] & 1;
1503	NextMetadataNo++;
1504	break;
1505	}
1506	case bitc::METADATA_ENUMERATOR: {
1507	if (Record.size() < 3)
1508	return error(Message: "Invalid record");
1509
1510	IsDistinct = Record[0] & 1;
1511	bool IsUnsigned = Record[0] & 2;
1512	bool IsBigInt = Record[0] & 4;
1513	APInt Value;
1514
1515	if (IsBigInt) {
1516	const uint64_t BitWidth = Record[1];
1517	const size_t NumWords = Record.size() - 3;
1518	Value = readWideAPInt(Vals: ArrayRef(&Record[3], NumWords), TypeBits: BitWidth);
1519	} else
1520	Value = APInt(64, unrotateSign(U: Record[1]), !IsUnsigned);
1521
1522	MetadataList.assignValue(
1523	GET_OR_DISTINCT(DIEnumerator,
1524	(Context, Value, IsUnsigned, getMDString(Record[2]))),
1525	Idx: NextMetadataNo);
1526	NextMetadataNo++;
1527	break;
1528	}
1529	case bitc::METADATA_BASIC_TYPE: {
1530	if (Record.size() < 6 || Record.size() > 7)
1531	return error(Message: "Invalid record");
1532
1533	IsDistinct = Record[0];
1534	DINode::DIFlags Flags = (Record.size() > 6)
1535	? static_cast<DINode::DIFlags>(Record[6])
1536	: DINode::FlagZero;
1537
1538	MetadataList.assignValue(
1539	GET_OR_DISTINCT(DIBasicType,
1540	(Context, Record[1], getMDString(Record[2]), Record[3],
1541	Record[4], Record[5], Flags)),
1542	Idx: NextMetadataNo);
1543	NextMetadataNo++;
1544	break;
1545	}
1546	case bitc::METADATA_STRING_TYPE: {
1547	if (Record.size() > 9 || Record.size() < 8)
1548	return error(Message: "Invalid record");
1549
1550	IsDistinct = Record[0];
1551	bool SizeIs8 = Record.size() == 8;
1552	// StringLocationExp (i.e. Record[5]) is added at a later time
1553	// than the other fields. The code here enables backward compatibility.
1554	Metadata *StringLocationExp = SizeIs8 ? nullptr : getMDOrNull(Record[5]);
1555	unsigned Offset = SizeIs8 ? 5 : 6;
1556	MetadataList.assignValue(
1557	GET_OR_DISTINCT(DIStringType,
1558	(Context, Record[1], getMDString(Record[2]),
1559	getMDOrNull(Record[3]), getMDOrNull(Record[4]),
1560	StringLocationExp, Record[Offset], Record[Offset + 1],
1561	Record[Offset + 2])),
1562	Idx: NextMetadataNo);
1563	NextMetadataNo++;
1564	break;
1565	}
1566	case bitc::METADATA_DERIVED_TYPE: {
1567	if (Record.size() < 12 || Record.size() > 15)
1568	return error(Message: "Invalid record");
1569
1570	// DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
1571	// that there is no DWARF address space associated with DIDerivedType.
1572	std::optional<unsigned> DWARFAddressSpace;
1573	if (Record.size() > 12 && Record[12])
1574	DWARFAddressSpace = Record[12] - 1;
1575
1576	Metadata *Annotations = nullptr;
1577	std::optional<DIDerivedType::PtrAuthData> PtrAuthData;
1578
1579	// Only look for annotations/ptrauth if both are allocated.
1580	// If not, we can't tell which was intended to be embedded, as both ptrauth
1581	// and annotations have been expected at Record[13] at various times.
1582	if (Record.size() > 14) {
1583	if (Record[13])
1584	Annotations = getMDOrNull(Record[13]);
1585	if (Record[14])
1586	PtrAuthData.emplace(args&: Record[14]);
1587	}
1588
1589	IsDistinct = Record[0];
1590	DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[10]);
1591	MetadataList.assignValue(
1592	GET_OR_DISTINCT(DIDerivedType,
1593	(Context, Record[1], getMDString(Record[2]),
1594	getMDOrNull(Record[3]), Record[4],
1595	getDITypeRefOrNull(Record[5]),
1596	getDITypeRefOrNull(Record[6]), Record[7], Record[8],
1597	Record[9], DWARFAddressSpace, PtrAuthData, Flags,
1598	getDITypeRefOrNull(Record[11]), Annotations)),
1599	Idx: NextMetadataNo);
1600	NextMetadataNo++;
1601	break;
1602	}
1603	case bitc::METADATA_COMPOSITE_TYPE: {
1604	if (Record.size() < 16 || Record.size() > 22)
1605	return error(Message: "Invalid record");
1606
1607	// If we have a UUID and this is not a forward declaration, lookup the
1608	// mapping.
1609	IsDistinct = Record[0] & 0x1;
1610	bool IsNotUsedInTypeRef = Record[0] >= 2;
1611	unsigned Tag = Record[1];
1612	MDString *Name = getMDString(Record[2]);
1613	Metadata *File = getMDOrNull(Record[3]);
1614	unsigned Line = Record[4];
1615	Metadata *Scope = getDITypeRefOrNull(Record[5]);
1616	Metadata *BaseType = nullptr;
1617	uint64_t SizeInBits = Record[7];
1618	if (Record[8] > (uint64_t)std::numeric_limits<uint32_t>::max())
1619	return error(Message: "Alignment value is too large");
1620	uint32_t AlignInBits = Record[8];
1621	uint64_t OffsetInBits = 0;
1622	DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[10]);
1623	Metadata *Elements = nullptr;
1624	unsigned RuntimeLang = Record[12];
1625	Metadata *VTableHolder = nullptr;
1626	Metadata *TemplateParams = nullptr;
1627	Metadata *Discriminator = nullptr;
1628	Metadata *DataLocation = nullptr;
1629	Metadata *Associated = nullptr;
1630	Metadata *Allocated = nullptr;
1631	Metadata *Rank = nullptr;
1632	Metadata *Annotations = nullptr;
1633	auto *Identifier = getMDString(Record[15]);
1634	// If this module is being parsed so that it can be ThinLTO imported
1635	// into another module, composite types only need to be imported as
1636	// type declarations (unless full type definitions are requested).
1637	// Create type declarations up front to save memory. This is only
1638	// done for types which have an Identifier, and are therefore
1639	// subject to the ODR.
1640	//
1641	// buildODRType handles the case where this is type ODRed with a
1642	// definition needed by the importing module, in which case the
1643	// existing definition is used.
1644	//
1645	// We always import full definitions for anonymous composite types,
1646	// as without a name, debuggers cannot easily resolve a declaration
1647	// to its definition.
1648	if (IsImporting && !ImportFullTypeDefinitions && Identifier && Name &&
1649	(Tag == dwarf::DW_TAG_enumeration_type ||
1650	Tag == dwarf::DW_TAG_class_type ||
1651	Tag == dwarf::DW_TAG_structure_type ||
1652	Tag == dwarf::DW_TAG_union_type)) {
1653	Flags = Flags | DINode::FlagFwdDecl;
1654	// This is a hack around preserving template parameters for simplified
1655	// template names - it should probably be replaced with a
1656	// DICompositeType flag specifying whether template parameters are
1657	// required on declarations of this type.
1658	StringRef NameStr = Name->getString();
1659	if (!NameStr.contains(C: '<') || NameStr.starts_with(Prefix: "_STN|"))
1660	TemplateParams = getMDOrNull(Record[14]);
1661	} else {
1662	BaseType = getDITypeRefOrNull(Record[6]);
1663	OffsetInBits = Record[9];
1664	Elements = getMDOrNull(Record[11]);
1665	VTableHolder = getDITypeRefOrNull(Record[13]);
1666	TemplateParams = getMDOrNull(Record[14]);
1667	if (Record.size() > 16)
1668	Discriminator = getMDOrNull(Record[16]);
1669	if (Record.size() > 17)
1670	DataLocation = getMDOrNull(Record[17]);
1671	if (Record.size() > 19) {
1672	Associated = getMDOrNull(Record[18]);
1673	Allocated = getMDOrNull(Record[19]);
1674	}
1675	if (Record.size() > 20) {
1676	Rank = getMDOrNull(Record[20]);
1677	}
1678	if (Record.size() > 21) {
1679	Annotations = getMDOrNull(Record[21]);
1680	}
1681	}
1682	DICompositeType *CT = nullptr;
1683	if (Identifier)
1684	CT = DICompositeType::buildODRType(
1685	Context, Identifier&: *Identifier, Tag, Name, File, Line, Scope, BaseType,
1686	SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang,
1687	VTableHolder, TemplateParams, Discriminator, DataLocation, Associated,
1688	Allocated, Rank, Annotations);
1689
1690	// Create a node if we didn't get a lazy ODR type.
1691	if (!CT)
1692	CT = GET_OR_DISTINCT(DICompositeType,
1693	(Context, Tag, Name, File, Line, Scope, BaseType,
1694	SizeInBits, AlignInBits, OffsetInBits, Flags,
1695	Elements, RuntimeLang, VTableHolder, TemplateParams,
1696	Identifier, Discriminator, DataLocation, Associated,
1697	Allocated, Rank, Annotations));
1698	if (!IsNotUsedInTypeRef && Identifier)
1699	MetadataList.addTypeRef(UUID&: *Identifier, CT&: *cast<DICompositeType>(Val: CT));
1700
1701	MetadataList.assignValue(MD: CT, Idx: NextMetadataNo);
1702	NextMetadataNo++;
1703	break;
1704	}
1705	case bitc::METADATA_SUBROUTINE_TYPE: {
1706	if (Record.size() < 3 || Record.size() > 4)
1707	return error(Message: "Invalid record");
1708	bool IsOldTypeRefArray = Record[0] < 2;
1709	unsigned CC = (Record.size() > 3) ? Record[3] : 0;
1710
1711	IsDistinct = Record[0] & 0x1;
1712	DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[1]);
1713	Metadata *Types = getMDOrNull(Record[2]);
1714	if (LLVM_UNLIKELY(IsOldTypeRefArray))
1715	Types = MetadataList.upgradeTypeRefArray(MaybeTuple: Types);
1716
1717	MetadataList.assignValue(
1718	GET_OR_DISTINCT(DISubroutineType, (Context, Flags, CC, Types)),
1719	Idx: NextMetadataNo);
1720	NextMetadataNo++;
1721	break;
1722	}
1723
1724	case bitc::METADATA_MODULE: {
1725	if (Record.size() < 5 || Record.size() > 9)
1726	return error(Message: "Invalid record");
1727
1728	unsigned Offset = Record.size() >= 8 ? 2 : 1;
1729	IsDistinct = Record[0];
1730	MetadataList.assignValue(
1731	GET_OR_DISTINCT(
1732	DIModule,
1733	(Context, Record.size() >= 8 ? getMDOrNull(Record[1]) : nullptr,
1734	getMDOrNull(Record[0 + Offset]), getMDString(Record[1 + Offset]),
1735	getMDString(Record[2 + Offset]), getMDString(Record[3 + Offset]),
1736	getMDString(Record[4 + Offset]),
1737	Record.size() <= 7 ? 0 : Record[7],
1738	Record.size() <= 8 ? false : Record[8])),
1739	Idx: NextMetadataNo);
1740	NextMetadataNo++;
1741	break;
1742	}
1743
1744	case bitc::METADATA_FILE: {
1745	if (Record.size() != 3 && Record.size() != 5 && Record.size() != 6)
1746	return error(Message: "Invalid record");
1747
1748	IsDistinct = Record[0];
1749	std::optional<DIFile::ChecksumInfo<MDString *>> Checksum;
1750	// The BitcodeWriter writes null bytes into Record[3:4] when the Checksum
1751	// is not present. This matches up with the old internal representation,
1752	// and the old encoding for CSK_None in the ChecksumKind. The new
1753	// representation reserves the value 0 in the ChecksumKind to continue to
1754	// encode None in a backwards-compatible way.
1755	if (Record.size() > 4 && Record[3] && Record[4])
1756	Checksum.emplace(args: static_cast<DIFile::ChecksumKind>(Record[3]),
1757	args: getMDString(Record[4]));
1758	MetadataList.assignValue(
1759	GET_OR_DISTINCT(DIFile,
1760	(Context, getMDString(Record[1]),
1761	getMDString(Record[2]), Checksum,
1762	Record.size() > 5 ? getMDString(Record[5]) : nullptr)),
1763	Idx: NextMetadataNo);
1764	NextMetadataNo++;
1765	break;
1766	}
1767	case bitc::METADATA_COMPILE_UNIT: {
1768	if (Record.size() < 14 || Record.size() > 22)
1769	return error(Message: "Invalid record");
1770
1771	// Ignore Record[0], which indicates whether this compile unit is
1772	// distinct. It's always distinct.
1773	IsDistinct = true;
1774	auto *CU = DICompileUnit::getDistinct(
1775	Context, SourceLanguage: Record[1], File: getMDOrNull(Record[2]), Producer: getMDString(Record[3]),
1776	IsOptimized: Record[4], Flags: getMDString(Record[5]), RuntimeVersion: Record[6], SplitDebugFilename: getMDString(Record[7]),
1777	EmissionKind: Record[8], EnumTypes: getMDOrNull(Record[9]), RetainedTypes: getMDOrNull(Record[10]),
1778	GlobalVariables: getMDOrNull(Record[12]), ImportedEntities: getMDOrNull(Record[13]),
1779	Macros: Record.size() <= 15 ? nullptr : getMDOrNull(Record[15]),
1780	DWOId: Record.size() <= 14 ? 0 : Record[14],
1781	SplitDebugInlining: Record.size() <= 16 ? true : Record[16],
1782	DebugInfoForProfiling: Record.size() <= 17 ? false : Record[17],
1783	NameTableKind: Record.size() <= 18 ? 0 : Record[18],
1784	RangesBaseAddress: Record.size() <= 19 ? false : Record[19],
1785	SysRoot: Record.size() <= 20 ? nullptr : getMDString(Record[20]),
1786	SDK: Record.size() <= 21 ? nullptr : getMDString(Record[21]));
1787
1788	MetadataList.assignValue(MD: CU, Idx: NextMetadataNo);
1789	NextMetadataNo++;
1790
1791	// Move the Upgrade the list of subprograms.
1792	if (Metadata *SPs = getMDOrNullWithoutPlaceholders(Record[11]))
1793	CUSubprograms.push_back(x: {CU, SPs});
1794	break;
1795	}
1796	case bitc::METADATA_SUBPROGRAM: {
1797	if (Record.size() < 18 || Record.size() > 21)
1798	return error(Message: "Invalid record");
1799
1800	bool HasSPFlags = Record[0] & 4;
1801
1802	DINode::DIFlags Flags;
1803	DISubprogram::DISPFlags SPFlags;
1804	if (!HasSPFlags)
1805	Flags = static_cast<DINode::DIFlags>(Record[11 + 2]);
1806	else {
1807	Flags = static_cast<DINode::DIFlags>(Record[11]);
1808	SPFlags = static_cast<DISubprogram::DISPFlags>(Record[9]);
1809	}
1810
1811	// Support for old metadata when
1812	// subprogram specific flags are placed in DIFlags.
1813	const unsigned DIFlagMainSubprogram = 1 << 21;
1814	bool HasOldMainSubprogramFlag = Flags & DIFlagMainSubprogram;
1815	if (HasOldMainSubprogramFlag)
1816	// Remove old DIFlagMainSubprogram from DIFlags.
1817	// Note: This assumes that any future use of bit 21 defaults to it
1818	// being 0.
1819	Flags &= ~static_cast<DINode::DIFlags>(DIFlagMainSubprogram);
1820
1821	if (HasOldMainSubprogramFlag && HasSPFlags)
1822	SPFlags |= DISubprogram::SPFlagMainSubprogram;
1823	else if (!HasSPFlags)
1824	SPFlags = DISubprogram::toSPFlags(
1825	/*IsLocalToUnit=*/Record[7], /*IsDefinition=*/Record[8],
1826	/*IsOptimized=*/Record[14], /*Virtuality=*/Record[11],
1827	/*IsMainSubprogram=*/HasOldMainSubprogramFlag);
1828
1829	// All definitions should be distinct.
1830	IsDistinct = (Record[0] & 1) || (SPFlags & DISubprogram::SPFlagDefinition);
1831	// Version 1 has a Function as Record[15].
1832	// Version 2 has removed Record[15].
1833	// Version 3 has the Unit as Record[15].
1834	// Version 4 added thisAdjustment.
1835	// Version 5 repacked flags into DISPFlags, changing many element numbers.
1836	bool HasUnit = Record[0] & 2;
1837	if (!HasSPFlags && HasUnit && Record.size() < 19)
1838	return error(Message: "Invalid record");
1839	if (HasSPFlags && !HasUnit)
1840	return error(Message: "Invalid record");
1841	// Accommodate older formats.
1842	bool HasFn = false;
1843	bool HasThisAdj = true;
1844	bool HasThrownTypes = true;
1845	bool HasAnnotations = false;
1846	bool HasTargetFuncName = false;
1847	unsigned OffsetA = 0;
1848	unsigned OffsetB = 0;
1849	if (!HasSPFlags) {
1850	OffsetA = 2;
1851	OffsetB = 2;
1852	if (Record.size() >= 19) {
1853	HasFn = !HasUnit;
1854	OffsetB++;
1855	}
1856	HasThisAdj = Record.size() >= 20;
1857	HasThrownTypes = Record.size() >= 21;
1858	} else {
1859	HasAnnotations = Record.size() >= 19;
1860	HasTargetFuncName = Record.size() >= 20;
1861	}
1862	Metadata *CUorFn = getMDOrNull(Record[12 + OffsetB]);
1863	DISubprogram *SP = GET_OR_DISTINCT(
1864	DISubprogram,
1865	(Context,
1866	getDITypeRefOrNull(Record[1]), // scope
1867	getMDString(Record[2]), // name
1868	getMDString(Record[3]), // linkageName
1869	getMDOrNull(Record[4]), // file
1870	Record[5], // line
1871	getMDOrNull(Record[6]), // type
1872	Record[7 + OffsetA], // scopeLine
1873	getDITypeRefOrNull(Record[8 + OffsetA]), // containingType
1874	Record[10 + OffsetA], // virtualIndex
1875	HasThisAdj ? Record[16 + OffsetB] : 0, // thisAdjustment
1876	Flags, // flags
1877	SPFlags, // SPFlags
1878	HasUnit ? CUorFn : nullptr, // unit
1879	getMDOrNull(Record[13 + OffsetB]), // templateParams
1880	getMDOrNull(Record[14 + OffsetB]), // declaration
1881	getMDOrNull(Record[15 + OffsetB]), // retainedNodes
1882	HasThrownTypes ? getMDOrNull(Record[17 + OffsetB])
1883	: nullptr, // thrownTypes
1884	HasAnnotations ? getMDOrNull(Record[18 + OffsetB])
1885	: nullptr, // annotations
1886	HasTargetFuncName ? getMDString(Record[19 + OffsetB])
1887	: nullptr // targetFuncName
1888	));
1889	MetadataList.assignValue(MD: SP, Idx: NextMetadataNo);
1890	NextMetadataNo++;
1891
1892	// Upgrade sp->function mapping to function->sp mapping.
1893	if (HasFn) {
1894	if (auto *CMD = dyn_cast_or_null<ConstantAsMetadata>(Val: CUorFn))
1895	if (auto *F = dyn_cast<Function>(Val: CMD->getValue())) {
1896	if (F->isMaterializable())
1897	// Defer until materialized; unmaterialized functions may not have
1898	// metadata.
1899	FunctionsWithSPs[F] = SP;
1900	else if (!F->empty())
1901	F->setSubprogram(SP);
1902	}
1903	}
1904	break;
1905	}
1906	case bitc::METADATA_LEXICAL_BLOCK: {
1907	if (Record.size() != 5)
1908	return error(Message: "Invalid record");
1909
1910	IsDistinct = Record[0];
1911	MetadataList.assignValue(
1912	GET_OR_DISTINCT(DILexicalBlock,
1913	(Context, getMDOrNull(Record[1]),
1914	getMDOrNull(Record[2]), Record[3], Record[4])),
1915	Idx: NextMetadataNo);
1916	NextMetadataNo++;
1917	break;
1918	}
1919	case bitc::METADATA_LEXICAL_BLOCK_FILE: {
1920	if (Record.size() != 4)
1921	return error(Message: "Invalid record");
1922
1923	IsDistinct = Record[0];
1924	MetadataList.assignValue(
1925	GET_OR_DISTINCT(DILexicalBlockFile,
1926	(Context, getMDOrNull(Record[1]),
1927	getMDOrNull(Record[2]), Record[3])),
1928	Idx: NextMetadataNo);
1929	NextMetadataNo++;
1930	break;
1931	}
1932	case bitc::METADATA_COMMON_BLOCK: {
1933	IsDistinct = Record[0] & 1;
1934	MetadataList.assignValue(
1935	GET_OR_DISTINCT(DICommonBlock,
1936	(Context, getMDOrNull(Record[1]),
1937	getMDOrNull(Record[2]), getMDString(Record[3]),
1938	getMDOrNull(Record[4]), Record[5])),
1939	Idx: NextMetadataNo);
1940	NextMetadataNo++;
1941	break;
1942	}
1943	case bitc::METADATA_NAMESPACE: {
1944	// Newer versions of DINamespace dropped file and line.
1945	MDString *Name;
1946	if (Record.size() == 3)
1947	Name = getMDString(Record[2]);
1948	else if (Record.size() == 5)
1949	Name = getMDString(Record[3]);
1950	else
1951	return error(Message: "Invalid record");
1952
1953	IsDistinct = Record[0] & 1;
1954	bool ExportSymbols = Record[0] & 2;
1955	MetadataList.assignValue(
1956	GET_OR_DISTINCT(DINamespace,
1957	(Context, getMDOrNull(Record[1]), Name, ExportSymbols)),
1958	Idx: NextMetadataNo);
1959	NextMetadataNo++;
1960	break;
1961	}
1962	case bitc::METADATA_MACRO: {
1963	if (Record.size() != 5)
1964	return error(Message: "Invalid record");
1965
1966	IsDistinct = Record[0];
1967	MetadataList.assignValue(
1968	GET_OR_DISTINCT(DIMacro,
1969	(Context, Record[1], Record[2], getMDString(Record[3]),
1970	getMDString(Record[4]))),
1971	Idx: NextMetadataNo);
1972	NextMetadataNo++;
1973	break;
1974	}
1975	case bitc::METADATA_MACRO_FILE: {
1976	if (Record.size() != 5)
1977	return error(Message: "Invalid record");
1978
1979	IsDistinct = Record[0];
1980	MetadataList.assignValue(
1981	GET_OR_DISTINCT(DIMacroFile,
1982	(Context, Record[1], Record[2], getMDOrNull(Record[3]),
1983	getMDOrNull(Record[4]))),
1984	Idx: NextMetadataNo);
1985	NextMetadataNo++;
1986	break;
1987	}
1988	case bitc::METADATA_TEMPLATE_TYPE: {
1989	if (Record.size() < 3 || Record.size() > 4)
1990	return error(Message: "Invalid record");
1991
1992	IsDistinct = Record[0];
1993	MetadataList.assignValue(
1994	GET_OR_DISTINCT(DITemplateTypeParameter,
1995	(Context, getMDString(Record[1]),
1996	getDITypeRefOrNull(Record[2]),
1997	(Record.size() == 4) ? getMDOrNull(Record[3])
1998	: getMDOrNull(false))),
1999	Idx: NextMetadataNo);
2000	NextMetadataNo++;
2001	break;
2002	}
2003	case bitc::METADATA_TEMPLATE_VALUE: {
2004	if (Record.size() < 5 || Record.size() > 6)
2005	return error(Message: "Invalid record");
2006
2007	IsDistinct = Record[0];
2008
2009	MetadataList.assignValue(
2010	GET_OR_DISTINCT(
2011	DITemplateValueParameter,
2012	(Context, Record[1], getMDString(Record[2]),
2013	getDITypeRefOrNull(Record[3]),
2014	(Record.size() == 6) ? getMDOrNull(Record[4]) : getMDOrNull(false),
2015	(Record.size() == 6) ? getMDOrNull(Record[5])
2016	: getMDOrNull(Record[4]))),
2017	Idx: NextMetadataNo);
2018	NextMetadataNo++;
2019	break;
2020	}
2021	case bitc::METADATA_GLOBAL_VAR: {
2022	if (Record.size() < 11 || Record.size() > 13)
2023	return error(Message: "Invalid record");
2024
2025	IsDistinct = Record[0] & 1;
2026	unsigned Version = Record[0] >> 1;
2027
2028	if (Version == 2) {
2029	Metadata *Annotations = nullptr;
2030	if (Record.size() > 12)
2031	Annotations = getMDOrNull(Record[12]);
2032
2033	MetadataList.assignValue(
2034	GET_OR_DISTINCT(DIGlobalVariable,
2035	(Context, getMDOrNull(Record[1]),
2036	getMDString(Record[2]), getMDString(Record[3]),
2037	getMDOrNull(Record[4]), Record[5],
2038	getDITypeRefOrNull(Record[6]), Record[7], Record[8],
2039	getMDOrNull(Record[9]), getMDOrNull(Record[10]),
2040	Record[11], Annotations)),
2041	Idx: NextMetadataNo);
2042
2043	NextMetadataNo++;
2044	} else if (Version == 1) {
2045	// No upgrade necessary. A null field will be introduced to indicate
2046	// that no parameter information is available.
2047	MetadataList.assignValue(
2048	GET_OR_DISTINCT(
2049	DIGlobalVariable,
2050	(Context, getMDOrNull(Record[1]), getMDString(Record[2]),
2051	getMDString(Record[3]), getMDOrNull(Record[4]), Record[5],
2052	getDITypeRefOrNull(Record[6]), Record[7], Record[8],
2053	getMDOrNull(Record[10]), nullptr, Record[11], nullptr)),
2054	Idx: NextMetadataNo);
2055
2056	NextMetadataNo++;
2057	} else if (Version == 0) {
2058	// Upgrade old metadata, which stored a global variable reference or a
2059	// ConstantInt here.
2060	NeedUpgradeToDIGlobalVariableExpression = true;
2061	Metadata *Expr = getMDOrNull(Record[9]);
2062	uint32_t AlignInBits = 0;
2063	if (Record.size() > 11) {
2064	if (Record[11] > (uint64_t)std::numeric_limits<uint32_t>::max())
2065	return error(Message: "Alignment value is too large");
2066	AlignInBits = Record[11];
2067	}
2068	GlobalVariable *Attach = nullptr;
2069	if (auto *CMD = dyn_cast_or_null<ConstantAsMetadata>(Val: Expr)) {
2070	if (auto *GV = dyn_cast<GlobalVariable>(Val: CMD->getValue())) {
2071	Attach = GV;
2072	Expr = nullptr;
2073	} else if (auto *CI = dyn_cast<ConstantInt>(Val: CMD->getValue())) {
2074	Expr = DIExpression::get(Context,
2075	Elements: {dwarf::DW_OP_constu, CI->getZExtValue(),
2076	dwarf::DW_OP_stack_value});
2077	} else {
2078	Expr = nullptr;
2079	}
2080	}
2081	DIGlobalVariable *DGV = GET_OR_DISTINCT(
2082	DIGlobalVariable,
2083	(Context, getMDOrNull(Record[1]), getMDString(Record[2]),
2084	getMDString(Record[3]), getMDOrNull(Record[4]), Record[5],
2085	getDITypeRefOrNull(Record[6]), Record[7], Record[8],
2086	getMDOrNull(Record[10]), nullptr, AlignInBits, nullptr));
2087
2088	DIGlobalVariableExpression *DGVE = nullptr;
2089	if (Attach || Expr)
2090	DGVE = DIGlobalVariableExpression::getDistinct(
2091	Context, Variable: DGV, Expression: Expr ? Expr : DIExpression::get(Context, Elements: {}));
2092	if (Attach)
2093	Attach->addDebugInfo(GV: DGVE);
2094
2095	auto *MDNode = Expr ? cast<Metadata>(Val: DGVE) : cast<Metadata>(Val: DGV);
2096	MetadataList.assignValue(MD: MDNode, Idx: NextMetadataNo);
2097	NextMetadataNo++;
2098	} else
2099	return error(Message: "Invalid record");
2100
2101	break;
2102	}
2103	case bitc::METADATA_ASSIGN_ID: {
2104	if (Record.size() != 1)
2105	return error(Message: "Invalid DIAssignID record.");
2106
2107	IsDistinct = Record[0] & 1;
2108	if (!IsDistinct)
2109	return error(Message: "Invalid DIAssignID record. Must be distinct");
2110
2111	MetadataList.assignValue(MD: DIAssignID::getDistinct(Context), Idx: NextMetadataNo);
2112	NextMetadataNo++;
2113	break;
2114	}
2115	case bitc::METADATA_LOCAL_VAR: {
2116	// 10th field is for the obseleted 'inlinedAt:' field.
2117	if (Record.size() < 8 || Record.size() > 10)
2118	return error(Message: "Invalid record");
2119
2120	IsDistinct = Record[0] & 1;
2121	bool HasAlignment = Record[0] & 2;
2122	// 2nd field used to be an artificial tag, either DW_TAG_auto_variable or
2123	// DW_TAG_arg_variable, if we have alignment flag encoded it means, that
2124	// this is newer version of record which doesn't have artificial tag.
2125	bool HasTag = !HasAlignment && Record.size() > 8;
2126	DINode::DIFlags Flags = static_cast<DINode::DIFlags>(Record[7 + HasTag]);
2127	uint32_t AlignInBits = 0;
2128	Metadata *Annotations = nullptr;
2129	if (HasAlignment) {
2130	if (Record[8] > (uint64_t)std::numeric_limits<uint32_t>::max())
2131	return error(Message: "Alignment value is too large");
2132	AlignInBits = Record[8];
2133	if (Record.size() > 9)
2134	Annotations = getMDOrNull(Record[9]);
2135	}
2136
2137	MetadataList.assignValue(
2138	GET_OR_DISTINCT(DILocalVariable,
2139	(Context, getMDOrNull(Record[1 + HasTag]),
2140	getMDString(Record[2 + HasTag]),
2141	getMDOrNull(Record[3 + HasTag]), Record[4 + HasTag],
2142	getDITypeRefOrNull(Record[5 + HasTag]),
2143	Record[6 + HasTag], Flags, AlignInBits, Annotations)),
2144	Idx: NextMetadataNo);
2145	NextMetadataNo++;
2146	break;
2147	}
2148	case bitc::METADATA_LABEL: {
2149	if (Record.size() != 5)
2150	return error(Message: "Invalid record");
2151
2152	IsDistinct = Record[0] & 1;
2153	MetadataList.assignValue(
2154	GET_OR_DISTINCT(DILabel, (Context, getMDOrNull(Record[1]),
2155	getMDString(Record[2]),
2156	getMDOrNull(Record[3]), Record[4])),
2157	Idx: NextMetadataNo);
2158	NextMetadataNo++;
2159	break;
2160	}
2161	case bitc::METADATA_EXPRESSION: {
2162	if (Record.size() < 1)
2163	return error(Message: "Invalid record");
2164
2165	IsDistinct = Record[0] & 1;
2166	uint64_t Version = Record[0] >> 1;
2167	auto Elts = MutableArrayRef<uint64_t>(Record).slice(N: 1);
2168
2169	SmallVector<uint64_t, 6> Buffer;
2170	if (Error Err = upgradeDIExpression(FromVersion: Version, Expr&: Elts, Buffer))
2171	return Err;
2172
2173	MetadataList.assignValue(GET_OR_DISTINCT(DIExpression, (Context, Elts)),
2174	Idx: NextMetadataNo);
2175	NextMetadataNo++;
2176	break;
2177	}
2178	case bitc::METADATA_GLOBAL_VAR_EXPR: {
2179	if (Record.size() != 3)
2180	return error(Message: "Invalid record");
2181
2182	IsDistinct = Record[0];
2183	Metadata *Expr = getMDOrNull(Record[2]);
2184	if (!Expr)
2185	Expr = DIExpression::get(Context, Elements: {});
2186	MetadataList.assignValue(
2187	GET_OR_DISTINCT(DIGlobalVariableExpression,
2188	(Context, getMDOrNull(Record[1]), Expr)),
2189	Idx: NextMetadataNo);
2190	NextMetadataNo++;
2191	break;
2192	}
2193	case bitc::METADATA_OBJC_PROPERTY: {
2194	if (Record.size() != 8)
2195	return error(Message: "Invalid record");
2196
2197	IsDistinct = Record[0];
2198	MetadataList.assignValue(
2199	GET_OR_DISTINCT(DIObjCProperty,
2200	(Context, getMDString(Record[1]),
2201	getMDOrNull(Record[2]), Record[3],
2202	getMDString(Record[4]), getMDString(Record[5]),
2203	Record[6], getDITypeRefOrNull(Record[7]))),
2204	Idx: NextMetadataNo);
2205	NextMetadataNo++;
2206	break;
2207	}
2208	case bitc::METADATA_IMPORTED_ENTITY: {
2209	if (Record.size() < 6 || Record.size() > 8)
2210	return error(Message: "Invalid DIImportedEntity record");
2211
2212	IsDistinct = Record[0];
2213	bool HasFile = (Record.size() >= 7);
2214	bool HasElements = (Record.size() >= 8);
2215	MetadataList.assignValue(
2216	GET_OR_DISTINCT(DIImportedEntity,
2217	(Context, Record[1], getMDOrNull(Record[2]),
2218	getDITypeRefOrNull(Record[3]),
2219	HasFile ? getMDOrNull(Record[6]) : nullptr,
2220	HasFile ? Record[4] : 0, getMDString(Record[5]),
2221	HasElements ? getMDOrNull(Record[7]) : nullptr)),
2222	Idx: NextMetadataNo);
2223	NextMetadataNo++;
2224	break;
2225	}
2226	case bitc::METADATA_STRING_OLD: {
2227	std::string String(Record.begin(), Record.end());
2228
2229	// Test for upgrading !llvm.loop.
2230	HasSeenOldLoopTags |= mayBeOldLoopAttachmentTag(Name: String);
2231	++NumMDStringLoaded;
2232	Metadata *MD = MDString::get(Context, Str: String);
2233	MetadataList.assignValue(MD, Idx: NextMetadataNo);
2234	NextMetadataNo++;
2235	break;
2236	}
2237	case bitc::METADATA_STRINGS: {
2238	auto CreateNextMDString = [&](StringRef Str) {
2239	++NumMDStringLoaded;
2240	MetadataList.assignValue(MD: MDString::get(Context, Str), Idx: NextMetadataNo);
2241	NextMetadataNo++;
2242	};
2243	if (Error Err = parseMetadataStrings(Record, Blob, CallBack: CreateNextMDString))
2244	return Err;
2245	break;
2246	}
2247	case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: {
2248	if (Record.size() % 2 == 0)
2249	return error(Message: "Invalid record");
2250	unsigned ValueID = Record[0];
2251	if (ValueID >= ValueList.size())
2252	return error(Message: "Invalid record");
2253	if (auto *GO = dyn_cast<GlobalObject>(Val: ValueList[ValueID]))
2254	if (Error Err = parseGlobalObjectAttachment(
2255	GO&: *GO, Record: ArrayRef<uint64_t>(Record).slice(N: 1)))
2256	return Err;
2257	break;
2258	}
2259	case bitc::METADATA_KIND: {
2260	// Support older bitcode files that had METADATA_KIND records in a
2261	// block with METADATA_BLOCK_ID.
2262	if (Error Err = parseMetadataKindRecord(Record))
2263	return Err;
2264	break;
2265	}
2266	case bitc::METADATA_ARG_LIST: {
2267	SmallVector<ValueAsMetadata *, 4> Elts;
2268	Elts.reserve(N: Record.size());
2269	for (uint64_t Elt : Record) {
2270	Metadata *MD = getMD(Elt);
2271	if (isa<MDNode>(Val: MD) && cast<MDNode>(Val: MD)->isTemporary())
2272	return error(
2273	Message: "Invalid record: DIArgList should not contain forward refs");
2274	if (!isa<ValueAsMetadata>(Val: MD))
2275	return error(Message: "Invalid record");
2276	Elts.push_back(Elt: cast<ValueAsMetadata>(Val: MD));
2277	}
2278
2279	MetadataList.assignValue(MD: DIArgList::get(Context, Args: Elts), Idx: NextMetadataNo);
2280	NextMetadataNo++;
2281	break;
2282	}
2283	}
2284	return Error::success();
2285	#undef GET_OR_DISTINCT
2286	}
2287
2288	Error MetadataLoader::MetadataLoaderImpl::parseMetadataStrings(
2289	ArrayRef<uint64_t> Record, StringRef Blob,
2290	function_ref<void(StringRef)> CallBack) {
2291	// All the MDStrings in the block are emitted together in a single
2292	// record. The strings are concatenated and stored in a blob along with
2293	// their sizes.
2294	if (Record.size() != 2)
2295	return error(Message: "Invalid record: metadata strings layout");
2296
2297	unsigned NumStrings = Record[0];
2298	unsigned StringsOffset = Record[1];
2299	if (!NumStrings)
2300	return error(Message: "Invalid record: metadata strings with no strings");
2301	if (StringsOffset > Blob.size())
2302	return error(Message: "Invalid record: metadata strings corrupt offset");
2303
2304	StringRef Lengths = Blob.slice(Start: 0, End: StringsOffset);
2305	SimpleBitstreamCursor R(Lengths);
2306
2307	StringRef Strings = Blob.drop_front(N: StringsOffset);
2308	do {
2309	if (R.AtEndOfStream())
2310	return error(Message: "Invalid record: metadata strings bad length");
2311
2312	uint32_t Size;
2313	if (Error E = R.ReadVBR(NumBits: 6).moveInto(Value&: Size))
2314	return E;
2315	if (Strings.size() < Size)
2316	return error(Message: "Invalid record: metadata strings truncated chars");
2317
2318	CallBack(Strings.slice(Start: 0, End: Size));
2319	Strings = Strings.drop_front(N: Size);
2320	} while (--NumStrings);
2321
2322	return Error::success();
2323	}
2324
2325	Error MetadataLoader::MetadataLoaderImpl::parseGlobalObjectAttachment(
2326	GlobalObject &GO, ArrayRef<uint64_t> Record) {
2327	assert(Record.size() % 2 == 0);
2328	for (unsigned I = 0, E = Record.size(); I != E; I += 2) {
2329	auto K = MDKindMap.find(Val: Record[I]);
2330	if (K == MDKindMap.end())
2331	return error(Message: "Invalid ID");
2332	MDNode *MD =
2333	dyn_cast_or_null<MDNode>(Val: getMetadataFwdRefOrLoad(ID: Record[I + 1]));
2334	if (!MD)
2335	return error(Message: "Invalid metadata attachment: expect fwd ref to MDNode");
2336	GO.addMetadata(KindID: K->second, MD&: *MD);
2337	}
2338	return Error::success();
2339	}
2340
2341	/// Parse metadata attachments.
2342	Error MetadataLoader::MetadataLoaderImpl::parseMetadataAttachment(
2343	Function &F, ArrayRef<Instruction *> InstructionList) {
2344	if (Error Err = Stream.EnterSubBlock(BlockID: bitc::METADATA_ATTACHMENT_ID))
2345	return Err;
2346
2347	SmallVector<uint64_t, 64> Record;
2348	PlaceholderQueue Placeholders;
2349
2350	while (true) {
2351	BitstreamEntry Entry;
2352	if (Error E = Stream.advanceSkippingSubblocks().moveInto(Value&: Entry))
2353	return E;
2354
2355	switch (Entry.Kind) {
2356	case BitstreamEntry::SubBlock: // Handled for us already.
2357	case BitstreamEntry::Error:
2358	return error(Message: "Malformed block");
2359	case BitstreamEntry::EndBlock:
2360	resolveForwardRefsAndPlaceholders(Placeholders);
2361	return Error::success();
2362	case BitstreamEntry::Record:
2363	// The interesting case.
2364	break;
2365	}
2366
2367	// Read a metadata attachment record.
2368	Record.clear();
2369	++NumMDRecordLoaded;
2370	Expected<unsigned> MaybeRecord = Stream.readRecord(AbbrevID: Entry.ID, Vals&: Record);
2371	if (!MaybeRecord)
2372	return MaybeRecord.takeError();
2373	switch (MaybeRecord.get()) {
2374	default: // Default behavior: ignore.
2375	break;
2376	case bitc::METADATA_ATTACHMENT: {
2377	unsigned RecordLength = Record.size();
2378	if (Record.empty())
2379	return error(Message: "Invalid record");
2380	if (RecordLength % 2 == 0) {
2381	// A function attachment.
2382	if (Error Err = parseGlobalObjectAttachment(GO&: F, Record))
2383	return Err;
2384	continue;
2385	}
2386
2387	// An instruction attachment.
2388	Instruction *Inst = InstructionList[Record[0]];
2389	for (unsigned i = 1; i != RecordLength; i = i + 2) {
2390	unsigned Kind = Record[i];
2391	DenseMap<unsigned, unsigned>::iterator I = MDKindMap.find(Val: Kind);
2392	if (I == MDKindMap.end())
2393	return error(Message: "Invalid ID");
2394	if (I->second == LLVMContext::MD_tbaa && StripTBAA)
2395	continue;
2396
2397	auto Idx = Record[i + 1];
2398	if (Idx < (MDStringRef.size() + GlobalMetadataBitPosIndex.size()) &&
2399	!MetadataList.lookup(I: Idx)) {
2400	// Load the attachment if it is in the lazy-loadable range and hasn't
2401	// been loaded yet.
2402	lazyLoadOneMetadata(ID: Idx, Placeholders);
2403	resolveForwardRefsAndPlaceholders(Placeholders);
2404	}
2405
2406	Metadata *Node = MetadataList.getMetadataFwdRef(Idx);
2407	if (isa<LocalAsMetadata>(Val: Node))
2408	// Drop the attachment. This used to be legal, but there's no
2409	// upgrade path.
2410	break;
2411	MDNode *MD = dyn_cast_or_null<MDNode>(Val: Node);
2412	if (!MD)
2413	return error(Message: "Invalid metadata attachment");
2414
2415	if (HasSeenOldLoopTags && I->second == LLVMContext::MD_loop)
2416	MD = upgradeInstructionLoopAttachment(N&: *MD);
2417
2418	if (I->second == LLVMContext::MD_tbaa) {
2419	assert(!MD->isTemporary() && "should load MDs before attachments");
2420	MD = UpgradeTBAANode(TBAANode&: *MD);
2421	}
2422	Inst->setMetadata(KindID: I->second, Node: MD);
2423	}
2424	break;
2425	}
2426	}
2427	}
2428	}
2429
2430	/// Parse a single METADATA_KIND record, inserting result in MDKindMap.
2431	Error MetadataLoader::MetadataLoaderImpl::parseMetadataKindRecord(
2432	SmallVectorImpl<uint64_t> &Record) {
2433	if (Record.size() < 2)
2434	return error(Message: "Invalid record");
2435
2436	unsigned Kind = Record[0];
2437	SmallString<8> Name(Record.begin() + 1, Record.end());
2438
2439	unsigned NewKind = TheModule.getMDKindID(Name: Name.str());
2440	if (!MDKindMap.insert(KV: std::make_pair(x&: Kind, y&: NewKind)).second)
2441	return error(Message: "Conflicting METADATA_KIND records");
2442	return Error::success();
2443	}
2444
2445	/// Parse the metadata kinds out of the METADATA_KIND_BLOCK.
2446	Error MetadataLoader::MetadataLoaderImpl::parseMetadataKinds() {
2447	if (Error Err = Stream.EnterSubBlock(BlockID: bitc::METADATA_KIND_BLOCK_ID))
2448	return Err;
2449
2450	SmallVector<uint64_t, 64> Record;
2451
2452	// Read all the records.
2453	while (true) {
2454	BitstreamEntry Entry;
2455	if (Error E = Stream.advanceSkippingSubblocks().moveInto(Value&: Entry))
2456	return E;
2457
2458	switch (Entry.Kind) {
2459	case BitstreamEntry::SubBlock: // Handled for us already.
2460	case BitstreamEntry::Error:
2461	return error(Message: "Malformed block");
2462	case BitstreamEntry::EndBlock:
2463	return Error::success();
2464	case BitstreamEntry::Record:
2465	// The interesting case.
2466	break;
2467	}
2468
2469	// Read a record.
2470	Record.clear();
2471	++NumMDRecordLoaded;
2472	Expected<unsigned> MaybeCode = Stream.readRecord(AbbrevID: Entry.ID, Vals&: Record);
2473	if (!MaybeCode)
2474	return MaybeCode.takeError();
2475	switch (MaybeCode.get()) {
2476	default: // Default behavior: ignore.
2477	break;
2478	case bitc::METADATA_KIND: {
2479	if (Error Err = parseMetadataKindRecord(Record))
2480	return Err;
2481	break;
2482	}
2483	}
2484	}
2485	}
2486
2487	MetadataLoader &MetadataLoader::operator=(MetadataLoader &&RHS) {
2488	Pimpl = std::move(RHS.Pimpl);
2489	return *this;
2490	}
2491	MetadataLoader::MetadataLoader(MetadataLoader &&RHS)
2492	: Pimpl(std::move(RHS.Pimpl)) {}
2493
2494	MetadataLoader::~MetadataLoader() = default;
2495	MetadataLoader::MetadataLoader(BitstreamCursor &Stream, Module &TheModule,
2496	BitcodeReaderValueList &ValueList,
2497	bool IsImporting,
2498	MetadataLoaderCallbacks Callbacks)
2499	: Pimpl(std::make_unique<MetadataLoaderImpl>(
2500	args&: Stream, args&: TheModule, args&: ValueList, args: std::move(Callbacks), args&: IsImporting)) {}
2501
2502	Error MetadataLoader::parseMetadata(bool ModuleLevel) {
2503	return Pimpl->parseMetadata(ModuleLevel);
2504	}
2505
2506	bool MetadataLoader::hasFwdRefs() const { return Pimpl->hasFwdRefs(); }
2507
2508	/// Return the given metadata, creating a replaceable forward reference if
2509	/// necessary.
2510	Metadata *MetadataLoader::getMetadataFwdRefOrLoad(unsigned Idx) {
2511	return Pimpl->getMetadataFwdRefOrLoad(ID: Idx);
2512	}
2513
2514	DISubprogram *MetadataLoader::lookupSubprogramForFunction(Function *F) {
2515	return Pimpl->lookupSubprogramForFunction(F);
2516	}
2517
2518	Error MetadataLoader::parseMetadataAttachment(
2519	Function &F, ArrayRef<Instruction *> InstructionList) {
2520	return Pimpl->parseMetadataAttachment(F, InstructionList);
2521	}
2522
2523	Error MetadataLoader::parseMetadataKinds() {
2524	return Pimpl->parseMetadataKinds();
2525	}
2526
2527	void MetadataLoader::setStripTBAA(bool StripTBAA) {
2528	return Pimpl->setStripTBAA(StripTBAA);
2529	}
2530
2531	bool MetadataLoader::isStrippingTBAA() { return Pimpl->isStrippingTBAA(); }
2532
2533	unsigned MetadataLoader::size() const { return Pimpl->size(); }
2534	void MetadataLoader::shrinkTo(unsigned N) { return Pimpl->shrinkTo(N); }
2535
2536	void MetadataLoader::upgradeDebugIntrinsics(Function &F) {
2537	return Pimpl->upgradeDebugIntrinsics(F);
2538	}
2539