1	//===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// Bitcode writer implementation.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "llvm/Bitcode/BitcodeWriter.h"
14	#include "ValueEnumerator.h"
15	#include "llvm/ADT/APFloat.h"
16	#include "llvm/ADT/APInt.h"
17	#include "llvm/ADT/ArrayRef.h"
18	#include "llvm/ADT/DenseMap.h"
19	#include "llvm/ADT/STLExtras.h"
20	#include "llvm/ADT/SetVector.h"
21	#include "llvm/ADT/SmallPtrSet.h"
22	#include "llvm/ADT/SmallString.h"
23	#include "llvm/ADT/SmallVector.h"
24	#include "llvm/ADT/StringMap.h"
25	#include "llvm/ADT/StringRef.h"
26	#include "llvm/Bitcode/BitcodeCommon.h"
27	#include "llvm/Bitcode/BitcodeReader.h"
28	#include "llvm/Bitcode/LLVMBitCodes.h"
29	#include "llvm/Bitstream/BitCodes.h"
30	#include "llvm/Bitstream/BitstreamWriter.h"
31	#include "llvm/Config/llvm-config.h"
32	#include "llvm/IR/Attributes.h"
33	#include "llvm/IR/BasicBlock.h"
34	#include "llvm/IR/Comdat.h"
35	#include "llvm/IR/Constant.h"
36	#include "llvm/IR/Constants.h"
37	#include "llvm/IR/DebugInfoMetadata.h"
38	#include "llvm/IR/DebugLoc.h"
39	#include "llvm/IR/DerivedTypes.h"
40	#include "llvm/IR/Function.h"
41	#include "llvm/IR/GlobalAlias.h"
42	#include "llvm/IR/GlobalIFunc.h"
43	#include "llvm/IR/GlobalObject.h"
44	#include "llvm/IR/GlobalValue.h"
45	#include "llvm/IR/GlobalVariable.h"
46	#include "llvm/IR/InlineAsm.h"
47	#include "llvm/IR/InstrTypes.h"
48	#include "llvm/IR/Instruction.h"
49	#include "llvm/IR/Instructions.h"
50	#include "llvm/IR/LLVMContext.h"
51	#include "llvm/IR/Metadata.h"
52	#include "llvm/IR/Module.h"
53	#include "llvm/IR/ModuleSummaryIndex.h"
54	#include "llvm/IR/Operator.h"
55	#include "llvm/IR/Type.h"
56	#include "llvm/IR/UseListOrder.h"
57	#include "llvm/IR/Value.h"
58	#include "llvm/IR/ValueSymbolTable.h"
59	#include "llvm/MC/StringTableBuilder.h"
60	#include "llvm/MC/TargetRegistry.h"
61	#include "llvm/Object/IRSymtab.h"
62	#include "llvm/Support/AtomicOrdering.h"
63	#include "llvm/Support/Casting.h"
64	#include "llvm/Support/CommandLine.h"
65	#include "llvm/Support/Endian.h"
66	#include "llvm/Support/Error.h"
67	#include "llvm/Support/ErrorHandling.h"
68	#include "llvm/Support/MathExtras.h"
69	#include "llvm/Support/SHA1.h"
70	#include "llvm/Support/raw_ostream.h"
71	#include "llvm/TargetParser/Triple.h"
72	#include <algorithm>
73	#include <cassert>
74	#include <cstddef>
75	#include <cstdint>
76	#include <iterator>
77	#include <map>
78	#include <memory>
79	#include <optional>
80	#include <string>
81	#include <utility>
82	#include <vector>
83
84	using namespace llvm;
85
86	static cl::opt<unsigned>
87	IndexThreshold("bitcode-mdindex-threshold", cl::Hidden, cl::init(Val: 25),
88	cl::desc("Number of metadatas above which we emit an index "
89	"to enable lazy-loading"));
90	static cl::opt<uint32_t> FlushThreshold(
91	"bitcode-flush-threshold", cl::Hidden, cl::init(Val: 512),
92	cl::desc("The threshold (unit M) for flushing LLVM bitcode."));
93
94	static cl::opt<bool> WriteRelBFToSummary(
95	"write-relbf-to-summary", cl::Hidden, cl::init(Val: false),
96	cl::desc("Write relative block frequency to function summary "));
97
98	namespace llvm {
99	extern FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold;
100	}
101
102	extern bool WriteNewDbgInfoFormatToBitcode;
103	extern llvm::cl::opt<bool> UseNewDbgInfoFormat;
104
105	namespace {
106
107	/// These are manifest constants used by the bitcode writer. They do not need to
108	/// be kept in sync with the reader, but need to be consistent within this file.
109	enum {
110	// VALUE_SYMTAB_BLOCK abbrev id's.
111	VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
112	VST_ENTRY_7_ABBREV,
113	VST_ENTRY_6_ABBREV,
114	VST_BBENTRY_6_ABBREV,
115
116	// CONSTANTS_BLOCK abbrev id's.
117	CONSTANTS_SETTYPE_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
118	CONSTANTS_INTEGER_ABBREV,
119	CONSTANTS_CE_CAST_Abbrev,
120	CONSTANTS_NULL_Abbrev,
121
122	// FUNCTION_BLOCK abbrev id's.
123	FUNCTION_INST_LOAD_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
124	FUNCTION_INST_UNOP_ABBREV,
125	FUNCTION_INST_UNOP_FLAGS_ABBREV,
126	FUNCTION_INST_BINOP_ABBREV,
127	FUNCTION_INST_BINOP_FLAGS_ABBREV,
128	FUNCTION_INST_CAST_ABBREV,
129	FUNCTION_INST_CAST_FLAGS_ABBREV,
130	FUNCTION_INST_RET_VOID_ABBREV,
131	FUNCTION_INST_RET_VAL_ABBREV,
132	FUNCTION_INST_UNREACHABLE_ABBREV,
133	FUNCTION_INST_GEP_ABBREV,
134	FUNCTION_DEBUG_RECORD_VALUE_ABBREV,
135	};
136
137	/// Abstract class to manage the bitcode writing, subclassed for each bitcode
138	/// file type.
139	class BitcodeWriterBase {
140	protected:
141	/// The stream created and owned by the client.
142	BitstreamWriter &Stream;
143
144	StringTableBuilder &StrtabBuilder;
145
146	public:
147	/// Constructs a BitcodeWriterBase object that writes to the provided
148	/// \p Stream.
149	BitcodeWriterBase(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder)
150	: Stream(Stream), StrtabBuilder(StrtabBuilder) {}
151
152	protected:
153	void writeModuleVersion();
154	};
155
156	void BitcodeWriterBase::writeModuleVersion() {
157	// VERSION: [version#]
158	Stream.EmitRecord(Code: bitc::MODULE_CODE_VERSION, Vals: ArrayRef<uint64_t>{2});
159	}
160
161	/// Base class to manage the module bitcode writing, currently subclassed for
162	/// ModuleBitcodeWriter and ThinLinkBitcodeWriter.
163	class ModuleBitcodeWriterBase : public BitcodeWriterBase {
164	protected:
165	/// The Module to write to bitcode.
166	const Module &M;
167
168	/// Enumerates ids for all values in the module.
169	ValueEnumerator VE;
170
171	/// Optional per-module index to write for ThinLTO.
172	const ModuleSummaryIndex *Index;
173
174	/// Map that holds the correspondence between GUIDs in the summary index,
175	/// that came from indirect call profiles, and a value id generated by this
176	/// class to use in the VST and summary block records.
177	std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
178
179	/// Tracks the last value id recorded in the GUIDToValueMap.
180	unsigned GlobalValueId;
181
182	/// Saves the offset of the VSTOffset record that must eventually be
183	/// backpatched with the offset of the actual VST.
184	uint64_t VSTOffsetPlaceholder = 0;
185
186	public:
187	/// Constructs a ModuleBitcodeWriterBase object for the given Module,
188	/// writing to the provided \p Buffer.
189	ModuleBitcodeWriterBase(const Module &M, StringTableBuilder &StrtabBuilder,
190	BitstreamWriter &Stream,
191	bool ShouldPreserveUseListOrder,
192	const ModuleSummaryIndex *Index)
193	: BitcodeWriterBase(Stream, StrtabBuilder), M(M),
194	VE(M, ShouldPreserveUseListOrder), Index(Index) {
195	// Assign ValueIds to any callee values in the index that came from
196	// indirect call profiles and were recorded as a GUID not a Value*
197	// (which would have been assigned an ID by the ValueEnumerator).
198	// The starting ValueId is just after the number of values in the
199	// ValueEnumerator, so that they can be emitted in the VST.
200	GlobalValueId = VE.getValues().size();
201	if (!Index)
202	return;
203	for (const auto &GUIDSummaryLists : *Index)
204	// Examine all summaries for this GUID.
205	for (auto &Summary : GUIDSummaryLists.second.SummaryList)
206	if (auto FS = dyn_cast<FunctionSummary>(Val: Summary.get())) {
207	// For each call in the function summary, see if the call
208	// is to a GUID (which means it is for an indirect call,
209	// otherwise we would have a Value for it). If so, synthesize
210	// a value id.
211	for (auto &CallEdge : FS->calls())
212	if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
213	assignValueId(ValGUID: CallEdge.first.getGUID());
214
215	// For each referenced variables in the function summary, see if the
216	// variable is represented by a GUID (as opposed to a symbol to
217	// declarations or definitions in the module). If so, synthesize a
218	// value id.
219	for (auto &RefEdge : FS->refs())
220	if (!RefEdge.haveGVs() || !RefEdge.getValue())
221	assignValueId(ValGUID: RefEdge.getGUID());
222	}
223	}
224
225	protected:
226	void writePerModuleGlobalValueSummary();
227
228	private:
229	void writePerModuleFunctionSummaryRecord(
230	SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
231	unsigned ValueID, unsigned FSCallsAbbrev, unsigned FSCallsProfileAbbrev,
232	unsigned CallsiteAbbrev, unsigned AllocAbbrev, const Function &F);
233	void writeModuleLevelReferences(const GlobalVariable &V,
234	SmallVector<uint64_t, 64> &NameVals,
235	unsigned FSModRefsAbbrev,
236	unsigned FSModVTableRefsAbbrev);
237
238	void assignValueId(GlobalValue::GUID ValGUID) {
239	GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
240	}
241
242	unsigned getValueId(GlobalValue::GUID ValGUID) {
243	const auto &VMI = GUIDToValueIdMap.find(x: ValGUID);
244	// Expect that any GUID value had a value Id assigned by an
245	// earlier call to assignValueId.
246	assert(VMI != GUIDToValueIdMap.end() &&
247	"GUID does not have assigned value Id");
248	return VMI->second;
249	}
250
251	// Helper to get the valueId for the type of value recorded in VI.
252	unsigned getValueId(ValueInfo VI) {
253	if (!VI.haveGVs() || !VI.getValue())
254	return getValueId(ValGUID: VI.getGUID());
255	return VE.getValueID(V: VI.getValue());
256	}
257
258	std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
259	};
260
261	/// Class to manage the bitcode writing for a module.
262	class ModuleBitcodeWriter : public ModuleBitcodeWriterBase {
263	/// Pointer to the buffer allocated by caller for bitcode writing.
264	const SmallVectorImpl<char> &Buffer;
265
266	/// True if a module hash record should be written.
267	bool GenerateHash;
268
269	/// If non-null, when GenerateHash is true, the resulting hash is written
270	/// into ModHash.
271	ModuleHash *ModHash;
272
273	SHA1 Hasher;
274
275	/// The start bit of the identification block.
276	uint64_t BitcodeStartBit;
277
278	public:
279	/// Constructs a ModuleBitcodeWriter object for the given Module,
280	/// writing to the provided \p Buffer.
281	ModuleBitcodeWriter(const Module &M, SmallVectorImpl<char> &Buffer,
282	StringTableBuilder &StrtabBuilder,
283	BitstreamWriter &Stream, bool ShouldPreserveUseListOrder,
284	const ModuleSummaryIndex *Index, bool GenerateHash,
285	ModuleHash *ModHash = nullptr)
286	: ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
287	ShouldPreserveUseListOrder, Index),
288	Buffer(Buffer), GenerateHash(GenerateHash), ModHash(ModHash),
289	BitcodeStartBit(Stream.GetCurrentBitNo()) {}
290
291	/// Emit the current module to the bitstream.
292	void write();
293
294	private:
295	uint64_t bitcodeStartBit() { return BitcodeStartBit; }
296
297	size_t addToStrtab(StringRef Str);
298
299	void writeAttributeGroupTable();
300	void writeAttributeTable();
301	void writeTypeTable();
302	void writeComdats();
303	void writeValueSymbolTableForwardDecl();
304	void writeModuleInfo();
305	void writeValueAsMetadata(const ValueAsMetadata *MD,
306	SmallVectorImpl<uint64_t> &Record);
307	void writeMDTuple(const MDTuple *N, SmallVectorImpl<uint64_t> &Record,
308	unsigned Abbrev);
309	unsigned createDILocationAbbrev();
310	void writeDILocation(const DILocation *N, SmallVectorImpl<uint64_t> &Record,
311	unsigned &Abbrev);
312	unsigned createGenericDINodeAbbrev();
313	void writeGenericDINode(const GenericDINode *N,
314	SmallVectorImpl<uint64_t> &Record, unsigned &Abbrev);
315	void writeDISubrange(const DISubrange *N, SmallVectorImpl<uint64_t> &Record,
316	unsigned Abbrev);
317	void writeDIGenericSubrange(const DIGenericSubrange *N,
318	SmallVectorImpl<uint64_t> &Record,
319	unsigned Abbrev);
320	void writeDIEnumerator(const DIEnumerator *N,
321	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
322	void writeDIBasicType(const DIBasicType *N, SmallVectorImpl<uint64_t> &Record,
323	unsigned Abbrev);
324	void writeDIStringType(const DIStringType *N,
325	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
326	void writeDIDerivedType(const DIDerivedType *N,
327	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
328	void writeDICompositeType(const DICompositeType *N,
329	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
330	void writeDISubroutineType(const DISubroutineType *N,
331	SmallVectorImpl<uint64_t> &Record,
332	unsigned Abbrev);
333	void writeDIFile(const DIFile *N, SmallVectorImpl<uint64_t> &Record,
334	unsigned Abbrev);
335	void writeDICompileUnit(const DICompileUnit *N,
336	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
337	void writeDISubprogram(const DISubprogram *N,
338	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
339	void writeDILexicalBlock(const DILexicalBlock *N,
340	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
341	void writeDILexicalBlockFile(const DILexicalBlockFile *N,
342	SmallVectorImpl<uint64_t> &Record,
343	unsigned Abbrev);
344	void writeDICommonBlock(const DICommonBlock *N,
345	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
346	void writeDINamespace(const DINamespace *N, SmallVectorImpl<uint64_t> &Record,
347	unsigned Abbrev);
348	void writeDIMacro(const DIMacro *N, SmallVectorImpl<uint64_t> &Record,
349	unsigned Abbrev);
350	void writeDIMacroFile(const DIMacroFile *N, SmallVectorImpl<uint64_t> &Record,
351	unsigned Abbrev);
352	void writeDIArgList(const DIArgList *N, SmallVectorImpl<uint64_t> &Record);
353	void writeDIModule(const DIModule *N, SmallVectorImpl<uint64_t> &Record,
354	unsigned Abbrev);
355	void writeDIAssignID(const DIAssignID *N, SmallVectorImpl<uint64_t> &Record,
356	unsigned Abbrev);
357	void writeDITemplateTypeParameter(const DITemplateTypeParameter *N,
358	SmallVectorImpl<uint64_t> &Record,
359	unsigned Abbrev);
360	void writeDITemplateValueParameter(const DITemplateValueParameter *N,
361	SmallVectorImpl<uint64_t> &Record,
362	unsigned Abbrev);
363	void writeDIGlobalVariable(const DIGlobalVariable *N,
364	SmallVectorImpl<uint64_t> &Record,
365	unsigned Abbrev);
366	void writeDILocalVariable(const DILocalVariable *N,
367	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
368	void writeDILabel(const DILabel *N,
369	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
370	void writeDIExpression(const DIExpression *N,
371	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
372	void writeDIGlobalVariableExpression(const DIGlobalVariableExpression *N,
373	SmallVectorImpl<uint64_t> &Record,
374	unsigned Abbrev);
375	void writeDIObjCProperty(const DIObjCProperty *N,
376	SmallVectorImpl<uint64_t> &Record, unsigned Abbrev);
377	void writeDIImportedEntity(const DIImportedEntity *N,
378	SmallVectorImpl<uint64_t> &Record,
379	unsigned Abbrev);
380	unsigned createNamedMetadataAbbrev();
381	void writeNamedMetadata(SmallVectorImpl<uint64_t> &Record);
382	unsigned createMetadataStringsAbbrev();
383	void writeMetadataStrings(ArrayRef<const Metadata *> Strings,
384	SmallVectorImpl<uint64_t> &Record);
385	void writeMetadataRecords(ArrayRef<const Metadata *> MDs,
386	SmallVectorImpl<uint64_t> &Record,
387	std::vector<unsigned> *MDAbbrevs = nullptr,
388	std::vector<uint64_t> *IndexPos = nullptr);
389	void writeModuleMetadata();
390	void writeFunctionMetadata(const Function &F);
391	void writeFunctionMetadataAttachment(const Function &F);
392	void pushGlobalMetadataAttachment(SmallVectorImpl<uint64_t> &Record,
393	const GlobalObject &GO);
394	void writeModuleMetadataKinds();
395	void writeOperandBundleTags();
396	void writeSyncScopeNames();
397	void writeConstants(unsigned FirstVal, unsigned LastVal, bool isGlobal);
398	void writeModuleConstants();
399	bool pushValueAndType(const Value *V, unsigned InstID,
400	SmallVectorImpl<unsigned> &Vals);
401	void writeOperandBundles(const CallBase &CB, unsigned InstID);
402	void pushValue(const Value *V, unsigned InstID,
403	SmallVectorImpl<unsigned> &Vals);
404	void pushValueSigned(const Value *V, unsigned InstID,
405	SmallVectorImpl<uint64_t> &Vals);
406	void writeInstruction(const Instruction &I, unsigned InstID,
407	SmallVectorImpl<unsigned> &Vals);
408	void writeFunctionLevelValueSymbolTable(const ValueSymbolTable &VST);
409	void writeGlobalValueSymbolTable(
410	DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
411	void writeUseList(UseListOrder &&Order);
412	void writeUseListBlock(const Function *F);
413	void
414	writeFunction(const Function &F,
415	DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex);
416	void writeBlockInfo();
417	void writeModuleHash(size_t BlockStartPos);
418
419	unsigned getEncodedSyncScopeID(SyncScope::ID SSID) {
420	return unsigned(SSID);
421	}
422
423	unsigned getEncodedAlign(MaybeAlign Alignment) { return encode(A: Alignment); }
424	};
425
426	/// Class to manage the bitcode writing for a combined index.
427	class IndexBitcodeWriter : public BitcodeWriterBase {
428	/// The combined index to write to bitcode.
429	const ModuleSummaryIndex &Index;
430
431	/// When writing a subset of the index for distributed backends, client
432	/// provides a map of modules to the corresponding GUIDs/summaries to write.
433	const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex;
434
435	/// Map that holds the correspondence between the GUID used in the combined
436	/// index and a value id generated by this class to use in references.
437	std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
438
439	// The sorted stack id indices actually used in the summary entries being
440	// written, which will be a subset of those in the full index in the case of
441	// distributed indexes.
442	std::vector<unsigned> StackIdIndices;
443
444	/// Tracks the last value id recorded in the GUIDToValueMap.
445	unsigned GlobalValueId = 0;
446
447	/// Tracks the assignment of module paths in the module path string table to
448	/// an id assigned for use in summary references to the module path.
449	DenseMap<StringRef, uint64_t> ModuleIdMap;
450
451	public:
452	/// Constructs a IndexBitcodeWriter object for the given combined index,
453	/// writing to the provided \p Buffer. When writing a subset of the index
454	/// for a distributed backend, provide a \p ModuleToSummariesForIndex map.
455	IndexBitcodeWriter(BitstreamWriter &Stream, StringTableBuilder &StrtabBuilder,
456	const ModuleSummaryIndex &Index,
457	const std::map<std::string, GVSummaryMapTy>
458	*ModuleToSummariesForIndex = nullptr)
459	: BitcodeWriterBase(Stream, StrtabBuilder), Index(Index),
460	ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
461	// Assign unique value ids to all summaries to be written, for use
462	// in writing out the call graph edges. Save the mapping from GUID
463	// to the new global value id to use when writing those edges, which
464	// are currently saved in the index in terms of GUID.
465	forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
466	GUIDToValueIdMap[I.first] = ++GlobalValueId;
467	if (IsAliasee)
468	return;
469	auto *FS = dyn_cast<FunctionSummary>(Val: I.second);
470	if (!FS)
471	return;
472	// Record all stack id indices actually used in the summary entries being
473	// written, so that we can compact them in the case of distributed ThinLTO
474	// indexes.
475	for (auto &CI : FS->callsites()) {
476	// If the stack id list is empty, this callsite info was synthesized for
477	// a missing tail call frame. Ensure that the callee's GUID gets a value
478	// id. Normally we only generate these for defined summaries, which in
479	// the case of distributed ThinLTO is only the functions already defined
480	// in the module or that we want to import. We don't bother to include
481	// all the callee symbols as they aren't normally needed in the backend.
482	// However, for the synthesized callsite infos we do need the callee
483	// GUID in the backend so that we can correlate the identified callee
484	// with this callsite info (which for non-tail calls is done by the
485	// ordering of the callsite infos and verified via stack ids).
486	if (CI.StackIdIndices.empty()) {
487	GUIDToValueIdMap[CI.Callee.getGUID()] = ++GlobalValueId;
488	continue;
489	}
490	for (auto Idx : CI.StackIdIndices)
491	StackIdIndices.push_back(x: Idx);
492	}
493	for (auto &AI : FS->allocs())
494	for (auto &MIB : AI.MIBs)
495	for (auto Idx : MIB.StackIdIndices)
496	StackIdIndices.push_back(x: Idx);
497	});
498	llvm::sort(C&: StackIdIndices);
499	StackIdIndices.erase(
500	first: std::unique(first: StackIdIndices.begin(), last: StackIdIndices.end()),
501	last: StackIdIndices.end());
502	}
503
504	/// The below iterator returns the GUID and associated summary.
505	using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
506
507	/// Calls the callback for each value GUID and summary to be written to
508	/// bitcode. This hides the details of whether they are being pulled from the
509	/// entire index or just those in a provided ModuleToSummariesForIndex map.
510	template<typename Functor>
511	void forEachSummary(Functor Callback) {
512	if (ModuleToSummariesForIndex) {
513	for (auto &M : *ModuleToSummariesForIndex)
514	for (auto &Summary : M.second) {
515	Callback(Summary, false);
516	// Ensure aliasee is handled, e.g. for assigning a valueId,
517	// even if we are not importing the aliasee directly (the
518	// imported alias will contain a copy of aliasee).
519	if (auto *AS = dyn_cast<AliasSummary>(Val: Summary.getSecond()))
520	Callback({AS->getAliaseeGUID(), &AS->getAliasee()}, true);
521	}
522	} else {
523	for (auto &Summaries : Index)
524	for (auto &Summary : Summaries.second.SummaryList)
525	Callback({Summaries.first, Summary.get()}, false);
526	}
527	}
528
529	/// Calls the callback for each entry in the modulePaths StringMap that
530	/// should be written to the module path string table. This hides the details
531	/// of whether they are being pulled from the entire index or just those in a
532	/// provided ModuleToSummariesForIndex map.
533	template <typename Functor> void forEachModule(Functor Callback) {
534	if (ModuleToSummariesForIndex) {
535	for (const auto &M : *ModuleToSummariesForIndex) {
536	const auto &MPI = Index.modulePaths().find(Key: M.first);
537	if (MPI == Index.modulePaths().end()) {
538	// This should only happen if the bitcode file was empty, in which
539	// case we shouldn't be importing (the ModuleToSummariesForIndex
540	// would only include the module we are writing and index for).
541	assert(ModuleToSummariesForIndex->size() == 1);
542	continue;
543	}
544	Callback(*MPI);
545	}
546	} else {
547	// Since StringMap iteration order isn't guaranteed, order by path string
548	// first.
549	// FIXME: Make this a vector of StringMapEntry instead to avoid the later
550	// map lookup.
551	std::vector<StringRef> ModulePaths;
552	for (auto &[ModPath, _] : Index.modulePaths())
553	ModulePaths.push_back(x: ModPath);
554	llvm::sort(Start: ModulePaths.begin(), End: ModulePaths.end());
555	for (auto &ModPath : ModulePaths)
556	Callback(*Index.modulePaths().find(Key: ModPath));
557	}
558	}
559
560	/// Main entry point for writing a combined index to bitcode.
561	void write();
562
563	private:
564	void writeModStrings();
565	void writeCombinedGlobalValueSummary();
566
567	std::optional<unsigned> getValueId(GlobalValue::GUID ValGUID) {
568	auto VMI = GUIDToValueIdMap.find(x: ValGUID);
569	if (VMI == GUIDToValueIdMap.end())
570	return std::nullopt;
571	return VMI->second;
572	}
573
574	std::map<GlobalValue::GUID, unsigned> &valueIds() { return GUIDToValueIdMap; }
575	};
576
577	} // end anonymous namespace
578
579	static unsigned getEncodedCastOpcode(unsigned Opcode) {
580	switch (Opcode) {
581	default: llvm_unreachable("Unknown cast instruction!");
582	case Instruction::Trunc : return bitc::CAST_TRUNC;
583	case Instruction::ZExt : return bitc::CAST_ZEXT;
584	case Instruction::SExt : return bitc::CAST_SEXT;
585	case Instruction::FPToUI : return bitc::CAST_FPTOUI;
586	case Instruction::FPToSI : return bitc::CAST_FPTOSI;
587	case Instruction::UIToFP : return bitc::CAST_UITOFP;
588	case Instruction::SIToFP : return bitc::CAST_SITOFP;
589	case Instruction::FPTrunc : return bitc::CAST_FPTRUNC;
590	case Instruction::FPExt : return bitc::CAST_FPEXT;
591	case Instruction::PtrToInt: return bitc::CAST_PTRTOINT;
592	case Instruction::IntToPtr: return bitc::CAST_INTTOPTR;
593	case Instruction::BitCast : return bitc::CAST_BITCAST;
594	case Instruction::AddrSpaceCast: return bitc::CAST_ADDRSPACECAST;
595	}
596	}
597
598	static unsigned getEncodedUnaryOpcode(unsigned Opcode) {
599	switch (Opcode) {
600	default: llvm_unreachable("Unknown binary instruction!");
601	case Instruction::FNeg: return bitc::UNOP_FNEG;
602	}
603	}
604
605	static unsigned getEncodedBinaryOpcode(unsigned Opcode) {
606	switch (Opcode) {
607	default: llvm_unreachable("Unknown binary instruction!");
608	case Instruction::Add:
609	case Instruction::FAdd: return bitc::BINOP_ADD;
610	case Instruction::Sub:
611	case Instruction::FSub: return bitc::BINOP_SUB;
612	case Instruction::Mul:
613	case Instruction::FMul: return bitc::BINOP_MUL;
614	case Instruction::UDiv: return bitc::BINOP_UDIV;
615	case Instruction::FDiv:
616	case Instruction::SDiv: return bitc::BINOP_SDIV;
617	case Instruction::URem: return bitc::BINOP_UREM;
618	case Instruction::FRem:
619	case Instruction::SRem: return bitc::BINOP_SREM;
620	case Instruction::Shl: return bitc::BINOP_SHL;
621	case Instruction::LShr: return bitc::BINOP_LSHR;
622	case Instruction::AShr: return bitc::BINOP_ASHR;
623	case Instruction::And: return bitc::BINOP_AND;
624	case Instruction::Or: return bitc::BINOP_OR;
625	case Instruction::Xor: return bitc::BINOP_XOR;
626	}
627	}
628
629	static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
630	switch (Op) {
631	default: llvm_unreachable("Unknown RMW operation!");
632	case AtomicRMWInst::Xchg: return bitc::RMW_XCHG;
633	case AtomicRMWInst::Add: return bitc::RMW_ADD;
634	case AtomicRMWInst::Sub: return bitc::RMW_SUB;
635	case AtomicRMWInst::And: return bitc::RMW_AND;
636	case AtomicRMWInst::Nand: return bitc::RMW_NAND;
637	case AtomicRMWInst::Or: return bitc::RMW_OR;
638	case AtomicRMWInst::Xor: return bitc::RMW_XOR;
639	case AtomicRMWInst::Max: return bitc::RMW_MAX;
640	case AtomicRMWInst::Min: return bitc::RMW_MIN;
641	case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
642	case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
643	case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
644	case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
645	case AtomicRMWInst::FMax: return bitc::RMW_FMAX;
646	case AtomicRMWInst::FMin: return bitc::RMW_FMIN;
647	case AtomicRMWInst::UIncWrap:
648	return bitc::RMW_UINC_WRAP;
649	case AtomicRMWInst::UDecWrap:
650	return bitc::RMW_UDEC_WRAP;
651	}
652	}
653
654	static unsigned getEncodedOrdering(AtomicOrdering Ordering) {
655	switch (Ordering) {
656	case AtomicOrdering::NotAtomic: return bitc::ORDERING_NOTATOMIC;
657	case AtomicOrdering::Unordered: return bitc::ORDERING_UNORDERED;
658	case AtomicOrdering::Monotonic: return bitc::ORDERING_MONOTONIC;
659	case AtomicOrdering::Acquire: return bitc::ORDERING_ACQUIRE;
660	case AtomicOrdering::Release: return bitc::ORDERING_RELEASE;
661	case AtomicOrdering::AcquireRelease: return bitc::ORDERING_ACQREL;
662	case AtomicOrdering::SequentiallyConsistent: return bitc::ORDERING_SEQCST;
663	}
664	llvm_unreachable("Invalid ordering");
665	}
666
667	static void writeStringRecord(BitstreamWriter &Stream, unsigned Code,
668	StringRef Str, unsigned AbbrevToUse) {
669	SmallVector<unsigned, 64> Vals;
670
671	// Code: [strchar x N]
672	for (char C : Str) {
673	if (AbbrevToUse && !BitCodeAbbrevOp::isChar6(C))
674	AbbrevToUse = 0;
675	Vals.push_back(Elt: C);
676	}
677
678	// Emit the finished record.
679	Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
680	}
681
682	static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
683	switch (Kind) {
684	case Attribute::Alignment:
685	return bitc::ATTR_KIND_ALIGNMENT;
686	case Attribute::AllocAlign:
687	return bitc::ATTR_KIND_ALLOC_ALIGN;
688	case Attribute::AllocSize:
689	return bitc::ATTR_KIND_ALLOC_SIZE;
690	case Attribute::AlwaysInline:
691	return bitc::ATTR_KIND_ALWAYS_INLINE;
692	case Attribute::Builtin:
693	return bitc::ATTR_KIND_BUILTIN;
694	case Attribute::ByVal:
695	return bitc::ATTR_KIND_BY_VAL;
696	case Attribute::Convergent:
697	return bitc::ATTR_KIND_CONVERGENT;
698	case Attribute::InAlloca:
699	return bitc::ATTR_KIND_IN_ALLOCA;
700	case Attribute::Cold:
701	return bitc::ATTR_KIND_COLD;
702	case Attribute::DisableSanitizerInstrumentation:
703	return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION;
704	case Attribute::FnRetThunkExtern:
705	return bitc::ATTR_KIND_FNRETTHUNK_EXTERN;
706	case Attribute::Hot:
707	return bitc::ATTR_KIND_HOT;
708	case Attribute::ElementType:
709	return bitc::ATTR_KIND_ELEMENTTYPE;
710	case Attribute::InlineHint:
711	return bitc::ATTR_KIND_INLINE_HINT;
712	case Attribute::InReg:
713	return bitc::ATTR_KIND_IN_REG;
714	case Attribute::JumpTable:
715	return bitc::ATTR_KIND_JUMP_TABLE;
716	case Attribute::MinSize:
717	return bitc::ATTR_KIND_MIN_SIZE;
718	case Attribute::AllocatedPointer:
719	return bitc::ATTR_KIND_ALLOCATED_POINTER;
720	case Attribute::AllocKind:
721	return bitc::ATTR_KIND_ALLOC_KIND;
722	case Attribute::Memory:
723	return bitc::ATTR_KIND_MEMORY;
724	case Attribute::NoFPClass:
725	return bitc::ATTR_KIND_NOFPCLASS;
726	case Attribute::Naked:
727	return bitc::ATTR_KIND_NAKED;
728	case Attribute::Nest:
729	return bitc::ATTR_KIND_NEST;
730	case Attribute::NoAlias:
731	return bitc::ATTR_KIND_NO_ALIAS;
732	case Attribute::NoBuiltin:
733	return bitc::ATTR_KIND_NO_BUILTIN;
734	case Attribute::NoCallback:
735	return bitc::ATTR_KIND_NO_CALLBACK;
736	case Attribute::NoCapture:
737	return bitc::ATTR_KIND_NO_CAPTURE;
738	case Attribute::NoDuplicate:
739	return bitc::ATTR_KIND_NO_DUPLICATE;
740	case Attribute::NoFree:
741	return bitc::ATTR_KIND_NOFREE;
742	case Attribute::NoImplicitFloat:
743	return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT;
744	case Attribute::NoInline:
745	return bitc::ATTR_KIND_NO_INLINE;
746	case Attribute::NoRecurse:
747	return bitc::ATTR_KIND_NO_RECURSE;
748	case Attribute::NoMerge:
749	return bitc::ATTR_KIND_NO_MERGE;
750	case Attribute::NonLazyBind:
751	return bitc::ATTR_KIND_NON_LAZY_BIND;
752	case Attribute::NonNull:
753	return bitc::ATTR_KIND_NON_NULL;
754	case Attribute::Dereferenceable:
755	return bitc::ATTR_KIND_DEREFERENCEABLE;
756	case Attribute::DereferenceableOrNull:
757	return bitc::ATTR_KIND_DEREFERENCEABLE_OR_NULL;
758	case Attribute::NoRedZone:
759	return bitc::ATTR_KIND_NO_RED_ZONE;
760	case Attribute::NoReturn:
761	return bitc::ATTR_KIND_NO_RETURN;
762	case Attribute::NoSync:
763	return bitc::ATTR_KIND_NOSYNC;
764	case Attribute::NoCfCheck:
765	return bitc::ATTR_KIND_NOCF_CHECK;
766	case Attribute::NoProfile:
767	return bitc::ATTR_KIND_NO_PROFILE;
768	case Attribute::SkipProfile:
769	return bitc::ATTR_KIND_SKIP_PROFILE;
770	case Attribute::NoUnwind:
771	return bitc::ATTR_KIND_NO_UNWIND;
772	case Attribute::NoSanitizeBounds:
773	return bitc::ATTR_KIND_NO_SANITIZE_BOUNDS;
774	case Attribute::NoSanitizeCoverage:
775	return bitc::ATTR_KIND_NO_SANITIZE_COVERAGE;
776	case Attribute::NullPointerIsValid:
777	return bitc::ATTR_KIND_NULL_POINTER_IS_VALID;
778	case Attribute::OptimizeForDebugging:
779	return bitc::ATTR_KIND_OPTIMIZE_FOR_DEBUGGING;
780	case Attribute::OptForFuzzing:
781	return bitc::ATTR_KIND_OPT_FOR_FUZZING;
782	case Attribute::OptimizeForSize:
783	return bitc::ATTR_KIND_OPTIMIZE_FOR_SIZE;
784	case Attribute::OptimizeNone:
785	return bitc::ATTR_KIND_OPTIMIZE_NONE;
786	case Attribute::ReadNone:
787	return bitc::ATTR_KIND_READ_NONE;
788	case Attribute::ReadOnly:
789	return bitc::ATTR_KIND_READ_ONLY;
790	case Attribute::Returned:
791	return bitc::ATTR_KIND_RETURNED;
792	case Attribute::ReturnsTwice:
793	return bitc::ATTR_KIND_RETURNS_TWICE;
794	case Attribute::SExt:
795	return bitc::ATTR_KIND_S_EXT;
796	case Attribute::Speculatable:
797	return bitc::ATTR_KIND_SPECULATABLE;
798	case Attribute::StackAlignment:
799	return bitc::ATTR_KIND_STACK_ALIGNMENT;
800	case Attribute::StackProtect:
801	return bitc::ATTR_KIND_STACK_PROTECT;
802	case Attribute::StackProtectReq:
803	return bitc::ATTR_KIND_STACK_PROTECT_REQ;
804	case Attribute::StackProtectStrong:
805	return bitc::ATTR_KIND_STACK_PROTECT_STRONG;
806	case Attribute::SafeStack:
807	return bitc::ATTR_KIND_SAFESTACK;
808	case Attribute::ShadowCallStack:
809	return bitc::ATTR_KIND_SHADOWCALLSTACK;
810	case Attribute::StrictFP:
811	return bitc::ATTR_KIND_STRICT_FP;
812	case Attribute::StructRet:
813	return bitc::ATTR_KIND_STRUCT_RET;
814	case Attribute::SanitizeAddress:
815	return bitc::ATTR_KIND_SANITIZE_ADDRESS;
816	case Attribute::SanitizeHWAddress:
817	return bitc::ATTR_KIND_SANITIZE_HWADDRESS;
818	case Attribute::SanitizeThread:
819	return bitc::ATTR_KIND_SANITIZE_THREAD;
820	case Attribute::SanitizeMemory:
821	return bitc::ATTR_KIND_SANITIZE_MEMORY;
822	case Attribute::SpeculativeLoadHardening:
823	return bitc::ATTR_KIND_SPECULATIVE_LOAD_HARDENING;
824	case Attribute::SwiftError:
825	return bitc::ATTR_KIND_SWIFT_ERROR;
826	case Attribute::SwiftSelf:
827	return bitc::ATTR_KIND_SWIFT_SELF;
828	case Attribute::SwiftAsync:
829	return bitc::ATTR_KIND_SWIFT_ASYNC;
830	case Attribute::UWTable:
831	return bitc::ATTR_KIND_UW_TABLE;
832	case Attribute::VScaleRange:
833	return bitc::ATTR_KIND_VSCALE_RANGE;
834	case Attribute::WillReturn:
835	return bitc::ATTR_KIND_WILLRETURN;
836	case Attribute::WriteOnly:
837	return bitc::ATTR_KIND_WRITEONLY;
838	case Attribute::ZExt:
839	return bitc::ATTR_KIND_Z_EXT;
840	case Attribute::ImmArg:
841	return bitc::ATTR_KIND_IMMARG;
842	case Attribute::SanitizeMemTag:
843	return bitc::ATTR_KIND_SANITIZE_MEMTAG;
844	case Attribute::Preallocated:
845	return bitc::ATTR_KIND_PREALLOCATED;
846	case Attribute::NoUndef:
847	return bitc::ATTR_KIND_NOUNDEF;
848	case Attribute::ByRef:
849	return bitc::ATTR_KIND_BYREF;
850	case Attribute::MustProgress:
851	return bitc::ATTR_KIND_MUSTPROGRESS;
852	case Attribute::PresplitCoroutine:
853	return bitc::ATTR_KIND_PRESPLIT_COROUTINE;
854	case Attribute::Writable:
855	return bitc::ATTR_KIND_WRITABLE;
856	case Attribute::CoroDestroyOnlyWhenComplete:
857	return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
858	case Attribute::DeadOnUnwind:
859	return bitc::ATTR_KIND_DEAD_ON_UNWIND;
860	case Attribute::Range:
861	return bitc::ATTR_KIND_RANGE;
862	case Attribute::EndAttrKinds:
863	llvm_unreachable("Can not encode end-attribute kinds marker.");
864	case Attribute::None:
865	llvm_unreachable("Can not encode none-attribute.");
866	case Attribute::EmptyKey:
867	case Attribute::TombstoneKey:
868	llvm_unreachable("Trying to encode EmptyKey/TombstoneKey");
869	}
870
871	llvm_unreachable("Trying to encode unknown attribute");
872	}
873
874	static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
875	if ((int64_t)V >= 0)
876	Vals.push_back(Elt: V << 1);
877	else
878	Vals.push_back(Elt: (-V << 1) | 1);
879	}
880
881	static void emitWideAPInt(SmallVectorImpl<uint64_t> &Vals, const APInt &A) {
882	// We have an arbitrary precision integer value to write whose
883	// bit width is > 64. However, in canonical unsigned integer
884	// format it is likely that the high bits are going to be zero.
885	// So, we only write the number of active words.
886	unsigned NumWords = A.getActiveWords();
887	const uint64_t *RawData = A.getRawData();
888	for (unsigned i = 0; i < NumWords; i++)
889	emitSignedInt64(Vals, V: RawData[i]);
890	}
891
892	static void emitConstantRange(SmallVectorImpl<uint64_t> &Record,
893	const ConstantRange &CR) {
894	unsigned BitWidth = CR.getBitWidth();
895	Record.push_back(Elt: BitWidth);
896	if (BitWidth > 64) {
897	Record.push_back(Elt: CR.getLower().getActiveWords() |
898	(uint64_t(CR.getUpper().getActiveWords()) << 32));
899	emitWideAPInt(Vals&: Record, A: CR.getLower());
900	emitWideAPInt(Vals&: Record, A: CR.getUpper());
901	} else {
902	emitSignedInt64(Vals&: Record, V: CR.getLower().getSExtValue());
903	emitSignedInt64(Vals&: Record, V: CR.getUpper().getSExtValue());
904	}
905	}
906
907	void ModuleBitcodeWriter::writeAttributeGroupTable() {
908	const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
909	VE.getAttributeGroups();
910	if (AttrGrps.empty()) return;
911
912	Stream.EnterSubblock(BlockID: bitc::PARAMATTR_GROUP_BLOCK_ID, CodeLen: 3);
913
914	SmallVector<uint64_t, 64> Record;
915	for (ValueEnumerator::IndexAndAttrSet Pair : AttrGrps) {
916	unsigned AttrListIndex = Pair.first;
917	AttributeSet AS = Pair.second;
918	Record.push_back(Elt: VE.getAttributeGroupID(Group: Pair));
919	Record.push_back(Elt: AttrListIndex);
920
921	for (Attribute Attr : AS) {
922	if (Attr.isEnumAttribute()) {
923	Record.push_back(Elt: 0);
924	Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
925	} else if (Attr.isIntAttribute()) {
926	Record.push_back(Elt: 1);
927	Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
928	Record.push_back(Elt: Attr.getValueAsInt());
929	} else if (Attr.isStringAttribute()) {
930	StringRef Kind = Attr.getKindAsString();
931	StringRef Val = Attr.getValueAsString();
932
933	Record.push_back(Elt: Val.empty() ? 3 : 4);
934	Record.append(in_start: Kind.begin(), in_end: Kind.end());
935	Record.push_back(Elt: 0);
936	if (!Val.empty()) {
937	Record.append(in_start: Val.begin(), in_end: Val.end());
938	Record.push_back(Elt: 0);
939	}
940	} else if (Attr.isTypeAttribute()) {
941	Type *Ty = Attr.getValueAsType();
942	Record.push_back(Elt: Ty ? 6 : 5);
943	Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
944	if (Ty)
945	Record.push_back(Elt: VE.getTypeID(T: Attr.getValueAsType()));
946	} else {
947	assert(Attr.isConstantRangeAttribute());
948	Record.push_back(Elt: 7);
949	Record.push_back(Elt: getAttrKindEncoding(Kind: Attr.getKindAsEnum()));
950	emitConstantRange(Record, CR: Attr.getValueAsConstantRange());
951	}
952	}
953
954	Stream.EmitRecord(Code: bitc::PARAMATTR_GRP_CODE_ENTRY, Vals: Record);
955	Record.clear();
956	}
957
958	Stream.ExitBlock();
959	}
960
961	void ModuleBitcodeWriter::writeAttributeTable() {
962	const std::vector<AttributeList> &Attrs = VE.getAttributeLists();
963	if (Attrs.empty()) return;
964
965	Stream.EnterSubblock(BlockID: bitc::PARAMATTR_BLOCK_ID, CodeLen: 3);
966
967	SmallVector<uint64_t, 64> Record;
968	for (const AttributeList &AL : Attrs) {
969	for (unsigned i : AL.indexes()) {
970	AttributeSet AS = AL.getAttributes(Index: i);
971	if (AS.hasAttributes())
972	Record.push_back(Elt: VE.getAttributeGroupID(Group: {i, AS}));
973	}
974
975	Stream.EmitRecord(Code: bitc::PARAMATTR_CODE_ENTRY, Vals: Record);
976	Record.clear();
977	}
978
979	Stream.ExitBlock();
980	}
981
982	/// WriteTypeTable - Write out the type table for a module.
983	void ModuleBitcodeWriter::writeTypeTable() {
984	const ValueEnumerator::TypeList &TypeList = VE.getTypes();
985
986	Stream.EnterSubblock(BlockID: bitc::TYPE_BLOCK_ID_NEW, CodeLen: 4 /*count from # abbrevs */);
987	SmallVector<uint64_t, 64> TypeVals;
988
989	uint64_t NumBits = VE.computeBitsRequiredForTypeIndicies();
990
991	// Abbrev for TYPE_CODE_OPAQUE_POINTER.
992	auto Abbv = std::make_shared<BitCodeAbbrev>();
993	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_OPAQUE_POINTER));
994	Abbv->Add(OpInfo: BitCodeAbbrevOp(0)); // Addrspace = 0
995	unsigned OpaquePtrAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
996
997	// Abbrev for TYPE_CODE_FUNCTION.
998	Abbv = std::make_shared<BitCodeAbbrev>();
999	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_FUNCTION));
1000	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isvararg
1001	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1002	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1003	unsigned FunctionAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1004
1005	// Abbrev for TYPE_CODE_STRUCT_ANON.
1006	Abbv = std::make_shared<BitCodeAbbrev>();
1007	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_ANON));
1008	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1009	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1010	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1011	unsigned StructAnonAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1012
1013	// Abbrev for TYPE_CODE_STRUCT_NAME.
1014	Abbv = std::make_shared<BitCodeAbbrev>();
1015	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAME));
1016	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1017	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
1018	unsigned StructNameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1019
1020	// Abbrev for TYPE_CODE_STRUCT_NAMED.
1021	Abbv = std::make_shared<BitCodeAbbrev>();
1022	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_STRUCT_NAMED));
1023	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // ispacked
1024	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1025	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1026	unsigned StructNamedAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1027
1028	// Abbrev for TYPE_CODE_ARRAY.
1029	Abbv = std::make_shared<BitCodeAbbrev>();
1030	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::TYPE_CODE_ARRAY));
1031	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // size
1032	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, NumBits));
1033	unsigned ArrayAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1034
1035	// Emit an entry count so the reader can reserve space.
1036	TypeVals.push_back(Elt: TypeList.size());
1037	Stream.EmitRecord(Code: bitc::TYPE_CODE_NUMENTRY, Vals: TypeVals);
1038	TypeVals.clear();
1039
1040	// Loop over all of the types, emitting each in turn.
1041	for (Type *T : TypeList) {
1042	int AbbrevToUse = 0;
1043	unsigned Code = 0;
1044
1045	switch (T->getTypeID()) {
1046	case Type::VoidTyID: Code = bitc::TYPE_CODE_VOID; break;
1047	case Type::HalfTyID: Code = bitc::TYPE_CODE_HALF; break;
1048	case Type::BFloatTyID: Code = bitc::TYPE_CODE_BFLOAT; break;
1049	case Type::FloatTyID: Code = bitc::TYPE_CODE_FLOAT; break;
1050	case Type::DoubleTyID: Code = bitc::TYPE_CODE_DOUBLE; break;
1051	case Type::X86_FP80TyID: Code = bitc::TYPE_CODE_X86_FP80; break;
1052	case Type::FP128TyID: Code = bitc::TYPE_CODE_FP128; break;
1053	case Type::PPC_FP128TyID: Code = bitc::TYPE_CODE_PPC_FP128; break;
1054	case Type::LabelTyID: Code = bitc::TYPE_CODE_LABEL; break;
1055	case Type::MetadataTyID: Code = bitc::TYPE_CODE_METADATA; break;
1056	case Type::X86_MMXTyID: Code = bitc::TYPE_CODE_X86_MMX; break;
1057	case Type::X86_AMXTyID: Code = bitc::TYPE_CODE_X86_AMX; break;
1058	case Type::TokenTyID: Code = bitc::TYPE_CODE_TOKEN; break;
1059	case Type::IntegerTyID:
1060	// INTEGER: [width]
1061	Code = bitc::TYPE_CODE_INTEGER;
1062	TypeVals.push_back(Elt: cast<IntegerType>(Val: T)->getBitWidth());
1063	break;
1064	case Type::PointerTyID: {
1065	PointerType *PTy = cast<PointerType>(Val: T);
1066	unsigned AddressSpace = PTy->getAddressSpace();
1067	// OPAQUE_POINTER: [address space]
1068	Code = bitc::TYPE_CODE_OPAQUE_POINTER;
1069	TypeVals.push_back(Elt: AddressSpace);
1070	if (AddressSpace == 0)
1071	AbbrevToUse = OpaquePtrAbbrev;
1072	break;
1073	}
1074	case Type::FunctionTyID: {
1075	FunctionType *FT = cast<FunctionType>(Val: T);
1076	// FUNCTION: [isvararg, retty, paramty x N]
1077	Code = bitc::TYPE_CODE_FUNCTION;
1078	TypeVals.push_back(Elt: FT->isVarArg());
1079	TypeVals.push_back(Elt: VE.getTypeID(T: FT->getReturnType()));
1080	for (unsigned i = 0, e = FT->getNumParams(); i != e; ++i)
1081	TypeVals.push_back(Elt: VE.getTypeID(T: FT->getParamType(i)));
1082	AbbrevToUse = FunctionAbbrev;
1083	break;
1084	}
1085	case Type::StructTyID: {
1086	StructType *ST = cast<StructType>(Val: T);
1087	// STRUCT: [ispacked, eltty x N]
1088	TypeVals.push_back(Elt: ST->isPacked());
1089	// Output all of the element types.
1090	for (Type *ET : ST->elements())
1091	TypeVals.push_back(Elt: VE.getTypeID(T: ET));
1092
1093	if (ST->isLiteral()) {
1094	Code = bitc::TYPE_CODE_STRUCT_ANON;
1095	AbbrevToUse = StructAnonAbbrev;
1096	} else {
1097	if (ST->isOpaque()) {
1098	Code = bitc::TYPE_CODE_OPAQUE;
1099	} else {
1100	Code = bitc::TYPE_CODE_STRUCT_NAMED;
1101	AbbrevToUse = StructNamedAbbrev;
1102	}
1103
1104	// Emit the name if it is present.
1105	if (!ST->getName().empty())
1106	writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: ST->getName(),
1107	AbbrevToUse: StructNameAbbrev);
1108	}
1109	break;
1110	}
1111	case Type::ArrayTyID: {
1112	ArrayType *AT = cast<ArrayType>(Val: T);
1113	// ARRAY: [numelts, eltty]
1114	Code = bitc::TYPE_CODE_ARRAY;
1115	TypeVals.push_back(Elt: AT->getNumElements());
1116	TypeVals.push_back(Elt: VE.getTypeID(T: AT->getElementType()));
1117	AbbrevToUse = ArrayAbbrev;
1118	break;
1119	}
1120	case Type::FixedVectorTyID:
1121	case Type::ScalableVectorTyID: {
1122	VectorType *VT = cast<VectorType>(Val: T);
1123	// VECTOR [numelts, eltty] or
1124	// [numelts, eltty, scalable]
1125	Code = bitc::TYPE_CODE_VECTOR;
1126	TypeVals.push_back(Elt: VT->getElementCount().getKnownMinValue());
1127	TypeVals.push_back(Elt: VE.getTypeID(T: VT->getElementType()));
1128	if (isa<ScalableVectorType>(Val: VT))
1129	TypeVals.push_back(Elt: true);
1130	break;
1131	}
1132	case Type::TargetExtTyID: {
1133	TargetExtType *TET = cast<TargetExtType>(Val: T);
1134	Code = bitc::TYPE_CODE_TARGET_TYPE;
1135	writeStringRecord(Stream, Code: bitc::TYPE_CODE_STRUCT_NAME, Str: TET->getName(),
1136	AbbrevToUse: StructNameAbbrev);
1137	TypeVals.push_back(Elt: TET->getNumTypeParameters());
1138	for (Type *InnerTy : TET->type_params())
1139	TypeVals.push_back(Elt: VE.getTypeID(T: InnerTy));
1140	for (unsigned IntParam : TET->int_params())
1141	TypeVals.push_back(Elt: IntParam);
1142	break;
1143	}
1144	case Type::TypedPointerTyID:
1145	llvm_unreachable("Typed pointers cannot be added to IR modules");
1146	}
1147
1148	// Emit the finished record.
1149	Stream.EmitRecord(Code, Vals: TypeVals, Abbrev: AbbrevToUse);
1150	TypeVals.clear();
1151	}
1152
1153	Stream.ExitBlock();
1154	}
1155
1156	static unsigned getEncodedLinkage(const GlobalValue::LinkageTypes Linkage) {
1157	switch (Linkage) {
1158	case GlobalValue::ExternalLinkage:
1159	return 0;
1160	case GlobalValue::WeakAnyLinkage:
1161	return 16;
1162	case GlobalValue::AppendingLinkage:
1163	return 2;
1164	case GlobalValue::InternalLinkage:
1165	return 3;
1166	case GlobalValue::LinkOnceAnyLinkage:
1167	return 18;
1168	case GlobalValue::ExternalWeakLinkage:
1169	return 7;
1170	case GlobalValue::CommonLinkage:
1171	return 8;
1172	case GlobalValue::PrivateLinkage:
1173	return 9;
1174	case GlobalValue::WeakODRLinkage:
1175	return 17;
1176	case GlobalValue::LinkOnceODRLinkage:
1177	return 19;
1178	case GlobalValue::AvailableExternallyLinkage:
1179	return 12;
1180	}
1181	llvm_unreachable("Invalid linkage");
1182	}
1183
1184	static unsigned getEncodedLinkage(const GlobalValue &GV) {
1185	return getEncodedLinkage(Linkage: GV.getLinkage());
1186	}
1187
1188	static uint64_t getEncodedFFlags(FunctionSummary::FFlags Flags) {
1189	uint64_t RawFlags = 0;
1190	RawFlags |= Flags.ReadNone;
1191	RawFlags |= (Flags.ReadOnly << 1);
1192	RawFlags |= (Flags.NoRecurse << 2);
1193	RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1194	RawFlags |= (Flags.NoInline << 4);
1195	RawFlags |= (Flags.AlwaysInline << 5);
1196	RawFlags |= (Flags.NoUnwind << 6);
1197	RawFlags |= (Flags.MayThrow << 7);
1198	RawFlags |= (Flags.HasUnknownCall << 8);
1199	RawFlags |= (Flags.MustBeUnreachable << 9);
1200	return RawFlags;
1201	}
1202
1203	// Decode the flags for GlobalValue in the summary. See getDecodedGVSummaryFlags
1204	// in BitcodeReader.cpp.
1205	static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) {
1206	uint64_t RawFlags = 0;
1207
1208	RawFlags |= Flags.NotEligibleToImport; // bool
1209	RawFlags |= (Flags.Live << 1);
1210	RawFlags |= (Flags.DSOLocal << 2);
1211	RawFlags |= (Flags.CanAutoHide << 3);
1212
1213	// Linkage don't need to be remapped at that time for the summary. Any future
1214	// change to the getEncodedLinkage() function will need to be taken into
1215	// account here as well.
1216	RawFlags = (RawFlags << 4) | Flags.Linkage; // 4 bits
1217
1218	RawFlags |= (Flags.Visibility << 8); // 2 bits
1219
1220	RawFlags |= (Flags.ImportType << 10); // 1 bit
1221
1222	return RawFlags;
1223	}
1224
1225	static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) {
1226	uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1227	(Flags.Constant << 2) | Flags.VCallVisibility << 3;
1228	return RawFlags;
1229	}
1230
1231	static uint64_t getEncodedHotnessCallEdgeInfo(const CalleeInfo &CI) {
1232	uint64_t RawFlags = 0;
1233
1234	RawFlags |= CI.Hotness; // 3 bits
1235	RawFlags |= (CI.HasTailCall << 3); // 1 bit
1236
1237	return RawFlags;
1238	}
1239
1240	static uint64_t getEncodedRelBFCallEdgeInfo(const CalleeInfo &CI) {
1241	uint64_t RawFlags = 0;
1242
1243	RawFlags |= CI.RelBlockFreq; // CalleeInfo::RelBlockFreqBits bits
1244	RawFlags |= (CI.HasTailCall << CalleeInfo::RelBlockFreqBits); // 1 bit
1245
1246	return RawFlags;
1247	}
1248
1249	static unsigned getEncodedVisibility(const GlobalValue &GV) {
1250	switch (GV.getVisibility()) {
1251	case GlobalValue::DefaultVisibility: return 0;
1252	case GlobalValue::HiddenVisibility: return 1;
1253	case GlobalValue::ProtectedVisibility: return 2;
1254	}
1255	llvm_unreachable("Invalid visibility");
1256	}
1257
1258	static unsigned getEncodedDLLStorageClass(const GlobalValue &GV) {
1259	switch (GV.getDLLStorageClass()) {
1260	case GlobalValue::DefaultStorageClass: return 0;
1261	case GlobalValue::DLLImportStorageClass: return 1;
1262	case GlobalValue::DLLExportStorageClass: return 2;
1263	}
1264	llvm_unreachable("Invalid DLL storage class");
1265	}
1266
1267	static unsigned getEncodedThreadLocalMode(const GlobalValue &GV) {
1268	switch (GV.getThreadLocalMode()) {
1269	case GlobalVariable::NotThreadLocal: return 0;
1270	case GlobalVariable::GeneralDynamicTLSModel: return 1;
1271	case GlobalVariable::LocalDynamicTLSModel: return 2;
1272	case GlobalVariable::InitialExecTLSModel: return 3;
1273	case GlobalVariable::LocalExecTLSModel: return 4;
1274	}
1275	llvm_unreachable("Invalid TLS model");
1276	}
1277
1278	static unsigned getEncodedComdatSelectionKind(const Comdat &C) {
1279	switch (C.getSelectionKind()) {
1280	case Comdat::Any:
1281	return bitc::COMDAT_SELECTION_KIND_ANY;
1282	case Comdat::ExactMatch:
1283	return bitc::COMDAT_SELECTION_KIND_EXACT_MATCH;
1284	case Comdat::Largest:
1285	return bitc::COMDAT_SELECTION_KIND_LARGEST;
1286	case Comdat::NoDeduplicate:
1287	return bitc::COMDAT_SELECTION_KIND_NO_DUPLICATES;
1288	case Comdat::SameSize:
1289	return bitc::COMDAT_SELECTION_KIND_SAME_SIZE;
1290	}
1291	llvm_unreachable("Invalid selection kind");
1292	}
1293
1294	static unsigned getEncodedUnnamedAddr(const GlobalValue &GV) {
1295	switch (GV.getUnnamedAddr()) {
1296	case GlobalValue::UnnamedAddr::None: return 0;
1297	case GlobalValue::UnnamedAddr::Local: return 2;
1298	case GlobalValue::UnnamedAddr::Global: return 1;
1299	}
1300	llvm_unreachable("Invalid unnamed_addr");
1301	}
1302
1303	size_t ModuleBitcodeWriter::addToStrtab(StringRef Str) {
1304	if (GenerateHash)
1305	Hasher.update(Str);
1306	return StrtabBuilder.add(S: Str);
1307	}
1308
1309	void ModuleBitcodeWriter::writeComdats() {
1310	SmallVector<unsigned, 64> Vals;
1311	for (const Comdat *C : VE.getComdats()) {
1312	// COMDAT: [strtab offset, strtab size, selection_kind]
1313	Vals.push_back(Elt: addToStrtab(Str: C->getName()));
1314	Vals.push_back(Elt: C->getName().size());
1315	Vals.push_back(Elt: getEncodedComdatSelectionKind(C: *C));
1316	Stream.EmitRecord(Code: bitc::MODULE_CODE_COMDAT, Vals, /*AbbrevToUse=*/Abbrev: 0);
1317	Vals.clear();
1318	}
1319	}
1320
1321	/// Write a record that will eventually hold the word offset of the
1322	/// module-level VST. For now the offset is 0, which will be backpatched
1323	/// after the real VST is written. Saves the bit offset to backpatch.
1324	void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1325	// Write a placeholder value in for the offset of the real VST,
1326	// which is written after the function blocks so that it can include
1327	// the offset of each function. The placeholder offset will be
1328	// updated when the real VST is written.
1329	auto Abbv = std::make_shared<BitCodeAbbrev>();
1330	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_VSTOFFSET));
1331	// Blocks are 32-bit aligned, so we can use a 32-bit word offset to
1332	// hold the real VST offset. Must use fixed instead of VBR as we don't
1333	// know how many VBR chunks to reserve ahead of time.
1334	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
1335	unsigned VSTOffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1336
1337	// Emit the placeholder
1338	uint64_t Vals[] = {bitc::MODULE_CODE_VSTOFFSET, 0};
1339	Stream.EmitRecordWithAbbrev(Abbrev: VSTOffsetAbbrev, Vals);
1340
1341	// Compute and save the bit offset to the placeholder, which will be
1342	// patched when the real VST is written. We can simply subtract the 32-bit
1343	// fixed size from the current bit number to get the location to backpatch.
1344	VSTOffsetPlaceholder = Stream.GetCurrentBitNo() - 32;
1345	}
1346
1347	enum StringEncoding { SE_Char6, SE_Fixed7, SE_Fixed8 };
1348
1349	/// Determine the encoding to use for the given string name and length.
1350	static StringEncoding getStringEncoding(StringRef Str) {
1351	bool isChar6 = true;
1352	for (char C : Str) {
1353	if (isChar6)
1354	isChar6 = BitCodeAbbrevOp::isChar6(C);
1355	if ((unsigned char)C & 128)
1356	// don't bother scanning the rest.
1357	return SE_Fixed8;
1358	}
1359	if (isChar6)
1360	return SE_Char6;
1361	return SE_Fixed7;
1362	}
1363
1364	static_assert(sizeof(GlobalValue::SanitizerMetadata) <= sizeof(unsigned),
1365	"Sanitizer Metadata is too large for naive serialization.");
1366	static unsigned
1367	serializeSanitizerMetadata(const GlobalValue::SanitizerMetadata &Meta) {
1368	return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1369	(Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1370	}
1371
1372	/// Emit top-level description of module, including target triple, inline asm,
1373	/// descriptors for global variables, and function prototype info.
1374	/// Returns the bit offset to backpatch with the location of the real VST.
1375	void ModuleBitcodeWriter::writeModuleInfo() {
1376	// Emit various pieces of data attached to a module.
1377	if (!M.getTargetTriple().empty())
1378	writeStringRecord(Stream, Code: bitc::MODULE_CODE_TRIPLE, Str: M.getTargetTriple(),
1379	AbbrevToUse: 0 /*TODO*/);
1380	const std::string &DL = M.getDataLayoutStr();
1381	if (!DL.empty())
1382	writeStringRecord(Stream, Code: bitc::MODULE_CODE_DATALAYOUT, Str: DL, AbbrevToUse: 0 /*TODO*/);
1383	if (!M.getModuleInlineAsm().empty())
1384	writeStringRecord(Stream, Code: bitc::MODULE_CODE_ASM, Str: M.getModuleInlineAsm(),
1385	AbbrevToUse: 0 /*TODO*/);
1386
1387	// Emit information about sections and GC, computing how many there are. Also
1388	// compute the maximum alignment value.
1389	std::map<std::string, unsigned> SectionMap;
1390	std::map<std::string, unsigned> GCMap;
1391	MaybeAlign MaxAlignment;
1392	unsigned MaxGlobalType = 0;
1393	const auto UpdateMaxAlignment = [&MaxAlignment](const MaybeAlign A) {
1394	if (A)
1395	MaxAlignment = !MaxAlignment ? *A : std::max(a: *MaxAlignment, b: *A);
1396	};
1397	for (const GlobalVariable &GV : M.globals()) {
1398	UpdateMaxAlignment(GV.getAlign());
1399	MaxGlobalType = std::max(a: MaxGlobalType, b: VE.getTypeID(T: GV.getValueType()));
1400	if (GV.hasSection()) {
1401	// Give section names unique ID's.
1402	unsigned &Entry = SectionMap[std::string(GV.getSection())];
1403	if (!Entry) {
1404	writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: GV.getSection(),
1405	AbbrevToUse: 0 /*TODO*/);
1406	Entry = SectionMap.size();
1407	}
1408	}
1409	}
1410	for (const Function &F : M) {
1411	UpdateMaxAlignment(F.getAlign());
1412	if (F.hasSection()) {
1413	// Give section names unique ID's.
1414	unsigned &Entry = SectionMap[std::string(F.getSection())];
1415	if (!Entry) {
1416	writeStringRecord(Stream, Code: bitc::MODULE_CODE_SECTIONNAME, Str: F.getSection(),
1417	AbbrevToUse: 0 /*TODO*/);
1418	Entry = SectionMap.size();
1419	}
1420	}
1421	if (F.hasGC()) {
1422	// Same for GC names.
1423	unsigned &Entry = GCMap[F.getGC()];
1424	if (!Entry) {
1425	writeStringRecord(Stream, Code: bitc::MODULE_CODE_GCNAME, Str: F.getGC(),
1426	AbbrevToUse: 0 /*TODO*/);
1427	Entry = GCMap.size();
1428	}
1429	}
1430	}
1431
1432	// Emit abbrev for globals, now that we know # sections and max alignment.
1433	unsigned SimpleGVarAbbrev = 0;
1434	if (!M.global_empty()) {
1435	// Add an abbrev for common globals with no visibility or thread localness.
1436	auto Abbv = std::make_shared<BitCodeAbbrev>();
1437	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_GLOBALVAR));
1438	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1439	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1440	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1441	Log2_32_Ceil(Value: MaxGlobalType+1)));
1442	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddrSpace << 2
1443	//| explicitType << 1
1444	//| constant
1445	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Initializer.
1446	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 5)); // Linkage.
1447	if (!MaxAlignment) // Alignment.
1448	Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1449	else {
1450	unsigned MaxEncAlignment = getEncodedAlign(Alignment: MaxAlignment);
1451	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1452	Log2_32_Ceil(Value: MaxEncAlignment+1)));
1453	}
1454	if (SectionMap.empty()) // Section.
1455	Abbv->Add(OpInfo: BitCodeAbbrevOp(0));
1456	else
1457	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
1458	Log2_32_Ceil(Value: SectionMap.size()+1)));
1459	// Don't bother emitting vis + thread local.
1460	SimpleGVarAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1461	}
1462
1463	SmallVector<unsigned, 64> Vals;
1464	// Emit the module's source file name.
1465	{
1466	StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
1467	BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
1468	if (Bits == SE_Char6)
1469	AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
1470	else if (Bits == SE_Fixed7)
1471	AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
1472
1473	// MODULE_CODE_SOURCE_FILENAME: [namechar x N]
1474	auto Abbv = std::make_shared<BitCodeAbbrev>();
1475	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
1476	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1477	Abbv->Add(OpInfo: AbbrevOpToUse);
1478	unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
1479
1480	for (const auto P : M.getSourceFileName())
1481	Vals.push_back(Elt: (unsigned char)P);
1482
1483	// Emit the finished record.
1484	Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
1485	Vals.clear();
1486	}
1487
1488	// Emit the global variable information.
1489	for (const GlobalVariable &GV : M.globals()) {
1490	unsigned AbbrevToUse = 0;
1491
1492	// GLOBALVAR: [strtab offset, strtab size, type, isconst, initid,
1493	// linkage, alignment, section, visibility, threadlocal,
1494	// unnamed_addr, externally_initialized, dllstorageclass,
1495	// comdat, attributes, DSO_Local, GlobalSanitizer, code_model]
1496	Vals.push_back(Elt: addToStrtab(Str: GV.getName()));
1497	Vals.push_back(Elt: GV.getName().size());
1498	Vals.push_back(Elt: VE.getTypeID(T: GV.getValueType()));
1499	Vals.push_back(Elt: GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1500	Vals.push_back(Elt: GV.isDeclaration() ? 0 :
1501	(VE.getValueID(V: GV.getInitializer()) + 1));
1502	Vals.push_back(Elt: getEncodedLinkage(GV));
1503	Vals.push_back(Elt: getEncodedAlign(Alignment: GV.getAlign()));
1504	Vals.push_back(Elt: GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1505	: 0);
1506	if (GV.isThreadLocal() ||
1507	GV.getVisibility() != GlobalValue::DefaultVisibility ||
1508	GV.getUnnamedAddr() != GlobalValue::UnnamedAddr::None ||
1509	GV.isExternallyInitialized() ||
1510	GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass ||
1511	GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1512	GV.hasPartition() || GV.hasSanitizerMetadata() || GV.getCodeModel()) {
1513	Vals.push_back(Elt: getEncodedVisibility(GV));
1514	Vals.push_back(Elt: getEncodedThreadLocalMode(GV));
1515	Vals.push_back(Elt: getEncodedUnnamedAddr(GV));
1516	Vals.push_back(Elt: GV.isExternallyInitialized());
1517	Vals.push_back(Elt: getEncodedDLLStorageClass(GV));
1518	Vals.push_back(Elt: GV.hasComdat() ? VE.getComdatID(C: GV.getComdat()) : 0);
1519
1520	auto AL = GV.getAttributesAsList(index: AttributeList::FunctionIndex);
1521	Vals.push_back(Elt: VE.getAttributeListID(PAL: AL));
1522
1523	Vals.push_back(Elt: GV.isDSOLocal());
1524	Vals.push_back(Elt: addToStrtab(Str: GV.getPartition()));
1525	Vals.push_back(Elt: GV.getPartition().size());
1526
1527	Vals.push_back(Elt: (GV.hasSanitizerMetadata() ? serializeSanitizerMetadata(
1528	Meta: GV.getSanitizerMetadata())
1529	: 0));
1530	Vals.push_back(Elt: GV.getCodeModelRaw());
1531	} else {
1532	AbbrevToUse = SimpleGVarAbbrev;
1533	}
1534
1535	Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals, Abbrev: AbbrevToUse);
1536	Vals.clear();
1537	}
1538
1539	// Emit the function proto information.
1540	for (const Function &F : M) {
1541	// FUNCTION: [strtab offset, strtab size, type, callingconv, isproto,
1542	// linkage, paramattrs, alignment, section, visibility, gc,
1543	// unnamed_addr, prologuedata, dllstorageclass, comdat,
1544	// prefixdata, personalityfn, DSO_Local, addrspace]
1545	Vals.push_back(Elt: addToStrtab(Str: F.getName()));
1546	Vals.push_back(Elt: F.getName().size());
1547	Vals.push_back(Elt: VE.getTypeID(T: F.getFunctionType()));
1548	Vals.push_back(Elt: F.getCallingConv());
1549	Vals.push_back(Elt: F.isDeclaration());
1550	Vals.push_back(Elt: getEncodedLinkage(GV: F));
1551	Vals.push_back(Elt: VE.getAttributeListID(PAL: F.getAttributes()));
1552	Vals.push_back(Elt: getEncodedAlign(Alignment: F.getAlign()));
1553	Vals.push_back(Elt: F.hasSection() ? SectionMap[std::string(F.getSection())]
1554	: 0);
1555	Vals.push_back(Elt: getEncodedVisibility(GV: F));
1556	Vals.push_back(Elt: F.hasGC() ? GCMap[F.getGC()] : 0);
1557	Vals.push_back(Elt: getEncodedUnnamedAddr(GV: F));
1558	Vals.push_back(Elt: F.hasPrologueData() ? (VE.getValueID(V: F.getPrologueData()) + 1)
1559	: 0);
1560	Vals.push_back(Elt: getEncodedDLLStorageClass(GV: F));
1561	Vals.push_back(Elt: F.hasComdat() ? VE.getComdatID(C: F.getComdat()) : 0);
1562	Vals.push_back(Elt: F.hasPrefixData() ? (VE.getValueID(V: F.getPrefixData()) + 1)
1563	: 0);
1564	Vals.push_back(
1565	Elt: F.hasPersonalityFn() ? (VE.getValueID(V: F.getPersonalityFn()) + 1) : 0);
1566
1567	Vals.push_back(Elt: F.isDSOLocal());
1568	Vals.push_back(Elt: F.getAddressSpace());
1569	Vals.push_back(Elt: addToStrtab(Str: F.getPartition()));
1570	Vals.push_back(Elt: F.getPartition().size());
1571
1572	unsigned AbbrevToUse = 0;
1573	Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals, Abbrev: AbbrevToUse);
1574	Vals.clear();
1575	}
1576
1577	// Emit the alias information.
1578	for (const GlobalAlias &A : M.aliases()) {
1579	// ALIAS: [strtab offset, strtab size, alias type, aliasee val#, linkage,
1580	// visibility, dllstorageclass, threadlocal, unnamed_addr,
1581	// DSO_Local]
1582	Vals.push_back(Elt: addToStrtab(Str: A.getName()));
1583	Vals.push_back(Elt: A.getName().size());
1584	Vals.push_back(Elt: VE.getTypeID(T: A.getValueType()));
1585	Vals.push_back(Elt: A.getType()->getAddressSpace());
1586	Vals.push_back(Elt: VE.getValueID(V: A.getAliasee()));
1587	Vals.push_back(Elt: getEncodedLinkage(GV: A));
1588	Vals.push_back(Elt: getEncodedVisibility(GV: A));
1589	Vals.push_back(Elt: getEncodedDLLStorageClass(GV: A));
1590	Vals.push_back(Elt: getEncodedThreadLocalMode(GV: A));
1591	Vals.push_back(Elt: getEncodedUnnamedAddr(GV: A));
1592	Vals.push_back(Elt: A.isDSOLocal());
1593	Vals.push_back(Elt: addToStrtab(Str: A.getPartition()));
1594	Vals.push_back(Elt: A.getPartition().size());
1595
1596	unsigned AbbrevToUse = 0;
1597	Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals, Abbrev: AbbrevToUse);
1598	Vals.clear();
1599	}
1600
1601	// Emit the ifunc information.
1602	for (const GlobalIFunc &I : M.ifuncs()) {
1603	// IFUNC: [strtab offset, strtab size, ifunc type, address space, resolver
1604	// val#, linkage, visibility, DSO_Local]
1605	Vals.push_back(Elt: addToStrtab(Str: I.getName()));
1606	Vals.push_back(Elt: I.getName().size());
1607	Vals.push_back(Elt: VE.getTypeID(T: I.getValueType()));
1608	Vals.push_back(Elt: I.getType()->getAddressSpace());
1609	Vals.push_back(Elt: VE.getValueID(V: I.getResolver()));
1610	Vals.push_back(Elt: getEncodedLinkage(GV: I));
1611	Vals.push_back(Elt: getEncodedVisibility(GV: I));
1612	Vals.push_back(Elt: I.isDSOLocal());
1613	Vals.push_back(Elt: addToStrtab(Str: I.getPartition()));
1614	Vals.push_back(Elt: I.getPartition().size());
1615	Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
1616	Vals.clear();
1617	}
1618
1619	writeValueSymbolTableForwardDecl();
1620	}
1621
1622	static uint64_t getOptimizationFlags(const Value *V) {
1623	uint64_t Flags = 0;
1624
1625	if (const auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: V)) {
1626	if (OBO->hasNoSignedWrap())
1627	Flags |= 1 << bitc::OBO_NO_SIGNED_WRAP;
1628	if (OBO->hasNoUnsignedWrap())
1629	Flags |= 1 << bitc::OBO_NO_UNSIGNED_WRAP;
1630	} else if (const auto *PEO = dyn_cast<PossiblyExactOperator>(Val: V)) {
1631	if (PEO->isExact())
1632	Flags |= 1 << bitc::PEO_EXACT;
1633	} else if (const auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: V)) {
1634	if (PDI->isDisjoint())
1635	Flags |= 1 << bitc::PDI_DISJOINT;
1636	} else if (const auto *FPMO = dyn_cast<FPMathOperator>(Val: V)) {
1637	if (FPMO->hasAllowReassoc())
1638	Flags |= bitc::AllowReassoc;
1639	if (FPMO->hasNoNaNs())
1640	Flags |= bitc::NoNaNs;
1641	if (FPMO->hasNoInfs())
1642	Flags |= bitc::NoInfs;
1643	if (FPMO->hasNoSignedZeros())
1644	Flags |= bitc::NoSignedZeros;
1645	if (FPMO->hasAllowReciprocal())
1646	Flags |= bitc::AllowReciprocal;
1647	if (FPMO->hasAllowContract())
1648	Flags |= bitc::AllowContract;
1649	if (FPMO->hasApproxFunc())
1650	Flags |= bitc::ApproxFunc;
1651	} else if (const auto *NNI = dyn_cast<PossiblyNonNegInst>(Val: V)) {
1652	if (NNI->hasNonNeg())
1653	Flags |= 1 << bitc::PNNI_NON_NEG;
1654	} else if (const auto *TI = dyn_cast<TruncInst>(Val: V)) {
1655	if (TI->hasNoSignedWrap())
1656	Flags |= 1 << bitc::TIO_NO_SIGNED_WRAP;
1657	if (TI->hasNoUnsignedWrap())
1658	Flags |= 1 << bitc::TIO_NO_UNSIGNED_WRAP;
1659	}
1660
1661	return Flags;
1662	}
1663
1664	void ModuleBitcodeWriter::writeValueAsMetadata(
1665	const ValueAsMetadata *MD, SmallVectorImpl<uint64_t> &Record) {
1666	// Mimic an MDNode with a value as one operand.
1667	Value *V = MD->getValue();
1668	Record.push_back(Elt: VE.getTypeID(T: V->getType()));
1669	Record.push_back(Elt: VE.getValueID(V));
1670	Stream.EmitRecord(Code: bitc::METADATA_VALUE, Vals: Record, Abbrev: 0);
1671	Record.clear();
1672	}
1673
1674	void ModuleBitcodeWriter::writeMDTuple(const MDTuple *N,
1675	SmallVectorImpl<uint64_t> &Record,
1676	unsigned Abbrev) {
1677	for (const MDOperand &MDO : N->operands()) {
1678	Metadata *MD = MDO;
1679	assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1680	"Unexpected function-local metadata");
1681	Record.push_back(Elt: VE.getMetadataOrNullID(MD));
1682	}
1683	Stream.EmitRecord(Code: N->isDistinct() ? bitc::METADATA_DISTINCT_NODE
1684	: bitc::METADATA_NODE,
1685	Vals: Record, Abbrev);
1686	Record.clear();
1687	}
1688
1689	unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1690	// Assume the column is usually under 128, and always output the inlined-at
1691	// location (it's never more expensive than building an array size 1).
1692	auto Abbv = std::make_shared<BitCodeAbbrev>();
1693	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_LOCATION));
1694	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1695	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1696	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
1697	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1698	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1699	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1700	return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1701	}
1702
1703	void ModuleBitcodeWriter::writeDILocation(const DILocation *N,
1704	SmallVectorImpl<uint64_t> &Record,
1705	unsigned &Abbrev) {
1706	if (!Abbrev)
1707	Abbrev = createDILocationAbbrev();
1708
1709	Record.push_back(Elt: N->isDistinct());
1710	Record.push_back(Elt: N->getLine());
1711	Record.push_back(Elt: N->getColumn());
1712	Record.push_back(Elt: VE.getMetadataID(MD: N->getScope()));
1713	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getInlinedAt()));
1714	Record.push_back(Elt: N->isImplicitCode());
1715
1716	Stream.EmitRecord(Code: bitc::METADATA_LOCATION, Vals: Record, Abbrev);
1717	Record.clear();
1718	}
1719
1720	unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1721	// Assume the column is usually under 128, and always output the inlined-at
1722	// location (it's never more expensive than building an array size 1).
1723	auto Abbv = std::make_shared<BitCodeAbbrev>();
1724	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_GENERIC_DEBUG));
1725	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1726	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1727	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
1728	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1729	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
1730	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
1731	return Stream.EmitAbbrev(Abbv: std::move(Abbv));
1732	}
1733
1734	void ModuleBitcodeWriter::writeGenericDINode(const GenericDINode *N,
1735	SmallVectorImpl<uint64_t> &Record,
1736	unsigned &Abbrev) {
1737	if (!Abbrev)
1738	Abbrev = createGenericDINodeAbbrev();
1739
1740	Record.push_back(Elt: N->isDistinct());
1741	Record.push_back(Elt: N->getTag());
1742	Record.push_back(Elt: 0); // Per-tag version field; unused for now.
1743
1744	for (auto &I : N->operands())
1745	Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
1746
1747	Stream.EmitRecord(Code: bitc::METADATA_GENERIC_DEBUG, Vals: Record, Abbrev);
1748	Record.clear();
1749	}
1750
1751	void ModuleBitcodeWriter::writeDISubrange(const DISubrange *N,
1752	SmallVectorImpl<uint64_t> &Record,
1753	unsigned Abbrev) {
1754	const uint64_t Version = 2 << 1;
1755	Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
1756	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1757	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1758	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1759	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1760
1761	Stream.EmitRecord(Code: bitc::METADATA_SUBRANGE, Vals: Record, Abbrev);
1762	Record.clear();
1763	}
1764
1765	void ModuleBitcodeWriter::writeDIGenericSubrange(
1766	const DIGenericSubrange *N, SmallVectorImpl<uint64_t> &Record,
1767	unsigned Abbrev) {
1768	Record.push_back(Elt: (uint64_t)N->isDistinct());
1769	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawCountNode()));
1770	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLowerBound()));
1771	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUpperBound()));
1772	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawStride()));
1773
1774	Stream.EmitRecord(Code: bitc::METADATA_GENERIC_SUBRANGE, Vals: Record, Abbrev);
1775	Record.clear();
1776	}
1777
1778	void ModuleBitcodeWriter::writeDIEnumerator(const DIEnumerator *N,
1779	SmallVectorImpl<uint64_t> &Record,
1780	unsigned Abbrev) {
1781	const uint64_t IsBigInt = 1 << 2;
1782	Record.push_back(Elt: IsBigInt | (N->isUnsigned() << 1) | N->isDistinct());
1783	Record.push_back(Elt: N->getValue().getBitWidth());
1784	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1785	emitWideAPInt(Vals&: Record, A: N->getValue());
1786
1787	Stream.EmitRecord(Code: bitc::METADATA_ENUMERATOR, Vals: Record, Abbrev);
1788	Record.clear();
1789	}
1790
1791	void ModuleBitcodeWriter::writeDIBasicType(const DIBasicType *N,
1792	SmallVectorImpl<uint64_t> &Record,
1793	unsigned Abbrev) {
1794	Record.push_back(Elt: N->isDistinct());
1795	Record.push_back(Elt: N->getTag());
1796	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1797	Record.push_back(Elt: N->getSizeInBits());
1798	Record.push_back(Elt: N->getAlignInBits());
1799	Record.push_back(Elt: N->getEncoding());
1800	Record.push_back(Elt: N->getFlags());
1801
1802	Stream.EmitRecord(Code: bitc::METADATA_BASIC_TYPE, Vals: Record, Abbrev);
1803	Record.clear();
1804	}
1805
1806	void ModuleBitcodeWriter::writeDIStringType(const DIStringType *N,
1807	SmallVectorImpl<uint64_t> &Record,
1808	unsigned Abbrev) {
1809	Record.push_back(Elt: N->isDistinct());
1810	Record.push_back(Elt: N->getTag());
1811	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1812	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLength()));
1813	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLengthExp()));
1814	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStringLocationExp()));
1815	Record.push_back(Elt: N->getSizeInBits());
1816	Record.push_back(Elt: N->getAlignInBits());
1817	Record.push_back(Elt: N->getEncoding());
1818
1819	Stream.EmitRecord(Code: bitc::METADATA_STRING_TYPE, Vals: Record, Abbrev);
1820	Record.clear();
1821	}
1822
1823	void ModuleBitcodeWriter::writeDIDerivedType(const DIDerivedType *N,
1824	SmallVectorImpl<uint64_t> &Record,
1825	unsigned Abbrev) {
1826	Record.push_back(Elt: N->isDistinct());
1827	Record.push_back(Elt: N->getTag());
1828	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1829	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1830	Record.push_back(Elt: N->getLine());
1831	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
1832	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
1833	Record.push_back(Elt: N->getSizeInBits());
1834	Record.push_back(Elt: N->getAlignInBits());
1835	Record.push_back(Elt: N->getOffsetInBits());
1836	Record.push_back(Elt: N->getFlags());
1837	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExtraData()));
1838
1839	// DWARF address space is encoded as N->getDWARFAddressSpace() + 1. 0 means
1840	// that there is no DWARF address space associated with DIDerivedType.
1841	if (const auto &DWARFAddressSpace = N->getDWARFAddressSpace())
1842	Record.push_back(Elt: *DWARFAddressSpace + 1);
1843	else
1844	Record.push_back(Elt: 0);
1845
1846	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
1847
1848	if (auto PtrAuthData = N->getPtrAuthData())
1849	Record.push_back(Elt: PtrAuthData->RawData);
1850	else
1851	Record.push_back(Elt: 0);
1852
1853	Stream.EmitRecord(Code: bitc::METADATA_DERIVED_TYPE, Vals: Record, Abbrev);
1854	Record.clear();
1855	}
1856
1857	void ModuleBitcodeWriter::writeDICompositeType(
1858	const DICompositeType *N, SmallVectorImpl<uint64_t> &Record,
1859	unsigned Abbrev) {
1860	const unsigned IsNotUsedInOldTypeRef = 0x2;
1861	Record.push_back(Elt: IsNotUsedInOldTypeRef | (unsigned)N->isDistinct());
1862	Record.push_back(Elt: N->getTag());
1863	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1864	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1865	Record.push_back(Elt: N->getLine());
1866	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
1867	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getBaseType()));
1868	Record.push_back(Elt: N->getSizeInBits());
1869	Record.push_back(Elt: N->getAlignInBits());
1870	Record.push_back(Elt: N->getOffsetInBits());
1871	Record.push_back(Elt: N->getFlags());
1872	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
1873	Record.push_back(Elt: N->getRuntimeLang());
1874	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVTableHolder()));
1875	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
1876	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawIdentifier()));
1877	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDiscriminator()));
1878	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDataLocation()));
1879	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAssociated()));
1880	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawAllocated()));
1881	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawRank()));
1882	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
1883
1884	Stream.EmitRecord(Code: bitc::METADATA_COMPOSITE_TYPE, Vals: Record, Abbrev);
1885	Record.clear();
1886	}
1887
1888	void ModuleBitcodeWriter::writeDISubroutineType(
1889	const DISubroutineType *N, SmallVectorImpl<uint64_t> &Record,
1890	unsigned Abbrev) {
1891	const unsigned HasNoOldTypeRefs = 0x2;
1892	Record.push_back(Elt: HasNoOldTypeRefs | (unsigned)N->isDistinct());
1893	Record.push_back(Elt: N->getFlags());
1894	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTypeArray().get()));
1895	Record.push_back(Elt: N->getCC());
1896
1897	Stream.EmitRecord(Code: bitc::METADATA_SUBROUTINE_TYPE, Vals: Record, Abbrev);
1898	Record.clear();
1899	}
1900
1901	void ModuleBitcodeWriter::writeDIFile(const DIFile *N,
1902	SmallVectorImpl<uint64_t> &Record,
1903	unsigned Abbrev) {
1904	Record.push_back(Elt: N->isDistinct());
1905	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFilename()));
1906	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawDirectory()));
1907	if (N->getRawChecksum()) {
1908	Record.push_back(Elt: N->getRawChecksum()->Kind);
1909	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawChecksum()->Value));
1910	} else {
1911	// Maintain backwards compatibility with the old internal representation of
1912	// CSK_None in ChecksumKind by writing nulls here when Checksum is None.
1913	Record.push_back(Elt: 0);
1914	Record.push_back(Elt: VE.getMetadataOrNullID(MD: nullptr));
1915	}
1916	auto Source = N->getRawSource();
1917	if (Source)
1918	Record.push_back(Elt: VE.getMetadataOrNullID(MD: Source));
1919
1920	Stream.EmitRecord(Code: bitc::METADATA_FILE, Vals: Record, Abbrev);
1921	Record.clear();
1922	}
1923
1924	void ModuleBitcodeWriter::writeDICompileUnit(const DICompileUnit *N,
1925	SmallVectorImpl<uint64_t> &Record,
1926	unsigned Abbrev) {
1927	assert(N->isDistinct() && "Expected distinct compile units");
1928	Record.push_back(/* IsDistinct */ Elt: true);
1929	Record.push_back(Elt: N->getSourceLanguage());
1930	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1931	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawProducer()));
1932	Record.push_back(Elt: N->isOptimized());
1933	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFlags()));
1934	Record.push_back(Elt: N->getRuntimeVersion());
1935	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSplitDebugFilename()));
1936	Record.push_back(Elt: N->getEmissionKind());
1937	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEnumTypes().get()));
1938	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedTypes().get()));
1939	Record.push_back(/* subprograms */ Elt: 0);
1940	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getGlobalVariables().get()));
1941	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getImportedEntities().get()));
1942	Record.push_back(Elt: N->getDWOId());
1943	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getMacros().get()));
1944	Record.push_back(Elt: N->getSplitDebugInlining());
1945	Record.push_back(Elt: N->getDebugInfoForProfiling());
1946	Record.push_back(Elt: (unsigned)N->getNameTableKind());
1947	Record.push_back(Elt: N->getRangesBaseAddress());
1948	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSysRoot()));
1949	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSDK()));
1950
1951	Stream.EmitRecord(Code: bitc::METADATA_COMPILE_UNIT, Vals: Record, Abbrev);
1952	Record.clear();
1953	}
1954
1955	void ModuleBitcodeWriter::writeDISubprogram(const DISubprogram *N,
1956	SmallVectorImpl<uint64_t> &Record,
1957	unsigned Abbrev) {
1958	const uint64_t HasUnitFlag = 1 << 1;
1959	const uint64_t HasSPFlagsFlag = 1 << 2;
1960	Record.push_back(Elt: uint64_t(N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
1961	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
1962	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
1963	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
1964	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1965	Record.push_back(Elt: N->getLine());
1966	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
1967	Record.push_back(Elt: N->getScopeLine());
1968	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getContainingType()));
1969	Record.push_back(Elt: N->getSPFlags());
1970	Record.push_back(Elt: N->getVirtualIndex());
1971	Record.push_back(Elt: N->getFlags());
1972	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawUnit()));
1973	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams().get()));
1974	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDeclaration()));
1975	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRetainedNodes().get()));
1976	Record.push_back(Elt: N->getThisAdjustment());
1977	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getThrownTypes().get()));
1978	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
1979	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawTargetFuncName()));
1980
1981	Stream.EmitRecord(Code: bitc::METADATA_SUBPROGRAM, Vals: Record, Abbrev);
1982	Record.clear();
1983	}
1984
1985	void ModuleBitcodeWriter::writeDILexicalBlock(const DILexicalBlock *N,
1986	SmallVectorImpl<uint64_t> &Record,
1987	unsigned Abbrev) {
1988	Record.push_back(Elt: N->isDistinct());
1989	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
1990	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
1991	Record.push_back(Elt: N->getLine());
1992	Record.push_back(Elt: N->getColumn());
1993
1994	Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK, Vals: Record, Abbrev);
1995	Record.clear();
1996	}
1997
1998	void ModuleBitcodeWriter::writeDILexicalBlockFile(
1999	const DILexicalBlockFile *N, SmallVectorImpl<uint64_t> &Record,
2000	unsigned Abbrev) {
2001	Record.push_back(Elt: N->isDistinct());
2002	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2003	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2004	Record.push_back(Elt: N->getDiscriminator());
2005
2006	Stream.EmitRecord(Code: bitc::METADATA_LEXICAL_BLOCK_FILE, Vals: Record, Abbrev);
2007	Record.clear();
2008	}
2009
2010	void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N,
2011	SmallVectorImpl<uint64_t> &Record,
2012	unsigned Abbrev) {
2013	Record.push_back(Elt: N->isDistinct());
2014	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2015	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getDecl()));
2016	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2017	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2018	Record.push_back(Elt: N->getLineNo());
2019
2020	Stream.EmitRecord(Code: bitc::METADATA_COMMON_BLOCK, Vals: Record, Abbrev);
2021	Record.clear();
2022	}
2023
2024	void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N,
2025	SmallVectorImpl<uint64_t> &Record,
2026	unsigned Abbrev) {
2027	Record.push_back(Elt: N->isDistinct() | N->getExportSymbols() << 1);
2028	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2029	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2030
2031	Stream.EmitRecord(Code: bitc::METADATA_NAMESPACE, Vals: Record, Abbrev);
2032	Record.clear();
2033	}
2034
2035	void ModuleBitcodeWriter::writeDIMacro(const DIMacro *N,
2036	SmallVectorImpl<uint64_t> &Record,
2037	unsigned Abbrev) {
2038	Record.push_back(Elt: N->isDistinct());
2039	Record.push_back(Elt: N->getMacinfoType());
2040	Record.push_back(Elt: N->getLine());
2041	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2042	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawValue()));
2043
2044	Stream.EmitRecord(Code: bitc::METADATA_MACRO, Vals: Record, Abbrev);
2045	Record.clear();
2046	}
2047
2048	void ModuleBitcodeWriter::writeDIMacroFile(const DIMacroFile *N,
2049	SmallVectorImpl<uint64_t> &Record,
2050	unsigned Abbrev) {
2051	Record.push_back(Elt: N->isDistinct());
2052	Record.push_back(Elt: N->getMacinfoType());
2053	Record.push_back(Elt: N->getLine());
2054	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2055	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2056
2057	Stream.EmitRecord(Code: bitc::METADATA_MACRO_FILE, Vals: Record, Abbrev);
2058	Record.clear();
2059	}
2060
2061	void ModuleBitcodeWriter::writeDIArgList(const DIArgList *N,
2062	SmallVectorImpl<uint64_t> &Record) {
2063	Record.reserve(N: N->getArgs().size());
2064	for (ValueAsMetadata *MD : N->getArgs())
2065	Record.push_back(Elt: VE.getMetadataID(MD));
2066
2067	Stream.EmitRecord(Code: bitc::METADATA_ARG_LIST, Vals: Record);
2068	Record.clear();
2069	}
2070
2071	void ModuleBitcodeWriter::writeDIModule(const DIModule *N,
2072	SmallVectorImpl<uint64_t> &Record,
2073	unsigned Abbrev) {
2074	Record.push_back(Elt: N->isDistinct());
2075	for (auto &I : N->operands())
2076	Record.push_back(Elt: VE.getMetadataOrNullID(MD: I));
2077	Record.push_back(Elt: N->getLineNo());
2078	Record.push_back(Elt: N->getIsDecl());
2079
2080	Stream.EmitRecord(Code: bitc::METADATA_MODULE, Vals: Record, Abbrev);
2081	Record.clear();
2082	}
2083
2084	void ModuleBitcodeWriter::writeDIAssignID(const DIAssignID *N,
2085	SmallVectorImpl<uint64_t> &Record,
2086	unsigned Abbrev) {
2087	// There are no arguments for this metadata type.
2088	Record.push_back(Elt: N->isDistinct());
2089	Stream.EmitRecord(Code: bitc::METADATA_ASSIGN_ID, Vals: Record, Abbrev);
2090	Record.clear();
2091	}
2092
2093	void ModuleBitcodeWriter::writeDITemplateTypeParameter(
2094	const DITemplateTypeParameter *N, SmallVectorImpl<uint64_t> &Record,
2095	unsigned Abbrev) {
2096	Record.push_back(Elt: N->isDistinct());
2097	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2098	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2099	Record.push_back(Elt: N->isDefault());
2100
2101	Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_TYPE, Vals: Record, Abbrev);
2102	Record.clear();
2103	}
2104
2105	void ModuleBitcodeWriter::writeDITemplateValueParameter(
2106	const DITemplateValueParameter *N, SmallVectorImpl<uint64_t> &Record,
2107	unsigned Abbrev) {
2108	Record.push_back(Elt: N->isDistinct());
2109	Record.push_back(Elt: N->getTag());
2110	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2111	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2112	Record.push_back(Elt: N->isDefault());
2113	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getValue()));
2114
2115	Stream.EmitRecord(Code: bitc::METADATA_TEMPLATE_VALUE, Vals: Record, Abbrev);
2116	Record.clear();
2117	}
2118
2119	void ModuleBitcodeWriter::writeDIGlobalVariable(
2120	const DIGlobalVariable *N, SmallVectorImpl<uint64_t> &Record,
2121	unsigned Abbrev) {
2122	const uint64_t Version = 2 << 1;
2123	Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2124	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2125	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2126	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawLinkageName()));
2127	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2128	Record.push_back(Elt: N->getLine());
2129	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2130	Record.push_back(Elt: N->isLocalToUnit());
2131	Record.push_back(Elt: N->isDefinition());
2132	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getStaticDataMemberDeclaration()));
2133	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getTemplateParams()));
2134	Record.push_back(Elt: N->getAlignInBits());
2135	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2136
2137	Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR, Vals: Record, Abbrev);
2138	Record.clear();
2139	}
2140
2141	void ModuleBitcodeWriter::writeDILocalVariable(
2142	const DILocalVariable *N, SmallVectorImpl<uint64_t> &Record,
2143	unsigned Abbrev) {
2144	// In order to support all possible bitcode formats in BitcodeReader we need
2145	// to distinguish the following cases:
2146	// 1) Record has no artificial tag (Record[1]),
2147	// has no obsolete inlinedAt field (Record[9]).
2148	// In this case Record size will be 8, HasAlignment flag is false.
2149	// 2) Record has artificial tag (Record[1]),
2150	// has no obsolete inlignedAt field (Record[9]).
2151	// In this case Record size will be 9, HasAlignment flag is false.
2152	// 3) Record has both artificial tag (Record[1]) and
2153	// obsolete inlignedAt field (Record[9]).
2154	// In this case Record size will be 10, HasAlignment flag is false.
2155	// 4) Record has neither artificial tag, nor inlignedAt field, but
2156	// HasAlignment flag is true and Record[8] contains alignment value.
2157	const uint64_t HasAlignmentFlag = 1 << 1;
2158	Record.push_back(Elt: (uint64_t)N->isDistinct() | HasAlignmentFlag);
2159	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2160	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2161	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2162	Record.push_back(Elt: N->getLine());
2163	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2164	Record.push_back(Elt: N->getArg());
2165	Record.push_back(Elt: N->getFlags());
2166	Record.push_back(Elt: N->getAlignInBits());
2167	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getAnnotations().get()));
2168
2169	Stream.EmitRecord(Code: bitc::METADATA_LOCAL_VAR, Vals: Record, Abbrev);
2170	Record.clear();
2171	}
2172
2173	void ModuleBitcodeWriter::writeDILabel(
2174	const DILabel *N, SmallVectorImpl<uint64_t> &Record,
2175	unsigned Abbrev) {
2176	Record.push_back(Elt: (uint64_t)N->isDistinct());
2177	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2178	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2179	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2180	Record.push_back(Elt: N->getLine());
2181
2182	Stream.EmitRecord(Code: bitc::METADATA_LABEL, Vals: Record, Abbrev);
2183	Record.clear();
2184	}
2185
2186	void ModuleBitcodeWriter::writeDIExpression(const DIExpression *N,
2187	SmallVectorImpl<uint64_t> &Record,
2188	unsigned Abbrev) {
2189	Record.reserve(N: N->getElements().size() + 1);
2190	const uint64_t Version = 3 << 1;
2191	Record.push_back(Elt: (uint64_t)N->isDistinct() | Version);
2192	Record.append(in_start: N->elements_begin(), in_end: N->elements_end());
2193
2194	Stream.EmitRecord(Code: bitc::METADATA_EXPRESSION, Vals: Record, Abbrev);
2195	Record.clear();
2196	}
2197
2198	void ModuleBitcodeWriter::writeDIGlobalVariableExpression(
2199	const DIGlobalVariableExpression *N, SmallVectorImpl<uint64_t> &Record,
2200	unsigned Abbrev) {
2201	Record.push_back(Elt: N->isDistinct());
2202	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getVariable()));
2203	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getExpression()));
2204
2205	Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_VAR_EXPR, Vals: Record, Abbrev);
2206	Record.clear();
2207	}
2208
2209	void ModuleBitcodeWriter::writeDIObjCProperty(const DIObjCProperty *N,
2210	SmallVectorImpl<uint64_t> &Record,
2211	unsigned Abbrev) {
2212	Record.push_back(Elt: N->isDistinct());
2213	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2214	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getFile()));
2215	Record.push_back(Elt: N->getLine());
2216	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawSetterName()));
2217	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawGetterName()));
2218	Record.push_back(Elt: N->getAttributes());
2219	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getType()));
2220
2221	Stream.EmitRecord(Code: bitc::METADATA_OBJC_PROPERTY, Vals: Record, Abbrev);
2222	Record.clear();
2223	}
2224
2225	void ModuleBitcodeWriter::writeDIImportedEntity(
2226	const DIImportedEntity *N, SmallVectorImpl<uint64_t> &Record,
2227	unsigned Abbrev) {
2228	Record.push_back(Elt: N->isDistinct());
2229	Record.push_back(Elt: N->getTag());
2230	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getScope()));
2231	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getEntity()));
2232	Record.push_back(Elt: N->getLine());
2233	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawName()));
2234	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getRawFile()));
2235	Record.push_back(Elt: VE.getMetadataOrNullID(MD: N->getElements().get()));
2236
2237	Stream.EmitRecord(Code: bitc::METADATA_IMPORTED_ENTITY, Vals: Record, Abbrev);
2238	Record.clear();
2239	}
2240
2241	unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2242	auto Abbv = std::make_shared<BitCodeAbbrev>();
2243	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_NAME));
2244	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2245	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2246	return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2247	}
2248
2249	void ModuleBitcodeWriter::writeNamedMetadata(
2250	SmallVectorImpl<uint64_t> &Record) {
2251	if (M.named_metadata_empty())
2252	return;
2253
2254	unsigned Abbrev = createNamedMetadataAbbrev();
2255	for (const NamedMDNode &NMD : M.named_metadata()) {
2256	// Write name.
2257	StringRef Str = NMD.getName();
2258	Record.append(in_start: Str.bytes_begin(), in_end: Str.bytes_end());
2259	Stream.EmitRecord(Code: bitc::METADATA_NAME, Vals: Record, Abbrev);
2260	Record.clear();
2261
2262	// Write named metadata operands.
2263	for (const MDNode *N : NMD.operands())
2264	Record.push_back(Elt: VE.getMetadataID(MD: N));
2265	Stream.EmitRecord(Code: bitc::METADATA_NAMED_NODE, Vals: Record, Abbrev: 0);
2266	Record.clear();
2267	}
2268	}
2269
2270	unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2271	auto Abbv = std::make_shared<BitCodeAbbrev>();
2272	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_STRINGS));
2273	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of strings
2274	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // offset to chars
2275	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
2276	return Stream.EmitAbbrev(Abbv: std::move(Abbv));
2277	}
2278
2279	/// Write out a record for MDString.
2280	///
2281	/// All the metadata strings in a metadata block are emitted in a single
2282	/// record. The sizes and strings themselves are shoved into a blob.
2283	void ModuleBitcodeWriter::writeMetadataStrings(
2284	ArrayRef<const Metadata *> Strings, SmallVectorImpl<uint64_t> &Record) {
2285	if (Strings.empty())
2286	return;
2287
2288	// Start the record with the number of strings.
2289	Record.push_back(Elt: bitc::METADATA_STRINGS);
2290	Record.push_back(Elt: Strings.size());
2291
2292	// Emit the sizes of the strings in the blob.
2293	SmallString<256> Blob;
2294	{
2295	BitstreamWriter W(Blob);
2296	for (const Metadata *MD : Strings)
2297	W.EmitVBR(Val: cast<MDString>(Val: MD)->getLength(), NumBits: 6);
2298	W.FlushToWord();
2299	}
2300
2301	// Add the offset to the strings to the record.
2302	Record.push_back(Elt: Blob.size());
2303
2304	// Add the strings to the blob.
2305	for (const Metadata *MD : Strings)
2306	Blob.append(RHS: cast<MDString>(Val: MD)->getString());
2307
2308	// Emit the final record.
2309	Stream.EmitRecordWithBlob(Abbrev: createMetadataStringsAbbrev(), Vals: Record, Blob);
2310	Record.clear();
2311	}
2312
2313	// Generates an enum to use as an index in the Abbrev array of Metadata record.
2314	enum MetadataAbbrev : unsigned {
2315	#define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2316	#include "llvm/IR/Metadata.def"
2317	LastPlusOne
2318	};
2319
2320	void ModuleBitcodeWriter::writeMetadataRecords(
2321	ArrayRef<const Metadata *> MDs, SmallVectorImpl<uint64_t> &Record,
2322	std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2323	if (MDs.empty())
2324	return;
2325
2326	// Initialize MDNode abbreviations.
2327	#define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2328	#include "llvm/IR/Metadata.def"
2329
2330	for (const Metadata *MD : MDs) {
2331	if (IndexPos)
2332	IndexPos->push_back(x: Stream.GetCurrentBitNo());
2333	if (const MDNode *N = dyn_cast<MDNode>(Val: MD)) {
2334	assert(N->isResolved() && "Expected forward references to be resolved");
2335
2336	switch (N->getMetadataID()) {
2337	default:
2338	llvm_unreachable("Invalid MDNode subclass");
2339	#define HANDLE_MDNODE_LEAF(CLASS) \
2340	case Metadata::CLASS##Kind: \
2341	if (MDAbbrevs) \
2342	write##CLASS(cast<CLASS>(N), Record, \
2343	(*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2344	else \
2345	write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2346	continue;
2347	#include "llvm/IR/Metadata.def"
2348	}
2349	}
2350	if (auto *AL = dyn_cast<DIArgList>(Val: MD)) {
2351	writeDIArgList(N: AL, Record);
2352	continue;
2353	}
2354	writeValueAsMetadata(MD: cast<ValueAsMetadata>(Val: MD), Record);
2355	}
2356	}
2357
2358	void ModuleBitcodeWriter::writeModuleMetadata() {
2359	if (!VE.hasMDs() && M.named_metadata_empty())
2360	return;
2361
2362	Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 4);
2363	SmallVector<uint64_t, 64> Record;
2364
2365	// Emit all abbrevs upfront, so that the reader can jump in the middle of the
2366	// block and load any metadata.
2367	std::vector<unsigned> MDAbbrevs;
2368
2369	MDAbbrevs.resize(new_size: MetadataAbbrev::LastPlusOne);
2370	MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2371	MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2372	createGenericDINodeAbbrev();
2373
2374	auto Abbv = std::make_shared<BitCodeAbbrev>();
2375	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX_OFFSET));
2376	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2377	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
2378	unsigned OffsetAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2379
2380	Abbv = std::make_shared<BitCodeAbbrev>();
2381	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::METADATA_INDEX));
2382	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2383	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
2384	unsigned IndexAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2385
2386	// Emit MDStrings together upfront.
2387	writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2388
2389	// We only emit an index for the metadata record if we have more than a given
2390	// (naive) threshold of metadatas, otherwise it is not worth it.
2391	if (VE.getNonMDStrings().size() > IndexThreshold) {
2392	// Write a placeholder value in for the offset of the metadata index,
2393	// which is written after the records, so that it can include
2394	// the offset of each entry. The placeholder offset will be
2395	// updated after all records are emitted.
2396	uint64_t Vals[] = {0, 0};
2397	Stream.EmitRecord(Code: bitc::METADATA_INDEX_OFFSET, Vals, Abbrev: OffsetAbbrev);
2398	}
2399
2400	// Compute and save the bit offset to the current position, which will be
2401	// patched when we emit the index later. We can simply subtract the 64-bit
2402	// fixed size from the current bit number to get the location to backpatch.
2403	uint64_t IndexOffsetRecordBitPos = Stream.GetCurrentBitNo();
2404
2405	// This index will contain the bitpos for each individual record.
2406	std::vector<uint64_t> IndexPos;
2407	IndexPos.reserve(n: VE.getNonMDStrings().size());
2408
2409	// Write all the records
2410	writeMetadataRecords(MDs: VE.getNonMDStrings(), Record, MDAbbrevs: &MDAbbrevs, IndexPos: &IndexPos);
2411
2412	if (VE.getNonMDStrings().size() > IndexThreshold) {
2413	// Now that we have emitted all the records we will emit the index. But
2414	// first
2415	// backpatch the forward reference so that the reader can skip the records
2416	// efficiently.
2417	Stream.BackpatchWord64(BitNo: IndexOffsetRecordBitPos - 64,
2418	Val: Stream.GetCurrentBitNo() - IndexOffsetRecordBitPos);
2419
2420	// Delta encode the index.
2421	uint64_t PreviousValue = IndexOffsetRecordBitPos;
2422	for (auto &Elt : IndexPos) {
2423	auto EltDelta = Elt - PreviousValue;
2424	PreviousValue = Elt;
2425	Elt = EltDelta;
2426	}
2427	// Emit the index record.
2428	Stream.EmitRecord(Code: bitc::METADATA_INDEX, Vals: IndexPos, Abbrev: IndexAbbrev);
2429	IndexPos.clear();
2430	}
2431
2432	// Write the named metadata now.
2433	writeNamedMetadata(Record);
2434
2435	auto AddDeclAttachedMetadata = [&](const GlobalObject &GO) {
2436	SmallVector<uint64_t, 4> Record;
2437	Record.push_back(Elt: VE.getValueID(V: &GO));
2438	pushGlobalMetadataAttachment(Record, GO);
2439	Stream.EmitRecord(Code: bitc::METADATA_GLOBAL_DECL_ATTACHMENT, Vals: Record);
2440	};
2441	for (const Function &F : M)
2442	if (F.isDeclaration() && F.hasMetadata())
2443	AddDeclAttachedMetadata(F);
2444	// FIXME: Only store metadata for declarations here, and move data for global
2445	// variable definitions to a separate block (PR28134).
2446	for (const GlobalVariable &GV : M.globals())
2447	if (GV.hasMetadata())
2448	AddDeclAttachedMetadata(GV);
2449
2450	Stream.ExitBlock();
2451	}
2452
2453	void ModuleBitcodeWriter::writeFunctionMetadata(const Function &F) {
2454	if (!VE.hasMDs())
2455	return;
2456
2457	Stream.EnterSubblock(BlockID: bitc::METADATA_BLOCK_ID, CodeLen: 3);
2458	SmallVector<uint64_t, 64> Record;
2459	writeMetadataStrings(Strings: VE.getMDStrings(), Record);
2460	writeMetadataRecords(MDs: VE.getNonMDStrings(), Record);
2461	Stream.ExitBlock();
2462	}
2463
2464	void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2465	SmallVectorImpl<uint64_t> &Record, const GlobalObject &GO) {
2466	// [n x [id, mdnode]]
2467	SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2468	GO.getAllMetadata(MDs);
2469	for (const auto &I : MDs) {
2470	Record.push_back(Elt: I.first);
2471	Record.push_back(Elt: VE.getMetadataID(MD: I.second));
2472	}
2473	}
2474
2475	void ModuleBitcodeWriter::writeFunctionMetadataAttachment(const Function &F) {
2476	Stream.EnterSubblock(BlockID: bitc::METADATA_ATTACHMENT_ID, CodeLen: 3);
2477
2478	SmallVector<uint64_t, 64> Record;
2479
2480	if (F.hasMetadata()) {
2481	pushGlobalMetadataAttachment(Record, GO: F);
2482	Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2483	Record.clear();
2484	}
2485
2486	// Write metadata attachments
2487	// METADATA_ATTACHMENT - [m x [value, [n x [id, mdnode]]]
2488	SmallVector<std::pair<unsigned, MDNode *>, 4> MDs;
2489	for (const BasicBlock &BB : F)
2490	for (const Instruction &I : BB) {
2491	MDs.clear();
2492	I.getAllMetadataOtherThanDebugLoc(MDs);
2493
2494	// If no metadata, ignore instruction.
2495	if (MDs.empty()) continue;
2496
2497	Record.push_back(Elt: VE.getInstructionID(I: &I));
2498
2499	for (unsigned i = 0, e = MDs.size(); i != e; ++i) {
2500	Record.push_back(Elt: MDs[i].first);
2501	Record.push_back(Elt: VE.getMetadataID(MD: MDs[i].second));
2502	}
2503	Stream.EmitRecord(Code: bitc::METADATA_ATTACHMENT, Vals: Record, Abbrev: 0);
2504	Record.clear();
2505	}
2506
2507	Stream.ExitBlock();
2508	}
2509
2510	void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2511	SmallVector<uint64_t, 64> Record;
2512
2513	// Write metadata kinds
2514	// METADATA_KIND - [n x [id, name]]
2515	SmallVector<StringRef, 8> Names;
2516	M.getMDKindNames(Result&: Names);
2517
2518	if (Names.empty()) return;
2519
2520	Stream.EnterSubblock(BlockID: bitc::METADATA_KIND_BLOCK_ID, CodeLen: 3);
2521
2522	for (unsigned MDKindID = 0, e = Names.size(); MDKindID != e; ++MDKindID) {
2523	Record.push_back(Elt: MDKindID);
2524	StringRef KName = Names[MDKindID];
2525	Record.append(in_start: KName.begin(), in_end: KName.end());
2526
2527	Stream.EmitRecord(Code: bitc::METADATA_KIND, Vals: Record, Abbrev: 0);
2528	Record.clear();
2529	}
2530
2531	Stream.ExitBlock();
2532	}
2533
2534	void ModuleBitcodeWriter::writeOperandBundleTags() {
2535	// Write metadata kinds
2536	//
2537	// OPERAND_BUNDLE_TAGS_BLOCK_ID : N x OPERAND_BUNDLE_TAG
2538	//
2539	// OPERAND_BUNDLE_TAG - [strchr x N]
2540
2541	SmallVector<StringRef, 8> Tags;
2542	M.getOperandBundleTags(Result&: Tags);
2543
2544	if (Tags.empty())
2545	return;
2546
2547	Stream.EnterSubblock(BlockID: bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID, CodeLen: 3);
2548
2549	SmallVector<uint64_t, 64> Record;
2550
2551	for (auto Tag : Tags) {
2552	Record.append(in_start: Tag.begin(), in_end: Tag.end());
2553
2554	Stream.EmitRecord(Code: bitc::OPERAND_BUNDLE_TAG, Vals: Record, Abbrev: 0);
2555	Record.clear();
2556	}
2557
2558	Stream.ExitBlock();
2559	}
2560
2561	void ModuleBitcodeWriter::writeSyncScopeNames() {
2562	SmallVector<StringRef, 8> SSNs;
2563	M.getContext().getSyncScopeNames(SSNs);
2564	if (SSNs.empty())
2565	return;
2566
2567	Stream.EnterSubblock(BlockID: bitc::SYNC_SCOPE_NAMES_BLOCK_ID, CodeLen: 2);
2568
2569	SmallVector<uint64_t, 64> Record;
2570	for (auto SSN : SSNs) {
2571	Record.append(in_start: SSN.begin(), in_end: SSN.end());
2572	Stream.EmitRecord(Code: bitc::SYNC_SCOPE_NAME, Vals: Record, Abbrev: 0);
2573	Record.clear();
2574	}
2575
2576	Stream.ExitBlock();
2577	}
2578
2579	void ModuleBitcodeWriter::writeConstants(unsigned FirstVal, unsigned LastVal,
2580	bool isGlobal) {
2581	if (FirstVal == LastVal) return;
2582
2583	Stream.EnterSubblock(BlockID: bitc::CONSTANTS_BLOCK_ID, CodeLen: 4);
2584
2585	unsigned AggregateAbbrev = 0;
2586	unsigned String8Abbrev = 0;
2587	unsigned CString7Abbrev = 0;
2588	unsigned CString6Abbrev = 0;
2589	// If this is a constant pool for the module, emit module-specific abbrevs.
2590	if (isGlobal) {
2591	// Abbrev for CST_CODE_AGGREGATE.
2592	auto Abbv = std::make_shared<BitCodeAbbrev>();
2593	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_AGGREGATE));
2594	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2595	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, Log2_32_Ceil(Value: LastVal+1)));
2596	AggregateAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2597
2598	// Abbrev for CST_CODE_STRING.
2599	Abbv = std::make_shared<BitCodeAbbrev>();
2600	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_STRING));
2601	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2602	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
2603	String8Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2604	// Abbrev for CST_CODE_CSTRING.
2605	Abbv = std::make_shared<BitCodeAbbrev>();
2606	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2607	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2608	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
2609	CString7Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2610	// Abbrev for CST_CODE_CSTRING.
2611	Abbv = std::make_shared<BitCodeAbbrev>();
2612	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CSTRING));
2613	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
2614	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
2615	CString6Abbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
2616	}
2617
2618	SmallVector<uint64_t, 64> Record;
2619
2620	const ValueEnumerator::ValueList &Vals = VE.getValues();
2621	Type *LastTy = nullptr;
2622	for (unsigned i = FirstVal; i != LastVal; ++i) {
2623	const Value *V = Vals[i].first;
2624	// If we need to switch types, do so now.
2625	if (V->getType() != LastTy) {
2626	LastTy = V->getType();
2627	Record.push_back(Elt: VE.getTypeID(T: LastTy));
2628	Stream.EmitRecord(Code: bitc::CST_CODE_SETTYPE, Vals: Record,
2629	Abbrev: CONSTANTS_SETTYPE_ABBREV);
2630	Record.clear();
2631	}
2632
2633	if (const InlineAsm *IA = dyn_cast<InlineAsm>(Val: V)) {
2634	Record.push_back(Elt: VE.getTypeID(T: IA->getFunctionType()));
2635	Record.push_back(
2636	Elt: unsigned(IA->hasSideEffects()) | unsigned(IA->isAlignStack()) << 1 |
2637	unsigned(IA->getDialect() & 1) << 2 | unsigned(IA->canThrow()) << 3);
2638
2639	// Add the asm string.
2640	const std::string &AsmStr = IA->getAsmString();
2641	Record.push_back(Elt: AsmStr.size());
2642	Record.append(in_start: AsmStr.begin(), in_end: AsmStr.end());
2643
2644	// Add the constraint string.
2645	const std::string &ConstraintStr = IA->getConstraintString();
2646	Record.push_back(Elt: ConstraintStr.size());
2647	Record.append(in_start: ConstraintStr.begin(), in_end: ConstraintStr.end());
2648	Stream.EmitRecord(Code: bitc::CST_CODE_INLINEASM, Vals: Record);
2649	Record.clear();
2650	continue;
2651	}
2652	const Constant *C = cast<Constant>(Val: V);
2653	unsigned Code = -1U;
2654	unsigned AbbrevToUse = 0;
2655	if (C->isNullValue()) {
2656	Code = bitc::CST_CODE_NULL;
2657	} else if (isa<PoisonValue>(Val: C)) {
2658	Code = bitc::CST_CODE_POISON;
2659	} else if (isa<UndefValue>(Val: C)) {
2660	Code = bitc::CST_CODE_UNDEF;
2661	} else if (const ConstantInt *IV = dyn_cast<ConstantInt>(Val: C)) {
2662	if (IV->getBitWidth() <= 64) {
2663	uint64_t V = IV->getSExtValue();
2664	emitSignedInt64(Vals&: Record, V);
2665	Code = bitc::CST_CODE_INTEGER;
2666	AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2667	} else { // Wide integers, > 64 bits in size.
2668	emitWideAPInt(Vals&: Record, A: IV->getValue());
2669	Code = bitc::CST_CODE_WIDE_INTEGER;
2670	}
2671	} else if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: C)) {
2672	Code = bitc::CST_CODE_FLOAT;
2673	Type *Ty = CFP->getType()->getScalarType();
2674	if (Ty->isHalfTy() || Ty->isBFloatTy() || Ty->isFloatTy() ||
2675	Ty->isDoubleTy()) {
2676	Record.push_back(Elt: CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2677	} else if (Ty->isX86_FP80Ty()) {
2678	// api needed to prevent premature destruction
2679	// bits are not in the same order as a normal i80 APInt, compensate.
2680	APInt api = CFP->getValueAPF().bitcastToAPInt();
2681	const uint64_t *p = api.getRawData();
2682	Record.push_back(Elt: (p[1] << 48) | (p[0] >> 16));
2683	Record.push_back(Elt: p[0] & 0xffffLL);
2684	} else if (Ty->isFP128Ty() || Ty->isPPC_FP128Ty()) {
2685	APInt api = CFP->getValueAPF().bitcastToAPInt();
2686	const uint64_t *p = api.getRawData();
2687	Record.push_back(Elt: p[0]);
2688	Record.push_back(Elt: p[1]);
2689	} else {
2690	assert(0 && "Unknown FP type!");
2691	}
2692	} else if (isa<ConstantDataSequential>(Val: C) &&
2693	cast<ConstantDataSequential>(Val: C)->isString()) {
2694	const ConstantDataSequential *Str = cast<ConstantDataSequential>(Val: C);
2695	// Emit constant strings specially.
2696	unsigned NumElts = Str->getNumElements();
2697	// If this is a null-terminated string, use the denser CSTRING encoding.
2698	if (Str->isCString()) {
2699	Code = bitc::CST_CODE_CSTRING;
2700	--NumElts; // Don't encode the null, which isn't allowed by char6.
2701	} else {
2702	Code = bitc::CST_CODE_STRING;
2703	AbbrevToUse = String8Abbrev;
2704	}
2705	bool isCStr7 = Code == bitc::CST_CODE_CSTRING;
2706	bool isCStrChar6 = Code == bitc::CST_CODE_CSTRING;
2707	for (unsigned i = 0; i != NumElts; ++i) {
2708	unsigned char V = Str->getElementAsInteger(i);
2709	Record.push_back(Elt: V);
2710	isCStr7 &= (V & 128) == 0;
2711	if (isCStrChar6)
2712	isCStrChar6 = BitCodeAbbrevOp::isChar6(C: V);
2713	}
2714
2715	if (isCStrChar6)
2716	AbbrevToUse = CString6Abbrev;
2717	else if (isCStr7)
2718	AbbrevToUse = CString7Abbrev;
2719	} else if (const ConstantDataSequential *CDS =
2720	dyn_cast<ConstantDataSequential>(Val: C)) {
2721	Code = bitc::CST_CODE_DATA;
2722	Type *EltTy = CDS->getElementType();
2723	if (isa<IntegerType>(Val: EltTy)) {
2724	for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
2725	Record.push_back(Elt: CDS->getElementAsInteger(i));
2726	} else {
2727	for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i)
2728	Record.push_back(
2729	Elt: CDS->getElementAsAPFloat(i).bitcastToAPInt().getLimitedValue());
2730	}
2731	} else if (isa<ConstantAggregate>(Val: C)) {
2732	Code = bitc::CST_CODE_AGGREGATE;
2733	for (const Value *Op : C->operands())
2734	Record.push_back(Elt: VE.getValueID(V: Op));
2735	AbbrevToUse = AggregateAbbrev;
2736	} else if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(Val: C)) {
2737	switch (CE->getOpcode()) {
2738	default:
2739	if (Instruction::isCast(Opcode: CE->getOpcode())) {
2740	Code = bitc::CST_CODE_CE_CAST;
2741	Record.push_back(Elt: getEncodedCastOpcode(Opcode: CE->getOpcode()));
2742	Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2743	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2744	AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2745	} else {
2746	assert(CE->getNumOperands() == 2 && "Unknown constant expr!");
2747	Code = bitc::CST_CODE_CE_BINOP;
2748	Record.push_back(Elt: getEncodedBinaryOpcode(Opcode: CE->getOpcode()));
2749	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2750	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2751	uint64_t Flags = getOptimizationFlags(V: CE);
2752	if (Flags != 0)
2753	Record.push_back(Elt: Flags);
2754	}
2755	break;
2756	case Instruction::FNeg: {
2757	assert(CE->getNumOperands() == 1 && "Unknown constant expr!");
2758	Code = bitc::CST_CODE_CE_UNOP;
2759	Record.push_back(Elt: getEncodedUnaryOpcode(Opcode: CE->getOpcode()));
2760	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2761	uint64_t Flags = getOptimizationFlags(V: CE);
2762	if (Flags != 0)
2763	Record.push_back(Elt: Flags);
2764	break;
2765	}
2766	case Instruction::GetElementPtr: {
2767	Code = bitc::CST_CODE_CE_GEP;
2768	const auto *GO = cast<GEPOperator>(Val: C);
2769	Record.push_back(Elt: VE.getTypeID(T: GO->getSourceElementType()));
2770	if (std::optional<ConstantRange> Range = GO->getInRange()) {
2771	Code = bitc::CST_CODE_CE_GEP_WITH_INRANGE;
2772	Record.push_back(Elt: GO->isInBounds());
2773	emitConstantRange(Record, CR: *Range);
2774	} else if (GO->isInBounds())
2775	Code = bitc::CST_CODE_CE_INBOUNDS_GEP;
2776	for (unsigned i = 0, e = CE->getNumOperands(); i != e; ++i) {
2777	Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i)->getType()));
2778	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i)));
2779	}
2780	break;
2781	}
2782	case Instruction::ExtractElement:
2783	Code = bitc::CST_CODE_CE_EXTRACTELT;
2784	Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2785	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2786	Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 1)->getType()));
2787	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2788	break;
2789	case Instruction::InsertElement:
2790	Code = bitc::CST_CODE_CE_INSERTELT;
2791	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2792	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2793	Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 2)->getType()));
2794	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 2)));
2795	break;
2796	case Instruction::ShuffleVector:
2797	// If the return type and argument types are the same, this is a
2798	// standard shufflevector instruction. If the types are different,
2799	// then the shuffle is widening or truncating the input vectors, and
2800	// the argument type must also be encoded.
2801	if (C->getType() == C->getOperand(i: 0)->getType()) {
2802	Code = bitc::CST_CODE_CE_SHUFFLEVEC;
2803	} else {
2804	Code = bitc::CST_CODE_CE_SHUFVEC_EX;
2805	Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2806	}
2807	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2808	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2809	Record.push_back(Elt: VE.getValueID(V: CE->getShuffleMaskForBitcode()));
2810	break;
2811	case Instruction::ICmp:
2812	case Instruction::FCmp:
2813	Code = bitc::CST_CODE_CE_CMP;
2814	Record.push_back(Elt: VE.getTypeID(T: C->getOperand(i: 0)->getType()));
2815	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 0)));
2816	Record.push_back(Elt: VE.getValueID(V: C->getOperand(i: 1)));
2817	Record.push_back(Elt: CE->getPredicate());
2818	break;
2819	}
2820	} else if (const BlockAddress *BA = dyn_cast<BlockAddress>(Val: C)) {
2821	Code = bitc::CST_CODE_BLOCKADDRESS;
2822	Record.push_back(Elt: VE.getTypeID(T: BA->getFunction()->getType()));
2823	Record.push_back(Elt: VE.getValueID(V: BA->getFunction()));
2824	Record.push_back(Elt: VE.getGlobalBasicBlockID(BB: BA->getBasicBlock()));
2825	} else if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(Val: C)) {
2826	Code = bitc::CST_CODE_DSO_LOCAL_EQUIVALENT;
2827	Record.push_back(Elt: VE.getTypeID(T: Equiv->getGlobalValue()->getType()));
2828	Record.push_back(Elt: VE.getValueID(V: Equiv->getGlobalValue()));
2829	} else if (const auto *NC = dyn_cast<NoCFIValue>(Val: C)) {
2830	Code = bitc::CST_CODE_NO_CFI_VALUE;
2831	Record.push_back(Elt: VE.getTypeID(T: NC->getGlobalValue()->getType()));
2832	Record.push_back(Elt: VE.getValueID(V: NC->getGlobalValue()));
2833	} else {
2834	#ifndef NDEBUG
2835	C->dump();
2836	#endif
2837	llvm_unreachable("Unknown constant!");
2838	}
2839	Stream.EmitRecord(Code, Vals: Record, Abbrev: AbbrevToUse);
2840	Record.clear();
2841	}
2842
2843	Stream.ExitBlock();
2844	}
2845
2846	void ModuleBitcodeWriter::writeModuleConstants() {
2847	const ValueEnumerator::ValueList &Vals = VE.getValues();
2848
2849	// Find the first constant to emit, which is the first non-globalvalue value.
2850	// We know globalvalues have been emitted by WriteModuleInfo.
2851	for (unsigned i = 0, e = Vals.size(); i != e; ++i) {
2852	if (!isa<GlobalValue>(Val: Vals[i].first)) {
2853	writeConstants(FirstVal: i, LastVal: Vals.size(), isGlobal: true);
2854	return;
2855	}
2856	}
2857	}
2858
2859	/// pushValueAndType - The file has to encode both the value and type id for
2860	/// many values, because we need to know what type to create for forward
2861	/// references. However, most operands are not forward references, so this type
2862	/// field is not needed.
2863	///
2864	/// This function adds V's value ID to Vals. If the value ID is higher than the
2865	/// instruction ID, then it is a forward reference, and it also includes the
2866	/// type ID. The value ID that is written is encoded relative to the InstID.
2867	bool ModuleBitcodeWriter::pushValueAndType(const Value *V, unsigned InstID,
2868	SmallVectorImpl<unsigned> &Vals) {
2869	unsigned ValID = VE.getValueID(V);
2870	// Make encoding relative to the InstID.
2871	Vals.push_back(Elt: InstID - ValID);
2872	if (ValID >= InstID) {
2873	Vals.push_back(Elt: VE.getTypeID(T: V->getType()));
2874	return true;
2875	}
2876	return false;
2877	}
2878
2879	void ModuleBitcodeWriter::writeOperandBundles(const CallBase &CS,
2880	unsigned InstID) {
2881	SmallVector<unsigned, 64> Record;
2882	LLVMContext &C = CS.getContext();
2883
2884	for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
2885	const auto &Bundle = CS.getOperandBundleAt(Index: i);
2886	Record.push_back(Elt: C.getOperandBundleTagID(Tag: Bundle.getTagName()));
2887
2888	for (auto &Input : Bundle.Inputs)
2889	pushValueAndType(V: Input, InstID, Vals&: Record);
2890
2891	Stream.EmitRecord(Code: bitc::FUNC_CODE_OPERAND_BUNDLE, Vals: Record);
2892	Record.clear();
2893	}
2894	}
2895
2896	/// pushValue - Like pushValueAndType, but where the type of the value is
2897	/// omitted (perhaps it was already encoded in an earlier operand).
2898	void ModuleBitcodeWriter::pushValue(const Value *V, unsigned InstID,
2899	SmallVectorImpl<unsigned> &Vals) {
2900	unsigned ValID = VE.getValueID(V);
2901	Vals.push_back(Elt: InstID - ValID);
2902	}
2903
2904	void ModuleBitcodeWriter::pushValueSigned(const Value *V, unsigned InstID,
2905	SmallVectorImpl<uint64_t> &Vals) {
2906	unsigned ValID = VE.getValueID(V);
2907	int64_t diff = ((int32_t)InstID - (int32_t)ValID);
2908	emitSignedInt64(Vals, V: diff);
2909	}
2910
2911	/// WriteInstruction - Emit an instruction to the specified stream.
2912	void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
2913	unsigned InstID,
2914	SmallVectorImpl<unsigned> &Vals) {
2915	unsigned Code = 0;
2916	unsigned AbbrevToUse = 0;
2917	VE.setInstructionID(&I);
2918	switch (I.getOpcode()) {
2919	default:
2920	if (Instruction::isCast(Opcode: I.getOpcode())) {
2921	Code = bitc::FUNC_CODE_INST_CAST;
2922	if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
2923	AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
2924	Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
2925	Vals.push_back(Elt: getEncodedCastOpcode(Opcode: I.getOpcode()));
2926	uint64_t Flags = getOptimizationFlags(V: &I);
2927	if (Flags != 0) {
2928	if (AbbrevToUse == FUNCTION_INST_CAST_ABBREV)
2929	AbbrevToUse = FUNCTION_INST_CAST_FLAGS_ABBREV;
2930	Vals.push_back(Elt: Flags);
2931	}
2932	} else {
2933	assert(isa<BinaryOperator>(I) && "Unknown instruction!");
2934	Code = bitc::FUNC_CODE_INST_BINOP;
2935	if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
2936	AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
2937	pushValue(V: I.getOperand(i: 1), InstID, Vals);
2938	Vals.push_back(Elt: getEncodedBinaryOpcode(Opcode: I.getOpcode()));
2939	uint64_t Flags = getOptimizationFlags(V: &I);
2940	if (Flags != 0) {
2941	if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
2942	AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
2943	Vals.push_back(Elt: Flags);
2944	}
2945	}
2946	break;
2947	case Instruction::FNeg: {
2948	Code = bitc::FUNC_CODE_INST_UNOP;
2949	if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
2950	AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
2951	Vals.push_back(Elt: getEncodedUnaryOpcode(Opcode: I.getOpcode()));
2952	uint64_t Flags = getOptimizationFlags(V: &I);
2953	if (Flags != 0) {
2954	if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
2955	AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
2956	Vals.push_back(Elt: Flags);
2957	}
2958	break;
2959	}
2960	case Instruction::GetElementPtr: {
2961	Code = bitc::FUNC_CODE_INST_GEP;
2962	AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
2963	auto &GEPInst = cast<GetElementPtrInst>(Val: I);
2964	Vals.push_back(Elt: GEPInst.isInBounds());
2965	Vals.push_back(Elt: VE.getTypeID(T: GEPInst.getSourceElementType()));
2966	for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i)
2967	pushValueAndType(V: I.getOperand(i), InstID, Vals);
2968	break;
2969	}
2970	case Instruction::ExtractValue: {
2971	Code = bitc::FUNC_CODE_INST_EXTRACTVAL;
2972	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
2973	const ExtractValueInst *EVI = cast<ExtractValueInst>(Val: &I);
2974	Vals.append(in_start: EVI->idx_begin(), in_end: EVI->idx_end());
2975	break;
2976	}
2977	case Instruction::InsertValue: {
2978	Code = bitc::FUNC_CODE_INST_INSERTVAL;
2979	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
2980	pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
2981	const InsertValueInst *IVI = cast<InsertValueInst>(Val: &I);
2982	Vals.append(in_start: IVI->idx_begin(), in_end: IVI->idx_end());
2983	break;
2984	}
2985	case Instruction::Select: {
2986	Code = bitc::FUNC_CODE_INST_VSELECT;
2987	pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
2988	pushValue(V: I.getOperand(i: 2), InstID, Vals);
2989	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
2990	uint64_t Flags = getOptimizationFlags(V: &I);
2991	if (Flags != 0)
2992	Vals.push_back(Elt: Flags);
2993	break;
2994	}
2995	case Instruction::ExtractElement:
2996	Code = bitc::FUNC_CODE_INST_EXTRACTELT;
2997	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
2998	pushValueAndType(V: I.getOperand(i: 1), InstID, Vals);
2999	break;
3000	case Instruction::InsertElement:
3001	Code = bitc::FUNC_CODE_INST_INSERTELT;
3002	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3003	pushValue(V: I.getOperand(i: 1), InstID, Vals);
3004	pushValueAndType(V: I.getOperand(i: 2), InstID, Vals);
3005	break;
3006	case Instruction::ShuffleVector:
3007	Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
3008	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3009	pushValue(V: I.getOperand(i: 1), InstID, Vals);
3010	pushValue(V: cast<ShuffleVectorInst>(Val: I).getShuffleMaskForBitcode(), InstID,
3011	Vals);
3012	break;
3013	case Instruction::ICmp:
3014	case Instruction::FCmp: {
3015	// compare returning Int1Ty or vector of Int1Ty
3016	Code = bitc::FUNC_CODE_INST_CMP2;
3017	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3018	pushValue(V: I.getOperand(i: 1), InstID, Vals);
3019	Vals.push_back(Elt: cast<CmpInst>(Val: I).getPredicate());
3020	uint64_t Flags = getOptimizationFlags(V: &I);
3021	if (Flags != 0)
3022	Vals.push_back(Elt: Flags);
3023	break;
3024	}
3025
3026	case Instruction::Ret:
3027	{
3028	Code = bitc::FUNC_CODE_INST_RET;
3029	unsigned NumOperands = I.getNumOperands();
3030	if (NumOperands == 0)
3031	AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
3032	else if (NumOperands == 1) {
3033	if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals))
3034	AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
3035	} else {
3036	for (unsigned i = 0, e = NumOperands; i != e; ++i)
3037	pushValueAndType(V: I.getOperand(i), InstID, Vals);
3038	}
3039	}
3040	break;
3041	case Instruction::Br:
3042	{
3043	Code = bitc::FUNC_CODE_INST_BR;
3044	const BranchInst &II = cast<BranchInst>(Val: I);
3045	Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 0)));
3046	if (II.isConditional()) {
3047	Vals.push_back(Elt: VE.getValueID(V: II.getSuccessor(i: 1)));
3048	pushValue(V: II.getCondition(), InstID, Vals);
3049	}
3050	}
3051	break;
3052	case Instruction::Switch:
3053	{
3054	Code = bitc::FUNC_CODE_INST_SWITCH;
3055	const SwitchInst &SI = cast<SwitchInst>(Val: I);
3056	Vals.push_back(Elt: VE.getTypeID(T: SI.getCondition()->getType()));
3057	pushValue(V: SI.getCondition(), InstID, Vals);
3058	Vals.push_back(Elt: VE.getValueID(V: SI.getDefaultDest()));
3059	for (auto Case : SI.cases()) {
3060	Vals.push_back(Elt: VE.getValueID(V: Case.getCaseValue()));
3061	Vals.push_back(Elt: VE.getValueID(V: Case.getCaseSuccessor()));
3062	}
3063	}
3064	break;
3065	case Instruction::IndirectBr:
3066	Code = bitc::FUNC_CODE_INST_INDIRECTBR;
3067	Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3068	// Encode the address operand as relative, but not the basic blocks.
3069	pushValue(V: I.getOperand(i: 0), InstID, Vals);
3070	for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i)
3071	Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i)));
3072	break;
3073
3074	case Instruction::Invoke: {
3075	const InvokeInst *II = cast<InvokeInst>(Val: &I);
3076	const Value *Callee = II->getCalledOperand();
3077	FunctionType *FTy = II->getFunctionType();
3078
3079	if (II->hasOperandBundles())
3080	writeOperandBundles(CS: *II, InstID);
3081
3082	Code = bitc::FUNC_CODE_INST_INVOKE;
3083
3084	Vals.push_back(Elt: VE.getAttributeListID(PAL: II->getAttributes()));
3085	Vals.push_back(Elt: II->getCallingConv() | 1 << 13);
3086	Vals.push_back(Elt: VE.getValueID(V: II->getNormalDest()));
3087	Vals.push_back(Elt: VE.getValueID(V: II->getUnwindDest()));
3088	Vals.push_back(Elt: VE.getTypeID(T: FTy));
3089	pushValueAndType(V: Callee, InstID, Vals);
3090
3091	// Emit value #'s for the fixed parameters.
3092	for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3093	pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3094
3095	// Emit type/value pairs for varargs params.
3096	if (FTy->isVarArg()) {
3097	for (unsigned i = FTy->getNumParams(), e = II->arg_size(); i != e; ++i)
3098	pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3099	}
3100	break;
3101	}
3102	case Instruction::Resume:
3103	Code = bitc::FUNC_CODE_INST_RESUME;
3104	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3105	break;
3106	case Instruction::CleanupRet: {
3107	Code = bitc::FUNC_CODE_INST_CLEANUPRET;
3108	const auto &CRI = cast<CleanupReturnInst>(Val: I);
3109	pushValue(V: CRI.getCleanupPad(), InstID, Vals);
3110	if (CRI.hasUnwindDest())
3111	Vals.push_back(Elt: VE.getValueID(V: CRI.getUnwindDest()));
3112	break;
3113	}
3114	case Instruction::CatchRet: {
3115	Code = bitc::FUNC_CODE_INST_CATCHRET;
3116	const auto &CRI = cast<CatchReturnInst>(Val: I);
3117	pushValue(V: CRI.getCatchPad(), InstID, Vals);
3118	Vals.push_back(Elt: VE.getValueID(V: CRI.getSuccessor()));
3119	break;
3120	}
3121	case Instruction::CleanupPad:
3122	case Instruction::CatchPad: {
3123	const auto &FuncletPad = cast<FuncletPadInst>(Val: I);
3124	Code = isa<CatchPadInst>(Val: FuncletPad) ? bitc::FUNC_CODE_INST_CATCHPAD
3125	: bitc::FUNC_CODE_INST_CLEANUPPAD;
3126	pushValue(V: FuncletPad.getParentPad(), InstID, Vals);
3127
3128	unsigned NumArgOperands = FuncletPad.arg_size();
3129	Vals.push_back(Elt: NumArgOperands);
3130	for (unsigned Op = 0; Op != NumArgOperands; ++Op)
3131	pushValueAndType(V: FuncletPad.getArgOperand(i: Op), InstID, Vals);
3132	break;
3133	}
3134	case Instruction::CatchSwitch: {
3135	Code = bitc::FUNC_CODE_INST_CATCHSWITCH;
3136	const auto &CatchSwitch = cast<CatchSwitchInst>(Val: I);
3137
3138	pushValue(V: CatchSwitch.getParentPad(), InstID, Vals);
3139
3140	unsigned NumHandlers = CatchSwitch.getNumHandlers();
3141	Vals.push_back(Elt: NumHandlers);
3142	for (const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3143	Vals.push_back(Elt: VE.getValueID(V: CatchPadBB));
3144
3145	if (CatchSwitch.hasUnwindDest())
3146	Vals.push_back(Elt: VE.getValueID(V: CatchSwitch.getUnwindDest()));
3147	break;
3148	}
3149	case Instruction::CallBr: {
3150	const CallBrInst *CBI = cast<CallBrInst>(Val: &I);
3151	const Value *Callee = CBI->getCalledOperand();
3152	FunctionType *FTy = CBI->getFunctionType();
3153
3154	if (CBI->hasOperandBundles())
3155	writeOperandBundles(CS: *CBI, InstID);
3156
3157	Code = bitc::FUNC_CODE_INST_CALLBR;
3158
3159	Vals.push_back(Elt: VE.getAttributeListID(PAL: CBI->getAttributes()));
3160
3161	Vals.push_back(Elt: CBI->getCallingConv() << bitc::CALL_CCONV |
3162	1 << bitc::CALL_EXPLICIT_TYPE);
3163
3164	Vals.push_back(Elt: VE.getValueID(V: CBI->getDefaultDest()));
3165	Vals.push_back(Elt: CBI->getNumIndirectDests());
3166	for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
3167	Vals.push_back(Elt: VE.getValueID(V: CBI->getIndirectDest(i)));
3168
3169	Vals.push_back(Elt: VE.getTypeID(T: FTy));
3170	pushValueAndType(V: Callee, InstID, Vals);
3171
3172	// Emit value #'s for the fixed parameters.
3173	for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
3174	pushValue(V: I.getOperand(i), InstID, Vals); // fixed param.
3175
3176	// Emit type/value pairs for varargs params.
3177	if (FTy->isVarArg()) {
3178	for (unsigned i = FTy->getNumParams(), e = CBI->arg_size(); i != e; ++i)
3179	pushValueAndType(V: I.getOperand(i), InstID, Vals); // vararg
3180	}
3181	break;
3182	}
3183	case Instruction::Unreachable:
3184	Code = bitc::FUNC_CODE_INST_UNREACHABLE;
3185	AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3186	break;
3187
3188	case Instruction::PHI: {
3189	const PHINode &PN = cast<PHINode>(Val: I);
3190	Code = bitc::FUNC_CODE_INST_PHI;
3191	// With the newer instruction encoding, forward references could give
3192	// negative valued IDs. This is most common for PHIs, so we use
3193	// signed VBRs.
3194	SmallVector<uint64_t, 128> Vals64;
3195	Vals64.push_back(Elt: VE.getTypeID(T: PN.getType()));
3196	for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
3197	pushValueSigned(V: PN.getIncomingValue(i), InstID, Vals&: Vals64);
3198	Vals64.push_back(Elt: VE.getValueID(V: PN.getIncomingBlock(i)));
3199	}
3200
3201	uint64_t Flags = getOptimizationFlags(V: &I);
3202	if (Flags != 0)
3203	Vals64.push_back(Elt: Flags);
3204
3205	// Emit a Vals64 vector and exit.
3206	Stream.EmitRecord(Code, Vals: Vals64, Abbrev: AbbrevToUse);
3207	Vals64.clear();
3208	return;
3209	}
3210
3211	case Instruction::LandingPad: {
3212	const LandingPadInst &LP = cast<LandingPadInst>(Val: I);
3213	Code = bitc::FUNC_CODE_INST_LANDINGPAD;
3214	Vals.push_back(Elt: VE.getTypeID(T: LP.getType()));
3215	Vals.push_back(Elt: LP.isCleanup());
3216	Vals.push_back(Elt: LP.getNumClauses());
3217	for (unsigned I = 0, E = LP.getNumClauses(); I != E; ++I) {
3218	if (LP.isCatch(Idx: I))
3219	Vals.push_back(Elt: LandingPadInst::Catch);
3220	else
3221	Vals.push_back(Elt: LandingPadInst::Filter);
3222	pushValueAndType(V: LP.getClause(Idx: I), InstID, Vals);
3223	}
3224	break;
3225	}
3226
3227	case Instruction::Alloca: {
3228	Code = bitc::FUNC_CODE_INST_ALLOCA;
3229	const AllocaInst &AI = cast<AllocaInst>(Val: I);
3230	Vals.push_back(Elt: VE.getTypeID(T: AI.getAllocatedType()));
3231	Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType()));
3232	Vals.push_back(Elt: VE.getValueID(V: I.getOperand(i: 0))); // size.
3233	using APV = AllocaPackedValues;
3234	unsigned Record = 0;
3235	unsigned EncodedAlign = getEncodedAlign(Alignment: AI.getAlign());
3236	Bitfield::set<APV::AlignLower>(
3237	Packed&: Record, Value: EncodedAlign & ((1 << APV::AlignLower::Bits) - 1));
3238	Bitfield::set<APV::AlignUpper>(Packed&: Record,
3239	Value: EncodedAlign >> APV::AlignLower::Bits);
3240	Bitfield::set<APV::UsedWithInAlloca>(Packed&: Record, Value: AI.isUsedWithInAlloca());
3241	Bitfield::set<APV::ExplicitType>(Packed&: Record, Value: true);
3242	Bitfield::set<APV::SwiftError>(Packed&: Record, Value: AI.isSwiftError());
3243	Vals.push_back(Elt: Record);
3244
3245	unsigned AS = AI.getAddressSpace();
3246	if (AS != M.getDataLayout().getAllocaAddrSpace())
3247	Vals.push_back(Elt: AS);
3248	break;
3249	}
3250
3251	case Instruction::Load:
3252	if (cast<LoadInst>(Val: I).isAtomic()) {
3253	Code = bitc::FUNC_CODE_INST_LOADATOMIC;
3254	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3255	} else {
3256	Code = bitc::FUNC_CODE_INST_LOAD;
3257	if (!pushValueAndType(V: I.getOperand(i: 0), InstID, Vals)) // ptr
3258	AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3259	}
3260	Vals.push_back(Elt: VE.getTypeID(T: I.getType()));
3261	Vals.push_back(Elt: getEncodedAlign(Alignment: cast<LoadInst>(Val: I).getAlign()));
3262	Vals.push_back(Elt: cast<LoadInst>(Val: I).isVolatile());
3263	if (cast<LoadInst>(Val: I).isAtomic()) {
3264	Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<LoadInst>(Val: I).getOrdering()));
3265	Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<LoadInst>(Val: I).getSyncScopeID()));
3266	}
3267	break;
3268	case Instruction::Store:
3269	if (cast<StoreInst>(Val: I).isAtomic())
3270	Code = bitc::FUNC_CODE_INST_STOREATOMIC;
3271	else
3272	Code = bitc::FUNC_CODE_INST_STORE;
3273	pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // ptrty + ptr
3274	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // valty + val
3275	Vals.push_back(Elt: getEncodedAlign(Alignment: cast<StoreInst>(Val: I).getAlign()));
3276	Vals.push_back(Elt: cast<StoreInst>(Val: I).isVolatile());
3277	if (cast<StoreInst>(Val: I).isAtomic()) {
3278	Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<StoreInst>(Val: I).getOrdering()));
3279	Vals.push_back(
3280	Elt: getEncodedSyncScopeID(SSID: cast<StoreInst>(Val: I).getSyncScopeID()));
3281	}
3282	break;
3283	case Instruction::AtomicCmpXchg:
3284	Code = bitc::FUNC_CODE_INST_CMPXCHG;
3285	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3286	pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // cmp.
3287	pushValue(V: I.getOperand(i: 2), InstID, Vals); // newval.
3288	Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isVolatile());
3289	Vals.push_back(
3290	Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getSuccessOrdering()));
3291	Vals.push_back(
3292	Elt: getEncodedSyncScopeID(SSID: cast<AtomicCmpXchgInst>(Val: I).getSyncScopeID()));
3293	Vals.push_back(
3294	Elt: getEncodedOrdering(Ordering: cast<AtomicCmpXchgInst>(Val: I).getFailureOrdering()));
3295	Vals.push_back(Elt: cast<AtomicCmpXchgInst>(Val: I).isWeak());
3296	Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicCmpXchgInst>(Val: I).getAlign()));
3297	break;
3298	case Instruction::AtomicRMW:
3299	Code = bitc::FUNC_CODE_INST_ATOMICRMW;
3300	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals); // ptrty + ptr
3301	pushValueAndType(V: I.getOperand(i: 1), InstID, Vals); // valty + val
3302	Vals.push_back(
3303	Elt: getEncodedRMWOperation(Op: cast<AtomicRMWInst>(Val: I).getOperation()));
3304	Vals.push_back(Elt: cast<AtomicRMWInst>(Val: I).isVolatile());
3305	Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<AtomicRMWInst>(Val: I).getOrdering()));
3306	Vals.push_back(
3307	Elt: getEncodedSyncScopeID(SSID: cast<AtomicRMWInst>(Val: I).getSyncScopeID()));
3308	Vals.push_back(Elt: getEncodedAlign(Alignment: cast<AtomicRMWInst>(Val: I).getAlign()));
3309	break;
3310	case Instruction::Fence:
3311	Code = bitc::FUNC_CODE_INST_FENCE;
3312	Vals.push_back(Elt: getEncodedOrdering(Ordering: cast<FenceInst>(Val: I).getOrdering()));
3313	Vals.push_back(Elt: getEncodedSyncScopeID(SSID: cast<FenceInst>(Val: I).getSyncScopeID()));
3314	break;
3315	case Instruction::Call: {
3316	const CallInst &CI = cast<CallInst>(Val: I);
3317	FunctionType *FTy = CI.getFunctionType();
3318
3319	if (CI.hasOperandBundles())
3320	writeOperandBundles(CS: CI, InstID);
3321
3322	Code = bitc::FUNC_CODE_INST_CALL;
3323
3324	Vals.push_back(Elt: VE.getAttributeListID(PAL: CI.getAttributes()));
3325
3326	unsigned Flags = getOptimizationFlags(V: &I);
3327	Vals.push_back(Elt: CI.getCallingConv() << bitc::CALL_CCONV |
3328	unsigned(CI.isTailCall()) << bitc::CALL_TAIL |
3329	unsigned(CI.isMustTailCall()) << bitc::CALL_MUSTTAIL |
3330	1 << bitc::CALL_EXPLICIT_TYPE |
3331	unsigned(CI.isNoTailCall()) << bitc::CALL_NOTAIL |
3332	unsigned(Flags != 0) << bitc::CALL_FMF);
3333	if (Flags != 0)
3334	Vals.push_back(Elt: Flags);
3335
3336	Vals.push_back(Elt: VE.getTypeID(T: FTy));
3337	pushValueAndType(V: CI.getCalledOperand(), InstID, Vals); // Callee
3338
3339	// Emit value #'s for the fixed parameters.
3340	for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) {
3341	// Check for labels (can happen with asm labels).
3342	if (FTy->getParamType(i)->isLabelTy())
3343	Vals.push_back(Elt: VE.getValueID(V: CI.getArgOperand(i)));
3344	else
3345	pushValue(V: CI.getArgOperand(i), InstID, Vals); // fixed param.
3346	}
3347
3348	// Emit type/value pairs for varargs params.
3349	if (FTy->isVarArg()) {
3350	for (unsigned i = FTy->getNumParams(), e = CI.arg_size(); i != e; ++i)
3351	pushValueAndType(V: CI.getArgOperand(i), InstID, Vals); // varargs
3352	}
3353	break;
3354	}
3355	case Instruction::VAArg:
3356	Code = bitc::FUNC_CODE_INST_VAARG;
3357	Vals.push_back(Elt: VE.getTypeID(T: I.getOperand(i: 0)->getType())); // valistty
3358	pushValue(V: I.getOperand(i: 0), InstID, Vals); // valist.
3359	Vals.push_back(Elt: VE.getTypeID(T: I.getType())); // restype.
3360	break;
3361	case Instruction::Freeze:
3362	Code = bitc::FUNC_CODE_INST_FREEZE;
3363	pushValueAndType(V: I.getOperand(i: 0), InstID, Vals);
3364	break;
3365	}
3366
3367	Stream.EmitRecord(Code, Vals, Abbrev: AbbrevToUse);
3368	Vals.clear();
3369	}
3370
3371	/// Write a GlobalValue VST to the module. The purpose of this data structure is
3372	/// to allow clients to efficiently find the function body.
3373	void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3374	DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3375	// Get the offset of the VST we are writing, and backpatch it into
3376	// the VST forward declaration record.
3377	uint64_t VSTOffset = Stream.GetCurrentBitNo();
3378	// The BitcodeStartBit was the stream offset of the identification block.
3379	VSTOffset -= bitcodeStartBit();
3380	assert((VSTOffset & 31) == 0 && "VST block not 32-bit aligned");
3381	// Note that we add 1 here because the offset is relative to one word
3382	// before the start of the identification block, which was historically
3383	// always the start of the regular bitcode header.
3384	Stream.BackpatchWord(BitNo: VSTOffsetPlaceholder, Val: VSTOffset / 32 + 1);
3385
3386	Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3387
3388	auto Abbv = std::make_shared<BitCodeAbbrev>();
3389	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_FNENTRY));
3390	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3391	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // funcoffset
3392	unsigned FnEntryAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3393
3394	for (const Function &F : M) {
3395	uint64_t Record[2];
3396
3397	if (F.isDeclaration())
3398	continue;
3399
3400	Record[0] = VE.getValueID(V: &F);
3401
3402	// Save the word offset of the function (from the start of the
3403	// actual bitcode written to the stream).
3404	uint64_t BitcodeIndex = FunctionToBitcodeIndex[&F] - bitcodeStartBit();
3405	assert((BitcodeIndex & 31) == 0 && "function block not 32-bit aligned");
3406	// Note that we add 1 here because the offset is relative to one word
3407	// before the start of the identification block, which was historically
3408	// always the start of the regular bitcode header.
3409	Record[1] = BitcodeIndex / 32 + 1;
3410
3411	Stream.EmitRecord(Code: bitc::VST_CODE_FNENTRY, Vals: Record, Abbrev: FnEntryAbbrev);
3412	}
3413
3414	Stream.ExitBlock();
3415	}
3416
3417	/// Emit names for arguments, instructions and basic blocks in a function.
3418	void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3419	const ValueSymbolTable &VST) {
3420	if (VST.empty())
3421	return;
3422
3423	Stream.EnterSubblock(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, CodeLen: 4);
3424
3425	// FIXME: Set up the abbrev, we know how many values there are!
3426	// FIXME: We know if the type names can use 7-bit ascii.
3427	SmallVector<uint64_t, 64> NameVals;
3428
3429	for (const ValueName &Name : VST) {
3430	// Figure out the encoding to use for the name.
3431	StringEncoding Bits = getStringEncoding(Str: Name.getKey());
3432
3433	unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3434	NameVals.push_back(Elt: VE.getValueID(V: Name.getValue()));
3435
3436	// VST_CODE_ENTRY: [valueid, namechar x N]
3437	// VST_CODE_BBENTRY: [bbid, namechar x N]
3438	unsigned Code;
3439	if (isa<BasicBlock>(Val: Name.getValue())) {
3440	Code = bitc::VST_CODE_BBENTRY;
3441	if (Bits == SE_Char6)
3442	AbbrevToUse = VST_BBENTRY_6_ABBREV;
3443	} else {
3444	Code = bitc::VST_CODE_ENTRY;
3445	if (Bits == SE_Char6)
3446	AbbrevToUse = VST_ENTRY_6_ABBREV;
3447	else if (Bits == SE_Fixed7)
3448	AbbrevToUse = VST_ENTRY_7_ABBREV;
3449	}
3450
3451	for (const auto P : Name.getKey())
3452	NameVals.push_back(Elt: (unsigned char)P);
3453
3454	// Emit the finished record.
3455	Stream.EmitRecord(Code, Vals: NameVals, Abbrev: AbbrevToUse);
3456	NameVals.clear();
3457	}
3458
3459	Stream.ExitBlock();
3460	}
3461
3462	void ModuleBitcodeWriter::writeUseList(UseListOrder &&Order) {
3463	assert(Order.Shuffle.size() >= 2 && "Shuffle too small");
3464	unsigned Code;
3465	if (isa<BasicBlock>(Val: Order.V))
3466	Code = bitc::USELIST_CODE_BB;
3467	else
3468	Code = bitc::USELIST_CODE_DEFAULT;
3469
3470	SmallVector<uint64_t, 64> Record(Order.Shuffle.begin(), Order.Shuffle.end());
3471	Record.push_back(Elt: VE.getValueID(V: Order.V));
3472	Stream.EmitRecord(Code, Vals: Record);
3473	}
3474
3475	void ModuleBitcodeWriter::writeUseListBlock(const Function *F) {
3476	assert(VE.shouldPreserveUseListOrder() &&
3477	"Expected to be preserving use-list order");
3478
3479	auto hasMore = [&]() {
3480	return !VE.UseListOrders.empty() && VE.UseListOrders.back().F == F;
3481	};
3482	if (!hasMore())
3483	// Nothing to do.
3484	return;
3485
3486	Stream.EnterSubblock(BlockID: bitc::USELIST_BLOCK_ID, CodeLen: 3);
3487	while (hasMore()) {
3488	writeUseList(Order: std::move(VE.UseListOrders.back()));
3489	VE.UseListOrders.pop_back();
3490	}
3491	Stream.ExitBlock();
3492	}
3493
3494	/// Emit a function body to the module stream.
3495	void ModuleBitcodeWriter::writeFunction(
3496	const Function &F,
3497	DenseMap<const Function *, uint64_t> &FunctionToBitcodeIndex) {
3498	// Save the bitcode index of the start of this function block for recording
3499	// in the VST.
3500	FunctionToBitcodeIndex[&F] = Stream.GetCurrentBitNo();
3501
3502	Stream.EnterSubblock(BlockID: bitc::FUNCTION_BLOCK_ID, CodeLen: 4);
3503	VE.incorporateFunction(F);
3504
3505	SmallVector<unsigned, 64> Vals;
3506
3507	// Emit the number of basic blocks, so the reader can create them ahead of
3508	// time.
3509	Vals.push_back(Elt: VE.getBasicBlocks().size());
3510	Stream.EmitRecord(Code: bitc::FUNC_CODE_DECLAREBLOCKS, Vals);
3511	Vals.clear();
3512
3513	// If there are function-local constants, emit them now.
3514	unsigned CstStart, CstEnd;
3515	VE.getFunctionConstantRange(Start&: CstStart, End&: CstEnd);
3516	writeConstants(FirstVal: CstStart, LastVal: CstEnd, isGlobal: false);
3517
3518	// If there is function-local metadata, emit it now.
3519	writeFunctionMetadata(F);
3520
3521	// Keep a running idea of what the instruction ID is.
3522	unsigned InstID = CstEnd;
3523
3524	bool NeedsMetadataAttachment = F.hasMetadata();
3525
3526	DILocation *LastDL = nullptr;
3527	SmallSetVector<Function *, 4> BlockAddressUsers;
3528
3529	// Finally, emit all the instructions, in order.
3530	for (const BasicBlock &BB : F) {
3531	for (const Instruction &I : BB) {
3532	writeInstruction(I, InstID, Vals);
3533
3534	if (!I.getType()->isVoidTy())
3535	++InstID;
3536
3537	// If the instruction has metadata, write a metadata attachment later.
3538	NeedsMetadataAttachment |= I.hasMetadataOtherThanDebugLoc();
3539
3540	// If the instruction has a debug location, emit it.
3541	if (DILocation *DL = I.getDebugLoc()) {
3542	if (DL == LastDL) {
3543	// Just repeat the same debug loc as last time.
3544	Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC_AGAIN, Vals);
3545	} else {
3546	Vals.push_back(Elt: DL->getLine());
3547	Vals.push_back(Elt: DL->getColumn());
3548	Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getScope()));
3549	Vals.push_back(Elt: VE.getMetadataOrNullID(MD: DL->getInlinedAt()));
3550	Vals.push_back(Elt: DL->isImplicitCode());
3551	Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_LOC, Vals);
3552	Vals.clear();
3553	LastDL = DL;
3554	}
3555	}
3556
3557	// If the instruction has DbgRecords attached to it, emit them. Note that
3558	// they come after the instruction so that it's easy to attach them again
3559	// when reading the bitcode, even though conceptually the debug locations
3560	// start "before" the instruction.
3561	if (I.hasDbgRecords() && WriteNewDbgInfoFormatToBitcode) {
3562	/// Try to push the value only (unwrapped), otherwise push the
3563	/// metadata wrapped value. Returns true if the value was pushed
3564	/// without the ValueAsMetadata wrapper.
3565	auto PushValueOrMetadata = [&Vals, InstID,
3566	this](Metadata *RawLocation) {
3567	assert(RawLocation &&
3568	"RawLocation unexpectedly null in DbgVariableRecord");
3569	if (ValueAsMetadata *VAM = dyn_cast<ValueAsMetadata>(Val: RawLocation)) {
3570	SmallVector<unsigned, 2> ValAndType;
3571	// If the value is a fwd-ref the type is also pushed. We don't
3572	// want the type, so fwd-refs are kept wrapped (pushValueAndType
3573	// returns false if the value is pushed without type).
3574	if (!pushValueAndType(V: VAM->getValue(), InstID, Vals&: ValAndType)) {
3575	Vals.push_back(Elt: ValAndType[0]);
3576	return true;
3577	}
3578	}
3579	// The metadata is a DIArgList, or ValueAsMetadata wrapping a
3580	// fwd-ref. Push the metadata ID.
3581	Vals.push_back(Elt: VE.getMetadataID(MD: RawLocation));
3582	return false;
3583	};
3584
3585	// Write out non-instruction debug information attached to this
3586	// instruction. Write it after the instruction so that it's easy to
3587	// re-attach to the instruction reading the records in.
3588	for (DbgRecord &DR : I.DebugMarker->getDbgRecordRange()) {
3589	if (DbgLabelRecord *DLR = dyn_cast<DbgLabelRecord>(Val: &DR)) {
3590	Vals.push_back(Elt: VE.getMetadataID(MD: &*DLR->getDebugLoc()));
3591	Vals.push_back(Elt: VE.getMetadataID(MD: DLR->getLabel()));
3592	Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_LABEL, Vals);
3593	Vals.clear();
3594	continue;
3595	}
3596
3597	// First 3 fields are common to all kinds:
3598	// DILocation, DILocalVariable, DIExpression
3599	// dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE)
3600	// ..., LocationMetadata
3601	// dbg_value (FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE - abbrev'd)
3602	// ..., Value
3603	// dbg_declare (FUNC_CODE_DEBUG_RECORD_DECLARE)
3604	// ..., LocationMetadata
3605	// dbg_assign (FUNC_CODE_DEBUG_RECORD_ASSIGN)
3606	// ..., LocationMetadata, DIAssignID, DIExpression, LocationMetadata
3607	DbgVariableRecord &DVR = cast<DbgVariableRecord>(Val&: DR);
3608	Vals.push_back(Elt: VE.getMetadataID(MD: &*DVR.getDebugLoc()));
3609	Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getVariable()));
3610	Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getExpression()));
3611	if (DVR.isDbgValue()) {
3612	if (PushValueOrMetadata(DVR.getRawLocation()))
3613	Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE, Vals,
3614	Abbrev: FUNCTION_DEBUG_RECORD_VALUE_ABBREV);
3615	else
3616	Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_VALUE, Vals);
3617	} else if (DVR.isDbgDeclare()) {
3618	Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3619	Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_DECLARE, Vals);
3620	} else {
3621	assert(DVR.isDbgAssign() && "Unexpected DbgRecord kind");
3622	Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawLocation()));
3623	Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAssignID()));
3624	Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getAddressExpression()));
3625	Vals.push_back(Elt: VE.getMetadataID(MD: DVR.getRawAddress()));
3626	Stream.EmitRecord(Code: bitc::FUNC_CODE_DEBUG_RECORD_ASSIGN, Vals);
3627	}
3628	Vals.clear();
3629	}
3630	}
3631	}
3632
3633	if (BlockAddress *BA = BlockAddress::lookup(BB: &BB)) {
3634	SmallVector<Value *> Worklist{BA};
3635	SmallPtrSet<Value *, 8> Visited{BA};
3636	while (!Worklist.empty()) {
3637	Value *V = Worklist.pop_back_val();
3638	for (User *U : V->users()) {
3639	if (auto *I = dyn_cast<Instruction>(Val: U)) {
3640	Function *P = I->getFunction();
3641	if (P != &F)
3642	BlockAddressUsers.insert(X: P);
3643	} else if (isa<Constant>(Val: U) && !isa<GlobalValue>(Val: U) &&
3644	Visited.insert(Ptr: U).second)
3645	Worklist.push_back(Elt: U);
3646	}
3647	}
3648	}
3649	}
3650
3651	if (!BlockAddressUsers.empty()) {
3652	Vals.resize(N: BlockAddressUsers.size());
3653	for (auto I : llvm::enumerate(First&: BlockAddressUsers))
3654	Vals[I.index()] = VE.getValueID(V: I.value());
3655	Stream.EmitRecord(Code: bitc::FUNC_CODE_BLOCKADDR_USERS, Vals);
3656	Vals.clear();
3657	}
3658
3659	// Emit names for all the instructions etc.
3660	if (auto *Symtab = F.getValueSymbolTable())
3661	writeFunctionLevelValueSymbolTable(VST: *Symtab);
3662
3663	if (NeedsMetadataAttachment)
3664	writeFunctionMetadataAttachment(F);
3665	if (VE.shouldPreserveUseListOrder())
3666	writeUseListBlock(F: &F);
3667	VE.purgeFunction();
3668	Stream.ExitBlock();
3669	}
3670
3671	// Emit blockinfo, which defines the standard abbreviations etc.
3672	void ModuleBitcodeWriter::writeBlockInfo() {
3673	// We only want to emit block info records for blocks that have multiple
3674	// instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
3675	// Other blocks can define their abbrevs inline.
3676	Stream.EnterBlockInfoBlock();
3677
3678	{ // 8-bit fixed-width VST_CODE_ENTRY/VST_CODE_BBENTRY strings.
3679	auto Abbv = std::make_shared<BitCodeAbbrev>();
3680	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3));
3681	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3682	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3683	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3684	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3685	VST_ENTRY_8_ABBREV)
3686	llvm_unreachable("Unexpected abbrev ordering!");
3687	}
3688
3689	{ // 7-bit fixed width VST_CODE_ENTRY strings.
3690	auto Abbv = std::make_shared<BitCodeAbbrev>();
3691	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3692	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3693	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3694	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3695	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3696	VST_ENTRY_7_ABBREV)
3697	llvm_unreachable("Unexpected abbrev ordering!");
3698	}
3699	{ // 6-bit char6 VST_CODE_ENTRY strings.
3700	auto Abbv = std::make_shared<BitCodeAbbrev>();
3701	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_ENTRY));
3702	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3703	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3704	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3705	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3706	VST_ENTRY_6_ABBREV)
3707	llvm_unreachable("Unexpected abbrev ordering!");
3708	}
3709	{ // 6-bit char6 VST_CODE_BBENTRY strings.
3710	auto Abbv = std::make_shared<BitCodeAbbrev>();
3711	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::VST_CODE_BBENTRY));
3712	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3713	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3714	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3715	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::VALUE_SYMTAB_BLOCK_ID, Abbv) !=
3716	VST_BBENTRY_6_ABBREV)
3717	llvm_unreachable("Unexpected abbrev ordering!");
3718	}
3719
3720	{ // SETTYPE abbrev for CONSTANTS_BLOCK.
3721	auto Abbv = std::make_shared<BitCodeAbbrev>();
3722	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_SETTYPE));
3723	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed,
3724	VE.computeBitsRequiredForTypeIndicies()));
3725	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3726	CONSTANTS_SETTYPE_ABBREV)
3727	llvm_unreachable("Unexpected abbrev ordering!");
3728	}
3729
3730	{ // INTEGER abbrev for CONSTANTS_BLOCK.
3731	auto Abbv = std::make_shared<BitCodeAbbrev>();
3732	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_INTEGER));
3733	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3734	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3735	CONSTANTS_INTEGER_ABBREV)
3736	llvm_unreachable("Unexpected abbrev ordering!");
3737	}
3738
3739	{ // CE_CAST abbrev for CONSTANTS_BLOCK.
3740	auto Abbv = std::make_shared<BitCodeAbbrev>();
3741	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_CE_CAST));
3742	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // cast opc
3743	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // typeid
3744	VE.computeBitsRequiredForTypeIndicies()));
3745	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // value id
3746
3747	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3748	CONSTANTS_CE_CAST_Abbrev)
3749	llvm_unreachable("Unexpected abbrev ordering!");
3750	}
3751	{ // NULL abbrev for CONSTANTS_BLOCK.
3752	auto Abbv = std::make_shared<BitCodeAbbrev>();
3753	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::CST_CODE_NULL));
3754	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::CONSTANTS_BLOCK_ID, Abbv) !=
3755	CONSTANTS_NULL_Abbrev)
3756	llvm_unreachable("Unexpected abbrev ordering!");
3757	}
3758
3759	// FIXME: This should only use space for first class types!
3760
3761	{ // INST_LOAD abbrev for FUNCTION_BLOCK.
3762	auto Abbv = std::make_shared<BitCodeAbbrev>();
3763	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_LOAD));
3764	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Ptr
3765	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty
3766	VE.computeBitsRequiredForTypeIndicies()));
3767	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // Align
3768	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // volatile
3769	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3770	FUNCTION_INST_LOAD_ABBREV)
3771	llvm_unreachable("Unexpected abbrev ordering!");
3772	}
3773	{ // INST_UNOP abbrev for FUNCTION_BLOCK.
3774	auto Abbv = std::make_shared<BitCodeAbbrev>();
3775	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
3776	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3777	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3778	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3779	FUNCTION_INST_UNOP_ABBREV)
3780	llvm_unreachable("Unexpected abbrev ordering!");
3781	}
3782	{ // INST_UNOP_FLAGS abbrev for FUNCTION_BLOCK.
3783	auto Abbv = std::make_shared<BitCodeAbbrev>();
3784	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNOP));
3785	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3786	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3787	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
3788	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3789	FUNCTION_INST_UNOP_FLAGS_ABBREV)
3790	llvm_unreachable("Unexpected abbrev ordering!");
3791	}
3792	{ // INST_BINOP abbrev for FUNCTION_BLOCK.
3793	auto Abbv = std::make_shared<BitCodeAbbrev>();
3794	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
3795	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3796	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
3797	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3798	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3799	FUNCTION_INST_BINOP_ABBREV)
3800	llvm_unreachable("Unexpected abbrev ordering!");
3801	}
3802	{ // INST_BINOP_FLAGS abbrev for FUNCTION_BLOCK.
3803	auto Abbv = std::make_shared<BitCodeAbbrev>();
3804	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_BINOP));
3805	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // LHS
3806	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // RHS
3807	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3808	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
3809	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3810	FUNCTION_INST_BINOP_FLAGS_ABBREV)
3811	llvm_unreachable("Unexpected abbrev ordering!");
3812	}
3813	{ // INST_CAST abbrev for FUNCTION_BLOCK.
3814	auto Abbv = std::make_shared<BitCodeAbbrev>();
3815	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
3816	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal
3817	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty
3818	VE.computeBitsRequiredForTypeIndicies()));
3819	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3820	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3821	FUNCTION_INST_CAST_ABBREV)
3822	llvm_unreachable("Unexpected abbrev ordering!");
3823	}
3824	{ // INST_CAST_FLAGS abbrev for FUNCTION_BLOCK.
3825	auto Abbv = std::make_shared<BitCodeAbbrev>();
3826	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_CAST));
3827	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // OpVal
3828	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty
3829	VE.computeBitsRequiredForTypeIndicies()));
3830	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 4)); // opc
3831	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8)); // flags
3832	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3833	FUNCTION_INST_CAST_FLAGS_ABBREV)
3834	llvm_unreachable("Unexpected abbrev ordering!");
3835	}
3836
3837	{ // INST_RET abbrev for FUNCTION_BLOCK.
3838	auto Abbv = std::make_shared<BitCodeAbbrev>();
3839	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
3840	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3841	FUNCTION_INST_RET_VOID_ABBREV)
3842	llvm_unreachable("Unexpected abbrev ordering!");
3843	}
3844	{ // INST_RET abbrev for FUNCTION_BLOCK.
3845	auto Abbv = std::make_shared<BitCodeAbbrev>();
3846	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_RET));
3847	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ValID
3848	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3849	FUNCTION_INST_RET_VAL_ABBREV)
3850	llvm_unreachable("Unexpected abbrev ordering!");
3851	}
3852	{ // INST_UNREACHABLE abbrev for FUNCTION_BLOCK.
3853	auto Abbv = std::make_shared<BitCodeAbbrev>();
3854	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_UNREACHABLE));
3855	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3856	FUNCTION_INST_UNREACHABLE_ABBREV)
3857	llvm_unreachable("Unexpected abbrev ordering!");
3858	}
3859	{
3860	auto Abbv = std::make_shared<BitCodeAbbrev>();
3861	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_INST_GEP));
3862	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1));
3863	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, // dest ty
3864	Log2_32_Ceil(Value: VE.getTypes().size() + 1)));
3865	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3866	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
3867	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3868	FUNCTION_INST_GEP_ABBREV)
3869	llvm_unreachable("Unexpected abbrev ordering!");
3870	}
3871	{
3872	auto Abbv = std::make_shared<BitCodeAbbrev>();
3873	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FUNC_CODE_DEBUG_RECORD_VALUE_SIMPLE));
3874	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // dbgloc
3875	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // var
3876	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 7)); // expr
3877	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // val
3878	if (Stream.EmitBlockInfoAbbrev(BlockID: bitc::FUNCTION_BLOCK_ID, Abbv) !=
3879	FUNCTION_DEBUG_RECORD_VALUE_ABBREV)
3880	llvm_unreachable("Unexpected abbrev ordering! 1");
3881	}
3882	Stream.ExitBlock();
3883	}
3884
3885	/// Write the module path strings, currently only used when generating
3886	/// a combined index file.
3887	void IndexBitcodeWriter::writeModStrings() {
3888	Stream.EnterSubblock(BlockID: bitc::MODULE_STRTAB_BLOCK_ID, CodeLen: 3);
3889
3890	// TODO: See which abbrev sizes we actually need to emit
3891
3892	// 8-bit fixed-width MST_ENTRY strings.
3893	auto Abbv = std::make_shared<BitCodeAbbrev>();
3894	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
3895	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3896	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3897	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8));
3898	unsigned Abbrev8Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3899
3900	// 7-bit fixed width MST_ENTRY strings.
3901	Abbv = std::make_shared<BitCodeAbbrev>();
3902	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
3903	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3904	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3905	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7));
3906	unsigned Abbrev7Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3907
3908	// 6-bit char6 MST_ENTRY strings.
3909	Abbv = std::make_shared<BitCodeAbbrev>();
3910	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_ENTRY));
3911	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
3912	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
3913	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
3914	unsigned Abbrev6Bit = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3915
3916	// Module Hash, 160 bits SHA1. Optionally, emitted after each MST_CODE_ENTRY.
3917	Abbv = std::make_shared<BitCodeAbbrev>();
3918	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MST_CODE_HASH));
3919	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3920	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3921	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3922	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3923	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
3924	unsigned AbbrevHash = Stream.EmitAbbrev(Abbv: std::move(Abbv));
3925
3926	SmallVector<unsigned, 64> Vals;
3927	forEachModule(Callback: [&](const StringMapEntry<ModuleHash> &MPSE) {
3928	StringRef Key = MPSE.getKey();
3929	const auto &Hash = MPSE.getValue();
3930	StringEncoding Bits = getStringEncoding(Str: Key);
3931	unsigned AbbrevToUse = Abbrev8Bit;
3932	if (Bits == SE_Char6)
3933	AbbrevToUse = Abbrev6Bit;
3934	else if (Bits == SE_Fixed7)
3935	AbbrevToUse = Abbrev7Bit;
3936
3937	auto ModuleId = ModuleIdMap.size();
3938	ModuleIdMap[Key] = ModuleId;
3939	Vals.push_back(Elt: ModuleId);
3940	Vals.append(in_start: Key.begin(), in_end: Key.end());
3941
3942	// Emit the finished record.
3943	Stream.EmitRecord(Code: bitc::MST_CODE_ENTRY, Vals, Abbrev: AbbrevToUse);
3944
3945	// Emit an optional hash for the module now
3946	if (llvm::any_of(Range: Hash, P: [](uint32_t H) { return H; })) {
3947	Vals.assign(in_start: Hash.begin(), in_end: Hash.end());
3948	// Emit the hash record.
3949	Stream.EmitRecord(Code: bitc::MST_CODE_HASH, Vals, Abbrev: AbbrevHash);
3950	}
3951
3952	Vals.clear();
3953	});
3954	Stream.ExitBlock();
3955	}
3956
3957	/// Write the function type metadata related records that need to appear before
3958	/// a function summary entry (whether per-module or combined).
3959	template <typename Fn>
3960	static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
3961	FunctionSummary *FS,
3962	Fn GetValueID) {
3963	if (!FS->type_tests().empty())
3964	Stream.EmitRecord(Code: bitc::FS_TYPE_TESTS, Vals: FS->type_tests());
3965
3966	SmallVector<uint64_t, 64> Record;
3967
3968	auto WriteVFuncIdVec = [&](uint64_t Ty,
3969	ArrayRef<FunctionSummary::VFuncId> VFs) {
3970	if (VFs.empty())
3971	return;
3972	Record.clear();
3973	for (auto &VF : VFs) {
3974	Record.push_back(Elt: VF.GUID);
3975	Record.push_back(Elt: VF.Offset);
3976	}
3977	Stream.EmitRecord(Code: Ty, Vals: Record);
3978	};
3979
3980	WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
3981	FS->type_test_assume_vcalls());
3982	WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
3983	FS->type_checked_load_vcalls());
3984
3985	auto WriteConstVCallVec = [&](uint64_t Ty,
3986	ArrayRef<FunctionSummary::ConstVCall> VCs) {
3987	for (auto &VC : VCs) {
3988	Record.clear();
3989	Record.push_back(Elt: VC.VFunc.GUID);
3990	Record.push_back(Elt: VC.VFunc.Offset);
3991	llvm::append_range(C&: Record, R: VC.Args);
3992	Stream.EmitRecord(Code: Ty, Vals: Record);
3993	}
3994	};
3995
3996	WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
3997	FS->type_test_assume_const_vcalls());
3998	WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
3999	FS->type_checked_load_const_vcalls());
4000
4001	auto WriteRange = [&](ConstantRange Range) {
4002	Range = Range.sextOrTrunc(BitWidth: FunctionSummary::ParamAccess::RangeWidth);
4003	assert(Range.getLower().getNumWords() == 1);
4004	assert(Range.getUpper().getNumWords() == 1);
4005	emitSignedInt64(Vals&: Record, V: *Range.getLower().getRawData());
4006	emitSignedInt64(Vals&: Record, V: *Range.getUpper().getRawData());
4007	};
4008
4009	if (!FS->paramAccesses().empty()) {
4010	Record.clear();
4011	for (auto &Arg : FS->paramAccesses()) {
4012	size_t UndoSize = Record.size();
4013	Record.push_back(Elt: Arg.ParamNo);
4014	WriteRange(Arg.Use);
4015	Record.push_back(Elt: Arg.Calls.size());
4016	for (auto &Call : Arg.Calls) {
4017	Record.push_back(Elt: Call.ParamNo);
4018	std::optional<unsigned> ValueID = GetValueID(Call.Callee);
4019	if (!ValueID) {
4020	// If ValueID is unknown we can't drop just this call, we must drop
4021	// entire parameter.
4022	Record.resize(N: UndoSize);
4023	break;
4024	}
4025	Record.push_back(Elt: *ValueID);
4026	WriteRange(Call.Offsets);
4027	}
4028	}
4029	if (!Record.empty())
4030	Stream.EmitRecord(Code: bitc::FS_PARAM_ACCESS, Vals: Record);
4031	}
4032	}
4033
4034	/// Collect type IDs from type tests used by function.
4035	static void
4036	getReferencedTypeIds(FunctionSummary *FS,
4037	std::set<GlobalValue::GUID> &ReferencedTypeIds) {
4038	if (!FS->type_tests().empty())
4039	for (auto &TT : FS->type_tests())
4040	ReferencedTypeIds.insert(x: TT);
4041
4042	auto GetReferencedTypesFromVFuncIdVec =
4043	[&](ArrayRef<FunctionSummary::VFuncId> VFs) {
4044	for (auto &VF : VFs)
4045	ReferencedTypeIds.insert(x: VF.GUID);
4046	};
4047
4048	GetReferencedTypesFromVFuncIdVec(FS->type_test_assume_vcalls());
4049	GetReferencedTypesFromVFuncIdVec(FS->type_checked_load_vcalls());
4050
4051	auto GetReferencedTypesFromConstVCallVec =
4052	[&](ArrayRef<FunctionSummary::ConstVCall> VCs) {
4053	for (auto &VC : VCs)
4054	ReferencedTypeIds.insert(x: VC.VFunc.GUID);
4055	};
4056
4057	GetReferencedTypesFromConstVCallVec(FS->type_test_assume_const_vcalls());
4058	GetReferencedTypesFromConstVCallVec(FS->type_checked_load_const_vcalls());
4059	}
4060
4061	static void writeWholeProgramDevirtResolutionByArg(
4062	SmallVector<uint64_t, 64> &NameVals, const std::vector<uint64_t> &args,
4063	const WholeProgramDevirtResolution::ByArg &ByArg) {
4064	NameVals.push_back(Elt: args.size());
4065	llvm::append_range(C&: NameVals, R: args);
4066
4067	NameVals.push_back(Elt: ByArg.TheKind);
4068	NameVals.push_back(Elt: ByArg.Info);
4069	NameVals.push_back(Elt: ByArg.Byte);
4070	NameVals.push_back(Elt: ByArg.Bit);
4071	}
4072
4073	static void writeWholeProgramDevirtResolution(
4074	SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4075	uint64_t Id, const WholeProgramDevirtResolution &Wpd) {
4076	NameVals.push_back(Elt: Id);
4077
4078	NameVals.push_back(Elt: Wpd.TheKind);
4079	NameVals.push_back(Elt: StrtabBuilder.add(S: Wpd.SingleImplName));
4080	NameVals.push_back(Elt: Wpd.SingleImplName.size());
4081
4082	NameVals.push_back(Elt: Wpd.ResByArg.size());
4083	for (auto &A : Wpd.ResByArg)
4084	writeWholeProgramDevirtResolutionByArg(NameVals, args: A.first, ByArg: A.second);
4085	}
4086
4087	static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
4088	StringTableBuilder &StrtabBuilder,
4089	const std::string &Id,
4090	const TypeIdSummary &Summary) {
4091	NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4092	NameVals.push_back(Elt: Id.size());
4093
4094	NameVals.push_back(Elt: Summary.TTRes.TheKind);
4095	NameVals.push_back(Elt: Summary.TTRes.SizeM1BitWidth);
4096	NameVals.push_back(Elt: Summary.TTRes.AlignLog2);
4097	NameVals.push_back(Elt: Summary.TTRes.SizeM1);
4098	NameVals.push_back(Elt: Summary.TTRes.BitMask);
4099	NameVals.push_back(Elt: Summary.TTRes.InlineBits);
4100
4101	for (auto &W : Summary.WPDRes)
4102	writeWholeProgramDevirtResolution(NameVals, StrtabBuilder, Id: W.first,
4103	Wpd: W.second);
4104	}
4105
4106	static void writeTypeIdCompatibleVtableSummaryRecord(
4107	SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
4108	const std::string &Id, const TypeIdCompatibleVtableInfo &Summary,
4109	ValueEnumerator &VE) {
4110	NameVals.push_back(Elt: StrtabBuilder.add(S: Id));
4111	NameVals.push_back(Elt: Id.size());
4112
4113	for (auto &P : Summary) {
4114	NameVals.push_back(Elt: P.AddressPointOffset);
4115	NameVals.push_back(Elt: VE.getValueID(V: P.VTableVI.getValue()));
4116	}
4117	}
4118
4119	static void writeFunctionHeapProfileRecords(
4120	BitstreamWriter &Stream, FunctionSummary *FS, unsigned CallsiteAbbrev,
4121	unsigned AllocAbbrev, bool PerModule,
4122	std::function<unsigned(const ValueInfo &VI)> GetValueID,
4123	std::function<unsigned(unsigned)> GetStackIndex) {
4124	SmallVector<uint64_t> Record;
4125
4126	for (auto &CI : FS->callsites()) {
4127	Record.clear();
4128	// Per module callsite clones should always have a single entry of
4129	// value 0.
4130	assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
4131	Record.push_back(Elt: GetValueID(CI.Callee));
4132	if (!PerModule) {
4133	Record.push_back(Elt: CI.StackIdIndices.size());
4134	Record.push_back(Elt: CI.Clones.size());
4135	}
4136	for (auto Id : CI.StackIdIndices)
4137	Record.push_back(Elt: GetStackIndex(Id));
4138	if (!PerModule) {
4139	for (auto V : CI.Clones)
4140	Record.push_back(Elt: V);
4141	}
4142	Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_CALLSITE_INFO
4143	: bitc::FS_COMBINED_CALLSITE_INFO,
4144	Vals: Record, Abbrev: CallsiteAbbrev);
4145	}
4146
4147	for (auto &AI : FS->allocs()) {
4148	Record.clear();
4149	// Per module alloc versions should always have a single entry of
4150	// value 0.
4151	assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
4152	if (!PerModule) {
4153	Record.push_back(Elt: AI.MIBs.size());
4154	Record.push_back(Elt: AI.Versions.size());
4155	}
4156	for (auto &MIB : AI.MIBs) {
4157	Record.push_back(Elt: (uint8_t)MIB.AllocType);
4158	Record.push_back(Elt: MIB.StackIdIndices.size());
4159	for (auto Id : MIB.StackIdIndices)
4160	Record.push_back(Elt: GetStackIndex(Id));
4161	}
4162	if (!PerModule) {
4163	for (auto V : AI.Versions)
4164	Record.push_back(Elt: V);
4165	}
4166	Stream.EmitRecord(Code: PerModule ? bitc::FS_PERMODULE_ALLOC_INFO
4167	: bitc::FS_COMBINED_ALLOC_INFO,
4168	Vals: Record, Abbrev: AllocAbbrev);
4169	}
4170	}
4171
4172	// Helper to emit a single function summary record.
4173	void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
4174	SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
4175	unsigned ValueID, unsigned FSCallsRelBFAbbrev,
4176	unsigned FSCallsProfileAbbrev, unsigned CallsiteAbbrev,
4177	unsigned AllocAbbrev, const Function &F) {
4178	NameVals.push_back(Elt: ValueID);
4179
4180	FunctionSummary *FS = cast<FunctionSummary>(Val: Summary);
4181
4182	writeFunctionTypeMetadataRecords(
4183	Stream, FS, GetValueID: [&](const ValueInfo &VI) -> std::optional<unsigned> {
4184	return {VE.getValueID(V: VI.getValue())};
4185	});
4186
4187	writeFunctionHeapProfileRecords(
4188	Stream, FS, CallsiteAbbrev, AllocAbbrev,
4189	/*PerModule*/ true,
4190	/*GetValueId*/ GetValueID: [&](const ValueInfo &VI) { return getValueId(VI); },
4191	/*GetStackIndex*/ [&](unsigned I) { return I; });
4192
4193	auto SpecialRefCnts = FS->specialRefCounts();
4194	NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4195	NameVals.push_back(Elt: FS->instCount());
4196	NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4197	NameVals.push_back(Elt: FS->refs().size());
4198	NameVals.push_back(Elt: SpecialRefCnts.first); // rorefcnt
4199	NameVals.push_back(Elt: SpecialRefCnts.second); // worefcnt
4200
4201	for (auto &RI : FS->refs())
4202	NameVals.push_back(Elt: getValueId(VI: RI));
4203
4204	const bool UseRelBFRecord =
4205	WriteRelBFToSummary && !F.hasProfileData() &&
4206	ForceSummaryEdgesCold == FunctionSummary::FSHT_None;
4207	for (auto &ECI : FS->calls()) {
4208	NameVals.push_back(Elt: getValueId(VI: ECI.first));
4209	if (UseRelBFRecord)
4210	NameVals.push_back(Elt: getEncodedRelBFCallEdgeInfo(CI: ECI.second));
4211	else
4212	NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: ECI.second));
4213	}
4214
4215	unsigned FSAbbrev =
4216	(UseRelBFRecord ? FSCallsRelBFAbbrev : FSCallsProfileAbbrev);
4217	unsigned Code =
4218	(UseRelBFRecord ? bitc::FS_PERMODULE_RELBF : bitc::FS_PERMODULE_PROFILE);
4219
4220	// Emit the finished record.
4221	Stream.EmitRecord(Code, Vals: NameVals, Abbrev: FSAbbrev);
4222	NameVals.clear();
4223	}
4224
4225	// Collect the global value references in the given variable's initializer,
4226	// and emit them in a summary record.
4227	void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4228	const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
4229	unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
4230	auto VI = Index->getValueInfo(GUID: V.getGUID());
4231	if (!VI || VI.getSummaryList().empty()) {
4232	// Only declarations should not have a summary (a declaration might however
4233	// have a summary if the def was in module level asm).
4234	assert(V.isDeclaration());
4235	return;
4236	}
4237	auto *Summary = VI.getSummaryList()[0].get();
4238	NameVals.push_back(Elt: VE.getValueID(V: &V));
4239	GlobalVarSummary *VS = cast<GlobalVarSummary>(Val: Summary);
4240	NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4241	NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4242
4243	auto VTableFuncs = VS->vTableFuncs();
4244	if (!VTableFuncs.empty())
4245	NameVals.push_back(Elt: VS->refs().size());
4246
4247	unsigned SizeBeforeRefs = NameVals.size();
4248	for (auto &RI : VS->refs())
4249	NameVals.push_back(Elt: VE.getValueID(V: RI.getValue()));
4250	// Sort the refs for determinism output, the vector returned by FS->refs() has
4251	// been initialized from a DenseSet.
4252	llvm::sort(C: drop_begin(RangeOrContainer&: NameVals, N: SizeBeforeRefs));
4253
4254	if (VTableFuncs.empty())
4255	Stream.EmitRecord(Code: bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4256	Abbrev: FSModRefsAbbrev);
4257	else {
4258	// VTableFuncs pairs should already be sorted by offset.
4259	for (auto &P : VTableFuncs) {
4260	NameVals.push_back(Elt: VE.getValueID(V: P.FuncVI.getValue()));
4261	NameVals.push_back(Elt: P.VTableOffset);
4262	}
4263
4264	Stream.EmitRecord(Code: bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, Vals: NameVals,
4265	Abbrev: FSModVTableRefsAbbrev);
4266	}
4267	NameVals.clear();
4268	}
4269
4270	/// Emit the per-module summary section alongside the rest of
4271	/// the module's bitcode.
4272	void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4273	// By default we compile with ThinLTO if the module has a summary, but the
4274	// client can request full LTO with a module flag.
4275	bool IsThinLTO = true;
4276	if (auto *MD =
4277	mdconst::extract_or_null<ConstantInt>(MD: M.getModuleFlag(Key: "ThinLTO")))
4278	IsThinLTO = MD->getZExtValue();
4279	Stream.EnterSubblock(BlockID: IsThinLTO ? bitc::GLOBALVAL_SUMMARY_BLOCK_ID
4280	: bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID,
4281	CodeLen: 4);
4282
4283	Stream.EmitRecord(
4284	Code: bitc::FS_VERSION,
4285	Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4286
4287	// Write the index flags.
4288	uint64_t Flags = 0;
4289	// Bits 1-3 are set only in the combined index, skip them.
4290	if (Index->enableSplitLTOUnit())
4291	Flags |= 0x8;
4292	if (Index->hasUnifiedLTO())
4293	Flags |= 0x200;
4294
4295	Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Flags});
4296
4297	if (Index->begin() == Index->end()) {
4298	Stream.ExitBlock();
4299	return;
4300	}
4301
4302	for (const auto &GVI : valueIds()) {
4303	Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4304	Vals: ArrayRef<uint64_t>{GVI.second, GVI.first});
4305	}
4306
4307	if (!Index->stackIds().empty()) {
4308	auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4309	StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4310	// numids x stackid
4311	StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4312	StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4313	unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4314	Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals: Index->stackIds(), Abbrev: StackIdAbbvId);
4315	}
4316
4317	// Abbrev for FS_PERMODULE_PROFILE.
4318	auto Abbv = std::make_shared<BitCodeAbbrev>();
4319	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_PROFILE));
4320	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4321	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // flags
4322	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4323	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4324	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4325	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4326	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4327	// numrefs x valueid, n x (valueid, hotness+tailcall flags)
4328	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4329	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4330	unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4331
4332	// Abbrev for FS_PERMODULE_RELBF.
4333	Abbv = std::make_shared<BitCodeAbbrev>();
4334	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_RELBF));
4335	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4336	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4337	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4338	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4339	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4340	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4341	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4342	// numrefs x valueid, n x (valueid, rel_block_freq+tailcall])
4343	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4344	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4345	unsigned FSCallsRelBFAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4346
4347	// Abbrev for FS_PERMODULE_GLOBALVAR_INIT_REFS.
4348	Abbv = std::make_shared<BitCodeAbbrev>();
4349	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS));
4350	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4351	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4352	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4353	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4354	unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4355
4356	// Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
4357	Abbv = std::make_shared<BitCodeAbbrev>();
4358	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
4359	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4360	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4361	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4362	// numrefs x valueid, n x (valueid , offset)
4363	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4364	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4365	unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4366
4367	// Abbrev for FS_ALIAS.
4368	Abbv = std::make_shared<BitCodeAbbrev>();
4369	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_ALIAS));
4370	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4371	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4372	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4373	unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4374
4375	// Abbrev for FS_TYPE_ID_METADATA
4376	Abbv = std::make_shared<BitCodeAbbrev>();
4377	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA));
4378	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index
4379	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length
4380	// n x (valueid , offset)
4381	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4382	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4383	unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4384
4385	Abbv = std::make_shared<BitCodeAbbrev>();
4386	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_CALLSITE_INFO));
4387	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4388	// n x stackidindex
4389	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4390	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4391	unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4392
4393	Abbv = std::make_shared<BitCodeAbbrev>();
4394	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_PERMODULE_ALLOC_INFO));
4395	// n x (alloc type, numstackids, numstackids x stackidindex)
4396	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4397	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4398	unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4399
4400	SmallVector<uint64_t, 64> NameVals;
4401	// Iterate over the list of functions instead of the Index to
4402	// ensure the ordering is stable.
4403	for (const Function &F : M) {
4404	// Summary emission does not support anonymous functions, they have to
4405	// renamed using the anonymous function renaming pass.
4406	if (!F.hasName())
4407	report_fatal_error(reason: "Unexpected anonymous function when writing summary");
4408
4409	ValueInfo VI = Index->getValueInfo(GUID: F.getGUID());
4410	if (!VI || VI.getSummaryList().empty()) {
4411	// Only declarations should not have a summary (a declaration might
4412	// however have a summary if the def was in module level asm).
4413	assert(F.isDeclaration());
4414	continue;
4415	}
4416	auto *Summary = VI.getSummaryList()[0].get();
4417	writePerModuleFunctionSummaryRecord(
4418	NameVals, Summary, ValueID: VE.getValueID(V: &F), FSCallsRelBFAbbrev,
4419	FSCallsProfileAbbrev, CallsiteAbbrev, AllocAbbrev, F);
4420	}
4421
4422	// Capture references from GlobalVariable initializers, which are outside
4423	// of a function scope.
4424	for (const GlobalVariable &G : M.globals())
4425	writeModuleLevelReferences(V: G, NameVals, FSModRefsAbbrev,
4426	FSModVTableRefsAbbrev);
4427
4428	for (const GlobalAlias &A : M.aliases()) {
4429	auto *Aliasee = A.getAliaseeObject();
4430	// Skip ifunc and nameless functions which don't have an entry in the
4431	// summary.
4432	if (!Aliasee->hasName() || isa<GlobalIFunc>(Val: Aliasee))
4433	continue;
4434	auto AliasId = VE.getValueID(V: &A);
4435	auto AliaseeId = VE.getValueID(V: Aliasee);
4436	NameVals.push_back(Elt: AliasId);
4437	auto *Summary = Index->getGlobalValueSummary(GV: A);
4438	AliasSummary *AS = cast<AliasSummary>(Val: Summary);
4439	NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
4440	NameVals.push_back(Elt: AliaseeId);
4441	Stream.EmitRecord(Code: bitc::FS_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
4442	NameVals.clear();
4443	}
4444
4445	for (auto &S : Index->typeIdCompatibleVtableMap()) {
4446	writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, Id: S.first,
4447	Summary: S.second, VE);
4448	Stream.EmitRecord(Code: bitc::FS_TYPE_ID_METADATA, Vals: NameVals,
4449	Abbrev: TypeIdCompatibleVtableAbbrev);
4450	NameVals.clear();
4451	}
4452
4453	if (Index->getBlockCount())
4454	Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
4455	Vals: ArrayRef<uint64_t>{Index->getBlockCount()});
4456
4457	Stream.ExitBlock();
4458	}
4459
4460	/// Emit the combined summary section into the combined index file.
4461	void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4462	Stream.EnterSubblock(BlockID: bitc::GLOBALVAL_SUMMARY_BLOCK_ID, CodeLen: 4);
4463	Stream.EmitRecord(
4464	Code: bitc::FS_VERSION,
4465	Vals: ArrayRef<uint64_t>{ModuleSummaryIndex::BitcodeSummaryVersion});
4466
4467	// Write the index flags.
4468	Stream.EmitRecord(Code: bitc::FS_FLAGS, Vals: ArrayRef<uint64_t>{Index.getFlags()});
4469
4470	for (const auto &GVI : valueIds()) {
4471	Stream.EmitRecord(Code: bitc::FS_VALUE_GUID,
4472	Vals: ArrayRef<uint64_t>{GVI.second, GVI.first});
4473	}
4474
4475	if (!StackIdIndices.empty()) {
4476	auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4477	StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_STACK_IDS));
4478	// numids x stackid
4479	StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4480	StackIdAbbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4481	unsigned StackIdAbbvId = Stream.EmitAbbrev(Abbv: std::move(StackIdAbbv));
4482	// Write the stack ids used by this index, which will be a subset of those in
4483	// the full index in the case of distributed indexes.
4484	std::vector<uint64_t> StackIds;
4485	for (auto &I : StackIdIndices)
4486	StackIds.push_back(x: Index.getStackIdAtIndex(Index: I));
4487	Stream.EmitRecord(Code: bitc::FS_STACK_IDS, Vals: StackIds, Abbrev: StackIdAbbvId);
4488	}
4489
4490	// Abbrev for FS_COMBINED_PROFILE.
4491	auto Abbv = std::make_shared<BitCodeAbbrev>();
4492	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE));
4493	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4494	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4495	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4496	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount
4497	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags
4498	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount
4499	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
4500	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt
4501	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt
4502	// numrefs x valueid, n x (valueid, hotness+tailcall flags)
4503	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4504	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4505	unsigned FSCallsProfileAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4506
4507	// Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS.
4508	Abbv = std::make_shared<BitCodeAbbrev>();
4509	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS));
4510	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4511	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4512	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4513	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids
4514	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4515	unsigned FSModRefsAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4516
4517	// Abbrev for FS_COMBINED_ALIAS.
4518	Abbv = std::make_shared<BitCodeAbbrev>();
4519	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS));
4520	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4521	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid
4522	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
4523	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4524	unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4525
4526	Abbv = std::make_shared<BitCodeAbbrev>();
4527	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_CALLSITE_INFO));
4528	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
4529	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numstackindices
4530	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4531	// numstackindices x stackidindex, numver x version
4532	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4533	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4534	unsigned CallsiteAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4535
4536	Abbv = std::make_shared<BitCodeAbbrev>();
4537	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::FS_COMBINED_ALLOC_INFO));
4538	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // nummib
4539	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numver
4540	// nummib x (alloc type, numstackids, numstackids x stackidindex),
4541	// numver x version
4542	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4543	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
4544	unsigned AllocAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4545
4546	// The aliases are emitted as a post-pass, and will point to the value
4547	// id of the aliasee. Save them in a vector for post-processing.
4548	SmallVector<AliasSummary *, 64> Aliases;
4549
4550	// Save the value id for each summary for alias emission.
4551	DenseMap<const GlobalValueSummary *, unsigned> SummaryToValueIdMap;
4552
4553	SmallVector<uint64_t, 64> NameVals;
4554
4555	// Set that will be populated during call to writeFunctionTypeMetadataRecords
4556	// with the type ids referenced by this index file.
4557	std::set<GlobalValue::GUID> ReferencedTypeIds;
4558
4559	// For local linkage, we also emit the original name separately
4560	// immediately after the record.
4561	auto MaybeEmitOriginalName = [&](GlobalValueSummary &S) {
4562	// We don't need to emit the original name if we are writing the index for
4563	// distributed backends (in which case ModuleToSummariesForIndex is
4564	// non-null). The original name is only needed during the thin link, since
4565	// for SamplePGO the indirect call targets for local functions have
4566	// have the original name annotated in profile.
4567	// Continue to emit it when writing out the entire combined index, which is
4568	// used in testing the thin link via llvm-lto.
4569	if (ModuleToSummariesForIndex || !GlobalValue::isLocalLinkage(Linkage: S.linkage()))
4570	return;
4571	NameVals.push_back(Elt: S.getOriginalName());
4572	Stream.EmitRecord(Code: bitc::FS_COMBINED_ORIGINAL_NAME, Vals: NameVals);
4573	NameVals.clear();
4574	};
4575
4576	std::set<GlobalValue::GUID> DefOrUseGUIDs;
4577	forEachSummary(Callback: [&](GVInfo I, bool IsAliasee) {
4578	GlobalValueSummary *S = I.second;
4579	assert(S);
4580	DefOrUseGUIDs.insert(x: I.first);
4581	for (const ValueInfo &VI : S->refs())
4582	DefOrUseGUIDs.insert(x: VI.getGUID());
4583
4584	auto ValueId = getValueId(ValGUID: I.first);
4585	assert(ValueId);
4586	SummaryToValueIdMap[S] = *ValueId;
4587
4588	// If this is invoked for an aliasee, we want to record the above
4589	// mapping, but then not emit a summary entry (if the aliasee is
4590	// to be imported, we will invoke this separately with IsAliasee=false).
4591	if (IsAliasee)
4592	return;
4593
4594	if (auto *AS = dyn_cast<AliasSummary>(Val: S)) {
4595	// Will process aliases as a post-pass because the reader wants all
4596	// global to be loaded first.
4597	Aliases.push_back(Elt: AS);
4598	return;
4599	}
4600
4601	if (auto *VS = dyn_cast<GlobalVarSummary>(Val: S)) {
4602	NameVals.push_back(Elt: *ValueId);
4603	assert(ModuleIdMap.count(VS->modulePath()));
4604	NameVals.push_back(Elt: ModuleIdMap[VS->modulePath()]);
4605	NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: VS->flags()));
4606	NameVals.push_back(Elt: getEncodedGVarFlags(Flags: VS->varflags()));
4607	for (auto &RI : VS->refs()) {
4608	auto RefValueId = getValueId(ValGUID: RI.getGUID());
4609	if (!RefValueId)
4610	continue;
4611	NameVals.push_back(Elt: *RefValueId);
4612	}
4613
4614	// Emit the finished record.
4615	Stream.EmitRecord(Code: bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, Vals: NameVals,
4616	Abbrev: FSModRefsAbbrev);
4617	NameVals.clear();
4618	MaybeEmitOriginalName(*S);
4619	return;
4620	}
4621
4622	auto GetValueId = [&](const ValueInfo &VI) -> std::optional<unsigned> {
4623	if (!VI)
4624	return std::nullopt;
4625	return getValueId(ValGUID: VI.getGUID());
4626	};
4627
4628	auto *FS = cast<FunctionSummary>(Val: S);
4629	writeFunctionTypeMetadataRecords(Stream, FS, GetValueID: GetValueId);
4630	getReferencedTypeIds(FS, ReferencedTypeIds);
4631
4632	writeFunctionHeapProfileRecords(
4633	Stream, FS, CallsiteAbbrev, AllocAbbrev,
4634	/*PerModule*/ false,
4635	/*GetValueId*/ GetValueID: [&](const ValueInfo &VI) -> unsigned {
4636	std::optional<unsigned> ValueID = GetValueId(VI);
4637	// This can happen in shared index files for distributed ThinLTO if
4638	// the callee function summary is not included. Record 0 which we
4639	// will have to deal with conservatively when doing any kind of
4640	// validation in the ThinLTO backends.
4641	if (!ValueID)
4642	return 0;
4643	return *ValueID;
4644	},
4645	/*GetStackIndex*/ [&](unsigned I) {
4646	// Get the corresponding index into the list of StackIdIndices
4647	// actually being written for this combined index (which may be a
4648	// subset in the case of distributed indexes).
4649	auto Lower = llvm::lower_bound(Range&: StackIdIndices, Value&: I);
4650	return std::distance(first: StackIdIndices.begin(), last: Lower);
4651	});
4652
4653	NameVals.push_back(Elt: *ValueId);
4654	assert(ModuleIdMap.count(FS->modulePath()));
4655	NameVals.push_back(Elt: ModuleIdMap[FS->modulePath()]);
4656	NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: FS->flags()));
4657	NameVals.push_back(Elt: FS->instCount());
4658	NameVals.push_back(Elt: getEncodedFFlags(Flags: FS->fflags()));
4659	NameVals.push_back(Elt: FS->entryCount());
4660
4661	// Fill in below
4662	NameVals.push_back(Elt: 0); // numrefs
4663	NameVals.push_back(Elt: 0); // rorefcnt
4664	NameVals.push_back(Elt: 0); // worefcnt
4665
4666	unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
4667	for (auto &RI : FS->refs()) {
4668	auto RefValueId = getValueId(ValGUID: RI.getGUID());
4669	if (!RefValueId)
4670	continue;
4671	NameVals.push_back(Elt: *RefValueId);
4672	if (RI.isReadOnly())
4673	RORefCnt++;
4674	else if (RI.isWriteOnly())
4675	WORefCnt++;
4676	Count++;
4677	}
4678	NameVals[6] = Count;
4679	NameVals[7] = RORefCnt;
4680	NameVals[8] = WORefCnt;
4681
4682	for (auto &EI : FS->calls()) {
4683	// If this GUID doesn't have a value id, it doesn't have a function
4684	// summary and we don't need to record any calls to it.
4685	std::optional<unsigned> CallValueId = GetValueId(EI.first);
4686	if (!CallValueId)
4687	continue;
4688	NameVals.push_back(Elt: *CallValueId);
4689	NameVals.push_back(Elt: getEncodedHotnessCallEdgeInfo(CI: EI.second));
4690	}
4691
4692	// Emit the finished record.
4693	Stream.EmitRecord(Code: bitc::FS_COMBINED_PROFILE, Vals: NameVals,
4694	Abbrev: FSCallsProfileAbbrev);
4695	NameVals.clear();
4696	MaybeEmitOriginalName(*S);
4697	});
4698
4699	for (auto *AS : Aliases) {
4700	auto AliasValueId = SummaryToValueIdMap[AS];
4701	assert(AliasValueId);
4702	NameVals.push_back(Elt: AliasValueId);
4703	assert(ModuleIdMap.count(AS->modulePath()));
4704	NameVals.push_back(Elt: ModuleIdMap[AS->modulePath()]);
4705	NameVals.push_back(Elt: getEncodedGVSummaryFlags(Flags: AS->flags()));
4706	auto AliaseeValueId = SummaryToValueIdMap[&AS->getAliasee()];
4707	assert(AliaseeValueId);
4708	NameVals.push_back(Elt: AliaseeValueId);
4709
4710	// Emit the finished record.
4711	Stream.EmitRecord(Code: bitc::FS_COMBINED_ALIAS, Vals: NameVals, Abbrev: FSAliasAbbrev);
4712	NameVals.clear();
4713	MaybeEmitOriginalName(*AS);
4714
4715	if (auto *FS = dyn_cast<FunctionSummary>(Val: &AS->getAliasee()))
4716	getReferencedTypeIds(FS, ReferencedTypeIds);
4717	}
4718
4719	if (!Index.cfiFunctionDefs().empty()) {
4720	for (auto &S : Index.cfiFunctionDefs()) {
4721	if (DefOrUseGUIDs.count(
4722	x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name: S)))) {
4723	NameVals.push_back(Elt: StrtabBuilder.add(S));
4724	NameVals.push_back(Elt: S.size());
4725	}
4726	}
4727	if (!NameVals.empty()) {
4728	Stream.EmitRecord(Code: bitc::FS_CFI_FUNCTION_DEFS, Vals: NameVals);
4729	NameVals.clear();
4730	}
4731	}
4732
4733	if (!Index.cfiFunctionDecls().empty()) {
4734	for (auto &S : Index.cfiFunctionDecls()) {
4735	if (DefOrUseGUIDs.count(
4736	x: GlobalValue::getGUID(GlobalName: GlobalValue::dropLLVMManglingEscape(Name: S)))) {
4737	NameVals.push_back(Elt: StrtabBuilder.add(S));
4738	NameVals.push_back(Elt: S.size());
4739	}
4740	}
4741	if (!NameVals.empty()) {
4742	Stream.EmitRecord(Code: bitc::FS_CFI_FUNCTION_DECLS, Vals: NameVals);
4743	NameVals.clear();
4744	}
4745	}
4746
4747	// Walk the GUIDs that were referenced, and write the
4748	// corresponding type id records.
4749	for (auto &T : ReferencedTypeIds) {
4750	auto TidIter = Index.typeIds().equal_range(x: T);
4751	for (auto It = TidIter.first; It != TidIter.second; ++It) {
4752	writeTypeIdSummaryRecord(NameVals, StrtabBuilder, Id: It->second.first,
4753	Summary: It->second.second);
4754	Stream.EmitRecord(Code: bitc::FS_TYPE_ID, Vals: NameVals);
4755	NameVals.clear();
4756	}
4757	}
4758
4759	if (Index.getBlockCount())
4760	Stream.EmitRecord(Code: bitc::FS_BLOCK_COUNT,
4761	Vals: ArrayRef<uint64_t>{Index.getBlockCount()});
4762
4763	Stream.ExitBlock();
4764	}
4765
4766	/// Create the "IDENTIFICATION_BLOCK_ID" containing a single string with the
4767	/// current llvm version, and a record for the epoch number.
4768	static void writeIdentificationBlock(BitstreamWriter &Stream) {
4769	Stream.EnterSubblock(BlockID: bitc::IDENTIFICATION_BLOCK_ID, CodeLen: 5);
4770
4771	// Write the "user readable" string identifying the bitcode producer
4772	auto Abbv = std::make_shared<BitCodeAbbrev>();
4773	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_STRING));
4774	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
4775	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Char6));
4776	auto StringAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4777	writeStringRecord(Stream, Code: bitc::IDENTIFICATION_CODE_STRING,
4778	Str: "LLVM" LLVM_VERSION_STRING, AbbrevToUse: StringAbbrev);
4779
4780	// Write the epoch version
4781	Abbv = std::make_shared<BitCodeAbbrev>();
4782	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::IDENTIFICATION_CODE_EPOCH));
4783	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6));
4784	auto EpochAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
4785	constexpr std::array<unsigned, 1> Vals = {._M_elems: {bitc::BITCODE_CURRENT_EPOCH}};
4786	Stream.EmitRecord(Code: bitc::IDENTIFICATION_CODE_EPOCH, Vals, Abbrev: EpochAbbrev);
4787	Stream.ExitBlock();
4788	}
4789
4790	void ModuleBitcodeWriter::writeModuleHash(size_t BlockStartPos) {
4791	// Emit the module's hash.
4792	// MODULE_CODE_HASH: [5*i32]
4793	if (GenerateHash) {
4794	uint32_t Vals[5];
4795	Hasher.update(Data: ArrayRef<uint8_t>((const uint8_t *)&(Buffer)[BlockStartPos],
4796	Buffer.size() - BlockStartPos));
4797	std::array<uint8_t, 20> Hash = Hasher.result();
4798	for (int Pos = 0; Pos < 20; Pos += 4) {
4799	Vals[Pos / 4] = support::endian::read32be(P: Hash.data() + Pos);
4800	}
4801
4802	// Emit the finished record.
4803	Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals);
4804
4805	if (ModHash)
4806	// Save the written hash value.
4807	llvm::copy(Range&: Vals, Out: std::begin(cont&: *ModHash));
4808	}
4809	}
4810
4811	void ModuleBitcodeWriter::write() {
4812	writeIdentificationBlock(Stream);
4813
4814	Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
4815	size_t BlockStartPos = Buffer.size();
4816
4817	writeModuleVersion();
4818
4819	// Emit blockinfo, which defines the standard abbreviations etc.
4820	writeBlockInfo();
4821
4822	// Emit information describing all of the types in the module.
4823	writeTypeTable();
4824
4825	// Emit information about attribute groups.
4826	writeAttributeGroupTable();
4827
4828	// Emit information about parameter attributes.
4829	writeAttributeTable();
4830
4831	writeComdats();
4832
4833	// Emit top-level description of module, including target triple, inline asm,
4834	// descriptors for global variables, and function prototype info.
4835	writeModuleInfo();
4836
4837	// Emit constants.
4838	writeModuleConstants();
4839
4840	// Emit metadata kind names.
4841	writeModuleMetadataKinds();
4842
4843	// Emit metadata.
4844	writeModuleMetadata();
4845
4846	// Emit module-level use-lists.
4847	if (VE.shouldPreserveUseListOrder())
4848	writeUseListBlock(F: nullptr);
4849
4850	writeOperandBundleTags();
4851	writeSyncScopeNames();
4852
4853	// Emit function bodies.
4854	DenseMap<const Function *, uint64_t> FunctionToBitcodeIndex;
4855	for (const Function &F : M)
4856	if (!F.isDeclaration())
4857	writeFunction(F, FunctionToBitcodeIndex);
4858
4859	// Need to write after the above call to WriteFunction which populates
4860	// the summary information in the index.
4861	if (Index)
4862	writePerModuleGlobalValueSummary();
4863
4864	writeGlobalValueSymbolTable(FunctionToBitcodeIndex);
4865
4866	writeModuleHash(BlockStartPos);
4867
4868	Stream.ExitBlock();
4869	}
4870
4871	static void writeInt32ToBuffer(uint32_t Value, SmallVectorImpl<char> &Buffer,
4872	uint32_t &Position) {
4873	support::endian::write32le(P: &Buffer[Position], V: Value);
4874	Position += 4;
4875	}
4876
4877	/// If generating a bc file on darwin, we have to emit a
4878	/// header and trailer to make it compatible with the system archiver. To do
4879	/// this we emit the following header, and then emit a trailer that pads the
4880	/// file out to be a multiple of 16 bytes.
4881	///
4882	/// struct bc_header {
4883	/// uint32_t Magic; // 0x0B17C0DE
4884	/// uint32_t Version; // Version, currently always 0.
4885	/// uint32_t BitcodeOffset; // Offset to traditional bitcode file.
4886	/// uint32_t BitcodeSize; // Size of traditional bitcode file.
4887	/// uint32_t CPUType; // CPU specifier.
4888	/// ... potentially more later ...
4889	/// };
4890	static void emitDarwinBCHeaderAndTrailer(SmallVectorImpl<char> &Buffer,
4891	const Triple &TT) {
4892	unsigned CPUType = ~0U;
4893
4894	// Match x86_64-*, i[3-9]86-*, powerpc-*, powerpc64-*, arm-*, thumb-*,
4895	// armv[0-9]-*, thumbv[0-9]-*, armv5te-*, or armv6t2-*. The CPUType is a magic
4896	// number from /usr/include/mach/machine.h. It is ok to reproduce the
4897	// specific constants here because they are implicitly part of the Darwin ABI.
4898	enum {
4899	DARWIN_CPU_ARCH_ABI64 = 0x01000000,
4900	DARWIN_CPU_TYPE_X86 = 7,
4901	DARWIN_CPU_TYPE_ARM = 12,
4902	DARWIN_CPU_TYPE_POWERPC = 18
4903	};
4904
4905	Triple::ArchType Arch = TT.getArch();
4906	if (Arch == Triple::x86_64)
4907	CPUType = DARWIN_CPU_TYPE_X86 | DARWIN_CPU_ARCH_ABI64;
4908	else if (Arch == Triple::x86)
4909	CPUType = DARWIN_CPU_TYPE_X86;
4910	else if (Arch == Triple::ppc)
4911	CPUType = DARWIN_CPU_TYPE_POWERPC;
4912	else if (Arch == Triple::ppc64)
4913	CPUType = DARWIN_CPU_TYPE_POWERPC | DARWIN_CPU_ARCH_ABI64;
4914	else if (Arch == Triple::arm || Arch == Triple::thumb)
4915	CPUType = DARWIN_CPU_TYPE_ARM;
4916
4917	// Traditional Bitcode starts after header.
4918	assert(Buffer.size() >= BWH_HeaderSize &&
4919	"Expected header size to be reserved");
4920	unsigned BCOffset = BWH_HeaderSize;
4921	unsigned BCSize = Buffer.size() - BWH_HeaderSize;
4922
4923	// Write the magic and version.
4924	unsigned Position = 0;
4925	writeInt32ToBuffer(Value: 0x0B17C0DE, Buffer, Position);
4926	writeInt32ToBuffer(Value: 0, Buffer, Position); // Version.
4927	writeInt32ToBuffer(Value: BCOffset, Buffer, Position);
4928	writeInt32ToBuffer(Value: BCSize, Buffer, Position);
4929	writeInt32ToBuffer(Value: CPUType, Buffer, Position);
4930
4931	// If the file is not a multiple of 16 bytes, insert dummy padding.
4932	while (Buffer.size() & 15)
4933	Buffer.push_back(Elt: 0);
4934	}
4935
4936	/// Helper to write the header common to all bitcode files.
4937	static void writeBitcodeHeader(BitstreamWriter &Stream) {
4938	// Emit the file header.
4939	Stream.Emit(Val: (unsigned)'B', NumBits: 8);
4940	Stream.Emit(Val: (unsigned)'C', NumBits: 8);
4941	Stream.Emit(Val: 0x0, NumBits: 4);
4942	Stream.Emit(Val: 0xC, NumBits: 4);
4943	Stream.Emit(Val: 0xE, NumBits: 4);
4944	Stream.Emit(Val: 0xD, NumBits: 4);
4945	}
4946
4947	BitcodeWriter::BitcodeWriter(SmallVectorImpl<char> &Buffer, raw_fd_stream *FS)
4948	: Buffer(Buffer), Stream(new BitstreamWriter(Buffer, FS, FlushThreshold)) {
4949	writeBitcodeHeader(Stream&: *Stream);
4950	}
4951
4952	BitcodeWriter::~BitcodeWriter() { assert(WroteStrtab); }
4953
4954	void BitcodeWriter::writeBlob(unsigned Block, unsigned Record, StringRef Blob) {
4955	Stream->EnterSubblock(BlockID: Block, CodeLen: 3);
4956
4957	auto Abbv = std::make_shared<BitCodeAbbrev>();
4958	Abbv->Add(OpInfo: BitCodeAbbrevOp(Record));
4959	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
4960	auto AbbrevNo = Stream->EmitAbbrev(Abbv: std::move(Abbv));
4961
4962	Stream->EmitRecordWithBlob(Abbrev: AbbrevNo, Vals: ArrayRef<uint64_t>{Record}, Blob);
4963
4964	Stream->ExitBlock();
4965	}
4966
4967	void BitcodeWriter::writeSymtab() {
4968	assert(!WroteStrtab && !WroteSymtab);
4969
4970	// If any module has module-level inline asm, we will require a registered asm
4971	// parser for the target so that we can create an accurate symbol table for
4972	// the module.
4973	for (Module *M : Mods) {
4974	if (M->getModuleInlineAsm().empty())
4975	continue;
4976
4977	std::string Err;
4978	const Triple TT(M->getTargetTriple());
4979	const Target *T = TargetRegistry::lookupTarget(Triple: TT.str(), Error&: Err);
4980	if (!T || !T->hasMCAsmParser())
4981	return;
4982	}
4983
4984	WroteSymtab = true;
4985	SmallVector<char, 0> Symtab;
4986	// The irsymtab::build function may be unable to create a symbol table if the
4987	// module is malformed (e.g. it contains an invalid alias). Writing a symbol
4988	// table is not required for correctness, but we still want to be able to
4989	// write malformed modules to bitcode files, so swallow the error.
4990	if (Error E = irsymtab::build(Mods, Symtab, StrtabBuilder, Alloc)) {
4991	consumeError(Err: std::move(E));
4992	return;
4993	}
4994
4995	writeBlob(Block: bitc::SYMTAB_BLOCK_ID, Record: bitc::SYMTAB_BLOB,
4996	Blob: {Symtab.data(), Symtab.size()});
4997	}
4998
4999	void BitcodeWriter::writeStrtab() {
5000	assert(!WroteStrtab);
5001
5002	std::vector<char> Strtab;
5003	StrtabBuilder.finalizeInOrder();
5004	Strtab.resize(new_size: StrtabBuilder.getSize());
5005	StrtabBuilder.write(Buf: (uint8_t *)Strtab.data());
5006
5007	writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB,
5008	Blob: {Strtab.data(), Strtab.size()});
5009
5010	WroteStrtab = true;
5011	}
5012
5013	void BitcodeWriter::copyStrtab(StringRef Strtab) {
5014	writeBlob(Block: bitc::STRTAB_BLOCK_ID, Record: bitc::STRTAB_BLOB, Blob: Strtab);
5015	WroteStrtab = true;
5016	}
5017
5018	void BitcodeWriter::writeModule(const Module &M,
5019	bool ShouldPreserveUseListOrder,
5020	const ModuleSummaryIndex *Index,
5021	bool GenerateHash, ModuleHash *ModHash) {
5022	assert(!WroteStrtab);
5023
5024	// The Mods vector is used by irsymtab::build, which requires non-const
5025	// Modules in case it needs to materialize metadata. But the bitcode writer
5026	// requires that the module is materialized, so we can cast to non-const here,
5027	// after checking that it is in fact materialized.
5028	assert(M.isMaterialized());
5029	Mods.push_back(x: const_cast<Module *>(&M));
5030
5031	ModuleBitcodeWriter ModuleWriter(M, Buffer, StrtabBuilder, *Stream,
5032	ShouldPreserveUseListOrder, Index,
5033	GenerateHash, ModHash);
5034	ModuleWriter.write();
5035	}
5036
5037	void BitcodeWriter::writeIndex(
5038	const ModuleSummaryIndex *Index,
5039	const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) {
5040	IndexBitcodeWriter IndexWriter(*Stream, StrtabBuilder, *Index,
5041	ModuleToSummariesForIndex);
5042	IndexWriter.write();
5043	}
5044
5045	/// Write the specified module to the specified output stream.
5046	void llvm::WriteBitcodeToFile(const Module &M, raw_ostream &Out,
5047	bool ShouldPreserveUseListOrder,
5048	const ModuleSummaryIndex *Index,
5049	bool GenerateHash, ModuleHash *ModHash) {
5050	SmallVector<char, 0> Buffer;
5051	Buffer.reserve(N: 256*1024);
5052
5053	// If this is darwin or another generic macho target, reserve space for the
5054	// header.
5055	Triple TT(M.getTargetTriple());
5056	if (TT.isOSDarwin() || TT.isOSBinFormatMachO())
5057	Buffer.insert(I: Buffer.begin(), NumToInsert: BWH_HeaderSize, Elt: 0);
5058
5059	BitcodeWriter Writer(Buffer, dyn_cast<raw_fd_stream>(Val: &Out));
5060	Writer.writeModule(M, ShouldPreserveUseListOrder, Index, GenerateHash,
5061	ModHash);
5062	Writer.writeSymtab();
5063	Writer.writeStrtab();
5064
5065	if (TT.isOSDarwin() || TT.isOSBinFormatMachO())
5066	emitDarwinBCHeaderAndTrailer(Buffer, TT);
5067
5068	// Write the generated bitstream to "Out".
5069	if (!Buffer.empty())
5070	Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5071	}
5072
5073	void IndexBitcodeWriter::write() {
5074	Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5075
5076	writeModuleVersion();
5077
5078	// Write the module paths in the combined index.
5079	writeModStrings();
5080
5081	// Write the summary combined index records.
5082	writeCombinedGlobalValueSummary();
5083
5084	Stream.ExitBlock();
5085	}
5086
5087	// Write the specified module summary index to the given raw output stream,
5088	// where it will be written in a new bitcode block. This is used when
5089	// writing the combined index file for ThinLTO. When writing a subset of the
5090	// index for a distributed backend, provide a \p ModuleToSummariesForIndex map.
5091	void llvm::writeIndexToFile(
5092	const ModuleSummaryIndex &Index, raw_ostream &Out,
5093	const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex) {
5094	SmallVector<char, 0> Buffer;
5095	Buffer.reserve(N: 256 * 1024);
5096
5097	BitcodeWriter Writer(Buffer);
5098	Writer.writeIndex(Index: &Index, ModuleToSummariesForIndex);
5099	Writer.writeStrtab();
5100
5101	Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5102	}
5103
5104	namespace {
5105
5106	/// Class to manage the bitcode writing for a thin link bitcode file.
5107	class ThinLinkBitcodeWriter : public ModuleBitcodeWriterBase {
5108	/// ModHash is for use in ThinLTO incremental build, generated while writing
5109	/// the module bitcode file.
5110	const ModuleHash *ModHash;
5111
5112	public:
5113	ThinLinkBitcodeWriter(const Module &M, StringTableBuilder &StrtabBuilder,
5114	BitstreamWriter &Stream,
5115	const ModuleSummaryIndex &Index,
5116	const ModuleHash &ModHash)
5117	: ModuleBitcodeWriterBase(M, StrtabBuilder, Stream,
5118	/*ShouldPreserveUseListOrder=*/false, &Index),
5119	ModHash(&ModHash) {}
5120
5121	void write();
5122
5123	private:
5124	void writeSimplifiedModuleInfo();
5125	};
5126
5127	} // end anonymous namespace
5128
5129	// This function writes a simpilified module info for thin link bitcode file.
5130	// It only contains the source file name along with the name(the offset and
5131	// size in strtab) and linkage for global values. For the global value info
5132	// entry, in order to keep linkage at offset 5, there are three zeros used
5133	// as padding.
5134	void ThinLinkBitcodeWriter::writeSimplifiedModuleInfo() {
5135	SmallVector<unsigned, 64> Vals;
5136	// Emit the module's source file name.
5137	{
5138	StringEncoding Bits = getStringEncoding(Str: M.getSourceFileName());
5139	BitCodeAbbrevOp AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 8);
5140	if (Bits == SE_Char6)
5141	AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Char6);
5142	else if (Bits == SE_Fixed7)
5143	AbbrevOpToUse = BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 7);
5144
5145	// MODULE_CODE_SOURCE_FILENAME: [namechar x N]
5146	auto Abbv = std::make_shared<BitCodeAbbrev>();
5147	Abbv->Add(OpInfo: BitCodeAbbrevOp(bitc::MODULE_CODE_SOURCE_FILENAME));
5148	Abbv->Add(OpInfo: BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
5149	Abbv->Add(OpInfo: AbbrevOpToUse);
5150	unsigned FilenameAbbrev = Stream.EmitAbbrev(Abbv: std::move(Abbv));
5151
5152	for (const auto P : M.getSourceFileName())
5153	Vals.push_back(Elt: (unsigned char)P);
5154
5155	Stream.EmitRecord(Code: bitc::MODULE_CODE_SOURCE_FILENAME, Vals, Abbrev: FilenameAbbrev);
5156	Vals.clear();
5157	}
5158
5159	// Emit the global variable information.
5160	for (const GlobalVariable &GV : M.globals()) {
5161	// GLOBALVAR: [strtab offset, strtab size, 0, 0, 0, linkage]
5162	Vals.push_back(Elt: StrtabBuilder.add(S: GV.getName()));
5163	Vals.push_back(Elt: GV.getName().size());
5164	Vals.push_back(Elt: 0);
5165	Vals.push_back(Elt: 0);
5166	Vals.push_back(Elt: 0);
5167	Vals.push_back(Elt: getEncodedLinkage(GV));
5168
5169	Stream.EmitRecord(Code: bitc::MODULE_CODE_GLOBALVAR, Vals);
5170	Vals.clear();
5171	}
5172
5173	// Emit the function proto information.
5174	for (const Function &F : M) {
5175	// FUNCTION: [strtab offset, strtab size, 0, 0, 0, linkage]
5176	Vals.push_back(Elt: StrtabBuilder.add(S: F.getName()));
5177	Vals.push_back(Elt: F.getName().size());
5178	Vals.push_back(Elt: 0);
5179	Vals.push_back(Elt: 0);
5180	Vals.push_back(Elt: 0);
5181	Vals.push_back(Elt: getEncodedLinkage(GV: F));
5182
5183	Stream.EmitRecord(Code: bitc::MODULE_CODE_FUNCTION, Vals);
5184	Vals.clear();
5185	}
5186
5187	// Emit the alias information.
5188	for (const GlobalAlias &A : M.aliases()) {
5189	// ALIAS: [strtab offset, strtab size, 0, 0, 0, linkage]
5190	Vals.push_back(Elt: StrtabBuilder.add(S: A.getName()));
5191	Vals.push_back(Elt: A.getName().size());
5192	Vals.push_back(Elt: 0);
5193	Vals.push_back(Elt: 0);
5194	Vals.push_back(Elt: 0);
5195	Vals.push_back(Elt: getEncodedLinkage(GV: A));
5196
5197	Stream.EmitRecord(Code: bitc::MODULE_CODE_ALIAS, Vals);
5198	Vals.clear();
5199	}
5200
5201	// Emit the ifunc information.
5202	for (const GlobalIFunc &I : M.ifuncs()) {
5203	// IFUNC: [strtab offset, strtab size, 0, 0, 0, linkage]
5204	Vals.push_back(Elt: StrtabBuilder.add(S: I.getName()));
5205	Vals.push_back(Elt: I.getName().size());
5206	Vals.push_back(Elt: 0);
5207	Vals.push_back(Elt: 0);
5208	Vals.push_back(Elt: 0);
5209	Vals.push_back(Elt: getEncodedLinkage(GV: I));
5210
5211	Stream.EmitRecord(Code: bitc::MODULE_CODE_IFUNC, Vals);
5212	Vals.clear();
5213	}
5214	}
5215
5216	void ThinLinkBitcodeWriter::write() {
5217	Stream.EnterSubblock(BlockID: bitc::MODULE_BLOCK_ID, CodeLen: 3);
5218
5219	writeModuleVersion();
5220
5221	writeSimplifiedModuleInfo();
5222
5223	writePerModuleGlobalValueSummary();
5224
5225	// Write module hash.
5226	Stream.EmitRecord(Code: bitc::MODULE_CODE_HASH, Vals: ArrayRef<uint32_t>(*ModHash));
5227
5228	Stream.ExitBlock();
5229	}
5230
5231	void BitcodeWriter::writeThinLinkBitcode(const Module &M,
5232	const ModuleSummaryIndex &Index,
5233	const ModuleHash &ModHash) {
5234	assert(!WroteStrtab);
5235
5236	// The Mods vector is used by irsymtab::build, which requires non-const
5237	// Modules in case it needs to materialize metadata. But the bitcode writer
5238	// requires that the module is materialized, so we can cast to non-const here,
5239	// after checking that it is in fact materialized.
5240	assert(M.isMaterialized());
5241	Mods.push_back(x: const_cast<Module *>(&M));
5242
5243	ThinLinkBitcodeWriter ThinLinkWriter(M, StrtabBuilder, *Stream, Index,
5244	ModHash);
5245	ThinLinkWriter.write();
5246	}
5247
5248	// Write the specified thin link bitcode file to the given raw output stream,
5249	// where it will be written in a new bitcode block. This is used when
5250	// writing the per-module index file for ThinLTO.
5251	void llvm::writeThinLinkBitcodeToFile(const Module &M, raw_ostream &Out,
5252	const ModuleSummaryIndex &Index,
5253	const ModuleHash &ModHash) {
5254	SmallVector<char, 0> Buffer;
5255	Buffer.reserve(N: 256 * 1024);
5256
5257	BitcodeWriter Writer(Buffer);
5258	Writer.writeThinLinkBitcode(M, Index, ModHash);
5259	Writer.writeSymtab();
5260	Writer.writeStrtab();
5261
5262	Out.write(Ptr: (char *)&Buffer.front(), Size: Buffer.size());
5263	}
5264
5265	static const char *getSectionNameForBitcode(const Triple &T) {
5266	switch (T.getObjectFormat()) {
5267	case Triple::MachO:
5268	return "__LLVM,__bitcode";
5269	case Triple::COFF:
5270	case Triple::ELF:
5271	case Triple::Wasm:
5272	case Triple::UnknownObjectFormat:
5273	return ".llvmbc";
5274	case Triple::GOFF:
5275	llvm_unreachable("GOFF is not yet implemented");
5276	break;
5277	case Triple::SPIRV:
5278	llvm_unreachable("SPIRV is not yet implemented");
5279	break;
5280	case Triple::XCOFF:
5281	llvm_unreachable("XCOFF is not yet implemented");
5282	break;
5283	case Triple::DXContainer:
5284	llvm_unreachable("DXContainer is not yet implemented");
5285	break;
5286	}
5287	llvm_unreachable("Unimplemented ObjectFormatType");
5288	}
5289
5290	static const char *getSectionNameForCommandline(const Triple &T) {
5291	switch (T.getObjectFormat()) {
5292	case Triple::MachO:
5293	return "__LLVM,__cmdline";
5294	case Triple::COFF:
5295	case Triple::ELF:
5296	case Triple::Wasm:
5297	case Triple::UnknownObjectFormat:
5298	return ".llvmcmd";
5299	case Triple::GOFF:
5300	llvm_unreachable("GOFF is not yet implemented");
5301	break;
5302	case Triple::SPIRV:
5303	llvm_unreachable("SPIRV is not yet implemented");
5304	break;
5305	case Triple::XCOFF:
5306	llvm_unreachable("XCOFF is not yet implemented");
5307	break;
5308	case Triple::DXContainer:
5309	llvm_unreachable("DXC is not yet implemented");
5310	break;
5311	}
5312	llvm_unreachable("Unimplemented ObjectFormatType");
5313	}
5314
5315	void llvm::embedBitcodeInModule(llvm::Module &M, llvm::MemoryBufferRef Buf,
5316	bool EmbedBitcode, bool EmbedCmdline,
5317	const std::vector<uint8_t> &CmdArgs) {
5318	// Save llvm.compiler.used and remove it.
5319	SmallVector<Constant *, 2> UsedArray;
5320	SmallVector<GlobalValue *, 4> UsedGlobals;
5321	Type *UsedElementType = PointerType::getUnqual(C&: M.getContext());
5322	GlobalVariable *Used = collectUsedGlobalVariables(M, Vec&: UsedGlobals, CompilerUsed: true);
5323	for (auto *GV : UsedGlobals) {
5324	if (GV->getName() != "llvm.embedded.module" &&
5325	GV->getName() != "llvm.cmdline")
5326	UsedArray.push_back(
5327	Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5328	}
5329	if (Used)
5330	Used->eraseFromParent();
5331
5332	// Embed the bitcode for the llvm module.
5333	std::string Data;
5334	ArrayRef<uint8_t> ModuleData;
5335	Triple T(M.getTargetTriple());
5336
5337	if (EmbedBitcode) {
5338	if (Buf.getBufferSize() == 0 ||
5339	!isBitcode(BufPtr: (const unsigned char *)Buf.getBufferStart(),
5340	BufEnd: (const unsigned char *)Buf.getBufferEnd())) {
5341	// If the input is LLVM Assembly, bitcode is produced by serializing
5342	// the module. Use-lists order need to be preserved in this case.
5343	llvm::raw_string_ostream OS(Data);
5344	llvm::WriteBitcodeToFile(M, Out&: OS, /* ShouldPreserveUseListOrder */ true);
5345	ModuleData =
5346	ArrayRef<uint8_t>((const uint8_t *)OS.str().data(), OS.str().size());
5347	} else
5348	// If the input is LLVM bitcode, write the input byte stream directly.
5349	ModuleData = ArrayRef<uint8_t>((const uint8_t *)Buf.getBufferStart(),
5350	Buf.getBufferSize());
5351	}
5352	llvm::Constant *ModuleConstant =
5353	llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: ModuleData);
5354	llvm::GlobalVariable *GV = new llvm::GlobalVariable(
5355	M, ModuleConstant->getType(), true, llvm::GlobalValue::PrivateLinkage,
5356	ModuleConstant);
5357	GV->setSection(getSectionNameForBitcode(T));
5358	// Set alignment to 1 to prevent padding between two contributions from input
5359	// sections after linking.
5360	GV->setAlignment(Align(1));
5361	UsedArray.push_back(
5362	Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5363	if (llvm::GlobalVariable *Old =
5364	M.getGlobalVariable(Name: "llvm.embedded.module", AllowInternal: true)) {
5365	assert(Old->hasZeroLiveUses() &&
5366	"llvm.embedded.module can only be used once in llvm.compiler.used");
5367	GV->takeName(V: Old);
5368	Old->eraseFromParent();
5369	} else {
5370	GV->setName("llvm.embedded.module");
5371	}
5372
5373	// Skip if only bitcode needs to be embedded.
5374	if (EmbedCmdline) {
5375	// Embed command-line options.
5376	ArrayRef<uint8_t> CmdData(const_cast<uint8_t *>(CmdArgs.data()),
5377	CmdArgs.size());
5378	llvm::Constant *CmdConstant =
5379	llvm::ConstantDataArray::get(Context&: M.getContext(), Elts: CmdData);
5380	GV = new llvm::GlobalVariable(M, CmdConstant->getType(), true,
5381	llvm::GlobalValue::PrivateLinkage,
5382	CmdConstant);
5383	GV->setSection(getSectionNameForCommandline(T));
5384	GV->setAlignment(Align(1));
5385	UsedArray.push_back(
5386	Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast(C: GV, Ty: UsedElementType));
5387	if (llvm::GlobalVariable *Old = M.getGlobalVariable(Name: "llvm.cmdline", AllowInternal: true)) {
5388	assert(Old->hasZeroLiveUses() &&
5389	"llvm.cmdline can only be used once in llvm.compiler.used");
5390	GV->takeName(V: Old);
5391	Old->eraseFromParent();
5392	} else {
5393	GV->setName("llvm.cmdline");
5394	}
5395	}
5396
5397	if (UsedArray.empty())
5398	return;
5399
5400	// Recreate llvm.compiler.used.
5401	ArrayType *ATy = ArrayType::get(ElementType: UsedElementType, NumElements: UsedArray.size());
5402	auto *NewUsed = new GlobalVariable(
5403	M, ATy, false, llvm::GlobalValue::AppendingLinkage,
5404	llvm::ConstantArray::get(T: ATy, V: UsedArray), "llvm.compiler.used");
5405	NewUsed->setSection("llvm.metadata");
5406	}
5407