1	//===--- CGExprConstant.cpp - Emit LLVM Code from Constant Expressions ----===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code to emit Constant Expr nodes as LLVM code.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGCXXABI.h"
14	#include "CGObjCRuntime.h"
15	#include "CGRecordLayout.h"
16	#include "CodeGenFunction.h"
17	#include "CodeGenModule.h"
18	#include "ConstantEmitter.h"
19	#include "TargetInfo.h"
20	#include "clang/AST/APValue.h"
21	#include "clang/AST/ASTContext.h"
22	#include "clang/AST/Attr.h"
23	#include "clang/AST/RecordLayout.h"
24	#include "clang/AST/StmtVisitor.h"
25	#include "clang/Basic/Builtins.h"
26	#include "llvm/ADT/STLExtras.h"
27	#include "llvm/ADT/Sequence.h"
28	#include "llvm/Analysis/ConstantFolding.h"
29	#include "llvm/IR/Constants.h"
30	#include "llvm/IR/DataLayout.h"
31	#include "llvm/IR/Function.h"
32	#include "llvm/IR/GlobalVariable.h"
33	#include <optional>
34	using namespace clang;
35	using namespace CodeGen;
36
37	//===----------------------------------------------------------------------===//
38	// ConstantAggregateBuilder
39	//===----------------------------------------------------------------------===//
40
41	namespace {
42	class ConstExprEmitter;
43
44	struct ConstantAggregateBuilderUtils {
45	CodeGenModule &CGM;
46
47	ConstantAggregateBuilderUtils(CodeGenModule &CGM) : CGM(CGM) {}
48
49	CharUnits getAlignment(const llvm::Constant *C) const {
50	return CharUnits::fromQuantity(
51	Quantity: CGM.getDataLayout().getABITypeAlign(Ty: C->getType()));
52	}
53
54	CharUnits getSize(llvm::Type *Ty) const {
55	return CharUnits::fromQuantity(Quantity: CGM.getDataLayout().getTypeAllocSize(Ty));
56	}
57
58	CharUnits getSize(const llvm::Constant *C) const {
59	return getSize(Ty: C->getType());
60	}
61
62	llvm::Constant *getPadding(CharUnits PadSize) const {
63	llvm::Type *Ty = CGM.CharTy;
64	if (PadSize > CharUnits::One())
65	Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: PadSize.getQuantity());
66	return llvm::UndefValue::get(T: Ty);
67	}
68
69	llvm::Constant *getZeroes(CharUnits ZeroSize) const {
70	llvm::Type *Ty = llvm::ArrayType::get(ElementType: CGM.CharTy, NumElements: ZeroSize.getQuantity());
71	return llvm::ConstantAggregateZero::get(Ty);
72	}
73	};
74
75	/// Incremental builder for an llvm::Constant* holding a struct or array
76	/// constant.
77	class ConstantAggregateBuilder : private ConstantAggregateBuilderUtils {
78	/// The elements of the constant. These two arrays must have the same size;
79	/// Offsets[i] describes the offset of Elems[i] within the constant. The
80	/// elements are kept in increasing offset order, and we ensure that there
81	/// is no overlap: Offsets[i+1] >= Offsets[i] + getSize(Elemes[i]).
82	///
83	/// This may contain explicit padding elements (in order to create a
84	/// natural layout), but need not. Gaps between elements are implicitly
85	/// considered to be filled with undef.
86	llvm::SmallVector<llvm::Constant*, 32> Elems;
87	llvm::SmallVector<CharUnits, 32> Offsets;
88
89	/// The size of the constant (the maximum end offset of any added element).
90	/// May be larger than the end of Elems.back() if we split the last element
91	/// and removed some trailing undefs.
92	CharUnits Size = CharUnits::Zero();
93
94	/// This is true only if laying out Elems in order as the elements of a
95	/// non-packed LLVM struct will give the correct layout.
96	bool NaturalLayout = true;
97
98	bool split(size_t Index, CharUnits Hint);
99	std::optional<size_t> splitAt(CharUnits Pos);
100
101	static llvm::Constant *buildFrom(CodeGenModule &CGM,
102	ArrayRef<llvm::Constant *> Elems,
103	ArrayRef<CharUnits> Offsets,
104	CharUnits StartOffset, CharUnits Size,
105	bool NaturalLayout, llvm::Type *DesiredTy,
106	bool AllowOversized);
107
108	public:
109	ConstantAggregateBuilder(CodeGenModule &CGM)
110	: ConstantAggregateBuilderUtils(CGM) {}
111
112	/// Update or overwrite the value starting at \p Offset with \c C.
113	///
114	/// \param AllowOverwrite If \c true, this constant might overwrite (part of)
115	/// a constant that has already been added. This flag is only used to
116	/// detect bugs.
117	bool add(llvm::Constant *C, CharUnits Offset, bool AllowOverwrite);
118
119	/// Update or overwrite the bits starting at \p OffsetInBits with \p Bits.
120	bool addBits(llvm::APInt Bits, uint64_t OffsetInBits, bool AllowOverwrite);
121
122	/// Attempt to condense the value starting at \p Offset to a constant of type
123	/// \p DesiredTy.
124	void condense(CharUnits Offset, llvm::Type *DesiredTy);
125
126	/// Produce a constant representing the entire accumulated value, ideally of
127	/// the specified type. If \p AllowOversized, the constant might be larger
128	/// than implied by \p DesiredTy (eg, if there is a flexible array member).
129	/// Otherwise, the constant will be of exactly the same size as \p DesiredTy
130	/// even if we can't represent it as that type.
131	llvm::Constant *build(llvm::Type *DesiredTy, bool AllowOversized) const {
132	return buildFrom(CGM, Elems, Offsets, StartOffset: CharUnits::Zero(), Size,
133	NaturalLayout, DesiredTy, AllowOversized);
134	}
135	};
136
137	template<typename Container, typename Range = std::initializer_list<
138	typename Container::value_type>>
139	static void replace(Container &C, size_t BeginOff, size_t EndOff, Range Vals) {
140	assert(BeginOff <= EndOff && "invalid replacement range");
141	llvm::replace(C, C.begin() + BeginOff, C.begin() + EndOff, Vals);
142	}
143
144	bool ConstantAggregateBuilder::add(llvm::Constant *C, CharUnits Offset,
145	bool AllowOverwrite) {
146	// Common case: appending to a layout.
147	if (Offset >= Size) {
148	CharUnits Align = getAlignment(C);
149	CharUnits AlignedSize = Size.alignTo(Align);
150	if (AlignedSize > Offset || Offset.alignTo(Align) != Offset)
151	NaturalLayout = false;
152	else if (AlignedSize < Offset) {
153	Elems.push_back(Elt: getPadding(PadSize: Offset - Size));
154	Offsets.push_back(Elt: Size);
155	}
156	Elems.push_back(Elt: C);
157	Offsets.push_back(Elt: Offset);
158	Size = Offset + getSize(C);
159	return true;
160	}
161
162	// Uncommon case: constant overlaps what we've already created.
163	std::optional<size_t> FirstElemToReplace = splitAt(Pos: Offset);
164	if (!FirstElemToReplace)
165	return false;
166
167	CharUnits CSize = getSize(C);
168	std::optional<size_t> LastElemToReplace = splitAt(Pos: Offset + CSize);
169	if (!LastElemToReplace)
170	return false;
171
172	assert((FirstElemToReplace == LastElemToReplace || AllowOverwrite) &&
173	"unexpectedly overwriting field");
174
175	replace(C&: Elems, BeginOff: *FirstElemToReplace, EndOff: *LastElemToReplace, Vals: {C});
176	replace(C&: Offsets, BeginOff: *FirstElemToReplace, EndOff: *LastElemToReplace, Vals: {Offset});
177	Size = std::max(a: Size, b: Offset + CSize);
178	NaturalLayout = false;
179	return true;
180	}
181
182	bool ConstantAggregateBuilder::addBits(llvm::APInt Bits, uint64_t OffsetInBits,
183	bool AllowOverwrite) {
184	const ASTContext &Context = CGM.getContext();
185	const uint64_t CharWidth = CGM.getContext().getCharWidth();
186
187	// Offset of where we want the first bit to go within the bits of the
188	// current char.
189	unsigned OffsetWithinChar = OffsetInBits % CharWidth;
190
191	// We split bit-fields up into individual bytes. Walk over the bytes and
192	// update them.
193	for (CharUnits OffsetInChars =
194	Context.toCharUnitsFromBits(BitSize: OffsetInBits - OffsetWithinChar);
195	/**/; ++OffsetInChars) {
196	// Number of bits we want to fill in this char.
197	unsigned WantedBits =
198	std::min(a: (uint64_t)Bits.getBitWidth(), b: CharWidth - OffsetWithinChar);
199
200	// Get a char containing the bits we want in the right places. The other
201	// bits have unspecified values.
202	llvm::APInt BitsThisChar = Bits;
203	if (BitsThisChar.getBitWidth() < CharWidth)
204	BitsThisChar = BitsThisChar.zext(width: CharWidth);
205	if (CGM.getDataLayout().isBigEndian()) {
206	// Figure out how much to shift by. We may need to left-shift if we have
207	// less than one byte of Bits left.
208	int Shift = Bits.getBitWidth() - CharWidth + OffsetWithinChar;
209	if (Shift > 0)
210	BitsThisChar.lshrInPlace(ShiftAmt: Shift);
211	else if (Shift < 0)
212	BitsThisChar = BitsThisChar.shl(shiftAmt: -Shift);
213	} else {
214	BitsThisChar = BitsThisChar.shl(shiftAmt: OffsetWithinChar);
215	}
216	if (BitsThisChar.getBitWidth() > CharWidth)
217	BitsThisChar = BitsThisChar.trunc(width: CharWidth);
218
219	if (WantedBits == CharWidth) {
220	// Got a full byte: just add it directly.
221	add(C: llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar),
222	Offset: OffsetInChars, AllowOverwrite);
223	} else {
224	// Partial byte: update the existing integer if there is one. If we
225	// can't split out a 1-CharUnit range to update, then we can't add
226	// these bits and fail the entire constant emission.
227	std::optional<size_t> FirstElemToUpdate = splitAt(Pos: OffsetInChars);
228	if (!FirstElemToUpdate)
229	return false;
230	std::optional<size_t> LastElemToUpdate =
231	splitAt(Pos: OffsetInChars + CharUnits::One());
232	if (!LastElemToUpdate)
233	return false;
234	assert(*LastElemToUpdate - *FirstElemToUpdate < 2 &&
235	"should have at most one element covering one byte");
236
237	// Figure out which bits we want and discard the rest.
238	llvm::APInt UpdateMask(CharWidth, 0);
239	if (CGM.getDataLayout().isBigEndian())
240	UpdateMask.setBits(loBit: CharWidth - OffsetWithinChar - WantedBits,
241	hiBit: CharWidth - OffsetWithinChar);
242	else
243	UpdateMask.setBits(loBit: OffsetWithinChar, hiBit: OffsetWithinChar + WantedBits);
244	BitsThisChar &= UpdateMask;
245
246	if (*FirstElemToUpdate == *LastElemToUpdate ||
247	Elems[*FirstElemToUpdate]->isNullValue() ||
248	isa<llvm::UndefValue>(Val: Elems[*FirstElemToUpdate])) {
249	// All existing bits are either zero or undef.
250	add(C: llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar),
251	Offset: OffsetInChars, /*AllowOverwrite*/ true);
252	} else {
253	llvm::Constant *&ToUpdate = Elems[*FirstElemToUpdate];
254	// In order to perform a partial update, we need the existing bitwise
255	// value, which we can only extract for a constant int.
256	auto *CI = dyn_cast<llvm::ConstantInt>(Val: ToUpdate);
257	if (!CI)
258	return false;
259	// Because this is a 1-CharUnit range, the constant occupying it must
260	// be exactly one CharUnit wide.
261	assert(CI->getBitWidth() == CharWidth && "splitAt failed");
262	assert((!(CI->getValue() & UpdateMask) || AllowOverwrite) &&
263	"unexpectedly overwriting bitfield");
264	BitsThisChar |= (CI->getValue() & ~UpdateMask);
265	ToUpdate = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: BitsThisChar);
266	}
267	}
268
269	// Stop if we've added all the bits.
270	if (WantedBits == Bits.getBitWidth())
271	break;
272
273	// Remove the consumed bits from Bits.
274	if (!CGM.getDataLayout().isBigEndian())
275	Bits.lshrInPlace(ShiftAmt: WantedBits);
276	Bits = Bits.trunc(width: Bits.getBitWidth() - WantedBits);
277
278	// The remanining bits go at the start of the following bytes.
279	OffsetWithinChar = 0;
280	}
281
282	return true;
283	}
284
285	/// Returns a position within Elems and Offsets such that all elements
286	/// before the returned index end before Pos and all elements at or after
287	/// the returned index begin at or after Pos. Splits elements as necessary
288	/// to ensure this. Returns std::nullopt if we find something we can't split.
289	std::optional<size_t> ConstantAggregateBuilder::splitAt(CharUnits Pos) {
290	if (Pos >= Size)
291	return Offsets.size();
292
293	while (true) {
294	auto FirstAfterPos = llvm::upper_bound(Range&: Offsets, Value&: Pos);
295	if (FirstAfterPos == Offsets.begin())
296	return 0;
297
298	// If we already have an element starting at Pos, we're done.
299	size_t LastAtOrBeforePosIndex = FirstAfterPos - Offsets.begin() - 1;
300	if (Offsets[LastAtOrBeforePosIndex] == Pos)
301	return LastAtOrBeforePosIndex;
302
303	// We found an element starting before Pos. Check for overlap.
304	if (Offsets[LastAtOrBeforePosIndex] +
305	getSize(C: Elems[LastAtOrBeforePosIndex]) <= Pos)
306	return LastAtOrBeforePosIndex + 1;
307
308	// Try to decompose it into smaller constants.
309	if (!split(Index: LastAtOrBeforePosIndex, Hint: Pos))
310	return std::nullopt;
311	}
312	}
313
314	/// Split the constant at index Index, if possible. Return true if we did.
315	/// Hint indicates the location at which we'd like to split, but may be
316	/// ignored.
317	bool ConstantAggregateBuilder::split(size_t Index, CharUnits Hint) {
318	NaturalLayout = false;
319	llvm::Constant *C = Elems[Index];
320	CharUnits Offset = Offsets[Index];
321
322	if (auto *CA = dyn_cast<llvm::ConstantAggregate>(Val: C)) {
323	// Expand the sequence into its contained elements.
324	// FIXME: This assumes vector elements are byte-sized.
325	replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
326	Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CA->getNumOperands()),
327	F: [&](unsigned Op) { return CA->getOperand(i_nocapture: Op); }));
328	if (isa<llvm::ArrayType>(Val: CA->getType()) ||
329	isa<llvm::VectorType>(Val: CA->getType())) {
330	// Array or vector.
331	llvm::Type *ElemTy =
332	llvm::GetElementPtrInst::getTypeAtIndex(Ty: CA->getType(), Idx: (uint64_t)0);
333	CharUnits ElemSize = getSize(Ty: ElemTy);
334	replace(
335	C&: Offsets, BeginOff: Index, EndOff: Index + 1,
336	Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CA->getNumOperands()),
337	F: [&](unsigned Op) { return Offset + Op * ElemSize; }));
338	} else {
339	// Must be a struct.
340	auto *ST = cast<llvm::StructType>(Val: CA->getType());
341	const llvm::StructLayout *Layout =
342	CGM.getDataLayout().getStructLayout(Ty: ST);
343	replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1,
344	Vals: llvm::map_range(
345	C: llvm::seq(Begin: 0u, End: CA->getNumOperands()), F: [&](unsigned Op) {
346	return Offset + CharUnits::fromQuantity(
347	Quantity: Layout->getElementOffset(Idx: Op));
348	}));
349	}
350	return true;
351	}
352
353	if (auto *CDS = dyn_cast<llvm::ConstantDataSequential>(Val: C)) {
354	// Expand the sequence into its contained elements.
355	// FIXME: This assumes vector elements are byte-sized.
356	// FIXME: If possible, split into two ConstantDataSequentials at Hint.
357	CharUnits ElemSize = getSize(Ty: CDS->getElementType());
358	replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
359	Vals: llvm::map_range(C: llvm::seq(Begin: 0u, End: CDS->getNumElements()),
360	F: [&](unsigned Elem) {
361	return CDS->getElementAsConstant(i: Elem);
362	}));
363	replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1,
364	Vals: llvm::map_range(
365	C: llvm::seq(Begin: 0u, End: CDS->getNumElements()),
366	F: [&](unsigned Elem) { return Offset + Elem * ElemSize; }));
367	return true;
368	}
369
370	if (isa<llvm::ConstantAggregateZero>(Val: C)) {
371	// Split into two zeros at the hinted offset.
372	CharUnits ElemSize = getSize(C);
373	assert(Hint > Offset && Hint < Offset + ElemSize && "nothing to split");
374	replace(C&: Elems, BeginOff: Index, EndOff: Index + 1,
375	Vals: {getZeroes(ZeroSize: Hint - Offset), getZeroes(ZeroSize: Offset + ElemSize - Hint)});
376	replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1, Vals: {Offset, Hint});
377	return true;
378	}
379
380	if (isa<llvm::UndefValue>(Val: C)) {
381	// Drop undef; it doesn't contribute to the final layout.
382	replace(C&: Elems, BeginOff: Index, EndOff: Index + 1, Vals: {});
383	replace(C&: Offsets, BeginOff: Index, EndOff: Index + 1, Vals: {});
384	return true;
385	}
386
387	// FIXME: We could split a ConstantInt if the need ever arose.
388	// We don't need to do this to handle bit-fields because we always eagerly
389	// split them into 1-byte chunks.
390
391	return false;
392	}
393
394	static llvm::Constant *
395	EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
396	llvm::Type *CommonElementType, unsigned ArrayBound,
397	SmallVectorImpl<llvm::Constant *> &Elements,
398	llvm::Constant *Filler);
399
400	llvm::Constant *ConstantAggregateBuilder::buildFrom(
401	CodeGenModule &CGM, ArrayRef<llvm::Constant *> Elems,
402	ArrayRef<CharUnits> Offsets, CharUnits StartOffset, CharUnits Size,
403	bool NaturalLayout, llvm::Type *DesiredTy, bool AllowOversized) {
404	ConstantAggregateBuilderUtils Utils(CGM);
405
406	if (Elems.empty())
407	return llvm::UndefValue::get(T: DesiredTy);
408
409	auto Offset = [&](size_t I) { return Offsets[I] - StartOffset; };
410
411	// If we want an array type, see if all the elements are the same type and
412	// appropriately spaced.
413	if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(Val: DesiredTy)) {
414	assert(!AllowOversized && "oversized array emission not supported");
415
416	bool CanEmitArray = true;
417	llvm::Type *CommonType = Elems[0]->getType();
418	llvm::Constant *Filler = llvm::Constant::getNullValue(Ty: CommonType);
419	CharUnits ElemSize = Utils.getSize(Ty: ATy->getElementType());
420	SmallVector<llvm::Constant*, 32> ArrayElements;
421	for (size_t I = 0; I != Elems.size(); ++I) {
422	// Skip zeroes; we'll use a zero value as our array filler.
423	if (Elems[I]->isNullValue())
424	continue;
425
426	// All remaining elements must be the same type.
427	if (Elems[I]->getType() != CommonType ||
428	Offset(I) % ElemSize != 0) {
429	CanEmitArray = false;
430	break;
431	}
432	ArrayElements.resize(N: Offset(I) / ElemSize + 1, NV: Filler);
433	ArrayElements.back() = Elems[I];
434	}
435
436	if (CanEmitArray) {
437	return EmitArrayConstant(CGM, DesiredType: ATy, CommonElementType: CommonType, ArrayBound: ATy->getNumElements(),
438	Elements&: ArrayElements, Filler);
439	}
440
441	// Can't emit as an array, carry on to emit as a struct.
442	}
443
444	// The size of the constant we plan to generate. This is usually just
445	// the size of the initialized type, but in AllowOversized mode (i.e.
446	// flexible array init), it can be larger.
447	CharUnits DesiredSize = Utils.getSize(Ty: DesiredTy);
448	if (Size > DesiredSize) {
449	assert(AllowOversized && "Elems are oversized");
450	DesiredSize = Size;
451	}
452
453	// The natural alignment of an unpacked LLVM struct with the given elements.
454	CharUnits Align = CharUnits::One();
455	for (llvm::Constant *C : Elems)
456	Align = std::max(a: Align, b: Utils.getAlignment(C));
457
458	// The natural size of an unpacked LLVM struct with the given elements.
459	CharUnits AlignedSize = Size.alignTo(Align);
460
461	bool Packed = false;
462	ArrayRef<llvm::Constant*> UnpackedElems = Elems;
463	llvm::SmallVector<llvm::Constant*, 32> UnpackedElemStorage;
464	if (DesiredSize < AlignedSize || DesiredSize.alignTo(Align) != DesiredSize) {
465	// The natural layout would be too big; force use of a packed layout.
466	NaturalLayout = false;
467	Packed = true;
468	} else if (DesiredSize > AlignedSize) {
469	// The natural layout would be too small. Add padding to fix it. (This
470	// is ignored if we choose a packed layout.)
471	UnpackedElemStorage.assign(in_start: Elems.begin(), in_end: Elems.end());
472	UnpackedElemStorage.push_back(Elt: Utils.getPadding(PadSize: DesiredSize - Size));
473	UnpackedElems = UnpackedElemStorage;
474	}
475
476	// If we don't have a natural layout, insert padding as necessary.
477	// As we go, double-check to see if we can actually just emit Elems
478	// as a non-packed struct and do so opportunistically if possible.
479	llvm::SmallVector<llvm::Constant*, 32> PackedElems;
480	if (!NaturalLayout) {
481	CharUnits SizeSoFar = CharUnits::Zero();
482	for (size_t I = 0; I != Elems.size(); ++I) {
483	CharUnits Align = Utils.getAlignment(C: Elems[I]);
484	CharUnits NaturalOffset = SizeSoFar.alignTo(Align);
485	CharUnits DesiredOffset = Offset(I);
486	assert(DesiredOffset >= SizeSoFar && "elements out of order");
487
488	if (DesiredOffset != NaturalOffset)
489	Packed = true;
490	if (DesiredOffset != SizeSoFar)
491	PackedElems.push_back(Elt: Utils.getPadding(PadSize: DesiredOffset - SizeSoFar));
492	PackedElems.push_back(Elt: Elems[I]);
493	SizeSoFar = DesiredOffset + Utils.getSize(C: Elems[I]);
494	}
495	// If we're using the packed layout, pad it out to the desired size if
496	// necessary.
497	if (Packed) {
498	assert(SizeSoFar <= DesiredSize &&
499	"requested size is too small for contents");
500	if (SizeSoFar < DesiredSize)
501	PackedElems.push_back(Elt: Utils.getPadding(PadSize: DesiredSize - SizeSoFar));
502	}
503	}
504
505	llvm::StructType *STy = llvm::ConstantStruct::getTypeForElements(
506	Ctx&: CGM.getLLVMContext(), V: Packed ? PackedElems : UnpackedElems, Packed);
507
508	// Pick the type to use. If the type is layout identical to the desired
509	// type then use it, otherwise use whatever the builder produced for us.
510	if (llvm::StructType *DesiredSTy = dyn_cast<llvm::StructType>(Val: DesiredTy)) {
511	if (DesiredSTy->isLayoutIdentical(Other: STy))
512	STy = DesiredSTy;
513	}
514
515	return llvm::ConstantStruct::get(T: STy, V: Packed ? PackedElems : UnpackedElems);
516	}
517
518	void ConstantAggregateBuilder::condense(CharUnits Offset,
519	llvm::Type *DesiredTy) {
520	CharUnits Size = getSize(Ty: DesiredTy);
521
522	std::optional<size_t> FirstElemToReplace = splitAt(Pos: Offset);
523	if (!FirstElemToReplace)
524	return;
525	size_t First = *FirstElemToReplace;
526
527	std::optional<size_t> LastElemToReplace = splitAt(Pos: Offset + Size);
528	if (!LastElemToReplace)
529	return;
530	size_t Last = *LastElemToReplace;
531
532	size_t Length = Last - First;
533	if (Length == 0)
534	return;
535
536	if (Length == 1 && Offsets[First] == Offset &&
537	getSize(C: Elems[First]) == Size) {
538	// Re-wrap single element structs if necessary. Otherwise, leave any single
539	// element constant of the right size alone even if it has the wrong type.
540	auto *STy = dyn_cast<llvm::StructType>(Val: DesiredTy);
541	if (STy && STy->getNumElements() == 1 &&
542	STy->getElementType(N: 0) == Elems[First]->getType())
543	Elems[First] = llvm::ConstantStruct::get(T: STy, Vs: Elems[First]);
544	return;
545	}
546
547	llvm::Constant *Replacement = buildFrom(
548	CGM, Elems: ArrayRef(Elems).slice(N: First, M: Length),
549	Offsets: ArrayRef(Offsets).slice(N: First, M: Length), StartOffset: Offset, Size: getSize(Ty: DesiredTy),
550	/*known to have natural layout=*/NaturalLayout: false, DesiredTy, AllowOversized: false);
551	replace(C&: Elems, BeginOff: First, EndOff: Last, Vals: {Replacement});
552	replace(C&: Offsets, BeginOff: First, EndOff: Last, Vals: {Offset});
553	}
554
555	//===----------------------------------------------------------------------===//
556	// ConstStructBuilder
557	//===----------------------------------------------------------------------===//
558
559	class ConstStructBuilder {
560	CodeGenModule &CGM;
561	ConstantEmitter &Emitter;
562	ConstantAggregateBuilder &Builder;
563	CharUnits StartOffset;
564
565	public:
566	static llvm::Constant *BuildStruct(ConstantEmitter &Emitter,
567	const InitListExpr *ILE,
568	QualType StructTy);
569	static llvm::Constant *BuildStruct(ConstantEmitter &Emitter,
570	const APValue &Value, QualType ValTy);
571	static bool UpdateStruct(ConstantEmitter &Emitter,
572	ConstantAggregateBuilder &Const, CharUnits Offset,
573	const InitListExpr *Updater);
574
575	private:
576	ConstStructBuilder(ConstantEmitter &Emitter,
577	ConstantAggregateBuilder &Builder, CharUnits StartOffset)
578	: CGM(Emitter.CGM), Emitter(Emitter), Builder(Builder),
579	StartOffset(StartOffset) {}
580
581	bool AppendField(const FieldDecl *Field, uint64_t FieldOffset,
582	llvm::Constant *InitExpr, bool AllowOverwrite = false);
583
584	bool AppendBytes(CharUnits FieldOffsetInChars, llvm::Constant *InitCst,
585	bool AllowOverwrite = false);
586
587	bool AppendBitField(const FieldDecl *Field, uint64_t FieldOffset,
588	llvm::ConstantInt *InitExpr, bool AllowOverwrite = false);
589
590	bool Build(const InitListExpr *ILE, bool AllowOverwrite);
591	bool Build(const APValue &Val, const RecordDecl *RD, bool IsPrimaryBase,
592	const CXXRecordDecl *VTableClass, CharUnits BaseOffset);
593	llvm::Constant *Finalize(QualType Ty);
594	};
595
596	bool ConstStructBuilder::AppendField(
597	const FieldDecl *Field, uint64_t FieldOffset, llvm::Constant *InitCst,
598	bool AllowOverwrite) {
599	const ASTContext &Context = CGM.getContext();
600
601	CharUnits FieldOffsetInChars = Context.toCharUnitsFromBits(BitSize: FieldOffset);
602
603	return AppendBytes(FieldOffsetInChars, InitCst, AllowOverwrite);
604	}
605
606	bool ConstStructBuilder::AppendBytes(CharUnits FieldOffsetInChars,
607	llvm::Constant *InitCst,
608	bool AllowOverwrite) {
609	return Builder.add(C: InitCst, Offset: StartOffset + FieldOffsetInChars, AllowOverwrite);
610	}
611
612	bool ConstStructBuilder::AppendBitField(
613	const FieldDecl *Field, uint64_t FieldOffset, llvm::ConstantInt *CI,
614	bool AllowOverwrite) {
615	const CGRecordLayout &RL =
616	CGM.getTypes().getCGRecordLayout(Field->getParent());
617	const CGBitFieldInfo &Info = RL.getBitFieldInfo(FD: Field);
618	llvm::APInt FieldValue = CI->getValue();
619
620	// Promote the size of FieldValue if necessary
621	// FIXME: This should never occur, but currently it can because initializer
622	// constants are cast to bool, and because clang is not enforcing bitfield
623	// width limits.
624	if (Info.Size > FieldValue.getBitWidth())
625	FieldValue = FieldValue.zext(width: Info.Size);
626
627	// Truncate the size of FieldValue to the bit field size.
628	if (Info.Size < FieldValue.getBitWidth())
629	FieldValue = FieldValue.trunc(width: Info.Size);
630
631	return Builder.addBits(Bits: FieldValue,
632	OffsetInBits: CGM.getContext().toBits(CharSize: StartOffset) + FieldOffset,
633	AllowOverwrite);
634	}
635
636	static bool EmitDesignatedInitUpdater(ConstantEmitter &Emitter,
637	ConstantAggregateBuilder &Const,
638	CharUnits Offset, QualType Type,
639	const InitListExpr *Updater) {
640	if (Type->isRecordType())
641	return ConstStructBuilder::UpdateStruct(Emitter, Const, Offset, Updater);
642
643	auto CAT = Emitter.CGM.getContext().getAsConstantArrayType(T: Type);
644	if (!CAT)
645	return false;
646	QualType ElemType = CAT->getElementType();
647	CharUnits ElemSize = Emitter.CGM.getContext().getTypeSizeInChars(T: ElemType);
648	llvm::Type *ElemTy = Emitter.CGM.getTypes().ConvertTypeForMem(T: ElemType);
649
650	llvm::Constant *FillC = nullptr;
651	if (const Expr *Filler = Updater->getArrayFiller()) {
652	if (!isa<NoInitExpr>(Val: Filler)) {
653	FillC = Emitter.tryEmitAbstractForMemory(E: Filler, T: ElemType);
654	if (!FillC)
655	return false;
656	}
657	}
658
659	unsigned NumElementsToUpdate =
660	FillC ? CAT->getZExtSize() : Updater->getNumInits();
661	for (unsigned I = 0; I != NumElementsToUpdate; ++I, Offset += ElemSize) {
662	const Expr *Init = nullptr;
663	if (I < Updater->getNumInits())
664	Init = Updater->getInit(Init: I);
665
666	if (!Init && FillC) {
667	if (!Const.add(C: FillC, Offset, AllowOverwrite: true))
668	return false;
669	} else if (!Init || isa<NoInitExpr>(Val: Init)) {
670	continue;
671	} else if (const auto *ChildILE = dyn_cast<InitListExpr>(Val: Init)) {
672	if (!EmitDesignatedInitUpdater(Emitter, Const, Offset, Type: ElemType,
673	Updater: ChildILE))
674	return false;
675	// Attempt to reduce the array element to a single constant if necessary.
676	Const.condense(Offset, DesiredTy: ElemTy);
677	} else {
678	llvm::Constant *Val = Emitter.tryEmitPrivateForMemory(E: Init, T: ElemType);
679	if (!Const.add(C: Val, Offset, AllowOverwrite: true))
680	return false;
681	}
682	}
683
684	return true;
685	}
686
687	bool ConstStructBuilder::Build(const InitListExpr *ILE, bool AllowOverwrite) {
688	RecordDecl *RD = ILE->getType()->castAs<RecordType>()->getDecl();
689	const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
690
691	unsigned FieldNo = -1;
692	unsigned ElementNo = 0;
693
694	// Bail out if we have base classes. We could support these, but they only
695	// arise in C++1z where we will have already constant folded most interesting
696	// cases. FIXME: There are still a few more cases we can handle this way.
697	if (auto *CXXRD = dyn_cast<CXXRecordDecl>(RD))
698	if (CXXRD->getNumBases())
699	return false;
700
701	for (FieldDecl *Field : RD->fields()) {
702	++FieldNo;
703
704	// If this is a union, skip all the fields that aren't being initialized.
705	if (RD->isUnion() &&
706	!declaresSameEntity(ILE->getInitializedFieldInUnion(), Field))
707	continue;
708
709	// Don't emit anonymous bitfields.
710	if (Field->isUnnamedBitField())
711	continue;
712
713	// Get the initializer. A struct can include fields without initializers,
714	// we just use explicit null values for them.
715	const Expr *Init = nullptr;
716	if (ElementNo < ILE->getNumInits())
717	Init = ILE->getInit(ElementNo++);
718	if (Init && isa<NoInitExpr>(Init))
719	continue;
720
721	// Zero-sized fields are not emitted, but their initializers may still
722	// prevent emission of this struct as a constant.
723	if (Field->isZeroSize(CGM.getContext())) {
724	if (Init->HasSideEffects(CGM.getContext()))
725	return false;
726	continue;
727	}
728
729	// When emitting a DesignatedInitUpdateExpr, a nested InitListExpr
730	// represents additional overwriting of our current constant value, and not
731	// a new constant to emit independently.
732	if (AllowOverwrite &&
733	(Field->getType()->isArrayType() || Field->getType()->isRecordType())) {
734	if (auto *SubILE = dyn_cast<InitListExpr>(Init)) {
735	CharUnits Offset = CGM.getContext().toCharUnitsFromBits(
736	Layout.getFieldOffset(FieldNo));
737	if (!EmitDesignatedInitUpdater(Emitter, Builder, StartOffset + Offset,
738	Field->getType(), SubILE))
739	return false;
740	// If we split apart the field's value, try to collapse it down to a
741	// single value now.
742	Builder.condense(StartOffset + Offset,
743	CGM.getTypes().ConvertTypeForMem(Field->getType()));
744	continue;
745	}
746	}
747
748	llvm::Constant *EltInit =
749	Init ? Emitter.tryEmitPrivateForMemory(Init, Field->getType())
750	: Emitter.emitNullForMemory(Field->getType());
751	if (!EltInit)
752	return false;
753
754	if (!Field->isBitField()) {
755	// Handle non-bitfield members.
756	if (!AppendField(Field, Layout.getFieldOffset(FieldNo), EltInit,
757	AllowOverwrite))
758	return false;
759	// After emitting a non-empty field with [[no_unique_address]], we may
760	// need to overwrite its tail padding.
761	if (Field->hasAttr<NoUniqueAddressAttr>())
762	AllowOverwrite = true;
763	} else {
764	// Otherwise we have a bitfield.
765	if (auto *CI = dyn_cast<llvm::ConstantInt>(EltInit)) {
766	if (!AppendBitField(Field, Layout.getFieldOffset(FieldNo), CI,
767	AllowOverwrite))
768	return false;
769	} else {
770	// We are trying to initialize a bitfield with a non-trivial constant,
771	// this must require run-time code.
772	return false;
773	}
774	}
775	}
776
777	return true;
778	}
779
780	namespace {
781	struct BaseInfo {
782	BaseInfo(const CXXRecordDecl *Decl, CharUnits Offset, unsigned Index)
783	: Decl(Decl), Offset(Offset), Index(Index) {
784	}
785
786	const CXXRecordDecl *Decl;
787	CharUnits Offset;
788	unsigned Index;
789
790	bool operator<(const BaseInfo &O) const { return Offset < O.Offset; }
791	};
792	}
793
794	bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
795	bool IsPrimaryBase,
796	const CXXRecordDecl *VTableClass,
797	CharUnits Offset) {
798	const ASTRecordLayout &Layout = CGM.getContext().getASTRecordLayout(D: RD);
799
800	if (const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(Val: RD)) {
801	// Add a vtable pointer, if we need one and it hasn't already been added.
802	if (Layout.hasOwnVFPtr()) {
803	llvm::Constant *VTableAddressPoint =
804	CGM.getCXXABI().getVTableAddressPoint(Base: BaseSubobject(CD, Offset),
805	VTableClass);
806	if (!AppendBytes(FieldOffsetInChars: Offset, InitCst: VTableAddressPoint))
807	return false;
808	}
809
810	// Accumulate and sort bases, in order to visit them in address order, which
811	// may not be the same as declaration order.
812	SmallVector<BaseInfo, 8> Bases;
813	Bases.reserve(N: CD->getNumBases());
814	unsigned BaseNo = 0;
815	for (CXXRecordDecl::base_class_const_iterator Base = CD->bases_begin(),
816	BaseEnd = CD->bases_end(); Base != BaseEnd; ++Base, ++BaseNo) {
817	assert(!Base->isVirtual() && "should not have virtual bases here");
818	const CXXRecordDecl *BD = Base->getType()->getAsCXXRecordDecl();
819	CharUnits BaseOffset = Layout.getBaseClassOffset(Base: BD);
820	Bases.push_back(Elt: BaseInfo(BD, BaseOffset, BaseNo));
821	}
822	llvm::stable_sort(Range&: Bases);
823
824	for (unsigned I = 0, N = Bases.size(); I != N; ++I) {
825	BaseInfo &Base = Bases[I];
826
827	bool IsPrimaryBase = Layout.getPrimaryBase() == Base.Decl;
828	Build(Val.getStructBase(i: Base.Index), Base.Decl, IsPrimaryBase,
829	VTableClass, Offset + Base.Offset);
830	}
831	}
832
833	unsigned FieldNo = 0;
834	uint64_t OffsetBits = CGM.getContext().toBits(CharSize: Offset);
835
836	bool AllowOverwrite = false;
837	for (RecordDecl::field_iterator Field = RD->field_begin(),
838	FieldEnd = RD->field_end(); Field != FieldEnd; ++Field, ++FieldNo) {
839	// If this is a union, skip all the fields that aren't being initialized.
840	if (RD->isUnion() && !declaresSameEntity(Val.getUnionField(), *Field))
841	continue;
842
843	// Don't emit anonymous bitfields or zero-sized fields.
844	if (Field->isUnnamedBitField() || Field->isZeroSize(Ctx: CGM.getContext()))
845	continue;
846
847	// Emit the value of the initializer.
848	const APValue &FieldValue =
849	RD->isUnion() ? Val.getUnionValue() : Val.getStructField(i: FieldNo);
850	llvm::Constant *EltInit =
851	Emitter.tryEmitPrivateForMemory(FieldValue, Field->getType());
852	if (!EltInit)
853	return false;
854
855	if (!Field->isBitField()) {
856	// Handle non-bitfield members.
857	if (!AppendField(Field: *Field, FieldOffset: Layout.getFieldOffset(FieldNo) + OffsetBits,
858	InitCst: EltInit, AllowOverwrite))
859	return false;
860	// After emitting a non-empty field with [[no_unique_address]], we may
861	// need to overwrite its tail padding.
862	if (Field->hasAttr<NoUniqueAddressAttr>())
863	AllowOverwrite = true;
864	} else {
865	// Otherwise we have a bitfield.
866	if (!AppendBitField(Field: *Field, FieldOffset: Layout.getFieldOffset(FieldNo) + OffsetBits,
867	CI: cast<llvm::ConstantInt>(Val: EltInit), AllowOverwrite))
868	return false;
869	}
870	}
871
872	return true;
873	}
874
875	llvm::Constant *ConstStructBuilder::Finalize(QualType Type) {
876	Type = Type.getNonReferenceType();
877	RecordDecl *RD = Type->castAs<RecordType>()->getDecl();
878	llvm::Type *ValTy = CGM.getTypes().ConvertType(T: Type);
879	return Builder.build(DesiredTy: ValTy, AllowOversized: RD->hasFlexibleArrayMember());
880	}
881
882	llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
883	const InitListExpr *ILE,
884	QualType ValTy) {
885	ConstantAggregateBuilder Const(Emitter.CGM);
886	ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
887
888	if (!Builder.Build(ILE, /*AllowOverwrite*/false))
889	return nullptr;
890
891	return Builder.Finalize(Type: ValTy);
892	}
893
894	llvm::Constant *ConstStructBuilder::BuildStruct(ConstantEmitter &Emitter,
895	const APValue &Val,
896	QualType ValTy) {
897	ConstantAggregateBuilder Const(Emitter.CGM);
898	ConstStructBuilder Builder(Emitter, Const, CharUnits::Zero());
899
900	const RecordDecl *RD = ValTy->castAs<RecordType>()->getDecl();
901	const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(Val: RD);
902	if (!Builder.Build(Val, RD, IsPrimaryBase: false, VTableClass: CD, Offset: CharUnits::Zero()))
903	return nullptr;
904
905	return Builder.Finalize(Type: ValTy);
906	}
907
908	bool ConstStructBuilder::UpdateStruct(ConstantEmitter &Emitter,
909	ConstantAggregateBuilder &Const,
910	CharUnits Offset,
911	const InitListExpr *Updater) {
912	return ConstStructBuilder(Emitter, Const, Offset)
913	.Build(ILE: Updater, /*AllowOverwrite*/ true);
914	}
915
916	//===----------------------------------------------------------------------===//
917	// ConstExprEmitter
918	//===----------------------------------------------------------------------===//
919
920	static ConstantAddress
921	tryEmitGlobalCompoundLiteral(ConstantEmitter &emitter,
922	const CompoundLiteralExpr *E) {
923	CodeGenModule &CGM = emitter.CGM;
924	CharUnits Align = CGM.getContext().getTypeAlignInChars(E->getType());
925	if (llvm::GlobalVariable *Addr =
926	CGM.getAddrOfConstantCompoundLiteralIfEmitted(E))
927	return ConstantAddress(Addr, Addr->getValueType(), Align);
928
929	LangAS addressSpace = E->getType().getAddressSpace();
930	llvm::Constant *C = emitter.tryEmitForInitializer(E->getInitializer(),
931	addressSpace, E->getType());
932	if (!C) {
933	assert(!E->isFileScope() &&
934	"file-scope compound literal did not have constant initializer!");
935	return ConstantAddress::invalid();
936	}
937
938	auto GV = new llvm::GlobalVariable(
939	CGM.getModule(), C->getType(),
940	E->getType().isConstantStorage(CGM.getContext(), true, false),
941	llvm::GlobalValue::InternalLinkage, C, ".compoundliteral", nullptr,
942	llvm::GlobalVariable::NotThreadLocal,
943	CGM.getContext().getTargetAddressSpace(AS: addressSpace));
944	emitter.finalize(global: GV);
945	GV->setAlignment(Align.getAsAlign());
946	CGM.setAddrOfConstantCompoundLiteral(CLE: E, GV: GV);
947	return ConstantAddress(GV, GV->getValueType(), Align);
948	}
949
950	static llvm::Constant *
951	EmitArrayConstant(CodeGenModule &CGM, llvm::ArrayType *DesiredType,
952	llvm::Type *CommonElementType, unsigned ArrayBound,
953	SmallVectorImpl<llvm::Constant *> &Elements,
954	llvm::Constant *Filler) {
955	// Figure out how long the initial prefix of non-zero elements is.
956	unsigned NonzeroLength = ArrayBound;
957	if (Elements.size() < NonzeroLength && Filler->isNullValue())
958	NonzeroLength = Elements.size();
959	if (NonzeroLength == Elements.size()) {
960	while (NonzeroLength > 0 && Elements[NonzeroLength - 1]->isNullValue())
961	--NonzeroLength;
962	}
963
964	if (NonzeroLength == 0)
965	return llvm::ConstantAggregateZero::get(Ty: DesiredType);
966
967	// Add a zeroinitializer array filler if we have lots of trailing zeroes.
968	unsigned TrailingZeroes = ArrayBound - NonzeroLength;
969	if (TrailingZeroes >= 8) {
970	assert(Elements.size() >= NonzeroLength &&
971	"missing initializer for non-zero element");
972
973	// If all the elements had the same type up to the trailing zeroes, emit a
974	// struct of two arrays (the nonzero data and the zeroinitializer).
975	if (CommonElementType && NonzeroLength >= 8) {
976	llvm::Constant *Initial = llvm::ConstantArray::get(
977	T: llvm::ArrayType::get(ElementType: CommonElementType, NumElements: NonzeroLength),
978	V: ArrayRef(Elements).take_front(N: NonzeroLength));
979	Elements.resize(N: 2);
980	Elements[0] = Initial;
981	} else {
982	Elements.resize(N: NonzeroLength + 1);
983	}
984
985	auto *FillerType =
986	CommonElementType ? CommonElementType : DesiredType->getElementType();
987	FillerType = llvm::ArrayType::get(ElementType: FillerType, NumElements: TrailingZeroes);
988	Elements.back() = llvm::ConstantAggregateZero::get(Ty: FillerType);
989	CommonElementType = nullptr;
990	} else if (Elements.size() != ArrayBound) {
991	// Otherwise pad to the right size with the filler if necessary.
992	Elements.resize(N: ArrayBound, NV: Filler);
993	if (Filler->getType() != CommonElementType)
994	CommonElementType = nullptr;
995	}
996
997	// If all elements have the same type, just emit an array constant.
998	if (CommonElementType)
999	return llvm::ConstantArray::get(
1000	T: llvm::ArrayType::get(ElementType: CommonElementType, NumElements: ArrayBound), V: Elements);
1001
1002	// We have mixed types. Use a packed struct.
1003	llvm::SmallVector<llvm::Type *, 16> Types;
1004	Types.reserve(N: Elements.size());
1005	for (llvm::Constant *Elt : Elements)
1006	Types.push_back(Elt: Elt->getType());
1007	llvm::StructType *SType =
1008	llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: Types, isPacked: true);
1009	return llvm::ConstantStruct::get(T: SType, V: Elements);
1010	}
1011
1012	// This class only needs to handle arrays, structs and unions. Outside C++11
1013	// mode, we don't currently constant fold those types. All other types are
1014	// handled by constant folding.
1015	//
1016	// Constant folding is currently missing support for a few features supported
1017	// here: CK_ToUnion, CK_ReinterpretMemberPointer, and DesignatedInitUpdateExpr.
1018	class ConstExprEmitter
1019	: public ConstStmtVisitor<ConstExprEmitter, llvm::Constant *, QualType> {
1020	CodeGenModule &CGM;
1021	ConstantEmitter &Emitter;
1022	llvm::LLVMContext &VMContext;
1023	public:
1024	ConstExprEmitter(ConstantEmitter &emitter)
1025	: CGM(emitter.CGM), Emitter(emitter), VMContext(CGM.getLLVMContext()) {
1026	}
1027
1028	//===--------------------------------------------------------------------===//
1029	// Visitor Methods
1030	//===--------------------------------------------------------------------===//
1031
1032	llvm::Constant *VisitStmt(const Stmt *S, QualType T) { return nullptr; }
1033
1034	llvm::Constant *VisitConstantExpr(const ConstantExpr *CE, QualType T) {
1035	if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE))
1036	return Result;
1037	return Visit(CE->getSubExpr(), T);
1038	}
1039
1040	llvm::Constant *VisitParenExpr(const ParenExpr *PE, QualType T) {
1041	return Visit(PE->getSubExpr(), T);
1042	}
1043
1044	llvm::Constant *
1045	VisitSubstNonTypeTemplateParmExpr(const SubstNonTypeTemplateParmExpr *PE,
1046	QualType T) {
1047	return Visit(PE->getReplacement(), T);
1048	}
1049
1050	llvm::Constant *VisitGenericSelectionExpr(const GenericSelectionExpr *GE,
1051	QualType T) {
1052	return Visit(GE->getResultExpr(), T);
1053	}
1054
1055	llvm::Constant *VisitChooseExpr(const ChooseExpr *CE, QualType T) {
1056	return Visit(CE->getChosenSubExpr(), T);
1057	}
1058
1059	llvm::Constant *VisitCompoundLiteralExpr(const CompoundLiteralExpr *E,
1060	QualType T) {
1061	return Visit(E->getInitializer(), T);
1062	}
1063
1064	llvm::Constant *VisitCastExpr(const CastExpr *E, QualType destType) {
1065	if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
1066	CGM.EmitExplicitCastExprType(E: ECE, CGF: Emitter.CGF);
1067	const Expr *subExpr = E->getSubExpr();
1068
1069	switch (E->getCastKind()) {
1070	case CK_ToUnion: {
1071	// GCC cast to union extension
1072	assert(E->getType()->isUnionType() &&
1073	"Destination type is not union type!");
1074
1075	auto field = E->getTargetUnionField();
1076
1077	auto C = Emitter.tryEmitPrivateForMemory(subExpr, field->getType());
1078	if (!C) return nullptr;
1079
1080	auto destTy = ConvertType(T: destType);
1081	if (C->getType() == destTy) return C;
1082
1083	// Build a struct with the union sub-element as the first member,
1084	// and padded to the appropriate size.
1085	SmallVector<llvm::Constant*, 2> Elts;
1086	SmallVector<llvm::Type*, 2> Types;
1087	Elts.push_back(Elt: C);
1088	Types.push_back(Elt: C->getType());
1089	unsigned CurSize = CGM.getDataLayout().getTypeAllocSize(Ty: C->getType());
1090	unsigned TotalSize = CGM.getDataLayout().getTypeAllocSize(Ty: destTy);
1091
1092	assert(CurSize <= TotalSize && "Union size mismatch!");
1093	if (unsigned NumPadBytes = TotalSize - CurSize) {
1094	llvm::Type *Ty = CGM.CharTy;
1095	if (NumPadBytes > 1)
1096	Ty = llvm::ArrayType::get(ElementType: Ty, NumElements: NumPadBytes);
1097
1098	Elts.push_back(Elt: llvm::UndefValue::get(T: Ty));
1099	Types.push_back(Elt: Ty);
1100	}
1101
1102	llvm::StructType *STy = llvm::StructType::get(Context&: VMContext, Elements: Types, isPacked: false);
1103	return llvm::ConstantStruct::get(T: STy, V: Elts);
1104	}
1105
1106	case CK_AddressSpaceConversion: {
1107	auto C = Emitter.tryEmitPrivate(E: subExpr, T: subExpr->getType());
1108	if (!C) return nullptr;
1109	LangAS destAS = E->getType()->getPointeeType().getAddressSpace();
1110	LangAS srcAS = subExpr->getType()->getPointeeType().getAddressSpace();
1111	llvm::Type *destTy = ConvertType(T: E->getType());
1112	return CGM.getTargetCodeGenInfo().performAddrSpaceCast(CGM, V: C, SrcAddr: srcAS,
1113	DestAddr: destAS, DestTy: destTy);
1114	}
1115
1116	case CK_LValueToRValue: {
1117	// We don't really support doing lvalue-to-rvalue conversions here; any
1118	// interesting conversions should be done in Evaluate(). But as a
1119	// special case, allow compound literals to support the gcc extension
1120	// allowing "struct x {int x;} x = (struct x) {};".
1121	if (const auto *E =
1122	dyn_cast<CompoundLiteralExpr>(Val: subExpr->IgnoreParens()))
1123	return Visit(E->getInitializer(), destType);
1124	return nullptr;
1125	}
1126
1127	case CK_AtomicToNonAtomic:
1128	case CK_NonAtomicToAtomic:
1129	case CK_NoOp:
1130	case CK_ConstructorConversion:
1131	return Visit(subExpr, destType);
1132
1133	case CK_ArrayToPointerDecay:
1134	if (const auto *S = dyn_cast<StringLiteral>(Val: subExpr))
1135	return CGM.GetAddrOfConstantStringFromLiteral(S).getPointer();
1136	return nullptr;
1137	case CK_NullToPointer:
1138	if (Visit(subExpr, destType))
1139	return CGM.EmitNullConstant(T: destType);
1140	return nullptr;
1141
1142	case CK_IntToOCLSampler:
1143	llvm_unreachable("global sampler variables are not generated");
1144
1145	case CK_IntegralCast: {
1146	QualType FromType = subExpr->getType();
1147	// See also HandleIntToIntCast in ExprConstant.cpp
1148	if (FromType->isIntegerType())
1149	if (llvm::Constant *C = Visit(subExpr, FromType))
1150	if (auto *CI = dyn_cast<llvm::ConstantInt>(C)) {
1151	unsigned SrcWidth = CGM.getContext().getIntWidth(T: FromType);
1152	unsigned DstWidth = CGM.getContext().getIntWidth(T: destType);
1153	if (DstWidth == SrcWidth)
1154	return CI;
1155	llvm::APInt A = FromType->isSignedIntegerType()
1156	? CI->getValue().sextOrTrunc(DstWidth)
1157	: CI->getValue().zextOrTrunc(DstWidth);
1158	return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: A);
1159	}
1160	return nullptr;
1161	}
1162
1163	case CK_Dependent: llvm_unreachable("saw dependent cast!");
1164
1165	case CK_BuiltinFnToFnPtr:
1166	llvm_unreachable("builtin functions are handled elsewhere");
1167
1168	case CK_ReinterpretMemberPointer:
1169	case CK_DerivedToBaseMemberPointer:
1170	case CK_BaseToDerivedMemberPointer: {
1171	auto C = Emitter.tryEmitPrivate(E: subExpr, T: subExpr->getType());
1172	if (!C) return nullptr;
1173	return CGM.getCXXABI().EmitMemberPointerConversion(E, Src: C);
1174	}
1175
1176	// These will never be supported.
1177	case CK_ObjCObjectLValueCast:
1178	case CK_ARCProduceObject:
1179	case CK_ARCConsumeObject:
1180	case CK_ARCReclaimReturnedObject:
1181	case CK_ARCExtendBlockObject:
1182	case CK_CopyAndAutoreleaseBlockObject:
1183	return nullptr;
1184
1185	// These don't need to be handled here because Evaluate knows how to
1186	// evaluate them in the cases where they can be folded.
1187	case CK_BitCast:
1188	case CK_ToVoid:
1189	case CK_Dynamic:
1190	case CK_LValueBitCast:
1191	case CK_LValueToRValueBitCast:
1192	case CK_NullToMemberPointer:
1193	case CK_UserDefinedConversion:
1194	case CK_CPointerToObjCPointerCast:
1195	case CK_BlockPointerToObjCPointerCast:
1196	case CK_AnyPointerToBlockPointerCast:
1197	case CK_FunctionToPointerDecay:
1198	case CK_BaseToDerived:
1199	case CK_DerivedToBase:
1200	case CK_UncheckedDerivedToBase:
1201	case CK_MemberPointerToBoolean:
1202	case CK_VectorSplat:
1203	case CK_FloatingRealToComplex:
1204	case CK_FloatingComplexToReal:
1205	case CK_FloatingComplexToBoolean:
1206	case CK_FloatingComplexCast:
1207	case CK_FloatingComplexToIntegralComplex:
1208	case CK_IntegralRealToComplex:
1209	case CK_IntegralComplexToReal:
1210	case CK_IntegralComplexToBoolean:
1211	case CK_IntegralComplexCast:
1212	case CK_IntegralComplexToFloatingComplex:
1213	case CK_PointerToIntegral:
1214	case CK_PointerToBoolean:
1215	case CK_BooleanToSignedIntegral:
1216	case CK_IntegralToPointer:
1217	case CK_IntegralToBoolean:
1218	case CK_IntegralToFloating:
1219	case CK_FloatingToIntegral:
1220	case CK_FloatingToBoolean:
1221	case CK_FloatingCast:
1222	case CK_FloatingToFixedPoint:
1223	case CK_FixedPointToFloating:
1224	case CK_FixedPointCast:
1225	case CK_FixedPointToBoolean:
1226	case CK_FixedPointToIntegral:
1227	case CK_IntegralToFixedPoint:
1228	case CK_ZeroToOCLOpaqueType:
1229	case CK_MatrixCast:
1230	case CK_HLSLVectorTruncation:
1231	case CK_HLSLArrayRValue:
1232	return nullptr;
1233	}
1234	llvm_unreachable("Invalid CastKind");
1235	}
1236
1237	llvm::Constant *VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *DIE,
1238	QualType T) {
1239	// No need for a DefaultInitExprScope: we don't handle 'this' in a
1240	// constant expression.
1241	return Visit(DIE->getExpr(), T);
1242	}
1243
1244	llvm::Constant *VisitExprWithCleanups(const ExprWithCleanups *E, QualType T) {
1245	return Visit(E->getSubExpr(), T);
1246	}
1247
1248	llvm::Constant *VisitIntegerLiteral(const IntegerLiteral *I, QualType T) {
1249	return llvm::ConstantInt::get(CGM.getLLVMContext(), I->getValue());
1250	}
1251
1252	llvm::Constant *EmitArrayInitialization(const InitListExpr *ILE, QualType T) {
1253	auto *CAT = CGM.getContext().getAsConstantArrayType(T: ILE->getType());
1254	assert(CAT && "can't emit array init for non-constant-bound array");
1255	unsigned NumInitElements = ILE->getNumInits();
1256	unsigned NumElements = CAT->getZExtSize();
1257
1258	// Initialising an array requires us to automatically
1259	// initialise any elements that have not been initialised explicitly
1260	unsigned NumInitableElts = std::min(a: NumInitElements, b: NumElements);
1261
1262	QualType EltType = CAT->getElementType();
1263
1264	// Initialize remaining array elements.
1265	llvm::Constant *fillC = nullptr;
1266	if (const Expr *filler = ILE->getArrayFiller()) {
1267	fillC = Emitter.tryEmitAbstractForMemory(E: filler, T: EltType);
1268	if (!fillC)
1269	return nullptr;
1270	}
1271
1272	// Copy initializer elements.
1273	SmallVector<llvm::Constant*, 16> Elts;
1274	if (fillC && fillC->isNullValue())
1275	Elts.reserve(N: NumInitableElts + 1);
1276	else
1277	Elts.reserve(N: NumElements);
1278
1279	llvm::Type *CommonElementType = nullptr;
1280	for (unsigned i = 0; i < NumInitableElts; ++i) {
1281	const Expr *Init = ILE->getInit(Init: i);
1282	llvm::Constant *C = Emitter.tryEmitPrivateForMemory(E: Init, T: EltType);
1283	if (!C)
1284	return nullptr;
1285	if (i == 0)
1286	CommonElementType = C->getType();
1287	else if (C->getType() != CommonElementType)
1288	CommonElementType = nullptr;
1289	Elts.push_back(Elt: C);
1290	}
1291
1292	llvm::ArrayType *Desired =
1293	cast<llvm::ArrayType>(CGM.getTypes().ConvertType(T: ILE->getType()));
1294	return EmitArrayConstant(CGM, DesiredType: Desired, CommonElementType, ArrayBound: NumElements, Elements&: Elts,
1295	Filler: fillC);
1296	}
1297
1298	llvm::Constant *EmitRecordInitialization(const InitListExpr *ILE,
1299	QualType T) {
1300	return ConstStructBuilder::BuildStruct(Emitter, ILE, ValTy: T);
1301	}
1302
1303	llvm::Constant *VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E,
1304	QualType T) {
1305	return CGM.EmitNullConstant(T);
1306	}
1307
1308	llvm::Constant *VisitInitListExpr(const InitListExpr *ILE, QualType T) {
1309	if (ILE->isTransparent())
1310	return Visit(ILE->getInit(Init: 0), T);
1311
1312	if (ILE->getType()->isArrayType())
1313	return EmitArrayInitialization(ILE, T);
1314
1315	if (ILE->getType()->isRecordType())
1316	return EmitRecordInitialization(ILE, T);
1317
1318	return nullptr;
1319	}
1320
1321	llvm::Constant *
1322	VisitDesignatedInitUpdateExpr(const DesignatedInitUpdateExpr *E,
1323	QualType destType) {
1324	auto C = Visit(E->getBase(), destType);
1325	if (!C)
1326	return nullptr;
1327
1328	ConstantAggregateBuilder Const(CGM);
1329	Const.add(C: C, Offset: CharUnits::Zero(), AllowOverwrite: false);
1330
1331	if (!EmitDesignatedInitUpdater(Emitter, Const, Offset: CharUnits::Zero(), Type: destType,
1332	Updater: E->getUpdater()))
1333	return nullptr;
1334
1335	llvm::Type *ValTy = CGM.getTypes().ConvertType(T: destType);
1336	bool HasFlexibleArray = false;
1337	if (const auto *RT = destType->getAs<RecordType>())
1338	HasFlexibleArray = RT->getDecl()->hasFlexibleArrayMember();
1339	return Const.build(DesiredTy: ValTy, AllowOversized: HasFlexibleArray);
1340	}
1341
1342	llvm::Constant *VisitCXXConstructExpr(const CXXConstructExpr *E,
1343	QualType Ty) {
1344	if (!E->getConstructor()->isTrivial())
1345	return nullptr;
1346
1347	// Only default and copy/move constructors can be trivial.
1348	if (E->getNumArgs()) {
1349	assert(E->getNumArgs() == 1 && "trivial ctor with > 1 argument");
1350	assert(E->getConstructor()->isCopyOrMoveConstructor() &&
1351	"trivial ctor has argument but isn't a copy/move ctor");
1352
1353	const Expr *Arg = E->getArg(Arg: 0);
1354	assert(CGM.getContext().hasSameUnqualifiedType(Ty, Arg->getType()) &&
1355	"argument to copy ctor is of wrong type");
1356
1357	// Look through the temporary; it's just converting the value to an
1358	// lvalue to pass it to the constructor.
1359	if (const auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: Arg))
1360	return Visit(MTE->getSubExpr(), Ty);
1361	// Don't try to support arbitrary lvalue-to-rvalue conversions for now.
1362	return nullptr;
1363	}
1364
1365	return CGM.EmitNullConstant(T: Ty);
1366	}
1367
1368	llvm::Constant *VisitStringLiteral(const StringLiteral *E, QualType T) {
1369	// This is a string literal initializing an array in an initializer.
1370	return CGM.GetConstantArrayFromStringLiteral(E);
1371	}
1372
1373	llvm::Constant *VisitObjCEncodeExpr(const ObjCEncodeExpr *E, QualType T) {
1374	// This must be an @encode initializing an array in a static initializer.
1375	// Don't emit it as the address of the string, emit the string data itself
1376	// as an inline array.
1377	std::string Str;
1378	CGM.getContext().getObjCEncodingForType(T: E->getEncodedType(), S&: Str);
1379	const ConstantArrayType *CAT = CGM.getContext().getAsConstantArrayType(T);
1380	assert(CAT && "String data not of constant array type!");
1381
1382	// Resize the string to the right size, adding zeros at the end, or
1383	// truncating as needed.
1384	Str.resize(n: CAT->getZExtSize(), c: '\0');
1385	return llvm::ConstantDataArray::getString(Context&: VMContext, Initializer: Str, AddNull: false);
1386	}
1387
1388	llvm::Constant *VisitUnaryExtension(const UnaryOperator *E, QualType T) {
1389	return Visit(E->getSubExpr(), T);
1390	}
1391
1392	llvm::Constant *VisitUnaryMinus(const UnaryOperator *U, QualType T) {
1393	if (llvm::Constant *C = Visit(U->getSubExpr(), T))
1394	if (auto *CI = dyn_cast<llvm::ConstantInt>(C))
1395	return llvm::ConstantInt::get(CGM.getLLVMContext(), -CI->getValue());
1396	return nullptr;
1397	}
1398
1399	llvm::Constant *VisitPackIndexingExpr(const PackIndexingExpr *E, QualType T) {
1400	return Visit(E->getSelectedExpr(), T);
1401	}
1402
1403	// Utility methods
1404	llvm::Type *ConvertType(QualType T) {
1405	return CGM.getTypes().ConvertType(T);
1406	}
1407	};
1408
1409	} // end anonymous namespace.
1410
1411	llvm::Constant *ConstantEmitter::validateAndPopAbstract(llvm::Constant *C,
1412	AbstractState saved) {
1413	Abstract = saved.OldValue;
1414
1415	assert(saved.OldPlaceholdersSize == PlaceholderAddresses.size() &&
1416	"created a placeholder while doing an abstract emission?");
1417
1418	// No validation necessary for now.
1419	// No cleanup to do for now.
1420	return C;
1421	}
1422
1423	llvm::Constant *
1424	ConstantEmitter::tryEmitAbstractForInitializer(const VarDecl &D) {
1425	auto state = pushAbstract();
1426	auto C = tryEmitPrivateForVarInit(D);
1427	return validateAndPopAbstract(C, saved: state);
1428	}
1429
1430	llvm::Constant *
1431	ConstantEmitter::tryEmitAbstract(const Expr *E, QualType destType) {
1432	auto state = pushAbstract();
1433	auto C = tryEmitPrivate(E, T: destType);
1434	return validateAndPopAbstract(C, saved: state);
1435	}
1436
1437	llvm::Constant *
1438	ConstantEmitter::tryEmitAbstract(const APValue &value, QualType destType) {
1439	auto state = pushAbstract();
1440	auto C = tryEmitPrivate(value, T: destType);
1441	return validateAndPopAbstract(C, saved: state);
1442	}
1443
1444	llvm::Constant *ConstantEmitter::tryEmitConstantExpr(const ConstantExpr *CE) {
1445	if (!CE->hasAPValueResult())
1446	return nullptr;
1447
1448	QualType RetType = CE->getType();
1449	if (CE->isGLValue())
1450	RetType = CGM.getContext().getLValueReferenceType(T: RetType);
1451
1452	return emitAbstract(loc: CE->getBeginLoc(), value: CE->getAPValueResult(), T: RetType);
1453	}
1454
1455	llvm::Constant *
1456	ConstantEmitter::emitAbstract(const Expr *E, QualType destType) {
1457	auto state = pushAbstract();
1458	auto C = tryEmitPrivate(E, T: destType);
1459	C = validateAndPopAbstract(C, saved: state);
1460	if (!C) {
1461	CGM.Error(loc: E->getExprLoc(),
1462	error: "internal error: could not emit constant value \"abstractly\"");
1463	C = CGM.EmitNullConstant(T: destType);
1464	}
1465	return C;
1466	}
1467
1468	llvm::Constant *
1469	ConstantEmitter::emitAbstract(SourceLocation loc, const APValue &value,
1470	QualType destType) {
1471	auto state = pushAbstract();
1472	auto C = tryEmitPrivate(value, T: destType);
1473	C = validateAndPopAbstract(C, saved: state);
1474	if (!C) {
1475	CGM.Error(loc,
1476	error: "internal error: could not emit constant value \"abstractly\"");
1477	C = CGM.EmitNullConstant(T: destType);
1478	}
1479	return C;
1480	}
1481
1482	llvm::Constant *ConstantEmitter::tryEmitForInitializer(const VarDecl &D) {
1483	initializeNonAbstract(destAS: D.getType().getAddressSpace());
1484	return markIfFailed(init: tryEmitPrivateForVarInit(D));
1485	}
1486
1487	llvm::Constant *ConstantEmitter::tryEmitForInitializer(const Expr *E,
1488	LangAS destAddrSpace,
1489	QualType destType) {
1490	initializeNonAbstract(destAS: destAddrSpace);
1491	return markIfFailed(init: tryEmitPrivateForMemory(E, T: destType));
1492	}
1493
1494	llvm::Constant *ConstantEmitter::emitForInitializer(const APValue &value,
1495	LangAS destAddrSpace,
1496	QualType destType) {
1497	initializeNonAbstract(destAS: destAddrSpace);
1498	auto C = tryEmitPrivateForMemory(value, T: destType);
1499	assert(C && "couldn't emit constant value non-abstractly?");
1500	return C;
1501	}
1502
1503	llvm::GlobalValue *ConstantEmitter::getCurrentAddrPrivate() {
1504	assert(!Abstract && "cannot get current address for abstract constant");
1505
1506
1507
1508	// Make an obviously ill-formed global that should blow up compilation
1509	// if it survives.
1510	auto global = new llvm::GlobalVariable(CGM.getModule(), CGM.Int8Ty, true,
1511	llvm::GlobalValue::PrivateLinkage,
1512	/*init*/ nullptr,
1513	/*name*/ "",
1514	/*before*/ nullptr,
1515	llvm::GlobalVariable::NotThreadLocal,
1516	CGM.getContext().getTargetAddressSpace(AS: DestAddressSpace));
1517
1518	PlaceholderAddresses.push_back(Elt: std::make_pair(x: nullptr, y&: global));
1519
1520	return global;
1521	}
1522
1523	void ConstantEmitter::registerCurrentAddrPrivate(llvm::Constant *signal,
1524	llvm::GlobalValue *placeholder) {
1525	assert(!PlaceholderAddresses.empty());
1526	assert(PlaceholderAddresses.back().first == nullptr);
1527	assert(PlaceholderAddresses.back().second == placeholder);
1528	PlaceholderAddresses.back().first = signal;
1529	}
1530
1531	namespace {
1532	struct ReplacePlaceholders {
1533	CodeGenModule &CGM;
1534
1535	/// The base address of the global.
1536	llvm::Constant *Base;
1537	llvm::Type *BaseValueTy = nullptr;
1538
1539	/// The placeholder addresses that were registered during emission.
1540	llvm::DenseMap<llvm::Constant*, llvm::GlobalVariable*> PlaceholderAddresses;
1541
1542	/// The locations of the placeholder signals.
1543	llvm::DenseMap<llvm::GlobalVariable*, llvm::Constant*> Locations;
1544
1545	/// The current index stack. We use a simple unsigned stack because
1546	/// we assume that placeholders will be relatively sparse in the
1547	/// initializer, but we cache the index values we find just in case.
1548	llvm::SmallVector<unsigned, 8> Indices;
1549	llvm::SmallVector<llvm::Constant*, 8> IndexValues;
1550
1551	ReplacePlaceholders(CodeGenModule &CGM, llvm::Constant *base,
1552	ArrayRef<std::pair<llvm::Constant*,
1553	llvm::GlobalVariable*>> addresses)
1554	: CGM(CGM), Base(base),
1555	PlaceholderAddresses(addresses.begin(), addresses.end()) {
1556	}
1557
1558	void replaceInInitializer(llvm::Constant *init) {
1559	// Remember the type of the top-most initializer.
1560	BaseValueTy = init->getType();
1561
1562	// Initialize the stack.
1563	Indices.push_back(Elt: 0);
1564	IndexValues.push_back(Elt: nullptr);
1565
1566	// Recurse into the initializer.
1567	findLocations(init);
1568
1569	// Check invariants.
1570	assert(IndexValues.size() == Indices.size() && "mismatch");
1571	assert(Indices.size() == 1 && "didn't pop all indices");
1572
1573	// Do the replacement; this basically invalidates 'init'.
1574	assert(Locations.size() == PlaceholderAddresses.size() &&
1575	"missed a placeholder?");
1576
1577	// We're iterating over a hashtable, so this would be a source of
1578	// non-determinism in compiler output *except* that we're just
1579	// messing around with llvm::Constant structures, which never itself
1580	// does anything that should be visible in compiler output.
1581	for (auto &entry : Locations) {
1582	assert(entry.first->getParent() == nullptr && "not a placeholder!");
1583	entry.first->replaceAllUsesWith(V: entry.second);
1584	entry.first->eraseFromParent();
1585	}
1586	}
1587
1588	private:
1589	void findLocations(llvm::Constant *init) {
1590	// Recurse into aggregates.
1591	if (auto agg = dyn_cast<llvm::ConstantAggregate>(Val: init)) {
1592	for (unsigned i = 0, e = agg->getNumOperands(); i != e; ++i) {
1593	Indices.push_back(Elt: i);
1594	IndexValues.push_back(Elt: nullptr);
1595
1596	findLocations(init: agg->getOperand(i_nocapture: i));
1597
1598	IndexValues.pop_back();
1599	Indices.pop_back();
1600	}
1601	return;
1602	}
1603
1604	// Otherwise, check for registered constants.
1605	while (true) {
1606	auto it = PlaceholderAddresses.find(Val: init);
1607	if (it != PlaceholderAddresses.end()) {
1608	setLocation(it->second);
1609	break;
1610	}
1611
1612	// Look through bitcasts or other expressions.
1613	if (auto expr = dyn_cast<llvm::ConstantExpr>(Val: init)) {
1614	init = expr->getOperand(i_nocapture: 0);
1615	} else {
1616	break;
1617	}
1618	}
1619	}
1620
1621	void setLocation(llvm::GlobalVariable *placeholder) {
1622	assert(!Locations.contains(placeholder) &&
1623	"already found location for placeholder!");
1624
1625	// Lazily fill in IndexValues with the values from Indices.
1626	// We do this in reverse because we should always have a strict
1627	// prefix of indices from the start.
1628	assert(Indices.size() == IndexValues.size());
1629	for (size_t i = Indices.size() - 1; i != size_t(-1); --i) {
1630	if (IndexValues[i]) {
1631	#ifndef NDEBUG
1632	for (size_t j = 0; j != i + 1; ++j) {
1633	assert(IndexValues[j] &&
1634	isa<llvm::ConstantInt>(IndexValues[j]) &&
1635	cast<llvm::ConstantInt>(IndexValues[j])->getZExtValue()
1636	== Indices[j]);
1637	}
1638	#endif
1639	break;
1640	}
1641
1642	IndexValues[i] = llvm::ConstantInt::get(Ty: CGM.Int32Ty, V: Indices[i]);
1643	}
1644
1645	llvm::Constant *location = llvm::ConstantExpr::getInBoundsGetElementPtr(
1646	Ty: BaseValueTy, C: Base, IdxList: IndexValues);
1647
1648	Locations.insert(KV: {placeholder, location});
1649	}
1650	};
1651	}
1652
1653	void ConstantEmitter::finalize(llvm::GlobalVariable *global) {
1654	assert(InitializedNonAbstract &&
1655	"finalizing emitter that was used for abstract emission?");
1656	assert(!Finalized && "finalizing emitter multiple times");
1657	assert(global->getInitializer());
1658
1659	// Note that we might also be Failed.
1660	Finalized = true;
1661
1662	if (!PlaceholderAddresses.empty()) {
1663	ReplacePlaceholders(CGM, global, PlaceholderAddresses)
1664	.replaceInInitializer(init: global->getInitializer());
1665	PlaceholderAddresses.clear(); // satisfy
1666	}
1667	}
1668
1669	ConstantEmitter::~ConstantEmitter() {
1670	assert((!InitializedNonAbstract || Finalized || Failed) &&
1671	"not finalized after being initialized for non-abstract emission");
1672	assert(PlaceholderAddresses.empty() && "unhandled placeholders");
1673	}
1674
1675	static QualType getNonMemoryType(CodeGenModule &CGM, QualType type) {
1676	if (auto AT = type->getAs<AtomicType>()) {
1677	return CGM.getContext().getQualifiedType(T: AT->getValueType(),
1678	Qs: type.getQualifiers());
1679	}
1680	return type;
1681	}
1682
1683	llvm::Constant *ConstantEmitter::tryEmitPrivateForVarInit(const VarDecl &D) {
1684	// Make a quick check if variable can be default NULL initialized
1685	// and avoid going through rest of code which may do, for c++11,
1686	// initialization of memory to all NULLs.
1687	if (!D.hasLocalStorage()) {
1688	QualType Ty = CGM.getContext().getBaseElementType(D.getType());
1689	if (Ty->isRecordType())
1690	if (const CXXConstructExpr *E =
1691	dyn_cast_or_null<CXXConstructExpr>(Val: D.getInit())) {
1692	const CXXConstructorDecl *CD = E->getConstructor();
1693	if (CD->isTrivial() && CD->isDefaultConstructor())
1694	return CGM.EmitNullConstant(T: D.getType());
1695	}
1696	}
1697	InConstantContext = D.hasConstantInitialization();
1698
1699	QualType destType = D.getType();
1700	const Expr *E = D.getInit();
1701	assert(E && "No initializer to emit");
1702
1703	if (!destType->isReferenceType()) {
1704	QualType nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1705	if (llvm::Constant *C = ConstExprEmitter(*this).Visit(E, nonMemoryDestType))
1706	return emitForMemory(C, T: destType);
1707	}
1708
1709	// Try to emit the initializer. Note that this can allow some things that
1710	// are not allowed by tryEmitPrivateForMemory alone.
1711	if (APValue *value = D.evaluateValue())
1712	return tryEmitPrivateForMemory(value: *value, T: destType);
1713
1714	return nullptr;
1715	}
1716
1717	llvm::Constant *
1718	ConstantEmitter::tryEmitAbstractForMemory(const Expr *E, QualType destType) {
1719	auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1720	auto C = tryEmitAbstract(E, destType: nonMemoryDestType);
1721	return (C ? emitForMemory(C, T: destType) : nullptr);
1722	}
1723
1724	llvm::Constant *
1725	ConstantEmitter::tryEmitAbstractForMemory(const APValue &value,
1726	QualType destType) {
1727	auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1728	auto C = tryEmitAbstract(value, destType: nonMemoryDestType);
1729	return (C ? emitForMemory(C, T: destType) : nullptr);
1730	}
1731
1732	llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const Expr *E,
1733	QualType destType) {
1734	auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1735	llvm::Constant *C = tryEmitPrivate(E, T: nonMemoryDestType);
1736	return (C ? emitForMemory(C, T: destType) : nullptr);
1737	}
1738
1739	llvm::Constant *ConstantEmitter::tryEmitPrivateForMemory(const APValue &value,
1740	QualType destType) {
1741	auto nonMemoryDestType = getNonMemoryType(CGM, type: destType);
1742	auto C = tryEmitPrivate(value, T: nonMemoryDestType);
1743	return (C ? emitForMemory(C, T: destType) : nullptr);
1744	}
1745
1746	llvm::Constant *ConstantEmitter::emitForMemory(CodeGenModule &CGM,
1747	llvm::Constant *C,
1748	QualType destType) {
1749	// For an _Atomic-qualified constant, we may need to add tail padding.
1750	if (auto AT = destType->getAs<AtomicType>()) {
1751	QualType destValueType = AT->getValueType();
1752	C = emitForMemory(CGM, C, destType: destValueType);
1753
1754	uint64_t innerSize = CGM.getContext().getTypeSize(T: destValueType);
1755	uint64_t outerSize = CGM.getContext().getTypeSize(T: destType);
1756	if (innerSize == outerSize)
1757	return C;
1758
1759	assert(innerSize < outerSize && "emitted over-large constant for atomic");
1760	llvm::Constant *elts[] = {
1761	C,
1762	llvm::ConstantAggregateZero::get(
1763	Ty: llvm::ArrayType::get(ElementType: CGM.Int8Ty, NumElements: (outerSize - innerSize) / 8))
1764	};
1765	return llvm::ConstantStruct::getAnon(V: elts);
1766	}
1767
1768	// Zero-extend bool.
1769	if (C->getType()->isIntegerTy(Bitwidth: 1) && !destType->isBitIntType()) {
1770	llvm::Type *boolTy = CGM.getTypes().ConvertTypeForMem(T: destType);
1771	llvm::Constant *Res = llvm::ConstantFoldCastOperand(
1772	Opcode: llvm::Instruction::ZExt, C, DestTy: boolTy, DL: CGM.getDataLayout());
1773	assert(Res && "Constant folding must succeed");
1774	return Res;
1775	}
1776
1777	return C;
1778	}
1779
1780	llvm::Constant *ConstantEmitter::tryEmitPrivate(const Expr *E,
1781	QualType destType) {
1782	assert(!destType->isVoidType() && "can't emit a void constant");
1783
1784	if (!destType->isReferenceType())
1785	if (llvm::Constant *C = ConstExprEmitter(*this).Visit(E, destType))
1786	return C;
1787
1788	Expr::EvalResult Result;
1789
1790	bool Success = false;
1791
1792	if (destType->isReferenceType())
1793	Success = E->EvaluateAsLValue(Result, Ctx: CGM.getContext());
1794	else
1795	Success = E->EvaluateAsRValue(Result, Ctx: CGM.getContext(), InConstantContext);
1796
1797	if (Success && !Result.HasSideEffects)
1798	return tryEmitPrivate(value: Result.Val, T: destType);
1799
1800	return nullptr;
1801	}
1802
1803	llvm::Constant *CodeGenModule::getNullPointer(llvm::PointerType *T, QualType QT) {
1804	return getTargetCodeGenInfo().getNullPointer(CGM: *this, T, QT);
1805	}
1806
1807	namespace {
1808	/// A struct which can be used to peephole certain kinds of finalization
1809	/// that normally happen during l-value emission.
1810	struct ConstantLValue {
1811	llvm::Constant *Value;
1812	bool HasOffsetApplied;
1813
1814	/*implicit*/ ConstantLValue(llvm::Constant *value,
1815	bool hasOffsetApplied = false)
1816	: Value(value), HasOffsetApplied(hasOffsetApplied) {}
1817
1818	/*implicit*/ ConstantLValue(ConstantAddress address)
1819	: ConstantLValue(address.getPointer()) {}
1820	};
1821
1822	/// A helper class for emitting constant l-values.
1823	class ConstantLValueEmitter : public ConstStmtVisitor<ConstantLValueEmitter,
1824	ConstantLValue> {
1825	CodeGenModule &CGM;
1826	ConstantEmitter &Emitter;
1827	const APValue &Value;
1828	QualType DestType;
1829
1830	// Befriend StmtVisitorBase so that we don't have to expose Visit*.
1831	friend StmtVisitorBase;
1832
1833	public:
1834	ConstantLValueEmitter(ConstantEmitter &emitter, const APValue &value,
1835	QualType destType)
1836	: CGM(emitter.CGM), Emitter(emitter), Value(value), DestType(destType) {}
1837
1838	llvm::Constant *tryEmit();
1839
1840	private:
1841	llvm::Constant *tryEmitAbsolute(llvm::Type *destTy);
1842	ConstantLValue tryEmitBase(const APValue::LValueBase &base);
1843
1844	ConstantLValue VisitStmt(const Stmt *S) { return nullptr; }
1845	ConstantLValue VisitConstantExpr(const ConstantExpr *E);
1846	ConstantLValue VisitCompoundLiteralExpr(const CompoundLiteralExpr *E);
1847	ConstantLValue VisitStringLiteral(const StringLiteral *E);
1848	ConstantLValue VisitObjCBoxedExpr(const ObjCBoxedExpr *E);
1849	ConstantLValue VisitObjCEncodeExpr(const ObjCEncodeExpr *E);
1850	ConstantLValue VisitObjCStringLiteral(const ObjCStringLiteral *E);
1851	ConstantLValue VisitPredefinedExpr(const PredefinedExpr *E);
1852	ConstantLValue VisitAddrLabelExpr(const AddrLabelExpr *E);
1853	ConstantLValue VisitCallExpr(const CallExpr *E);
1854	ConstantLValue VisitBlockExpr(const BlockExpr *E);
1855	ConstantLValue VisitCXXTypeidExpr(const CXXTypeidExpr *E);
1856	ConstantLValue VisitMaterializeTemporaryExpr(
1857	const MaterializeTemporaryExpr *E);
1858
1859	bool hasNonZeroOffset() const {
1860	return !Value.getLValueOffset().isZero();
1861	}
1862
1863	/// Return the value offset.
1864	llvm::Constant *getOffset() {
1865	return llvm::ConstantInt::get(Ty: CGM.Int64Ty,
1866	V: Value.getLValueOffset().getQuantity());
1867	}
1868
1869	/// Apply the value offset to the given constant.
1870	llvm::Constant *applyOffset(llvm::Constant *C) {
1871	if (!hasNonZeroOffset())
1872	return C;
1873
1874	return llvm::ConstantExpr::getGetElementPtr(Ty: CGM.Int8Ty, C, Idx: getOffset());
1875	}
1876	};
1877
1878	}
1879
1880	llvm::Constant *ConstantLValueEmitter::tryEmit() {
1881	const APValue::LValueBase &base = Value.getLValueBase();
1882
1883	// The destination type should be a pointer or reference
1884	// type, but it might also be a cast thereof.
1885	//
1886	// FIXME: the chain of casts required should be reflected in the APValue.
1887	// We need this in order to correctly handle things like a ptrtoint of a
1888	// non-zero null pointer and addrspace casts that aren't trivially
1889	// represented in LLVM IR.
1890	auto destTy = CGM.getTypes().ConvertTypeForMem(DestType);
1891	assert(isa<llvm::IntegerType>(destTy) || isa<llvm::PointerType>(destTy));
1892
1893	// If there's no base at all, this is a null or absolute pointer,
1894	// possibly cast back to an integer type.
1895	if (!base) {
1896	return tryEmitAbsolute(destTy: destTy);
1897	}
1898
1899	// Otherwise, try to emit the base.
1900	ConstantLValue result = tryEmitBase(base);
1901
1902	// If that failed, we're done.
1903	llvm::Constant *value = result.Value;
1904	if (!value) return nullptr;
1905
1906	// Apply the offset if necessary and not already done.
1907	if (!result.HasOffsetApplied) {
1908	value = applyOffset(C: value);
1909	}
1910
1911	// Convert to the appropriate type; this could be an lvalue for
1912	// an integer. FIXME: performAddrSpaceCast
1913	if (isa<llvm::PointerType>(destTy))
1914	return llvm::ConstantExpr::getPointerCast(C: value, Ty: destTy);
1915
1916	return llvm::ConstantExpr::getPtrToInt(C: value, Ty: destTy);
1917	}
1918
1919	/// Try to emit an absolute l-value, such as a null pointer or an integer
1920	/// bitcast to pointer type.
1921	llvm::Constant *
1922	ConstantLValueEmitter::tryEmitAbsolute(llvm::Type *destTy) {
1923	// If we're producing a pointer, this is easy.
1924	auto destPtrTy = cast<llvm::PointerType>(Val: destTy);
1925	if (Value.isNullPointer()) {
1926	// FIXME: integer offsets from non-zero null pointers.
1927	return CGM.getNullPointer(destPtrTy, DestType);
1928	}
1929
1930	// Convert the integer to a pointer-sized integer before converting it
1931	// to a pointer.
1932	// FIXME: signedness depends on the original integer type.
1933	auto intptrTy = CGM.getDataLayout().getIntPtrType(destPtrTy);
1934	llvm::Constant *C;
1935	C = llvm::ConstantFoldIntegerCast(C: getOffset(), DestTy: intptrTy, /*isSigned*/ IsSigned: false,
1936	DL: CGM.getDataLayout());
1937	assert(C && "Must have folded, as Offset is a ConstantInt");
1938	C = llvm::ConstantExpr::getIntToPtr(C, Ty: destPtrTy);
1939	return C;
1940	}
1941
1942	ConstantLValue
1943	ConstantLValueEmitter::tryEmitBase(const APValue::LValueBase &base) {
1944	// Handle values.
1945	if (const ValueDecl *D = base.dyn_cast<const ValueDecl*>()) {
1946	// The constant always points to the canonical declaration. We want to look
1947	// at properties of the most recent declaration at the point of emission.
1948	D = cast<ValueDecl>(D->getMostRecentDecl());
1949
1950	if (D->hasAttr<WeakRefAttr>())
1951	return CGM.GetWeakRefReference(VD: D).getPointer();
1952
1953	if (auto FD = dyn_cast<FunctionDecl>(Val: D))
1954	return CGM.GetAddrOfFunction(GD: FD);
1955
1956	if (auto VD = dyn_cast<VarDecl>(Val: D)) {
1957	// We can never refer to a variable with local storage.
1958	if (!VD->hasLocalStorage()) {
1959	if (VD->isFileVarDecl() || VD->hasExternalStorage())
1960	return CGM.GetAddrOfGlobalVar(D: VD);
1961
1962	if (VD->isLocalVarDecl()) {
1963	return CGM.getOrCreateStaticVarDecl(
1964	D: *VD, Linkage: CGM.getLLVMLinkageVarDefinition(VD));
1965	}
1966	}
1967	}
1968
1969	if (auto *GD = dyn_cast<MSGuidDecl>(Val: D))
1970	return CGM.GetAddrOfMSGuidDecl(GD);
1971
1972	if (auto *GCD = dyn_cast<UnnamedGlobalConstantDecl>(Val: D))
1973	return CGM.GetAddrOfUnnamedGlobalConstantDecl(GCD);
1974
1975	if (auto *TPO = dyn_cast<TemplateParamObjectDecl>(Val: D))
1976	return CGM.GetAddrOfTemplateParamObject(TPO);
1977
1978	return nullptr;
1979	}
1980
1981	// Handle typeid(T).
1982	if (TypeInfoLValue TI = base.dyn_cast<TypeInfoLValue>())
1983	return CGM.GetAddrOfRTTIDescriptor(Ty: QualType(TI.getType(), 0));
1984
1985	// Otherwise, it must be an expression.
1986	return Visit(base.get<const Expr*>());
1987	}
1988
1989	ConstantLValue
1990	ConstantLValueEmitter::VisitConstantExpr(const ConstantExpr *E) {
1991	if (llvm::Constant *Result = Emitter.tryEmitConstantExpr(CE: E))
1992	return Result;
1993	return Visit(E->getSubExpr());
1994	}
1995
1996	ConstantLValue
1997	ConstantLValueEmitter::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
1998	ConstantEmitter CompoundLiteralEmitter(CGM, Emitter.CGF);
1999	CompoundLiteralEmitter.setInConstantContext(Emitter.isInConstantContext());
2000	return tryEmitGlobalCompoundLiteral(emitter&: CompoundLiteralEmitter, E);
2001	}
2002
2003	ConstantLValue
2004	ConstantLValueEmitter::VisitStringLiteral(const StringLiteral *E) {
2005	return CGM.GetAddrOfConstantStringFromLiteral(S: E);
2006	}
2007
2008	ConstantLValue
2009	ConstantLValueEmitter::VisitObjCEncodeExpr(const ObjCEncodeExpr *E) {
2010	return CGM.GetAddrOfConstantStringFromObjCEncode(E);
2011	}
2012
2013	static ConstantLValue emitConstantObjCStringLiteral(const StringLiteral *S,
2014	QualType T,
2015	CodeGenModule &CGM) {
2016	auto C = CGM.getObjCRuntime().GenerateConstantString(S);
2017	return C.withElementType(ElemTy: CGM.getTypes().ConvertTypeForMem(T));
2018	}
2019
2020	ConstantLValue
2021	ConstantLValueEmitter::VisitObjCStringLiteral(const ObjCStringLiteral *E) {
2022	return emitConstantObjCStringLiteral(E->getString(), E->getType(), CGM);
2023	}
2024
2025	ConstantLValue
2026	ConstantLValueEmitter::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) {
2027	assert(E->isExpressibleAsConstantInitializer() &&
2028	"this boxed expression can't be emitted as a compile-time constant");
2029	const auto *SL = cast<StringLiteral>(Val: E->getSubExpr()->IgnoreParenCasts());
2030	return emitConstantObjCStringLiteral(SL, E->getType(), CGM);
2031	}
2032
2033	ConstantLValue
2034	ConstantLValueEmitter::VisitPredefinedExpr(const PredefinedExpr *E) {
2035	return CGM.GetAddrOfConstantStringFromLiteral(S: E->getFunctionName());
2036	}
2037
2038	ConstantLValue
2039	ConstantLValueEmitter::VisitAddrLabelExpr(const AddrLabelExpr *E) {
2040	assert(Emitter.CGF && "Invalid address of label expression outside function");
2041	llvm::Constant *Ptr = Emitter.CGF->GetAddrOfLabel(L: E->getLabel());
2042	return Ptr;
2043	}
2044
2045	ConstantLValue
2046	ConstantLValueEmitter::VisitCallExpr(const CallExpr *E) {
2047	unsigned builtin = E->getBuiltinCallee();
2048	if (builtin == Builtin::BI__builtin_function_start)
2049	return CGM.GetFunctionStart(
2050	Decl: E->getArg(Arg: 0)->getAsBuiltinConstantDeclRef(Context: CGM.getContext()));
2051	if (builtin != Builtin::BI__builtin___CFStringMakeConstantString &&
2052	builtin != Builtin::BI__builtin___NSStringMakeConstantString)
2053	return nullptr;
2054
2055	const auto *Literal = cast<StringLiteral>(Val: E->getArg(Arg: 0)->IgnoreParenCasts());
2056	if (builtin == Builtin::BI__builtin___NSStringMakeConstantString) {
2057	return CGM.getObjCRuntime().GenerateConstantString(Literal);
2058	} else {
2059	// FIXME: need to deal with UCN conversion issues.
2060	return CGM.GetAddrOfConstantCFString(Literal);
2061	}
2062	}
2063
2064	ConstantLValue
2065	ConstantLValueEmitter::VisitBlockExpr(const BlockExpr *E) {
2066	StringRef functionName;
2067	if (auto CGF = Emitter.CGF)
2068	functionName = CGF->CurFn->getName();
2069	else
2070	functionName = "global";
2071
2072	return CGM.GetAddrOfGlobalBlock(BE: E, Name: functionName);
2073	}
2074
2075	ConstantLValue
2076	ConstantLValueEmitter::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
2077	QualType T;
2078	if (E->isTypeOperand())
2079	T = E->getTypeOperand(Context&: CGM.getContext());
2080	else
2081	T = E->getExprOperand()->getType();
2082	return CGM.GetAddrOfRTTIDescriptor(Ty: T);
2083	}
2084
2085	ConstantLValue
2086	ConstantLValueEmitter::VisitMaterializeTemporaryExpr(
2087	const MaterializeTemporaryExpr *E) {
2088	assert(E->getStorageDuration() == SD_Static);
2089	const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments();
2090	return CGM.GetAddrOfGlobalTemporary(E, Inner);
2091	}
2092
2093	llvm::Constant *ConstantEmitter::tryEmitPrivate(const APValue &Value,
2094	QualType DestType) {
2095	switch (Value.getKind()) {
2096	case APValue::None:
2097	case APValue::Indeterminate:
2098	// Out-of-lifetime and indeterminate values can be modeled as 'undef'.
2099	return llvm::UndefValue::get(T: CGM.getTypes().ConvertType(T: DestType));
2100	case APValue::LValue:
2101	return ConstantLValueEmitter(*this, Value, DestType).tryEmit();
2102	case APValue::Int:
2103	return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Value.getInt());
2104	case APValue::FixedPoint:
2105	return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2106	V: Value.getFixedPoint().getValue());
2107	case APValue::ComplexInt: {
2108	llvm::Constant *Complex[2];
2109
2110	Complex[0] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2111	V: Value.getComplexIntReal());
2112	Complex[1] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2113	V: Value.getComplexIntImag());
2114
2115	// FIXME: the target may want to specify that this is packed.
2116	llvm::StructType *STy =
2117	llvm::StructType::get(elt1: Complex[0]->getType(), elts: Complex[1]->getType());
2118	return llvm::ConstantStruct::get(T: STy, V: Complex);
2119	}
2120	case APValue::Float: {
2121	const llvm::APFloat &Init = Value.getFloat();
2122	if (&Init.getSemantics() == &llvm::APFloat::IEEEhalf() &&
2123	!CGM.getContext().getLangOpts().NativeHalfType &&
2124	CGM.getContext().getTargetInfo().useFP16ConversionIntrinsics())
2125	return llvm::ConstantInt::get(Context&: CGM.getLLVMContext(),
2126	V: Init.bitcastToAPInt());
2127	else
2128	return llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Init);
2129	}
2130	case APValue::ComplexFloat: {
2131	llvm::Constant *Complex[2];
2132
2133	Complex[0] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(),
2134	V: Value.getComplexFloatReal());
2135	Complex[1] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(),
2136	V: Value.getComplexFloatImag());
2137
2138	// FIXME: the target may want to specify that this is packed.
2139	llvm::StructType *STy =
2140	llvm::StructType::get(elt1: Complex[0]->getType(), elts: Complex[1]->getType());
2141	return llvm::ConstantStruct::get(T: STy, V: Complex);
2142	}
2143	case APValue::Vector: {
2144	unsigned NumElts = Value.getVectorLength();
2145	SmallVector<llvm::Constant *, 4> Inits(NumElts);
2146
2147	for (unsigned I = 0; I != NumElts; ++I) {
2148	const APValue &Elt = Value.getVectorElt(I);
2149	if (Elt.isInt())
2150	Inits[I] = llvm::ConstantInt::get(Context&: CGM.getLLVMContext(), V: Elt.getInt());
2151	else if (Elt.isFloat())
2152	Inits[I] = llvm::ConstantFP::get(Context&: CGM.getLLVMContext(), V: Elt.getFloat());
2153	else if (Elt.isIndeterminate())
2154	Inits[I] = llvm::UndefValue::get(T: CGM.getTypes().ConvertType(
2155	T: DestType->castAs<VectorType>()->getElementType()));
2156	else
2157	llvm_unreachable("unsupported vector element type");
2158	}
2159	return llvm::ConstantVector::get(V: Inits);
2160	}
2161	case APValue::AddrLabelDiff: {
2162	const AddrLabelExpr *LHSExpr = Value.getAddrLabelDiffLHS();
2163	const AddrLabelExpr *RHSExpr = Value.getAddrLabelDiffRHS();
2164	llvm::Constant *LHS = tryEmitPrivate(LHSExpr, LHSExpr->getType());
2165	llvm::Constant *RHS = tryEmitPrivate(RHSExpr, RHSExpr->getType());
2166	if (!LHS || !RHS) return nullptr;
2167
2168	// Compute difference
2169	llvm::Type *ResultType = CGM.getTypes().ConvertType(T: DestType);
2170	LHS = llvm::ConstantExpr::getPtrToInt(C: LHS, Ty: CGM.IntPtrTy);
2171	RHS = llvm::ConstantExpr::getPtrToInt(C: RHS, Ty: CGM.IntPtrTy);
2172	llvm::Constant *AddrLabelDiff = llvm::ConstantExpr::getSub(C1: LHS, C2: RHS);
2173
2174	// LLVM is a bit sensitive about the exact format of the
2175	// address-of-label difference; make sure to truncate after
2176	// the subtraction.
2177	return llvm::ConstantExpr::getTruncOrBitCast(C: AddrLabelDiff, Ty: ResultType);
2178	}
2179	case APValue::Struct:
2180	case APValue::Union:
2181	return ConstStructBuilder::BuildStruct(Emitter&: *this, Val: Value, ValTy: DestType);
2182	case APValue::Array: {
2183	const ArrayType *ArrayTy = CGM.getContext().getAsArrayType(T: DestType);
2184	unsigned NumElements = Value.getArraySize();
2185	unsigned NumInitElts = Value.getArrayInitializedElts();
2186
2187	// Emit array filler, if there is one.
2188	llvm::Constant *Filler = nullptr;
2189	if (Value.hasArrayFiller()) {
2190	Filler = tryEmitAbstractForMemory(value: Value.getArrayFiller(),
2191	destType: ArrayTy->getElementType());
2192	if (!Filler)
2193	return nullptr;
2194	}
2195
2196	// Emit initializer elements.
2197	SmallVector<llvm::Constant*, 16> Elts;
2198	if (Filler && Filler->isNullValue())
2199	Elts.reserve(N: NumInitElts + 1);
2200	else
2201	Elts.reserve(N: NumElements);
2202
2203	llvm::Type *CommonElementType = nullptr;
2204	for (unsigned I = 0; I < NumInitElts; ++I) {
2205	llvm::Constant *C = tryEmitPrivateForMemory(
2206	value: Value.getArrayInitializedElt(I), destType: ArrayTy->getElementType());
2207	if (!C) return nullptr;
2208
2209	if (I == 0)
2210	CommonElementType = C->getType();
2211	else if (C->getType() != CommonElementType)
2212	CommonElementType = nullptr;
2213	Elts.push_back(Elt: C);
2214	}
2215
2216	llvm::ArrayType *Desired =
2217	cast<llvm::ArrayType>(Val: CGM.getTypes().ConvertType(T: DestType));
2218
2219	// Fix the type of incomplete arrays if the initializer isn't empty.
2220	if (DestType->isIncompleteArrayType() && !Elts.empty())
2221	Desired = llvm::ArrayType::get(ElementType: Desired->getElementType(), NumElements: Elts.size());
2222
2223	return EmitArrayConstant(CGM, DesiredType: Desired, CommonElementType, ArrayBound: NumElements, Elements&: Elts,
2224	Filler);
2225	}
2226	case APValue::MemberPointer:
2227	return CGM.getCXXABI().EmitMemberPointer(MP: Value, MPT: DestType);
2228	}
2229	llvm_unreachable("Unknown APValue kind");
2230	}
2231
2232	llvm::GlobalVariable *CodeGenModule::getAddrOfConstantCompoundLiteralIfEmitted(
2233	const CompoundLiteralExpr *E) {
2234	return EmittedCompoundLiterals.lookup(Val: E);
2235	}
2236
2237	void CodeGenModule::setAddrOfConstantCompoundLiteral(
2238	const CompoundLiteralExpr *CLE, llvm::GlobalVariable *GV) {
2239	bool Ok = EmittedCompoundLiterals.insert(KV: std::make_pair(x&: CLE, y&: GV)).second;
2240	(void)Ok;
2241	assert(Ok && "CLE has already been emitted!");
2242	}
2243
2244	ConstantAddress
2245	CodeGenModule::GetAddrOfConstantCompoundLiteral(const CompoundLiteralExpr *E) {
2246	assert(E->isFileScope() && "not a file-scope compound literal expr");
2247	ConstantEmitter emitter(*this);
2248	return tryEmitGlobalCompoundLiteral(emitter, E);
2249	}
2250
2251	llvm::Constant *
2252	CodeGenModule::getMemberPointerConstant(const UnaryOperator *uo) {
2253	// Member pointer constants always have a very particular form.
2254	const MemberPointerType *type = cast<MemberPointerType>(uo->getType());
2255	const ValueDecl *decl = cast<DeclRefExpr>(Val: uo->getSubExpr())->getDecl();
2256
2257	// A member function pointer.
2258	if (const CXXMethodDecl *method = dyn_cast<CXXMethodDecl>(Val: decl))
2259	return getCXXABI().EmitMemberFunctionPointer(MD: method);
2260
2261	// Otherwise, a member data pointer.
2262	uint64_t fieldOffset = getContext().getFieldOffset(FD: decl);
2263	CharUnits chars = getContext().toCharUnitsFromBits(BitSize: (int64_t) fieldOffset);
2264	return getCXXABI().EmitMemberDataPointer(MPT: type, offset: chars);
2265	}
2266
2267	static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
2268	llvm::Type *baseType,
2269	const CXXRecordDecl *base);
2270
2271	static llvm::Constant *EmitNullConstant(CodeGenModule &CGM,
2272	const RecordDecl *record,
2273	bool asCompleteObject) {
2274	const CGRecordLayout &layout = CGM.getTypes().getCGRecordLayout(record);
2275	llvm::StructType *structure =
2276	(asCompleteObject ? layout.getLLVMType()
2277	: layout.getBaseSubobjectLLVMType());
2278
2279	unsigned numElements = structure->getNumElements();
2280	std::vector<llvm::Constant *> elements(numElements);
2281
2282	auto CXXR = dyn_cast<CXXRecordDecl>(Val: record);
2283	// Fill in all the bases.
2284	if (CXXR) {
2285	for (const auto &I : CXXR->bases()) {
2286	if (I.isVirtual()) {
2287	// Ignore virtual bases; if we're laying out for a complete
2288	// object, we'll lay these out later.
2289	continue;
2290	}
2291
2292	const CXXRecordDecl *base =
2293	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2294
2295	// Ignore empty bases.
2296	if (base->isEmpty() ||
2297	CGM.getContext().getASTRecordLayout(base).getNonVirtualSize()
2298	.isZero())
2299	continue;
2300
2301	unsigned fieldIndex = layout.getNonVirtualBaseLLVMFieldNo(RD: base);
2302	llvm::Type *baseType = structure->getElementType(N: fieldIndex);
2303	elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
2304	}
2305	}
2306
2307	// Fill in all the fields.
2308	for (const auto *Field : record->fields()) {
2309	// Fill in non-bitfields. (Bitfields always use a zero pattern, which we
2310	// will fill in later.)
2311	if (!Field->isBitField() && !Field->isZeroSize(Ctx: CGM.getContext())) {
2312	unsigned fieldIndex = layout.getLLVMFieldNo(FD: Field);
2313	elements[fieldIndex] = CGM.EmitNullConstant(T: Field->getType());
2314	}
2315
2316	// For unions, stop after the first named field.
2317	if (record->isUnion()) {
2318	if (Field->getIdentifier())
2319	break;
2320	if (const auto *FieldRD = Field->getType()->getAsRecordDecl())
2321	if (FieldRD->findFirstNamedDataMember())
2322	break;
2323	}
2324	}
2325
2326	// Fill in the virtual bases, if we're working with the complete object.
2327	if (CXXR && asCompleteObject) {
2328	for (const auto &I : CXXR->vbases()) {
2329	const CXXRecordDecl *base =
2330	cast<CXXRecordDecl>(Val: I.getType()->castAs<RecordType>()->getDecl());
2331
2332	// Ignore empty bases.
2333	if (base->isEmpty())
2334	continue;
2335
2336	unsigned fieldIndex = layout.getVirtualBaseIndex(base);
2337
2338	// We might have already laid this field out.
2339	if (elements[fieldIndex]) continue;
2340
2341	llvm::Type *baseType = structure->getElementType(N: fieldIndex);
2342	elements[fieldIndex] = EmitNullConstantForBase(CGM, baseType, base);
2343	}
2344	}
2345
2346	// Now go through all other fields and zero them out.
2347	for (unsigned i = 0; i != numElements; ++i) {
2348	if (!elements[i])
2349	elements[i] = llvm::Constant::getNullValue(Ty: structure->getElementType(N: i));
2350	}
2351
2352	return llvm::ConstantStruct::get(T: structure, V: elements);
2353	}
2354
2355	/// Emit the null constant for a base subobject.
2356	static llvm::Constant *EmitNullConstantForBase(CodeGenModule &CGM,
2357	llvm::Type *baseType,
2358	const CXXRecordDecl *base) {
2359	const CGRecordLayout &baseLayout = CGM.getTypes().getCGRecordLayout(base);
2360
2361	// Just zero out bases that don't have any pointer to data members.
2362	if (baseLayout.isZeroInitializableAsBase())
2363	return llvm::Constant::getNullValue(Ty: baseType);
2364
2365	// Otherwise, we can just use its null constant.
2366	return EmitNullConstant(CGM, base, /*asCompleteObject=*/false);
2367	}
2368
2369	llvm::Constant *ConstantEmitter::emitNullForMemory(CodeGenModule &CGM,
2370	QualType T) {
2371	return emitForMemory(CGM, C: CGM.EmitNullConstant(T), destType: T);
2372	}
2373
2374	llvm::Constant *CodeGenModule::EmitNullConstant(QualType T) {
2375	if (T->getAs<PointerType>())
2376	return getNullPointer(
2377	T: cast<llvm::PointerType>(Val: getTypes().ConvertTypeForMem(T)), QT: T);
2378
2379	if (getTypes().isZeroInitializable(T))
2380	return llvm::Constant::getNullValue(Ty: getTypes().ConvertTypeForMem(T));
2381
2382	if (const ConstantArrayType *CAT = Context.getAsConstantArrayType(T)) {
2383	llvm::ArrayType *ATy =
2384	cast<llvm::ArrayType>(Val: getTypes().ConvertTypeForMem(T));
2385
2386	QualType ElementTy = CAT->getElementType();
2387
2388	llvm::Constant *Element =
2389	ConstantEmitter::emitNullForMemory(CGM&: *this, T: ElementTy);
2390	unsigned NumElements = CAT->getZExtSize();
2391	SmallVector<llvm::Constant *, 8> Array(NumElements, Element);
2392	return llvm::ConstantArray::get(T: ATy, V: Array);
2393	}
2394
2395	if (const RecordType *RT = T->getAs<RecordType>())
2396	return ::EmitNullConstant(CGM&: *this, record: RT->getDecl(), /*complete object*/ asCompleteObject: true);
2397
2398	assert(T->isMemberDataPointerType() &&
2399	"Should only see pointers to data members here!");
2400
2401	return getCXXABI().EmitNullMemberPointer(MPT: T->castAs<MemberPointerType>());
2402	}
2403
2404	llvm::Constant *
2405	CodeGenModule::EmitNullConstantForBase(const CXXRecordDecl *Record) {
2406	return ::EmitNullConstant(*this, Record, false);
2407	}
2408