1	//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code to emit Aggregate Expr nodes as LLVM code.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGCXXABI.h"
14	#include "CGObjCRuntime.h"
15	#include "CodeGenFunction.h"
16	#include "CodeGenModule.h"
17	#include "ConstantEmitter.h"
18	#include "TargetInfo.h"
19	#include "clang/AST/ASTContext.h"
20	#include "clang/AST/Attr.h"
21	#include "clang/AST/DeclCXX.h"
22	#include "clang/AST/DeclTemplate.h"
23	#include "clang/AST/StmtVisitor.h"
24	#include "llvm/IR/Constants.h"
25	#include "llvm/IR/Function.h"
26	#include "llvm/IR/GlobalVariable.h"
27	#include "llvm/IR/IntrinsicInst.h"
28	#include "llvm/IR/Intrinsics.h"
29	using namespace clang;
30	using namespace CodeGen;
31
32	//===----------------------------------------------------------------------===//
33	// Aggregate Expression Emitter
34	//===----------------------------------------------------------------------===//
35
36	namespace llvm {
37	extern cl::opt<bool> EnableSingleByteCoverage;
38	} // namespace llvm
39
40	namespace {
41	class AggExprEmitter : public StmtVisitor<AggExprEmitter> {
42	CodeGenFunction &CGF;
43	CGBuilderTy &Builder;
44	AggValueSlot Dest;
45	bool IsResultUnused;
46
47	AggValueSlot EnsureSlot(QualType T) {
48	if (!Dest.isIgnored()) return Dest;
49	return CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
50	}
51	void EnsureDest(QualType T) {
52	if (!Dest.isIgnored()) return;
53	Dest = CGF.CreateAggTemp(T, Name: "agg.tmp.ensured");
54	}
55
56	// Calls `Fn` with a valid return value slot, potentially creating a temporary
57	// to do so. If a temporary is created, an appropriate copy into `Dest` will
58	// be emitted, as will lifetime markers.
59	//
60	// The given function should take a ReturnValueSlot, and return an RValue that
61	// points to said slot.
62	void withReturnValueSlot(const Expr *E,
63	llvm::function_ref<RValue(ReturnValueSlot)> Fn);
64
65	public:
66	AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused)
67	: CGF(cgf), Builder(CGF.Builder), Dest(Dest),
68	IsResultUnused(IsResultUnused) { }
69
70	//===--------------------------------------------------------------------===//
71	// Utilities
72	//===--------------------------------------------------------------------===//
73
74	/// EmitAggLoadOfLValue - Given an expression with aggregate type that
75	/// represents a value lvalue, this method emits the address of the lvalue,
76	/// then loads the result into DestPtr.
77	void EmitAggLoadOfLValue(const Expr *E);
78
79	enum ExprValueKind {
80	EVK_RValue,
81	EVK_NonRValue
82	};
83
84	/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
85	/// SrcIsRValue is true if source comes from an RValue.
86	void EmitFinalDestCopy(QualType type, const LValue &src,
87	ExprValueKind SrcValueKind = EVK_NonRValue);
88	void EmitFinalDestCopy(QualType type, RValue src);
89	void EmitCopy(QualType type, const AggValueSlot &dest,
90	const AggValueSlot &src);
91
92	void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, QualType ArrayQTy,
93	Expr *ExprToVisit, ArrayRef<Expr *> Args,
94	Expr *ArrayFiller);
95
96	AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) {
97	if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T))
98	return AggValueSlot::NeedsGCBarriers;
99	return AggValueSlot::DoesNotNeedGCBarriers;
100	}
101
102	bool TypeRequiresGCollection(QualType T);
103
104	//===--------------------------------------------------------------------===//
105	// Visitor Methods
106	//===--------------------------------------------------------------------===//
107
108	void Visit(Expr *E) {
109	ApplyDebugLocation DL(CGF, E);
110	StmtVisitor<AggExprEmitter>::Visit(E);
111	}
112
113	void VisitStmt(Stmt *S) {
114	CGF.ErrorUnsupported(S, Type: "aggregate expression");
115	}
116	void VisitParenExpr(ParenExpr *PE) { Visit(E: PE->getSubExpr()); }
117	void VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
118	Visit(E: GE->getResultExpr());
119	}
120	void VisitCoawaitExpr(CoawaitExpr *E) {
121	CGF.EmitCoawaitExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
122	}
123	void VisitCoyieldExpr(CoyieldExpr *E) {
124	CGF.EmitCoyieldExpr(E: *E, aggSlot: Dest, ignoreResult: IsResultUnused);
125	}
126	void VisitUnaryCoawait(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
127	void VisitUnaryExtension(UnaryOperator *E) { Visit(E: E->getSubExpr()); }
128	void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) {
129	return Visit(E: E->getReplacement());
130	}
131
132	void VisitConstantExpr(ConstantExpr *E) {
133	EnsureDest(T: E->getType());
134
135	if (llvm::Value *Result = ConstantEmitter(CGF).tryEmitConstantExpr(CE: E)) {
136	Address StoreDest = Dest.getAddress();
137	// The emitted value is guaranteed to have the same size as the
138	// destination but can have a different type. Just do a bitcast in this
139	// case to avoid incorrect GEPs.
140	if (Result->getType() != StoreDest.getType())
141	StoreDest = StoreDest.withElementType(ElemTy: Result->getType());
142
143	CGF.EmitAggregateStore(Val: Result, Dest: StoreDest,
144	DestIsVolatile: E->getType().isVolatileQualified());
145	return;
146	}
147	return Visit(E: E->getSubExpr());
148	}
149
150	// l-values.
151	void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); }
152	void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(ME); }
153	void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); }
154	void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); }
155	void VisitCompoundLiteralExpr(CompoundLiteralExpr *E);
156	void VisitArraySubscriptExpr(ArraySubscriptExpr *E) {
157	EmitAggLoadOfLValue(E);
158	}
159	void VisitPredefinedExpr(const PredefinedExpr *E) {
160	EmitAggLoadOfLValue(E);
161	}
162
163	// Operators.
164	void VisitCastExpr(CastExpr *E);
165	void VisitCallExpr(const CallExpr *E);
166	void VisitStmtExpr(const StmtExpr *E);
167	void VisitBinaryOperator(const BinaryOperator *BO);
168	void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO);
169	void VisitBinAssign(const BinaryOperator *E);
170	void VisitBinComma(const BinaryOperator *E);
171	void VisitBinCmp(const BinaryOperator *E);
172	void VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
173	Visit(E: E->getSemanticForm());
174	}
175
176	void VisitObjCMessageExpr(ObjCMessageExpr *E);
177	void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
178	EmitAggLoadOfLValue(E);
179	}
180
181	void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E);
182	void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
183	void VisitChooseExpr(const ChooseExpr *CE);
184	void VisitInitListExpr(InitListExpr *E);
185	void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
186	FieldDecl *InitializedFieldInUnion,
187	Expr *ArrayFiller);
188	void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
189	llvm::Value *outerBegin = nullptr);
190	void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E);
191	void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing.
192	void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
193	CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
194	Visit(E: DAE->getExpr());
195	}
196	void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
197	CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
198	Visit(E: DIE->getExpr());
199	}
200	void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E);
201	void VisitCXXConstructExpr(const CXXConstructExpr *E);
202	void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
203	void VisitLambdaExpr(LambdaExpr *E);
204	void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E);
205	void VisitExprWithCleanups(ExprWithCleanups *E);
206	void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E);
207	void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); }
208	void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E);
209	void VisitOpaqueValueExpr(OpaqueValueExpr *E);
210
211	void VisitPseudoObjectExpr(PseudoObjectExpr *E) {
212	if (E->isGLValue()) {
213	LValue LV = CGF.EmitPseudoObjectLValue(e: E);
214	return EmitFinalDestCopy(E->getType(), LV);
215	}
216
217	AggValueSlot Slot = EnsureSlot(T: E->getType());
218	bool NeedsDestruction =
219	!Slot.isExternallyDestructed() &&
220	E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
221	if (NeedsDestruction)
222	Slot.setExternallyDestructed();
223	CGF.EmitPseudoObjectRValue(e: E, slot: Slot);
224	if (NeedsDestruction)
225	CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Slot.getAddress(),
226	E->getType());
227	}
228
229	void VisitVAArgExpr(VAArgExpr *E);
230	void VisitCXXParenListInitExpr(CXXParenListInitExpr *E);
231	void VisitCXXParenListOrInitListExpr(Expr *ExprToVisit, ArrayRef<Expr *> Args,
232	Expr *ArrayFiller);
233
234	void EmitInitializationToLValue(Expr *E, LValue Address);
235	void EmitNullInitializationToLValue(LValue Address);
236	// case Expr::ChooseExprClass:
237	void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); }
238	void VisitAtomicExpr(AtomicExpr *E) {
239	RValue Res = CGF.EmitAtomicExpr(E);
240	EmitFinalDestCopy(E->getType(), Res);
241	}
242	void VisitPackIndexingExpr(PackIndexingExpr *E) {
243	Visit(E: E->getSelectedExpr());
244	}
245	};
246	} // end anonymous namespace.
247
248	//===----------------------------------------------------------------------===//
249	// Utilities
250	//===----------------------------------------------------------------------===//
251
252	/// EmitAggLoadOfLValue - Given an expression with aggregate type that
253	/// represents a value lvalue, this method emits the address of the lvalue,
254	/// then loads the result into DestPtr.
255	void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) {
256	LValue LV = CGF.EmitLValue(E);
257
258	// If the type of the l-value is atomic, then do an atomic load.
259	if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(Src: LV)) {
260	CGF.EmitAtomicLoad(LV, SL: E->getExprLoc(), Slot: Dest);
261	return;
262	}
263
264	EmitFinalDestCopy(type: E->getType(), src: LV);
265	}
266
267	/// True if the given aggregate type requires special GC API calls.
268	bool AggExprEmitter::TypeRequiresGCollection(QualType T) {
269	// Only record types have members that might require garbage collection.
270	const RecordType *RecordTy = T->getAs<RecordType>();
271	if (!RecordTy) return false;
272
273	// Don't mess with non-trivial C++ types.
274	RecordDecl *Record = RecordTy->getDecl();
275	if (isa<CXXRecordDecl>(Val: Record) &&
276	(cast<CXXRecordDecl>(Val: Record)->hasNonTrivialCopyConstructor() ||
277	!cast<CXXRecordDecl>(Val: Record)->hasTrivialDestructor()))
278	return false;
279
280	// Check whether the type has an object member.
281	return Record->hasObjectMember();
282	}
283
284	void AggExprEmitter::withReturnValueSlot(
285	const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) {
286	QualType RetTy = E->getType();
287	bool RequiresDestruction =
288	!Dest.isExternallyDestructed() &&
289	RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct;
290
291	// If it makes no observable difference, save a memcpy + temporary.
292	//
293	// We need to always provide our own temporary if destruction is required.
294	// Otherwise, EmitCall will emit its own, notice that it's "unused", and end
295	// its lifetime before we have the chance to emit a proper destructor call.
296	bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() ||
297	(RequiresDestruction && Dest.isIgnored());
298
299	Address RetAddr = Address::invalid();
300	RawAddress RetAllocaAddr = RawAddress::invalid();
301
302	EHScopeStack::stable_iterator LifetimeEndBlock;
303	llvm::Value *LifetimeSizePtr = nullptr;
304	llvm::IntrinsicInst *LifetimeStartInst = nullptr;
305	if (!UseTemp) {
306	RetAddr = Dest.getAddress();
307	} else {
308	RetAddr = CGF.CreateMemTemp(T: RetTy, Name: "tmp", Alloca: &RetAllocaAddr);
309	llvm::TypeSize Size =
310	CGF.CGM.getDataLayout().getTypeAllocSize(Ty: CGF.ConvertTypeForMem(T: RetTy));
311	LifetimeSizePtr = CGF.EmitLifetimeStart(Size, Addr: RetAllocaAddr.getPointer());
312	if (LifetimeSizePtr) {
313	LifetimeStartInst =
314	cast<llvm::IntrinsicInst>(Val: std::prev(x: Builder.GetInsertPoint()));
315	assert(LifetimeStartInst->getIntrinsicID() ==
316	llvm::Intrinsic::lifetime_start &&
317	"Last insertion wasn't a lifetime.start?");
318
319	CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>(
320	kind: NormalEHLifetimeMarker, A: RetAllocaAddr, A: LifetimeSizePtr);
321	LifetimeEndBlock = CGF.EHStack.stable_begin();
322	}
323	}
324
325	RValue Src =
326	EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused,
327	Dest.isExternallyDestructed()));
328
329	if (!UseTemp)
330	return;
331
332	assert(Dest.isIgnored() || Dest.emitRawPointer(CGF) !=
333	Src.getAggregatePointer(E->getType(), CGF));
334	EmitFinalDestCopy(type: E->getType(), src: Src);
335
336	if (!RequiresDestruction && LifetimeStartInst) {
337	// If there's no dtor to run, the copy was the last use of our temporary.
338	// Since we're not guaranteed to be in an ExprWithCleanups, clean up
339	// eagerly.
340	CGF.DeactivateCleanupBlock(Cleanup: LifetimeEndBlock, DominatingIP: LifetimeStartInst);
341	CGF.EmitLifetimeEnd(Size: LifetimeSizePtr, Addr: RetAllocaAddr.getPointer());
342	}
343	}
344
345	/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
346	void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) {
347	assert(src.isAggregate() && "value must be aggregate value!");
348	LValue srcLV = CGF.MakeAddrLValue(Addr: src.getAggregateAddress(), T: type);
349	EmitFinalDestCopy(type, src: srcLV, SrcValueKind: EVK_RValue);
350	}
351
352	/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired.
353	void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src,
354	ExprValueKind SrcValueKind) {
355	// If Dest is ignored, then we're evaluating an aggregate expression
356	// in a context that doesn't care about the result. Note that loads
357	// from volatile l-values force the existence of a non-ignored
358	// destination.
359	if (Dest.isIgnored())
360	return;
361
362	// Copy non-trivial C structs here.
363	LValue DstLV = CGF.MakeAddrLValue(
364	Addr: Dest.getAddress(), T: Dest.isVolatile() ? type.withVolatile() : type);
365
366	if (SrcValueKind == EVK_RValue) {
367	if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) {
368	if (Dest.isPotentiallyAliased())
369	CGF.callCStructMoveAssignmentOperator(Dst: DstLV, Src: src);
370	else
371	CGF.callCStructMoveConstructor(Dst: DstLV, Src: src);
372	return;
373	}
374	} else {
375	if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) {
376	if (Dest.isPotentiallyAliased())
377	CGF.callCStructCopyAssignmentOperator(Dst: DstLV, Src: src);
378	else
379	CGF.callCStructCopyConstructor(Dst: DstLV, Src: src);
380	return;
381	}
382	}
383
384	AggValueSlot srcAgg = AggValueSlot::forLValue(
385	LV: src, CGF, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: type),
386	isAliased: AggValueSlot::IsAliased, mayOverlap: AggValueSlot::MayOverlap);
387	EmitCopy(type, dest: Dest, src: srcAgg);
388	}
389
390	/// Perform a copy from the source into the destination.
391	///
392	/// \param type - the type of the aggregate being copied; qualifiers are
393	/// ignored
394	void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest,
395	const AggValueSlot &src) {
396	if (dest.requiresGCollection()) {
397	CharUnits sz = dest.getPreferredSize(Ctx&: CGF.getContext(), Type: type);
398	llvm::Value *size = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: sz.getQuantity());
399	CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF,
400	DestPtr: dest.getAddress(),
401	SrcPtr: src.getAddress(),
402	Size: size);
403	return;
404	}
405
406	// If the result of the assignment is used, copy the LHS there also.
407	// It's volatile if either side is. Use the minimum alignment of
408	// the two sides.
409	LValue DestLV = CGF.MakeAddrLValue(Addr: dest.getAddress(), T: type);
410	LValue SrcLV = CGF.MakeAddrLValue(Addr: src.getAddress(), T: type);
411	CGF.EmitAggregateCopy(Dest: DestLV, Src: SrcLV, EltTy: type, MayOverlap: dest.mayOverlap(),
412	isVolatile: dest.isVolatile() || src.isVolatile());
413	}
414
415	/// Emit the initializer for a std::initializer_list initialized with a
416	/// real initializer list.
417	void
418	AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) {
419	// Emit an array containing the elements. The array is externally destructed
420	// if the std::initializer_list object is.
421	ASTContext &Ctx = CGF.getContext();
422	LValue Array = CGF.EmitLValue(E: E->getSubExpr());
423	assert(Array.isSimple() && "initializer_list array not a simple lvalue");
424	Address ArrayPtr = Array.getAddress(CGF);
425
426	const ConstantArrayType *ArrayType =
427	Ctx.getAsConstantArrayType(T: E->getSubExpr()->getType());
428	assert(ArrayType && "std::initializer_list constructed from non-array");
429
430	// FIXME: Perform the checks on the field types in SemaInit.
431	RecordDecl *Record = E->getType()->castAs<RecordType>()->getDecl();
432	RecordDecl::field_iterator Field = Record->field_begin();
433	if (Field == Record->field_end()) {
434	CGF.ErrorUnsupported(E, "weird std::initializer_list");
435	return;
436	}
437
438	// Start pointer.
439	if (!Field->getType()->isPointerType() ||
440	!Ctx.hasSameType(Field->getType()->getPointeeType(),
441	ArrayType->getElementType())) {
442	CGF.ErrorUnsupported(E, "weird std::initializer_list");
443	return;
444	}
445
446	AggValueSlot Dest = EnsureSlot(T: E->getType());
447	LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
448	LValue Start = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
449	llvm::Value *Zero = llvm::ConstantInt::get(Ty: CGF.PtrDiffTy, V: 0);
450	llvm::Value *IdxStart[] = { Zero, Zero };
451	llvm::Value *ArrayStart = Builder.CreateInBoundsGEP(
452	Ty: ArrayPtr.getElementType(), Ptr: ArrayPtr.emitRawPointer(CGF), IdxList: IdxStart,
453	Name: "arraystart");
454	CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayStart), Dst: Start);
455	++Field;
456
457	if (Field == Record->field_end()) {
458	CGF.ErrorUnsupported(E, "weird std::initializer_list");
459	return;
460	}
461
462	llvm::Value *Size = Builder.getInt(AI: ArrayType->getSize());
463	LValue EndOrLength = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: *Field);
464	if (Field->getType()->isPointerType() &&
465	Ctx.hasSameType(Field->getType()->getPointeeType(),
466	ArrayType->getElementType())) {
467	// End pointer.
468	llvm::Value *IdxEnd[] = { Zero, Size };
469	llvm::Value *ArrayEnd = Builder.CreateInBoundsGEP(
470	Ty: ArrayPtr.getElementType(), Ptr: ArrayPtr.emitRawPointer(CGF), IdxList: IdxEnd,
471	Name: "arrayend");
472	CGF.EmitStoreThroughLValue(Src: RValue::get(V: ArrayEnd), Dst: EndOrLength);
473	} else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) {
474	// Length.
475	CGF.EmitStoreThroughLValue(Src: RValue::get(V: Size), Dst: EndOrLength);
476	} else {
477	CGF.ErrorUnsupported(E, "weird std::initializer_list");
478	return;
479	}
480	}
481
482	/// Determine if E is a trivial array filler, that is, one that is
483	/// equivalent to zero-initialization.
484	static bool isTrivialFiller(Expr *E) {
485	if (!E)
486	return true;
487
488	if (isa<ImplicitValueInitExpr>(Val: E))
489	return true;
490
491	if (auto *ILE = dyn_cast<InitListExpr>(Val: E)) {
492	if (ILE->getNumInits())
493	return false;
494	return isTrivialFiller(E: ILE->getArrayFiller());
495	}
496
497	if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(Val: E))
498	return Cons->getConstructor()->isDefaultConstructor() &&
499	Cons->getConstructor()->isTrivial();
500
501	// FIXME: Are there other cases where we can avoid emitting an initializer?
502	return false;
503	}
504
505	/// Emit initialization of an array from an initializer list. ExprToVisit must
506	/// be either an InitListEpxr a CXXParenInitListExpr.
507	void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
508	QualType ArrayQTy, Expr *ExprToVisit,
509	ArrayRef<Expr *> Args, Expr *ArrayFiller) {
510	uint64_t NumInitElements = Args.size();
511
512	uint64_t NumArrayElements = AType->getNumElements();
513	assert(NumInitElements <= NumArrayElements);
514
515	QualType elementType =
516	CGF.getContext().getAsArrayType(T: ArrayQTy)->getElementType();
517
518	// DestPtr is an array*. Construct an elementType* by drilling
519	// down a level.
520	llvm::Value *zero = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
521	llvm::Value *indices[] = { zero, zero };
522	llvm::Value *begin = Builder.CreateInBoundsGEP(Ty: DestPtr.getElementType(),
523	Ptr: DestPtr.emitRawPointer(CGF),
524	IdxList: indices, Name: "arrayinit.begin");
525
526	CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
527	CharUnits elementAlign =
528	DestPtr.getAlignment().alignmentOfArrayElement(elementSize);
529	llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
530
531	// Consider initializing the array by copying from a global. For this to be
532	// more efficient than per-element initialization, the size of the elements
533	// with explicit initializers should be large enough.
534	if (NumInitElements * elementSize.getQuantity() > 16 &&
535	elementType.isTriviallyCopyableType(Context: CGF.getContext())) {
536	CodeGen::CodeGenModule &CGM = CGF.CGM;
537	ConstantEmitter Emitter(CGF);
538	LangAS AS = ArrayQTy.getAddressSpace();
539	if (llvm::Constant *C =
540	Emitter.tryEmitForInitializer(E: ExprToVisit, destAddrSpace: AS, destType: ArrayQTy)) {
541	auto GV = new llvm::GlobalVariable(
542	CGM.getModule(), C->getType(),
543	/* isConstant= */ true, llvm::GlobalValue::PrivateLinkage, C,
544	"constinit",
545	/* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal,
546	CGM.getContext().getTargetAddressSpace(AS));
547	Emitter.finalize(global: GV);
548	CharUnits Align = CGM.getContext().getTypeAlignInChars(T: ArrayQTy);
549	GV->setAlignment(Align.getAsAlign());
550	Address GVAddr(GV, GV->getValueType(), Align);
551	EmitFinalDestCopy(type: ArrayQTy, src: CGF.MakeAddrLValue(Addr: GVAddr, T: ArrayQTy));
552	return;
553	}
554	}
555
556	// Exception safety requires us to destroy all the
557	// already-constructed members if an initializer throws.
558	// For that, we'll need an EH cleanup.
559	QualType::DestructionKind dtorKind = elementType.isDestructedType();
560	Address endOfInit = Address::invalid();
561	EHScopeStack::stable_iterator cleanup;
562	llvm::Instruction *cleanupDominator = nullptr;
563	if (CGF.needsEHCleanup(kind: dtorKind)) {
564	// In principle we could tell the cleanup where we are more
565	// directly, but the control flow can get so varied here that it
566	// would actually be quite complex. Therefore we go through an
567	// alloca.
568	endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(),
569	"arrayinit.endOfInit");
570	cleanupDominator = Builder.CreateStore(Val: begin, Addr: endOfInit);
571	CGF.pushIrregularPartialArrayCleanup(arrayBegin: begin, arrayEndPointer: endOfInit, elementType,
572	elementAlignment: elementAlign,
573	destroyer: CGF.getDestroyer(destructionKind: dtorKind));
574	cleanup = CGF.EHStack.stable_begin();
575
576	// Otherwise, remember that we didn't need a cleanup.
577	} else {
578	dtorKind = QualType::DK_none;
579	}
580
581	llvm::Value *one = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1);
582
583	// The 'current element to initialize'. The invariants on this
584	// variable are complicated. Essentially, after each iteration of
585	// the loop, it points to the last initialized element, except
586	// that it points to the beginning of the array before any
587	// elements have been initialized.
588	llvm::Value *element = begin;
589
590	// Emit the explicit initializers.
591	for (uint64_t i = 0; i != NumInitElements; ++i) {
592	// Advance to the next element.
593	if (i > 0) {
594	element = Builder.CreateInBoundsGEP(
595	Ty: llvmElementType, Ptr: element, IdxList: one, Name: "arrayinit.element");
596
597	// Tell the cleanup that it needs to destroy up to this
598	// element. TODO: some of these stores can be trivially
599	// observed to be unnecessary.
600	if (endOfInit.isValid()) Builder.CreateStore(Val: element, Addr: endOfInit);
601	}
602
603	LValue elementLV = CGF.MakeAddrLValue(
604	Addr: Address(element, llvmElementType, elementAlign), T: elementType);
605	EmitInitializationToLValue(E: Args[i], Address: elementLV);
606	}
607
608	// Check whether there's a non-trivial array-fill expression.
609	bool hasTrivialFiller = isTrivialFiller(E: ArrayFiller);
610
611	// Any remaining elements need to be zero-initialized, possibly
612	// using the filler expression. We can skip this if the we're
613	// emitting to zeroed memory.
614	if (NumInitElements != NumArrayElements &&
615	!(Dest.isZeroed() && hasTrivialFiller &&
616	CGF.getTypes().isZeroInitializable(T: elementType))) {
617
618	// Use an actual loop. This is basically
619	// do { *array++ = filler; } while (array != end);
620
621	// Advance to the start of the rest of the array.
622	if (NumInitElements) {
623	element = Builder.CreateInBoundsGEP(
624	Ty: llvmElementType, Ptr: element, IdxList: one, Name: "arrayinit.start");
625	if (endOfInit.isValid()) Builder.CreateStore(Val: element, Addr: endOfInit);
626	}
627
628	// Compute the end of the array.
629	llvm::Value *end = Builder.CreateInBoundsGEP(
630	Ty: llvmElementType, Ptr: begin,
631	IdxList: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: NumArrayElements), Name: "arrayinit.end");
632
633	llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
634	llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
635
636	// Jump into the body.
637	CGF.EmitBlock(BB: bodyBB);
638	llvm::PHINode *currentElement =
639	Builder.CreatePHI(Ty: element->getType(), NumReservedValues: 2, Name: "arrayinit.cur");
640	currentElement->addIncoming(V: element, BB: entryBB);
641
642	// Emit the actual filler expression.
643	{
644	// C++1z [class.temporary]p5:
645	// when a default constructor is called to initialize an element of
646	// an array with no corresponding initializer [...] the destruction of
647	// every temporary created in a default argument is sequenced before
648	// the construction of the next array element, if any
649	CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
650	LValue elementLV = CGF.MakeAddrLValue(
651	Addr: Address(currentElement, llvmElementType, elementAlign), T: elementType);
652	if (ArrayFiller)
653	EmitInitializationToLValue(E: ArrayFiller, Address: elementLV);
654	else
655	EmitNullInitializationToLValue(Address: elementLV);
656	}
657
658	// Move on to the next element.
659	llvm::Value *nextElement = Builder.CreateInBoundsGEP(
660	Ty: llvmElementType, Ptr: currentElement, IdxList: one, Name: "arrayinit.next");
661
662	// Tell the EH cleanup that we finished with the last element.
663	if (endOfInit.isValid()) Builder.CreateStore(Val: nextElement, Addr: endOfInit);
664
665	// Leave the loop if we're done.
666	llvm::Value *done = Builder.CreateICmpEQ(LHS: nextElement, RHS: end,
667	Name: "arrayinit.done");
668	llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
669	Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
670	currentElement->addIncoming(V: nextElement, BB: Builder.GetInsertBlock());
671
672	CGF.EmitBlock(BB: endBB);
673	}
674
675	// Leave the partial-array cleanup if we entered one.
676	if (dtorKind) CGF.DeactivateCleanupBlock(Cleanup: cleanup, DominatingIP: cleanupDominator);
677	}
678
679	//===----------------------------------------------------------------------===//
680	// Visitor Methods
681	//===----------------------------------------------------------------------===//
682
683	void AggExprEmitter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E){
684	Visit(E: E->getSubExpr());
685	}
686
687	void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) {
688	// If this is a unique OVE, just visit its source expression.
689	if (e->isUnique())
690	Visit(E: e->getSourceExpr());
691	else
692	EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e));
693	}
694
695	void
696	AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
697	if (Dest.isPotentiallyAliased() &&
698	E->getType().isPODType(CGF.getContext())) {
699	// For a POD type, just emit a load of the lvalue + a copy, because our
700	// compound literal might alias the destination.
701	EmitAggLoadOfLValue(E);
702	return;
703	}
704
705	AggValueSlot Slot = EnsureSlot(T: E->getType());
706
707	// Block-scope compound literals are destroyed at the end of the enclosing
708	// scope in C.
709	bool Destruct =
710	!CGF.getLangOpts().CPlusPlus && !Slot.isExternallyDestructed();
711	if (Destruct)
712	Slot.setExternallyDestructed();
713
714	CGF.EmitAggExpr(E: E->getInitializer(), AS: Slot);
715
716	if (Destruct)
717	if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
718	CGF.pushLifetimeExtendedDestroy(
719	kind: CGF.getCleanupKind(kind: DtorKind), addr: Slot.getAddress(), type: E->getType(),
720	destroyer: CGF.getDestroyer(destructionKind: DtorKind), useEHCleanupForArray: DtorKind & EHCleanup);
721	}
722
723	/// Attempt to look through various unimportant expressions to find a
724	/// cast of the given kind.
725	static Expr *findPeephole(Expr *op, CastKind kind, const ASTContext &ctx) {
726	op = op->IgnoreParenNoopCasts(Ctx: ctx);
727	if (auto castE = dyn_cast<CastExpr>(Val: op)) {
728	if (castE->getCastKind() == kind)
729	return castE->getSubExpr();
730	}
731	return nullptr;
732	}
733
734	void AggExprEmitter::VisitCastExpr(CastExpr *E) {
735	if (const auto *ECE = dyn_cast<ExplicitCastExpr>(Val: E))
736	CGF.CGM.EmitExplicitCastExprType(E: ECE, CGF: &CGF);
737	switch (E->getCastKind()) {
738	case CK_Dynamic: {
739	// FIXME: Can this actually happen? We have no test coverage for it.
740	assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
741	LValue LV = CGF.EmitCheckedLValue(E: E->getSubExpr(),
742	TCK: CodeGenFunction::TCK_Load);
743	// FIXME: Do we also need to handle property references here?
744	if (LV.isSimple())
745	CGF.EmitDynamicCast(V: LV.getAddress(CGF), DCE: cast<CXXDynamicCastExpr>(Val: E));
746	else
747	CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast");
748
749	if (!Dest.isIgnored())
750	CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination");
751	break;
752	}
753
754	case CK_ToUnion: {
755	// Evaluate even if the destination is ignored.
756	if (Dest.isIgnored()) {
757	CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
758	/*ignoreResult=*/true);
759	break;
760	}
761
762	// GCC union extension
763	QualType Ty = E->getSubExpr()->getType();
764	Address CastPtr = Dest.getAddress().withElementType(ElemTy: CGF.ConvertType(T: Ty));
765	EmitInitializationToLValue(E: E->getSubExpr(),
766	Address: CGF.MakeAddrLValue(Addr: CastPtr, T: Ty));
767	break;
768	}
769
770	case CK_LValueToRValueBitCast: {
771	if (Dest.isIgnored()) {
772	CGF.EmitAnyExpr(E: E->getSubExpr(), aggSlot: AggValueSlot::ignored(),
773	/*ignoreResult=*/true);
774	break;
775	}
776
777	LValue SourceLV = CGF.EmitLValue(E: E->getSubExpr());
778	Address SourceAddress =
779	SourceLV.getAddress(CGF).withElementType(ElemTy: CGF.Int8Ty);
780	Address DestAddress = Dest.getAddress().withElementType(ElemTy: CGF.Int8Ty);
781	llvm::Value *SizeVal = llvm::ConstantInt::get(
782	CGF.SizeTy,
783	CGF.getContext().getTypeSizeInChars(E->getType()).getQuantity());
784	Builder.CreateMemCpy(Dest: DestAddress, Src: SourceAddress, Size: SizeVal);
785	break;
786	}
787
788	case CK_DerivedToBase:
789	case CK_BaseToDerived:
790	case CK_UncheckedDerivedToBase: {
791	llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: "
792	"should have been unpacked before we got here");
793	}
794
795	case CK_NonAtomicToAtomic:
796	case CK_AtomicToNonAtomic: {
797	bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic);
798
799	// Determine the atomic and value types.
800	QualType atomicType = E->getSubExpr()->getType();
801	QualType valueType = E->getType();
802	if (isToAtomic) std::swap(a&: atomicType, b&: valueType);
803
804	assert(atomicType->isAtomicType());
805	assert(CGF.getContext().hasSameUnqualifiedType(valueType,
806	atomicType->castAs<AtomicType>()->getValueType()));
807
808	// Just recurse normally if we're ignoring the result or the
809	// atomic type doesn't change representation.
810	if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(type: atomicType)) {
811	return Visit(E: E->getSubExpr());
812	}
813
814	CastKind peepholeTarget =
815	(isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic);
816
817	// These two cases are reverses of each other; try to peephole them.
818	if (Expr *op =
819	findPeephole(op: E->getSubExpr(), kind: peepholeTarget, ctx: CGF.getContext())) {
820	assert(CGF.getContext().hasSameUnqualifiedType(op->getType(),
821	E->getType()) &&
822	"peephole significantly changed types?");
823	return Visit(E: op);
824	}
825
826	// If we're converting an r-value of non-atomic type to an r-value
827	// of atomic type, just emit directly into the relevant sub-object.
828	if (isToAtomic) {
829	AggValueSlot valueDest = Dest;
830	if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(type: atomicType)) {
831	// Zero-initialize. (Strictly speaking, we only need to initialize
832	// the padding at the end, but this is simpler.)
833	if (!Dest.isZeroed())
834	CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: atomicType);
835
836	// Build a GEP to refer to the subobject.
837	Address valueAddr =
838	CGF.Builder.CreateStructGEP(Addr: valueDest.getAddress(), Index: 0);
839	valueDest = AggValueSlot::forAddr(addr: valueAddr,
840	quals: valueDest.getQualifiers(),
841	isDestructed: valueDest.isExternallyDestructed(),
842	needsGC: valueDest.requiresGCollection(),
843	isAliased: valueDest.isPotentiallyAliased(),
844	mayOverlap: AggValueSlot::DoesNotOverlap,
845	isZeroed: AggValueSlot::IsZeroed);
846	}
847
848	CGF.EmitAggExpr(E: E->getSubExpr(), AS: valueDest);
849	return;
850	}
851
852	// Otherwise, we're converting an atomic type to a non-atomic type.
853	// Make an atomic temporary, emit into that, and then copy the value out.
854	AggValueSlot atomicSlot =
855	CGF.CreateAggTemp(T: atomicType, Name: "atomic-to-nonatomic.temp");
856	CGF.EmitAggExpr(E: E->getSubExpr(), AS: atomicSlot);
857
858	Address valueAddr = Builder.CreateStructGEP(Addr: atomicSlot.getAddress(), Index: 0);
859	RValue rvalue = RValue::getAggregate(addr: valueAddr, isVolatile: atomicSlot.isVolatile());
860	return EmitFinalDestCopy(type: valueType, src: rvalue);
861	}
862	case CK_AddressSpaceConversion:
863	return Visit(E: E->getSubExpr());
864
865	case CK_LValueToRValue:
866	// If we're loading from a volatile type, force the destination
867	// into existence.
868	if (E->getSubExpr()->getType().isVolatileQualified()) {
869	bool Destruct =
870	!Dest.isExternallyDestructed() &&
871	E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
872	if (Destruct)
873	Dest.setExternallyDestructed();
874	EnsureDest(T: E->getType());
875	Visit(E: E->getSubExpr());
876
877	if (Destruct)
878	CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
879	E->getType());
880
881	return;
882	}
883
884	[[fallthrough]];
885
886	case CK_HLSLArrayRValue:
887	Visit(E: E->getSubExpr());
888	break;
889
890	case CK_NoOp:
891	case CK_UserDefinedConversion:
892	case CK_ConstructorConversion:
893	assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(),
894	E->getType()) &&
895	"Implicit cast types must be compatible");
896	Visit(E: E->getSubExpr());
897	break;
898
899	case CK_LValueBitCast:
900	llvm_unreachable("should not be emitting lvalue bitcast as rvalue");
901
902	case CK_Dependent:
903	case CK_BitCast:
904	case CK_ArrayToPointerDecay:
905	case CK_FunctionToPointerDecay:
906	case CK_NullToPointer:
907	case CK_NullToMemberPointer:
908	case CK_BaseToDerivedMemberPointer:
909	case CK_DerivedToBaseMemberPointer:
910	case CK_MemberPointerToBoolean:
911	case CK_ReinterpretMemberPointer:
912	case CK_IntegralToPointer:
913	case CK_PointerToIntegral:
914	case CK_PointerToBoolean:
915	case CK_ToVoid:
916	case CK_VectorSplat:
917	case CK_IntegralCast:
918	case CK_BooleanToSignedIntegral:
919	case CK_IntegralToBoolean:
920	case CK_IntegralToFloating:
921	case CK_FloatingToIntegral:
922	case CK_FloatingToBoolean:
923	case CK_FloatingCast:
924	case CK_CPointerToObjCPointerCast:
925	case CK_BlockPointerToObjCPointerCast:
926	case CK_AnyPointerToBlockPointerCast:
927	case CK_ObjCObjectLValueCast:
928	case CK_FloatingRealToComplex:
929	case CK_FloatingComplexToReal:
930	case CK_FloatingComplexToBoolean:
931	case CK_FloatingComplexCast:
932	case CK_FloatingComplexToIntegralComplex:
933	case CK_IntegralRealToComplex:
934	case CK_IntegralComplexToReal:
935	case CK_IntegralComplexToBoolean:
936	case CK_IntegralComplexCast:
937	case CK_IntegralComplexToFloatingComplex:
938	case CK_ARCProduceObject:
939	case CK_ARCConsumeObject:
940	case CK_ARCReclaimReturnedObject:
941	case CK_ARCExtendBlockObject:
942	case CK_CopyAndAutoreleaseBlockObject:
943	case CK_BuiltinFnToFnPtr:
944	case CK_ZeroToOCLOpaqueType:
945	case CK_MatrixCast:
946	case CK_HLSLVectorTruncation:
947
948	case CK_IntToOCLSampler:
949	case CK_FloatingToFixedPoint:
950	case CK_FixedPointToFloating:
951	case CK_FixedPointCast:
952	case CK_FixedPointToBoolean:
953	case CK_FixedPointToIntegral:
954	case CK_IntegralToFixedPoint:
955	llvm_unreachable("cast kind invalid for aggregate types");
956	}
957	}
958
959	void AggExprEmitter::VisitCallExpr(const CallExpr *E) {
960	if (E->getCallReturnType(Ctx: CGF.getContext())->isReferenceType()) {
961	EmitAggLoadOfLValue(E);
962	return;
963	}
964
965	withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
966	return CGF.EmitCallExpr(E, ReturnValue: Slot);
967	});
968	}
969
970	void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) {
971	withReturnValueSlot(E, [&](ReturnValueSlot Slot) {
972	return CGF.EmitObjCMessageExpr(E, Return: Slot);
973	});
974	}
975
976	void AggExprEmitter::VisitBinComma(const BinaryOperator *E) {
977	CGF.EmitIgnoredExpr(E: E->getLHS());
978	Visit(E: E->getRHS());
979	}
980
981	void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) {
982	CodeGenFunction::StmtExprEvaluation eval(CGF);
983	CGF.EmitCompoundStmt(S: *E->getSubStmt(), GetLast: true, AVS: Dest);
984	}
985
986	enum CompareKind {
987	CK_Less,
988	CK_Greater,
989	CK_Equal,
990	};
991
992	static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF,
993	const BinaryOperator *E, llvm::Value *LHS,
994	llvm::Value *RHS, CompareKind Kind,
995	const char *NameSuffix = "") {
996	QualType ArgTy = E->getLHS()->getType();
997	if (const ComplexType *CT = ArgTy->getAs<ComplexType>())
998	ArgTy = CT->getElementType();
999
1000	if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) {
1001	assert(Kind == CK_Equal &&
1002	"member pointers may only be compared for equality");
1003	return CGF.CGM.getCXXABI().EmitMemberPointerComparison(
1004	CGF, L: LHS, R: RHS, MPT, /*IsInequality*/ Inequality: false);
1005	}
1006
1007	// Compute the comparison instructions for the specified comparison kind.
1008	struct CmpInstInfo {
1009	const char *Name;
1010	llvm::CmpInst::Predicate FCmp;
1011	llvm::CmpInst::Predicate SCmp;
1012	llvm::CmpInst::Predicate UCmp;
1013	};
1014	CmpInstInfo InstInfo = [&]() -> CmpInstInfo {
1015	using FI = llvm::FCmpInst;
1016	using II = llvm::ICmpInst;
1017	switch (Kind) {
1018	case CK_Less:
1019	return {.Name: "cmp.lt", .FCmp: FI::FCMP_OLT, .SCmp: II::ICMP_SLT, .UCmp: II::ICMP_ULT};
1020	case CK_Greater:
1021	return {.Name: "cmp.gt", .FCmp: FI::FCMP_OGT, .SCmp: II::ICMP_SGT, .UCmp: II::ICMP_UGT};
1022	case CK_Equal:
1023	return {.Name: "cmp.eq", .FCmp: FI::FCMP_OEQ, .SCmp: II::ICMP_EQ, .UCmp: II::ICMP_EQ};
1024	}
1025	llvm_unreachable("Unrecognised CompareKind enum");
1026	}();
1027
1028	if (ArgTy->hasFloatingRepresentation())
1029	return Builder.CreateFCmp(P: InstInfo.FCmp, LHS, RHS,
1030	Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1031	if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) {
1032	auto Inst =
1033	ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp;
1034	return Builder.CreateICmp(P: Inst, LHS, RHS,
1035	Name: llvm::Twine(InstInfo.Name) + NameSuffix);
1036	}
1037
1038	llvm_unreachable("unsupported aggregate binary expression should have "
1039	"already been handled");
1040	}
1041
1042	void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) {
1043	using llvm::BasicBlock;
1044	using llvm::PHINode;
1045	using llvm::Value;
1046	assert(CGF.getContext().hasSameType(E->getLHS()->getType(),
1047	E->getRHS()->getType()));
1048	const ComparisonCategoryInfo &CmpInfo =
1049	CGF.getContext().CompCategories.getInfoForType(Ty: E->getType());
1050	assert(CmpInfo.Record->isTriviallyCopyable() &&
1051	"cannot copy non-trivially copyable aggregate");
1052
1053	QualType ArgTy = E->getLHS()->getType();
1054
1055	if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() &&
1056	!ArgTy->isNullPtrType() && !ArgTy->isPointerType() &&
1057	!ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) {
1058	return CGF.ErrorUnsupported(E, "aggregate three-way comparison");
1059	}
1060	bool IsComplex = ArgTy->isAnyComplexType();
1061
1062	// Evaluate the operands to the expression and extract their values.
1063	auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> {
1064	RValue RV = CGF.EmitAnyExpr(E);
1065	if (RV.isScalar())
1066	return {RV.getScalarVal(), nullptr};
1067	if (RV.isAggregate())
1068	return {RV.getAggregatePointer(PointeeType: E->getType(), CGF), nullptr};
1069	assert(RV.isComplex());
1070	return RV.getComplexVal();
1071	};
1072	auto LHSValues = EmitOperand(E->getLHS()),
1073	RHSValues = EmitOperand(E->getRHS());
1074
1075	auto EmitCmp = [&](CompareKind K) {
1076	Value *Cmp = EmitCompare(Builder, CGF, E, LHS: LHSValues.first, RHS: RHSValues.first,
1077	Kind: K, NameSuffix: IsComplex ? ".r" : "");
1078	if (!IsComplex)
1079	return Cmp;
1080	assert(K == CompareKind::CK_Equal);
1081	Value *CmpImag = EmitCompare(Builder, CGF, E, LHS: LHSValues.second,
1082	RHS: RHSValues.second, Kind: K, NameSuffix: ".i");
1083	return Builder.CreateAnd(LHS: Cmp, RHS: CmpImag, Name: "and.eq");
1084	};
1085	auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) {
1086	return Builder.getInt(AI: VInfo->getIntValue());
1087	};
1088
1089	Value *Select;
1090	if (ArgTy->isNullPtrType()) {
1091	Select = EmitCmpRes(CmpInfo.getEqualOrEquiv());
1092	} else if (!CmpInfo.isPartial()) {
1093	Value *SelectOne =
1094	Builder.CreateSelect(C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()),
1095	False: EmitCmpRes(CmpInfo.getGreater()), Name: "sel.lt");
1096	Select = Builder.CreateSelect(C: EmitCmp(CK_Equal),
1097	True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1098	False: SelectOne, Name: "sel.eq");
1099	} else {
1100	Value *SelectEq = Builder.CreateSelect(
1101	C: EmitCmp(CK_Equal), True: EmitCmpRes(CmpInfo.getEqualOrEquiv()),
1102	False: EmitCmpRes(CmpInfo.getUnordered()), Name: "sel.eq");
1103	Value *SelectGT = Builder.CreateSelect(C: EmitCmp(CK_Greater),
1104	True: EmitCmpRes(CmpInfo.getGreater()),
1105	False: SelectEq, Name: "sel.gt");
1106	Select = Builder.CreateSelect(
1107	C: EmitCmp(CK_Less), True: EmitCmpRes(CmpInfo.getLess()), False: SelectGT, Name: "sel.lt");
1108	}
1109	// Create the return value in the destination slot.
1110	EnsureDest(T: E->getType());
1111	LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1112
1113	// Emit the address of the first (and only) field in the comparison category
1114	// type, and initialize it from the constant integer value selected above.
1115	LValue FieldLV = CGF.EmitLValueForFieldInitialization(
1116	Base: DestLV, Field: *CmpInfo.Record->field_begin());
1117	CGF.EmitStoreThroughLValue(Src: RValue::get(V: Select), Dst: FieldLV, /*IsInit*/ isInit: true);
1118
1119	// All done! The result is in the Dest slot.
1120	}
1121
1122	void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) {
1123	if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI)
1124	VisitPointerToDataMemberBinaryOperator(BO: E);
1125	else
1126	CGF.ErrorUnsupported(E, "aggregate binary expression");
1127	}
1128
1129	void AggExprEmitter::VisitPointerToDataMemberBinaryOperator(
1130	const BinaryOperator *E) {
1131	LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E);
1132	EmitFinalDestCopy(E->getType(), LV);
1133	}
1134
1135	/// Is the value of the given expression possibly a reference to or
1136	/// into a __block variable?
1137	static bool isBlockVarRef(const Expr *E) {
1138	// Make sure we look through parens.
1139	E = E->IgnoreParens();
1140
1141	// Check for a direct reference to a __block variable.
1142	if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Val: E)) {
1143	const VarDecl *var = dyn_cast<VarDecl>(Val: DRE->getDecl());
1144	return (var && var->hasAttr<BlocksAttr>());
1145	}
1146
1147	// More complicated stuff.
1148
1149	// Binary operators.
1150	if (const BinaryOperator *op = dyn_cast<BinaryOperator>(Val: E)) {
1151	// For an assignment or pointer-to-member operation, just care
1152	// about the LHS.
1153	if (op->isAssignmentOp() || op->isPtrMemOp())
1154	return isBlockVarRef(E: op->getLHS());
1155
1156	// For a comma, just care about the RHS.
1157	if (op->getOpcode() == BO_Comma)
1158	return isBlockVarRef(E: op->getRHS());
1159
1160	// FIXME: pointer arithmetic?
1161	return false;
1162
1163	// Check both sides of a conditional operator.
1164	} else if (const AbstractConditionalOperator *op
1165	= dyn_cast<AbstractConditionalOperator>(Val: E)) {
1166	return isBlockVarRef(E: op->getTrueExpr())
1167	|| isBlockVarRef(E: op->getFalseExpr());
1168
1169	// OVEs are required to support BinaryConditionalOperators.
1170	} else if (const OpaqueValueExpr *op
1171	= dyn_cast<OpaqueValueExpr>(Val: E)) {
1172	if (const Expr *src = op->getSourceExpr())
1173	return isBlockVarRef(E: src);
1174
1175	// Casts are necessary to get things like (*(int*)&var) = foo().
1176	// We don't really care about the kind of cast here, except
1177	// we don't want to look through l2r casts, because it's okay
1178	// to get the *value* in a __block variable.
1179	} else if (const CastExpr *cast = dyn_cast<CastExpr>(Val: E)) {
1180	if (cast->getCastKind() == CK_LValueToRValue)
1181	return false;
1182	return isBlockVarRef(E: cast->getSubExpr());
1183
1184	// Handle unary operators. Again, just aggressively look through
1185	// it, ignoring the operation.
1186	} else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(Val: E)) {
1187	return isBlockVarRef(E: uop->getSubExpr());
1188
1189	// Look into the base of a field access.
1190	} else if (const MemberExpr *mem = dyn_cast<MemberExpr>(Val: E)) {
1191	return isBlockVarRef(E: mem->getBase());
1192
1193	// Look into the base of a subscript.
1194	} else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(Val: E)) {
1195	return isBlockVarRef(E: sub->getBase());
1196	}
1197
1198	return false;
1199	}
1200
1201	void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1202	// For an assignment to work, the value on the right has
1203	// to be compatible with the value on the left.
1204	assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1205	E->getRHS()->getType())
1206	&& "Invalid assignment");
1207
1208	// If the LHS might be a __block variable, and the RHS can
1209	// potentially cause a block copy, we need to evaluate the RHS first
1210	// so that the assignment goes the right place.
1211	// This is pretty semantically fragile.
1212	if (isBlockVarRef(E: E->getLHS()) &&
1213	E->getRHS()->HasSideEffects(Ctx: CGF.getContext())) {
1214	// Ensure that we have a destination, and evaluate the RHS into that.
1215	EnsureDest(T: E->getRHS()->getType());
1216	Visit(E: E->getRHS());
1217
1218	// Now emit the LHS and copy into it.
1219	LValue LHS = CGF.EmitCheckedLValue(E: E->getLHS(), TCK: CodeGenFunction::TCK_Store);
1220
1221	// That copy is an atomic copy if the LHS is atomic.
1222	if (LHS.getType()->isAtomicType() ||
1223	CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1224	CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1225	return;
1226	}
1227
1228	EmitCopy(type: E->getLHS()->getType(),
1229	dest: AggValueSlot::forLValue(LV: LHS, CGF, isDestructed: AggValueSlot::IsDestructed,
1230	needsGC: needsGC(T: E->getLHS()->getType()),
1231	isAliased: AggValueSlot::IsAliased,
1232	mayOverlap: AggValueSlot::MayOverlap),
1233	src: Dest);
1234	return;
1235	}
1236
1237	LValue LHS = CGF.EmitLValue(E: E->getLHS());
1238
1239	// If we have an atomic type, evaluate into the destination and then
1240	// do an atomic copy.
1241	if (LHS.getType()->isAtomicType() ||
1242	CGF.LValueIsSuitableForInlineAtomic(Src: LHS)) {
1243	EnsureDest(T: E->getRHS()->getType());
1244	Visit(E: E->getRHS());
1245	CGF.EmitAtomicStore(rvalue: Dest.asRValue(), lvalue: LHS, /*isInit*/ false);
1246	return;
1247	}
1248
1249	// Codegen the RHS so that it stores directly into the LHS.
1250	AggValueSlot LHSSlot = AggValueSlot::forLValue(
1251	LV: LHS, CGF, isDestructed: AggValueSlot::IsDestructed, needsGC: needsGC(T: E->getLHS()->getType()),
1252	isAliased: AggValueSlot::IsAliased, mayOverlap: AggValueSlot::MayOverlap);
1253	// A non-volatile aggregate destination might have volatile member.
1254	if (!LHSSlot.isVolatile() &&
1255	CGF.hasVolatileMember(T: E->getLHS()->getType()))
1256	LHSSlot.setVolatile(true);
1257
1258	CGF.EmitAggExpr(E: E->getRHS(), AS: LHSSlot);
1259
1260	// Copy into the destination if the assignment isn't ignored.
1261	EmitFinalDestCopy(E->getType(), LHS);
1262
1263	if (!Dest.isIgnored() && !Dest.isExternallyDestructed() &&
1264	E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct)
1265	CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
1266	E->getType());
1267	}
1268
1269	void AggExprEmitter::
1270	VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
1271	llvm::BasicBlock *LHSBlock = CGF.createBasicBlock(name: "cond.true");
1272	llvm::BasicBlock *RHSBlock = CGF.createBasicBlock(name: "cond.false");
1273	llvm::BasicBlock *ContBlock = CGF.createBasicBlock(name: "cond.end");
1274
1275	// Bind the common expression if necessary.
1276	CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1277
1278	CodeGenFunction::ConditionalEvaluation eval(CGF);
1279	CGF.EmitBranchOnBoolExpr(Cond: E->getCond(), TrueBlock: LHSBlock, FalseBlock: RHSBlock,
1280	TrueCount: CGF.getProfileCount(E));
1281
1282	// Save whether the destination's lifetime is externally managed.
1283	bool isExternallyDestructed = Dest.isExternallyDestructed();
1284	bool destructNonTrivialCStruct =
1285	!isExternallyDestructed &&
1286	E->getType().isDestructedType() == QualType::DK_nontrivial_c_struct;
1287	isExternallyDestructed |= destructNonTrivialCStruct;
1288	Dest.setExternallyDestructed(isExternallyDestructed);
1289
1290	eval.begin(CGF);
1291	CGF.EmitBlock(BB: LHSBlock);
1292	if (llvm::EnableSingleByteCoverage)
1293	CGF.incrementProfileCounter(E->getTrueExpr());
1294	else
1295	CGF.incrementProfileCounter(E);
1296	Visit(E: E->getTrueExpr());
1297	eval.end(CGF);
1298
1299	assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!");
1300	CGF.Builder.CreateBr(Dest: ContBlock);
1301
1302	// If the result of an agg expression is unused, then the emission
1303	// of the LHS might need to create a destination slot. That's fine
1304	// with us, and we can safely emit the RHS into the same slot, but
1305	// we shouldn't claim that it's already being destructed.
1306	Dest.setExternallyDestructed(isExternallyDestructed);
1307
1308	eval.begin(CGF);
1309	CGF.EmitBlock(BB: RHSBlock);
1310	if (llvm::EnableSingleByteCoverage)
1311	CGF.incrementProfileCounter(E->getFalseExpr());
1312	Visit(E: E->getFalseExpr());
1313	eval.end(CGF);
1314
1315	if (destructNonTrivialCStruct)
1316	CGF.pushDestroy(QualType::DK_nontrivial_c_struct, Dest.getAddress(),
1317	E->getType());
1318
1319	CGF.EmitBlock(BB: ContBlock);
1320	if (llvm::EnableSingleByteCoverage)
1321	CGF.incrementProfileCounter(E);
1322	}
1323
1324	void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) {
1325	Visit(E: CE->getChosenSubExpr());
1326	}
1327
1328	void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) {
1329	Address ArgValue = Address::invalid();
1330	Address ArgPtr = CGF.EmitVAArg(VE, VAListAddr&: ArgValue);
1331
1332	// If EmitVAArg fails, emit an error.
1333	if (!ArgPtr.isValid()) {
1334	CGF.ErrorUnsupported(VE, "aggregate va_arg expression");
1335	return;
1336	}
1337
1338	EmitFinalDestCopy(VE->getType(), CGF.MakeAddrLValue(ArgPtr, VE->getType()));
1339	}
1340
1341	void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
1342	// Ensure that we have a slot, but if we already do, remember
1343	// whether it was externally destructed.
1344	bool wasExternallyDestructed = Dest.isExternallyDestructed();
1345	EnsureDest(T: E->getType());
1346
1347	// We're going to push a destructor if there isn't already one.
1348	Dest.setExternallyDestructed();
1349
1350	Visit(E: E->getSubExpr());
1351
1352	// Push that destructor we promised.
1353	if (!wasExternallyDestructed)
1354	CGF.EmitCXXTemporary(Temporary: E->getTemporary(), TempType: E->getType(), Ptr: Dest.getAddress());
1355	}
1356
1357	void
1358	AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) {
1359	AggValueSlot Slot = EnsureSlot(T: E->getType());
1360	CGF.EmitCXXConstructExpr(E, Dest: Slot);
1361	}
1362
1363	void AggExprEmitter::VisitCXXInheritedCtorInitExpr(
1364	const CXXInheritedCtorInitExpr *E) {
1365	AggValueSlot Slot = EnsureSlot(T: E->getType());
1366	CGF.EmitInheritedCXXConstructorCall(
1367	D: E->getConstructor(), ForVirtualBase: E->constructsVBase(), This: Slot.getAddress(),
1368	InheritedFromVBase: E->inheritedFromVBase(), E);
1369	}
1370
1371	void
1372	AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
1373	AggValueSlot Slot = EnsureSlot(T: E->getType());
1374	LValue SlotLV = CGF.MakeAddrLValue(Slot.getAddress(), E->getType());
1375
1376	// We'll need to enter cleanup scopes in case any of the element
1377	// initializers throws an exception.
1378	SmallVector<EHScopeStack::stable_iterator, 16> Cleanups;
1379	llvm::Instruction *CleanupDominator = nullptr;
1380
1381	CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
1382	for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
1383	e = E->capture_init_end();
1384	i != e; ++i, ++CurField) {
1385	// Emit initialization
1386	LValue LV = CGF.EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
1387	if (CurField->hasCapturedVLAType()) {
1388	CGF.EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
1389	continue;
1390	}
1391
1392	EmitInitializationToLValue(E: *i, Address: LV);
1393
1394	// Push a destructor if necessary.
1395	if (QualType::DestructionKind DtorKind =
1396	CurField->getType().isDestructedType()) {
1397	assert(LV.isSimple());
1398	if (CGF.needsEHCleanup(kind: DtorKind)) {
1399	if (!CleanupDominator)
1400	CleanupDominator = CGF.Builder.CreateAlignedLoad(
1401	Ty: CGF.Int8Ty,
1402	Addr: llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy),
1403	Align: CharUnits::One()); // placeholder
1404
1405	CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), CurField->getType(),
1406	CGF.getDestroyer(destructionKind: DtorKind), false);
1407	Cleanups.push_back(Elt: CGF.EHStack.stable_begin());
1408	}
1409	}
1410	}
1411
1412	// Deactivate all the partial cleanups in reverse order, which
1413	// generally means popping them.
1414	for (unsigned i = Cleanups.size(); i != 0; --i)
1415	CGF.DeactivateCleanupBlock(Cleanup: Cleanups[i-1], DominatingIP: CleanupDominator);
1416
1417	// Destroy the placeholder if we made one.
1418	if (CleanupDominator)
1419	CleanupDominator->eraseFromParent();
1420	}
1421
1422	void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
1423	CodeGenFunction::RunCleanupsScope cleanups(CGF);
1424	Visit(E: E->getSubExpr());
1425	}
1426
1427	void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
1428	QualType T = E->getType();
1429	AggValueSlot Slot = EnsureSlot(T);
1430	EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1431	}
1432
1433	void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
1434	QualType T = E->getType();
1435	AggValueSlot Slot = EnsureSlot(T);
1436	EmitNullInitializationToLValue(Address: CGF.MakeAddrLValue(Addr: Slot.getAddress(), T));
1437	}
1438
1439	/// Determine whether the given cast kind is known to always convert values
1440	/// with all zero bits in their value representation to values with all zero
1441	/// bits in their value representation.
1442	static bool castPreservesZero(const CastExpr *CE) {
1443	switch (CE->getCastKind()) {
1444	// No-ops.
1445	case CK_NoOp:
1446	case CK_UserDefinedConversion:
1447	case CK_ConstructorConversion:
1448	case CK_BitCast:
1449	case CK_ToUnion:
1450	case CK_ToVoid:
1451	// Conversions between (possibly-complex) integral, (possibly-complex)
1452	// floating-point, and bool.
1453	case CK_BooleanToSignedIntegral:
1454	case CK_FloatingCast:
1455	case CK_FloatingComplexCast:
1456	case CK_FloatingComplexToBoolean:
1457	case CK_FloatingComplexToIntegralComplex:
1458	case CK_FloatingComplexToReal:
1459	case CK_FloatingRealToComplex:
1460	case CK_FloatingToBoolean:
1461	case CK_FloatingToIntegral:
1462	case CK_IntegralCast:
1463	case CK_IntegralComplexCast:
1464	case CK_IntegralComplexToBoolean:
1465	case CK_IntegralComplexToFloatingComplex:
1466	case CK_IntegralComplexToReal:
1467	case CK_IntegralRealToComplex:
1468	case CK_IntegralToBoolean:
1469	case CK_IntegralToFloating:
1470	// Reinterpreting integers as pointers and vice versa.
1471	case CK_IntegralToPointer:
1472	case CK_PointerToIntegral:
1473	// Language extensions.
1474	case CK_VectorSplat:
1475	case CK_MatrixCast:
1476	case CK_NonAtomicToAtomic:
1477	case CK_AtomicToNonAtomic:
1478	case CK_HLSLVectorTruncation:
1479	return true;
1480
1481	case CK_BaseToDerivedMemberPointer:
1482	case CK_DerivedToBaseMemberPointer:
1483	case CK_MemberPointerToBoolean:
1484	case CK_NullToMemberPointer:
1485	case CK_ReinterpretMemberPointer:
1486	// FIXME: ABI-dependent.
1487	return false;
1488
1489	case CK_AnyPointerToBlockPointerCast:
1490	case CK_BlockPointerToObjCPointerCast:
1491	case CK_CPointerToObjCPointerCast:
1492	case CK_ObjCObjectLValueCast:
1493	case CK_IntToOCLSampler:
1494	case CK_ZeroToOCLOpaqueType:
1495	// FIXME: Check these.
1496	return false;
1497
1498	case CK_FixedPointCast:
1499	case CK_FixedPointToBoolean:
1500	case CK_FixedPointToFloating:
1501	case CK_FixedPointToIntegral:
1502	case CK_FloatingToFixedPoint:
1503	case CK_IntegralToFixedPoint:
1504	// FIXME: Do all fixed-point types represent zero as all 0 bits?
1505	return false;
1506
1507	case CK_AddressSpaceConversion:
1508	case CK_BaseToDerived:
1509	case CK_DerivedToBase:
1510	case CK_Dynamic:
1511	case CK_NullToPointer:
1512	case CK_PointerToBoolean:
1513	// FIXME: Preserves zeroes only if zero pointers and null pointers have the
1514	// same representation in all involved address spaces.
1515	return false;
1516
1517	case CK_ARCConsumeObject:
1518	case CK_ARCExtendBlockObject:
1519	case CK_ARCProduceObject:
1520	case CK_ARCReclaimReturnedObject:
1521	case CK_CopyAndAutoreleaseBlockObject:
1522	case CK_ArrayToPointerDecay:
1523	case CK_FunctionToPointerDecay:
1524	case CK_BuiltinFnToFnPtr:
1525	case CK_Dependent:
1526	case CK_LValueBitCast:
1527	case CK_LValueToRValue:
1528	case CK_LValueToRValueBitCast:
1529	case CK_UncheckedDerivedToBase:
1530	case CK_HLSLArrayRValue:
1531	return false;
1532	}
1533	llvm_unreachable("Unhandled clang::CastKind enum");
1534	}
1535
1536	/// isSimpleZero - If emitting this value will obviously just cause a store of
1537	/// zero to memory, return true. This can return false if uncertain, so it just
1538	/// handles simple cases.
1539	static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) {
1540	E = E->IgnoreParens();
1541	while (auto *CE = dyn_cast<CastExpr>(Val: E)) {
1542	if (!castPreservesZero(CE))
1543	break;
1544	E = CE->getSubExpr()->IgnoreParens();
1545	}
1546
1547	// 0
1548	if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(Val: E))
1549	return IL->getValue() == 0;
1550	// +0.0
1551	if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(Val: E))
1552	return FL->getValue().isPosZero();
1553	// int()
1554	if ((isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) &&
1555	CGF.getTypes().isZeroInitializable(T: E->getType()))
1556	return true;
1557	// (int*)0 - Null pointer expressions.
1558	if (const CastExpr *ICE = dyn_cast<CastExpr>(Val: E))
1559	return ICE->getCastKind() == CK_NullToPointer &&
1560	CGF.getTypes().isPointerZeroInitializable(T: E->getType()) &&
1561	!E->HasSideEffects(Ctx: CGF.getContext());
1562	// '\0'
1563	if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(Val: E))
1564	return CL->getValue() == 0;
1565
1566	// Otherwise, hard case: conservatively return false.
1567	return false;
1568	}
1569
1570
1571	void
1572	AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) {
1573	QualType type = LV.getType();
1574	// FIXME: Ignore result?
1575	// FIXME: Are initializers affected by volatile?
1576	if (Dest.isZeroed() && isSimpleZero(E, CGF)) {
1577	// Storing "i32 0" to a zero'd memory location is a noop.
1578	return;
1579	} else if (isa<ImplicitValueInitExpr>(Val: E) || isa<CXXScalarValueInitExpr>(Val: E)) {
1580	return EmitNullInitializationToLValue(Address: LV);
1581	} else if (isa<NoInitExpr>(Val: E)) {
1582	// Do nothing.
1583	return;
1584	} else if (type->isReferenceType()) {
1585	RValue RV = CGF.EmitReferenceBindingToExpr(E);
1586	return CGF.EmitStoreThroughLValue(Src: RV, Dst: LV);
1587	}
1588
1589	switch (CGF.getEvaluationKind(T: type)) {
1590	case TEK_Complex:
1591	CGF.EmitComplexExprIntoLValue(E, dest: LV, /*isInit*/ true);
1592	return;
1593	case TEK_Aggregate:
1594	CGF.EmitAggExpr(
1595	E, AS: AggValueSlot::forLValue(LV, CGF, isDestructed: AggValueSlot::IsDestructed,
1596	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1597	isAliased: AggValueSlot::IsNotAliased,
1598	mayOverlap: AggValueSlot::MayOverlap, isZeroed: Dest.isZeroed()));
1599	return;
1600	case TEK_Scalar:
1601	if (LV.isSimple()) {
1602	CGF.EmitScalarInit(init: E, /*D=*/nullptr, lvalue: LV, /*Captured=*/capturedByInit: false);
1603	} else {
1604	CGF.EmitStoreThroughLValue(Src: RValue::get(V: CGF.EmitScalarExpr(E)), Dst: LV);
1605	}
1606	return;
1607	}
1608	llvm_unreachable("bad evaluation kind");
1609	}
1610
1611	void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) {
1612	QualType type = lv.getType();
1613
1614	// If the destination slot is already zeroed out before the aggregate is
1615	// copied into it, we don't have to emit any zeros here.
1616	if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(T: type))
1617	return;
1618
1619	if (CGF.hasScalarEvaluationKind(T: type)) {
1620	// For non-aggregates, we can store the appropriate null constant.
1621	llvm::Value *null = CGF.CGM.EmitNullConstant(T: type);
1622	// Note that the following is not equivalent to
1623	// EmitStoreThroughBitfieldLValue for ARC types.
1624	if (lv.isBitField()) {
1625	CGF.EmitStoreThroughBitfieldLValue(Src: RValue::get(V: null), Dst: lv);
1626	} else {
1627	assert(lv.isSimple());
1628	CGF.EmitStoreOfScalar(value: null, lvalue: lv, /* isInitialization */ isInit: true);
1629	}
1630	} else {
1631	// There's a potential optimization opportunity in combining
1632	// memsets; that would be easy for arrays, but relatively
1633	// difficult for structures with the current code.
1634	CGF.EmitNullInitialization(DestPtr: lv.getAddress(CGF), Ty: lv.getType());
1635	}
1636	}
1637
1638	void AggExprEmitter::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
1639	VisitCXXParenListOrInitListExpr(E, E->getInitExprs(),
1640	E->getInitializedFieldInUnion(),
1641	E->getArrayFiller());
1642	}
1643
1644	void AggExprEmitter::VisitInitListExpr(InitListExpr *E) {
1645	if (E->hadArrayRangeDesignator())
1646	CGF.ErrorUnsupported(E, "GNU array range designator extension");
1647
1648	if (E->isTransparent())
1649	return Visit(E: E->getInit(Init: 0));
1650
1651	VisitCXXParenListOrInitListExpr(
1652	E, E->inits(), E->getInitializedFieldInUnion(), E->getArrayFiller());
1653	}
1654
1655	void AggExprEmitter::VisitCXXParenListOrInitListExpr(
1656	Expr *ExprToVisit, ArrayRef<Expr *> InitExprs,
1657	FieldDecl *InitializedFieldInUnion, Expr *ArrayFiller) {
1658	#if 0
1659	// FIXME: Assess perf here? Figure out what cases are worth optimizing here
1660	// (Length of globals? Chunks of zeroed-out space?).
1661	//
1662	// If we can, prefer a copy from a global; this is a lot less code for long
1663	// globals, and it's easier for the current optimizers to analyze.
1664	if (llvm::Constant *C =
1665	CGF.CGM.EmitConstantExpr(ExprToVisit, ExprToVisit->getType(), &CGF)) {
1666	llvm::GlobalVariable* GV =
1667	new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true,
1668	llvm::GlobalValue::InternalLinkage, C, "");
1669	EmitFinalDestCopy(ExprToVisit->getType(),
1670	CGF.MakeAddrLValue(GV, ExprToVisit->getType()));
1671	return;
1672	}
1673	#endif
1674
1675	AggValueSlot Dest = EnsureSlot(T: ExprToVisit->getType());
1676
1677	LValue DestLV = CGF.MakeAddrLValue(Addr: Dest.getAddress(), T: ExprToVisit->getType());
1678
1679	// Handle initialization of an array.
1680	if (ExprToVisit->getType()->isConstantArrayType()) {
1681	auto AType = cast<llvm::ArrayType>(Val: Dest.getAddress().getElementType());
1682	EmitArrayInit(DestPtr: Dest.getAddress(), AType, ArrayQTy: ExprToVisit->getType(), ExprToVisit,
1683	Args: InitExprs, ArrayFiller);
1684	return;
1685	} else if (ExprToVisit->getType()->isVariableArrayType()) {
1686	// A variable array type that has an initializer can only do empty
1687	// initialization. And because this feature is not exposed as an extension
1688	// in C++, we can safely memset the array memory to zero.
1689	assert(InitExprs.size() == 0 &&
1690	"you can only use an empty initializer with VLAs");
1691	CGF.EmitNullInitialization(DestPtr: Dest.getAddress(), Ty: ExprToVisit->getType());
1692	return;
1693	}
1694
1695	assert(ExprToVisit->getType()->isRecordType() &&
1696	"Only support structs/unions here!");
1697
1698	// Do struct initialization; this code just sets each individual member
1699	// to the approprate value. This makes bitfield support automatic;
1700	// the disadvantage is that the generated code is more difficult for
1701	// the optimizer, especially with bitfields.
1702	unsigned NumInitElements = InitExprs.size();
1703	RecordDecl *record = ExprToVisit->getType()->castAs<RecordType>()->getDecl();
1704
1705	// We'll need to enter cleanup scopes in case any of the element
1706	// initializers throws an exception.
1707	SmallVector<EHScopeStack::stable_iterator, 16> cleanups;
1708	llvm::Instruction *cleanupDominator = nullptr;
1709	auto addCleanup = [&](const EHScopeStack::stable_iterator &cleanup) {
1710	cleanups.push_back(Elt: cleanup);
1711	if (!cleanupDominator) // create placeholder once needed
1712	cleanupDominator = CGF.Builder.CreateAlignedLoad(
1713	Ty: CGF.Int8Ty, Addr: llvm::Constant::getNullValue(Ty: CGF.Int8PtrTy),
1714	Align: CharUnits::One());
1715	};
1716
1717	unsigned curInitIndex = 0;
1718
1719	// Emit initialization of base classes.
1720	if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: record)) {
1721	assert(NumInitElements >= CXXRD->getNumBases() &&
1722	"missing initializer for base class");
1723	for (auto &Base : CXXRD->bases()) {
1724	assert(!Base.isVirtual() && "should not see vbases here");
1725	auto *BaseRD = Base.getType()->getAsCXXRecordDecl();
1726	Address V = CGF.GetAddressOfDirectBaseInCompleteClass(
1727	Value: Dest.getAddress(), Derived: CXXRD, Base: BaseRD,
1728	/*isBaseVirtual*/ BaseIsVirtual: false);
1729	AggValueSlot AggSlot = AggValueSlot::forAddr(
1730	addr: V, quals: Qualifiers(),
1731	isDestructed: AggValueSlot::IsDestructed,
1732	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1733	isAliased: AggValueSlot::IsNotAliased,
1734	mayOverlap: CGF.getOverlapForBaseInit(RD: CXXRD, BaseRD, IsVirtual: Base.isVirtual()));
1735	CGF.EmitAggExpr(E: InitExprs[curInitIndex++], AS: AggSlot);
1736
1737	if (QualType::DestructionKind dtorKind =
1738	Base.getType().isDestructedType()) {
1739	CGF.pushDestroy(dtorKind, addr: V, type: Base.getType());
1740	addCleanup(CGF.EHStack.stable_begin());
1741	}
1742	}
1743	}
1744
1745	// Prepare a 'this' for CXXDefaultInitExprs.
1746	CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress());
1747
1748	if (record->isUnion()) {
1749	// Only initialize one field of a union. The field itself is
1750	// specified by the initializer list.
1751	if (!InitializedFieldInUnion) {
1752	// Empty union; we have nothing to do.
1753
1754	#ifndef NDEBUG
1755	// Make sure that it's really an empty and not a failure of
1756	// semantic analysis.
1757	for (const auto *Field : record->fields())
1758	assert(
1759	(Field->isUnnamedBitField() || Field->isAnonymousStructOrUnion()) &&
1760	"Only unnamed bitfields or ananymous class allowed");
1761	#endif
1762	return;
1763	}
1764
1765	// FIXME: volatility
1766	FieldDecl *Field = InitializedFieldInUnion;
1767
1768	LValue FieldLoc = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field);
1769	if (NumInitElements) {
1770	// Store the initializer into the field
1771	EmitInitializationToLValue(E: InitExprs[0], LV: FieldLoc);
1772	} else {
1773	// Default-initialize to null.
1774	EmitNullInitializationToLValue(lv: FieldLoc);
1775	}
1776
1777	return;
1778	}
1779
1780	// Here we iterate over the fields; this makes it simpler to both
1781	// default-initialize fields and skip over unnamed fields.
1782	for (const auto *field : record->fields()) {
1783	// We're done once we hit the flexible array member.
1784	if (field->getType()->isIncompleteArrayType())
1785	break;
1786
1787	// Always skip anonymous bitfields.
1788	if (field->isUnnamedBitField())
1789	continue;
1790
1791	// We're done if we reach the end of the explicit initializers, we
1792	// have a zeroed object, and the rest of the fields are
1793	// zero-initializable.
1794	if (curInitIndex == NumInitElements && Dest.isZeroed() &&
1795	CGF.getTypes().isZeroInitializable(T: ExprToVisit->getType()))
1796	break;
1797
1798
1799	LValue LV = CGF.EmitLValueForFieldInitialization(Base: DestLV, Field: field);
1800	// We never generate write-barries for initialized fields.
1801	LV.setNonGC(true);
1802
1803	if (curInitIndex < NumInitElements) {
1804	// Store the initializer into the field.
1805	EmitInitializationToLValue(E: InitExprs[curInitIndex++], LV);
1806	} else {
1807	// We're out of initializers; default-initialize to null
1808	EmitNullInitializationToLValue(lv: LV);
1809	}
1810
1811	// Push a destructor if necessary.
1812	// FIXME: if we have an array of structures, all explicitly
1813	// initialized, we can end up pushing a linear number of cleanups.
1814	bool pushedCleanup = false;
1815	if (QualType::DestructionKind dtorKind
1816	= field->getType().isDestructedType()) {
1817	assert(LV.isSimple());
1818	if (CGF.needsEHCleanup(kind: dtorKind)) {
1819	CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), field->getType(),
1820	CGF.getDestroyer(destructionKind: dtorKind), false);
1821	addCleanup(CGF.EHStack.stable_begin());
1822	pushedCleanup = true;
1823	}
1824	}
1825
1826	// If the GEP didn't get used because of a dead zero init or something
1827	// else, clean it up for -O0 builds and general tidiness.
1828	if (!pushedCleanup && LV.isSimple())
1829	if (llvm::GetElementPtrInst *GEP =
1830	dyn_cast<llvm::GetElementPtrInst>(Val: LV.emitRawPointer(CGF)))
1831	if (GEP->use_empty())
1832	GEP->eraseFromParent();
1833	}
1834
1835	// Deactivate all the partial cleanups in reverse order, which
1836	// generally means popping them.
1837	assert((cleanupDominator || cleanups.empty()) &&
1838	"Missing cleanupDominator before deactivating cleanup blocks");
1839	for (unsigned i = cleanups.size(); i != 0; --i)
1840	CGF.DeactivateCleanupBlock(Cleanup: cleanups[i-1], DominatingIP: cleanupDominator);
1841
1842	// Destroy the placeholder if we made one.
1843	if (cleanupDominator)
1844	cleanupDominator->eraseFromParent();
1845	}
1846
1847	void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
1848	llvm::Value *outerBegin) {
1849	// Emit the common subexpression.
1850	CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr());
1851
1852	Address destPtr = EnsureSlot(T: E->getType()).getAddress();
1853	uint64_t numElements = E->getArraySize().getZExtValue();
1854
1855	if (!numElements)
1856	return;
1857
1858	// destPtr is an array*. Construct an elementType* by drilling down a level.
1859	llvm::Value *zero = llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 0);
1860	llvm::Value *indices[] = {zero, zero};
1861	llvm::Value *begin = Builder.CreateInBoundsGEP(Ty: destPtr.getElementType(),
1862	Ptr: destPtr.emitRawPointer(CGF),
1863	IdxList: indices, Name: "arrayinit.begin");
1864
1865	// Prepare to special-case multidimensional array initialization: we avoid
1866	// emitting multiple destructor loops in that case.
1867	if (!outerBegin)
1868	outerBegin = begin;
1869	ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(Val: E->getSubExpr());
1870
1871	QualType elementType =
1872	CGF.getContext().getAsArrayType(T: E->getType())->getElementType();
1873	CharUnits elementSize = CGF.getContext().getTypeSizeInChars(T: elementType);
1874	CharUnits elementAlign =
1875	destPtr.getAlignment().alignmentOfArrayElement(elementSize);
1876	llvm::Type *llvmElementType = CGF.ConvertTypeForMem(T: elementType);
1877
1878	llvm::BasicBlock *entryBB = Builder.GetInsertBlock();
1879	llvm::BasicBlock *bodyBB = CGF.createBasicBlock(name: "arrayinit.body");
1880
1881	// Jump into the body.
1882	CGF.EmitBlock(BB: bodyBB);
1883	llvm::PHINode *index =
1884	Builder.CreatePHI(Ty: zero->getType(), NumReservedValues: 2, Name: "arrayinit.index");
1885	index->addIncoming(V: zero, BB: entryBB);
1886	llvm::Value *element =
1887	Builder.CreateInBoundsGEP(Ty: llvmElementType, Ptr: begin, IdxList: index);
1888
1889	// Prepare for a cleanup.
1890	QualType::DestructionKind dtorKind = elementType.isDestructedType();
1891	EHScopeStack::stable_iterator cleanup;
1892	if (CGF.needsEHCleanup(kind: dtorKind) && !InnerLoop) {
1893	if (outerBegin->getType() != element->getType())
1894	outerBegin = Builder.CreateBitCast(V: outerBegin, DestTy: element->getType());
1895	CGF.pushRegularPartialArrayCleanup(arrayBegin: outerBegin, arrayEnd: element, elementType,
1896	elementAlignment: elementAlign,
1897	destroyer: CGF.getDestroyer(destructionKind: dtorKind));
1898	cleanup = CGF.EHStack.stable_begin();
1899	} else {
1900	dtorKind = QualType::DK_none;
1901	}
1902
1903	// Emit the actual filler expression.
1904	{
1905	// Temporaries created in an array initialization loop are destroyed
1906	// at the end of each iteration.
1907	CodeGenFunction::RunCleanupsScope CleanupsScope(CGF);
1908	CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index);
1909	LValue elementLV = CGF.MakeAddrLValue(
1910	Addr: Address(element, llvmElementType, elementAlign), T: elementType);
1911
1912	if (InnerLoop) {
1913	// If the subexpression is an ArrayInitLoopExpr, share its cleanup.
1914	auto elementSlot = AggValueSlot::forLValue(
1915	LV: elementLV, CGF, isDestructed: AggValueSlot::IsDestructed,
1916	needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
1917	mayOverlap: AggValueSlot::DoesNotOverlap);
1918	AggExprEmitter(CGF, elementSlot, false)
1919	.VisitArrayInitLoopExpr(E: InnerLoop, outerBegin);
1920	} else
1921	EmitInitializationToLValue(E: E->getSubExpr(), LV: elementLV);
1922	}
1923
1924	// Move on to the next element.
1925	llvm::Value *nextIndex = Builder.CreateNUWAdd(
1926	LHS: index, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: 1), Name: "arrayinit.next");
1927	index->addIncoming(V: nextIndex, BB: Builder.GetInsertBlock());
1928
1929	// Leave the loop if we're done.
1930	llvm::Value *done = Builder.CreateICmpEQ(
1931	LHS: nextIndex, RHS: llvm::ConstantInt::get(Ty: CGF.SizeTy, V: numElements),
1932	Name: "arrayinit.done");
1933	llvm::BasicBlock *endBB = CGF.createBasicBlock(name: "arrayinit.end");
1934	Builder.CreateCondBr(Cond: done, True: endBB, False: bodyBB);
1935
1936	CGF.EmitBlock(BB: endBB);
1937
1938	// Leave the partial-array cleanup if we entered one.
1939	if (dtorKind)
1940	CGF.DeactivateCleanupBlock(Cleanup: cleanup, DominatingIP: index);
1941	}
1942
1943	void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) {
1944	AggValueSlot Dest = EnsureSlot(T: E->getType());
1945
1946	LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType());
1947	EmitInitializationToLValue(E: E->getBase(), LV: DestLV);
1948	VisitInitListExpr(E: E->getUpdater());
1949	}
1950
1951	//===----------------------------------------------------------------------===//
1952	// Entry Points into this File
1953	//===----------------------------------------------------------------------===//
1954
1955	/// GetNumNonZeroBytesInInit - Get an approximate count of the number of
1956	/// non-zero bytes that will be stored when outputting the initializer for the
1957	/// specified initializer expression.
1958	static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
1959	if (auto *MTE = dyn_cast<MaterializeTemporaryExpr>(Val: E))
1960	E = MTE->getSubExpr();
1961	E = E->IgnoreParenNoopCasts(Ctx: CGF.getContext());
1962
1963	// 0 and 0.0 won't require any non-zero stores!
1964	if (isSimpleZero(E, CGF)) return CharUnits::Zero();
1965
1966	// If this is an initlist expr, sum up the size of sizes of the (present)
1967	// elements. If this is something weird, assume the whole thing is non-zero.
1968	const InitListExpr *ILE = dyn_cast<InitListExpr>(Val: E);
1969	while (ILE && ILE->isTransparent())
1970	ILE = dyn_cast<InitListExpr>(Val: ILE->getInit(Init: 0));
1971	if (!ILE || !CGF.getTypes().isZeroInitializable(ILE->getType()))
1972	return CGF.getContext().getTypeSizeInChars(T: E->getType());
1973
1974	// InitListExprs for structs have to be handled carefully. If there are
1975	// reference members, we need to consider the size of the reference, not the
1976	// referencee. InitListExprs for unions and arrays can't have references.
1977	if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
1978	if (!RT->isUnionType()) {
1979	RecordDecl *SD = RT->getDecl();
1980	CharUnits NumNonZeroBytes = CharUnits::Zero();
1981
1982	unsigned ILEElement = 0;
1983	if (auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: SD))
1984	while (ILEElement != CXXRD->getNumBases())
1985	NumNonZeroBytes +=
1986	GetNumNonZeroBytesInInit(E: ILE->getInit(Init: ILEElement++), CGF);
1987	for (const auto *Field : SD->fields()) {
1988	// We're done once we hit the flexible array member or run out of
1989	// InitListExpr elements.
1990	if (Field->getType()->isIncompleteArrayType() ||
1991	ILEElement == ILE->getNumInits())
1992	break;
1993	if (Field->isUnnamedBitField())
1994	continue;
1995
1996	const Expr *E = ILE->getInit(Init: ILEElement++);
1997
1998	// Reference values are always non-null and have the width of a pointer.
1999	if (Field->getType()->isReferenceType())
2000	NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits(
2001	BitSize: CGF.getTarget().getPointerWidth(AddrSpace: LangAS::Default));
2002	else
2003	NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF);
2004	}
2005
2006	return NumNonZeroBytes;
2007	}
2008	}
2009
2010	// FIXME: This overestimates the number of non-zero bytes for bit-fields.
2011	CharUnits NumNonZeroBytes = CharUnits::Zero();
2012	for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i)
2013	NumNonZeroBytes += GetNumNonZeroBytesInInit(E: ILE->getInit(Init: i), CGF);
2014	return NumNonZeroBytes;
2015	}
2016
2017	/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of
2018	/// zeros in it, emit a memset and avoid storing the individual zeros.
2019	///
2020	static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
2021	CodeGenFunction &CGF) {
2022	// If the slot is already known to be zeroed, nothing to do. Don't mess with
2023	// volatile stores.
2024	if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid())
2025	return;
2026
2027	// C++ objects with a user-declared constructor don't need zero'ing.
2028	if (CGF.getLangOpts().CPlusPlus)
2029	if (const RecordType *RT = CGF.getContext()
2030	.getBaseElementType(QT: E->getType())->getAs<RecordType>()) {
2031	const CXXRecordDecl *RD = cast<CXXRecordDecl>(Val: RT->getDecl());
2032	if (RD->hasUserDeclaredConstructor())
2033	return;
2034	}
2035
2036	// If the type is 16-bytes or smaller, prefer individual stores over memset.
2037	CharUnits Size = Slot.getPreferredSize(Ctx&: CGF.getContext(), Type: E->getType());
2038	if (Size <= CharUnits::fromQuantity(Quantity: 16))
2039	return;
2040
2041	// Check to see if over 3/4 of the initializer are known to be zero. If so,
2042	// we prefer to emit memset + individual stores for the rest.
2043	CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF);
2044	if (NumNonZeroBytes*4 > Size)
2045	return;
2046
2047	// Okay, it seems like a good idea to use an initial memset, emit the call.
2048	llvm::Constant *SizeVal = CGF.Builder.getInt64(C: Size.getQuantity());
2049
2050	Address Loc = Slot.getAddress().withElementType(ElemTy: CGF.Int8Ty);
2051	CGF.Builder.CreateMemSet(Dest: Loc, Value: CGF.Builder.getInt8(C: 0), Size: SizeVal, IsVolatile: false);
2052
2053	// Tell the AggExprEmitter that the slot is known zero.
2054	Slot.setZeroed();
2055	}
2056
2057
2058
2059
2060	/// EmitAggExpr - Emit the computation of the specified expression of aggregate
2061	/// type. The result is computed into DestPtr. Note that if DestPtr is null,
2062	/// the value of the aggregate expression is not needed. If VolatileDest is
2063	/// true, DestPtr cannot be 0.
2064	void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) {
2065	assert(E && hasAggregateEvaluationKind(E->getType()) &&
2066	"Invalid aggregate expression to emit");
2067	assert((Slot.getAddress().isValid() || Slot.isIgnored()) &&
2068	"slot has bits but no address");
2069
2070	// Optimize the slot if possible.
2071	CheckAggExprForMemSetUse(Slot, E, CGF&: *this);
2072
2073	AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(E: const_cast<Expr*>(E));
2074	}
2075
2076	LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
2077	assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!");
2078	Address Temp = CreateMemTemp(T: E->getType());
2079	LValue LV = MakeAddrLValue(Addr: Temp, T: E->getType());
2080	EmitAggExpr(E, Slot: AggValueSlot::forLValue(
2081	LV, CGF&: *this, isDestructed: AggValueSlot::IsNotDestructed,
2082	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
2083	isAliased: AggValueSlot::IsNotAliased, mayOverlap: AggValueSlot::DoesNotOverlap));
2084	return LV;
2085	}
2086
2087	AggValueSlot::Overlap_t
2088	CodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) {
2089	if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType())
2090	return AggValueSlot::DoesNotOverlap;
2091
2092	// If the field lies entirely within the enclosing class's nvsize, its tail
2093	// padding cannot overlap any already-initialized object. (The only subobjects
2094	// with greater addresses that might already be initialized are vbases.)
2095	const RecordDecl *ClassRD = FD->getParent();
2096	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D: ClassRD);
2097	if (Layout.getFieldOffset(FieldNo: FD->getFieldIndex()) +
2098	getContext().getTypeSize(FD->getType()) <=
2099	(uint64_t)getContext().toBits(CharSize: Layout.getNonVirtualSize()))
2100	return AggValueSlot::DoesNotOverlap;
2101
2102	// The tail padding may contain values we need to preserve.
2103	return AggValueSlot::MayOverlap;
2104	}
2105
2106	AggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit(
2107	const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) {
2108	// If the most-derived object is a field declared with [[no_unique_address]],
2109	// the tail padding of any virtual base could be reused for other subobjects
2110	// of that field's class.
2111	if (IsVirtual)
2112	return AggValueSlot::MayOverlap;
2113
2114	// If the base class is laid out entirely within the nvsize of the derived
2115	// class, its tail padding cannot yet be initialized, so we can issue
2116	// stores at the full width of the base class.
2117	const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD);
2118	if (Layout.getBaseClassOffset(Base: BaseRD) +
2119	getContext().getASTRecordLayout(BaseRD).getSize() <=
2120	Layout.getNonVirtualSize())
2121	return AggValueSlot::DoesNotOverlap;
2122
2123	// The tail padding may contain values we need to preserve.
2124	return AggValueSlot::MayOverlap;
2125	}
2126
2127	void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty,
2128	AggValueSlot::Overlap_t MayOverlap,
2129	bool isVolatile) {
2130	assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
2131
2132	Address DestPtr = Dest.getAddress(CGF&: *this);
2133	Address SrcPtr = Src.getAddress(CGF&: *this);
2134
2135	if (getLangOpts().CPlusPlus) {
2136	if (const RecordType *RT = Ty->getAs<RecordType>()) {
2137	CXXRecordDecl *Record = cast<CXXRecordDecl>(Val: RT->getDecl());
2138	assert((Record->hasTrivialCopyConstructor() ||
2139	Record->hasTrivialCopyAssignment() ||
2140	Record->hasTrivialMoveConstructor() ||
2141	Record->hasTrivialMoveAssignment() ||
2142	Record->hasAttr<TrivialABIAttr>() || Record->isUnion()) &&
2143	"Trying to aggregate-copy a type without a trivial copy/move "
2144	"constructor or assignment operator");
2145	// Ignore empty classes in C++.
2146	if (Record->isEmpty())
2147	return;
2148	}
2149	}
2150
2151	if (getLangOpts().CUDAIsDevice) {
2152	if (Ty->isCUDADeviceBuiltinSurfaceType()) {
2153	if (getTargetHooks().emitCUDADeviceBuiltinSurfaceDeviceCopy(CGF&: *this, Dst: Dest,
2154	Src))
2155	return;
2156	} else if (Ty->isCUDADeviceBuiltinTextureType()) {
2157	if (getTargetHooks().emitCUDADeviceBuiltinTextureDeviceCopy(CGF&: *this, Dst: Dest,
2158	Src))
2159	return;
2160	}
2161	}
2162
2163	// Aggregate assignment turns into llvm.memcpy. This is almost valid per
2164	// C99 6.5.16.1p3, which states "If the value being stored in an object is
2165	// read from another object that overlaps in anyway the storage of the first
2166	// object, then the overlap shall be exact and the two objects shall have
2167	// qualified or unqualified versions of a compatible type."
2168	//
2169	// memcpy is not defined if the source and destination pointers are exactly
2170	// equal, but other compilers do this optimization, and almost every memcpy
2171	// implementation handles this case safely. If there is a libc that does not
2172	// safely handle this, we can add a target hook.
2173
2174	// Get data size info for this aggregate. Don't copy the tail padding if this
2175	// might be a potentially-overlapping subobject, since the tail padding might
2176	// be occupied by a different object. Otherwise, copying it is fine.
2177	TypeInfoChars TypeInfo;
2178	if (MayOverlap)
2179	TypeInfo = getContext().getTypeInfoDataSizeInChars(T: Ty);
2180	else
2181	TypeInfo = getContext().getTypeInfoInChars(T: Ty);
2182
2183	llvm::Value *SizeVal = nullptr;
2184	if (TypeInfo.Width.isZero()) {
2185	// But note that getTypeInfo returns 0 for a VLA.
2186	if (auto *VAT = dyn_cast_or_null<VariableArrayType>(
2187	Val: getContext().getAsArrayType(T: Ty))) {
2188	QualType BaseEltTy;
2189	SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr);
2190	TypeInfo = getContext().getTypeInfoInChars(T: BaseEltTy);
2191	assert(!TypeInfo.Width.isZero());
2192	SizeVal = Builder.CreateNUWMul(
2193	LHS: SizeVal,
2194	RHS: llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity()));
2195	}
2196	}
2197	if (!SizeVal) {
2198	SizeVal = llvm::ConstantInt::get(Ty: SizeTy, V: TypeInfo.Width.getQuantity());
2199	}
2200
2201	// FIXME: If we have a volatile struct, the optimizer can remove what might
2202	// appear to be `extra' memory ops:
2203	//
2204	// volatile struct { int i; } a, b;
2205	//
2206	// int main() {
2207	// a = b;
2208	// a = b;
2209	// }
2210	//
2211	// we need to use a different call here. We use isVolatile to indicate when
2212	// either the source or the destination is volatile.
2213
2214	DestPtr = DestPtr.withElementType(ElemTy: Int8Ty);
2215	SrcPtr = SrcPtr.withElementType(ElemTy: Int8Ty);
2216
2217	// Don't do any of the memmove_collectable tests if GC isn't set.
2218	if (CGM.getLangOpts().getGC() == LangOptions::NonGC) {
2219	// fall through
2220	} else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) {
2221	RecordDecl *Record = RecordTy->getDecl();
2222	if (Record->hasObjectMember()) {
2223	CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2224	Size: SizeVal);
2225	return;
2226	}
2227	} else if (Ty->isArrayType()) {
2228	QualType BaseType = getContext().getBaseElementType(QT: Ty);
2229	if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) {
2230	if (RecordTy->getDecl()->hasObjectMember()) {
2231	CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF&: *this, DestPtr, SrcPtr,
2232	Size: SizeVal);
2233	return;
2234	}
2235	}
2236	}
2237
2238	auto Inst = Builder.CreateMemCpy(Dest: DestPtr, Src: SrcPtr, Size: SizeVal, IsVolatile: isVolatile);
2239
2240	// Determine the metadata to describe the position of any padding in this
2241	// memcpy, as well as the TBAA tags for the members of the struct, in case
2242	// the optimizer wishes to expand it in to scalar memory operations.
2243	if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(QTy: Ty))
2244	Inst->setMetadata(KindID: llvm::LLVMContext::MD_tbaa_struct, Node: TBAAStructTag);
2245
2246	if (CGM.getCodeGenOpts().NewStructPathTBAA) {
2247	TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer(
2248	DestInfo: Dest.getTBAAInfo(), SrcInfo: Src.getTBAAInfo());
2249	CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo);
2250	}
2251	}
2252