1	//===--- Compiler.cpp - Code generator for expressions ---*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "Compiler.h"
10	#include "ByteCodeEmitter.h"
11	#include "Context.h"
12	#include "FixedPoint.h"
13	#include "Floating.h"
14	#include "Function.h"
15	#include "InterpShared.h"
16	#include "PrimType.h"
17	#include "Program.h"
18	#include "clang/AST/Attr.h"
19
20	using namespace clang;
21	using namespace clang::interp;
22
23	using APSInt = llvm::APSInt;
24
25	namespace clang {
26	namespace interp {
27
28	static std::optional<bool> getBoolValue(const Expr *E) {
29	if (const auto *CE = dyn_cast_if_present<ConstantExpr>(Val: E);
30	CE && CE->hasAPValueResult() &&
31	CE->getResultAPValueKind() == APValue::ValueKind::Int) {
32	return CE->getResultAsAPSInt().getBoolValue();
33	}
34
35	return std::nullopt;
36	}
37
38	/// Scope used to handle temporaries in toplevel variable declarations.
39	template <class Emitter> class DeclScope final : public LocalScope<Emitter> {
40	public:
41	DeclScope(Compiler<Emitter> *Ctx, const ValueDecl *VD)
42	: LocalScope<Emitter>(Ctx, VD), Scope(Ctx->P),
43	OldInitializingDecl(Ctx->InitializingDecl) {
44	Ctx->InitializingDecl = VD;
45	Ctx->InitStack.push_back(InitLink::Decl(D: VD));
46	}
47
48	~DeclScope() {
49	this->Ctx->InitializingDecl = OldInitializingDecl;
50	this->Ctx->InitStack.pop_back();
51	}
52
53	private:
54	Program::DeclScope Scope;
55	const ValueDecl *OldInitializingDecl;
56	};
57
58	/// Scope used to handle initialization methods.
59	template <class Emitter> class OptionScope final {
60	public:
61	/// Root constructor, compiling or discarding primitives.
62	OptionScope(Compiler<Emitter> *Ctx, bool NewDiscardResult,
63	bool NewInitializing)
64	: Ctx(Ctx), OldDiscardResult(Ctx->DiscardResult),
65	OldInitializing(Ctx->Initializing) {
66	Ctx->DiscardResult = NewDiscardResult;
67	Ctx->Initializing = NewInitializing;
68	}
69
70	~OptionScope() {
71	Ctx->DiscardResult = OldDiscardResult;
72	Ctx->Initializing = OldInitializing;
73	}
74
75	private:
76	/// Parent context.
77	Compiler<Emitter> *Ctx;
78	/// Old discard flag to restore.
79	bool OldDiscardResult;
80	bool OldInitializing;
81	};
82
83	template <class Emitter>
84	bool InitLink::emit(Compiler<Emitter> *Ctx, const Expr *E) const {
85	switch (Kind) {
86	case K_This:
87	return Ctx->emitThis(E);
88	case K_Field:
89	// We're assuming there's a base pointer on the stack already.
90	return Ctx->emitGetPtrFieldPop(Offset, E);
91	case K_Temp:
92	return Ctx->emitGetPtrLocal(Offset, E);
93	case K_Decl:
94	return Ctx->visitDeclRef(D, E);
95	case K_Elem:
96	if (!Ctx->emitConstUint32(Offset, E))
97	return false;
98	return Ctx->emitArrayElemPtrPopUint32(E);
99	case K_RVO:
100	return Ctx->emitRVOPtr(E);
101	case K_InitList:
102	return true;
103	default:
104	llvm_unreachable("Unhandled InitLink kind");
105	}
106	return true;
107	}
108
109	/// Scope managing label targets.
110	template <class Emitter> class LabelScope {
111	public:
112	virtual ~LabelScope() {}
113
114	protected:
115	LabelScope(Compiler<Emitter> *Ctx) : Ctx(Ctx) {}
116	/// Compiler instance.
117	Compiler<Emitter> *Ctx;
118	};
119
120	/// Sets the context for break/continue statements.
121	template <class Emitter> class LoopScope final : public LabelScope<Emitter> {
122	public:
123	using LabelTy = typename Compiler<Emitter>::LabelTy;
124	using OptLabelTy = typename Compiler<Emitter>::OptLabelTy;
125
126	LoopScope(Compiler<Emitter> *Ctx, LabelTy BreakLabel, LabelTy ContinueLabel)
127	: LabelScope<Emitter>(Ctx), OldBreakLabel(Ctx->BreakLabel),
128	OldContinueLabel(Ctx->ContinueLabel),
129	OldBreakVarScope(Ctx->BreakVarScope),
130	OldContinueVarScope(Ctx->ContinueVarScope) {
131	this->Ctx->BreakLabel = BreakLabel;
132	this->Ctx->ContinueLabel = ContinueLabel;
133	this->Ctx->BreakVarScope = this->Ctx->VarScope;
134	this->Ctx->ContinueVarScope = this->Ctx->VarScope;
135	}
136
137	~LoopScope() {
138	this->Ctx->BreakLabel = OldBreakLabel;
139	this->Ctx->ContinueLabel = OldContinueLabel;
140	this->Ctx->ContinueVarScope = OldContinueVarScope;
141	this->Ctx->BreakVarScope = OldBreakVarScope;
142	}
143
144	private:
145	OptLabelTy OldBreakLabel;
146	OptLabelTy OldContinueLabel;
147	VariableScope<Emitter> *OldBreakVarScope;
148	VariableScope<Emitter> *OldContinueVarScope;
149	};
150
151	// Sets the context for a switch scope, mapping labels.
152	template <class Emitter> class SwitchScope final : public LabelScope<Emitter> {
153	public:
154	using LabelTy = typename Compiler<Emitter>::LabelTy;
155	using OptLabelTy = typename Compiler<Emitter>::OptLabelTy;
156	using CaseMap = typename Compiler<Emitter>::CaseMap;
157
158	SwitchScope(Compiler<Emitter> *Ctx, CaseMap &&CaseLabels, LabelTy BreakLabel,
159	OptLabelTy DefaultLabel)
160	: LabelScope<Emitter>(Ctx), OldBreakLabel(Ctx->BreakLabel),
161	OldDefaultLabel(this->Ctx->DefaultLabel),
162	OldCaseLabels(std::move(this->Ctx->CaseLabels)),
163	OldLabelVarScope(Ctx->BreakVarScope) {
164	this->Ctx->BreakLabel = BreakLabel;
165	this->Ctx->DefaultLabel = DefaultLabel;
166	this->Ctx->CaseLabels = std::move(CaseLabels);
167	this->Ctx->BreakVarScope = this->Ctx->VarScope;
168	}
169
170	~SwitchScope() {
171	this->Ctx->BreakLabel = OldBreakLabel;
172	this->Ctx->DefaultLabel = OldDefaultLabel;
173	this->Ctx->CaseLabels = std::move(OldCaseLabels);
174	this->Ctx->BreakVarScope = OldLabelVarScope;
175	}
176
177	private:
178	OptLabelTy OldBreakLabel;
179	OptLabelTy OldDefaultLabel;
180	CaseMap OldCaseLabels;
181	VariableScope<Emitter> *OldLabelVarScope;
182	};
183
184	template <class Emitter> class StmtExprScope final {
185	public:
186	StmtExprScope(Compiler<Emitter> *Ctx) : Ctx(Ctx), OldFlag(Ctx->InStmtExpr) {
187	Ctx->InStmtExpr = true;
188	}
189
190	~StmtExprScope() { Ctx->InStmtExpr = OldFlag; }
191
192	private:
193	Compiler<Emitter> *Ctx;
194	bool OldFlag;
195	};
196
197	} // namespace interp
198	} // namespace clang
199
200	template <class Emitter>
201	bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
202	const Expr *SubExpr = CE->getSubExpr();
203
204	if (DiscardResult)
205	return this->delegate(SubExpr);
206
207	switch (CE->getCastKind()) {
208	case CK_LValueToRValue: {
209	if (SubExpr->getType().isVolatileQualified())
210	return this->emitInvalidCast(CastKind::Volatile, /*Fatal=*/true, CE);
211
212	std::optional<PrimType> SubExprT = classify(SubExpr->getType());
213	// Prepare storage for the result.
214	if (!Initializing && !SubExprT) {
215	std::optional<unsigned> LocalIndex = allocateLocal(Decl: SubExpr);
216	if (!LocalIndex)
217	return false;
218	if (!this->emitGetPtrLocal(*LocalIndex, CE))
219	return false;
220	}
221
222	if (!this->visit(SubExpr))
223	return false;
224
225	if (SubExprT)
226	return this->emitLoadPop(*SubExprT, CE);
227
228	// If the subexpr type is not primitive, we need to perform a copy here.
229	// This happens for example in C when dereferencing a pointer of struct
230	// type.
231	return this->emitMemcpy(CE);
232	}
233
234	case CK_DerivedToBaseMemberPointer: {
235	assert(classifyPrim(CE->getType()) == PT_MemberPtr);
236	assert(classifyPrim(SubExpr->getType()) == PT_MemberPtr);
237	const auto *FromMP = SubExpr->getType()->castAs<MemberPointerType>();
238	const auto *ToMP = CE->getType()->castAs<MemberPointerType>();
239
240	unsigned DerivedOffset =
241	Ctx.collectBaseOffset(BaseDecl: ToMP->getMostRecentCXXRecordDecl(),
242	DerivedDecl: FromMP->getMostRecentCXXRecordDecl());
243
244	if (!this->delegate(SubExpr))
245	return false;
246
247	return this->emitGetMemberPtrBasePop(DerivedOffset, CE);
248	}
249
250	case CK_BaseToDerivedMemberPointer: {
251	assert(classifyPrim(CE) == PT_MemberPtr);
252	assert(classifyPrim(SubExpr) == PT_MemberPtr);
253	const auto *FromMP = SubExpr->getType()->castAs<MemberPointerType>();
254	const auto *ToMP = CE->getType()->castAs<MemberPointerType>();
255
256	unsigned DerivedOffset =
257	Ctx.collectBaseOffset(BaseDecl: FromMP->getMostRecentCXXRecordDecl(),
258	DerivedDecl: ToMP->getMostRecentCXXRecordDecl());
259
260	if (!this->delegate(SubExpr))
261	return false;
262	return this->emitGetMemberPtrBasePop(-DerivedOffset, CE);
263	}
264
265	case CK_UncheckedDerivedToBase:
266	case CK_DerivedToBase: {
267	if (!this->delegate(SubExpr))
268	return false;
269
270	const auto extractRecordDecl = [](QualType Ty) -> const CXXRecordDecl * {
271	if (const auto *PT = dyn_cast<PointerType>(Val&: Ty))
272	return PT->getPointeeType()->getAsCXXRecordDecl();
273	return Ty->getAsCXXRecordDecl();
274	};
275
276	// FIXME: We can express a series of non-virtual casts as a single
277	// GetPtrBasePop op.
278	QualType CurType = SubExpr->getType();
279	for (const CXXBaseSpecifier *B : CE->path()) {
280	if (B->isVirtual()) {
281	if (!this->emitGetPtrVirtBasePop(extractRecordDecl(B->getType()), CE))
282	return false;
283	CurType = B->getType();
284	} else {
285	unsigned DerivedOffset = collectBaseOffset(BaseType: B->getType(), DerivedType: CurType);
286	if (!this->emitGetPtrBasePop(
287	DerivedOffset, /*NullOK=*/CE->getType()->isPointerType(), CE))
288	return false;
289	CurType = B->getType();
290	}
291	}
292
293	return true;
294	}
295
296	case CK_BaseToDerived: {
297	if (!this->delegate(SubExpr))
298	return false;
299	unsigned DerivedOffset =
300	collectBaseOffset(BaseType: SubExpr->getType(), DerivedType: CE->getType());
301
302	const Type *TargetType = CE->getType().getTypePtr();
303	if (TargetType->isPointerOrReferenceType())
304	TargetType = TargetType->getPointeeType().getTypePtr();
305	return this->emitGetPtrDerivedPop(DerivedOffset,
306	/*NullOK=*/CE->getType()->isPointerType(),
307	TargetType, CE);
308	}
309
310	case CK_FloatingCast: {
311	// HLSL uses CK_FloatingCast to cast between vectors.
312	if (!SubExpr->getType()->isFloatingType() ||
313	!CE->getType()->isFloatingType())
314	return false;
315	if (!this->visit(SubExpr))
316	return false;
317	const auto *TargetSemantics = &Ctx.getFloatSemantics(T: CE->getType());
318	return this->emitCastFP(TargetSemantics, getRoundingMode(E: CE), CE);
319	}
320
321	case CK_IntegralToFloating: {
322	if (!CE->getType()->isRealFloatingType())
323	return false;
324	if (!this->visit(SubExpr))
325	return false;
326	const auto *TargetSemantics = &Ctx.getFloatSemantics(T: CE->getType());
327	return this->emitCastIntegralFloating(
328	classifyPrim(SubExpr), TargetSemantics, getFPOptions(E: CE), CE);
329	}
330
331	case CK_FloatingToBoolean: {
332	if (!SubExpr->getType()->isRealFloatingType() ||
333	!CE->getType()->isBooleanType())
334	return false;
335	if (const auto *FL = dyn_cast<FloatingLiteral>(Val: SubExpr))
336	return this->emitConstBool(FL->getValue().isNonZero(), CE);
337	if (!this->visit(SubExpr))
338	return false;
339	return this->emitCastFloatingIntegralBool(getFPOptions(E: CE), CE);
340	}
341
342	case CK_FloatingToIntegral: {
343	if (!this->visit(SubExpr))
344	return false;
345	PrimType ToT = classifyPrim(CE);
346	if (ToT == PT_IntAP)
347	return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(T: CE->getType()),
348	getFPOptions(E: CE), CE);
349	if (ToT == PT_IntAPS)
350	return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(T: CE->getType()),
351	getFPOptions(E: CE), CE);
352
353	return this->emitCastFloatingIntegral(ToT, getFPOptions(E: CE), CE);
354	}
355
356	case CK_NullToPointer:
357	case CK_NullToMemberPointer: {
358	if (!this->discard(SubExpr))
359	return false;
360	const Descriptor *Desc = nullptr;
361	const QualType PointeeType = CE->getType()->getPointeeType();
362	if (!PointeeType.isNull()) {
363	if (std::optional<PrimType> T = classify(PointeeType))
364	Desc = P.createDescriptor(D: SubExpr, T: *T);
365	else
366	Desc = P.createDescriptor(D: SubExpr, Ty: PointeeType.getTypePtr(),
367	MDSize: std::nullopt, /*IsConst=*/true);
368	}
369
370	uint64_t Val = Ctx.getASTContext().getTargetNullPointerValue(QT: CE->getType());
371	return this->emitNull(classifyPrim(CE->getType()), Val, Desc, CE);
372	}
373
374	case CK_PointerToIntegral: {
375	if (!this->visit(SubExpr))
376	return false;
377
378	// If SubExpr doesn't result in a pointer, make it one.
379	if (PrimType FromT = classifyPrim(SubExpr->getType()); FromT != PT_Ptr) {
380	assert(isPtrType(FromT));
381	if (!this->emitDecayPtr(FromT, PT_Ptr, CE))
382	return false;
383	}
384
385	PrimType T = classifyPrim(CE->getType());
386	if (T == PT_IntAP)
387	return this->emitCastPointerIntegralAP(Ctx.getBitWidth(T: CE->getType()),
388	CE);
389	if (T == PT_IntAPS)
390	return this->emitCastPointerIntegralAPS(Ctx.getBitWidth(T: CE->getType()),
391	CE);
392	return this->emitCastPointerIntegral(T, CE);
393	}
394
395	case CK_ArrayToPointerDecay: {
396	if (!this->visit(SubExpr))
397	return false;
398	return this->emitArrayDecay(CE);
399	}
400
401	case CK_IntegralToPointer: {
402	QualType IntType = SubExpr->getType();
403	assert(IntType->isIntegralOrEnumerationType());
404	if (!this->visit(SubExpr))
405	return false;
406	// FIXME: I think the discard is wrong since the int->ptr cast might cause a
407	// diagnostic.
408	PrimType T = classifyPrim(IntType);
409	QualType PtrType = CE->getType();
410	const Descriptor *Desc;
411	if (std::optional<PrimType> T = classify(PtrType->getPointeeType()))
412	Desc = P.createDescriptor(D: SubExpr, T: *T);
413	else if (PtrType->getPointeeType()->isVoidType())
414	Desc = nullptr;
415	else
416	Desc = P.createDescriptor(CE, PtrType->getPointeeType().getTypePtr(),
417	Descriptor::InlineDescMD, /*IsConst=*/true);
418
419	if (!this->emitGetIntPtr(T, Desc, CE))
420	return false;
421
422	PrimType DestPtrT = classifyPrim(PtrType);
423	if (DestPtrT == PT_Ptr)
424	return true;
425
426	// In case we're converting the integer to a non-Pointer.
427	return this->emitDecayPtr(PT_Ptr, DestPtrT, CE);
428	}
429
430	case CK_AtomicToNonAtomic:
431	case CK_ConstructorConversion:
432	case CK_FunctionToPointerDecay:
433	case CK_NonAtomicToAtomic:
434	case CK_NoOp:
435	case CK_UserDefinedConversion:
436	case CK_AddressSpaceConversion:
437	case CK_CPointerToObjCPointerCast:
438	return this->delegate(SubExpr);
439
440	case CK_BitCast: {
441	// Reject bitcasts to atomic types.
442	if (CE->getType()->isAtomicType()) {
443	if (!this->discard(SubExpr))
444	return false;
445	return this->emitInvalidCast(CastKind::Reinterpret, /*Fatal=*/true, CE);
446	}
447	QualType SubExprTy = SubExpr->getType();
448	std::optional<PrimType> FromT = classify(SubExprTy);
449	// Casts from integer/vector to vector.
450	if (CE->getType()->isVectorType())
451	return this->emitBuiltinBitCast(CE);
452
453	std::optional<PrimType> ToT = classify(CE->getType());
454	if (!FromT || !ToT)
455	return false;
456
457	assert(isPtrType(*FromT));
458	assert(isPtrType(*ToT));
459	if (FromT == ToT) {
460	if (CE->getType()->isVoidPointerType())
461	return this->delegate(SubExpr);
462
463	if (!this->visit(SubExpr))
464	return false;
465	if (CE->getType()->isFunctionPointerType())
466	return true;
467	if (FromT == PT_Ptr)
468	return this->emitPtrPtrCast(SubExprTy->isVoidPointerType(), CE);
469	return true;
470	}
471
472	if (!this->visit(SubExpr))
473	return false;
474	return this->emitDecayPtr(*FromT, *ToT, CE);
475	}
476	case CK_IntegralToBoolean:
477	case CK_FixedPointToBoolean: {
478	// HLSL uses this to cast to one-element vectors.
479	std::optional<PrimType> FromT = classify(SubExpr->getType());
480	if (!FromT)
481	return false;
482
483	if (const auto *IL = dyn_cast<IntegerLiteral>(Val: SubExpr))
484	return this->emitConst(IL->getValue(), CE);
485	if (!this->visit(SubExpr))
486	return false;
487	return this->emitCast(*FromT, classifyPrim(CE), CE);
488	}
489
490	case CK_BooleanToSignedIntegral:
491	case CK_IntegralCast: {
492	std::optional<PrimType> FromT = classify(SubExpr->getType());
493	std::optional<PrimType> ToT = classify(CE->getType());
494	if (!FromT || !ToT)
495	return false;
496
497	// Try to emit a casted known constant value directly.
498	if (const auto *IL = dyn_cast<IntegerLiteral>(Val: SubExpr)) {
499	if (ToT != PT_IntAP && ToT != PT_IntAPS && FromT != PT_IntAP &&
500	FromT != PT_IntAPS && !CE->getType()->isEnumeralType())
501	return this->emitConst(IL->getValue(), CE);
502	if (!this->emitConst(IL->getValue(), SubExpr))
503	return false;
504	} else {
505	if (!this->visit(SubExpr))
506	return false;
507	}
508
509	// Possibly diagnose casts to enum types if the target type does not
510	// have a fixed size.
511	if (Ctx.getLangOpts().CPlusPlus && CE->getType()->isEnumeralType()) {
512	if (const auto *ET = CE->getType().getCanonicalType()->castAs<EnumType>();
513	!ET->getDecl()->isFixed()) {
514	if (!this->emitCheckEnumValue(*FromT, ET->getDecl(), CE))
515	return false;
516	}
517	}
518
519	if (ToT == PT_IntAP) {
520	if (!this->emitCastAP(*FromT, Ctx.getBitWidth(T: CE->getType()), CE))
521	return false;
522	} else if (ToT == PT_IntAPS) {
523	if (!this->emitCastAPS(*FromT, Ctx.getBitWidth(T: CE->getType()), CE))
524	return false;
525	} else {
526	if (FromT == ToT)
527	return true;
528	if (!this->emitCast(*FromT, *ToT, CE))
529	return false;
530	}
531	if (CE->getCastKind() == CK_BooleanToSignedIntegral)
532	return this->emitNeg(*ToT, CE);
533	return true;
534	}
535
536	case CK_PointerToBoolean:
537	case CK_MemberPointerToBoolean: {
538	PrimType PtrT = classifyPrim(SubExpr->getType());
539
540	if (!this->visit(SubExpr))
541	return false;
542	return this->emitIsNonNull(PtrT, CE);
543	}
544
545	case CK_IntegralComplexToBoolean:
546	case CK_FloatingComplexToBoolean: {
547	if (!this->visit(SubExpr))
548	return false;
549	return this->emitComplexBoolCast(SubExpr);
550	}
551
552	case CK_IntegralComplexToReal:
553	case CK_FloatingComplexToReal:
554	return this->emitComplexReal(SubExpr);
555
556	case CK_IntegralRealToComplex:
557	case CK_FloatingRealToComplex: {
558	// We're creating a complex value here, so we need to
559	// allocate storage for it.
560	if (!Initializing) {
561	std::optional<unsigned> LocalIndex = allocateTemporary(E: CE);
562	if (!LocalIndex)
563	return false;
564	if (!this->emitGetPtrLocal(*LocalIndex, CE))
565	return false;
566	}
567
568	PrimType T = classifyPrim(SubExpr->getType());
569	// Init the complex value to {SubExpr, 0}.
570	if (!this->visitArrayElemInit(0, SubExpr, T))
571	return false;
572	// Zero-init the second element.
573	if (!this->visitZeroInitializer(T, SubExpr->getType(), SubExpr))
574	return false;
575	return this->emitInitElem(T, 1, SubExpr);
576	}
577
578	case CK_IntegralComplexCast:
579	case CK_FloatingComplexCast:
580	case CK_IntegralComplexToFloatingComplex:
581	case CK_FloatingComplexToIntegralComplex: {
582	assert(CE->getType()->isAnyComplexType());
583	assert(SubExpr->getType()->isAnyComplexType());
584	if (!Initializing) {
585	std::optional<unsigned> LocalIndex = allocateLocal(Decl: CE);
586	if (!LocalIndex)
587	return false;
588	if (!this->emitGetPtrLocal(*LocalIndex, CE))
589	return false;
590	}
591
592	// Location for the SubExpr.
593	// Since SubExpr is of complex type, visiting it results in a pointer
594	// anyway, so we just create a temporary pointer variable.
595	unsigned SubExprOffset =
596	allocateLocalPrimitive(Decl: SubExpr, Ty: PT_Ptr, /*IsConst=*/true);
597	if (!this->visit(SubExpr))
598	return false;
599	if (!this->emitSetLocal(PT_Ptr, SubExprOffset, CE))
600	return false;
601
602	PrimType SourceElemT = classifyComplexElementType(T: SubExpr->getType());
603	QualType DestElemType =
604	CE->getType()->getAs<ComplexType>()->getElementType();
605	PrimType DestElemT = classifyPrim(DestElemType);
606	// Cast both elements individually.
607	for (unsigned I = 0; I != 2; ++I) {
608	if (!this->emitGetLocal(PT_Ptr, SubExprOffset, CE))
609	return false;
610	if (!this->emitArrayElemPop(SourceElemT, I, CE))
611	return false;
612
613	// Do the cast.
614	if (!this->emitPrimCast(SourceElemT, DestElemT, DestElemType, CE))
615	return false;
616
617	// Save the value.
618	if (!this->emitInitElem(DestElemT, I, CE))
619	return false;
620	}
621	return true;
622	}
623
624	case CK_VectorSplat: {
625	assert(!classify(CE->getType()));
626	assert(classify(SubExpr->getType()));
627	assert(CE->getType()->isVectorType());
628
629	if (!Initializing) {
630	std::optional<unsigned> LocalIndex = allocateLocal(Decl: CE);
631	if (!LocalIndex)
632	return false;
633	if (!this->emitGetPtrLocal(*LocalIndex, CE))
634	return false;
635	}
636
637	const auto *VT = CE->getType()->getAs<VectorType>();
638	PrimType ElemT = classifyPrim(SubExpr->getType());
639	unsigned ElemOffset =
640	allocateLocalPrimitive(Decl: SubExpr, Ty: ElemT, /*IsConst=*/true);
641
642	// Prepare a local variable for the scalar value.
643	if (!this->visit(SubExpr))
644	return false;
645	if (classifyPrim(SubExpr) == PT_Ptr && !this->emitLoadPop(ElemT, CE))
646	return false;
647
648	if (!this->emitSetLocal(ElemT, ElemOffset, CE))
649	return false;
650
651	for (unsigned I = 0; I != VT->getNumElements(); ++I) {
652	if (!this->emitGetLocal(ElemT, ElemOffset, CE))
653	return false;
654	if (!this->emitInitElem(ElemT, I, CE))
655	return false;
656	}
657
658	return true;
659	}
660
661	case CK_HLSLVectorTruncation: {
662	assert(SubExpr->getType()->isVectorType());
663	if (std::optional<PrimType> ResultT = classify(CE)) {
664	assert(!DiscardResult);
665	// Result must be either a float or integer. Take the first element.
666	if (!this->visit(SubExpr))
667	return false;
668	return this->emitArrayElemPop(*ResultT, 0, CE);
669	}
670	// Otherwise, this truncates from one vector type to another.
671	assert(CE->getType()->isVectorType());
672
673	if (!Initializing) {
674	std::optional<unsigned> LocalIndex = allocateTemporary(E: CE);
675	if (!LocalIndex)
676	return false;
677	if (!this->emitGetPtrLocal(*LocalIndex, CE))
678	return false;
679	}
680	unsigned ToSize = CE->getType()->getAs<VectorType>()->getNumElements();
681	assert(SubExpr->getType()->getAs<VectorType>()->getNumElements() > ToSize);
682	if (!this->visit(SubExpr))
683	return false;
684	return this->emitCopyArray(classifyVectorElementType(T: CE->getType()), 0, 0,
685	ToSize, CE);
686	};
687
688	case CK_IntegralToFixedPoint: {
689	if (!this->visit(SubExpr))
690	return false;
691
692	auto Sem =
693	Ctx.getASTContext().getFixedPointSemantics(Ty: CE->getType()).toOpaqueInt();
694	return this->emitCastIntegralFixedPoint(classifyPrim(SubExpr->getType()),
695	Sem, CE);
696	}
697	case CK_FloatingToFixedPoint: {
698	if (!this->visit(SubExpr))
699	return false;
700
701	auto Sem =
702	Ctx.getASTContext().getFixedPointSemantics(Ty: CE->getType()).toOpaqueInt();
703	return this->emitCastFloatingFixedPoint(Sem, CE);
704	}
705	case CK_FixedPointToFloating: {
706	if (!this->visit(SubExpr))
707	return false;
708	const auto *TargetSemantics = &Ctx.getFloatSemantics(T: CE->getType());
709	return this->emitCastFixedPointFloating(TargetSemantics, CE);
710	}
711	case CK_FixedPointToIntegral: {
712	if (!this->visit(SubExpr))
713	return false;
714	return this->emitCastFixedPointIntegral(classifyPrim(CE->getType()), CE);
715	}
716	case CK_FixedPointCast: {
717	if (!this->visit(SubExpr))
718	return false;
719	auto Sem =
720	Ctx.getASTContext().getFixedPointSemantics(Ty: CE->getType()).toOpaqueInt();
721	return this->emitCastFixedPoint(Sem, CE);
722	}
723
724	case CK_ToVoid:
725	return discard(E: SubExpr);
726
727	default:
728	return this->emitInvalid(CE);
729	}
730	llvm_unreachable("Unhandled clang::CastKind enum");
731	}
732
733	template <class Emitter>
734	bool Compiler<Emitter>::VisitBuiltinBitCastExpr(const BuiltinBitCastExpr *E) {
735	return this->emitBuiltinBitCast(E);
736	}
737
738	template <class Emitter>
739	bool Compiler<Emitter>::VisitIntegerLiteral(const IntegerLiteral *LE) {
740	if (DiscardResult)
741	return true;
742
743	return this->emitConst(LE->getValue(), LE);
744	}
745
746	template <class Emitter>
747	bool Compiler<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
748	if (DiscardResult)
749	return true;
750
751	return this->emitConstFloat(E->getValue(), E);
752	}
753
754	template <class Emitter>
755	bool Compiler<Emitter>::VisitImaginaryLiteral(const ImaginaryLiteral *E) {
756	assert(E->getType()->isAnyComplexType());
757	if (DiscardResult)
758	return true;
759
760	if (!Initializing) {
761	std::optional<unsigned> LocalIndex = allocateTemporary(E);
762	if (!LocalIndex)
763	return false;
764	if (!this->emitGetPtrLocal(*LocalIndex, E))
765	return false;
766	}
767
768	const Expr *SubExpr = E->getSubExpr();
769	PrimType SubExprT = classifyPrim(SubExpr->getType());
770
771	if (!this->visitZeroInitializer(SubExprT, SubExpr->getType(), SubExpr))
772	return false;
773	if (!this->emitInitElem(SubExprT, 0, SubExpr))
774	return false;
775	return this->visitArrayElemInit(1, SubExpr, SubExprT);
776	}
777
778	template <class Emitter>
779	bool Compiler<Emitter>::VisitFixedPointLiteral(const FixedPointLiteral *E) {
780	assert(E->getType()->isFixedPointType());
781	assert(classifyPrim(E) == PT_FixedPoint);
782
783	if (DiscardResult)
784	return true;
785
786	auto Sem = Ctx.getASTContext().getFixedPointSemantics(Ty: E->getType());
787	APInt Value = E->getValue();
788	return this->emitConstFixedPoint(FixedPoint(Value, Sem), E);
789	}
790
791	template <class Emitter>
792	bool Compiler<Emitter>::VisitParenExpr(const ParenExpr *E) {
793	return this->delegate(E->getSubExpr());
794	}
795
796	template <class Emitter>
797	bool Compiler<Emitter>::VisitBinaryOperator(const BinaryOperator *BO) {
798	// Need short-circuiting for these.
799	if (BO->isLogicalOp() && !BO->getType()->isVectorType())
800	return this->VisitLogicalBinOp(BO);
801
802	const Expr *LHS = BO->getLHS();
803	const Expr *RHS = BO->getRHS();
804
805	// Handle comma operators. Just discard the LHS
806	// and delegate to RHS.
807	if (BO->isCommaOp()) {
808	if (!this->discard(LHS))
809	return false;
810	if (RHS->getType()->isVoidType())
811	return this->discard(RHS);
812
813	return this->delegate(RHS);
814	}
815
816	if (BO->getType()->isAnyComplexType())
817	return this->VisitComplexBinOp(BO);
818	if (BO->getType()->isVectorType())
819	return this->VisitVectorBinOp(BO);
820	if ((LHS->getType()->isAnyComplexType() ||
821	RHS->getType()->isAnyComplexType()) &&
822	BO->isComparisonOp())
823	return this->emitComplexComparison(LHS, RHS, BO);
824	if (LHS->getType()->isFixedPointType() || RHS->getType()->isFixedPointType())
825	return this->VisitFixedPointBinOp(BO);
826
827	if (BO->isPtrMemOp()) {
828	if (!this->visit(LHS))
829	return false;
830
831	if (!this->visit(RHS))
832	return false;
833
834	if (!this->emitToMemberPtr(BO))
835	return false;
836
837	if (classifyPrim(BO) == PT_MemberPtr)
838	return true;
839
840	if (!this->emitCastMemberPtrPtr(BO))
841	return false;
842	return DiscardResult ? this->emitPopPtr(BO) : true;
843	}
844
845	// Typecheck the args.
846	std::optional<PrimType> LT = classify(LHS);
847	std::optional<PrimType> RT = classify(RHS);
848	std::optional<PrimType> T = classify(BO->getType());
849
850	// Special case for C++'s three-way/spaceship operator <=>, which
851	// returns a std::{strong,weak,partial}_ordering (which is a class, so doesn't
852	// have a PrimType).
853	if (!T && BO->getOpcode() == BO_Cmp) {
854	if (DiscardResult)
855	return true;
856	const ComparisonCategoryInfo *CmpInfo =
857	Ctx.getASTContext().CompCategories.lookupInfoForType(Ty: BO->getType());
858	assert(CmpInfo);
859
860	// We need a temporary variable holding our return value.
861	if (!Initializing) {
862	std::optional<unsigned> ResultIndex = this->allocateLocal(BO);
863	if (!this->emitGetPtrLocal(*ResultIndex, BO))
864	return false;
865	}
866
867	if (!visit(E: LHS) || !visit(E: RHS))
868	return false;
869
870	return this->emitCMP3(*LT, CmpInfo, BO);
871	}
872
873	if (!LT || !RT || !T)
874	return false;
875
876	// Pointer arithmetic special case.
877	if (BO->getOpcode() == BO_Add || BO->getOpcode() == BO_Sub) {
878	if (isPtrType(T: *T) || (isPtrType(T: *LT) && isPtrType(T: *RT)))
879	return this->VisitPointerArithBinOp(BO);
880	}
881
882	// Assignments require us to evalute the RHS first.
883	if (BO->getOpcode() == BO_Assign) {
884
885	if (!visit(E: RHS) || !visit(E: LHS))
886	return false;
887
888	// We don't support assignments in C.
889	if (!Ctx.getLangOpts().CPlusPlus && !this->emitInvalid(BO))
890	return false;
891
892	if (!this->emitFlip(*LT, *RT, BO))
893	return false;
894	} else {
895	if (!visit(E: LHS) || !visit(E: RHS))
896	return false;
897	}
898
899	// For languages such as C, cast the result of one
900	// of our comparision opcodes to T (which is usually int).
901	auto MaybeCastToBool = [this, T, BO](bool Result) {
902	if (!Result)
903	return false;
904	if (DiscardResult)
905	return this->emitPop(*T, BO);
906	if (T != PT_Bool)
907	return this->emitCast(PT_Bool, *T, BO);
908	return true;
909	};
910
911	auto Discard = [this, T, BO](bool Result) {
912	if (!Result)
913	return false;
914	return DiscardResult ? this->emitPop(*T, BO) : true;
915	};
916
917	switch (BO->getOpcode()) {
918	case BO_EQ:
919	return MaybeCastToBool(this->emitEQ(*LT, BO));
920	case BO_NE:
921	return MaybeCastToBool(this->emitNE(*LT, BO));
922	case BO_LT:
923	return MaybeCastToBool(this->emitLT(*LT, BO));
924	case BO_LE:
925	return MaybeCastToBool(this->emitLE(*LT, BO));
926	case BO_GT:
927	return MaybeCastToBool(this->emitGT(*LT, BO));
928	case BO_GE:
929	return MaybeCastToBool(this->emitGE(*LT, BO));
930	case BO_Sub:
931	if (BO->getType()->isFloatingType())
932	return Discard(this->emitSubf(getFPOptions(E: BO), BO));
933	return Discard(this->emitSub(*T, BO));
934	case BO_Add:
935	if (BO->getType()->isFloatingType())
936	return Discard(this->emitAddf(getFPOptions(E: BO), BO));
937	return Discard(this->emitAdd(*T, BO));
938	case BO_Mul:
939	if (BO->getType()->isFloatingType())
940	return Discard(this->emitMulf(getFPOptions(E: BO), BO));
941	return Discard(this->emitMul(*T, BO));
942	case BO_Rem:
943	return Discard(this->emitRem(*T, BO));
944	case BO_Div:
945	if (BO->getType()->isFloatingType())
946	return Discard(this->emitDivf(getFPOptions(E: BO), BO));
947	return Discard(this->emitDiv(*T, BO));
948	case BO_Assign:
949	if (DiscardResult)
950	return LHS->refersToBitField() ? this->emitStoreBitFieldPop(*T, BO)
951	: this->emitStorePop(*T, BO);
952	if (LHS->refersToBitField()) {
953	if (!this->emitStoreBitField(*T, BO))
954	return false;
955	} else {
956	if (!this->emitStore(*T, BO))
957	return false;
958	}
959	// Assignments aren't necessarily lvalues in C.
960	// Load from them in that case.
961	if (!BO->isLValue())
962	return this->emitLoadPop(*T, BO);
963	return true;
964	case BO_And:
965	return Discard(this->emitBitAnd(*T, BO));
966	case BO_Or:
967	return Discard(this->emitBitOr(*T, BO));
968	case BO_Shl:
969	return Discard(this->emitShl(*LT, *RT, BO));
970	case BO_Shr:
971	return Discard(this->emitShr(*LT, *RT, BO));
972	case BO_Xor:
973	return Discard(this->emitBitXor(*T, BO));
974	case BO_LOr:
975	case BO_LAnd:
976	llvm_unreachable("Already handled earlier");
977	default:
978	return false;
979	}
980
981	llvm_unreachable("Unhandled binary op");
982	}
983
984	/// Perform addition/subtraction of a pointer and an integer or
985	/// subtraction of two pointers.
986	template <class Emitter>
987	bool Compiler<Emitter>::VisitPointerArithBinOp(const BinaryOperator *E) {
988	BinaryOperatorKind Op = E->getOpcode();
989	const Expr *LHS = E->getLHS();
990	const Expr *RHS = E->getRHS();
991
992	if ((Op != BO_Add && Op != BO_Sub) ||
993	(!LHS->getType()->isPointerType() && !RHS->getType()->isPointerType()))
994	return false;
995
996	std::optional<PrimType> LT = classify(LHS);
997	std::optional<PrimType> RT = classify(RHS);
998
999	if (!LT || !RT)
1000	return false;
1001
1002	// Visit the given pointer expression and optionally convert to a PT_Ptr.
1003	auto visitAsPointer = [&](const Expr *E, PrimType T) -> bool {
1004	if (!this->visit(E))
1005	return false;
1006	if (T != PT_Ptr)
1007	return this->emitDecayPtr(T, PT_Ptr, E);
1008	return true;
1009	};
1010
1011	if (LHS->getType()->isPointerType() && RHS->getType()->isPointerType()) {
1012	if (Op != BO_Sub)
1013	return false;
1014
1015	assert(E->getType()->isIntegerType());
1016	if (!visitAsPointer(RHS, *RT) || !visitAsPointer(LHS, *LT))
1017	return false;
1018
1019	PrimType IntT = classifyPrim(E->getType());
1020	if (!this->emitSubPtr(IntT, E))
1021	return false;
1022	return DiscardResult ? this->emitPop(IntT, E) : true;
1023	}
1024
1025	PrimType OffsetType;
1026	if (LHS->getType()->isIntegerType()) {
1027	if (!visitAsPointer(RHS, *RT))
1028	return false;
1029	if (!this->visit(LHS))
1030	return false;
1031	OffsetType = *LT;
1032	} else if (RHS->getType()->isIntegerType()) {
1033	if (!visitAsPointer(LHS, *LT))
1034	return false;
1035	if (!this->visit(RHS))
1036	return false;
1037	OffsetType = *RT;
1038	} else {
1039	return false;
1040	}
1041
1042	// Do the operation and optionally transform to
1043	// result pointer type.
1044	if (Op == BO_Add) {
1045	if (!this->emitAddOffset(OffsetType, E))
1046	return false;
1047
1048	if (classifyPrim(E) != PT_Ptr)
1049	return this->emitDecayPtr(PT_Ptr, classifyPrim(E), E);
1050	return true;
1051	} else if (Op == BO_Sub) {
1052	if (!this->emitSubOffset(OffsetType, E))
1053	return false;
1054
1055	if (classifyPrim(E) != PT_Ptr)
1056	return this->emitDecayPtr(PT_Ptr, classifyPrim(E), E);
1057	return true;
1058	}
1059
1060	return false;
1061	}
1062
1063	template <class Emitter>
1064	bool Compiler<Emitter>::VisitLogicalBinOp(const BinaryOperator *E) {
1065	assert(E->isLogicalOp());
1066	BinaryOperatorKind Op = E->getOpcode();
1067	const Expr *LHS = E->getLHS();
1068	const Expr *RHS = E->getRHS();
1069	std::optional<PrimType> T = classify(E->getType());
1070
1071	if (Op == BO_LOr) {
1072	// Logical OR. Visit LHS and only evaluate RHS if LHS was FALSE.
1073	LabelTy LabelTrue = this->getLabel();
1074	LabelTy LabelEnd = this->getLabel();
1075
1076	if (!this->visitBool(LHS))
1077	return false;
1078	if (!this->jumpTrue(LabelTrue))
1079	return false;
1080
1081	if (!this->visitBool(RHS))
1082	return false;
1083	if (!this->jump(LabelEnd))
1084	return false;
1085
1086	this->emitLabel(LabelTrue);
1087	this->emitConstBool(true, E);
1088	this->fallthrough(LabelEnd);
1089	this->emitLabel(LabelEnd);
1090
1091	} else {
1092	assert(Op == BO_LAnd);
1093	// Logical AND.
1094	// Visit LHS. Only visit RHS if LHS was TRUE.
1095	LabelTy LabelFalse = this->getLabel();
1096	LabelTy LabelEnd = this->getLabel();
1097
1098	if (!this->visitBool(LHS))
1099	return false;
1100	if (!this->jumpFalse(LabelFalse))
1101	return false;
1102
1103	if (!this->visitBool(RHS))
1104	return false;
1105	if (!this->jump(LabelEnd))
1106	return false;
1107
1108	this->emitLabel(LabelFalse);
1109	this->emitConstBool(false, E);
1110	this->fallthrough(LabelEnd);
1111	this->emitLabel(LabelEnd);
1112	}
1113
1114	if (DiscardResult)
1115	return this->emitPopBool(E);
1116
1117	// For C, cast back to integer type.
1118	assert(T);
1119	if (T != PT_Bool)
1120	return this->emitCast(PT_Bool, *T, E);
1121	return true;
1122	}
1123
1124	template <class Emitter>
1125	bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
1126	// Prepare storage for result.
1127	if (!Initializing) {
1128	std::optional<unsigned> LocalIndex = allocateTemporary(E);
1129	if (!LocalIndex)
1130	return false;
1131	if (!this->emitGetPtrLocal(*LocalIndex, E))
1132	return false;
1133	}
1134
1135	// Both LHS and RHS might _not_ be of complex type, but one of them
1136	// needs to be.
1137	const Expr *LHS = E->getLHS();
1138	const Expr *RHS = E->getRHS();
1139
1140	PrimType ResultElemT = this->classifyComplexElementType(E->getType());
1141	unsigned ResultOffset = ~0u;
1142	if (!DiscardResult)
1143	ResultOffset = this->allocateLocalPrimitive(E, PT_Ptr, /*IsConst=*/true);
1144
1145	// Save result pointer in ResultOffset
1146	if (!this->DiscardResult) {
1147	if (!this->emitDupPtr(E))
1148	return false;
1149	if (!this->emitSetLocal(PT_Ptr, ResultOffset, E))
1150	return false;
1151	}
1152	QualType LHSType = LHS->getType();
1153	if (const auto *AT = LHSType->getAs<AtomicType>())
1154	LHSType = AT->getValueType();
1155	QualType RHSType = RHS->getType();
1156	if (const auto *AT = RHSType->getAs<AtomicType>())
1157	RHSType = AT->getValueType();
1158
1159	bool LHSIsComplex = LHSType->isAnyComplexType();
1160	unsigned LHSOffset;
1161	bool RHSIsComplex = RHSType->isAnyComplexType();
1162
1163	// For ComplexComplex Mul, we have special ops to make their implementation
1164	// easier.
1165	BinaryOperatorKind Op = E->getOpcode();
1166	if (Op == BO_Mul && LHSIsComplex && RHSIsComplex) {
1167	assert(classifyPrim(LHSType->getAs<ComplexType>()->getElementType()) ==
1168	classifyPrim(RHSType->getAs<ComplexType>()->getElementType()));
1169	PrimType ElemT =
1170	classifyPrim(LHSType->getAs<ComplexType>()->getElementType());
1171	if (!this->visit(LHS))
1172	return false;
1173	if (!this->visit(RHS))
1174	return false;
1175	return this->emitMulc(ElemT, E);
1176	}
1177
1178	if (Op == BO_Div && RHSIsComplex) {
1179	QualType ElemQT = RHSType->getAs<ComplexType>()->getElementType();
1180	PrimType ElemT = classifyPrim(ElemQT);
1181	// If the LHS is not complex, we still need to do the full complex
1182	// division, so just stub create a complex value and stub it out with
1183	// the LHS and a zero.
1184
1185	if (!LHSIsComplex) {
1186	// This is using the RHS type for the fake-complex LHS.
1187	std::optional<unsigned> LocalIndex = allocateTemporary(E: RHS);
1188	if (!LocalIndex)
1189	return false;
1190	LHSOffset = *LocalIndex;
1191
1192	if (!this->emitGetPtrLocal(LHSOffset, E))
1193	return false;
1194
1195	if (!this->visit(LHS))
1196	return false;
1197	// real is LHS
1198	if (!this->emitInitElem(ElemT, 0, E))
1199	return false;
1200	// imag is zero
1201	if (!this->visitZeroInitializer(ElemT, ElemQT, E))
1202	return false;
1203	if (!this->emitInitElem(ElemT, 1, E))
1204	return false;
1205	} else {
1206	if (!this->visit(LHS))
1207	return false;
1208	}
1209
1210	if (!this->visit(RHS))
1211	return false;
1212	return this->emitDivc(ElemT, E);
1213	}
1214
1215	// Evaluate LHS and save value to LHSOffset.
1216	if (LHSType->isAnyComplexType()) {
1217	LHSOffset = this->allocateLocalPrimitive(LHS, PT_Ptr, /*IsConst=*/true);
1218	if (!this->visit(LHS))
1219	return false;
1220	if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
1221	return false;
1222	} else {
1223	PrimType LHST = classifyPrim(LHSType);
1224	LHSOffset = this->allocateLocalPrimitive(LHS, LHST, /*IsConst=*/true);
1225	if (!this->visit(LHS))
1226	return false;
1227	if (!this->emitSetLocal(LHST, LHSOffset, E))
1228	return false;
1229	}
1230
1231	// Same with RHS.
1232	unsigned RHSOffset;
1233	if (RHSType->isAnyComplexType()) {
1234	RHSOffset = this->allocateLocalPrimitive(RHS, PT_Ptr, /*IsConst=*/true);
1235	if (!this->visit(RHS))
1236	return false;
1237	if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
1238	return false;
1239	} else {
1240	PrimType RHST = classifyPrim(RHSType);
1241	RHSOffset = this->allocateLocalPrimitive(RHS, RHST, /*IsConst=*/true);
1242	if (!this->visit(RHS))
1243	return false;
1244	if (!this->emitSetLocal(RHST, RHSOffset, E))
1245	return false;
1246	}
1247
1248	// For both LHS and RHS, either load the value from the complex pointer, or
1249	// directly from the local variable. For index 1 (i.e. the imaginary part),
1250	// just load 0 and do the operation anyway.
1251	auto loadComplexValue = [this](bool IsComplex, bool LoadZero,
1252	unsigned ElemIndex, unsigned Offset,
1253	const Expr *E) -> bool {
1254	if (IsComplex) {
1255	if (!this->emitGetLocal(PT_Ptr, Offset, E))
1256	return false;
1257	return this->emitArrayElemPop(classifyComplexElementType(T: E->getType()),
1258	ElemIndex, E);
1259	}
1260	if (ElemIndex == 0 || !LoadZero)
1261	return this->emitGetLocal(classifyPrim(E->getType()), Offset, E);
1262	return this->visitZeroInitializer(classifyPrim(E->getType()), E->getType(),
1263	E);
1264	};
1265
1266	// Now we can get pointers to the LHS and RHS from the offsets above.
1267	for (unsigned ElemIndex = 0; ElemIndex != 2; ++ElemIndex) {
1268	// Result pointer for the store later.
1269	if (!this->DiscardResult) {
1270	if (!this->emitGetLocal(PT_Ptr, ResultOffset, E))
1271	return false;
1272	}
1273
1274	// The actual operation.
1275	switch (Op) {
1276	case BO_Add:
1277	if (!loadComplexValue(LHSIsComplex, true, ElemIndex, LHSOffset, LHS))
1278	return false;
1279
1280	if (!loadComplexValue(RHSIsComplex, true, ElemIndex, RHSOffset, RHS))
1281	return false;
1282	if (ResultElemT == PT_Float) {
1283	if (!this->emitAddf(getFPOptions(E), E))
1284	return false;
1285	} else {
1286	if (!this->emitAdd(ResultElemT, E))
1287	return false;
1288	}
1289	break;
1290	case BO_Sub:
1291	if (!loadComplexValue(LHSIsComplex, true, ElemIndex, LHSOffset, LHS))
1292	return false;
1293
1294	if (!loadComplexValue(RHSIsComplex, true, ElemIndex, RHSOffset, RHS))
1295	return false;
1296	if (ResultElemT == PT_Float) {
1297	if (!this->emitSubf(getFPOptions(E), E))
1298	return false;
1299	} else {
1300	if (!this->emitSub(ResultElemT, E))
1301	return false;
1302	}
1303	break;
1304	case BO_Mul:
1305	if (!loadComplexValue(LHSIsComplex, false, ElemIndex, LHSOffset, LHS))
1306	return false;
1307
1308	if (!loadComplexValue(RHSIsComplex, false, ElemIndex, RHSOffset, RHS))
1309	return false;
1310
1311	if (ResultElemT == PT_Float) {
1312	if (!this->emitMulf(getFPOptions(E), E))
1313	return false;
1314	} else {
1315	if (!this->emitMul(ResultElemT, E))
1316	return false;
1317	}
1318	break;
1319	case BO_Div:
1320	assert(!RHSIsComplex);
1321	if (!loadComplexValue(LHSIsComplex, false, ElemIndex, LHSOffset, LHS))
1322	return false;
1323
1324	if (!loadComplexValue(RHSIsComplex, false, ElemIndex, RHSOffset, RHS))
1325	return false;
1326
1327	if (ResultElemT == PT_Float) {
1328	if (!this->emitDivf(getFPOptions(E), E))
1329	return false;
1330	} else {
1331	if (!this->emitDiv(ResultElemT, E))
1332	return false;
1333	}
1334	break;
1335
1336	default:
1337	return false;
1338	}
1339
1340	if (!this->DiscardResult) {
1341	// Initialize array element with the value we just computed.
1342	if (!this->emitInitElemPop(ResultElemT, ElemIndex, E))
1343	return false;
1344	} else {
1345	if (!this->emitPop(ResultElemT, E))
1346	return false;
1347	}
1348	}
1349	return true;
1350	}
1351
1352	template <class Emitter>
1353	bool Compiler<Emitter>::VisitVectorBinOp(const BinaryOperator *E) {
1354	assert(!E->isCommaOp() &&
1355	"Comma op should be handled in VisitBinaryOperator");
1356	assert(E->getType()->isVectorType());
1357	assert(E->getLHS()->getType()->isVectorType());
1358	assert(E->getRHS()->getType()->isVectorType());
1359
1360	// Prepare storage for result.
1361	if (!Initializing && !E->isCompoundAssignmentOp()) {
1362	std::optional<unsigned> LocalIndex = allocateTemporary(E);
1363	if (!LocalIndex)
1364	return false;
1365	if (!this->emitGetPtrLocal(*LocalIndex, E))
1366	return false;
1367	}
1368
1369	const Expr *LHS = E->getLHS();
1370	const Expr *RHS = E->getRHS();
1371	const auto *VecTy = E->getType()->getAs<VectorType>();
1372	auto Op = E->isCompoundAssignmentOp()
1373	? BinaryOperator::getOpForCompoundAssignment(Opc: E->getOpcode())
1374	: E->getOpcode();
1375
1376	PrimType ElemT = this->classifyVectorElementType(LHS->getType());
1377	PrimType RHSElemT = this->classifyVectorElementType(RHS->getType());
1378	PrimType ResultElemT = this->classifyVectorElementType(E->getType());
1379
1380	// Evaluate LHS and save value to LHSOffset.
1381	unsigned LHSOffset =
1382	this->allocateLocalPrimitive(LHS, PT_Ptr, /*IsConst=*/true);
1383	if (!this->visit(LHS))
1384	return false;
1385	if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
1386	return false;
1387
1388	// Evaluate RHS and save value to RHSOffset.
1389	unsigned RHSOffset =
1390	this->allocateLocalPrimitive(RHS, PT_Ptr, /*IsConst=*/true);
1391	if (!this->visit(RHS))
1392	return false;
1393	if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
1394	return false;
1395
1396	if (E->isCompoundAssignmentOp() && !this->emitGetLocal(PT_Ptr, LHSOffset, E))
1397	return false;
1398
1399	// BitAdd/BitOr/BitXor/Shl/Shr doesn't support bool type, we need perform the
1400	// integer promotion.
1401	bool NeedIntPromot = ElemT == PT_Bool && (E->isBitwiseOp() || E->isShiftOp());
1402	QualType PromotTy =
1403	Ctx.getASTContext().getPromotedIntegerType(PromotableType: Ctx.getASTContext().BoolTy);
1404	PrimType PromotT = classifyPrim(PromotTy);
1405	PrimType OpT = NeedIntPromot ? PromotT : ElemT;
1406
1407	auto getElem = [=](unsigned Offset, PrimType ElemT, unsigned Index) {
1408	if (!this->emitGetLocal(PT_Ptr, Offset, E))
1409	return false;
1410	if (!this->emitArrayElemPop(ElemT, Index, E))
1411	return false;
1412	if (E->isLogicalOp()) {
1413	if (!this->emitPrimCast(ElemT, PT_Bool, Ctx.getASTContext().BoolTy, E))
1414	return false;
1415	if (!this->emitPrimCast(PT_Bool, ResultElemT, VecTy->getElementType(), E))
1416	return false;
1417	} else if (NeedIntPromot) {
1418	if (!this->emitPrimCast(ElemT, PromotT, PromotTy, E))
1419	return false;
1420	}
1421	return true;
1422	};
1423
1424	#define EMIT_ARITH_OP(OP) \
1425	{ \
1426	if (ElemT == PT_Float) { \
1427	if (!this->emit##OP##f(getFPOptions(E), E)) \
1428	return false; \
1429	} else { \
1430	if (!this->emit##OP(ElemT, E)) \
1431	return false; \
1432	} \
1433	break; \
1434	}
1435
1436	for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
1437	if (!getElem(LHSOffset, ElemT, I))
1438	return false;
1439	if (!getElem(RHSOffset, RHSElemT, I))
1440	return false;
1441	switch (Op) {
1442	case BO_Add:
1443	EMIT_ARITH_OP(Add)
1444	case BO_Sub:
1445	EMIT_ARITH_OP(Sub)
1446	case BO_Mul:
1447	EMIT_ARITH_OP(Mul)
1448	case BO_Div:
1449	EMIT_ARITH_OP(Div)
1450	case BO_Rem:
1451	if (!this->emitRem(ElemT, E))
1452	return false;
1453	break;
1454	case BO_And:
1455	if (!this->emitBitAnd(OpT, E))
1456	return false;
1457	break;
1458	case BO_Or:
1459	if (!this->emitBitOr(OpT, E))
1460	return false;
1461	break;
1462	case BO_Xor:
1463	if (!this->emitBitXor(OpT, E))
1464	return false;
1465	break;
1466	case BO_Shl:
1467	if (!this->emitShl(OpT, RHSElemT, E))
1468	return false;
1469	break;
1470	case BO_Shr:
1471	if (!this->emitShr(OpT, RHSElemT, E))
1472	return false;
1473	break;
1474	case BO_EQ:
1475	if (!this->emitEQ(ElemT, E))
1476	return false;
1477	break;
1478	case BO_NE:
1479	if (!this->emitNE(ElemT, E))
1480	return false;
1481	break;
1482	case BO_LE:
1483	if (!this->emitLE(ElemT, E))
1484	return false;
1485	break;
1486	case BO_LT:
1487	if (!this->emitLT(ElemT, E))
1488	return false;
1489	break;
1490	case BO_GE:
1491	if (!this->emitGE(ElemT, E))
1492	return false;
1493	break;
1494	case BO_GT:
1495	if (!this->emitGT(ElemT, E))
1496	return false;
1497	break;
1498	case BO_LAnd:
1499	// a && b is equivalent to a!=0 & b!=0
1500	if (!this->emitBitAnd(ResultElemT, E))
1501	return false;
1502	break;
1503	case BO_LOr:
1504	// a || b is equivalent to a!=0 | b!=0
1505	if (!this->emitBitOr(ResultElemT, E))
1506	return false;
1507	break;
1508	default:
1509	return this->emitInvalid(E);
1510	}
1511
1512	// The result of the comparison is a vector of the same width and number
1513	// of elements as the comparison operands with a signed integral element
1514	// type.
1515	//
1516	// https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
1517	if (E->isComparisonOp()) {
1518	if (!this->emitPrimCast(PT_Bool, ResultElemT, VecTy->getElementType(), E))
1519	return false;
1520	if (!this->emitNeg(ResultElemT, E))
1521	return false;
1522	}
1523
1524	// If we performed an integer promotion, we need to cast the compute result
1525	// into result vector element type.
1526	if (NeedIntPromot &&
1527	!this->emitPrimCast(PromotT, ResultElemT, VecTy->getElementType(), E))
1528	return false;
1529
1530	// Initialize array element with the value we just computed.
1531	if (!this->emitInitElem(ResultElemT, I, E))
1532	return false;
1533	}
1534
1535	if (DiscardResult && E->isCompoundAssignmentOp() && !this->emitPopPtr(E))
1536	return false;
1537	return true;
1538	}
1539
1540	template <class Emitter>
1541	bool Compiler<Emitter>::VisitFixedPointBinOp(const BinaryOperator *E) {
1542	const Expr *LHS = E->getLHS();
1543	const Expr *RHS = E->getRHS();
1544	const ASTContext &ASTCtx = Ctx.getASTContext();
1545
1546	assert(LHS->getType()->isFixedPointType() ||
1547	RHS->getType()->isFixedPointType());
1548
1549	auto LHSSema = ASTCtx.getFixedPointSemantics(Ty: LHS->getType());
1550	auto LHSSemaInt = LHSSema.toOpaqueInt();
1551	auto RHSSema = ASTCtx.getFixedPointSemantics(Ty: RHS->getType());
1552	auto RHSSemaInt = RHSSema.toOpaqueInt();
1553
1554	if (!this->visit(LHS))
1555	return false;
1556	if (!LHS->getType()->isFixedPointType()) {
1557	if (!this->emitCastIntegralFixedPoint(classifyPrim(LHS->getType()),
1558	LHSSemaInt, E))
1559	return false;
1560	}
1561
1562	if (!this->visit(RHS))
1563	return false;
1564	if (!RHS->getType()->isFixedPointType()) {
1565	if (!this->emitCastIntegralFixedPoint(classifyPrim(RHS->getType()),
1566	RHSSemaInt, E))
1567	return false;
1568	}
1569
1570	// Convert the result to the target semantics.
1571	auto ConvertResult = [&](bool R) -> bool {
1572	if (!R)
1573	return false;
1574	auto ResultSema = ASTCtx.getFixedPointSemantics(Ty: E->getType()).toOpaqueInt();
1575	auto CommonSema = LHSSema.getCommonSemantics(Other: RHSSema).toOpaqueInt();
1576	if (ResultSema != CommonSema)
1577	return this->emitCastFixedPoint(ResultSema, E);
1578	return true;
1579	};
1580
1581	auto MaybeCastToBool = [&](bool Result) {
1582	if (!Result)
1583	return false;
1584	PrimType T = classifyPrim(E);
1585	if (DiscardResult)
1586	return this->emitPop(T, E);
1587	if (T != PT_Bool)
1588	return this->emitCast(PT_Bool, T, E);
1589	return true;
1590	};
1591
1592	switch (E->getOpcode()) {
1593	case BO_EQ:
1594	return MaybeCastToBool(this->emitEQFixedPoint(E));
1595	case BO_NE:
1596	return MaybeCastToBool(this->emitNEFixedPoint(E));
1597	case BO_LT:
1598	return MaybeCastToBool(this->emitLTFixedPoint(E));
1599	case BO_LE:
1600	return MaybeCastToBool(this->emitLEFixedPoint(E));
1601	case BO_GT:
1602	return MaybeCastToBool(this->emitGTFixedPoint(E));
1603	case BO_GE:
1604	return MaybeCastToBool(this->emitGEFixedPoint(E));
1605	case BO_Add:
1606	return ConvertResult(this->emitAddFixedPoint(E));
1607	case BO_Sub:
1608	return ConvertResult(this->emitSubFixedPoint(E));
1609	case BO_Mul:
1610	return ConvertResult(this->emitMulFixedPoint(E));
1611	case BO_Div:
1612	return ConvertResult(this->emitDivFixedPoint(E));
1613	case BO_Shl:
1614	return ConvertResult(this->emitShiftFixedPoint(/*Left=*/true, E));
1615	case BO_Shr:
1616	return ConvertResult(this->emitShiftFixedPoint(/*Left=*/false, E));
1617
1618	default:
1619	return this->emitInvalid(E);
1620	}
1621
1622	llvm_unreachable("unhandled binop opcode");
1623	}
1624
1625	template <class Emitter>
1626	bool Compiler<Emitter>::VisitFixedPointUnaryOperator(const UnaryOperator *E) {
1627	const Expr *SubExpr = E->getSubExpr();
1628	assert(SubExpr->getType()->isFixedPointType());
1629
1630	switch (E->getOpcode()) {
1631	case UO_Plus:
1632	return this->delegate(SubExpr);
1633	case UO_Minus:
1634	if (!this->visit(SubExpr))
1635	return false;
1636	return this->emitNegFixedPoint(E);
1637	default:
1638	return false;
1639	}
1640
1641	llvm_unreachable("Unhandled unary opcode");
1642	}
1643
1644	template <class Emitter>
1645	bool Compiler<Emitter>::VisitImplicitValueInitExpr(
1646	const ImplicitValueInitExpr *E) {
1647	QualType QT = E->getType();
1648
1649	if (std::optional<PrimType> T = classify(QT))
1650	return this->visitZeroInitializer(*T, QT, E);
1651
1652	if (QT->isRecordType()) {
1653	const RecordDecl *RD = QT->getAsRecordDecl();
1654	assert(RD);
1655	if (RD->isInvalidDecl())
1656	return false;
1657
1658	if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD);
1659	CXXRD && CXXRD->getNumVBases() > 0) {
1660	// TODO: Diagnose.
1661	return false;
1662	}
1663
1664	const Record *R = getRecord(QT);
1665	if (!R)
1666	return false;
1667
1668	assert(Initializing);
1669	return this->visitZeroRecordInitializer(R, E);
1670	}
1671
1672	if (QT->isIncompleteArrayType())
1673	return true;
1674
1675	if (QT->isArrayType())
1676	return this->visitZeroArrayInitializer(QT, E);
1677
1678	if (const auto *ComplexTy = E->getType()->getAs<ComplexType>()) {
1679	assert(Initializing);
1680	QualType ElemQT = ComplexTy->getElementType();
1681	PrimType ElemT = classifyPrim(ElemQT);
1682	for (unsigned I = 0; I < 2; ++I) {
1683	if (!this->visitZeroInitializer(ElemT, ElemQT, E))
1684	return false;
1685	if (!this->emitInitElem(ElemT, I, E))
1686	return false;
1687	}
1688	return true;
1689	}
1690
1691	if (const auto *VecT = E->getType()->getAs<VectorType>()) {
1692	unsigned NumVecElements = VecT->getNumElements();
1693	QualType ElemQT = VecT->getElementType();
1694	PrimType ElemT = classifyPrim(ElemQT);
1695
1696	for (unsigned I = 0; I < NumVecElements; ++I) {
1697	if (!this->visitZeroInitializer(ElemT, ElemQT, E))
1698	return false;
1699	if (!this->emitInitElem(ElemT, I, E))
1700	return false;
1701	}
1702	return true;
1703	}
1704
1705	return false;
1706	}
1707
1708	template <class Emitter>
1709	bool Compiler<Emitter>::VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1710	const Expr *LHS = E->getLHS();
1711	const Expr *RHS = E->getRHS();
1712	const Expr *Index = E->getIdx();
1713	const Expr *Base = E->getBase();
1714
1715	// C++17's rules require us to evaluate the LHS first, regardless of which
1716	// side is the base.
1717	bool Success = true;
1718	for (const Expr *SubExpr : {LHS, RHS}) {
1719	if (!this->visit(SubExpr)) {
1720	Success = false;
1721	continue;
1722	}
1723
1724	// Expand the base if this is a subscript on a
1725	// pointer expression.
1726	if (SubExpr == Base && Base->getType()->isPointerType()) {
1727	if (!this->emitExpandPtr(E))
1728	Success = false;
1729	}
1730	}
1731
1732	if (!Success)
1733	return false;
1734
1735	std::optional<PrimType> IndexT = classify(Index->getType());
1736	// In error-recovery cases, the index expression has a dependent type.
1737	if (!IndexT)
1738	return this->emitError(E);
1739	// If the index is first, we need to change that.
1740	if (LHS == Index) {
1741	if (!this->emitFlip(PT_Ptr, *IndexT, E))
1742	return false;
1743	}
1744
1745	if (!this->emitArrayElemPtrPop(*IndexT, E))
1746	return false;
1747	if (DiscardResult)
1748	return this->emitPopPtr(E);
1749	return true;
1750	}
1751
1752	template <class Emitter>
1753	bool Compiler<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
1754	const Expr *ArrayFiller, const Expr *E) {
1755	InitLinkScope<Emitter> ILS(this, InitLink::InitList());
1756
1757	QualType QT = E->getType();
1758	if (const auto *AT = QT->getAs<AtomicType>())
1759	QT = AT->getValueType();
1760
1761	if (QT->isVoidType()) {
1762	if (Inits.size() == 0)
1763	return true;
1764	return this->emitInvalid(E);
1765	}
1766
1767	// Handle discarding first.
1768	if (DiscardResult) {
1769	for (const Expr *Init : Inits) {
1770	if (!this->discard(Init))
1771	return false;
1772	}
1773	return true;
1774	}
1775
1776	// Primitive values.
1777	if (std::optional<PrimType> T = classify(QT)) {
1778	assert(!DiscardResult);
1779	if (Inits.size() == 0)
1780	return this->visitZeroInitializer(*T, QT, E);
1781	assert(Inits.size() == 1);
1782	return this->delegate(Inits[0]);
1783	}
1784
1785	if (QT->isRecordType()) {
1786	const Record *R = getRecord(QT);
1787
1788	if (Inits.size() == 1 && E->getType() == Inits[0]->getType())
1789	return this->delegate(Inits[0]);
1790
1791	auto initPrimitiveField = [=](const Record::Field *FieldToInit,
1792	const Expr *Init, PrimType T) -> bool {
1793	InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(Val: Init));
1794	InitLinkScope<Emitter> ILS(this, InitLink::Field(Offset: FieldToInit->Offset));
1795	if (!this->visit(Init))
1796	return false;
1797
1798	if (FieldToInit->isBitField())
1799	return this->emitInitBitField(T, FieldToInit, E);
1800	return this->emitInitField(T, FieldToInit->Offset, E);
1801	};
1802
1803	auto initCompositeField = [=](const Record::Field *FieldToInit,
1804	const Expr *Init) -> bool {
1805	InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(Val: Init));
1806	InitLinkScope<Emitter> ILS(this, InitLink::Field(Offset: FieldToInit->Offset));
1807
1808	// Non-primitive case. Get a pointer to the field-to-initialize
1809	// on the stack and recurse into visitInitializer().
1810	if (!this->emitGetPtrField(FieldToInit->Offset, Init))
1811	return false;
1812	if (!this->visitInitializer(Init))
1813	return false;
1814	return this->emitPopPtr(E);
1815	};
1816
1817	if (R->isUnion()) {
1818	if (Inits.size() == 0) {
1819	if (!this->visitZeroRecordInitializer(R, E))
1820	return false;
1821	} else {
1822	const Expr *Init = Inits[0];
1823	const FieldDecl *FToInit = nullptr;
1824	if (const auto *ILE = dyn_cast<InitListExpr>(Val: E))
1825	FToInit = ILE->getInitializedFieldInUnion();
1826	else
1827	FToInit = cast<CXXParenListInitExpr>(Val: E)->getInitializedFieldInUnion();
1828
1829	const Record::Field *FieldToInit = R->getField(FD: FToInit);
1830	if (std::optional<PrimType> T = classify(Init)) {
1831	if (!initPrimitiveField(FieldToInit, Init, *T))
1832	return false;
1833	} else {
1834	if (!initCompositeField(FieldToInit, Init))
1835	return false;
1836	}
1837	}
1838	return this->emitFinishInit(E);
1839	}
1840
1841	assert(!R->isUnion());
1842	unsigned InitIndex = 0;
1843	for (const Expr *Init : Inits) {
1844	// Skip unnamed bitfields.
1845	while (InitIndex < R->getNumFields() &&
1846	R->getField(I: InitIndex)->isUnnamedBitField())
1847	++InitIndex;
1848
1849	if (std::optional<PrimType> T = classify(Init)) {
1850	const Record::Field *FieldToInit = R->getField(I: InitIndex);
1851	if (!initPrimitiveField(FieldToInit, Init, *T))
1852	return false;
1853	++InitIndex;
1854	} else {
1855	// Initializer for a direct base class.
1856	if (const Record::Base *B = R->getBase(T: Init->getType())) {
1857	if (!this->emitGetPtrBase(B->Offset, Init))
1858	return false;
1859
1860	if (!this->visitInitializer(Init))
1861	return false;
1862
1863	if (!this->emitFinishInitPop(E))
1864	return false;
1865	// Base initializers don't increase InitIndex, since they don't count
1866	// into the Record's fields.
1867	} else {
1868	const Record::Field *FieldToInit = R->getField(I: InitIndex);
1869	if (!initCompositeField(FieldToInit, Init))
1870	return false;
1871	++InitIndex;
1872	}
1873	}
1874	}
1875	return this->emitFinishInit(E);
1876	}
1877
1878	if (QT->isArrayType()) {
1879	if (Inits.size() == 1 && QT == Inits[0]->getType())
1880	return this->delegate(Inits[0]);
1881
1882	const ConstantArrayType *CAT =
1883	Ctx.getASTContext().getAsConstantArrayType(T: QT);
1884	uint64_t NumElems = CAT->getZExtSize();
1885
1886	if (!this->emitCheckArraySize(NumElems, E))
1887	return false;
1888
1889	std::optional<PrimType> InitT = classify(CAT->getElementType());
1890	unsigned ElementIndex = 0;
1891	for (const Expr *Init : Inits) {
1892	if (const auto *EmbedS =
1893	dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
1894	PrimType TargetT = classifyPrim(Init->getType());
1895
1896	auto Eval = [&](const Expr *Init, unsigned ElemIndex) {
1897	PrimType InitT = classifyPrim(Init->getType());
1898	if (!this->visit(Init))
1899	return false;
1900	if (InitT != TargetT) {
1901	if (!this->emitCast(InitT, TargetT, E))
1902	return false;
1903	}
1904	return this->emitInitElem(TargetT, ElemIndex, Init);
1905	};
1906	if (!EmbedS->doForEachDataElement(Eval, ElementIndex))
1907	return false;
1908	} else {
1909	if (!this->visitArrayElemInit(ElementIndex, Init, InitT))
1910	return false;
1911	++ElementIndex;
1912	}
1913	}
1914
1915	// Expand the filler expression.
1916	// FIXME: This should go away.
1917	if (ArrayFiller) {
1918	for (; ElementIndex != NumElems; ++ElementIndex) {
1919	if (!this->visitArrayElemInit(ElementIndex, ArrayFiller, InitT))
1920	return false;
1921	}
1922	}
1923
1924	return this->emitFinishInit(E);
1925	}
1926
1927	if (const auto *ComplexTy = QT->getAs<ComplexType>()) {
1928	unsigned NumInits = Inits.size();
1929
1930	if (NumInits == 1)
1931	return this->delegate(Inits[0]);
1932
1933	QualType ElemQT = ComplexTy->getElementType();
1934	PrimType ElemT = classifyPrim(ElemQT);
1935	if (NumInits == 0) {
1936	// Zero-initialize both elements.
1937	for (unsigned I = 0; I < 2; ++I) {
1938	if (!this->visitZeroInitializer(ElemT, ElemQT, E))
1939	return false;
1940	if (!this->emitInitElem(ElemT, I, E))
1941	return false;
1942	}
1943	} else if (NumInits == 2) {
1944	unsigned InitIndex = 0;
1945	for (const Expr *Init : Inits) {
1946	if (!this->visit(Init))
1947	return false;
1948
1949	if (!this->emitInitElem(ElemT, InitIndex, E))
1950	return false;
1951	++InitIndex;
1952	}
1953	}
1954	return true;
1955	}
1956
1957	if (const auto *VecT = QT->getAs<VectorType>()) {
1958	unsigned NumVecElements = VecT->getNumElements();
1959	assert(NumVecElements >= Inits.size());
1960
1961	QualType ElemQT = VecT->getElementType();
1962	PrimType ElemT = classifyPrim(ElemQT);
1963
1964	// All initializer elements.
1965	unsigned InitIndex = 0;
1966	for (const Expr *Init : Inits) {
1967	if (!this->visit(Init))
1968	return false;
1969
1970	// If the initializer is of vector type itself, we have to deconstruct
1971	// that and initialize all the target fields from the initializer fields.
1972	if (const auto *InitVecT = Init->getType()->getAs<VectorType>()) {
1973	if (!this->emitCopyArray(ElemT, 0, InitIndex,
1974	InitVecT->getNumElements(), E))
1975	return false;
1976	InitIndex += InitVecT->getNumElements();
1977	} else {
1978	if (!this->emitInitElem(ElemT, InitIndex, E))
1979	return false;
1980	++InitIndex;
1981	}
1982	}
1983
1984	assert(InitIndex <= NumVecElements);
1985
1986	// Fill the rest with zeroes.
1987	for (; InitIndex != NumVecElements; ++InitIndex) {
1988	if (!this->visitZeroInitializer(ElemT, ElemQT, E))
1989	return false;
1990	if (!this->emitInitElem(ElemT, InitIndex, E))
1991	return false;
1992	}
1993	return true;
1994	}
1995
1996	return false;
1997	}
1998
1999	/// Pointer to the array(not the element!) must be on the stack when calling
2000	/// this.
2001	template <class Emitter>
2002	bool Compiler<Emitter>::visitArrayElemInit(unsigned ElemIndex, const Expr *Init,
2003	std::optional<PrimType> InitT) {
2004	if (InitT) {
2005	// Visit the primitive element like normal.
2006	if (!this->visit(Init))
2007	return false;
2008	return this->emitInitElem(*InitT, ElemIndex, Init);
2009	}
2010
2011	InitLinkScope<Emitter> ILS(this, InitLink::Elem(Index: ElemIndex));
2012	// Advance the pointer currently on the stack to the given
2013	// dimension.
2014	if (!this->emitConstUint32(ElemIndex, Init))
2015	return false;
2016	if (!this->emitArrayElemPtrUint32(Init))
2017	return false;
2018	if (!this->visitInitializer(Init))
2019	return false;
2020	return this->emitFinishInitPop(Init);
2021	}
2022
2023	template <class Emitter>
2024	bool Compiler<Emitter>::visitCallArgs(ArrayRef<const Expr *> Args,
2025	const FunctionDecl *FuncDecl) {
2026	assert(VarScope->getKind() == ScopeKind::Call);
2027	llvm::BitVector NonNullArgs = collectNonNullArgs(F: FuncDecl, Args);
2028
2029	unsigned ArgIndex = 0;
2030	for (const Expr *Arg : Args) {
2031	if (std::optional<PrimType> T = classify(Arg)) {
2032	if (!this->visit(Arg))
2033	return false;
2034	} else {
2035
2036	std::optional<unsigned> LocalIndex = allocateLocal(
2037	Decl: Arg, Ty: Arg->getType(), /*ExtendingDecl=*/nullptr, ScopeKind::Call);
2038	if (!LocalIndex)
2039	return false;
2040
2041	if (!this->emitGetPtrLocal(*LocalIndex, Arg))
2042	return false;
2043	InitLinkScope<Emitter> ILS(this, InitLink::Temp(Offset: *LocalIndex));
2044	if (!this->visitInitializer(Arg))
2045	return false;
2046	}
2047
2048	if (FuncDecl && NonNullArgs[ArgIndex]) {
2049	PrimType ArgT = classify(Arg).value_or(PT_Ptr);
2050	if (ArgT == PT_Ptr) {
2051	if (!this->emitCheckNonNullArg(ArgT, Arg))
2052	return false;
2053	}
2054	}
2055
2056	++ArgIndex;
2057	}
2058
2059	return true;
2060	}
2061
2062	template <class Emitter>
2063	bool Compiler<Emitter>::VisitInitListExpr(const InitListExpr *E) {
2064	return this->visitInitList(E->inits(), E->getArrayFiller(), E);
2065	}
2066
2067	template <class Emitter>
2068	bool Compiler<Emitter>::VisitCXXParenListInitExpr(
2069	const CXXParenListInitExpr *E) {
2070	return this->visitInitList(E->getInitExprs(), E->getArrayFiller(), E);
2071	}
2072
2073	template <class Emitter>
2074	bool Compiler<Emitter>::VisitSubstNonTypeTemplateParmExpr(
2075	const SubstNonTypeTemplateParmExpr *E) {
2076	return this->delegate(E->getReplacement());
2077	}
2078
2079	template <class Emitter>
2080	bool Compiler<Emitter>::VisitConstantExpr(const ConstantExpr *E) {
2081	std::optional<PrimType> T = classify(E->getType());
2082	if (T && E->hasAPValueResult()) {
2083	// Try to emit the APValue directly, without visiting the subexpr.
2084	// This will only fail if we can't emit the APValue, so won't emit any
2085	// diagnostics or any double values.
2086	if (DiscardResult)
2087	return true;
2088
2089	if (this->visitAPValue(E->getAPValueResult(), *T, E))
2090	return true;
2091	}
2092	return this->delegate(E->getSubExpr());
2093	}
2094
2095	template <class Emitter>
2096	bool Compiler<Emitter>::VisitEmbedExpr(const EmbedExpr *E) {
2097	auto It = E->begin();
2098	return this->visit(*It);
2099	}
2100
2101	static CharUnits AlignOfType(QualType T, const ASTContext &ASTCtx,
2102	UnaryExprOrTypeTrait Kind) {
2103	bool AlignOfReturnsPreferred =
2104	ASTCtx.getLangOpts().getClangABICompat() <= LangOptions::ClangABI::Ver7;
2105
2106	// C++ [expr.alignof]p3:
2107	// When alignof is applied to a reference type, the result is the
2108	// alignment of the referenced type.
2109	if (const auto *Ref = T->getAs<ReferenceType>())
2110	T = Ref->getPointeeType();
2111
2112	if (T.getQualifiers().hasUnaligned())
2113	return CharUnits::One();
2114
2115	// __alignof is defined to return the preferred alignment.
2116	// Before 8, clang returned the preferred alignment for alignof and
2117	// _Alignof as well.
2118	if (Kind == UETT_PreferredAlignOf || AlignOfReturnsPreferred)
2119	return ASTCtx.toCharUnitsFromBits(BitSize: ASTCtx.getPreferredTypeAlign(T));
2120
2121	return ASTCtx.getTypeAlignInChars(T);
2122	}
2123
2124	template <class Emitter>
2125	bool Compiler<Emitter>::VisitUnaryExprOrTypeTraitExpr(
2126	const UnaryExprOrTypeTraitExpr *E) {
2127	UnaryExprOrTypeTrait Kind = E->getKind();
2128	const ASTContext &ASTCtx = Ctx.getASTContext();
2129
2130	if (Kind == UETT_SizeOf || Kind == UETT_DataSizeOf) {
2131	QualType ArgType = E->getTypeOfArgument();
2132
2133	// C++ [expr.sizeof]p2: "When applied to a reference or a reference type,
2134	// the result is the size of the referenced type."
2135	if (const auto *Ref = ArgType->getAs<ReferenceType>())
2136	ArgType = Ref->getPointeeType();
2137
2138	CharUnits Size;
2139	if (ArgType->isVoidType() || ArgType->isFunctionType())
2140	Size = CharUnits::One();
2141	else {
2142	if (ArgType->isDependentType() || !ArgType->isConstantSizeType())
2143	return this->emitInvalid(E);
2144
2145	if (Kind == UETT_SizeOf)
2146	Size = ASTCtx.getTypeSizeInChars(T: ArgType);
2147	else
2148	Size = ASTCtx.getTypeInfoDataSizeInChars(T: ArgType).Width;
2149	}
2150
2151	if (DiscardResult)
2152	return true;
2153
2154	return this->emitConst(Size.getQuantity(), E);
2155	}
2156
2157	if (Kind == UETT_CountOf) {
2158	QualType Ty = E->getTypeOfArgument();
2159	assert(Ty->isArrayType());
2160
2161	// We don't need to worry about array element qualifiers, so getting the
2162	// unsafe array type is fine.
2163	if (const auto *CAT =
2164	dyn_cast<ConstantArrayType>(Ty->getAsArrayTypeUnsafe())) {
2165	if (DiscardResult)
2166	return true;
2167	return this->emitConst(CAT->getSize(), E);
2168	}
2169
2170	assert(!Ty->isConstantSizeType());
2171
2172	// If it's a variable-length array type, we need to check whether it is a
2173	// multidimensional array. If so, we need to check the size expression of
2174	// the VLA to see if it's a constant size. If so, we can return that value.
2175	const auto *VAT = ASTCtx.getAsVariableArrayType(T: Ty);
2176	assert(VAT);
2177	if (VAT->getElementType()->isArrayType()) {
2178	std::optional<APSInt> Res =
2179	VAT->getSizeExpr()->getIntegerConstantExpr(Ctx: ASTCtx);
2180	if (Res) {
2181	if (DiscardResult)
2182	return true;
2183	return this->emitConst(*Res, E);
2184	}
2185	}
2186	}
2187
2188	if (Kind == UETT_AlignOf || Kind == UETT_PreferredAlignOf) {
2189	CharUnits Size;
2190
2191	if (E->isArgumentType()) {
2192	QualType ArgType = E->getTypeOfArgument();
2193
2194	Size = AlignOfType(T: ArgType, ASTCtx, Kind);
2195	} else {
2196	// Argument is an expression, not a type.
2197	const Expr *Arg = E->getArgumentExpr()->IgnoreParens();
2198
2199	// The kinds of expressions that we have special-case logic here for
2200	// should be kept up to date with the special checks for those
2201	// expressions in Sema.
2202
2203	// alignof decl is always accepted, even if it doesn't make sense: we
2204	// default to 1 in those cases.
2205	if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Arg))
2206	Size = ASTCtx.getDeclAlign(DRE->getDecl(),
2207	/*RefAsPointee*/ true);
2208	else if (const auto *ME = dyn_cast<MemberExpr>(Val: Arg))
2209	Size = ASTCtx.getDeclAlign(ME->getMemberDecl(),
2210	/*RefAsPointee*/ true);
2211	else
2212	Size = AlignOfType(T: Arg->getType(), ASTCtx, Kind);
2213	}
2214
2215	if (DiscardResult)
2216	return true;
2217
2218	return this->emitConst(Size.getQuantity(), E);
2219	}
2220
2221	if (Kind == UETT_VectorElements) {
2222	if (const auto *VT = E->getTypeOfArgument()->getAs<VectorType>())
2223	return this->emitConst(VT->getNumElements(), E);
2224	assert(E->getTypeOfArgument()->isSizelessVectorType());
2225	return this->emitSizelessVectorElementSize(E);
2226	}
2227
2228	if (Kind == UETT_VecStep) {
2229	if (const auto *VT = E->getTypeOfArgument()->getAs<VectorType>()) {
2230	unsigned N = VT->getNumElements();
2231
2232	// The vec_step built-in functions that take a 3-component
2233	// vector return 4. (OpenCL 1.1 spec 6.11.12)
2234	if (N == 3)
2235	N = 4;
2236
2237	return this->emitConst(N, E);
2238	}
2239	return this->emitConst(1, E);
2240	}
2241
2242	if (Kind == UETT_OpenMPRequiredSimdAlign) {
2243	assert(E->isArgumentType());
2244	unsigned Bits = ASTCtx.getOpenMPDefaultSimdAlign(T: E->getArgumentType());
2245
2246	return this->emitConst(ASTCtx.toCharUnitsFromBits(BitSize: Bits).getQuantity(), E);
2247	}
2248
2249	if (Kind == UETT_PtrAuthTypeDiscriminator) {
2250	if (E->getArgumentType()->isDependentType())
2251	return this->emitInvalid(E);
2252
2253	return this->emitConst(
2254	const_cast<ASTContext &>(ASTCtx).getPointerAuthTypeDiscriminator(
2255	T: E->getArgumentType()),
2256	E);
2257	}
2258
2259	return false;
2260	}
2261
2262	template <class Emitter>
2263	bool Compiler<Emitter>::VisitMemberExpr(const MemberExpr *E) {
2264	// 'Base.Member'
2265	const Expr *Base = E->getBase();
2266	const ValueDecl *Member = E->getMemberDecl();
2267
2268	if (DiscardResult)
2269	return this->discard(Base);
2270
2271	// MemberExprs are almost always lvalues, in which case we don't need to
2272	// do the load. But sometimes they aren't.
2273	const auto maybeLoadValue = [&]() -> bool {
2274	if (E->isGLValue())
2275	return true;
2276	if (std::optional<PrimType> T = classify(E))
2277	return this->emitLoadPop(*T, E);
2278	return false;
2279	};
2280
2281	if (const auto *VD = dyn_cast<VarDecl>(Val: Member)) {
2282	// I am almost confident in saying that a var decl must be static
2283	// and therefore registered as a global variable. But this will probably
2284	// turn out to be wrong some time in the future, as always.
2285	if (auto GlobalIndex = P.getGlobal(VD))
2286	return this->emitGetPtrGlobal(*GlobalIndex, E) && maybeLoadValue();
2287	return false;
2288	}
2289
2290	if (!isa<FieldDecl>(Val: Member)) {
2291	if (!this->discard(Base) && !this->emitSideEffect(E))
2292	return false;
2293
2294	return this->visitDeclRef(Member, E);
2295	}
2296
2297	if (Initializing) {
2298	if (!this->delegate(Base))
2299	return false;
2300	} else {
2301	if (!this->visit(Base))
2302	return false;
2303	}
2304
2305	// Base above gives us a pointer on the stack.
2306	const auto *FD = cast<FieldDecl>(Val: Member);
2307	const RecordDecl *RD = FD->getParent();
2308	const Record *R = getRecord(RD);
2309	if (!R)
2310	return false;
2311	const Record::Field *F = R->getField(FD);
2312	// Leave a pointer to the field on the stack.
2313	if (F->Decl->getType()->isReferenceType())
2314	return this->emitGetFieldPop(PT_Ptr, F->Offset, E) && maybeLoadValue();
2315	return this->emitGetPtrFieldPop(F->Offset, E) && maybeLoadValue();
2316	}
2317
2318	template <class Emitter>
2319	bool Compiler<Emitter>::VisitArrayInitIndexExpr(const ArrayInitIndexExpr *E) {
2320	// ArrayIndex might not be set if a ArrayInitIndexExpr is being evaluated
2321	// stand-alone, e.g. via EvaluateAsInt().
2322	if (!ArrayIndex)
2323	return false;
2324	return this->emitConst(*ArrayIndex, E);
2325	}
2326
2327	template <class Emitter>
2328	bool Compiler<Emitter>::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E) {
2329	assert(Initializing);
2330	assert(!DiscardResult);
2331
2332	// We visit the common opaque expression here once so we have its value
2333	// cached.
2334	if (!this->discard(E->getCommonExpr()))
2335	return false;
2336
2337	// TODO: This compiles to quite a lot of bytecode if the array is larger.
2338	// Investigate compiling this to a loop.
2339	const Expr *SubExpr = E->getSubExpr();
2340	size_t Size = E->getArraySize().getZExtValue();
2341	std::optional<PrimType> SubExprT = classify(SubExpr);
2342
2343	// So, every iteration, we execute an assignment here
2344	// where the LHS is on the stack (the target array)
2345	// and the RHS is our SubExpr.
2346	for (size_t I = 0; I != Size; ++I) {
2347	ArrayIndexScope<Emitter> IndexScope(this, I);
2348	BlockScope<Emitter> BS(this);
2349
2350	if (!this->visitArrayElemInit(I, SubExpr, SubExprT))
2351	return false;
2352	if (!BS.destroyLocals())
2353	return false;
2354	}
2355	return true;
2356	}
2357
2358	template <class Emitter>
2359	bool Compiler<Emitter>::VisitOpaqueValueExpr(const OpaqueValueExpr *E) {
2360	const Expr *SourceExpr = E->getSourceExpr();
2361	if (!SourceExpr)
2362	return false;
2363
2364	if (Initializing)
2365	return this->visitInitializer(SourceExpr);
2366
2367	PrimType SubExprT = classify(SourceExpr).value_or(PT_Ptr);
2368	if (auto It = OpaqueExprs.find(Val: E); It != OpaqueExprs.end())
2369	return this->emitGetLocal(SubExprT, It->second, E);
2370
2371	if (!this->visit(SourceExpr))
2372	return false;
2373
2374	// At this point we either have the evaluated source expression or a pointer
2375	// to an object on the stack. We want to create a local variable that stores
2376	// this value.
2377	unsigned LocalIndex = allocateLocalPrimitive(Decl: E, Ty: SubExprT, /*IsConst=*/true);
2378	if (!this->emitSetLocal(SubExprT, LocalIndex, E))
2379	return false;
2380
2381	// Here the local variable is created but the value is removed from the stack,
2382	// so we put it back if the caller needs it.
2383	if (!DiscardResult) {
2384	if (!this->emitGetLocal(SubExprT, LocalIndex, E))
2385	return false;
2386	}
2387
2388	// This is cleaned up when the local variable is destroyed.
2389	OpaqueExprs.insert(KV: {E, LocalIndex});
2390
2391	return true;
2392	}
2393
2394	template <class Emitter>
2395	bool Compiler<Emitter>::VisitAbstractConditionalOperator(
2396	const AbstractConditionalOperator *E) {
2397	const Expr *Condition = E->getCond();
2398	const Expr *TrueExpr = E->getTrueExpr();
2399	const Expr *FalseExpr = E->getFalseExpr();
2400
2401	auto visitChildExpr = [&](const Expr *E) -> bool {
2402	LocalScope<Emitter> S(this);
2403	if (!this->delegate(E))
2404	return false;
2405	return S.destroyLocals();
2406	};
2407
2408	if (std::optional<bool> BoolValue = getBoolValue(E: Condition)) {
2409	if (BoolValue)
2410	return visitChildExpr(TrueExpr);
2411	return visitChildExpr(FalseExpr);
2412	}
2413
2414	bool IsBcpCall = false;
2415	if (const auto *CE = dyn_cast<CallExpr>(Val: Condition->IgnoreParenCasts());
2416	CE && CE->getBuiltinCallee() == Builtin::BI__builtin_constant_p) {
2417	IsBcpCall = true;
2418	}
2419
2420	LabelTy LabelEnd = this->getLabel(); // Label after the operator.
2421	LabelTy LabelFalse = this->getLabel(); // Label for the false expr.
2422
2423	if (IsBcpCall) {
2424	if (!this->emitStartSpeculation(E))
2425	return false;
2426	}
2427
2428	if (!this->visitBool(Condition)) {
2429	// If the condition failed and we're checking for undefined behavior
2430	// (which only happens with EvalEmitter) check the TrueExpr and FalseExpr
2431	// as well.
2432	if (this->checkingForUndefinedBehavior()) {
2433	if (!this->discard(TrueExpr))
2434	return false;
2435	if (!this->discard(FalseExpr))
2436	return false;
2437	}
2438	return false;
2439	}
2440
2441	if (!this->jumpFalse(LabelFalse))
2442	return false;
2443	if (!visitChildExpr(TrueExpr))
2444	return false;
2445	if (!this->jump(LabelEnd))
2446	return false;
2447	this->emitLabel(LabelFalse);
2448	if (!visitChildExpr(FalseExpr))
2449	return false;
2450	this->fallthrough(LabelEnd);
2451	this->emitLabel(LabelEnd);
2452
2453	if (IsBcpCall)
2454	return this->emitEndSpeculation(E);
2455	return true;
2456	}
2457
2458	template <class Emitter>
2459	bool Compiler<Emitter>::VisitStringLiteral(const StringLiteral *E) {
2460	if (DiscardResult)
2461	return true;
2462
2463	if (!Initializing) {
2464	unsigned StringIndex = P.createGlobalString(S: E);
2465	return this->emitGetPtrGlobal(StringIndex, E);
2466	}
2467
2468	// We are initializing an array on the stack.
2469	const ConstantArrayType *CAT =
2470	Ctx.getASTContext().getAsConstantArrayType(T: E->getType());
2471	assert(CAT && "a string literal that's not a constant array?");
2472
2473	// If the initializer string is too long, a diagnostic has already been
2474	// emitted. Read only the array length from the string literal.
2475	unsigned ArraySize = CAT->getZExtSize();
2476	unsigned N = std::min(a: ArraySize, b: E->getLength());
2477	unsigned CharWidth = E->getCharByteWidth();
2478
2479	for (unsigned I = 0; I != N; ++I) {
2480	uint32_t CodeUnit = E->getCodeUnit(i: I);
2481
2482	if (CharWidth == 1) {
2483	this->emitConstSint8(CodeUnit, E);
2484	this->emitInitElemSint8(I, E);
2485	} else if (CharWidth == 2) {
2486	this->emitConstUint16(CodeUnit, E);
2487	this->emitInitElemUint16(I, E);
2488	} else if (CharWidth == 4) {
2489	this->emitConstUint32(CodeUnit, E);
2490	this->emitInitElemUint32(I, E);
2491	} else {
2492	llvm_unreachable("unsupported character width");
2493	}
2494	}
2495
2496	// Fill up the rest of the char array with NUL bytes.
2497	for (unsigned I = N; I != ArraySize; ++I) {
2498	if (CharWidth == 1) {
2499	this->emitConstSint8(0, E);
2500	this->emitInitElemSint8(I, E);
2501	} else if (CharWidth == 2) {
2502	this->emitConstUint16(0, E);
2503	this->emitInitElemUint16(I, E);
2504	} else if (CharWidth == 4) {
2505	this->emitConstUint32(0, E);
2506	this->emitInitElemUint32(I, E);
2507	} else {
2508	llvm_unreachable("unsupported character width");
2509	}
2510	}
2511
2512	return true;
2513	}
2514
2515	template <class Emitter>
2516	bool Compiler<Emitter>::VisitObjCStringLiteral(const ObjCStringLiteral *E) {
2517	if (DiscardResult)
2518	return true;
2519	return this->emitDummyPtr(E, E);
2520	}
2521
2522	template <class Emitter>
2523	bool Compiler<Emitter>::VisitObjCEncodeExpr(const ObjCEncodeExpr *E) {
2524	auto &A = Ctx.getASTContext();
2525	std::string Str;
2526	A.getObjCEncodingForType(T: E->getEncodedType(), S&: Str);
2527	StringLiteral *SL =
2528	StringLiteral::Create(A, Str, StringLiteralKind::Ordinary,
2529	/*Pascal=*/false, E->getType(), E->getAtLoc());
2530	return this->delegate(SL);
2531	}
2532
2533	template <class Emitter>
2534	bool Compiler<Emitter>::VisitSYCLUniqueStableNameExpr(
2535	const SYCLUniqueStableNameExpr *E) {
2536	if (DiscardResult)
2537	return true;
2538
2539	assert(!Initializing);
2540
2541	auto &A = Ctx.getASTContext();
2542	std::string ResultStr = E->ComputeName(Context&: A);
2543
2544	QualType CharTy = A.CharTy.withConst();
2545	APInt Size(A.getTypeSize(T: A.getSizeType()), ResultStr.size() + 1);
2546	QualType ArrayTy = A.getConstantArrayType(EltTy: CharTy, ArySize: Size, SizeExpr: nullptr,
2547	ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
2548
2549	StringLiteral *SL =
2550	StringLiteral::Create(Ctx: A, Str: ResultStr, Kind: StringLiteralKind::Ordinary,
2551	/*Pascal=*/false, Ty: ArrayTy, Loc: E->getLocation());
2552
2553	unsigned StringIndex = P.createGlobalString(S: SL);
2554	return this->emitGetPtrGlobal(StringIndex, E);
2555	}
2556
2557	template <class Emitter>
2558	bool Compiler<Emitter>::VisitCharacterLiteral(const CharacterLiteral *E) {
2559	if (DiscardResult)
2560	return true;
2561	return this->emitConst(E->getValue(), E);
2562	}
2563
2564	template <class Emitter>
2565	bool Compiler<Emitter>::VisitFloatCompoundAssignOperator(
2566	const CompoundAssignOperator *E) {
2567
2568	const Expr *LHS = E->getLHS();
2569	const Expr *RHS = E->getRHS();
2570	QualType LHSType = LHS->getType();
2571	QualType LHSComputationType = E->getComputationLHSType();
2572	QualType ResultType = E->getComputationResultType();
2573	std::optional<PrimType> LT = classify(LHSComputationType);
2574	std::optional<PrimType> RT = classify(ResultType);
2575
2576	assert(ResultType->isFloatingType());
2577
2578	if (!LT || !RT)
2579	return false;
2580
2581	PrimType LHST = classifyPrim(LHSType);
2582
2583	// C++17 onwards require that we evaluate the RHS first.
2584	// Compute RHS and save it in a temporary variable so we can
2585	// load it again later.
2586	if (!visit(E: RHS))
2587	return false;
2588
2589	unsigned TempOffset = this->allocateLocalPrimitive(E, *RT, /*IsConst=*/true);
2590	if (!this->emitSetLocal(*RT, TempOffset, E))
2591	return false;
2592
2593	// First, visit LHS.
2594	if (!visit(E: LHS))
2595	return false;
2596	if (!this->emitLoad(LHST, E))
2597	return false;
2598
2599	// If necessary, convert LHS to its computation type.
2600	if (!this->emitPrimCast(LHST, classifyPrim(LHSComputationType),
2601	LHSComputationType, E))
2602	return false;
2603
2604	// Now load RHS.
2605	if (!this->emitGetLocal(*RT, TempOffset, E))
2606	return false;
2607
2608	switch (E->getOpcode()) {
2609	case BO_AddAssign:
2610	if (!this->emitAddf(getFPOptions(E), E))
2611	return false;
2612	break;
2613	case BO_SubAssign:
2614	if (!this->emitSubf(getFPOptions(E), E))
2615	return false;
2616	break;
2617	case BO_MulAssign:
2618	if (!this->emitMulf(getFPOptions(E), E))
2619	return false;
2620	break;
2621	case BO_DivAssign:
2622	if (!this->emitDivf(getFPOptions(E), E))
2623	return false;
2624	break;
2625	default:
2626	return false;
2627	}
2628
2629	if (!this->emitPrimCast(classifyPrim(ResultType), LHST, LHS->getType(), E))
2630	return false;
2631
2632	if (DiscardResult)
2633	return this->emitStorePop(LHST, E);
2634	return this->emitStore(LHST, E);
2635	}
2636
2637	template <class Emitter>
2638	bool Compiler<Emitter>::VisitPointerCompoundAssignOperator(
2639	const CompoundAssignOperator *E) {
2640	BinaryOperatorKind Op = E->getOpcode();
2641	const Expr *LHS = E->getLHS();
2642	const Expr *RHS = E->getRHS();
2643	std::optional<PrimType> LT = classify(LHS->getType());
2644	std::optional<PrimType> RT = classify(RHS->getType());
2645
2646	if (Op != BO_AddAssign && Op != BO_SubAssign)
2647	return false;
2648
2649	if (!LT || !RT)
2650	return false;
2651
2652	if (!visit(E: LHS))
2653	return false;
2654
2655	if (!this->emitLoad(*LT, LHS))
2656	return false;
2657
2658	if (!visit(E: RHS))
2659	return false;
2660
2661	if (Op == BO_AddAssign) {
2662	if (!this->emitAddOffset(*RT, E))
2663	return false;
2664	} else {
2665	if (!this->emitSubOffset(*RT, E))
2666	return false;
2667	}
2668
2669	if (DiscardResult)
2670	return this->emitStorePopPtr(E);
2671	return this->emitStorePtr(E);
2672	}
2673
2674	template <class Emitter>
2675	bool Compiler<Emitter>::VisitCompoundAssignOperator(
2676	const CompoundAssignOperator *E) {
2677	if (E->getType()->isVectorType())
2678	return VisitVectorBinOp(E);
2679
2680	const Expr *LHS = E->getLHS();
2681	const Expr *RHS = E->getRHS();
2682	std::optional<PrimType> LHSComputationT =
2683	classify(E->getComputationLHSType());
2684	std::optional<PrimType> LT = classify(LHS->getType());
2685	std::optional<PrimType> RT = classify(RHS->getType());
2686	std::optional<PrimType> ResultT = classify(E->getType());
2687
2688	if (!Ctx.getLangOpts().CPlusPlus14)
2689	return this->visit(RHS) && this->visit(LHS) && this->emitError(E);
2690
2691	if (!LT || !RT || !ResultT || !LHSComputationT)
2692	return false;
2693
2694	// Handle floating point operations separately here, since they
2695	// require special care.
2696
2697	if (ResultT == PT_Float || RT == PT_Float)
2698	return VisitFloatCompoundAssignOperator(E);
2699
2700	if (E->getType()->isPointerType())
2701	return VisitPointerCompoundAssignOperator(E);
2702
2703	assert(!E->getType()->isPointerType() && "Handled above");
2704	assert(!E->getType()->isFloatingType() && "Handled above");
2705
2706	// C++17 onwards require that we evaluate the RHS first.
2707	// Compute RHS and save it in a temporary variable so we can
2708	// load it again later.
2709	// FIXME: Compound assignments are unsequenced in C, so we might
2710	// have to figure out how to reject them.
2711	if (!visit(E: RHS))
2712	return false;
2713
2714	unsigned TempOffset = this->allocateLocalPrimitive(E, *RT, /*IsConst=*/true);
2715
2716	if (!this->emitSetLocal(*RT, TempOffset, E))
2717	return false;
2718
2719	// Get LHS pointer, load its value and cast it to the
2720	// computation type if necessary.
2721	if (!visit(E: LHS))
2722	return false;
2723	if (!this->emitLoad(*LT, E))
2724	return false;
2725	if (LT != LHSComputationT) {
2726	if (!this->emitCast(*LT, *LHSComputationT, E))
2727	return false;
2728	}
2729
2730	// Get the RHS value on the stack.
2731	if (!this->emitGetLocal(*RT, TempOffset, E))
2732	return false;
2733
2734	// Perform operation.
2735	switch (E->getOpcode()) {
2736	case BO_AddAssign:
2737	if (!this->emitAdd(*LHSComputationT, E))
2738	return false;
2739	break;
2740	case BO_SubAssign:
2741	if (!this->emitSub(*LHSComputationT, E))
2742	return false;
2743	break;
2744	case BO_MulAssign:
2745	if (!this->emitMul(*LHSComputationT, E))
2746	return false;
2747	break;
2748	case BO_DivAssign:
2749	if (!this->emitDiv(*LHSComputationT, E))
2750	return false;
2751	break;
2752	case BO_RemAssign:
2753	if (!this->emitRem(*LHSComputationT, E))
2754	return false;
2755	break;
2756	case BO_ShlAssign:
2757	if (!this->emitShl(*LHSComputationT, *RT, E))
2758	return false;
2759	break;
2760	case BO_ShrAssign:
2761	if (!this->emitShr(*LHSComputationT, *RT, E))
2762	return false;
2763	break;
2764	case BO_AndAssign:
2765	if (!this->emitBitAnd(*LHSComputationT, E))
2766	return false;
2767	break;
2768	case BO_XorAssign:
2769	if (!this->emitBitXor(*LHSComputationT, E))
2770	return false;
2771	break;
2772	case BO_OrAssign:
2773	if (!this->emitBitOr(*LHSComputationT, E))
2774	return false;
2775	break;
2776	default:
2777	llvm_unreachable("Unimplemented compound assign operator");
2778	}
2779
2780	// And now cast from LHSComputationT to ResultT.
2781	if (ResultT != LHSComputationT) {
2782	if (!this->emitCast(*LHSComputationT, *ResultT, E))
2783	return false;
2784	}
2785
2786	// And store the result in LHS.
2787	if (DiscardResult) {
2788	if (LHS->refersToBitField())
2789	return this->emitStoreBitFieldPop(*ResultT, E);
2790	return this->emitStorePop(*ResultT, E);
2791	}
2792	if (LHS->refersToBitField())
2793	return this->emitStoreBitField(*ResultT, E);
2794	return this->emitStore(*ResultT, E);
2795	}
2796
2797	template <class Emitter>
2798	bool Compiler<Emitter>::VisitExprWithCleanups(const ExprWithCleanups *E) {
2799	LocalScope<Emitter> ES(this);
2800	const Expr *SubExpr = E->getSubExpr();
2801
2802	return this->delegate(SubExpr) && ES.destroyLocals(E);
2803	}
2804
2805	template <class Emitter>
2806	bool Compiler<Emitter>::VisitMaterializeTemporaryExpr(
2807	const MaterializeTemporaryExpr *E) {
2808	const Expr *SubExpr = E->getSubExpr();
2809
2810	if (Initializing) {
2811	// We already have a value, just initialize that.
2812	return this->delegate(SubExpr);
2813	}
2814	// If we don't end up using the materialized temporary anyway, don't
2815	// bother creating it.
2816	if (DiscardResult)
2817	return this->discard(SubExpr);
2818
2819	// When we're initializing a global variable *or* the storage duration of
2820	// the temporary is explicitly static, create a global variable.
2821	std::optional<PrimType> SubExprT = classify(SubExpr);
2822	bool IsStatic = E->getStorageDuration() == SD_Static;
2823	if (IsStatic) {
2824	std::optional<unsigned> GlobalIndex = P.createGlobal(E);
2825	if (!GlobalIndex)
2826	return false;
2827
2828	const LifetimeExtendedTemporaryDecl *TempDecl =
2829	E->getLifetimeExtendedTemporaryDecl();
2830	if (IsStatic)
2831	assert(TempDecl);
2832
2833	if (SubExprT) {
2834	if (!this->visit(SubExpr))
2835	return false;
2836	if (IsStatic) {
2837	if (!this->emitInitGlobalTemp(*SubExprT, *GlobalIndex, TempDecl, E))
2838	return false;
2839	} else {
2840	if (!this->emitInitGlobal(*SubExprT, *GlobalIndex, E))
2841	return false;
2842	}
2843	return this->emitGetPtrGlobal(*GlobalIndex, E);
2844	}
2845
2846	if (!this->checkLiteralType(SubExpr))
2847	return false;
2848	// Non-primitive values.
2849	if (!this->emitGetPtrGlobal(*GlobalIndex, E))
2850	return false;
2851	if (!this->visitInitializer(SubExpr))
2852	return false;
2853	if (IsStatic)
2854	return this->emitInitGlobalTempComp(TempDecl, E);
2855	return true;
2856	}
2857
2858	// For everyhing else, use local variables.
2859	if (SubExprT) {
2860	bool IsConst = SubExpr->getType().isConstQualified();
2861	unsigned LocalIndex =
2862	allocateLocalPrimitive(Decl: E, Ty: *SubExprT, IsConst, ExtendingDecl: E->getExtendingDecl());
2863	if (!this->visit(SubExpr))
2864	return false;
2865	if (!this->emitSetLocal(*SubExprT, LocalIndex, E))
2866	return false;
2867	return this->emitGetPtrLocal(LocalIndex, E);
2868	} else {
2869
2870	if (!this->checkLiteralType(SubExpr))
2871	return false;
2872
2873	const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments();
2874	if (std::optional<unsigned> LocalIndex =
2875	allocateLocal(Decl: E, Ty: Inner->getType(), ExtendingDecl: E->getExtendingDecl())) {
2876	InitLinkScope<Emitter> ILS(this, InitLink::Temp(Offset: *LocalIndex));
2877	if (!this->emitGetPtrLocal(*LocalIndex, E))
2878	return false;
2879	return this->visitInitializer(SubExpr) && this->emitFinishInit(E);
2880	}
2881	}
2882	return false;
2883	}
2884
2885	template <class Emitter>
2886	bool Compiler<Emitter>::VisitCXXBindTemporaryExpr(
2887	const CXXBindTemporaryExpr *E) {
2888	return this->delegate(E->getSubExpr());
2889	}
2890
2891	template <class Emitter>
2892	bool Compiler<Emitter>::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
2893	const Expr *Init = E->getInitializer();
2894	if (DiscardResult)
2895	return this->discard(Init);
2896
2897	if (Initializing) {
2898	// We already have a value, just initialize that.
2899	return this->visitInitializer(Init) && this->emitFinishInit(E);
2900	}
2901
2902	std::optional<PrimType> T = classify(E->getType());
2903	if (E->isFileScope()) {
2904	// Avoid creating a variable if this is a primitive RValue anyway.
2905	if (T && !E->isLValue())
2906	return this->delegate(Init);
2907
2908	if (std::optional<unsigned> GlobalIndex = P.createGlobal(E)) {
2909	if (!this->emitGetPtrGlobal(*GlobalIndex, E))
2910	return false;
2911
2912	if (T) {
2913	if (!this->visit(Init))
2914	return false;
2915	return this->emitInitGlobal(*T, *GlobalIndex, E);
2916	}
2917
2918	return this->visitInitializer(Init) && this->emitFinishInit(E);
2919	}
2920
2921	return false;
2922	}
2923
2924	// Otherwise, use a local variable.
2925	if (T && !E->isLValue()) {
2926	// For primitive types, we just visit the initializer.
2927	return this->delegate(Init);
2928	}
2929
2930	unsigned LocalIndex;
2931	if (T)
2932	LocalIndex = this->allocateLocalPrimitive(Init, *T, /*IsConst=*/false);
2933	else if (std::optional<unsigned> MaybeIndex = this->allocateLocal(Init))
2934	LocalIndex = *MaybeIndex;
2935	else
2936	return false;
2937
2938	if (!this->emitGetPtrLocal(LocalIndex, E))
2939	return false;
2940
2941	if (T)
2942	return this->visit(Init) && this->emitInit(*T, E);
2943	return this->visitInitializer(Init) && this->emitFinishInit(E);
2944	}
2945
2946	template <class Emitter>
2947	bool Compiler<Emitter>::VisitTypeTraitExpr(const TypeTraitExpr *E) {
2948	if (DiscardResult)
2949	return true;
2950	if (E->isStoredAsBoolean()) {
2951	if (E->getType()->isBooleanType())
2952	return this->emitConstBool(E->getBoolValue(), E);
2953	return this->emitConst(E->getBoolValue(), E);
2954	}
2955	PrimType T = classifyPrim(E->getType());
2956	return this->visitAPValue(E->getAPValue(), T, E);
2957	}
2958
2959	template <class Emitter>
2960	bool Compiler<Emitter>::VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *E) {
2961	if (DiscardResult)
2962	return true;
2963	return this->emitConst(E->getValue(), E);
2964	}
2965
2966	template <class Emitter>
2967	bool Compiler<Emitter>::VisitLambdaExpr(const LambdaExpr *E) {
2968	if (DiscardResult)
2969	return true;
2970
2971	assert(Initializing);
2972	const Record *R = P.getOrCreateRecord(E->getLambdaClass());
2973	if (!R)
2974	return false;
2975
2976	auto *CaptureInitIt = E->capture_init_begin();
2977	// Initialize all fields (which represent lambda captures) of the
2978	// record with their initializers.
2979	for (const Record::Field &F : R->fields()) {
2980	const Expr *Init = *CaptureInitIt;
2981	if (!Init || Init->containsErrors())
2982	continue;
2983	++CaptureInitIt;
2984
2985	if (std::optional<PrimType> T = classify(Init)) {
2986	if (!this->visit(Init))
2987	return false;
2988
2989	if (!this->emitInitField(*T, F.Offset, E))
2990	return false;
2991	} else {
2992	if (!this->emitGetPtrField(F.Offset, E))
2993	return false;
2994
2995	if (!this->visitInitializer(Init))
2996	return false;
2997
2998	if (!this->emitPopPtr(E))
2999	return false;
3000	}
3001	}
3002
3003	return true;
3004	}
3005
3006	template <class Emitter>
3007	bool Compiler<Emitter>::VisitPredefinedExpr(const PredefinedExpr *E) {
3008	if (DiscardResult)
3009	return true;
3010
3011	if (!Initializing) {
3012	unsigned StringIndex = P.createGlobalString(E->getFunctionName(), E);
3013	return this->emitGetPtrGlobal(StringIndex, E);
3014	}
3015
3016	return this->delegate(E->getFunctionName());
3017	}
3018
3019	template <class Emitter>
3020	bool Compiler<Emitter>::VisitCXXThrowExpr(const CXXThrowExpr *E) {
3021	if (E->getSubExpr() && !this->discard(E->getSubExpr()))
3022	return false;
3023
3024	return this->emitInvalid(E);
3025	}
3026
3027	template <class Emitter>
3028	bool Compiler<Emitter>::VisitCXXReinterpretCastExpr(
3029	const CXXReinterpretCastExpr *E) {
3030	const Expr *SubExpr = E->getSubExpr();
3031
3032	std::optional<PrimType> FromT = classify(SubExpr);
3033	std::optional<PrimType> ToT = classify(E);
3034
3035	if (!FromT || !ToT)
3036	return this->emitInvalidCast(CastKind::Reinterpret, /*Fatal=*/true, E);
3037
3038	if (FromT == PT_Ptr || ToT == PT_Ptr) {
3039	// Both types could be PT_Ptr because their expressions are glvalues.
3040	std::optional<PrimType> PointeeFromT;
3041	if (SubExpr->getType()->isPointerOrReferenceType())
3042	PointeeFromT = classify(SubExpr->getType()->getPointeeType());
3043	else
3044	PointeeFromT = classify(SubExpr->getType());
3045
3046	std::optional<PrimType> PointeeToT;
3047	if (E->getType()->isPointerOrReferenceType())
3048	PointeeToT = classify(E->getType()->getPointeeType());
3049	else
3050	PointeeToT = classify(E->getType());
3051
3052	bool Fatal = true;
3053	if (PointeeToT && PointeeFromT) {
3054	if (isIntegralType(T: *PointeeFromT) && isIntegralType(T: *PointeeToT))
3055	Fatal = false;
3056	} else {
3057	Fatal = SubExpr->getType().getTypePtr() != E->getType().getTypePtr();
3058	}
3059
3060	if (!this->emitInvalidCast(CastKind::Reinterpret, Fatal, E))
3061	return false;
3062
3063	if (E->getCastKind() == CK_LValueBitCast)
3064	return this->delegate(SubExpr);
3065	return this->VisitCastExpr(E);
3066	}
3067
3068	// Try to actually do the cast.
3069	bool Fatal = (ToT != FromT);
3070	if (!this->emitInvalidCast(CastKind::Reinterpret, Fatal, E))
3071	return false;
3072
3073	return this->VisitCastExpr(E);
3074	}
3075
3076	template <class Emitter>
3077	bool Compiler<Emitter>::VisitCXXDynamicCastExpr(const CXXDynamicCastExpr *E) {
3078
3079	if (!Ctx.getLangOpts().CPlusPlus20) {
3080	if (!this->emitInvalidCast(CastKind::Dynamic, /*Fatal=*/false, E))
3081	return false;
3082	}
3083
3084	return this->VisitCastExpr(E);
3085	}
3086
3087	template <class Emitter>
3088	bool Compiler<Emitter>::VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
3089	assert(E->getType()->isBooleanType());
3090
3091	if (DiscardResult)
3092	return true;
3093	return this->emitConstBool(E->getValue(), E);
3094	}
3095
3096	template <class Emitter>
3097	bool Compiler<Emitter>::VisitCXXConstructExpr(const CXXConstructExpr *E) {
3098	QualType T = E->getType();
3099	assert(!classify(T));
3100
3101	if (T->isRecordType()) {
3102	const CXXConstructorDecl *Ctor = E->getConstructor();
3103
3104	// Trivial copy/move constructor. Avoid copy.
3105	if (Ctor->isDefaulted() && Ctor->isCopyOrMoveConstructor() &&
3106	Ctor->isTrivial() &&
3107	E->getArg(Arg: 0)->isTemporaryObject(Ctx&: Ctx.getASTContext(),
3108	TempTy: T->getAsCXXRecordDecl()))
3109	return this->visitInitializer(E->getArg(Arg: 0));
3110
3111	// If we're discarding a construct expression, we still need
3112	// to allocate a variable and call the constructor and destructor.
3113	if (DiscardResult) {
3114	if (Ctor->isTrivial())
3115	return true;
3116	assert(!Initializing);
3117	std::optional<unsigned> LocalIndex = allocateLocal(Decl: E);
3118
3119	if (!LocalIndex)
3120	return false;
3121
3122	if (!this->emitGetPtrLocal(*LocalIndex, E))
3123	return false;
3124	}
3125
3126	// Zero initialization.
3127	if (E->requiresZeroInitialization()) {
3128	const Record *R = getRecord(E->getType());
3129
3130	if (!this->visitZeroRecordInitializer(R, E))
3131	return false;
3132
3133	// If the constructor is trivial anyway, we're done.
3134	if (Ctor->isTrivial())
3135	return true;
3136	}
3137
3138	const Function *Func = getFunction(FD: Ctor);
3139
3140	if (!Func)
3141	return false;
3142
3143	assert(Func->hasThisPointer());
3144	assert(!Func->hasRVO());
3145
3146	// The This pointer is already on the stack because this is an initializer,
3147	// but we need to dup() so the call() below has its own copy.
3148	if (!this->emitDupPtr(E))
3149	return false;
3150
3151	// Constructor arguments.
3152	for (const auto *Arg : E->arguments()) {
3153	if (!this->visit(Arg))
3154	return false;
3155	}
3156
3157	if (Func->isVariadic()) {
3158	uint32_t VarArgSize = 0;
3159	unsigned NumParams = Func->getNumWrittenParams();
3160	for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I) {
3161	VarArgSize +=
3162	align(primSize(classify(E->getArg(Arg: I)->getType()).value_or(PT_Ptr)));
3163	}
3164	if (!this->emitCallVar(Func, VarArgSize, E))
3165	return false;
3166	} else {
3167	if (!this->emitCall(Func, 0, E)) {
3168	// When discarding, we don't need the result anyway, so clean up
3169	// the instance dup we did earlier in case surrounding code wants
3170	// to keep evaluating.
3171	if (DiscardResult)
3172	(void)this->emitPopPtr(E);
3173	return false;
3174	}
3175	}
3176
3177	if (DiscardResult)
3178	return this->emitPopPtr(E);
3179	return this->emitFinishInit(E);
3180	}
3181
3182	if (T->isArrayType()) {
3183	const ConstantArrayType *CAT =
3184	Ctx.getASTContext().getAsConstantArrayType(T: E->getType());
3185	if (!CAT)
3186	return false;
3187
3188	size_t NumElems = CAT->getZExtSize();
3189	const Function *Func = getFunction(FD: E->getConstructor());
3190	if (!Func)
3191	return false;
3192
3193	// FIXME(perf): We're calling the constructor once per array element here,
3194	// in the old intepreter we had a special-case for trivial constructors.
3195	for (size_t I = 0; I != NumElems; ++I) {
3196	if (!this->emitConstUint64(I, E))
3197	return false;
3198	if (!this->emitArrayElemPtrUint64(E))
3199	return false;
3200
3201	// Constructor arguments.
3202	for (const auto *Arg : E->arguments()) {
3203	if (!this->visit(Arg))
3204	return false;
3205	}
3206
3207	if (!this->emitCall(Func, 0, E))
3208	return false;
3209	}
3210	return true;
3211	}
3212
3213	return false;
3214	}
3215
3216	template <class Emitter>
3217	bool Compiler<Emitter>::VisitSourceLocExpr(const SourceLocExpr *E) {
3218	if (DiscardResult)
3219	return true;
3220
3221	const APValue Val =
3222	E->EvaluateInContext(Ctx: Ctx.getASTContext(), DefaultExpr: SourceLocDefaultExpr);
3223
3224	// Things like __builtin_LINE().
3225	if (E->getType()->isIntegerType()) {
3226	assert(Val.isInt());
3227	const APSInt &I = Val.getInt();
3228	return this->emitConst(I, E);
3229	}
3230	// Otherwise, the APValue is an LValue, with only one element.
3231	// Theoretically, we don't need the APValue at all of course.
3232	assert(E->getType()->isPointerType());
3233	assert(Val.isLValue());
3234	const APValue::LValueBase &Base = Val.getLValueBase();
3235	if (const Expr *LValueExpr = Base.dyn_cast<const Expr *>())
3236	return this->visit(LValueExpr);
3237
3238	// Otherwise, we have a decl (which is the case for
3239	// __builtin_source_location).
3240	assert(Base.is<const ValueDecl *>());
3241	assert(Val.getLValuePath().size() == 0);
3242	const auto *BaseDecl = Base.dyn_cast<const ValueDecl *>();
3243	assert(BaseDecl);
3244
3245	auto *UGCD = cast<UnnamedGlobalConstantDecl>(Val: BaseDecl);
3246
3247	std::optional<unsigned> GlobalIndex = P.getOrCreateGlobal(UGCD);
3248	if (!GlobalIndex)
3249	return false;
3250
3251	if (!this->emitGetPtrGlobal(*GlobalIndex, E))
3252	return false;
3253
3254	const Record *R = getRecord(E->getType());
3255	const APValue &V = UGCD->getValue();
3256	for (unsigned I = 0, N = R->getNumFields(); I != N; ++I) {
3257	const Record::Field *F = R->getField(I);
3258	const APValue &FieldValue = V.getStructField(i: I);
3259
3260	PrimType FieldT = classifyPrim(F->Decl->getType());
3261
3262	if (!this->visitAPValue(FieldValue, FieldT, E))
3263	return false;
3264	if (!this->emitInitField(FieldT, F->Offset, E))
3265	return false;
3266	}
3267
3268	// Leave the pointer to the global on the stack.
3269	return true;
3270	}
3271
3272	template <class Emitter>
3273	bool Compiler<Emitter>::VisitOffsetOfExpr(const OffsetOfExpr *E) {
3274	unsigned N = E->getNumComponents();
3275	if (N == 0)
3276	return false;
3277
3278	for (unsigned I = 0; I != N; ++I) {
3279	const OffsetOfNode &Node = E->getComponent(Idx: I);
3280	if (Node.getKind() == OffsetOfNode::Array) {
3281	const Expr *ArrayIndexExpr = E->getIndexExpr(Idx: Node.getArrayExprIndex());
3282	PrimType IndexT = classifyPrim(ArrayIndexExpr->getType());
3283
3284	if (DiscardResult) {
3285	if (!this->discard(ArrayIndexExpr))
3286	return false;
3287	continue;
3288	}
3289
3290	if (!this->visit(ArrayIndexExpr))
3291	return false;
3292	// Cast to Sint64.
3293	if (IndexT != PT_Sint64) {
3294	if (!this->emitCast(IndexT, PT_Sint64, E))
3295	return false;
3296	}
3297	}
3298	}
3299
3300	if (DiscardResult)
3301	return true;
3302
3303	PrimType T = classifyPrim(E->getType());
3304	return this->emitOffsetOf(T, E, E);
3305	}
3306
3307	template <class Emitter>
3308	bool Compiler<Emitter>::VisitCXXScalarValueInitExpr(
3309	const CXXScalarValueInitExpr *E) {
3310	QualType Ty = E->getType();
3311
3312	if (DiscardResult || Ty->isVoidType())
3313	return true;
3314
3315	if (std::optional<PrimType> T = classify(Ty))
3316	return this->visitZeroInitializer(*T, Ty, E);
3317
3318	if (const auto *CT = Ty->getAs<ComplexType>()) {
3319	if (!Initializing) {
3320	std::optional<unsigned> LocalIndex = allocateLocal(Decl: E);
3321	if (!LocalIndex)
3322	return false;
3323	if (!this->emitGetPtrLocal(*LocalIndex, E))
3324	return false;
3325	}
3326
3327	// Initialize both fields to 0.
3328	QualType ElemQT = CT->getElementType();
3329	PrimType ElemT = classifyPrim(ElemQT);
3330
3331	for (unsigned I = 0; I != 2; ++I) {
3332	if (!this->visitZeroInitializer(ElemT, ElemQT, E))
3333	return false;
3334	if (!this->emitInitElem(ElemT, I, E))
3335	return false;
3336	}
3337	return true;
3338	}
3339
3340	if (const auto *VT = Ty->getAs<VectorType>()) {
3341	// FIXME: Code duplication with the _Complex case above.
3342	if (!Initializing) {
3343	std::optional<unsigned> LocalIndex = allocateLocal(Decl: E);
3344	if (!LocalIndex)
3345	return false;
3346	if (!this->emitGetPtrLocal(*LocalIndex, E))
3347	return false;
3348	}
3349
3350	// Initialize all fields to 0.
3351	QualType ElemQT = VT->getElementType();
3352	PrimType ElemT = classifyPrim(ElemQT);
3353
3354	for (unsigned I = 0, N = VT->getNumElements(); I != N; ++I) {
3355	if (!this->visitZeroInitializer(ElemT, ElemQT, E))
3356	return false;
3357	if (!this->emitInitElem(ElemT, I, E))
3358	return false;
3359	}
3360	return true;
3361	}
3362
3363	return false;
3364	}
3365
3366	template <class Emitter>
3367	bool Compiler<Emitter>::VisitSizeOfPackExpr(const SizeOfPackExpr *E) {
3368	return this->emitConst(E->getPackLength(), E);
3369	}
3370
3371	template <class Emitter>
3372	bool Compiler<Emitter>::VisitGenericSelectionExpr(
3373	const GenericSelectionExpr *E) {
3374	return this->delegate(E->getResultExpr());
3375	}
3376
3377	template <class Emitter>
3378	bool Compiler<Emitter>::VisitChooseExpr(const ChooseExpr *E) {
3379	return this->delegate(E->getChosenSubExpr());
3380	}
3381
3382	template <class Emitter>
3383	bool Compiler<Emitter>::VisitObjCBoolLiteralExpr(const ObjCBoolLiteralExpr *E) {
3384	if (DiscardResult)
3385	return true;
3386
3387	return this->emitConst(E->getValue(), E);
3388	}
3389
3390	template <class Emitter>
3391	bool Compiler<Emitter>::VisitCXXInheritedCtorInitExpr(
3392	const CXXInheritedCtorInitExpr *E) {
3393	const CXXConstructorDecl *Ctor = E->getConstructor();
3394	assert(!Ctor->isTrivial() &&
3395	"Trivial CXXInheritedCtorInitExpr, implement. (possible?)");
3396	const Function *F = this->getFunction(Ctor);
3397	assert(F);
3398	assert(!F->hasRVO());
3399	assert(F->hasThisPointer());
3400
3401	if (!this->emitDupPtr(SourceInfo{}))
3402	return false;
3403
3404	// Forward all arguments of the current function (which should be a
3405	// constructor itself) to the inherited ctor.
3406	// This is necessary because the calling code has pushed the pointer
3407	// of the correct base for us already, but the arguments need
3408	// to come after.
3409	unsigned Offset = align(Size: primSize(Type: PT_Ptr)); // instance pointer.
3410	for (const ParmVarDecl *PD : Ctor->parameters()) {
3411	PrimType PT = this->classify(PD->getType()).value_or(PT_Ptr);
3412
3413	if (!this->emitGetParam(PT, Offset, E))
3414	return false;
3415	Offset += align(primSize(PT));
3416	}
3417
3418	return this->emitCall(F, 0, E);
3419	}
3420
3421	// FIXME: This function has become rather unwieldy, especially
3422	// the part where we initialize an array allocation of dynamic size.
3423	template <class Emitter>
3424	bool Compiler<Emitter>::VisitCXXNewExpr(const CXXNewExpr *E) {
3425	assert(classifyPrim(E->getType()) == PT_Ptr);
3426	const Expr *Init = E->getInitializer();
3427	QualType ElementType = E->getAllocatedType();
3428	std::optional<PrimType> ElemT = classify(ElementType);
3429	unsigned PlacementArgs = E->getNumPlacementArgs();
3430	const FunctionDecl *OperatorNew = E->getOperatorNew();
3431	const Expr *PlacementDest = nullptr;
3432	bool IsNoThrow = false;
3433
3434	if (PlacementArgs != 0) {
3435	// FIXME: There is no restriction on this, but it's not clear that any
3436	// other form makes any sense. We get here for cases such as:
3437	//
3438	// new (std::align_val_t{N}) X(int)
3439	//
3440	// (which should presumably be valid only if N is a multiple of
3441	// alignof(int), and in any case can't be deallocated unless N is
3442	// alignof(X) and X has new-extended alignment).
3443	if (PlacementArgs == 1) {
3444	const Expr *Arg1 = E->getPlacementArg(I: 0);
3445	if (Arg1->getType()->isNothrowT()) {
3446	if (!this->discard(Arg1))
3447	return false;
3448	IsNoThrow = true;
3449	} else {
3450	// Invalid unless we have C++26 or are in a std:: function.
3451	if (!this->emitInvalidNewDeleteExpr(E, E))
3452	return false;
3453
3454	// If we have a placement-new destination, we'll later use that instead
3455	// of allocating.
3456	if (OperatorNew->isReservedGlobalPlacementOperator())
3457	PlacementDest = Arg1;
3458	}
3459	} else {
3460	// Always invalid.
3461	return this->emitInvalid(E);
3462	}
3463	} else if (!OperatorNew
3464	->isUsableAsGlobalAllocationFunctionInConstantEvaluation())
3465	return this->emitInvalidNewDeleteExpr(E, E);
3466
3467	const Descriptor *Desc;
3468	if (!PlacementDest) {
3469	if (ElemT) {
3470	if (E->isArray())
3471	Desc = nullptr; // We're not going to use it in this case.
3472	else
3473	Desc = P.createDescriptor(E, *ElemT, /*SourceTy=*/nullptr,
3474	Descriptor::InlineDescMD);
3475	} else {
3476	Desc = P.createDescriptor(
3477	E, ElementType.getTypePtr(),
3478	E->isArray() ? std::nullopt : Descriptor::InlineDescMD,
3479	/*IsConst=*/false, /*IsTemporary=*/false, /*IsMutable=*/false,
3480	/*IsVolatile=*/false, Init);
3481	}
3482	}
3483
3484	if (E->isArray()) {
3485	std::optional<const Expr *> ArraySizeExpr = E->getArraySize();
3486	if (!ArraySizeExpr)
3487	return false;
3488
3489	const Expr *Stripped = *ArraySizeExpr;
3490	for (; auto *ICE = dyn_cast<ImplicitCastExpr>(Val: Stripped);
3491	Stripped = ICE->getSubExpr())
3492	if (ICE->getCastKind() != CK_NoOp &&
3493	ICE->getCastKind() != CK_IntegralCast)
3494	break;
3495
3496	PrimType SizeT = classifyPrim(Stripped->getType());
3497
3498	// Save evaluated array size to a variable.
3499	unsigned ArrayLen =
3500	allocateLocalPrimitive(Decl: Stripped, Ty: SizeT, /*IsConst=*/false);
3501	if (!this->visit(Stripped))
3502	return false;
3503	if (!this->emitSetLocal(SizeT, ArrayLen, E))
3504	return false;
3505
3506	if (PlacementDest) {
3507	if (!this->visit(PlacementDest))
3508	return false;
3509	if (!this->emitStartLifetime(E))
3510	return false;
3511	if (!this->emitGetLocal(SizeT, ArrayLen, E))
3512	return false;
3513	if (!this->emitCheckNewTypeMismatchArray(SizeT, E, E))
3514	return false;
3515	} else {
3516	if (!this->emitGetLocal(SizeT, ArrayLen, E))
3517	return false;
3518
3519	if (ElemT) {
3520	// N primitive elements.
3521	if (!this->emitAllocN(SizeT, *ElemT, E, IsNoThrow, E))
3522	return false;
3523	} else {
3524	// N Composite elements.
3525	if (!this->emitAllocCN(SizeT, Desc, IsNoThrow, E))
3526	return false;
3527	}
3528	}
3529
3530	if (Init) {
3531	QualType InitType = Init->getType();
3532	size_t StaticInitElems = 0;
3533	const Expr *DynamicInit = nullptr;
3534	if (const ConstantArrayType *CAT =
3535	Ctx.getASTContext().getAsConstantArrayType(T: InitType)) {
3536	StaticInitElems = CAT->getZExtSize();
3537	if (!this->visitInitializer(Init))
3538	return false;
3539
3540	if (const auto *ILE = dyn_cast<InitListExpr>(Val: Init);
3541	ILE && ILE->hasArrayFiller())
3542	DynamicInit = ILE->getArrayFiller();
3543	}
3544
3545	// The initializer initializes a certain number of elements, S.
3546	// However, the complete number of elements, N, might be larger than that.
3547	// In this case, we need to get an initializer for the remaining elements.
3548	// There are to cases:
3549	// 1) For the form 'new Struct[n];', the initializer is a
3550	// CXXConstructExpr and its type is an IncompleteArrayType.
3551	// 2) For the form 'new Struct[n]{1,2,3}', the initializer is an
3552	// InitListExpr and the initializer for the remaining elements
3553	// is the array filler.
3554
3555	if (DynamicInit || InitType->isIncompleteArrayType()) {
3556	const Function *CtorFunc = nullptr;
3557	if (const auto *CE = dyn_cast<CXXConstructExpr>(Val: Init)) {
3558	CtorFunc = getFunction(FD: CE->getConstructor());
3559	if (!CtorFunc)
3560	return false;
3561	} else if (!DynamicInit)
3562	DynamicInit = Init;
3563
3564	LabelTy EndLabel = this->getLabel();
3565	LabelTy StartLabel = this->getLabel();
3566
3567	// In the nothrow case, the alloc above might have returned nullptr.
3568	// Don't call any constructors that case.
3569	if (IsNoThrow) {
3570	if (!this->emitDupPtr(E))
3571	return false;
3572	if (!this->emitNullPtr(0, nullptr, E))
3573	return false;
3574	if (!this->emitEQPtr(E))
3575	return false;
3576	if (!this->jumpTrue(EndLabel))
3577	return false;
3578	}
3579
3580	// Create loop variables.
3581	unsigned Iter =
3582	allocateLocalPrimitive(Decl: Stripped, Ty: SizeT, /*IsConst=*/false);
3583	if (!this->emitConst(StaticInitElems, SizeT, E))
3584	return false;
3585	if (!this->emitSetLocal(SizeT, Iter, E))
3586	return false;
3587
3588	this->fallthrough(StartLabel);
3589	this->emitLabel(StartLabel);
3590	// Condition. Iter < ArrayLen?
3591	if (!this->emitGetLocal(SizeT, Iter, E))
3592	return false;
3593	if (!this->emitGetLocal(SizeT, ArrayLen, E))
3594	return false;
3595	if (!this->emitLT(SizeT, E))
3596	return false;
3597	if (!this->jumpFalse(EndLabel))
3598	return false;
3599
3600	// Pointer to the allocated array is already on the stack.
3601	if (!this->emitGetLocal(SizeT, Iter, E))
3602	return false;
3603	if (!this->emitArrayElemPtr(SizeT, E))
3604	return false;
3605
3606	if (isa_and_nonnull<ImplicitValueInitExpr>(Val: DynamicInit) &&
3607	DynamicInit->getType()->isArrayType()) {
3608	QualType ElemType =
3609	DynamicInit->getType()->getAsArrayTypeUnsafe()->getElementType();
3610	PrimType InitT = classifyPrim(ElemType);
3611	if (!this->visitZeroInitializer(InitT, ElemType, E))
3612	return false;
3613	if (!this->emitStorePop(InitT, E))
3614	return false;
3615	} else if (DynamicInit) {
3616	if (std::optional<PrimType> InitT = classify(DynamicInit)) {
3617	if (!this->visit(DynamicInit))
3618	return false;
3619	if (!this->emitStorePop(*InitT, E))
3620	return false;
3621	} else {
3622	if (!this->visitInitializer(DynamicInit))
3623	return false;
3624	if (!this->emitPopPtr(E))
3625	return false;
3626	}
3627	} else {
3628	assert(CtorFunc);
3629	if (!this->emitCall(CtorFunc, 0, E))
3630	return false;
3631	}
3632
3633	// ++Iter;
3634	if (!this->emitGetPtrLocal(Iter, E))
3635	return false;
3636	if (!this->emitIncPop(SizeT, false, E))
3637	return false;
3638
3639	if (!this->jump(StartLabel))
3640	return false;
3641
3642	this->fallthrough(EndLabel);
3643	this->emitLabel(EndLabel);
3644	}
3645	}
3646	} else { // Non-array.
3647	if (PlacementDest) {
3648	if (!this->visit(PlacementDest))
3649	return false;
3650	if (!this->emitStartLifetime(E))
3651	return false;
3652	if (!this->emitCheckNewTypeMismatch(E, E))
3653	return false;
3654	} else {
3655	// Allocate just one element.
3656	if (!this->emitAlloc(Desc, E))
3657	return false;
3658	}
3659
3660	if (Init) {
3661	if (ElemT) {
3662	if (!this->visit(Init))
3663	return false;
3664
3665	if (!this->emitInit(*ElemT, E))
3666	return false;
3667	} else {
3668	// Composite.
3669	if (!this->visitInitializer(Init))
3670	return false;
3671	}
3672	}
3673	}
3674
3675	if (DiscardResult)
3676	return this->emitPopPtr(E);
3677
3678	return true;
3679	}
3680
3681	template <class Emitter>
3682	bool Compiler<Emitter>::VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
3683	const Expr *Arg = E->getArgument();
3684
3685	const FunctionDecl *OperatorDelete = E->getOperatorDelete();
3686
3687	if (!OperatorDelete->isUsableAsGlobalAllocationFunctionInConstantEvaluation())
3688	return this->emitInvalidNewDeleteExpr(E, E);
3689
3690	// Arg must be an lvalue.
3691	if (!this->visit(Arg))
3692	return false;
3693
3694	return this->emitFree(E->isArrayForm(), E->isGlobalDelete(), E);
3695	}
3696
3697	template <class Emitter>
3698	bool Compiler<Emitter>::VisitBlockExpr(const BlockExpr *E) {
3699	if (DiscardResult)
3700	return true;
3701
3702	const Function *Func = nullptr;
3703	if (auto F = Ctx.getOrCreateObjCBlock(E))
3704	Func = F;
3705
3706	if (!Func)
3707	return false;
3708	return this->emitGetFnPtr(Func, E);
3709	}
3710
3711	template <class Emitter>
3712	bool Compiler<Emitter>::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
3713	const Type *TypeInfoType = E->getType().getTypePtr();
3714
3715	auto canonType = [](const Type *T) {
3716	return T->getCanonicalTypeUnqualified().getTypePtr();
3717	};
3718
3719	if (!E->isPotentiallyEvaluated()) {
3720	if (DiscardResult)
3721	return true;
3722
3723	if (E->isTypeOperand())
3724	return this->emitGetTypeid(
3725	canonType(E->getTypeOperand(Context: Ctx.getASTContext()).getTypePtr()),
3726	TypeInfoType, E);
3727
3728	return this->emitGetTypeid(
3729	canonType(E->getExprOperand()->getType().getTypePtr()), TypeInfoType,
3730	E);
3731	}
3732
3733	// Otherwise, we need to evaluate the expression operand.
3734	assert(E->getExprOperand());
3735	assert(E->getExprOperand()->isLValue());
3736
3737	if (!Ctx.getLangOpts().CPlusPlus20 && !this->emitDiagTypeid(E))
3738	return false;
3739
3740	if (!this->visit(E->getExprOperand()))
3741	return false;
3742
3743	if (!this->emitGetTypeidPtr(TypeInfoType, E))
3744	return false;
3745	if (DiscardResult)
3746	return this->emitPopPtr(E);
3747	return true;
3748	}
3749
3750	template <class Emitter>
3751	bool Compiler<Emitter>::VisitExpressionTraitExpr(const ExpressionTraitExpr *E) {
3752	assert(Ctx.getLangOpts().CPlusPlus);
3753	return this->emitConstBool(E->getValue(), E);
3754	}
3755
3756	template <class Emitter>
3757	bool Compiler<Emitter>::VisitCXXUuidofExpr(const CXXUuidofExpr *E) {
3758	if (DiscardResult)
3759	return true;
3760	assert(!Initializing);
3761
3762	const MSGuidDecl *GuidDecl = E->getGuidDecl();
3763	const RecordDecl *RD = GuidDecl->getType()->getAsRecordDecl();
3764	assert(RD);
3765	// If the definiton of the result type is incomplete, just return a dummy.
3766	// If (and when) that is read from, we will fail, but not now.
3767	if (!RD->isCompleteDefinition())
3768	return this->emitDummyPtr(GuidDecl, E);
3769
3770	std::optional<unsigned> GlobalIndex = P.getOrCreateGlobal(GuidDecl);
3771	if (!GlobalIndex)
3772	return false;
3773	if (!this->emitGetPtrGlobal(*GlobalIndex, E))
3774	return false;
3775
3776	assert(this->getRecord(E->getType()));
3777
3778	const APValue &V = GuidDecl->getAsAPValue();
3779	if (V.getKind() == APValue::None)
3780	return true;
3781
3782	assert(V.isStruct());
3783	assert(V.getStructNumBases() == 0);
3784	if (!this->visitAPValueInitializer(V, E, E->getType()))
3785	return false;
3786
3787	return this->emitFinishInit(E);
3788	}
3789
3790	template <class Emitter>
3791	bool Compiler<Emitter>::VisitRequiresExpr(const RequiresExpr *E) {
3792	assert(classifyPrim(E->getType()) == PT_Bool);
3793	if (DiscardResult)
3794	return true;
3795	return this->emitConstBool(E->isSatisfied(), E);
3796	}
3797
3798	template <class Emitter>
3799	bool Compiler<Emitter>::VisitConceptSpecializationExpr(
3800	const ConceptSpecializationExpr *E) {
3801	assert(classifyPrim(E->getType()) == PT_Bool);
3802	if (DiscardResult)
3803	return true;
3804	return this->emitConstBool(E->isSatisfied(), E);
3805	}
3806
3807	template <class Emitter>
3808	bool Compiler<Emitter>::VisitCXXRewrittenBinaryOperator(
3809	const CXXRewrittenBinaryOperator *E) {
3810	return this->delegate(E->getSemanticForm());
3811	}
3812
3813	template <class Emitter>
3814	bool Compiler<Emitter>::VisitPseudoObjectExpr(const PseudoObjectExpr *E) {
3815
3816	for (const Expr *SemE : E->semantics()) {
3817	if (auto *OVE = dyn_cast<OpaqueValueExpr>(Val: SemE)) {
3818	if (SemE == E->getResultExpr())
3819	return false;
3820
3821	if (OVE->isUnique())
3822	continue;
3823
3824	if (!this->discard(OVE))
3825	return false;
3826	} else if (SemE == E->getResultExpr()) {
3827	if (!this->delegate(SemE))
3828	return false;
3829	} else {
3830	if (!this->discard(SemE))
3831	return false;
3832	}
3833	}
3834	return true;
3835	}
3836
3837	template <class Emitter>
3838	bool Compiler<Emitter>::VisitPackIndexingExpr(const PackIndexingExpr *E) {
3839	return this->delegate(E->getSelectedExpr());
3840	}
3841
3842	template <class Emitter>
3843	bool Compiler<Emitter>::VisitRecoveryExpr(const RecoveryExpr *E) {
3844	return this->emitError(E);
3845	}
3846
3847	template <class Emitter>
3848	bool Compiler<Emitter>::VisitAddrLabelExpr(const AddrLabelExpr *E) {
3849	assert(E->getType()->isVoidPointerType());
3850
3851	unsigned Offset =
3852	allocateLocalPrimitive(Decl: E->getLabel(), Ty: PT_Ptr, /*IsConst=*/true);
3853
3854	return this->emitGetLocal(PT_Ptr, Offset, E);
3855	}
3856
3857	template <class Emitter>
3858	bool Compiler<Emitter>::VisitConvertVectorExpr(const ConvertVectorExpr *E) {
3859	assert(Initializing);
3860	const auto *VT = E->getType()->castAs<VectorType>();
3861	QualType ElemType = VT->getElementType();
3862	PrimType ElemT = classifyPrim(ElemType);
3863	const Expr *Src = E->getSrcExpr();
3864	QualType SrcType = Src->getType();
3865	PrimType SrcElemT = classifyVectorElementType(T: SrcType);
3866
3867	unsigned SrcOffset =
3868	this->allocateLocalPrimitive(Src, PT_Ptr, /*IsConst=*/true);
3869	if (!this->visit(Src))
3870	return false;
3871	if (!this->emitSetLocal(PT_Ptr, SrcOffset, E))
3872	return false;
3873
3874	for (unsigned I = 0; I != VT->getNumElements(); ++I) {
3875	if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
3876	return false;
3877	if (!this->emitArrayElemPop(SrcElemT, I, E))
3878	return false;
3879
3880	// Cast to the desired result element type.
3881	if (SrcElemT != ElemT) {
3882	if (!this->emitPrimCast(SrcElemT, ElemT, ElemType, E))
3883	return false;
3884	} else if (ElemType->isFloatingType() && SrcType != ElemType) {
3885	const auto *TargetSemantics = &Ctx.getFloatSemantics(T: ElemType);
3886	if (!this->emitCastFP(TargetSemantics, getRoundingMode(E), E))
3887	return false;
3888	}
3889	if (!this->emitInitElem(ElemT, I, E))
3890	return false;
3891	}
3892
3893	return true;
3894	}
3895
3896	template <class Emitter>
3897	bool Compiler<Emitter>::VisitShuffleVectorExpr(const ShuffleVectorExpr *E) {
3898	assert(Initializing);
3899	assert(E->getNumSubExprs() > 2);
3900
3901	const Expr *Vecs[] = {E->getExpr(Index: 0), E->getExpr(Index: 1)};
3902	const VectorType *VT = Vecs[0]->getType()->castAs<VectorType>();
3903	PrimType ElemT = classifyPrim(VT->getElementType());
3904	unsigned NumInputElems = VT->getNumElements();
3905	unsigned NumOutputElems = E->getNumSubExprs() - 2;
3906	assert(NumOutputElems > 0);
3907
3908	// Save both input vectors to a local variable.
3909	unsigned VectorOffsets[2];
3910	for (unsigned I = 0; I != 2; ++I) {
3911	VectorOffsets[I] =
3912	this->allocateLocalPrimitive(Vecs[I], PT_Ptr, /*IsConst=*/true);
3913	if (!this->visit(Vecs[I]))
3914	return false;
3915	if (!this->emitSetLocal(PT_Ptr, VectorOffsets[I], E))
3916	return false;
3917	}
3918	for (unsigned I = 0; I != NumOutputElems; ++I) {
3919	APSInt ShuffleIndex = E->getShuffleMaskIdx(N: I);
3920	assert(ShuffleIndex >= -1);
3921	if (ShuffleIndex == -1)
3922	return this->emitInvalidShuffleVectorIndex(I, E);
3923
3924	assert(ShuffleIndex < (NumInputElems * 2));
3925	if (!this->emitGetLocal(PT_Ptr,
3926	VectorOffsets[ShuffleIndex >= NumInputElems], E))
3927	return false;
3928	unsigned InputVectorIndex = ShuffleIndex.getZExtValue() % NumInputElems;
3929	if (!this->emitArrayElemPop(ElemT, InputVectorIndex, E))
3930	return false;
3931
3932	if (!this->emitInitElem(ElemT, I, E))
3933	return false;
3934	}
3935
3936	return true;
3937	}
3938
3939	template <class Emitter>
3940	bool Compiler<Emitter>::VisitExtVectorElementExpr(
3941	const ExtVectorElementExpr *E) {
3942	const Expr *Base = E->getBase();
3943	assert(
3944	Base->getType()->isVectorType() ||
3945	Base->getType()->getAs<PointerType>()->getPointeeType()->isVectorType());
3946
3947	SmallVector<uint32_t, 4> Indices;
3948	E->getEncodedElementAccess(Elts&: Indices);
3949
3950	if (Indices.size() == 1) {
3951	if (!this->visit(Base))
3952	return false;
3953
3954	if (E->isGLValue()) {
3955	if (!this->emitConstUint32(Indices[0], E))
3956	return false;
3957	return this->emitArrayElemPtrPop(PT_Uint32, E);
3958	}
3959	// Else, also load the value.
3960	return this->emitArrayElemPop(classifyPrim(E->getType()), Indices[0], E);
3961	}
3962
3963	// Create a local variable for the base.
3964	unsigned BaseOffset = allocateLocalPrimitive(Decl: Base, Ty: PT_Ptr, /*IsConst=*/true);
3965	if (!this->visit(Base))
3966	return false;
3967	if (!this->emitSetLocal(PT_Ptr, BaseOffset, E))
3968	return false;
3969
3970	// Now the vector variable for the return value.
3971	if (!Initializing) {
3972	std::optional<unsigned> ResultIndex;
3973	ResultIndex = allocateLocal(Decl: E);
3974	if (!ResultIndex)
3975	return false;
3976	if (!this->emitGetPtrLocal(*ResultIndex, E))
3977	return false;
3978	}
3979
3980	assert(Indices.size() == E->getType()->getAs<VectorType>()->getNumElements());
3981
3982	PrimType ElemT =
3983	classifyPrim(E->getType()->getAs<VectorType>()->getElementType());
3984	uint32_t DstIndex = 0;
3985	for (uint32_t I : Indices) {
3986	if (!this->emitGetLocal(PT_Ptr, BaseOffset, E))
3987	return false;
3988	if (!this->emitArrayElemPop(ElemT, I, E))
3989	return false;
3990	if (!this->emitInitElem(ElemT, DstIndex, E))
3991	return false;
3992	++DstIndex;
3993	}
3994
3995	// Leave the result pointer on the stack.
3996	assert(!DiscardResult);
3997	return true;
3998	}
3999
4000	template <class Emitter>
4001	bool Compiler<Emitter>::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) {
4002	const Expr *SubExpr = E->getSubExpr();
4003	if (!E->isExpressibleAsConstantInitializer())
4004	return this->discard(SubExpr) && this->emitInvalid(E);
4005
4006	if (DiscardResult)
4007	return true;
4008
4009	assert(classifyPrim(E) == PT_Ptr);
4010	return this->emitDummyPtr(E, E);
4011	}
4012
4013	template <class Emitter>
4014	bool Compiler<Emitter>::VisitCXXStdInitializerListExpr(
4015	const CXXStdInitializerListExpr *E) {
4016	const Expr *SubExpr = E->getSubExpr();
4017	const ConstantArrayType *ArrayType =
4018	Ctx.getASTContext().getAsConstantArrayType(T: SubExpr->getType());
4019	const Record *R = getRecord(E->getType());
4020	assert(Initializing);
4021	assert(SubExpr->isGLValue());
4022
4023	if (!this->visit(SubExpr))
4024	return false;
4025	if (!this->emitConstUint8(0, E))
4026	return false;
4027	if (!this->emitArrayElemPtrPopUint8(E))
4028	return false;
4029	if (!this->emitInitFieldPtr(R->getField(I: 0u)->Offset, E))
4030	return false;
4031
4032	PrimType SecondFieldT = classifyPrim(R->getField(I: 1u)->Decl->getType());
4033	if (isIntegralType(T: SecondFieldT)) {
4034	if (!this->emitConst(static_cast<APSInt>(ArrayType->getSize()),
4035	SecondFieldT, E))
4036	return false;
4037	return this->emitInitField(SecondFieldT, R->getField(I: 1u)->Offset, E);
4038	}
4039	assert(SecondFieldT == PT_Ptr);
4040
4041	if (!this->emitGetFieldPtr(R->getField(I: 0u)->Offset, E))
4042	return false;
4043	if (!this->emitExpandPtr(E))
4044	return false;
4045	if (!this->emitConst(static_cast<APSInt>(ArrayType->getSize()), PT_Uint64, E))
4046	return false;
4047	if (!this->emitArrayElemPtrPop(PT_Uint64, E))
4048	return false;
4049	return this->emitInitFieldPtr(R->getField(I: 1u)->Offset, E);
4050	}
4051
4052	template <class Emitter>
4053	bool Compiler<Emitter>::VisitStmtExpr(const StmtExpr *E) {
4054	BlockScope<Emitter> BS(this);
4055	StmtExprScope<Emitter> SS(this);
4056
4057	const CompoundStmt *CS = E->getSubStmt();
4058	const Stmt *Result = CS->getStmtExprResult();
4059	for (const Stmt *S : CS->body()) {
4060	if (S != Result) {
4061	if (!this->visitStmt(S))
4062	return false;
4063	continue;
4064	}
4065
4066	assert(S == Result);
4067	if (const Expr *ResultExpr = dyn_cast<Expr>(Val: S))
4068	return this->delegate(ResultExpr);
4069	return this->emitUnsupported(E);
4070	}
4071
4072	return BS.destroyLocals();
4073	}
4074
4075	template <class Emitter> bool Compiler<Emitter>::discard(const Expr *E) {
4076	OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
4077	/*NewInitializing=*/false);
4078	return this->Visit(E);
4079	}
4080
4081	template <class Emitter> bool Compiler<Emitter>::delegate(const Expr *E) {
4082	// We're basically doing:
4083	// OptionScope<Emitter> Scope(this, DicardResult, Initializing);
4084	// but that's unnecessary of course.
4085	return this->Visit(E);
4086	}
4087
4088	template <class Emitter> bool Compiler<Emitter>::visit(const Expr *E) {
4089	if (E->getType().isNull())
4090	return false;
4091
4092	if (E->getType()->isVoidType())
4093	return this->discard(E);
4094
4095	// Create local variable to hold the return value.
4096	if (!E->isGLValue() && !E->getType()->isAnyComplexType() &&
4097	!classify(E->getType())) {
4098	std::optional<unsigned> LocalIndex = allocateLocal(Decl: E);
4099	if (!LocalIndex)
4100	return false;
4101
4102	if (!this->emitGetPtrLocal(*LocalIndex, E))
4103	return false;
4104	InitLinkScope<Emitter> ILS(this, InitLink::Temp(Offset: *LocalIndex));
4105	return this->visitInitializer(E);
4106	}
4107
4108	// Otherwise,we have a primitive return value, produce the value directly
4109	// and push it on the stack.
4110	OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
4111	/*NewInitializing=*/false);
4112	return this->Visit(E);
4113	}
4114
4115	template <class Emitter>
4116	bool Compiler<Emitter>::visitInitializer(const Expr *E) {
4117	assert(!classify(E->getType()));
4118
4119	OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
4120	/*NewInitializing=*/true);
4121	return this->Visit(E);
4122	}
4123
4124	template <class Emitter> bool Compiler<Emitter>::visitBool(const Expr *E) {
4125	std::optional<PrimType> T = classify(E->getType());
4126	if (!T) {
4127	// Convert complex values to bool.
4128	if (E->getType()->isAnyComplexType()) {
4129	if (!this->visit(E))
4130	return false;
4131	return this->emitComplexBoolCast(E);
4132	}
4133	return false;
4134	}
4135
4136	if (!this->visit(E))
4137	return false;
4138
4139	if (T == PT_Bool)
4140	return true;
4141
4142	// Convert pointers to bool.
4143	if (T == PT_Ptr)
4144	return this->emitIsNonNullPtr(E);
4145
4146	// Or Floats.
4147	if (T == PT_Float)
4148	return this->emitCastFloatingIntegralBool(getFPOptions(E), E);
4149
4150	// Or anything else we can.
4151	return this->emitCast(*T, PT_Bool, E);
4152	}
4153
4154	template <class Emitter>
4155	bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
4156	const Expr *E) {
4157	if (const auto *AT = QT->getAs<AtomicType>())
4158	QT = AT->getValueType();
4159
4160	switch (T) {
4161	case PT_Bool:
4162	return this->emitZeroBool(E);
4163	case PT_Sint8:
4164	return this->emitZeroSint8(E);
4165	case PT_Uint8:
4166	return this->emitZeroUint8(E);
4167	case PT_Sint16:
4168	return this->emitZeroSint16(E);
4169	case PT_Uint16:
4170	return this->emitZeroUint16(E);
4171	case PT_Sint32:
4172	return this->emitZeroSint32(E);
4173	case PT_Uint32:
4174	return this->emitZeroUint32(E);
4175	case PT_Sint64:
4176	return this->emitZeroSint64(E);
4177	case PT_Uint64:
4178	return this->emitZeroUint64(E);
4179	case PT_IntAP:
4180	return this->emitZeroIntAP(Ctx.getBitWidth(T: QT), E);
4181	case PT_IntAPS:
4182	return this->emitZeroIntAPS(Ctx.getBitWidth(T: QT), E);
4183	case PT_Ptr:
4184	return this->emitNullPtr(Ctx.getASTContext().getTargetNullPointerValue(QT),
4185	nullptr, E);
4186	case PT_MemberPtr:
4187	return this->emitNullMemberPtr(0, nullptr, E);
4188	case PT_Float:
4189	return this->emitConstFloat(APFloat::getZero(Sem: Ctx.getFloatSemantics(T: QT)), E);
4190	case PT_FixedPoint: {
4191	auto Sem = Ctx.getASTContext().getFixedPointSemantics(Ty: E->getType());
4192	return this->emitConstFixedPoint(FixedPoint::zero(Sem), E);
4193	}
4194	llvm_unreachable("Implement");
4195	}
4196	llvm_unreachable("unknown primitive type");
4197	}
4198
4199	template <class Emitter>
4200	bool Compiler<Emitter>::visitZeroRecordInitializer(const Record *R,
4201	const Expr *E) {
4202	assert(E);
4203	assert(R);
4204	// Fields
4205	for (const Record::Field &Field : R->fields()) {
4206	if (Field.isUnnamedBitField())
4207	continue;
4208
4209	const Descriptor *D = Field.Desc;
4210	if (D->isPrimitive()) {
4211	QualType QT = D->getType();
4212	PrimType T = classifyPrim(D->getType());
4213	if (!this->visitZeroInitializer(T, QT, E))
4214	return false;
4215	if (!this->emitInitField(T, Field.Offset, E))
4216	return false;
4217	if (R->isUnion())
4218	break;
4219	continue;
4220	}
4221
4222	if (!this->emitGetPtrField(Field.Offset, E))
4223	return false;
4224
4225	if (D->isPrimitiveArray()) {
4226	QualType ET = D->getElemQualType();
4227	PrimType T = classifyPrim(ET);
4228	for (uint32_t I = 0, N = D->getNumElems(); I != N; ++I) {
4229	if (!this->visitZeroInitializer(T, ET, E))
4230	return false;
4231	if (!this->emitInitElem(T, I, E))
4232	return false;
4233	}
4234	} else if (D->isCompositeArray()) {
4235	// Can't be a vector or complex field.
4236	if (!this->visitZeroArrayInitializer(D->getType(), E))
4237	return false;
4238	} else if (D->isRecord()) {
4239	if (!this->visitZeroRecordInitializer(D->ElemRecord, E))
4240	return false;
4241	} else
4242	return false;
4243
4244	if (!this->emitFinishInitPop(E))
4245	return false;
4246
4247	// C++11 [dcl.init]p5: If T is a (possibly cv-qualified) union type, the
4248	// object's first non-static named data member is zero-initialized
4249	if (R->isUnion())
4250	break;
4251	}
4252
4253	for (const Record::Base &B : R->bases()) {
4254	if (!this->emitGetPtrBase(B.Offset, E))
4255	return false;
4256	if (!this->visitZeroRecordInitializer(B.R, E))
4257	return false;
4258	if (!this->emitFinishInitPop(E))
4259	return false;
4260	}
4261
4262	// FIXME: Virtual bases.
4263
4264	return true;
4265	}
4266
4267	template <class Emitter>
4268	bool Compiler<Emitter>::visitZeroArrayInitializer(QualType T, const Expr *E) {
4269	assert(T->isArrayType() || T->isAnyComplexType() || T->isVectorType());
4270	const ArrayType *AT = T->getAsArrayTypeUnsafe();
4271	QualType ElemType = AT->getElementType();
4272	size_t NumElems = cast<ConstantArrayType>(Val: AT)->getZExtSize();
4273
4274	if (std::optional<PrimType> ElemT = classify(ElemType)) {
4275	for (size_t I = 0; I != NumElems; ++I) {
4276	if (!this->visitZeroInitializer(*ElemT, ElemType, E))
4277	return false;
4278	if (!this->emitInitElem(*ElemT, I, E))
4279	return false;
4280	}
4281	return true;
4282	} else if (ElemType->isRecordType()) {
4283	const Record *R = getRecord(ElemType);
4284
4285	for (size_t I = 0; I != NumElems; ++I) {
4286	if (!this->emitConstUint32(I, E))
4287	return false;
4288	if (!this->emitArrayElemPtr(PT_Uint32, E))
4289	return false;
4290	if (!this->visitZeroRecordInitializer(R, E))
4291	return false;
4292	if (!this->emitPopPtr(E))
4293	return false;
4294	}
4295	return true;
4296	} else if (ElemType->isArrayType()) {
4297	for (size_t I = 0; I != NumElems; ++I) {
4298	if (!this->emitConstUint32(I, E))
4299	return false;
4300	if (!this->emitArrayElemPtr(PT_Uint32, E))
4301	return false;
4302	if (!this->visitZeroArrayInitializer(ElemType, E))
4303	return false;
4304	if (!this->emitPopPtr(E))
4305	return false;
4306	}
4307	return true;
4308	}
4309
4310	return false;
4311	}
4312
4313	template <class Emitter>
4314	template <typename T>
4315	bool Compiler<Emitter>::emitConst(T Value, PrimType Ty, const Expr *E) {
4316	switch (Ty) {
4317	case PT_Sint8:
4318	return this->emitConstSint8(Value, E);
4319	case PT_Uint8:
4320	return this->emitConstUint8(Value, E);
4321	case PT_Sint16:
4322	return this->emitConstSint16(Value, E);
4323	case PT_Uint16:
4324	return this->emitConstUint16(Value, E);
4325	case PT_Sint32:
4326	return this->emitConstSint32(Value, E);
4327	case PT_Uint32:
4328	return this->emitConstUint32(Value, E);
4329	case PT_Sint64:
4330	return this->emitConstSint64(Value, E);
4331	case PT_Uint64:
4332	return this->emitConstUint64(Value, E);
4333	case PT_Bool:
4334	return this->emitConstBool(Value, E);
4335	case PT_Ptr:
4336	case PT_MemberPtr:
4337	case PT_Float:
4338	case PT_IntAP:
4339	case PT_IntAPS:
4340	case PT_FixedPoint:
4341	llvm_unreachable("Invalid integral type");
4342	break;
4343	}
4344	llvm_unreachable("unknown primitive type");
4345	}
4346
4347	template <class Emitter>
4348	template <typename T>
4349	bool Compiler<Emitter>::emitConst(T Value, const Expr *E) {
4350	return this->emitConst(Value, classifyPrim(E->getType()), E);
4351	}
4352
4353	template <class Emitter>
4354	bool Compiler<Emitter>::emitConst(const APSInt &Value, PrimType Ty,
4355	const Expr *E) {
4356	if (Ty == PT_IntAPS)
4357	return this->emitConstIntAPS(Value, E);
4358	if (Ty == PT_IntAP)
4359	return this->emitConstIntAP(Value, E);
4360
4361	if (Value.isSigned())
4362	return this->emitConst(Value.getSExtValue(), Ty, E);
4363	return this->emitConst(Value.getZExtValue(), Ty, E);
4364	}
4365
4366	template <class Emitter>
4367	bool Compiler<Emitter>::emitConst(const APSInt &Value, const Expr *E) {
4368	return this->emitConst(Value, classifyPrim(E->getType()), E);
4369	}
4370
4371	template <class Emitter>
4372	unsigned Compiler<Emitter>::allocateLocalPrimitive(
4373	DeclTy &&Src, PrimType Ty, bool IsConst, const ValueDecl *ExtendingDecl,
4374	ScopeKind SC, bool IsConstexprUnknown) {
4375	// Make sure we don't accidentally register the same decl twice.
4376	if (const auto *VD =
4377	dyn_cast_if_present<ValueDecl>(Val: Src.dyn_cast<const Decl *>())) {
4378	assert(!P.getGlobal(VD));
4379	assert(!Locals.contains(VD));
4380	(void)VD;
4381	}
4382
4383	// FIXME: There are cases where Src.is<Expr*>() is wrong, e.g.
4384	// (int){12} in C. Consider using Expr::isTemporaryObject() instead
4385	// or isa<MaterializeTemporaryExpr>().
4386	Descriptor *D = P.createDescriptor(D: Src, T: Ty, SourceTy: nullptr, MDSize: Descriptor::InlineDescMD,
4387	IsConst, IsTemporary: isa<const Expr *>(Val: Src));
4388	D->IsConstexprUnknown = IsConstexprUnknown;
4389	Scope::Local Local = this->createLocal(D);
4390	if (auto *VD = dyn_cast_if_present<ValueDecl>(Val: Src.dyn_cast<const Decl *>()))
4391	Locals.insert(KV: {VD, Local});
4392	if (ExtendingDecl)
4393	VarScope->addExtended(Local, ExtendingDecl);
4394	else
4395	VarScope->addForScopeKind(Local, SC);
4396	return Local.Offset;
4397	}
4398
4399	template <class Emitter>
4400	std::optional<unsigned>
4401	Compiler<Emitter>::allocateLocal(DeclTy &&Src, QualType Ty,
4402	const ValueDecl *ExtendingDecl, ScopeKind SC,
4403	bool IsConstexprUnknown) {
4404	// Make sure we don't accidentally register the same decl twice.
4405	if ([[maybe_unused]] const auto *VD =
4406	dyn_cast_if_present<ValueDecl>(Val: Src.dyn_cast<const Decl *>())) {
4407	assert(!P.getGlobal(VD));
4408	assert(!Locals.contains(VD));
4409	}
4410
4411	const ValueDecl *Key = nullptr;
4412	const Expr *Init = nullptr;
4413	bool IsTemporary = false;
4414	if (auto *VD = dyn_cast_if_present<ValueDecl>(Val: Src.dyn_cast<const Decl *>())) {
4415	Key = VD;
4416	Ty = VD->getType();
4417
4418	if (const auto *VarD = dyn_cast<VarDecl>(Val: VD))
4419	Init = VarD->getInit();
4420	}
4421	if (auto *E = Src.dyn_cast<const Expr *>()) {
4422	IsTemporary = true;
4423	if (Ty.isNull())
4424	Ty = E->getType();
4425	}
4426
4427	Descriptor *D = P.createDescriptor(
4428	D: Src, Ty: Ty.getTypePtr(), MDSize: Descriptor::InlineDescMD, IsConst: Ty.isConstQualified(),
4429	IsTemporary, /*IsMutable=*/false, /*IsVolatile=*/false, Init);
4430	if (!D)
4431	return std::nullopt;
4432	D->IsConstexprUnknown = IsConstexprUnknown;
4433
4434	Scope::Local Local = this->createLocal(D);
4435	if (Key)
4436	Locals.insert(KV: {Key, Local});
4437	if (ExtendingDecl)
4438	VarScope->addExtended(Local, ExtendingDecl);
4439	else
4440	VarScope->addForScopeKind(Local, SC);
4441	return Local.Offset;
4442	}
4443
4444	template <class Emitter>
4445	std::optional<unsigned> Compiler<Emitter>::allocateTemporary(const Expr *E) {
4446	QualType Ty = E->getType();
4447	assert(!Ty->isRecordType());
4448
4449	Descriptor *D = P.createDescriptor(
4450	D: E, Ty: Ty.getTypePtr(), MDSize: Descriptor::InlineDescMD, IsConst: Ty.isConstQualified(),
4451	/*IsTemporary=*/true);
4452
4453	if (!D)
4454	return std::nullopt;
4455
4456	Scope::Local Local = this->createLocal(D);
4457	VariableScope<Emitter> *S = VarScope;
4458	assert(S);
4459	// Attach to topmost scope.
4460	while (S->getParent())
4461	S = S->getParent();
4462	assert(S && !S->getParent());
4463	S->addLocal(Local);
4464	return Local.Offset;
4465	}
4466
4467	template <class Emitter>
4468	const RecordType *Compiler<Emitter>::getRecordTy(QualType Ty) {
4469	if (const PointerType *PT = dyn_cast<PointerType>(Val&: Ty))
4470	return PT->getPointeeType()->getAs<RecordType>();
4471	return Ty->getAs<RecordType>();
4472	}
4473
4474	template <class Emitter> Record *Compiler<Emitter>::getRecord(QualType Ty) {
4475	if (const auto *RecordTy = getRecordTy(Ty))
4476	return getRecord(RecordTy->getDecl());
4477	return nullptr;
4478	}
4479
4480	template <class Emitter>
4481	Record *Compiler<Emitter>::getRecord(const RecordDecl *RD) {
4482	return P.getOrCreateRecord(RD);
4483	}
4484
4485	template <class Emitter>
4486	const Function *Compiler<Emitter>::getFunction(const FunctionDecl *FD) {
4487	return Ctx.getOrCreateFunction(FuncDecl: FD);
4488	}
4489
4490	template <class Emitter>
4491	bool Compiler<Emitter>::visitExpr(const Expr *E, bool DestroyToplevelScope) {
4492	LocalScope<Emitter> RootScope(this);
4493
4494	// If we won't destroy the toplevel scope, check for memory leaks first.
4495	if (!DestroyToplevelScope) {
4496	if (!this->emitCheckAllocations(E))
4497	return false;
4498	}
4499
4500	auto maybeDestroyLocals = [&]() -> bool {
4501	if (DestroyToplevelScope)
4502	return RootScope.destroyLocals() && this->emitCheckAllocations(E);
4503	return this->emitCheckAllocations(E);
4504	};
4505
4506	// Void expressions.
4507	if (E->getType()->isVoidType()) {
4508	if (!visit(E))
4509	return false;
4510	return this->emitRetVoid(E) && maybeDestroyLocals();
4511	}
4512
4513	// Expressions with a primitive return type.
4514	if (std::optional<PrimType> T = classify(E)) {
4515	if (!visit(E))
4516	return false;
4517
4518	return this->emitRet(*T, E) && maybeDestroyLocals();
4519	}
4520
4521	// Expressions with a composite return type.
4522	// For us, that means everything we don't
4523	// have a PrimType for.
4524	if (std::optional<unsigned> LocalOffset = this->allocateLocal(E)) {
4525	InitLinkScope<Emitter> ILS(this, InitLink::Temp(Offset: *LocalOffset));
4526	if (!this->emitGetPtrLocal(*LocalOffset, E))
4527	return false;
4528
4529	if (!visitInitializer(E))
4530	return false;
4531
4532	if (!this->emitFinishInit(E))
4533	return false;
4534	// We are destroying the locals AFTER the Ret op.
4535	// The Ret op needs to copy the (alive) values, but the
4536	// destructors may still turn the entire expression invalid.
4537	return this->emitRetValue(E) && maybeDestroyLocals();
4538	}
4539
4540	return maybeDestroyLocals() && this->emitCheckAllocations(E) && false;
4541	}
4542
4543	template <class Emitter>
4544	VarCreationState Compiler<Emitter>::visitDecl(const VarDecl *VD,
4545	bool IsConstexprUnknown) {
4546
4547	auto R = this->visitVarDecl(VD, /*Toplevel=*/true, IsConstexprUnknown);
4548
4549	if (R.notCreated())
4550	return R;
4551
4552	if (R)
4553	return true;
4554
4555	if (!R && Context::shouldBeGloballyIndexed(VD)) {
4556	if (auto GlobalIndex = P.getGlobal(VD)) {
4557	Block *GlobalBlock = P.getGlobal(*GlobalIndex);
4558	GlobalInlineDescriptor &GD =
4559	*reinterpret_cast<GlobalInlineDescriptor *>(GlobalBlock->rawData());
4560
4561	GD.InitState = GlobalInitState::InitializerFailed;
4562	GlobalBlock->invokeDtor();
4563	}
4564	}
4565
4566	return R;
4567	}
4568
4569	/// Toplevel visitDeclAndReturn().
4570	/// We get here from evaluateAsInitializer().
4571	/// We need to evaluate the initializer and return its value.
4572	template <class Emitter>
4573	bool Compiler<Emitter>::visitDeclAndReturn(const VarDecl *VD,
4574	bool ConstantContext) {
4575	std::optional<PrimType> VarT = classify(VD->getType());
4576
4577	// We only create variables if we're evaluating in a constant context.
4578	// Otherwise, just evaluate the initializer and return it.
4579	if (!ConstantContext) {
4580	DeclScope<Emitter> LS(this, VD);
4581	if (!this->visit(VD->getAnyInitializer()))
4582	return false;
4583	return this->emitRet(VarT.value_or(u: PT_Ptr), VD) && LS.destroyLocals() &&
4584	this->emitCheckAllocations(VD);
4585	}
4586
4587	LocalScope<Emitter> VDScope(this, VD);
4588	if (!this->visitVarDecl(VD, /*Toplevel=*/true))
4589	return false;
4590
4591	if (Context::shouldBeGloballyIndexed(VD)) {
4592	auto GlobalIndex = P.getGlobal(VD);
4593	assert(GlobalIndex); // visitVarDecl() didn't return false.
4594	if (VarT) {
4595	if (!this->emitGetGlobalUnchecked(*VarT, *GlobalIndex, VD))
4596	return false;
4597	} else {
4598	if (!this->emitGetPtrGlobal(*GlobalIndex, VD))
4599	return false;
4600	}
4601	} else {
4602	auto Local = Locals.find(VD);
4603	assert(Local != Locals.end()); // Same here.
4604	if (VarT) {
4605	if (!this->emitGetLocal(*VarT, Local->second.Offset, VD))
4606	return false;
4607	} else {
4608	if (!this->emitGetPtrLocal(Local->second.Offset, VD))
4609	return false;
4610	}
4611	}
4612
4613	// Return the value.
4614	if (!this->emitRet(VarT.value_or(u: PT_Ptr), VD)) {
4615	// If the Ret above failed and this is a global variable, mark it as
4616	// uninitialized, even everything else succeeded.
4617	if (Context::shouldBeGloballyIndexed(VD)) {
4618	auto GlobalIndex = P.getGlobal(VD);
4619	assert(GlobalIndex);
4620	Block *GlobalBlock = P.getGlobal(*GlobalIndex);
4621	GlobalInlineDescriptor &GD =
4622	*reinterpret_cast<GlobalInlineDescriptor *>(GlobalBlock->rawData());
4623
4624	GD.InitState = GlobalInitState::InitializerFailed;
4625	GlobalBlock->invokeDtor();
4626	}
4627	return false;
4628	}
4629
4630	return VDScope.destroyLocals() && this->emitCheckAllocations(VD);
4631	}
4632
4633	template <class Emitter>
4634	VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
4635	bool Toplevel,
4636	bool IsConstexprUnknown) {
4637	// We don't know what to do with these, so just return false.
4638	if (VD->getType().isNull())
4639	return false;
4640
4641	// This case is EvalEmitter-only. If we won't create any instructions for the
4642	// initializer anyway, don't bother creating the variable in the first place.
4643	if (!this->isActive())
4644	return VarCreationState::NotCreated();
4645
4646	const Expr *Init = VD->getInit();
4647	std::optional<PrimType> VarT = classify(VD->getType());
4648
4649	if (Init && Init->isValueDependent())
4650	return false;
4651
4652	if (Context::shouldBeGloballyIndexed(VD)) {
4653	auto checkDecl = [&]() -> bool {
4654	bool NeedsOp = !Toplevel && VD->isLocalVarDecl() && VD->isStaticLocal();
4655	return !NeedsOp || this->emitCheckDecl(VD, VD);
4656	};
4657
4658	auto initGlobal = [&](unsigned GlobalIndex) -> bool {
4659	assert(Init);
4660
4661	if (VarT) {
4662	if (!this->visit(Init))
4663	return checkDecl() && false;
4664
4665	return checkDecl() && this->emitInitGlobal(*VarT, GlobalIndex, VD);
4666	}
4667
4668	if (!checkDecl())
4669	return false;
4670
4671	if (!this->emitGetPtrGlobal(GlobalIndex, Init))
4672	return false;
4673
4674	if (!visitInitializer(E: Init))
4675	return false;
4676
4677	if (!this->emitFinishInit(Init))
4678	return false;
4679
4680	return this->emitPopPtr(Init);
4681	};
4682
4683	DeclScope<Emitter> LocalScope(this, VD);
4684
4685	// We've already seen and initialized this global.
4686	if (std::optional<unsigned> GlobalIndex = P.getGlobal(VD)) {
4687	if (P.getPtrGlobal(Idx: *GlobalIndex).isInitialized())
4688	return checkDecl();
4689
4690	// The previous attempt at initialization might've been unsuccessful,
4691	// so let's try this one.
4692	return Init && checkDecl() && initGlobal(*GlobalIndex);
4693	}
4694
4695	std::optional<unsigned> GlobalIndex = P.createGlobal(VD, Init);
4696
4697	if (!GlobalIndex)
4698	return false;
4699
4700	return !Init || (checkDecl() && initGlobal(*GlobalIndex));
4701	} else {
4702	InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
4703
4704	if (VarT) {
4705	unsigned Offset = this->allocateLocalPrimitive(
4706	VD, *VarT, VD->getType().isConstQualified(), nullptr,
4707	ScopeKind::Block, IsConstexprUnknown);
4708	if (Init) {
4709	// If this is a toplevel declaration, create a scope for the
4710	// initializer.
4711	if (Toplevel) {
4712	LocalScope<Emitter> Scope(this);
4713	if (!this->visit(Init))
4714	return false;
4715	return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
4716	} else {
4717	if (!this->visit(Init))
4718	return false;
4719	return this->emitSetLocal(*VarT, Offset, VD);
4720	}
4721	}
4722	} else {
4723	if (std::optional<unsigned> Offset =
4724	this->allocateLocal(VD, VD->getType(), nullptr, ScopeKind::Block,
4725	IsConstexprUnknown)) {
4726	if (!Init)
4727	return true;
4728
4729	if (!this->emitGetPtrLocal(*Offset, Init))
4730	return false;
4731
4732	if (!visitInitializer(E: Init))
4733	return false;
4734
4735	if (!this->emitFinishInit(Init))
4736	return false;
4737
4738	return this->emitPopPtr(Init);
4739	}
4740	return false;
4741	}
4742	return true;
4743	}
4744
4745	return false;
4746	}
4747
4748	template <class Emitter>
4749	bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
4750	const Expr *E) {
4751	assert(!DiscardResult);
4752	if (Val.isInt())
4753	return this->emitConst(Val.getInt(), ValType, E);
4754	else if (Val.isFloat())
4755	return this->emitConstFloat(Val.getFloat(), E);
4756
4757	if (Val.isLValue()) {
4758	if (Val.isNullPointer())
4759	return this->emitNull(ValType, 0, nullptr, E);
4760	APValue::LValueBase Base = Val.getLValueBase();
4761	if (const Expr *BaseExpr = Base.dyn_cast<const Expr *>())
4762	return this->visit(BaseExpr);
4763	else if (const auto *VD = Base.dyn_cast<const ValueDecl *>()) {
4764	return this->visitDeclRef(VD, E);
4765	}
4766	} else if (Val.isMemberPointer()) {
4767	if (const ValueDecl *MemberDecl = Val.getMemberPointerDecl())
4768	return this->emitGetMemberPtr(MemberDecl, E);
4769	return this->emitNullMemberPtr(0, nullptr, E);
4770	}
4771
4772	return false;
4773	}
4774
4775	template <class Emitter>
4776	bool Compiler<Emitter>::visitAPValueInitializer(const APValue &Val,
4777	const Expr *E, QualType T) {
4778	if (Val.isStruct()) {
4779	const Record *R = this->getRecord(T);
4780	assert(R);
4781	for (unsigned I = 0, N = Val.getStructNumFields(); I != N; ++I) {
4782	const APValue &F = Val.getStructField(i: I);
4783	const Record::Field *RF = R->getField(I);
4784	QualType FieldType = RF->Decl->getType();
4785
4786	if (std::optional<PrimType> PT = classify(FieldType)) {
4787	if (!this->visitAPValue(F, *PT, E))
4788	return false;
4789	if (!this->emitInitField(*PT, RF->Offset, E))
4790	return false;
4791	} else {
4792	if (!this->emitGetPtrField(RF->Offset, E))
4793	return false;
4794	if (!this->visitAPValueInitializer(F, E, FieldType))
4795	return false;
4796	if (!this->emitPopPtr(E))
4797	return false;
4798	}
4799	}
4800	return true;
4801	} else if (Val.isUnion()) {
4802	const FieldDecl *UnionField = Val.getUnionField();
4803	const Record *R = this->getRecord(UnionField->getParent());
4804	assert(R);
4805	const APValue &F = Val.getUnionValue();
4806	const Record::Field *RF = R->getField(FD: UnionField);
4807	PrimType T = classifyPrim(RF->Decl->getType());
4808	if (!this->visitAPValue(F, T, E))
4809	return false;
4810	return this->emitInitField(T, RF->Offset, E);
4811	} else if (Val.isArray()) {
4812	const auto *ArrType = T->getAsArrayTypeUnsafe();
4813	QualType ElemType = ArrType->getElementType();
4814	for (unsigned A = 0, AN = Val.getArraySize(); A != AN; ++A) {
4815	const APValue &Elem = Val.getArrayInitializedElt(I: A);
4816	if (std::optional<PrimType> ElemT = classify(ElemType)) {
4817	if (!this->visitAPValue(Elem, *ElemT, E))
4818	return false;
4819	if (!this->emitInitElem(*ElemT, A, E))
4820	return false;
4821	} else {
4822	if (!this->emitConstUint32(A, E))
4823	return false;
4824	if (!this->emitArrayElemPtrUint32(E))
4825	return false;
4826	if (!this->visitAPValueInitializer(Elem, E, ElemType))
4827	return false;
4828	if (!this->emitPopPtr(E))
4829	return false;
4830	}
4831	}
4832	return true;
4833	}
4834	// TODO: Other types.
4835
4836	return false;
4837	}
4838
4839	template <class Emitter>
4840	bool Compiler<Emitter>::VisitBuiltinCallExpr(const CallExpr *E,
4841	unsigned BuiltinID) {
4842
4843	if (BuiltinID == Builtin::BI__builtin_constant_p) {
4844	// Void argument is always invalid and harder to handle later.
4845	if (E->getArg(Arg: 0)->getType()->isVoidType()) {
4846	if (DiscardResult)
4847	return true;
4848	return this->emitConst(0, E);
4849	}
4850
4851	if (!this->emitStartSpeculation(E))
4852	return false;
4853	LabelTy EndLabel = this->getLabel();
4854	if (!this->speculate(E, EndLabel))
4855	return false;
4856	this->fallthrough(EndLabel);
4857	if (!this->emitEndSpeculation(E))
4858	return false;
4859	if (DiscardResult)
4860	return this->emitPop(classifyPrim(E), E);
4861	return true;
4862	}
4863
4864	// For these, we're expected to ultimately return an APValue pointing
4865	// to the CallExpr. This is needed to get the correct codegen.
4866	if (BuiltinID == Builtin::BI__builtin___CFStringMakeConstantString ||
4867	BuiltinID == Builtin::BI__builtin___NSStringMakeConstantString ||
4868	BuiltinID == Builtin::BI__builtin_ptrauth_sign_constant ||
4869	BuiltinID == Builtin::BI__builtin_function_start) {
4870	if (DiscardResult)
4871	return true;
4872	return this->emitDummyPtr(E, E);
4873	}
4874
4875	QualType ReturnType = E->getType();
4876	std::optional<PrimType> ReturnT = classify(E);
4877
4878	// Non-primitive return type. Prepare storage.
4879	if (!Initializing && !ReturnT && !ReturnType->isVoidType()) {
4880	std::optional<unsigned> LocalIndex = allocateLocal(Src: E);
4881	if (!LocalIndex)
4882	return false;
4883	if (!this->emitGetPtrLocal(*LocalIndex, E))
4884	return false;
4885	}
4886
4887	if (!Context::isUnevaluatedBuiltin(ID: BuiltinID)) {
4888	// Put arguments on the stack.
4889	for (const auto *Arg : E->arguments()) {
4890	if (!this->visit(Arg))
4891	return false;
4892	}
4893	}
4894
4895	if (!this->emitCallBI(E, BuiltinID, E))
4896	return false;
4897
4898	if (DiscardResult && !ReturnType->isVoidType()) {
4899	assert(ReturnT);
4900	return this->emitPop(*ReturnT, E);
4901	}
4902
4903	return true;
4904	}
4905
4906	template <class Emitter>
4907	bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
4908	const FunctionDecl *FuncDecl = E->getDirectCallee();
4909
4910	if (FuncDecl) {
4911	if (unsigned BuiltinID = FuncDecl->getBuiltinID())
4912	return VisitBuiltinCallExpr(E, BuiltinID);
4913
4914	// Calls to replaceable operator new/operator delete.
4915	if (FuncDecl->isUsableAsGlobalAllocationFunctionInConstantEvaluation()) {
4916	if (FuncDecl->getDeclName().isAnyOperatorNew()) {
4917	return VisitBuiltinCallExpr(E, Builtin::BI__builtin_operator_new);
4918	} else {
4919	assert(FuncDecl->getDeclName().getCXXOverloadedOperator() == OO_Delete);
4920	return VisitBuiltinCallExpr(E, Builtin::BI__builtin_operator_delete);
4921	}
4922	}
4923
4924	// Explicit calls to trivial destructors
4925	if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: FuncDecl);
4926	DD && DD->isTrivial()) {
4927	const auto *MemberCall = cast<CXXMemberCallExpr>(Val: E);
4928	if (!this->visit(MemberCall->getImplicitObjectArgument()))
4929	return false;
4930	return this->emitCheckDestruction(E) && this->emitEndLifetime(E) &&
4931	this->emitPopPtr(E);
4932	}
4933	}
4934
4935	BlockScope<Emitter> CallScope(this, ScopeKind::Call);
4936
4937	QualType ReturnType = E->getCallReturnType(Ctx: Ctx.getASTContext());
4938	std::optional<PrimType> T = classify(ReturnType);
4939	bool HasRVO = !ReturnType->isVoidType() && !T;
4940
4941	if (HasRVO) {
4942	if (DiscardResult) {
4943	// If we need to discard the return value but the function returns its
4944	// value via an RVO pointer, we need to create one such pointer just
4945	// for this call.
4946	if (std::optional<unsigned> LocalIndex = allocateLocal(Src: E)) {
4947	if (!this->emitGetPtrLocal(*LocalIndex, E))
4948	return false;
4949	}
4950	} else {
4951	// We need the result. Prepare a pointer to return or
4952	// dup the current one.
4953	if (!Initializing) {
4954	if (std::optional<unsigned> LocalIndex = allocateLocal(Src: E)) {
4955	if (!this->emitGetPtrLocal(*LocalIndex, E))
4956	return false;
4957	}
4958	}
4959	if (!this->emitDupPtr(E))
4960	return false;
4961	}
4962	}
4963
4964	SmallVector<const Expr *, 8> Args(
4965	llvm::ArrayRef(E->getArgs(), E->getNumArgs()));
4966
4967	bool IsAssignmentOperatorCall = false;
4968	if (const auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: E);
4969	OCE && OCE->isAssignmentOp()) {
4970	// Just like with regular assignments, we need to special-case assignment
4971	// operators here and evaluate the RHS (the second arg) before the LHS (the
4972	// first arg). We fix this by using a Flip op later.
4973	assert(Args.size() == 2);
4974	IsAssignmentOperatorCall = true;
4975	std::reverse(first: Args.begin(), last: Args.end());
4976	}
4977	// Calling a static operator will still
4978	// pass the instance, but we don't need it.
4979	// Discard it here.
4980	if (isa<CXXOperatorCallExpr>(Val: E)) {
4981	if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: FuncDecl);
4982	MD && MD->isStatic()) {
4983	if (!this->discard(E->getArg(Arg: 0)))
4984	return false;
4985	// Drop first arg.
4986	Args.erase(CI: Args.begin());
4987	}
4988	}
4989
4990	std::optional<unsigned> CalleeOffset;
4991	// Add the (optional, implicit) This pointer.
4992	if (const auto *MC = dyn_cast<CXXMemberCallExpr>(Val: E)) {
4993	if (!FuncDecl && classifyPrim(E->getCallee()) == PT_MemberPtr) {
4994	// If we end up creating a CallPtr op for this, we need the base of the
4995	// member pointer as the instance pointer, and later extract the function
4996	// decl as the function pointer.
4997	const Expr *Callee = E->getCallee();
4998	CalleeOffset =
4999	this->allocateLocalPrimitive(Callee, PT_MemberPtr, /*IsConst=*/true);
5000	if (!this->visit(Callee))
5001	return false;
5002	if (!this->emitSetLocal(PT_MemberPtr, *CalleeOffset, E))
5003	return false;
5004	if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E))
5005	return false;
5006	if (!this->emitGetMemberPtrBase(E))
5007	return false;
5008	} else if (!this->visit(MC->getImplicitObjectArgument())) {
5009	return false;
5010	}
5011	} else if (const auto *PD =
5012	dyn_cast<CXXPseudoDestructorExpr>(Val: E->getCallee())) {
5013	if (!this->emitCheckPseudoDtor(E))
5014	return false;
5015	const Expr *Base = PD->getBase();
5016	if (!Base->isGLValue())
5017	return this->discard(Base);
5018	if (!this->visit(Base))
5019	return false;
5020	return this->emitEndLifetimePop(E);
5021	} else if (!FuncDecl) {
5022	const Expr *Callee = E->getCallee();
5023	CalleeOffset =
5024	this->allocateLocalPrimitive(Callee, PT_Ptr, /*IsConst=*/true);
5025	if (!this->visit(Callee))
5026	return false;
5027	if (!this->emitSetLocal(PT_Ptr, *CalleeOffset, E))
5028	return false;
5029	}
5030
5031	if (!this->visitCallArgs(Args, FuncDecl))
5032	return false;
5033
5034	// Undo the argument reversal we did earlier.
5035	if (IsAssignmentOperatorCall) {
5036	assert(Args.size() == 2);
5037	PrimType Arg1T = classify(Args[0]).value_or(PT_Ptr);
5038	PrimType Arg2T = classify(Args[1]).value_or(PT_Ptr);
5039	if (!this->emitFlip(Arg2T, Arg1T, E))
5040	return false;
5041	}
5042
5043	if (FuncDecl) {
5044	const Function *Func = getFunction(FD: FuncDecl);
5045	if (!Func)
5046	return false;
5047	assert(HasRVO == Func->hasRVO());
5048
5049	bool HasQualifier = false;
5050	if (const auto *ME = dyn_cast<MemberExpr>(Val: E->getCallee()))
5051	HasQualifier = ME->hasQualifier();
5052
5053	bool IsVirtual = false;
5054	if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FuncDecl))
5055	IsVirtual = MD->isVirtual();
5056
5057	// In any case call the function. The return value will end up on the stack
5058	// and if the function has RVO, we already have the pointer on the stack to
5059	// write the result into.
5060	if (IsVirtual && !HasQualifier) {
5061	uint32_t VarArgSize = 0;
5062	unsigned NumParams =
5063	Func->getNumWrittenParams() + isa<CXXOperatorCallExpr>(Val: E);
5064	for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I)
5065	VarArgSize += align(primSize(classify(E->getArg(Arg: I)).value_or(PT_Ptr)));
5066
5067	if (!this->emitCallVirt(Func, VarArgSize, E))
5068	return false;
5069	} else if (Func->isVariadic()) {
5070	uint32_t VarArgSize = 0;
5071	unsigned NumParams =
5072	Func->getNumWrittenParams() + isa<CXXOperatorCallExpr>(Val: E);
5073	for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I)
5074	VarArgSize += align(primSize(classify(E->getArg(Arg: I)).value_or(PT_Ptr)));
5075	if (!this->emitCallVar(Func, VarArgSize, E))
5076	return false;
5077	} else {
5078	if (!this->emitCall(Func, 0, E))
5079	return false;
5080	}
5081	} else {
5082	// Indirect call. Visit the callee, which will leave a FunctionPointer on
5083	// the stack. Cleanup of the returned value if necessary will be done after
5084	// the function call completed.
5085
5086	// Sum the size of all args from the call expr.
5087	uint32_t ArgSize = 0;
5088	for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
5089	ArgSize += align(primSize(classify(E->getArg(Arg: I)).value_or(PT_Ptr)));
5090
5091	// Get the callee, either from a member pointer or function pointer saved in
5092	// CalleeOffset.
5093	if (isa<CXXMemberCallExpr>(Val: E) && CalleeOffset) {
5094	if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E))
5095	return false;
5096	if (!this->emitGetMemberPtrDecl(E))
5097	return false;
5098	} else {
5099	if (!this->emitGetLocal(PT_Ptr, *CalleeOffset, E))
5100	return false;
5101	}
5102	if (!this->emitCallPtr(ArgSize, E, E))
5103	return false;
5104	}
5105
5106	// Cleanup for discarded return values.
5107	if (DiscardResult && !ReturnType->isVoidType() && T)
5108	return this->emitPop(*T, E) && CallScope.destroyLocals();
5109
5110	return CallScope.destroyLocals();
5111	}
5112
5113	template <class Emitter>
5114	bool Compiler<Emitter>::VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *E) {
5115	SourceLocScope<Emitter> SLS(this, E);
5116
5117	return this->delegate(E->getExpr());
5118	}
5119
5120	template <class Emitter>
5121	bool Compiler<Emitter>::VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *E) {
5122	SourceLocScope<Emitter> SLS(this, E);
5123
5124	return this->delegate(E->getExpr());
5125	}
5126
5127	template <class Emitter>
5128	bool Compiler<Emitter>::VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *E) {
5129	if (DiscardResult)
5130	return true;
5131
5132	return this->emitConstBool(E->getValue(), E);
5133	}
5134
5135	template <class Emitter>
5136	bool Compiler<Emitter>::VisitCXXNullPtrLiteralExpr(
5137	const CXXNullPtrLiteralExpr *E) {
5138	if (DiscardResult)
5139	return true;
5140
5141	uint64_t Val = Ctx.getASTContext().getTargetNullPointerValue(QT: E->getType());
5142	return this->emitNullPtr(Val, nullptr, E);
5143	}
5144
5145	template <class Emitter>
5146	bool Compiler<Emitter>::VisitGNUNullExpr(const GNUNullExpr *E) {
5147	if (DiscardResult)
5148	return true;
5149
5150	assert(E->getType()->isIntegerType());
5151
5152	PrimType T = classifyPrim(E->getType());
5153	return this->emitZero(T, E);
5154	}
5155
5156	template <class Emitter>
5157	bool Compiler<Emitter>::VisitCXXThisExpr(const CXXThisExpr *E) {
5158	if (DiscardResult)
5159	return true;
5160
5161	if (this->LambdaThisCapture.Offset > 0) {
5162	if (this->LambdaThisCapture.IsPtr)
5163	return this->emitGetThisFieldPtr(this->LambdaThisCapture.Offset, E);
5164	return this->emitGetPtrThisField(this->LambdaThisCapture.Offset, E);
5165	}
5166
5167	// In some circumstances, the 'this' pointer does not actually refer to the
5168	// instance pointer of the current function frame, but e.g. to the declaration
5169	// currently being initialized. Here we emit the necessary instruction(s) for
5170	// this scenario.
5171	if (!InitStackActive)
5172	return this->emitThis(E);
5173
5174	if (!InitStack.empty()) {
5175	// If our init stack is, for example:
5176	// 0 Stack: 3 (decl)
5177	// 1 Stack: 6 (init list)
5178	// 2 Stack: 1 (field)
5179	// 3 Stack: 6 (init list)
5180	// 4 Stack: 1 (field)
5181	//
5182	// We want to find the LAST element in it that's an init list,
5183	// which is marked with the K_InitList marker. The index right
5184	// before that points to an init list. We need to find the
5185	// elements before the K_InitList element that point to a base
5186	// (e.g. a decl or This), optionally followed by field, elem, etc.
5187	// In the example above, we want to emit elements [0..2].
5188	unsigned StartIndex = 0;
5189	unsigned EndIndex = 0;
5190	// Find the init list.
5191	for (StartIndex = InitStack.size() - 1; StartIndex > 0; --StartIndex) {
5192	if (InitStack[StartIndex].Kind == InitLink::K_InitList ||
5193	InitStack[StartIndex].Kind == InitLink::K_This) {
5194	EndIndex = StartIndex;
5195	--StartIndex;
5196	break;
5197	}
5198	}
5199
5200	// Walk backwards to find the base.
5201	for (; StartIndex > 0; --StartIndex) {
5202	if (InitStack[StartIndex].Kind == InitLink::K_InitList)
5203	continue;
5204
5205	if (InitStack[StartIndex].Kind != InitLink::K_Field &&
5206	InitStack[StartIndex].Kind != InitLink::K_Elem)
5207	break;
5208	}
5209
5210	// Emit the instructions.
5211	for (unsigned I = StartIndex; I != EndIndex; ++I) {
5212	if (InitStack[I].Kind == InitLink::K_InitList)
5213	continue;
5214	if (!InitStack[I].template emit<Emitter>(this, E))
5215	return false;
5216	}
5217	return true;
5218	}
5219	return this->emitThis(E);
5220	}
5221
5222	template <class Emitter> bool Compiler<Emitter>::visitStmt(const Stmt *S) {
5223	switch (S->getStmtClass()) {
5224	case Stmt::CompoundStmtClass:
5225	return visitCompoundStmt(S: cast<CompoundStmt>(Val: S));
5226	case Stmt::DeclStmtClass:
5227	return visitDeclStmt(DS: cast<DeclStmt>(Val: S), /*EvaluateConditionDecl=*/true);
5228	case Stmt::ReturnStmtClass:
5229	return visitReturnStmt(RS: cast<ReturnStmt>(Val: S));
5230	case Stmt::IfStmtClass:
5231	return visitIfStmt(IS: cast<IfStmt>(Val: S));
5232	case Stmt::WhileStmtClass:
5233	return visitWhileStmt(S: cast<WhileStmt>(Val: S));
5234	case Stmt::DoStmtClass:
5235	return visitDoStmt(S: cast<DoStmt>(Val: S));
5236	case Stmt::ForStmtClass:
5237	return visitForStmt(S: cast<ForStmt>(Val: S));
5238	case Stmt::CXXForRangeStmtClass:
5239	return visitCXXForRangeStmt(S: cast<CXXForRangeStmt>(Val: S));
5240	case Stmt::BreakStmtClass:
5241	return visitBreakStmt(S: cast<BreakStmt>(Val: S));
5242	case Stmt::ContinueStmtClass:
5243	return visitContinueStmt(S: cast<ContinueStmt>(Val: S));
5244	case Stmt::SwitchStmtClass:
5245	return visitSwitchStmt(S: cast<SwitchStmt>(Val: S));
5246	case Stmt::CaseStmtClass:
5247	return visitCaseStmt(S: cast<CaseStmt>(Val: S));
5248	case Stmt::DefaultStmtClass:
5249	return visitDefaultStmt(S: cast<DefaultStmt>(Val: S));
5250	case Stmt::AttributedStmtClass:
5251	return visitAttributedStmt(S: cast<AttributedStmt>(Val: S));
5252	case Stmt::CXXTryStmtClass:
5253	return visitCXXTryStmt(S: cast<CXXTryStmt>(Val: S));
5254	case Stmt::NullStmtClass:
5255	return true;
5256	// Always invalid statements.
5257	case Stmt::GCCAsmStmtClass:
5258	case Stmt::MSAsmStmtClass:
5259	case Stmt::GotoStmtClass:
5260	return this->emitInvalid(S);
5261	case Stmt::LabelStmtClass:
5262	return this->visitStmt(cast<LabelStmt>(Val: S)->getSubStmt());
5263	default: {
5264	if (const auto *E = dyn_cast<Expr>(Val: S))
5265	return this->discard(E);
5266	return false;
5267	}
5268	}
5269	}
5270
5271	template <class Emitter>
5272	bool Compiler<Emitter>::visitCompoundStmt(const CompoundStmt *S) {
5273	BlockScope<Emitter> Scope(this);
5274	for (const auto *InnerStmt : S->body())
5275	if (!visitStmt(S: InnerStmt))
5276	return false;
5277	return Scope.destroyLocals();
5278	}
5279
5280	template <class Emitter>
5281	bool Compiler<Emitter>::maybeEmitDeferredVarInit(const VarDecl *VD) {
5282	if (auto *DD = dyn_cast_if_present<DecompositionDecl>(Val: VD)) {
5283	for (auto *BD : DD->bindings())
5284	if (auto *KD = BD->getHoldingVar(); KD && !this->visitVarDecl(KD))
5285	return false;
5286	}
5287	return true;
5288	}
5289
5290	template <class Emitter>
5291	bool Compiler<Emitter>::visitDeclStmt(const DeclStmt *DS,
5292	bool EvaluateConditionDecl) {
5293	for (const auto *D : DS->decls()) {
5294	if (isa<StaticAssertDecl, TagDecl, TypedefNameDecl, BaseUsingDecl,
5295	FunctionDecl, NamespaceAliasDecl, UsingDirectiveDecl>(Val: D))
5296	continue;
5297
5298	const auto *VD = dyn_cast<VarDecl>(Val: D);
5299	if (!VD)
5300	return false;
5301	if (!this->visitVarDecl(VD))
5302	return false;
5303
5304	// Register decomposition decl holding vars.
5305	if (EvaluateConditionDecl && !this->maybeEmitDeferredVarInit(VD))
5306	return false;
5307	}
5308
5309	return true;
5310	}
5311
5312	template <class Emitter>
5313	bool Compiler<Emitter>::visitReturnStmt(const ReturnStmt *RS) {
5314	if (this->InStmtExpr)
5315	return this->emitUnsupported(RS);
5316
5317	if (const Expr *RE = RS->getRetValue()) {
5318	LocalScope<Emitter> RetScope(this);
5319	if (ReturnType) {
5320	// Primitive types are simply returned.
5321	if (!this->visit(RE))
5322	return false;
5323	this->emitCleanup();
5324	return this->emitRet(*ReturnType, RS);
5325	} else if (RE->getType()->isVoidType()) {
5326	if (!this->visit(RE))
5327	return false;
5328	} else {
5329	InitLinkScope<Emitter> ILS(this, InitLink::RVO());
5330	// RVO - construct the value in the return location.
5331	if (!this->emitRVOPtr(RE))
5332	return false;
5333	if (!this->visitInitializer(RE))
5334	return false;
5335	if (!this->emitPopPtr(RE))
5336	return false;
5337
5338	this->emitCleanup();
5339	return this->emitRetVoid(RS);
5340	}
5341	}
5342
5343	// Void return.
5344	this->emitCleanup();
5345	return this->emitRetVoid(RS);
5346	}
5347
5348	template <class Emitter> bool Compiler<Emitter>::visitIfStmt(const IfStmt *IS) {
5349	auto visitChildStmt = [&](const Stmt *S) -> bool {
5350	LocalScope<Emitter> SScope(this);
5351	if (!visitStmt(S))
5352	return false;
5353	return SScope.destroyLocals();
5354	};
5355	if (auto *CondInit = IS->getInit())
5356	if (!visitStmt(S: CondInit))
5357	return false;
5358
5359	if (const DeclStmt *CondDecl = IS->getConditionVariableDeclStmt())
5360	if (!visitDeclStmt(DS: CondDecl))
5361	return false;
5362
5363	// Save ourselves compiling some code and the jumps, etc. if the condition is
5364	// stataically known to be either true or false. We could look at more cases
5365	// here, but I think all the ones that actually happen are using a
5366	// ConstantExpr.
5367	if (std::optional<bool> BoolValue = getBoolValue(E: IS->getCond())) {
5368	if (*BoolValue)
5369	return visitChildStmt(IS->getThen());
5370	else if (const Stmt *Else = IS->getElse())
5371	return visitChildStmt(Else);
5372	return true;
5373	}
5374
5375	// Otherwise, compile the condition.
5376	if (IS->isNonNegatedConsteval()) {
5377	if (!this->emitIsConstantContext(IS))
5378	return false;
5379	} else if (IS->isNegatedConsteval()) {
5380	if (!this->emitIsConstantContext(IS))
5381	return false;
5382	if (!this->emitInv(IS))
5383	return false;
5384	} else {
5385	if (!this->visitBool(IS->getCond()))
5386	return false;
5387	}
5388
5389	if (!this->maybeEmitDeferredVarInit(IS->getConditionVariable()))
5390	return false;
5391
5392	if (const Stmt *Else = IS->getElse()) {
5393	LabelTy LabelElse = this->getLabel();
5394	LabelTy LabelEnd = this->getLabel();
5395	if (!this->jumpFalse(LabelElse))
5396	return false;
5397	if (!visitChildStmt(IS->getThen()))
5398	return false;
5399	if (!this->jump(LabelEnd))
5400	return false;
5401	this->emitLabel(LabelElse);
5402	if (!visitChildStmt(Else))
5403	return false;
5404	this->emitLabel(LabelEnd);
5405	} else {
5406	LabelTy LabelEnd = this->getLabel();
5407	if (!this->jumpFalse(LabelEnd))
5408	return false;
5409	if (!visitChildStmt(IS->getThen()))
5410	return false;
5411	this->emitLabel(LabelEnd);
5412	}
5413
5414	return true;
5415	}
5416
5417	template <class Emitter>
5418	bool Compiler<Emitter>::visitWhileStmt(const WhileStmt *S) {
5419	const Expr *Cond = S->getCond();
5420	const Stmt *Body = S->getBody();
5421
5422	LabelTy CondLabel = this->getLabel(); // Label before the condition.
5423	LabelTy EndLabel = this->getLabel(); // Label after the loop.
5424	LoopScope<Emitter> LS(this, EndLabel, CondLabel);
5425
5426	this->fallthrough(CondLabel);
5427	this->emitLabel(CondLabel);
5428
5429	{
5430	LocalScope<Emitter> CondScope(this);
5431	if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt())
5432	if (!visitDeclStmt(DS: CondDecl))
5433	return false;
5434
5435	if (!this->visitBool(Cond))
5436	return false;
5437
5438	if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
5439	return false;
5440
5441	if (!this->jumpFalse(EndLabel))
5442	return false;
5443
5444	if (!this->visitStmt(Body))
5445	return false;
5446
5447	if (!CondScope.destroyLocals())
5448	return false;
5449	}
5450	if (!this->jump(CondLabel))
5451	return false;
5452	this->fallthrough(EndLabel);
5453	this->emitLabel(EndLabel);
5454
5455	return true;
5456	}
5457
5458	template <class Emitter> bool Compiler<Emitter>::visitDoStmt(const DoStmt *S) {
5459	const Expr *Cond = S->getCond();
5460	const Stmt *Body = S->getBody();
5461
5462	LabelTy StartLabel = this->getLabel();
5463	LabelTy EndLabel = this->getLabel();
5464	LabelTy CondLabel = this->getLabel();
5465	LoopScope<Emitter> LS(this, EndLabel, CondLabel);
5466
5467	this->fallthrough(StartLabel);
5468	this->emitLabel(StartLabel);
5469
5470	{
5471	LocalScope<Emitter> CondScope(this);
5472	if (!this->visitStmt(Body))
5473	return false;
5474	this->fallthrough(CondLabel);
5475	this->emitLabel(CondLabel);
5476	if (!this->visitBool(Cond))
5477	return false;
5478
5479	if (!CondScope.destroyLocals())
5480	return false;
5481	}
5482	if (!this->jumpTrue(StartLabel))
5483	return false;
5484
5485	this->fallthrough(EndLabel);
5486	this->emitLabel(EndLabel);
5487	return true;
5488	}
5489
5490	template <class Emitter>
5491	bool Compiler<Emitter>::visitForStmt(const ForStmt *S) {
5492	// for (Init; Cond; Inc) { Body }
5493	const Stmt *Init = S->getInit();
5494	const Expr *Cond = S->getCond();
5495	const Expr *Inc = S->getInc();
5496	const Stmt *Body = S->getBody();
5497
5498	LabelTy EndLabel = this->getLabel();
5499	LabelTy CondLabel = this->getLabel();
5500	LabelTy IncLabel = this->getLabel();
5501	LoopScope<Emitter> LS(this, EndLabel, IncLabel);
5502
5503	if (Init && !this->visitStmt(Init))
5504	return false;
5505
5506	this->fallthrough(CondLabel);
5507	this->emitLabel(CondLabel);
5508
5509	// Start of loop body.
5510	LocalScope<Emitter> CondScope(this);
5511	if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt())
5512	if (!visitDeclStmt(DS: CondDecl))
5513	return false;
5514
5515	if (Cond) {
5516	if (!this->visitBool(Cond))
5517	return false;
5518	if (!this->jumpFalse(EndLabel))
5519	return false;
5520	}
5521	if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
5522	return false;
5523
5524	if (Body && !this->visitStmt(Body))
5525	return false;
5526
5527	this->fallthrough(IncLabel);
5528	this->emitLabel(IncLabel);
5529	if (Inc && !this->discard(Inc))
5530	return false;
5531
5532	if (!CondScope.destroyLocals())
5533	return false;
5534	if (!this->jump(CondLabel))
5535	return false;
5536	// End of loop body.
5537
5538	this->emitLabel(EndLabel);
5539	// If we jumped out of the loop above, we still need to clean up the condition
5540	// scope.
5541	return CondScope.destroyLocals();
5542	}
5543
5544	template <class Emitter>
5545	bool Compiler<Emitter>::visitCXXForRangeStmt(const CXXForRangeStmt *S) {
5546	const Stmt *Init = S->getInit();
5547	const Expr *Cond = S->getCond();
5548	const Expr *Inc = S->getInc();
5549	const Stmt *Body = S->getBody();
5550	const Stmt *BeginStmt = S->getBeginStmt();
5551	const Stmt *RangeStmt = S->getRangeStmt();
5552	const Stmt *EndStmt = S->getEndStmt();
5553	const VarDecl *LoopVar = S->getLoopVariable();
5554
5555	LabelTy EndLabel = this->getLabel();
5556	LabelTy CondLabel = this->getLabel();
5557	LabelTy IncLabel = this->getLabel();
5558	LoopScope<Emitter> LS(this, EndLabel, IncLabel);
5559
5560	// Emit declarations needed in the loop.
5561	if (Init && !this->visitStmt(Init))
5562	return false;
5563	if (!this->visitStmt(RangeStmt))
5564	return false;
5565	if (!this->visitStmt(BeginStmt))
5566	return false;
5567	if (!this->visitStmt(EndStmt))
5568	return false;
5569
5570	// Now the condition as well as the loop variable assignment.
5571	this->fallthrough(CondLabel);
5572	this->emitLabel(CondLabel);
5573	if (!this->visitBool(Cond))
5574	return false;
5575	if (!this->jumpFalse(EndLabel))
5576	return false;
5577
5578	if (!this->visitVarDecl(LoopVar))
5579	return false;
5580
5581	// Body.
5582	{
5583	if (!this->visitStmt(Body))
5584	return false;
5585
5586	this->fallthrough(IncLabel);
5587	this->emitLabel(IncLabel);
5588	if (!this->discard(Inc))
5589	return false;
5590	}
5591
5592	if (!this->jump(CondLabel))
5593	return false;
5594
5595	this->fallthrough(EndLabel);
5596	this->emitLabel(EndLabel);
5597	return true;
5598	}
5599
5600	template <class Emitter>
5601	bool Compiler<Emitter>::visitBreakStmt(const BreakStmt *S) {
5602	if (!BreakLabel)
5603	return false;
5604
5605	for (VariableScope<Emitter> *C = VarScope; C != BreakVarScope;
5606	C = C->getParent())
5607	C->emitDestruction();
5608	return this->jump(*BreakLabel);
5609	}
5610
5611	template <class Emitter>
5612	bool Compiler<Emitter>::visitContinueStmt(const ContinueStmt *S) {
5613	if (!ContinueLabel)
5614	return false;
5615
5616	for (VariableScope<Emitter> *C = VarScope;
5617	C && C->getParent() != ContinueVarScope; C = C->getParent())
5618	C->emitDestruction();
5619	return this->jump(*ContinueLabel);
5620	}
5621
5622	template <class Emitter>
5623	bool Compiler<Emitter>::visitSwitchStmt(const SwitchStmt *S) {
5624	const Expr *Cond = S->getCond();
5625	PrimType CondT = this->classifyPrim(Cond->getType());
5626	LocalScope<Emitter> LS(this);
5627
5628	LabelTy EndLabel = this->getLabel();
5629	OptLabelTy DefaultLabel = std::nullopt;
5630	unsigned CondVar =
5631	this->allocateLocalPrimitive(Cond, CondT, /*IsConst=*/true);
5632
5633	if (const auto *CondInit = S->getInit())
5634	if (!visitStmt(S: CondInit))
5635	return false;
5636
5637	if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt())
5638	if (!visitDeclStmt(DS: CondDecl))
5639	return false;
5640
5641	// Initialize condition variable.
5642	if (!this->visit(Cond))
5643	return false;
5644	if (!this->emitSetLocal(CondT, CondVar, S))
5645	return false;
5646
5647	if (!this->maybeEmitDeferredVarInit(S->getConditionVariable()))
5648	return false;
5649
5650	CaseMap CaseLabels;
5651	// Create labels and comparison ops for all case statements.
5652	for (const SwitchCase *SC = S->getSwitchCaseList(); SC;
5653	SC = SC->getNextSwitchCase()) {
5654	if (const auto *CS = dyn_cast<CaseStmt>(Val: SC)) {
5655	// FIXME: Implement ranges.
5656	if (CS->caseStmtIsGNURange())
5657	return false;
5658	CaseLabels[SC] = this->getLabel();
5659
5660	const Expr *Value = CS->getLHS();
5661	PrimType ValueT = this->classifyPrim(Value->getType());
5662
5663	// Compare the case statement's value to the switch condition.
5664	if (!this->emitGetLocal(CondT, CondVar, CS))
5665	return false;
5666	if (!this->visit(Value))
5667	return false;
5668
5669	// Compare and jump to the case label.
5670	if (!this->emitEQ(ValueT, S))
5671	return false;
5672	if (!this->jumpTrue(CaseLabels[CS]))
5673	return false;
5674	} else {
5675	assert(!DefaultLabel);
5676	DefaultLabel = this->getLabel();
5677	}
5678	}
5679
5680	// If none of the conditions above were true, fall through to the default
5681	// statement or jump after the switch statement.
5682	if (DefaultLabel) {
5683	if (!this->jump(*DefaultLabel))
5684	return false;
5685	} else {
5686	if (!this->jump(EndLabel))
5687	return false;
5688	}
5689
5690	SwitchScope<Emitter> SS(this, std::move(CaseLabels), EndLabel, DefaultLabel);
5691	if (!this->visitStmt(S->getBody()))
5692	return false;
5693	this->emitLabel(EndLabel);
5694
5695	return LS.destroyLocals();
5696	}
5697
5698	template <class Emitter>
5699	bool Compiler<Emitter>::visitCaseStmt(const CaseStmt *S) {
5700	this->emitLabel(CaseLabels[S]);
5701	return this->visitStmt(S->getSubStmt());
5702	}
5703
5704	template <class Emitter>
5705	bool Compiler<Emitter>::visitDefaultStmt(const DefaultStmt *S) {
5706	this->emitLabel(*DefaultLabel);
5707	return this->visitStmt(S->getSubStmt());
5708	}
5709
5710	template <class Emitter>
5711	bool Compiler<Emitter>::visitAttributedStmt(const AttributedStmt *S) {
5712	if (this->Ctx.getLangOpts().CXXAssumptions &&
5713	!this->Ctx.getLangOpts().MSVCCompat) {
5714	for (const Attr *A : S->getAttrs()) {
5715	auto *AA = dyn_cast<CXXAssumeAttr>(A);
5716	if (!AA)
5717	continue;
5718
5719	assert(isa<NullStmt>(S->getSubStmt()));
5720
5721	const Expr *Assumption = AA->getAssumption();
5722	if (Assumption->isValueDependent())
5723	return false;
5724
5725	if (Assumption->HasSideEffects(Ctx: this->Ctx.getASTContext()))
5726	continue;
5727
5728	// Evaluate assumption.
5729	if (!this->visitBool(Assumption))
5730	return false;
5731
5732	if (!this->emitAssume(Assumption))
5733	return false;
5734	}
5735	}
5736
5737	// Ignore other attributes.
5738	return this->visitStmt(S->getSubStmt());
5739	}
5740
5741	template <class Emitter>
5742	bool Compiler<Emitter>::visitCXXTryStmt(const CXXTryStmt *S) {
5743	// Ignore all handlers.
5744	return this->visitStmt(S->getTryBlock());
5745	}
5746
5747	template <class Emitter>
5748	bool Compiler<Emitter>::emitLambdaStaticInvokerBody(const CXXMethodDecl *MD) {
5749	assert(MD->isLambdaStaticInvoker());
5750	assert(MD->hasBody());
5751	assert(cast<CompoundStmt>(MD->getBody())->body_empty());
5752
5753	const CXXRecordDecl *ClosureClass = MD->getParent();
5754	const CXXMethodDecl *LambdaCallOp = ClosureClass->getLambdaCallOperator();
5755	assert(ClosureClass->captures_begin() == ClosureClass->captures_end());
5756	const Function *Func = this->getFunction(LambdaCallOp);
5757	if (!Func)
5758	return false;
5759	assert(Func->hasThisPointer());
5760	assert(Func->getNumParams() == (MD->getNumParams() + 1 + Func->hasRVO()));
5761
5762	if (Func->hasRVO()) {
5763	if (!this->emitRVOPtr(MD))
5764	return false;
5765	}
5766
5767	// The lambda call operator needs an instance pointer, but we don't have
5768	// one here, and we don't need one either because the lambda cannot have
5769	// any captures, as verified above. Emit a null pointer. This is then
5770	// special-cased when interpreting to not emit any misleading diagnostics.
5771	if (!this->emitNullPtr(0, nullptr, MD))
5772	return false;
5773
5774	// Forward all arguments from the static invoker to the lambda call operator.
5775	for (const ParmVarDecl *PVD : MD->parameters()) {
5776	auto It = this->Params.find(PVD);
5777	assert(It != this->Params.end());
5778
5779	// We do the lvalue-to-rvalue conversion manually here, so no need
5780	// to care about references.
5781	PrimType ParamType = this->classify(PVD->getType()).value_or(PT_Ptr);
5782	if (!this->emitGetParam(ParamType, It->second.Offset, MD))
5783	return false;
5784	}
5785
5786	if (!this->emitCall(Func, 0, LambdaCallOp))
5787	return false;
5788
5789	this->emitCleanup();
5790	if (ReturnType)
5791	return this->emitRet(*ReturnType, MD);
5792
5793	// Nothing to do, since we emitted the RVO pointer above.
5794	return this->emitRetVoid(MD);
5795	}
5796
5797	template <class Emitter>
5798	bool Compiler<Emitter>::checkLiteralType(const Expr *E) {
5799	if (Ctx.getLangOpts().CPlusPlus23)
5800	return true;
5801
5802	if (!E->isPRValue() || E->getType()->isLiteralType(Ctx: Ctx.getASTContext()))
5803	return true;
5804
5805	return this->emitCheckLiteralType(E->getType().getTypePtr(), E);
5806	}
5807
5808	template <class Emitter>
5809	bool Compiler<Emitter>::compileConstructor(const CXXConstructorDecl *Ctor) {
5810	assert(!ReturnType);
5811
5812	auto emitFieldInitializer = [&](const Record::Field *F, unsigned FieldOffset,
5813	const Expr *InitExpr) -> bool {
5814	// We don't know what to do with these, so just return false.
5815	if (InitExpr->getType().isNull())
5816	return false;
5817
5818	if (std::optional<PrimType> T = this->classify(InitExpr)) {
5819	if (!this->visit(InitExpr))
5820	return false;
5821
5822	if (F->isBitField())
5823	return this->emitInitThisBitField(*T, F, FieldOffset, InitExpr);
5824	return this->emitInitThisField(*T, FieldOffset, InitExpr);
5825	}
5826	// Non-primitive case. Get a pointer to the field-to-initialize
5827	// on the stack and call visitInitialzer() for it.
5828	InitLinkScope<Emitter> FieldScope(this, InitLink::Field(Offset: F->Offset));
5829	if (!this->emitGetPtrThisField(FieldOffset, InitExpr))
5830	return false;
5831
5832	if (!this->visitInitializer(InitExpr))
5833	return false;
5834
5835	return this->emitFinishInitPop(InitExpr);
5836	};
5837
5838	const RecordDecl *RD = Ctor->getParent();
5839	const Record *R = this->getRecord(RD);
5840	if (!R)
5841	return false;
5842
5843	if (R->isUnion() && Ctor->isCopyOrMoveConstructor()) {
5844	// union copy and move ctors are special.
5845	assert(cast<CompoundStmt>(Ctor->getBody())->body_empty());
5846	if (!this->emitThis(Ctor))
5847	return false;
5848
5849	auto PVD = Ctor->getParamDecl(0);
5850	ParamOffset PO = this->Params[PVD]; // Must exist.
5851
5852	if (!this->emitGetParam(PT_Ptr, PO.Offset, Ctor))
5853	return false;
5854
5855	return this->emitMemcpy(Ctor) && this->emitPopPtr(Ctor) &&
5856	this->emitRetVoid(Ctor);
5857	}
5858
5859	InitLinkScope<Emitter> InitScope(this, InitLink::This());
5860	for (const auto *Init : Ctor->inits()) {
5861	// Scope needed for the initializers.
5862	BlockScope<Emitter> Scope(this);
5863
5864	const Expr *InitExpr = Init->getInit();
5865	if (const FieldDecl *Member = Init->getMember()) {
5866	const Record::Field *F = R->getField(FD: Member);
5867
5868	if (!emitFieldInitializer(F, F->Offset, InitExpr))
5869	return false;
5870	} else if (const Type *Base = Init->getBaseClass()) {
5871	const auto *BaseDecl = Base->getAsCXXRecordDecl();
5872	assert(BaseDecl);
5873
5874	if (Init->isBaseVirtual()) {
5875	assert(R->getVirtualBase(BaseDecl));
5876	if (!this->emitGetPtrThisVirtBase(BaseDecl, InitExpr))
5877	return false;
5878
5879	} else {
5880	// Base class initializer.
5881	// Get This Base and call initializer on it.
5882	const Record::Base *B = R->getBase(BaseDecl);
5883	assert(B);
5884	if (!this->emitGetPtrThisBase(B->Offset, InitExpr))
5885	return false;
5886	}
5887
5888	if (!this->visitInitializer(InitExpr))
5889	return false;
5890	if (!this->emitFinishInitPop(InitExpr))
5891	return false;
5892	} else if (const IndirectFieldDecl *IFD = Init->getIndirectMember()) {
5893	assert(IFD->getChainingSize() >= 2);
5894
5895	unsigned NestedFieldOffset = 0;
5896	const Record::Field *NestedField = nullptr;
5897	for (const NamedDecl *ND : IFD->chain()) {
5898	const auto *FD = cast<FieldDecl>(Val: ND);
5899	const Record *FieldRecord = this->P.getOrCreateRecord(FD->getParent());
5900	assert(FieldRecord);
5901
5902	NestedField = FieldRecord->getField(FD);
5903	assert(NestedField);
5904
5905	NestedFieldOffset += NestedField->Offset;
5906	}
5907	assert(NestedField);
5908
5909	if (!emitFieldInitializer(NestedField, NestedFieldOffset, InitExpr))
5910	return false;
5911
5912	// Mark all chain links as initialized.
5913	unsigned InitFieldOffset = 0;
5914	for (const NamedDecl *ND : IFD->chain().drop_back()) {
5915	const auto *FD = cast<FieldDecl>(Val: ND);
5916	const Record *FieldRecord = this->P.getOrCreateRecord(FD->getParent());
5917	assert(FieldRecord);
5918	NestedField = FieldRecord->getField(FD);
5919	InitFieldOffset += NestedField->Offset;
5920	assert(NestedField);
5921	if (!this->emitGetPtrThisField(InitFieldOffset, InitExpr))
5922	return false;
5923	if (!this->emitFinishInitPop(InitExpr))
5924	return false;
5925	}
5926
5927	} else {
5928	assert(Init->isDelegatingInitializer());
5929	if (!this->emitThis(InitExpr))
5930	return false;
5931	if (!this->visitInitializer(Init->getInit()))
5932	return false;
5933	if (!this->emitPopPtr(InitExpr))
5934	return false;
5935	}
5936
5937	if (!Scope.destroyLocals())
5938	return false;
5939	}
5940
5941	if (const auto *Body = Ctor->getBody())
5942	if (!visitStmt(S: Body))
5943	return false;
5944
5945	return this->emitRetVoid(SourceInfo{});
5946	}
5947
5948	template <class Emitter>
5949	bool Compiler<Emitter>::compileDestructor(const CXXDestructorDecl *Dtor) {
5950	const RecordDecl *RD = Dtor->getParent();
5951	const Record *R = this->getRecord(RD);
5952	if (!R)
5953	return false;
5954
5955	if (!Dtor->isTrivial() && Dtor->getBody()) {
5956	if (!this->visitStmt(Dtor->getBody()))
5957	return false;
5958	}
5959
5960	if (!this->emitThis(Dtor))
5961	return false;
5962
5963	if (!this->emitCheckDestruction(Dtor))
5964	return false;
5965
5966	assert(R);
5967	if (!R->isUnion()) {
5968	// First, destroy all fields.
5969	for (const Record::Field &Field : llvm::reverse(R->fields())) {
5970	const Descriptor *D = Field.Desc;
5971	if (!D->isPrimitive() && !D->isPrimitiveArray()) {
5972	if (!this->emitGetPtrField(Field.Offset, SourceInfo{}))
5973	return false;
5974	if (!this->emitDestruction(D, SourceInfo{}))
5975	return false;
5976	if (!this->emitPopPtr(SourceInfo{}))
5977	return false;
5978	}
5979	}
5980	}
5981
5982	for (const Record::Base &Base : llvm::reverse(R->bases())) {
5983	if (Base.R->isAnonymousUnion())
5984	continue;
5985
5986	if (!this->emitGetPtrBase(Base.Offset, SourceInfo{}))
5987	return false;
5988	if (!this->emitRecordDestruction(Base.R, {}))
5989	return false;
5990	if (!this->emitPopPtr(SourceInfo{}))
5991	return false;
5992	}
5993
5994	// FIXME: Virtual bases.
5995	return this->emitPopPtr(Dtor) && this->emitRetVoid(Dtor);
5996	}
5997
5998	template <class Emitter>
5999	bool Compiler<Emitter>::compileUnionAssignmentOperator(
6000	const CXXMethodDecl *MD) {
6001	if (!this->emitThis(MD))
6002	return false;
6003
6004	auto PVD = MD->getParamDecl(0);
6005	ParamOffset PO = this->Params[PVD]; // Must exist.
6006
6007	if (!this->emitGetParam(PT_Ptr, PO.Offset, MD))
6008	return false;
6009
6010	return this->emitMemcpy(MD) && this->emitRet(PT_Ptr, MD);
6011	}
6012
6013	template <class Emitter>
6014	bool Compiler<Emitter>::visitFunc(const FunctionDecl *F) {
6015	// Classify the return type.
6016	ReturnType = this->classify(F->getReturnType());
6017
6018	if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(Val: F))
6019	return this->compileConstructor(Ctor);
6020	if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(Val: F))
6021	return this->compileDestructor(Dtor);
6022
6023	// Emit custom code if this is a lambda static invoker.
6024	if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: F)) {
6025	const RecordDecl *RD = MD->getParent();
6026
6027	if (RD->isUnion() &&
6028	(MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()))
6029	return this->compileUnionAssignmentOperator(MD);
6030
6031	if (MD->isLambdaStaticInvoker())
6032	return this->emitLambdaStaticInvokerBody(MD);
6033	}
6034
6035	// Regular functions.
6036	if (const auto *Body = F->getBody())
6037	if (!visitStmt(S: Body))
6038	return false;
6039
6040	// Emit a guard return to protect against a code path missing one.
6041	if (F->getReturnType()->isVoidType())
6042	return this->emitRetVoid(SourceInfo{});
6043	return this->emitNoRet(SourceInfo{});
6044	}
6045
6046	template <class Emitter>
6047	bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
6048	const Expr *SubExpr = E->getSubExpr();
6049	if (SubExpr->getType()->isAnyComplexType())
6050	return this->VisitComplexUnaryOperator(E);
6051	if (SubExpr->getType()->isVectorType())
6052	return this->VisitVectorUnaryOperator(E);
6053	if (SubExpr->getType()->isFixedPointType())
6054	return this->VisitFixedPointUnaryOperator(E);
6055	std::optional<PrimType> T = classify(SubExpr->getType());
6056
6057	switch (E->getOpcode()) {
6058	case UO_PostInc: { // x++
6059	if (!Ctx.getLangOpts().CPlusPlus14)
6060	return this->emitInvalid(E);
6061	if (!T)
6062	return this->emitError(E);
6063
6064	if (!this->visit(SubExpr))
6065	return false;
6066
6067	if (T == PT_Ptr) {
6068	if (!this->emitIncPtr(E))
6069	return false;
6070
6071	return DiscardResult ? this->emitPopPtr(E) : true;
6072	}
6073
6074	if (T == PT_Float) {
6075	return DiscardResult ? this->emitIncfPop(getFPOptions(E), E)
6076	: this->emitIncf(getFPOptions(E), E);
6077	}
6078
6079	return DiscardResult ? this->emitIncPop(*T, E->canOverflow(), E)
6080	: this->emitInc(*T, E->canOverflow(), E);
6081	}
6082	case UO_PostDec: { // x--
6083	if (!Ctx.getLangOpts().CPlusPlus14)
6084	return this->emitInvalid(E);
6085	if (!T)
6086	return this->emitError(E);
6087
6088	if (!this->visit(SubExpr))
6089	return false;
6090
6091	if (T == PT_Ptr) {
6092	if (!this->emitDecPtr(E))
6093	return false;
6094
6095	return DiscardResult ? this->emitPopPtr(E) : true;
6096	}
6097
6098	if (T == PT_Float) {
6099	return DiscardResult ? this->emitDecfPop(getFPOptions(E), E)
6100	: this->emitDecf(getFPOptions(E), E);
6101	}
6102
6103	return DiscardResult ? this->emitDecPop(*T, E->canOverflow(), E)
6104	: this->emitDec(*T, E->canOverflow(), E);
6105	}
6106	case UO_PreInc: { // ++x
6107	if (!Ctx.getLangOpts().CPlusPlus14)
6108	return this->emitInvalid(E);
6109	if (!T)
6110	return this->emitError(E);
6111
6112	if (!this->visit(SubExpr))
6113	return false;
6114
6115	if (T == PT_Ptr) {
6116	if (!this->emitLoadPtr(E))
6117	return false;
6118	if (!this->emitConstUint8(1, E))
6119	return false;
6120	if (!this->emitAddOffsetUint8(E))
6121	return false;
6122	return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
6123	}
6124
6125	// Post-inc and pre-inc are the same if the value is to be discarded.
6126	if (DiscardResult) {
6127	if (T == PT_Float)
6128	return this->emitIncfPop(getFPOptions(E), E);
6129	return this->emitIncPop(*T, E->canOverflow(), E);
6130	}
6131
6132	if (T == PT_Float) {
6133	const auto &TargetSemantics = Ctx.getFloatSemantics(T: E->getType());
6134	if (!this->emitLoadFloat(E))
6135	return false;
6136	if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
6137	return false;
6138	if (!this->emitAddf(getFPOptions(E), E))
6139	return false;
6140	if (!this->emitStoreFloat(E))
6141	return false;
6142	} else {
6143	assert(isIntegralType(*T));
6144	if (!this->emitPreInc(*T, E->canOverflow(), E))
6145	return false;
6146	}
6147	return E->isGLValue() || this->emitLoadPop(*T, E);
6148	}
6149	case UO_PreDec: { // --x
6150	if (!Ctx.getLangOpts().CPlusPlus14)
6151	return this->emitInvalid(E);
6152	if (!T)
6153	return this->emitError(E);
6154
6155	if (!this->visit(SubExpr))
6156	return false;
6157
6158	if (T == PT_Ptr) {
6159	if (!this->emitLoadPtr(E))
6160	return false;
6161	if (!this->emitConstUint8(1, E))
6162	return false;
6163	if (!this->emitSubOffsetUint8(E))
6164	return false;
6165	return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
6166	}
6167
6168	// Post-dec and pre-dec are the same if the value is to be discarded.
6169	if (DiscardResult) {
6170	if (T == PT_Float)
6171	return this->emitDecfPop(getFPOptions(E), E);
6172	return this->emitDecPop(*T, E->canOverflow(), E);
6173	}
6174
6175	if (T == PT_Float) {
6176	const auto &TargetSemantics = Ctx.getFloatSemantics(T: E->getType());
6177	if (!this->emitLoadFloat(E))
6178	return false;
6179	if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
6180	return false;
6181	if (!this->emitSubf(getFPOptions(E), E))
6182	return false;
6183	if (!this->emitStoreFloat(E))
6184	return false;
6185	} else {
6186	assert(isIntegralType(*T));
6187	if (!this->emitPreDec(*T, E->canOverflow(), E))
6188	return false;
6189	}
6190	return E->isGLValue() || this->emitLoadPop(*T, E);
6191	}
6192	case UO_LNot: // !x
6193	if (!T)
6194	return this->emitError(E);
6195
6196	if (DiscardResult)
6197	return this->discard(SubExpr);
6198
6199	if (!this->visitBool(SubExpr))
6200	return false;
6201
6202	if (!this->emitInv(E))
6203	return false;
6204
6205	if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
6206	return this->emitCast(PT_Bool, ET, E);
6207	return true;
6208	case UO_Minus: // -x
6209	if (!T)
6210	return this->emitError(E);
6211
6212	if (!this->visit(SubExpr))
6213	return false;
6214	return DiscardResult ? this->emitPop(*T, E) : this->emitNeg(*T, E);
6215	case UO_Plus: // +x
6216	if (!T)
6217	return this->emitError(E);
6218
6219	if (!this->visit(SubExpr)) // noop
6220	return false;
6221	return DiscardResult ? this->emitPop(*T, E) : true;
6222	case UO_AddrOf: // &x
6223	if (E->getType()->isMemberPointerType()) {
6224	// C++11 [expr.unary.op]p3 has very strict rules on how the address of a
6225	// member can be formed.
6226	return this->emitGetMemberPtr(cast<DeclRefExpr>(Val: SubExpr)->getDecl(), E);
6227	}
6228	// We should already have a pointer when we get here.
6229	return this->delegate(SubExpr);
6230	case UO_Deref: // *x
6231	if (DiscardResult)
6232	return this->discard(SubExpr);
6233
6234	if (!this->visit(SubExpr))
6235	return false;
6236
6237	if (classifyPrim(SubExpr) == PT_Ptr)
6238	return this->emitNarrowPtr(E);
6239	return true;
6240
6241	case UO_Not: // ~x
6242	if (!T)
6243	return this->emitError(E);
6244
6245	if (!this->visit(SubExpr))
6246	return false;
6247	return DiscardResult ? this->emitPop(*T, E) : this->emitComp(*T, E);
6248	case UO_Real: // __real x
6249	assert(T);
6250	return this->delegate(SubExpr);
6251	case UO_Imag: { // __imag x
6252	assert(T);
6253	if (!this->discard(SubExpr))
6254	return false;
6255	return this->visitZeroInitializer(*T, SubExpr->getType(), SubExpr);
6256	}
6257	case UO_Extension:
6258	return this->delegate(SubExpr);
6259	case UO_Coawait:
6260	assert(false && "Unhandled opcode");
6261	}
6262
6263	return false;
6264	}
6265
6266	template <class Emitter>
6267	bool Compiler<Emitter>::VisitComplexUnaryOperator(const UnaryOperator *E) {
6268	const Expr *SubExpr = E->getSubExpr();
6269	assert(SubExpr->getType()->isAnyComplexType());
6270
6271	if (DiscardResult)
6272	return this->discard(SubExpr);
6273
6274	std::optional<PrimType> ResT = classify(E);
6275	auto prepareResult = [=]() -> bool {
6276	if (!ResT && !Initializing) {
6277	std::optional<unsigned> LocalIndex = allocateLocal(Src: SubExpr);
6278	if (!LocalIndex)
6279	return false;
6280	return this->emitGetPtrLocal(*LocalIndex, E);
6281	}
6282
6283	return true;
6284	};
6285
6286	// The offset of the temporary, if we created one.
6287	unsigned SubExprOffset = ~0u;
6288	auto createTemp = [=, &SubExprOffset]() -> bool {
6289	SubExprOffset =
6290	this->allocateLocalPrimitive(SubExpr, PT_Ptr, /*IsConst=*/true);
6291	if (!this->visit(SubExpr))
6292	return false;
6293	return this->emitSetLocal(PT_Ptr, SubExprOffset, E);
6294	};
6295
6296	PrimType ElemT = classifyComplexElementType(T: SubExpr->getType());
6297	auto getElem = [=](unsigned Offset, unsigned Index) -> bool {
6298	if (!this->emitGetLocal(PT_Ptr, Offset, E))
6299	return false;
6300	return this->emitArrayElemPop(ElemT, Index, E);
6301	};
6302
6303	switch (E->getOpcode()) {
6304	case UO_Minus:
6305	if (!prepareResult())
6306	return false;
6307	if (!createTemp())
6308	return false;
6309	for (unsigned I = 0; I != 2; ++I) {
6310	if (!getElem(SubExprOffset, I))
6311	return false;
6312	if (!this->emitNeg(ElemT, E))
6313	return false;
6314	if (!this->emitInitElem(ElemT, I, E))
6315	return false;
6316	}
6317	break;
6318
6319	case UO_Plus: // +x
6320	case UO_AddrOf: // &x
6321	case UO_Deref: // *x
6322	return this->delegate(SubExpr);
6323
6324	case UO_LNot:
6325	if (!this->visit(SubExpr))
6326	return false;
6327	if (!this->emitComplexBoolCast(SubExpr))
6328	return false;
6329	if (!this->emitInv(E))
6330	return false;
6331	if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
6332	return this->emitCast(PT_Bool, ET, E);
6333	return true;
6334
6335	case UO_Real:
6336	return this->emitComplexReal(SubExpr);
6337
6338	case UO_Imag:
6339	if (!this->visit(SubExpr))
6340	return false;
6341
6342	if (SubExpr->isLValue()) {
6343	if (!this->emitConstUint8(1, E))
6344	return false;
6345	return this->emitArrayElemPtrPopUint8(E);
6346	}
6347
6348	// Since our _Complex implementation does not map to a primitive type,
6349	// we sometimes have to do the lvalue-to-rvalue conversion here manually.
6350	return this->emitArrayElemPop(classifyPrim(E->getType()), 1, E);
6351
6352	case UO_Not: // ~x
6353	if (!this->visit(SubExpr))
6354	return false;
6355	// Negate the imaginary component.
6356	if (!this->emitArrayElem(ElemT, 1, E))
6357	return false;
6358	if (!this->emitNeg(ElemT, E))
6359	return false;
6360	if (!this->emitInitElem(ElemT, 1, E))
6361	return false;
6362	return DiscardResult ? this->emitPopPtr(E) : true;
6363
6364	case UO_Extension:
6365	return this->delegate(SubExpr);
6366
6367	default:
6368	return this->emitInvalid(E);
6369	}
6370
6371	return true;
6372	}
6373
6374	template <class Emitter>
6375	bool Compiler<Emitter>::VisitVectorUnaryOperator(const UnaryOperator *E) {
6376	const Expr *SubExpr = E->getSubExpr();
6377	assert(SubExpr->getType()->isVectorType());
6378
6379	if (DiscardResult)
6380	return this->discard(SubExpr);
6381
6382	auto UnaryOp = E->getOpcode();
6383	if (UnaryOp == UO_Extension)
6384	return this->delegate(SubExpr);
6385
6386	if (UnaryOp != UO_Plus && UnaryOp != UO_Minus && UnaryOp != UO_LNot &&
6387	UnaryOp != UO_Not && UnaryOp != UO_AddrOf)
6388	return this->emitInvalid(E);
6389
6390	// Nothing to do here.
6391	if (UnaryOp == UO_Plus || UnaryOp == UO_AddrOf)
6392	return this->delegate(SubExpr);
6393
6394	if (!Initializing) {
6395	std::optional<unsigned> LocalIndex = allocateLocal(Src: SubExpr);
6396	if (!LocalIndex)
6397	return false;
6398	if (!this->emitGetPtrLocal(*LocalIndex, E))
6399	return false;
6400	}
6401
6402	// The offset of the temporary, if we created one.
6403	unsigned SubExprOffset =
6404	this->allocateLocalPrimitive(SubExpr, PT_Ptr, /*IsConst=*/true);
6405	if (!this->visit(SubExpr))
6406	return false;
6407	if (!this->emitSetLocal(PT_Ptr, SubExprOffset, E))
6408	return false;
6409
6410	const auto *VecTy = SubExpr->getType()->getAs<VectorType>();
6411	PrimType ElemT = classifyVectorElementType(T: SubExpr->getType());
6412	auto getElem = [=](unsigned Offset, unsigned Index) -> bool {
6413	if (!this->emitGetLocal(PT_Ptr, Offset, E))
6414	return false;
6415	return this->emitArrayElemPop(ElemT, Index, E);
6416	};
6417
6418	switch (UnaryOp) {
6419	case UO_Minus:
6420	for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
6421	if (!getElem(SubExprOffset, I))
6422	return false;
6423	if (!this->emitNeg(ElemT, E))
6424	return false;
6425	if (!this->emitInitElem(ElemT, I, E))
6426	return false;
6427	}
6428	break;
6429	case UO_LNot: { // !x
6430	// In C++, the logic operators !, &&, || are available for vectors. !v is
6431	// equivalent to v == 0.
6432	//
6433	// The result of the comparison is a vector of the same width and number of
6434	// elements as the comparison operands with a signed integral element type.
6435	//
6436	// https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
6437	QualType ResultVecTy = E->getType();
6438	PrimType ResultVecElemT =
6439	classifyPrim(ResultVecTy->getAs<VectorType>()->getElementType());
6440	for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
6441	if (!getElem(SubExprOffset, I))
6442	return false;
6443	// operator ! on vectors returns -1 for 'truth', so negate it.
6444	if (!this->emitPrimCast(ElemT, PT_Bool, Ctx.getASTContext().BoolTy, E))
6445	return false;
6446	if (!this->emitInv(E))
6447	return false;
6448	if (!this->emitPrimCast(PT_Bool, ElemT, VecTy->getElementType(), E))
6449	return false;
6450	if (!this->emitNeg(ElemT, E))
6451	return false;
6452	if (ElemT != ResultVecElemT &&
6453	!this->emitPrimCast(ElemT, ResultVecElemT, ResultVecTy, E))
6454	return false;
6455	if (!this->emitInitElem(ResultVecElemT, I, E))
6456	return false;
6457	}
6458	break;
6459	}
6460	case UO_Not: // ~x
6461	for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
6462	if (!getElem(SubExprOffset, I))
6463	return false;
6464	if (ElemT == PT_Bool) {
6465	if (!this->emitInv(E))
6466	return false;
6467	} else {
6468	if (!this->emitComp(ElemT, E))
6469	return false;
6470	}
6471	if (!this->emitInitElem(ElemT, I, E))
6472	return false;
6473	}
6474	break;
6475	default:
6476	llvm_unreachable("Unsupported unary operators should be handled up front");
6477	}
6478	return true;
6479	}
6480
6481	template <class Emitter>
6482	bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
6483	if (DiscardResult)
6484	return true;
6485
6486	if (const auto *ECD = dyn_cast<EnumConstantDecl>(Val: D)) {
6487	return this->emitConst(ECD->getInitVal(), E);
6488	} else if (const auto *BD = dyn_cast<BindingDecl>(Val: D)) {
6489	return this->visit(BD->getBinding());
6490	} else if (const auto *FuncDecl = dyn_cast<FunctionDecl>(Val: D)) {
6491	const Function *F = getFunction(FD: FuncDecl);
6492	return F && this->emitGetFnPtr(F, E);
6493	} else if (const auto *TPOD = dyn_cast<TemplateParamObjectDecl>(Val: D)) {
6494	if (std::optional<unsigned> Index = P.getOrCreateGlobal(VD: D)) {
6495	if (!this->emitGetPtrGlobal(*Index, E))
6496	return false;
6497	if (std::optional<PrimType> T = classify(E->getType())) {
6498	if (!this->visitAPValue(TPOD->getValue(), *T, E))
6499	return false;
6500	return this->emitInitGlobal(*T, *Index, E);
6501	}
6502	return this->visitAPValueInitializer(TPOD->getValue(), E,
6503	TPOD->getType());
6504	}
6505	return false;
6506	}
6507
6508	// References are implemented via pointers, so when we see a DeclRefExpr
6509	// pointing to a reference, we need to get its value directly (i.e. the
6510	// pointer to the actual value) instead of a pointer to the pointer to the
6511	// value.
6512	bool IsReference = D->getType()->isReferenceType();
6513
6514	// Check for local/global variables and parameters.
6515	if (auto It = Locals.find(Val: D); It != Locals.end()) {
6516	const unsigned Offset = It->second.Offset;
6517	if (IsReference)
6518	return this->emitGetLocal(classifyPrim(E), Offset, E);
6519	return this->emitGetPtrLocal(Offset, E);
6520	} else if (auto GlobalIndex = P.getGlobal(VD: D)) {
6521	if (IsReference) {
6522	if (!Ctx.getLangOpts().CPlusPlus11)
6523	return this->emitGetGlobal(classifyPrim(E), *GlobalIndex, E);
6524	return this->emitGetGlobalUnchecked(classifyPrim(E), *GlobalIndex, E);
6525	}
6526
6527	return this->emitGetPtrGlobal(*GlobalIndex, E);
6528	} else if (const auto *PVD = dyn_cast<ParmVarDecl>(Val: D)) {
6529	if (auto It = this->Params.find(PVD); It != this->Params.end()) {
6530	if (IsReference || !It->second.IsPtr)
6531	return this->emitGetParam(classifyPrim(E), It->second.Offset, E);
6532
6533	return this->emitGetPtrParam(It->second.Offset, E);
6534	}
6535	}
6536
6537	// In case we need to re-visit a declaration.
6538	auto revisit = [&](const VarDecl *VD) -> bool {
6539	if (!this->emitPushCC(VD->hasConstantInitialization(), E))
6540	return false;
6541	auto VarState = this->visitDecl(VD, /*IsConstexprUnknown=*/true);
6542
6543	if (!this->emitPopCC(E))
6544	return false;
6545
6546	if (VarState.notCreated())
6547	return true;
6548	if (!VarState)
6549	return false;
6550	// Retry.
6551	return this->visitDeclRef(D, E);
6552	};
6553
6554	// Handle lambda captures.
6555	if (auto It = this->LambdaCaptures.find(D);
6556	It != this->LambdaCaptures.end()) {
6557	auto [Offset, IsPtr] = It->second;
6558
6559	if (IsPtr)
6560	return this->emitGetThisFieldPtr(Offset, E);
6561	return this->emitGetPtrThisField(Offset, E);
6562	} else if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E);
6563	DRE && DRE->refersToEnclosingVariableOrCapture()) {
6564	if (const auto *VD = dyn_cast<VarDecl>(Val: D); VD && VD->isInitCapture())
6565	return revisit(VD);
6566	}
6567
6568	// Avoid infinite recursion.
6569	if (D == InitializingDecl)
6570	return this->emitDummyPtr(D, E);
6571
6572	// Try to lazily visit (or emit dummy pointers for) declarations
6573	// we haven't seen yet.
6574	// For C.
6575	if (!Ctx.getLangOpts().CPlusPlus) {
6576	if (const auto *VD = dyn_cast<VarDecl>(Val: D);
6577	VD && VD->getAnyInitializer() &&
6578	VD->getType().isConstant(Ctx.getASTContext()) && !VD->isWeak())
6579	return revisit(VD);
6580	return this->emitDummyPtr(D, E);
6581	}
6582
6583	// ... and C++.
6584	const auto *VD = dyn_cast<VarDecl>(Val: D);
6585	if (!VD)
6586	return this->emitDummyPtr(D, E);
6587
6588	const auto typeShouldBeVisited = [&](QualType T) -> bool {
6589	if (T.isConstant(Ctx: Ctx.getASTContext()))
6590	return true;
6591	return T->isReferenceType();
6592	};
6593
6594	// DecompositionDecls are just proxies for us.
6595	if (isa<DecompositionDecl>(Val: VD))
6596	return revisit(VD);
6597
6598	if ((VD->hasGlobalStorage() || VD->isStaticDataMember()) &&
6599	typeShouldBeVisited(VD->getType())) {
6600	if (const Expr *Init = VD->getAnyInitializer();
6601	Init && !Init->isValueDependent()) {
6602	// Whether or not the evaluation is successul doesn't really matter
6603	// here -- we will create a global variable in any case, and that
6604	// will have the state of initializer evaluation attached.
6605	APValue V;
6606	SmallVector<PartialDiagnosticAt> Notes;
6607	(void)Init->EvaluateAsInitializer(Result&: V, Ctx: Ctx.getASTContext(), VD, Notes,
6608	IsConstantInitializer: true);
6609	return this->visitDeclRef(D, E);
6610	}
6611	return revisit(VD);
6612	}
6613
6614	// FIXME: The evaluateValue() check here is a little ridiculous, since
6615	// it will ultimately call into Context::evaluateAsInitializer(). In
6616	// other words, we're evaluating the initializer, just to know if we can
6617	// evaluate the initializer.
6618	if (VD->isLocalVarDecl() && typeShouldBeVisited(VD->getType()) &&
6619	VD->getInit() && !VD->getInit()->isValueDependent()) {
6620
6621	if (VD->evaluateValue())
6622	return revisit(VD);
6623
6624	if (!D->getType()->isReferenceType())
6625	return this->emitDummyPtr(D, E);
6626
6627	return this->emitInvalidDeclRef(cast<DeclRefExpr>(Val: E),
6628	/*InitializerFailed=*/true, E);
6629	}
6630
6631	return this->emitDummyPtr(D, E);
6632	}
6633
6634	template <class Emitter>
6635	bool Compiler<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
6636	const auto *D = E->getDecl();
6637	return this->visitDeclRef(D, E);
6638	}
6639
6640	template <class Emitter> void Compiler<Emitter>::emitCleanup() {
6641	for (VariableScope<Emitter> *C = VarScope; C; C = C->getParent())
6642	C->emitDestruction();
6643	}
6644
6645	template <class Emitter>
6646	unsigned Compiler<Emitter>::collectBaseOffset(const QualType BaseType,
6647	const QualType DerivedType) {
6648	const auto extractRecordDecl = [](QualType Ty) -> const CXXRecordDecl * {
6649	if (const auto *R = Ty->getPointeeCXXRecordDecl())
6650	return R;
6651	return Ty->getAsCXXRecordDecl();
6652	};
6653	const CXXRecordDecl *BaseDecl = extractRecordDecl(BaseType);
6654	const CXXRecordDecl *DerivedDecl = extractRecordDecl(DerivedType);
6655
6656	return Ctx.collectBaseOffset(BaseDecl, DerivedDecl);
6657	}
6658
6659	/// Emit casts from a PrimType to another PrimType.
6660	template <class Emitter>
6661	bool Compiler<Emitter>::emitPrimCast(PrimType FromT, PrimType ToT,
6662	QualType ToQT, const Expr *E) {
6663
6664	if (FromT == PT_Float) {
6665	// Floating to floating.
6666	if (ToT == PT_Float) {
6667	const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(T: ToQT);
6668	return this->emitCastFP(ToSem, getRoundingMode(E), E);
6669	}
6670
6671	if (ToT == PT_IntAP)
6672	return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(T: ToQT),
6673	getFPOptions(E), E);
6674	if (ToT == PT_IntAPS)
6675	return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(T: ToQT),
6676	getFPOptions(E), E);
6677
6678	// Float to integral.
6679	if (isIntegralType(T: ToT) || ToT == PT_Bool)
6680	return this->emitCastFloatingIntegral(ToT, getFPOptions(E), E);
6681	}
6682
6683	if (isIntegralType(T: FromT) || FromT == PT_Bool) {
6684	if (ToT == PT_IntAP)
6685	return this->emitCastAP(FromT, Ctx.getBitWidth(T: ToQT), E);
6686	if (ToT == PT_IntAPS)
6687	return this->emitCastAPS(FromT, Ctx.getBitWidth(T: ToQT), E);
6688
6689	// Integral to integral.
6690	if (isIntegralType(T: ToT) || ToT == PT_Bool)
6691	return FromT != ToT ? this->emitCast(FromT, ToT, E) : true;
6692
6693	if (ToT == PT_Float) {
6694	// Integral to floating.
6695	const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(T: ToQT);
6696	return this->emitCastIntegralFloating(FromT, ToSem, getFPOptions(E), E);
6697	}
6698	}
6699
6700	return false;
6701	}
6702
6703	/// Emits __real(SubExpr)
6704	template <class Emitter>
6705	bool Compiler<Emitter>::emitComplexReal(const Expr *SubExpr) {
6706	assert(SubExpr->getType()->isAnyComplexType());
6707
6708	if (DiscardResult)
6709	return this->discard(SubExpr);
6710
6711	if (!this->visit(SubExpr))
6712	return false;
6713	if (SubExpr->isLValue()) {
6714	if (!this->emitConstUint8(0, SubExpr))
6715	return false;
6716	return this->emitArrayElemPtrPopUint8(SubExpr);
6717	}
6718
6719	// Rvalue, load the actual element.
6720	return this->emitArrayElemPop(classifyComplexElementType(T: SubExpr->getType()),
6721	0, SubExpr);
6722	}
6723
6724	template <class Emitter>
6725	bool Compiler<Emitter>::emitComplexBoolCast(const Expr *E) {
6726	assert(!DiscardResult);
6727	PrimType ElemT = classifyComplexElementType(T: E->getType());
6728	// We emit the expression (__real(E) != 0 || __imag(E) != 0)
6729	// for us, that means (bool)E[0] || (bool)E[1]
6730	if (!this->emitArrayElem(ElemT, 0, E))
6731	return false;
6732	if (ElemT == PT_Float) {
6733	if (!this->emitCastFloatingIntegral(PT_Bool, getFPOptions(E), E))
6734	return false;
6735	} else {
6736	if (!this->emitCast(ElemT, PT_Bool, E))
6737	return false;
6738	}
6739
6740	// We now have the bool value of E[0] on the stack.
6741	LabelTy LabelTrue = this->getLabel();
6742	if (!this->jumpTrue(LabelTrue))
6743	return false;
6744
6745	if (!this->emitArrayElemPop(ElemT, 1, E))
6746	return false;
6747	if (ElemT == PT_Float) {
6748	if (!this->emitCastFloatingIntegral(PT_Bool, getFPOptions(E), E))
6749	return false;
6750	} else {
6751	if (!this->emitCast(ElemT, PT_Bool, E))
6752	return false;
6753	}
6754	// Leave the boolean value of E[1] on the stack.
6755	LabelTy EndLabel = this->getLabel();
6756	this->jump(EndLabel);
6757
6758	this->emitLabel(LabelTrue);
6759	if (!this->emitPopPtr(E))
6760	return false;
6761	if (!this->emitConstBool(true, E))
6762	return false;
6763
6764	this->fallthrough(EndLabel);
6765	this->emitLabel(EndLabel);
6766
6767	return true;
6768	}
6769
6770	template <class Emitter>
6771	bool Compiler<Emitter>::emitComplexComparison(const Expr *LHS, const Expr *RHS,
6772	const BinaryOperator *E) {
6773	assert(E->isComparisonOp());
6774	assert(!Initializing);
6775	assert(!DiscardResult);
6776
6777	PrimType ElemT;
6778	bool LHSIsComplex;
6779	unsigned LHSOffset;
6780	if (LHS->getType()->isAnyComplexType()) {
6781	LHSIsComplex = true;
6782	ElemT = classifyComplexElementType(T: LHS->getType());
6783	LHSOffset = allocateLocalPrimitive(Src: LHS, Ty: PT_Ptr, /*IsConst=*/true);
6784	if (!this->visit(LHS))
6785	return false;
6786	if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
6787	return false;
6788	} else {
6789	LHSIsComplex = false;
6790	PrimType LHST = classifyPrim(LHS->getType());
6791	LHSOffset = this->allocateLocalPrimitive(LHS, LHST, /*IsConst=*/true);
6792	if (!this->visit(LHS))
6793	return false;
6794	if (!this->emitSetLocal(LHST, LHSOffset, E))
6795	return false;
6796	}
6797
6798	bool RHSIsComplex;
6799	unsigned RHSOffset;
6800	if (RHS->getType()->isAnyComplexType()) {
6801	RHSIsComplex = true;
6802	ElemT = classifyComplexElementType(T: RHS->getType());
6803	RHSOffset = allocateLocalPrimitive(Src: RHS, Ty: PT_Ptr, /*IsConst=*/true);
6804	if (!this->visit(RHS))
6805	return false;
6806	if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
6807	return false;
6808	} else {
6809	RHSIsComplex = false;
6810	PrimType RHST = classifyPrim(RHS->getType());
6811	RHSOffset = this->allocateLocalPrimitive(RHS, RHST, /*IsConst=*/true);
6812	if (!this->visit(RHS))
6813	return false;
6814	if (!this->emitSetLocal(RHST, RHSOffset, E))
6815	return false;
6816	}
6817
6818	auto getElem = [&](unsigned LocalOffset, unsigned Index,
6819	bool IsComplex) -> bool {
6820	if (IsComplex) {
6821	if (!this->emitGetLocal(PT_Ptr, LocalOffset, E))
6822	return false;
6823	return this->emitArrayElemPop(ElemT, Index, E);
6824	}
6825	return this->emitGetLocal(ElemT, LocalOffset, E);
6826	};
6827
6828	for (unsigned I = 0; I != 2; ++I) {
6829	// Get both values.
6830	if (!getElem(LHSOffset, I, LHSIsComplex))
6831	return false;
6832	if (!getElem(RHSOffset, I, RHSIsComplex))
6833	return false;
6834	// And compare them.
6835	if (!this->emitEQ(ElemT, E))
6836	return false;
6837
6838	if (!this->emitCastBoolUint8(E))
6839	return false;
6840	}
6841
6842	// We now have two bool values on the stack. Compare those.
6843	if (!this->emitAddUint8(E))
6844	return false;
6845	if (!this->emitConstUint8(2, E))
6846	return false;
6847
6848	if (E->getOpcode() == BO_EQ) {
6849	if (!this->emitEQUint8(E))
6850	return false;
6851	} else if (E->getOpcode() == BO_NE) {
6852	if (!this->emitNEUint8(E))
6853	return false;
6854	} else
6855	return false;
6856
6857	// In C, this returns an int.
6858	if (PrimType ResT = classifyPrim(E->getType()); ResT != PT_Bool)
6859	return this->emitCast(PT_Bool, ResT, E);
6860	return true;
6861	}
6862
6863	/// When calling this, we have a pointer of the local-to-destroy
6864	/// on the stack.
6865	/// Emit destruction of record types (or arrays of record types).
6866	template <class Emitter>
6867	bool Compiler<Emitter>::emitRecordDestruction(const Record *R, SourceInfo Loc) {
6868	assert(R);
6869	assert(!R->isAnonymousUnion());
6870	const CXXDestructorDecl *Dtor = R->getDestructor();
6871	if (!Dtor || Dtor->isTrivial())
6872	return true;
6873
6874	assert(Dtor);
6875	const Function *DtorFunc = getFunction(FD: Dtor);
6876	if (!DtorFunc)
6877	return false;
6878	assert(DtorFunc->hasThisPointer());
6879	assert(DtorFunc->getNumParams() == 1);
6880	if (!this->emitDupPtr(Loc))
6881	return false;
6882	return this->emitCall(DtorFunc, 0, Loc);
6883	}
6884	/// When calling this, we have a pointer of the local-to-destroy
6885	/// on the stack.
6886	/// Emit destruction of record types (or arrays of record types).
6887	template <class Emitter>
6888	bool Compiler<Emitter>::emitDestruction(const Descriptor *Desc,
6889	SourceInfo Loc) {
6890	assert(Desc);
6891	assert(!Desc->isPrimitive());
6892	assert(!Desc->isPrimitiveArray());
6893
6894	// Can happen if the decl is invalid.
6895	if (Desc->isDummy())
6896	return true;
6897
6898	// Arrays.
6899	if (Desc->isArray()) {
6900	const Descriptor *ElemDesc = Desc->ElemDesc;
6901	assert(ElemDesc);
6902
6903	// Don't need to do anything for these.
6904	if (ElemDesc->isPrimitiveArray())
6905	return true;
6906
6907	// If this is an array of record types, check if we need
6908	// to call the element destructors at all. If not, try
6909	// to save the work.
6910	if (const Record *ElemRecord = ElemDesc->ElemRecord) {
6911	if (const CXXDestructorDecl *Dtor = ElemRecord->getDestructor();
6912	!Dtor || Dtor->isTrivial())
6913	return true;
6914	}
6915
6916	if (unsigned N = Desc->getNumElems()) {
6917	for (ssize_t I = N - 1; I >= 0; --I) {
6918	if (!this->emitConstUint64(I, Loc))
6919	return false;
6920	if (!this->emitArrayElemPtrUint64(Loc))
6921	return false;
6922	if (!this->emitDestruction(ElemDesc, Loc))
6923	return false;
6924	if (!this->emitPopPtr(Loc))
6925	return false;
6926	}
6927	}
6928	return true;
6929	}
6930
6931	assert(Desc->ElemRecord);
6932	if (Desc->ElemRecord->isAnonymousUnion())
6933	return true;
6934
6935	return this->emitRecordDestruction(Desc->ElemRecord, Loc);
6936	}
6937
6938	/// Create a dummy pointer for the given decl (or expr) and
6939	/// push a pointer to it on the stack.
6940	template <class Emitter>
6941	bool Compiler<Emitter>::emitDummyPtr(const DeclTy &D, const Expr *E) {
6942	assert(!DiscardResult && "Should've been checked before");
6943
6944	unsigned DummyID = P.getOrCreateDummy(D);
6945
6946	if (!this->emitGetPtrGlobal(DummyID, E))
6947	return false;
6948	if (E->getType()->isVoidType())
6949	return true;
6950
6951	// Convert the dummy pointer to another pointer type if we have to.
6952	if (PrimType PT = classifyPrim(E); PT != PT_Ptr) {
6953	if (isPtrType(T: PT))
6954	return this->emitDecayPtr(PT_Ptr, PT, E);
6955	return false;
6956	}
6957	return true;
6958	}
6959
6960	// This function is constexpr if and only if To, From, and the types of
6961	// all subobjects of To and From are types T such that...
6962	// (3.1) - is_union_v<T> is false;
6963	// (3.2) - is_pointer_v<T> is false;
6964	// (3.3) - is_member_pointer_v<T> is false;
6965	// (3.4) - is_volatile_v<T> is false; and
6966	// (3.5) - T has no non-static data members of reference type
6967	template <class Emitter>
6968	bool Compiler<Emitter>::emitBuiltinBitCast(const CastExpr *E) {
6969	const Expr *SubExpr = E->getSubExpr();
6970	QualType FromType = SubExpr->getType();
6971	QualType ToType = E->getType();
6972	std::optional<PrimType> ToT = classify(ToType);
6973
6974	assert(!ToType->isReferenceType());
6975
6976	// Prepare storage for the result in case we discard.
6977	if (DiscardResult && !Initializing && !ToT) {
6978	std::optional<unsigned> LocalIndex = allocateLocal(Src: E);
6979	if (!LocalIndex)
6980	return false;
6981	if (!this->emitGetPtrLocal(*LocalIndex, E))
6982	return false;
6983	}
6984
6985	// Get a pointer to the value-to-cast on the stack.
6986	// For CK_LValueToRValueBitCast, this is always an lvalue and
6987	// we later assume it to be one (i.e. a PT_Ptr). However,
6988	// we call this function for other utility methods where
6989	// a bitcast might be useful, so convert it to a PT_Ptr in that case.
6990	if (SubExpr->isGLValue() || FromType->isVectorType()) {
6991	if (!this->visit(SubExpr))
6992	return false;
6993	} else if (std::optional<PrimType> FromT = classify(SubExpr)) {
6994	unsigned TempOffset =
6995	allocateLocalPrimitive(Src: SubExpr, Ty: *FromT, /*IsConst=*/true);
6996	if (!this->visit(SubExpr))
6997	return false;
6998	if (!this->emitSetLocal(*FromT, TempOffset, E))
6999	return false;
7000	if (!this->emitGetPtrLocal(TempOffset, E))
7001	return false;
7002	} else {
7003	return false;
7004	}
7005
7006	if (!ToT) {
7007	if (!this->emitBitCast(E))
7008	return false;
7009	return DiscardResult ? this->emitPopPtr(E) : true;
7010	}
7011	assert(ToT);
7012
7013	const llvm::fltSemantics *TargetSemantics = nullptr;
7014	if (ToT == PT_Float)
7015	TargetSemantics = &Ctx.getFloatSemantics(T: ToType);
7016
7017	// Conversion to a primitive type. FromType can be another
7018	// primitive type, or a record/array.
7019	bool ToTypeIsUChar = (ToType->isSpecificBuiltinType(K: BuiltinType::UChar) ||
7020	ToType->isSpecificBuiltinType(K: BuiltinType::Char_U));
7021	uint32_t ResultBitWidth = std::max(a: Ctx.getBitWidth(T: ToType), b: 8u);
7022
7023	if (!this->emitBitCastPrim(*ToT, ToTypeIsUChar || ToType->isStdByteType(),
7024	ResultBitWidth, TargetSemantics, E))
7025	return false;
7026
7027	if (DiscardResult)
7028	return this->emitPop(*ToT, E);
7029
7030	return true;
7031	}
7032
7033	namespace clang {
7034	namespace interp {
7035
7036	template class Compiler<ByteCodeEmitter>;
7037	template class Compiler<EvalEmitter>;
7038
7039	} // namespace interp
7040	} // namespace clang
7041