1	//===--- Compiler.cpp - Code generator for expressions ---*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8
9	#include "Compiler.h"
10	#include "ByteCodeEmitter.h"
11	#include "Context.h"
12	#include "FixedPoint.h"
13	#include "Floating.h"
14	#include "Function.h"
15	#include "InterpShared.h"
16	#include "PrimType.h"
17	#include "Program.h"
18	#include "clang/AST/Attr.h"
19
20	using namespace clang;
21	using namespace clang::interp;
22
23	using APSInt = llvm::APSInt;
24
25	namespace clang {
26	namespace interp {
27
28	static std::optional<bool> getBoolValue(const Expr *E) {
29	if (const auto *CE = dyn_cast_if_present<ConstantExpr>(Val: E);
30	CE && CE->hasAPValueResult() &&
31	CE->getResultAPValueKind() == APValue::ValueKind::Int) {
32	return CE->getResultAsAPSInt().getBoolValue();
33	}
34
35	return std::nullopt;
36	}
37
38	/// Scope used to handle temporaries in toplevel variable declarations.
39	template <class Emitter> class DeclScope final : public LocalScope<Emitter> {
40	public:
41	DeclScope(Compiler<Emitter> *Ctx, const ValueDecl *VD)
42	: LocalScope<Emitter>(Ctx, VD), Scope(Ctx->P),
43	OldInitializingDecl(Ctx->InitializingDecl) {
44	Ctx->InitializingDecl = VD;
45	Ctx->InitStack.push_back(InitLink::Decl(D: VD));
46	}
47
48	~DeclScope() {
49	this->Ctx->InitializingDecl = OldInitializingDecl;
50	this->Ctx->InitStack.pop_back();
51	}
52
53	private:
54	Program::DeclScope Scope;
55	const ValueDecl *OldInitializingDecl;
56	};
57
58	/// Scope used to handle initialization methods.
59	template <class Emitter> class OptionScope final {
60	public:
61	/// Root constructor, compiling or discarding primitives.
62	OptionScope(Compiler<Emitter> *Ctx, bool NewDiscardResult,
63	bool NewInitializing)
64	: Ctx(Ctx), OldDiscardResult(Ctx->DiscardResult),
65	OldInitializing(Ctx->Initializing) {
66	Ctx->DiscardResult = NewDiscardResult;
67	Ctx->Initializing = NewInitializing;
68	}
69
70	~OptionScope() {
71	Ctx->DiscardResult = OldDiscardResult;
72	Ctx->Initializing = OldInitializing;
73	}
74
75	private:
76	/// Parent context.
77	Compiler<Emitter> *Ctx;
78	/// Old discard flag to restore.
79	bool OldDiscardResult;
80	bool OldInitializing;
81	};
82
83	template <class Emitter>
84	bool InitLink::emit(Compiler<Emitter> *Ctx, const Expr *E) const {
85	switch (Kind) {
86	case K_This:
87	return Ctx->emitThis(E);
88	case K_Field:
89	// We're assuming there's a base pointer on the stack already.
90	return Ctx->emitGetPtrFieldPop(Offset, E);
91	case K_Temp:
92	return Ctx->emitGetPtrLocal(Offset, E);
93	case K_Decl:
94	return Ctx->visitDeclRef(D, E);
95	case K_Elem:
96	if (!Ctx->emitConstUint32(Offset, E))
97	return false;
98	return Ctx->emitArrayElemPtrPopUint32(E);
99	case K_RVO:
100	return Ctx->emitRVOPtr(E);
101	case K_InitList:
102	return true;
103	default:
104	llvm_unreachable("Unhandled InitLink kind");
105	}
106	return true;
107	}
108
109	/// Scope managing label targets.
110	template <class Emitter> class LabelScope {
111	public:
112	virtual ~LabelScope() {}
113
114	protected:
115	LabelScope(Compiler<Emitter> *Ctx) : Ctx(Ctx) {}
116	/// Compiler instance.
117	Compiler<Emitter> *Ctx;
118	};
119
120	/// Sets the context for break/continue statements.
121	template <class Emitter> class LoopScope final : public LabelScope<Emitter> {
122	public:
123	using LabelTy = typename Compiler<Emitter>::LabelTy;
124	using OptLabelTy = typename Compiler<Emitter>::OptLabelTy;
125
126	LoopScope(Compiler<Emitter> *Ctx, LabelTy BreakLabel, LabelTy ContinueLabel)
127	: LabelScope<Emitter>(Ctx), OldBreakLabel(Ctx->BreakLabel),
128	OldContinueLabel(Ctx->ContinueLabel),
129	OldBreakVarScope(Ctx->BreakVarScope),
130	OldContinueVarScope(Ctx->ContinueVarScope) {
131	this->Ctx->BreakLabel = BreakLabel;
132	this->Ctx->ContinueLabel = ContinueLabel;
133	this->Ctx->BreakVarScope = this->Ctx->VarScope;
134	this->Ctx->ContinueVarScope = this->Ctx->VarScope;
135	}
136
137	~LoopScope() {
138	this->Ctx->BreakLabel = OldBreakLabel;
139	this->Ctx->ContinueLabel = OldContinueLabel;
140	this->Ctx->ContinueVarScope = OldContinueVarScope;
141	this->Ctx->BreakVarScope = OldBreakVarScope;
142	}
143
144	private:
145	OptLabelTy OldBreakLabel;
146	OptLabelTy OldContinueLabel;
147	VariableScope<Emitter> *OldBreakVarScope;
148	VariableScope<Emitter> *OldContinueVarScope;
149	};
150
151	// Sets the context for a switch scope, mapping labels.
152	template <class Emitter> class SwitchScope final : public LabelScope<Emitter> {
153	public:
154	using LabelTy = typename Compiler<Emitter>::LabelTy;
155	using OptLabelTy = typename Compiler<Emitter>::OptLabelTy;
156	using CaseMap = typename Compiler<Emitter>::CaseMap;
157
158	SwitchScope(Compiler<Emitter> *Ctx, CaseMap &&CaseLabels, LabelTy BreakLabel,
159	OptLabelTy DefaultLabel)
160	: LabelScope<Emitter>(Ctx), OldBreakLabel(Ctx->BreakLabel),
161	OldDefaultLabel(this->Ctx->DefaultLabel),
162	OldCaseLabels(std::move(this->Ctx->CaseLabels)),
163	OldLabelVarScope(Ctx->BreakVarScope) {
164	this->Ctx->BreakLabel = BreakLabel;
165	this->Ctx->DefaultLabel = DefaultLabel;
166	this->Ctx->CaseLabels = std::move(CaseLabels);
167	this->Ctx->BreakVarScope = this->Ctx->VarScope;
168	}
169
170	~SwitchScope() {
171	this->Ctx->BreakLabel = OldBreakLabel;
172	this->Ctx->DefaultLabel = OldDefaultLabel;
173	this->Ctx->CaseLabels = std::move(OldCaseLabels);
174	this->Ctx->BreakVarScope = OldLabelVarScope;
175	}
176
177	private:
178	OptLabelTy OldBreakLabel;
179	OptLabelTy OldDefaultLabel;
180	CaseMap OldCaseLabels;
181	VariableScope<Emitter> *OldLabelVarScope;
182	};
183
184	template <class Emitter> class StmtExprScope final {
185	public:
186	StmtExprScope(Compiler<Emitter> *Ctx) : Ctx(Ctx), OldFlag(Ctx->InStmtExpr) {
187	Ctx->InStmtExpr = true;
188	}
189
190	~StmtExprScope() { Ctx->InStmtExpr = OldFlag; }
191
192	private:
193	Compiler<Emitter> *Ctx;
194	bool OldFlag;
195	};
196
197	} // namespace interp
198	} // namespace clang
199
200	template <class Emitter>
201	bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
202	const Expr *SubExpr = CE->getSubExpr();
203
204	if (DiscardResult)
205	return this->delegate(E: SubExpr);
206
207	switch (CE->getCastKind()) {
208	case CK_LValueToRValue: {
209	if (SubExpr->getType().isVolatileQualified())
210	return this->emitInvalidCast(CastKind::Volatile, /*Fatal=*/true, CE);
211
212	std::optional<PrimType> SubExprT = classify(SubExpr->getType());
213	// Prepare storage for the result.
214	if (!Initializing && !SubExprT) {
215	std::optional<unsigned> LocalIndex = allocateLocal(Decl: SubExpr);
216	if (!LocalIndex)
217	return false;
218	if (!this->emitGetPtrLocal(*LocalIndex, CE))
219	return false;
220	}
221
222	if (!this->visit(E: SubExpr))
223	return false;
224
225	if (SubExprT)
226	return this->emitLoadPop(*SubExprT, CE);
227
228	// If the subexpr type is not primitive, we need to perform a copy here.
229	// This happens for example in C when dereferencing a pointer of struct
230	// type.
231	return this->emitMemcpy(CE);
232	}
233
234	case CK_DerivedToBaseMemberPointer: {
235	assert(classifyPrim(CE->getType()) == PT_MemberPtr);
236	assert(classifyPrim(SubExpr->getType()) == PT_MemberPtr);
237	const auto *FromMP = SubExpr->getType()->castAs<MemberPointerType>();
238	const auto *ToMP = CE->getType()->castAs<MemberPointerType>();
239
240	unsigned DerivedOffset =
241	Ctx.collectBaseOffset(BaseDecl: ToMP->getMostRecentCXXRecordDecl(),
242	DerivedDecl: FromMP->getMostRecentCXXRecordDecl());
243
244	if (!this->delegate(E: SubExpr))
245	return false;
246
247	return this->emitGetMemberPtrBasePop(DerivedOffset, CE);
248	}
249
250	case CK_BaseToDerivedMemberPointer: {
251	assert(classifyPrim(CE) == PT_MemberPtr);
252	assert(classifyPrim(SubExpr) == PT_MemberPtr);
253	const auto *FromMP = SubExpr->getType()->castAs<MemberPointerType>();
254	const auto *ToMP = CE->getType()->castAs<MemberPointerType>();
255
256	unsigned DerivedOffset =
257	Ctx.collectBaseOffset(BaseDecl: FromMP->getMostRecentCXXRecordDecl(),
258	DerivedDecl: ToMP->getMostRecentCXXRecordDecl());
259
260	if (!this->delegate(E: SubExpr))
261	return false;
262	return this->emitGetMemberPtrBasePop(-DerivedOffset, CE);
263	}
264
265	case CK_UncheckedDerivedToBase:
266	case CK_DerivedToBase: {
267	if (!this->delegate(E: SubExpr))
268	return false;
269
270	const auto extractRecordDecl = [](QualType Ty) -> const CXXRecordDecl * {
271	if (const auto *PT = dyn_cast<PointerType>(Val&: Ty))
272	return PT->getPointeeType()->getAsCXXRecordDecl();
273	return Ty->getAsCXXRecordDecl();
274	};
275
276	// FIXME: We can express a series of non-virtual casts as a single
277	// GetPtrBasePop op.
278	QualType CurType = SubExpr->getType();
279	for (const CXXBaseSpecifier *B : CE->path()) {
280	if (B->isVirtual()) {
281	if (!this->emitGetPtrVirtBasePop(extractRecordDecl(B->getType()), CE))
282	return false;
283	CurType = B->getType();
284	} else {
285	unsigned DerivedOffset = collectBaseOffset(BaseType: B->getType(), DerivedType: CurType);
286	if (!this->emitGetPtrBasePop(
287	DerivedOffset, /*NullOK=*/CE->getType()->isPointerType(), CE))
288	return false;
289	CurType = B->getType();
290	}
291	}
292
293	return true;
294	}
295
296	case CK_BaseToDerived: {
297	if (!this->delegate(E: SubExpr))
298	return false;
299	unsigned DerivedOffset =
300	collectBaseOffset(BaseType: SubExpr->getType(), DerivedType: CE->getType());
301
302	const Type *TargetType = CE->getType().getTypePtr();
303	if (TargetType->isPointerOrReferenceType())
304	TargetType = TargetType->getPointeeType().getTypePtr();
305	return this->emitGetPtrDerivedPop(DerivedOffset,
306	/*NullOK=*/CE->getType()->isPointerType(),
307	TargetType, CE);
308	}
309
310	case CK_FloatingCast: {
311	// HLSL uses CK_FloatingCast to cast between vectors.
312	if (!SubExpr->getType()->isFloatingType() ||
313	!CE->getType()->isFloatingType())
314	return false;
315	if (!this->visit(E: SubExpr))
316	return false;
317	const auto *TargetSemantics = &Ctx.getFloatSemantics(T: CE->getType());
318	return this->emitCastFP(TargetSemantics, getRoundingMode(E: CE), CE);
319	}
320
321	case CK_IntegralToFloating: {
322	if (!CE->getType()->isRealFloatingType())
323	return false;
324	if (!this->visit(E: SubExpr))
325	return false;
326	const auto *TargetSemantics = &Ctx.getFloatSemantics(T: CE->getType());
327	return this->emitCastIntegralFloating(
328	classifyPrim(SubExpr), TargetSemantics, getFPOptions(E: CE), CE);
329	}
330
331	case CK_FloatingToBoolean: {
332	if (!SubExpr->getType()->isRealFloatingType() ||
333	!CE->getType()->isBooleanType())
334	return false;
335	if (const auto *FL = dyn_cast<FloatingLiteral>(Val: SubExpr))
336	return this->emitConstBool(FL->getValue().isNonZero(), CE);
337	if (!this->visit(E: SubExpr))
338	return false;
339	return this->emitCastFloatingIntegralBool(getFPOptions(E: CE), CE);
340	}
341
342	case CK_FloatingToIntegral: {
343	if (!this->visit(E: SubExpr))
344	return false;
345	PrimType ToT = classifyPrim(CE);
346	if (ToT == PT_IntAP)
347	return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(T: CE->getType()),
348	getFPOptions(E: CE), CE);
349	if (ToT == PT_IntAPS)
350	return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(T: CE->getType()),
351	getFPOptions(E: CE), CE);
352
353	return this->emitCastFloatingIntegral(ToT, getFPOptions(E: CE), CE);
354	}
355
356	case CK_NullToPointer:
357	case CK_NullToMemberPointer: {
358	if (!this->discard(E: SubExpr))
359	return false;
360	const Descriptor *Desc = nullptr;
361	const QualType PointeeType = CE->getType()->getPointeeType();
362	if (!PointeeType.isNull()) {
363	if (std::optional<PrimType> T = classify(PointeeType))
364	Desc = P.createDescriptor(D: SubExpr, T: *T);
365	else
366	Desc = P.createDescriptor(D: SubExpr, Ty: PointeeType.getTypePtr(),
367	MDSize: std::nullopt, /*IsConst=*/true);
368	}
369
370	uint64_t Val = Ctx.getASTContext().getTargetNullPointerValue(QT: CE->getType());
371	return this->emitNull(classifyPrim(CE->getType()), Val, Desc, CE);
372	}
373
374	case CK_PointerToIntegral: {
375	if (!this->visit(E: SubExpr))
376	return false;
377
378	// If SubExpr doesn't result in a pointer, make it one.
379	if (PrimType FromT = classifyPrim(SubExpr->getType()); FromT != PT_Ptr) {
380	assert(isPtrType(FromT));
381	if (!this->emitDecayPtr(FromT, PT_Ptr, CE))
382	return false;
383	}
384
385	PrimType T = classifyPrim(CE->getType());
386	if (T == PT_IntAP)
387	return this->emitCastPointerIntegralAP(Ctx.getBitWidth(T: CE->getType()),
388	CE);
389	if (T == PT_IntAPS)
390	return this->emitCastPointerIntegralAPS(Ctx.getBitWidth(T: CE->getType()),
391	CE);
392	return this->emitCastPointerIntegral(T, CE);
393	}
394
395	case CK_ArrayToPointerDecay: {
396	if (!this->visit(E: SubExpr))
397	return false;
398	return this->emitArrayDecay(CE);
399	}
400
401	case CK_IntegralToPointer: {
402	QualType IntType = SubExpr->getType();
403	assert(IntType->isIntegralOrEnumerationType());
404	if (!this->visit(E: SubExpr))
405	return false;
406	// FIXME: I think the discard is wrong since the int->ptr cast might cause a
407	// diagnostic.
408	PrimType T = classifyPrim(IntType);
409	QualType PtrType = CE->getType();
410	const Descriptor *Desc;
411	if (std::optional<PrimType> T = classify(PtrType->getPointeeType()))
412	Desc = P.createDescriptor(D: SubExpr, T: *T);
413	else if (PtrType->getPointeeType()->isVoidType())
414	Desc = nullptr;
415	else
416	Desc = P.createDescriptor(D: CE, Ty: PtrType->getPointeeType().getTypePtr(),
417	MDSize: Descriptor::InlineDescMD, /*IsConst=*/true);
418
419	if (!this->emitGetIntPtr(T, Desc, CE))
420	return false;
421
422	PrimType DestPtrT = classifyPrim(PtrType);
423	if (DestPtrT == PT_Ptr)
424	return true;
425
426	// In case we're converting the integer to a non-Pointer.
427	return this->emitDecayPtr(PT_Ptr, DestPtrT, CE);
428	}
429
430	case CK_AtomicToNonAtomic:
431	case CK_ConstructorConversion:
432	case CK_FunctionToPointerDecay:
433	case CK_NonAtomicToAtomic:
434	case CK_NoOp:
435	case CK_UserDefinedConversion:
436	case CK_AddressSpaceConversion:
437	case CK_CPointerToObjCPointerCast:
438	return this->delegate(E: SubExpr);
439
440	case CK_BitCast: {
441	// Reject bitcasts to atomic types.
442	if (CE->getType()->isAtomicType()) {
443	if (!this->discard(E: SubExpr))
444	return false;
445	return this->emitInvalidCast(CastKind::Reinterpret, /*Fatal=*/true, CE);
446	}
447	QualType SubExprTy = SubExpr->getType();
448	std::optional<PrimType> FromT = classify(SubExprTy);
449	// Casts from integer/vector to vector.
450	if (CE->getType()->isVectorType())
451	return this->emitBuiltinBitCast(E: CE);
452
453	std::optional<PrimType> ToT = classify(CE->getType());
454	if (!FromT || !ToT)
455	return false;
456
457	assert(isPtrType(*FromT));
458	assert(isPtrType(*ToT));
459	if (FromT == ToT) {
460	if (CE->getType()->isVoidPointerType())
461	return this->delegate(E: SubExpr);
462
463	if (!this->visit(E: SubExpr))
464	return false;
465	if (CE->getType()->isFunctionPointerType())
466	return true;
467	if (FromT == PT_Ptr)
468	return this->emitPtrPtrCast(SubExprTy->isVoidPointerType(), CE);
469	return true;
470	}
471
472	if (!this->visit(E: SubExpr))
473	return false;
474	return this->emitDecayPtr(*FromT, *ToT, CE);
475	}
476	case CK_IntegralToBoolean:
477	case CK_FixedPointToBoolean: {
478	// HLSL uses this to cast to one-element vectors.
479	std::optional<PrimType> FromT = classify(SubExpr->getType());
480	if (!FromT)
481	return false;
482
483	if (const auto *IL = dyn_cast<IntegerLiteral>(Val: SubExpr))
484	return this->emitConst(IL->getValue(), CE);
485	if (!this->visit(E: SubExpr))
486	return false;
487	return this->emitCast(*FromT, classifyPrim(CE), CE);
488	}
489
490	case CK_BooleanToSignedIntegral:
491	case CK_IntegralCast: {
492	std::optional<PrimType> FromT = classify(SubExpr->getType());
493	std::optional<PrimType> ToT = classify(CE->getType());
494	if (!FromT || !ToT)
495	return false;
496
497	// Try to emit a casted known constant value directly.
498	if (const auto *IL = dyn_cast<IntegerLiteral>(Val: SubExpr)) {
499	if (ToT != PT_IntAP && ToT != PT_IntAPS && FromT != PT_IntAP &&
500	FromT != PT_IntAPS && !CE->getType()->isEnumeralType())
501	return this->emitConst(IL->getValue(), CE);
502	if (!this->emitConst(IL->getValue(), SubExpr))
503	return false;
504	} else {
505	if (!this->visit(E: SubExpr))
506	return false;
507	}
508
509	// Possibly diagnose casts to enum types if the target type does not
510	// have a fixed size.
511	if (Ctx.getLangOpts().CPlusPlus && CE->getType()->isEnumeralType()) {
512	if (const auto *ET = CE->getType().getCanonicalType()->castAs<EnumType>();
513	!ET->getDecl()->isFixed()) {
514	if (!this->emitCheckEnumValue(*FromT, ET->getDecl(), CE))
515	return false;
516	}
517	}
518
519	if (ToT == PT_IntAP) {
520	if (!this->emitCastAP(*FromT, Ctx.getBitWidth(T: CE->getType()), CE))
521	return false;
522	} else if (ToT == PT_IntAPS) {
523	if (!this->emitCastAPS(*FromT, Ctx.getBitWidth(T: CE->getType()), CE))
524	return false;
525	} else {
526	if (FromT == ToT)
527	return true;
528	if (!this->emitCast(*FromT, *ToT, CE))
529	return false;
530	}
531	if (CE->getCastKind() == CK_BooleanToSignedIntegral)
532	return this->emitNeg(*ToT, CE);
533	return true;
534	}
535
536	case CK_PointerToBoolean:
537	case CK_MemberPointerToBoolean: {
538	PrimType PtrT = classifyPrim(SubExpr->getType());
539
540	if (!this->visit(E: SubExpr))
541	return false;
542	return this->emitIsNonNull(PtrT, CE);
543	}
544
545	case CK_IntegralComplexToBoolean:
546	case CK_FloatingComplexToBoolean: {
547	if (!this->visit(E: SubExpr))
548	return false;
549	return this->emitComplexBoolCast(E: SubExpr);
550	}
551
552	case CK_IntegralComplexToReal:
553	case CK_FloatingComplexToReal:
554	return this->emitComplexReal(SubExpr);
555
556	case CK_IntegralRealToComplex:
557	case CK_FloatingRealToComplex: {
558	// We're creating a complex value here, so we need to
559	// allocate storage for it.
560	if (!Initializing) {
561	std::optional<unsigned> LocalIndex = allocateTemporary(E: CE);
562	if (!LocalIndex)
563	return false;
564	if (!this->emitGetPtrLocal(*LocalIndex, CE))
565	return false;
566	}
567
568	PrimType T = classifyPrim(SubExpr->getType());
569	// Init the complex value to {SubExpr, 0}.
570	if (!this->visitArrayElemInit(ElemIndex: 0, Init: SubExpr, InitT: T))
571	return false;
572	// Zero-init the second element.
573	if (!this->visitZeroInitializer(T, QT: SubExpr->getType(), E: SubExpr))
574	return false;
575	return this->emitInitElem(T, 1, SubExpr);
576	}
577
578	case CK_IntegralComplexCast:
579	case CK_FloatingComplexCast:
580	case CK_IntegralComplexToFloatingComplex:
581	case CK_FloatingComplexToIntegralComplex: {
582	assert(CE->getType()->isAnyComplexType());
583	assert(SubExpr->getType()->isAnyComplexType());
584	if (!Initializing) {
585	std::optional<unsigned> LocalIndex = allocateLocal(Decl: CE);
586	if (!LocalIndex)
587	return false;
588	if (!this->emitGetPtrLocal(*LocalIndex, CE))
589	return false;
590	}
591
592	// Location for the SubExpr.
593	// Since SubExpr is of complex type, visiting it results in a pointer
594	// anyway, so we just create a temporary pointer variable.
595	unsigned SubExprOffset =
596	allocateLocalPrimitive(Decl: SubExpr, Ty: PT_Ptr, /*IsConst=*/true);
597	if (!this->visit(E: SubExpr))
598	return false;
599	if (!this->emitSetLocal(PT_Ptr, SubExprOffset, CE))
600	return false;
601
602	PrimType SourceElemT = classifyComplexElementType(T: SubExpr->getType());
603	QualType DestElemType =
604	CE->getType()->getAs<ComplexType>()->getElementType();
605	PrimType DestElemT = classifyPrim(DestElemType);
606	// Cast both elements individually.
607	for (unsigned I = 0; I != 2; ++I) {
608	if (!this->emitGetLocal(PT_Ptr, SubExprOffset, CE))
609	return false;
610	if (!this->emitArrayElemPop(SourceElemT, I, CE))
611	return false;
612
613	// Do the cast.
614	if (!this->emitPrimCast(FromT: SourceElemT, ToT: DestElemT, ToQT: DestElemType, E: CE))
615	return false;
616
617	// Save the value.
618	if (!this->emitInitElem(DestElemT, I, CE))
619	return false;
620	}
621	return true;
622	}
623
624	case CK_VectorSplat: {
625	assert(!classify(CE->getType()));
626	assert(classify(SubExpr->getType()));
627	assert(CE->getType()->isVectorType());
628
629	if (!Initializing) {
630	std::optional<unsigned> LocalIndex = allocateLocal(Decl: CE);
631	if (!LocalIndex)
632	return false;
633	if (!this->emitGetPtrLocal(*LocalIndex, CE))
634	return false;
635	}
636
637	const auto *VT = CE->getType()->getAs<VectorType>();
638	PrimType ElemT = classifyPrim(SubExpr->getType());
639	unsigned ElemOffset =
640	allocateLocalPrimitive(Decl: SubExpr, Ty: ElemT, /*IsConst=*/true);
641
642	// Prepare a local variable for the scalar value.
643	if (!this->visit(E: SubExpr))
644	return false;
645	if (classifyPrim(SubExpr) == PT_Ptr && !this->emitLoadPop(ElemT, CE))
646	return false;
647
648	if (!this->emitSetLocal(ElemT, ElemOffset, CE))
649	return false;
650
651	for (unsigned I = 0; I != VT->getNumElements(); ++I) {
652	if (!this->emitGetLocal(ElemT, ElemOffset, CE))
653	return false;
654	if (!this->emitInitElem(ElemT, I, CE))
655	return false;
656	}
657
658	return true;
659	}
660
661	case CK_HLSLVectorTruncation: {
662	assert(SubExpr->getType()->isVectorType());
663	if (std::optional<PrimType> ResultT = classify(CE)) {
664	assert(!DiscardResult);
665	// Result must be either a float or integer. Take the first element.
666	if (!this->visit(E: SubExpr))
667	return false;
668	return this->emitArrayElemPop(*ResultT, 0, CE);
669	}
670	// Otherwise, this truncates from one vector type to another.
671	assert(CE->getType()->isVectorType());
672
673	if (!Initializing) {
674	std::optional<unsigned> LocalIndex = allocateTemporary(E: CE);
675	if (!LocalIndex)
676	return false;
677	if (!this->emitGetPtrLocal(*LocalIndex, CE))
678	return false;
679	}
680	unsigned ToSize = CE->getType()->getAs<VectorType>()->getNumElements();
681	assert(SubExpr->getType()->getAs<VectorType>()->getNumElements() > ToSize);
682	if (!this->visit(E: SubExpr))
683	return false;
684	return this->emitCopyArray(classifyVectorElementType(T: CE->getType()), 0, 0,
685	ToSize, CE);
686	};
687
688	case CK_IntegralToFixedPoint: {
689	if (!this->visit(E: SubExpr))
690	return false;
691
692	auto Sem =
693	Ctx.getASTContext().getFixedPointSemantics(Ty: CE->getType()).toOpaqueInt();
694	return this->emitCastIntegralFixedPoint(classifyPrim(SubExpr->getType()),
695	Sem, CE);
696	}
697	case CK_FloatingToFixedPoint: {
698	if (!this->visit(E: SubExpr))
699	return false;
700
701	auto Sem =
702	Ctx.getASTContext().getFixedPointSemantics(Ty: CE->getType()).toOpaqueInt();
703	return this->emitCastFloatingFixedPoint(Sem, CE);
704	}
705	case CK_FixedPointToFloating: {
706	if (!this->visit(E: SubExpr))
707	return false;
708	const auto *TargetSemantics = &Ctx.getFloatSemantics(T: CE->getType());
709	return this->emitCastFixedPointFloating(TargetSemantics, CE);
710	}
711	case CK_FixedPointToIntegral: {
712	if (!this->visit(E: SubExpr))
713	return false;
714	return this->emitCastFixedPointIntegral(classifyPrim(CE->getType()), CE);
715	}
716	case CK_FixedPointCast: {
717	if (!this->visit(E: SubExpr))
718	return false;
719	auto Sem =
720	Ctx.getASTContext().getFixedPointSemantics(Ty: CE->getType()).toOpaqueInt();
721	return this->emitCastFixedPoint(Sem, CE);
722	}
723
724	case CK_ToVoid:
725	return discard(E: SubExpr);
726
727	default:
728	return this->emitInvalid(CE);
729	}
730	llvm_unreachable("Unhandled clang::CastKind enum");
731	}
732
733	template <class Emitter>
734	bool Compiler<Emitter>::VisitBuiltinBitCastExpr(const BuiltinBitCastExpr *E) {
735	return this->emitBuiltinBitCast(E);
736	}
737
738	template <class Emitter>
739	bool Compiler<Emitter>::VisitIntegerLiteral(const IntegerLiteral *LE) {
740	if (DiscardResult)
741	return true;
742
743	return this->emitConst(LE->getValue(), LE);
744	}
745
746	template <class Emitter>
747	bool Compiler<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
748	if (DiscardResult)
749	return true;
750
751	APFloat F = E->getValue();
752	return this->emitFloat(F, E);
753	}
754
755	template <class Emitter>
756	bool Compiler<Emitter>::VisitImaginaryLiteral(const ImaginaryLiteral *E) {
757	assert(E->getType()->isAnyComplexType());
758	if (DiscardResult)
759	return true;
760
761	if (!Initializing) {
762	std::optional<unsigned> LocalIndex = allocateTemporary(E);
763	if (!LocalIndex)
764	return false;
765	if (!this->emitGetPtrLocal(*LocalIndex, E))
766	return false;
767	}
768
769	const Expr *SubExpr = E->getSubExpr();
770	PrimType SubExprT = classifyPrim(SubExpr->getType());
771
772	if (!this->visitZeroInitializer(T: SubExprT, QT: SubExpr->getType(), E: SubExpr))
773	return false;
774	if (!this->emitInitElem(SubExprT, 0, SubExpr))
775	return false;
776	return this->visitArrayElemInit(ElemIndex: 1, Init: SubExpr, InitT: SubExprT);
777	}
778
779	template <class Emitter>
780	bool Compiler<Emitter>::VisitFixedPointLiteral(const FixedPointLiteral *E) {
781	assert(E->getType()->isFixedPointType());
782	assert(classifyPrim(E) == PT_FixedPoint);
783
784	if (DiscardResult)
785	return true;
786
787	auto Sem = Ctx.getASTContext().getFixedPointSemantics(Ty: E->getType());
788	APInt Value = E->getValue();
789	return this->emitConstFixedPoint(FixedPoint(Value, Sem), E);
790	}
791
792	template <class Emitter>
793	bool Compiler<Emitter>::VisitParenExpr(const ParenExpr *E) {
794	return this->delegate(E: E->getSubExpr());
795	}
796
797	template <class Emitter>
798	bool Compiler<Emitter>::VisitBinaryOperator(const BinaryOperator *BO) {
799	// Need short-circuiting for these.
800	if (BO->isLogicalOp() && !BO->getType()->isVectorType())
801	return this->VisitLogicalBinOp(E: BO);
802
803	const Expr *LHS = BO->getLHS();
804	const Expr *RHS = BO->getRHS();
805
806	// Handle comma operators. Just discard the LHS
807	// and delegate to RHS.
808	if (BO->isCommaOp()) {
809	if (!this->discard(E: LHS))
810	return false;
811	if (RHS->getType()->isVoidType())
812	return this->discard(E: RHS);
813
814	return this->delegate(E: RHS);
815	}
816
817	if (BO->getType()->isAnyComplexType())
818	return this->VisitComplexBinOp(E: BO);
819	if (BO->getType()->isVectorType())
820	return this->VisitVectorBinOp(E: BO);
821	if ((LHS->getType()->isAnyComplexType() ||
822	RHS->getType()->isAnyComplexType()) &&
823	BO->isComparisonOp())
824	return this->emitComplexComparison(LHS, RHS, E: BO);
825	if (LHS->getType()->isFixedPointType() || RHS->getType()->isFixedPointType())
826	return this->VisitFixedPointBinOp(E: BO);
827
828	if (BO->isPtrMemOp()) {
829	if (!this->visit(E: LHS))
830	return false;
831
832	if (!this->visit(E: RHS))
833	return false;
834
835	if (!this->emitToMemberPtr(BO))
836	return false;
837
838	if (classifyPrim(BO) == PT_MemberPtr)
839	return true;
840
841	if (!this->emitCastMemberPtrPtr(BO))
842	return false;
843	return DiscardResult ? this->emitPopPtr(BO) : true;
844	}
845
846	// Typecheck the args.
847	std::optional<PrimType> LT = classify(LHS);
848	std::optional<PrimType> RT = classify(RHS);
849	std::optional<PrimType> T = classify(BO->getType());
850
851	// Special case for C++'s three-way/spaceship operator <=>, which
852	// returns a std::{strong,weak,partial}_ordering (which is a class, so doesn't
853	// have a PrimType).
854	if (!T && BO->getOpcode() == BO_Cmp) {
855	if (DiscardResult)
856	return true;
857	const ComparisonCategoryInfo *CmpInfo =
858	Ctx.getASTContext().CompCategories.lookupInfoForType(Ty: BO->getType());
859	assert(CmpInfo);
860
861	// We need a temporary variable holding our return value.
862	if (!Initializing) {
863	std::optional<unsigned> ResultIndex = this->allocateLocal(Decl: BO);
864	if (!this->emitGetPtrLocal(*ResultIndex, BO))
865	return false;
866	}
867
868	if (!visit(E: LHS) || !visit(E: RHS))
869	return false;
870
871	return this->emitCMP3(*LT, CmpInfo, BO);
872	}
873
874	if (!LT || !RT || !T)
875	return false;
876
877	// Pointer arithmetic special case.
878	if (BO->getOpcode() == BO_Add || BO->getOpcode() == BO_Sub) {
879	if (isPtrType(T: *T) || (isPtrType(T: *LT) && isPtrType(T: *RT)))
880	return this->VisitPointerArithBinOp(E: BO);
881	}
882
883	// Assignments require us to evalute the RHS first.
884	if (BO->getOpcode() == BO_Assign) {
885
886	if (!visit(E: RHS) || !visit(E: LHS))
887	return false;
888
889	// We don't support assignments in C.
890	if (!Ctx.getLangOpts().CPlusPlus && !this->emitInvalid(BO))
891	return false;
892
893	if (!this->emitFlip(*LT, *RT, BO))
894	return false;
895	} else {
896	if (!visit(E: LHS) || !visit(E: RHS))
897	return false;
898	}
899
900	// For languages such as C, cast the result of one
901	// of our comparision opcodes to T (which is usually int).
902	auto MaybeCastToBool = [this, T, BO](bool Result) {
903	if (!Result)
904	return false;
905	if (DiscardResult)
906	return this->emitPop(*T, BO);
907	if (T != PT_Bool)
908	return this->emitCast(PT_Bool, *T, BO);
909	return true;
910	};
911
912	auto Discard = [this, T, BO](bool Result) {
913	if (!Result)
914	return false;
915	return DiscardResult ? this->emitPop(*T, BO) : true;
916	};
917
918	switch (BO->getOpcode()) {
919	case BO_EQ:
920	return MaybeCastToBool(this->emitEQ(*LT, BO));
921	case BO_NE:
922	return MaybeCastToBool(this->emitNE(*LT, BO));
923	case BO_LT:
924	return MaybeCastToBool(this->emitLT(*LT, BO));
925	case BO_LE:
926	return MaybeCastToBool(this->emitLE(*LT, BO));
927	case BO_GT:
928	return MaybeCastToBool(this->emitGT(*LT, BO));
929	case BO_GE:
930	return MaybeCastToBool(this->emitGE(*LT, BO));
931	case BO_Sub:
932	if (BO->getType()->isFloatingType())
933	return Discard(this->emitSubf(getFPOptions(E: BO), BO));
934	return Discard(this->emitSub(*T, BO));
935	case BO_Add:
936	if (BO->getType()->isFloatingType())
937	return Discard(this->emitAddf(getFPOptions(E: BO), BO));
938	return Discard(this->emitAdd(*T, BO));
939	case BO_Mul:
940	if (BO->getType()->isFloatingType())
941	return Discard(this->emitMulf(getFPOptions(E: BO), BO));
942	return Discard(this->emitMul(*T, BO));
943	case BO_Rem:
944	return Discard(this->emitRem(*T, BO));
945	case BO_Div:
946	if (BO->getType()->isFloatingType())
947	return Discard(this->emitDivf(getFPOptions(E: BO), BO));
948	return Discard(this->emitDiv(*T, BO));
949	case BO_Assign:
950	if (DiscardResult)
951	return LHS->refersToBitField() ? this->emitStoreBitFieldPop(*T, BO)
952	: this->emitStorePop(*T, BO);
953	if (LHS->refersToBitField()) {
954	if (!this->emitStoreBitField(*T, BO))
955	return false;
956	} else {
957	if (!this->emitStore(*T, BO))
958	return false;
959	}
960	// Assignments aren't necessarily lvalues in C.
961	// Load from them in that case.
962	if (!BO->isLValue())
963	return this->emitLoadPop(*T, BO);
964	return true;
965	case BO_And:
966	return Discard(this->emitBitAnd(*T, BO));
967	case BO_Or:
968	return Discard(this->emitBitOr(*T, BO));
969	case BO_Shl:
970	return Discard(this->emitShl(*LT, *RT, BO));
971	case BO_Shr:
972	return Discard(this->emitShr(*LT, *RT, BO));
973	case BO_Xor:
974	return Discard(this->emitBitXor(*T, BO));
975	case BO_LOr:
976	case BO_LAnd:
977	llvm_unreachable("Already handled earlier");
978	default:
979	return false;
980	}
981
982	llvm_unreachable("Unhandled binary op");
983	}
984
985	/// Perform addition/subtraction of a pointer and an integer or
986	/// subtraction of two pointers.
987	template <class Emitter>
988	bool Compiler<Emitter>::VisitPointerArithBinOp(const BinaryOperator *E) {
989	BinaryOperatorKind Op = E->getOpcode();
990	const Expr *LHS = E->getLHS();
991	const Expr *RHS = E->getRHS();
992
993	if ((Op != BO_Add && Op != BO_Sub) ||
994	(!LHS->getType()->isPointerType() && !RHS->getType()->isPointerType()))
995	return false;
996
997	std::optional<PrimType> LT = classify(LHS);
998	std::optional<PrimType> RT = classify(RHS);
999
1000	if (!LT || !RT)
1001	return false;
1002
1003	// Visit the given pointer expression and optionally convert to a PT_Ptr.
1004	auto visitAsPointer = [&](const Expr *E, PrimType T) -> bool {
1005	if (!this->visit(E))
1006	return false;
1007	if (T != PT_Ptr)
1008	return this->emitDecayPtr(T, PT_Ptr, E);
1009	return true;
1010	};
1011
1012	if (LHS->getType()->isPointerType() && RHS->getType()->isPointerType()) {
1013	if (Op != BO_Sub)
1014	return false;
1015
1016	assert(E->getType()->isIntegerType());
1017	if (!visitAsPointer(RHS, *RT) || !visitAsPointer(LHS, *LT))
1018	return false;
1019
1020	PrimType IntT = classifyPrim(E->getType());
1021	if (!this->emitSubPtr(IntT, E))
1022	return false;
1023	return DiscardResult ? this->emitPop(IntT, E) : true;
1024	}
1025
1026	PrimType OffsetType;
1027	if (LHS->getType()->isIntegerType()) {
1028	if (!visitAsPointer(RHS, *RT))
1029	return false;
1030	if (!this->visit(E: LHS))
1031	return false;
1032	OffsetType = *LT;
1033	} else if (RHS->getType()->isIntegerType()) {
1034	if (!visitAsPointer(LHS, *LT))
1035	return false;
1036	if (!this->visit(E: RHS))
1037	return false;
1038	OffsetType = *RT;
1039	} else {
1040	return false;
1041	}
1042
1043	// Do the operation and optionally transform to
1044	// result pointer type.
1045	if (Op == BO_Add) {
1046	if (!this->emitAddOffset(OffsetType, E))
1047	return false;
1048
1049	if (classifyPrim(E) != PT_Ptr)
1050	return this->emitDecayPtr(PT_Ptr, classifyPrim(E), E);
1051	return true;
1052	} else if (Op == BO_Sub) {
1053	if (!this->emitSubOffset(OffsetType, E))
1054	return false;
1055
1056	if (classifyPrim(E) != PT_Ptr)
1057	return this->emitDecayPtr(PT_Ptr, classifyPrim(E), E);
1058	return true;
1059	}
1060
1061	return false;
1062	}
1063
1064	template <class Emitter>
1065	bool Compiler<Emitter>::VisitLogicalBinOp(const BinaryOperator *E) {
1066	assert(E->isLogicalOp());
1067	BinaryOperatorKind Op = E->getOpcode();
1068	const Expr *LHS = E->getLHS();
1069	const Expr *RHS = E->getRHS();
1070	std::optional<PrimType> T = classify(E->getType());
1071
1072	if (Op == BO_LOr) {
1073	// Logical OR. Visit LHS and only evaluate RHS if LHS was FALSE.
1074	LabelTy LabelTrue = this->getLabel();
1075	LabelTy LabelEnd = this->getLabel();
1076
1077	if (!this->visitBool(E: LHS))
1078	return false;
1079	if (!this->jumpTrue(LabelTrue))
1080	return false;
1081
1082	if (!this->visitBool(E: RHS))
1083	return false;
1084	if (!this->jump(LabelEnd))
1085	return false;
1086
1087	this->emitLabel(LabelTrue);
1088	this->emitConstBool(true, E);
1089	this->fallthrough(LabelEnd);
1090	this->emitLabel(LabelEnd);
1091
1092	} else {
1093	assert(Op == BO_LAnd);
1094	// Logical AND.
1095	// Visit LHS. Only visit RHS if LHS was TRUE.
1096	LabelTy LabelFalse = this->getLabel();
1097	LabelTy LabelEnd = this->getLabel();
1098
1099	if (!this->visitBool(E: LHS))
1100	return false;
1101	if (!this->jumpFalse(LabelFalse))
1102	return false;
1103
1104	if (!this->visitBool(E: RHS))
1105	return false;
1106	if (!this->jump(LabelEnd))
1107	return false;
1108
1109	this->emitLabel(LabelFalse);
1110	this->emitConstBool(false, E);
1111	this->fallthrough(LabelEnd);
1112	this->emitLabel(LabelEnd);
1113	}
1114
1115	if (DiscardResult)
1116	return this->emitPopBool(E);
1117
1118	// For C, cast back to integer type.
1119	assert(T);
1120	if (T != PT_Bool)
1121	return this->emitCast(PT_Bool, *T, E);
1122	return true;
1123	}
1124
1125	template <class Emitter>
1126	bool Compiler<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
1127	// Prepare storage for result.
1128	if (!Initializing) {
1129	std::optional<unsigned> LocalIndex = allocateTemporary(E);
1130	if (!LocalIndex)
1131	return false;
1132	if (!this->emitGetPtrLocal(*LocalIndex, E))
1133	return false;
1134	}
1135
1136	// Both LHS and RHS might _not_ be of complex type, but one of them
1137	// needs to be.
1138	const Expr *LHS = E->getLHS();
1139	const Expr *RHS = E->getRHS();
1140
1141	PrimType ResultElemT = this->classifyComplexElementType(T: E->getType());
1142	unsigned ResultOffset = ~0u;
1143	if (!DiscardResult)
1144	ResultOffset = this->allocateLocalPrimitive(Decl: E, Ty: PT_Ptr, /*IsConst=*/true);
1145
1146	// Save result pointer in ResultOffset
1147	if (!this->DiscardResult) {
1148	if (!this->emitDupPtr(E))
1149	return false;
1150	if (!this->emitSetLocal(PT_Ptr, ResultOffset, E))
1151	return false;
1152	}
1153	QualType LHSType = LHS->getType();
1154	if (const auto *AT = LHSType->getAs<AtomicType>())
1155	LHSType = AT->getValueType();
1156	QualType RHSType = RHS->getType();
1157	if (const auto *AT = RHSType->getAs<AtomicType>())
1158	RHSType = AT->getValueType();
1159
1160	bool LHSIsComplex = LHSType->isAnyComplexType();
1161	unsigned LHSOffset;
1162	bool RHSIsComplex = RHSType->isAnyComplexType();
1163
1164	// For ComplexComplex Mul, we have special ops to make their implementation
1165	// easier.
1166	BinaryOperatorKind Op = E->getOpcode();
1167	if (Op == BO_Mul && LHSIsComplex && RHSIsComplex) {
1168	assert(classifyPrim(LHSType->getAs<ComplexType>()->getElementType()) ==
1169	classifyPrim(RHSType->getAs<ComplexType>()->getElementType()));
1170	PrimType ElemT =
1171	classifyPrim(LHSType->getAs<ComplexType>()->getElementType());
1172	if (!this->visit(E: LHS))
1173	return false;
1174	if (!this->visit(E: RHS))
1175	return false;
1176	return this->emitMulc(ElemT, E);
1177	}
1178
1179	if (Op == BO_Div && RHSIsComplex) {
1180	QualType ElemQT = RHSType->getAs<ComplexType>()->getElementType();
1181	PrimType ElemT = classifyPrim(ElemQT);
1182	// If the LHS is not complex, we still need to do the full complex
1183	// division, so just stub create a complex value and stub it out with
1184	// the LHS and a zero.
1185
1186	if (!LHSIsComplex) {
1187	// This is using the RHS type for the fake-complex LHS.
1188	std::optional<unsigned> LocalIndex = allocateTemporary(E: RHS);
1189	if (!LocalIndex)
1190	return false;
1191	LHSOffset = *LocalIndex;
1192
1193	if (!this->emitGetPtrLocal(LHSOffset, E))
1194	return false;
1195
1196	if (!this->visit(E: LHS))
1197	return false;
1198	// real is LHS
1199	if (!this->emitInitElem(ElemT, 0, E))
1200	return false;
1201	// imag is zero
1202	if (!this->visitZeroInitializer(T: ElemT, QT: ElemQT, E))
1203	return false;
1204	if (!this->emitInitElem(ElemT, 1, E))
1205	return false;
1206	} else {
1207	if (!this->visit(E: LHS))
1208	return false;
1209	}
1210
1211	if (!this->visit(E: RHS))
1212	return false;
1213	return this->emitDivc(ElemT, E);
1214	}
1215
1216	// Evaluate LHS and save value to LHSOffset.
1217	if (LHSType->isAnyComplexType()) {
1218	LHSOffset = this->allocateLocalPrimitive(Decl: LHS, Ty: PT_Ptr, /*IsConst=*/true);
1219	if (!this->visit(E: LHS))
1220	return false;
1221	if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
1222	return false;
1223	} else {
1224	PrimType LHST = classifyPrim(LHSType);
1225	LHSOffset = this->allocateLocalPrimitive(Decl: LHS, Ty: LHST, /*IsConst=*/true);
1226	if (!this->visit(E: LHS))
1227	return false;
1228	if (!this->emitSetLocal(LHST, LHSOffset, E))
1229	return false;
1230	}
1231
1232	// Same with RHS.
1233	unsigned RHSOffset;
1234	if (RHSType->isAnyComplexType()) {
1235	RHSOffset = this->allocateLocalPrimitive(Decl: RHS, Ty: PT_Ptr, /*IsConst=*/true);
1236	if (!this->visit(E: RHS))
1237	return false;
1238	if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
1239	return false;
1240	} else {
1241	PrimType RHST = classifyPrim(RHSType);
1242	RHSOffset = this->allocateLocalPrimitive(Decl: RHS, Ty: RHST, /*IsConst=*/true);
1243	if (!this->visit(E: RHS))
1244	return false;
1245	if (!this->emitSetLocal(RHST, RHSOffset, E))
1246	return false;
1247	}
1248
1249	// For both LHS and RHS, either load the value from the complex pointer, or
1250	// directly from the local variable. For index 1 (i.e. the imaginary part),
1251	// just load 0 and do the operation anyway.
1252	auto loadComplexValue = [this](bool IsComplex, bool LoadZero,
1253	unsigned ElemIndex, unsigned Offset,
1254	const Expr *E) -> bool {
1255	if (IsComplex) {
1256	if (!this->emitGetLocal(PT_Ptr, Offset, E))
1257	return false;
1258	return this->emitArrayElemPop(classifyComplexElementType(T: E->getType()),
1259	ElemIndex, E);
1260	}
1261	if (ElemIndex == 0 || !LoadZero)
1262	return this->emitGetLocal(classifyPrim(E->getType()), Offset, E);
1263	return this->visitZeroInitializer(T: classifyPrim(E->getType()), QT: E->getType(),
1264	E);
1265	};
1266
1267	// Now we can get pointers to the LHS and RHS from the offsets above.
1268	for (unsigned ElemIndex = 0; ElemIndex != 2; ++ElemIndex) {
1269	// Result pointer for the store later.
1270	if (!this->DiscardResult) {
1271	if (!this->emitGetLocal(PT_Ptr, ResultOffset, E))
1272	return false;
1273	}
1274
1275	// The actual operation.
1276	switch (Op) {
1277	case BO_Add:
1278	if (!loadComplexValue(LHSIsComplex, true, ElemIndex, LHSOffset, LHS))
1279	return false;
1280
1281	if (!loadComplexValue(RHSIsComplex, true, ElemIndex, RHSOffset, RHS))
1282	return false;
1283	if (ResultElemT == PT_Float) {
1284	if (!this->emitAddf(getFPOptions(E), E))
1285	return false;
1286	} else {
1287	if (!this->emitAdd(ResultElemT, E))
1288	return false;
1289	}
1290	break;
1291	case BO_Sub:
1292	if (!loadComplexValue(LHSIsComplex, true, ElemIndex, LHSOffset, LHS))
1293	return false;
1294
1295	if (!loadComplexValue(RHSIsComplex, true, ElemIndex, RHSOffset, RHS))
1296	return false;
1297	if (ResultElemT == PT_Float) {
1298	if (!this->emitSubf(getFPOptions(E), E))
1299	return false;
1300	} else {
1301	if (!this->emitSub(ResultElemT, E))
1302	return false;
1303	}
1304	break;
1305	case BO_Mul:
1306	if (!loadComplexValue(LHSIsComplex, false, ElemIndex, LHSOffset, LHS))
1307	return false;
1308
1309	if (!loadComplexValue(RHSIsComplex, false, ElemIndex, RHSOffset, RHS))
1310	return false;
1311
1312	if (ResultElemT == PT_Float) {
1313	if (!this->emitMulf(getFPOptions(E), E))
1314	return false;
1315	} else {
1316	if (!this->emitMul(ResultElemT, E))
1317	return false;
1318	}
1319	break;
1320	case BO_Div:
1321	assert(!RHSIsComplex);
1322	if (!loadComplexValue(LHSIsComplex, false, ElemIndex, LHSOffset, LHS))
1323	return false;
1324
1325	if (!loadComplexValue(RHSIsComplex, false, ElemIndex, RHSOffset, RHS))
1326	return false;
1327
1328	if (ResultElemT == PT_Float) {
1329	if (!this->emitDivf(getFPOptions(E), E))
1330	return false;
1331	} else {
1332	if (!this->emitDiv(ResultElemT, E))
1333	return false;
1334	}
1335	break;
1336
1337	default:
1338	return false;
1339	}
1340
1341	if (!this->DiscardResult) {
1342	// Initialize array element with the value we just computed.
1343	if (!this->emitInitElemPop(ResultElemT, ElemIndex, E))
1344	return false;
1345	} else {
1346	if (!this->emitPop(ResultElemT, E))
1347	return false;
1348	}
1349	}
1350	return true;
1351	}
1352
1353	template <class Emitter>
1354	bool Compiler<Emitter>::VisitVectorBinOp(const BinaryOperator *E) {
1355	assert(!E->isCommaOp() &&
1356	"Comma op should be handled in VisitBinaryOperator");
1357	assert(E->getType()->isVectorType());
1358	assert(E->getLHS()->getType()->isVectorType());
1359	assert(E->getRHS()->getType()->isVectorType());
1360
1361	// Prepare storage for result.
1362	if (!Initializing && !E->isCompoundAssignmentOp()) {
1363	std::optional<unsigned> LocalIndex = allocateTemporary(E);
1364	if (!LocalIndex)
1365	return false;
1366	if (!this->emitGetPtrLocal(*LocalIndex, E))
1367	return false;
1368	}
1369
1370	const Expr *LHS = E->getLHS();
1371	const Expr *RHS = E->getRHS();
1372	const auto *VecTy = E->getType()->getAs<VectorType>();
1373	auto Op = E->isCompoundAssignmentOp()
1374	? BinaryOperator::getOpForCompoundAssignment(Opc: E->getOpcode())
1375	: E->getOpcode();
1376
1377	PrimType ElemT = this->classifyVectorElementType(T: LHS->getType());
1378	PrimType RHSElemT = this->classifyVectorElementType(T: RHS->getType());
1379	PrimType ResultElemT = this->classifyVectorElementType(T: E->getType());
1380
1381	// Evaluate LHS and save value to LHSOffset.
1382	unsigned LHSOffset =
1383	this->allocateLocalPrimitive(Decl: LHS, Ty: PT_Ptr, /*IsConst=*/true);
1384	if (!this->visit(E: LHS))
1385	return false;
1386	if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
1387	return false;
1388
1389	// Evaluate RHS and save value to RHSOffset.
1390	unsigned RHSOffset =
1391	this->allocateLocalPrimitive(Decl: RHS, Ty: PT_Ptr, /*IsConst=*/true);
1392	if (!this->visit(E: RHS))
1393	return false;
1394	if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
1395	return false;
1396
1397	if (E->isCompoundAssignmentOp() && !this->emitGetLocal(PT_Ptr, LHSOffset, E))
1398	return false;
1399
1400	// BitAdd/BitOr/BitXor/Shl/Shr doesn't support bool type, we need perform the
1401	// integer promotion.
1402	bool NeedIntPromot = ElemT == PT_Bool && (E->isBitwiseOp() || E->isShiftOp());
1403	QualType PromotTy =
1404	Ctx.getASTContext().getPromotedIntegerType(PromotableType: Ctx.getASTContext().BoolTy);
1405	PrimType PromotT = classifyPrim(PromotTy);
1406	PrimType OpT = NeedIntPromot ? PromotT : ElemT;
1407
1408	auto getElem = [=](unsigned Offset, PrimType ElemT, unsigned Index) {
1409	if (!this->emitGetLocal(PT_Ptr, Offset, E))
1410	return false;
1411	if (!this->emitArrayElemPop(ElemT, Index, E))
1412	return false;
1413	if (E->isLogicalOp()) {
1414	if (!this->emitPrimCast(FromT: ElemT, ToT: PT_Bool, ToQT: Ctx.getASTContext().BoolTy, E))
1415	return false;
1416	if (!this->emitPrimCast(FromT: PT_Bool, ToT: ResultElemT, ToQT: VecTy->getElementType(), E))
1417	return false;
1418	} else if (NeedIntPromot) {
1419	if (!this->emitPrimCast(FromT: ElemT, ToT: PromotT, ToQT: PromotTy, E))
1420	return false;
1421	}
1422	return true;
1423	};
1424
1425	#define EMIT_ARITH_OP(OP) \
1426	{ \
1427	if (ElemT == PT_Float) { \
1428	if (!this->emit##OP##f(getFPOptions(E), E)) \
1429	return false; \
1430	} else { \
1431	if (!this->emit##OP(ElemT, E)) \
1432	return false; \
1433	} \
1434	break; \
1435	}
1436
1437	for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
1438	if (!getElem(LHSOffset, ElemT, I))
1439	return false;
1440	if (!getElem(RHSOffset, RHSElemT, I))
1441	return false;
1442	switch (Op) {
1443	case BO_Add:
1444	EMIT_ARITH_OP(Add)
1445	case BO_Sub:
1446	EMIT_ARITH_OP(Sub)
1447	case BO_Mul:
1448	EMIT_ARITH_OP(Mul)
1449	case BO_Div:
1450	EMIT_ARITH_OP(Div)
1451	case BO_Rem:
1452	if (!this->emitRem(ElemT, E))
1453	return false;
1454	break;
1455	case BO_And:
1456	if (!this->emitBitAnd(OpT, E))
1457	return false;
1458	break;
1459	case BO_Or:
1460	if (!this->emitBitOr(OpT, E))
1461	return false;
1462	break;
1463	case BO_Xor:
1464	if (!this->emitBitXor(OpT, E))
1465	return false;
1466	break;
1467	case BO_Shl:
1468	if (!this->emitShl(OpT, RHSElemT, E))
1469	return false;
1470	break;
1471	case BO_Shr:
1472	if (!this->emitShr(OpT, RHSElemT, E))
1473	return false;
1474	break;
1475	case BO_EQ:
1476	if (!this->emitEQ(ElemT, E))
1477	return false;
1478	break;
1479	case BO_NE:
1480	if (!this->emitNE(ElemT, E))
1481	return false;
1482	break;
1483	case BO_LE:
1484	if (!this->emitLE(ElemT, E))
1485	return false;
1486	break;
1487	case BO_LT:
1488	if (!this->emitLT(ElemT, E))
1489	return false;
1490	break;
1491	case BO_GE:
1492	if (!this->emitGE(ElemT, E))
1493	return false;
1494	break;
1495	case BO_GT:
1496	if (!this->emitGT(ElemT, E))
1497	return false;
1498	break;
1499	case BO_LAnd:
1500	// a && b is equivalent to a!=0 & b!=0
1501	if (!this->emitBitAnd(ResultElemT, E))
1502	return false;
1503	break;
1504	case BO_LOr:
1505	// a || b is equivalent to a!=0 | b!=0
1506	if (!this->emitBitOr(ResultElemT, E))
1507	return false;
1508	break;
1509	default:
1510	return this->emitInvalid(E);
1511	}
1512
1513	// The result of the comparison is a vector of the same width and number
1514	// of elements as the comparison operands with a signed integral element
1515	// type.
1516	//
1517	// https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
1518	if (E->isComparisonOp()) {
1519	if (!this->emitPrimCast(FromT: PT_Bool, ToT: ResultElemT, ToQT: VecTy->getElementType(), E))
1520	return false;
1521	if (!this->emitNeg(ResultElemT, E))
1522	return false;
1523	}
1524
1525	// If we performed an integer promotion, we need to cast the compute result
1526	// into result vector element type.
1527	if (NeedIntPromot &&
1528	!this->emitPrimCast(FromT: PromotT, ToT: ResultElemT, ToQT: VecTy->getElementType(), E))
1529	return false;
1530
1531	// Initialize array element with the value we just computed.
1532	if (!this->emitInitElem(ResultElemT, I, E))
1533	return false;
1534	}
1535
1536	if (DiscardResult && E->isCompoundAssignmentOp() && !this->emitPopPtr(E))
1537	return false;
1538	return true;
1539	}
1540
1541	template <class Emitter>
1542	bool Compiler<Emitter>::VisitFixedPointBinOp(const BinaryOperator *E) {
1543	const Expr *LHS = E->getLHS();
1544	const Expr *RHS = E->getRHS();
1545	const ASTContext &ASTCtx = Ctx.getASTContext();
1546
1547	assert(LHS->getType()->isFixedPointType() ||
1548	RHS->getType()->isFixedPointType());
1549
1550	auto LHSSema = ASTCtx.getFixedPointSemantics(Ty: LHS->getType());
1551	auto LHSSemaInt = LHSSema.toOpaqueInt();
1552	auto RHSSema = ASTCtx.getFixedPointSemantics(Ty: RHS->getType());
1553	auto RHSSemaInt = RHSSema.toOpaqueInt();
1554
1555	if (!this->visit(E: LHS))
1556	return false;
1557	if (!LHS->getType()->isFixedPointType()) {
1558	if (!this->emitCastIntegralFixedPoint(classifyPrim(LHS->getType()),
1559	LHSSemaInt, E))
1560	return false;
1561	}
1562
1563	if (!this->visit(E: RHS))
1564	return false;
1565	if (!RHS->getType()->isFixedPointType()) {
1566	if (!this->emitCastIntegralFixedPoint(classifyPrim(RHS->getType()),
1567	RHSSemaInt, E))
1568	return false;
1569	}
1570
1571	// Convert the result to the target semantics.
1572	auto ConvertResult = [&](bool R) -> bool {
1573	if (!R)
1574	return false;
1575	auto ResultSema = ASTCtx.getFixedPointSemantics(Ty: E->getType()).toOpaqueInt();
1576	auto CommonSema = LHSSema.getCommonSemantics(Other: RHSSema).toOpaqueInt();
1577	if (ResultSema != CommonSema)
1578	return this->emitCastFixedPoint(ResultSema, E);
1579	return true;
1580	};
1581
1582	auto MaybeCastToBool = [&](bool Result) {
1583	if (!Result)
1584	return false;
1585	PrimType T = classifyPrim(E);
1586	if (DiscardResult)
1587	return this->emitPop(T, E);
1588	if (T != PT_Bool)
1589	return this->emitCast(PT_Bool, T, E);
1590	return true;
1591	};
1592
1593	switch (E->getOpcode()) {
1594	case BO_EQ:
1595	return MaybeCastToBool(this->emitEQFixedPoint(E));
1596	case BO_NE:
1597	return MaybeCastToBool(this->emitNEFixedPoint(E));
1598	case BO_LT:
1599	return MaybeCastToBool(this->emitLTFixedPoint(E));
1600	case BO_LE:
1601	return MaybeCastToBool(this->emitLEFixedPoint(E));
1602	case BO_GT:
1603	return MaybeCastToBool(this->emitGTFixedPoint(E));
1604	case BO_GE:
1605	return MaybeCastToBool(this->emitGEFixedPoint(E));
1606	case BO_Add:
1607	return ConvertResult(this->emitAddFixedPoint(E));
1608	case BO_Sub:
1609	return ConvertResult(this->emitSubFixedPoint(E));
1610	case BO_Mul:
1611	return ConvertResult(this->emitMulFixedPoint(E));
1612	case BO_Div:
1613	return ConvertResult(this->emitDivFixedPoint(E));
1614	case BO_Shl:
1615	return ConvertResult(this->emitShiftFixedPoint(/*Left=*/true, E));
1616	case BO_Shr:
1617	return ConvertResult(this->emitShiftFixedPoint(/*Left=*/false, E));
1618
1619	default:
1620	return this->emitInvalid(E);
1621	}
1622
1623	llvm_unreachable("unhandled binop opcode");
1624	}
1625
1626	template <class Emitter>
1627	bool Compiler<Emitter>::VisitFixedPointUnaryOperator(const UnaryOperator *E) {
1628	const Expr *SubExpr = E->getSubExpr();
1629	assert(SubExpr->getType()->isFixedPointType());
1630
1631	switch (E->getOpcode()) {
1632	case UO_Plus:
1633	return this->delegate(E: SubExpr);
1634	case UO_Minus:
1635	if (!this->visit(E: SubExpr))
1636	return false;
1637	return this->emitNegFixedPoint(E);
1638	default:
1639	return false;
1640	}
1641
1642	llvm_unreachable("Unhandled unary opcode");
1643	}
1644
1645	template <class Emitter>
1646	bool Compiler<Emitter>::VisitImplicitValueInitExpr(
1647	const ImplicitValueInitExpr *E) {
1648	QualType QT = E->getType();
1649
1650	if (std::optional<PrimType> T = classify(QT))
1651	return this->visitZeroInitializer(T: *T, QT, E);
1652
1653	if (QT->isRecordType()) {
1654	const RecordDecl *RD = QT->getAsRecordDecl();
1655	assert(RD);
1656	if (RD->isInvalidDecl())
1657	return false;
1658
1659	if (const auto *CXXRD = dyn_cast<CXXRecordDecl>(Val: RD);
1660	CXXRD && CXXRD->getNumVBases() > 0) {
1661	// TODO: Diagnose.
1662	return false;
1663	}
1664
1665	const Record *R = getRecord(QT);
1666	if (!R)
1667	return false;
1668
1669	assert(Initializing);
1670	return this->visitZeroRecordInitializer(R, E);
1671	}
1672
1673	if (QT->isIncompleteArrayType())
1674	return true;
1675
1676	if (QT->isArrayType())
1677	return this->visitZeroArrayInitializer(T: QT, E);
1678
1679	if (const auto *ComplexTy = E->getType()->getAs<ComplexType>()) {
1680	assert(Initializing);
1681	QualType ElemQT = ComplexTy->getElementType();
1682	PrimType ElemT = classifyPrim(ElemQT);
1683	for (unsigned I = 0; I < 2; ++I) {
1684	if (!this->visitZeroInitializer(T: ElemT, QT: ElemQT, E))
1685	return false;
1686	if (!this->emitInitElem(ElemT, I, E))
1687	return false;
1688	}
1689	return true;
1690	}
1691
1692	if (const auto *VecT = E->getType()->getAs<VectorType>()) {
1693	unsigned NumVecElements = VecT->getNumElements();
1694	QualType ElemQT = VecT->getElementType();
1695	PrimType ElemT = classifyPrim(ElemQT);
1696
1697	for (unsigned I = 0; I < NumVecElements; ++I) {
1698	if (!this->visitZeroInitializer(T: ElemT, QT: ElemQT, E))
1699	return false;
1700	if (!this->emitInitElem(ElemT, I, E))
1701	return false;
1702	}
1703	return true;
1704	}
1705
1706	return false;
1707	}
1708
1709	template <class Emitter>
1710	bool Compiler<Emitter>::VisitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1711	const Expr *LHS = E->getLHS();
1712	const Expr *RHS = E->getRHS();
1713	const Expr *Index = E->getIdx();
1714	const Expr *Base = E->getBase();
1715
1716	// C++17's rules require us to evaluate the LHS first, regardless of which
1717	// side is the base.
1718	bool Success = true;
1719	for (const Expr *SubExpr : {LHS, RHS}) {
1720	if (!this->visit(E: SubExpr)) {
1721	Success = false;
1722	continue;
1723	}
1724
1725	// Expand the base if this is a subscript on a
1726	// pointer expression.
1727	if (SubExpr == Base && Base->getType()->isPointerType()) {
1728	if (!this->emitExpandPtr(E))
1729	Success = false;
1730	}
1731	}
1732
1733	if (!Success)
1734	return false;
1735
1736	std::optional<PrimType> IndexT = classify(Index->getType());
1737	// In error-recovery cases, the index expression has a dependent type.
1738	if (!IndexT)
1739	return this->emitError(E);
1740	// If the index is first, we need to change that.
1741	if (LHS == Index) {
1742	if (!this->emitFlip(PT_Ptr, *IndexT, E))
1743	return false;
1744	}
1745
1746	if (!this->emitArrayElemPtrPop(*IndexT, E))
1747	return false;
1748	if (DiscardResult)
1749	return this->emitPopPtr(E);
1750	return true;
1751	}
1752
1753	template <class Emitter>
1754	bool Compiler<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
1755	const Expr *ArrayFiller, const Expr *E) {
1756	InitLinkScope<Emitter> ILS(this, InitLink::InitList());
1757
1758	QualType QT = E->getType();
1759	if (const auto *AT = QT->getAs<AtomicType>())
1760	QT = AT->getValueType();
1761
1762	if (QT->isVoidType()) {
1763	if (Inits.size() == 0)
1764	return true;
1765	return this->emitInvalid(E);
1766	}
1767
1768	// Handle discarding first.
1769	if (DiscardResult) {
1770	for (const Expr *Init : Inits) {
1771	if (!this->discard(E: Init))
1772	return false;
1773	}
1774	return true;
1775	}
1776
1777	// Primitive values.
1778	if (std::optional<PrimType> T = classify(QT)) {
1779	assert(!DiscardResult);
1780	if (Inits.size() == 0)
1781	return this->visitZeroInitializer(T: *T, QT, E);
1782	assert(Inits.size() == 1);
1783	return this->delegate(E: Inits[0]);
1784	}
1785
1786	if (QT->isRecordType()) {
1787	const Record *R = getRecord(QT);
1788
1789	if (Inits.size() == 1 && E->getType() == Inits[0]->getType())
1790	return this->delegate(E: Inits[0]);
1791
1792	auto initPrimitiveField = [=](const Record::Field *FieldToInit,
1793	const Expr *Init, PrimType T) -> bool {
1794	InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(Val: Init));
1795	InitLinkScope<Emitter> ILS(this, InitLink::Field(Offset: FieldToInit->Offset));
1796	if (!this->visit(E: Init))
1797	return false;
1798
1799	if (FieldToInit->isBitField())
1800	return this->emitInitBitField(T, FieldToInit, E);
1801	return this->emitInitField(T, FieldToInit->Offset, E);
1802	};
1803
1804	auto initCompositeField = [=](const Record::Field *FieldToInit,
1805	const Expr *Init) -> bool {
1806	InitStackScope<Emitter> ISS(this, isa<CXXDefaultInitExpr>(Val: Init));
1807	InitLinkScope<Emitter> ILS(this, InitLink::Field(Offset: FieldToInit->Offset));
1808
1809	// Non-primitive case. Get a pointer to the field-to-initialize
1810	// on the stack and recurse into visitInitializer().
1811	if (!this->emitGetPtrField(FieldToInit->Offset, Init))
1812	return false;
1813	if (!this->visitInitializer(E: Init))
1814	return false;
1815	return this->emitPopPtr(E);
1816	};
1817
1818	if (R->isUnion()) {
1819	if (Inits.size() == 0) {
1820	if (!this->visitZeroRecordInitializer(R, E))
1821	return false;
1822	} else {
1823	const Expr *Init = Inits[0];
1824	const FieldDecl *FToInit = nullptr;
1825	if (const auto *ILE = dyn_cast<InitListExpr>(Val: E))
1826	FToInit = ILE->getInitializedFieldInUnion();
1827	else
1828	FToInit = cast<CXXParenListInitExpr>(Val: E)->getInitializedFieldInUnion();
1829
1830	const Record::Field *FieldToInit = R->getField(FD: FToInit);
1831	if (std::optional<PrimType> T = classify(Init)) {
1832	if (!initPrimitiveField(FieldToInit, Init, *T))
1833	return false;
1834	} else {
1835	if (!initCompositeField(FieldToInit, Init))
1836	return false;
1837	}
1838	}
1839	return this->emitFinishInit(E);
1840	}
1841
1842	assert(!R->isUnion());
1843	unsigned InitIndex = 0;
1844	for (const Expr *Init : Inits) {
1845	// Skip unnamed bitfields.
1846	while (InitIndex < R->getNumFields() &&
1847	R->getField(I: InitIndex)->isUnnamedBitField())
1848	++InitIndex;
1849
1850	if (std::optional<PrimType> T = classify(Init)) {
1851	const Record::Field *FieldToInit = R->getField(I: InitIndex);
1852	if (!initPrimitiveField(FieldToInit, Init, *T))
1853	return false;
1854	++InitIndex;
1855	} else {
1856	// Initializer for a direct base class.
1857	if (const Record::Base *B = R->getBase(T: Init->getType())) {
1858	if (!this->emitGetPtrBase(B->Offset, Init))
1859	return false;
1860
1861	if (!this->visitInitializer(E: Init))
1862	return false;
1863
1864	if (!this->emitFinishInitPop(E))
1865	return false;
1866	// Base initializers don't increase InitIndex, since they don't count
1867	// into the Record's fields.
1868	} else {
1869	const Record::Field *FieldToInit = R->getField(I: InitIndex);
1870	if (!initCompositeField(FieldToInit, Init))
1871	return false;
1872	++InitIndex;
1873	}
1874	}
1875	}
1876	return this->emitFinishInit(E);
1877	}
1878
1879	if (QT->isArrayType()) {
1880	if (Inits.size() == 1 && QT == Inits[0]->getType())
1881	return this->delegate(E: Inits[0]);
1882
1883	const ConstantArrayType *CAT =
1884	Ctx.getASTContext().getAsConstantArrayType(T: QT);
1885	uint64_t NumElems = CAT->getZExtSize();
1886
1887	if (!this->emitCheckArraySize(NumElems, E))
1888	return false;
1889
1890	std::optional<PrimType> InitT = classify(CAT->getElementType());
1891	unsigned ElementIndex = 0;
1892	for (const Expr *Init : Inits) {
1893	if (const auto *EmbedS =
1894	dyn_cast<EmbedExpr>(Val: Init->IgnoreParenImpCasts())) {
1895	PrimType TargetT = classifyPrim(Init->getType());
1896
1897	auto Eval = [&](const Expr *Init, unsigned ElemIndex) {
1898	PrimType InitT = classifyPrim(Init->getType());
1899	if (!this->visit(E: Init))
1900	return false;
1901	if (InitT != TargetT) {
1902	if (!this->emitCast(InitT, TargetT, E))
1903	return false;
1904	}
1905	return this->emitInitElem(TargetT, ElemIndex, Init);
1906	};
1907	if (!EmbedS->doForEachDataElement(Eval, ElementIndex))
1908	return false;
1909	} else {
1910	if (!this->visitArrayElemInit(ElemIndex: ElementIndex, Init, InitT))
1911	return false;
1912	++ElementIndex;
1913	}
1914	}
1915
1916	// Expand the filler expression.
1917	// FIXME: This should go away.
1918	if (ArrayFiller) {
1919	for (; ElementIndex != NumElems; ++ElementIndex) {
1920	if (!this->visitArrayElemInit(ElemIndex: ElementIndex, Init: ArrayFiller, InitT))
1921	return false;
1922	}
1923	}
1924
1925	return this->emitFinishInit(E);
1926	}
1927
1928	if (const auto *ComplexTy = QT->getAs<ComplexType>()) {
1929	unsigned NumInits = Inits.size();
1930
1931	if (NumInits == 1)
1932	return this->delegate(E: Inits[0]);
1933
1934	QualType ElemQT = ComplexTy->getElementType();
1935	PrimType ElemT = classifyPrim(ElemQT);
1936	if (NumInits == 0) {
1937	// Zero-initialize both elements.
1938	for (unsigned I = 0; I < 2; ++I) {
1939	if (!this->visitZeroInitializer(T: ElemT, QT: ElemQT, E))
1940	return false;
1941	if (!this->emitInitElem(ElemT, I, E))
1942	return false;
1943	}
1944	} else if (NumInits == 2) {
1945	unsigned InitIndex = 0;
1946	for (const Expr *Init : Inits) {
1947	if (!this->visit(E: Init))
1948	return false;
1949
1950	if (!this->emitInitElem(ElemT, InitIndex, E))
1951	return false;
1952	++InitIndex;
1953	}
1954	}
1955	return true;
1956	}
1957
1958	if (const auto *VecT = QT->getAs<VectorType>()) {
1959	unsigned NumVecElements = VecT->getNumElements();
1960	assert(NumVecElements >= Inits.size());
1961
1962	QualType ElemQT = VecT->getElementType();
1963	PrimType ElemT = classifyPrim(ElemQT);
1964
1965	// All initializer elements.
1966	unsigned InitIndex = 0;
1967	for (const Expr *Init : Inits) {
1968	if (!this->visit(E: Init))
1969	return false;
1970
1971	// If the initializer is of vector type itself, we have to deconstruct
1972	// that and initialize all the target fields from the initializer fields.
1973	if (const auto *InitVecT = Init->getType()->getAs<VectorType>()) {
1974	if (!this->emitCopyArray(ElemT, 0, InitIndex,
1975	InitVecT->getNumElements(), E))
1976	return false;
1977	InitIndex += InitVecT->getNumElements();
1978	} else {
1979	if (!this->emitInitElem(ElemT, InitIndex, E))
1980	return false;
1981	++InitIndex;
1982	}
1983	}
1984
1985	assert(InitIndex <= NumVecElements);
1986
1987	// Fill the rest with zeroes.
1988	for (; InitIndex != NumVecElements; ++InitIndex) {
1989	if (!this->visitZeroInitializer(T: ElemT, QT: ElemQT, E))
1990	return false;
1991	if (!this->emitInitElem(ElemT, InitIndex, E))
1992	return false;
1993	}
1994	return true;
1995	}
1996
1997	return false;
1998	}
1999
2000	/// Pointer to the array(not the element!) must be on the stack when calling
2001	/// this.
2002	template <class Emitter>
2003	bool Compiler<Emitter>::visitArrayElemInit(unsigned ElemIndex, const Expr *Init,
2004	std::optional<PrimType> InitT) {
2005	if (InitT) {
2006	// Visit the primitive element like normal.
2007	if (!this->visit(E: Init))
2008	return false;
2009	return this->emitInitElem(*InitT, ElemIndex, Init);
2010	}
2011
2012	InitLinkScope<Emitter> ILS(this, InitLink::Elem(Index: ElemIndex));
2013	// Advance the pointer currently on the stack to the given
2014	// dimension.
2015	if (!this->emitConstUint32(ElemIndex, Init))
2016	return false;
2017	if (!this->emitArrayElemPtrUint32(Init))
2018	return false;
2019	if (!this->visitInitializer(E: Init))
2020	return false;
2021	return this->emitFinishInitPop(Init);
2022	}
2023
2024	template <class Emitter>
2025	bool Compiler<Emitter>::visitCallArgs(ArrayRef<const Expr *> Args,
2026	const FunctionDecl *FuncDecl) {
2027	assert(VarScope->getKind() == ScopeKind::Call);
2028	llvm::BitVector NonNullArgs = collectNonNullArgs(F: FuncDecl, Args);
2029
2030	unsigned ArgIndex = 0;
2031	for (const Expr *Arg : Args) {
2032	if (std::optional<PrimType> T = classify(Arg)) {
2033	if (!this->visit(E: Arg))
2034	return false;
2035	} else {
2036
2037	std::optional<unsigned> LocalIndex = allocateLocal(
2038	Decl: Arg, Ty: Arg->getType(), /*ExtendingDecl=*/nullptr, ScopeKind::Call);
2039	if (!LocalIndex)
2040	return false;
2041
2042	if (!this->emitGetPtrLocal(*LocalIndex, Arg))
2043	return false;
2044	InitLinkScope<Emitter> ILS(this, InitLink::Temp(Offset: *LocalIndex));
2045	if (!this->visitInitializer(E: Arg))
2046	return false;
2047	}
2048
2049	if (FuncDecl && NonNullArgs[ArgIndex]) {
2050	PrimType ArgT = classify(Arg).value_or(PT_Ptr);
2051	if (ArgT == PT_Ptr) {
2052	if (!this->emitCheckNonNullArg(ArgT, Arg))
2053	return false;
2054	}
2055	}
2056
2057	++ArgIndex;
2058	}
2059
2060	return true;
2061	}
2062
2063	template <class Emitter>
2064	bool Compiler<Emitter>::VisitInitListExpr(const InitListExpr *E) {
2065	return this->visitInitList(Inits: E->inits(), ArrayFiller: E->getArrayFiller(), E);
2066	}
2067
2068	template <class Emitter>
2069	bool Compiler<Emitter>::VisitCXXParenListInitExpr(
2070	const CXXParenListInitExpr *E) {
2071	return this->visitInitList(Inits: E->getInitExprs(), ArrayFiller: E->getArrayFiller(), E);
2072	}
2073
2074	template <class Emitter>
2075	bool Compiler<Emitter>::VisitSubstNonTypeTemplateParmExpr(
2076	const SubstNonTypeTemplateParmExpr *E) {
2077	return this->delegate(E: E->getReplacement());
2078	}
2079
2080	template <class Emitter>
2081	bool Compiler<Emitter>::VisitConstantExpr(const ConstantExpr *E) {
2082	std::optional<PrimType> T = classify(E->getType());
2083	if (T && E->hasAPValueResult()) {
2084	// Try to emit the APValue directly, without visiting the subexpr.
2085	// This will only fail if we can't emit the APValue, so won't emit any
2086	// diagnostics or any double values.
2087	if (DiscardResult)
2088	return true;
2089
2090	if (this->visitAPValue(Val: E->getAPValueResult(), ValType: *T, E))
2091	return true;
2092	}
2093	return this->delegate(E: E->getSubExpr());
2094	}
2095
2096	template <class Emitter>
2097	bool Compiler<Emitter>::VisitEmbedExpr(const EmbedExpr *E) {
2098	auto It = E->begin();
2099	return this->visit(E: *It);
2100	}
2101
2102	static CharUnits AlignOfType(QualType T, const ASTContext &ASTCtx,
2103	UnaryExprOrTypeTrait Kind) {
2104	bool AlignOfReturnsPreferred =
2105	ASTCtx.getLangOpts().getClangABICompat() <= LangOptions::ClangABI::Ver7;
2106
2107	// C++ [expr.alignof]p3:
2108	// When alignof is applied to a reference type, the result is the
2109	// alignment of the referenced type.
2110	if (const auto *Ref = T->getAs<ReferenceType>())
2111	T = Ref->getPointeeType();
2112
2113	if (T.getQualifiers().hasUnaligned())
2114	return CharUnits::One();
2115
2116	// __alignof is defined to return the preferred alignment.
2117	// Before 8, clang returned the preferred alignment for alignof and
2118	// _Alignof as well.
2119	if (Kind == UETT_PreferredAlignOf || AlignOfReturnsPreferred)
2120	return ASTCtx.toCharUnitsFromBits(BitSize: ASTCtx.getPreferredTypeAlign(T));
2121
2122	return ASTCtx.getTypeAlignInChars(T);
2123	}
2124
2125	template <class Emitter>
2126	bool Compiler<Emitter>::VisitUnaryExprOrTypeTraitExpr(
2127	const UnaryExprOrTypeTraitExpr *E) {
2128	UnaryExprOrTypeTrait Kind = E->getKind();
2129	const ASTContext &ASTCtx = Ctx.getASTContext();
2130
2131	if (Kind == UETT_SizeOf || Kind == UETT_DataSizeOf) {
2132	QualType ArgType = E->getTypeOfArgument();
2133
2134	// C++ [expr.sizeof]p2: "When applied to a reference or a reference type,
2135	// the result is the size of the referenced type."
2136	if (const auto *Ref = ArgType->getAs<ReferenceType>())
2137	ArgType = Ref->getPointeeType();
2138
2139	CharUnits Size;
2140	if (ArgType->isVoidType() || ArgType->isFunctionType())
2141	Size = CharUnits::One();
2142	else {
2143	if (ArgType->isDependentType() || !ArgType->isConstantSizeType())
2144	return this->emitInvalid(E);
2145
2146	if (Kind == UETT_SizeOf)
2147	Size = ASTCtx.getTypeSizeInChars(T: ArgType);
2148	else
2149	Size = ASTCtx.getTypeInfoDataSizeInChars(T: ArgType).Width;
2150	}
2151
2152	if (DiscardResult)
2153	return true;
2154
2155	return this->emitConst(Size.getQuantity(), E);
2156	}
2157
2158	if (Kind == UETT_CountOf) {
2159	QualType Ty = E->getTypeOfArgument();
2160	assert(Ty->isArrayType());
2161
2162	// We don't need to worry about array element qualifiers, so getting the
2163	// unsafe array type is fine.
2164	if (const auto *CAT =
2165	dyn_cast<ConstantArrayType>(Val: Ty->getAsArrayTypeUnsafe())) {
2166	if (DiscardResult)
2167	return true;
2168	return this->emitConst(CAT->getSize(), E);
2169	}
2170
2171	assert(!Ty->isConstantSizeType());
2172
2173	// If it's a variable-length array type, we need to check whether it is a
2174	// multidimensional array. If so, we need to check the size expression of
2175	// the VLA to see if it's a constant size. If so, we can return that value.
2176	const auto *VAT = ASTCtx.getAsVariableArrayType(T: Ty);
2177	assert(VAT);
2178	if (VAT->getElementType()->isArrayType()) {
2179	std::optional<APSInt> Res =
2180	VAT->getSizeExpr()->getIntegerConstantExpr(Ctx: ASTCtx);
2181	if (Res) {
2182	if (DiscardResult)
2183	return true;
2184	return this->emitConst(*Res, E);
2185	}
2186	}
2187	}
2188
2189	if (Kind == UETT_AlignOf || Kind == UETT_PreferredAlignOf) {
2190	CharUnits Size;
2191
2192	if (E->isArgumentType()) {
2193	QualType ArgType = E->getTypeOfArgument();
2194
2195	Size = AlignOfType(T: ArgType, ASTCtx, Kind);
2196	} else {
2197	// Argument is an expression, not a type.
2198	const Expr *Arg = E->getArgumentExpr()->IgnoreParens();
2199
2200	// The kinds of expressions that we have special-case logic here for
2201	// should be kept up to date with the special checks for those
2202	// expressions in Sema.
2203
2204	// alignof decl is always accepted, even if it doesn't make sense: we
2205	// default to 1 in those cases.
2206	if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: Arg))
2207	Size = ASTCtx.getDeclAlign(D: DRE->getDecl(),
2208	/*RefAsPointee*/ ForAlignof: true);
2209	else if (const auto *ME = dyn_cast<MemberExpr>(Val: Arg))
2210	Size = ASTCtx.getDeclAlign(D: ME->getMemberDecl(),
2211	/*RefAsPointee*/ ForAlignof: true);
2212	else
2213	Size = AlignOfType(T: Arg->getType(), ASTCtx, Kind);
2214	}
2215
2216	if (DiscardResult)
2217	return true;
2218
2219	return this->emitConst(Size.getQuantity(), E);
2220	}
2221
2222	if (Kind == UETT_VectorElements) {
2223	if (const auto *VT = E->getTypeOfArgument()->getAs<VectorType>())
2224	return this->emitConst(VT->getNumElements(), E);
2225	assert(E->getTypeOfArgument()->isSizelessVectorType());
2226	return this->emitSizelessVectorElementSize(E);
2227	}
2228
2229	if (Kind == UETT_VecStep) {
2230	if (const auto *VT = E->getTypeOfArgument()->getAs<VectorType>()) {
2231	unsigned N = VT->getNumElements();
2232
2233	// The vec_step built-in functions that take a 3-component
2234	// vector return 4. (OpenCL 1.1 spec 6.11.12)
2235	if (N == 3)
2236	N = 4;
2237
2238	return this->emitConst(N, E);
2239	}
2240	return this->emitConst(1, E);
2241	}
2242
2243	if (Kind == UETT_OpenMPRequiredSimdAlign) {
2244	assert(E->isArgumentType());
2245	unsigned Bits = ASTCtx.getOpenMPDefaultSimdAlign(T: E->getArgumentType());
2246
2247	return this->emitConst(ASTCtx.toCharUnitsFromBits(BitSize: Bits).getQuantity(), E);
2248	}
2249
2250	if (Kind == UETT_PtrAuthTypeDiscriminator) {
2251	if (E->getArgumentType()->isDependentType())
2252	return this->emitInvalid(E);
2253
2254	return this->emitConst(
2255	const_cast<ASTContext &>(ASTCtx).getPointerAuthTypeDiscriminator(
2256	T: E->getArgumentType()),
2257	E);
2258	}
2259
2260	return false;
2261	}
2262
2263	template <class Emitter>
2264	bool Compiler<Emitter>::VisitMemberExpr(const MemberExpr *E) {
2265	// 'Base.Member'
2266	const Expr *Base = E->getBase();
2267	const ValueDecl *Member = E->getMemberDecl();
2268
2269	if (DiscardResult)
2270	return this->discard(E: Base);
2271
2272	// MemberExprs are almost always lvalues, in which case we don't need to
2273	// do the load. But sometimes they aren't.
2274	const auto maybeLoadValue = [&]() -> bool {
2275	if (E->isGLValue())
2276	return true;
2277	if (std::optional<PrimType> T = classify(E))
2278	return this->emitLoadPop(*T, E);
2279	return false;
2280	};
2281
2282	if (const auto *VD = dyn_cast<VarDecl>(Val: Member)) {
2283	// I am almost confident in saying that a var decl must be static
2284	// and therefore registered as a global variable. But this will probably
2285	// turn out to be wrong some time in the future, as always.
2286	if (auto GlobalIndex = P.getGlobal(VD))
2287	return this->emitGetPtrGlobal(*GlobalIndex, E) && maybeLoadValue();
2288	return false;
2289	}
2290
2291	if (!isa<FieldDecl>(Val: Member)) {
2292	if (!this->discard(E: Base) && !this->emitSideEffect(E))
2293	return false;
2294
2295	return this->visitDeclRef(D: Member, E);
2296	}
2297
2298	if (Initializing) {
2299	if (!this->delegate(E: Base))
2300	return false;
2301	} else {
2302	if (!this->visit(E: Base))
2303	return false;
2304	}
2305
2306	// Base above gives us a pointer on the stack.
2307	const auto *FD = cast<FieldDecl>(Val: Member);
2308	const RecordDecl *RD = FD->getParent();
2309	const Record *R = getRecord(RD);
2310	if (!R)
2311	return false;
2312	const Record::Field *F = R->getField(FD);
2313	// Leave a pointer to the field on the stack.
2314	if (F->Decl->getType()->isReferenceType())
2315	return this->emitGetFieldPop(PT_Ptr, F->Offset, E) && maybeLoadValue();
2316	return this->emitGetPtrFieldPop(F->Offset, E) && maybeLoadValue();
2317	}
2318
2319	template <class Emitter>
2320	bool Compiler<Emitter>::VisitArrayInitIndexExpr(const ArrayInitIndexExpr *E) {
2321	// ArrayIndex might not be set if a ArrayInitIndexExpr is being evaluated
2322	// stand-alone, e.g. via EvaluateAsInt().
2323	if (!ArrayIndex)
2324	return false;
2325	return this->emitConst(*ArrayIndex, E);
2326	}
2327
2328	template <class Emitter>
2329	bool Compiler<Emitter>::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E) {
2330	assert(Initializing);
2331	assert(!DiscardResult);
2332
2333	// We visit the common opaque expression here once so we have its value
2334	// cached.
2335	if (!this->discard(E: E->getCommonExpr()))
2336	return false;
2337
2338	// TODO: This compiles to quite a lot of bytecode if the array is larger.
2339	// Investigate compiling this to a loop.
2340	const Expr *SubExpr = E->getSubExpr();
2341	size_t Size = E->getArraySize().getZExtValue();
2342	std::optional<PrimType> SubExprT = classify(SubExpr);
2343
2344	// So, every iteration, we execute an assignment here
2345	// where the LHS is on the stack (the target array)
2346	// and the RHS is our SubExpr.
2347	for (size_t I = 0; I != Size; ++I) {
2348	ArrayIndexScope<Emitter> IndexScope(this, I);
2349	BlockScope<Emitter> BS(this);
2350
2351	if (!this->visitArrayElemInit(ElemIndex: I, Init: SubExpr, InitT: SubExprT))
2352	return false;
2353	if (!BS.destroyLocals())
2354	return false;
2355	}
2356	return true;
2357	}
2358
2359	template <class Emitter>
2360	bool Compiler<Emitter>::VisitOpaqueValueExpr(const OpaqueValueExpr *E) {
2361	const Expr *SourceExpr = E->getSourceExpr();
2362	if (!SourceExpr)
2363	return false;
2364
2365	if (Initializing)
2366	return this->visitInitializer(E: SourceExpr);
2367
2368	PrimType SubExprT = classify(SourceExpr).value_or(PT_Ptr);
2369	if (auto It = OpaqueExprs.find(Val: E); It != OpaqueExprs.end())
2370	return this->emitGetLocal(SubExprT, It->second, E);
2371
2372	if (!this->visit(E: SourceExpr))
2373	return false;
2374
2375	// At this point we either have the evaluated source expression or a pointer
2376	// to an object on the stack. We want to create a local variable that stores
2377	// this value.
2378	unsigned LocalIndex = allocateLocalPrimitive(Decl: E, Ty: SubExprT, /*IsConst=*/true);
2379	if (!this->emitSetLocal(SubExprT, LocalIndex, E))
2380	return false;
2381
2382	// Here the local variable is created but the value is removed from the stack,
2383	// so we put it back if the caller needs it.
2384	if (!DiscardResult) {
2385	if (!this->emitGetLocal(SubExprT, LocalIndex, E))
2386	return false;
2387	}
2388
2389	// This is cleaned up when the local variable is destroyed.
2390	OpaqueExprs.insert(KV: {E, LocalIndex});
2391
2392	return true;
2393	}
2394
2395	template <class Emitter>
2396	bool Compiler<Emitter>::VisitAbstractConditionalOperator(
2397	const AbstractConditionalOperator *E) {
2398	const Expr *Condition = E->getCond();
2399	const Expr *TrueExpr = E->getTrueExpr();
2400	const Expr *FalseExpr = E->getFalseExpr();
2401
2402	auto visitChildExpr = [&](const Expr *E) -> bool {
2403	LocalScope<Emitter> S(this);
2404	if (!this->delegate(E))
2405	return false;
2406	return S.destroyLocals();
2407	};
2408
2409	if (std::optional<bool> BoolValue = getBoolValue(E: Condition)) {
2410	if (BoolValue)
2411	return visitChildExpr(TrueExpr);
2412	return visitChildExpr(FalseExpr);
2413	}
2414
2415	bool IsBcpCall = false;
2416	if (const auto *CE = dyn_cast<CallExpr>(Val: Condition->IgnoreParenCasts());
2417	CE && CE->getBuiltinCallee() == Builtin::BI__builtin_constant_p) {
2418	IsBcpCall = true;
2419	}
2420
2421	LabelTy LabelEnd = this->getLabel(); // Label after the operator.
2422	LabelTy LabelFalse = this->getLabel(); // Label for the false expr.
2423
2424	if (IsBcpCall) {
2425	if (!this->emitStartSpeculation(E))
2426	return false;
2427	}
2428
2429	if (!this->visitBool(E: Condition)) {
2430	// If the condition failed and we're checking for undefined behavior
2431	// (which only happens with EvalEmitter) check the TrueExpr and FalseExpr
2432	// as well.
2433	if (this->checkingForUndefinedBehavior()) {
2434	if (!this->discard(E: TrueExpr))
2435	return false;
2436	if (!this->discard(E: FalseExpr))
2437	return false;
2438	}
2439	return false;
2440	}
2441
2442	if (!this->jumpFalse(LabelFalse))
2443	return false;
2444	if (!visitChildExpr(TrueExpr))
2445	return false;
2446	if (!this->jump(LabelEnd))
2447	return false;
2448	this->emitLabel(LabelFalse);
2449	if (!visitChildExpr(FalseExpr))
2450	return false;
2451	this->fallthrough(LabelEnd);
2452	this->emitLabel(LabelEnd);
2453
2454	if (IsBcpCall)
2455	return this->emitEndSpeculation(E);
2456	return true;
2457	}
2458
2459	template <class Emitter>
2460	bool Compiler<Emitter>::VisitStringLiteral(const StringLiteral *E) {
2461	if (DiscardResult)
2462	return true;
2463
2464	if (!Initializing) {
2465	unsigned StringIndex = P.createGlobalString(S: E);
2466	return this->emitGetPtrGlobal(StringIndex, E);
2467	}
2468
2469	// We are initializing an array on the stack.
2470	const ConstantArrayType *CAT =
2471	Ctx.getASTContext().getAsConstantArrayType(T: E->getType());
2472	assert(CAT && "a string literal that's not a constant array?");
2473
2474	// If the initializer string is too long, a diagnostic has already been
2475	// emitted. Read only the array length from the string literal.
2476	unsigned ArraySize = CAT->getZExtSize();
2477	unsigned N = std::min(a: ArraySize, b: E->getLength());
2478	unsigned CharWidth = E->getCharByteWidth();
2479
2480	for (unsigned I = 0; I != N; ++I) {
2481	uint32_t CodeUnit = E->getCodeUnit(i: I);
2482
2483	if (CharWidth == 1) {
2484	this->emitConstSint8(CodeUnit, E);
2485	this->emitInitElemSint8(I, E);
2486	} else if (CharWidth == 2) {
2487	this->emitConstUint16(CodeUnit, E);
2488	this->emitInitElemUint16(I, E);
2489	} else if (CharWidth == 4) {
2490	this->emitConstUint32(CodeUnit, E);
2491	this->emitInitElemUint32(I, E);
2492	} else {
2493	llvm_unreachable("unsupported character width");
2494	}
2495	}
2496
2497	// Fill up the rest of the char array with NUL bytes.
2498	for (unsigned I = N; I != ArraySize; ++I) {
2499	if (CharWidth == 1) {
2500	this->emitConstSint8(0, E);
2501	this->emitInitElemSint8(I, E);
2502	} else if (CharWidth == 2) {
2503	this->emitConstUint16(0, E);
2504	this->emitInitElemUint16(I, E);
2505	} else if (CharWidth == 4) {
2506	this->emitConstUint32(0, E);
2507	this->emitInitElemUint32(I, E);
2508	} else {
2509	llvm_unreachable("unsupported character width");
2510	}
2511	}
2512
2513	return true;
2514	}
2515
2516	template <class Emitter>
2517	bool Compiler<Emitter>::VisitObjCStringLiteral(const ObjCStringLiteral *E) {
2518	if (DiscardResult)
2519	return true;
2520	return this->emitDummyPtr(D: E, E);
2521	}
2522
2523	template <class Emitter>
2524	bool Compiler<Emitter>::VisitObjCEncodeExpr(const ObjCEncodeExpr *E) {
2525	auto &A = Ctx.getASTContext();
2526	std::string Str;
2527	A.getObjCEncodingForType(T: E->getEncodedType(), S&: Str);
2528	StringLiteral *SL =
2529	StringLiteral::Create(Ctx: A, Str, Kind: StringLiteralKind::Ordinary,
2530	/*Pascal=*/false, Ty: E->getType(), Locs: E->getAtLoc());
2531	return this->delegate(E: SL);
2532	}
2533
2534	template <class Emitter>
2535	bool Compiler<Emitter>::VisitSYCLUniqueStableNameExpr(
2536	const SYCLUniqueStableNameExpr *E) {
2537	if (DiscardResult)
2538	return true;
2539
2540	assert(!Initializing);
2541
2542	auto &A = Ctx.getASTContext();
2543	std::string ResultStr = E->ComputeName(Context&: A);
2544
2545	QualType CharTy = A.CharTy.withConst();
2546	APInt Size(A.getTypeSize(T: A.getSizeType()), ResultStr.size() + 1);
2547	QualType ArrayTy = A.getConstantArrayType(EltTy: CharTy, ArySize: Size, SizeExpr: nullptr,
2548	ASM: ArraySizeModifier::Normal, IndexTypeQuals: 0);
2549
2550	StringLiteral *SL =
2551	StringLiteral::Create(Ctx: A, Str: ResultStr, Kind: StringLiteralKind::Ordinary,
2552	/*Pascal=*/false, Ty: ArrayTy, Locs: E->getLocation());
2553
2554	unsigned StringIndex = P.createGlobalString(S: SL);
2555	return this->emitGetPtrGlobal(StringIndex, E);
2556	}
2557
2558	template <class Emitter>
2559	bool Compiler<Emitter>::VisitCharacterLiteral(const CharacterLiteral *E) {
2560	if (DiscardResult)
2561	return true;
2562	return this->emitConst(E->getValue(), E);
2563	}
2564
2565	template <class Emitter>
2566	bool Compiler<Emitter>::VisitFloatCompoundAssignOperator(
2567	const CompoundAssignOperator *E) {
2568
2569	const Expr *LHS = E->getLHS();
2570	const Expr *RHS = E->getRHS();
2571	QualType LHSType = LHS->getType();
2572	QualType LHSComputationType = E->getComputationLHSType();
2573	QualType ResultType = E->getComputationResultType();
2574	std::optional<PrimType> LT = classify(LHSComputationType);
2575	std::optional<PrimType> RT = classify(ResultType);
2576
2577	assert(ResultType->isFloatingType());
2578
2579	if (!LT || !RT)
2580	return false;
2581
2582	PrimType LHST = classifyPrim(LHSType);
2583
2584	// C++17 onwards require that we evaluate the RHS first.
2585	// Compute RHS and save it in a temporary variable so we can
2586	// load it again later.
2587	if (!visit(E: RHS))
2588	return false;
2589
2590	unsigned TempOffset = this->allocateLocalPrimitive(Decl: E, Ty: *RT, /*IsConst=*/true);
2591	if (!this->emitSetLocal(*RT, TempOffset, E))
2592	return false;
2593
2594	// First, visit LHS.
2595	if (!visit(E: LHS))
2596	return false;
2597	if (!this->emitLoad(LHST, E))
2598	return false;
2599
2600	// If necessary, convert LHS to its computation type.
2601	if (!this->emitPrimCast(FromT: LHST, ToT: classifyPrim(LHSComputationType),
2602	ToQT: LHSComputationType, E))
2603	return false;
2604
2605	// Now load RHS.
2606	if (!this->emitGetLocal(*RT, TempOffset, E))
2607	return false;
2608
2609	switch (E->getOpcode()) {
2610	case BO_AddAssign:
2611	if (!this->emitAddf(getFPOptions(E), E))
2612	return false;
2613	break;
2614	case BO_SubAssign:
2615	if (!this->emitSubf(getFPOptions(E), E))
2616	return false;
2617	break;
2618	case BO_MulAssign:
2619	if (!this->emitMulf(getFPOptions(E), E))
2620	return false;
2621	break;
2622	case BO_DivAssign:
2623	if (!this->emitDivf(getFPOptions(E), E))
2624	return false;
2625	break;
2626	default:
2627	return false;
2628	}
2629
2630	if (!this->emitPrimCast(FromT: classifyPrim(ResultType), ToT: LHST, ToQT: LHS->getType(), E))
2631	return false;
2632
2633	if (DiscardResult)
2634	return this->emitStorePop(LHST, E);
2635	return this->emitStore(LHST, E);
2636	}
2637
2638	template <class Emitter>
2639	bool Compiler<Emitter>::VisitPointerCompoundAssignOperator(
2640	const CompoundAssignOperator *E) {
2641	BinaryOperatorKind Op = E->getOpcode();
2642	const Expr *LHS = E->getLHS();
2643	const Expr *RHS = E->getRHS();
2644	std::optional<PrimType> LT = classify(LHS->getType());
2645	std::optional<PrimType> RT = classify(RHS->getType());
2646
2647	if (Op != BO_AddAssign && Op != BO_SubAssign)
2648	return false;
2649
2650	if (!LT || !RT)
2651	return false;
2652
2653	if (!visit(E: LHS))
2654	return false;
2655
2656	if (!this->emitLoad(*LT, LHS))
2657	return false;
2658
2659	if (!visit(E: RHS))
2660	return false;
2661
2662	if (Op == BO_AddAssign) {
2663	if (!this->emitAddOffset(*RT, E))
2664	return false;
2665	} else {
2666	if (!this->emitSubOffset(*RT, E))
2667	return false;
2668	}
2669
2670	if (DiscardResult)
2671	return this->emitStorePopPtr(E);
2672	return this->emitStorePtr(E);
2673	}
2674
2675	template <class Emitter>
2676	bool Compiler<Emitter>::VisitCompoundAssignOperator(
2677	const CompoundAssignOperator *E) {
2678	if (E->getType()->isVectorType())
2679	return VisitVectorBinOp(E);
2680
2681	const Expr *LHS = E->getLHS();
2682	const Expr *RHS = E->getRHS();
2683	std::optional<PrimType> LHSComputationT =
2684	classify(E->getComputationLHSType());
2685	std::optional<PrimType> LT = classify(LHS->getType());
2686	std::optional<PrimType> RT = classify(RHS->getType());
2687	std::optional<PrimType> ResultT = classify(E->getType());
2688
2689	if (!Ctx.getLangOpts().CPlusPlus14)
2690	return this->visit(E: RHS) && this->visit(E: LHS) && this->emitError(E);
2691
2692	if (!LT || !RT || !ResultT || !LHSComputationT)
2693	return false;
2694
2695	// Handle floating point operations separately here, since they
2696	// require special care.
2697
2698	if (ResultT == PT_Float || RT == PT_Float)
2699	return VisitFloatCompoundAssignOperator(E);
2700
2701	if (E->getType()->isPointerType())
2702	return VisitPointerCompoundAssignOperator(E);
2703
2704	assert(!E->getType()->isPointerType() && "Handled above");
2705	assert(!E->getType()->isFloatingType() && "Handled above");
2706
2707	// C++17 onwards require that we evaluate the RHS first.
2708	// Compute RHS and save it in a temporary variable so we can
2709	// load it again later.
2710	// FIXME: Compound assignments are unsequenced in C, so we might
2711	// have to figure out how to reject them.
2712	if (!visit(E: RHS))
2713	return false;
2714
2715	unsigned TempOffset = this->allocateLocalPrimitive(Decl: E, Ty: *RT, /*IsConst=*/true);
2716
2717	if (!this->emitSetLocal(*RT, TempOffset, E))
2718	return false;
2719
2720	// Get LHS pointer, load its value and cast it to the
2721	// computation type if necessary.
2722	if (!visit(E: LHS))
2723	return false;
2724	if (!this->emitLoad(*LT, E))
2725	return false;
2726	if (LT != LHSComputationT) {
2727	if (!this->emitCast(*LT, *LHSComputationT, E))
2728	return false;
2729	}
2730
2731	// Get the RHS value on the stack.
2732	if (!this->emitGetLocal(*RT, TempOffset, E))
2733	return false;
2734
2735	// Perform operation.
2736	switch (E->getOpcode()) {
2737	case BO_AddAssign:
2738	if (!this->emitAdd(*LHSComputationT, E))
2739	return false;
2740	break;
2741	case BO_SubAssign:
2742	if (!this->emitSub(*LHSComputationT, E))
2743	return false;
2744	break;
2745	case BO_MulAssign:
2746	if (!this->emitMul(*LHSComputationT, E))
2747	return false;
2748	break;
2749	case BO_DivAssign:
2750	if (!this->emitDiv(*LHSComputationT, E))
2751	return false;
2752	break;
2753	case BO_RemAssign:
2754	if (!this->emitRem(*LHSComputationT, E))
2755	return false;
2756	break;
2757	case BO_ShlAssign:
2758	if (!this->emitShl(*LHSComputationT, *RT, E))
2759	return false;
2760	break;
2761	case BO_ShrAssign:
2762	if (!this->emitShr(*LHSComputationT, *RT, E))
2763	return false;
2764	break;
2765	case BO_AndAssign:
2766	if (!this->emitBitAnd(*LHSComputationT, E))
2767	return false;
2768	break;
2769	case BO_XorAssign:
2770	if (!this->emitBitXor(*LHSComputationT, E))
2771	return false;
2772	break;
2773	case BO_OrAssign:
2774	if (!this->emitBitOr(*LHSComputationT, E))
2775	return false;
2776	break;
2777	default:
2778	llvm_unreachable("Unimplemented compound assign operator");
2779	}
2780
2781	// And now cast from LHSComputationT to ResultT.
2782	if (ResultT != LHSComputationT) {
2783	if (!this->emitCast(*LHSComputationT, *ResultT, E))
2784	return false;
2785	}
2786
2787	// And store the result in LHS.
2788	if (DiscardResult) {
2789	if (LHS->refersToBitField())
2790	return this->emitStoreBitFieldPop(*ResultT, E);
2791	return this->emitStorePop(*ResultT, E);
2792	}
2793	if (LHS->refersToBitField())
2794	return this->emitStoreBitField(*ResultT, E);
2795	return this->emitStore(*ResultT, E);
2796	}
2797
2798	template <class Emitter>
2799	bool Compiler<Emitter>::VisitExprWithCleanups(const ExprWithCleanups *E) {
2800	LocalScope<Emitter> ES(this);
2801	const Expr *SubExpr = E->getSubExpr();
2802
2803	return this->delegate(E: SubExpr) && ES.destroyLocals(E);
2804	}
2805
2806	template <class Emitter>
2807	bool Compiler<Emitter>::VisitMaterializeTemporaryExpr(
2808	const MaterializeTemporaryExpr *E) {
2809	const Expr *SubExpr = E->getSubExpr();
2810
2811	if (Initializing) {
2812	// We already have a value, just initialize that.
2813	return this->delegate(E: SubExpr);
2814	}
2815	// If we don't end up using the materialized temporary anyway, don't
2816	// bother creating it.
2817	if (DiscardResult)
2818	return this->discard(E: SubExpr);
2819
2820	// When we're initializing a global variable *or* the storage duration of
2821	// the temporary is explicitly static, create a global variable.
2822	std::optional<PrimType> SubExprT = classify(SubExpr);
2823	bool IsStatic = E->getStorageDuration() == SD_Static;
2824	if (IsStatic) {
2825	std::optional<unsigned> GlobalIndex = P.createGlobal(E);
2826	if (!GlobalIndex)
2827	return false;
2828
2829	const LifetimeExtendedTemporaryDecl *TempDecl =
2830	E->getLifetimeExtendedTemporaryDecl();
2831	if (IsStatic)
2832	assert(TempDecl);
2833
2834	if (SubExprT) {
2835	if (!this->visit(E: SubExpr))
2836	return false;
2837	if (IsStatic) {
2838	if (!this->emitInitGlobalTemp(*SubExprT, *GlobalIndex, TempDecl, E))
2839	return false;
2840	} else {
2841	if (!this->emitInitGlobal(*SubExprT, *GlobalIndex, E))
2842	return false;
2843	}
2844	return this->emitGetPtrGlobal(*GlobalIndex, E);
2845	}
2846
2847	if (!this->checkLiteralType(E: SubExpr))
2848	return false;
2849	// Non-primitive values.
2850	if (!this->emitGetPtrGlobal(*GlobalIndex, E))
2851	return false;
2852	if (!this->visitInitializer(E: SubExpr))
2853	return false;
2854	if (IsStatic)
2855	return this->emitInitGlobalTempComp(TempDecl, E);
2856	return true;
2857	}
2858
2859	// For everyhing else, use local variables.
2860	if (SubExprT) {
2861	bool IsConst = SubExpr->getType().isConstQualified();
2862	unsigned LocalIndex =
2863	allocateLocalPrimitive(Decl: E, Ty: *SubExprT, IsConst, ExtendingDecl: E->getExtendingDecl());
2864	if (!this->visit(E: SubExpr))
2865	return false;
2866	if (!this->emitSetLocal(*SubExprT, LocalIndex, E))
2867	return false;
2868	return this->emitGetPtrLocal(LocalIndex, E);
2869	} else {
2870
2871	if (!this->checkLiteralType(E: SubExpr))
2872	return false;
2873
2874	const Expr *Inner = E->getSubExpr()->skipRValueSubobjectAdjustments();
2875	if (std::optional<unsigned> LocalIndex =
2876	allocateLocal(Decl: E, Ty: Inner->getType(), ExtendingDecl: E->getExtendingDecl())) {
2877	InitLinkScope<Emitter> ILS(this, InitLink::Temp(Offset: *LocalIndex));
2878	if (!this->emitGetPtrLocal(*LocalIndex, E))
2879	return false;
2880	return this->visitInitializer(E: SubExpr) && this->emitFinishInit(E);
2881	}
2882	}
2883	return false;
2884	}
2885
2886	template <class Emitter>
2887	bool Compiler<Emitter>::VisitCXXBindTemporaryExpr(
2888	const CXXBindTemporaryExpr *E) {
2889	const Expr *SubExpr = E->getSubExpr();
2890
2891	if (Initializing)
2892	return this->delegate(E: SubExpr);
2893
2894	// Make sure we create a temporary even if we're discarding, since that will
2895	// make sure we will also call the destructor.
2896
2897	if (!this->visit(E: SubExpr))
2898	return false;
2899
2900	if (DiscardResult)
2901	return this->emitPopPtr(E);
2902	return true;
2903	}
2904
2905	template <class Emitter>
2906	bool Compiler<Emitter>::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
2907	const Expr *Init = E->getInitializer();
2908	if (DiscardResult)
2909	return this->discard(E: Init);
2910
2911	if (Initializing) {
2912	// We already have a value, just initialize that.
2913	return this->visitInitializer(E: Init) && this->emitFinishInit(E);
2914	}
2915
2916	std::optional<PrimType> T = classify(E->getType());
2917	if (E->isFileScope()) {
2918	// Avoid creating a variable if this is a primitive RValue anyway.
2919	if (T && !E->isLValue())
2920	return this->delegate(E: Init);
2921
2922	if (std::optional<unsigned> GlobalIndex = P.createGlobal(E)) {
2923	if (!this->emitGetPtrGlobal(*GlobalIndex, E))
2924	return false;
2925
2926	if (T) {
2927	if (!this->visit(E: Init))
2928	return false;
2929	return this->emitInitGlobal(*T, *GlobalIndex, E);
2930	}
2931
2932	return this->visitInitializer(E: Init) && this->emitFinishInit(E);
2933	}
2934
2935	return false;
2936	}
2937
2938	// Otherwise, use a local variable.
2939	if (T && !E->isLValue()) {
2940	// For primitive types, we just visit the initializer.
2941	return this->delegate(E: Init);
2942	}
2943
2944	unsigned LocalIndex;
2945	if (T)
2946	LocalIndex = this->allocateLocalPrimitive(Decl: Init, Ty: *T, /*IsConst=*/false);
2947	else if (std::optional<unsigned> MaybeIndex = this->allocateLocal(Decl: Init))
2948	LocalIndex = *MaybeIndex;
2949	else
2950	return false;
2951
2952	if (!this->emitGetPtrLocal(LocalIndex, E))
2953	return false;
2954
2955	if (T)
2956	return this->visit(E: Init) && this->emitInit(*T, E);
2957	return this->visitInitializer(E: Init) && this->emitFinishInit(E);
2958	}
2959
2960	template <class Emitter>
2961	bool Compiler<Emitter>::VisitTypeTraitExpr(const TypeTraitExpr *E) {
2962	if (DiscardResult)
2963	return true;
2964	if (E->isStoredAsBoolean()) {
2965	if (E->getType()->isBooleanType())
2966	return this->emitConstBool(E->getBoolValue(), E);
2967	return this->emitConst(E->getBoolValue(), E);
2968	}
2969	PrimType T = classifyPrim(E->getType());
2970	return this->visitAPValue(Val: E->getAPValue(), ValType: T, E);
2971	}
2972
2973	template <class Emitter>
2974	bool Compiler<Emitter>::VisitArrayTypeTraitExpr(const ArrayTypeTraitExpr *E) {
2975	if (DiscardResult)
2976	return true;
2977	return this->emitConst(E->getValue(), E);
2978	}
2979
2980	template <class Emitter>
2981	bool Compiler<Emitter>::VisitLambdaExpr(const LambdaExpr *E) {
2982	if (DiscardResult)
2983	return true;
2984
2985	assert(Initializing);
2986	const Record *R = P.getOrCreateRecord(RD: E->getLambdaClass());
2987	if (!R)
2988	return false;
2989
2990	auto *CaptureInitIt = E->capture_init_begin();
2991	// Initialize all fields (which represent lambda captures) of the
2992	// record with their initializers.
2993	for (const Record::Field &F : R->fields()) {
2994	const Expr *Init = *CaptureInitIt;
2995	if (!Init || Init->containsErrors())
2996	continue;
2997	++CaptureInitIt;
2998
2999	if (std::optional<PrimType> T = classify(Init)) {
3000	if (!this->visit(E: Init))
3001	return false;
3002
3003	if (!this->emitInitField(*T, F.Offset, E))
3004	return false;
3005	} else {
3006	if (!this->emitGetPtrField(F.Offset, E))
3007	return false;
3008
3009	if (!this->visitInitializer(E: Init))
3010	return false;
3011
3012	if (!this->emitPopPtr(E))
3013	return false;
3014	}
3015	}
3016
3017	return true;
3018	}
3019
3020	template <class Emitter>
3021	bool Compiler<Emitter>::VisitPredefinedExpr(const PredefinedExpr *E) {
3022	if (DiscardResult)
3023	return true;
3024
3025	if (!Initializing) {
3026	unsigned StringIndex = P.createGlobalString(S: E->getFunctionName(), Base: E);
3027	return this->emitGetPtrGlobal(StringIndex, E);
3028	}
3029
3030	return this->delegate(E: E->getFunctionName());
3031	}
3032
3033	template <class Emitter>
3034	bool Compiler<Emitter>::VisitCXXThrowExpr(const CXXThrowExpr *E) {
3035	if (E->getSubExpr() && !this->discard(E: E->getSubExpr()))
3036	return false;
3037
3038	return this->emitInvalid(E);
3039	}
3040
3041	template <class Emitter>
3042	bool Compiler<Emitter>::VisitCXXReinterpretCastExpr(
3043	const CXXReinterpretCastExpr *E) {
3044	const Expr *SubExpr = E->getSubExpr();
3045
3046	std::optional<PrimType> FromT = classify(SubExpr);
3047	std::optional<PrimType> ToT = classify(E);
3048
3049	if (!FromT || !ToT)
3050	return this->emitInvalidCast(CastKind::Reinterpret, /*Fatal=*/true, E);
3051
3052	if (FromT == PT_Ptr || ToT == PT_Ptr) {
3053	// Both types could be PT_Ptr because their expressions are glvalues.
3054	std::optional<PrimType> PointeeFromT;
3055	if (SubExpr->getType()->isPointerOrReferenceType())
3056	PointeeFromT = classify(SubExpr->getType()->getPointeeType());
3057	else
3058	PointeeFromT = classify(SubExpr->getType());
3059
3060	std::optional<PrimType> PointeeToT;
3061	if (E->getType()->isPointerOrReferenceType())
3062	PointeeToT = classify(E->getType()->getPointeeType());
3063	else
3064	PointeeToT = classify(E->getType());
3065
3066	bool Fatal = true;
3067	if (PointeeToT && PointeeFromT) {
3068	if (isIntegralType(T: *PointeeFromT) && isIntegralType(T: *PointeeToT))
3069	Fatal = false;
3070	} else {
3071	Fatal = SubExpr->getType().getTypePtr() != E->getType().getTypePtr();
3072	}
3073
3074	if (!this->emitInvalidCast(CastKind::Reinterpret, Fatal, E))
3075	return false;
3076
3077	if (E->getCastKind() == CK_LValueBitCast)
3078	return this->delegate(E: SubExpr);
3079	return this->VisitCastExpr(CE: E);
3080	}
3081
3082	// Try to actually do the cast.
3083	bool Fatal = (ToT != FromT);
3084	if (!this->emitInvalidCast(CastKind::Reinterpret, Fatal, E))
3085	return false;
3086
3087	return this->VisitCastExpr(CE: E);
3088	}
3089
3090	template <class Emitter>
3091	bool Compiler<Emitter>::VisitCXXDynamicCastExpr(const CXXDynamicCastExpr *E) {
3092
3093	if (!Ctx.getLangOpts().CPlusPlus20) {
3094	if (!this->emitInvalidCast(CastKind::Dynamic, /*Fatal=*/false, E))
3095	return false;
3096	}
3097
3098	return this->VisitCastExpr(CE: E);
3099	}
3100
3101	template <class Emitter>
3102	bool Compiler<Emitter>::VisitCXXNoexceptExpr(const CXXNoexceptExpr *E) {
3103	assert(E->getType()->isBooleanType());
3104
3105	if (DiscardResult)
3106	return true;
3107	return this->emitConstBool(E->getValue(), E);
3108	}
3109
3110	template <class Emitter>
3111	bool Compiler<Emitter>::VisitCXXConstructExpr(const CXXConstructExpr *E) {
3112	QualType T = E->getType();
3113	assert(!classify(T));
3114
3115	if (T->isRecordType()) {
3116	const CXXConstructorDecl *Ctor = E->getConstructor();
3117
3118	// Trivial copy/move constructor. Avoid copy.
3119	if (Ctor->isDefaulted() && Ctor->isCopyOrMoveConstructor() &&
3120	Ctor->isTrivial() &&
3121	E->getArg(Arg: 0)->isTemporaryObject(Ctx&: Ctx.getASTContext(),
3122	TempTy: T->getAsCXXRecordDecl()))
3123	return this->visitInitializer(E: E->getArg(Arg: 0));
3124
3125	// If we're discarding a construct expression, we still need
3126	// to allocate a variable and call the constructor and destructor.
3127	if (DiscardResult) {
3128	if (Ctor->isTrivial())
3129	return true;
3130	assert(!Initializing);
3131	std::optional<unsigned> LocalIndex = allocateLocal(Decl: E);
3132
3133	if (!LocalIndex)
3134	return false;
3135
3136	if (!this->emitGetPtrLocal(*LocalIndex, E))
3137	return false;
3138	}
3139
3140	// Zero initialization.
3141	if (E->requiresZeroInitialization()) {
3142	const Record *R = getRecord(E->getType());
3143
3144	if (!this->visitZeroRecordInitializer(R, E))
3145	return false;
3146
3147	// If the constructor is trivial anyway, we're done.
3148	if (Ctor->isTrivial())
3149	return true;
3150	}
3151
3152	const Function *Func = getFunction(FD: Ctor);
3153
3154	if (!Func)
3155	return false;
3156
3157	assert(Func->hasThisPointer());
3158	assert(!Func->hasRVO());
3159
3160	// The This pointer is already on the stack because this is an initializer,
3161	// but we need to dup() so the call() below has its own copy.
3162	if (!this->emitDupPtr(E))
3163	return false;
3164
3165	// Constructor arguments.
3166	for (const auto *Arg : E->arguments()) {
3167	if (!this->visit(E: Arg))
3168	return false;
3169	}
3170
3171	if (Func->isVariadic()) {
3172	uint32_t VarArgSize = 0;
3173	unsigned NumParams = Func->getNumWrittenParams();
3174	for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I) {
3175	VarArgSize +=
3176	align(primSize(classify(E->getArg(Arg: I)->getType()).value_or(PT_Ptr)));
3177	}
3178	if (!this->emitCallVar(Func, VarArgSize, E))
3179	return false;
3180	} else {
3181	if (!this->emitCall(Func, 0, E)) {
3182	// When discarding, we don't need the result anyway, so clean up
3183	// the instance dup we did earlier in case surrounding code wants
3184	// to keep evaluating.
3185	if (DiscardResult)
3186	(void)this->emitPopPtr(E);
3187	return false;
3188	}
3189	}
3190
3191	if (DiscardResult)
3192	return this->emitPopPtr(E);
3193	return this->emitFinishInit(E);
3194	}
3195
3196	if (T->isArrayType()) {
3197	const ConstantArrayType *CAT =
3198	Ctx.getASTContext().getAsConstantArrayType(T: E->getType());
3199	if (!CAT)
3200	return false;
3201
3202	size_t NumElems = CAT->getZExtSize();
3203	const Function *Func = getFunction(FD: E->getConstructor());
3204	if (!Func)
3205	return false;
3206
3207	// FIXME(perf): We're calling the constructor once per array element here,
3208	// in the old intepreter we had a special-case for trivial constructors.
3209	for (size_t I = 0; I != NumElems; ++I) {
3210	if (!this->emitConstUint64(I, E))
3211	return false;
3212	if (!this->emitArrayElemPtrUint64(E))
3213	return false;
3214
3215	// Constructor arguments.
3216	for (const auto *Arg : E->arguments()) {
3217	if (!this->visit(E: Arg))
3218	return false;
3219	}
3220
3221	if (!this->emitCall(Func, 0, E))
3222	return false;
3223	}
3224	return true;
3225	}
3226
3227	return false;
3228	}
3229
3230	template <class Emitter>
3231	bool Compiler<Emitter>::VisitSourceLocExpr(const SourceLocExpr *E) {
3232	if (DiscardResult)
3233	return true;
3234
3235	const APValue Val =
3236	E->EvaluateInContext(Ctx: Ctx.getASTContext(), DefaultExpr: SourceLocDefaultExpr);
3237
3238	// Things like __builtin_LINE().
3239	if (E->getType()->isIntegerType()) {
3240	assert(Val.isInt());
3241	const APSInt &I = Val.getInt();
3242	return this->emitConst(I, E);
3243	}
3244	// Otherwise, the APValue is an LValue, with only one element.
3245	// Theoretically, we don't need the APValue at all of course.
3246	assert(E->getType()->isPointerType());
3247	assert(Val.isLValue());
3248	const APValue::LValueBase &Base = Val.getLValueBase();
3249	if (const Expr *LValueExpr = Base.dyn_cast<const Expr *>())
3250	return this->visit(E: LValueExpr);
3251
3252	// Otherwise, we have a decl (which is the case for
3253	// __builtin_source_location).
3254	assert(Base.is<const ValueDecl *>());
3255	assert(Val.getLValuePath().size() == 0);
3256	const auto *BaseDecl = Base.dyn_cast<const ValueDecl *>();
3257	assert(BaseDecl);
3258
3259	auto *UGCD = cast<UnnamedGlobalConstantDecl>(Val: BaseDecl);
3260
3261	std::optional<unsigned> GlobalIndex = P.getOrCreateGlobal(VD: UGCD);
3262	if (!GlobalIndex)
3263	return false;
3264
3265	if (!this->emitGetPtrGlobal(*GlobalIndex, E))
3266	return false;
3267
3268	const Record *R = getRecord(E->getType());
3269	const APValue &V = UGCD->getValue();
3270	for (unsigned I = 0, N = R->getNumFields(); I != N; ++I) {
3271	const Record::Field *F = R->getField(I);
3272	const APValue &FieldValue = V.getStructField(i: I);
3273
3274	PrimType FieldT = classifyPrim(F->Decl->getType());
3275
3276	if (!this->visitAPValue(Val: FieldValue, ValType: FieldT, E))
3277	return false;
3278	if (!this->emitInitField(FieldT, F->Offset, E))
3279	return false;
3280	}
3281
3282	// Leave the pointer to the global on the stack.
3283	return true;
3284	}
3285
3286	template <class Emitter>
3287	bool Compiler<Emitter>::VisitOffsetOfExpr(const OffsetOfExpr *E) {
3288	unsigned N = E->getNumComponents();
3289	if (N == 0)
3290	return false;
3291
3292	for (unsigned I = 0; I != N; ++I) {
3293	const OffsetOfNode &Node = E->getComponent(Idx: I);
3294	if (Node.getKind() == OffsetOfNode::Array) {
3295	const Expr *ArrayIndexExpr = E->getIndexExpr(Idx: Node.getArrayExprIndex());
3296	PrimType IndexT = classifyPrim(ArrayIndexExpr->getType());
3297
3298	if (DiscardResult) {
3299	if (!this->discard(E: ArrayIndexExpr))
3300	return false;
3301	continue;
3302	}
3303
3304	if (!this->visit(E: ArrayIndexExpr))
3305	return false;
3306	// Cast to Sint64.
3307	if (IndexT != PT_Sint64) {
3308	if (!this->emitCast(IndexT, PT_Sint64, E))
3309	return false;
3310	}
3311	}
3312	}
3313
3314	if (DiscardResult)
3315	return true;
3316
3317	PrimType T = classifyPrim(E->getType());
3318	return this->emitOffsetOf(T, E, E);
3319	}
3320
3321	template <class Emitter>
3322	bool Compiler<Emitter>::VisitCXXScalarValueInitExpr(
3323	const CXXScalarValueInitExpr *E) {
3324	QualType Ty = E->getType();
3325
3326	if (DiscardResult || Ty->isVoidType())
3327	return true;
3328
3329	if (std::optional<PrimType> T = classify(Ty))
3330	return this->visitZeroInitializer(T: *T, QT: Ty, E);
3331
3332	if (const auto *CT = Ty->getAs<ComplexType>()) {
3333	if (!Initializing) {
3334	std::optional<unsigned> LocalIndex = allocateLocal(Decl: E);
3335	if (!LocalIndex)
3336	return false;
3337	if (!this->emitGetPtrLocal(*LocalIndex, E))
3338	return false;
3339	}
3340
3341	// Initialize both fields to 0.
3342	QualType ElemQT = CT->getElementType();
3343	PrimType ElemT = classifyPrim(ElemQT);
3344
3345	for (unsigned I = 0; I != 2; ++I) {
3346	if (!this->visitZeroInitializer(T: ElemT, QT: ElemQT, E))
3347	return false;
3348	if (!this->emitInitElem(ElemT, I, E))
3349	return false;
3350	}
3351	return true;
3352	}
3353
3354	if (const auto *VT = Ty->getAs<VectorType>()) {
3355	// FIXME: Code duplication with the _Complex case above.
3356	if (!Initializing) {
3357	std::optional<unsigned> LocalIndex = allocateLocal(Decl: E);
3358	if (!LocalIndex)
3359	return false;
3360	if (!this->emitGetPtrLocal(*LocalIndex, E))
3361	return false;
3362	}
3363
3364	// Initialize all fields to 0.
3365	QualType ElemQT = VT->getElementType();
3366	PrimType ElemT = classifyPrim(ElemQT);
3367
3368	for (unsigned I = 0, N = VT->getNumElements(); I != N; ++I) {
3369	if (!this->visitZeroInitializer(T: ElemT, QT: ElemQT, E))
3370	return false;
3371	if (!this->emitInitElem(ElemT, I, E))
3372	return false;
3373	}
3374	return true;
3375	}
3376
3377	return false;
3378	}
3379
3380	template <class Emitter>
3381	bool Compiler<Emitter>::VisitSizeOfPackExpr(const SizeOfPackExpr *E) {
3382	return this->emitConst(E->getPackLength(), E);
3383	}
3384
3385	template <class Emitter>
3386	bool Compiler<Emitter>::VisitGenericSelectionExpr(
3387	const GenericSelectionExpr *E) {
3388	return this->delegate(E: E->getResultExpr());
3389	}
3390
3391	template <class Emitter>
3392	bool Compiler<Emitter>::VisitChooseExpr(const ChooseExpr *E) {
3393	return this->delegate(E: E->getChosenSubExpr());
3394	}
3395
3396	template <class Emitter>
3397	bool Compiler<Emitter>::VisitObjCBoolLiteralExpr(const ObjCBoolLiteralExpr *E) {
3398	if (DiscardResult)
3399	return true;
3400
3401	return this->emitConst(E->getValue(), E);
3402	}
3403
3404	template <class Emitter>
3405	bool Compiler<Emitter>::VisitCXXInheritedCtorInitExpr(
3406	const CXXInheritedCtorInitExpr *E) {
3407	const CXXConstructorDecl *Ctor = E->getConstructor();
3408	assert(!Ctor->isTrivial() &&
3409	"Trivial CXXInheritedCtorInitExpr, implement. (possible?)");
3410	const Function *F = this->getFunction(FD: Ctor);
3411	assert(F);
3412	assert(!F->hasRVO());
3413	assert(F->hasThisPointer());
3414
3415	if (!this->emitDupPtr(SourceInfo{}))
3416	return false;
3417
3418	// Forward all arguments of the current function (which should be a
3419	// constructor itself) to the inherited ctor.
3420	// This is necessary because the calling code has pushed the pointer
3421	// of the correct base for us already, but the arguments need
3422	// to come after.
3423	unsigned Offset = align(Size: primSize(Type: PT_Ptr)); // instance pointer.
3424	for (const ParmVarDecl *PD : Ctor->parameters()) {
3425	PrimType PT = this->classify(PD->getType()).value_or(PT_Ptr);
3426
3427	if (!this->emitGetParam(PT, Offset, E))
3428	return false;
3429	Offset += align(Size: primSize(Type: PT));
3430	}
3431
3432	return this->emitCall(F, 0, E);
3433	}
3434
3435	// FIXME: This function has become rather unwieldy, especially
3436	// the part where we initialize an array allocation of dynamic size.
3437	template <class Emitter>
3438	bool Compiler<Emitter>::VisitCXXNewExpr(const CXXNewExpr *E) {
3439	assert(classifyPrim(E->getType()) == PT_Ptr);
3440	const Expr *Init = E->getInitializer();
3441	QualType ElementType = E->getAllocatedType();
3442	std::optional<PrimType> ElemT = classify(ElementType);
3443	unsigned PlacementArgs = E->getNumPlacementArgs();
3444	const FunctionDecl *OperatorNew = E->getOperatorNew();
3445	const Expr *PlacementDest = nullptr;
3446	bool IsNoThrow = false;
3447
3448	if (PlacementArgs != 0) {
3449	// FIXME: There is no restriction on this, but it's not clear that any
3450	// other form makes any sense. We get here for cases such as:
3451	//
3452	// new (std::align_val_t{N}) X(int)
3453	//
3454	// (which should presumably be valid only if N is a multiple of
3455	// alignof(int), and in any case can't be deallocated unless N is
3456	// alignof(X) and X has new-extended alignment).
3457	if (PlacementArgs == 1) {
3458	const Expr *Arg1 = E->getPlacementArg(I: 0);
3459	if (Arg1->getType()->isNothrowT()) {
3460	if (!this->discard(E: Arg1))
3461	return false;
3462	IsNoThrow = true;
3463	} else {
3464	// Invalid unless we have C++26 or are in a std:: function.
3465	if (!this->emitInvalidNewDeleteExpr(E, E))
3466	return false;
3467
3468	// If we have a placement-new destination, we'll later use that instead
3469	// of allocating.
3470	if (OperatorNew->isReservedGlobalPlacementOperator())
3471	PlacementDest = Arg1;
3472	}
3473	} else {
3474	// Always invalid.
3475	return this->emitInvalid(E);
3476	}
3477	} else if (!OperatorNew
3478	->isUsableAsGlobalAllocationFunctionInConstantEvaluation())
3479	return this->emitInvalidNewDeleteExpr(E, E);
3480
3481	const Descriptor *Desc;
3482	if (!PlacementDest) {
3483	if (ElemT) {
3484	if (E->isArray())
3485	Desc = nullptr; // We're not going to use it in this case.
3486	else
3487	Desc = P.createDescriptor(D: E, T: *ElemT, /*SourceTy=*/nullptr,
3488	MDSize: Descriptor::InlineDescMD);
3489	} else {
3490	Desc = P.createDescriptor(
3491	D: E, Ty: ElementType.getTypePtr(),
3492	MDSize: E->isArray() ? std::nullopt : Descriptor::InlineDescMD,
3493	/*IsConst=*/false, /*IsTemporary=*/false, /*IsMutable=*/false,
3494	/*IsVolatile=*/false, Init);
3495	}
3496	}
3497
3498	if (E->isArray()) {
3499	std::optional<const Expr *> ArraySizeExpr = E->getArraySize();
3500	if (!ArraySizeExpr)
3501	return false;
3502
3503	const Expr *Stripped = *ArraySizeExpr;
3504	for (; auto *ICE = dyn_cast<ImplicitCastExpr>(Val: Stripped);
3505	Stripped = ICE->getSubExpr())
3506	if (ICE->getCastKind() != CK_NoOp &&
3507	ICE->getCastKind() != CK_IntegralCast)
3508	break;
3509
3510	PrimType SizeT = classifyPrim(Stripped->getType());
3511
3512	// Save evaluated array size to a variable.
3513	unsigned ArrayLen =
3514	allocateLocalPrimitive(Decl: Stripped, Ty: SizeT, /*IsConst=*/false);
3515	if (!this->visit(E: Stripped))
3516	return false;
3517	if (!this->emitSetLocal(SizeT, ArrayLen, E))
3518	return false;
3519
3520	if (PlacementDest) {
3521	if (!this->visit(E: PlacementDest))
3522	return false;
3523	if (!this->emitStartLifetime(E))
3524	return false;
3525	if (!this->emitGetLocal(SizeT, ArrayLen, E))
3526	return false;
3527	if (!this->emitCheckNewTypeMismatchArray(SizeT, E, E))
3528	return false;
3529	} else {
3530	if (!this->emitGetLocal(SizeT, ArrayLen, E))
3531	return false;
3532
3533	if (ElemT) {
3534	// N primitive elements.
3535	if (!this->emitAllocN(SizeT, *ElemT, E, IsNoThrow, E))
3536	return false;
3537	} else {
3538	// N Composite elements.
3539	if (!this->emitAllocCN(SizeT, Desc, IsNoThrow, E))
3540	return false;
3541	}
3542	}
3543
3544	if (Init) {
3545	QualType InitType = Init->getType();
3546	size_t StaticInitElems = 0;
3547	const Expr *DynamicInit = nullptr;
3548	if (const ConstantArrayType *CAT =
3549	Ctx.getASTContext().getAsConstantArrayType(T: InitType)) {
3550	StaticInitElems = CAT->getZExtSize();
3551	if (!this->visitInitializer(E: Init))
3552	return false;
3553
3554	if (const auto *ILE = dyn_cast<InitListExpr>(Val: Init);
3555	ILE && ILE->hasArrayFiller())
3556	DynamicInit = ILE->getArrayFiller();
3557	}
3558
3559	// The initializer initializes a certain number of elements, S.
3560	// However, the complete number of elements, N, might be larger than that.
3561	// In this case, we need to get an initializer for the remaining elements.
3562	// There are to cases:
3563	// 1) For the form 'new Struct[n];', the initializer is a
3564	// CXXConstructExpr and its type is an IncompleteArrayType.
3565	// 2) For the form 'new Struct[n]{1,2,3}', the initializer is an
3566	// InitListExpr and the initializer for the remaining elements
3567	// is the array filler.
3568
3569	if (DynamicInit || InitType->isIncompleteArrayType()) {
3570	const Function *CtorFunc = nullptr;
3571	if (const auto *CE = dyn_cast<CXXConstructExpr>(Val: Init)) {
3572	CtorFunc = getFunction(FD: CE->getConstructor());
3573	if (!CtorFunc)
3574	return false;
3575	} else if (!DynamicInit)
3576	DynamicInit = Init;
3577
3578	LabelTy EndLabel = this->getLabel();
3579	LabelTy StartLabel = this->getLabel();
3580
3581	// In the nothrow case, the alloc above might have returned nullptr.
3582	// Don't call any constructors that case.
3583	if (IsNoThrow) {
3584	if (!this->emitDupPtr(E))
3585	return false;
3586	if (!this->emitNullPtr(0, nullptr, E))
3587	return false;
3588	if (!this->emitEQPtr(E))
3589	return false;
3590	if (!this->jumpTrue(EndLabel))
3591	return false;
3592	}
3593
3594	// Create loop variables.
3595	unsigned Iter =
3596	allocateLocalPrimitive(Decl: Stripped, Ty: SizeT, /*IsConst=*/false);
3597	if (!this->emitConst(StaticInitElems, SizeT, E))
3598	return false;
3599	if (!this->emitSetLocal(SizeT, Iter, E))
3600	return false;
3601
3602	this->fallthrough(StartLabel);
3603	this->emitLabel(StartLabel);
3604	// Condition. Iter < ArrayLen?
3605	if (!this->emitGetLocal(SizeT, Iter, E))
3606	return false;
3607	if (!this->emitGetLocal(SizeT, ArrayLen, E))
3608	return false;
3609	if (!this->emitLT(SizeT, E))
3610	return false;
3611	if (!this->jumpFalse(EndLabel))
3612	return false;
3613
3614	// Pointer to the allocated array is already on the stack.
3615	if (!this->emitGetLocal(SizeT, Iter, E))
3616	return false;
3617	if (!this->emitArrayElemPtr(SizeT, E))
3618	return false;
3619
3620	if (isa_and_nonnull<ImplicitValueInitExpr>(Val: DynamicInit) &&
3621	DynamicInit->getType()->isArrayType()) {
3622	QualType ElemType =
3623	DynamicInit->getType()->getAsArrayTypeUnsafe()->getElementType();
3624	PrimType InitT = classifyPrim(ElemType);
3625	if (!this->visitZeroInitializer(T: InitT, QT: ElemType, E))
3626	return false;
3627	if (!this->emitStorePop(InitT, E))
3628	return false;
3629	} else if (DynamicInit) {
3630	if (std::optional<PrimType> InitT = classify(DynamicInit)) {
3631	if (!this->visit(E: DynamicInit))
3632	return false;
3633	if (!this->emitStorePop(*InitT, E))
3634	return false;
3635	} else {
3636	if (!this->visitInitializer(E: DynamicInit))
3637	return false;
3638	if (!this->emitPopPtr(E))
3639	return false;
3640	}
3641	} else {
3642	assert(CtorFunc);
3643	if (!this->emitCall(CtorFunc, 0, E))
3644	return false;
3645	}
3646
3647	// ++Iter;
3648	if (!this->emitGetPtrLocal(Iter, E))
3649	return false;
3650	if (!this->emitIncPop(SizeT, false, E))
3651	return false;
3652
3653	if (!this->jump(StartLabel))
3654	return false;
3655
3656	this->fallthrough(EndLabel);
3657	this->emitLabel(EndLabel);
3658	}
3659	}
3660	} else { // Non-array.
3661	if (PlacementDest) {
3662	if (!this->visit(E: PlacementDest))
3663	return false;
3664	if (!this->emitStartLifetime(E))
3665	return false;
3666	if (!this->emitCheckNewTypeMismatch(E, E))
3667	return false;
3668	} else {
3669	// Allocate just one element.
3670	if (!this->emitAlloc(Desc, E))
3671	return false;
3672	}
3673
3674	if (Init) {
3675	if (ElemT) {
3676	if (!this->visit(E: Init))
3677	return false;
3678
3679	if (!this->emitInit(*ElemT, E))
3680	return false;
3681	} else {
3682	// Composite.
3683	if (!this->visitInitializer(E: Init))
3684	return false;
3685	}
3686	}
3687	}
3688
3689	if (DiscardResult)
3690	return this->emitPopPtr(E);
3691
3692	return true;
3693	}
3694
3695	template <class Emitter>
3696	bool Compiler<Emitter>::VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
3697	const Expr *Arg = E->getArgument();
3698
3699	const FunctionDecl *OperatorDelete = E->getOperatorDelete();
3700
3701	if (!OperatorDelete->isUsableAsGlobalAllocationFunctionInConstantEvaluation())
3702	return this->emitInvalidNewDeleteExpr(E, E);
3703
3704	// Arg must be an lvalue.
3705	if (!this->visit(E: Arg))
3706	return false;
3707
3708	return this->emitFree(E->isArrayForm(), E->isGlobalDelete(), E);
3709	}
3710
3711	template <class Emitter>
3712	bool Compiler<Emitter>::VisitBlockExpr(const BlockExpr *E) {
3713	if (DiscardResult)
3714	return true;
3715
3716	const Function *Func = nullptr;
3717	if (auto F = Ctx.getOrCreateObjCBlock(E))
3718	Func = F;
3719
3720	if (!Func)
3721	return false;
3722	return this->emitGetFnPtr(Func, E);
3723	}
3724
3725	template <class Emitter>
3726	bool Compiler<Emitter>::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
3727	const Type *TypeInfoType = E->getType().getTypePtr();
3728
3729	auto canonType = [](const Type *T) {
3730	return T->getCanonicalTypeUnqualified().getTypePtr();
3731	};
3732
3733	if (!E->isPotentiallyEvaluated()) {
3734	if (DiscardResult)
3735	return true;
3736
3737	if (E->isTypeOperand())
3738	return this->emitGetTypeid(
3739	canonType(E->getTypeOperand(Context: Ctx.getASTContext()).getTypePtr()),
3740	TypeInfoType, E);
3741
3742	return this->emitGetTypeid(
3743	canonType(E->getExprOperand()->getType().getTypePtr()), TypeInfoType,
3744	E);
3745	}
3746
3747	// Otherwise, we need to evaluate the expression operand.
3748	assert(E->getExprOperand());
3749	assert(E->getExprOperand()->isLValue());
3750
3751	if (!Ctx.getLangOpts().CPlusPlus20 && !this->emitDiagTypeid(E))
3752	return false;
3753
3754	if (!this->visit(E: E->getExprOperand()))
3755	return false;
3756
3757	if (!this->emitGetTypeidPtr(TypeInfoType, E))
3758	return false;
3759	if (DiscardResult)
3760	return this->emitPopPtr(E);
3761	return true;
3762	}
3763
3764	template <class Emitter>
3765	bool Compiler<Emitter>::VisitExpressionTraitExpr(const ExpressionTraitExpr *E) {
3766	assert(Ctx.getLangOpts().CPlusPlus);
3767	return this->emitConstBool(E->getValue(), E);
3768	}
3769
3770	template <class Emitter>
3771	bool Compiler<Emitter>::VisitCXXUuidofExpr(const CXXUuidofExpr *E) {
3772	if (DiscardResult)
3773	return true;
3774	assert(!Initializing);
3775
3776	const MSGuidDecl *GuidDecl = E->getGuidDecl();
3777	const RecordDecl *RD = GuidDecl->getType()->getAsRecordDecl();
3778	assert(RD);
3779	// If the definiton of the result type is incomplete, just return a dummy.
3780	// If (and when) that is read from, we will fail, but not now.
3781	if (!RD->isCompleteDefinition())
3782	return this->emitDummyPtr(D: GuidDecl, E);
3783
3784	std::optional<unsigned> GlobalIndex = P.getOrCreateGlobal(VD: GuidDecl);
3785	if (!GlobalIndex)
3786	return false;
3787	if (!this->emitGetPtrGlobal(*GlobalIndex, E))
3788	return false;
3789
3790	assert(this->getRecord(E->getType()));
3791
3792	const APValue &V = GuidDecl->getAsAPValue();
3793	if (V.getKind() == APValue::None)
3794	return true;
3795
3796	assert(V.isStruct());
3797	assert(V.getStructNumBases() == 0);
3798	if (!this->visitAPValueInitializer(Val: V, E, T: E->getType()))
3799	return false;
3800
3801	return this->emitFinishInit(E);
3802	}
3803
3804	template <class Emitter>
3805	bool Compiler<Emitter>::VisitRequiresExpr(const RequiresExpr *E) {
3806	assert(classifyPrim(E->getType()) == PT_Bool);
3807	if (DiscardResult)
3808	return true;
3809	return this->emitConstBool(E->isSatisfied(), E);
3810	}
3811
3812	template <class Emitter>
3813	bool Compiler<Emitter>::VisitConceptSpecializationExpr(
3814	const ConceptSpecializationExpr *E) {
3815	assert(classifyPrim(E->getType()) == PT_Bool);
3816	if (DiscardResult)
3817	return true;
3818	return this->emitConstBool(E->isSatisfied(), E);
3819	}
3820
3821	template <class Emitter>
3822	bool Compiler<Emitter>::VisitCXXRewrittenBinaryOperator(
3823	const CXXRewrittenBinaryOperator *E) {
3824	return this->delegate(E: E->getSemanticForm());
3825	}
3826
3827	template <class Emitter>
3828	bool Compiler<Emitter>::VisitPseudoObjectExpr(const PseudoObjectExpr *E) {
3829
3830	for (const Expr *SemE : E->semantics()) {
3831	if (auto *OVE = dyn_cast<OpaqueValueExpr>(Val: SemE)) {
3832	if (SemE == E->getResultExpr())
3833	return false;
3834
3835	if (OVE->isUnique())
3836	continue;
3837
3838	if (!this->discard(E: OVE))
3839	return false;
3840	} else if (SemE == E->getResultExpr()) {
3841	if (!this->delegate(E: SemE))
3842	return false;
3843	} else {
3844	if (!this->discard(E: SemE))
3845	return false;
3846	}
3847	}
3848	return true;
3849	}
3850
3851	template <class Emitter>
3852	bool Compiler<Emitter>::VisitPackIndexingExpr(const PackIndexingExpr *E) {
3853	return this->delegate(E: E->getSelectedExpr());
3854	}
3855
3856	template <class Emitter>
3857	bool Compiler<Emitter>::VisitRecoveryExpr(const RecoveryExpr *E) {
3858	return this->emitError(E);
3859	}
3860
3861	template <class Emitter>
3862	bool Compiler<Emitter>::VisitAddrLabelExpr(const AddrLabelExpr *E) {
3863	assert(E->getType()->isVoidPointerType());
3864
3865	unsigned Offset =
3866	allocateLocalPrimitive(Decl: E->getLabel(), Ty: PT_Ptr, /*IsConst=*/true);
3867
3868	return this->emitGetLocal(PT_Ptr, Offset, E);
3869	}
3870
3871	template <class Emitter>
3872	bool Compiler<Emitter>::VisitConvertVectorExpr(const ConvertVectorExpr *E) {
3873	assert(Initializing);
3874	const auto *VT = E->getType()->castAs<VectorType>();
3875	QualType ElemType = VT->getElementType();
3876	PrimType ElemT = classifyPrim(ElemType);
3877	const Expr *Src = E->getSrcExpr();
3878	QualType SrcType = Src->getType();
3879	PrimType SrcElemT = classifyVectorElementType(T: SrcType);
3880
3881	unsigned SrcOffset =
3882	this->allocateLocalPrimitive(Decl: Src, Ty: PT_Ptr, /*IsConst=*/true);
3883	if (!this->visit(E: Src))
3884	return false;
3885	if (!this->emitSetLocal(PT_Ptr, SrcOffset, E))
3886	return false;
3887
3888	for (unsigned I = 0; I != VT->getNumElements(); ++I) {
3889	if (!this->emitGetLocal(PT_Ptr, SrcOffset, E))
3890	return false;
3891	if (!this->emitArrayElemPop(SrcElemT, I, E))
3892	return false;
3893
3894	// Cast to the desired result element type.
3895	if (SrcElemT != ElemT) {
3896	if (!this->emitPrimCast(FromT: SrcElemT, ToT: ElemT, ToQT: ElemType, E))
3897	return false;
3898	} else if (ElemType->isFloatingType() && SrcType != ElemType) {
3899	const auto *TargetSemantics = &Ctx.getFloatSemantics(T: ElemType);
3900	if (!this->emitCastFP(TargetSemantics, getRoundingMode(E), E))
3901	return false;
3902	}
3903	if (!this->emitInitElem(ElemT, I, E))
3904	return false;
3905	}
3906
3907	return true;
3908	}
3909
3910	template <class Emitter>
3911	bool Compiler<Emitter>::VisitShuffleVectorExpr(const ShuffleVectorExpr *E) {
3912	assert(Initializing);
3913	assert(E->getNumSubExprs() > 2);
3914
3915	const Expr *Vecs[] = {E->getExpr(Index: 0), E->getExpr(Index: 1)};
3916	const VectorType *VT = Vecs[0]->getType()->castAs<VectorType>();
3917	PrimType ElemT = classifyPrim(VT->getElementType());
3918	unsigned NumInputElems = VT->getNumElements();
3919	unsigned NumOutputElems = E->getNumSubExprs() - 2;
3920	assert(NumOutputElems > 0);
3921
3922	// Save both input vectors to a local variable.
3923	unsigned VectorOffsets[2];
3924	for (unsigned I = 0; I != 2; ++I) {
3925	VectorOffsets[I] =
3926	this->allocateLocalPrimitive(Decl: Vecs[I], Ty: PT_Ptr, /*IsConst=*/true);
3927	if (!this->visit(E: Vecs[I]))
3928	return false;
3929	if (!this->emitSetLocal(PT_Ptr, VectorOffsets[I], E))
3930	return false;
3931	}
3932	for (unsigned I = 0; I != NumOutputElems; ++I) {
3933	APSInt ShuffleIndex = E->getShuffleMaskIdx(N: I);
3934	assert(ShuffleIndex >= -1);
3935	if (ShuffleIndex == -1)
3936	return this->emitInvalidShuffleVectorIndex(I, E);
3937
3938	assert(ShuffleIndex < (NumInputElems * 2));
3939	if (!this->emitGetLocal(PT_Ptr,
3940	VectorOffsets[ShuffleIndex >= NumInputElems], E))
3941	return false;
3942	unsigned InputVectorIndex = ShuffleIndex.getZExtValue() % NumInputElems;
3943	if (!this->emitArrayElemPop(ElemT, InputVectorIndex, E))
3944	return false;
3945
3946	if (!this->emitInitElem(ElemT, I, E))
3947	return false;
3948	}
3949
3950	return true;
3951	}
3952
3953	template <class Emitter>
3954	bool Compiler<Emitter>::VisitExtVectorElementExpr(
3955	const ExtVectorElementExpr *E) {
3956	const Expr *Base = E->getBase();
3957	assert(
3958	Base->getType()->isVectorType() ||
3959	Base->getType()->getAs<PointerType>()->getPointeeType()->isVectorType());
3960
3961	SmallVector<uint32_t, 4> Indices;
3962	E->getEncodedElementAccess(Elts&: Indices);
3963
3964	if (Indices.size() == 1) {
3965	if (!this->visit(E: Base))
3966	return false;
3967
3968	if (E->isGLValue()) {
3969	if (!this->emitConstUint32(Indices[0], E))
3970	return false;
3971	return this->emitArrayElemPtrPop(PT_Uint32, E);
3972	}
3973	// Else, also load the value.
3974	return this->emitArrayElemPop(classifyPrim(E->getType()), Indices[0], E);
3975	}
3976
3977	// Create a local variable for the base.
3978	unsigned BaseOffset = allocateLocalPrimitive(Decl: Base, Ty: PT_Ptr, /*IsConst=*/true);
3979	if (!this->visit(E: Base))
3980	return false;
3981	if (!this->emitSetLocal(PT_Ptr, BaseOffset, E))
3982	return false;
3983
3984	// Now the vector variable for the return value.
3985	if (!Initializing) {
3986	std::optional<unsigned> ResultIndex;
3987	ResultIndex = allocateLocal(Decl: E);
3988	if (!ResultIndex)
3989	return false;
3990	if (!this->emitGetPtrLocal(*ResultIndex, E))
3991	return false;
3992	}
3993
3994	assert(Indices.size() == E->getType()->getAs<VectorType>()->getNumElements());
3995
3996	PrimType ElemT =
3997	classifyPrim(E->getType()->getAs<VectorType>()->getElementType());
3998	uint32_t DstIndex = 0;
3999	for (uint32_t I : Indices) {
4000	if (!this->emitGetLocal(PT_Ptr, BaseOffset, E))
4001	return false;
4002	if (!this->emitArrayElemPop(ElemT, I, E))
4003	return false;
4004	if (!this->emitInitElem(ElemT, DstIndex, E))
4005	return false;
4006	++DstIndex;
4007	}
4008
4009	// Leave the result pointer on the stack.
4010	assert(!DiscardResult);
4011	return true;
4012	}
4013
4014	template <class Emitter>
4015	bool Compiler<Emitter>::VisitObjCBoxedExpr(const ObjCBoxedExpr *E) {
4016	const Expr *SubExpr = E->getSubExpr();
4017	if (!E->isExpressibleAsConstantInitializer())
4018	return this->discard(E: SubExpr) && this->emitInvalid(E);
4019
4020	if (DiscardResult)
4021	return true;
4022
4023	assert(classifyPrim(E) == PT_Ptr);
4024	return this->emitDummyPtr(D: E, E);
4025	}
4026
4027	template <class Emitter>
4028	bool Compiler<Emitter>::VisitCXXStdInitializerListExpr(
4029	const CXXStdInitializerListExpr *E) {
4030	const Expr *SubExpr = E->getSubExpr();
4031	const ConstantArrayType *ArrayType =
4032	Ctx.getASTContext().getAsConstantArrayType(T: SubExpr->getType());
4033	const Record *R = getRecord(E->getType());
4034	assert(Initializing);
4035	assert(SubExpr->isGLValue());
4036
4037	if (!this->visit(E: SubExpr))
4038	return false;
4039	if (!this->emitConstUint8(0, E))
4040	return false;
4041	if (!this->emitArrayElemPtrPopUint8(E))
4042	return false;
4043	if (!this->emitInitFieldPtr(R->getField(I: 0u)->Offset, E))
4044	return false;
4045
4046	PrimType SecondFieldT = classifyPrim(R->getField(I: 1u)->Decl->getType());
4047	if (isIntegralType(T: SecondFieldT)) {
4048	if (!this->emitConst(static_cast<APSInt>(ArrayType->getSize()),
4049	SecondFieldT, E))
4050	return false;
4051	return this->emitInitField(SecondFieldT, R->getField(I: 1u)->Offset, E);
4052	}
4053	assert(SecondFieldT == PT_Ptr);
4054
4055	if (!this->emitGetFieldPtr(R->getField(I: 0u)->Offset, E))
4056	return false;
4057	if (!this->emitExpandPtr(E))
4058	return false;
4059	if (!this->emitConst(static_cast<APSInt>(ArrayType->getSize()), PT_Uint64, E))
4060	return false;
4061	if (!this->emitArrayElemPtrPop(PT_Uint64, E))
4062	return false;
4063	return this->emitInitFieldPtr(R->getField(I: 1u)->Offset, E);
4064	}
4065
4066	template <class Emitter>
4067	bool Compiler<Emitter>::VisitStmtExpr(const StmtExpr *E) {
4068	BlockScope<Emitter> BS(this);
4069	StmtExprScope<Emitter> SS(this);
4070
4071	const CompoundStmt *CS = E->getSubStmt();
4072	const Stmt *Result = CS->getStmtExprResult();
4073	for (const Stmt *S : CS->body()) {
4074	if (S != Result) {
4075	if (!this->visitStmt(S))
4076	return false;
4077	continue;
4078	}
4079
4080	assert(S == Result);
4081	if (const Expr *ResultExpr = dyn_cast<Expr>(Val: S))
4082	return this->delegate(E: ResultExpr);
4083	return this->emitUnsupported(E);
4084	}
4085
4086	return BS.destroyLocals();
4087	}
4088
4089	template <class Emitter> bool Compiler<Emitter>::discard(const Expr *E) {
4090	OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/true,
4091	/*NewInitializing=*/false);
4092	return this->Visit(E);
4093	}
4094
4095	template <class Emitter> bool Compiler<Emitter>::delegate(const Expr *E) {
4096	// We're basically doing:
4097	// OptionScope<Emitter> Scope(this, DicardResult, Initializing);
4098	// but that's unnecessary of course.
4099	return this->Visit(E);
4100	}
4101
4102	template <class Emitter> bool Compiler<Emitter>::visit(const Expr *E) {
4103	if (E->getType().isNull())
4104	return false;
4105
4106	if (E->getType()->isVoidType())
4107	return this->discard(E);
4108
4109	// Create local variable to hold the return value.
4110	if (!E->isGLValue() && !E->getType()->isAnyComplexType() &&
4111	!classify(E->getType())) {
4112	std::optional<unsigned> LocalIndex = allocateLocal(Decl: E);
4113	if (!LocalIndex)
4114	return false;
4115
4116	if (!this->emitGetPtrLocal(*LocalIndex, E))
4117	return false;
4118	InitLinkScope<Emitter> ILS(this, InitLink::Temp(Offset: *LocalIndex));
4119	return this->visitInitializer(E);
4120	}
4121
4122	// Otherwise,we have a primitive return value, produce the value directly
4123	// and push it on the stack.
4124	OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
4125	/*NewInitializing=*/false);
4126	return this->Visit(E);
4127	}
4128
4129	template <class Emitter>
4130	bool Compiler<Emitter>::visitInitializer(const Expr *E) {
4131	assert(!classify(E->getType()));
4132
4133	OptionScope<Emitter> Scope(this, /*NewDiscardResult=*/false,
4134	/*NewInitializing=*/true);
4135	return this->Visit(E);
4136	}
4137
4138	template <class Emitter> bool Compiler<Emitter>::visitBool(const Expr *E) {
4139	std::optional<PrimType> T = classify(E->getType());
4140	if (!T) {
4141	// Convert complex values to bool.
4142	if (E->getType()->isAnyComplexType()) {
4143	if (!this->visit(E))
4144	return false;
4145	return this->emitComplexBoolCast(E);
4146	}
4147	return false;
4148	}
4149
4150	if (!this->visit(E))
4151	return false;
4152
4153	if (T == PT_Bool)
4154	return true;
4155
4156	// Convert pointers to bool.
4157	if (T == PT_Ptr)
4158	return this->emitIsNonNullPtr(E);
4159
4160	// Or Floats.
4161	if (T == PT_Float)
4162	return this->emitCastFloatingIntegralBool(getFPOptions(E), E);
4163
4164	// Or anything else we can.
4165	return this->emitCast(*T, PT_Bool, E);
4166	}
4167
4168	template <class Emitter>
4169	bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
4170	const Expr *E) {
4171	if (const auto *AT = QT->getAs<AtomicType>())
4172	QT = AT->getValueType();
4173
4174	switch (T) {
4175	case PT_Bool:
4176	return this->emitZeroBool(E);
4177	case PT_Sint8:
4178	return this->emitZeroSint8(E);
4179	case PT_Uint8:
4180	return this->emitZeroUint8(E);
4181	case PT_Sint16:
4182	return this->emitZeroSint16(E);
4183	case PT_Uint16:
4184	return this->emitZeroUint16(E);
4185	case PT_Sint32:
4186	return this->emitZeroSint32(E);
4187	case PT_Uint32:
4188	return this->emitZeroUint32(E);
4189	case PT_Sint64:
4190	return this->emitZeroSint64(E);
4191	case PT_Uint64:
4192	return this->emitZeroUint64(E);
4193	case PT_IntAP:
4194	return this->emitZeroIntAP(Ctx.getBitWidth(T: QT), E);
4195	case PT_IntAPS:
4196	return this->emitZeroIntAPS(Ctx.getBitWidth(T: QT), E);
4197	case PT_Ptr:
4198	return this->emitNullPtr(Ctx.getASTContext().getTargetNullPointerValue(QT),
4199	nullptr, E);
4200	case PT_MemberPtr:
4201	return this->emitNullMemberPtr(0, nullptr, E);
4202	case PT_Float: {
4203	APFloat F = APFloat::getZero(Sem: Ctx.getFloatSemantics(T: QT));
4204	return this->emitFloat(F, E);
4205	}
4206	case PT_FixedPoint: {
4207	auto Sem = Ctx.getASTContext().getFixedPointSemantics(Ty: E->getType());
4208	return this->emitConstFixedPoint(FixedPoint::zero(Sem), E);
4209	}
4210	}
4211	llvm_unreachable("unknown primitive type");
4212	}
4213
4214	template <class Emitter>
4215	bool Compiler<Emitter>::visitZeroRecordInitializer(const Record *R,
4216	const Expr *E) {
4217	assert(E);
4218	assert(R);
4219	// Fields
4220	for (const Record::Field &Field : R->fields()) {
4221	if (Field.isUnnamedBitField())
4222	continue;
4223
4224	const Descriptor *D = Field.Desc;
4225	if (D->isPrimitive()) {
4226	QualType QT = D->getType();
4227	PrimType T = classifyPrim(D->getType());
4228	if (!this->visitZeroInitializer(T, QT, E))
4229	return false;
4230	if (!this->emitInitField(T, Field.Offset, E))
4231	return false;
4232	if (R->isUnion())
4233	break;
4234	continue;
4235	}
4236
4237	if (!this->emitGetPtrField(Field.Offset, E))
4238	return false;
4239
4240	if (D->isPrimitiveArray()) {
4241	QualType ET = D->getElemQualType();
4242	PrimType T = classifyPrim(ET);
4243	for (uint32_t I = 0, N = D->getNumElems(); I != N; ++I) {
4244	if (!this->visitZeroInitializer(T, QT: ET, E))
4245	return false;
4246	if (!this->emitInitElem(T, I, E))
4247	return false;
4248	}
4249	} else if (D->isCompositeArray()) {
4250	// Can't be a vector or complex field.
4251	if (!this->visitZeroArrayInitializer(T: D->getType(), E))
4252	return false;
4253	} else if (D->isRecord()) {
4254	if (!this->visitZeroRecordInitializer(R: D->ElemRecord, E))
4255	return false;
4256	} else
4257	return false;
4258
4259	if (!this->emitFinishInitPop(E))
4260	return false;
4261
4262	// C++11 [dcl.init]p5: If T is a (possibly cv-qualified) union type, the
4263	// object's first non-static named data member is zero-initialized
4264	if (R->isUnion())
4265	break;
4266	}
4267
4268	for (const Record::Base &B : R->bases()) {
4269	if (!this->emitGetPtrBase(B.Offset, E))
4270	return false;
4271	if (!this->visitZeroRecordInitializer(R: B.R, E))
4272	return false;
4273	if (!this->emitFinishInitPop(E))
4274	return false;
4275	}
4276
4277	// FIXME: Virtual bases.
4278
4279	return true;
4280	}
4281
4282	template <class Emitter>
4283	bool Compiler<Emitter>::visitZeroArrayInitializer(QualType T, const Expr *E) {
4284	assert(T->isArrayType() || T->isAnyComplexType() || T->isVectorType());
4285	const ArrayType *AT = T->getAsArrayTypeUnsafe();
4286	QualType ElemType = AT->getElementType();
4287	size_t NumElems = cast<ConstantArrayType>(Val: AT)->getZExtSize();
4288
4289	if (std::optional<PrimType> ElemT = classify(ElemType)) {
4290	for (size_t I = 0; I != NumElems; ++I) {
4291	if (!this->visitZeroInitializer(T: *ElemT, QT: ElemType, E))
4292	return false;
4293	if (!this->emitInitElem(*ElemT, I, E))
4294	return false;
4295	}
4296	return true;
4297	} else if (ElemType->isRecordType()) {
4298	const Record *R = getRecord(ElemType);
4299
4300	for (size_t I = 0; I != NumElems; ++I) {
4301	if (!this->emitConstUint32(I, E))
4302	return false;
4303	if (!this->emitArrayElemPtr(PT_Uint32, E))
4304	return false;
4305	if (!this->visitZeroRecordInitializer(R, E))
4306	return false;
4307	if (!this->emitPopPtr(E))
4308	return false;
4309	}
4310	return true;
4311	} else if (ElemType->isArrayType()) {
4312	for (size_t I = 0; I != NumElems; ++I) {
4313	if (!this->emitConstUint32(I, E))
4314	return false;
4315	if (!this->emitArrayElemPtr(PT_Uint32, E))
4316	return false;
4317	if (!this->visitZeroArrayInitializer(T: ElemType, E))
4318	return false;
4319	if (!this->emitPopPtr(E))
4320	return false;
4321	}
4322	return true;
4323	}
4324
4325	return false;
4326	}
4327
4328	template <class Emitter>
4329	template <typename T>
4330	bool Compiler<Emitter>::emitConst(T Value, PrimType Ty, const Expr *E) {
4331	switch (Ty) {
4332	case PT_Sint8:
4333	return this->emitConstSint8(Value, E);
4334	case PT_Uint8:
4335	return this->emitConstUint8(Value, E);
4336	case PT_Sint16:
4337	return this->emitConstSint16(Value, E);
4338	case PT_Uint16:
4339	return this->emitConstUint16(Value, E);
4340	case PT_Sint32:
4341	return this->emitConstSint32(Value, E);
4342	case PT_Uint32:
4343	return this->emitConstUint32(Value, E);
4344	case PT_Sint64:
4345	return this->emitConstSint64(Value, E);
4346	case PT_Uint64:
4347	return this->emitConstUint64(Value, E);
4348	case PT_Bool:
4349	return this->emitConstBool(Value, E);
4350	case PT_Ptr:
4351	case PT_MemberPtr:
4352	case PT_Float:
4353	case PT_IntAP:
4354	case PT_IntAPS:
4355	case PT_FixedPoint:
4356	llvm_unreachable("Invalid integral type");
4357	break;
4358	}
4359	llvm_unreachable("unknown primitive type");
4360	}
4361
4362	template <class Emitter>
4363	template <typename T>
4364	bool Compiler<Emitter>::emitConst(T Value, const Expr *E) {
4365	return this->emitConst(Value, classifyPrim(E->getType()), E);
4366	}
4367
4368	template <class Emitter>
4369	bool Compiler<Emitter>::emitConst(const APSInt &Value, PrimType Ty,
4370	const Expr *E) {
4371	if (Ty == PT_IntAPS)
4372	return this->emitConstIntAPS(Value, E);
4373	if (Ty == PT_IntAP)
4374	return this->emitConstIntAP(Value, E);
4375
4376	if (Value.isSigned())
4377	return this->emitConst(Value.getSExtValue(), Ty, E);
4378	return this->emitConst(Value.getZExtValue(), Ty, E);
4379	}
4380
4381	template <class Emitter>
4382	bool Compiler<Emitter>::emitConst(const APSInt &Value, const Expr *E) {
4383	return this->emitConst(Value, classifyPrim(E->getType()), E);
4384	}
4385
4386	template <class Emitter>
4387	unsigned Compiler<Emitter>::allocateLocalPrimitive(
4388	DeclTy &&Src, PrimType Ty, bool IsConst, const ValueDecl *ExtendingDecl,
4389	ScopeKind SC, bool IsConstexprUnknown) {
4390	// Make sure we don't accidentally register the same decl twice.
4391	if (const auto *VD =
4392	dyn_cast_if_present<ValueDecl>(Val: Src.dyn_cast<const Decl *>())) {
4393	assert(!P.getGlobal(VD));
4394	assert(!Locals.contains(VD));
4395	(void)VD;
4396	}
4397
4398	// FIXME: There are cases where Src.is<Expr*>() is wrong, e.g.
4399	// (int){12} in C. Consider using Expr::isTemporaryObject() instead
4400	// or isa<MaterializeTemporaryExpr>().
4401	Descriptor *D = P.createDescriptor(D: Src, T: Ty, SourceTy: nullptr, MDSize: Descriptor::InlineDescMD,
4402	IsConst, IsTemporary: isa<const Expr *>(Val: Src));
4403	D->IsConstexprUnknown = IsConstexprUnknown;
4404	Scope::Local Local = this->createLocal(D);
4405	if (auto *VD = dyn_cast_if_present<ValueDecl>(Val: Src.dyn_cast<const Decl *>()))
4406	Locals.insert(KV: {VD, Local});
4407	if (ExtendingDecl)
4408	VarScope->addExtended(Local, ExtendingDecl);
4409	else
4410	VarScope->addForScopeKind(Local, SC);
4411	return Local.Offset;
4412	}
4413
4414	template <class Emitter>
4415	std::optional<unsigned>
4416	Compiler<Emitter>::allocateLocal(DeclTy &&Src, QualType Ty,
4417	const ValueDecl *ExtendingDecl, ScopeKind SC,
4418	bool IsConstexprUnknown) {
4419	// Make sure we don't accidentally register the same decl twice.
4420	if ([[maybe_unused]] const auto *VD =
4421	dyn_cast_if_present<ValueDecl>(Val: Src.dyn_cast<const Decl *>())) {
4422	assert(!P.getGlobal(VD));
4423	assert(!Locals.contains(VD));
4424	}
4425
4426	const ValueDecl *Key = nullptr;
4427	const Expr *Init = nullptr;
4428	bool IsTemporary = false;
4429	if (auto *VD = dyn_cast_if_present<ValueDecl>(Val: Src.dyn_cast<const Decl *>())) {
4430	Key = VD;
4431	Ty = VD->getType();
4432
4433	if (const auto *VarD = dyn_cast<VarDecl>(Val: VD))
4434	Init = VarD->getInit();
4435	}
4436	if (auto *E = Src.dyn_cast<const Expr *>()) {
4437	IsTemporary = true;
4438	if (Ty.isNull())
4439	Ty = E->getType();
4440	}
4441
4442	Descriptor *D = P.createDescriptor(
4443	D: Src, Ty: Ty.getTypePtr(), MDSize: Descriptor::InlineDescMD, IsConst: Ty.isConstQualified(),
4444	IsTemporary, /*IsMutable=*/false, /*IsVolatile=*/false, Init);
4445	if (!D)
4446	return std::nullopt;
4447	D->IsConstexprUnknown = IsConstexprUnknown;
4448
4449	Scope::Local Local = this->createLocal(D);
4450	if (Key)
4451	Locals.insert(KV: {Key, Local});
4452	if (ExtendingDecl)
4453	VarScope->addExtended(Local, ExtendingDecl);
4454	else
4455	VarScope->addForScopeKind(Local, SC);
4456	return Local.Offset;
4457	}
4458
4459	template <class Emitter>
4460	std::optional<unsigned> Compiler<Emitter>::allocateTemporary(const Expr *E) {
4461	QualType Ty = E->getType();
4462	assert(!Ty->isRecordType());
4463
4464	Descriptor *D = P.createDescriptor(
4465	D: E, Ty: Ty.getTypePtr(), MDSize: Descriptor::InlineDescMD, IsConst: Ty.isConstQualified(),
4466	/*IsTemporary=*/true);
4467
4468	if (!D)
4469	return std::nullopt;
4470
4471	Scope::Local Local = this->createLocal(D);
4472	VariableScope<Emitter> *S = VarScope;
4473	assert(S);
4474	// Attach to topmost scope.
4475	while (S->getParent())
4476	S = S->getParent();
4477	assert(S && !S->getParent());
4478	S->addLocal(Local);
4479	return Local.Offset;
4480	}
4481
4482	template <class Emitter>
4483	const RecordType *Compiler<Emitter>::getRecordTy(QualType Ty) {
4484	if (const PointerType *PT = dyn_cast<PointerType>(Val&: Ty))
4485	return PT->getPointeeType()->getAs<RecordType>();
4486	return Ty->getAs<RecordType>();
4487	}
4488
4489	template <class Emitter> Record *Compiler<Emitter>::getRecord(QualType Ty) {
4490	if (const auto *RecordTy = getRecordTy(Ty))
4491	return getRecord(RecordTy->getDecl());
4492	return nullptr;
4493	}
4494
4495	template <class Emitter>
4496	Record *Compiler<Emitter>::getRecord(const RecordDecl *RD) {
4497	return P.getOrCreateRecord(RD);
4498	}
4499
4500	template <class Emitter>
4501	const Function *Compiler<Emitter>::getFunction(const FunctionDecl *FD) {
4502	return Ctx.getOrCreateFunction(FuncDecl: FD);
4503	}
4504
4505	template <class Emitter>
4506	bool Compiler<Emitter>::visitExpr(const Expr *E, bool DestroyToplevelScope) {
4507	LocalScope<Emitter> RootScope(this);
4508
4509	// If we won't destroy the toplevel scope, check for memory leaks first.
4510	if (!DestroyToplevelScope) {
4511	if (!this->emitCheckAllocations(E))
4512	return false;
4513	}
4514
4515	auto maybeDestroyLocals = [&]() -> bool {
4516	if (DestroyToplevelScope)
4517	return RootScope.destroyLocals() && this->emitCheckAllocations(E);
4518	return this->emitCheckAllocations(E);
4519	};
4520
4521	// Void expressions.
4522	if (E->getType()->isVoidType()) {
4523	if (!visit(E))
4524	return false;
4525	return this->emitRetVoid(E) && maybeDestroyLocals();
4526	}
4527
4528	// Expressions with a primitive return type.
4529	if (std::optional<PrimType> T = classify(E)) {
4530	if (!visit(E))
4531	return false;
4532
4533	return this->emitRet(*T, E) && maybeDestroyLocals();
4534	}
4535
4536	// Expressions with a composite return type.
4537	// For us, that means everything we don't
4538	// have a PrimType for.
4539	if (std::optional<unsigned> LocalOffset = this->allocateLocal(Src: E)) {
4540	InitLinkScope<Emitter> ILS(this, InitLink::Temp(Offset: *LocalOffset));
4541	if (!this->emitGetPtrLocal(*LocalOffset, E))
4542	return false;
4543
4544	if (!visitInitializer(E))
4545	return false;
4546
4547	if (!this->emitFinishInit(E))
4548	return false;
4549	// We are destroying the locals AFTER the Ret op.
4550	// The Ret op needs to copy the (alive) values, but the
4551	// destructors may still turn the entire expression invalid.
4552	return this->emitRetValue(E) && maybeDestroyLocals();
4553	}
4554
4555	return maybeDestroyLocals() && this->emitCheckAllocations(E) && false;
4556	}
4557
4558	template <class Emitter>
4559	VarCreationState Compiler<Emitter>::visitDecl(const VarDecl *VD,
4560	bool IsConstexprUnknown) {
4561
4562	auto R = this->visitVarDecl(VD, /*Toplevel=*/true, IsConstexprUnknown);
4563
4564	if (R.notCreated())
4565	return R;
4566
4567	if (R)
4568	return true;
4569
4570	if (!R && Context::shouldBeGloballyIndexed(VD)) {
4571	if (auto GlobalIndex = P.getGlobal(VD)) {
4572	Block *GlobalBlock = P.getGlobal(Idx: *GlobalIndex);
4573	GlobalInlineDescriptor &GD =
4574	*reinterpret_cast<GlobalInlineDescriptor *>(GlobalBlock->rawData());
4575
4576	GD.InitState = GlobalInitState::InitializerFailed;
4577	GlobalBlock->invokeDtor();
4578	}
4579	}
4580
4581	return R;
4582	}
4583
4584	/// Toplevel visitDeclAndReturn().
4585	/// We get here from evaluateAsInitializer().
4586	/// We need to evaluate the initializer and return its value.
4587	template <class Emitter>
4588	bool Compiler<Emitter>::visitDeclAndReturn(const VarDecl *VD,
4589	bool ConstantContext) {
4590
4591	// We only create variables if we're evaluating in a constant context.
4592	// Otherwise, just evaluate the initializer and return it.
4593	if (!ConstantContext) {
4594	DeclScope<Emitter> LS(this, VD);
4595	const Expr *Init = VD->getInit();
4596	if (!this->visit(E: Init))
4597	return false;
4598	return this->emitRet(classify(Init).value_or(PT_Ptr), VD) &&
4599	LS.destroyLocals() && this->emitCheckAllocations(VD);
4600	}
4601
4602	LocalScope<Emitter> VDScope(this, VD);
4603	if (!this->visitVarDecl(VD, /*Toplevel=*/true))
4604	return false;
4605
4606	std::optional<PrimType> VarT = classify(VD->getType());
4607	if (Context::shouldBeGloballyIndexed(VD)) {
4608	auto GlobalIndex = P.getGlobal(VD);
4609	assert(GlobalIndex); // visitVarDecl() didn't return false.
4610	if (VarT) {
4611	if (!this->emitGetGlobalUnchecked(*VarT, *GlobalIndex, VD))
4612	return false;
4613	} else {
4614	if (!this->emitGetPtrGlobal(*GlobalIndex, VD))
4615	return false;
4616	}
4617	} else {
4618	auto Local = Locals.find(Val: VD);
4619	assert(Local != Locals.end()); // Same here.
4620	if (VarT) {
4621	if (!this->emitGetLocal(*VarT, Local->second.Offset, VD))
4622	return false;
4623	} else {
4624	if (!this->emitGetPtrLocal(Local->second.Offset, VD))
4625	return false;
4626	}
4627	}
4628
4629	// Return the value.
4630	if (!this->emitRet(VarT.value_or(u: PT_Ptr), VD)) {
4631	// If the Ret above failed and this is a global variable, mark it as
4632	// uninitialized, even everything else succeeded.
4633	if (Context::shouldBeGloballyIndexed(VD)) {
4634	auto GlobalIndex = P.getGlobal(VD);
4635	assert(GlobalIndex);
4636	Block *GlobalBlock = P.getGlobal(Idx: *GlobalIndex);
4637	GlobalInlineDescriptor &GD =
4638	*reinterpret_cast<GlobalInlineDescriptor *>(GlobalBlock->rawData());
4639
4640	GD.InitState = GlobalInitState::InitializerFailed;
4641	GlobalBlock->invokeDtor();
4642	}
4643	return false;
4644	}
4645
4646	return VDScope.destroyLocals() && this->emitCheckAllocations(VD);
4647	}
4648
4649	template <class Emitter>
4650	VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
4651	bool Toplevel,
4652	bool IsConstexprUnknown) {
4653	// We don't know what to do with these, so just return false.
4654	if (VD->getType().isNull())
4655	return false;
4656
4657	// This case is EvalEmitter-only. If we won't create any instructions for the
4658	// initializer anyway, don't bother creating the variable in the first place.
4659	if (!this->isActive())
4660	return VarCreationState::NotCreated();
4661
4662	const Expr *Init = VD->getInit();
4663	std::optional<PrimType> VarT = classify(VD->getType());
4664
4665	if (Init && Init->isValueDependent())
4666	return false;
4667
4668	if (Context::shouldBeGloballyIndexed(VD)) {
4669	auto checkDecl = [&]() -> bool {
4670	bool NeedsOp = !Toplevel && VD->isLocalVarDecl() && VD->isStaticLocal();
4671	return !NeedsOp || this->emitCheckDecl(VD, VD);
4672	};
4673
4674	auto initGlobal = [&](unsigned GlobalIndex) -> bool {
4675	assert(Init);
4676
4677	if (VarT) {
4678	if (!this->visit(E: Init))
4679	return checkDecl() && false;
4680
4681	return checkDecl() && this->emitInitGlobal(*VarT, GlobalIndex, VD);
4682	}
4683
4684	if (!checkDecl())
4685	return false;
4686
4687	if (!this->emitGetPtrGlobal(GlobalIndex, Init))
4688	return false;
4689
4690	if (!visitInitializer(E: Init))
4691	return false;
4692
4693	return this->emitFinishInitGlobal(Init);
4694	};
4695
4696	DeclScope<Emitter> LocalScope(this, VD);
4697
4698	// We've already seen and initialized this global.
4699	if (std::optional<unsigned> GlobalIndex = P.getGlobal(VD)) {
4700	if (P.getPtrGlobal(Idx: *GlobalIndex).isInitialized())
4701	return checkDecl();
4702
4703	// The previous attempt at initialization might've been unsuccessful,
4704	// so let's try this one.
4705	return Init && checkDecl() && initGlobal(*GlobalIndex);
4706	}
4707
4708	std::optional<unsigned> GlobalIndex = P.createGlobal(VD, Init);
4709
4710	if (!GlobalIndex)
4711	return false;
4712
4713	return !Init || (checkDecl() && initGlobal(*GlobalIndex));
4714	}
4715	// Local variables.
4716	InitLinkScope<Emitter> ILS(this, InitLink::Decl(D: VD));
4717
4718	if (VarT) {
4719	unsigned Offset = this->allocateLocalPrimitive(
4720	Src: VD, Ty: *VarT, IsConst: VD->getType().isConstQualified(), ExtendingDecl: nullptr, SC: ScopeKind::Block,
4721	IsConstexprUnknown);
4722	if (Init) {
4723	// If this is a toplevel declaration, create a scope for the
4724	// initializer.
4725	if (Toplevel) {
4726	LocalScope<Emitter> Scope(this);
4727	if (!this->visit(E: Init))
4728	return false;
4729	return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
4730	} else {
4731	if (!this->visit(E: Init))
4732	return false;
4733	return this->emitSetLocal(*VarT, Offset, VD);
4734	}
4735	}
4736	} else {
4737	if (std::optional<unsigned> Offset = this->allocateLocal(
4738	Src: VD, Ty: VD->getType(), ExtendingDecl: nullptr, SC: ScopeKind::Block, IsConstexprUnknown)) {
4739	if (!Init)
4740	return true;
4741
4742	if (!this->emitGetPtrLocal(*Offset, Init))
4743	return false;
4744
4745	if (!visitInitializer(E: Init))
4746	return false;
4747
4748	return this->emitFinishInitPop(Init);
4749	}
4750	return false;
4751	}
4752	return true;
4753	}
4754
4755	template <class Emitter>
4756	bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
4757	const Expr *E) {
4758	assert(!DiscardResult);
4759	if (Val.isInt())
4760	return this->emitConst(Val.getInt(), ValType, E);
4761	else if (Val.isFloat()) {
4762	APFloat F = Val.getFloat();
4763	return this->emitFloat(F, E);
4764	}
4765
4766	if (Val.isLValue()) {
4767	if (Val.isNullPointer())
4768	return this->emitNull(ValType, 0, nullptr, E);
4769	APValue::LValueBase Base = Val.getLValueBase();
4770	if (const Expr *BaseExpr = Base.dyn_cast<const Expr *>())
4771	return this->visit(E: BaseExpr);
4772	else if (const auto *VD = Base.dyn_cast<const ValueDecl *>()) {
4773	return this->visitDeclRef(D: VD, E);
4774	}
4775	} else if (Val.isMemberPointer()) {
4776	if (const ValueDecl *MemberDecl = Val.getMemberPointerDecl())
4777	return this->emitGetMemberPtr(MemberDecl, E);
4778	return this->emitNullMemberPtr(0, nullptr, E);
4779	}
4780
4781	return false;
4782	}
4783
4784	template <class Emitter>
4785	bool Compiler<Emitter>::visitAPValueInitializer(const APValue &Val,
4786	const Expr *E, QualType T) {
4787	if (Val.isStruct()) {
4788	const Record *R = this->getRecord(T);
4789	assert(R);
4790	for (unsigned I = 0, N = Val.getStructNumFields(); I != N; ++I) {
4791	const APValue &F = Val.getStructField(i: I);
4792	const Record::Field *RF = R->getField(I);
4793	QualType FieldType = RF->Decl->getType();
4794
4795	if (std::optional<PrimType> PT = classify(FieldType)) {
4796	if (!this->visitAPValue(Val: F, ValType: *PT, E))
4797	return false;
4798	if (!this->emitInitField(*PT, RF->Offset, E))
4799	return false;
4800	} else {
4801	if (!this->emitGetPtrField(RF->Offset, E))
4802	return false;
4803	if (!this->visitAPValueInitializer(Val: F, E, T: FieldType))
4804	return false;
4805	if (!this->emitPopPtr(E))
4806	return false;
4807	}
4808	}
4809	return true;
4810	} else if (Val.isUnion()) {
4811	const FieldDecl *UnionField = Val.getUnionField();
4812	const Record *R = this->getRecord(UnionField->getParent());
4813	assert(R);
4814	const APValue &F = Val.getUnionValue();
4815	const Record::Field *RF = R->getField(FD: UnionField);
4816	PrimType T = classifyPrim(RF->Decl->getType());
4817	if (!this->visitAPValue(Val: F, ValType: T, E))
4818	return false;
4819	return this->emitInitField(T, RF->Offset, E);
4820	} else if (Val.isArray()) {
4821	const auto *ArrType = T->getAsArrayTypeUnsafe();
4822	QualType ElemType = ArrType->getElementType();
4823	for (unsigned A = 0, AN = Val.getArraySize(); A != AN; ++A) {
4824	const APValue &Elem = Val.getArrayInitializedElt(I: A);
4825	if (std::optional<PrimType> ElemT = classify(ElemType)) {
4826	if (!this->visitAPValue(Val: Elem, ValType: *ElemT, E))
4827	return false;
4828	if (!this->emitInitElem(*ElemT, A, E))
4829	return false;
4830	} else {
4831	if (!this->emitConstUint32(A, E))
4832	return false;
4833	if (!this->emitArrayElemPtrUint32(E))
4834	return false;
4835	if (!this->visitAPValueInitializer(Val: Elem, E, T: ElemType))
4836	return false;
4837	if (!this->emitPopPtr(E))
4838	return false;
4839	}
4840	}
4841	return true;
4842	}
4843	// TODO: Other types.
4844
4845	return false;
4846	}
4847
4848	template <class Emitter>
4849	bool Compiler<Emitter>::VisitBuiltinCallExpr(const CallExpr *E,
4850	unsigned BuiltinID) {
4851
4852	if (BuiltinID == Builtin::BI__builtin_constant_p) {
4853	// Void argument is always invalid and harder to handle later.
4854	if (E->getArg(Arg: 0)->getType()->isVoidType()) {
4855	if (DiscardResult)
4856	return true;
4857	return this->emitConst(0, E);
4858	}
4859
4860	if (!this->emitStartSpeculation(E))
4861	return false;
4862	LabelTy EndLabel = this->getLabel();
4863	if (!this->speculate(E, EndLabel))
4864	return false;
4865	this->fallthrough(EndLabel);
4866	if (!this->emitEndSpeculation(E))
4867	return false;
4868	if (DiscardResult)
4869	return this->emitPop(classifyPrim(E), E);
4870	return true;
4871	}
4872
4873	// For these, we're expected to ultimately return an APValue pointing
4874	// to the CallExpr. This is needed to get the correct codegen.
4875	if (BuiltinID == Builtin::BI__builtin___CFStringMakeConstantString ||
4876	BuiltinID == Builtin::BI__builtin___NSStringMakeConstantString ||
4877	BuiltinID == Builtin::BI__builtin_ptrauth_sign_constant ||
4878	BuiltinID == Builtin::BI__builtin_function_start) {
4879	if (DiscardResult)
4880	return true;
4881	return this->emitDummyPtr(D: E, E);
4882	}
4883
4884	QualType ReturnType = E->getType();
4885	std::optional<PrimType> ReturnT = classify(E);
4886
4887	// Non-primitive return type. Prepare storage.
4888	if (!Initializing && !ReturnT && !ReturnType->isVoidType()) {
4889	std::optional<unsigned> LocalIndex = allocateLocal(Src: E);
4890	if (!LocalIndex)
4891	return false;
4892	if (!this->emitGetPtrLocal(*LocalIndex, E))
4893	return false;
4894	}
4895
4896	if (!Context::isUnevaluatedBuiltin(ID: BuiltinID)) {
4897	// Put arguments on the stack.
4898	for (const auto *Arg : E->arguments()) {
4899	if (!this->visit(E: Arg))
4900	return false;
4901	}
4902	}
4903
4904	if (!this->emitCallBI(E, BuiltinID, E))
4905	return false;
4906
4907	if (DiscardResult && !ReturnType->isVoidType()) {
4908	assert(ReturnT);
4909	return this->emitPop(*ReturnT, E);
4910	}
4911
4912	return true;
4913	}
4914
4915	static const Expr *stripDerivedToBaseCasts(const Expr *E) {
4916	if (const auto *PE = dyn_cast<ParenExpr>(Val: E))
4917	return stripDerivedToBaseCasts(E: PE->getSubExpr());
4918
4919	if (const auto *CE = dyn_cast<CastExpr>(Val: E);
4920	CE &&
4921	(CE->getCastKind() == CK_DerivedToBase || CE->getCastKind() == CK_NoOp))
4922	return stripDerivedToBaseCasts(E: CE->getSubExpr());
4923
4924	return E;
4925	}
4926
4927	template <class Emitter>
4928	bool Compiler<Emitter>::VisitCallExpr(const CallExpr *E) {
4929	const FunctionDecl *FuncDecl = E->getDirectCallee();
4930
4931	if (FuncDecl) {
4932	if (unsigned BuiltinID = FuncDecl->getBuiltinID())
4933	return VisitBuiltinCallExpr(E, BuiltinID);
4934
4935	// Calls to replaceable operator new/operator delete.
4936	if (FuncDecl->isUsableAsGlobalAllocationFunctionInConstantEvaluation()) {
4937	if (FuncDecl->getDeclName().isAnyOperatorNew()) {
4938	return VisitBuiltinCallExpr(E, BuiltinID: Builtin::BI__builtin_operator_new);
4939	} else {
4940	assert(FuncDecl->getDeclName().getCXXOverloadedOperator() == OO_Delete);
4941	return VisitBuiltinCallExpr(E, BuiltinID: Builtin::BI__builtin_operator_delete);
4942	}
4943	}
4944
4945	// Explicit calls to trivial destructors
4946	if (const auto *DD = dyn_cast<CXXDestructorDecl>(Val: FuncDecl);
4947	DD && DD->isTrivial()) {
4948	const auto *MemberCall = cast<CXXMemberCallExpr>(Val: E);
4949	if (!this->visit(E: MemberCall->getImplicitObjectArgument()))
4950	return false;
4951	return this->emitCheckDestruction(E) && this->emitEndLifetime(E) &&
4952	this->emitPopPtr(E);
4953	}
4954	}
4955
4956	BlockScope<Emitter> CallScope(this, ScopeKind::Call);
4957
4958	QualType ReturnType = E->getCallReturnType(Ctx: Ctx.getASTContext());
4959	std::optional<PrimType> T = classify(ReturnType);
4960	bool HasRVO = !ReturnType->isVoidType() && !T;
4961
4962	if (HasRVO) {
4963	if (DiscardResult) {
4964	// If we need to discard the return value but the function returns its
4965	// value via an RVO pointer, we need to create one such pointer just
4966	// for this call.
4967	if (std::optional<unsigned> LocalIndex = allocateLocal(Src: E)) {
4968	if (!this->emitGetPtrLocal(*LocalIndex, E))
4969	return false;
4970	}
4971	} else {
4972	// We need the result. Prepare a pointer to return or
4973	// dup the current one.
4974	if (!Initializing) {
4975	if (std::optional<unsigned> LocalIndex = allocateLocal(Src: E)) {
4976	if (!this->emitGetPtrLocal(*LocalIndex, E))
4977	return false;
4978	}
4979	}
4980	if (!this->emitDupPtr(E))
4981	return false;
4982	}
4983	}
4984
4985	SmallVector<const Expr *, 8> Args(ArrayRef(E->getArgs(), E->getNumArgs()));
4986
4987	bool IsAssignmentOperatorCall = false;
4988	if (const auto *OCE = dyn_cast<CXXOperatorCallExpr>(Val: E);
4989	OCE && OCE->isAssignmentOp()) {
4990	// Just like with regular assignments, we need to special-case assignment
4991	// operators here and evaluate the RHS (the second arg) before the LHS (the
4992	// first arg). We fix this by using a Flip op later.
4993	assert(Args.size() == 2);
4994	IsAssignmentOperatorCall = true;
4995	std::reverse(first: Args.begin(), last: Args.end());
4996	}
4997	// Calling a static operator will still
4998	// pass the instance, but we don't need it.
4999	// Discard it here.
5000	if (isa<CXXOperatorCallExpr>(Val: E)) {
5001	if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: FuncDecl);
5002	MD && MD->isStatic()) {
5003	if (!this->discard(E: E->getArg(Arg: 0)))
5004	return false;
5005	// Drop first arg.
5006	Args.erase(CI: Args.begin());
5007	}
5008	}
5009
5010	bool Devirtualized = false;
5011	std::optional<unsigned> CalleeOffset;
5012	// Add the (optional, implicit) This pointer.
5013	if (const auto *MC = dyn_cast<CXXMemberCallExpr>(Val: E)) {
5014	if (!FuncDecl && classifyPrim(E->getCallee()) == PT_MemberPtr) {
5015	// If we end up creating a CallPtr op for this, we need the base of the
5016	// member pointer as the instance pointer, and later extract the function
5017	// decl as the function pointer.
5018	const Expr *Callee = E->getCallee();
5019	CalleeOffset =
5020	this->allocateLocalPrimitive(Src: Callee, Ty: PT_MemberPtr, /*IsConst=*/true);
5021	if (!this->visit(E: Callee))
5022	return false;
5023	if (!this->emitSetLocal(PT_MemberPtr, *CalleeOffset, E))
5024	return false;
5025	if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E))
5026	return false;
5027	if (!this->emitGetMemberPtrBase(E))
5028	return false;
5029	} else {
5030	const auto *InstancePtr = MC->getImplicitObjectArgument();
5031	if (isa_and_nonnull<CXXDestructorDecl>(Val: CompilingFunction) ||
5032	isa_and_nonnull<CXXConstructorDecl>(Val: CompilingFunction)) {
5033	const auto *Stripped = stripDerivedToBaseCasts(E: InstancePtr);
5034	if (isa<CXXThisExpr>(Val: Stripped)) {
5035	FuncDecl =
5036	cast<CXXMethodDecl>(Val: FuncDecl)->getCorrespondingMethodInClass(
5037	RD: Stripped->getType()->getPointeeType()->getAsCXXRecordDecl());
5038	Devirtualized = true;
5039	if (!this->visit(E: Stripped))
5040	return false;
5041	} else {
5042	if (!this->visit(E: InstancePtr))
5043	return false;
5044	}
5045	} else {
5046	if (!this->visit(E: InstancePtr))
5047	return false;
5048	}
5049	}
5050	} else if (const auto *PD =
5051	dyn_cast<CXXPseudoDestructorExpr>(Val: E->getCallee())) {
5052	if (!this->emitCheckPseudoDtor(E))
5053	return false;
5054	const Expr *Base = PD->getBase();
5055	if (!Base->isGLValue())
5056	return this->discard(E: Base);
5057	if (!this->visit(E: Base))
5058	return false;
5059	return this->emitEndLifetimePop(E);
5060	} else if (!FuncDecl) {
5061	const Expr *Callee = E->getCallee();
5062	CalleeOffset =
5063	this->allocateLocalPrimitive(Src: Callee, Ty: PT_Ptr, /*IsConst=*/true);
5064	if (!this->visit(E: Callee))
5065	return false;
5066	if (!this->emitSetLocal(PT_Ptr, *CalleeOffset, E))
5067	return false;
5068	}
5069
5070	if (!this->visitCallArgs(Args, FuncDecl))
5071	return false;
5072
5073	// Undo the argument reversal we did earlier.
5074	if (IsAssignmentOperatorCall) {
5075	assert(Args.size() == 2);
5076	PrimType Arg1T = classify(Args[0]).value_or(PT_Ptr);
5077	PrimType Arg2T = classify(Args[1]).value_or(PT_Ptr);
5078	if (!this->emitFlip(Arg2T, Arg1T, E))
5079	return false;
5080	}
5081
5082	if (FuncDecl) {
5083	const Function *Func = getFunction(FD: FuncDecl);
5084	if (!Func)
5085	return false;
5086	assert(HasRVO == Func->hasRVO());
5087
5088	bool HasQualifier = false;
5089	if (const auto *ME = dyn_cast<MemberExpr>(Val: E->getCallee()))
5090	HasQualifier = ME->hasQualifier();
5091
5092	bool IsVirtual = false;
5093	if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: FuncDecl))
5094	IsVirtual = !Devirtualized && MD->isVirtual();
5095
5096	// In any case call the function. The return value will end up on the stack
5097	// and if the function has RVO, we already have the pointer on the stack to
5098	// write the result into.
5099	if (IsVirtual && !HasQualifier) {
5100	uint32_t VarArgSize = 0;
5101	unsigned NumParams =
5102	Func->getNumWrittenParams() + isa<CXXOperatorCallExpr>(Val: E);
5103	for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I)
5104	VarArgSize += align(primSize(classify(E->getArg(Arg: I)).value_or(PT_Ptr)));
5105
5106	if (!this->emitCallVirt(Func, VarArgSize, E))
5107	return false;
5108	} else if (Func->isVariadic()) {
5109	uint32_t VarArgSize = 0;
5110	unsigned NumParams =
5111	Func->getNumWrittenParams() + isa<CXXOperatorCallExpr>(Val: E);
5112	for (unsigned I = NumParams, N = E->getNumArgs(); I != N; ++I)
5113	VarArgSize += align(primSize(classify(E->getArg(Arg: I)).value_or(PT_Ptr)));
5114	if (!this->emitCallVar(Func, VarArgSize, E))
5115	return false;
5116	} else {
5117	if (!this->emitCall(Func, 0, E))
5118	return false;
5119	}
5120	} else {
5121	// Indirect call. Visit the callee, which will leave a FunctionPointer on
5122	// the stack. Cleanup of the returned value if necessary will be done after
5123	// the function call completed.
5124
5125	// Sum the size of all args from the call expr.
5126	uint32_t ArgSize = 0;
5127	for (unsigned I = 0, N = E->getNumArgs(); I != N; ++I)
5128	ArgSize += align(primSize(classify(E->getArg(Arg: I)).value_or(PT_Ptr)));
5129
5130	// Get the callee, either from a member pointer or function pointer saved in
5131	// CalleeOffset.
5132	if (isa<CXXMemberCallExpr>(Val: E) && CalleeOffset) {
5133	if (!this->emitGetLocal(PT_MemberPtr, *CalleeOffset, E))
5134	return false;
5135	if (!this->emitGetMemberPtrDecl(E))
5136	return false;
5137	} else {
5138	if (!this->emitGetLocal(PT_Ptr, *CalleeOffset, E))
5139	return false;
5140	}
5141	if (!this->emitCallPtr(ArgSize, E, E))
5142	return false;
5143	}
5144
5145	// Cleanup for discarded return values.
5146	if (DiscardResult && !ReturnType->isVoidType() && T)
5147	return this->emitPop(*T, E) && CallScope.destroyLocals();
5148
5149	return CallScope.destroyLocals();
5150	}
5151
5152	template <class Emitter>
5153	bool Compiler<Emitter>::VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *E) {
5154	SourceLocScope<Emitter> SLS(this, E);
5155
5156	return this->delegate(E: E->getExpr());
5157	}
5158
5159	template <class Emitter>
5160	bool Compiler<Emitter>::VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *E) {
5161	SourceLocScope<Emitter> SLS(this, E);
5162
5163	return this->delegate(E: E->getExpr());
5164	}
5165
5166	template <class Emitter>
5167	bool Compiler<Emitter>::VisitCXXBoolLiteralExpr(const CXXBoolLiteralExpr *E) {
5168	if (DiscardResult)
5169	return true;
5170
5171	return this->emitConstBool(E->getValue(), E);
5172	}
5173
5174	template <class Emitter>
5175	bool Compiler<Emitter>::VisitCXXNullPtrLiteralExpr(
5176	const CXXNullPtrLiteralExpr *E) {
5177	if (DiscardResult)
5178	return true;
5179
5180	uint64_t Val = Ctx.getASTContext().getTargetNullPointerValue(QT: E->getType());
5181	return this->emitNullPtr(Val, nullptr, E);
5182	}
5183
5184	template <class Emitter>
5185	bool Compiler<Emitter>::VisitGNUNullExpr(const GNUNullExpr *E) {
5186	if (DiscardResult)
5187	return true;
5188
5189	assert(E->getType()->isIntegerType());
5190
5191	PrimType T = classifyPrim(E->getType());
5192	return this->emitZero(T, E);
5193	}
5194
5195	template <class Emitter>
5196	bool Compiler<Emitter>::VisitCXXThisExpr(const CXXThisExpr *E) {
5197	if (DiscardResult)
5198	return true;
5199
5200	if (this->LambdaThisCapture.Offset > 0) {
5201	if (this->LambdaThisCapture.IsPtr)
5202	return this->emitGetThisFieldPtr(this->LambdaThisCapture.Offset, E);
5203	return this->emitGetPtrThisField(this->LambdaThisCapture.Offset, E);
5204	}
5205
5206	// In some circumstances, the 'this' pointer does not actually refer to the
5207	// instance pointer of the current function frame, but e.g. to the declaration
5208	// currently being initialized. Here we emit the necessary instruction(s) for
5209	// this scenario.
5210	if (!InitStackActive)
5211	return this->emitThis(E);
5212
5213	if (!InitStack.empty()) {
5214	// If our init stack is, for example:
5215	// 0 Stack: 3 (decl)
5216	// 1 Stack: 6 (init list)
5217	// 2 Stack: 1 (field)
5218	// 3 Stack: 6 (init list)
5219	// 4 Stack: 1 (field)
5220	//
5221	// We want to find the LAST element in it that's an init list,
5222	// which is marked with the K_InitList marker. The index right
5223	// before that points to an init list. We need to find the
5224	// elements before the K_InitList element that point to a base
5225	// (e.g. a decl or This), optionally followed by field, elem, etc.
5226	// In the example above, we want to emit elements [0..2].
5227	unsigned StartIndex = 0;
5228	unsigned EndIndex = 0;
5229	// Find the init list.
5230	for (StartIndex = InitStack.size() - 1; StartIndex > 0; --StartIndex) {
5231	if (InitStack[StartIndex].Kind == InitLink::K_InitList ||
5232	InitStack[StartIndex].Kind == InitLink::K_This) {
5233	EndIndex = StartIndex;
5234	--StartIndex;
5235	break;
5236	}
5237	}
5238
5239	// Walk backwards to find the base.
5240	for (; StartIndex > 0; --StartIndex) {
5241	if (InitStack[StartIndex].Kind == InitLink::K_InitList)
5242	continue;
5243
5244	if (InitStack[StartIndex].Kind != InitLink::K_Field &&
5245	InitStack[StartIndex].Kind != InitLink::K_Elem)
5246	break;
5247	}
5248
5249	// Emit the instructions.
5250	for (unsigned I = StartIndex; I != EndIndex; ++I) {
5251	if (InitStack[I].Kind == InitLink::K_InitList)
5252	continue;
5253	if (!InitStack[I].template emit<Emitter>(this, E))
5254	return false;
5255	}
5256	return true;
5257	}
5258	return this->emitThis(E);
5259	}
5260
5261	template <class Emitter> bool Compiler<Emitter>::visitStmt(const Stmt *S) {
5262	switch (S->getStmtClass()) {
5263	case Stmt::CompoundStmtClass:
5264	return visitCompoundStmt(S: cast<CompoundStmt>(Val: S));
5265	case Stmt::DeclStmtClass:
5266	return visitDeclStmt(DS: cast<DeclStmt>(Val: S), /*EvaluateConditionDecl=*/true);
5267	case Stmt::ReturnStmtClass:
5268	return visitReturnStmt(RS: cast<ReturnStmt>(Val: S));
5269	case Stmt::IfStmtClass:
5270	return visitIfStmt(IS: cast<IfStmt>(Val: S));
5271	case Stmt::WhileStmtClass:
5272	return visitWhileStmt(S: cast<WhileStmt>(Val: S));
5273	case Stmt::DoStmtClass:
5274	return visitDoStmt(S: cast<DoStmt>(Val: S));
5275	case Stmt::ForStmtClass:
5276	return visitForStmt(S: cast<ForStmt>(Val: S));
5277	case Stmt::CXXForRangeStmtClass:
5278	return visitCXXForRangeStmt(S: cast<CXXForRangeStmt>(Val: S));
5279	case Stmt::BreakStmtClass:
5280	return visitBreakStmt(S: cast<BreakStmt>(Val: S));
5281	case Stmt::ContinueStmtClass:
5282	return visitContinueStmt(S: cast<ContinueStmt>(Val: S));
5283	case Stmt::SwitchStmtClass:
5284	return visitSwitchStmt(S: cast<SwitchStmt>(Val: S));
5285	case Stmt::CaseStmtClass:
5286	return visitCaseStmt(S: cast<CaseStmt>(Val: S));
5287	case Stmt::DefaultStmtClass:
5288	return visitDefaultStmt(S: cast<DefaultStmt>(Val: S));
5289	case Stmt::AttributedStmtClass:
5290	return visitAttributedStmt(S: cast<AttributedStmt>(Val: S));
5291	case Stmt::CXXTryStmtClass:
5292	return visitCXXTryStmt(S: cast<CXXTryStmt>(Val: S));
5293	case Stmt::NullStmtClass:
5294	return true;
5295	// Always invalid statements.
5296	case Stmt::GCCAsmStmtClass:
5297	case Stmt::MSAsmStmtClass:
5298	case Stmt::GotoStmtClass:
5299	return this->emitInvalid(S);
5300	case Stmt::LabelStmtClass:
5301	return this->visitStmt(S: cast<LabelStmt>(Val: S)->getSubStmt());
5302	default: {
5303	if (const auto *E = dyn_cast<Expr>(Val: S))
5304	return this->discard(E);
5305	return false;
5306	}
5307	}
5308	}
5309
5310	template <class Emitter>
5311	bool Compiler<Emitter>::visitCompoundStmt(const CompoundStmt *S) {
5312	BlockScope<Emitter> Scope(this);
5313	for (const auto *InnerStmt : S->body())
5314	if (!visitStmt(S: InnerStmt))
5315	return false;
5316	return Scope.destroyLocals();
5317	}
5318
5319	template <class Emitter>
5320	bool Compiler<Emitter>::maybeEmitDeferredVarInit(const VarDecl *VD) {
5321	if (auto *DD = dyn_cast_if_present<DecompositionDecl>(Val: VD)) {
5322	for (auto *BD : DD->flat_bindings())
5323	if (auto *KD = BD->getHoldingVar(); KD && !this->visitVarDecl(VD: KD))
5324	return false;
5325	}
5326	return true;
5327	}
5328
5329	template <class Emitter>
5330	bool Compiler<Emitter>::visitDeclStmt(const DeclStmt *DS,
5331	bool EvaluateConditionDecl) {
5332	for (const auto *D : DS->decls()) {
5333	if (isa<StaticAssertDecl, TagDecl, TypedefNameDecl, BaseUsingDecl,
5334	FunctionDecl, NamespaceAliasDecl, UsingDirectiveDecl>(Val: D))
5335	continue;
5336
5337	const auto *VD = dyn_cast<VarDecl>(Val: D);
5338	if (!VD)
5339	return false;
5340	if (!this->visitVarDecl(VD))
5341	return false;
5342
5343	// Register decomposition decl holding vars.
5344	if (EvaluateConditionDecl && !this->maybeEmitDeferredVarInit(VD))
5345	return false;
5346	}
5347
5348	return true;
5349	}
5350
5351	template <class Emitter>
5352	bool Compiler<Emitter>::visitReturnStmt(const ReturnStmt *RS) {
5353	if (this->InStmtExpr)
5354	return this->emitUnsupported(RS);
5355
5356	if (const Expr *RE = RS->getRetValue()) {
5357	LocalScope<Emitter> RetScope(this);
5358	if (ReturnType) {
5359	// Primitive types are simply returned.
5360	if (!this->visit(E: RE))
5361	return false;
5362	this->emitCleanup();
5363	return this->emitRet(*ReturnType, RS);
5364	} else if (RE->getType()->isVoidType()) {
5365	if (!this->visit(E: RE))
5366	return false;
5367	} else {
5368	InitLinkScope<Emitter> ILS(this, InitLink::RVO());
5369	// RVO - construct the value in the return location.
5370	if (!this->emitRVOPtr(RE))
5371	return false;
5372	if (!this->visitInitializer(E: RE))
5373	return false;
5374	if (!this->emitPopPtr(RE))
5375	return false;
5376
5377	this->emitCleanup();
5378	return this->emitRetVoid(RS);
5379	}
5380	}
5381
5382	// Void return.
5383	this->emitCleanup();
5384	return this->emitRetVoid(RS);
5385	}
5386
5387	template <class Emitter> bool Compiler<Emitter>::visitIfStmt(const IfStmt *IS) {
5388	auto visitChildStmt = [&](const Stmt *S) -> bool {
5389	LocalScope<Emitter> SScope(this);
5390	if (!visitStmt(S))
5391	return false;
5392	return SScope.destroyLocals();
5393	};
5394	if (auto *CondInit = IS->getInit())
5395	if (!visitStmt(S: CondInit))
5396	return false;
5397
5398	if (const DeclStmt *CondDecl = IS->getConditionVariableDeclStmt())
5399	if (!visitDeclStmt(DS: CondDecl))
5400	return false;
5401
5402	// Save ourselves compiling some code and the jumps, etc. if the condition is
5403	// stataically known to be either true or false. We could look at more cases
5404	// here, but I think all the ones that actually happen are using a
5405	// ConstantExpr.
5406	if (std::optional<bool> BoolValue = getBoolValue(E: IS->getCond())) {
5407	if (*BoolValue)
5408	return visitChildStmt(IS->getThen());
5409	else if (const Stmt *Else = IS->getElse())
5410	return visitChildStmt(Else);
5411	return true;
5412	}
5413
5414	// Otherwise, compile the condition.
5415	if (IS->isNonNegatedConsteval()) {
5416	if (!this->emitIsConstantContext(IS))
5417	return false;
5418	} else if (IS->isNegatedConsteval()) {
5419	if (!this->emitIsConstantContext(IS))
5420	return false;
5421	if (!this->emitInv(IS))
5422	return false;
5423	} else {
5424	if (!this->visitBool(E: IS->getCond()))
5425	return false;
5426	}
5427
5428	if (!this->maybeEmitDeferredVarInit(VD: IS->getConditionVariable()))
5429	return false;
5430
5431	if (const Stmt *Else = IS->getElse()) {
5432	LabelTy LabelElse = this->getLabel();
5433	LabelTy LabelEnd = this->getLabel();
5434	if (!this->jumpFalse(LabelElse))
5435	return false;
5436	if (!visitChildStmt(IS->getThen()))
5437	return false;
5438	if (!this->jump(LabelEnd))
5439	return false;
5440	this->emitLabel(LabelElse);
5441	if (!visitChildStmt(Else))
5442	return false;
5443	this->emitLabel(LabelEnd);
5444	} else {
5445	LabelTy LabelEnd = this->getLabel();
5446	if (!this->jumpFalse(LabelEnd))
5447	return false;
5448	if (!visitChildStmt(IS->getThen()))
5449	return false;
5450	this->emitLabel(LabelEnd);
5451	}
5452
5453	return true;
5454	}
5455
5456	template <class Emitter>
5457	bool Compiler<Emitter>::visitWhileStmt(const WhileStmt *S) {
5458	const Expr *Cond = S->getCond();
5459	const Stmt *Body = S->getBody();
5460
5461	LabelTy CondLabel = this->getLabel(); // Label before the condition.
5462	LabelTy EndLabel = this->getLabel(); // Label after the loop.
5463	LoopScope<Emitter> LS(this, EndLabel, CondLabel);
5464
5465	this->fallthrough(CondLabel);
5466	this->emitLabel(CondLabel);
5467
5468	{
5469	LocalScope<Emitter> CondScope(this);
5470	if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt())
5471	if (!visitDeclStmt(DS: CondDecl))
5472	return false;
5473
5474	if (!this->visitBool(E: Cond))
5475	return false;
5476
5477	if (!this->maybeEmitDeferredVarInit(VD: S->getConditionVariable()))
5478	return false;
5479
5480	if (!this->jumpFalse(EndLabel))
5481	return false;
5482
5483	if (!this->visitStmt(S: Body))
5484	return false;
5485
5486	if (!CondScope.destroyLocals())
5487	return false;
5488	}
5489	if (!this->jump(CondLabel))
5490	return false;
5491	this->fallthrough(EndLabel);
5492	this->emitLabel(EndLabel);
5493
5494	return true;
5495	}
5496
5497	template <class Emitter> bool Compiler<Emitter>::visitDoStmt(const DoStmt *S) {
5498	const Expr *Cond = S->getCond();
5499	const Stmt *Body = S->getBody();
5500
5501	LabelTy StartLabel = this->getLabel();
5502	LabelTy EndLabel = this->getLabel();
5503	LabelTy CondLabel = this->getLabel();
5504	LoopScope<Emitter> LS(this, EndLabel, CondLabel);
5505
5506	this->fallthrough(StartLabel);
5507	this->emitLabel(StartLabel);
5508
5509	{
5510	LocalScope<Emitter> CondScope(this);
5511	if (!this->visitStmt(S: Body))
5512	return false;
5513	this->fallthrough(CondLabel);
5514	this->emitLabel(CondLabel);
5515	if (!this->visitBool(E: Cond))
5516	return false;
5517
5518	if (!CondScope.destroyLocals())
5519	return false;
5520	}
5521	if (!this->jumpTrue(StartLabel))
5522	return false;
5523
5524	this->fallthrough(EndLabel);
5525	this->emitLabel(EndLabel);
5526	return true;
5527	}
5528
5529	template <class Emitter>
5530	bool Compiler<Emitter>::visitForStmt(const ForStmt *S) {
5531	// for (Init; Cond; Inc) { Body }
5532	const Stmt *Init = S->getInit();
5533	const Expr *Cond = S->getCond();
5534	const Expr *Inc = S->getInc();
5535	const Stmt *Body = S->getBody();
5536
5537	LabelTy EndLabel = this->getLabel();
5538	LabelTy CondLabel = this->getLabel();
5539	LabelTy IncLabel = this->getLabel();
5540	LoopScope<Emitter> LS(this, EndLabel, IncLabel);
5541
5542	if (Init && !this->visitStmt(S: Init))
5543	return false;
5544
5545	this->fallthrough(CondLabel);
5546	this->emitLabel(CondLabel);
5547
5548	// Start of loop body.
5549	LocalScope<Emitter> CondScope(this);
5550	if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt())
5551	if (!visitDeclStmt(DS: CondDecl))
5552	return false;
5553
5554	if (Cond) {
5555	if (!this->visitBool(E: Cond))
5556	return false;
5557	if (!this->jumpFalse(EndLabel))
5558	return false;
5559	}
5560	if (!this->maybeEmitDeferredVarInit(VD: S->getConditionVariable()))
5561	return false;
5562
5563	if (Body && !this->visitStmt(S: Body))
5564	return false;
5565
5566	this->fallthrough(IncLabel);
5567	this->emitLabel(IncLabel);
5568	if (Inc && !this->discard(E: Inc))
5569	return false;
5570
5571	if (!CondScope.destroyLocals())
5572	return false;
5573	if (!this->jump(CondLabel))
5574	return false;
5575	// End of loop body.
5576
5577	this->emitLabel(EndLabel);
5578	// If we jumped out of the loop above, we still need to clean up the condition
5579	// scope.
5580	return CondScope.destroyLocals();
5581	}
5582
5583	template <class Emitter>
5584	bool Compiler<Emitter>::visitCXXForRangeStmt(const CXXForRangeStmt *S) {
5585	const Stmt *Init = S->getInit();
5586	const Expr *Cond = S->getCond();
5587	const Expr *Inc = S->getInc();
5588	const Stmt *Body = S->getBody();
5589	const Stmt *BeginStmt = S->getBeginStmt();
5590	const Stmt *RangeStmt = S->getRangeStmt();
5591	const Stmt *EndStmt = S->getEndStmt();
5592
5593	LabelTy EndLabel = this->getLabel();
5594	LabelTy CondLabel = this->getLabel();
5595	LabelTy IncLabel = this->getLabel();
5596	LoopScope<Emitter> LS(this, EndLabel, IncLabel);
5597
5598	// Emit declarations needed in the loop.
5599	if (Init && !this->visitStmt(S: Init))
5600	return false;
5601	if (!this->visitStmt(S: RangeStmt))
5602	return false;
5603	if (!this->visitStmt(S: BeginStmt))
5604	return false;
5605	if (!this->visitStmt(S: EndStmt))
5606	return false;
5607
5608	// Now the condition as well as the loop variable assignment.
5609	this->fallthrough(CondLabel);
5610	this->emitLabel(CondLabel);
5611	if (!this->visitBool(E: Cond))
5612	return false;
5613	if (!this->jumpFalse(EndLabel))
5614	return false;
5615
5616	if (!this->visitDeclStmt(DS: S->getLoopVarStmt(), /*EvaluateConditionDecl=*/true))
5617	return false;
5618
5619	// Body.
5620	{
5621	if (!this->visitStmt(S: Body))
5622	return false;
5623
5624	this->fallthrough(IncLabel);
5625	this->emitLabel(IncLabel);
5626	if (!this->discard(E: Inc))
5627	return false;
5628	}
5629
5630	if (!this->jump(CondLabel))
5631	return false;
5632
5633	this->fallthrough(EndLabel);
5634	this->emitLabel(EndLabel);
5635	return true;
5636	}
5637
5638	template <class Emitter>
5639	bool Compiler<Emitter>::visitBreakStmt(const BreakStmt *S) {
5640	if (!BreakLabel)
5641	return false;
5642
5643	for (VariableScope<Emitter> *C = VarScope; C != BreakVarScope;
5644	C = C->getParent())
5645	C->emitDestruction();
5646	return this->jump(*BreakLabel);
5647	}
5648
5649	template <class Emitter>
5650	bool Compiler<Emitter>::visitContinueStmt(const ContinueStmt *S) {
5651	if (!ContinueLabel)
5652	return false;
5653
5654	for (VariableScope<Emitter> *C = VarScope;
5655	C && C->getParent() != ContinueVarScope; C = C->getParent())
5656	C->emitDestruction();
5657	return this->jump(*ContinueLabel);
5658	}
5659
5660	template <class Emitter>
5661	bool Compiler<Emitter>::visitSwitchStmt(const SwitchStmt *S) {
5662	const Expr *Cond = S->getCond();
5663	if (Cond->containsErrors())
5664	return false;
5665
5666	PrimType CondT = this->classifyPrim(Cond->getType());
5667	LocalScope<Emitter> LS(this);
5668
5669	LabelTy EndLabel = this->getLabel();
5670	OptLabelTy DefaultLabel = std::nullopt;
5671	unsigned CondVar =
5672	this->allocateLocalPrimitive(Src: Cond, Ty: CondT, /*IsConst=*/true);
5673
5674	if (const auto *CondInit = S->getInit())
5675	if (!visitStmt(S: CondInit))
5676	return false;
5677
5678	if (const DeclStmt *CondDecl = S->getConditionVariableDeclStmt())
5679	if (!visitDeclStmt(DS: CondDecl))
5680	return false;
5681
5682	// Initialize condition variable.
5683	if (!this->visit(E: Cond))
5684	return false;
5685	if (!this->emitSetLocal(CondT, CondVar, S))
5686	return false;
5687
5688	if (!this->maybeEmitDeferredVarInit(VD: S->getConditionVariable()))
5689	return false;
5690
5691	CaseMap CaseLabels;
5692	// Create labels and comparison ops for all case statements.
5693	for (const SwitchCase *SC = S->getSwitchCaseList(); SC;
5694	SC = SC->getNextSwitchCase()) {
5695	if (const auto *CS = dyn_cast<CaseStmt>(Val: SC)) {
5696	// FIXME: Implement ranges.
5697	if (CS->caseStmtIsGNURange())
5698	return false;
5699	CaseLabels[SC] = this->getLabel();
5700
5701	const Expr *Value = CS->getLHS();
5702	PrimType ValueT = this->classifyPrim(Value->getType());
5703
5704	// Compare the case statement's value to the switch condition.
5705	if (!this->emitGetLocal(CondT, CondVar, CS))
5706	return false;
5707	if (!this->visit(E: Value))
5708	return false;
5709
5710	// Compare and jump to the case label.
5711	if (!this->emitEQ(ValueT, S))
5712	return false;
5713	if (!this->jumpTrue(CaseLabels[CS]))
5714	return false;
5715	} else {
5716	assert(!DefaultLabel);
5717	DefaultLabel = this->getLabel();
5718	}
5719	}
5720
5721	// If none of the conditions above were true, fall through to the default
5722	// statement or jump after the switch statement.
5723	if (DefaultLabel) {
5724	if (!this->jump(*DefaultLabel))
5725	return false;
5726	} else {
5727	if (!this->jump(EndLabel))
5728	return false;
5729	}
5730
5731	SwitchScope<Emitter> SS(this, std::move(CaseLabels), EndLabel, DefaultLabel);
5732	if (!this->visitStmt(S: S->getBody()))
5733	return false;
5734	this->emitLabel(EndLabel);
5735
5736	return LS.destroyLocals();
5737	}
5738
5739	template <class Emitter>
5740	bool Compiler<Emitter>::visitCaseStmt(const CaseStmt *S) {
5741	this->emitLabel(CaseLabels[S]);
5742	return this->visitStmt(S: S->getSubStmt());
5743	}
5744
5745	template <class Emitter>
5746	bool Compiler<Emitter>::visitDefaultStmt(const DefaultStmt *S) {
5747	this->emitLabel(*DefaultLabel);
5748	return this->visitStmt(S: S->getSubStmt());
5749	}
5750
5751	template <class Emitter>
5752	bool Compiler<Emitter>::visitAttributedStmt(const AttributedStmt *S) {
5753	if (this->Ctx.getLangOpts().CXXAssumptions &&
5754	!this->Ctx.getLangOpts().MSVCCompat) {
5755	for (const Attr *A : S->getAttrs()) {
5756	auto *AA = dyn_cast<CXXAssumeAttr>(Val: A);
5757	if (!AA)
5758	continue;
5759
5760	assert(isa<NullStmt>(S->getSubStmt()));
5761
5762	const Expr *Assumption = AA->getAssumption();
5763	if (Assumption->isValueDependent())
5764	return false;
5765
5766	if (Assumption->HasSideEffects(Ctx: this->Ctx.getASTContext()))
5767	continue;
5768
5769	// Evaluate assumption.
5770	if (!this->visitBool(E: Assumption))
5771	return false;
5772
5773	if (!this->emitAssume(Assumption))
5774	return false;
5775	}
5776	}
5777
5778	// Ignore other attributes.
5779	return this->visitStmt(S: S->getSubStmt());
5780	}
5781
5782	template <class Emitter>
5783	bool Compiler<Emitter>::visitCXXTryStmt(const CXXTryStmt *S) {
5784	// Ignore all handlers.
5785	return this->visitStmt(S: S->getTryBlock());
5786	}
5787
5788	template <class Emitter>
5789	bool Compiler<Emitter>::emitLambdaStaticInvokerBody(const CXXMethodDecl *MD) {
5790	assert(MD->isLambdaStaticInvoker());
5791	assert(MD->hasBody());
5792	assert(cast<CompoundStmt>(MD->getBody())->body_empty());
5793
5794	const CXXRecordDecl *ClosureClass = MD->getParent();
5795	const CXXMethodDecl *LambdaCallOp = ClosureClass->getLambdaCallOperator();
5796	assert(ClosureClass->captures_begin() == ClosureClass->captures_end());
5797	const Function *Func = this->getFunction(FD: LambdaCallOp);
5798	if (!Func)
5799	return false;
5800	assert(Func->hasThisPointer());
5801	assert(Func->getNumParams() == (MD->getNumParams() + 1 + Func->hasRVO()));
5802
5803	if (Func->hasRVO()) {
5804	if (!this->emitRVOPtr(MD))
5805	return false;
5806	}
5807
5808	// The lambda call operator needs an instance pointer, but we don't have
5809	// one here, and we don't need one either because the lambda cannot have
5810	// any captures, as verified above. Emit a null pointer. This is then
5811	// special-cased when interpreting to not emit any misleading diagnostics.
5812	if (!this->emitNullPtr(0, nullptr, MD))
5813	return false;
5814
5815	// Forward all arguments from the static invoker to the lambda call operator.
5816	for (const ParmVarDecl *PVD : MD->parameters()) {
5817	auto It = this->Params.find(PVD);
5818	assert(It != this->Params.end());
5819
5820	// We do the lvalue-to-rvalue conversion manually here, so no need
5821	// to care about references.
5822	PrimType ParamType = this->classify(PVD->getType()).value_or(PT_Ptr);
5823	if (!this->emitGetParam(ParamType, It->second.Offset, MD))
5824	return false;
5825	}
5826
5827	if (!this->emitCall(Func, 0, LambdaCallOp))
5828	return false;
5829
5830	this->emitCleanup();
5831	if (ReturnType)
5832	return this->emitRet(*ReturnType, MD);
5833
5834	// Nothing to do, since we emitted the RVO pointer above.
5835	return this->emitRetVoid(MD);
5836	}
5837
5838	template <class Emitter>
5839	bool Compiler<Emitter>::checkLiteralType(const Expr *E) {
5840	if (Ctx.getLangOpts().CPlusPlus23)
5841	return true;
5842
5843	if (!E->isPRValue() || E->getType()->isLiteralType(Ctx: Ctx.getASTContext()))
5844	return true;
5845
5846	return this->emitCheckLiteralType(E->getType().getTypePtr(), E);
5847	}
5848
5849	template <class Emitter>
5850	bool Compiler<Emitter>::compileConstructor(const CXXConstructorDecl *Ctor) {
5851	assert(!ReturnType);
5852
5853	auto emitFieldInitializer = [&](const Record::Field *F, unsigned FieldOffset,
5854	const Expr *InitExpr) -> bool {
5855	// We don't know what to do with these, so just return false.
5856	if (InitExpr->getType().isNull())
5857	return false;
5858
5859	if (std::optional<PrimType> T = this->classify(InitExpr)) {
5860	if (!this->visit(E: InitExpr))
5861	return false;
5862
5863	if (F->isBitField())
5864	return this->emitInitThisBitField(*T, F, FieldOffset, InitExpr);
5865	return this->emitInitThisField(*T, FieldOffset, InitExpr);
5866	}
5867	// Non-primitive case. Get a pointer to the field-to-initialize
5868	// on the stack and call visitInitialzer() for it.
5869	InitLinkScope<Emitter> FieldScope(this, InitLink::Field(Offset: F->Offset));
5870	if (!this->emitGetPtrThisField(FieldOffset, InitExpr))
5871	return false;
5872
5873	if (!this->visitInitializer(E: InitExpr))
5874	return false;
5875
5876	return this->emitFinishInitPop(InitExpr);
5877	};
5878
5879	const RecordDecl *RD = Ctor->getParent();
5880	const Record *R = this->getRecord(RD);
5881	if (!R)
5882	return false;
5883
5884	if (R->isUnion() && Ctor->isCopyOrMoveConstructor()) {
5885	if (R->getNumFields() == 0)
5886	return this->emitRetVoid(Ctor);
5887	// union copy and move ctors are special.
5888	assert(cast<CompoundStmt>(Ctor->getBody())->body_empty());
5889	if (!this->emitThis(Ctor))
5890	return false;
5891
5892	auto PVD = Ctor->getParamDecl(i: 0);
5893	ParamOffset PO = this->Params[PVD]; // Must exist.
5894
5895	if (!this->emitGetParam(PT_Ptr, PO.Offset, Ctor))
5896	return false;
5897
5898	return this->emitMemcpy(Ctor) && this->emitPopPtr(Ctor) &&
5899	this->emitRetVoid(Ctor);
5900	}
5901
5902	InitLinkScope<Emitter> InitScope(this, InitLink::This());
5903	for (const auto *Init : Ctor->inits()) {
5904	// Scope needed for the initializers.
5905	BlockScope<Emitter> Scope(this);
5906
5907	const Expr *InitExpr = Init->getInit();
5908	if (const FieldDecl *Member = Init->getMember()) {
5909	const Record::Field *F = R->getField(FD: Member);
5910
5911	if (!emitFieldInitializer(F, F->Offset, InitExpr))
5912	return false;
5913	} else if (const Type *Base = Init->getBaseClass()) {
5914	const auto *BaseDecl = Base->getAsCXXRecordDecl();
5915	assert(BaseDecl);
5916
5917	if (Init->isBaseVirtual()) {
5918	assert(R->getVirtualBase(BaseDecl));
5919	if (!this->emitGetPtrThisVirtBase(BaseDecl, InitExpr))
5920	return false;
5921
5922	} else {
5923	// Base class initializer.
5924	// Get This Base and call initializer on it.
5925	const Record::Base *B = R->getBase(FD: BaseDecl);
5926	assert(B);
5927	if (!this->emitGetPtrThisBase(B->Offset, InitExpr))
5928	return false;
5929	}
5930
5931	if (!this->visitInitializer(E: InitExpr))
5932	return false;
5933	if (!this->emitFinishInitPop(InitExpr))
5934	return false;
5935	} else if (const IndirectFieldDecl *IFD = Init->getIndirectMember()) {
5936	assert(IFD->getChainingSize() >= 2);
5937
5938	unsigned NestedFieldOffset = 0;
5939	const Record::Field *NestedField = nullptr;
5940	for (const NamedDecl *ND : IFD->chain()) {
5941	const auto *FD = cast<FieldDecl>(Val: ND);
5942	const Record *FieldRecord = this->P.getOrCreateRecord(RD: FD->getParent());
5943	assert(FieldRecord);
5944
5945	NestedField = FieldRecord->getField(FD);
5946	assert(NestedField);
5947
5948	NestedFieldOffset += NestedField->Offset;
5949	}
5950	assert(NestedField);
5951
5952	if (!emitFieldInitializer(NestedField, NestedFieldOffset, InitExpr))
5953	return false;
5954
5955	// Mark all chain links as initialized.
5956	unsigned InitFieldOffset = 0;
5957	for (const NamedDecl *ND : IFD->chain().drop_back()) {
5958	const auto *FD = cast<FieldDecl>(Val: ND);
5959	const Record *FieldRecord = this->P.getOrCreateRecord(RD: FD->getParent());
5960	assert(FieldRecord);
5961	NestedField = FieldRecord->getField(FD);
5962	InitFieldOffset += NestedField->Offset;
5963	assert(NestedField);
5964	if (!this->emitGetPtrThisField(InitFieldOffset, InitExpr))
5965	return false;
5966	if (!this->emitFinishInitPop(InitExpr))
5967	return false;
5968	}
5969
5970	} else {
5971	assert(Init->isDelegatingInitializer());
5972	if (!this->emitThis(InitExpr))
5973	return false;
5974	if (!this->visitInitializer(E: Init->getInit()))
5975	return false;
5976	if (!this->emitPopPtr(InitExpr))
5977	return false;
5978	}
5979
5980	if (!Scope.destroyLocals())
5981	return false;
5982	}
5983
5984	if (const auto *Body = Ctor->getBody())
5985	if (!visitStmt(S: Body))
5986	return false;
5987
5988	return this->emitRetVoid(SourceInfo{});
5989	}
5990
5991	template <class Emitter>
5992	bool Compiler<Emitter>::compileDestructor(const CXXDestructorDecl *Dtor) {
5993	const RecordDecl *RD = Dtor->getParent();
5994	const Record *R = this->getRecord(RD);
5995	if (!R)
5996	return false;
5997
5998	if (!Dtor->isTrivial() && Dtor->getBody()) {
5999	if (!this->visitStmt(S: Dtor->getBody()))
6000	return false;
6001	}
6002
6003	if (!this->emitThis(Dtor))
6004	return false;
6005
6006	if (!this->emitCheckDestruction(Dtor))
6007	return false;
6008
6009	assert(R);
6010	if (!R->isUnion()) {
6011	// First, destroy all fields.
6012	for (const Record::Field &Field : llvm::reverse(C: R->fields())) {
6013	const Descriptor *D = Field.Desc;
6014	if (!D->isPrimitive() && !D->isPrimitiveArray()) {
6015	if (!this->emitGetPtrField(Field.Offset, SourceInfo{}))
6016	return false;
6017	if (!this->emitDestruction(Desc: D, Loc: SourceInfo{}))
6018	return false;
6019	if (!this->emitPopPtr(SourceInfo{}))
6020	return false;
6021	}
6022	}
6023	}
6024
6025	for (const Record::Base &Base : llvm::reverse(C: R->bases())) {
6026	if (Base.R->isAnonymousUnion())
6027	continue;
6028
6029	if (!this->emitGetPtrBase(Base.Offset, SourceInfo{}))
6030	return false;
6031	if (!this->emitRecordDestruction(R: Base.R, Loc: {}))
6032	return false;
6033	if (!this->emitPopPtr(SourceInfo{}))
6034	return false;
6035	}
6036
6037	// FIXME: Virtual bases.
6038	return this->emitPopPtr(Dtor) && this->emitRetVoid(Dtor);
6039	}
6040
6041	template <class Emitter>
6042	bool Compiler<Emitter>::compileUnionAssignmentOperator(
6043	const CXXMethodDecl *MD) {
6044	if (!this->emitThis(MD))
6045	return false;
6046
6047	auto PVD = MD->getParamDecl(i: 0);
6048	ParamOffset PO = this->Params[PVD]; // Must exist.
6049
6050	if (!this->emitGetParam(PT_Ptr, PO.Offset, MD))
6051	return false;
6052
6053	return this->emitMemcpy(MD) && this->emitRet(PT_Ptr, MD);
6054	}
6055
6056	template <class Emitter>
6057	bool Compiler<Emitter>::visitFunc(const FunctionDecl *F) {
6058	// Classify the return type.
6059	ReturnType = this->classify(F->getReturnType());
6060
6061	this->CompilingFunction = F;
6062
6063	if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(Val: F))
6064	return this->compileConstructor(Ctor);
6065	if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(Val: F))
6066	return this->compileDestructor(Dtor);
6067
6068	// Emit custom code if this is a lambda static invoker.
6069	if (const auto *MD = dyn_cast<CXXMethodDecl>(Val: F)) {
6070	const RecordDecl *RD = MD->getParent();
6071
6072	if (RD->isUnion() &&
6073	(MD->isCopyAssignmentOperator() || MD->isMoveAssignmentOperator()))
6074	return this->compileUnionAssignmentOperator(MD);
6075
6076	if (MD->isLambdaStaticInvoker())
6077	return this->emitLambdaStaticInvokerBody(MD);
6078	}
6079
6080	// Regular functions.
6081	if (const auto *Body = F->getBody())
6082	if (!visitStmt(S: Body))
6083	return false;
6084
6085	// Emit a guard return to protect against a code path missing one.
6086	if (F->getReturnType()->isVoidType())
6087	return this->emitRetVoid(SourceInfo{});
6088	return this->emitNoRet(SourceInfo{});
6089	}
6090
6091	template <class Emitter>
6092	bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
6093	const Expr *SubExpr = E->getSubExpr();
6094	if (SubExpr->getType()->isAnyComplexType())
6095	return this->VisitComplexUnaryOperator(E);
6096	if (SubExpr->getType()->isVectorType())
6097	return this->VisitVectorUnaryOperator(E);
6098	if (SubExpr->getType()->isFixedPointType())
6099	return this->VisitFixedPointUnaryOperator(E);
6100	std::optional<PrimType> T = classify(SubExpr->getType());
6101
6102	switch (E->getOpcode()) {
6103	case UO_PostInc: { // x++
6104	if (!Ctx.getLangOpts().CPlusPlus14)
6105	return this->emitInvalid(E);
6106	if (!T)
6107	return this->emitError(E);
6108
6109	if (!this->visit(E: SubExpr))
6110	return false;
6111
6112	if (T == PT_Ptr) {
6113	if (!this->emitIncPtr(E))
6114	return false;
6115
6116	return DiscardResult ? this->emitPopPtr(E) : true;
6117	}
6118
6119	if (T == PT_Float) {
6120	return DiscardResult ? this->emitIncfPop(getFPOptions(E), E)
6121	: this->emitIncf(getFPOptions(E), E);
6122	}
6123
6124	return DiscardResult ? this->emitIncPop(*T, E->canOverflow(), E)
6125	: this->emitInc(*T, E->canOverflow(), E);
6126	}
6127	case UO_PostDec: { // x--
6128	if (!Ctx.getLangOpts().CPlusPlus14)
6129	return this->emitInvalid(E);
6130	if (!T)
6131	return this->emitError(E);
6132
6133	if (!this->visit(E: SubExpr))
6134	return false;
6135
6136	if (T == PT_Ptr) {
6137	if (!this->emitDecPtr(E))
6138	return false;
6139
6140	return DiscardResult ? this->emitPopPtr(E) : true;
6141	}
6142
6143	if (T == PT_Float) {
6144	return DiscardResult ? this->emitDecfPop(getFPOptions(E), E)
6145	: this->emitDecf(getFPOptions(E), E);
6146	}
6147
6148	return DiscardResult ? this->emitDecPop(*T, E->canOverflow(), E)
6149	: this->emitDec(*T, E->canOverflow(), E);
6150	}
6151	case UO_PreInc: { // ++x
6152	if (!Ctx.getLangOpts().CPlusPlus14)
6153	return this->emitInvalid(E);
6154	if (!T)
6155	return this->emitError(E);
6156
6157	if (!this->visit(E: SubExpr))
6158	return false;
6159
6160	if (T == PT_Ptr) {
6161	if (!this->emitLoadPtr(E))
6162	return false;
6163	if (!this->emitConstUint8(1, E))
6164	return false;
6165	if (!this->emitAddOffsetUint8(E))
6166	return false;
6167	return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
6168	}
6169
6170	// Post-inc and pre-inc are the same if the value is to be discarded.
6171	if (DiscardResult) {
6172	if (T == PT_Float)
6173	return this->emitIncfPop(getFPOptions(E), E);
6174	return this->emitIncPop(*T, E->canOverflow(), E);
6175	}
6176
6177	if (T == PT_Float) {
6178	const auto &TargetSemantics = Ctx.getFloatSemantics(T: E->getType());
6179	if (!this->emitLoadFloat(E))
6180	return false;
6181	APFloat F(TargetSemantics, 1);
6182	if (!this->emitFloat(F, E))
6183	return false;
6184
6185	if (!this->emitAddf(getFPOptions(E), E))
6186	return false;
6187	if (!this->emitStoreFloat(E))
6188	return false;
6189	} else {
6190	assert(isIntegralType(*T));
6191	if (!this->emitPreInc(*T, E->canOverflow(), E))
6192	return false;
6193	}
6194	return E->isGLValue() || this->emitLoadPop(*T, E);
6195	}
6196	case UO_PreDec: { // --x
6197	if (!Ctx.getLangOpts().CPlusPlus14)
6198	return this->emitInvalid(E);
6199	if (!T)
6200	return this->emitError(E);
6201
6202	if (!this->visit(E: SubExpr))
6203	return false;
6204
6205	if (T == PT_Ptr) {
6206	if (!this->emitLoadPtr(E))
6207	return false;
6208	if (!this->emitConstUint8(1, E))
6209	return false;
6210	if (!this->emitSubOffsetUint8(E))
6211	return false;
6212	return DiscardResult ? this->emitStorePopPtr(E) : this->emitStorePtr(E);
6213	}
6214
6215	// Post-dec and pre-dec are the same if the value is to be discarded.
6216	if (DiscardResult) {
6217	if (T == PT_Float)
6218	return this->emitDecfPop(getFPOptions(E), E);
6219	return this->emitDecPop(*T, E->canOverflow(), E);
6220	}
6221
6222	if (T == PT_Float) {
6223	const auto &TargetSemantics = Ctx.getFloatSemantics(T: E->getType());
6224	if (!this->emitLoadFloat(E))
6225	return false;
6226	APFloat F(TargetSemantics, 1);
6227	if (!this->emitFloat(F, E))
6228	return false;
6229
6230	if (!this->emitSubf(getFPOptions(E), E))
6231	return false;
6232	if (!this->emitStoreFloat(E))
6233	return false;
6234	} else {
6235	assert(isIntegralType(*T));
6236	if (!this->emitPreDec(*T, E->canOverflow(), E))
6237	return false;
6238	}
6239	return E->isGLValue() || this->emitLoadPop(*T, E);
6240	}
6241	case UO_LNot: // !x
6242	if (!T)
6243	return this->emitError(E);
6244
6245	if (DiscardResult)
6246	return this->discard(E: SubExpr);
6247
6248	if (!this->visitBool(E: SubExpr))
6249	return false;
6250
6251	if (!this->emitInv(E))
6252	return false;
6253
6254	if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
6255	return this->emitCast(PT_Bool, ET, E);
6256	return true;
6257	case UO_Minus: // -x
6258	if (!T)
6259	return this->emitError(E);
6260
6261	if (!this->visit(E: SubExpr))
6262	return false;
6263	return DiscardResult ? this->emitPop(*T, E) : this->emitNeg(*T, E);
6264	case UO_Plus: // +x
6265	if (!T)
6266	return this->emitError(E);
6267
6268	if (!this->visit(E: SubExpr)) // noop
6269	return false;
6270	return DiscardResult ? this->emitPop(*T, E) : true;
6271	case UO_AddrOf: // &x
6272	if (E->getType()->isMemberPointerType()) {
6273	// C++11 [expr.unary.op]p3 has very strict rules on how the address of a
6274	// member can be formed.
6275	return this->emitGetMemberPtr(cast<DeclRefExpr>(Val: SubExpr)->getDecl(), E);
6276	}
6277	// We should already have a pointer when we get here.
6278	return this->delegate(E: SubExpr);
6279	case UO_Deref: // *x
6280	if (DiscardResult)
6281	return this->discard(E: SubExpr);
6282
6283	if (!this->visit(E: SubExpr))
6284	return false;
6285
6286	if (classifyPrim(SubExpr) == PT_Ptr)
6287	return this->emitNarrowPtr(E);
6288	return true;
6289
6290	case UO_Not: // ~x
6291	if (!T)
6292	return this->emitError(E);
6293
6294	if (!this->visit(E: SubExpr))
6295	return false;
6296	return DiscardResult ? this->emitPop(*T, E) : this->emitComp(*T, E);
6297	case UO_Real: // __real x
6298	assert(T);
6299	return this->delegate(E: SubExpr);
6300	case UO_Imag: { // __imag x
6301	assert(T);
6302	if (!this->discard(E: SubExpr))
6303	return false;
6304	return this->visitZeroInitializer(T: *T, QT: SubExpr->getType(), E: SubExpr);
6305	}
6306	case UO_Extension:
6307	return this->delegate(E: SubExpr);
6308	case UO_Coawait:
6309	assert(false && "Unhandled opcode");
6310	}
6311
6312	return false;
6313	}
6314
6315	template <class Emitter>
6316	bool Compiler<Emitter>::VisitComplexUnaryOperator(const UnaryOperator *E) {
6317	const Expr *SubExpr = E->getSubExpr();
6318	assert(SubExpr->getType()->isAnyComplexType());
6319
6320	if (DiscardResult)
6321	return this->discard(E: SubExpr);
6322
6323	std::optional<PrimType> ResT = classify(E);
6324	auto prepareResult = [=]() -> bool {
6325	if (!ResT && !Initializing) {
6326	std::optional<unsigned> LocalIndex = allocateLocal(Src: SubExpr);
6327	if (!LocalIndex)
6328	return false;
6329	return this->emitGetPtrLocal(*LocalIndex, E);
6330	}
6331
6332	return true;
6333	};
6334
6335	// The offset of the temporary, if we created one.
6336	unsigned SubExprOffset = ~0u;
6337	auto createTemp = [=, &SubExprOffset]() -> bool {
6338	SubExprOffset =
6339	this->allocateLocalPrimitive(Src: SubExpr, Ty: PT_Ptr, /*IsConst=*/true);
6340	if (!this->visit(E: SubExpr))
6341	return false;
6342	return this->emitSetLocal(PT_Ptr, SubExprOffset, E);
6343	};
6344
6345	PrimType ElemT = classifyComplexElementType(T: SubExpr->getType());
6346	auto getElem = [=](unsigned Offset, unsigned Index) -> bool {
6347	if (!this->emitGetLocal(PT_Ptr, Offset, E))
6348	return false;
6349	return this->emitArrayElemPop(ElemT, Index, E);
6350	};
6351
6352	switch (E->getOpcode()) {
6353	case UO_Minus:
6354	if (!prepareResult())
6355	return false;
6356	if (!createTemp())
6357	return false;
6358	for (unsigned I = 0; I != 2; ++I) {
6359	if (!getElem(SubExprOffset, I))
6360	return false;
6361	if (!this->emitNeg(ElemT, E))
6362	return false;
6363	if (!this->emitInitElem(ElemT, I, E))
6364	return false;
6365	}
6366	break;
6367
6368	case UO_Plus: // +x
6369	case UO_AddrOf: // &x
6370	case UO_Deref: // *x
6371	return this->delegate(E: SubExpr);
6372
6373	case UO_LNot:
6374	if (!this->visit(E: SubExpr))
6375	return false;
6376	if (!this->emitComplexBoolCast(E: SubExpr))
6377	return false;
6378	if (!this->emitInv(E))
6379	return false;
6380	if (PrimType ET = classifyPrim(E->getType()); ET != PT_Bool)
6381	return this->emitCast(PT_Bool, ET, E);
6382	return true;
6383
6384	case UO_Real:
6385	return this->emitComplexReal(SubExpr);
6386
6387	case UO_Imag:
6388	if (!this->visit(E: SubExpr))
6389	return false;
6390
6391	if (SubExpr->isLValue()) {
6392	if (!this->emitConstUint8(1, E))
6393	return false;
6394	return this->emitArrayElemPtrPopUint8(E);
6395	}
6396
6397	// Since our _Complex implementation does not map to a primitive type,
6398	// we sometimes have to do the lvalue-to-rvalue conversion here manually.
6399	return this->emitArrayElemPop(classifyPrim(E->getType()), 1, E);
6400
6401	case UO_Not: // ~x
6402	if (!this->visit(E: SubExpr))
6403	return false;
6404	// Negate the imaginary component.
6405	if (!this->emitArrayElem(ElemT, 1, E))
6406	return false;
6407	if (!this->emitNeg(ElemT, E))
6408	return false;
6409	if (!this->emitInitElem(ElemT, 1, E))
6410	return false;
6411	return DiscardResult ? this->emitPopPtr(E) : true;
6412
6413	case UO_Extension:
6414	return this->delegate(E: SubExpr);
6415
6416	default:
6417	return this->emitInvalid(E);
6418	}
6419
6420	return true;
6421	}
6422
6423	template <class Emitter>
6424	bool Compiler<Emitter>::VisitVectorUnaryOperator(const UnaryOperator *E) {
6425	const Expr *SubExpr = E->getSubExpr();
6426	assert(SubExpr->getType()->isVectorType());
6427
6428	if (DiscardResult)
6429	return this->discard(E: SubExpr);
6430
6431	auto UnaryOp = E->getOpcode();
6432	if (UnaryOp == UO_Extension)
6433	return this->delegate(E: SubExpr);
6434
6435	if (UnaryOp != UO_Plus && UnaryOp != UO_Minus && UnaryOp != UO_LNot &&
6436	UnaryOp != UO_Not && UnaryOp != UO_AddrOf)
6437	return this->emitInvalid(E);
6438
6439	// Nothing to do here.
6440	if (UnaryOp == UO_Plus || UnaryOp == UO_AddrOf)
6441	return this->delegate(E: SubExpr);
6442
6443	if (!Initializing) {
6444	std::optional<unsigned> LocalIndex = allocateLocal(Src: SubExpr);
6445	if (!LocalIndex)
6446	return false;
6447	if (!this->emitGetPtrLocal(*LocalIndex, E))
6448	return false;
6449	}
6450
6451	// The offset of the temporary, if we created one.
6452	unsigned SubExprOffset =
6453	this->allocateLocalPrimitive(Src: SubExpr, Ty: PT_Ptr, /*IsConst=*/true);
6454	if (!this->visit(E: SubExpr))
6455	return false;
6456	if (!this->emitSetLocal(PT_Ptr, SubExprOffset, E))
6457	return false;
6458
6459	const auto *VecTy = SubExpr->getType()->getAs<VectorType>();
6460	PrimType ElemT = classifyVectorElementType(T: SubExpr->getType());
6461	auto getElem = [=](unsigned Offset, unsigned Index) -> bool {
6462	if (!this->emitGetLocal(PT_Ptr, Offset, E))
6463	return false;
6464	return this->emitArrayElemPop(ElemT, Index, E);
6465	};
6466
6467	switch (UnaryOp) {
6468	case UO_Minus:
6469	for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
6470	if (!getElem(SubExprOffset, I))
6471	return false;
6472	if (!this->emitNeg(ElemT, E))
6473	return false;
6474	if (!this->emitInitElem(ElemT, I, E))
6475	return false;
6476	}
6477	break;
6478	case UO_LNot: { // !x
6479	// In C++, the logic operators !, &&, || are available for vectors. !v is
6480	// equivalent to v == 0.
6481	//
6482	// The result of the comparison is a vector of the same width and number of
6483	// elements as the comparison operands with a signed integral element type.
6484	//
6485	// https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html
6486	QualType ResultVecTy = E->getType();
6487	PrimType ResultVecElemT =
6488	classifyPrim(ResultVecTy->getAs<VectorType>()->getElementType());
6489	for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
6490	if (!getElem(SubExprOffset, I))
6491	return false;
6492	// operator ! on vectors returns -1 for 'truth', so negate it.
6493	if (!this->emitPrimCast(FromT: ElemT, ToT: PT_Bool, ToQT: Ctx.getASTContext().BoolTy, E))
6494	return false;
6495	if (!this->emitInv(E))
6496	return false;
6497	if (!this->emitPrimCast(FromT: PT_Bool, ToT: ElemT, ToQT: VecTy->getElementType(), E))
6498	return false;
6499	if (!this->emitNeg(ElemT, E))
6500	return false;
6501	if (ElemT != ResultVecElemT &&
6502	!this->emitPrimCast(FromT: ElemT, ToT: ResultVecElemT, ToQT: ResultVecTy, E))
6503	return false;
6504	if (!this->emitInitElem(ResultVecElemT, I, E))
6505	return false;
6506	}
6507	break;
6508	}
6509	case UO_Not: // ~x
6510	for (unsigned I = 0; I != VecTy->getNumElements(); ++I) {
6511	if (!getElem(SubExprOffset, I))
6512	return false;
6513	if (ElemT == PT_Bool) {
6514	if (!this->emitInv(E))
6515	return false;
6516	} else {
6517	if (!this->emitComp(ElemT, E))
6518	return false;
6519	}
6520	if (!this->emitInitElem(ElemT, I, E))
6521	return false;
6522	}
6523	break;
6524	default:
6525	llvm_unreachable("Unsupported unary operators should be handled up front");
6526	}
6527	return true;
6528	}
6529
6530	template <class Emitter>
6531	bool Compiler<Emitter>::visitDeclRef(const ValueDecl *D, const Expr *E) {
6532	if (DiscardResult)
6533	return true;
6534
6535	if (const auto *ECD = dyn_cast<EnumConstantDecl>(Val: D))
6536	return this->emitConst(ECD->getInitVal(), E);
6537	if (const auto *FuncDecl = dyn_cast<FunctionDecl>(Val: D)) {
6538	const Function *F = getFunction(FD: FuncDecl);
6539	return F && this->emitGetFnPtr(F, E);
6540	}
6541	if (const auto *TPOD = dyn_cast<TemplateParamObjectDecl>(Val: D)) {
6542	if (std::optional<unsigned> Index = P.getOrCreateGlobal(VD: D)) {
6543	if (!this->emitGetPtrGlobal(*Index, E))
6544	return false;
6545	if (std::optional<PrimType> T = classify(E->getType())) {
6546	if (!this->visitAPValue(Val: TPOD->getValue(), ValType: *T, E))
6547	return false;
6548	return this->emitInitGlobal(*T, *Index, E);
6549	}
6550	return this->visitAPValueInitializer(Val: TPOD->getValue(), E,
6551	T: TPOD->getType());
6552	}
6553	return false;
6554	}
6555
6556	// References are implemented via pointers, so when we see a DeclRefExpr
6557	// pointing to a reference, we need to get its value directly (i.e. the
6558	// pointer to the actual value) instead of a pointer to the pointer to the
6559	// value.
6560	bool IsReference = D->getType()->isReferenceType();
6561
6562	// Local variables.
6563	if (auto It = Locals.find(Val: D); It != Locals.end()) {
6564	const unsigned Offset = It->second.Offset;
6565	if (IsReference)
6566	return this->emitGetLocal(classifyPrim(E), Offset, E);
6567	return this->emitGetPtrLocal(Offset, E);
6568	}
6569	// Global variables.
6570	if (auto GlobalIndex = P.getGlobal(VD: D)) {
6571	if (IsReference) {
6572	if (!Ctx.getLangOpts().CPlusPlus11)
6573	return this->emitGetGlobal(classifyPrim(E), *GlobalIndex, E);
6574	return this->emitGetGlobalUnchecked(classifyPrim(E), *GlobalIndex, E);
6575	}
6576
6577	return this->emitGetPtrGlobal(*GlobalIndex, E);
6578	}
6579	// Function parameters.
6580	if (const auto *PVD = dyn_cast<ParmVarDecl>(Val: D)) {
6581	if (auto It = this->Params.find(PVD); It != this->Params.end()) {
6582	if (IsReference || !It->second.IsPtr)
6583	return this->emitGetParam(classifyPrim(E), It->second.Offset, E);
6584
6585	return this->emitGetPtrParam(It->second.Offset, E);
6586	}
6587	}
6588
6589	// In case we need to re-visit a declaration.
6590	auto revisit = [&](const VarDecl *VD) -> bool {
6591	if (!this->emitPushCC(VD->hasConstantInitialization(), E))
6592	return false;
6593	auto VarState = this->visitDecl(VD, /*IsConstexprUnknown=*/true);
6594
6595	if (!this->emitPopCC(E))
6596	return false;
6597
6598	if (VarState.notCreated())
6599	return true;
6600	if (!VarState)
6601	return false;
6602	// Retry.
6603	return this->visitDeclRef(D, E);
6604	};
6605
6606	// Lambda captures.
6607	if (auto It = this->LambdaCaptures.find(D);
6608	It != this->LambdaCaptures.end()) {
6609	auto [Offset, IsPtr] = It->second;
6610
6611	if (IsPtr)
6612	return this->emitGetThisFieldPtr(Offset, E);
6613	return this->emitGetPtrThisField(Offset, E);
6614	}
6615
6616	if (const auto *DRE = dyn_cast<DeclRefExpr>(Val: E);
6617	DRE && DRE->refersToEnclosingVariableOrCapture()) {
6618	if (const auto *VD = dyn_cast<VarDecl>(Val: D); VD && VD->isInitCapture())
6619	return revisit(VD);
6620	}
6621
6622	if (const auto *BD = dyn_cast<BindingDecl>(Val: D))
6623	return this->visit(E: BD->getBinding());
6624
6625	// Avoid infinite recursion.
6626	if (D == InitializingDecl)
6627	return this->emitDummyPtr(D, E);
6628
6629	// Try to lazily visit (or emit dummy pointers for) declarations
6630	// we haven't seen yet.
6631	// For C.
6632	if (!Ctx.getLangOpts().CPlusPlus) {
6633	if (const auto *VD = dyn_cast<VarDecl>(Val: D);
6634	VD && VD->getAnyInitializer() &&
6635	VD->getType().isConstant(Ctx: Ctx.getASTContext()) && !VD->isWeak())
6636	return revisit(VD);
6637	return this->emitDummyPtr(D, E);
6638	}
6639
6640	// ... and C++.
6641	const auto *VD = dyn_cast<VarDecl>(Val: D);
6642	if (!VD)
6643	return this->emitDummyPtr(D, E);
6644
6645	const auto typeShouldBeVisited = [&](QualType T) -> bool {
6646	if (T.isConstant(Ctx: Ctx.getASTContext()))
6647	return true;
6648	return T->isReferenceType();
6649	};
6650
6651	if ((VD->hasGlobalStorage() || VD->isStaticDataMember()) &&
6652	typeShouldBeVisited(VD->getType())) {
6653	if (const Expr *Init = VD->getAnyInitializer();
6654	Init && !Init->isValueDependent()) {
6655	// Whether or not the evaluation is successul doesn't really matter
6656	// here -- we will create a global variable in any case, and that
6657	// will have the state of initializer evaluation attached.
6658	APValue V;
6659	SmallVector<PartialDiagnosticAt> Notes;
6660	(void)Init->EvaluateAsInitializer(Result&: V, Ctx: Ctx.getASTContext(), VD, Notes,
6661	IsConstantInitializer: true);
6662	return this->visitDeclRef(D, E);
6663	}
6664	return revisit(VD);
6665	}
6666
6667	// FIXME: The evaluateValue() check here is a little ridiculous, since
6668	// it will ultimately call into Context::evaluateAsInitializer(). In
6669	// other words, we're evaluating the initializer, just to know if we can
6670	// evaluate the initializer.
6671	if (VD->isLocalVarDecl() && typeShouldBeVisited(VD->getType()) &&
6672	VD->getInit() && !VD->getInit()->isValueDependent()) {
6673
6674	if (VD->evaluateValue())
6675	return revisit(VD);
6676
6677	if (!IsReference)
6678	return this->emitDummyPtr(D, E);
6679
6680	return this->emitInvalidDeclRef(cast<DeclRefExpr>(Val: E),
6681	/*InitializerFailed=*/true, E);
6682	}
6683
6684	return this->emitDummyPtr(D, E);
6685	}
6686
6687	template <class Emitter>
6688	bool Compiler<Emitter>::VisitDeclRefExpr(const DeclRefExpr *E) {
6689	const auto *D = E->getDecl();
6690	return this->visitDeclRef(D, E);
6691	}
6692
6693	template <class Emitter> void Compiler<Emitter>::emitCleanup() {
6694	for (VariableScope<Emitter> *C = VarScope; C; C = C->getParent())
6695	C->emitDestruction();
6696	}
6697
6698	template <class Emitter>
6699	unsigned Compiler<Emitter>::collectBaseOffset(const QualType BaseType,
6700	const QualType DerivedType) {
6701	const auto extractRecordDecl = [](QualType Ty) -> const CXXRecordDecl * {
6702	if (const auto *R = Ty->getPointeeCXXRecordDecl())
6703	return R;
6704	return Ty->getAsCXXRecordDecl();
6705	};
6706	const CXXRecordDecl *BaseDecl = extractRecordDecl(BaseType);
6707	const CXXRecordDecl *DerivedDecl = extractRecordDecl(DerivedType);
6708
6709	return Ctx.collectBaseOffset(BaseDecl, DerivedDecl);
6710	}
6711
6712	/// Emit casts from a PrimType to another PrimType.
6713	template <class Emitter>
6714	bool Compiler<Emitter>::emitPrimCast(PrimType FromT, PrimType ToT,
6715	QualType ToQT, const Expr *E) {
6716
6717	if (FromT == PT_Float) {
6718	// Floating to floating.
6719	if (ToT == PT_Float) {
6720	const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(T: ToQT);
6721	return this->emitCastFP(ToSem, getRoundingMode(E), E);
6722	}
6723
6724	if (ToT == PT_IntAP)
6725	return this->emitCastFloatingIntegralAP(Ctx.getBitWidth(T: ToQT),
6726	getFPOptions(E), E);
6727	if (ToT == PT_IntAPS)
6728	return this->emitCastFloatingIntegralAPS(Ctx.getBitWidth(T: ToQT),
6729	getFPOptions(E), E);
6730
6731	// Float to integral.
6732	if (isIntegralType(T: ToT) || ToT == PT_Bool)
6733	return this->emitCastFloatingIntegral(ToT, getFPOptions(E), E);
6734	}
6735
6736	if (isIntegralType(T: FromT) || FromT == PT_Bool) {
6737	if (ToT == PT_IntAP)
6738	return this->emitCastAP(FromT, Ctx.getBitWidth(T: ToQT), E);
6739	if (ToT == PT_IntAPS)
6740	return this->emitCastAPS(FromT, Ctx.getBitWidth(T: ToQT), E);
6741
6742	// Integral to integral.
6743	if (isIntegralType(T: ToT) || ToT == PT_Bool)
6744	return FromT != ToT ? this->emitCast(FromT, ToT, E) : true;
6745
6746	if (ToT == PT_Float) {
6747	// Integral to floating.
6748	const llvm::fltSemantics *ToSem = &Ctx.getFloatSemantics(T: ToQT);
6749	return this->emitCastIntegralFloating(FromT, ToSem, getFPOptions(E), E);
6750	}
6751	}
6752
6753	return false;
6754	}
6755
6756	/// Emits __real(SubExpr)
6757	template <class Emitter>
6758	bool Compiler<Emitter>::emitComplexReal(const Expr *SubExpr) {
6759	assert(SubExpr->getType()->isAnyComplexType());
6760
6761	if (DiscardResult)
6762	return this->discard(E: SubExpr);
6763
6764	if (!this->visit(E: SubExpr))
6765	return false;
6766	if (SubExpr->isLValue()) {
6767	if (!this->emitConstUint8(0, SubExpr))
6768	return false;
6769	return this->emitArrayElemPtrPopUint8(SubExpr);
6770	}
6771
6772	// Rvalue, load the actual element.
6773	return this->emitArrayElemPop(classifyComplexElementType(T: SubExpr->getType()),
6774	0, SubExpr);
6775	}
6776
6777	template <class Emitter>
6778	bool Compiler<Emitter>::emitComplexBoolCast(const Expr *E) {
6779	assert(!DiscardResult);
6780	PrimType ElemT = classifyComplexElementType(T: E->getType());
6781	// We emit the expression (__real(E) != 0 || __imag(E) != 0)
6782	// for us, that means (bool)E[0] || (bool)E[1]
6783	if (!this->emitArrayElem(ElemT, 0, E))
6784	return false;
6785	if (ElemT == PT_Float) {
6786	if (!this->emitCastFloatingIntegral(PT_Bool, getFPOptions(E), E))
6787	return false;
6788	} else {
6789	if (!this->emitCast(ElemT, PT_Bool, E))
6790	return false;
6791	}
6792
6793	// We now have the bool value of E[0] on the stack.
6794	LabelTy LabelTrue = this->getLabel();
6795	if (!this->jumpTrue(LabelTrue))
6796	return false;
6797
6798	if (!this->emitArrayElemPop(ElemT, 1, E))
6799	return false;
6800	if (ElemT == PT_Float) {
6801	if (!this->emitCastFloatingIntegral(PT_Bool, getFPOptions(E), E))
6802	return false;
6803	} else {
6804	if (!this->emitCast(ElemT, PT_Bool, E))
6805	return false;
6806	}
6807	// Leave the boolean value of E[1] on the stack.
6808	LabelTy EndLabel = this->getLabel();
6809	this->jump(EndLabel);
6810
6811	this->emitLabel(LabelTrue);
6812	if (!this->emitPopPtr(E))
6813	return false;
6814	if (!this->emitConstBool(true, E))
6815	return false;
6816
6817	this->fallthrough(EndLabel);
6818	this->emitLabel(EndLabel);
6819
6820	return true;
6821	}
6822
6823	template <class Emitter>
6824	bool Compiler<Emitter>::emitComplexComparison(const Expr *LHS, const Expr *RHS,
6825	const BinaryOperator *E) {
6826	assert(E->isComparisonOp());
6827	assert(!Initializing);
6828	assert(!DiscardResult);
6829
6830	PrimType ElemT;
6831	bool LHSIsComplex;
6832	unsigned LHSOffset;
6833	if (LHS->getType()->isAnyComplexType()) {
6834	LHSIsComplex = true;
6835	ElemT = classifyComplexElementType(T: LHS->getType());
6836	LHSOffset = allocateLocalPrimitive(Src: LHS, Ty: PT_Ptr, /*IsConst=*/true);
6837	if (!this->visit(E: LHS))
6838	return false;
6839	if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
6840	return false;
6841	} else {
6842	LHSIsComplex = false;
6843	PrimType LHST = classifyPrim(LHS->getType());
6844	LHSOffset = this->allocateLocalPrimitive(Src: LHS, Ty: LHST, /*IsConst=*/true);
6845	if (!this->visit(E: LHS))
6846	return false;
6847	if (!this->emitSetLocal(LHST, LHSOffset, E))
6848	return false;
6849	}
6850
6851	bool RHSIsComplex;
6852	unsigned RHSOffset;
6853	if (RHS->getType()->isAnyComplexType()) {
6854	RHSIsComplex = true;
6855	ElemT = classifyComplexElementType(T: RHS->getType());
6856	RHSOffset = allocateLocalPrimitive(Src: RHS, Ty: PT_Ptr, /*IsConst=*/true);
6857	if (!this->visit(E: RHS))
6858	return false;
6859	if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
6860	return false;
6861	} else {
6862	RHSIsComplex = false;
6863	PrimType RHST = classifyPrim(RHS->getType());
6864	RHSOffset = this->allocateLocalPrimitive(Src: RHS, Ty: RHST, /*IsConst=*/true);
6865	if (!this->visit(E: RHS))
6866	return false;
6867	if (!this->emitSetLocal(RHST, RHSOffset, E))
6868	return false;
6869	}
6870
6871	auto getElem = [&](unsigned LocalOffset, unsigned Index,
6872	bool IsComplex) -> bool {
6873	if (IsComplex) {
6874	if (!this->emitGetLocal(PT_Ptr, LocalOffset, E))
6875	return false;
6876	return this->emitArrayElemPop(ElemT, Index, E);
6877	}
6878	return this->emitGetLocal(ElemT, LocalOffset, E);
6879	};
6880
6881	for (unsigned I = 0; I != 2; ++I) {
6882	// Get both values.
6883	if (!getElem(LHSOffset, I, LHSIsComplex))
6884	return false;
6885	if (!getElem(RHSOffset, I, RHSIsComplex))
6886	return false;
6887	// And compare them.
6888	if (!this->emitEQ(ElemT, E))
6889	return false;
6890
6891	if (!this->emitCastBoolUint8(E))
6892	return false;
6893	}
6894
6895	// We now have two bool values on the stack. Compare those.
6896	if (!this->emitAddUint8(E))
6897	return false;
6898	if (!this->emitConstUint8(2, E))
6899	return false;
6900
6901	if (E->getOpcode() == BO_EQ) {
6902	if (!this->emitEQUint8(E))
6903	return false;
6904	} else if (E->getOpcode() == BO_NE) {
6905	if (!this->emitNEUint8(E))
6906	return false;
6907	} else
6908	return false;
6909
6910	// In C, this returns an int.
6911	if (PrimType ResT = classifyPrim(E->getType()); ResT != PT_Bool)
6912	return this->emitCast(PT_Bool, ResT, E);
6913	return true;
6914	}
6915
6916	/// When calling this, we have a pointer of the local-to-destroy
6917	/// on the stack.
6918	/// Emit destruction of record types (or arrays of record types).
6919	template <class Emitter>
6920	bool Compiler<Emitter>::emitRecordDestruction(const Record *R, SourceInfo Loc) {
6921	assert(R);
6922	assert(!R->isAnonymousUnion());
6923	const CXXDestructorDecl *Dtor = R->getDestructor();
6924	if (!Dtor || Dtor->isTrivial())
6925	return true;
6926
6927	assert(Dtor);
6928	const Function *DtorFunc = getFunction(FD: Dtor);
6929	if (!DtorFunc)
6930	return false;
6931	assert(DtorFunc->hasThisPointer());
6932	assert(DtorFunc->getNumParams() == 1);
6933	if (!this->emitDupPtr(Loc))
6934	return false;
6935	return this->emitCall(DtorFunc, 0, Loc);
6936	}
6937	/// When calling this, we have a pointer of the local-to-destroy
6938	/// on the stack.
6939	/// Emit destruction of record types (or arrays of record types).
6940	template <class Emitter>
6941	bool Compiler<Emitter>::emitDestruction(const Descriptor *Desc,
6942	SourceInfo Loc) {
6943	assert(Desc);
6944	assert(!Desc->isPrimitive());
6945	assert(!Desc->isPrimitiveArray());
6946
6947	// Can happen if the decl is invalid.
6948	if (Desc->isDummy())
6949	return true;
6950
6951	// Arrays.
6952	if (Desc->isArray()) {
6953	const Descriptor *ElemDesc = Desc->ElemDesc;
6954	assert(ElemDesc);
6955
6956	// Don't need to do anything for these.
6957	if (ElemDesc->isPrimitiveArray())
6958	return true;
6959
6960	// If this is an array of record types, check if we need
6961	// to call the element destructors at all. If not, try
6962	// to save the work.
6963	if (const Record *ElemRecord = ElemDesc->ElemRecord) {
6964	if (const CXXDestructorDecl *Dtor = ElemRecord->getDestructor();
6965	!Dtor || Dtor->isTrivial())
6966	return true;
6967	}
6968
6969	if (unsigned N = Desc->getNumElems()) {
6970	for (ssize_t I = N - 1; I >= 0; --I) {
6971	if (!this->emitConstUint64(I, Loc))
6972	return false;
6973	if (!this->emitArrayElemPtrUint64(Loc))
6974	return false;
6975	if (!this->emitDestruction(Desc: ElemDesc, Loc))
6976	return false;
6977	if (!this->emitPopPtr(Loc))
6978	return false;
6979	}
6980	}
6981	return true;
6982	}
6983
6984	assert(Desc->ElemRecord);
6985	if (Desc->ElemRecord->isAnonymousUnion())
6986	return true;
6987
6988	return this->emitRecordDestruction(R: Desc->ElemRecord, Loc);
6989	}
6990
6991	/// Create a dummy pointer for the given decl (or expr) and
6992	/// push a pointer to it on the stack.
6993	template <class Emitter>
6994	bool Compiler<Emitter>::emitDummyPtr(const DeclTy &D, const Expr *E) {
6995	assert(!DiscardResult && "Should've been checked before");
6996
6997	unsigned DummyID = P.getOrCreateDummy(D);
6998
6999	if (!this->emitGetPtrGlobal(DummyID, E))
7000	return false;
7001	if (E->getType()->isVoidType())
7002	return true;
7003
7004	// Convert the dummy pointer to another pointer type if we have to.
7005	if (PrimType PT = classifyPrim(E); PT != PT_Ptr) {
7006	if (isPtrType(T: PT))
7007	return this->emitDecayPtr(PT_Ptr, PT, E);
7008	return false;
7009	}
7010	return true;
7011	}
7012
7013	template <class Emitter>
7014	bool Compiler<Emitter>::emitFloat(const APFloat &F, const Expr *E) {
7015	assert(!DiscardResult && "Should've been checked before");
7016
7017	if (Floating::singleWord(F.getSemantics()))
7018	return this->emitConstFloat(Floating(F), E);
7019
7020	APInt I = F.bitcastToAPInt();
7021	return this->emitConstFloat(
7022	Floating(const_cast<uint64_t *>(I.getRawData()),
7023	llvm::APFloatBase::SemanticsToEnum(Sem: F.getSemantics())),
7024	E);
7025	}
7026
7027	// This function is constexpr if and only if To, From, and the types of
7028	// all subobjects of To and From are types T such that...
7029	// (3.1) - is_union_v<T> is false;
7030	// (3.2) - is_pointer_v<T> is false;
7031	// (3.3) - is_member_pointer_v<T> is false;
7032	// (3.4) - is_volatile_v<T> is false; and
7033	// (3.5) - T has no non-static data members of reference type
7034	template <class Emitter>
7035	bool Compiler<Emitter>::emitBuiltinBitCast(const CastExpr *E) {
7036	const Expr *SubExpr = E->getSubExpr();
7037	QualType FromType = SubExpr->getType();
7038	QualType ToType = E->getType();
7039	std::optional<PrimType> ToT = classify(ToType);
7040
7041	assert(!ToType->isReferenceType());
7042
7043	// Prepare storage for the result in case we discard.
7044	if (DiscardResult && !Initializing && !ToT) {
7045	std::optional<unsigned> LocalIndex = allocateLocal(Src: E);
7046	if (!LocalIndex)
7047	return false;
7048	if (!this->emitGetPtrLocal(*LocalIndex, E))
7049	return false;
7050	}
7051
7052	// Get a pointer to the value-to-cast on the stack.
7053	// For CK_LValueToRValueBitCast, this is always an lvalue and
7054	// we later assume it to be one (i.e. a PT_Ptr). However,
7055	// we call this function for other utility methods where
7056	// a bitcast might be useful, so convert it to a PT_Ptr in that case.
7057	if (SubExpr->isGLValue() || FromType->isVectorType()) {
7058	if (!this->visit(E: SubExpr))
7059	return false;
7060	} else if (std::optional<PrimType> FromT = classify(SubExpr)) {
7061	unsigned TempOffset =
7062	allocateLocalPrimitive(Src: SubExpr, Ty: *FromT, /*IsConst=*/true);
7063	if (!this->visit(E: SubExpr))
7064	return false;
7065	if (!this->emitSetLocal(*FromT, TempOffset, E))
7066	return false;
7067	if (!this->emitGetPtrLocal(TempOffset, E))
7068	return false;
7069	} else {
7070	return false;
7071	}
7072
7073	if (!ToT) {
7074	if (!this->emitBitCast(E))
7075	return false;
7076	return DiscardResult ? this->emitPopPtr(E) : true;
7077	}
7078	assert(ToT);
7079
7080	const llvm::fltSemantics *TargetSemantics = nullptr;
7081	if (ToT == PT_Float)
7082	TargetSemantics = &Ctx.getFloatSemantics(T: ToType);
7083
7084	// Conversion to a primitive type. FromType can be another
7085	// primitive type, or a record/array.
7086	bool ToTypeIsUChar = (ToType->isSpecificBuiltinType(K: BuiltinType::UChar) ||
7087	ToType->isSpecificBuiltinType(K: BuiltinType::Char_U));
7088	uint32_t ResultBitWidth = std::max(a: Ctx.getBitWidth(T: ToType), b: 8u);
7089
7090	if (!this->emitBitCastPrim(*ToT, ToTypeIsUChar || ToType->isStdByteType(),
7091	ResultBitWidth, TargetSemantics, E))
7092	return false;
7093
7094	if (DiscardResult)
7095	return this->emitPop(*ToT, E);
7096
7097	return true;
7098	}
7099
7100	namespace clang {
7101	namespace interp {
7102
7103	template class Compiler<ByteCodeEmitter>;
7104	template class Compiler<EvalEmitter>;
7105
7106	} // namespace interp
7107	} // namespace clang
7108