1	//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code to emit Stmt nodes as LLVM code.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGDebugInfo.h"
14	#include "CGOpenMPRuntime.h"
15	#include "CodeGenFunction.h"
16	#include "CodeGenModule.h"
17	#include "CodeGenPGO.h"
18	#include "TargetInfo.h"
19	#include "clang/AST/Attr.h"
20	#include "clang/AST/Expr.h"
21	#include "clang/AST/Stmt.h"
22	#include "clang/AST/StmtVisitor.h"
23	#include "clang/Basic/Builtins.h"
24	#include "clang/Basic/DiagnosticSema.h"
25	#include "clang/Basic/PrettyStackTrace.h"
26	#include "clang/Basic/SourceManager.h"
27	#include "clang/Basic/TargetInfo.h"
28	#include "llvm/ADT/ArrayRef.h"
29	#include "llvm/ADT/DenseMap.h"
30	#include "llvm/ADT/SmallSet.h"
31	#include "llvm/ADT/StringExtras.h"
32	#include "llvm/IR/Assumptions.h"
33	#include "llvm/IR/DataLayout.h"
34	#include "llvm/IR/InlineAsm.h"
35	#include "llvm/IR/Intrinsics.h"
36	#include "llvm/IR/MDBuilder.h"
37	#include "llvm/Support/SaveAndRestore.h"
38	#include <optional>
39
40	using namespace clang;
41	using namespace CodeGen;
42
43	//===----------------------------------------------------------------------===//
44	// Statement Emission
45	//===----------------------------------------------------------------------===//
46
47	namespace llvm {
48	extern cl::opt<bool> EnableSingleByteCoverage;
49	} // namespace llvm
50
51	void CodeGenFunction::EmitStopPoint(const Stmt *S) {
52	if (CGDebugInfo *DI = getDebugInfo()) {
53	SourceLocation Loc;
54	Loc = S->getBeginLoc();
55	DI->EmitLocation(Builder, Loc);
56
57	LastStopPoint = Loc;
58	}
59	}
60
61	void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
62	assert(S && "Null statement?");
63	PGO->setCurrentStmt(S);
64
65	// These statements have their own debug info handling.
66	if (EmitSimpleStmt(S, Attrs))
67	return;
68
69	// Check if we are generating unreachable code.
70	if (!HaveInsertPoint()) {
71	// If so, and the statement doesn't contain a label, then we do not need to
72	// generate actual code. This is safe because (1) the current point is
73	// unreachable, so we don't need to execute the code, and (2) we've already
74	// handled the statements which update internal data structures (like the
75	// local variable map) which could be used by subsequent statements.
76	if (!ContainsLabel(S)) {
77	// Verify that any decl statements were handled as simple, they may be in
78	// scope of subsequent reachable statements.
79	assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
80	PGO->markStmtMaybeUsed(S);
81	return;
82	}
83
84	// Otherwise, make a new block to hold the code.
85	EnsureInsertPoint();
86	}
87
88	// Generate a stoppoint if we are emitting debug info.
89	EmitStopPoint(S);
90
91	// Ignore all OpenMP directives except for simd if OpenMP with Simd is
92	// enabled.
93	if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
94	if (const auto *D = dyn_cast<OMPExecutableDirective>(Val: S)) {
95	EmitSimpleOMPExecutableDirective(D: *D);
96	return;
97	}
98	}
99
100	switch (S->getStmtClass()) {
101	case Stmt::NoStmtClass:
102	case Stmt::CXXCatchStmtClass:
103	case Stmt::SEHExceptStmtClass:
104	case Stmt::SEHFinallyStmtClass:
105	case Stmt::MSDependentExistsStmtClass:
106	llvm_unreachable("invalid statement class to emit generically");
107	case Stmt::NullStmtClass:
108	case Stmt::CompoundStmtClass:
109	case Stmt::DeclStmtClass:
110	case Stmt::LabelStmtClass:
111	case Stmt::AttributedStmtClass:
112	case Stmt::GotoStmtClass:
113	case Stmt::BreakStmtClass:
114	case Stmt::ContinueStmtClass:
115	case Stmt::DefaultStmtClass:
116	case Stmt::CaseStmtClass:
117	case Stmt::SEHLeaveStmtClass:
118	case Stmt::SYCLKernelCallStmtClass:
119	llvm_unreachable("should have emitted these statements as simple");
120
121	#define STMT(Type, Base)
122	#define ABSTRACT_STMT(Op)
123	#define EXPR(Type, Base) \
124	case Stmt::Type##Class:
125	#include "clang/AST/StmtNodes.inc"
126	{
127	// Remember the block we came in on.
128	llvm::BasicBlock *incoming = Builder.GetInsertBlock();
129	assert(incoming && "expression emission must have an insertion point");
130
131	EmitIgnoredExpr(E: cast<Expr>(S));
132
133	llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
134	assert(outgoing && "expression emission cleared block!");
135
136	// The expression emitters assume (reasonably!) that the insertion
137	// point is always set. To maintain that, the call-emission code
138	// for noreturn functions has to enter a new block with no
139	// predecessors. We want to kill that block and mark the current
140	// insertion point unreachable in the common case of a call like
141	// "exit();". Since expression emission doesn't otherwise create
142	// blocks with no predecessors, we can just test for that.
143	// However, we must be careful not to do this to our incoming
144	// block, because *statement* emission does sometimes create
145	// reachable blocks which will have no predecessors until later in
146	// the function. This occurs with, e.g., labels that are not
147	// reachable by fallthrough.
148	if (incoming != outgoing && outgoing->use_empty()) {
149	outgoing->eraseFromParent();
150	Builder.ClearInsertionPoint();
151	}
152	break;
153	}
154
155	case Stmt::IndirectGotoStmtClass:
156	EmitIndirectGotoStmt(S: cast<IndirectGotoStmt>(*S)); break;
157
158	case Stmt::IfStmtClass: EmitIfStmt(S: cast<IfStmt>(*S)); break;
159	case Stmt::WhileStmtClass: EmitWhileStmt(S: cast<WhileStmt>(*S), Attrs); break;
160	case Stmt::DoStmtClass: EmitDoStmt(S: cast<DoStmt>(*S), Attrs); break;
161	case Stmt::ForStmtClass: EmitForStmt(S: cast<ForStmt>(*S), Attrs); break;
162
163	case Stmt::ReturnStmtClass: EmitReturnStmt(S: cast<ReturnStmt>(*S)); break;
164
165	case Stmt::SwitchStmtClass: EmitSwitchStmt(S: cast<SwitchStmt>(*S)); break;
166	case Stmt::GCCAsmStmtClass: // Intentional fall-through.
167	case Stmt::MSAsmStmtClass: EmitAsmStmt(S: cast<AsmStmt>(*S)); break;
168	case Stmt::CoroutineBodyStmtClass:
169	EmitCoroutineBody(S: cast<CoroutineBodyStmt>(*S));
170	break;
171	case Stmt::CoreturnStmtClass:
172	EmitCoreturnStmt(S: cast<CoreturnStmt>(*S));
173	break;
174	case Stmt::CapturedStmtClass: {
175	const CapturedStmt *CS = cast<CapturedStmt>(S);
176	EmitCapturedStmt(S: *CS, K: CS->getCapturedRegionKind());
177	}
178	break;
179	case Stmt::ObjCAtTryStmtClass:
180	EmitObjCAtTryStmt(S: cast<ObjCAtTryStmt>(*S));
181	break;
182	case Stmt::ObjCAtCatchStmtClass:
183	llvm_unreachable(
184	"@catch statements should be handled by EmitObjCAtTryStmt");
185	case Stmt::ObjCAtFinallyStmtClass:
186	llvm_unreachable(
187	"@finally statements should be handled by EmitObjCAtTryStmt");
188	case Stmt::ObjCAtThrowStmtClass:
189	EmitObjCAtThrowStmt(S: cast<ObjCAtThrowStmt>(*S));
190	break;
191	case Stmt::ObjCAtSynchronizedStmtClass:
192	EmitObjCAtSynchronizedStmt(S: cast<ObjCAtSynchronizedStmt>(*S));
193	break;
194	case Stmt::ObjCForCollectionStmtClass:
195	EmitObjCForCollectionStmt(S: cast<ObjCForCollectionStmt>(*S));
196	break;
197	case Stmt::ObjCAutoreleasePoolStmtClass:
198	EmitObjCAutoreleasePoolStmt(S: cast<ObjCAutoreleasePoolStmt>(*S));
199	break;
200
201	case Stmt::CXXTryStmtClass:
202	EmitCXXTryStmt(S: cast<CXXTryStmt>(*S));
203	break;
204	case Stmt::CXXForRangeStmtClass:
205	EmitCXXForRangeStmt(S: cast<CXXForRangeStmt>(*S), Attrs);
206	break;
207	case Stmt::SEHTryStmtClass:
208	EmitSEHTryStmt(S: cast<SEHTryStmt>(*S));
209	break;
210	case Stmt::OMPMetaDirectiveClass:
211	EmitOMPMetaDirective(S: cast<OMPMetaDirective>(*S));
212	break;
213	case Stmt::OMPCanonicalLoopClass:
214	EmitOMPCanonicalLoop(S: cast<OMPCanonicalLoop>(S));
215	break;
216	case Stmt::OMPParallelDirectiveClass:
217	EmitOMPParallelDirective(S: cast<OMPParallelDirective>(*S));
218	break;
219	case Stmt::OMPSimdDirectiveClass:
220	EmitOMPSimdDirective(S: cast<OMPSimdDirective>(*S));
221	break;
222	case Stmt::OMPTileDirectiveClass:
223	EmitOMPTileDirective(S: cast<OMPTileDirective>(*S));
224	break;
225	case Stmt::OMPStripeDirectiveClass:
226	EmitOMPStripeDirective(S: cast<OMPStripeDirective>(*S));
227	break;
228	case Stmt::OMPUnrollDirectiveClass:
229	EmitOMPUnrollDirective(S: cast<OMPUnrollDirective>(*S));
230	break;
231	case Stmt::OMPReverseDirectiveClass:
232	EmitOMPReverseDirective(S: cast<OMPReverseDirective>(*S));
233	break;
234	case Stmt::OMPInterchangeDirectiveClass:
235	EmitOMPInterchangeDirective(S: cast<OMPInterchangeDirective>(*S));
236	break;
237	case Stmt::OMPForDirectiveClass:
238	EmitOMPForDirective(S: cast<OMPForDirective>(*S));
239	break;
240	case Stmt::OMPForSimdDirectiveClass:
241	EmitOMPForSimdDirective(S: cast<OMPForSimdDirective>(*S));
242	break;
243	case Stmt::OMPSectionsDirectiveClass:
244	EmitOMPSectionsDirective(S: cast<OMPSectionsDirective>(*S));
245	break;
246	case Stmt::OMPSectionDirectiveClass:
247	EmitOMPSectionDirective(S: cast<OMPSectionDirective>(*S));
248	break;
249	case Stmt::OMPSingleDirectiveClass:
250	EmitOMPSingleDirective(S: cast<OMPSingleDirective>(*S));
251	break;
252	case Stmt::OMPMasterDirectiveClass:
253	EmitOMPMasterDirective(S: cast<OMPMasterDirective>(*S));
254	break;
255	case Stmt::OMPCriticalDirectiveClass:
256	EmitOMPCriticalDirective(S: cast<OMPCriticalDirective>(*S));
257	break;
258	case Stmt::OMPParallelForDirectiveClass:
259	EmitOMPParallelForDirective(S: cast<OMPParallelForDirective>(*S));
260	break;
261	case Stmt::OMPParallelForSimdDirectiveClass:
262	EmitOMPParallelForSimdDirective(S: cast<OMPParallelForSimdDirective>(*S));
263	break;
264	case Stmt::OMPParallelMasterDirectiveClass:
265	EmitOMPParallelMasterDirective(S: cast<OMPParallelMasterDirective>(*S));
266	break;
267	case Stmt::OMPParallelSectionsDirectiveClass:
268	EmitOMPParallelSectionsDirective(S: cast<OMPParallelSectionsDirective>(*S));
269	break;
270	case Stmt::OMPTaskDirectiveClass:
271	EmitOMPTaskDirective(S: cast<OMPTaskDirective>(*S));
272	break;
273	case Stmt::OMPTaskyieldDirectiveClass:
274	EmitOMPTaskyieldDirective(S: cast<OMPTaskyieldDirective>(*S));
275	break;
276	case Stmt::OMPErrorDirectiveClass:
277	EmitOMPErrorDirective(S: cast<OMPErrorDirective>(*S));
278	break;
279	case Stmt::OMPBarrierDirectiveClass:
280	EmitOMPBarrierDirective(S: cast<OMPBarrierDirective>(*S));
281	break;
282	case Stmt::OMPTaskwaitDirectiveClass:
283	EmitOMPTaskwaitDirective(S: cast<OMPTaskwaitDirective>(*S));
284	break;
285	case Stmt::OMPTaskgroupDirectiveClass:
286	EmitOMPTaskgroupDirective(S: cast<OMPTaskgroupDirective>(*S));
287	break;
288	case Stmt::OMPFlushDirectiveClass:
289	EmitOMPFlushDirective(S: cast<OMPFlushDirective>(*S));
290	break;
291	case Stmt::OMPDepobjDirectiveClass:
292	EmitOMPDepobjDirective(S: cast<OMPDepobjDirective>(*S));
293	break;
294	case Stmt::OMPScanDirectiveClass:
295	EmitOMPScanDirective(S: cast<OMPScanDirective>(*S));
296	break;
297	case Stmt::OMPOrderedDirectiveClass:
298	EmitOMPOrderedDirective(S: cast<OMPOrderedDirective>(*S));
299	break;
300	case Stmt::OMPAtomicDirectiveClass:
301	EmitOMPAtomicDirective(S: cast<OMPAtomicDirective>(*S));
302	break;
303	case Stmt::OMPTargetDirectiveClass:
304	EmitOMPTargetDirective(S: cast<OMPTargetDirective>(*S));
305	break;
306	case Stmt::OMPTeamsDirectiveClass:
307	EmitOMPTeamsDirective(S: cast<OMPTeamsDirective>(*S));
308	break;
309	case Stmt::OMPCancellationPointDirectiveClass:
310	EmitOMPCancellationPointDirective(S: cast<OMPCancellationPointDirective>(*S));
311	break;
312	case Stmt::OMPCancelDirectiveClass:
313	EmitOMPCancelDirective(S: cast<OMPCancelDirective>(*S));
314	break;
315	case Stmt::OMPTargetDataDirectiveClass:
316	EmitOMPTargetDataDirective(S: cast<OMPTargetDataDirective>(*S));
317	break;
318	case Stmt::OMPTargetEnterDataDirectiveClass:
319	EmitOMPTargetEnterDataDirective(S: cast<OMPTargetEnterDataDirective>(*S));
320	break;
321	case Stmt::OMPTargetExitDataDirectiveClass:
322	EmitOMPTargetExitDataDirective(S: cast<OMPTargetExitDataDirective>(*S));
323	break;
324	case Stmt::OMPTargetParallelDirectiveClass:
325	EmitOMPTargetParallelDirective(S: cast<OMPTargetParallelDirective>(*S));
326	break;
327	case Stmt::OMPTargetParallelForDirectiveClass:
328	EmitOMPTargetParallelForDirective(S: cast<OMPTargetParallelForDirective>(*S));
329	break;
330	case Stmt::OMPTaskLoopDirectiveClass:
331	EmitOMPTaskLoopDirective(S: cast<OMPTaskLoopDirective>(*S));
332	break;
333	case Stmt::OMPTaskLoopSimdDirectiveClass:
334	EmitOMPTaskLoopSimdDirective(S: cast<OMPTaskLoopSimdDirective>(*S));
335	break;
336	case Stmt::OMPMasterTaskLoopDirectiveClass:
337	EmitOMPMasterTaskLoopDirective(S: cast<OMPMasterTaskLoopDirective>(*S));
338	break;
339	case Stmt::OMPMaskedTaskLoopDirectiveClass:
340	EmitOMPMaskedTaskLoopDirective(S: cast<OMPMaskedTaskLoopDirective>(*S));
341	break;
342	case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
343	EmitOMPMasterTaskLoopSimdDirective(
344	S: cast<OMPMasterTaskLoopSimdDirective>(*S));
345	break;
346	case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:
347	EmitOMPMaskedTaskLoopSimdDirective(
348	S: cast<OMPMaskedTaskLoopSimdDirective>(*S));
349	break;
350	case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
351	EmitOMPParallelMasterTaskLoopDirective(
352	S: cast<OMPParallelMasterTaskLoopDirective>(*S));
353	break;
354	case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:
355	EmitOMPParallelMaskedTaskLoopDirective(
356	S: cast<OMPParallelMaskedTaskLoopDirective>(*S));
357	break;
358	case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
359	EmitOMPParallelMasterTaskLoopSimdDirective(
360	S: cast<OMPParallelMasterTaskLoopSimdDirective>(*S));
361	break;
362	case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:
363	EmitOMPParallelMaskedTaskLoopSimdDirective(
364	S: cast<OMPParallelMaskedTaskLoopSimdDirective>(*S));
365	break;
366	case Stmt::OMPDistributeDirectiveClass:
367	EmitOMPDistributeDirective(S: cast<OMPDistributeDirective>(*S));
368	break;
369	case Stmt::OMPTargetUpdateDirectiveClass:
370	EmitOMPTargetUpdateDirective(S: cast<OMPTargetUpdateDirective>(*S));
371	break;
372	case Stmt::OMPDistributeParallelForDirectiveClass:
373	EmitOMPDistributeParallelForDirective(
374	S: cast<OMPDistributeParallelForDirective>(*S));
375	break;
376	case Stmt::OMPDistributeParallelForSimdDirectiveClass:
377	EmitOMPDistributeParallelForSimdDirective(
378	S: cast<OMPDistributeParallelForSimdDirective>(*S));
379	break;
380	case Stmt::OMPDistributeSimdDirectiveClass:
381	EmitOMPDistributeSimdDirective(S: cast<OMPDistributeSimdDirective>(*S));
382	break;
383	case Stmt::OMPTargetParallelForSimdDirectiveClass:
384	EmitOMPTargetParallelForSimdDirective(
385	S: cast<OMPTargetParallelForSimdDirective>(*S));
386	break;
387	case Stmt::OMPTargetSimdDirectiveClass:
388	EmitOMPTargetSimdDirective(S: cast<OMPTargetSimdDirective>(*S));
389	break;
390	case Stmt::OMPTeamsDistributeDirectiveClass:
391	EmitOMPTeamsDistributeDirective(S: cast<OMPTeamsDistributeDirective>(*S));
392	break;
393	case Stmt::OMPTeamsDistributeSimdDirectiveClass:
394	EmitOMPTeamsDistributeSimdDirective(
395	S: cast<OMPTeamsDistributeSimdDirective>(*S));
396	break;
397	case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
398	EmitOMPTeamsDistributeParallelForSimdDirective(
399	S: cast<OMPTeamsDistributeParallelForSimdDirective>(*S));
400	break;
401	case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
402	EmitOMPTeamsDistributeParallelForDirective(
403	S: cast<OMPTeamsDistributeParallelForDirective>(*S));
404	break;
405	case Stmt::OMPTargetTeamsDirectiveClass:
406	EmitOMPTargetTeamsDirective(S: cast<OMPTargetTeamsDirective>(*S));
407	break;
408	case Stmt::OMPTargetTeamsDistributeDirectiveClass:
409	EmitOMPTargetTeamsDistributeDirective(
410	S: cast<OMPTargetTeamsDistributeDirective>(*S));
411	break;
412	case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
413	EmitOMPTargetTeamsDistributeParallelForDirective(
414	S: cast<OMPTargetTeamsDistributeParallelForDirective>(*S));
415	break;
416	case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
417	EmitOMPTargetTeamsDistributeParallelForSimdDirective(
418	S: cast<OMPTargetTeamsDistributeParallelForSimdDirective>(*S));
419	break;
420	case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
421	EmitOMPTargetTeamsDistributeSimdDirective(
422	S: cast<OMPTargetTeamsDistributeSimdDirective>(*S));
423	break;
424	case Stmt::OMPInteropDirectiveClass:
425	EmitOMPInteropDirective(S: cast<OMPInteropDirective>(*S));
426	break;
427	case Stmt::OMPDispatchDirectiveClass:
428	CGM.ErrorUnsupported(S, Type: "OpenMP dispatch directive");
429	break;
430	case Stmt::OMPScopeDirectiveClass:
431	EmitOMPScopeDirective(S: cast<OMPScopeDirective>(*S));
432	break;
433	case Stmt::OMPMaskedDirectiveClass:
434	EmitOMPMaskedDirective(S: cast<OMPMaskedDirective>(*S));
435	break;
436	case Stmt::OMPGenericLoopDirectiveClass:
437	EmitOMPGenericLoopDirective(S: cast<OMPGenericLoopDirective>(*S));
438	break;
439	case Stmt::OMPTeamsGenericLoopDirectiveClass:
440	EmitOMPTeamsGenericLoopDirective(S: cast<OMPTeamsGenericLoopDirective>(*S));
441	break;
442	case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
443	EmitOMPTargetTeamsGenericLoopDirective(
444	S: cast<OMPTargetTeamsGenericLoopDirective>(*S));
445	break;
446	case Stmt::OMPParallelGenericLoopDirectiveClass:
447	EmitOMPParallelGenericLoopDirective(
448	S: cast<OMPParallelGenericLoopDirective>(*S));
449	break;
450	case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
451	EmitOMPTargetParallelGenericLoopDirective(
452	S: cast<OMPTargetParallelGenericLoopDirective>(*S));
453	break;
454	case Stmt::OMPParallelMaskedDirectiveClass:
455	EmitOMPParallelMaskedDirective(S: cast<OMPParallelMaskedDirective>(*S));
456	break;
457	case Stmt::OMPAssumeDirectiveClass:
458	EmitOMPAssumeDirective(S: cast<OMPAssumeDirective>(*S));
459	break;
460	case Stmt::OpenACCComputeConstructClass:
461	EmitOpenACCComputeConstruct(S: cast<OpenACCComputeConstruct>(*S));
462	break;
463	case Stmt::OpenACCLoopConstructClass:
464	EmitOpenACCLoopConstruct(S: cast<OpenACCLoopConstruct>(*S));
465	break;
466	case Stmt::OpenACCCombinedConstructClass:
467	EmitOpenACCCombinedConstruct(S: cast<OpenACCCombinedConstruct>(*S));
468	break;
469	case Stmt::OpenACCDataConstructClass:
470	EmitOpenACCDataConstruct(S: cast<OpenACCDataConstruct>(*S));
471	break;
472	case Stmt::OpenACCEnterDataConstructClass:
473	EmitOpenACCEnterDataConstruct(S: cast<OpenACCEnterDataConstruct>(*S));
474	break;
475	case Stmt::OpenACCExitDataConstructClass:
476	EmitOpenACCExitDataConstruct(S: cast<OpenACCExitDataConstruct>(*S));
477	break;
478	case Stmt::OpenACCHostDataConstructClass:
479	EmitOpenACCHostDataConstruct(S: cast<OpenACCHostDataConstruct>(*S));
480	break;
481	case Stmt::OpenACCWaitConstructClass:
482	EmitOpenACCWaitConstruct(S: cast<OpenACCWaitConstruct>(*S));
483	break;
484	case Stmt::OpenACCInitConstructClass:
485	EmitOpenACCInitConstruct(S: cast<OpenACCInitConstruct>(*S));
486	break;
487	case Stmt::OpenACCShutdownConstructClass:
488	EmitOpenACCShutdownConstruct(S: cast<OpenACCShutdownConstruct>(*S));
489	break;
490	case Stmt::OpenACCSetConstructClass:
491	EmitOpenACCSetConstruct(S: cast<OpenACCSetConstruct>(*S));
492	break;
493	case Stmt::OpenACCUpdateConstructClass:
494	EmitOpenACCUpdateConstruct(S: cast<OpenACCUpdateConstruct>(*S));
495	break;
496	case Stmt::OpenACCAtomicConstructClass:
497	EmitOpenACCAtomicConstruct(S: cast<OpenACCAtomicConstruct>(*S));
498	break;
499	case Stmt::OpenACCCacheConstructClass:
500	EmitOpenACCCacheConstruct(S: cast<OpenACCCacheConstruct>(*S));
501	break;
502	}
503	}
504
505	bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
506	ArrayRef<const Attr *> Attrs) {
507	switch (S->getStmtClass()) {
508	default:
509	return false;
510	case Stmt::NullStmtClass:
511	break;
512	case Stmt::CompoundStmtClass:
513	EmitCompoundStmt(S: cast<CompoundStmt>(Val: *S));
514	break;
515	case Stmt::DeclStmtClass:
516	EmitDeclStmt(S: cast<DeclStmt>(Val: *S));
517	break;
518	case Stmt::LabelStmtClass:
519	EmitLabelStmt(S: cast<LabelStmt>(Val: *S));
520	break;
521	case Stmt::AttributedStmtClass:
522	EmitAttributedStmt(S: cast<AttributedStmt>(Val: *S));
523	break;
524	case Stmt::GotoStmtClass:
525	EmitGotoStmt(S: cast<GotoStmt>(Val: *S));
526	break;
527	case Stmt::BreakStmtClass:
528	EmitBreakStmt(S: cast<BreakStmt>(Val: *S));
529	break;
530	case Stmt::ContinueStmtClass:
531	EmitContinueStmt(S: cast<ContinueStmt>(Val: *S));
532	break;
533	case Stmt::DefaultStmtClass:
534	EmitDefaultStmt(S: cast<DefaultStmt>(Val: *S), Attrs);
535	break;
536	case Stmt::CaseStmtClass:
537	EmitCaseStmt(S: cast<CaseStmt>(Val: *S), Attrs);
538	break;
539	case Stmt::SEHLeaveStmtClass:
540	EmitSEHLeaveStmt(S: cast<SEHLeaveStmt>(Val: *S));
541	break;
542	case Stmt::SYCLKernelCallStmtClass:
543	// SYCL kernel call statements are generated as wrappers around the body
544	// of functions declared with the sycl_kernel_entry_point attribute. Such
545	// functions are used to specify how a SYCL kernel (a function object) is
546	// to be invoked; the SYCL kernel call statement contains a transformed
547	// variation of the function body and is used to generate a SYCL kernel
548	// caller function; a function that serves as the device side entry point
549	// used to execute the SYCL kernel. The sycl_kernel_entry_point attributed
550	// function is invoked by host code in order to trigger emission of the
551	// device side SYCL kernel caller function and to generate metadata needed
552	// by SYCL run-time library implementations; the function is otherwise
553	// intended to have no effect. As such, the function body is not evaluated
554	// as part of the invocation during host compilation (and the function
555	// should not be called or emitted during device compilation); the SYCL
556	// kernel call statement is thus handled as a null statement for the
557	// purpose of code generation.
558	break;
559	}
560	return true;
561	}
562
563	/// EmitCompoundStmt - Emit a compound statement {..} node. If GetLast is true,
564	/// this captures the expression result of the last sub-statement and returns it
565	/// (for use by the statement expression extension).
566	Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
567	AggValueSlot AggSlot) {
568	PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
569	"LLVM IR generation of compound statement ('{}')");
570
571	// Keep track of the current cleanup stack depth, including debug scopes.
572	LexicalScope Scope(*this, S.getSourceRange());
573
574	return EmitCompoundStmtWithoutScope(S, GetLast, AVS: AggSlot);
575	}
576
577	Address
578	CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
579	bool GetLast,
580	AggValueSlot AggSlot) {
581
582	const Stmt *ExprResult = S.getStmtExprResult();
583	assert((!GetLast || (GetLast && ExprResult)) &&
584	"If GetLast is true then the CompoundStmt must have a StmtExprResult");
585
586	Address RetAlloca = Address::invalid();
587
588	for (auto *CurStmt : S.body()) {
589	if (GetLast && ExprResult == CurStmt) {
590	// We have to special case labels here. They are statements, but when put
591	// at the end of a statement expression, they yield the value of their
592	// subexpression. Handle this by walking through all labels we encounter,
593	// emitting them before we evaluate the subexpr.
594	// Similar issues arise for attributed statements.
595	while (!isa<Expr>(Val: ExprResult)) {
596	if (const auto *LS = dyn_cast<LabelStmt>(Val: ExprResult)) {
597	EmitLabel(D: LS->getDecl());
598	ExprResult = LS->getSubStmt();
599	} else if (const auto *AS = dyn_cast<AttributedStmt>(Val: ExprResult)) {
600	// FIXME: Update this if we ever have attributes that affect the
601	// semantics of an expression.
602	ExprResult = AS->getSubStmt();
603	} else {
604	llvm_unreachable("unknown value statement");
605	}
606	}
607
608	EnsureInsertPoint();
609
610	const Expr *E = cast<Expr>(Val: ExprResult);
611	QualType ExprTy = E->getType();
612	if (hasAggregateEvaluationKind(T: ExprTy)) {
613	EmitAggExpr(E, AS: AggSlot);
614	} else {
615	// We can't return an RValue here because there might be cleanups at
616	// the end of the StmtExpr. Because of that, we have to emit the result
617	// here into a temporary alloca.
618	RetAlloca = CreateMemTemp(T: ExprTy);
619	EmitAnyExprToMem(E, Location: RetAlloca, Quals: Qualifiers(),
620	/*IsInit*/ IsInitializer: false);
621	}
622	} else {
623	EmitStmt(S: CurStmt);
624	}
625	}
626
627	return RetAlloca;
628	}
629
630	void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
631	llvm::BranchInst *BI = dyn_cast<llvm::BranchInst>(Val: BB->getTerminator());
632
633	// If there is a cleanup stack, then we it isn't worth trying to
634	// simplify this block (we would need to remove it from the scope map
635	// and cleanup entry).
636	if (!EHStack.empty())
637	return;
638
639	// Can only simplify direct branches.
640	if (!BI || !BI->isUnconditional())
641	return;
642
643	// Can only simplify empty blocks.
644	if (BI->getIterator() != BB->begin())
645	return;
646
647	BB->replaceAllUsesWith(V: BI->getSuccessor(i: 0));
648	BI->eraseFromParent();
649	BB->eraseFromParent();
650	}
651
652	void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
653	llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
654
655	// Fall out of the current block (if necessary).
656	EmitBranch(Block: BB);
657
658	if (IsFinished && BB->use_empty()) {
659	delete BB;
660	return;
661	}
662
663	// Place the block after the current block, if possible, or else at
664	// the end of the function.
665	if (CurBB && CurBB->getParent())
666	CurFn->insert(Position: std::next(x: CurBB->getIterator()), BB);
667	else
668	CurFn->insert(Position: CurFn->end(), BB);
669	Builder.SetInsertPoint(BB);
670	}
671
672	void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
673	// Emit a branch from the current block to the target one if this
674	// was a real block. If this was just a fall-through block after a
675	// terminator, don't emit it.
676	llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
677
678	if (!CurBB || CurBB->getTerminator()) {
679	// If there is no insert point or the previous block is already
680	// terminated, don't touch it.
681	} else {
682	// Otherwise, create a fall-through branch.
683	Builder.CreateBr(Dest: Target);
684	}
685
686	Builder.ClearInsertionPoint();
687	}
688
689	void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
690	bool inserted = false;
691	for (llvm::User *u : block->users()) {
692	if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(Val: u)) {
693	CurFn->insert(Position: std::next(x: insn->getParent()->getIterator()), BB: block);
694	inserted = true;
695	break;
696	}
697	}
698
699	if (!inserted)
700	CurFn->insert(Position: CurFn->end(), BB: block);
701
702	Builder.SetInsertPoint(block);
703	}
704
705	CodeGenFunction::JumpDest
706	CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
707	JumpDest &Dest = LabelMap[D];
708	if (Dest.isValid()) return Dest;
709
710	// Create, but don't insert, the new block.
711	Dest = JumpDest(createBasicBlock(name: D->getName()),
712	EHScopeStack::stable_iterator::invalid(),
713	NextCleanupDestIndex++);
714	return Dest;
715	}
716
717	void CodeGenFunction::EmitLabel(const LabelDecl *D) {
718	// Add this label to the current lexical scope if we're within any
719	// normal cleanups. Jumps "in" to this label --- when permitted by
720	// the language --- may need to be routed around such cleanups.
721	if (EHStack.hasNormalCleanups() && CurLexicalScope)
722	CurLexicalScope->addLabel(label: D);
723
724	JumpDest &Dest = LabelMap[D];
725
726	// If we didn't need a forward reference to this label, just go
727	// ahead and create a destination at the current scope.
728	if (!Dest.isValid()) {
729	Dest = getJumpDestInCurrentScope(D->getName());
730
731	// Otherwise, we need to give this label a target depth and remove
732	// it from the branch-fixups list.
733	} else {
734	assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
735	Dest.setScopeDepth(EHStack.stable_begin());
736	ResolveBranchFixups(Target: Dest.getBlock());
737	}
738
739	EmitBlock(BB: Dest.getBlock());
740
741	// Emit debug info for labels.
742	if (CGDebugInfo *DI = getDebugInfo()) {
743	if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
744	DI->setLocation(D->getLocation());
745	DI->EmitLabel(D, Builder);
746	}
747	}
748
749	incrementProfileCounter(S: D->getStmt());
750	}
751
752	/// Change the cleanup scope of the labels in this lexical scope to
753	/// match the scope of the enclosing context.
754	void CodeGenFunction::LexicalScope::rescopeLabels() {
755	assert(!Labels.empty());
756	EHScopeStack::stable_iterator innermostScope
757	= CGF.EHStack.getInnermostNormalCleanup();
758
759	// Change the scope depth of all the labels.
760	for (SmallVectorImpl<const LabelDecl*>::const_iterator
761	i = Labels.begin(), e = Labels.end(); i != e; ++i) {
762	assert(CGF.LabelMap.count(*i));
763	JumpDest &dest = CGF.LabelMap.find(Val: *i)->second;
764	assert(dest.getScopeDepth().isValid());
765	assert(innermostScope.encloses(dest.getScopeDepth()));
766	dest.setScopeDepth(innermostScope);
767	}
768
769	// Reparent the labels if the new scope also has cleanups.
770	if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
771	ParentScope->Labels.append(in_start: Labels.begin(), in_end: Labels.end());
772	}
773	}
774
775
776	void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
777	EmitLabel(D: S.getDecl());
778
779	// IsEHa - emit eha.scope.begin if it's a side entry of a scope
780	if (getLangOpts().EHAsynch && S.isSideEntry())
781	EmitSehCppScopeBegin();
782
783	EmitStmt(S: S.getSubStmt());
784	}
785
786	void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
787	bool nomerge = false;
788	bool noinline = false;
789	bool alwaysinline = false;
790	bool noconvergent = false;
791	HLSLControlFlowHintAttr::Spelling flattenOrBranch =
792	HLSLControlFlowHintAttr::SpellingNotCalculated;
793	const CallExpr *musttail = nullptr;
794	const AtomicAttr *AA = nullptr;
795
796	for (const auto *A : S.getAttrs()) {
797	switch (A->getKind()) {
798	default:
799	break;
800	case attr::NoMerge:
801	nomerge = true;
802	break;
803	case attr::NoInline:
804	noinline = true;
805	break;
806	case attr::AlwaysInline:
807	alwaysinline = true;
808	break;
809	case attr::NoConvergent:
810	noconvergent = true;
811	break;
812	case attr::MustTail: {
813	const Stmt *Sub = S.getSubStmt();
814	const ReturnStmt *R = cast<ReturnStmt>(Val: Sub);
815	musttail = cast<CallExpr>(Val: R->getRetValue()->IgnoreParens());
816	} break;
817	case attr::CXXAssume: {
818	const Expr *Assumption = cast<CXXAssumeAttr>(A)->getAssumption();
819	if (getLangOpts().CXXAssumptions && Builder.GetInsertBlock() &&
820	!Assumption->HasSideEffects(Ctx: getContext())) {
821	llvm::Value *AssumptionVal = EmitCheckedArgForAssume(E: Assumption);
822	Builder.CreateAssumption(Cond: AssumptionVal);
823	}
824	} break;
825	case attr::Atomic:
826	AA = cast<AtomicAttr>(A);
827	break;
828	case attr::HLSLControlFlowHint: {
829	flattenOrBranch = cast<HLSLControlFlowHintAttr>(A)->getSemanticSpelling();
830	} break;
831	}
832	}
833	SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge);
834	SaveAndRestore save_noinline(InNoInlineAttributedStmt, noinline);
835	SaveAndRestore save_alwaysinline(InAlwaysInlineAttributedStmt, alwaysinline);
836	SaveAndRestore save_noconvergent(InNoConvergentAttributedStmt, noconvergent);
837	SaveAndRestore save_musttail(MustTailCall, musttail);
838	SaveAndRestore save_flattenOrBranch(HLSLControlFlowAttr, flattenOrBranch);
839	CGAtomicOptionsRAII AORAII(CGM, AA);
840	EmitStmt(S: S.getSubStmt(), Attrs: S.getAttrs());
841	}
842
843	void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
844	// If this code is reachable then emit a stop point (if generating
845	// debug info). We have to do this ourselves because we are on the
846	// "simple" statement path.
847	if (HaveInsertPoint())
848	EmitStopPoint(S: &S);
849
850	EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: S.getLabel()));
851	}
852
853
854	void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
855	if (const LabelDecl *Target = S.getConstantTarget()) {
856	EmitBranchThroughCleanup(Dest: getJumpDestForLabel(D: Target));
857	return;
858	}
859
860	// Ensure that we have an i8* for our PHI node.
861	llvm::Value *V = Builder.CreateBitCast(V: EmitScalarExpr(E: S.getTarget()),
862	DestTy: Int8PtrTy, Name: "addr");
863	llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
864
865	// Get the basic block for the indirect goto.
866	llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
867
868	// The first instruction in the block has to be the PHI for the switch dest,
869	// add an entry for this branch.
870	cast<llvm::PHINode>(Val: IndGotoBB->begin())->addIncoming(V, BB: CurBB);
871
872	EmitBranch(Target: IndGotoBB);
873	}
874
875	void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
876	const Stmt *Else = S.getElse();
877
878	// The else branch of a consteval if statement is always the only branch that
879	// can be runtime evaluated.
880	if (S.isConsteval()) {
881	const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : Else;
882	if (Executed) {
883	RunCleanupsScope ExecutedScope(*this);
884	EmitStmt(S: Executed);
885	}
886	return;
887	}
888
889	// C99 6.8.4.1: The first substatement is executed if the expression compares
890	// unequal to 0. The condition must be a scalar type.
891	LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
892	ApplyDebugLocation DL(*this, S.getCond());
893
894	if (S.getInit())
895	EmitStmt(S: S.getInit());
896
897	if (S.getConditionVariable())
898	EmitDecl(*S.getConditionVariable());
899
900	// If the condition constant folds and can be elided, try to avoid emitting
901	// the condition and the dead arm of the if/else.
902	bool CondConstant;
903	if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: CondConstant,
904	AllowLabels: S.isConstexpr())) {
905	// Figure out which block (then or else) is executed.
906	const Stmt *Executed = S.getThen();
907	const Stmt *Skipped = Else;
908	if (!CondConstant) // Condition false?
909	std::swap(a&: Executed, b&: Skipped);
910
911	// If the skipped block has no labels in it, just emit the executed block.
912	// This avoids emitting dead code and simplifies the CFG substantially.
913	if (S.isConstexpr() || !ContainsLabel(S: Skipped)) {
914	if (CondConstant)
915	incrementProfileCounter(&S);
916	if (Executed) {
917	MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
918	RunCleanupsScope ExecutedScope(*this);
919	EmitStmt(S: Executed);
920	}
921	PGO->markStmtMaybeUsed(S: Skipped);
922	return;
923	}
924	}
925
926	// Otherwise, the condition did not fold, or we couldn't elide it. Just emit
927	// the conditional branch.
928	llvm::BasicBlock *ThenBlock = createBasicBlock(name: "if.then");
929	llvm::BasicBlock *ContBlock = createBasicBlock(name: "if.end");
930	llvm::BasicBlock *ElseBlock = ContBlock;
931	if (Else)
932	ElseBlock = createBasicBlock(name: "if.else");
933
934	// Prefer the PGO based weights over the likelihood attribute.
935	// When the build isn't optimized the metadata isn't used, so don't generate
936	// it.
937	// Also, differentiate between disabled PGO and a never executed branch with
938	// PGO. Assuming PGO is in use:
939	// - we want to ignore the [[likely]] attribute if the branch is never
940	// executed,
941	// - assuming the profile is poor, preserving the attribute may still be
942	// beneficial.
943	// As an approximation, preserve the attribute only if both the branch and the
944	// parent context were not executed.
945	Stmt::Likelihood LH = Stmt::LH_None;
946	uint64_t ThenCount = getProfileCount(S: S.getThen());
947	if (!ThenCount && !getCurrentProfileCount() &&
948	CGM.getCodeGenOpts().OptimizationLevel)
949	LH = Stmt::getLikelihood(Then: S.getThen(), Else);
950
951	// When measuring MC/DC, always fully evaluate the condition up front using
952	// EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
953	// executing the body of the if.then or if.else. This is useful for when
954	// there is a 'return' within the body, but this is particularly beneficial
955	// when one if-stmt is nested within another if-stmt so that all of the MC/DC
956	// updates are kept linear and consistent.
957	if (!CGM.getCodeGenOpts().MCDCCoverage) {
958	EmitBranchOnBoolExpr(Cond: S.getCond(), TrueBlock: ThenBlock, FalseBlock: ElseBlock, TrueCount: ThenCount, LH,
959	/*ConditionalOp=*/nullptr,
960	/*ConditionalDecl=*/S.getConditionVariable());
961	} else {
962	llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
963	MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
964	Builder.CreateCondBr(Cond: BoolCondVal, True: ThenBlock, False: ElseBlock);
965	}
966
967	// Emit the 'then' code.
968	EmitBlock(BB: ThenBlock);
969	if (llvm::EnableSingleByteCoverage)
970	incrementProfileCounter(S: S.getThen());
971	else
972	incrementProfileCounter(&S);
973	{
974	RunCleanupsScope ThenScope(*this);
975	EmitStmt(S: S.getThen());
976	}
977	EmitBranch(Target: ContBlock);
978
979	// Emit the 'else' code if present.
980	if (Else) {
981	{
982	// There is no need to emit line number for an unconditional branch.
983	auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
984	EmitBlock(BB: ElseBlock);
985	}
986	// When single byte coverage mode is enabled, add a counter to else block.
987	if (llvm::EnableSingleByteCoverage)
988	incrementProfileCounter(S: Else);
989	{
990	RunCleanupsScope ElseScope(*this);
991	EmitStmt(S: Else);
992	}
993	{
994	// There is no need to emit line number for an unconditional branch.
995	auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
996	EmitBranch(Target: ContBlock);
997	}
998	}
999
1000	// Emit the continuation block for code after the if.
1001	EmitBlock(BB: ContBlock, IsFinished: true);
1002
1003	// When single byte coverage mode is enabled, add a counter to continuation
1004	// block.
1005	if (llvm::EnableSingleByteCoverage)
1006	incrementProfileCounter(&S);
1007	}
1008
1009	bool CodeGenFunction::checkIfLoopMustProgress(const Expr *ControllingExpression,
1010	bool HasEmptyBody) {
1011	if (CGM.getCodeGenOpts().getFiniteLoops() ==
1012	CodeGenOptions::FiniteLoopsKind::Never)
1013	return false;
1014
1015	// Now apply rules for plain C (see 6.8.5.6 in C11).
1016	// Loops with constant conditions do not have to make progress in any C
1017	// version.
1018	// As an extension, we consisider loops whose constant expression
1019	// can be constant-folded.
1020	Expr::EvalResult Result;
1021	bool CondIsConstInt =
1022	!ControllingExpression ||
1023	(ControllingExpression->EvaluateAsInt(Result, Ctx: getContext()) &&
1024	Result.Val.isInt());
1025
1026	bool CondIsTrue = CondIsConstInt && (!ControllingExpression ||
1027	Result.Val.getInt().getBoolValue());
1028
1029	// Loops with non-constant conditions must make progress in C11 and later.
1030	if (getLangOpts().C11 && !CondIsConstInt)
1031	return true;
1032
1033	// [C++26][intro.progress] (DR)
1034	// The implementation may assume that any thread will eventually do one of the
1035	// following:
1036	// [...]
1037	// - continue execution of a trivial infinite loop ([stmt.iter.general]).
1038	if (CGM.getCodeGenOpts().getFiniteLoops() ==
1039	CodeGenOptions::FiniteLoopsKind::Always ||
1040	getLangOpts().CPlusPlus11) {
1041	if (HasEmptyBody && CondIsTrue) {
1042	CurFn->removeFnAttr(llvm::Attribute::MustProgress);
1043	return false;
1044	}
1045	return true;
1046	}
1047	return false;
1048	}
1049
1050	// [C++26][stmt.iter.general] (DR)
1051	// A trivially empty iteration statement is an iteration statement matching one
1052	// of the following forms:
1053	// - while ( expression ) ;
1054	// - while ( expression ) { }
1055	// - do ; while ( expression ) ;
1056	// - do { } while ( expression ) ;
1057	// - for ( init-statement expression(opt); ) ;
1058	// - for ( init-statement expression(opt); ) { }
1059	template <typename LoopStmt> static bool hasEmptyLoopBody(const LoopStmt &S) {
1060	if constexpr (std::is_same_v<LoopStmt, ForStmt>) {
1061	if (S.getInc())
1062	return false;
1063	}
1064	const Stmt *Body = S.getBody();
1065	if (!Body || isa<NullStmt>(Val: Body))
1066	return true;
1067	if (const CompoundStmt *Compound = dyn_cast<CompoundStmt>(Val: Body))
1068	return Compound->body_empty();
1069	return false;
1070	}
1071
1072	void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
1073	ArrayRef<const Attr *> WhileAttrs) {
1074	// Emit the header for the loop, which will also become
1075	// the continue target.
1076	JumpDest LoopHeader = getJumpDestInCurrentScope(Name: "while.cond");
1077	EmitBlock(BB: LoopHeader.getBlock());
1078
1079	if (CGM.shouldEmitConvergenceTokens())
1080	ConvergenceTokenStack.push_back(
1081	Elt: emitConvergenceLoopToken(BB: LoopHeader.getBlock()));
1082
1083	// Create an exit block for when the condition fails, which will
1084	// also become the break target.
1085	JumpDest LoopExit = getJumpDestInCurrentScope(Name: "while.end");
1086
1087	// Store the blocks to use for break and continue.
1088	BreakContinueStack.push_back(Elt: BreakContinue(LoopExit, LoopHeader));
1089
1090	// C++ [stmt.while]p2:
1091	// When the condition of a while statement is a declaration, the
1092	// scope of the variable that is declared extends from its point
1093	// of declaration (3.3.2) to the end of the while statement.
1094	// [...]
1095	// The object created in a condition is destroyed and created
1096	// with each iteration of the loop.
1097	RunCleanupsScope ConditionScope(*this);
1098
1099	if (S.getConditionVariable())
1100	EmitDecl(*S.getConditionVariable());
1101
1102	// Evaluate the conditional in the while header. C99 6.8.5.1: The
1103	// evaluation of the controlling expression takes place before each
1104	// execution of the loop body.
1105	llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1106
1107	MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1108
1109	// while(1) is common, avoid extra exit blocks. Be sure
1110	// to correctly handle break/continue though.
1111	llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1112	bool EmitBoolCondBranch = !C || !C->isOne();
1113	const SourceRange &R = S.getSourceRange();
1114	LoopStack.push(Header: LoopHeader.getBlock(), Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(),
1115	Attrs: WhileAttrs, StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1116	EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1117	MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1118
1119	// When single byte coverage mode is enabled, add a counter to loop condition.
1120	if (llvm::EnableSingleByteCoverage)
1121	incrementProfileCounter(S.getCond());
1122
1123	// As long as the condition is true, go to the loop body.
1124	llvm::BasicBlock *LoopBody = createBasicBlock(name: "while.body");
1125	if (EmitBoolCondBranch) {
1126	llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1127	if (ConditionScope.requiresCleanups())
1128	ExitBlock = createBasicBlock(name: "while.exit");
1129	llvm::MDNode *Weights =
1130	createProfileWeightsForLoop(S.getCond(), getProfileCount(S: S.getBody()));
1131	if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1132	BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1133	Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1134	auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: LoopBody, False: ExitBlock, BranchWeights: Weights);
1135	// Key Instructions: Emit the condition and branch as separate source
1136	// location atoms otherwise we may omit a step onto the loop condition in
1137	// favour of the `while` keyword.
1138	// FIXME: We could have the branch as the backup location for the condition,
1139	// which would probably be a better experience. Explore this later.
1140	if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1141	addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1142	addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1143
1144	if (ExitBlock != LoopExit.getBlock()) {
1145	EmitBlock(BB: ExitBlock);
1146	EmitBranchThroughCleanup(Dest: LoopExit);
1147	}
1148	} else if (const Attr *A = Stmt::getLikelihoodAttr(S: S.getBody())) {
1149	CGM.getDiags().Report(A->getLocation(),
1150	diag::warn_attribute_has_no_effect_on_infinite_loop)
1151	<< A << A->getRange();
1152	CGM.getDiags().Report(
1153	S.getWhileLoc(),
1154	diag::note_attribute_has_no_effect_on_infinite_loop_here)
1155	<< SourceRange(S.getWhileLoc(), S.getRParenLoc());
1156	}
1157
1158	// Emit the loop body. We have to emit this in a cleanup scope
1159	// because it might be a singleton DeclStmt.
1160	{
1161	RunCleanupsScope BodyScope(*this);
1162	EmitBlock(BB: LoopBody);
1163	// When single byte coverage mode is enabled, add a counter to the body.
1164	if (llvm::EnableSingleByteCoverage)
1165	incrementProfileCounter(S: S.getBody());
1166	else
1167	incrementProfileCounter(&S);
1168	EmitStmt(S: S.getBody());
1169	}
1170
1171	BreakContinueStack.pop_back();
1172
1173	// Immediately force cleanup.
1174	ConditionScope.ForceCleanup();
1175
1176	EmitStopPoint(&S);
1177	// Branch to the loop header again.
1178	EmitBranch(Target: LoopHeader.getBlock());
1179
1180	LoopStack.pop();
1181
1182	// Emit the exit block.
1183	EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1184
1185	// The LoopHeader typically is just a branch if we skipped emitting
1186	// a branch, try to erase it.
1187	if (!EmitBoolCondBranch)
1188	SimplifyForwardingBlocks(BB: LoopHeader.getBlock());
1189
1190	// When single byte coverage mode is enabled, add a counter to continuation
1191	// block.
1192	if (llvm::EnableSingleByteCoverage)
1193	incrementProfileCounter(&S);
1194
1195	if (CGM.shouldEmitConvergenceTokens())
1196	ConvergenceTokenStack.pop_back();
1197	}
1198
1199	void CodeGenFunction::EmitDoStmt(const DoStmt &S,
1200	ArrayRef<const Attr *> DoAttrs) {
1201	JumpDest LoopExit = getJumpDestInCurrentScope(Name: "do.end");
1202	JumpDest LoopCond = getJumpDestInCurrentScope(Name: "do.cond");
1203
1204	uint64_t ParentCount = getCurrentProfileCount();
1205
1206	// Store the blocks to use for break and continue.
1207	BreakContinueStack.push_back(Elt: BreakContinue(LoopExit, LoopCond));
1208
1209	// Emit the body of the loop.
1210	llvm::BasicBlock *LoopBody = createBasicBlock(name: "do.body");
1211
1212	if (llvm::EnableSingleByteCoverage)
1213	EmitBlockWithFallThrough(BB: LoopBody, S: S.getBody());
1214	else
1215	EmitBlockWithFallThrough(BB: LoopBody, S: &S);
1216
1217	if (CGM.shouldEmitConvergenceTokens())
1218	ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: LoopBody));
1219
1220	{
1221	RunCleanupsScope BodyScope(*this);
1222	EmitStmt(S: S.getBody());
1223	}
1224
1225	EmitBlock(BB: LoopCond.getBlock());
1226	// When single byte coverage mode is enabled, add a counter to loop condition.
1227	if (llvm::EnableSingleByteCoverage)
1228	incrementProfileCounter(S.getCond());
1229
1230	// C99 6.8.5.2: "The evaluation of the controlling expression takes place
1231	// after each execution of the loop body."
1232
1233	// Evaluate the conditional in the while header.
1234	// C99 6.8.5p2/p4: The first substatement is executed if the expression
1235	// compares unequal to 0. The condition must be a scalar type.
1236	llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1237
1238	BreakContinueStack.pop_back();
1239
1240	// "do {} while (0)" is common in macros, avoid extra blocks. Be sure
1241	// to correctly handle break/continue though.
1242	llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(Val: BoolCondVal);
1243	bool EmitBoolCondBranch = !C || !C->isZero();
1244
1245	const SourceRange &R = S.getSourceRange();
1246	LoopStack.push(Header: LoopBody, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: DoAttrs,
1247	StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1248	EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1249	MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1250
1251	// As long as the condition is true, iterate the loop.
1252	if (EmitBoolCondBranch) {
1253	uint64_t BackedgeCount = getProfileCount(S: S.getBody()) - ParentCount;
1254	auto *I = Builder.CreateCondBr(
1255	Cond: BoolCondVal, True: LoopBody, False: LoopExit.getBlock(),
1256	BranchWeights: createProfileWeightsForLoop(S.getCond(), BackedgeCount));
1257
1258	// Key Instructions: Emit the condition and branch as separate source
1259	// location atoms otherwise we may omit a step onto the loop condition in
1260	// favour of the closing brace.
1261	// FIXME: We could have the branch as the backup location for the condition,
1262	// which would probably be a better experience (no jumping to the brace).
1263	if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1264	addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1265	addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1266	}
1267
1268	LoopStack.pop();
1269
1270	// Emit the exit block.
1271	EmitBlock(BB: LoopExit.getBlock());
1272
1273	// The DoCond block typically is just a branch if we skipped
1274	// emitting a branch, try to erase it.
1275	if (!EmitBoolCondBranch)
1276	SimplifyForwardingBlocks(BB: LoopCond.getBlock());
1277
1278	// When single byte coverage mode is enabled, add a counter to continuation
1279	// block.
1280	if (llvm::EnableSingleByteCoverage)
1281	incrementProfileCounter(S: &S);
1282
1283	if (CGM.shouldEmitConvergenceTokens())
1284	ConvergenceTokenStack.pop_back();
1285	}
1286
1287	void CodeGenFunction::EmitForStmt(const ForStmt &S,
1288	ArrayRef<const Attr *> ForAttrs) {
1289	JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1290
1291	LexicalScope ForScope(*this, S.getSourceRange());
1292
1293	// Evaluate the first part before the loop.
1294	if (S.getInit())
1295	EmitStmt(S: S.getInit());
1296
1297	// Start the loop with a block that tests the condition.
1298	// If there's an increment, the continue scope will be overwritten
1299	// later.
1300	JumpDest CondDest = getJumpDestInCurrentScope(Name: "for.cond");
1301	llvm::BasicBlock *CondBlock = CondDest.getBlock();
1302	EmitBlock(BB: CondBlock);
1303
1304	if (CGM.shouldEmitConvergenceTokens())
1305	ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1306
1307	const SourceRange &R = S.getSourceRange();
1308	LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1309	StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1310	EndLoc: SourceLocToDebugLoc(Location: R.getEnd()),
1311	MustProgress: checkIfLoopMustProgress(ControllingExpression: S.getCond(), HasEmptyBody: hasEmptyLoopBody(S)));
1312
1313	// Create a cleanup scope for the condition variable cleanups.
1314	LexicalScope ConditionScope(*this, S.getSourceRange());
1315
1316	// If the for loop doesn't have an increment we can just use the condition as
1317	// the continue block. Otherwise, if there is no condition variable, we can
1318	// form the continue block now. If there is a condition variable, we can't
1319	// form the continue block until after we've emitted the condition, because
1320	// the condition is in scope in the increment, but Sema's jump diagnostics
1321	// ensure that there are no continues from the condition variable that jump
1322	// to the loop increment.
1323	JumpDest Continue;
1324	if (!S.getInc())
1325	Continue = CondDest;
1326	else if (!S.getConditionVariable())
1327	Continue = getJumpDestInCurrentScope(Name: "for.inc");
1328	BreakContinueStack.push_back(Elt: BreakContinue(LoopExit, Continue));
1329
1330	if (S.getCond()) {
1331	// If the for statement has a condition scope, emit the local variable
1332	// declaration.
1333	if (S.getConditionVariable()) {
1334	EmitDecl(*S.getConditionVariable());
1335
1336	// We have entered the condition variable's scope, so we're now able to
1337	// jump to the continue block.
1338	Continue = S.getInc() ? getJumpDestInCurrentScope(Name: "for.inc") : CondDest;
1339	BreakContinueStack.back().ContinueBlock = Continue;
1340	}
1341
1342	// When single byte coverage mode is enabled, add a counter to loop
1343	// condition.
1344	if (llvm::EnableSingleByteCoverage)
1345	incrementProfileCounter(S.getCond());
1346
1347	llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1348	// If there are any cleanups between here and the loop-exit scope,
1349	// create a block to stage a loop exit along.
1350	if (ForScope.requiresCleanups())
1351	ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1352
1353	// As long as the condition is true, iterate the loop.
1354	llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1355
1356	// C99 6.8.5p2/p4: The first substatement is executed if the expression
1357	// compares unequal to 0. The condition must be a scalar type.
1358	llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1359
1360	MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
1361
1362	llvm::MDNode *Weights =
1363	createProfileWeightsForLoop(S.getCond(), getProfileCount(S: S.getBody()));
1364	if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1365	BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1366	Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1367
1368	auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1369	// Key Instructions: Emit the condition and branch as separate atoms to
1370	// match existing loop stepping behaviour. FIXME: We could have the branch
1371	// as the backup location for the condition, which would probably be a
1372	// better experience (no jumping to the brace).
1373	if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1374	addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1375	addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1376
1377	if (ExitBlock != LoopExit.getBlock()) {
1378	EmitBlock(BB: ExitBlock);
1379	EmitBranchThroughCleanup(Dest: LoopExit);
1380	}
1381
1382	EmitBlock(BB: ForBody);
1383	} else {
1384	// Treat it as a non-zero constant. Don't even create a new block for the
1385	// body, just fall into it.
1386	}
1387
1388	// When single byte coverage mode is enabled, add a counter to the body.
1389	if (llvm::EnableSingleByteCoverage)
1390	incrementProfileCounter(S: S.getBody());
1391	else
1392	incrementProfileCounter(S: &S);
1393	{
1394	// Create a separate cleanup scope for the body, in case it is not
1395	// a compound statement.
1396	RunCleanupsScope BodyScope(*this);
1397	EmitStmt(S: S.getBody());
1398	}
1399
1400	// The last block in the loop's body (which unconditionally branches to the
1401	// `inc` block if there is one).
1402	auto *FinalBodyBB = Builder.GetInsertBlock();
1403
1404	// If there is an increment, emit it next.
1405	if (S.getInc()) {
1406	EmitBlock(BB: Continue.getBlock());
1407	EmitStmt(S.getInc());
1408	if (llvm::EnableSingleByteCoverage)
1409	incrementProfileCounter(S.getInc());
1410	}
1411
1412	BreakContinueStack.pop_back();
1413
1414	ConditionScope.ForceCleanup();
1415
1416	EmitStopPoint(S: &S);
1417	EmitBranch(Target: CondBlock);
1418
1419	ForScope.ForceCleanup();
1420
1421	LoopStack.pop();
1422
1423	// Emit the fall-through block.
1424	EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1425
1426	// When single byte coverage mode is enabled, add a counter to continuation
1427	// block.
1428	if (llvm::EnableSingleByteCoverage)
1429	incrementProfileCounter(S: &S);
1430
1431	if (CGM.shouldEmitConvergenceTokens())
1432	ConvergenceTokenStack.pop_back();
1433
1434	if (FinalBodyBB) {
1435	// Key Instructions: We want the for closing brace to be step-able on to
1436	// match existing behaviour.
1437	addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1438	}
1439	}
1440
1441	void
1442	CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1443	ArrayRef<const Attr *> ForAttrs) {
1444	JumpDest LoopExit = getJumpDestInCurrentScope(Name: "for.end");
1445
1446	LexicalScope ForScope(*this, S.getSourceRange());
1447
1448	// Evaluate the first pieces before the loop.
1449	if (S.getInit())
1450	EmitStmt(S: S.getInit());
1451	EmitStmt(S: S.getRangeStmt());
1452	EmitStmt(S: S.getBeginStmt());
1453	EmitStmt(S: S.getEndStmt());
1454
1455	// Start the loop with a block that tests the condition.
1456	// If there's an increment, the continue scope will be overwritten
1457	// later.
1458	llvm::BasicBlock *CondBlock = createBasicBlock(name: "for.cond");
1459	EmitBlock(BB: CondBlock);
1460
1461	if (CGM.shouldEmitConvergenceTokens())
1462	ConvergenceTokenStack.push_back(Elt: emitConvergenceLoopToken(BB: CondBlock));
1463
1464	const SourceRange &R = S.getSourceRange();
1465	LoopStack.push(Header: CondBlock, Ctx&: CGM.getContext(), CGOpts: CGM.getCodeGenOpts(), Attrs: ForAttrs,
1466	StartLoc: SourceLocToDebugLoc(Location: R.getBegin()),
1467	EndLoc: SourceLocToDebugLoc(Location: R.getEnd()));
1468
1469	// If there are any cleanups between here and the loop-exit scope,
1470	// create a block to stage a loop exit along.
1471	llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1472	if (ForScope.requiresCleanups())
1473	ExitBlock = createBasicBlock(name: "for.cond.cleanup");
1474
1475	// The loop body, consisting of the specified body and the loop variable.
1476	llvm::BasicBlock *ForBody = createBasicBlock(name: "for.body");
1477
1478	// The body is executed if the expression, contextually converted
1479	// to bool, is true.
1480	llvm::Value *BoolCondVal = EvaluateExprAsBool(E: S.getCond());
1481	llvm::MDNode *Weights =
1482	createProfileWeightsForLoop(S.getCond(), getProfileCount(S: S.getBody()));
1483	if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1484	BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1485	Cond: BoolCondVal, LH: Stmt::getLikelihood(S: S.getBody()));
1486	auto *I = Builder.CreateCondBr(Cond: BoolCondVal, True: ForBody, False: ExitBlock, BranchWeights: Weights);
1487	// Key Instructions: Emit the condition and branch as separate atoms to
1488	// match existing loop stepping behaviour. FIXME: We could have the branch as
1489	// the backup location for the condition, which would probably be a better
1490	// experience.
1491	if (auto *CondI = dyn_cast<llvm::Instruction>(Val: BoolCondVal))
1492	addInstToNewSourceAtom(KeyInstruction: CondI, Backup: nullptr);
1493	addInstToNewSourceAtom(KeyInstruction: I, Backup: nullptr);
1494
1495	if (ExitBlock != LoopExit.getBlock()) {
1496	EmitBlock(BB: ExitBlock);
1497	EmitBranchThroughCleanup(Dest: LoopExit);
1498	}
1499
1500	EmitBlock(BB: ForBody);
1501	if (llvm::EnableSingleByteCoverage)
1502	incrementProfileCounter(S: S.getBody());
1503	else
1504	incrementProfileCounter(S: &S);
1505
1506	// Create a block for the increment. In case of a 'continue', we jump there.
1507	JumpDest Continue = getJumpDestInCurrentScope(Name: "for.inc");
1508
1509	// Store the blocks to use for break and continue.
1510	BreakContinueStack.push_back(Elt: BreakContinue(LoopExit, Continue));
1511
1512	{
1513	// Create a separate cleanup scope for the loop variable and body.
1514	LexicalScope BodyScope(*this, S.getSourceRange());
1515	EmitStmt(S: S.getLoopVarStmt());
1516	EmitStmt(S: S.getBody());
1517	}
1518	// The last block in the loop's body (which unconditionally branches to the
1519	// `inc` block if there is one).
1520	auto *FinalBodyBB = Builder.GetInsertBlock();
1521
1522	EmitStopPoint(S: &S);
1523	// If there is an increment, emit it next.
1524	EmitBlock(BB: Continue.getBlock());
1525	EmitStmt(S.getInc());
1526
1527	BreakContinueStack.pop_back();
1528
1529	EmitBranch(Target: CondBlock);
1530
1531	ForScope.ForceCleanup();
1532
1533	LoopStack.pop();
1534
1535	// Emit the fall-through block.
1536	EmitBlock(BB: LoopExit.getBlock(), IsFinished: true);
1537
1538	// When single byte coverage mode is enabled, add a counter to continuation
1539	// block.
1540	if (llvm::EnableSingleByteCoverage)
1541	incrementProfileCounter(S: &S);
1542
1543	if (CGM.shouldEmitConvergenceTokens())
1544	ConvergenceTokenStack.pop_back();
1545
1546	if (FinalBodyBB) {
1547	// We want the for closing brace to be step-able on to match existing
1548	// behaviour.
1549	addInstToNewSourceAtom(KeyInstruction: FinalBodyBB->getTerminator(), Backup: nullptr);
1550	}
1551	}
1552
1553	void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1554	if (RV.isScalar()) {
1555	Builder.CreateStore(Val: RV.getScalarVal(), Addr: ReturnValue);
1556	} else if (RV.isAggregate()) {
1557	LValue Dest = MakeAddrLValue(Addr: ReturnValue, T: Ty);
1558	LValue Src = MakeAddrLValue(Addr: RV.getAggregateAddress(), T: Ty);
1559	EmitAggregateCopy(Dest, Src, EltTy: Ty, MayOverlap: getOverlapForReturnValue());
1560	} else {
1561	EmitStoreOfComplex(V: RV.getComplexVal(), dest: MakeAddrLValue(Addr: ReturnValue, T: Ty),
1562	/*init*/ isInit: true);
1563	}
1564	EmitBranchThroughCleanup(Dest: ReturnBlock);
1565	}
1566
1567	namespace {
1568	// RAII struct used to save and restore a return statment's result expression.
1569	struct SaveRetExprRAII {
1570	SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1571	: OldRetExpr(CGF.RetExpr), CGF(CGF) {
1572	CGF.RetExpr = RetExpr;
1573	}
1574	~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1575	const Expr *OldRetExpr;
1576	CodeGenFunction &CGF;
1577	};
1578	} // namespace
1579
1580	/// Determine if the given call uses the swiftasync calling convention.
1581	static bool isSwiftAsyncCallee(const CallExpr *CE) {
1582	auto calleeQualType = CE->getCallee()->getType();
1583	const FunctionType *calleeType = nullptr;
1584	if (calleeQualType->isFunctionPointerType() ||
1585	calleeQualType->isFunctionReferenceType() ||
1586	calleeQualType->isBlockPointerType() ||
1587	calleeQualType->isMemberFunctionPointerType()) {
1588	calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();
1589	} else if (auto *ty = dyn_cast<FunctionType>(Val&: calleeQualType)) {
1590	calleeType = ty;
1591	} else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(Val: CE)) {
1592	if (auto methodDecl = CMCE->getMethodDecl()) {
1593	// getMethodDecl() doesn't handle member pointers at the moment.
1594	calleeType = methodDecl->getType()->castAs<FunctionType>();
1595	} else {
1596	return false;
1597	}
1598	} else {
1599	return false;
1600	}
1601	return calleeType->getCallConv() == CallingConv::CC_SwiftAsync;
1602	}
1603
1604	/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1605	/// if the function returns void, or may be missing one if the function returns
1606	/// non-void. Fun stuff :).
1607	void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1608	ApplyAtomGroup Grp(getDebugInfo());
1609	if (requiresReturnValueCheck()) {
1610	llvm::Constant *SLoc = EmitCheckSourceLocation(Loc: S.getBeginLoc());
1611	auto *SLocPtr =
1612	new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1613	llvm::GlobalVariable::PrivateLinkage, SLoc);
1614	SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1615	CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV: SLocPtr);
1616	assert(ReturnLocation.isValid() && "No valid return location");
1617	Builder.CreateStore(Val: SLocPtr, Addr: ReturnLocation);
1618	}
1619
1620	// Returning from an outlined SEH helper is UB, and we already warn on it.
1621	if (IsOutlinedSEHHelper) {
1622	Builder.CreateUnreachable();
1623	Builder.ClearInsertionPoint();
1624	}
1625
1626	// Emit the result value, even if unused, to evaluate the side effects.
1627	const Expr *RV = S.getRetValue();
1628
1629	// Record the result expression of the return statement. The recorded
1630	// expression is used to determine whether a block capture's lifetime should
1631	// end at the end of the full expression as opposed to the end of the scope
1632	// enclosing the block expression.
1633	//
1634	// This permits a small, easily-implemented exception to our over-conservative
1635	// rules about not jumping to statements following block literals with
1636	// non-trivial cleanups.
1637	SaveRetExprRAII SaveRetExpr(RV, *this);
1638
1639	RunCleanupsScope cleanupScope(*this);
1640	if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(Val: RV))
1641	RV = EWC->getSubExpr();
1642
1643	// If we're in a swiftasynccall function, and the return expression is a
1644	// call to a swiftasynccall function, mark the call as the musttail call.
1645	std::optional<llvm::SaveAndRestore<const CallExpr *>> SaveMustTail;
1646	if (RV && CurFnInfo &&
1647	CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync) {
1648	if (auto CE = dyn_cast<CallExpr>(Val: RV)) {
1649	if (isSwiftAsyncCallee(CE)) {
1650	SaveMustTail.emplace(args&: MustTailCall, args&: CE);
1651	}
1652	}
1653	}
1654
1655	// FIXME: Clean this up by using an LValue for ReturnTemp,
1656	// EmitStoreThroughLValue, and EmitAnyExpr.
1657	// Check if the NRVO candidate was not globalized in OpenMP mode.
1658	if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&
1659	S.getNRVOCandidate()->isNRVOVariable() &&
1660	(!getLangOpts().OpenMP ||
1661	!CGM.getOpenMPRuntime()
1662	.getAddressOfLocalVariable(CGF&: *this, VD: S.getNRVOCandidate())
1663	.isValid())) {
1664	// Apply the named return value optimization for this return statement,
1665	// which means doing nothing: the appropriate result has already been
1666	// constructed into the NRVO variable.
1667
1668	// If there is an NRVO flag for this variable, set it to 1 into indicate
1669	// that the cleanup code should not destroy the variable.
1670	if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1671	Builder.CreateFlagStore(Value: Builder.getTrue(), Addr: NRVOFlag);
1672	} else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1673	// Make sure not to return anything, but evaluate the expression
1674	// for side effects.
1675	if (RV) {
1676	EmitAnyExpr(E: RV);
1677	}
1678	} else if (!RV) {
1679	// Do nothing (return value is left uninitialized)
1680	} else if (FnRetTy->isReferenceType()) {
1681	// If this function returns a reference, take the address of the expression
1682	// rather than the value.
1683	RValue Result = EmitReferenceBindingToExpr(E: RV);
1684	auto *I = Builder.CreateStore(Val: Result.getScalarVal(), Addr: ReturnValue);
1685	addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1686	} else {
1687	switch (getEvaluationKind(T: RV->getType())) {
1688	case TEK_Scalar: {
1689	llvm::Value *Ret = EmitScalarExpr(E: RV);
1690	if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1691	EmitStoreOfScalar(value: Ret, lvalue: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1692	/*isInit*/ true);
1693	} else {
1694	auto *I = Builder.CreateStore(Val: Ret, Addr: ReturnValue);
1695	addInstToCurrentSourceAtom(KeyInstruction: I, Backup: I->getValueOperand());
1696	}
1697	break;
1698	}
1699	case TEK_Complex:
1700	EmitComplexExprIntoLValue(E: RV, dest: MakeAddrLValue(Addr: ReturnValue, T: RV->getType()),
1701	/*isInit*/ true);
1702	break;
1703	case TEK_Aggregate:
1704	EmitAggExpr(E: RV, AS: AggValueSlot::forAddr(
1705	addr: ReturnValue, quals: Qualifiers(),
1706	isDestructed: AggValueSlot::IsDestructed,
1707	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
1708	isAliased: AggValueSlot::IsNotAliased,
1709	mayOverlap: getOverlapForReturnValue()));
1710	break;
1711	}
1712	}
1713
1714	++NumReturnExprs;
1715	if (!RV || RV->isEvaluatable(Ctx: getContext()))
1716	++NumSimpleReturnExprs;
1717
1718	cleanupScope.ForceCleanup();
1719	EmitBranchThroughCleanup(Dest: ReturnBlock);
1720	}
1721
1722	void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1723	// As long as debug info is modeled with instructions, we have to ensure we
1724	// have a place to insert here and write the stop point here.
1725	if (HaveInsertPoint())
1726	EmitStopPoint(S: &S);
1727
1728	for (const auto *I : S.decls())
1729	EmitDecl(D: *I, /*EvaluateConditionDecl=*/true);
1730	}
1731
1732	void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1733	assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1734
1735	// If this code is reachable then emit a stop point (if generating
1736	// debug info). We have to do this ourselves because we are on the
1737	// "simple" statement path.
1738	if (HaveInsertPoint())
1739	EmitStopPoint(S: &S);
1740
1741	ApplyAtomGroup Grp(getDebugInfo());
1742	EmitBranchThroughCleanup(Dest: BreakContinueStack.back().BreakBlock);
1743	}
1744
1745	void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1746	assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1747
1748	// If this code is reachable then emit a stop point (if generating
1749	// debug info). We have to do this ourselves because we are on the
1750	// "simple" statement path.
1751	if (HaveInsertPoint())
1752	EmitStopPoint(S: &S);
1753
1754	ApplyAtomGroup Grp(getDebugInfo());
1755	EmitBranchThroughCleanup(Dest: BreakContinueStack.back().ContinueBlock);
1756	}
1757
1758	/// EmitCaseStmtRange - If case statement range is not too big then
1759	/// add multiple cases to switch instruction, one for each value within
1760	/// the range. If range is too big then emit "if" condition check.
1761	void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1762	ArrayRef<const Attr *> Attrs) {
1763	assert(S.getRHS() && "Expected RHS value in CaseStmt");
1764
1765	llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(Ctx: getContext());
1766	llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(Ctx: getContext());
1767
1768	// Emit the code for this case. We do this first to make sure it is
1769	// properly chained from our predecessor before generating the
1770	// switch machinery to enter this block.
1771	llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1772	EmitBlockWithFallThrough(CaseDest, &S);
1773	EmitStmt(S: S.getSubStmt());
1774
1775	// If range is empty, do nothing.
1776	if (LHS.isSigned() ? RHS.slt(RHS: LHS) : RHS.ult(RHS: LHS))
1777	return;
1778
1779	Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1780	llvm::APInt Range = RHS - LHS;
1781	// FIXME: parameters such as this should not be hardcoded.
1782	if (Range.ult(RHS: llvm::APInt(Range.getBitWidth(), 64))) {
1783	// Range is small enough to add multiple switch instruction cases.
1784	uint64_t Total = getProfileCount(&S);
1785	unsigned NCases = Range.getZExtValue() + 1;
1786	// We only have one region counter for the entire set of cases here, so we
1787	// need to divide the weights evenly between the generated cases, ensuring
1788	// that the total weight is preserved. E.g., a weight of 5 over three cases
1789	// will be distributed as weights of 2, 2, and 1.
1790	uint64_t Weight = Total / NCases, Rem = Total % NCases;
1791	for (unsigned I = 0; I != NCases; ++I) {
1792	if (SwitchWeights)
1793	SwitchWeights->push_back(Elt: Weight + (Rem ? 1 : 0));
1794	else if (SwitchLikelihood)
1795	SwitchLikelihood->push_back(Elt: LH);
1796
1797	if (Rem)
1798	Rem--;
1799	SwitchInsn->addCase(OnVal: Builder.getInt(AI: LHS), Dest: CaseDest);
1800	++LHS;
1801	}
1802	return;
1803	}
1804
1805	// The range is too big. Emit "if" condition into a new block,
1806	// making sure to save and restore the current insertion point.
1807	llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1808
1809	// Push this test onto the chain of range checks (which terminates
1810	// in the default basic block). The switch's default will be changed
1811	// to the top of this chain after switch emission is complete.
1812	llvm::BasicBlock *FalseDest = CaseRangeBlock;
1813	CaseRangeBlock = createBasicBlock(name: "sw.caserange");
1814
1815	CurFn->insert(Position: CurFn->end(), BB: CaseRangeBlock);
1816	Builder.SetInsertPoint(CaseRangeBlock);
1817
1818	// Emit range check.
1819	llvm::Value *Diff =
1820	Builder.CreateSub(LHS: SwitchInsn->getCondition(), RHS: Builder.getInt(AI: LHS));
1821	llvm::Value *Cond =
1822	Builder.CreateICmpULE(LHS: Diff, RHS: Builder.getInt(AI: Range), Name: "inbounds");
1823
1824	llvm::MDNode *Weights = nullptr;
1825	if (SwitchWeights) {
1826	uint64_t ThisCount = getProfileCount(&S);
1827	uint64_t DefaultCount = (*SwitchWeights)[0];
1828	Weights = createProfileWeights(TrueCount: ThisCount, FalseCount: DefaultCount);
1829
1830	// Since we're chaining the switch default through each large case range, we
1831	// need to update the weight for the default, ie, the first case, to include
1832	// this case.
1833	(*SwitchWeights)[0] += ThisCount;
1834	} else if (SwitchLikelihood)
1835	Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
1836
1837	Builder.CreateCondBr(Cond, True: CaseDest, False: FalseDest, BranchWeights: Weights);
1838
1839	// Restore the appropriate insertion point.
1840	if (RestoreBB)
1841	Builder.SetInsertPoint(RestoreBB);
1842	else
1843	Builder.ClearInsertionPoint();
1844	}
1845
1846	void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1847	ArrayRef<const Attr *> Attrs) {
1848	// If there is no enclosing switch instance that we're aware of, then this
1849	// case statement and its block can be elided. This situation only happens
1850	// when we've constant-folded the switch, are emitting the constant case,
1851	// and part of the constant case includes another case statement. For
1852	// instance: switch (4) { case 4: do { case 5: } while (1); }
1853	if (!SwitchInsn) {
1854	EmitStmt(S: S.getSubStmt());
1855	return;
1856	}
1857
1858	// Handle case ranges.
1859	if (S.getRHS()) {
1860	EmitCaseStmtRange(S, Attrs);
1861	return;
1862	}
1863
1864	llvm::ConstantInt *CaseVal =
1865	Builder.getInt(AI: S.getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1866
1867	// Emit debuginfo for the case value if it is an enum value.
1868	const ConstantExpr *CE;
1869	if (auto ICE = dyn_cast<ImplicitCastExpr>(Val: S.getLHS()))
1870	CE = dyn_cast<ConstantExpr>(ICE->getSubExpr());
1871	else
1872	CE = dyn_cast<ConstantExpr>(Val: S.getLHS());
1873	if (CE) {
1874	if (auto DE = dyn_cast<DeclRefExpr>(CE->getSubExpr()))
1875	if (CGDebugInfo *Dbg = getDebugInfo())
1876	if (CGM.getCodeGenOpts().hasReducedDebugInfo())
1877	Dbg->EmitGlobalVariable(DE->getDecl(),
1878	APValue(llvm::APSInt(CaseVal->getValue())));
1879	}
1880
1881	if (SwitchLikelihood)
1882	SwitchLikelihood->push_back(Elt: Stmt::getLikelihood(Attrs));
1883
1884	// If the body of the case is just a 'break', try to not emit an empty block.
1885	// If we're profiling or we're not optimizing, leave the block in for better
1886	// debug and coverage analysis.
1887	if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1888	CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1889	isa<BreakStmt>(Val: S.getSubStmt())) {
1890	JumpDest Block = BreakContinueStack.back().BreakBlock;
1891
1892	// Only do this optimization if there are no cleanups that need emitting.
1893	if (isObviouslyBranchWithoutCleanups(Dest: Block)) {
1894	if (SwitchWeights)
1895	SwitchWeights->push_back(Elt: getProfileCount(&S));
1896	SwitchInsn->addCase(OnVal: CaseVal, Dest: Block.getBlock());
1897
1898	// If there was a fallthrough into this case, make sure to redirect it to
1899	// the end of the switch as well.
1900	if (Builder.GetInsertBlock()) {
1901	Builder.CreateBr(Dest: Block.getBlock());
1902	Builder.ClearInsertionPoint();
1903	}
1904	return;
1905	}
1906	}
1907
1908	llvm::BasicBlock *CaseDest = createBasicBlock(name: "sw.bb");
1909	EmitBlockWithFallThrough(CaseDest, &S);
1910	if (SwitchWeights)
1911	SwitchWeights->push_back(Elt: getProfileCount(&S));
1912	SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1913
1914	// Recursively emitting the statement is acceptable, but is not wonderful for
1915	// code where we have many case statements nested together, i.e.:
1916	// case 1:
1917	// case 2:
1918	// case 3: etc.
1919	// Handling this recursively will create a new block for each case statement
1920	// that falls through to the next case which is IR intensive. It also causes
1921	// deep recursion which can run into stack depth limitations. Handle
1922	// sequential non-range case statements specially.
1923	//
1924	// TODO When the next case has a likelihood attribute the code returns to the
1925	// recursive algorithm. Maybe improve this case if it becomes common practice
1926	// to use a lot of attributes.
1927	const CaseStmt *CurCase = &S;
1928	const CaseStmt *NextCase = dyn_cast<CaseStmt>(Val: S.getSubStmt());
1929
1930	// Otherwise, iteratively add consecutive cases to this switch stmt.
1931	while (NextCase && NextCase->getRHS() == nullptr) {
1932	CurCase = NextCase;
1933	llvm::ConstantInt *CaseVal =
1934	Builder.getInt(AI: CurCase->getLHS()->EvaluateKnownConstInt(Ctx: getContext()));
1935
1936	if (SwitchWeights)
1937	SwitchWeights->push_back(Elt: getProfileCount(NextCase));
1938	if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1939	CaseDest = createBasicBlock(name: "sw.bb");
1940	EmitBlockWithFallThrough(CaseDest, CurCase);
1941	}
1942	// Since this loop is only executed when the CaseStmt has no attributes
1943	// use a hard-coded value.
1944	if (SwitchLikelihood)
1945	SwitchLikelihood->push_back(Elt: Stmt::LH_None);
1946
1947	SwitchInsn->addCase(OnVal: CaseVal, Dest: CaseDest);
1948	NextCase = dyn_cast<CaseStmt>(Val: CurCase->getSubStmt());
1949	}
1950
1951	// Generate a stop point for debug info if the case statement is
1952	// followed by a default statement. A fallthrough case before a
1953	// default case gets its own branch target.
1954	if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)
1955	EmitStopPoint(CurCase);
1956
1957	// Normal default recursion for non-cases.
1958	EmitStmt(S: CurCase->getSubStmt());
1959	}
1960
1961	void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1962	ArrayRef<const Attr *> Attrs) {
1963	// If there is no enclosing switch instance that we're aware of, then this
1964	// default statement can be elided. This situation only happens when we've
1965	// constant-folded the switch.
1966	if (!SwitchInsn) {
1967	EmitStmt(S: S.getSubStmt());
1968	return;
1969	}
1970
1971	llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1972	assert(DefaultBlock->empty() &&
1973	"EmitDefaultStmt: Default block already defined?");
1974
1975	if (SwitchLikelihood)
1976	SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
1977
1978	EmitBlockWithFallThrough(BB: DefaultBlock, S: &S);
1979
1980	EmitStmt(S: S.getSubStmt());
1981	}
1982
1983	/// CollectStatementsForCase - Given the body of a 'switch' statement and a
1984	/// constant value that is being switched on, see if we can dead code eliminate
1985	/// the body of the switch to a simple series of statements to emit. Basically,
1986	/// on a switch (5) we want to find these statements:
1987	/// case 5:
1988	/// printf(...); <--
1989	/// ++i; <--
1990	/// break;
1991	///
1992	/// and add them to the ResultStmts vector. If it is unsafe to do this
1993	/// transformation (for example, one of the elided statements contains a label
1994	/// that might be jumped to), return CSFC_Failure. If we handled it and 'S'
1995	/// should include statements after it (e.g. the printf() line is a substmt of
1996	/// the case) then return CSFC_FallThrough. If we handled it and found a break
1997	/// statement, then return CSFC_Success.
1998	///
1999	/// If Case is non-null, then we are looking for the specified case, checking
2000	/// that nothing we jump over contains labels. If Case is null, then we found
2001	/// the case and are looking for the break.
2002	///
2003	/// If the recursive walk actually finds our Case, then we set FoundCase to
2004	/// true.
2005	///
2006	enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
2007	static CSFC_Result CollectStatementsForCase(const Stmt *S,
2008	const SwitchCase *Case,
2009	bool &FoundCase,
2010	SmallVectorImpl<const Stmt*> &ResultStmts) {
2011	// If this is a null statement, just succeed.
2012	if (!S)
2013	return Case ? CSFC_Success : CSFC_FallThrough;
2014
2015	// If this is the switchcase (case 4: or default) that we're looking for, then
2016	// we're in business. Just add the substatement.
2017	if (const SwitchCase *SC = dyn_cast<SwitchCase>(Val: S)) {
2018	if (S == Case) {
2019	FoundCase = true;
2020	return CollectStatementsForCase(S: SC->getSubStmt(), Case: nullptr, FoundCase,
2021	ResultStmts);
2022	}
2023
2024	// Otherwise, this is some other case or default statement, just ignore it.
2025	return CollectStatementsForCase(S: SC->getSubStmt(), Case, FoundCase,
2026	ResultStmts);
2027	}
2028
2029	// If we are in the live part of the code and we found our break statement,
2030	// return a success!
2031	if (!Case && isa<BreakStmt>(Val: S))
2032	return CSFC_Success;
2033
2034	// If this is a switch statement, then it might contain the SwitchCase, the
2035	// break, or neither.
2036	if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(Val: S)) {
2037	// Handle this as two cases: we might be looking for the SwitchCase (if so
2038	// the skipped statements must be skippable) or we might already have it.
2039	CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
2040	bool StartedInLiveCode = FoundCase;
2041	unsigned StartSize = ResultStmts.size();
2042
2043	// If we've not found the case yet, scan through looking for it.
2044	if (Case) {
2045	// Keep track of whether we see a skipped declaration. The code could be
2046	// using the declaration even if it is skipped, so we can't optimize out
2047	// the decl if the kept statements might refer to it.
2048	bool HadSkippedDecl = false;
2049
2050	// If we're looking for the case, just see if we can skip each of the
2051	// substatements.
2052	for (; Case && I != E; ++I) {
2053	HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(S: *I);
2054
2055	switch (CollectStatementsForCase(S: *I, Case, FoundCase, ResultStmts)) {
2056	case CSFC_Failure: return CSFC_Failure;
2057	case CSFC_Success:
2058	// A successful result means that either 1) that the statement doesn't
2059	// have the case and is skippable, or 2) does contain the case value
2060	// and also contains the break to exit the switch. In the later case,
2061	// we just verify the rest of the statements are elidable.
2062	if (FoundCase) {
2063	// If we found the case and skipped declarations, we can't do the
2064	// optimization.
2065	if (HadSkippedDecl)
2066	return CSFC_Failure;
2067
2068	for (++I; I != E; ++I)
2069	if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2070	return CSFC_Failure;
2071	return CSFC_Success;
2072	}
2073	break;
2074	case CSFC_FallThrough:
2075	// If we have a fallthrough condition, then we must have found the
2076	// case started to include statements. Consider the rest of the
2077	// statements in the compound statement as candidates for inclusion.
2078	assert(FoundCase && "Didn't find case but returned fallthrough?");
2079	// We recursively found Case, so we're not looking for it anymore.
2080	Case = nullptr;
2081
2082	// If we found the case and skipped declarations, we can't do the
2083	// optimization.
2084	if (HadSkippedDecl)
2085	return CSFC_Failure;
2086	break;
2087	}
2088	}
2089
2090	if (!FoundCase)
2091	return CSFC_Success;
2092
2093	assert(!HadSkippedDecl && "fallthrough after skipping decl");
2094	}
2095
2096	// If we have statements in our range, then we know that the statements are
2097	// live and need to be added to the set of statements we're tracking.
2098	bool AnyDecls = false;
2099	for (; I != E; ++I) {
2100	AnyDecls |= CodeGenFunction::mightAddDeclToScope(S: *I);
2101
2102	switch (CollectStatementsForCase(S: *I, Case: nullptr, FoundCase, ResultStmts)) {
2103	case CSFC_Failure: return CSFC_Failure;
2104	case CSFC_FallThrough:
2105	// A fallthrough result means that the statement was simple and just
2106	// included in ResultStmt, keep adding them afterwards.
2107	break;
2108	case CSFC_Success:
2109	// A successful result means that we found the break statement and
2110	// stopped statement inclusion. We just ensure that any leftover stmts
2111	// are skippable and return success ourselves.
2112	for (++I; I != E; ++I)
2113	if (CodeGenFunction::ContainsLabel(S: *I, IgnoreCaseStmts: true))
2114	return CSFC_Failure;
2115	return CSFC_Success;
2116	}
2117	}
2118
2119	// If we're about to fall out of a scope without hitting a 'break;', we
2120	// can't perform the optimization if there were any decls in that scope
2121	// (we'd lose their end-of-lifetime).
2122	if (AnyDecls) {
2123	// If the entire compound statement was live, there's one more thing we
2124	// can try before giving up: emit the whole thing as a single statement.
2125	// We can do that unless the statement contains a 'break;'.
2126	// FIXME: Such a break must be at the end of a construct within this one.
2127	// We could emit this by just ignoring the BreakStmts entirely.
2128	if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
2129	ResultStmts.resize(N: StartSize);
2130	ResultStmts.push_back(Elt: S);
2131	} else {
2132	return CSFC_Failure;
2133	}
2134	}
2135
2136	return CSFC_FallThrough;
2137	}
2138
2139	// Okay, this is some other statement that we don't handle explicitly, like a
2140	// for statement or increment etc. If we are skipping over this statement,
2141	// just verify it doesn't have labels, which would make it invalid to elide.
2142	if (Case) {
2143	if (CodeGenFunction::ContainsLabel(S, IgnoreCaseStmts: true))
2144	return CSFC_Failure;
2145	return CSFC_Success;
2146	}
2147
2148	// Otherwise, we want to include this statement. Everything is cool with that
2149	// so long as it doesn't contain a break out of the switch we're in.
2150	if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
2151
2152	// Otherwise, everything is great. Include the statement and tell the caller
2153	// that we fall through and include the next statement as well.
2154	ResultStmts.push_back(Elt: S);
2155	return CSFC_FallThrough;
2156	}
2157
2158	/// FindCaseStatementsForValue - Find the case statement being jumped to and
2159	/// then invoke CollectStatementsForCase to find the list of statements to emit
2160	/// for a switch on constant. See the comment above CollectStatementsForCase
2161	/// for more details.
2162	static bool FindCaseStatementsForValue(const SwitchStmt &S,
2163	const llvm::APSInt &ConstantCondValue,
2164	SmallVectorImpl<const Stmt*> &ResultStmts,
2165	ASTContext &C,
2166	const SwitchCase *&ResultCase) {
2167	// First step, find the switch case that is being branched to. We can do this
2168	// efficiently by scanning the SwitchCase list.
2169	const SwitchCase *Case = S.getSwitchCaseList();
2170	const DefaultStmt *DefaultCase = nullptr;
2171
2172	for (; Case; Case = Case->getNextSwitchCase()) {
2173	// It's either a default or case. Just remember the default statement in
2174	// case we're not jumping to any numbered cases.
2175	if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Val: Case)) {
2176	DefaultCase = DS;
2177	continue;
2178	}
2179
2180	// Check to see if this case is the one we're looking for.
2181	const CaseStmt *CS = cast<CaseStmt>(Val: Case);
2182	// Don't handle case ranges yet.
2183	if (CS->getRHS()) return false;
2184
2185	// If we found our case, remember it as 'case'.
2186	if (CS->getLHS()->EvaluateKnownConstInt(Ctx: C) == ConstantCondValue)
2187	break;
2188	}
2189
2190	// If we didn't find a matching case, we use a default if it exists, or we
2191	// elide the whole switch body!
2192	if (!Case) {
2193	// It is safe to elide the body of the switch if it doesn't contain labels
2194	// etc. If it is safe, return successfully with an empty ResultStmts list.
2195	if (!DefaultCase)
2196	return !CodeGenFunction::ContainsLabel(&S);
2197	Case = DefaultCase;
2198	}
2199
2200	// Ok, we know which case is being jumped to, try to collect all the
2201	// statements that follow it. This can fail for a variety of reasons. Also,
2202	// check to see that the recursive walk actually found our case statement.
2203	// Insane cases like this can fail to find it in the recursive walk since we
2204	// don't handle every stmt kind:
2205	// switch (4) {
2206	// while (1) {
2207	// case 4: ...
2208	bool FoundCase = false;
2209	ResultCase = Case;
2210	return CollectStatementsForCase(S: S.getBody(), Case, FoundCase,
2211	ResultStmts) != CSFC_Failure &&
2212	FoundCase;
2213	}
2214
2215	static std::optional<SmallVector<uint64_t, 16>>
2216	getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
2217	// Are there enough branches to weight them?
2218	if (Likelihoods.size() <= 1)
2219	return std::nullopt;
2220
2221	uint64_t NumUnlikely = 0;
2222	uint64_t NumNone = 0;
2223	uint64_t NumLikely = 0;
2224	for (const auto LH : Likelihoods) {
2225	switch (LH) {
2226	case Stmt::LH_Unlikely:
2227	++NumUnlikely;
2228	break;
2229	case Stmt::LH_None:
2230	++NumNone;
2231	break;
2232	case Stmt::LH_Likely:
2233	++NumLikely;
2234	break;
2235	}
2236	}
2237
2238	// Is there a likelihood attribute used?
2239	if (NumUnlikely == 0 && NumLikely == 0)
2240	return std::nullopt;
2241
2242	// When multiple cases share the same code they can be combined during
2243	// optimization. In that case the weights of the branch will be the sum of
2244	// the individual weights. Make sure the combined sum of all neutral cases
2245	// doesn't exceed the value of a single likely attribute.
2246	// The additions both avoid divisions by 0 and make sure the weights of None
2247	// don't exceed the weight of Likely.
2248	const uint64_t Likely = INT32_MAX / (NumLikely + 2);
2249	const uint64_t None = Likely / (NumNone + 1);
2250	const uint64_t Unlikely = 0;
2251
2252	SmallVector<uint64_t, 16> Result;
2253	Result.reserve(N: Likelihoods.size());
2254	for (const auto LH : Likelihoods) {
2255	switch (LH) {
2256	case Stmt::LH_Unlikely:
2257	Result.push_back(Elt: Unlikely);
2258	break;
2259	case Stmt::LH_None:
2260	Result.push_back(Elt: None);
2261	break;
2262	case Stmt::LH_Likely:
2263	Result.push_back(Elt: Likely);
2264	break;
2265	}
2266	}
2267
2268	return Result;
2269	}
2270
2271	void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
2272	// Handle nested switch statements.
2273	llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
2274	SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
2275	SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
2276	llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
2277
2278	// See if we can constant fold the condition of the switch and therefore only
2279	// emit the live case statement (if any) of the switch.
2280	llvm::APSInt ConstantCondValue;
2281	if (ConstantFoldsToSimpleInteger(Cond: S.getCond(), Result&: ConstantCondValue)) {
2282	SmallVector<const Stmt*, 4> CaseStmts;
2283	const SwitchCase *Case = nullptr;
2284	if (FindCaseStatementsForValue(S, ConstantCondValue, ResultStmts&: CaseStmts,
2285	C&: getContext(), ResultCase&: Case)) {
2286	if (Case)
2287	incrementProfileCounter(S: Case);
2288	RunCleanupsScope ExecutedScope(*this);
2289
2290	if (S.getInit())
2291	EmitStmt(S: S.getInit());
2292
2293	// Emit the condition variable if needed inside the entire cleanup scope
2294	// used by this special case for constant folded switches.
2295	if (S.getConditionVariable())
2296	EmitDecl(*S.getConditionVariable(), /*EvaluateConditionDecl=*/true);
2297
2298	// At this point, we are no longer "within" a switch instance, so
2299	// we can temporarily enforce this to ensure that any embedded case
2300	// statements are not emitted.
2301	SwitchInsn = nullptr;
2302
2303	// Okay, we can dead code eliminate everything except this case. Emit the
2304	// specified series of statements and we're good.
2305	for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i)
2306	EmitStmt(S: CaseStmts[i]);
2307	incrementProfileCounter(&S);
2308	PGO->markStmtMaybeUsed(S: S.getBody());
2309
2310	// Now we want to restore the saved switch instance so that nested
2311	// switches continue to function properly
2312	SwitchInsn = SavedSwitchInsn;
2313
2314	return;
2315	}
2316	}
2317
2318	JumpDest SwitchExit = getJumpDestInCurrentScope(Name: "sw.epilog");
2319
2320	RunCleanupsScope ConditionScope(*this);
2321
2322	if (S.getInit())
2323	EmitStmt(S: S.getInit());
2324
2325	if (S.getConditionVariable())
2326	EmitDecl(*S.getConditionVariable());
2327	llvm::Value *CondV = EmitScalarExpr(E: S.getCond());
2328	MaybeEmitDeferredVarDeclInit(var: S.getConditionVariable());
2329
2330	// Create basic block to hold stuff that comes after switch
2331	// statement. We also need to create a default block now so that
2332	// explicit case ranges tests can have a place to jump to on
2333	// failure.
2334	llvm::BasicBlock *DefaultBlock = createBasicBlock(name: "sw.default");
2335	SwitchInsn = Builder.CreateSwitch(V: CondV, Dest: DefaultBlock);
2336	addInstToNewSourceAtom(KeyInstruction: SwitchInsn, Backup: CondV);
2337
2338	if (HLSLControlFlowAttr != HLSLControlFlowHintAttr::SpellingNotCalculated) {
2339	llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2340	llvm::ConstantInt *BranchHintConstant =
2341	HLSLControlFlowAttr ==
2342	HLSLControlFlowHintAttr::Spelling::Microsoft_branch
2343	? llvm::ConstantInt::get(CGM.Int32Ty, 1)
2344	: llvm::ConstantInt::get(CGM.Int32Ty, 2);
2345	llvm::Metadata *Vals[] = {MDHelper.createString(Str: "hlsl.controlflow.hint"),
2346	MDHelper.createConstant(C: BranchHintConstant)};
2347	SwitchInsn->setMetadata(Kind: "hlsl.controlflow.hint",
2348	Node: llvm::MDNode::get(Context&: CGM.getLLVMContext(), MDs: Vals));
2349	}
2350
2351	if (PGO->haveRegionCounts()) {
2352	// Walk the SwitchCase list to find how many there are.
2353	uint64_t DefaultCount = 0;
2354	unsigned NumCases = 0;
2355	for (const SwitchCase *Case = S.getSwitchCaseList();
2356	Case;
2357	Case = Case->getNextSwitchCase()) {
2358	if (isa<DefaultStmt>(Val: Case))
2359	DefaultCount = getProfileCount(S: Case);
2360	NumCases += 1;
2361	}
2362	SwitchWeights = new SmallVector<uint64_t, 16>();
2363	SwitchWeights->reserve(N: NumCases);
2364	// The default needs to be first. We store the edge count, so we already
2365	// know the right weight.
2366	SwitchWeights->push_back(Elt: DefaultCount);
2367	} else if (CGM.getCodeGenOpts().OptimizationLevel) {
2368	SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
2369	// Initialize the default case.
2370	SwitchLikelihood->push_back(Elt: Stmt::LH_None);
2371	}
2372
2373	CaseRangeBlock = DefaultBlock;
2374
2375	// Clear the insertion point to indicate we are in unreachable code.
2376	Builder.ClearInsertionPoint();
2377
2378	// All break statements jump to NextBlock. If BreakContinueStack is non-empty
2379	// then reuse last ContinueBlock.
2380	JumpDest OuterContinue;
2381	if (!BreakContinueStack.empty())
2382	OuterContinue = BreakContinueStack.back().ContinueBlock;
2383
2384	BreakContinueStack.push_back(Elt: BreakContinue(SwitchExit, OuterContinue));
2385
2386	// Emit switch body.
2387	EmitStmt(S: S.getBody());
2388
2389	BreakContinueStack.pop_back();
2390
2391	// Update the default block in case explicit case range tests have
2392	// been chained on top.
2393	SwitchInsn->setDefaultDest(CaseRangeBlock);
2394
2395	// If a default was never emitted:
2396	if (!DefaultBlock->getParent()) {
2397	// If we have cleanups, emit the default block so that there's a
2398	// place to jump through the cleanups from.
2399	if (ConditionScope.requiresCleanups()) {
2400	EmitBlock(BB: DefaultBlock);
2401
2402	// Otherwise, just forward the default block to the switch end.
2403	} else {
2404	DefaultBlock->replaceAllUsesWith(V: SwitchExit.getBlock());
2405	delete DefaultBlock;
2406	}
2407	}
2408
2409	ConditionScope.ForceCleanup();
2410
2411	// Emit continuation.
2412	EmitBlock(BB: SwitchExit.getBlock(), IsFinished: true);
2413	incrementProfileCounter(&S);
2414
2415	// If the switch has a condition wrapped by __builtin_unpredictable,
2416	// create metadata that specifies that the switch is unpredictable.
2417	// Don't bother if not optimizing because that metadata would not be used.
2418	auto *Call = dyn_cast<CallExpr>(Val: S.getCond());
2419	if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2420	auto *FD = dyn_cast_or_null<FunctionDecl>(Val: Call->getCalleeDecl());
2421	if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2422	llvm::MDBuilder MDHelper(getLLVMContext());
2423	SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_unpredictable,
2424	Node: MDHelper.createUnpredictable());
2425	}
2426	}
2427
2428	if (SwitchWeights) {
2429	assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
2430	"switch weights do not match switch cases");
2431	// If there's only one jump destination there's no sense weighting it.
2432	if (SwitchWeights->size() > 1)
2433	SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2434	Node: createProfileWeights(Weights: *SwitchWeights));
2435	delete SwitchWeights;
2436	} else if (SwitchLikelihood) {
2437	assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
2438	"switch likelihoods do not match switch cases");
2439	std::optional<SmallVector<uint64_t, 16>> LHW =
2440	getLikelihoodWeights(Likelihoods: *SwitchLikelihood);
2441	if (LHW) {
2442	llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2443	SwitchInsn->setMetadata(KindID: llvm::LLVMContext::MD_prof,
2444	Node: createProfileWeights(Weights: *LHW));
2445	}
2446	delete SwitchLikelihood;
2447	}
2448	SwitchInsn = SavedSwitchInsn;
2449	SwitchWeights = SavedSwitchWeights;
2450	SwitchLikelihood = SavedSwitchLikelihood;
2451	CaseRangeBlock = SavedCRBlock;
2452	}
2453
2454	static std::string
2455	SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
2456	SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=nullptr) {
2457	std::string Result;
2458
2459	while (*Constraint) {
2460	switch (*Constraint) {
2461	default:
2462	Result += Target.convertConstraint(Constraint);
2463	break;
2464	// Ignore these
2465	case '*':
2466	case '?':
2467	case '!':
2468	case '=': // Will see this and the following in mult-alt constraints.
2469	case '+':
2470	break;
2471	case '#': // Ignore the rest of the constraint alternative.
2472	while (Constraint[1] && Constraint[1] != ',')
2473	Constraint++;
2474	break;
2475	case '&':
2476	case '%':
2477	Result += *Constraint;
2478	while (Constraint[1] && Constraint[1] == *Constraint)
2479	Constraint++;
2480	break;
2481	case ',':
2482	Result += "|";
2483	break;
2484	case 'g':
2485	Result += "imr";
2486	break;
2487	case '[': {
2488	assert(OutCons &&
2489	"Must pass output names to constraints with a symbolic name");
2490	unsigned Index;
2491	bool result = Target.resolveSymbolicName(Name&: Constraint, OutputConstraints: *OutCons, Index);
2492	assert(result && "Could not resolve symbolic name"); (void)result;
2493	Result += llvm::utostr(X: Index);
2494	break;
2495	}
2496	}
2497
2498	Constraint++;
2499	}
2500
2501	return Result;
2502	}
2503
2504	/// AddVariableConstraints - Look at AsmExpr and if it is a variable declared
2505	/// as using a particular register add that as a constraint that will be used
2506	/// in this asm stmt.
2507	static std::string
2508	AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,
2509	const TargetInfo &Target, CodeGenModule &CGM,
2510	const AsmStmt &Stmt, const bool EarlyClobber,
2511	std::string *GCCReg = nullptr) {
2512	const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(Val: &AsmExpr);
2513	if (!AsmDeclRef)
2514	return Constraint;
2515	const ValueDecl &Value = *AsmDeclRef->getDecl();
2516	const VarDecl *Variable = dyn_cast<VarDecl>(Val: &Value);
2517	if (!Variable)
2518	return Constraint;
2519	if (Variable->getStorageClass() != SC_Register)
2520	return Constraint;
2521	AsmLabelAttr *Attr = Variable->getAttr<AsmLabelAttr>();
2522	if (!Attr)
2523	return Constraint;
2524	StringRef Register = Attr->getLabel();
2525	assert(Target.isValidGCCRegisterName(Register));
2526	// We're using validateOutputConstraint here because we only care if
2527	// this is a register constraint.
2528	TargetInfo::ConstraintInfo Info(Constraint, "");
2529	if (Target.validateOutputConstraint(Info) &&
2530	!Info.allowsRegister()) {
2531	CGM.ErrorUnsupported(S: &Stmt, Type: "__asm__");
2532	return Constraint;
2533	}
2534	// Canonicalize the register here before returning it.
2535	Register = Target.getNormalizedGCCRegisterName(Name: Register);
2536	if (GCCReg != nullptr)
2537	*GCCReg = Register.str();
2538	return (EarlyClobber ? "&{" : "{") + Register.str() + "}";
2539	}
2540
2541	std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
2542	const TargetInfo::ConstraintInfo &Info, LValue InputValue,
2543	QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {
2544	if (Info.allowsRegister() || !Info.allowsMemory()) {
2545	if (CodeGenFunction::hasScalarEvaluationKind(T: InputType))
2546	return {EmitLoadOfLValue(V: InputValue, Loc).getScalarVal(), nullptr};
2547
2548	llvm::Type *Ty = ConvertType(T: InputType);
2549	uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2550	if ((Size <= 64 && llvm::isPowerOf2_64(Value: Size)) ||
2551	getTargetHooks().isScalarizableAsmOperand(CGF&: *this, Ty)) {
2552	Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2553
2554	return {Builder.CreateLoad(Addr: InputValue.getAddress().withElementType(ElemTy: Ty)),
2555	nullptr};
2556	}
2557	}
2558
2559	Address Addr = InputValue.getAddress();
2560	ConstraintStr += '*';
2561	return {InputValue.getPointer(CGF&: *this), Addr.getElementType()};
2562	}
2563
2564	std::pair<llvm::Value *, llvm::Type *>
2565	CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
2566	const Expr *InputExpr,
2567	std::string &ConstraintStr) {
2568	// If this can't be a register or memory, i.e., has to be a constant
2569	// (immediate or symbolic), try to emit it as such.
2570	if (!Info.allowsRegister() && !Info.allowsMemory()) {
2571	if (Info.requiresImmediateConstant()) {
2572	Expr::EvalResult EVResult;
2573	InputExpr->EvaluateAsRValue(Result&: EVResult, Ctx: getContext(), InConstantContext: true);
2574
2575	llvm::APSInt IntResult;
2576	if (EVResult.Val.toIntegralConstant(Result&: IntResult, SrcTy: InputExpr->getType(),
2577	Ctx: getContext()))
2578	return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: IntResult), nullptr};
2579	}
2580
2581	Expr::EvalResult Result;
2582	if (InputExpr->EvaluateAsInt(Result, Ctx: getContext()))
2583	return {llvm::ConstantInt::get(Context&: getLLVMContext(), V: Result.Val.getInt()),
2584	nullptr};
2585	}
2586
2587	if (Info.allowsRegister() || !Info.allowsMemory())
2588	if (CodeGenFunction::hasScalarEvaluationKind(T: InputExpr->getType()))
2589	return {EmitScalarExpr(E: InputExpr), nullptr};
2590	if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2591	return {EmitScalarExpr(E: InputExpr), nullptr};
2592	InputExpr = InputExpr->IgnoreParenNoopCasts(Ctx: getContext());
2593	LValue Dest = EmitLValue(E: InputExpr);
2594	return EmitAsmInputLValue(Info, InputValue: Dest, InputType: InputExpr->getType(), ConstraintStr,
2595	Loc: InputExpr->getExprLoc());
2596	}
2597
2598	/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2599	/// asm call instruction. The !srcloc MDNode contains a list of constant
2600	/// integers which are the source locations of the start of each line in the
2601	/// asm.
2602	static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2603	CodeGenFunction &CGF) {
2604	SmallVector<llvm::Metadata *, 8> Locs;
2605	// Add the location of the first line to the MDNode.
2606	Locs.push_back(Elt: llvm::ConstantAsMetadata::get(C: llvm::ConstantInt::get(
2607	Ty: CGF.Int64Ty, V: Str->getBeginLoc().getRawEncoding())));
2608	StringRef StrVal = Str->getString();
2609	if (!StrVal.empty()) {
2610	const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2611	const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2612	unsigned StartToken = 0;
2613	unsigned ByteOffset = 0;
2614
2615	// Add the location of the start of each subsequent line of the asm to the
2616	// MDNode.
2617	for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
2618	if (StrVal[i] != '\n') continue;
2619	SourceLocation LineLoc = Str->getLocationOfByte(
2620	ByteNo: i + 1, SM, Features: LangOpts, Target: CGF.getTarget(), StartToken: &StartToken, StartTokenByteOffset: &ByteOffset);
2621	Locs.push_back(Elt: llvm::ConstantAsMetadata::get(
2622	C: llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: LineLoc.getRawEncoding())));
2623	}
2624	}
2625
2626	return llvm::MDNode::get(Context&: CGF.getLLVMContext(), MDs: Locs);
2627	}
2628
2629	static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2630	bool HasUnwindClobber, bool ReadOnly,
2631	bool ReadNone, bool NoMerge, bool NoConvergent,
2632	const AsmStmt &S,
2633	const std::vector<llvm::Type *> &ResultRegTypes,
2634	const std::vector<llvm::Type *> &ArgElemTypes,
2635	CodeGenFunction &CGF,
2636	std::vector<llvm::Value *> &RegResults) {
2637	if (!HasUnwindClobber)
2638	Result.addFnAttr(llvm::Attribute::NoUnwind);
2639
2640	if (NoMerge)
2641	Result.addFnAttr(llvm::Attribute::NoMerge);
2642	// Attach readnone and readonly attributes.
2643	if (!HasSideEffect) {
2644	if (ReadNone)
2645	Result.setDoesNotAccessMemory();
2646	else if (ReadOnly)
2647	Result.setOnlyReadsMemory();
2648	}
2649
2650	// Add elementtype attribute for indirect constraints.
2651	for (auto Pair : llvm::enumerate(First: ArgElemTypes)) {
2652	if (Pair.value()) {
2653	auto Attr = llvm::Attribute::get(
2654	CGF.getLLVMContext(), llvm::Attribute::ElementType, Pair.value());
2655	Result.addParamAttr(Pair.index(), Attr);
2656	}
2657	}
2658
2659	// Slap the source location of the inline asm into a !srcloc metadata on the
2660	// call.
2661	const StringLiteral *SL;
2662	if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(Val: &S);
2663	gccAsmStmt &&
2664	(SL = dyn_cast<StringLiteral>(Val: gccAsmStmt->getAsmStringExpr()))) {
2665	Result.setMetadata(Kind: "srcloc", Node: getAsmSrcLocInfo(Str: SL, CGF));
2666	} else {
2667	// At least put the line number on MS inline asm blobs and GCC asm constexpr
2668	// strings.
2669	llvm::Constant *Loc =
2670	llvm::ConstantInt::get(Ty: CGF.Int64Ty, V: S.getAsmLoc().getRawEncoding());
2671	Result.setMetadata(Kind: "srcloc",
2672	Node: llvm::MDNode::get(Context&: CGF.getLLVMContext(),
2673	MDs: llvm::ConstantAsMetadata::get(C: Loc)));
2674	}
2675
2676	if (!NoConvergent && CGF.getLangOpts().assumeFunctionsAreConvergent())
2677	// Conservatively, mark all inline asm blocks in CUDA or OpenCL as
2678	// convergent (meaning, they may call an intrinsically convergent op, such
2679	// as bar.sync, and so can't have certain optimizations applied around
2680	// them) unless it's explicitly marked 'noconvergent'.
2681	Result.addFnAttr(llvm::Attribute::Convergent);
2682	// Extract all of the register value results from the asm.
2683	if (ResultRegTypes.size() == 1) {
2684	RegResults.push_back(x: &Result);
2685	} else {
2686	for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
2687	llvm::Value *Tmp = CGF.Builder.CreateExtractValue(Agg: &Result, Idxs: i, Name: "asmresult");
2688	RegResults.push_back(x: Tmp);
2689	}
2690	}
2691	}
2692
2693	static void
2694	EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S,
2695	const llvm::ArrayRef<llvm::Value *> RegResults,
2696	const llvm::ArrayRef<llvm::Type *> ResultRegTypes,
2697	const llvm::ArrayRef<llvm::Type *> ResultTruncRegTypes,
2698	const llvm::ArrayRef<LValue> ResultRegDests,
2699	const llvm::ArrayRef<QualType> ResultRegQualTys,
2700	const llvm::BitVector &ResultTypeRequiresCast,
2701	const llvm::BitVector &ResultRegIsFlagReg) {
2702	CGBuilderTy &Builder = CGF.Builder;
2703	CodeGenModule &CGM = CGF.CGM;
2704	llvm::LLVMContext &CTX = CGF.getLLVMContext();
2705
2706	assert(RegResults.size() == ResultRegTypes.size());
2707	assert(RegResults.size() == ResultTruncRegTypes.size());
2708	assert(RegResults.size() == ResultRegDests.size());
2709	// ResultRegDests can be also populated by addReturnRegisterOutputs() above,
2710	// in which case its size may grow.
2711	assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
2712	assert(ResultRegIsFlagReg.size() <= ResultRegDests.size());
2713
2714	for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
2715	llvm::Value *Tmp = RegResults[i];
2716	llvm::Type *TruncTy = ResultTruncRegTypes[i];
2717
2718	if ((i < ResultRegIsFlagReg.size()) && ResultRegIsFlagReg[i]) {
2719	// Target must guarantee the Value `Tmp` here is lowered to a boolean
2720	// value.
2721	llvm::Constant *Two = llvm::ConstantInt::get(Ty: Tmp->getType(), V: 2);
2722	llvm::Value *IsBooleanValue =
2723	Builder.CreateCmp(Pred: llvm::CmpInst::ICMP_ULT, LHS: Tmp, RHS: Two);
2724	llvm::Function *FnAssume = CGM.getIntrinsic(llvm::Intrinsic::assume);
2725	Builder.CreateCall(Callee: FnAssume, Args: IsBooleanValue);
2726	}
2727
2728	// If the result type of the LLVM IR asm doesn't match the result type of
2729	// the expression, do the conversion.
2730	if (ResultRegTypes[i] != TruncTy) {
2731
2732	// Truncate the integer result to the right size, note that TruncTy can be
2733	// a pointer.
2734	if (TruncTy->isFloatingPointTy())
2735	Tmp = Builder.CreateFPTrunc(V: Tmp, DestTy: TruncTy);
2736	else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
2737	uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(Ty: TruncTy);
2738	Tmp = Builder.CreateTrunc(
2739	V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)ResSize));
2740	Tmp = Builder.CreateIntToPtr(V: Tmp, DestTy: TruncTy);
2741	} else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
2742	uint64_t TmpSize =
2743	CGM.getDataLayout().getTypeSizeInBits(Ty: Tmp->getType());
2744	Tmp = Builder.CreatePtrToInt(
2745	V: Tmp, DestTy: llvm::IntegerType::get(C&: CTX, NumBits: (unsigned)TmpSize));
2746	Tmp = Builder.CreateTrunc(V: Tmp, DestTy: TruncTy);
2747	} else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {
2748	Tmp = Builder.CreateZExtOrTrunc(V: Tmp, DestTy: TruncTy);
2749	} else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {
2750	Tmp = Builder.CreateBitCast(V: Tmp, DestTy: TruncTy);
2751	}
2752	}
2753
2754	LValue Dest = ResultRegDests[i];
2755	// ResultTypeRequiresCast elements correspond to the first
2756	// ResultTypeRequiresCast.size() elements of RegResults.
2757	if ((i < ResultTypeRequiresCast.size()) && ResultTypeRequiresCast[i]) {
2758	unsigned Size = CGF.getContext().getTypeSize(T: ResultRegQualTys[i]);
2759	Address A = Dest.getAddress().withElementType(ElemTy: ResultRegTypes[i]);
2760	if (CGF.getTargetHooks().isScalarizableAsmOperand(CGF, Ty: TruncTy)) {
2761	Builder.CreateStore(Val: Tmp, Addr: A);
2762	continue;
2763	}
2764
2765	QualType Ty =
2766	CGF.getContext().getIntTypeForBitwidth(DestWidth: Size, /*Signed=*/false);
2767	if (Ty.isNull()) {
2768	const Expr *OutExpr = S.getOutputExpr(i);
2769	CGM.getDiags().Report(OutExpr->getExprLoc(),
2770	diag::err_store_value_to_reg);
2771	return;
2772	}
2773	Dest = CGF.MakeAddrLValue(Addr: A, T: Ty);
2774	}
2775	CGF.EmitStoreThroughLValue(Src: RValue::get(V: Tmp), Dst: Dest);
2776	}
2777	}
2778
2779	static void EmitHipStdParUnsupportedAsm(CodeGenFunction *CGF,
2780	const AsmStmt &S) {
2781	constexpr auto Name = "__ASM__hipstdpar_unsupported";
2782
2783	std::string Asm;
2784	if (auto GCCAsm = dyn_cast<GCCAsmStmt>(Val: &S))
2785	Asm = GCCAsm->getAsmString();
2786
2787	auto &Ctx = CGF->CGM.getLLVMContext();
2788
2789	auto StrTy = llvm::ConstantDataArray::getString(Context&: Ctx, Initializer: Asm);
2790	auto FnTy = llvm::FunctionType::get(Result: llvm::Type::getVoidTy(C&: Ctx),
2791	Params: {StrTy->getType()}, isVarArg: false);
2792	auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, T: FnTy);
2793
2794	CGF->Builder.CreateCall(Callee: UBF, Args: {StrTy});
2795	}
2796
2797	void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2798	// Pop all cleanup blocks at the end of the asm statement.
2799	CodeGenFunction::RunCleanupsScope Cleanups(*this);
2800
2801	// Assemble the final asm string.
2802	std::string AsmString = S.generateAsmString(C: getContext());
2803
2804	// Get all the output and input constraints together.
2805	SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2806	SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2807
2808	bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2809	bool IsValidTargetAsm = true;
2810	for (unsigned i = 0, e = S.getNumOutputs(); i != e && IsValidTargetAsm; i++) {
2811	StringRef Name;
2812	if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2813	Name = GAS->getOutputName(i);
2814	TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);
2815	bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;
2816	if (IsHipStdPar && !IsValid)
2817	IsValidTargetAsm = false;
2818	else
2819	assert(IsValid && "Failed to parse output constraint");
2820	OutputConstraintInfos.push_back(Elt: Info);
2821	}
2822
2823	for (unsigned i = 0, e = S.getNumInputs(); i != e && IsValidTargetAsm; i++) {
2824	StringRef Name;
2825	if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(Val: &S))
2826	Name = GAS->getInputName(i);
2827	TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);
2828	bool IsValid =
2829	getTarget().validateInputConstraint(OutputConstraints: OutputConstraintInfos, info&: Info);
2830	if (IsHipStdPar && !IsValid)
2831	IsValidTargetAsm = false;
2832	else
2833	assert(IsValid && "Failed to parse input constraint");
2834	InputConstraintInfos.push_back(Elt: Info);
2835	}
2836
2837	if (!IsValidTargetAsm)
2838	return EmitHipStdParUnsupportedAsm(CGF: this, S);
2839
2840	std::string Constraints;
2841
2842	std::vector<LValue> ResultRegDests;
2843	std::vector<QualType> ResultRegQualTys;
2844	std::vector<llvm::Type *> ResultRegTypes;
2845	std::vector<llvm::Type *> ResultTruncRegTypes;
2846	std::vector<llvm::Type *> ArgTypes;
2847	std::vector<llvm::Type *> ArgElemTypes;
2848	std::vector<llvm::Value*> Args;
2849	llvm::BitVector ResultTypeRequiresCast;
2850	llvm::BitVector ResultRegIsFlagReg;
2851
2852	// Keep track of inout constraints.
2853	std::string InOutConstraints;
2854	std::vector<llvm::Value*> InOutArgs;
2855	std::vector<llvm::Type*> InOutArgTypes;
2856	std::vector<llvm::Type*> InOutArgElemTypes;
2857
2858	// Keep track of out constraints for tied input operand.
2859	std::vector<std::string> OutputConstraints;
2860
2861	// Keep track of defined physregs.
2862	llvm::SmallSet<std::string, 8> PhysRegOutputs;
2863
2864	// An inline asm can be marked readonly if it meets the following conditions:
2865	// - it doesn't have any sideeffects
2866	// - it doesn't clobber memory
2867	// - it doesn't return a value by-reference
2868	// It can be marked readnone if it doesn't have any input memory constraints
2869	// in addition to meeting the conditions listed above.
2870	bool ReadOnly = true, ReadNone = true;
2871
2872	for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
2873	TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];
2874
2875	// Simplify the output constraint.
2876	std::string OutputConstraint(S.getOutputConstraint(i));
2877	OutputConstraint = SimplifyConstraint(Constraint: OutputConstraint.c_str() + 1,
2878	Target: getTarget(), OutCons: &OutputConstraintInfos);
2879
2880	const Expr *OutExpr = S.getOutputExpr(i);
2881	OutExpr = OutExpr->IgnoreParenNoopCasts(Ctx: getContext());
2882
2883	std::string GCCReg;
2884	OutputConstraint = AddVariableConstraints(Constraint: OutputConstraint, AsmExpr: *OutExpr,
2885	Target: getTarget(), CGM, Stmt: S,
2886	EarlyClobber: Info.earlyClobber(),
2887	GCCReg: &GCCReg);
2888	// Give an error on multiple outputs to same physreg.
2889	if (!GCCReg.empty() && !PhysRegOutputs.insert(V: GCCReg).second)
2890	CGM.Error(loc: S.getAsmLoc(), error: "multiple outputs to hard register: " + GCCReg);
2891
2892	OutputConstraints.push_back(x: OutputConstraint);
2893	LValue Dest = EmitLValue(E: OutExpr);
2894	if (!Constraints.empty())
2895	Constraints += ',';
2896
2897	// If this is a register output, then make the inline asm return it
2898	// by-value. If this is a memory result, return the value by-reference.
2899	QualType QTy = OutExpr->getType();
2900	const bool IsScalarOrAggregate = hasScalarEvaluationKind(T: QTy) ||
2901	hasAggregateEvaluationKind(T: QTy);
2902	if (!Info.allowsMemory() && IsScalarOrAggregate) {
2903
2904	Constraints += "=" + OutputConstraint;
2905	ResultRegQualTys.push_back(x: QTy);
2906	ResultRegDests.push_back(x: Dest);
2907
2908	bool IsFlagReg = llvm::StringRef(OutputConstraint).starts_with(Prefix: "{@cc");
2909	ResultRegIsFlagReg.push_back(Val: IsFlagReg);
2910
2911	llvm::Type *Ty = ConvertTypeForMem(T: QTy);
2912	const bool RequiresCast = Info.allowsRegister() &&
2913	(getTargetHooks().isScalarizableAsmOperand(CGF&: *this, Ty) ||
2914	Ty->isAggregateType());
2915
2916	ResultTruncRegTypes.push_back(x: Ty);
2917	ResultTypeRequiresCast.push_back(Val: RequiresCast);
2918
2919	if (RequiresCast) {
2920	unsigned Size = getContext().getTypeSize(T: QTy);
2921	if (Size)
2922	Ty = llvm::IntegerType::get(C&: getLLVMContext(), NumBits: Size);
2923	else
2924	CGM.Error(loc: OutExpr->getExprLoc(), error: "output size should not be zero");
2925	}
2926	ResultRegTypes.push_back(x: Ty);
2927	// If this output is tied to an input, and if the input is larger, then
2928	// we need to set the actual result type of the inline asm node to be the
2929	// same as the input type.
2930	if (Info.hasMatchingInput()) {
2931	unsigned InputNo;
2932	for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
2933	TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2934	if (Input.hasTiedOperand() && Input.getTiedOperand() == i)
2935	break;
2936	}
2937	assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
2938
2939	QualType InputTy = S.getInputExpr(i: InputNo)->getType();
2940	QualType OutputType = OutExpr->getType();
2941
2942	uint64_t InputSize = getContext().getTypeSize(T: InputTy);
2943	if (getContext().getTypeSize(T: OutputType) < InputSize) {
2944	// Form the asm to return the value as a larger integer or fp type.
2945	ResultRegTypes.back() = ConvertType(T: InputTy);
2946	}
2947	}
2948	if (llvm::Type* AdjTy =
2949	getTargetHooks().adjustInlineAsmType(CGF&: *this, Constraint: OutputConstraint,
2950	Ty: ResultRegTypes.back()))
2951	ResultRegTypes.back() = AdjTy;
2952	else {
2953	CGM.getDiags().Report(S.getAsmLoc(),
2954	diag::err_asm_invalid_type_in_input)
2955	<< OutExpr->getType() << OutputConstraint;
2956	}
2957
2958	// Update largest vector width for any vector types.
2959	if (auto *VT = dyn_cast<llvm::VectorType>(Val: ResultRegTypes.back()))
2960	LargestVectorWidth =
2961	std::max(a: (uint64_t)LargestVectorWidth,
2962	b: VT->getPrimitiveSizeInBits().getKnownMinValue());
2963	} else {
2964	Address DestAddr = Dest.getAddress();
2965	// Matrix types in memory are represented by arrays, but accessed through
2966	// vector pointers, with the alignment specified on the access operation.
2967	// For inline assembly, update pointer arguments to use vector pointers.
2968	// Otherwise there will be a mis-match if the matrix is also an
2969	// input-argument which is represented as vector.
2970	if (isa<MatrixType>(Val: OutExpr->getType().getCanonicalType()))
2971	DestAddr = DestAddr.withElementType(ElemTy: ConvertType(T: OutExpr->getType()));
2972
2973	ArgTypes.push_back(x: DestAddr.getType());
2974	ArgElemTypes.push_back(x: DestAddr.getElementType());
2975	Args.push_back(x: DestAddr.emitRawPointer(CGF&: *this));
2976	Constraints += "=*";
2977	Constraints += OutputConstraint;
2978	ReadOnly = ReadNone = false;
2979	}
2980
2981	if (Info.isReadWrite()) {
2982	InOutConstraints += ',';
2983
2984	const Expr *InputExpr = S.getOutputExpr(i);
2985	llvm::Value *Arg;
2986	llvm::Type *ArgElemType;
2987	std::tie(args&: Arg, args&: ArgElemType) = EmitAsmInputLValue(
2988	Info, InputValue: Dest, InputType: InputExpr->getType(), ConstraintStr&: InOutConstraints,
2989	Loc: InputExpr->getExprLoc());
2990
2991	if (llvm::Type* AdjTy =
2992	getTargetHooks().adjustInlineAsmType(CGF&: *this, Constraint: OutputConstraint,
2993	Ty: Arg->getType()))
2994	Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
2995
2996	// Update largest vector width for any vector types.
2997	if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
2998	LargestVectorWidth =
2999	std::max(a: (uint64_t)LargestVectorWidth,
3000	b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3001	// Only tie earlyclobber physregs.
3002	if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))
3003	InOutConstraints += llvm::utostr(X: i);
3004	else
3005	InOutConstraints += OutputConstraint;
3006
3007	InOutArgTypes.push_back(x: Arg->getType());
3008	InOutArgElemTypes.push_back(x: ArgElemType);
3009	InOutArgs.push_back(x: Arg);
3010	}
3011	}
3012
3013	// If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
3014	// to the return value slot. Only do this when returning in registers.
3015	if (isa<MSAsmStmt>(Val: &S)) {
3016	const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo();
3017	if (RetAI.isDirect() || RetAI.isExtend()) {
3018	// Make a fake lvalue for the return value slot.
3019	LValue ReturnSlot = MakeAddrLValueWithoutTBAA(ReturnValue, FnRetTy);
3020	CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
3021	CGF&: *this, ReturnValue: ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
3022	ResultRegDests, AsmString, NumOutputs: S.getNumOutputs());
3023	SawAsmBlock = true;
3024	}
3025	}
3026
3027	for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
3028	const Expr *InputExpr = S.getInputExpr(i);
3029
3030	TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
3031
3032	if (Info.allowsMemory())
3033	ReadNone = false;
3034
3035	if (!Constraints.empty())
3036	Constraints += ',';
3037
3038	// Simplify the input constraint.
3039	std::string InputConstraint(S.getInputConstraint(i));
3040	InputConstraint = SimplifyConstraint(Constraint: InputConstraint.c_str(), Target: getTarget(),
3041	OutCons: &OutputConstraintInfos);
3042
3043	InputConstraint = AddVariableConstraints(
3044	Constraint: InputConstraint, AsmExpr: *InputExpr->IgnoreParenNoopCasts(Ctx: getContext()),
3045	Target: getTarget(), CGM, Stmt: S, EarlyClobber: false /* No EarlyClobber */);
3046
3047	std::string ReplaceConstraint (InputConstraint);
3048	llvm::Value *Arg;
3049	llvm::Type *ArgElemType;
3050	std::tie(args&: Arg, args&: ArgElemType) = EmitAsmInput(Info, InputExpr, ConstraintStr&: Constraints);
3051
3052	// If this input argument is tied to a larger output result, extend the
3053	// input to be the same size as the output. The LLVM backend wants to see
3054	// the input and output of a matching constraint be the same size. Note
3055	// that GCC does not define what the top bits are here. We use zext because
3056	// that is usually cheaper, but LLVM IR should really get an anyext someday.
3057	if (Info.hasTiedOperand()) {
3058	unsigned Output = Info.getTiedOperand();
3059	QualType OutputType = S.getOutputExpr(i: Output)->getType();
3060	QualType InputTy = InputExpr->getType();
3061
3062	if (getContext().getTypeSize(T: OutputType) >
3063	getContext().getTypeSize(T: InputTy)) {
3064	// Use ptrtoint as appropriate so that we can do our extension.
3065	if (isa<llvm::PointerType>(Val: Arg->getType()))
3066	Arg = Builder.CreatePtrToInt(V: Arg, DestTy: IntPtrTy);
3067	llvm::Type *OutputTy = ConvertType(T: OutputType);
3068	if (isa<llvm::IntegerType>(Val: OutputTy))
3069	Arg = Builder.CreateZExt(V: Arg, DestTy: OutputTy);
3070	else if (isa<llvm::PointerType>(Val: OutputTy))
3071	Arg = Builder.CreateZExt(V: Arg, DestTy: IntPtrTy);
3072	else if (OutputTy->isFloatingPointTy())
3073	Arg = Builder.CreateFPExt(V: Arg, DestTy: OutputTy);
3074	}
3075	// Deal with the tied operands' constraint code in adjustInlineAsmType.
3076	ReplaceConstraint = OutputConstraints[Output];
3077	}
3078	if (llvm::Type* AdjTy =
3079	getTargetHooks().adjustInlineAsmType(CGF&: *this, Constraint: ReplaceConstraint,
3080	Ty: Arg->getType()))
3081	Arg = Builder.CreateBitCast(V: Arg, DestTy: AdjTy);
3082	else
3083	CGM.getDiags().Report(S.getAsmLoc(), diag::err_asm_invalid_type_in_input)
3084	<< InputExpr->getType() << InputConstraint;
3085
3086	// Update largest vector width for any vector types.
3087	if (auto *VT = dyn_cast<llvm::VectorType>(Val: Arg->getType()))
3088	LargestVectorWidth =
3089	std::max(a: (uint64_t)LargestVectorWidth,
3090	b: VT->getPrimitiveSizeInBits().getKnownMinValue());
3091
3092	ArgTypes.push_back(x: Arg->getType());
3093	ArgElemTypes.push_back(x: ArgElemType);
3094	Args.push_back(x: Arg);
3095	Constraints += InputConstraint;
3096	}
3097
3098	// Append the "input" part of inout constraints.
3099	for (unsigned i = 0, e = InOutArgs.size(); i != e; i++) {
3100	ArgTypes.push_back(x: InOutArgTypes[i]);
3101	ArgElemTypes.push_back(x: InOutArgElemTypes[i]);
3102	Args.push_back(x: InOutArgs[i]);
3103	}
3104	Constraints += InOutConstraints;
3105
3106	// Labels
3107	SmallVector<llvm::BasicBlock *, 16> Transfer;
3108	llvm::BasicBlock *Fallthrough = nullptr;
3109	bool IsGCCAsmGoto = false;
3110	if (const auto *GS = dyn_cast<GCCAsmStmt>(Val: &S)) {
3111	IsGCCAsmGoto = GS->isAsmGoto();
3112	if (IsGCCAsmGoto) {
3113	for (const auto *E : GS->labels()) {
3114	JumpDest Dest = getJumpDestForLabel(E->getLabel());
3115	Transfer.push_back(Dest.getBlock());
3116	if (!Constraints.empty())
3117	Constraints += ',';
3118	Constraints += "!i";
3119	}
3120	Fallthrough = createBasicBlock(name: "asm.fallthrough");
3121	}
3122	}
3123
3124	bool HasUnwindClobber = false;
3125
3126	// Clobbers
3127	for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
3128	std::string Clobber = S.getClobber(i);
3129
3130	if (Clobber == "memory")
3131	ReadOnly = ReadNone = false;
3132	else if (Clobber == "unwind") {
3133	HasUnwindClobber = true;
3134	continue;
3135	} else if (Clobber != "cc") {
3136	Clobber = getTarget().getNormalizedGCCRegisterName(Name: Clobber);
3137	if (CGM.getCodeGenOpts().StackClashProtector &&
3138	getTarget().isSPRegName(Clobber)) {
3139	CGM.getDiags().Report(S.getAsmLoc(),
3140	diag::warn_stack_clash_protection_inline_asm);
3141	}
3142	}
3143
3144	if (isa<MSAsmStmt>(Val: &S)) {
3145	if (Clobber == "eax" || Clobber == "edx") {
3146	if (Constraints.find(s: "=&A") != std::string::npos)
3147	continue;
3148	std::string::size_type position1 =
3149	Constraints.find(str: "={" + Clobber + "}");
3150	if (position1 != std::string::npos) {
3151	Constraints.insert(pos: position1 + 1, s: "&");
3152	continue;
3153	}
3154	std::string::size_type position2 = Constraints.find(s: "=A");
3155	if (position2 != std::string::npos) {
3156	Constraints.insert(pos: position2 + 1, s: "&");
3157	continue;
3158	}
3159	}
3160	}
3161	if (!Constraints.empty())
3162	Constraints += ',';
3163
3164	Constraints += "~{";
3165	Constraints += Clobber;
3166	Constraints += '}';
3167	}
3168
3169	assert(!(HasUnwindClobber && IsGCCAsmGoto) &&
3170	"unwind clobber can't be used with asm goto");
3171
3172	// Add machine specific clobbers
3173	std::string_view MachineClobbers = getTarget().getClobbers();
3174	if (!MachineClobbers.empty()) {
3175	if (!Constraints.empty())
3176	Constraints += ',';
3177	Constraints += MachineClobbers;
3178	}
3179
3180	llvm::Type *ResultType;
3181	if (ResultRegTypes.empty())
3182	ResultType = VoidTy;
3183	else if (ResultRegTypes.size() == 1)
3184	ResultType = ResultRegTypes[0];
3185	else
3186	ResultType = llvm::StructType::get(Context&: getLLVMContext(), Elements: ResultRegTypes);
3187
3188	llvm::FunctionType *FTy =
3189	llvm::FunctionType::get(Result: ResultType, Params: ArgTypes, isVarArg: false);
3190
3191	bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
3192
3193	llvm::InlineAsm::AsmDialect GnuAsmDialect =
3194	CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
3195	? llvm::InlineAsm::AD_ATT
3196	: llvm::InlineAsm::AD_Intel;
3197	llvm::InlineAsm::AsmDialect AsmDialect = isa<MSAsmStmt>(Val: &S) ?
3198	llvm::InlineAsm::AD_Intel : GnuAsmDialect;
3199
3200	llvm::InlineAsm *IA = llvm::InlineAsm::get(
3201	Ty: FTy, AsmString, Constraints, hasSideEffects: HasSideEffect,
3202	/* IsAlignStack */ isAlignStack: false, asmDialect: AsmDialect, canThrow: HasUnwindClobber);
3203	std::vector<llvm::Value*> RegResults;
3204	llvm::CallBrInst *CBR;
3205	llvm::DenseMap<llvm::BasicBlock *, SmallVector<llvm::Value *, 4>>
3206	CBRRegResults;
3207	if (IsGCCAsmGoto) {
3208	CBR = Builder.CreateCallBr(Callee: IA, DefaultDest: Fallthrough, IndirectDests: Transfer, Args);
3209	EmitBlock(BB: Fallthrough);
3210	UpdateAsmCallInst(Result&: *CBR, HasSideEffect, /*HasUnwindClobber=*/false, ReadOnly,
3211	ReadNone, NoMerge: InNoMergeAttributedStmt,
3212	NoConvergent: InNoConvergentAttributedStmt, S, ResultRegTypes,
3213	ArgElemTypes, CGF&: *this, RegResults);
3214	// Because we are emitting code top to bottom, we don't have enough
3215	// information at this point to know precisely whether we have a critical
3216	// edge. If we have outputs, split all indirect destinations.
3217	if (!RegResults.empty()) {
3218	unsigned i = 0;
3219	for (llvm::BasicBlock *Dest : CBR->getIndirectDests()) {
3220	llvm::Twine SynthName = Dest->getName() + ".split";
3221	llvm::BasicBlock *SynthBB = createBasicBlock(name: SynthName);
3222	llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
3223	Builder.SetInsertPoint(SynthBB);
3224
3225	if (ResultRegTypes.size() == 1) {
3226	CBRRegResults[SynthBB].push_back(Elt: CBR);
3227	} else {
3228	for (unsigned j = 0, e = ResultRegTypes.size(); j != e; ++j) {
3229	llvm::Value *Tmp = Builder.CreateExtractValue(Agg: CBR, Idxs: j, Name: "asmresult");
3230	CBRRegResults[SynthBB].push_back(Elt: Tmp);
3231	}
3232	}
3233
3234	EmitBranch(Target: Dest);
3235	EmitBlock(BB: SynthBB);
3236	CBR->setIndirectDest(i: i++, B: SynthBB);
3237	}
3238	}
3239	} else if (HasUnwindClobber) {
3240	llvm::CallBase *Result = EmitCallOrInvoke(Callee: IA, Args, Name: "");
3241	UpdateAsmCallInst(Result&: *Result, HasSideEffect, /*HasUnwindClobber=*/true,
3242	ReadOnly, ReadNone, NoMerge: InNoMergeAttributedStmt,
3243	NoConvergent: InNoConvergentAttributedStmt, S, ResultRegTypes,
3244	ArgElemTypes, CGF&: *this, RegResults);
3245	} else {
3246	llvm::CallInst *Result =
3247	Builder.CreateCall(Callee: IA, Args, OpBundles: getBundlesForFunclet(Callee: IA));
3248	UpdateAsmCallInst(Result&: *Result, HasSideEffect, /*HasUnwindClobber=*/false,
3249	ReadOnly, ReadNone, NoMerge: InNoMergeAttributedStmt,
3250	NoConvergent: InNoConvergentAttributedStmt, S, ResultRegTypes,
3251	ArgElemTypes, CGF&: *this, RegResults);
3252	}
3253
3254	EmitAsmStores(CGF&: *this, S, RegResults, ResultRegTypes, ResultTruncRegTypes,
3255	ResultRegDests, ResultRegQualTys, ResultTypeRequiresCast,
3256	ResultRegIsFlagReg);
3257
3258	// If this is an asm goto with outputs, repeat EmitAsmStores, but with a
3259	// different insertion point; one for each indirect destination and with
3260	// CBRRegResults rather than RegResults.
3261	if (IsGCCAsmGoto && !CBRRegResults.empty()) {
3262	for (llvm::BasicBlock *Succ : CBR->getIndirectDests()) {
3263	llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
3264	Builder.SetInsertPoint(TheBB: Succ, IP: --(Succ->end()));
3265	EmitAsmStores(CGF&: *this, S, RegResults: CBRRegResults[Succ], ResultRegTypes,
3266	ResultTruncRegTypes, ResultRegDests, ResultRegQualTys,
3267	ResultTypeRequiresCast, ResultRegIsFlagReg);
3268	}
3269	}
3270	}
3271
3272	LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
3273	const RecordDecl *RD = S.getCapturedRecordDecl();
3274	QualType RecordTy = getContext().getRecordType(Decl: RD);
3275
3276	// Initialize the captured struct.
3277	LValue SlotLV =
3278	MakeAddrLValue(Addr: CreateMemTemp(T: RecordTy, Name: "agg.captured"), T: RecordTy);
3279
3280	RecordDecl::field_iterator CurField = RD->field_begin();
3281	for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
3282	E = S.capture_init_end();
3283	I != E; ++I, ++CurField) {
3284	LValue LV = EmitLValueForFieldInitialization(Base: SlotLV, Field: *CurField);
3285	if (CurField->hasCapturedVLAType()) {
3286	EmitLambdaVLACapture(VAT: CurField->getCapturedVLAType(), LV);
3287	} else {
3288	EmitInitializerForField(Field: *CurField, LHS: LV, Init: *I);
3289	}
3290	}
3291
3292	return SlotLV;
3293	}
3294
3295	/// Generate an outlined function for the body of a CapturedStmt, store any
3296	/// captured variables into the captured struct, and call the outlined function.
3297	llvm::Function *
3298	CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
3299	LValue CapStruct = InitCapturedStruct(S);
3300
3301	// Emit the CapturedDecl
3302	CodeGenFunction CGF(CGM, true);
3303	CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
3304	llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
3305	delete CGF.CapturedStmtInfo;
3306
3307	// Emit call to the helper function.
3308	EmitCallOrInvoke(Callee: F, Args: CapStruct.getPointer(CGF&: *this));
3309
3310	return F;
3311	}
3312
3313	Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
3314	LValue CapStruct = InitCapturedStruct(S);
3315	return CapStruct.getAddress();
3316	}
3317
3318	/// Creates the outlined function for a CapturedStmt.
3319	llvm::Function *
3320	CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
3321	assert(CapturedStmtInfo &&
3322	"CapturedStmtInfo should be set when generating the captured function");
3323	const CapturedDecl *CD = S.getCapturedDecl();
3324	const RecordDecl *RD = S.getCapturedRecordDecl();
3325	SourceLocation Loc = S.getBeginLoc();
3326	assert(CD->hasBody() && "missing CapturedDecl body");
3327
3328	// Build the argument list.
3329	ASTContext &Ctx = CGM.getContext();
3330	FunctionArgList Args;
3331	Args.append(in_start: CD->param_begin(), in_end: CD->param_end());
3332
3333	// Create the function declaration.
3334	const CGFunctionInfo &FuncInfo =
3335	CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
3336	llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(Info: FuncInfo);
3337
3338	llvm::Function *F =
3339	llvm::Function::Create(Ty: FuncLLVMTy, Linkage: llvm::GlobalValue::InternalLinkage,
3340	N: CapturedStmtInfo->getHelperName(), M: &CGM.getModule());
3341	CGM.SetInternalFunctionAttributes(GD: CD, F, FI: FuncInfo);
3342	if (CD->isNothrow())
3343	F->addFnAttr(llvm::Attribute::NoUnwind);
3344
3345	// Generate the function.
3346	StartFunction(GD: CD, RetTy: Ctx.VoidTy, Fn: F, FnInfo: FuncInfo, Args, Loc: CD->getLocation(),
3347	StartLoc: CD->getBody()->getBeginLoc());
3348	// Set the context parameter in CapturedStmtInfo.
3349	Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam());
3350	CapturedStmtInfo->setContextValue(Builder.CreateLoad(Addr: DeclPtr));
3351
3352	// Initialize variable-length arrays.
3353	LValue Base = MakeNaturalAlignRawAddrLValue(
3354	V: CapturedStmtInfo->getContextValue(), T: Ctx.getTagDeclType(RD));
3355	for (auto *FD : RD->fields()) {
3356	if (FD->hasCapturedVLAType()) {
3357	auto *ExprArg =
3358	EmitLoadOfLValue(V: EmitLValueForField(Base, Field: FD), Loc: S.getBeginLoc())
3359	.getScalarVal();
3360	auto VAT = FD->getCapturedVLAType();
3361	VLASizeMap[VAT->getSizeExpr()] = ExprArg;
3362	}
3363	}
3364
3365	// If 'this' is captured, load it into CXXThisValue.
3366	if (CapturedStmtInfo->isCXXThisExprCaptured()) {
3367	FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
3368	LValue ThisLValue = EmitLValueForField(Base, Field: FD);
3369	CXXThisValue = EmitLoadOfLValue(V: ThisLValue, Loc).getScalarVal();
3370	}
3371
3372	PGO->assignRegionCounters(GD: GlobalDecl(CD), Fn: F);
3373	CapturedStmtInfo->EmitBody(CGF&: *this, S: CD->getBody());
3374	FinishFunction(EndLoc: CD->getBodyRBrace());
3375
3376	return F;
3377	}
3378
3379	// Returns the first convergence entry/loop/anchor instruction found in |BB|.
3380	// std::nullptr otherwise.
3381	static llvm::ConvergenceControlInst *getConvergenceToken(llvm::BasicBlock *BB) {
3382	for (auto &I : *BB) {
3383	if (auto *CI = dyn_cast<llvm::ConvergenceControlInst>(Val: &I))
3384	return CI;
3385	}
3386	return nullptr;
3387	}
3388
3389	llvm::CallBase *
3390	CodeGenFunction::addConvergenceControlToken(llvm::CallBase *Input) {
3391	llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3392	assert(ParentToken);
3393
3394	llvm::Value *bundleArgs[] = {ParentToken};
3395	llvm::OperandBundleDef OB("convergencectrl", bundleArgs);
3396	auto *Output = llvm::CallBase::addOperandBundle(
3397	CB: Input, ID: llvm::LLVMContext::OB_convergencectrl, OB, InsertPt: Input->getIterator());
3398	Input->replaceAllUsesWith(V: Output);
3399	Input->eraseFromParent();
3400	return Output;
3401	}
3402
3403	llvm::ConvergenceControlInst *
3404	CodeGenFunction::emitConvergenceLoopToken(llvm::BasicBlock *BB) {
3405	llvm::ConvergenceControlInst *ParentToken = ConvergenceTokenStack.back();
3406	assert(ParentToken);
3407	return llvm::ConvergenceControlInst::CreateLoop(BB&: *BB, Parent: ParentToken);
3408	}
3409
3410	llvm::ConvergenceControlInst *
3411	CodeGenFunction::getOrEmitConvergenceEntryToken(llvm::Function *F) {
3412	llvm::BasicBlock *BB = &F->getEntryBlock();
3413	llvm::ConvergenceControlInst *Token = getConvergenceToken(BB);
3414	if (Token)
3415	return Token;
3416
3417	// Adding a convergence token requires the function to be marked as
3418	// convergent.
3419	F->setConvergent();
3420	return llvm::ConvergenceControlInst::CreateEntry(BB&: *BB);
3421	}
3422