1	//===-- SemaCoroutine.cpp - Semantic Analysis for Coroutines --------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file implements semantic analysis for C++ Coroutines.
10	//
11	// This file contains references to sections of the Coroutines TS, which
12	// can be found at http://wg21.link/coroutines.
13	//
14	//===----------------------------------------------------------------------===//
15
16	#include "CoroutineStmtBuilder.h"
17	#include "clang/AST/ASTLambda.h"
18	#include "clang/AST/Decl.h"
19	#include "clang/AST/Expr.h"
20	#include "clang/AST/ExprCXX.h"
21	#include "clang/AST/StmtCXX.h"
22	#include "clang/Basic/Builtins.h"
23	#include "clang/Lex/Preprocessor.h"
24	#include "clang/Sema/EnterExpressionEvaluationContext.h"
25	#include "clang/Sema/Initialization.h"
26	#include "clang/Sema/Overload.h"
27	#include "clang/Sema/ScopeInfo.h"
28	#include "clang/Sema/SemaInternal.h"
29	#include "llvm/ADT/SmallSet.h"
30
31	using namespace clang;
32	using namespace sema;
33
34	static LookupResult lookupMember(Sema &S, const char *Name, CXXRecordDecl *RD,
35	SourceLocation Loc, bool &Res) {
36	DeclarationName DN = S.PP.getIdentifierInfo(Name);
37	LookupResult LR(S, DN, Loc, Sema::LookupMemberName);
38	// Suppress diagnostics when a private member is selected. The same warnings
39	// will be produced again when building the call.
40	LR.suppressDiagnostics();
41	Res = S.LookupQualifiedName(LR, RD);
42	return LR;
43	}
44
45	static bool lookupMember(Sema &S, const char *Name, CXXRecordDecl *RD,
46	SourceLocation Loc) {
47	bool Res;
48	lookupMember(S, Name, RD, Loc, Res);
49	return Res;
50	}
51
52	/// Look up the std::coroutine_traits<...>::promise_type for the given
53	/// function type.
54	static QualType lookupPromiseType(Sema &S, const FunctionDecl *FD,
55	SourceLocation KwLoc) {
56	const FunctionProtoType *FnType = FD->getType()->castAs<FunctionProtoType>();
57	const SourceLocation FuncLoc = FD->getLocation();
58
59	ClassTemplateDecl *CoroTraits =
60	S.lookupCoroutineTraits(KwLoc, FuncLoc);
61	if (!CoroTraits)
62	return QualType();
63
64	// Form template argument list for coroutine_traits<R, P1, P2, ...> according
65	// to [dcl.fct.def.coroutine]3
66	TemplateArgumentListInfo Args(KwLoc, KwLoc);
67	auto AddArg = [&](QualType T) {
68	Args.addArgument(Loc: TemplateArgumentLoc(
69	TemplateArgument(T), S.Context.getTrivialTypeSourceInfo(T, Loc: KwLoc)));
70	};
71	AddArg(FnType->getReturnType());
72	// If the function is a non-static member function, add the type
73	// of the implicit object parameter before the formal parameters.
74	if (auto *MD = dyn_cast<CXXMethodDecl>(Val: FD)) {
75	if (MD->isImplicitObjectMemberFunction()) {
76	// [over.match.funcs]4
77	// For non-static member functions, the type of the implicit object
78	// parameter is
79	// -- "lvalue reference to cv X" for functions declared without a
80	// ref-qualifier or with the & ref-qualifier
81	// -- "rvalue reference to cv X" for functions declared with the &&
82	// ref-qualifier
83	QualType T = MD->getFunctionObjectParameterType();
84	T = FnType->getRefQualifier() == RQ_RValue
85	? S.Context.getRValueReferenceType(T)
86	: S.Context.getLValueReferenceType(T, /*SpelledAsLValue*/ true);
87	AddArg(T);
88	}
89	}
90	for (QualType T : FnType->getParamTypes())
91	AddArg(T);
92
93	// Build the template-id.
94	QualType CoroTrait =
95	S.CheckTemplateIdType(Template: TemplateName(CoroTraits), TemplateLoc: KwLoc, TemplateArgs&: Args);
96	if (CoroTrait.isNull())
97	return QualType();
98	if (S.RequireCompleteType(KwLoc, CoroTrait,
99	diag::err_coroutine_type_missing_specialization))
100	return QualType();
101
102	auto *RD = CoroTrait->getAsCXXRecordDecl();
103	assert(RD && "specialization of class template is not a class?");
104
105	// Look up the ::promise_type member.
106	LookupResult R(S, &S.PP.getIdentifierTable().get(Name: "promise_type"), KwLoc,
107	Sema::LookupOrdinaryName);
108	S.LookupQualifiedName(R, RD);
109	auto *Promise = R.getAsSingle<TypeDecl>();
110	if (!Promise) {
111	S.Diag(FuncLoc,
112	diag::err_implied_std_coroutine_traits_promise_type_not_found)
113	<< RD;
114	return QualType();
115	}
116	// The promise type is required to be a class type.
117	QualType PromiseType = S.Context.getTypeDeclType(Decl: Promise);
118
119	auto buildElaboratedType = [&]() {
120	auto *NNS = NestedNameSpecifier::Create(Context: S.Context, Prefix: nullptr, NS: S.getStdNamespace());
121	NNS = NestedNameSpecifier::Create(Context: S.Context, Prefix: NNS, Template: false,
122	T: CoroTrait.getTypePtr());
123	return S.Context.getElaboratedType(Keyword: ElaboratedTypeKeyword::None, NNS,
124	NamedType: PromiseType);
125	};
126
127	if (!PromiseType->getAsCXXRecordDecl()) {
128	S.Diag(FuncLoc,
129	diag::err_implied_std_coroutine_traits_promise_type_not_class)
130	<< buildElaboratedType();
131	return QualType();
132	}
133	if (S.RequireCompleteType(FuncLoc, buildElaboratedType(),
134	diag::err_coroutine_promise_type_incomplete))
135	return QualType();
136
137	return PromiseType;
138	}
139
140	/// Look up the std::coroutine_handle<PromiseType>.
141	static QualType lookupCoroutineHandleType(Sema &S, QualType PromiseType,
142	SourceLocation Loc) {
143	if (PromiseType.isNull())
144	return QualType();
145
146	NamespaceDecl *CoroNamespace = S.getStdNamespace();
147	assert(CoroNamespace && "Should already be diagnosed");
148
149	LookupResult Result(S, &S.PP.getIdentifierTable().get(Name: "coroutine_handle"),
150	Loc, Sema::LookupOrdinaryName);
151	if (!S.LookupQualifiedName(Result, CoroNamespace)) {
152	S.Diag(Loc, diag::err_implied_coroutine_type_not_found)
153	<< "std::coroutine_handle";
154	return QualType();
155	}
156
157	ClassTemplateDecl *CoroHandle = Result.getAsSingle<ClassTemplateDecl>();
158	if (!CoroHandle) {
159	Result.suppressDiagnostics();
160	// We found something weird. Complain about the first thing we found.
161	NamedDecl *Found = *Result.begin();
162	S.Diag(Found->getLocation(), diag::err_malformed_std_coroutine_handle);
163	return QualType();
164	}
165
166	// Form template argument list for coroutine_handle<Promise>.
167	TemplateArgumentListInfo Args(Loc, Loc);
168	Args.addArgument(Loc: TemplateArgumentLoc(
169	TemplateArgument(PromiseType),
170	S.Context.getTrivialTypeSourceInfo(T: PromiseType, Loc)));
171
172	// Build the template-id.
173	QualType CoroHandleType =
174	S.CheckTemplateIdType(Template: TemplateName(CoroHandle), TemplateLoc: Loc, TemplateArgs&: Args);
175	if (CoroHandleType.isNull())
176	return QualType();
177	if (S.RequireCompleteType(Loc, CoroHandleType,
178	diag::err_coroutine_type_missing_specialization))
179	return QualType();
180
181	return CoroHandleType;
182	}
183
184	static bool isValidCoroutineContext(Sema &S, SourceLocation Loc,
185	StringRef Keyword) {
186	// [expr.await]p2 dictates that 'co_await' and 'co_yield' must be used within
187	// a function body.
188	// FIXME: This also covers [expr.await]p2: "An await-expression shall not
189	// appear in a default argument." But the diagnostic QoI here could be
190	// improved to inform the user that default arguments specifically are not
191	// allowed.
192	auto *FD = dyn_cast<FunctionDecl>(Val: S.CurContext);
193	if (!FD) {
194	S.Diag(Loc, isa<ObjCMethodDecl>(S.CurContext)
195	? diag::err_coroutine_objc_method
196	: diag::err_coroutine_outside_function) << Keyword;
197	return false;
198	}
199
200	// An enumeration for mapping the diagnostic type to the correct diagnostic
201	// selection index.
202	enum InvalidFuncDiag {
203	DiagCtor = 0,
204	DiagDtor,
205	DiagMain,
206	DiagConstexpr,
207	DiagAutoRet,
208	DiagVarargs,
209	DiagConsteval,
210	};
211	bool Diagnosed = false;
212	auto DiagInvalid = [&](InvalidFuncDiag ID) {
213	S.Diag(Loc, diag::err_coroutine_invalid_func_context) << ID << Keyword;
214	Diagnosed = true;
215	return false;
216	};
217
218	// Diagnose when a constructor, destructor
219	// or the function 'main' are declared as a coroutine.
220	auto *MD = dyn_cast<CXXMethodDecl>(Val: FD);
221	// [class.ctor]p11: "A constructor shall not be a coroutine."
222	if (MD && isa<CXXConstructorDecl>(Val: MD))
223	return DiagInvalid(DiagCtor);
224	// [class.dtor]p17: "A destructor shall not be a coroutine."
225	else if (MD && isa<CXXDestructorDecl>(Val: MD))
226	return DiagInvalid(DiagDtor);
227	// [basic.start.main]p3: "The function main shall not be a coroutine."
228	else if (FD->isMain())
229	return DiagInvalid(DiagMain);
230
231	// Emit a diagnostics for each of the following conditions which is not met.
232	// [expr.const]p2: "An expression e is a core constant expression unless the
233	// evaluation of e [...] would evaluate one of the following expressions:
234	// [...] an await-expression [...] a yield-expression."
235	if (FD->isConstexpr())
236	DiagInvalid(FD->isConsteval() ? DiagConsteval : DiagConstexpr);
237	// [dcl.spec.auto]p15: "A function declared with a return type that uses a
238	// placeholder type shall not be a coroutine."
239	if (FD->getReturnType()->isUndeducedType())
240	DiagInvalid(DiagAutoRet);
241	// [dcl.fct.def.coroutine]p1
242	// The parameter-declaration-clause of the coroutine shall not terminate with
243	// an ellipsis that is not part of a parameter-declaration.
244	if (FD->isVariadic())
245	DiagInvalid(DiagVarargs);
246
247	return !Diagnosed;
248	}
249
250	/// Build a call to 'operator co_await' if there is a suitable operator for
251	/// the given expression.
252	ExprResult Sema::BuildOperatorCoawaitCall(SourceLocation Loc, Expr *E,
253	UnresolvedLookupExpr *Lookup) {
254	UnresolvedSet<16> Functions;
255	Functions.append(I: Lookup->decls_begin(), E: Lookup->decls_end());
256	return CreateOverloadedUnaryOp(OpLoc: Loc, Opc: UO_Coawait, Fns: Functions, input: E);
257	}
258
259	static ExprResult buildOperatorCoawaitCall(Sema &SemaRef, Scope *S,
260	SourceLocation Loc, Expr *E) {
261	ExprResult R = SemaRef.BuildOperatorCoawaitLookupExpr(S, Loc);
262	if (R.isInvalid())
263	return ExprError();
264	return SemaRef.BuildOperatorCoawaitCall(Loc, E,
265	Lookup: cast<UnresolvedLookupExpr>(Val: R.get()));
266	}
267
268	static ExprResult buildCoroutineHandle(Sema &S, QualType PromiseType,
269	SourceLocation Loc) {
270	QualType CoroHandleType = lookupCoroutineHandleType(S, PromiseType, Loc);
271	if (CoroHandleType.isNull())
272	return ExprError();
273
274	DeclContext *LookupCtx = S.computeDeclContext(T: CoroHandleType);
275	LookupResult Found(S, &S.PP.getIdentifierTable().get(Name: "from_address"), Loc,
276	Sema::LookupOrdinaryName);
277	if (!S.LookupQualifiedName(R&: Found, LookupCtx)) {
278	S.Diag(Loc, diag::err_coroutine_handle_missing_member)
279	<< "from_address";
280	return ExprError();
281	}
282
283	Expr *FramePtr =
284	S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_frame, {});
285
286	CXXScopeSpec SS;
287	ExprResult FromAddr =
288	S.BuildDeclarationNameExpr(SS, R&: Found, /*NeedsADL=*/false);
289	if (FromAddr.isInvalid())
290	return ExprError();
291
292	return S.BuildCallExpr(S: nullptr, Fn: FromAddr.get(), LParenLoc: Loc, ArgExprs: FramePtr, RParenLoc: Loc);
293	}
294
295	struct ReadySuspendResumeResult {
296	enum AwaitCallType { ACT_Ready, ACT_Suspend, ACT_Resume };
297	Expr *Results[3];
298	OpaqueValueExpr *OpaqueValue;
299	bool IsInvalid;
300	};
301
302	static ExprResult buildMemberCall(Sema &S, Expr *Base, SourceLocation Loc,
303	StringRef Name, MultiExprArg Args) {
304	DeclarationNameInfo NameInfo(&S.PP.getIdentifierTable().get(Name), Loc);
305
306	// FIXME: Fix BuildMemberReferenceExpr to take a const CXXScopeSpec&.
307	CXXScopeSpec SS;
308	ExprResult Result = S.BuildMemberReferenceExpr(
309	Base, BaseType: Base->getType(), OpLoc: Loc, /*IsPtr=*/IsArrow: false, SS,
310	TemplateKWLoc: SourceLocation(), FirstQualifierInScope: nullptr, NameInfo, /*TemplateArgs=*/nullptr,
311	/*Scope=*/S: nullptr);
312	if (Result.isInvalid())
313	return ExprError();
314
315	// We meant exactly what we asked for. No need for typo correction.
316	if (auto *TE = dyn_cast<TypoExpr>(Val: Result.get())) {
317	S.clearDelayedTypo(TE);
318	S.Diag(Loc, diag::err_no_member)
319	<< NameInfo.getName() << Base->getType()->getAsCXXRecordDecl()
320	<< Base->getSourceRange();
321	return ExprError();
322	}
323
324	auto EndLoc = Args.empty() ? Loc : Args.back()->getEndLoc();
325	return S.BuildCallExpr(S: nullptr, Fn: Result.get(), LParenLoc: Loc, ArgExprs: Args, RParenLoc: EndLoc, ExecConfig: nullptr);
326	}
327
328	// See if return type is coroutine-handle and if so, invoke builtin coro-resume
329	// on its address. This is to enable the support for coroutine-handle
330	// returning await_suspend that results in a guaranteed tail call to the target
331	// coroutine.
332	static Expr *maybeTailCall(Sema &S, QualType RetType, Expr *E,
333	SourceLocation Loc) {
334	if (RetType->isReferenceType())
335	return nullptr;
336	Type const *T = RetType.getTypePtr();
337	if (!T->isClassType() && !T->isStructureType())
338	return nullptr;
339
340	// FIXME: Add convertability check to coroutine_handle<>. Possibly via
341	// EvaluateBinaryTypeTrait(BTT_IsConvertible, ...) which is at the moment
342	// a private function in SemaExprCXX.cpp
343
344	ExprResult AddressExpr = buildMemberCall(S, Base: E, Loc, Name: "address", Args: std::nullopt);
345	if (AddressExpr.isInvalid())
346	return nullptr;
347
348	Expr *JustAddress = AddressExpr.get();
349
350	// FIXME: Without optimizations, the temporary result from `await_suspend()`
351	// may be put on the coroutine frame since the coroutine frame constructor
352	// will think the temporary variable will escape from the
353	// `coroutine_handle<>::address()` call. This is problematic since the
354	// coroutine should be considered to be suspended after it enters
355	// `await_suspend` so it shouldn't access/update the coroutine frame after
356	// that.
357	//
358	// See https://github.com/llvm/llvm-project/issues/65054 for the report.
359	//
360	// The long term solution may wrap the whole logic about `await-suspend`
361	// into a standalone function. This is similar to the proposed solution
362	// in tryMarkAwaitSuspendNoInline. See the comments there for details.
363	//
364	// The short term solution here is to mark `coroutine_handle<>::address()`
365	// function as always-inline so that the coroutine frame constructor won't
366	// think the temporary result is escaped incorrectly.
367	if (auto *FD = cast<CallExpr>(JustAddress)->getDirectCallee())
368	if (!FD->hasAttr<AlwaysInlineAttr>() && !FD->hasAttr<NoInlineAttr>())
369	FD->addAttr(AlwaysInlineAttr::CreateImplicit(S.getASTContext(),
370	FD->getLocation()));
371
372	// Check that the type of AddressExpr is void*
373	if (!JustAddress->getType().getTypePtr()->isVoidPointerType())
374	S.Diag(cast<CallExpr>(JustAddress)->getCalleeDecl()->getLocation(),
375	diag::warn_coroutine_handle_address_invalid_return_type)
376	<< JustAddress->getType();
377
378	// Clean up temporary objects so that they don't live across suspension points
379	// unnecessarily. We choose to clean up before the call to
380	// __builtin_coro_resume so that the cleanup code are not inserted in-between
381	// the resume call and return instruction, which would interfere with the
382	// musttail call contract.
383	JustAddress = S.MaybeCreateExprWithCleanups(SubExpr: JustAddress);
384	return S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_resume,
385	JustAddress);
386	}
387
388	/// The await_suspend call performed by co_await is essentially asynchronous
389	/// to the execution of the coroutine. Inlining it normally into an unsplit
390	/// coroutine can cause miscompilation because the coroutine CFG misrepresents
391	/// the true control flow of the program: things that happen in the
392	/// await_suspend are not guaranteed to happen prior to the resumption of the
393	/// coroutine, and things that happen after the resumption of the coroutine
394	/// (including its exit and the potential deallocation of the coroutine frame)
395	/// are not guaranteed to happen only after the end of await_suspend.
396	///
397	/// See https://github.com/llvm/llvm-project/issues/56301 and
398	/// https://reviews.llvm.org/D157070 for the example and the full discussion.
399	///
400	/// The short-term solution to this problem is to mark the call as uninlinable.
401	/// But we don't want to do this if the call is known to be trivial, which is
402	/// very common.
403	///
404	/// The long-term solution may introduce patterns like:
405	///
406	/// call @llvm.coro.await_suspend(ptr %awaiter, ptr %handle,
407	/// ptr @awaitSuspendFn)
408	///
409	/// Then it is much easier to perform the safety analysis in the middle end.
410	/// If it is safe to inline the call to awaitSuspend, we can replace it in the
411	/// CoroEarly pass. Otherwise we could replace it in the CoroSplit pass.
412	static void tryMarkAwaitSuspendNoInline(Sema &S, OpaqueValueExpr *Awaiter,
413	CallExpr *AwaitSuspend) {
414	// The method here to extract the awaiter decl is not precise.
415	// This is intentional. Since it is hard to perform the analysis in the
416	// frontend due to the complexity of C++'s type systems.
417	// And we prefer to perform such analysis in the middle end since it is
418	// easier to implement and more powerful.
419	CXXRecordDecl *AwaiterDecl =
420	Awaiter->getType().getNonReferenceType()->getAsCXXRecordDecl();
421
422	if (AwaiterDecl && AwaiterDecl->field_empty())
423	return;
424
425	FunctionDecl *FD = AwaitSuspend->getDirectCallee();
426
427	assert(FD);
428
429	// If the `await_suspend()` function is marked as `always_inline` explicitly,
430	// we should give the user the right to control the codegen.
431	if (FD->hasAttr<NoInlineAttr>() || FD->hasAttr<AlwaysInlineAttr>())
432	return;
433
434	// This is problematic if the user calls the await_suspend standalone. But on
435	// the on hand, it is not incorrect semantically since inlining is not part
436	// of the standard. On the other hand, it is relatively rare to call
437	// the await_suspend function standalone.
438	//
439	// And given we've already had the long-term plan, the current workaround
440	// looks relatively tolerant.
441	FD->addAttr(
442	NoInlineAttr::CreateImplicit(S.getASTContext(), FD->getLocation()));
443	}
444
445	/// Build calls to await_ready, await_suspend, and await_resume for a co_await
446	/// expression.
447	/// The generated AST tries to clean up temporary objects as early as
448	/// possible so that they don't live across suspension points if possible.
449	/// Having temporary objects living across suspension points unnecessarily can
450	/// lead to large frame size, and also lead to memory corruptions if the
451	/// coroutine frame is destroyed after coming back from suspension. This is done
452	/// by wrapping both the await_ready call and the await_suspend call with
453	/// ExprWithCleanups. In the end of this function, we also need to explicitly
454	/// set cleanup state so that the CoawaitExpr is also wrapped with an
455	/// ExprWithCleanups to clean up the awaiter associated with the co_await
456	/// expression.
457	static ReadySuspendResumeResult buildCoawaitCalls(Sema &S, VarDecl *CoroPromise,
458	SourceLocation Loc, Expr *E) {
459	OpaqueValueExpr *Operand = new (S.Context)
460	OpaqueValueExpr(Loc, E->getType(), VK_LValue, E->getObjectKind(), E);
461
462	// Assume valid until we see otherwise.
463	// Further operations are responsible for setting IsInalid to true.
464	ReadySuspendResumeResult Calls = {.Results: {}, .OpaqueValue: Operand, /*IsInvalid=*/false};
465
466	using ACT = ReadySuspendResumeResult::AwaitCallType;
467
468	auto BuildSubExpr = [&](ACT CallType, StringRef Func,
469	MultiExprArg Arg) -> Expr * {
470	ExprResult Result = buildMemberCall(S, Operand, Loc, Func, Arg);
471	if (Result.isInvalid()) {
472	Calls.IsInvalid = true;
473	return nullptr;
474	}
475	Calls.Results[CallType] = Result.get();
476	return Result.get();
477	};
478
479	CallExpr *AwaitReady = cast_or_null<CallExpr>(
480	Val: BuildSubExpr(ACT::ACT_Ready, "await_ready", std::nullopt));
481	if (!AwaitReady)
482	return Calls;
483	if (!AwaitReady->getType()->isDependentType()) {
484	// [expr.await]p3 [...]
485	// — await-ready is the expression e.await_ready(), contextually converted
486	// to bool.
487	ExprResult Conv = S.PerformContextuallyConvertToBool(AwaitReady);
488	if (Conv.isInvalid()) {
489	S.Diag(AwaitReady->getDirectCallee()->getBeginLoc(),
490	diag::note_await_ready_no_bool_conversion);
491	S.Diag(Loc, diag::note_coroutine_promise_call_implicitly_required)
492	<< AwaitReady->getDirectCallee() << E->getSourceRange();
493	Calls.IsInvalid = true;
494	} else
495	Calls.Results[ACT::ACT_Ready] = S.MaybeCreateExprWithCleanups(SubExpr: Conv.get());
496	}
497
498	ExprResult CoroHandleRes =
499	buildCoroutineHandle(S, CoroPromise->getType(), Loc);
500	if (CoroHandleRes.isInvalid()) {
501	Calls.IsInvalid = true;
502	return Calls;
503	}
504	Expr *CoroHandle = CoroHandleRes.get();
505	CallExpr *AwaitSuspend = cast_or_null<CallExpr>(
506	Val: BuildSubExpr(ACT::ACT_Suspend, "await_suspend", CoroHandle));
507	if (!AwaitSuspend)
508	return Calls;
509	if (!AwaitSuspend->getType()->isDependentType()) {
510	// [expr.await]p3 [...]
511	// - await-suspend is the expression e.await_suspend(h), which shall be
512	// a prvalue of type void, bool, or std::coroutine_handle<Z> for some
513	// type Z.
514	QualType RetType = AwaitSuspend->getCallReturnType(Ctx: S.Context);
515
516	// We need to mark await_suspend as noinline temporarily. See the comment
517	// of tryMarkAwaitSuspendNoInline for details.
518	tryMarkAwaitSuspendNoInline(S, Awaiter: Operand, AwaitSuspend);
519
520	// Support for coroutine_handle returning await_suspend.
521	if (Expr *TailCallSuspend =
522	maybeTailCall(S, RetType, AwaitSuspend, Loc))
523	// Note that we don't wrap the expression with ExprWithCleanups here
524	// because that might interfere with tailcall contract (e.g. inserting
525	// clean up instructions in-between tailcall and return). Instead
526	// ExprWithCleanups is wrapped within maybeTailCall() prior to the resume
527	// call.
528	Calls.Results[ACT::ACT_Suspend] = TailCallSuspend;
529	else {
530	// non-class prvalues always have cv-unqualified types
531	if (RetType->isReferenceType() ||
532	(!RetType->isBooleanType() && !RetType->isVoidType())) {
533	S.Diag(AwaitSuspend->getCalleeDecl()->getLocation(),
534	diag::err_await_suspend_invalid_return_type)
535	<< RetType;
536	S.Diag(Loc, diag::note_coroutine_promise_call_implicitly_required)
537	<< AwaitSuspend->getDirectCallee();
538	Calls.IsInvalid = true;
539	} else
540	Calls.Results[ACT::ACT_Suspend] =
541	S.MaybeCreateExprWithCleanups(AwaitSuspend);
542	}
543	}
544
545	BuildSubExpr(ACT::ACT_Resume, "await_resume", std::nullopt);
546
547	// Make sure the awaiter object gets a chance to be cleaned up.
548	S.Cleanup.setExprNeedsCleanups(true);
549
550	return Calls;
551	}
552
553	static ExprResult buildPromiseCall(Sema &S, VarDecl *Promise,
554	SourceLocation Loc, StringRef Name,
555	MultiExprArg Args) {
556
557	// Form a reference to the promise.
558	ExprResult PromiseRef = S.BuildDeclRefExpr(
559	Promise, Promise->getType().getNonReferenceType(), VK_LValue, Loc);
560	if (PromiseRef.isInvalid())
561	return ExprError();
562
563	return buildMemberCall(S, Base: PromiseRef.get(), Loc, Name, Args);
564	}
565
566	VarDecl *Sema::buildCoroutinePromise(SourceLocation Loc) {
567	assert(isa<FunctionDecl>(CurContext) && "not in a function scope");
568	auto *FD = cast<FunctionDecl>(Val: CurContext);
569	bool IsThisDependentType = [&] {
570	if (const auto *MD = dyn_cast_if_present<CXXMethodDecl>(Val: FD))
571	return MD->isImplicitObjectMemberFunction() &&
572	MD->getThisType()->isDependentType();
573	return false;
574	}();
575
576	QualType T = FD->getType()->isDependentType() || IsThisDependentType
577	? Context.DependentTy
578	: lookupPromiseType(S&: *this, FD, KwLoc: Loc);
579	if (T.isNull())
580	return nullptr;
581
582	auto *VD = VarDecl::Create(C&: Context, DC: FD, StartLoc: FD->getLocation(), IdLoc: FD->getLocation(),
583	Id: &PP.getIdentifierTable().get(Name: "__promise"), T,
584	TInfo: Context.getTrivialTypeSourceInfo(T, Loc), S: SC_None);
585	VD->setImplicit();
586	CheckVariableDeclarationType(NewVD: VD);
587	if (VD->isInvalidDecl())
588	return nullptr;
589
590	auto *ScopeInfo = getCurFunction();
591
592	// Build a list of arguments, based on the coroutine function's arguments,
593	// that if present will be passed to the promise type's constructor.
594	llvm::SmallVector<Expr *, 4> CtorArgExprs;
595
596	// Add implicit object parameter.
597	if (auto *MD = dyn_cast<CXXMethodDecl>(Val: FD)) {
598	if (MD->isImplicitObjectMemberFunction() && !isLambdaCallOperator(MD)) {
599	ExprResult ThisExpr = ActOnCXXThis(loc: Loc);
600	if (ThisExpr.isInvalid())
601	return nullptr;
602	ThisExpr = CreateBuiltinUnaryOp(OpLoc: Loc, Opc: UO_Deref, InputExpr: ThisExpr.get());
603	if (ThisExpr.isInvalid())
604	return nullptr;
605	CtorArgExprs.push_back(Elt: ThisExpr.get());
606	}
607	}
608
609	// Add the coroutine function's parameters.
610	auto &Moves = ScopeInfo->CoroutineParameterMoves;
611	for (auto *PD : FD->parameters()) {
612	if (PD->getType()->isDependentType())
613	continue;
614
615	auto RefExpr = ExprEmpty();
616	auto Move = Moves.find(Key: PD);
617	assert(Move != Moves.end() &&
618	"Coroutine function parameter not inserted into move map");
619	// If a reference to the function parameter exists in the coroutine
620	// frame, use that reference.
621	auto *MoveDecl =
622	cast<VarDecl>(Val: cast<DeclStmt>(Val: Move->second)->getSingleDecl());
623	RefExpr =
624	BuildDeclRefExpr(MoveDecl, MoveDecl->getType().getNonReferenceType(),
625	ExprValueKind::VK_LValue, FD->getLocation());
626	if (RefExpr.isInvalid())
627	return nullptr;
628	CtorArgExprs.push_back(Elt: RefExpr.get());
629	}
630
631	// If we have a non-zero number of constructor arguments, try to use them.
632	// Otherwise, fall back to the promise type's default constructor.
633	if (!CtorArgExprs.empty()) {
634	// Create an initialization sequence for the promise type using the
635	// constructor arguments, wrapped in a parenthesized list expression.
636	Expr *PLE = ParenListExpr::Create(Ctx: Context, LParenLoc: FD->getLocation(),
637	Exprs: CtorArgExprs, RParenLoc: FD->getLocation());
638	InitializedEntity Entity = InitializedEntity::InitializeVariable(Var: VD);
639	InitializationKind Kind = InitializationKind::CreateForInit(
640	Loc: VD->getLocation(), /*DirectInit=*/true, Init: PLE);
641	InitializationSequence InitSeq(*this, Entity, Kind, CtorArgExprs,
642	/*TopLevelOfInitList=*/false,
643	/*TreatUnavailableAsInvalid=*/false);
644
645	// [dcl.fct.def.coroutine]5.7
646	// promise-constructor-arguments is determined as follows: overload
647	// resolution is performed on a promise constructor call created by
648	// assembling an argument list q_1 ... q_n . If a viable constructor is
649	// found ([over.match.viable]), then promise-constructor-arguments is ( q_1
650	// , ..., q_n ), otherwise promise-constructor-arguments is empty.
651	if (InitSeq) {
652	ExprResult Result = InitSeq.Perform(S&: *this, Entity, Kind, Args: CtorArgExprs);
653	if (Result.isInvalid()) {
654	VD->setInvalidDecl();
655	} else if (Result.get()) {
656	VD->setInit(MaybeCreateExprWithCleanups(SubExpr: Result.get()));
657	VD->setInitStyle(VarDecl::CallInit);
658	CheckCompleteVariableDeclaration(VD: VD);
659	}
660	} else
661	ActOnUninitializedDecl(dcl: VD);
662	} else
663	ActOnUninitializedDecl(dcl: VD);
664
665	FD->addDecl(D: VD);
666	return VD;
667	}
668
669	/// Check that this is a context in which a coroutine suspension can appear.
670	static FunctionScopeInfo *checkCoroutineContext(Sema &S, SourceLocation Loc,
671	StringRef Keyword,
672	bool IsImplicit = false) {
673	if (!isValidCoroutineContext(S, Loc, Keyword))
674	return nullptr;
675
676	assert(isa<FunctionDecl>(S.CurContext) && "not in a function scope");
677
678	auto *ScopeInfo = S.getCurFunction();
679	assert(ScopeInfo && "missing function scope for function");
680
681	if (ScopeInfo->FirstCoroutineStmtLoc.isInvalid() && !IsImplicit)
682	ScopeInfo->setFirstCoroutineStmt(Loc, Keyword);
683
684	if (ScopeInfo->CoroutinePromise)
685	return ScopeInfo;
686
687	if (!S.buildCoroutineParameterMoves(Loc))
688	return nullptr;
689
690	ScopeInfo->CoroutinePromise = S.buildCoroutinePromise(Loc);
691	if (!ScopeInfo->CoroutinePromise)
692	return nullptr;
693
694	return ScopeInfo;
695	}
696
697	/// Recursively check \p E and all its children to see if any call target
698	/// (including constructor call) is declared noexcept. Also any value returned
699	/// from the call has a noexcept destructor.
700	static void checkNoThrow(Sema &S, const Stmt *E,
701	llvm::SmallPtrSetImpl<const Decl *> &ThrowingDecls) {
702	auto checkDeclNoexcept = [&](const Decl *D, bool IsDtor = false) {
703	// In the case of dtor, the call to dtor is implicit and hence we should
704	// pass nullptr to canCalleeThrow.
705	if (Sema::canCalleeThrow(S, E: IsDtor ? nullptr : cast<Expr>(Val: E), D)) {
706	if (const auto *FD = dyn_cast<FunctionDecl>(Val: D)) {
707	// co_await promise.final_suspend() could end up calling
708	// __builtin_coro_resume for symmetric transfer if await_suspend()
709	// returns a handle. In that case, even __builtin_coro_resume is not
710	// declared as noexcept and may throw, it does not throw _into_ the
711	// coroutine that just suspended, but rather throws back out from
712	// whoever called coroutine_handle::resume(), hence we claim that
713	// logically it does not throw.
714	if (FD->getBuiltinID() == Builtin::BI__builtin_coro_resume)
715	return;
716	}
717	if (ThrowingDecls.empty()) {
718	// [dcl.fct.def.coroutine]p15
719	// The expression co_await promise.final_suspend() shall not be
720	// potentially-throwing ([except.spec]).
721	//
722	// First time seeing an error, emit the error message.
723	S.Diag(cast<FunctionDecl>(S.CurContext)->getLocation(),
724	diag::err_coroutine_promise_final_suspend_requires_nothrow);
725	}
726	ThrowingDecls.insert(Ptr: D);
727	}
728	};
729
730	if (auto *CE = dyn_cast<CXXConstructExpr>(Val: E)) {
731	CXXConstructorDecl *Ctor = CE->getConstructor();
732	checkDeclNoexcept(Ctor);
733	// Check the corresponding destructor of the constructor.
734	checkDeclNoexcept(Ctor->getParent()->getDestructor(), /*IsDtor=*/true);
735	} else if (auto *CE = dyn_cast<CallExpr>(Val: E)) {
736	if (CE->isTypeDependent())
737	return;
738
739	checkDeclNoexcept(CE->getCalleeDecl());
740	QualType ReturnType = CE->getCallReturnType(Ctx: S.getASTContext());
741	// Check the destructor of the call return type, if any.
742	if (ReturnType.isDestructedType() ==
743	QualType::DestructionKind::DK_cxx_destructor) {
744	const auto *T =
745	cast<RecordType>(Val: ReturnType.getCanonicalType().getTypePtr());
746	checkDeclNoexcept(cast<CXXRecordDecl>(Val: T->getDecl())->getDestructor(),
747	/*IsDtor=*/true);
748	}
749	} else
750	for (const auto *Child : E->children()) {
751	if (!Child)
752	continue;
753	checkNoThrow(S, E: Child, ThrowingDecls);
754	}
755	}
756
757	bool Sema::checkFinalSuspendNoThrow(const Stmt *FinalSuspend) {
758	llvm::SmallPtrSet<const Decl *, 4> ThrowingDecls;
759	// We first collect all declarations that should not throw but not declared
760	// with noexcept. We then sort them based on the location before printing.
761	// This is to avoid emitting the same note multiple times on the same
762	// declaration, and also provide a deterministic order for the messages.
763	checkNoThrow(S&: *this, E: FinalSuspend, ThrowingDecls);
764	auto SortedDecls = llvm::SmallVector<const Decl *, 4>{ThrowingDecls.begin(),
765	ThrowingDecls.end()};
766	sort(C&: SortedDecls, Comp: [](const Decl *A, const Decl *B) {
767	return A->getEndLoc() < B->getEndLoc();
768	});
769	for (const auto *D : SortedDecls) {
770	Diag(D->getEndLoc(), diag::note_coroutine_function_declare_noexcept);
771	}
772	return ThrowingDecls.empty();
773	}
774
775	bool Sema::ActOnCoroutineBodyStart(Scope *SC, SourceLocation KWLoc,
776	StringRef Keyword) {
777	// Ignore previous expr evaluation contexts.
778	EnterExpressionEvaluationContext PotentiallyEvaluated(
779	*this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
780	if (!checkCoroutineContext(S&: *this, Loc: KWLoc, Keyword))
781	return false;
782	auto *ScopeInfo = getCurFunction();
783	assert(ScopeInfo->CoroutinePromise);
784
785	// If we have existing coroutine statements then we have already built
786	// the initial and final suspend points.
787	if (!ScopeInfo->NeedsCoroutineSuspends)
788	return true;
789
790	ScopeInfo->setNeedsCoroutineSuspends(false);
791
792	auto *Fn = cast<FunctionDecl>(Val: CurContext);
793	SourceLocation Loc = Fn->getLocation();
794	// Build the initial suspend point
795	auto buildSuspends = [&](StringRef Name) mutable -> StmtResult {
796	ExprResult Operand = buildPromiseCall(S&: *this, Promise: ScopeInfo->CoroutinePromise,
797	Loc, Name, Args: std::nullopt);
798	if (Operand.isInvalid())
799	return StmtError();
800	ExprResult Suspend =
801	buildOperatorCoawaitCall(SemaRef&: *this, S: SC, Loc, E: Operand.get());
802	if (Suspend.isInvalid())
803	return StmtError();
804	Suspend = BuildResolvedCoawaitExpr(KwLoc: Loc, Operand: Operand.get(), Awaiter: Suspend.get(),
805	/*IsImplicit*/ true);
806	Suspend = ActOnFinishFullExpr(Expr: Suspend.get(), /*DiscardedValue*/ false);
807	if (Suspend.isInvalid()) {
808	Diag(Loc, diag::note_coroutine_promise_suspend_implicitly_required)
809	<< ((Name == "initial_suspend") ? 0 : 1);
810	Diag(KWLoc, diag::note_declared_coroutine_here) << Keyword;
811	return StmtError();
812	}
813	return cast<Stmt>(Val: Suspend.get());
814	};
815
816	StmtResult InitSuspend = buildSuspends("initial_suspend");
817	if (InitSuspend.isInvalid())
818	return true;
819
820	StmtResult FinalSuspend = buildSuspends("final_suspend");
821	if (FinalSuspend.isInvalid() || !checkFinalSuspendNoThrow(FinalSuspend: FinalSuspend.get()))
822	return true;
823
824	ScopeInfo->setCoroutineSuspends(Initial: InitSuspend.get(), Final: FinalSuspend.get());
825
826	return true;
827	}
828
829	// Recursively walks up the scope hierarchy until either a 'catch' or a function
830	// scope is found, whichever comes first.
831	static bool isWithinCatchScope(Scope *S) {
832	// 'co_await' and 'co_yield' keywords are disallowed within catch blocks, but
833	// lambdas that use 'co_await' are allowed. The loop below ends when a
834	// function scope is found in order to ensure the following behavior:
835	//
836	// void foo() { // <- function scope
837	// try { //
838	// co_await x; // <- 'co_await' is OK within a function scope
839	// } catch { // <- catch scope
840	// co_await x; // <- 'co_await' is not OK within a catch scope
841	// []() { // <- function scope
842	// co_await x; // <- 'co_await' is OK within a function scope
843	// }();
844	// }
845	// }
846	while (S && !S->isFunctionScope()) {
847	if (S->isCatchScope())
848	return true;
849	S = S->getParent();
850	}
851	return false;
852	}
853
854	// [expr.await]p2, emphasis added: "An await-expression shall appear only in
855	// a *potentially evaluated* expression within the compound-statement of a
856	// function-body *outside of a handler* [...] A context within a function
857	// where an await-expression can appear is called a suspension context of the
858	// function."
859	static bool checkSuspensionContext(Sema &S, SourceLocation Loc,
860	StringRef Keyword) {
861	// First emphasis of [expr.await]p2: must be a potentially evaluated context.
862	// That is, 'co_await' and 'co_yield' cannot appear in subexpressions of
863	// \c sizeof.
864	if (S.isUnevaluatedContext()) {
865	S.Diag(Loc, diag::err_coroutine_unevaluated_context) << Keyword;
866	return false;
867	}
868
869	// Second emphasis of [expr.await]p2: must be outside of an exception handler.
870	if (isWithinCatchScope(S: S.getCurScope())) {
871	S.Diag(Loc, diag::err_coroutine_within_handler) << Keyword;
872	return false;
873	}
874
875	return true;
876	}
877
878	ExprResult Sema::ActOnCoawaitExpr(Scope *S, SourceLocation Loc, Expr *E) {
879	if (!checkSuspensionContext(S&: *this, Loc, Keyword: "co_await"))
880	return ExprError();
881
882	if (!ActOnCoroutineBodyStart(SC: S, KWLoc: Loc, Keyword: "co_await")) {
883	CorrectDelayedTyposInExpr(E);
884	return ExprError();
885	}
886
887	if (E->hasPlaceholderType()) {
888	ExprResult R = CheckPlaceholderExpr(E);
889	if (R.isInvalid()) return ExprError();
890	E = R.get();
891	}
892	ExprResult Lookup = BuildOperatorCoawaitLookupExpr(S, Loc);
893	if (Lookup.isInvalid())
894	return ExprError();
895	return BuildUnresolvedCoawaitExpr(KwLoc: Loc, Operand: E,
896	Lookup: cast<UnresolvedLookupExpr>(Val: Lookup.get()));
897	}
898
899	ExprResult Sema::BuildOperatorCoawaitLookupExpr(Scope *S, SourceLocation Loc) {
900	DeclarationName OpName =
901	Context.DeclarationNames.getCXXOperatorName(Op: OO_Coawait);
902	LookupResult Operators(*this, OpName, SourceLocation(),
903	Sema::LookupOperatorName);
904	LookupName(R&: Operators, S);
905
906	assert(!Operators.isAmbiguous() && "Operator lookup cannot be ambiguous");
907	const auto &Functions = Operators.asUnresolvedSet();
908	bool IsOverloaded =
909	Functions.size() > 1 ||
910	(Functions.size() == 1 && isa<FunctionTemplateDecl>(Val: *Functions.begin()));
911	Expr *CoawaitOp = UnresolvedLookupExpr::Create(
912	Context, /*NamingClass*/ nullptr, QualifierLoc: NestedNameSpecifierLoc(),
913	NameInfo: DeclarationNameInfo(OpName, Loc), /*RequiresADL*/ true, Overloaded: IsOverloaded,
914	Begin: Functions.begin(), End: Functions.end());
915	assert(CoawaitOp);
916	return CoawaitOp;
917	}
918
919	// Attempts to resolve and build a CoawaitExpr from "raw" inputs, bailing out to
920	// DependentCoawaitExpr if needed.
921	ExprResult Sema::BuildUnresolvedCoawaitExpr(SourceLocation Loc, Expr *Operand,
922	UnresolvedLookupExpr *Lookup) {
923	auto *FSI = checkCoroutineContext(S&: *this, Loc, Keyword: "co_await");
924	if (!FSI)
925	return ExprError();
926
927	if (Operand->hasPlaceholderType()) {
928	ExprResult R = CheckPlaceholderExpr(E: Operand);
929	if (R.isInvalid())
930	return ExprError();
931	Operand = R.get();
932	}
933
934	auto *Promise = FSI->CoroutinePromise;
935	if (Promise->getType()->isDependentType()) {
936	Expr *Res = new (Context)
937	DependentCoawaitExpr(Loc, Context.DependentTy, Operand, Lookup);
938	return Res;
939	}
940
941	auto *RD = Promise->getType()->getAsCXXRecordDecl();
942	auto *Transformed = Operand;
943	if (lookupMember(*this, "await_transform", RD, Loc)) {
944	ExprResult R =
945	buildPromiseCall(S&: *this, Promise, Loc, Name: "await_transform", Args: Operand);
946	if (R.isInvalid()) {
947	Diag(Loc,
948	diag::note_coroutine_promise_implicit_await_transform_required_here)
949	<< Operand->getSourceRange();
950	return ExprError();
951	}
952	Transformed = R.get();
953	}
954	ExprResult Awaiter = BuildOperatorCoawaitCall(Loc, E: Transformed, Lookup);
955	if (Awaiter.isInvalid())
956	return ExprError();
957
958	return BuildResolvedCoawaitExpr(KwLoc: Loc, Operand, Awaiter: Awaiter.get());
959	}
960
961	ExprResult Sema::BuildResolvedCoawaitExpr(SourceLocation Loc, Expr *Operand,
962	Expr *Awaiter, bool IsImplicit) {
963	auto *Coroutine = checkCoroutineContext(S&: *this, Loc, Keyword: "co_await", IsImplicit);
964	if (!Coroutine)
965	return ExprError();
966
967	if (Awaiter->hasPlaceholderType()) {
968	ExprResult R = CheckPlaceholderExpr(E: Awaiter);
969	if (R.isInvalid()) return ExprError();
970	Awaiter = R.get();
971	}
972
973	if (Awaiter->getType()->isDependentType()) {
974	Expr *Res = new (Context)
975	CoawaitExpr(Loc, Context.DependentTy, Operand, Awaiter, IsImplicit);
976	return Res;
977	}
978
979	// If the expression is a temporary, materialize it as an lvalue so that we
980	// can use it multiple times.
981	if (Awaiter->isPRValue())
982	Awaiter = CreateMaterializeTemporaryExpr(T: Awaiter->getType(), Temporary: Awaiter, BoundToLvalueReference: true);
983
984	// The location of the `co_await` token cannot be used when constructing
985	// the member call expressions since it's before the location of `Expr`, which
986	// is used as the start of the member call expression.
987	SourceLocation CallLoc = Awaiter->getExprLoc();
988
989	// Build the await_ready, await_suspend, await_resume calls.
990	ReadySuspendResumeResult RSS =
991	buildCoawaitCalls(S&: *this, CoroPromise: Coroutine->CoroutinePromise, Loc: CallLoc, E: Awaiter);
992	if (RSS.IsInvalid)
993	return ExprError();
994
995	Expr *Res = new (Context)
996	CoawaitExpr(Loc, Operand, Awaiter, RSS.Results[0], RSS.Results[1],
997	RSS.Results[2], RSS.OpaqueValue, IsImplicit);
998
999	return Res;
1000	}
1001
1002	ExprResult Sema::ActOnCoyieldExpr(Scope *S, SourceLocation Loc, Expr *E) {
1003	if (!checkSuspensionContext(S&: *this, Loc, Keyword: "co_yield"))
1004	return ExprError();
1005
1006	if (!ActOnCoroutineBodyStart(SC: S, KWLoc: Loc, Keyword: "co_yield")) {
1007	CorrectDelayedTyposInExpr(E);
1008	return ExprError();
1009	}
1010
1011	// Build yield_value call.
1012	ExprResult Awaitable = buildPromiseCall(
1013	S&: *this, Promise: getCurFunction()->CoroutinePromise, Loc, Name: "yield_value", Args: E);
1014	if (Awaitable.isInvalid())
1015	return ExprError();
1016
1017	// Build 'operator co_await' call.
1018	Awaitable = buildOperatorCoawaitCall(SemaRef&: *this, S, Loc, E: Awaitable.get());
1019	if (Awaitable.isInvalid())
1020	return ExprError();
1021
1022	return BuildCoyieldExpr(KwLoc: Loc, E: Awaitable.get());
1023	}
1024	ExprResult Sema::BuildCoyieldExpr(SourceLocation Loc, Expr *E) {
1025	auto *Coroutine = checkCoroutineContext(S&: *this, Loc, Keyword: "co_yield");
1026	if (!Coroutine)
1027	return ExprError();
1028
1029	if (E->hasPlaceholderType()) {
1030	ExprResult R = CheckPlaceholderExpr(E);
1031	if (R.isInvalid()) return ExprError();
1032	E = R.get();
1033	}
1034
1035	Expr *Operand = E;
1036
1037	if (E->getType()->isDependentType()) {
1038	Expr *Res = new (Context) CoyieldExpr(Loc, Context.DependentTy, Operand, E);
1039	return Res;
1040	}
1041
1042	// If the expression is a temporary, materialize it as an lvalue so that we
1043	// can use it multiple times.
1044	if (E->isPRValue())
1045	E = CreateMaterializeTemporaryExpr(T: E->getType(), Temporary: E, BoundToLvalueReference: true);
1046
1047	// Build the await_ready, await_suspend, await_resume calls.
1048	ReadySuspendResumeResult RSS = buildCoawaitCalls(
1049	S&: *this, CoroPromise: Coroutine->CoroutinePromise, Loc, E);
1050	if (RSS.IsInvalid)
1051	return ExprError();
1052
1053	Expr *Res =
1054	new (Context) CoyieldExpr(Loc, Operand, E, RSS.Results[0], RSS.Results[1],
1055	RSS.Results[2], RSS.OpaqueValue);
1056
1057	return Res;
1058	}
1059
1060	StmtResult Sema::ActOnCoreturnStmt(Scope *S, SourceLocation Loc, Expr *E) {
1061	if (!ActOnCoroutineBodyStart(SC: S, KWLoc: Loc, Keyword: "co_return")) {
1062	CorrectDelayedTyposInExpr(E);
1063	return StmtError();
1064	}
1065	return BuildCoreturnStmt(KwLoc: Loc, E);
1066	}
1067
1068	StmtResult Sema::BuildCoreturnStmt(SourceLocation Loc, Expr *E,
1069	bool IsImplicit) {
1070	auto *FSI = checkCoroutineContext(S&: *this, Loc, Keyword: "co_return", IsImplicit);
1071	if (!FSI)
1072	return StmtError();
1073
1074	if (E && E->hasPlaceholderType() &&
1075	!E->hasPlaceholderType(K: BuiltinType::Overload)) {
1076	ExprResult R = CheckPlaceholderExpr(E);
1077	if (R.isInvalid()) return StmtError();
1078	E = R.get();
1079	}
1080
1081	VarDecl *Promise = FSI->CoroutinePromise;
1082	ExprResult PC;
1083	if (E && (isa<InitListExpr>(Val: E) || !E->getType()->isVoidType())) {
1084	getNamedReturnInfo(E, Mode: SimplerImplicitMoveMode::ForceOn);
1085	PC = buildPromiseCall(S&: *this, Promise, Loc, Name: "return_value", Args: E);
1086	} else {
1087	E = MakeFullDiscardedValueExpr(Arg: E).get();
1088	PC = buildPromiseCall(S&: *this, Promise, Loc, Name: "return_void", Args: std::nullopt);
1089	}
1090	if (PC.isInvalid())
1091	return StmtError();
1092
1093	Expr *PCE = ActOnFinishFullExpr(Expr: PC.get(), /*DiscardedValue*/ false).get();
1094
1095	Stmt *Res = new (Context) CoreturnStmt(Loc, E, PCE, IsImplicit);
1096	return Res;
1097	}
1098
1099	/// Look up the std::nothrow object.
1100	static Expr *buildStdNoThrowDeclRef(Sema &S, SourceLocation Loc) {
1101	NamespaceDecl *Std = S.getStdNamespace();
1102	assert(Std && "Should already be diagnosed");
1103
1104	LookupResult Result(S, &S.PP.getIdentifierTable().get(Name: "nothrow"), Loc,
1105	Sema::LookupOrdinaryName);
1106	if (!S.LookupQualifiedName(Result, Std)) {
1107	// <coroutine> is not requred to include <new>, so we couldn't omit
1108	// the check here.
1109	S.Diag(Loc, diag::err_implicit_coroutine_std_nothrow_type_not_found);
1110	return nullptr;
1111	}
1112
1113	auto *VD = Result.getAsSingle<VarDecl>();
1114	if (!VD) {
1115	Result.suppressDiagnostics();
1116	// We found something weird. Complain about the first thing we found.
1117	NamedDecl *Found = *Result.begin();
1118	S.Diag(Found->getLocation(), diag::err_malformed_std_nothrow);
1119	return nullptr;
1120	}
1121
1122	ExprResult DR = S.BuildDeclRefExpr(VD, VD->getType(), VK_LValue, Loc);
1123	if (DR.isInvalid())
1124	return nullptr;
1125
1126	return DR.get();
1127	}
1128
1129	static TypeSourceInfo *getTypeSourceInfoForStdAlignValT(Sema &S,
1130	SourceLocation Loc) {
1131	EnumDecl *StdAlignValT = S.getStdAlignValT();
1132	QualType StdAlignValDecl = S.Context.getTypeDeclType(StdAlignValT);
1133	return S.Context.getTrivialTypeSourceInfo(T: StdAlignValDecl);
1134	}
1135
1136	// Find an appropriate delete for the promise.
1137	static bool findDeleteForPromise(Sema &S, SourceLocation Loc, QualType PromiseType,
1138	FunctionDecl *&OperatorDelete) {
1139	DeclarationName DeleteName =
1140	S.Context.DeclarationNames.getCXXOperatorName(Op: OO_Delete);
1141
1142	auto *PointeeRD = PromiseType->getAsCXXRecordDecl();
1143	assert(PointeeRD && "PromiseType must be a CxxRecordDecl type");
1144
1145	const bool Overaligned = S.getLangOpts().CoroAlignedAllocation;
1146
1147	// [dcl.fct.def.coroutine]p12
1148	// The deallocation function's name is looked up by searching for it in the
1149	// scope of the promise type. If nothing is found, a search is performed in
1150	// the global scope.
1151	if (S.FindDeallocationFunction(StartLoc: Loc, RD: PointeeRD, Name: DeleteName, Operator&: OperatorDelete,
1152	/*Diagnose*/ true, /*WantSize*/ true,
1153	/*WantAligned*/ Overaligned))
1154	return false;
1155
1156	// [dcl.fct.def.coroutine]p12
1157	// If both a usual deallocation function with only a pointer parameter and a
1158	// usual deallocation function with both a pointer parameter and a size
1159	// parameter are found, then the selected deallocation function shall be the
1160	// one with two parameters. Otherwise, the selected deallocation function
1161	// shall be the function with one parameter.
1162	if (!OperatorDelete) {
1163	// Look for a global declaration.
1164	// Coroutines can always provide their required size.
1165	const bool CanProvideSize = true;
1166	// Sema::FindUsualDeallocationFunction will try to find the one with two
1167	// parameters first. It will return the deallocation function with one
1168	// parameter if failed.
1169	OperatorDelete = S.FindUsualDeallocationFunction(StartLoc: Loc, CanProvideSize,
1170	Overaligned, Name: DeleteName);
1171
1172	if (!OperatorDelete)
1173	return false;
1174	}
1175
1176	S.MarkFunctionReferenced(Loc, Func: OperatorDelete);
1177	return true;
1178	}
1179
1180
1181	void Sema::CheckCompletedCoroutineBody(FunctionDecl *FD, Stmt *&Body) {
1182	FunctionScopeInfo *Fn = getCurFunction();
1183	assert(Fn && Fn->isCoroutine() && "not a coroutine");
1184	if (!Body) {
1185	assert(FD->isInvalidDecl() &&
1186	"a null body is only allowed for invalid declarations");
1187	return;
1188	}
1189	// We have a function that uses coroutine keywords, but we failed to build
1190	// the promise type.
1191	if (!Fn->CoroutinePromise)
1192	return FD->setInvalidDecl();
1193
1194	if (isa<CoroutineBodyStmt>(Val: Body)) {
1195	// Nothing todo. the body is already a transformed coroutine body statement.
1196	return;
1197	}
1198
1199	// The always_inline attribute doesn't reliably apply to a coroutine,
1200	// because the coroutine will be split into pieces and some pieces
1201	// might be called indirectly, as in a virtual call. Even the ramp
1202	// function cannot be inlined at -O0, due to pipeline ordering
1203	// problems (see https://llvm.org/PR53413). Tell the user about it.
1204	if (FD->hasAttr<AlwaysInlineAttr>())
1205	Diag(FD->getLocation(), diag::warn_always_inline_coroutine);
1206
1207	// The design of coroutines means we cannot allow use of VLAs within one, so
1208	// diagnose if we've seen a VLA in the body of this function.
1209	if (Fn->FirstVLALoc.isValid())
1210	Diag(Fn->FirstVLALoc, diag::err_vla_in_coroutine_unsupported);
1211
1212	// [stmt.return.coroutine]p1:
1213	// A coroutine shall not enclose a return statement ([stmt.return]).
1214	if (Fn->FirstReturnLoc.isValid()) {
1215	assert(Fn->FirstCoroutineStmtLoc.isValid() &&
1216	"first coroutine location not set");
1217	Diag(Fn->FirstReturnLoc, diag::err_return_in_coroutine);
1218	Diag(Fn->FirstCoroutineStmtLoc, diag::note_declared_coroutine_here)
1219	<< Fn->getFirstCoroutineStmtKeyword();
1220	}
1221
1222	// Coroutines will get splitted into pieces. The GNU address of label
1223	// extension wouldn't be meaningful in coroutines.
1224	for (AddrLabelExpr *ALE : Fn->AddrLabels)
1225	Diag(ALE->getBeginLoc(), diag::err_coro_invalid_addr_of_label);
1226
1227	CoroutineStmtBuilder Builder(*this, *FD, *Fn, Body);
1228	if (Builder.isInvalid() || !Builder.buildStatements())
1229	return FD->setInvalidDecl();
1230
1231	// Build body for the coroutine wrapper statement.
1232	Body = CoroutineBodyStmt::Create(C: Context, Args: Builder);
1233	}
1234
1235	static CompoundStmt *buildCoroutineBody(Stmt *Body, ASTContext &Context) {
1236	if (auto *CS = dyn_cast<CompoundStmt>(Val: Body))
1237	return CS;
1238
1239	// The body of the coroutine may be a try statement if it is in
1240	// 'function-try-block' syntax. Here we wrap it into a compound
1241	// statement for consistency.
1242	assert(isa<CXXTryStmt>(Body) && "Unimaged coroutine body type");
1243	return CompoundStmt::Create(C: Context, Stmts: {Body}, FPFeatures: FPOptionsOverride(),
1244	LB: SourceLocation(), RB: SourceLocation());
1245	}
1246
1247	CoroutineStmtBuilder::CoroutineStmtBuilder(Sema &S, FunctionDecl &FD,
1248	sema::FunctionScopeInfo &Fn,
1249	Stmt *Body)
1250	: S(S), FD(FD), Fn(Fn), Loc(FD.getLocation()),
1251	IsPromiseDependentType(
1252	!Fn.CoroutinePromise ||
1253	Fn.CoroutinePromise->getType()->isDependentType()) {
1254	this->Body = buildCoroutineBody(Body, Context&: S.getASTContext());
1255
1256	for (auto KV : Fn.CoroutineParameterMoves)
1257	this->ParamMovesVector.push_back(Elt: KV.second);
1258	this->ParamMoves = this->ParamMovesVector;
1259
1260	if (!IsPromiseDependentType) {
1261	PromiseRecordDecl = Fn.CoroutinePromise->getType()->getAsCXXRecordDecl();
1262	assert(PromiseRecordDecl && "Type should have already been checked");
1263	}
1264	this->IsValid = makePromiseStmt() && makeInitialAndFinalSuspend();
1265	}
1266
1267	bool CoroutineStmtBuilder::buildStatements() {
1268	assert(this->IsValid && "coroutine already invalid");
1269	this->IsValid = makeReturnObject();
1270	if (this->IsValid && !IsPromiseDependentType)
1271	buildDependentStatements();
1272	return this->IsValid;
1273	}
1274
1275	bool CoroutineStmtBuilder::buildDependentStatements() {
1276	assert(this->IsValid && "coroutine already invalid");
1277	assert(!this->IsPromiseDependentType &&
1278	"coroutine cannot have a dependent promise type");
1279	this->IsValid = makeOnException() && makeOnFallthrough() &&
1280	makeGroDeclAndReturnStmt() && makeReturnOnAllocFailure() &&
1281	makeNewAndDeleteExpr();
1282	return this->IsValid;
1283	}
1284
1285	bool CoroutineStmtBuilder::makePromiseStmt() {
1286	// Form a declaration statement for the promise declaration, so that AST
1287	// visitors can more easily find it.
1288	StmtResult PromiseStmt =
1289	S.ActOnDeclStmt(Decl: S.ConvertDeclToDeclGroup(Fn.CoroutinePromise), StartLoc: Loc, EndLoc: Loc);
1290	if (PromiseStmt.isInvalid())
1291	return false;
1292
1293	this->Promise = PromiseStmt.get();
1294	return true;
1295	}
1296
1297	bool CoroutineStmtBuilder::makeInitialAndFinalSuspend() {
1298	if (Fn.hasInvalidCoroutineSuspends())
1299	return false;
1300	this->InitialSuspend = cast<Expr>(Val: Fn.CoroutineSuspends.first);
1301	this->FinalSuspend = cast<Expr>(Val: Fn.CoroutineSuspends.second);
1302	return true;
1303	}
1304
1305	static bool diagReturnOnAllocFailure(Sema &S, Expr *E,
1306	CXXRecordDecl *PromiseRecordDecl,
1307	FunctionScopeInfo &Fn) {
1308	auto Loc = E->getExprLoc();
1309	if (auto *DeclRef = dyn_cast_or_null<DeclRefExpr>(Val: E)) {
1310	auto *Decl = DeclRef->getDecl();
1311	if (CXXMethodDecl *Method = dyn_cast_or_null<CXXMethodDecl>(Val: Decl)) {
1312	if (Method->isStatic())
1313	return true;
1314	else
1315	Loc = Decl->getLocation();
1316	}
1317	}
1318
1319	S.Diag(
1320	Loc,
1321	diag::err_coroutine_promise_get_return_object_on_allocation_failure)
1322	<< PromiseRecordDecl;
1323	S.Diag(Fn.FirstCoroutineStmtLoc, diag::note_declared_coroutine_here)
1324	<< Fn.getFirstCoroutineStmtKeyword();
1325	return false;
1326	}
1327
1328	bool CoroutineStmtBuilder::makeReturnOnAllocFailure() {
1329	assert(!IsPromiseDependentType &&
1330	"cannot make statement while the promise type is dependent");
1331
1332	// [dcl.fct.def.coroutine]p10
1333	// If a search for the name get_return_object_on_allocation_failure in
1334	// the scope of the promise type ([class.member.lookup]) finds any
1335	// declarations, then the result of a call to an allocation function used to
1336	// obtain storage for the coroutine state is assumed to return nullptr if it
1337	// fails to obtain storage, ... If the allocation function returns nullptr,
1338	// ... and the return value is obtained by a call to
1339	// T::get_return_object_on_allocation_failure(), where T is the
1340	// promise type.
1341	DeclarationName DN =
1342	S.PP.getIdentifierInfo(Name: "get_return_object_on_allocation_failure");
1343	LookupResult Found(S, DN, Loc, Sema::LookupMemberName);
1344	if (!S.LookupQualifiedName(Found, PromiseRecordDecl))
1345	return true;
1346
1347	CXXScopeSpec SS;
1348	ExprResult DeclNameExpr =
1349	S.BuildDeclarationNameExpr(SS, R&: Found, /*NeedsADL=*/false);
1350	if (DeclNameExpr.isInvalid())
1351	return false;
1352
1353	if (!diagReturnOnAllocFailure(S, E: DeclNameExpr.get(), PromiseRecordDecl, Fn))
1354	return false;
1355
1356	ExprResult ReturnObjectOnAllocationFailure =
1357	S.BuildCallExpr(S: nullptr, Fn: DeclNameExpr.get(), LParenLoc: Loc, ArgExprs: {}, RParenLoc: Loc);
1358	if (ReturnObjectOnAllocationFailure.isInvalid())
1359	return false;
1360
1361	StmtResult ReturnStmt =
1362	S.BuildReturnStmt(ReturnLoc: Loc, RetValExp: ReturnObjectOnAllocationFailure.get());
1363	if (ReturnStmt.isInvalid()) {
1364	S.Diag(Found.getFoundDecl()->getLocation(), diag::note_member_declared_here)
1365	<< DN;
1366	S.Diag(Fn.FirstCoroutineStmtLoc, diag::note_declared_coroutine_here)
1367	<< Fn.getFirstCoroutineStmtKeyword();
1368	return false;
1369	}
1370
1371	this->ReturnStmtOnAllocFailure = ReturnStmt.get();
1372	return true;
1373	}
1374
1375	// Collect placement arguments for allocation function of coroutine FD.
1376	// Return true if we collect placement arguments succesfully. Return false,
1377	// otherwise.
1378	static bool collectPlacementArgs(Sema &S, FunctionDecl &FD, SourceLocation Loc,
1379	SmallVectorImpl<Expr *> &PlacementArgs) {
1380	if (auto *MD = dyn_cast<CXXMethodDecl>(Val: &FD)) {
1381	if (MD->isImplicitObjectMemberFunction() && !isLambdaCallOperator(MD)) {
1382	ExprResult ThisExpr = S.ActOnCXXThis(loc: Loc);
1383	if (ThisExpr.isInvalid())
1384	return false;
1385	ThisExpr = S.CreateBuiltinUnaryOp(OpLoc: Loc, Opc: UO_Deref, InputExpr: ThisExpr.get());
1386	if (ThisExpr.isInvalid())
1387	return false;
1388	PlacementArgs.push_back(Elt: ThisExpr.get());
1389	}
1390	}
1391
1392	for (auto *PD : FD.parameters()) {
1393	if (PD->getType()->isDependentType())
1394	continue;
1395
1396	// Build a reference to the parameter.
1397	auto PDLoc = PD->getLocation();
1398	ExprResult PDRefExpr =
1399	S.BuildDeclRefExpr(PD, PD->getOriginalType().getNonReferenceType(),
1400	ExprValueKind::VK_LValue, PDLoc);
1401	if (PDRefExpr.isInvalid())
1402	return false;
1403
1404	PlacementArgs.push_back(Elt: PDRefExpr.get());
1405	}
1406
1407	return true;
1408	}
1409
1410	bool CoroutineStmtBuilder::makeNewAndDeleteExpr() {
1411	// Form and check allocation and deallocation calls.
1412	assert(!IsPromiseDependentType &&
1413	"cannot make statement while the promise type is dependent");
1414	QualType PromiseType = Fn.CoroutinePromise->getType();
1415
1416	if (S.RequireCompleteType(Loc, PromiseType, diag::err_incomplete_type))
1417	return false;
1418
1419	const bool RequiresNoThrowAlloc = ReturnStmtOnAllocFailure != nullptr;
1420
1421	// According to [dcl.fct.def.coroutine]p9, Lookup allocation functions using a
1422	// parameter list composed of the requested size of the coroutine state being
1423	// allocated, followed by the coroutine function's arguments. If a matching
1424	// allocation function exists, use it. Otherwise, use an allocation function
1425	// that just takes the requested size.
1426	//
1427	// [dcl.fct.def.coroutine]p9
1428	// An implementation may need to allocate additional storage for a
1429	// coroutine.
1430	// This storage is known as the coroutine state and is obtained by calling a
1431	// non-array allocation function ([basic.stc.dynamic.allocation]). The
1432	// allocation function's name is looked up by searching for it in the scope of
1433	// the promise type.
1434	// - If any declarations are found, overload resolution is performed on a
1435	// function call created by assembling an argument list. The first argument is
1436	// the amount of space requested, and has type std::size_t. The
1437	// lvalues p1 ... pn are the succeeding arguments.
1438	//
1439	// ...where "p1 ... pn" are defined earlier as:
1440	//
1441	// [dcl.fct.def.coroutine]p3
1442	// The promise type of a coroutine is `std::coroutine_traits<R, P1, ...,
1443	// Pn>`
1444	// , where R is the return type of the function, and `P1, ..., Pn` are the
1445	// sequence of types of the non-object function parameters, preceded by the
1446	// type of the object parameter ([dcl.fct]) if the coroutine is a non-static
1447	// member function. [dcl.fct.def.coroutine]p4 In the following, p_i is an
1448	// lvalue of type P_i, where p1 denotes the object parameter and p_i+1 denotes
1449	// the i-th non-object function parameter for a non-static member function,
1450	// and p_i denotes the i-th function parameter otherwise. For a non-static
1451	// member function, q_1 is an lvalue that denotes *this; any other q_i is an
1452	// lvalue that denotes the parameter copy corresponding to p_i.
1453
1454	FunctionDecl *OperatorNew = nullptr;
1455	SmallVector<Expr *, 1> PlacementArgs;
1456
1457	const bool PromiseContainsNew = [this, &PromiseType]() -> bool {
1458	DeclarationName NewName =
1459	S.getASTContext().DeclarationNames.getCXXOperatorName(Op: OO_New);
1460	LookupResult R(S, NewName, Loc, Sema::LookupOrdinaryName);
1461
1462	if (PromiseType->isRecordType())
1463	S.LookupQualifiedName(R, PromiseType->getAsCXXRecordDecl());
1464
1465	return !R.empty() && !R.isAmbiguous();
1466	}();
1467
1468	// Helper function to indicate whether the last lookup found the aligned
1469	// allocation function.
1470	bool PassAlignment = S.getLangOpts().CoroAlignedAllocation;
1471	auto LookupAllocationFunction = [&](Sema::AllocationFunctionScope NewScope =
1472	Sema::AFS_Both,
1473	bool WithoutPlacementArgs = false,
1474	bool ForceNonAligned = false) {
1475	// [dcl.fct.def.coroutine]p9
1476	// The allocation function's name is looked up by searching for it in the
1477	// scope of the promise type.
1478	// - If any declarations are found, ...
1479	// - If no declarations are found in the scope of the promise type, a search
1480	// is performed in the global scope.
1481	if (NewScope == Sema::AFS_Both)
1482	NewScope = PromiseContainsNew ? Sema::AFS_Class : Sema::AFS_Global;
1483
1484	PassAlignment = !ForceNonAligned && S.getLangOpts().CoroAlignedAllocation;
1485	FunctionDecl *UnusedResult = nullptr;
1486	S.FindAllocationFunctions(StartLoc: Loc, Range: SourceRange(), NewScope,
1487	/*DeleteScope*/ Sema::AFS_Both, AllocType: PromiseType,
1488	/*isArray*/ IsArray: false, PassAlignment,
1489	PlaceArgs: WithoutPlacementArgs ? MultiExprArg{}
1490	: PlacementArgs,
1491	OperatorNew, OperatorDelete&: UnusedResult, /*Diagnose*/ false);
1492	};
1493
1494	// We don't expect to call to global operator new with (size, p0, …, pn).
1495	// So if we choose to lookup the allocation function in global scope, we
1496	// shouldn't lookup placement arguments.
1497	if (PromiseContainsNew && !collectPlacementArgs(S, FD, Loc, PlacementArgs))
1498	return false;
1499
1500	LookupAllocationFunction();
1501
1502	if (PromiseContainsNew && !PlacementArgs.empty()) {
1503	// [dcl.fct.def.coroutine]p9
1504	// If no viable function is found ([over.match.viable]), overload
1505	// resolution
1506	// is performed again on a function call created by passing just the amount
1507	// of space required as an argument of type std::size_t.
1508	//
1509	// Proposed Change of [dcl.fct.def.coroutine]p9 in P2014R0:
1510	// Otherwise, overload resolution is performed again on a function call
1511	// created
1512	// by passing the amount of space requested as an argument of type
1513	// std::size_t as the first argument, and the requested alignment as
1514	// an argument of type std:align_val_t as the second argument.
1515	if (!OperatorNew ||
1516	(S.getLangOpts().CoroAlignedAllocation && !PassAlignment))
1517	LookupAllocationFunction(/*NewScope*/ Sema::AFS_Class,
1518	/*WithoutPlacementArgs*/ true);
1519	}
1520
1521	// Proposed Change of [dcl.fct.def.coroutine]p12 in P2014R0:
1522	// Otherwise, overload resolution is performed again on a function call
1523	// created
1524	// by passing the amount of space requested as an argument of type
1525	// std::size_t as the first argument, and the lvalues p1 ... pn as the
1526	// succeeding arguments. Otherwise, overload resolution is performed again
1527	// on a function call created by passing just the amount of space required as
1528	// an argument of type std::size_t.
1529	//
1530	// So within the proposed change in P2014RO, the priority order of aligned
1531	// allocation functions wiht promise_type is:
1532	//
1533	// void* operator new( std::size_t, std::align_val_t, placement_args... );
1534	// void* operator new( std::size_t, std::align_val_t);
1535	// void* operator new( std::size_t, placement_args... );
1536	// void* operator new( std::size_t);
1537
1538	// Helper variable to emit warnings.
1539	bool FoundNonAlignedInPromise = false;
1540	if (PromiseContainsNew && S.getLangOpts().CoroAlignedAllocation)
1541	if (!OperatorNew || !PassAlignment) {
1542	FoundNonAlignedInPromise = OperatorNew;
1543
1544	LookupAllocationFunction(/*NewScope*/ Sema::AFS_Class,
1545	/*WithoutPlacementArgs*/ false,
1546	/*ForceNonAligned*/ true);
1547
1548	if (!OperatorNew && !PlacementArgs.empty())
1549	LookupAllocationFunction(/*NewScope*/ Sema::AFS_Class,
1550	/*WithoutPlacementArgs*/ true,
1551	/*ForceNonAligned*/ true);
1552	}
1553
1554	bool IsGlobalOverload =
1555	OperatorNew && !isa<CXXRecordDecl>(OperatorNew->getDeclContext());
1556	// If we didn't find a class-local new declaration and non-throwing new
1557	// was is required then we need to lookup the non-throwing global operator
1558	// instead.
1559	if (RequiresNoThrowAlloc && (!OperatorNew || IsGlobalOverload)) {
1560	auto *StdNoThrow = buildStdNoThrowDeclRef(S, Loc);
1561	if (!StdNoThrow)
1562	return false;
1563	PlacementArgs = {StdNoThrow};
1564	OperatorNew = nullptr;
1565	LookupAllocationFunction(Sema::AFS_Global);
1566	}
1567
1568	// If we found a non-aligned allocation function in the promise_type,
1569	// it indicates the user forgot to update the allocation function. Let's emit
1570	// a warning here.
1571	if (FoundNonAlignedInPromise) {
1572	S.Diag(OperatorNew->getLocation(),
1573	diag::warn_non_aligned_allocation_function)
1574	<< &FD;
1575	}
1576
1577	if (!OperatorNew) {
1578	if (PromiseContainsNew)
1579	S.Diag(Loc, diag::err_coroutine_unusable_new) << PromiseType << &FD;
1580	else if (RequiresNoThrowAlloc)
1581	S.Diag(Loc, diag::err_coroutine_unfound_nothrow_new)
1582	<< &FD << S.getLangOpts().CoroAlignedAllocation;
1583
1584	return false;
1585	}
1586
1587	if (RequiresNoThrowAlloc) {
1588	const auto *FT = OperatorNew->getType()->castAs<FunctionProtoType>();
1589	if (!FT->isNothrow(/*ResultIfDependent*/ false)) {
1590	S.Diag(OperatorNew->getLocation(),
1591	diag::err_coroutine_promise_new_requires_nothrow)
1592	<< OperatorNew;
1593	S.Diag(Loc, diag::note_coroutine_promise_call_implicitly_required)
1594	<< OperatorNew;
1595	return false;
1596	}
1597	}
1598
1599	FunctionDecl *OperatorDelete = nullptr;
1600	if (!findDeleteForPromise(S, Loc, PromiseType, OperatorDelete)) {
1601	// FIXME: We should add an error here. According to:
1602	// [dcl.fct.def.coroutine]p12
1603	// If no usual deallocation function is found, the program is ill-formed.
1604	return false;
1605	}
1606
1607	Expr *FramePtr =
1608	S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_frame, {});
1609
1610	Expr *FrameSize =
1611	S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_size, {});
1612
1613	Expr *FrameAlignment = nullptr;
1614
1615	if (S.getLangOpts().CoroAlignedAllocation) {
1616	FrameAlignment =
1617	S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_align, {});
1618
1619	TypeSourceInfo *AlignValTy = getTypeSourceInfoForStdAlignValT(S, Loc);
1620	if (!AlignValTy)
1621	return false;
1622
1623	FrameAlignment = S.BuildCXXNamedCast(OpLoc: Loc, Kind: tok::kw_static_cast, Ty: AlignValTy,
1624	E: FrameAlignment, AngleBrackets: SourceRange(Loc, Loc),
1625	Parens: SourceRange(Loc, Loc))
1626	.get();
1627	}
1628
1629	// Make new call.
1630	ExprResult NewRef =
1631	S.BuildDeclRefExpr(OperatorNew, OperatorNew->getType(), VK_LValue, Loc);
1632	if (NewRef.isInvalid())
1633	return false;
1634
1635	SmallVector<Expr *, 2> NewArgs(1, FrameSize);
1636	if (S.getLangOpts().CoroAlignedAllocation && PassAlignment)
1637	NewArgs.push_back(Elt: FrameAlignment);
1638
1639	if (OperatorNew->getNumParams() > NewArgs.size())
1640	llvm::append_range(C&: NewArgs, R&: PlacementArgs);
1641
1642	ExprResult NewExpr =
1643	S.BuildCallExpr(S: S.getCurScope(), Fn: NewRef.get(), LParenLoc: Loc, ArgExprs: NewArgs, RParenLoc: Loc);
1644	NewExpr = S.ActOnFinishFullExpr(Expr: NewExpr.get(), /*DiscardedValue*/ false);
1645	if (NewExpr.isInvalid())
1646	return false;
1647
1648	// Make delete call.
1649
1650	QualType OpDeleteQualType = OperatorDelete->getType();
1651
1652	ExprResult DeleteRef =
1653	S.BuildDeclRefExpr(OperatorDelete, OpDeleteQualType, VK_LValue, Loc);
1654	if (DeleteRef.isInvalid())
1655	return false;
1656
1657	Expr *CoroFree =
1658	S.BuildBuiltinCallExpr(Loc, Builtin::BI__builtin_coro_free, {FramePtr});
1659
1660	SmallVector<Expr *, 2> DeleteArgs{CoroFree};
1661
1662	// [dcl.fct.def.coroutine]p12
1663	// The selected deallocation function shall be called with the address of
1664	// the block of storage to be reclaimed as its first argument. If a
1665	// deallocation function with a parameter of type std::size_t is
1666	// used, the size of the block is passed as the corresponding argument.
1667	const auto *OpDeleteType =
1668	OpDeleteQualType.getTypePtr()->castAs<FunctionProtoType>();
1669	if (OpDeleteType->getNumParams() > DeleteArgs.size() &&
1670	S.getASTContext().hasSameUnqualifiedType(
1671	T1: OpDeleteType->getParamType(DeleteArgs.size()), T2: FrameSize->getType()))
1672	DeleteArgs.push_back(Elt: FrameSize);
1673
1674	// Proposed Change of [dcl.fct.def.coroutine]p12 in P2014R0:
1675	// If deallocation function lookup finds a usual deallocation function with
1676	// a pointer parameter, size parameter and alignment parameter then this
1677	// will be the selected deallocation function, otherwise if lookup finds a
1678	// usual deallocation function with both a pointer parameter and a size
1679	// parameter, then this will be the selected deallocation function.
1680	// Otherwise, if lookup finds a usual deallocation function with only a
1681	// pointer parameter, then this will be the selected deallocation
1682	// function.
1683	//
1684	// So we are not forced to pass alignment to the deallocation function.
1685	if (S.getLangOpts().CoroAlignedAllocation &&
1686	OpDeleteType->getNumParams() > DeleteArgs.size() &&
1687	S.getASTContext().hasSameUnqualifiedType(
1688	T1: OpDeleteType->getParamType(DeleteArgs.size()),
1689	T2: FrameAlignment->getType()))
1690	DeleteArgs.push_back(Elt: FrameAlignment);
1691
1692	ExprResult DeleteExpr =
1693	S.BuildCallExpr(S: S.getCurScope(), Fn: DeleteRef.get(), LParenLoc: Loc, ArgExprs: DeleteArgs, RParenLoc: Loc);
1694	DeleteExpr =
1695	S.ActOnFinishFullExpr(Expr: DeleteExpr.get(), /*DiscardedValue*/ false);
1696	if (DeleteExpr.isInvalid())
1697	return false;
1698
1699	this->Allocate = NewExpr.get();
1700	this->Deallocate = DeleteExpr.get();
1701
1702	return true;
1703	}
1704
1705	bool CoroutineStmtBuilder::makeOnFallthrough() {
1706	assert(!IsPromiseDependentType &&
1707	"cannot make statement while the promise type is dependent");
1708
1709	// [dcl.fct.def.coroutine]/p6
1710	// If searches for the names return_void and return_value in the scope of
1711	// the promise type each find any declarations, the program is ill-formed.
1712	// [Note 1: If return_void is found, flowing off the end of a coroutine is
1713	// equivalent to a co_return with no operand. Otherwise, flowing off the end
1714	// of a coroutine results in undefined behavior ([stmt.return.coroutine]). —
1715	// end note]
1716	bool HasRVoid, HasRValue;
1717	LookupResult LRVoid =
1718	lookupMember(S, Name: "return_void", RD: PromiseRecordDecl, Loc, Res&: HasRVoid);
1719	LookupResult LRValue =
1720	lookupMember(S, Name: "return_value", RD: PromiseRecordDecl, Loc, Res&: HasRValue);
1721
1722	StmtResult Fallthrough;
1723	if (HasRVoid && HasRValue) {
1724	// FIXME Improve this diagnostic
1725	S.Diag(FD.getLocation(),
1726	diag::err_coroutine_promise_incompatible_return_functions)
1727	<< PromiseRecordDecl;
1728	S.Diag(LRVoid.getRepresentativeDecl()->getLocation(),
1729	diag::note_member_first_declared_here)
1730	<< LRVoid.getLookupName();
1731	S.Diag(LRValue.getRepresentativeDecl()->getLocation(),
1732	diag::note_member_first_declared_here)
1733	<< LRValue.getLookupName();
1734	return false;
1735	} else if (!HasRVoid && !HasRValue) {
1736	// We need to set 'Fallthrough'. Otherwise the other analysis part might
1737	// think the coroutine has defined a return_value method. So it might emit
1738	// **false** positive warning. e.g.,
1739	//
1740	// promise_without_return_func foo() {
1741	// co_await something();
1742	// }
1743	//
1744	// Then AnalysisBasedWarning would emit a warning about `foo()` lacking a
1745	// co_return statements, which isn't correct.
1746	Fallthrough = S.ActOnNullStmt(SemiLoc: PromiseRecordDecl->getLocation());
1747	if (Fallthrough.isInvalid())
1748	return false;
1749	} else if (HasRVoid) {
1750	Fallthrough = S.BuildCoreturnStmt(Loc: FD.getLocation(), E: nullptr,
1751	/*IsImplicit=*/true);
1752	Fallthrough = S.ActOnFinishFullStmt(Stmt: Fallthrough.get());
1753	if (Fallthrough.isInvalid())
1754	return false;
1755	}
1756
1757	this->OnFallthrough = Fallthrough.get();
1758	return true;
1759	}
1760
1761	bool CoroutineStmtBuilder::makeOnException() {
1762	// Try to form 'p.unhandled_exception();'
1763	assert(!IsPromiseDependentType &&
1764	"cannot make statement while the promise type is dependent");
1765
1766	const bool RequireUnhandledException = S.getLangOpts().CXXExceptions;
1767
1768	if (!lookupMember(S, Name: "unhandled_exception", RD: PromiseRecordDecl, Loc)) {
1769	auto DiagID =
1770	RequireUnhandledException
1771	? diag::err_coroutine_promise_unhandled_exception_required
1772	: diag::
1773	warn_coroutine_promise_unhandled_exception_required_with_exceptions;
1774	S.Diag(Loc, DiagID) << PromiseRecordDecl;
1775	S.Diag(PromiseRecordDecl->getLocation(), diag::note_defined_here)
1776	<< PromiseRecordDecl;
1777	return !RequireUnhandledException;
1778	}
1779
1780	// If exceptions are disabled, don't try to build OnException.
1781	if (!S.getLangOpts().CXXExceptions)
1782	return true;
1783
1784	ExprResult UnhandledException = buildPromiseCall(
1785	S, Promise: Fn.CoroutinePromise, Loc, Name: "unhandled_exception", Args: std::nullopt);
1786	UnhandledException = S.ActOnFinishFullExpr(Expr: UnhandledException.get(), CC: Loc,
1787	/*DiscardedValue*/ false);
1788	if (UnhandledException.isInvalid())
1789	return false;
1790
1791	// Since the body of the coroutine will be wrapped in try-catch, it will
1792	// be incompatible with SEH __try if present in a function.
1793	if (!S.getLangOpts().Borland && Fn.FirstSEHTryLoc.isValid()) {
1794	S.Diag(Fn.FirstSEHTryLoc, diag::err_seh_in_a_coroutine_with_cxx_exceptions);
1795	S.Diag(Fn.FirstCoroutineStmtLoc, diag::note_declared_coroutine_here)
1796	<< Fn.getFirstCoroutineStmtKeyword();
1797	return false;
1798	}
1799
1800	this->OnException = UnhandledException.get();
1801	return true;
1802	}
1803
1804	bool CoroutineStmtBuilder::makeReturnObject() {
1805	// [dcl.fct.def.coroutine]p7
1806	// The expression promise.get_return_object() is used to initialize the
1807	// returned reference or prvalue result object of a call to a coroutine.
1808	ExprResult ReturnObject = buildPromiseCall(S, Promise: Fn.CoroutinePromise, Loc,
1809	Name: "get_return_object", Args: std::nullopt);
1810	if (ReturnObject.isInvalid())
1811	return false;
1812
1813	this->ReturnValue = ReturnObject.get();
1814	return true;
1815	}
1816
1817	static void noteMemberDeclaredHere(Sema &S, Expr *E, FunctionScopeInfo &Fn) {
1818	if (auto *MbrRef = dyn_cast<CXXMemberCallExpr>(Val: E)) {
1819	auto *MethodDecl = MbrRef->getMethodDecl();
1820	S.Diag(MethodDecl->getLocation(), diag::note_member_declared_here)
1821	<< MethodDecl;
1822	}
1823	S.Diag(Fn.FirstCoroutineStmtLoc, diag::note_declared_coroutine_here)
1824	<< Fn.getFirstCoroutineStmtKeyword();
1825	}
1826
1827	bool CoroutineStmtBuilder::makeGroDeclAndReturnStmt() {
1828	assert(!IsPromiseDependentType &&
1829	"cannot make statement while the promise type is dependent");
1830	assert(this->ReturnValue && "ReturnValue must be already formed");
1831
1832	QualType const GroType = this->ReturnValue->getType();
1833	assert(!GroType->isDependentType() &&
1834	"get_return_object type must no longer be dependent");
1835
1836	QualType const FnRetType = FD.getReturnType();
1837	assert(!FnRetType->isDependentType() &&
1838	"get_return_object type must no longer be dependent");
1839
1840	// The call to get_­return_­object is sequenced before the call to
1841	// initial_­suspend and is invoked at most once, but there are caveats
1842	// regarding on whether the prvalue result object may be initialized
1843	// directly/eager or delayed, depending on the types involved.
1844	//
1845	// More info at https://github.com/cplusplus/papers/issues/1414
1846	bool GroMatchesRetType = S.getASTContext().hasSameType(T1: GroType, T2: FnRetType);
1847
1848	if (FnRetType->isVoidType()) {
1849	ExprResult Res =
1850	S.ActOnFinishFullExpr(Expr: this->ReturnValue, CC: Loc, /*DiscardedValue*/ false);
1851	if (Res.isInvalid())
1852	return false;
1853
1854	if (!GroMatchesRetType)
1855	this->ResultDecl = Res.get();
1856	return true;
1857	}
1858
1859	if (GroType->isVoidType()) {
1860	// Trigger a nice error message.
1861	InitializedEntity Entity =
1862	InitializedEntity::InitializeResult(ReturnLoc: Loc, Type: FnRetType);
1863	S.PerformCopyInitialization(Entity, EqualLoc: SourceLocation(), Init: ReturnValue);
1864	noteMemberDeclaredHere(S, E: ReturnValue, Fn);
1865	return false;
1866	}
1867
1868	StmtResult ReturnStmt;
1869	clang::VarDecl *GroDecl = nullptr;
1870	if (GroMatchesRetType) {
1871	ReturnStmt = S.BuildReturnStmt(ReturnLoc: Loc, RetValExp: ReturnValue);
1872	} else {
1873	GroDecl = VarDecl::Create(
1874	C&: S.Context, DC: &FD, StartLoc: FD.getLocation(), IdLoc: FD.getLocation(),
1875	Id: &S.PP.getIdentifierTable().get(Name: "__coro_gro"), T: GroType,
1876	TInfo: S.Context.getTrivialTypeSourceInfo(T: GroType, Loc), S: SC_None);
1877	GroDecl->setImplicit();
1878
1879	S.CheckVariableDeclarationType(NewVD: GroDecl);
1880	if (GroDecl->isInvalidDecl())
1881	return false;
1882
1883	InitializedEntity Entity = InitializedEntity::InitializeVariable(Var: GroDecl);
1884	ExprResult Res =
1885	S.PerformCopyInitialization(Entity, EqualLoc: SourceLocation(), Init: ReturnValue);
1886	if (Res.isInvalid())
1887	return false;
1888
1889	Res = S.ActOnFinishFullExpr(Expr: Res.get(), /*DiscardedValue*/ false);
1890	if (Res.isInvalid())
1891	return false;
1892
1893	S.AddInitializerToDecl(GroDecl, Res.get(),
1894	/*DirectInit=*/false);
1895
1896	S.FinalizeDeclaration(GroDecl);
1897
1898	// Form a declaration statement for the return declaration, so that AST
1899	// visitors can more easily find it.
1900	StmtResult GroDeclStmt =
1901	S.ActOnDeclStmt(Decl: S.ConvertDeclToDeclGroup(GroDecl), StartLoc: Loc, EndLoc: Loc);
1902	if (GroDeclStmt.isInvalid())
1903	return false;
1904
1905	this->ResultDecl = GroDeclStmt.get();
1906
1907	ExprResult declRef = S.BuildDeclRefExpr(GroDecl, GroType, VK_LValue, Loc);
1908	if (declRef.isInvalid())
1909	return false;
1910
1911	ReturnStmt = S.BuildReturnStmt(ReturnLoc: Loc, RetValExp: declRef.get());
1912	}
1913
1914	if (ReturnStmt.isInvalid()) {
1915	noteMemberDeclaredHere(S, E: ReturnValue, Fn);
1916	return false;
1917	}
1918
1919	if (!GroMatchesRetType &&
1920	cast<clang::ReturnStmt>(Val: ReturnStmt.get())->getNRVOCandidate() == GroDecl)
1921	GroDecl->setNRVOVariable(true);
1922
1923	this->ReturnStmt = ReturnStmt.get();
1924	return true;
1925	}
1926
1927	// Create a static_cast\<T&&>(expr).
1928	static Expr *castForMoving(Sema &S, Expr *E, QualType T = QualType()) {
1929	if (T.isNull())
1930	T = E->getType();
1931	QualType TargetType = S.BuildReferenceType(
1932	T, /*SpelledAsLValue*/ LValueRef: false, Loc: SourceLocation(), Entity: DeclarationName());
1933	SourceLocation ExprLoc = E->getBeginLoc();
1934	TypeSourceInfo *TargetLoc =
1935	S.Context.getTrivialTypeSourceInfo(T: TargetType, Loc: ExprLoc);
1936
1937	return S
1938	.BuildCXXNamedCast(OpLoc: ExprLoc, Kind: tok::kw_static_cast, Ty: TargetLoc, E,
1939	AngleBrackets: SourceRange(ExprLoc, ExprLoc), Parens: E->getSourceRange())
1940	.get();
1941	}
1942
1943	/// Build a variable declaration for move parameter.
1944	static VarDecl *buildVarDecl(Sema &S, SourceLocation Loc, QualType Type,
1945	IdentifierInfo *II) {
1946	TypeSourceInfo *TInfo = S.Context.getTrivialTypeSourceInfo(T: Type, Loc);
1947	VarDecl *Decl = VarDecl::Create(C&: S.Context, DC: S.CurContext, StartLoc: Loc, IdLoc: Loc, Id: II, T: Type,
1948	TInfo, S: SC_None);
1949	Decl->setImplicit();
1950	return Decl;
1951	}
1952
1953	// Build statements that move coroutine function parameters to the coroutine
1954	// frame, and store them on the function scope info.
1955	bool Sema::buildCoroutineParameterMoves(SourceLocation Loc) {
1956	assert(isa<FunctionDecl>(CurContext) && "not in a function scope");
1957	auto *FD = cast<FunctionDecl>(Val: CurContext);
1958
1959	auto *ScopeInfo = getCurFunction();
1960	if (!ScopeInfo->CoroutineParameterMoves.empty())
1961	return false;
1962
1963	// [dcl.fct.def.coroutine]p13
1964	// When a coroutine is invoked, after initializing its parameters
1965	// ([expr.call]), a copy is created for each coroutine parameter. For a
1966	// parameter of type cv T, the copy is a variable of type cv T with
1967	// automatic storage duration that is direct-initialized from an xvalue of
1968	// type T referring to the parameter.
1969	for (auto *PD : FD->parameters()) {
1970	if (PD->getType()->isDependentType())
1971	continue;
1972
1973	// Preserve the referenced state for unused parameter diagnostics.
1974	bool DeclReferenced = PD->isReferenced();
1975
1976	ExprResult PDRefExpr =
1977	BuildDeclRefExpr(PD, PD->getType().getNonReferenceType(),
1978	ExprValueKind::VK_LValue, Loc); // FIXME: scope?
1979
1980	PD->setReferenced(DeclReferenced);
1981
1982	if (PDRefExpr.isInvalid())
1983	return false;
1984
1985	Expr *CExpr = nullptr;
1986	if (PD->getType()->getAsCXXRecordDecl() ||
1987	PD->getType()->isRValueReferenceType())
1988	CExpr = castForMoving(S&: *this, E: PDRefExpr.get());
1989	else
1990	CExpr = PDRefExpr.get();
1991	// [dcl.fct.def.coroutine]p13
1992	// The initialization and destruction of each parameter copy occurs in the
1993	// context of the called coroutine.
1994	auto *D = buildVarDecl(*this, Loc, PD->getType(), PD->getIdentifier());
1995	AddInitializerToDecl(dcl: D, init: CExpr, /*DirectInit=*/true);
1996
1997	// Convert decl to a statement.
1998	StmtResult Stmt = ActOnDeclStmt(Decl: ConvertDeclToDeclGroup(Ptr: D), StartLoc: Loc, EndLoc: Loc);
1999	if (Stmt.isInvalid())
2000	return false;
2001
2002	ScopeInfo->CoroutineParameterMoves.insert(KV: std::make_pair(x&: PD, y: Stmt.get()));
2003	}
2004	return true;
2005	}
2006
2007	StmtResult Sema::BuildCoroutineBodyStmt(CoroutineBodyStmt::CtorArgs Args) {
2008	CoroutineBodyStmt *Res = CoroutineBodyStmt::Create(C: Context, Args);
2009	if (!Res)
2010	return StmtError();
2011	return Res;
2012	}
2013
2014	ClassTemplateDecl *Sema::lookupCoroutineTraits(SourceLocation KwLoc,
2015	SourceLocation FuncLoc) {
2016	if (StdCoroutineTraitsCache)
2017	return StdCoroutineTraitsCache;
2018
2019	IdentifierInfo const &TraitIdent =
2020	PP.getIdentifierTable().get(Name: "coroutine_traits");
2021
2022	NamespaceDecl *StdSpace = getStdNamespace();
2023	LookupResult Result(*this, &TraitIdent, FuncLoc, LookupOrdinaryName);
2024	bool Found = StdSpace && LookupQualifiedName(Result, StdSpace);
2025
2026	if (!Found) {
2027	// The goggles, we found nothing!
2028	Diag(KwLoc, diag::err_implied_coroutine_type_not_found)
2029	<< "std::coroutine_traits";
2030	return nullptr;
2031	}
2032
2033	// coroutine_traits is required to be a class template.
2034	StdCoroutineTraitsCache = Result.getAsSingle<ClassTemplateDecl>();
2035	if (!StdCoroutineTraitsCache) {
2036	Result.suppressDiagnostics();
2037	NamedDecl *Found = *Result.begin();
2038	Diag(Found->getLocation(), diag::err_malformed_std_coroutine_traits);
2039	return nullptr;
2040	}
2041
2042	return StdCoroutineTraitsCache;
2043	}
2044