1	//===--- CGBlocks.cpp - Emit LLVM Code for declarations ---------*- C++ -*-===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This contains code to emit blocks.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGBlocks.h"
14	#include "CGCXXABI.h"
15	#include "CGDebugInfo.h"
16	#include "CGObjCRuntime.h"
17	#include "CGOpenCLRuntime.h"
18	#include "CodeGenFunction.h"
19	#include "CodeGenModule.h"
20	#include "CodeGenPGO.h"
21	#include "ConstantEmitter.h"
22	#include "TargetInfo.h"
23	#include "clang/AST/Attr.h"
24	#include "clang/AST/DeclObjC.h"
25	#include "clang/CodeGen/ConstantInitBuilder.h"
26	#include "llvm/IR/DataLayout.h"
27	#include "llvm/IR/Module.h"
28	#include "llvm/Support/ScopedPrinter.h"
29	#include <algorithm>
30	#include <cstdio>
31
32	using namespace clang;
33	using namespace CodeGen;
34
35	CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
36	: Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
37	NoEscape(false), HasCXXObject(false), UsesStret(false),
38	HasCapturedVariableLayout(false), CapturesNonExternalType(false),
39	LocalAddress(RawAddress::invalid()), StructureType(nullptr),
40	Block(block) {
41
42	// Skip asm prefix, if any. 'name' is usually taken directly from
43	// the mangled name of the enclosing function.
44	name.consume_front(Prefix: "\01");
45	}
46
47	// Anchor the vtable to this translation unit.
48	BlockByrefHelpers::~BlockByrefHelpers() {}
49
50	/// Build the given block as a global block.
51	static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
52	const CGBlockInfo &blockInfo,
53	llvm::Constant *blockFn);
54
55	/// Build the helper function to copy a block.
56	static llvm::Constant *buildCopyHelper(CodeGenModule &CGM,
57	const CGBlockInfo &blockInfo) {
58	return CodeGenFunction(CGM).GenerateCopyHelperFunction(blockInfo);
59	}
60
61	/// Build the helper function to dispose of a block.
62	static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,
63	const CGBlockInfo &blockInfo) {
64	return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);
65	}
66
67	namespace {
68
69	enum class CaptureStrKind {
70	// String for the copy helper.
71	CopyHelper,
72	// String for the dispose helper.
73	DisposeHelper,
74	// Merge the strings for the copy helper and dispose helper.
75	Merged
76	};
77
78	} // end anonymous namespace
79
80	static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
81	CaptureStrKind StrKind,
82	CharUnits BlockAlignment,
83	CodeGenModule &CGM);
84
85	static std::string getBlockDescriptorName(const CGBlockInfo &BlockInfo,
86	CodeGenModule &CGM) {
87	std::string Name = "__block_descriptor_";
88	Name += llvm::to_string(Value: BlockInfo.BlockSize.getQuantity()) + "_";
89
90	if (BlockInfo.NeedsCopyDispose) {
91	if (CGM.getLangOpts().Exceptions)
92	Name += "e";
93	if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
94	Name += "a";
95	Name += llvm::to_string(Value: BlockInfo.BlockAlign.getQuantity()) + "_";
96
97	for (auto &Cap : BlockInfo.SortedCaptures) {
98	if (Cap.isConstantOrTrivial())
99	continue;
100
101	Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
102
103	if (Cap.CopyKind == Cap.DisposeKind) {
104	// If CopyKind and DisposeKind are the same, merge the capture
105	// information.
106	assert(Cap.CopyKind != BlockCaptureEntityKind::None &&
107	"shouldn't see BlockCaptureManagedEntity that is None");
108	Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::Merged,
109	BlockAlignment: BlockInfo.BlockAlign, CGM);
110	} else {
111	// If CopyKind and DisposeKind are not the same, which can happen when
112	// either Kind is None or the captured object is a __strong block,
113	// concatenate the copy and dispose strings.
114	Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::CopyHelper,
115	BlockAlignment: BlockInfo.BlockAlign, CGM);
116	Name += getBlockCaptureStr(Cap, StrKind: CaptureStrKind::DisposeHelper,
117	BlockAlignment: BlockInfo.BlockAlign, CGM);
118	}
119	}
120	Name += "_";
121	}
122
123	std::string TypeAtEncoding;
124
125	if (!CGM.getCodeGenOpts().DisableBlockSignatureString) {
126	TypeAtEncoding =
127	CGM.getContext().getObjCEncodingForBlock(blockExpr: BlockInfo.getBlockExpr());
128	/// Replace occurrences of '@' with '\1'. '@' is reserved on ELF platforms
129	/// as a separator between symbol name and symbol version.
130	llvm::replace(Range&: TypeAtEncoding, OldValue: '@', NewValue: '\1');
131	}
132	Name += "e" + llvm::to_string(Value: TypeAtEncoding.size()) + "_" + TypeAtEncoding;
133	Name += "l" + CGM.getObjCRuntime().getRCBlockLayoutStr(CGM, blockInfo: BlockInfo);
134	return Name;
135	}
136
137	/// buildBlockDescriptor - Build the block descriptor meta-data for a block.
138	/// buildBlockDescriptor is accessed from 5th field of the Block_literal
139	/// meta-data and contains stationary information about the block literal.
140	/// Its definition will have 4 (or optionally 6) words.
141	/// \code
142	/// struct Block_descriptor {
143	/// unsigned long reserved;
144	/// unsigned long size; // size of Block_literal metadata in bytes.
145	/// void *copy_func_helper_decl; // optional copy helper.
146	/// void *destroy_func_decl; // optional destructor helper.
147	/// void *block_method_encoding_address; // @encode for block literal signature.
148	/// void *block_layout_info; // encoding of captured block variables.
149	/// };
150	/// \endcode
151	static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
152	const CGBlockInfo &blockInfo) {
153	ASTContext &C = CGM.getContext();
154
155	llvm::IntegerType *ulong =
156	cast<llvm::IntegerType>(Val: CGM.getTypes().ConvertType(T: C.UnsignedLongTy));
157	llvm::PointerType *i8p = nullptr;
158	if (CGM.getLangOpts().OpenCL)
159	i8p = llvm::PointerType::get(
160	C&: CGM.getLLVMContext(), AddressSpace: C.getTargetAddressSpace(AS: LangAS::opencl_constant));
161	else
162	i8p = CGM.VoidPtrTy;
163
164	std::string descName;
165
166	// If an equivalent block descriptor global variable exists, return it.
167	if (C.getLangOpts().ObjC &&
168	CGM.getLangOpts().getGC() == LangOptions::NonGC) {
169	descName = getBlockDescriptorName(BlockInfo: blockInfo, CGM);
170	if (llvm::GlobalValue *desc = CGM.getModule().getNamedValue(Name: descName))
171	return desc;
172	}
173
174	// If there isn't an equivalent block descriptor global variable, create a new
175	// one.
176	ConstantInitBuilder builder(CGM);
177	auto elements = builder.beginStruct();
178
179	// reserved
180	elements.addInt(intTy: ulong, value: 0);
181
182	// Size
183	// FIXME: What is the right way to say this doesn't fit? We should give
184	// a user diagnostic in that case. Better fix would be to change the
185	// API to size_t.
186	elements.addInt(intTy: ulong, value: blockInfo.BlockSize.getQuantity());
187
188	// Optional copy/dispose helpers.
189	bool hasInternalHelper = false;
190	if (blockInfo.NeedsCopyDispose) {
191	auto &Schema = CGM.getCodeGenOpts().PointerAuth.BlockHelperFunctionPointers;
192	// copy_func_helper_decl
193	llvm::Constant *copyHelper = buildCopyHelper(CGM, blockInfo);
194	elements.addSignedPointer(Pointer: copyHelper, Schema, CalleeDecl: GlobalDecl(), CalleeType: QualType());
195
196	// destroy_func_decl
197	llvm::Constant *disposeHelper = buildDisposeHelper(CGM, blockInfo);
198	elements.addSignedPointer(Pointer: disposeHelper, Schema, CalleeDecl: GlobalDecl(), CalleeType: QualType());
199
200	if (cast<llvm::Function>(Val: copyHelper->stripPointerCasts())
201	->hasInternalLinkage() ||
202	cast<llvm::Function>(Val: disposeHelper->stripPointerCasts())
203	->hasInternalLinkage())
204	hasInternalHelper = true;
205	}
206
207	// Signature. Mandatory ObjC-style method descriptor @encode sequence.
208	if (CGM.getCodeGenOpts().DisableBlockSignatureString) {
209	elements.addNullPointer(ptrTy: i8p);
210	} else {
211	std::string typeAtEncoding =
212	CGM.getContext().getObjCEncodingForBlock(blockExpr: blockInfo.getBlockExpr());
213	elements.add(value: CGM.GetAddrOfConstantCString(Str: typeAtEncoding).getPointer());
214	}
215
216	// GC layout.
217	if (C.getLangOpts().ObjC) {
218	if (CGM.getLangOpts().getGC() != LangOptions::NonGC)
219	elements.add(value: CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
220	else
221	elements.add(value: CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo));
222	}
223	else
224	elements.addNullPointer(ptrTy: i8p);
225
226	unsigned AddrSpace = 0;
227	if (C.getLangOpts().OpenCL)
228	AddrSpace = C.getTargetAddressSpace(AS: LangAS::opencl_constant);
229
230	llvm::GlobalValue::LinkageTypes linkage;
231	if (descName.empty()) {
232	linkage = llvm::GlobalValue::InternalLinkage;
233	descName = "__block_descriptor_tmp";
234	} else if (hasInternalHelper) {
235	// If either the copy helper or the dispose helper has internal linkage,
236	// the block descriptor must have internal linkage too.
237	linkage = llvm::GlobalValue::InternalLinkage;
238	} else {
239	linkage = llvm::GlobalValue::LinkOnceODRLinkage;
240	}
241
242	llvm::GlobalVariable *global =
243	elements.finishAndCreateGlobal(args&: descName, args: CGM.getPointerAlign(),
244	/*constant*/ args: true, args&: linkage, args&: AddrSpace);
245
246	if (linkage == llvm::GlobalValue::LinkOnceODRLinkage) {
247	if (CGM.supportsCOMDAT())
248	global->setComdat(CGM.getModule().getOrInsertComdat(Name: descName));
249	global->setVisibility(llvm::GlobalValue::HiddenVisibility);
250	global->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
251	}
252
253	return global;
254	}
255
256	/*
257	Purely notional variadic template describing the layout of a block.
258
259	template <class _ResultType, class... _ParamTypes, class... _CaptureTypes>
260	struct Block_literal {
261	/// Initialized to one of:
262	/// extern void *_NSConcreteStackBlock[];
263	/// extern void *_NSConcreteGlobalBlock[];
264	///
265	/// In theory, we could start one off malloc'ed by setting
266	/// BLOCK_NEEDS_FREE, giving it a refcount of 1, and using
267	/// this isa:
268	/// extern void *_NSConcreteMallocBlock[];
269	struct objc_class *isa;
270
271	/// These are the flags (with corresponding bit number) that the
272	/// compiler is actually supposed to know about.
273	/// 23. BLOCK_IS_NOESCAPE - indicates that the block is non-escaping
274	/// 25. BLOCK_HAS_COPY_DISPOSE - indicates that the block
275	/// descriptor provides copy and dispose helper functions
276	/// 26. BLOCK_HAS_CXX_OBJ - indicates that there's a captured
277	/// object with a nontrivial destructor or copy constructor
278	/// 28. BLOCK_IS_GLOBAL - indicates that the block is allocated
279	/// as global memory
280	/// 29. BLOCK_USE_STRET - indicates that the block function
281	/// uses stret, which objc_msgSend needs to know about
282	/// 30. BLOCK_HAS_SIGNATURE - indicates that the block has an
283	/// @encoded signature string
284	/// And we're not supposed to manipulate these:
285	/// 24. BLOCK_NEEDS_FREE - indicates that the block has been moved
286	/// to malloc'ed memory
287	/// 27. BLOCK_IS_GC - indicates that the block has been moved to
288	/// to GC-allocated memory
289	/// Additionally, the bottom 16 bits are a reference count which
290	/// should be zero on the stack.
291	int flags;
292
293	/// Reserved; should be zero-initialized.
294	int reserved;
295
296	/// Function pointer generated from block literal.
297	_ResultType (*invoke)(Block_literal *, _ParamTypes...);
298
299	/// Block description metadata generated from block literal.
300	struct Block_descriptor *block_descriptor;
301
302	/// Captured values follow.
303	_CapturesTypes captures...;
304	};
305	*/
306
307	namespace {
308	/// A chunk of data that we actually have to capture in the block.
309	struct BlockLayoutChunk {
310	CharUnits Alignment;
311	CharUnits Size;
312	const BlockDecl::Capture *Capture; // null for 'this'
313	llvm::Type *Type;
314	QualType FieldType;
315	BlockCaptureEntityKind CopyKind, DisposeKind;
316	BlockFieldFlags CopyFlags, DisposeFlags;
317
318	BlockLayoutChunk(CharUnits align, CharUnits size,
319	const BlockDecl::Capture *capture, llvm::Type *type,
320	QualType fieldType, BlockCaptureEntityKind CopyKind,
321	BlockFieldFlags CopyFlags,
322	BlockCaptureEntityKind DisposeKind,
323	BlockFieldFlags DisposeFlags)
324	: Alignment(align), Size(size), Capture(capture), Type(type),
325	FieldType(fieldType), CopyKind(CopyKind), DisposeKind(DisposeKind),
326	CopyFlags(CopyFlags), DisposeFlags(DisposeFlags) {}
327
328	/// Tell the block info that this chunk has the given field index.
329	void setIndex(CGBlockInfo &info, unsigned index, CharUnits offset) {
330	if (!Capture) {
331	info.CXXThisIndex = index;
332	info.CXXThisOffset = offset;
333	} else {
334	info.SortedCaptures.push_back(Elt: CGBlockInfo::Capture::makeIndex(
335	index, offset, FieldType, CopyKind, CopyFlags, DisposeKind,
336	DisposeFlags, Cap: Capture));
337	}
338	}
339
340	bool isTrivial() const {
341	return CopyKind == BlockCaptureEntityKind::None &&
342	DisposeKind == BlockCaptureEntityKind::None;
343	}
344	};
345
346	/// Order by 1) all __strong together 2) next, all block together 3) next,
347	/// all byref together 4) next, all __weak together. Preserve descending
348	/// alignment in all situations.
349	bool operator<(const BlockLayoutChunk &left, const BlockLayoutChunk &right) {
350	if (left.Alignment != right.Alignment)
351	return left.Alignment > right.Alignment;
352
353	auto getPrefOrder = [](const BlockLayoutChunk &chunk) {
354	switch (chunk.CopyKind) {
355	case BlockCaptureEntityKind::ARCStrong:
356	return 0;
357	case BlockCaptureEntityKind::BlockObject:
358	switch (chunk.CopyFlags.getBitMask()) {
359	case BLOCK_FIELD_IS_OBJECT:
360	return 0;
361	case BLOCK_FIELD_IS_BLOCK:
362	return 1;
363	case BLOCK_FIELD_IS_BYREF:
364	return 2;
365	default:
366	break;
367	}
368	break;
369	case BlockCaptureEntityKind::ARCWeak:
370	return 3;
371	default:
372	break;
373	}
374	return 4;
375	};
376
377	return getPrefOrder(left) < getPrefOrder(right);
378	}
379	} // end anonymous namespace
380
381	static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
382	computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
383	const LangOptions &LangOpts);
384
385	static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
386	computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
387	const LangOptions &LangOpts);
388
389	static void addBlockLayout(CharUnits align, CharUnits size,
390	const BlockDecl::Capture *capture, llvm::Type *type,
391	QualType fieldType,
392	SmallVectorImpl<BlockLayoutChunk> &Layout,
393	CGBlockInfo &Info, CodeGenModule &CGM) {
394	if (!capture) {
395	// 'this' capture.
396	Layout.push_back(Elt: BlockLayoutChunk(
397	align, size, capture, type, fieldType, BlockCaptureEntityKind::None,
398	BlockFieldFlags(), BlockCaptureEntityKind::None, BlockFieldFlags()));
399	return;
400	}
401
402	const LangOptions &LangOpts = CGM.getLangOpts();
403	BlockCaptureEntityKind CopyKind, DisposeKind;
404	BlockFieldFlags CopyFlags, DisposeFlags;
405
406	std::tie(args&: CopyKind, args&: CopyFlags) =
407	computeCopyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
408	std::tie(args&: DisposeKind, args&: DisposeFlags) =
409	computeDestroyInfoForBlockCapture(CI: *capture, T: fieldType, LangOpts);
410	Layout.push_back(Elt: BlockLayoutChunk(align, size, capture, type, fieldType,
411	CopyKind, CopyFlags, DisposeKind,
412	DisposeFlags));
413
414	if (Info.NoEscape)
415	return;
416
417	if (!Layout.back().isTrivial())
418	Info.NeedsCopyDispose = true;
419	}
420
421	/// Determines if the given type is safe for constant capture in C++.
422	static bool isSafeForCXXConstantCapture(QualType type) {
423	const RecordType *recordType =
424	type->getBaseElementTypeUnsafe()->getAs<RecordType>();
425
426	// Only records can be unsafe.
427	if (!recordType) return true;
428
429	const auto *record = cast<CXXRecordDecl>(Val: recordType->getDecl());
430
431	// Maintain semantics for classes with non-trivial dtors or copy ctors.
432	if (!record->hasTrivialDestructor()) return false;
433	if (record->hasNonTrivialCopyConstructor()) return false;
434
435	// Otherwise, we just have to make sure there aren't any mutable
436	// fields that might have changed since initialization.
437	return !record->hasMutableFields();
438	}
439
440	/// It is illegal to modify a const object after initialization.
441	/// Therefore, if a const object has a constant initializer, we don't
442	/// actually need to keep storage for it in the block; we'll just
443	/// rematerialize it at the start of the block function. This is
444	/// acceptable because we make no promises about address stability of
445	/// captured variables.
446	static llvm::Constant *tryCaptureAsConstant(CodeGenModule &CGM,
447	CodeGenFunction *CGF,
448	const VarDecl *var) {
449	// Return if this is a function parameter. We shouldn't try to
450	// rematerialize default arguments of function parameters.
451	if (isa<ParmVarDecl>(Val: var))
452	return nullptr;
453
454	QualType type = var->getType();
455
456	// We can only do this if the variable is const.
457	if (!type.isConstQualified()) return nullptr;
458
459	// Furthermore, in C++ we have to worry about mutable fields:
460	// C++ [dcl.type.cv]p4:
461	// Except that any class member declared mutable can be
462	// modified, any attempt to modify a const object during its
463	// lifetime results in undefined behavior.
464	if (CGM.getLangOpts().CPlusPlus && !isSafeForCXXConstantCapture(type))
465	return nullptr;
466
467	// If the variable doesn't have any initializer (shouldn't this be
468	// invalid?), it's not clear what we should do. Maybe capture as
469	// zero?
470	const Expr *init = var->getInit();
471	if (!init) return nullptr;
472
473	return ConstantEmitter(CGM, CGF).tryEmitAbstractForInitializer(D: *var);
474	}
475
476	/// Get the low bit of a nonzero character count. This is the
477	/// alignment of the nth byte if the 0th byte is universally aligned.
478	static CharUnits getLowBit(CharUnits v) {
479	return CharUnits::fromQuantity(Quantity: v.getQuantity() & (~v.getQuantity() + 1));
480	}
481
482	static void initializeForBlockHeader(CodeGenModule &CGM, CGBlockInfo &info,
483	SmallVectorImpl<llvm::Type*> &elementTypes) {
484
485	assert(elementTypes.empty());
486	if (CGM.getLangOpts().OpenCL) {
487	// The header is basically 'struct { int; int; generic void *;
488	// custom_fields; }'. Assert that struct is packed.
489	auto GenPtrAlign = CharUnits::fromQuantity(
490	Quantity: CGM.getTarget().getPointerAlign(AddrSpace: LangAS::opencl_generic) / 8);
491	auto GenPtrSize = CharUnits::fromQuantity(
492	Quantity: CGM.getTarget().getPointerWidth(AddrSpace: LangAS::opencl_generic) / 8);
493	assert(CGM.getIntSize() <= GenPtrSize);
494	assert(CGM.getIntAlign() <= GenPtrAlign);
495	assert((2 * CGM.getIntSize()).isMultipleOf(GenPtrAlign));
496	elementTypes.push_back(Elt: CGM.IntTy); /* total size */
497	elementTypes.push_back(Elt: CGM.IntTy); /* align */
498	elementTypes.push_back(
499	Elt: CGM.getOpenCLRuntime()
500	.getGenericVoidPointerType()); /* invoke function */
501	unsigned Offset =
502	2 * CGM.getIntSize().getQuantity() + GenPtrSize.getQuantity();
503	unsigned BlockAlign = GenPtrAlign.getQuantity();
504	if (auto *Helper =
505	CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
506	for (auto *I : Helper->getCustomFieldTypes()) /* custom fields */ {
507	// TargetOpenCLBlockHelp needs to make sure the struct is packed.
508	// If necessary, add padding fields to the custom fields.
509	unsigned Align = CGM.getDataLayout().getABITypeAlign(Ty: I).value();
510	if (BlockAlign < Align)
511	BlockAlign = Align;
512	assert(Offset % Align == 0);
513	Offset += CGM.getDataLayout().getTypeAllocSize(Ty: I);
514	elementTypes.push_back(Elt: I);
515	}
516	}
517	info.BlockAlign = CharUnits::fromQuantity(Quantity: BlockAlign);
518	info.BlockSize = CharUnits::fromQuantity(Quantity: Offset);
519	} else {
520	// The header is basically 'struct { void *; int; int; void *; void *; }'.
521	// Assert that the struct is packed.
522	assert(CGM.getIntSize() <= CGM.getPointerSize());
523	assert(CGM.getIntAlign() <= CGM.getPointerAlign());
524	assert((2 * CGM.getIntSize()).isMultipleOf(CGM.getPointerAlign()));
525	info.BlockAlign = CGM.getPointerAlign();
526	info.BlockSize = 3 * CGM.getPointerSize() + 2 * CGM.getIntSize();
527	elementTypes.push_back(Elt: CGM.VoidPtrTy);
528	elementTypes.push_back(Elt: CGM.IntTy);
529	elementTypes.push_back(Elt: CGM.IntTy);
530	elementTypes.push_back(Elt: CGM.VoidPtrTy);
531	elementTypes.push_back(Elt: CGM.getBlockDescriptorType());
532	}
533	}
534
535	static QualType getCaptureFieldType(const CodeGenFunction &CGF,
536	const BlockDecl::Capture &CI) {
537	const VarDecl *VD = CI.getVariable();
538
539	// If the variable is captured by an enclosing block or lambda expression,
540	// use the type of the capture field.
541	if (CGF.BlockInfo && CI.isNested())
542	return CGF.BlockInfo->getCapture(var: VD).fieldType();
543	if (auto *FD = CGF.LambdaCaptureFields.lookup(Val: VD))
544	return FD->getType();
545	// If the captured variable is a non-escaping __block variable, the field
546	// type is the reference type. If the variable is a __block variable that
547	// already has a reference type, the field type is the variable's type.
548	return VD->isNonEscapingByref() ?
549	CGF.getContext().getLValueReferenceType(T: VD->getType()) : VD->getType();
550	}
551
552	/// Compute the layout of the given block. Attempts to lay the block
553	/// out with minimal space requirements.
554	static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
555	CGBlockInfo &info) {
556	ASTContext &C = CGM.getContext();
557	const BlockDecl *block = info.getBlockDecl();
558
559	SmallVector<llvm::Type*, 8> elementTypes;
560	initializeForBlockHeader(CGM, info, elementTypes);
561	bool hasNonConstantCustomFields = false;
562	if (auto *OpenCLHelper =
563	CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper())
564	hasNonConstantCustomFields =
565	!OpenCLHelper->areAllCustomFieldValuesConstant(Info: info);
566	if (!block->hasCaptures() && !hasNonConstantCustomFields) {
567	info.StructureType =
568	llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
569	info.CanBeGlobal = true;
570	return;
571	} else if (C.getLangOpts().ObjC &&
572	CGM.getLangOpts().getGC() == LangOptions::NonGC)
573	info.HasCapturedVariableLayout = true;
574
575	if (block->doesNotEscape())
576	info.NoEscape = true;
577
578	// Collect the layout chunks.
579	SmallVector<BlockLayoutChunk, 16> layout;
580	layout.reserve(N: block->capturesCXXThis() +
581	(block->capture_end() - block->capture_begin()));
582
583	CharUnits maxFieldAlign;
584
585	// First, 'this'.
586	if (block->capturesCXXThis()) {
587	assert(CGF && isa_and_nonnull<CXXMethodDecl>(CGF->CurFuncDecl) &&
588	"Can't capture 'this' outside a method");
589	QualType thisType = cast<CXXMethodDecl>(Val: CGF->CurFuncDecl)->getThisType();
590
591	// Theoretically, this could be in a different address space, so
592	// don't assume standard pointer size/align.
593	llvm::Type *llvmType = CGM.getTypes().ConvertType(T: thisType);
594	auto TInfo = CGM.getContext().getTypeInfoInChars(T: thisType);
595	maxFieldAlign = std::max(a: maxFieldAlign, b: TInfo.Align);
596
597	addBlockLayout(align: TInfo.Align, size: TInfo.Width, capture: nullptr, type: llvmType, fieldType: thisType,
598	Layout&: layout, Info&: info, CGM);
599	}
600
601	// Next, all the block captures.
602	for (const auto &CI : block->captures()) {
603	const VarDecl *variable = CI.getVariable();
604
605	if (CI.isEscapingByref()) {
606	// Just use void* instead of a pointer to the byref type.
607	CharUnits align = CGM.getPointerAlign();
608	maxFieldAlign = std::max(a: maxFieldAlign, b: align);
609
610	// Since a __block variable cannot be captured by lambdas, its type and
611	// the capture field type should always match.
612	assert(CGF && getCaptureFieldType(*CGF, CI) == variable->getType() &&
613	"capture type differs from the variable type");
614	addBlockLayout(align, size: CGM.getPointerSize(), capture: &CI, type: CGM.VoidPtrTy,
615	fieldType: variable->getType(), Layout&: layout, Info&: info, CGM);
616	continue;
617	}
618
619	// Otherwise, build a layout chunk with the size and alignment of
620	// the declaration.
621	if (llvm::Constant *constant = tryCaptureAsConstant(CGM, CGF, var: variable)) {
622	info.SortedCaptures.push_back(
623	Elt: CGBlockInfo::Capture::makeConstant(value: constant, Cap: &CI));
624	continue;
625	}
626
627	QualType VT = getCaptureFieldType(CGF: *CGF, CI);
628
629	if (CGM.getLangOpts().CPlusPlus)
630	if (const CXXRecordDecl *record = VT->getAsCXXRecordDecl())
631	if (CI.hasCopyExpr() || !record->hasTrivialDestructor()) {
632	info.HasCXXObject = true;
633	if (!record->isExternallyVisible())
634	info.CapturesNonExternalType = true;
635	}
636
637	CharUnits size = C.getTypeSizeInChars(T: VT);
638	CharUnits align = C.getDeclAlign(D: variable);
639
640	maxFieldAlign = std::max(a: maxFieldAlign, b: align);
641
642	llvm::Type *llvmType =
643	CGM.getTypes().ConvertTypeForMem(T: VT);
644
645	addBlockLayout(align, size, capture: &CI, type: llvmType, fieldType: VT, Layout&: layout, Info&: info, CGM);
646	}
647
648	// If that was everything, we're done here.
649	if (layout.empty()) {
650	info.StructureType =
651	llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
652	info.CanBeGlobal = true;
653	info.buildCaptureMap();
654	return;
655	}
656
657	// Sort the layout by alignment. We have to use a stable sort here
658	// to get reproducible results. There should probably be an
659	// llvm::array_pod_stable_sort.
660	llvm::stable_sort(Range&: layout);
661
662	// Needed for blocks layout info.
663	info.BlockHeaderForcedGapOffset = info.BlockSize;
664	info.BlockHeaderForcedGapSize = CharUnits::Zero();
665
666	CharUnits &blockSize = info.BlockSize;
667	info.BlockAlign = std::max(a: maxFieldAlign, b: info.BlockAlign);
668
669	// Assuming that the first byte in the header is maximally aligned,
670	// get the alignment of the first byte following the header.
671	CharUnits endAlign = getLowBit(v: blockSize);
672
673	// If the end of the header isn't satisfactorily aligned for the
674	// maximum thing, look for things that are okay with the header-end
675	// alignment, and keep appending them until we get something that's
676	// aligned right. This algorithm is only guaranteed optimal if
677	// that condition is satisfied at some point; otherwise we can get
678	// things like:
679	// header // next byte has alignment 4
680	// something_with_size_5; // next byte has alignment 1
681	// something_with_alignment_8;
682	// which has 7 bytes of padding, as opposed to the naive solution
683	// which might have less (?).
684	if (endAlign < maxFieldAlign) {
685	SmallVectorImpl<BlockLayoutChunk>::iterator
686	li = layout.begin() + 1, le = layout.end();
687
688	// Look for something that the header end is already
689	// satisfactorily aligned for.
690	for (; li != le && endAlign < li->Alignment; ++li)
691	;
692
693	// If we found something that's naturally aligned for the end of
694	// the header, keep adding things...
695	if (li != le) {
696	SmallVectorImpl<BlockLayoutChunk>::iterator first = li;
697	for (; li != le; ++li) {
698	assert(endAlign >= li->Alignment);
699
700	li->setIndex(info, index: elementTypes.size(), offset: blockSize);
701	elementTypes.push_back(Elt: li->Type);
702	blockSize += li->Size;
703	endAlign = getLowBit(v: blockSize);
704
705	// ...until we get to the alignment of the maximum field.
706	if (endAlign >= maxFieldAlign) {
707	++li;
708	break;
709	}
710	}
711	// Don't re-append everything we just appended.
712	layout.erase(CS: first, CE: li);
713	}
714	}
715
716	assert(endAlign == getLowBit(blockSize));
717
718	// At this point, we just have to add padding if the end align still
719	// isn't aligned right.
720	if (endAlign < maxFieldAlign) {
721	CharUnits newBlockSize = blockSize.alignTo(Align: maxFieldAlign);
722	CharUnits padding = newBlockSize - blockSize;
723
724	// If we haven't yet added any fields, remember that there was an
725	// initial gap; this need to go into the block layout bit map.
726	if (blockSize == info.BlockHeaderForcedGapOffset) {
727	info.BlockHeaderForcedGapSize = padding;
728	}
729
730	elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
731	NumElements: padding.getQuantity()));
732	blockSize = newBlockSize;
733	endAlign = getLowBit(v: blockSize); // might be > maxFieldAlign
734	}
735
736	assert(endAlign >= maxFieldAlign);
737	assert(endAlign == getLowBit(blockSize));
738	// Slam everything else on now. This works because they have
739	// strictly decreasing alignment and we expect that size is always a
740	// multiple of alignment.
741	for (SmallVectorImpl<BlockLayoutChunk>::iterator
742	li = layout.begin(), le = layout.end(); li != le; ++li) {
743	if (endAlign < li->Alignment) {
744	// size may not be multiple of alignment. This can only happen with
745	// an over-aligned variable. We will be adding a padding field to
746	// make the size be multiple of alignment.
747	CharUnits padding = li->Alignment - endAlign;
748	elementTypes.push_back(Elt: llvm::ArrayType::get(ElementType: CGM.Int8Ty,
749	NumElements: padding.getQuantity()));
750	blockSize += padding;
751	endAlign = getLowBit(v: blockSize);
752	}
753	assert(endAlign >= li->Alignment);
754	li->setIndex(info, index: elementTypes.size(), offset: blockSize);
755	elementTypes.push_back(Elt: li->Type);
756	blockSize += li->Size;
757	endAlign = getLowBit(v: blockSize);
758	}
759
760	info.buildCaptureMap();
761	info.StructureType =
762	llvm::StructType::get(Context&: CGM.getLLVMContext(), Elements: elementTypes, isPacked: true);
763	}
764
765	/// Emit a block literal expression in the current function.
766	llvm::Value *CodeGenFunction::EmitBlockLiteral(const BlockExpr *blockExpr) {
767	// If the block has no captures, we won't have a pre-computed
768	// layout for it.
769	if (!blockExpr->getBlockDecl()->hasCaptures())
770	// The block literal is emitted as a global variable, and the block invoke
771	// function has to be extracted from its initializer.
772	if (llvm::Constant *Block = CGM.getAddrOfGlobalBlockIfEmitted(BE: blockExpr))
773	return Block;
774
775	CGBlockInfo blockInfo(blockExpr->getBlockDecl(), CurFn->getName());
776	computeBlockInfo(CGM, CGF: this, info&: blockInfo);
777	blockInfo.BlockExpression = blockExpr;
778	if (!blockInfo.CanBeGlobal)
779	blockInfo.LocalAddress = CreateTempAlloca(Ty: blockInfo.StructureType,
780	align: blockInfo.BlockAlign, Name: "block");
781	return EmitBlockLiteral(Info: blockInfo);
782	}
783
784	llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
785	bool IsOpenCL = CGM.getContext().getLangOpts().OpenCL;
786	llvm::PointerType *GenVoidPtrTy =
787	IsOpenCL ? CGM.getOpenCLRuntime().getGenericVoidPointerType() : VoidPtrTy;
788	LangAS GenVoidPtrAddr = IsOpenCL ? LangAS::opencl_generic : LangAS::Default;
789	auto GenVoidPtrSize = CharUnits::fromQuantity(
790	Quantity: CGM.getTarget().getPointerWidth(AddrSpace: GenVoidPtrAddr) / 8);
791	// Using the computed layout, generate the actual block function.
792	bool isLambdaConv = blockInfo.getBlockDecl()->isConversionFromLambda();
793	CodeGenFunction BlockCGF{CGM, true};
794	BlockCGF.SanOpts = SanOpts;
795	auto *InvokeFn = BlockCGF.GenerateBlockFunction(
796	GD: CurGD, Info: blockInfo, ldm: LocalDeclMap, IsLambdaConversionToBlock: isLambdaConv, BuildGlobalBlock: blockInfo.CanBeGlobal);
797	auto *blockFn = llvm::ConstantExpr::getPointerCast(C: InvokeFn, Ty: GenVoidPtrTy);
798
799	// If there is nothing to capture, we can emit this as a global block.
800	if (blockInfo.CanBeGlobal)
801	return CGM.getAddrOfGlobalBlockIfEmitted(BE: blockInfo.BlockExpression);
802
803	// Otherwise, we have to emit this as a local block.
804
805	RawAddress blockAddr = blockInfo.LocalAddress;
806	assert(blockAddr.isValid() && "block has no address!");
807
808	llvm::Constant *isa;
809	llvm::Constant *descriptor;
810	BlockFlags flags;
811	if (!IsOpenCL) {
812	// If the block is non-escaping, set field 'isa 'to NSConcreteGlobalBlock
813	// and set the BLOCK_IS_GLOBAL bit of field 'flags'. Copying a non-escaping
814	// block just returns the original block and releasing it is a no-op.
815	llvm::Constant *blockISA = blockInfo.NoEscape
816	? CGM.getNSConcreteGlobalBlock()
817	: CGM.getNSConcreteStackBlock();
818	isa = blockISA;
819
820	// Compute the initial on-stack block flags.
821	if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
822	flags = BLOCK_HAS_SIGNATURE;
823	if (blockInfo.HasCapturedVariableLayout)
824	flags |= BLOCK_HAS_EXTENDED_LAYOUT;
825	if (blockInfo.NeedsCopyDispose)
826	flags |= BLOCK_HAS_COPY_DISPOSE;
827	if (blockInfo.HasCXXObject)
828	flags |= BLOCK_HAS_CXX_OBJ;
829	if (blockInfo.UsesStret)
830	flags |= BLOCK_USE_STRET;
831	if (blockInfo.NoEscape)
832	flags |= BLOCK_IS_NOESCAPE | BLOCK_IS_GLOBAL;
833
834	// Build the block descriptor.
835	descriptor = buildBlockDescriptor(CGM, blockInfo);
836	}
837
838	auto projectField = [&](unsigned index, const Twine &name) -> Address {
839	return Builder.CreateStructGEP(Addr: blockAddr, Index: index, Name: name);
840	};
841	auto storeField = [&](llvm::Value *value, unsigned index, const Twine &name) {
842	Builder.CreateStore(Val: value, Addr: projectField(index, name));
843	};
844
845	// Initialize the block header.
846	{
847	// We assume all the header fields are densely packed.
848	unsigned index = 0;
849	CharUnits offset;
850	auto addHeaderField = [&](llvm::Value *value, CharUnits size,
851	const Twine &name) {
852	storeField(value, index, name);
853	offset += size;
854	index++;
855	};
856	auto addSignedHeaderField =
857	[&](llvm::Value *Value, const PointerAuthSchema &Schema,
858	GlobalDecl Decl, QualType Type, CharUnits Size, const Twine &Name) {
859	auto StorageAddress = projectField(index, Name);
860	if (Schema) {
861	auto AuthInfo = EmitPointerAuthInfo(
862	Schema, StorageAddress: StorageAddress.emitRawPointer(CGF&: *this), SchemaDecl: Decl, SchemaType: Type);
863	Value = EmitPointerAuthSign(Info: AuthInfo, Pointer: Value);
864	}
865	Builder.CreateStore(Val: Value, Addr: StorageAddress);
866	offset += Size;
867	index++;
868	};
869
870	if (!IsOpenCL) {
871	addSignedHeaderField(
872	isa, CGM.getCodeGenOpts().PointerAuth.ObjCIsaPointers, GlobalDecl(),
873	QualType(), getPointerSize(), "block.isa");
874	addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
875	getIntSize(), "block.flags");
876	addHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: 0), getIntSize(),
877	"block.reserved");
878	} else {
879	addHeaderField(
880	llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockSize.getQuantity()),
881	getIntSize(), "block.size");
882	addHeaderField(
883	llvm::ConstantInt::get(Ty: IntTy, V: blockInfo.BlockAlign.getQuantity()),
884	getIntSize(), "block.align");
885	}
886
887	if (!IsOpenCL) {
888	llvm::Value *blockFnPtr =
889	llvm::ConstantExpr::getBitCast(C: InvokeFn, Ty: VoidPtrTy);
890	QualType type = blockInfo.getBlockExpr()
891	->getType()
892	->castAs<BlockPointerType>()
893	->getPointeeType();
894	addSignedHeaderField(
895	blockFnPtr,
896	CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers,
897	GlobalDecl(), type, getPointerSize(), "block.invoke");
898
899	addSignedHeaderField(
900	descriptor, CGM.getCodeGenOpts().PointerAuth.BlockDescriptorPointers,
901	GlobalDecl(), type, getPointerSize(), "block.descriptor");
902	} else if (auto *Helper =
903	CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
904	addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
905	for (auto I : Helper->getCustomFieldValues(CGF&: *this, Info: blockInfo)) {
906	addHeaderField(
907	I.first,
908	CharUnits::fromQuantity(
909	Quantity: CGM.getDataLayout().getTypeAllocSize(Ty: I.first->getType())),
910	I.second);
911	}
912	} else
913	addHeaderField(blockFn, GenVoidPtrSize, "block.invoke");
914	}
915
916	// Finally, capture all the values into the block.
917	const BlockDecl *blockDecl = blockInfo.getBlockDecl();
918
919	// First, 'this'.
920	if (blockDecl->capturesCXXThis()) {
921	Address addr =
922	projectField(blockInfo.CXXThisIndex, "block.captured-this.addr");
923	Builder.CreateStore(Val: LoadCXXThis(), Addr: addr);
924	}
925
926	// Next, captured variables.
927	for (const auto &CI : blockDecl->captures()) {
928	const VarDecl *variable = CI.getVariable();
929	const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
930
931	// Ignore constant captures.
932	if (capture.isConstant()) continue;
933
934	QualType type = capture.fieldType();
935
936	// This will be a [[type]]*, except that a byref entry will just be
937	// an i8**.
938	Address blockField = projectField(capture.getIndex(), "block.captured");
939
940	// Compute the address of the thing we're going to move into the
941	// block literal.
942	Address src = Address::invalid();
943
944	if (blockDecl->isConversionFromLambda()) {
945	// The lambda capture in a lambda's conversion-to-block-pointer is
946	// special; we'll simply emit it directly.
947	src = Address::invalid();
948	} else if (CI.isEscapingByref()) {
949	if (BlockInfo && CI.isNested()) {
950	// We need to use the capture from the enclosing block.
951	const CGBlockInfo::Capture &enclosingCapture =
952	BlockInfo->getCapture(var: variable);
953
954	// This is a [[type]]*, except that a byref entry will just be an i8**.
955	src = Builder.CreateStructGEP(Addr: LoadBlockStruct(),
956	Index: enclosingCapture.getIndex(),
957	Name: "block.capture.addr");
958	} else {
959	auto I = LocalDeclMap.find(Val: variable);
960	assert(I != LocalDeclMap.end());
961	src = I->second;
962	}
963	} else {
964	DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
965	/*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
966	type.getNonReferenceType(), VK_LValue,
967	SourceLocation());
968	src = EmitDeclRefLValue(E: &declRef).getAddress();
969	};
970
971	// For byrefs, we just write the pointer to the byref struct into
972	// the block field. There's no need to chase the forwarding
973	// pointer at this point, since we're building something that will
974	// live a shorter life than the stack byref anyway.
975	if (CI.isEscapingByref()) {
976	// Get a void* that points to the byref struct.
977	llvm::Value *byrefPointer;
978	if (CI.isNested())
979	byrefPointer = Builder.CreateLoad(Addr: src, Name: "byref.capture");
980	else
981	byrefPointer = src.emitRawPointer(CGF&: *this);
982
983	// Write that void* into the capture field.
984	Builder.CreateStore(Val: byrefPointer, Addr: blockField);
985
986	// If we have a copy constructor, evaluate that into the block field.
987	} else if (const Expr *copyExpr = CI.getCopyExpr()) {
988	if (blockDecl->isConversionFromLambda()) {
989	// If we have a lambda conversion, emit the expression
990	// directly into the block instead.
991	AggValueSlot Slot =
992	AggValueSlot::forAddr(addr: blockField, quals: Qualifiers(),
993	isDestructed: AggValueSlot::IsDestructed,
994	needsGC: AggValueSlot::DoesNotNeedGCBarriers,
995	isAliased: AggValueSlot::IsNotAliased,
996	mayOverlap: AggValueSlot::DoesNotOverlap);
997	EmitAggExpr(E: copyExpr, AS: Slot);
998	} else {
999	EmitSynthesizedCXXCopyCtor(Dest: blockField, Src: src, Exp: copyExpr);
1000	}
1001
1002	// If it's a reference variable, copy the reference into the block field.
1003	} else if (type->getAs<ReferenceType>()) {
1004	Builder.CreateStore(Val: src.emitRawPointer(CGF&: *this), Addr: blockField);
1005
1006	// If type is const-qualified, copy the value into the block field.
1007	} else if (type.isConstQualified() &&
1008	type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1009	CGM.getCodeGenOpts().OptimizationLevel != 0) {
1010	llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "captured");
1011	Builder.CreateStore(Val: value, Addr: blockField);
1012
1013	// If this is an ARC __strong block-pointer variable, don't do a
1014	// block copy.
1015	//
1016	// TODO: this can be generalized into the normal initialization logic:
1017	// we should never need to do a block-copy when initializing a local
1018	// variable, because the local variable's lifetime should be strictly
1019	// contained within the stack block's.
1020	} else if (type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1021	type->isBlockPointerType()) {
1022	// Load the block and do a simple retain.
1023	llvm::Value *value = Builder.CreateLoad(Addr: src, Name: "block.captured_block");
1024	value = EmitARCRetainNonBlock(value);
1025
1026	// Do a primitive store to the block field.
1027	Builder.CreateStore(Val: value, Addr: blockField);
1028
1029	// Otherwise, fake up a POD copy into the block field.
1030	} else {
1031	// Fake up a new variable so that EmitScalarInit doesn't think
1032	// we're referring to the variable in its own initializer.
1033	ImplicitParamDecl BlockFieldPseudoVar(getContext(), type,
1034	ImplicitParamKind::Other);
1035
1036	// We use one of these or the other depending on whether the
1037	// reference is nested.
1038	DeclRefExpr declRef(getContext(), const_cast<VarDecl *>(variable),
1039	/*RefersToEnclosingVariableOrCapture*/ CI.isNested(),
1040	type, VK_LValue, SourceLocation());
1041
1042	ImplicitCastExpr l2r(ImplicitCastExpr::OnStack, type, CK_LValueToRValue,
1043	&declRef, VK_PRValue, FPOptionsOverride());
1044	// FIXME: Pass a specific location for the expr init so that the store is
1045	// attributed to a reasonable location - otherwise it may be attributed to
1046	// locations of subexpressions in the initialization.
1047	EmitExprAsInit(init: &l2r, D: &BlockFieldPseudoVar,
1048	lvalue: MakeAddrLValue(Addr: blockField, T: type, Source: AlignmentSource::Decl),
1049	/*captured by init*/ capturedByInit: false);
1050	}
1051
1052	// Push a cleanup for the capture if necessary.
1053	if (!blockInfo.NoEscape && !blockInfo.NeedsCopyDispose)
1054	continue;
1055
1056	// Ignore __block captures; there's nothing special in the on-stack block
1057	// that we need to do for them.
1058	if (CI.isByRef())
1059	continue;
1060
1061	// Ignore objects that aren't destructed.
1062	QualType::DestructionKind dtorKind = type.isDestructedType();
1063	if (dtorKind == QualType::DK_none)
1064	continue;
1065
1066	CodeGenFunction::Destroyer *destroyer;
1067
1068	// Block captures count as local values and have imprecise semantics.
1069	// They also can't be arrays, so need to worry about that.
1070	//
1071	// For const-qualified captures, emit clang.arc.use to ensure the captured
1072	// object doesn't get released while we are still depending on its validity
1073	// within the block.
1074	if (type.isConstQualified() &&
1075	type.getObjCLifetime() == Qualifiers::OCL_Strong &&
1076	CGM.getCodeGenOpts().OptimizationLevel != 0) {
1077	assert(CGM.getLangOpts().ObjCAutoRefCount &&
1078	"expected ObjC ARC to be enabled");
1079	destroyer = emitARCIntrinsicUse;
1080	} else if (dtorKind == QualType::DK_objc_strong_lifetime) {
1081	destroyer = destroyARCStrongImprecise;
1082	} else {
1083	destroyer = getDestroyer(destructionKind: dtorKind);
1084	}
1085
1086	CleanupKind cleanupKind = NormalCleanup;
1087	bool useArrayEHCleanup = needsEHCleanup(kind: dtorKind);
1088	if (useArrayEHCleanup)
1089	cleanupKind = NormalAndEHCleanup;
1090
1091	// Extend the lifetime of the capture to the end of the scope enclosing the
1092	// block expression except when the block decl is in the list of RetExpr's
1093	// cleanup objects, in which case its lifetime ends after the full
1094	// expression.
1095	auto IsBlockDeclInRetExpr = [&]() {
1096	auto *EWC = llvm::dyn_cast_or_null<ExprWithCleanups>(Val: RetExpr);
1097	if (EWC)
1098	for (auto &C : EWC->getObjects())
1099	if (auto *BD = C.dyn_cast<BlockDecl *>())
1100	if (BD == blockDecl)
1101	return true;
1102	return false;
1103	};
1104
1105	if (IsBlockDeclInRetExpr())
1106	pushDestroy(kind: cleanupKind, addr: blockField, type, destroyer, useEHCleanupForArray: useArrayEHCleanup);
1107	else
1108	pushLifetimeExtendedDestroy(kind: cleanupKind, addr: blockField, type, destroyer,
1109	useEHCleanupForArray: useArrayEHCleanup);
1110	}
1111
1112	// Cast to the converted block-pointer type, which happens (somewhat
1113	// unfortunately) to be a pointer to function type.
1114	llvm::Value *result = Builder.CreatePointerCast(
1115	V: blockAddr.getPointer(), DestTy: ConvertType(T: blockInfo.getBlockExpr()->getType()));
1116
1117	if (IsOpenCL) {
1118	CGM.getOpenCLRuntime().recordBlockInfo(E: blockInfo.BlockExpression, InvokeF: InvokeFn,
1119	Block: result, BlockTy: blockInfo.StructureType);
1120	}
1121
1122	return result;
1123	}
1124
1125
1126	llvm::Type *CodeGenModule::getBlockDescriptorType() {
1127	if (BlockDescriptorType)
1128	return BlockDescriptorType;
1129
1130	unsigned AddrSpace = 0;
1131	if (getLangOpts().OpenCL)
1132	AddrSpace = getContext().getTargetAddressSpace(AS: LangAS::opencl_constant);
1133	BlockDescriptorType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: AddrSpace);
1134	return BlockDescriptorType;
1135	}
1136
1137	llvm::Type *CodeGenModule::getGenericBlockLiteralType() {
1138	if (GenericBlockLiteralType)
1139	return GenericBlockLiteralType;
1140
1141	llvm::Type *BlockDescPtrTy = getBlockDescriptorType();
1142
1143	if (getLangOpts().OpenCL) {
1144	// struct __opencl_block_literal_generic {
1145	// int __size;
1146	// int __align;
1147	// __generic void *__invoke;
1148	// /* custom fields */
1149	// };
1150	SmallVector<llvm::Type *, 8> StructFields(
1151	{IntTy, IntTy, getOpenCLRuntime().getGenericVoidPointerType()});
1152	if (auto *Helper = getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1153	llvm::append_range(C&: StructFields, R: Helper->getCustomFieldTypes());
1154	}
1155	GenericBlockLiteralType = llvm::StructType::create(
1156	Elements: StructFields, Name: "struct.__opencl_block_literal_generic");
1157	} else {
1158	// struct __block_literal_generic {
1159	// void *__isa;
1160	// int __flags;
1161	// int __reserved;
1162	// void (*__invoke)(void *);
1163	// struct __block_descriptor *__descriptor;
1164	// };
1165	GenericBlockLiteralType =
1166	llvm::StructType::create(Name: "struct.__block_literal_generic", elt1: VoidPtrTy,
1167	elts: IntTy, elts: IntTy, elts: VoidPtrTy, elts: BlockDescPtrTy);
1168	}
1169
1170	return GenericBlockLiteralType;
1171	}
1172
1173	RValue CodeGenFunction::EmitBlockCallExpr(const CallExpr *E,
1174	ReturnValueSlot ReturnValue,
1175	llvm::CallBase **CallOrInvoke) {
1176	const auto *BPT = E->getCallee()->getType()->castAs<BlockPointerType>();
1177	llvm::Value *BlockPtr = EmitScalarExpr(E: E->getCallee());
1178	llvm::Type *GenBlockTy = CGM.getGenericBlockLiteralType();
1179	llvm::Value *Func = nullptr;
1180	QualType FnType = BPT->getPointeeType();
1181	ASTContext &Ctx = getContext();
1182	CallArgList Args;
1183
1184	llvm::Value *FuncPtr = nullptr;
1185
1186	if (getLangOpts().OpenCL) {
1187	// For OpenCL, BlockPtr is already casted to generic block literal.
1188
1189	// First argument of a block call is a generic block literal casted to
1190	// generic void pointer, i.e. i8 addrspace(4)*
1191	llvm::Type *GenericVoidPtrTy =
1192	CGM.getOpenCLRuntime().getGenericVoidPointerType();
1193	llvm::Value *BlockDescriptor = Builder.CreatePointerCast(
1194	V: BlockPtr, DestTy: GenericVoidPtrTy);
1195	QualType VoidPtrQualTy = Ctx.getPointerType(
1196	T: Ctx.getAddrSpaceQualType(T: Ctx.VoidTy, AddressSpace: LangAS::opencl_generic));
1197	Args.add(rvalue: RValue::get(V: BlockDescriptor), type: VoidPtrQualTy);
1198	// And the rest of the arguments.
1199	EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1200
1201	// We *can* call the block directly unless it is a function argument.
1202	if (!isa<ParmVarDecl>(Val: E->getCalleeDecl()))
1203	Func = CGM.getOpenCLRuntime().getInvokeFunction(E: E->getCallee());
1204	else {
1205	FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 2);
1206	Func = Builder.CreateAlignedLoad(Ty: GenericVoidPtrTy, Addr: FuncPtr,
1207	Align: getPointerAlign());
1208	}
1209	} else {
1210	// Bitcast the block literal to a generic block literal.
1211	BlockPtr =
1212	Builder.CreatePointerCast(V: BlockPtr, DestTy: UnqualPtrTy, Name: "block.literal");
1213	// Get pointer to the block invoke function
1214	FuncPtr = Builder.CreateStructGEP(Ty: GenBlockTy, Ptr: BlockPtr, Idx: 3);
1215
1216	// First argument is a block literal casted to a void pointer
1217	BlockPtr = Builder.CreatePointerCast(V: BlockPtr, DestTy: VoidPtrTy);
1218	Args.add(rvalue: RValue::get(V: BlockPtr), type: Ctx.VoidPtrTy);
1219	// And the rest of the arguments.
1220	EmitCallArgs(Args, Prototype: FnType->getAs<FunctionProtoType>(), ArgRange: E->arguments());
1221
1222	// Load the function.
1223	Func = Builder.CreateAlignedLoad(Ty: VoidPtrTy, Addr: FuncPtr, Align: getPointerAlign());
1224	}
1225
1226	const FunctionType *FuncTy = FnType->castAs<FunctionType>();
1227	const CGFunctionInfo &FnInfo =
1228	CGM.getTypes().arrangeBlockFunctionCall(args: Args, type: FuncTy);
1229
1230	// Prepare the callee.
1231	CGPointerAuthInfo PointerAuth;
1232	if (auto &AuthSchema =
1233	CGM.getCodeGenOpts().PointerAuth.BlockInvocationFunctionPointers) {
1234	assert(FuncPtr != nullptr && "Missing function pointer for AuthInfo");
1235	PointerAuth =
1236	EmitPointerAuthInfo(Schema: AuthSchema, StorageAddress: FuncPtr, SchemaDecl: GlobalDecl(), SchemaType: FnType);
1237	}
1238
1239	CGCallee Callee(CGCalleeInfo(), Func, PointerAuth);
1240
1241	// And call the block.
1242	return EmitCall(CallInfo: FnInfo, Callee, ReturnValue, Args, CallOrInvoke);
1243	}
1244
1245	Address CodeGenFunction::GetAddrOfBlockDecl(const VarDecl *variable) {
1246	assert(BlockInfo && "evaluating block ref without block information?");
1247	const CGBlockInfo::Capture &capture = BlockInfo->getCapture(var: variable);
1248
1249	// Handle constant captures.
1250	if (capture.isConstant()) return LocalDeclMap.find(Val: variable)->second;
1251
1252	Address addr = Builder.CreateStructGEP(Addr: LoadBlockStruct(), Index: capture.getIndex(),
1253	Name: "block.capture.addr");
1254
1255	if (variable->isEscapingByref()) {
1256	// addr should be a void** right now. Load, then cast the result
1257	// to byref*.
1258
1259	auto &byrefInfo = getBlockByrefInfo(var: variable);
1260	addr = Address(Builder.CreateLoad(Addr: addr), byrefInfo.Type,
1261	byrefInfo.ByrefAlignment);
1262
1263	addr = emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, /*follow*/ followForward: true,
1264	name: variable->getName());
1265	}
1266
1267	assert((!variable->isNonEscapingByref() ||
1268	capture.fieldType()->isReferenceType()) &&
1269	"the capture field of a non-escaping variable should have a "
1270	"reference type");
1271	if (capture.fieldType()->isReferenceType())
1272	addr = EmitLoadOfReference(RefLVal: MakeAddrLValue(Addr: addr, T: capture.fieldType()));
1273
1274	return addr;
1275	}
1276
1277	void CodeGenModule::setAddrOfGlobalBlock(const BlockExpr *BE,
1278	llvm::Constant *Addr) {
1279	bool Ok = EmittedGlobalBlocks.insert(KV: std::make_pair(x&: BE, y&: Addr)).second;
1280	(void)Ok;
1281	assert(Ok && "Trying to replace an already-existing global block!");
1282	}
1283
1284	llvm::Constant *
1285	CodeGenModule::GetAddrOfGlobalBlock(const BlockExpr *BE,
1286	StringRef Name) {
1287	if (llvm::Constant *Block = getAddrOfGlobalBlockIfEmitted(BE))
1288	return Block;
1289
1290	CGBlockInfo blockInfo(BE->getBlockDecl(), Name);
1291	blockInfo.BlockExpression = BE;
1292
1293	// Compute information about the layout, etc., of this block.
1294	computeBlockInfo(CGM&: *this, CGF: nullptr, info&: blockInfo);
1295
1296	// Using that metadata, generate the actual block function.
1297	{
1298	CodeGenFunction::DeclMapTy LocalDeclMap;
1299	CodeGenFunction(*this).GenerateBlockFunction(
1300	GD: GlobalDecl(), Info: blockInfo, ldm: LocalDeclMap,
1301	/*IsLambdaConversionToBlock*/ false, /*BuildGlobalBlock*/ true);
1302	}
1303
1304	return getAddrOfGlobalBlockIfEmitted(BE);
1305	}
1306
1307	static llvm::Constant *buildGlobalBlock(CodeGenModule &CGM,
1308	const CGBlockInfo &blockInfo,
1309	llvm::Constant *blockFn) {
1310	assert(blockInfo.CanBeGlobal);
1311	// Callers should detect this case on their own: calling this function
1312	// generally requires computing layout information, which is a waste of time
1313	// if we've already emitted this block.
1314	assert(!CGM.getAddrOfGlobalBlockIfEmitted(blockInfo.BlockExpression) &&
1315	"Refusing to re-emit a global block.");
1316
1317	// Generate the constants for the block literal initializer.
1318	ConstantInitBuilder builder(CGM);
1319	auto fields = builder.beginStruct();
1320
1321	bool IsOpenCL = CGM.getLangOpts().OpenCL;
1322	bool IsWindows = CGM.getTarget().getTriple().isOSWindows();
1323	auto &CGOPointerAuth = CGM.getCodeGenOpts().PointerAuth;
1324	if (!IsOpenCL) {
1325	// isa
1326	if (IsWindows)
1327	fields.addNullPointer(ptrTy: CGM.Int8PtrPtrTy);
1328	else
1329	fields.addSignedPointer(Pointer: CGM.getNSConcreteGlobalBlock(),
1330	Schema: CGOPointerAuth.ObjCIsaPointers, CalleeDecl: GlobalDecl(),
1331	CalleeType: QualType());
1332
1333	// __flags
1334	BlockFlags flags = BLOCK_IS_GLOBAL;
1335	if (!CGM.getCodeGenOpts().DisableBlockSignatureString)
1336	flags |= BLOCK_HAS_SIGNATURE;
1337	if (blockInfo.UsesStret)
1338	flags |= BLOCK_USE_STRET;
1339
1340	fields.addInt(intTy: CGM.IntTy, value: flags.getBitMask());
1341
1342	// Reserved
1343	fields.addInt(intTy: CGM.IntTy, value: 0);
1344	} else {
1345	fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockSize.getQuantity());
1346	fields.addInt(intTy: CGM.IntTy, value: blockInfo.BlockAlign.getQuantity());
1347	}
1348
1349	// Function
1350	if (auto &Schema = CGOPointerAuth.BlockInvocationFunctionPointers) {
1351	QualType FnType = blockInfo.getBlockExpr()
1352	->getType()
1353	->castAs<BlockPointerType>()
1354	->getPointeeType();
1355	fields.addSignedPointer(Pointer: blockFn, Schema, CalleeDecl: GlobalDecl(), CalleeType: FnType);
1356	} else
1357	fields.add(value: blockFn);
1358
1359	if (!IsOpenCL) {
1360	// Descriptor
1361	llvm::Constant *Descriptor = buildBlockDescriptor(CGM, blockInfo);
1362	fields.addSignedPointer(Pointer: Descriptor, Schema: CGOPointerAuth.BlockDescriptorPointers,
1363	CalleeDecl: GlobalDecl(), CalleeType: QualType());
1364	} else if (auto *Helper =
1365	CGM.getTargetCodeGenInfo().getTargetOpenCLBlockHelper()) {
1366	for (auto *I : Helper->getCustomFieldValues(CGM, Info: blockInfo)) {
1367	fields.add(value: I);
1368	}
1369	}
1370
1371	unsigned AddrSpace = 0;
1372	if (CGM.getContext().getLangOpts().OpenCL)
1373	AddrSpace = CGM.getContext().getTargetAddressSpace(AS: LangAS::opencl_global);
1374
1375	llvm::GlobalVariable *literal = fields.finishAndCreateGlobal(
1376	args: "__block_literal_global", args: blockInfo.BlockAlign,
1377	/*constant*/ args: !IsWindows, args: llvm::GlobalVariable::InternalLinkage, args&: AddrSpace);
1378
1379	literal->addAttribute(Kind: "objc_arc_inert");
1380
1381	// Windows does not allow globals to be initialised to point to globals in
1382	// different DLLs. Any such variables must run code to initialise them.
1383	if (IsWindows) {
1384	auto *Init = llvm::Function::Create(Ty: llvm::FunctionType::get(Result: CGM.VoidTy,
1385	isVarArg: {}), Linkage: llvm::GlobalValue::InternalLinkage, N: ".block_isa_init",
1386	M: &CGM.getModule());
1387	llvm::IRBuilder<> b(llvm::BasicBlock::Create(Context&: CGM.getLLVMContext(), Name: "entry",
1388	Parent: Init));
1389	b.CreateAlignedStore(Val: CGM.getNSConcreteGlobalBlock(),
1390	Ptr: b.CreateStructGEP(Ty: literal->getValueType(), Ptr: literal, Idx: 0),
1391	Align: CGM.getPointerAlign().getAsAlign());
1392	b.CreateRetVoid();
1393	// We can't use the normal LLVM global initialisation array, because we
1394	// need to specify that this runs early in library initialisation.
1395	auto *InitVar = new llvm::GlobalVariable(CGM.getModule(), Init->getType(),
1396	/*isConstant*/true, llvm::GlobalValue::InternalLinkage,
1397	Init, ".block_isa_init_ptr");
1398	InitVar->setSection(".CRT$XCLa");
1399	CGM.addUsedGlobal(GV: InitVar);
1400	}
1401
1402	// Return a constant of the appropriately-casted type.
1403	llvm::Type *RequiredType =
1404	CGM.getTypes().ConvertType(T: blockInfo.getBlockExpr()->getType());
1405	llvm::Constant *Result =
1406	llvm::ConstantExpr::getPointerCast(C: literal, Ty: RequiredType);
1407	CGM.setAddrOfGlobalBlock(BE: blockInfo.BlockExpression, Addr: Result);
1408	if (CGM.getContext().getLangOpts().OpenCL)
1409	CGM.getOpenCLRuntime().recordBlockInfo(
1410	E: blockInfo.BlockExpression,
1411	InvokeF: cast<llvm::Function>(Val: blockFn->stripPointerCasts()), Block: Result,
1412	BlockTy: literal->getValueType());
1413	return Result;
1414	}
1415
1416	void CodeGenFunction::setBlockContextParameter(const ImplicitParamDecl *D,
1417	unsigned argNum,
1418	llvm::Value *arg) {
1419	assert(BlockInfo && "not emitting prologue of block invocation function?!");
1420
1421	// Allocate a stack slot like for any local variable to guarantee optimal
1422	// debug info at -O0. The mem2reg pass will eliminate it when optimizing.
1423	RawAddress alloc = CreateMemTemp(T: D->getType(), Name: D->getName() + ".addr");
1424	Builder.CreateStore(Val: arg, Addr: alloc);
1425	if (CGDebugInfo *DI = getDebugInfo()) {
1426	if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1427	DI->setLocation(D->getLocation());
1428	DI->EmitDeclareOfBlockLiteralArgVariable(
1429	block: *BlockInfo, Name: D->getName(), ArgNo: argNum,
1430	LocalAddr: cast<llvm::AllocaInst>(Val: alloc.getPointer()->stripPointerCasts()),
1431	Builder);
1432	}
1433	}
1434
1435	SourceLocation StartLoc = BlockInfo->getBlockExpr()->getBody()->getBeginLoc();
1436	ApplyDebugLocation Scope(*this, StartLoc);
1437
1438	// Instead of messing around with LocalDeclMap, just set the value
1439	// directly as BlockPointer.
1440	BlockPointer = Builder.CreatePointerCast(
1441	V: arg,
1442	DestTy: llvm::PointerType::get(
1443	C&: getLLVMContext(),
1444	AddressSpace: getContext().getLangOpts().OpenCL
1445	? getContext().getTargetAddressSpace(AS: LangAS::opencl_generic)
1446	: 0),
1447	Name: "block");
1448	}
1449
1450	Address CodeGenFunction::LoadBlockStruct() {
1451	assert(BlockInfo && "not in a block invocation function!");
1452	assert(BlockPointer && "no block pointer set!");
1453	return Address(BlockPointer, BlockInfo->StructureType, BlockInfo->BlockAlign);
1454	}
1455
1456	llvm::Function *CodeGenFunction::GenerateBlockFunction(
1457	GlobalDecl GD, const CGBlockInfo &blockInfo, const DeclMapTy &ldm,
1458	bool IsLambdaConversionToBlock, bool BuildGlobalBlock) {
1459	const BlockDecl *blockDecl = blockInfo.getBlockDecl();
1460
1461	CurGD = GD;
1462
1463	CurEHLocation = blockInfo.getBlockExpr()->getEndLoc();
1464
1465	BlockInfo = &blockInfo;
1466
1467	// Arrange for local static and local extern declarations to appear
1468	// to be local to this function as well, in case they're directly
1469	// referenced in a block.
1470	for (const auto &KV : ldm) {
1471	const auto *var = dyn_cast<VarDecl>(Val: KV.first);
1472	if (var && !var->hasLocalStorage())
1473	setAddrOfLocalVar(VD: var, Addr: KV.second);
1474	}
1475
1476	// Begin building the function declaration.
1477
1478	// Build the argument list.
1479	FunctionArgList args;
1480
1481	// The first argument is the block pointer. Just take it as a void*
1482	// and cast it later.
1483	QualType selfTy = getContext().VoidPtrTy;
1484
1485	// For OpenCL passed block pointer can be private AS local variable or
1486	// global AS program scope variable (for the case with and without captures).
1487	// Generic AS is used therefore to be able to accommodate both private and
1488	// generic AS in one implementation.
1489	if (getLangOpts().OpenCL)
1490	selfTy = getContext().getPointerType(T: getContext().getAddrSpaceQualType(
1491	T: getContext().VoidTy, AddressSpace: LangAS::opencl_generic));
1492
1493	const IdentifierInfo *II = &CGM.getContext().Idents.get(Name: ".block_descriptor");
1494
1495	ImplicitParamDecl SelfDecl(getContext(), const_cast<BlockDecl *>(blockDecl),
1496	SourceLocation(), II, selfTy,
1497	ImplicitParamKind::ObjCSelf);
1498	args.push_back(Elt: &SelfDecl);
1499
1500	// Now add the rest of the parameters.
1501	args.append(in_start: blockDecl->param_begin(), in_end: blockDecl->param_end());
1502
1503	// Create the function declaration.
1504	const FunctionProtoType *fnType = blockInfo.getBlockExpr()->getFunctionType();
1505	const CGFunctionInfo &fnInfo =
1506	CGM.getTypes().arrangeBlockFunctionDeclaration(type: fnType, args);
1507	if (CGM.ReturnSlotInterferesWithArgs(FI: fnInfo))
1508	blockInfo.UsesStret = true;
1509
1510	llvm::FunctionType *fnLLVMType = CGM.getTypes().GetFunctionType(Info: fnInfo);
1511
1512	StringRef name = CGM.getBlockMangledName(GD, BD: blockDecl);
1513	llvm::Function *fn = llvm::Function::Create(
1514	Ty: fnLLVMType, Linkage: llvm::GlobalValue::InternalLinkage, N: name, M: &CGM.getModule());
1515	CGM.SetInternalFunctionAttributes(GD: blockDecl, F: fn, FI: fnInfo);
1516
1517	if (BuildGlobalBlock) {
1518	auto GenVoidPtrTy = getContext().getLangOpts().OpenCL
1519	? CGM.getOpenCLRuntime().getGenericVoidPointerType()
1520	: VoidPtrTy;
1521	buildGlobalBlock(CGM, blockInfo,
1522	blockFn: llvm::ConstantExpr::getPointerCast(C: fn, Ty: GenVoidPtrTy));
1523	}
1524
1525	// Begin generating the function.
1526	StartFunction(GD: blockDecl, RetTy: fnType->getReturnType(), Fn: fn, FnInfo: fnInfo, Args: args,
1527	Loc: blockDecl->getLocation(),
1528	StartLoc: blockInfo.getBlockExpr()->getBody()->getBeginLoc());
1529
1530	// Okay. Undo some of what StartFunction did.
1531
1532	// At -O0 we generate an explicit alloca for the BlockPointer, so the RA
1533	// won't delete the dbg.declare intrinsics for captured variables.
1534	llvm::Value *BlockPointerDbgLoc = BlockPointer;
1535	if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
1536	// Allocate a stack slot for it, so we can point the debugger to it
1537	Address Alloca = CreateTempAlloca(Ty: BlockPointer->getType(),
1538	align: getPointerAlign(),
1539	Name: "block.addr");
1540	// Set the DebugLocation to empty, so the store is recognized as a
1541	// frame setup instruction by llvm::DwarfDebug::beginFunction().
1542	auto NL = ApplyDebugLocation::CreateEmpty(CGF&: *this);
1543	Builder.CreateStore(Val: BlockPointer, Addr: Alloca);
1544	BlockPointerDbgLoc = Alloca.emitRawPointer(CGF&: *this);
1545	}
1546
1547	// If we have a C++ 'this' reference, go ahead and force it into
1548	// existence now.
1549	if (blockDecl->capturesCXXThis()) {
1550	Address addr = Builder.CreateStructGEP(
1551	Addr: LoadBlockStruct(), Index: blockInfo.CXXThisIndex, Name: "block.captured-this");
1552	CXXThisValue = Builder.CreateLoad(Addr: addr, Name: "this");
1553	}
1554
1555	// Also force all the constant captures.
1556	for (const auto &CI : blockDecl->captures()) {
1557	const VarDecl *variable = CI.getVariable();
1558	const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1559	if (!capture.isConstant()) continue;
1560
1561	CharUnits align = getContext().getDeclAlign(D: variable);
1562	Address alloca =
1563	CreateMemTemp(T: variable->getType(), Align: align, Name: "block.captured-const");
1564
1565	Builder.CreateStore(Val: capture.getConstant(), Addr: alloca);
1566
1567	setAddrOfLocalVar(VD: variable, Addr: alloca);
1568	}
1569
1570	// Save a spot to insert the debug information for all the DeclRefExprs.
1571	llvm::BasicBlock *entry = Builder.GetInsertBlock();
1572	llvm::BasicBlock::iterator entry_ptr = Builder.GetInsertPoint();
1573	--entry_ptr;
1574
1575	if (IsLambdaConversionToBlock)
1576	EmitLambdaBlockInvokeBody();
1577	else {
1578	PGO->assignRegionCounters(GD: GlobalDecl(blockDecl), Fn: fn);
1579	incrementProfileCounter(S: blockDecl->getBody());
1580	EmitStmt(S: blockDecl->getBody());
1581	}
1582
1583	// Remember where we were...
1584	llvm::BasicBlock *resume = Builder.GetInsertBlock();
1585
1586	// Go back to the entry.
1587	if (entry_ptr->getNextNonDebugInstruction())
1588	entry_ptr = entry_ptr->getNextNonDebugInstruction()->getIterator();
1589	else
1590	entry_ptr = entry->end();
1591	Builder.SetInsertPoint(TheBB: entry, IP: entry_ptr);
1592
1593	// Emit debug information for all the DeclRefExprs.
1594	// FIXME: also for 'this'
1595	if (CGDebugInfo *DI = getDebugInfo()) {
1596	for (const auto &CI : blockDecl->captures()) {
1597	const VarDecl *variable = CI.getVariable();
1598	DI->EmitLocation(Builder, Loc: variable->getLocation());
1599
1600	if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
1601	const CGBlockInfo::Capture &capture = blockInfo.getCapture(var: variable);
1602	if (capture.isConstant()) {
1603	auto addr = LocalDeclMap.find(Val: variable)->second;
1604	(void)DI->EmitDeclareOfAutoVariable(
1605	Decl: variable, AI: addr.emitRawPointer(CGF&: *this), Builder);
1606	continue;
1607	}
1608
1609	DI->EmitDeclareOfBlockDeclRefVariable(
1610	variable, storage: BlockPointerDbgLoc, Builder, blockInfo,
1611	InsertPoint: entry_ptr == entry->end() ? nullptr : &*entry_ptr);
1612	}
1613	}
1614	// Recover location if it was changed in the above loop.
1615	DI->EmitLocation(Builder,
1616	Loc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1617	}
1618
1619	// And resume where we left off.
1620	if (resume == nullptr)
1621	Builder.ClearInsertionPoint();
1622	else
1623	Builder.SetInsertPoint(resume);
1624
1625	FinishFunction(EndLoc: cast<CompoundStmt>(Val: blockDecl->getBody())->getRBracLoc());
1626
1627	return fn;
1628	}
1629
1630	static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
1631	computeCopyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
1632	const LangOptions &LangOpts) {
1633	if (CI.getCopyExpr()) {
1634	assert(!CI.isByRef());
1635	// don't bother computing flags
1636	return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
1637	}
1638	BlockFieldFlags Flags;
1639	if (CI.isEscapingByref()) {
1640	Flags = BLOCK_FIELD_IS_BYREF;
1641	if (T.isObjCGCWeak())
1642	Flags |= BLOCK_FIELD_IS_WEAK;
1643	return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1644	}
1645
1646	if (T.hasAddressDiscriminatedPointerAuth())
1647	return std::make_pair(
1648	x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth, y&: Flags);
1649
1650	Flags = BLOCK_FIELD_IS_OBJECT;
1651	bool isBlockPointer = T->isBlockPointerType();
1652	if (isBlockPointer)
1653	Flags = BLOCK_FIELD_IS_BLOCK;
1654
1655	switch (T.isNonTrivialToPrimitiveCopy()) {
1656	case QualType::PCK_Struct:
1657	return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
1658	y: BlockFieldFlags());
1659	case QualType::PCK_ARCWeak:
1660	// We need to register __weak direct captures with the runtime.
1661	return std::make_pair(x: BlockCaptureEntityKind::ARCWeak, y&: Flags);
1662	case QualType::PCK_ARCStrong:
1663	// We need to retain the copied value for __strong direct captures.
1664	// If it's a block pointer, we have to copy the block and assign that to
1665	// the destination pointer, so we might as well use _Block_object_assign.
1666	// Otherwise we can avoid that.
1667	return std::make_pair(x: !isBlockPointer ? BlockCaptureEntityKind::ARCStrong
1668	: BlockCaptureEntityKind::BlockObject,
1669	y&: Flags);
1670	case QualType::PCK_PtrAuth:
1671	return std::make_pair(
1672	x: BlockCaptureEntityKind::AddressDiscriminatedPointerAuth,
1673	y: BlockFieldFlags());
1674	case QualType::PCK_Trivial:
1675	case QualType::PCK_VolatileTrivial: {
1676	if (!T->isObjCRetainableType())
1677	// For all other types, the memcpy is fine.
1678	return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1679
1680	// Honor the inert __unsafe_unretained qualifier, which doesn't actually
1681	// make it into the type system.
1682	if (T->isObjCInertUnsafeUnretainedType())
1683	return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1684
1685	// Special rules for ARC captures:
1686	Qualifiers QS = T.getQualifiers();
1687
1688	// Non-ARC captures of retainable pointers are strong and
1689	// therefore require a call to _Block_object_assign.
1690	if (!QS.getObjCLifetime() && !LangOpts.ObjCAutoRefCount)
1691	return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
1692
1693	// Otherwise the memcpy is fine.
1694	return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
1695	}
1696	}
1697	llvm_unreachable("after exhaustive PrimitiveCopyKind switch");
1698	}
1699
1700	namespace {
1701	/// Release a __block variable.
1702	struct CallBlockRelease final : EHScopeStack::Cleanup {
1703	Address Addr;
1704	BlockFieldFlags FieldFlags;
1705	bool LoadBlockVarAddr, CanThrow;
1706
1707	CallBlockRelease(Address Addr, BlockFieldFlags Flags, bool LoadValue,
1708	bool CT)
1709	: Addr(Addr), FieldFlags(Flags), LoadBlockVarAddr(LoadValue),
1710	CanThrow(CT) {}
1711
1712	void Emit(CodeGenFunction &CGF, Flags flags) override {
1713	llvm::Value *BlockVarAddr;
1714	if (LoadBlockVarAddr) {
1715	BlockVarAddr = CGF.Builder.CreateLoad(Addr);
1716	} else {
1717	BlockVarAddr = Addr.emitRawPointer(CGF);
1718	}
1719
1720	CGF.BuildBlockRelease(DeclPtr: BlockVarAddr, flags: FieldFlags, CanThrow);
1721	}
1722	};
1723	} // end anonymous namespace
1724
1725	/// Check if \p T is a C++ class that has a destructor that can throw.
1726	bool CodeGenFunction::cxxDestructorCanThrow(QualType T) {
1727	if (const auto *RD = T->getAsCXXRecordDecl())
1728	if (const CXXDestructorDecl *DD = RD->getDestructor())
1729	return DD->getType()->castAs<FunctionProtoType>()->canThrow();
1730	return false;
1731	}
1732
1733	// Return a string that has the information about a capture.
1734	static std::string getBlockCaptureStr(const CGBlockInfo::Capture &Cap,
1735	CaptureStrKind StrKind,
1736	CharUnits BlockAlignment,
1737	CodeGenModule &CGM) {
1738	std::string Str;
1739	ASTContext &Ctx = CGM.getContext();
1740	const BlockDecl::Capture &CI = *Cap.Cap;
1741	QualType CaptureTy = CI.getVariable()->getType();
1742
1743	BlockCaptureEntityKind Kind;
1744	BlockFieldFlags Flags;
1745
1746	// CaptureStrKind::Merged should be passed only when the operations and the
1747	// flags are the same for copy and dispose.
1748	assert((StrKind != CaptureStrKind::Merged ||
1749	(Cap.CopyKind == Cap.DisposeKind &&
1750	Cap.CopyFlags == Cap.DisposeFlags)) &&
1751	"different operations and flags");
1752
1753	if (StrKind == CaptureStrKind::DisposeHelper) {
1754	Kind = Cap.DisposeKind;
1755	Flags = Cap.DisposeFlags;
1756	} else {
1757	Kind = Cap.CopyKind;
1758	Flags = Cap.CopyFlags;
1759	}
1760
1761	switch (Kind) {
1762	case BlockCaptureEntityKind::CXXRecord: {
1763	Str += "c";
1764	SmallString<256> TyStr;
1765	llvm::raw_svector_ostream Out(TyStr);
1766	CGM.getCXXABI().getMangleContext().mangleCanonicalTypeName(T: CaptureTy, Out);
1767	Str += llvm::to_string(Value: TyStr.size()) + TyStr.c_str();
1768	break;
1769	}
1770	case BlockCaptureEntityKind::ARCWeak:
1771	Str += "w";
1772	break;
1773	case BlockCaptureEntityKind::ARCStrong:
1774	Str += "s";
1775	break;
1776	case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1777	auto PtrAuth = CaptureTy.getPointerAuth();
1778	assert(PtrAuth && PtrAuth.isAddressDiscriminated());
1779	Str += "p" + llvm::to_string(Value: PtrAuth.getKey()) + "d" +
1780	llvm::to_string(Value: PtrAuth.getExtraDiscriminator());
1781	break;
1782	}
1783	case BlockCaptureEntityKind::BlockObject: {
1784	const VarDecl *Var = CI.getVariable();
1785	unsigned F = Flags.getBitMask();
1786	if (F & BLOCK_FIELD_IS_BYREF) {
1787	Str += "r";
1788	if (F & BLOCK_FIELD_IS_WEAK)
1789	Str += "w";
1790	else {
1791	// If CaptureStrKind::Merged is passed, check both the copy expression
1792	// and the destructor.
1793	if (StrKind != CaptureStrKind::DisposeHelper) {
1794	if (Ctx.getBlockVarCopyInit(VD: Var).canThrow())
1795	Str += "c";
1796	}
1797	if (StrKind != CaptureStrKind::CopyHelper) {
1798	if (CodeGenFunction::cxxDestructorCanThrow(T: CaptureTy))
1799	Str += "d";
1800	}
1801	}
1802	} else {
1803	assert((F & BLOCK_FIELD_IS_OBJECT) && "unexpected flag value");
1804	if (F == BLOCK_FIELD_IS_BLOCK)
1805	Str += "b";
1806	else
1807	Str += "o";
1808	}
1809	break;
1810	}
1811	case BlockCaptureEntityKind::NonTrivialCStruct: {
1812	bool IsVolatile = CaptureTy.isVolatileQualified();
1813	CharUnits Alignment = BlockAlignment.alignmentAtOffset(offset: Cap.getOffset());
1814
1815	Str += "n";
1816	std::string FuncStr;
1817	if (StrKind == CaptureStrKind::DisposeHelper)
1818	FuncStr = CodeGenFunction::getNonTrivialDestructorStr(
1819	QT: CaptureTy, Alignment, IsVolatile, Ctx);
1820	else
1821	// If CaptureStrKind::Merged is passed, use the copy constructor string.
1822	// It has all the information that the destructor string has.
1823	FuncStr = CodeGenFunction::getNonTrivialCopyConstructorStr(
1824	QT: CaptureTy, Alignment, IsVolatile, Ctx);
1825	// The underscore is necessary here because non-trivial copy constructor
1826	// and destructor strings can start with a number.
1827	Str += llvm::to_string(Value: FuncStr.size()) + "_" + FuncStr;
1828	break;
1829	}
1830	case BlockCaptureEntityKind::None:
1831	break;
1832	}
1833
1834	return Str;
1835	}
1836
1837	static std::string getCopyDestroyHelperFuncName(
1838	const SmallVectorImpl<CGBlockInfo::Capture> &Captures,
1839	CharUnits BlockAlignment, CaptureStrKind StrKind, CodeGenModule &CGM) {
1840	assert((StrKind == CaptureStrKind::CopyHelper ||
1841	StrKind == CaptureStrKind::DisposeHelper) &&
1842	"unexpected CaptureStrKind");
1843	std::string Name = StrKind == CaptureStrKind::CopyHelper
1844	? "__copy_helper_block_"
1845	: "__destroy_helper_block_";
1846	if (CGM.getLangOpts().Exceptions)
1847	Name += "e";
1848	if (CGM.getCodeGenOpts().ObjCAutoRefCountExceptions)
1849	Name += "a";
1850	Name += llvm::to_string(Value: BlockAlignment.getQuantity()) + "_";
1851
1852	for (auto &Cap : Captures) {
1853	if (Cap.isConstantOrTrivial())
1854	continue;
1855	Name += llvm::to_string(Value: Cap.getOffset().getQuantity());
1856	Name += getBlockCaptureStr(Cap, StrKind, BlockAlignment, CGM);
1857	}
1858
1859	return Name;
1860	}
1861
1862	static void pushCaptureCleanup(BlockCaptureEntityKind CaptureKind,
1863	Address Field, QualType CaptureType,
1864	BlockFieldFlags Flags, bool ForCopyHelper,
1865	VarDecl *Var, CodeGenFunction &CGF) {
1866	bool EHOnly = ForCopyHelper;
1867
1868	switch (CaptureKind) {
1869	case BlockCaptureEntityKind::CXXRecord:
1870	case BlockCaptureEntityKind::ARCWeak:
1871	case BlockCaptureEntityKind::NonTrivialCStruct:
1872	case BlockCaptureEntityKind::ARCStrong: {
1873	if (CaptureType.isDestructedType() &&
1874	(!EHOnly || CGF.needsEHCleanup(kind: CaptureType.isDestructedType()))) {
1875	CodeGenFunction::Destroyer *Destroyer =
1876	CaptureKind == BlockCaptureEntityKind::ARCStrong
1877	? CodeGenFunction::destroyARCStrongImprecise
1878	: CGF.getDestroyer(destructionKind: CaptureType.isDestructedType());
1879	CleanupKind Kind =
1880	EHOnly ? EHCleanup
1881	: CGF.getCleanupKind(kind: CaptureType.isDestructedType());
1882	CGF.pushDestroy(kind: Kind, addr: Field, type: CaptureType, destroyer: Destroyer, useEHCleanupForArray: Kind & EHCleanup);
1883	}
1884	break;
1885	}
1886	case BlockCaptureEntityKind::BlockObject: {
1887	if (!EHOnly || CGF.getLangOpts().Exceptions) {
1888	CleanupKind Kind = EHOnly ? EHCleanup : NormalAndEHCleanup;
1889	// Calls to _Block_object_dispose along the EH path in the copy helper
1890	// function don't throw as newly-copied __block variables always have a
1891	// reference count of 2.
1892	bool CanThrow =
1893	!ForCopyHelper && CGF.cxxDestructorCanThrow(T: CaptureType);
1894	CGF.enterByrefCleanup(Kind, Addr: Field, Flags, /*LoadBlockVarAddr*/ true,
1895	CanThrow);
1896	}
1897	break;
1898	}
1899	case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth:
1900	case BlockCaptureEntityKind::None:
1901	break;
1902	}
1903	}
1904
1905	static void setBlockHelperAttributesVisibility(bool CapturesNonExternalType,
1906	llvm::Function *Fn,
1907	const CGFunctionInfo &FI,
1908	CodeGenModule &CGM) {
1909	if (CapturesNonExternalType) {
1910	CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
1911	} else {
1912	Fn->setVisibility(llvm::GlobalValue::HiddenVisibility);
1913	Fn->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1914	CGM.SetLLVMFunctionAttributes(GD: GlobalDecl(), Info: FI, F: Fn, /*IsThunk=*/false);
1915	CGM.SetLLVMFunctionAttributesForDefinition(D: nullptr, F: Fn);
1916	}
1917	}
1918	/// Generate the copy-helper function for a block closure object:
1919	/// static void block_copy_helper(block_t *dst, block_t *src);
1920	/// The runtime will have previously initialized 'dst' by doing a
1921	/// bit-copy of 'src'.
1922	///
1923	/// Note that this copies an entire block closure object to the heap;
1924	/// it should not be confused with a 'byref copy helper', which moves
1925	/// the contents of an individual __block variable to the heap.
1926	llvm::Constant *
1927	CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
1928	std::string FuncName = getCopyDestroyHelperFuncName(
1929	Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
1930	StrKind: CaptureStrKind::CopyHelper, CGM);
1931
1932	if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
1933	return Func;
1934
1935	ASTContext &C = getContext();
1936
1937	QualType ReturnTy = C.VoidTy;
1938
1939	FunctionArgList args;
1940	ImplicitParamDecl DstDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
1941	args.push_back(Elt: &DstDecl);
1942	ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
1943	args.push_back(Elt: &SrcDecl);
1944
1945	const CGFunctionInfo &FI =
1946	CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
1947
1948	// FIXME: it would be nice if these were mergeable with things with
1949	// identical semantics.
1950	llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
1951
1952	llvm::Function *Fn =
1953	llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
1954	N: FuncName, M: &CGM.getModule());
1955	if (CGM.supportsCOMDAT())
1956	Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
1957
1958	SmallVector<QualType, 2> ArgTys;
1959	ArgTys.push_back(Elt: C.VoidPtrTy);
1960	ArgTys.push_back(Elt: C.VoidPtrTy);
1961
1962	setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
1963	CGM);
1964	StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
1965	auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
1966
1967	Address src = GetAddrOfLocalVar(VD: &SrcDecl);
1968	src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
1969	blockInfo.BlockAlign);
1970
1971	Address dst = GetAddrOfLocalVar(VD: &DstDecl);
1972	dst = Address(Builder.CreateLoad(Addr: dst), blockInfo.StructureType,
1973	blockInfo.BlockAlign);
1974
1975	for (auto &capture : blockInfo.SortedCaptures) {
1976	if (capture.isConstantOrTrivial())
1977	continue;
1978
1979	const BlockDecl::Capture &CI = *capture.Cap;
1980	QualType captureType = CI.getVariable()->getType();
1981	BlockFieldFlags flags = capture.CopyFlags;
1982
1983	unsigned index = capture.getIndex();
1984	Address srcField = Builder.CreateStructGEP(Addr: src, Index: index);
1985	Address dstField = Builder.CreateStructGEP(Addr: dst, Index: index);
1986
1987	switch (capture.CopyKind) {
1988	case BlockCaptureEntityKind::CXXRecord:
1989	// If there's an explicit copy expression, we do that.
1990	assert(CI.getCopyExpr() && "copy expression for variable is missing");
1991	EmitSynthesizedCXXCopyCtor(Dest: dstField, Src: srcField, Exp: CI.getCopyExpr());
1992	break;
1993	case BlockCaptureEntityKind::ARCWeak:
1994	EmitARCCopyWeak(dst: dstField, src: srcField);
1995	break;
1996	case BlockCaptureEntityKind::AddressDiscriminatedPointerAuth: {
1997	QualType Type = CI.getVariable()->getType();
1998	PointerAuthQualifier PointerAuth = Type.getPointerAuth();
1999	assert(PointerAuth && PointerAuth.isAddressDiscriminated());
2000	EmitPointerAuthCopy(Qualifier: PointerAuth, Type, DestField: dstField, SrcField: srcField);
2001	// We don't need to push cleanups for ptrauth types.
2002	continue;
2003	}
2004	case BlockCaptureEntityKind::NonTrivialCStruct: {
2005	// If this is a C struct that requires non-trivial copy construction,
2006	// emit a call to its copy constructor.
2007	QualType varType = CI.getVariable()->getType();
2008	callCStructCopyConstructor(Dst: MakeAddrLValue(Addr: dstField, T: varType),
2009	Src: MakeAddrLValue(Addr: srcField, T: varType));
2010	break;
2011	}
2012	case BlockCaptureEntityKind::ARCStrong: {
2013	llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
2014	// At -O0, store null into the destination field (so that the
2015	// storeStrong doesn't over-release) and then call storeStrong.
2016	// This is a workaround to not having an initStrong call.
2017	if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
2018	auto *ty = cast<llvm::PointerType>(Val: srcValue->getType());
2019	llvm::Value *null = llvm::ConstantPointerNull::get(T: ty);
2020	Builder.CreateStore(Val: null, Addr: dstField);
2021	EmitARCStoreStrongCall(addr: dstField, value: srcValue, resultIgnored: true);
2022
2023	// With optimization enabled, take advantage of the fact that
2024	// the blocks runtime guarantees a memcpy of the block data, and
2025	// just emit a retain of the src field.
2026	} else {
2027	EmitARCRetainNonBlock(value: srcValue);
2028
2029	// Unless EH cleanup is required, we don't need this anymore, so kill
2030	// it. It's not quite worth the annoyance to avoid creating it in the
2031	// first place.
2032	if (!needsEHCleanup(kind: captureType.isDestructedType()))
2033	if (auto *I = cast_or_null<llvm::Instruction>(
2034	Val: dstField.getPointerIfNotSigned()))
2035	I->eraseFromParent();
2036	}
2037	break;
2038	}
2039	case BlockCaptureEntityKind::BlockObject: {
2040	llvm::Value *srcValue = Builder.CreateLoad(Addr: srcField, Name: "blockcopy.src");
2041	llvm::Value *dstAddr = dstField.emitRawPointer(CGF&: *this);
2042	llvm::Value *args[] = {
2043	dstAddr, srcValue, llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())
2044	};
2045
2046	if (CI.isByRef() && C.getBlockVarCopyInit(VD: CI.getVariable()).canThrow())
2047	EmitRuntimeCallOrInvoke(callee: CGM.getBlockObjectAssign(), args);
2048	else
2049	EmitNounwindRuntimeCall(callee: CGM.getBlockObjectAssign(), args);
2050	break;
2051	}
2052	case BlockCaptureEntityKind::None:
2053	continue;
2054	}
2055
2056	// Ensure that we destroy the copied object if an exception is thrown later
2057	// in the helper function.
2058	pushCaptureCleanup(CaptureKind: capture.CopyKind, Field: dstField, CaptureType: captureType, Flags: flags,
2059	/*ForCopyHelper*/ true, Var: CI.getVariable(), CGF&: *this);
2060	}
2061
2062	FinishFunction();
2063
2064	return Fn;
2065	}
2066
2067	static BlockFieldFlags
2068	getBlockFieldFlagsForObjCObjectPointer(const BlockDecl::Capture &CI,
2069	QualType T) {
2070	BlockFieldFlags Flags = BLOCK_FIELD_IS_OBJECT;
2071	if (T->isBlockPointerType())
2072	Flags = BLOCK_FIELD_IS_BLOCK;
2073	return Flags;
2074	}
2075
2076	static std::pair<BlockCaptureEntityKind, BlockFieldFlags>
2077	computeDestroyInfoForBlockCapture(const BlockDecl::Capture &CI, QualType T,
2078	const LangOptions &LangOpts) {
2079	if (CI.isEscapingByref()) {
2080	BlockFieldFlags Flags = BLOCK_FIELD_IS_BYREF;
2081	if (T.isObjCGCWeak())
2082	Flags |= BLOCK_FIELD_IS_WEAK;
2083	return std::make_pair(x: BlockCaptureEntityKind::BlockObject, y&: Flags);
2084	}
2085
2086	switch (T.isDestructedType()) {
2087	case QualType::DK_cxx_destructor:
2088	return std::make_pair(x: BlockCaptureEntityKind::CXXRecord, y: BlockFieldFlags());
2089	case QualType::DK_objc_strong_lifetime:
2090	// Use objc_storeStrong for __strong direct captures; the
2091	// dynamic tools really like it when we do this.
2092	return std::make_pair(x: BlockCaptureEntityKind::ARCStrong,
2093	y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2094	case QualType::DK_objc_weak_lifetime:
2095	// Support __weak direct captures.
2096	return std::make_pair(x: BlockCaptureEntityKind::ARCWeak,
2097	y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2098	case QualType::DK_nontrivial_c_struct:
2099	return std::make_pair(x: BlockCaptureEntityKind::NonTrivialCStruct,
2100	y: BlockFieldFlags());
2101	case QualType::DK_none: {
2102	// Non-ARC captures are strong, and we need to use _Block_object_dispose.
2103	// But honor the inert __unsafe_unretained qualifier, which doesn't actually
2104	// make it into the type system.
2105	if (T->isObjCRetainableType() && !T.getQualifiers().hasObjCLifetime() &&
2106	!LangOpts.ObjCAutoRefCount && !T->isObjCInertUnsafeUnretainedType())
2107	return std::make_pair(x: BlockCaptureEntityKind::BlockObject,
2108	y: getBlockFieldFlagsForObjCObjectPointer(CI, T));
2109	// Otherwise, we have nothing to do.
2110	return std::make_pair(x: BlockCaptureEntityKind::None, y: BlockFieldFlags());
2111	}
2112	}
2113	llvm_unreachable("after exhaustive DestructionKind switch");
2114	}
2115
2116	/// Generate the destroy-helper function for a block closure object:
2117	/// static void block_destroy_helper(block_t *theBlock);
2118	///
2119	/// Note that this destroys a heap-allocated block closure object;
2120	/// it should not be confused with a 'byref destroy helper', which
2121	/// destroys the heap-allocated contents of an individual __block
2122	/// variable.
2123	llvm::Constant *
2124	CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
2125	std::string FuncName = getCopyDestroyHelperFuncName(
2126	Captures: blockInfo.SortedCaptures, BlockAlignment: blockInfo.BlockAlign,
2127	StrKind: CaptureStrKind::DisposeHelper, CGM);
2128
2129	if (llvm::GlobalValue *Func = CGM.getModule().getNamedValue(Name: FuncName))
2130	return Func;
2131
2132	ASTContext &C = getContext();
2133
2134	QualType ReturnTy = C.VoidTy;
2135
2136	FunctionArgList args;
2137	ImplicitParamDecl SrcDecl(C, C.VoidPtrTy, ImplicitParamKind::Other);
2138	args.push_back(Elt: &SrcDecl);
2139
2140	const CGFunctionInfo &FI =
2141	CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2142
2143	// FIXME: We'd like to put these into a mergable by content, with
2144	// internal linkage.
2145	llvm::FunctionType *LTy = CGM.getTypes().GetFunctionType(Info: FI);
2146
2147	llvm::Function *Fn =
2148	llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::LinkOnceODRLinkage,
2149	N: FuncName, M: &CGM.getModule());
2150	if (CGM.supportsCOMDAT())
2151	Fn->setComdat(CGM.getModule().getOrInsertComdat(Name: FuncName));
2152
2153	SmallVector<QualType, 1> ArgTys;
2154	ArgTys.push_back(Elt: C.VoidPtrTy);
2155
2156	setBlockHelperAttributesVisibility(CapturesNonExternalType: blockInfo.CapturesNonExternalType, Fn, FI,
2157	CGM);
2158	StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2159	markAsIgnoreThreadCheckingAtRuntime(Fn);
2160
2161	auto AL = ApplyDebugLocation::CreateArtificial(CGF&: *this);
2162
2163	Address src = GetAddrOfLocalVar(VD: &SrcDecl);
2164	src = Address(Builder.CreateLoad(Addr: src), blockInfo.StructureType,
2165	blockInfo.BlockAlign);
2166
2167	CodeGenFunction::RunCleanupsScope cleanups(*this);
2168
2169	for (auto &capture : blockInfo.SortedCaptures) {
2170	if (capture.isConstantOrTrivial())
2171	continue;
2172
2173	const BlockDecl::Capture &CI = *capture.Cap;
2174	BlockFieldFlags flags = capture.DisposeFlags;
2175
2176	Address srcField = Builder.CreateStructGEP(Addr: src, Index: capture.getIndex());
2177
2178	pushCaptureCleanup(CaptureKind: capture.DisposeKind, Field: srcField,
2179	CaptureType: CI.getVariable()->getType(), Flags: flags,
2180	/*ForCopyHelper*/ false, Var: CI.getVariable(), CGF&: *this);
2181	}
2182
2183	cleanups.ForceCleanup();
2184
2185	FinishFunction();
2186
2187	return Fn;
2188	}
2189
2190	namespace {
2191
2192	/// Emits the copy/dispose helper functions for a __block object of id type.
2193	class ObjectByrefHelpers final : public BlockByrefHelpers {
2194	BlockFieldFlags Flags;
2195
2196	public:
2197	ObjectByrefHelpers(CharUnits alignment, BlockFieldFlags flags)
2198	: BlockByrefHelpers(alignment), Flags(flags) {}
2199
2200	void emitCopy(CodeGenFunction &CGF, Address destField,
2201	Address srcField) override {
2202	destField = destField.withElementType(ElemTy: CGF.Int8Ty);
2203
2204	srcField = srcField.withElementType(ElemTy: CGF.Int8PtrTy);
2205	llvm::Value *srcValue = CGF.Builder.CreateLoad(Addr: srcField);
2206
2207	unsigned flags = (Flags | BLOCK_BYREF_CALLER).getBitMask();
2208
2209	llvm::Value *flagsVal = llvm::ConstantInt::get(Ty: CGF.Int32Ty, V: flags);
2210	llvm::FunctionCallee fn = CGF.CGM.getBlockObjectAssign();
2211
2212	llvm::Value *args[] = {destField.emitRawPointer(CGF), srcValue, flagsVal};
2213	CGF.EmitNounwindRuntimeCall(callee: fn, args);
2214	}
2215
2216	void emitDispose(CodeGenFunction &CGF, Address field) override {
2217	field = field.withElementType(ElemTy: CGF.Int8PtrTy);
2218	llvm::Value *value = CGF.Builder.CreateLoad(Addr: field);
2219
2220	CGF.BuildBlockRelease(DeclPtr: value, flags: Flags | BLOCK_BYREF_CALLER, CanThrow: false);
2221	}
2222
2223	void profileImpl(llvm::FoldingSetNodeID &id) const override {
2224	id.AddInteger(I: Flags.getBitMask());
2225	}
2226	};
2227
2228	/// Emits the copy/dispose helpers for an ARC __block __weak variable.
2229	class ARCWeakByrefHelpers final : public BlockByrefHelpers {
2230	public:
2231	ARCWeakByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2232
2233	void emitCopy(CodeGenFunction &CGF, Address destField,
2234	Address srcField) override {
2235	CGF.EmitARCMoveWeak(dst: destField, src: srcField);
2236	}
2237
2238	void emitDispose(CodeGenFunction &CGF, Address field) override {
2239	CGF.EmitARCDestroyWeak(addr: field);
2240	}
2241
2242	void profileImpl(llvm::FoldingSetNodeID &id) const override {
2243	// 0 is distinguishable from all pointers and byref flags
2244	id.AddInteger(I: 0);
2245	}
2246	};
2247
2248	/// Emits the copy/dispose helpers for an ARC __block __strong variable
2249	/// that's not of block-pointer type.
2250	class ARCStrongByrefHelpers final : public BlockByrefHelpers {
2251	public:
2252	ARCStrongByrefHelpers(CharUnits alignment) : BlockByrefHelpers(alignment) {}
2253
2254	void emitCopy(CodeGenFunction &CGF, Address destField,
2255	Address srcField) override {
2256	// Do a "move" by copying the value and then zeroing out the old
2257	// variable.
2258
2259	llvm::Value *value = CGF.Builder.CreateLoad(Addr: srcField);
2260
2261	llvm::Value *null =
2262	llvm::ConstantPointerNull::get(T: cast<llvm::PointerType>(Val: value->getType()));
2263
2264	if (CGF.CGM.getCodeGenOpts().OptimizationLevel == 0) {
2265	CGF.Builder.CreateStore(Val: null, Addr: destField);
2266	CGF.EmitARCStoreStrongCall(addr: destField, value, /*ignored*/ resultIgnored: true);
2267	CGF.EmitARCStoreStrongCall(addr: srcField, value: null, /*ignored*/ resultIgnored: true);
2268	return;
2269	}
2270	CGF.Builder.CreateStore(Val: value, Addr: destField);
2271	CGF.Builder.CreateStore(Val: null, Addr: srcField);
2272	}
2273
2274	void emitDispose(CodeGenFunction &CGF, Address field) override {
2275	CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2276	}
2277
2278	void profileImpl(llvm::FoldingSetNodeID &id) const override {
2279	// 1 is distinguishable from all pointers and byref flags
2280	id.AddInteger(I: 1);
2281	}
2282	};
2283
2284	/// Emits the copy/dispose helpers for an ARC __block __strong
2285	/// variable that's of block-pointer type.
2286	class ARCStrongBlockByrefHelpers final : public BlockByrefHelpers {
2287	public:
2288	ARCStrongBlockByrefHelpers(CharUnits alignment)
2289	: BlockByrefHelpers(alignment) {}
2290
2291	void emitCopy(CodeGenFunction &CGF, Address destField,
2292	Address srcField) override {
2293	// Do the copy with objc_retainBlock; that's all that
2294	// _Block_object_assign would do anyway, and we'd have to pass the
2295	// right arguments to make sure it doesn't get no-op'ed.
2296	llvm::Value *oldValue = CGF.Builder.CreateLoad(Addr: srcField);
2297	llvm::Value *copy = CGF.EmitARCRetainBlock(value: oldValue, /*mandatory*/ true);
2298	CGF.Builder.CreateStore(Val: copy, Addr: destField);
2299	}
2300
2301	void emitDispose(CodeGenFunction &CGF, Address field) override {
2302	CGF.EmitARCDestroyStrong(addr: field, precise: ARCImpreciseLifetime);
2303	}
2304
2305	void profileImpl(llvm::FoldingSetNodeID &id) const override {
2306	// 2 is distinguishable from all pointers and byref flags
2307	id.AddInteger(I: 2);
2308	}
2309	};
2310
2311	/// Emits the copy/dispose helpers for a __block variable with a
2312	/// nontrivial copy constructor or destructor.
2313	class CXXByrefHelpers final : public BlockByrefHelpers {
2314	QualType VarType;
2315	const Expr *CopyExpr;
2316
2317	public:
2318	CXXByrefHelpers(CharUnits alignment, QualType type,
2319	const Expr *copyExpr)
2320	: BlockByrefHelpers(alignment), VarType(type), CopyExpr(copyExpr) {}
2321
2322	bool needsCopy() const override { return CopyExpr != nullptr; }
2323	void emitCopy(CodeGenFunction &CGF, Address destField,
2324	Address srcField) override {
2325	if (!CopyExpr) return;
2326	CGF.EmitSynthesizedCXXCopyCtor(Dest: destField, Src: srcField, Exp: CopyExpr);
2327	}
2328
2329	void emitDispose(CodeGenFunction &CGF, Address field) override {
2330	EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2331	CGF.PushDestructorCleanup(T: VarType, Addr: field);
2332	CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2333	}
2334
2335	void profileImpl(llvm::FoldingSetNodeID &id) const override {
2336	id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2337	}
2338	};
2339
2340	/// Emits the copy/dispose helpers for a __block variable with
2341	/// address-discriminated pointer authentication.
2342	class AddressDiscriminatedByrefHelpers final : public BlockByrefHelpers {
2343	QualType VarType;
2344
2345	public:
2346	AddressDiscriminatedByrefHelpers(CharUnits Alignment, QualType Type)
2347	: BlockByrefHelpers(Alignment), VarType(Type) {
2348	assert(Type.hasAddressDiscriminatedPointerAuth());
2349	}
2350
2351	void emitCopy(CodeGenFunction &CGF, Address DestField,
2352	Address SrcField) override {
2353	CGF.EmitPointerAuthCopy(Qualifier: VarType.getPointerAuth(), Type: VarType, DestField,
2354	SrcField);
2355	}
2356
2357	bool needsDispose() const override { return false; }
2358	void emitDispose(CodeGenFunction &CGF, Address Field) override {
2359	llvm_unreachable("should never be called");
2360	}
2361
2362	void profileImpl(llvm::FoldingSetNodeID &ID) const override {
2363	ID.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2364	}
2365	};
2366
2367	/// Emits the copy/dispose helpers for a __block variable that is a non-trivial
2368	/// C struct.
2369	class NonTrivialCStructByrefHelpers final : public BlockByrefHelpers {
2370	QualType VarType;
2371
2372	public:
2373	NonTrivialCStructByrefHelpers(CharUnits alignment, QualType type)
2374	: BlockByrefHelpers(alignment), VarType(type) {}
2375
2376	void emitCopy(CodeGenFunction &CGF, Address destField,
2377	Address srcField) override {
2378	CGF.callCStructMoveConstructor(Dst: CGF.MakeAddrLValue(Addr: destField, T: VarType),
2379	Src: CGF.MakeAddrLValue(Addr: srcField, T: VarType));
2380	}
2381
2382	bool needsDispose() const override {
2383	return VarType.isDestructedType();
2384	}
2385
2386	void emitDispose(CodeGenFunction &CGF, Address field) override {
2387	EHScopeStack::stable_iterator cleanupDepth = CGF.EHStack.stable_begin();
2388	CGF.pushDestroy(dtorKind: VarType.isDestructedType(), addr: field, type: VarType);
2389	CGF.PopCleanupBlocks(OldCleanupStackSize: cleanupDepth);
2390	}
2391
2392	void profileImpl(llvm::FoldingSetNodeID &id) const override {
2393	id.AddPointer(Ptr: VarType.getCanonicalType().getAsOpaquePtr());
2394	}
2395	};
2396	} // end anonymous namespace
2397
2398	static llvm::Constant *
2399	generateByrefCopyHelper(CodeGenFunction &CGF, const BlockByrefInfo &byrefInfo,
2400	BlockByrefHelpers &generator) {
2401	ASTContext &Context = CGF.getContext();
2402
2403	QualType ReturnTy = Context.VoidTy;
2404
2405	FunctionArgList args;
2406	ImplicitParamDecl Dst(Context, Context.VoidPtrTy, ImplicitParamKind::Other);
2407	args.push_back(Elt: &Dst);
2408
2409	ImplicitParamDecl Src(Context, Context.VoidPtrTy, ImplicitParamKind::Other);
2410	args.push_back(Elt: &Src);
2411
2412	const CGFunctionInfo &FI =
2413	CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: ReturnTy, args);
2414
2415	llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2416
2417	// FIXME: We'd like to put these into a mergable by content, with
2418	// internal linkage.
2419	llvm::Function *Fn =
2420	llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2421	N: "__Block_byref_object_copy_", M: &CGF.CGM.getModule());
2422
2423	SmallVector<QualType, 2> ArgTys;
2424	ArgTys.push_back(Elt: Context.VoidPtrTy);
2425	ArgTys.push_back(Elt: Context.VoidPtrTy);
2426
2427	CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2428
2429	CGF.StartFunction(GD: GlobalDecl(), RetTy: ReturnTy, Fn, FnInfo: FI, Args: args);
2430	// Create a scope with an artificial location for the body of this function.
2431	auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2432
2433	if (generator.needsCopy()) {
2434	// dst->x
2435	Address destField = CGF.GetAddrOfLocalVar(VD: &Dst);
2436	destField = Address(CGF.Builder.CreateLoad(Addr: destField), byrefInfo.Type,
2437	byrefInfo.ByrefAlignment);
2438	destField =
2439	CGF.emitBlockByrefAddress(baseAddr: destField, info: byrefInfo, followForward: false, name: "dest-object");
2440
2441	// src->x
2442	Address srcField = CGF.GetAddrOfLocalVar(VD: &Src);
2443	srcField = Address(CGF.Builder.CreateLoad(Addr: srcField), byrefInfo.Type,
2444	byrefInfo.ByrefAlignment);
2445	srcField =
2446	CGF.emitBlockByrefAddress(baseAddr: srcField, info: byrefInfo, followForward: false, name: "src-object");
2447
2448	generator.emitCopy(CGF, dest: destField, src: srcField);
2449	}
2450
2451	CGF.FinishFunction();
2452
2453	return Fn;
2454	}
2455
2456	/// Build the copy helper for a __block variable.
2457	static llvm::Constant *buildByrefCopyHelper(CodeGenModule &CGM,
2458	const BlockByrefInfo &byrefInfo,
2459	BlockByrefHelpers &generator) {
2460	CodeGenFunction CGF(CGM);
2461	return generateByrefCopyHelper(CGF, byrefInfo, generator);
2462	}
2463
2464	/// Generate code for a __block variable's dispose helper.
2465	static llvm::Constant *
2466	generateByrefDisposeHelper(CodeGenFunction &CGF,
2467	const BlockByrefInfo &byrefInfo,
2468	BlockByrefHelpers &generator) {
2469	ASTContext &Context = CGF.getContext();
2470	QualType R = Context.VoidTy;
2471
2472	FunctionArgList args;
2473	ImplicitParamDecl Src(CGF.getContext(), Context.VoidPtrTy,
2474	ImplicitParamKind::Other);
2475	args.push_back(Elt: &Src);
2476
2477	const CGFunctionInfo &FI =
2478	CGF.CGM.getTypes().arrangeBuiltinFunctionDeclaration(resultType: R, args);
2479
2480	llvm::FunctionType *LTy = CGF.CGM.getTypes().GetFunctionType(Info: FI);
2481
2482	// FIXME: We'd like to put these into a mergable by content, with
2483	// internal linkage.
2484	llvm::Function *Fn =
2485	llvm::Function::Create(Ty: LTy, Linkage: llvm::GlobalValue::InternalLinkage,
2486	N: "__Block_byref_object_dispose_",
2487	M: &CGF.CGM.getModule());
2488
2489	SmallVector<QualType, 1> ArgTys;
2490	ArgTys.push_back(Elt: Context.VoidPtrTy);
2491
2492	CGF.CGM.SetInternalFunctionAttributes(GD: GlobalDecl(), F: Fn, FI);
2493
2494	CGF.StartFunction(GD: GlobalDecl(), RetTy: R, Fn, FnInfo: FI, Args: args);
2495	// Create a scope with an artificial location for the body of this function.
2496	auto AL = ApplyDebugLocation::CreateArtificial(CGF);
2497
2498	if (generator.needsDispose()) {
2499	Address addr = CGF.GetAddrOfLocalVar(VD: &Src);
2500	addr = Address(CGF.Builder.CreateLoad(Addr: addr), byrefInfo.Type,
2501	byrefInfo.ByrefAlignment);
2502	addr = CGF.emitBlockByrefAddress(baseAddr: addr, info: byrefInfo, followForward: false, name: "object");
2503
2504	generator.emitDispose(CGF, field: addr);
2505	}
2506
2507	CGF.FinishFunction();
2508
2509	return Fn;
2510	}
2511
2512	/// Build the dispose helper for a __block variable.
2513	static llvm::Constant *buildByrefDisposeHelper(CodeGenModule &CGM,
2514	const BlockByrefInfo &byrefInfo,
2515	BlockByrefHelpers &generator) {
2516	CodeGenFunction CGF(CGM);
2517	return generateByrefDisposeHelper(CGF, byrefInfo, generator);
2518	}
2519
2520	/// Lazily build the copy and dispose helpers for a __block variable
2521	/// with the given information.
2522	template <class T>
2523	static T *buildByrefHelpers(CodeGenModule &CGM, const BlockByrefInfo &byrefInfo,
2524	T &&generator) {
2525	llvm::FoldingSetNodeID id;
2526	generator.Profile(id);
2527
2528	void *insertPos;
2529	BlockByrefHelpers *node
2530	= CGM.ByrefHelpersCache.FindNodeOrInsertPos(ID: id, InsertPos&: insertPos);
2531	if (node) return static_cast<T*>(node);
2532
2533	generator.CopyHelper = buildByrefCopyHelper(CGM, byrefInfo, generator);
2534	generator.DisposeHelper = buildByrefDisposeHelper(CGM, byrefInfo, generator);
2535
2536	T *copy = new (CGM.getContext()) T(std::forward<T>(generator));
2537	CGM.ByrefHelpersCache.InsertNode(copy, insertPos);
2538	return copy;
2539	}
2540
2541	/// Build the copy and dispose helpers for the given __block variable
2542	/// emission. Places the helpers in the global cache. Returns null
2543	/// if no helpers are required.
2544	BlockByrefHelpers *
2545	CodeGenFunction::buildByrefHelpers(llvm::StructType &byrefType,
2546	const AutoVarEmission &emission) {
2547	const VarDecl &var = *emission.Variable;
2548	assert(var.isEscapingByref() &&
2549	"only escaping __block variables need byref helpers");
2550
2551	QualType type = var.getType();
2552
2553	auto &byrefInfo = getBlockByrefInfo(var: &var);
2554
2555	// The alignment we care about for the purposes of uniquing byref
2556	// helpers is the alignment of the actual byref value field.
2557	CharUnits valueAlignment =
2558	byrefInfo.ByrefAlignment.alignmentAtOffset(offset: byrefInfo.FieldOffset);
2559
2560	if (const CXXRecordDecl *record = type->getAsCXXRecordDecl()) {
2561	const Expr *copyExpr =
2562	CGM.getContext().getBlockVarCopyInit(VD: &var).getCopyExpr();
2563	if (!copyExpr && record->hasTrivialDestructor()) return nullptr;
2564
2565	return ::buildByrefHelpers(
2566	CGM, byrefInfo, generator: CXXByrefHelpers(valueAlignment, type, copyExpr));
2567	}
2568	if (type.hasAddressDiscriminatedPointerAuth()) {
2569	return ::buildByrefHelpers(
2570	CGM, byrefInfo, generator: AddressDiscriminatedByrefHelpers(valueAlignment, type));
2571	}
2572	// If type is a non-trivial C struct type that is non-trivial to
2573	// destructly move or destroy, build the copy and dispose helpers.
2574	if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct ||
2575	type.isDestructedType() == QualType::DK_nontrivial_c_struct)
2576	return ::buildByrefHelpers(
2577	CGM, byrefInfo, generator: NonTrivialCStructByrefHelpers(valueAlignment, type));
2578
2579	// Otherwise, if we don't have a retainable type, there's nothing to do.
2580	// that the runtime does extra copies.
2581	if (!type->isObjCRetainableType()) return nullptr;
2582
2583	Qualifiers qs = type.getQualifiers();
2584
2585	// If we have lifetime, that dominates.
2586	if (Qualifiers::ObjCLifetime lifetime = qs.getObjCLifetime()) {
2587	switch (lifetime) {
2588	case Qualifiers::OCL_None: llvm_unreachable("impossible");
2589
2590	// These are just bits as far as the runtime is concerned.
2591	case Qualifiers::OCL_ExplicitNone:
2592	case Qualifiers::OCL_Autoreleasing:
2593	return nullptr;
2594
2595	// Tell the runtime that this is ARC __weak, called by the
2596	// byref routines.
2597	case Qualifiers::OCL_Weak:
2598	return ::buildByrefHelpers(CGM, byrefInfo,
2599	generator: ARCWeakByrefHelpers(valueAlignment));
2600
2601	// ARC __strong __block variables need to be retained.
2602	case Qualifiers::OCL_Strong:
2603	// Block pointers need to be copied, and there's no direct
2604	// transfer possible.
2605	if (type->isBlockPointerType()) {
2606	return ::buildByrefHelpers(CGM, byrefInfo,
2607	generator: ARCStrongBlockByrefHelpers(valueAlignment));
2608
2609	// Otherwise, we transfer ownership of the retain from the stack
2610	// to the heap.
2611	} else {
2612	return ::buildByrefHelpers(CGM, byrefInfo,
2613	generator: ARCStrongByrefHelpers(valueAlignment));
2614	}
2615	}
2616	llvm_unreachable("fell out of lifetime switch!");
2617	}
2618
2619	BlockFieldFlags flags;
2620	if (type->isBlockPointerType()) {
2621	flags |= BLOCK_FIELD_IS_BLOCK;
2622	} else if (CGM.getContext().isObjCNSObjectType(Ty: type) ||
2623	type->isObjCObjectPointerType()) {
2624	flags |= BLOCK_FIELD_IS_OBJECT;
2625	} else {
2626	return nullptr;
2627	}
2628
2629	if (type.isObjCGCWeak())
2630	flags |= BLOCK_FIELD_IS_WEAK;
2631
2632	return ::buildByrefHelpers(CGM, byrefInfo,
2633	generator: ObjectByrefHelpers(valueAlignment, flags));
2634	}
2635
2636	Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2637	const VarDecl *var,
2638	bool followForward) {
2639	auto &info = getBlockByrefInfo(var);
2640	return emitBlockByrefAddress(baseAddr, info, followForward, name: var->getName());
2641	}
2642
2643	Address CodeGenFunction::emitBlockByrefAddress(Address baseAddr,
2644	const BlockByrefInfo &info,
2645	bool followForward,
2646	const llvm::Twine &name) {
2647	// Chase the forwarding address if requested.
2648	if (followForward) {
2649	Address forwardingAddr = Builder.CreateStructGEP(Addr: baseAddr, Index: 1, Name: "forwarding");
2650	baseAddr = Address(Builder.CreateLoad(Addr: forwardingAddr), info.Type,
2651	info.ByrefAlignment);
2652	}
2653
2654	return Builder.CreateStructGEP(Addr: baseAddr, Index: info.FieldIndex, Name: name);
2655	}
2656
2657	/// BuildByrefInfo - This routine changes a __block variable declared as T x
2658	/// into:
2659	///
2660	/// struct {
2661	/// void *__isa;
2662	/// void *__forwarding;
2663	/// int32_t __flags;
2664	/// int32_t __size;
2665	/// void *__copy_helper; // only if needed
2666	/// void *__destroy_helper; // only if needed
2667	/// void *__byref_variable_layout;// only if needed
2668	/// char padding[X]; // only if needed
2669	/// T x;
2670	/// } x
2671	///
2672	const BlockByrefInfo &CodeGenFunction::getBlockByrefInfo(const VarDecl *D) {
2673	auto it = BlockByrefInfos.find(Val: D);
2674	if (it != BlockByrefInfos.end())
2675	return it->second;
2676
2677	QualType Ty = D->getType();
2678
2679	CharUnits size;
2680	SmallVector<llvm::Type *, 8> types;
2681
2682	// void *__isa;
2683	types.push_back(Elt: VoidPtrTy);
2684	size += getPointerSize();
2685
2686	// void *__forwarding;
2687	types.push_back(Elt: VoidPtrTy);
2688	size += getPointerSize();
2689
2690	// int32_t __flags;
2691	types.push_back(Elt: Int32Ty);
2692	size += CharUnits::fromQuantity(Quantity: 4);
2693
2694	// int32_t __size;
2695	types.push_back(Elt: Int32Ty);
2696	size += CharUnits::fromQuantity(Quantity: 4);
2697
2698	// Note that this must match *exactly* the logic in buildByrefHelpers.
2699	bool hasCopyAndDispose = getContext().BlockRequiresCopying(Ty, D);
2700	if (hasCopyAndDispose) {
2701	/// void *__copy_helper;
2702	types.push_back(Elt: VoidPtrTy);
2703	size += getPointerSize();
2704
2705	/// void *__destroy_helper;
2706	types.push_back(Elt: VoidPtrTy);
2707	size += getPointerSize();
2708	}
2709
2710	bool HasByrefExtendedLayout = false;
2711	Qualifiers::ObjCLifetime Lifetime = Qualifiers::OCL_None;
2712	if (getContext().getByrefLifetime(Ty, Lifetime, HasByrefExtendedLayout) &&
2713	HasByrefExtendedLayout) {
2714	/// void *__byref_variable_layout;
2715	types.push_back(Elt: VoidPtrTy);
2716	size += CharUnits::fromQuantity(Quantity: PointerSizeInBytes);
2717	}
2718
2719	// T x;
2720	llvm::Type *varTy = ConvertTypeForMem(T: Ty);
2721
2722	bool packed = false;
2723	CharUnits varAlign = getContext().getDeclAlign(D);
2724	CharUnits varOffset = size.alignTo(Align: varAlign);
2725
2726	// We may have to insert padding.
2727	if (varOffset != size) {
2728	llvm::Type *paddingTy =
2729	llvm::ArrayType::get(ElementType: Int8Ty, NumElements: (varOffset - size).getQuantity());
2730
2731	types.push_back(Elt: paddingTy);
2732	size = varOffset;
2733
2734	// Conversely, we might have to prevent LLVM from inserting padding.
2735	} else if (CGM.getDataLayout().getABITypeAlign(Ty: varTy) >
2736	uint64_t(varAlign.getQuantity())) {
2737	packed = true;
2738	}
2739	types.push_back(Elt: varTy);
2740
2741	llvm::StructType *byrefType = llvm::StructType::create(
2742	Context&: getLLVMContext(), Elements: types, Name: "struct.__block_byref_" + D->getNameAsString(),
2743	isPacked: packed);
2744
2745	BlockByrefInfo info;
2746	info.Type = byrefType;
2747	info.FieldIndex = types.size() - 1;
2748	info.FieldOffset = varOffset;
2749	info.ByrefAlignment = std::max(a: varAlign, b: getPointerAlign());
2750
2751	auto pair = BlockByrefInfos.insert(KV: {D, info});
2752	assert(pair.second && "info was inserted recursively?");
2753	return pair.first->second;
2754	}
2755
2756	/// Initialize the structural components of a __block variable, i.e.
2757	/// everything but the actual object.
2758	void CodeGenFunction::emitByrefStructureInit(const AutoVarEmission &emission) {
2759	// Find the address of the local.
2760	Address addr = emission.Addr;
2761
2762	// That's an alloca of the byref structure type.
2763	llvm::StructType *byrefType = cast<llvm::StructType>(Val: addr.getElementType());
2764
2765	unsigned nextHeaderIndex = 0;
2766	CharUnits nextHeaderOffset;
2767	auto storeHeaderField = [&](llvm::Value *value, CharUnits fieldSize,
2768	const Twine &name, bool isFunction = false) {
2769	auto fieldAddr = Builder.CreateStructGEP(Addr: addr, Index: nextHeaderIndex, Name: name);
2770	if (isFunction) {
2771	if (auto &Schema = CGM.getCodeGenOpts()
2772	.PointerAuth.BlockByrefHelperFunctionPointers) {
2773	auto PointerAuth = EmitPointerAuthInfo(
2774	Schema, StorageAddress: fieldAddr.emitRawPointer(CGF&: *this), SchemaDecl: GlobalDecl(), SchemaType: QualType());
2775	value = EmitPointerAuthSign(Info: PointerAuth, Pointer: value);
2776	}
2777	}
2778	Builder.CreateStore(Val: value, Addr: fieldAddr);
2779
2780	nextHeaderIndex++;
2781	nextHeaderOffset += fieldSize;
2782	};
2783
2784	// Build the byref helpers if necessary. This is null if we don't need any.
2785	BlockByrefHelpers *helpers = buildByrefHelpers(byrefType&: *byrefType, emission);
2786
2787	const VarDecl &D = *emission.Variable;
2788	QualType type = D.getType();
2789
2790	bool HasByrefExtendedLayout = false;
2791	Qualifiers::ObjCLifetime ByrefLifetime = Qualifiers::OCL_None;
2792	bool ByRefHasLifetime =
2793	getContext().getByrefLifetime(Ty: type, Lifetime&: ByrefLifetime, HasByrefExtendedLayout);
2794
2795	llvm::Value *V;
2796
2797	// Initialize the 'isa', which is just 0 or 1.
2798	int isa = 0;
2799	if (type.isObjCGCWeak())
2800	isa = 1;
2801	V = Builder.CreateIntToPtr(V: Builder.getInt32(C: isa), DestTy: Int8PtrTy, Name: "isa");
2802	storeHeaderField(V, getPointerSize(), "byref.isa");
2803
2804	// Store the address of the variable into its own forwarding pointer.
2805	storeHeaderField(addr.emitRawPointer(CGF&: *this), getPointerSize(),
2806	"byref.forwarding");
2807
2808	// Blocks ABI:
2809	// c) the flags field is set to either 0 if no helper functions are
2810	// needed or BLOCK_BYREF_HAS_COPY_DISPOSE if they are,
2811	BlockFlags flags;
2812	if (helpers) flags |= BLOCK_BYREF_HAS_COPY_DISPOSE;
2813	if (ByRefHasLifetime) {
2814	if (HasByrefExtendedLayout) flags |= BLOCK_BYREF_LAYOUT_EXTENDED;
2815	else switch (ByrefLifetime) {
2816	case Qualifiers::OCL_Strong:
2817	flags |= BLOCK_BYREF_LAYOUT_STRONG;
2818	break;
2819	case Qualifiers::OCL_Weak:
2820	flags |= BLOCK_BYREF_LAYOUT_WEAK;
2821	break;
2822	case Qualifiers::OCL_ExplicitNone:
2823	flags |= BLOCK_BYREF_LAYOUT_UNRETAINED;
2824	break;
2825	case Qualifiers::OCL_None:
2826	if (!type->isObjCObjectPointerType() && !type->isBlockPointerType())
2827	flags |= BLOCK_BYREF_LAYOUT_NON_OBJECT;
2828	break;
2829	default:
2830	break;
2831	}
2832	if (CGM.getLangOpts().ObjCGCBitmapPrint) {
2833	printf(format: "\n Inline flag for BYREF variable layout (%d):", flags.getBitMask());
2834	if (flags & BLOCK_BYREF_HAS_COPY_DISPOSE)
2835	printf(format: " BLOCK_BYREF_HAS_COPY_DISPOSE");
2836	if (flags & BLOCK_BYREF_LAYOUT_MASK) {
2837	BlockFlags ThisFlag(flags.getBitMask() & BLOCK_BYREF_LAYOUT_MASK);
2838	if (ThisFlag == BLOCK_BYREF_LAYOUT_EXTENDED)
2839	printf(format: " BLOCK_BYREF_LAYOUT_EXTENDED");
2840	if (ThisFlag == BLOCK_BYREF_LAYOUT_STRONG)
2841	printf(format: " BLOCK_BYREF_LAYOUT_STRONG");
2842	if (ThisFlag == BLOCK_BYREF_LAYOUT_WEAK)
2843	printf(format: " BLOCK_BYREF_LAYOUT_WEAK");
2844	if (ThisFlag == BLOCK_BYREF_LAYOUT_UNRETAINED)
2845	printf(format: " BLOCK_BYREF_LAYOUT_UNRETAINED");
2846	if (ThisFlag == BLOCK_BYREF_LAYOUT_NON_OBJECT)
2847	printf(format: " BLOCK_BYREF_LAYOUT_NON_OBJECT");
2848	}
2849	printf(format: "\n");
2850	}
2851	}
2852	storeHeaderField(llvm::ConstantInt::get(Ty: IntTy, V: flags.getBitMask()),
2853	getIntSize(), "byref.flags");
2854
2855	CharUnits byrefSize = CGM.GetTargetTypeStoreSize(Ty: byrefType);
2856	V = llvm::ConstantInt::get(Ty: IntTy, V: byrefSize.getQuantity());
2857	storeHeaderField(V, getIntSize(), "byref.size");
2858
2859	if (helpers) {
2860	storeHeaderField(helpers->CopyHelper, getPointerSize(), "byref.copyHelper",
2861	/*isFunction=*/true);
2862	storeHeaderField(helpers->DisposeHelper, getPointerSize(),
2863	"byref.disposeHelper", /*isFunction=*/true);
2864	}
2865
2866	if (ByRefHasLifetime && HasByrefExtendedLayout) {
2867	auto layoutInfo = CGM.getObjCRuntime().BuildByrefLayout(CGM, T: type);
2868	storeHeaderField(layoutInfo, getPointerSize(), "byref.layout");
2869	}
2870	}
2871
2872	void CodeGenFunction::BuildBlockRelease(llvm::Value *V, BlockFieldFlags flags,
2873	bool CanThrow) {
2874	llvm::FunctionCallee F = CGM.getBlockObjectDispose();
2875	llvm::Value *args[] = {V,
2876	llvm::ConstantInt::get(Ty: Int32Ty, V: flags.getBitMask())};
2877
2878	if (CanThrow)
2879	EmitRuntimeCallOrInvoke(callee: F, args);
2880	else
2881	EmitNounwindRuntimeCall(callee: F, args);
2882	}
2883
2884	void CodeGenFunction::enterByrefCleanup(CleanupKind Kind, Address Addr,
2885	BlockFieldFlags Flags,
2886	bool LoadBlockVarAddr, bool CanThrow) {
2887	EHStack.pushCleanup<CallBlockRelease>(Kind, A: Addr, A: Flags, A: LoadBlockVarAddr,
2888	A: CanThrow);
2889	}
2890
2891	/// Adjust the declaration of something from the blocks API.
2892	static void configureBlocksRuntimeObject(CodeGenModule &CGM,
2893	llvm::Constant *C) {
2894	auto *GV = cast<llvm::GlobalValue>(Val: C->stripPointerCasts());
2895
2896	if (!CGM.getCodeGenOpts().StaticClosure &&
2897	CGM.getTarget().getTriple().isOSBinFormatCOFF()) {
2898	const IdentifierInfo &II = CGM.getContext().Idents.get(Name: C->getName());
2899	TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
2900	DeclContext *DC = TranslationUnitDecl::castToDeclContext(D: TUDecl);
2901
2902	assert((isa<llvm::Function>(C->stripPointerCasts()) ||
2903	isa<llvm::GlobalVariable>(C->stripPointerCasts())) &&
2904	"expected Function or GlobalVariable");
2905
2906	const NamedDecl *ND = nullptr;
2907	for (const auto *Result : DC->lookup(Name: &II))
2908	if ((ND = dyn_cast<FunctionDecl>(Val: Result)) ||
2909	(ND = dyn_cast<VarDecl>(Val: Result)))
2910	break;
2911
2912	if (GV->isDeclaration() && (!ND || !ND->hasAttr<DLLExportAttr>())) {
2913	GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
2914	GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2915	} else {
2916	GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
2917	GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
2918	}
2919	}
2920
2921	if (CGM.getLangOpts().BlocksRuntimeOptional && GV->isDeclaration() &&
2922	GV->hasExternalLinkage())
2923	GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
2924
2925	CGM.setDSOLocal(GV);
2926	}
2927
2928	llvm::FunctionCallee CodeGenModule::getBlockObjectDispose() {
2929	if (BlockObjectDispose)
2930	return BlockObjectDispose;
2931
2932	QualType args[] = {Context.VoidPtrTy, Context.IntTy};
2933	BlockObjectDispose =
2934	CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_dispose");
2935	configureBlocksRuntimeObject(
2936	CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectDispose.getCallee()));
2937	return BlockObjectDispose;
2938	}
2939
2940	llvm::FunctionCallee CodeGenModule::getBlockObjectAssign() {
2941	if (BlockObjectAssign)
2942	return BlockObjectAssign;
2943
2944	QualType args[] = {Context.VoidPtrTy, Context.VoidPtrTy, Context.IntTy};
2945	BlockObjectAssign =
2946	CreateRuntimeFunction(ReturnTy: Context.VoidTy, ArgTys: args, Name: "_Block_object_assign");
2947	configureBlocksRuntimeObject(
2948	CGM&: *this, C: cast<llvm::Constant>(Val: BlockObjectAssign.getCallee()));
2949	return BlockObjectAssign;
2950	}
2951
2952	llvm::Constant *CodeGenModule::getNSConcreteGlobalBlock() {
2953	if (NSConcreteGlobalBlock)
2954	return NSConcreteGlobalBlock;
2955
2956	NSConcreteGlobalBlock = GetOrCreateLLVMGlobal(
2957	MangledName: "_NSConcreteGlobalBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2958	configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteGlobalBlock);
2959	return NSConcreteGlobalBlock;
2960	}
2961
2962	llvm::Constant *CodeGenModule::getNSConcreteStackBlock() {
2963	if (NSConcreteStackBlock)
2964	return NSConcreteStackBlock;
2965
2966	NSConcreteStackBlock = GetOrCreateLLVMGlobal(
2967	MangledName: "_NSConcreteStackBlock", Ty: Int8PtrTy, AddrSpace: LangAS::Default, D: nullptr);
2968	configureBlocksRuntimeObject(CGM&: *this, C: NSConcreteStackBlock);
2969	return NSConcreteStackBlock;
2970	}
2971