1	//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This file contains the code for emitting atomic operations.
10	//
11	//===----------------------------------------------------------------------===//
12
13	#include "CGCall.h"
14	#include "CGRecordLayout.h"
15	#include "CodeGenFunction.h"
16	#include "CodeGenModule.h"
17	#include "TargetInfo.h"
18	#include "clang/AST/ASTContext.h"
19	#include "clang/CodeGen/CGFunctionInfo.h"
20	#include "clang/Frontend/FrontendDiagnostic.h"
21	#include "llvm/ADT/DenseMap.h"
22	#include "llvm/IR/DataLayout.h"
23	#include "llvm/IR/Intrinsics.h"
24
25	using namespace clang;
26	using namespace CodeGen;
27
28	namespace {
29	class AtomicInfo {
30	CodeGenFunction &CGF;
31	QualType AtomicTy;
32	QualType ValueTy;
33	uint64_t AtomicSizeInBits;
34	uint64_t ValueSizeInBits;
35	CharUnits AtomicAlign;
36	CharUnits ValueAlign;
37	TypeEvaluationKind EvaluationKind;
38	bool UseLibcall;
39	LValue LVal;
40	CGBitFieldInfo BFI;
41	public:
42	AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
43	: CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
44	EvaluationKind(TEK_Scalar), UseLibcall(true) {
45	assert(!lvalue.isGlobalReg());
46	ASTContext &C = CGF.getContext();
47	if (lvalue.isSimple()) {
48	AtomicTy = lvalue.getType();
49	if (auto *ATy = AtomicTy->getAs<AtomicType>())
50	ValueTy = ATy->getValueType();
51	else
52	ValueTy = AtomicTy;
53	EvaluationKind = CGF.getEvaluationKind(ValueTy);
54
55	uint64_t ValueAlignInBits;
56	uint64_t AtomicAlignInBits;
57	TypeInfo ValueTI = C.getTypeInfo(ValueTy);
58	ValueSizeInBits = ValueTI.Width;
59	ValueAlignInBits = ValueTI.Align;
60
61	TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
62	AtomicSizeInBits = AtomicTI.Width;
63	AtomicAlignInBits = AtomicTI.Align;
64
65	assert(ValueSizeInBits <= AtomicSizeInBits);
66	assert(ValueAlignInBits <= AtomicAlignInBits);
67
68	AtomicAlign = C.toCharUnitsFromBits(BitSize: AtomicAlignInBits);
69	ValueAlign = C.toCharUnitsFromBits(BitSize: ValueAlignInBits);
70	if (lvalue.getAlignment().isZero())
71	lvalue.setAlignment(AtomicAlign);
72
73	LVal = lvalue;
74	} else if (lvalue.isBitField()) {
75	ValueTy = lvalue.getType();
76	ValueSizeInBits = C.getTypeSize(ValueTy);
77	auto &OrigBFI = lvalue.getBitFieldInfo();
78	auto Offset = OrigBFI.Offset % C.toBits(CharSize: lvalue.getAlignment());
79	AtomicSizeInBits = C.toBits(
80	CharSize: C.toCharUnitsFromBits(BitSize: Offset + OrigBFI.Size + C.getCharWidth() - 1)
81	.alignTo(Align: lvalue.getAlignment()));
82	llvm::Value *BitFieldPtr = lvalue.getRawBitFieldPointer(CGF);
83	auto OffsetInChars =
84	(C.toCharUnitsFromBits(BitSize: OrigBFI.Offset) / lvalue.getAlignment()) *
85	lvalue.getAlignment();
86	llvm::Value *StoragePtr = CGF.Builder.CreateConstGEP1_64(
87	Ty: CGF.Int8Ty, Ptr: BitFieldPtr, Idx0: OffsetInChars.getQuantity());
88	StoragePtr = CGF.Builder.CreateAddrSpaceCast(
89	V: StoragePtr, DestTy: CGF.UnqualPtrTy, Name: "atomic_bitfield_base");
90	BFI = OrigBFI;
91	BFI.Offset = Offset;
92	BFI.StorageSize = AtomicSizeInBits;
93	BFI.StorageOffset += OffsetInChars;
94	llvm::Type *StorageTy = CGF.Builder.getIntNTy(N: AtomicSizeInBits);
95	LVal = LValue::MakeBitfield(
96	Address(StoragePtr, StorageTy, lvalue.getAlignment()), BFI,
97	lvalue.getType(), lvalue.getBaseInfo(), lvalue.getTBAAInfo());
98	AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
99	if (AtomicTy.isNull()) {
100	llvm::APInt Size(
101	/*numBits=*/32,
102	C.toCharUnitsFromBits(BitSize: AtomicSizeInBits).getQuantity());
103	AtomicTy = C.getConstantArrayType(C.CharTy, Size, nullptr,
104	ArraySizeModifier::Normal,
105	/*IndexTypeQuals=*/0);
106	}
107	AtomicAlign = ValueAlign = lvalue.getAlignment();
108	} else if (lvalue.isVectorElt()) {
109	ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
110	ValueSizeInBits = C.getTypeSize(ValueTy);
111	AtomicTy = lvalue.getType();
112	AtomicSizeInBits = C.getTypeSize(AtomicTy);
113	AtomicAlign = ValueAlign = lvalue.getAlignment();
114	LVal = lvalue;
115	} else {
116	assert(lvalue.isExtVectorElt());
117	ValueTy = lvalue.getType();
118	ValueSizeInBits = C.getTypeSize(ValueTy);
119	AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
120	lvalue.getType(), cast<llvm::FixedVectorType>(
121	lvalue.getExtVectorAddress().getElementType())
122	->getNumElements());
123	AtomicSizeInBits = C.getTypeSize(AtomicTy);
124	AtomicAlign = ValueAlign = lvalue.getAlignment();
125	LVal = lvalue;
126	}
127	UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
128	AtomicSizeInBits, AlignmentInBits: C.toBits(CharSize: lvalue.getAlignment()));
129	}
130
131	QualType getAtomicType() const { return AtomicTy; }
132	QualType getValueType() const { return ValueTy; }
133	CharUnits getAtomicAlignment() const { return AtomicAlign; }
134	uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
135	uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
136	TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
137	bool shouldUseLibcall() const { return UseLibcall; }
138	const LValue &getAtomicLValue() const { return LVal; }
139	llvm::Value *getAtomicPointer() const {
140	if (LVal.isSimple())
141	return LVal.emitRawPointer(CGF);
142	else if (LVal.isBitField())
143	return LVal.getRawBitFieldPointer(CGF);
144	else if (LVal.isVectorElt())
145	return LVal.getRawVectorPointer(CGF);
146	assert(LVal.isExtVectorElt());
147	return LVal.getRawExtVectorPointer(CGF);
148	}
149	Address getAtomicAddress() const {
150	llvm::Type *ElTy;
151	if (LVal.isSimple())
152	ElTy = LVal.getAddress().getElementType();
153	else if (LVal.isBitField())
154	ElTy = LVal.getBitFieldAddress().getElementType();
155	else if (LVal.isVectorElt())
156	ElTy = LVal.getVectorAddress().getElementType();
157	else
158	ElTy = LVal.getExtVectorAddress().getElementType();
159	return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
160	}
161
162	Address getAtomicAddressAsAtomicIntPointer() const {
163	return castToAtomicIntPointer(Addr: getAtomicAddress());
164	}
165
166	/// Is the atomic size larger than the underlying value type?
167	///
168	/// Note that the absence of padding does not mean that atomic
169	/// objects are completely interchangeable with non-atomic
170	/// objects: we might have promoted the alignment of a type
171	/// without making it bigger.
172	bool hasPadding() const {
173	return (ValueSizeInBits != AtomicSizeInBits);
174	}
175
176	bool emitMemSetZeroIfNecessary() const;
177
178	llvm::Value *getAtomicSizeValue() const {
179	CharUnits size = CGF.getContext().toCharUnitsFromBits(BitSize: AtomicSizeInBits);
180	return CGF.CGM.getSize(numChars: size);
181	}
182
183	/// Cast the given pointer to an integer pointer suitable for atomic
184	/// operations if the source.
185	Address castToAtomicIntPointer(Address Addr) const;
186
187	/// If Addr is compatible with the iN that will be used for an atomic
188	/// operation, bitcast it. Otherwise, create a temporary that is suitable
189	/// and copy the value across.
190	Address convertToAtomicIntPointer(Address Addr) const;
191
192	/// Turn an atomic-layout object into an r-value.
193	RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
194	SourceLocation loc, bool AsValue) const;
195
196	llvm::Value *getScalarRValValueOrNull(RValue RVal) const;
197
198	/// Converts an rvalue to integer value if needed.
199	llvm::Value *convertRValueToInt(RValue RVal, bool CmpXchg = false) const;
200
201	RValue ConvertToValueOrAtomic(llvm::Value *IntVal, AggValueSlot ResultSlot,
202	SourceLocation Loc, bool AsValue,
203	bool CmpXchg = false) const;
204
205	/// Copy an atomic r-value into atomic-layout memory.
206	void emitCopyIntoMemory(RValue rvalue) const;
207
208	/// Project an l-value down to the value field.
209	LValue projectValue() const {
210	assert(LVal.isSimple());
211	Address addr = getAtomicAddress();
212	if (hasPadding())
213	addr = CGF.Builder.CreateStructGEP(Addr: addr, Index: 0);
214
215	return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
216	LVal.getBaseInfo(), LVal.getTBAAInfo());
217	}
218
219	/// Emits atomic load.
220	/// \returns Loaded value.
221	RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
222	bool AsValue, llvm::AtomicOrdering AO,
223	bool IsVolatile);
224
225	/// Emits atomic compare-and-exchange sequence.
226	/// \param Expected Expected value.
227	/// \param Desired Desired value.
228	/// \param Success Atomic ordering for success operation.
229	/// \param Failure Atomic ordering for failed operation.
230	/// \param IsWeak true if atomic operation is weak, false otherwise.
231	/// \returns Pair of values: previous value from storage (value type) and
232	/// boolean flag (i1 type) with true if success and false otherwise.
233	std::pair<RValue, llvm::Value *>
234	EmitAtomicCompareExchange(RValue Expected, RValue Desired,
235	llvm::AtomicOrdering Success =
236	llvm::AtomicOrdering::SequentiallyConsistent,
237	llvm::AtomicOrdering Failure =
238	llvm::AtomicOrdering::SequentiallyConsistent,
239	bool IsWeak = false);
240
241	/// Emits atomic update.
242	/// \param AO Atomic ordering.
243	/// \param UpdateOp Update operation for the current lvalue.
244	void EmitAtomicUpdate(llvm::AtomicOrdering AO,
245	const llvm::function_ref<RValue(RValue)> &UpdateOp,
246	bool IsVolatile);
247	/// Emits atomic update.
248	/// \param AO Atomic ordering.
249	void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
250	bool IsVolatile);
251
252	/// Materialize an atomic r-value in atomic-layout memory.
253	Address materializeRValue(RValue rvalue) const;
254
255	/// Creates temp alloca for intermediate operations on atomic value.
256	Address CreateTempAlloca() const;
257	private:
258	bool requiresMemSetZero(llvm::Type *type) const;
259
260
261	/// Emits atomic load as a libcall.
262	void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
263	llvm::AtomicOrdering AO, bool IsVolatile);
264	/// Emits atomic load as LLVM instruction.
265	llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile,
266	bool CmpXchg = false);
267	/// Emits atomic compare-and-exchange op as a libcall.
268	llvm::Value *EmitAtomicCompareExchangeLibcall(
269	llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
270	llvm::AtomicOrdering Success =
271	llvm::AtomicOrdering::SequentiallyConsistent,
272	llvm::AtomicOrdering Failure =
273	llvm::AtomicOrdering::SequentiallyConsistent);
274	/// Emits atomic compare-and-exchange op as LLVM instruction.
275	std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
276	llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
277	llvm::AtomicOrdering Success =
278	llvm::AtomicOrdering::SequentiallyConsistent,
279	llvm::AtomicOrdering Failure =
280	llvm::AtomicOrdering::SequentiallyConsistent,
281	bool IsWeak = false);
282	/// Emit atomic update as libcalls.
283	void
284	EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
285	const llvm::function_ref<RValue(RValue)> &UpdateOp,
286	bool IsVolatile);
287	/// Emit atomic update as LLVM instructions.
288	void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
289	const llvm::function_ref<RValue(RValue)> &UpdateOp,
290	bool IsVolatile);
291	/// Emit atomic update as libcalls.
292	void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
293	bool IsVolatile);
294	/// Emit atomic update as LLVM instructions.
295	void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
296	bool IsVolatile);
297	};
298	}
299
300	Address AtomicInfo::CreateTempAlloca() const {
301	Address TempAlloca = CGF.CreateMemTemp(
302	(LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
303	: AtomicTy,
304	getAtomicAlignment(),
305	"atomic-temp");
306	// Cast to pointer to value type for bitfields.
307	if (LVal.isBitField())
308	return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
309	Addr: TempAlloca, Ty: getAtomicAddress().getType(),
310	ElementTy: getAtomicAddress().getElementType());
311	return TempAlloca;
312	}
313
314	static RValue emitAtomicLibcall(CodeGenFunction &CGF,
315	StringRef fnName,
316	QualType resultType,
317	CallArgList &args) {
318	const CGFunctionInfo &fnInfo =
319	CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
320	llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(Info: fnInfo);
321	llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
322	fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
323	fnAttrB.addAttribute(llvm::Attribute::WillReturn);
324	llvm::AttributeList fnAttrs = llvm::AttributeList::get(
325	C&: CGF.getLLVMContext(), Index: llvm::AttributeList::FunctionIndex, B: fnAttrB);
326
327	llvm::FunctionCallee fn =
328	CGF.CGM.CreateRuntimeFunction(Ty: fnTy, Name: fnName, ExtraAttrs: fnAttrs);
329	auto callee = CGCallee::forDirect(functionPtr: fn);
330	return CGF.EmitCall(CallInfo: fnInfo, Callee: callee, ReturnValue: ReturnValueSlot(), Args: args);
331	}
332
333	/// Does a store of the given IR type modify the full expected width?
334	static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
335	uint64_t expectedSize) {
336	return (CGM.getDataLayout().getTypeStoreSize(Ty: type) * 8 == expectedSize);
337	}
338
339	/// Does the atomic type require memsetting to zero before initialization?
340	///
341	/// The IR type is provided as a way of making certain queries faster.
342	bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
343	// If the atomic type has size padding, we definitely need a memset.
344	if (hasPadding()) return true;
345
346	// Otherwise, do some simple heuristics to try to avoid it:
347	switch (getEvaluationKind()) {
348	// For scalars and complexes, check whether the store size of the
349	// type uses the full size.
350	case TEK_Scalar:
351	return !isFullSizeType(CGM&: CGF.CGM, type, expectedSize: AtomicSizeInBits);
352	case TEK_Complex:
353	return !isFullSizeType(CGM&: CGF.CGM, type: type->getStructElementType(N: 0),
354	expectedSize: AtomicSizeInBits / 2);
355
356	// Padding in structs has an undefined bit pattern. User beware.
357	case TEK_Aggregate:
358	return false;
359	}
360	llvm_unreachable("bad evaluation kind");
361	}
362
363	bool AtomicInfo::emitMemSetZeroIfNecessary() const {
364	assert(LVal.isSimple());
365	Address addr = LVal.getAddress();
366	if (!requiresMemSetZero(type: addr.getElementType()))
367	return false;
368
369	CGF.Builder.CreateMemSet(
370	addr.emitRawPointer(CGF), llvm::ConstantInt::get(CGF.Int8Ty, 0),
371	CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
372	LVal.getAlignment().getAsAlign());
373	return true;
374	}
375
376	static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
377	Address Dest, Address Ptr,
378	Address Val1, Address Val2,
379	uint64_t Size,
380	llvm::AtomicOrdering SuccessOrder,
381	llvm::AtomicOrdering FailureOrder,
382	llvm::SyncScope::ID Scope) {
383	// Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
384	llvm::Value *Expected = CGF.Builder.CreateLoad(Addr: Val1);
385	llvm::Value *Desired = CGF.Builder.CreateLoad(Addr: Val2);
386
387	llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
388	Addr: Ptr, Cmp: Expected, New: Desired, SuccessOrdering: SuccessOrder, FailureOrdering: FailureOrder, SSID: Scope);
389	Pair->setVolatile(E->isVolatile());
390	Pair->setWeak(IsWeak);
391	CGF.getTargetHooks().setTargetAtomicMetadata(CGF, AtomicInst&: *Pair, Expr: E);
392
393	// Cmp holds the result of the compare-exchange operation: true on success,
394	// false on failure.
395	llvm::Value *Old = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 0);
396	llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Agg: Pair, Idxs: 1);
397
398	// This basic block is used to hold the store instruction if the operation
399	// failed.
400	llvm::BasicBlock *StoreExpectedBB =
401	CGF.createBasicBlock(name: "cmpxchg.store_expected", parent: CGF.CurFn);
402
403	// This basic block is the exit point of the operation, we should end up
404	// here regardless of whether or not the operation succeeded.
405	llvm::BasicBlock *ContinueBB =
406	CGF.createBasicBlock(name: "cmpxchg.continue", parent: CGF.CurFn);
407
408	// Update Expected if Expected isn't equal to Old, otherwise branch to the
409	// exit point.
410	CGF.Builder.CreateCondBr(Cond: Cmp, True: ContinueBB, False: StoreExpectedBB);
411
412	CGF.Builder.SetInsertPoint(StoreExpectedBB);
413	// Update the memory at Expected with Old's value.
414	CGF.Builder.CreateStore(Val: Old, Addr: Val1);
415	// Finally, branch to the exit point.
416	CGF.Builder.CreateBr(Dest: ContinueBB);
417
418	CGF.Builder.SetInsertPoint(ContinueBB);
419	// Update the memory at Dest with Cmp's value.
420	CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
421	}
422
423	/// Given an ordering required on success, emit all possible cmpxchg
424	/// instructions to cope with the provided (but possibly only dynamically known)
425	/// FailureOrder.
426	static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
427	bool IsWeak, Address Dest, Address Ptr,
428	Address Val1, Address Val2,
429	llvm::Value *FailureOrderVal,
430	uint64_t Size,
431	llvm::AtomicOrdering SuccessOrder,
432	llvm::SyncScope::ID Scope) {
433	llvm::AtomicOrdering FailureOrder;
434	if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(Val: FailureOrderVal)) {
435	auto FOS = FO->getSExtValue();
436	if (!llvm::isValidAtomicOrderingCABI(I: FOS))
437	FailureOrder = llvm::AtomicOrdering::Monotonic;
438	else
439	switch ((llvm::AtomicOrderingCABI)FOS) {
440	case llvm::AtomicOrderingCABI::relaxed:
441	// 31.7.2.18: "The failure argument shall not be memory_order_release
442	// nor memory_order_acq_rel". Fallback to monotonic.
443	case llvm::AtomicOrderingCABI::release:
444	case llvm::AtomicOrderingCABI::acq_rel:
445	FailureOrder = llvm::AtomicOrdering::Monotonic;
446	break;
447	case llvm::AtomicOrderingCABI::consume:
448	case llvm::AtomicOrderingCABI::acquire:
449	FailureOrder = llvm::AtomicOrdering::Acquire;
450	break;
451	case llvm::AtomicOrderingCABI::seq_cst:
452	FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
453	break;
454	}
455	// Prior to c++17, "the failure argument shall be no stronger than the
456	// success argument". This condition has been lifted and the only
457	// precondition is 31.7.2.18. Effectively treat this as a DR and skip
458	// language version checks.
459	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
460	FailureOrder, Scope);
461	return;
462	}
463
464	// Create all the relevant BB's
465	auto *MonotonicBB = CGF.createBasicBlock(name: "monotonic_fail", parent: CGF.CurFn);
466	auto *AcquireBB = CGF.createBasicBlock(name: "acquire_fail", parent: CGF.CurFn);
467	auto *SeqCstBB = CGF.createBasicBlock(name: "seqcst_fail", parent: CGF.CurFn);
468	auto *ContBB = CGF.createBasicBlock(name: "atomic.continue", parent: CGF.CurFn);
469
470	// MonotonicBB is arbitrarily chosen as the default case; in practice, this
471	// doesn't matter unless someone is crazy enough to use something that
472	// doesn't fold to a constant for the ordering.
473	llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(V: FailureOrderVal, Dest: MonotonicBB);
474	// Implemented as acquire, since it's the closest in LLVM.
475	SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::consume),
476	Dest: AcquireBB);
477	SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acquire),
478	Dest: AcquireBB);
479	SI->addCase(OnVal: CGF.Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::seq_cst),
480	Dest: SeqCstBB);
481
482	// Emit all the different atomics
483	CGF.Builder.SetInsertPoint(MonotonicBB);
484	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
485	Size, SuccessOrder, FailureOrder: llvm::AtomicOrdering::Monotonic, Scope);
486	CGF.Builder.CreateBr(Dest: ContBB);
487
488	CGF.Builder.SetInsertPoint(AcquireBB);
489	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
490	FailureOrder: llvm::AtomicOrdering::Acquire, Scope);
491	CGF.Builder.CreateBr(Dest: ContBB);
492
493	CGF.Builder.SetInsertPoint(SeqCstBB);
494	emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
495	FailureOrder: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
496	CGF.Builder.CreateBr(Dest: ContBB);
497
498	CGF.Builder.SetInsertPoint(ContBB);
499	}
500
501	/// Duplicate the atomic min/max operation in conventional IR for the builtin
502	/// variants that return the new rather than the original value.
503	static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
504	AtomicExpr::AtomicOp Op,
505	bool IsSigned,
506	llvm::Value *OldVal,
507	llvm::Value *RHS) {
508	llvm::CmpInst::Predicate Pred;
509	switch (Op) {
510	default:
511	llvm_unreachable("Unexpected min/max operation");
512	case AtomicExpr::AO__atomic_max_fetch:
513	case AtomicExpr::AO__scoped_atomic_max_fetch:
514	Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
515	break;
516	case AtomicExpr::AO__atomic_min_fetch:
517	case AtomicExpr::AO__scoped_atomic_min_fetch:
518	Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
519	break;
520	}
521	llvm::Value *Cmp = Builder.CreateICmp(P: Pred, LHS: OldVal, RHS, Name: "tst");
522	return Builder.CreateSelect(C: Cmp, True: OldVal, False: RHS, Name: "newval");
523	}
524
525	static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
526	Address Ptr, Address Val1, Address Val2,
527	llvm::Value *IsWeak, llvm::Value *FailureOrder,
528	uint64_t Size, llvm::AtomicOrdering Order,
529	llvm::SyncScope::ID Scope) {
530	llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
531	bool PostOpMinMax = false;
532	unsigned PostOp = 0;
533
534	switch (E->getOp()) {
535	case AtomicExpr::AO__c11_atomic_init:
536	case AtomicExpr::AO__opencl_atomic_init:
537	llvm_unreachable("Already handled!");
538
539	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
540	case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
541	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
542	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: false, Dest, Ptr, Val1, Val2,
543	FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
544	return;
545	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
546	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
547	case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
548	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: true, Dest, Ptr, Val1, Val2,
549	FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
550	return;
551	case AtomicExpr::AO__atomic_compare_exchange:
552	case AtomicExpr::AO__atomic_compare_exchange_n:
553	case AtomicExpr::AO__scoped_atomic_compare_exchange:
554	case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
555	if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(Val: IsWeak)) {
556	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: IsWeakC->getZExtValue(), Dest, Ptr,
557	Val1, Val2, FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
558	} else {
559	// Create all the relevant BB's
560	llvm::BasicBlock *StrongBB =
561	CGF.createBasicBlock(name: "cmpxchg.strong", parent: CGF.CurFn);
562	llvm::BasicBlock *WeakBB = CGF.createBasicBlock(name: "cmxchg.weak", parent: CGF.CurFn);
563	llvm::BasicBlock *ContBB =
564	CGF.createBasicBlock(name: "cmpxchg.continue", parent: CGF.CurFn);
565
566	llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(V: IsWeak, Dest: WeakBB);
567	SI->addCase(OnVal: CGF.Builder.getInt1(V: false), Dest: StrongBB);
568
569	CGF.Builder.SetInsertPoint(StrongBB);
570	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: false, Dest, Ptr, Val1, Val2,
571	FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
572	CGF.Builder.CreateBr(Dest: ContBB);
573
574	CGF.Builder.SetInsertPoint(WeakBB);
575	emitAtomicCmpXchgFailureSet(CGF, E, IsWeak: true, Dest, Ptr, Val1, Val2,
576	FailureOrderVal: FailureOrder, Size, SuccessOrder: Order, Scope);
577	CGF.Builder.CreateBr(Dest: ContBB);
578
579	CGF.Builder.SetInsertPoint(ContBB);
580	}
581	return;
582	}
583	case AtomicExpr::AO__c11_atomic_load:
584	case AtomicExpr::AO__opencl_atomic_load:
585	case AtomicExpr::AO__hip_atomic_load:
586	case AtomicExpr::AO__atomic_load_n:
587	case AtomicExpr::AO__atomic_load:
588	case AtomicExpr::AO__scoped_atomic_load_n:
589	case AtomicExpr::AO__scoped_atomic_load: {
590	llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr: Ptr);
591	Load->setAtomic(Ordering: Order, SSID: Scope);
592	Load->setVolatile(E->isVolatile());
593	CGF.maybeAttachRangeForLoad(Load, Ty: E->getValueType(), Loc: E->getExprLoc());
594	CGF.Builder.CreateStore(Val: Load, Addr: Dest);
595	return;
596	}
597
598	case AtomicExpr::AO__c11_atomic_store:
599	case AtomicExpr::AO__opencl_atomic_store:
600	case AtomicExpr::AO__hip_atomic_store:
601	case AtomicExpr::AO__atomic_store:
602	case AtomicExpr::AO__atomic_store_n:
603	case AtomicExpr::AO__scoped_atomic_store:
604	case AtomicExpr::AO__scoped_atomic_store_n: {
605	llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Addr: Val1);
606	llvm::StoreInst *Store = CGF.Builder.CreateStore(Val: LoadVal1, Addr: Ptr);
607	Store->setAtomic(Ordering: Order, SSID: Scope);
608	Store->setVolatile(E->isVolatile());
609	return;
610	}
611
612	case AtomicExpr::AO__c11_atomic_exchange:
613	case AtomicExpr::AO__hip_atomic_exchange:
614	case AtomicExpr::AO__opencl_atomic_exchange:
615	case AtomicExpr::AO__atomic_exchange_n:
616	case AtomicExpr::AO__atomic_exchange:
617	case AtomicExpr::AO__scoped_atomic_exchange_n:
618	case AtomicExpr::AO__scoped_atomic_exchange:
619	Op = llvm::AtomicRMWInst::Xchg;
620	break;
621
622	case AtomicExpr::AO__atomic_add_fetch:
623	case AtomicExpr::AO__scoped_atomic_add_fetch:
624	PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
625	: llvm::Instruction::Add;
626	[[fallthrough]];
627	case AtomicExpr::AO__c11_atomic_fetch_add:
628	case AtomicExpr::AO__hip_atomic_fetch_add:
629	case AtomicExpr::AO__opencl_atomic_fetch_add:
630	case AtomicExpr::AO__atomic_fetch_add:
631	case AtomicExpr::AO__scoped_atomic_fetch_add:
632	Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
633	: llvm::AtomicRMWInst::Add;
634	break;
635
636	case AtomicExpr::AO__atomic_sub_fetch:
637	case AtomicExpr::AO__scoped_atomic_sub_fetch:
638	PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
639	: llvm::Instruction::Sub;
640	[[fallthrough]];
641	case AtomicExpr::AO__c11_atomic_fetch_sub:
642	case AtomicExpr::AO__hip_atomic_fetch_sub:
643	case AtomicExpr::AO__opencl_atomic_fetch_sub:
644	case AtomicExpr::AO__atomic_fetch_sub:
645	case AtomicExpr::AO__scoped_atomic_fetch_sub:
646	Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
647	: llvm::AtomicRMWInst::Sub;
648	break;
649
650	case AtomicExpr::AO__atomic_min_fetch:
651	case AtomicExpr::AO__scoped_atomic_min_fetch:
652	PostOpMinMax = true;
653	[[fallthrough]];
654	case AtomicExpr::AO__c11_atomic_fetch_min:
655	case AtomicExpr::AO__hip_atomic_fetch_min:
656	case AtomicExpr::AO__opencl_atomic_fetch_min:
657	case AtomicExpr::AO__atomic_fetch_min:
658	case AtomicExpr::AO__scoped_atomic_fetch_min:
659	Op = E->getValueType()->isFloatingType()
660	? llvm::AtomicRMWInst::FMin
661	: (E->getValueType()->isSignedIntegerType()
662	? llvm::AtomicRMWInst::Min
663	: llvm::AtomicRMWInst::UMin);
664	break;
665
666	case AtomicExpr::AO__atomic_max_fetch:
667	case AtomicExpr::AO__scoped_atomic_max_fetch:
668	PostOpMinMax = true;
669	[[fallthrough]];
670	case AtomicExpr::AO__c11_atomic_fetch_max:
671	case AtomicExpr::AO__hip_atomic_fetch_max:
672	case AtomicExpr::AO__opencl_atomic_fetch_max:
673	case AtomicExpr::AO__atomic_fetch_max:
674	case AtomicExpr::AO__scoped_atomic_fetch_max:
675	Op = E->getValueType()->isFloatingType()
676	? llvm::AtomicRMWInst::FMax
677	: (E->getValueType()->isSignedIntegerType()
678	? llvm::AtomicRMWInst::Max
679	: llvm::AtomicRMWInst::UMax);
680	break;
681
682	case AtomicExpr::AO__atomic_and_fetch:
683	case AtomicExpr::AO__scoped_atomic_and_fetch:
684	PostOp = llvm::Instruction::And;
685	[[fallthrough]];
686	case AtomicExpr::AO__c11_atomic_fetch_and:
687	case AtomicExpr::AO__hip_atomic_fetch_and:
688	case AtomicExpr::AO__opencl_atomic_fetch_and:
689	case AtomicExpr::AO__atomic_fetch_and:
690	case AtomicExpr::AO__scoped_atomic_fetch_and:
691	Op = llvm::AtomicRMWInst::And;
692	break;
693
694	case AtomicExpr::AO__atomic_or_fetch:
695	case AtomicExpr::AO__scoped_atomic_or_fetch:
696	PostOp = llvm::Instruction::Or;
697	[[fallthrough]];
698	case AtomicExpr::AO__c11_atomic_fetch_or:
699	case AtomicExpr::AO__hip_atomic_fetch_or:
700	case AtomicExpr::AO__opencl_atomic_fetch_or:
701	case AtomicExpr::AO__atomic_fetch_or:
702	case AtomicExpr::AO__scoped_atomic_fetch_or:
703	Op = llvm::AtomicRMWInst::Or;
704	break;
705
706	case AtomicExpr::AO__atomic_xor_fetch:
707	case AtomicExpr::AO__scoped_atomic_xor_fetch:
708	PostOp = llvm::Instruction::Xor;
709	[[fallthrough]];
710	case AtomicExpr::AO__c11_atomic_fetch_xor:
711	case AtomicExpr::AO__hip_atomic_fetch_xor:
712	case AtomicExpr::AO__opencl_atomic_fetch_xor:
713	case AtomicExpr::AO__atomic_fetch_xor:
714	case AtomicExpr::AO__scoped_atomic_fetch_xor:
715	Op = llvm::AtomicRMWInst::Xor;
716	break;
717
718	case AtomicExpr::AO__atomic_nand_fetch:
719	case AtomicExpr::AO__scoped_atomic_nand_fetch:
720	PostOp = llvm::Instruction::And; // the NOT is special cased below
721	[[fallthrough]];
722	case AtomicExpr::AO__c11_atomic_fetch_nand:
723	case AtomicExpr::AO__atomic_fetch_nand:
724	case AtomicExpr::AO__scoped_atomic_fetch_nand:
725	Op = llvm::AtomicRMWInst::Nand;
726	break;
727
728	case AtomicExpr::AO__atomic_test_and_set: {
729	llvm::AtomicRMWInst *RMWI =
730	CGF.emitAtomicRMWInst(Op: llvm::AtomicRMWInst::Xchg, Addr: Ptr,
731	Val: CGF.Builder.getInt8(C: 1), Order, SSID: Scope, AE: E);
732	RMWI->setVolatile(E->isVolatile());
733	llvm::Value *Result = CGF.Builder.CreateIsNotNull(Arg: RMWI, Name: "tobool");
734	CGF.Builder.CreateStore(Val: Result, Addr: Dest);
735	return;
736	}
737
738	case AtomicExpr::AO__atomic_clear: {
739	llvm::StoreInst *Store =
740	CGF.Builder.CreateStore(Val: CGF.Builder.getInt8(C: 0), Addr: Ptr);
741	Store->setAtomic(Ordering: Order, SSID: Scope);
742	Store->setVolatile(E->isVolatile());
743	return;
744	}
745	}
746
747	llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Addr: Val1);
748	llvm::AtomicRMWInst *RMWI =
749	CGF.emitAtomicRMWInst(Op, Addr: Ptr, Val: LoadVal1, Order, SSID: Scope, AE: E);
750	RMWI->setVolatile(E->isVolatile());
751
752	// For __atomic_*_fetch operations, perform the operation again to
753	// determine the value which was written.
754	llvm::Value *Result = RMWI;
755	if (PostOpMinMax)
756	Result = EmitPostAtomicMinMax(Builder&: CGF.Builder, Op: E->getOp(),
757	IsSigned: E->getValueType()->isSignedIntegerType(),
758	OldVal: RMWI, RHS: LoadVal1);
759	else if (PostOp)
760	Result = CGF.Builder.CreateBinOp(Opc: (llvm::Instruction::BinaryOps)PostOp, LHS: RMWI,
761	RHS: LoadVal1);
762	if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch ||
763	E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch)
764	Result = CGF.Builder.CreateNot(V: Result);
765	CGF.Builder.CreateStore(Val: Result, Addr: Dest);
766	}
767
768	// This function emits any expression (scalar, complex, or aggregate)
769	// into a temporary alloca.
770	static Address
771	EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
772	Address DeclPtr = CGF.CreateMemTemp(T: E->getType(), Name: ".atomictmp");
773	CGF.EmitAnyExprToMem(E, Location: DeclPtr, Quals: E->getType().getQualifiers(),
774	/*Init*/ IsInitializer: true);
775	return DeclPtr;
776	}
777
778	static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
779	Address Ptr, Address Val1, Address Val2,
780	llvm::Value *IsWeak, llvm::Value *FailureOrder,
781	uint64_t Size, llvm::AtomicOrdering Order,
782	llvm::Value *Scope) {
783	auto ScopeModel = Expr->getScopeModel();
784
785	// LLVM atomic instructions always have sync scope. If clang atomic
786	// expression has no scope operand, use default LLVM sync scope.
787	if (!ScopeModel) {
788	llvm::SyncScope::ID SS;
789	if (CGF.getLangOpts().OpenCL)
790	// OpenCL approach is: "The functions that do not have memory_scope
791	// argument have the same semantics as the corresponding functions with
792	// the memory_scope argument set to memory_scope_device." See ref.:
793	// https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
794	SS = CGF.getTargetHooks().getLLVMSyncScopeID(LangOpts: CGF.getLangOpts(),
795	Scope: SyncScope::OpenCLDevice,
796	Ordering: Order, Ctx&: CGF.getLLVMContext());
797	else
798	SS = llvm::SyncScope::System;
799	EmitAtomicOp(CGF, E: Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
800	Order, Scope: SS);
801	return;
802	}
803
804	// Handle constant scope.
805	if (auto SC = dyn_cast<llvm::ConstantInt>(Val: Scope)) {
806	auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
807	LangOpts: CGF.CGM.getLangOpts(), Scope: ScopeModel->map(SC->getZExtValue()),
808	Ordering: Order, Ctx&: CGF.CGM.getLLVMContext());
809	EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
810	Order, SCID);
811	return;
812	}
813
814	// Handle non-constant scope.
815	auto &Builder = CGF.Builder;
816	auto Scopes = ScopeModel->getRuntimeValues();
817	llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
818	for (auto S : Scopes)
819	BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
820
821	llvm::BasicBlock *ContBB =
822	CGF.createBasicBlock(name: "atomic.scope.continue", parent: CGF.CurFn);
823
824	auto *SC = Builder.CreateIntCast(V: Scope, DestTy: Builder.getInt32Ty(), isSigned: false);
825	// If unsupported sync scope is encountered at run time, assume a fallback
826	// sync scope value.
827	auto FallBack = ScopeModel->getFallBackValue();
828	llvm::SwitchInst *SI = Builder.CreateSwitch(V: SC, Dest: BB[FallBack]);
829	for (auto S : Scopes) {
830	auto *B = BB[S];
831	if (S != FallBack)
832	SI->addCase(Builder.getInt32(S), B);
833
834	Builder.SetInsertPoint(B);
835	EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
836	Order,
837	CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
838	ScopeModel->map(S),
839	Order,
840	CGF.getLLVMContext()));
841	Builder.CreateBr(ContBB);
842	}
843
844	Builder.SetInsertPoint(ContBB);
845	}
846
847	RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
848	QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
849	QualType MemTy = AtomicTy;
850	if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
851	MemTy = AT->getValueType();
852	llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
853
854	Address Val1 = Address::invalid();
855	Address Val2 = Address::invalid();
856	Address Dest = Address::invalid();
857	Address Ptr = EmitPointerWithAlignment(Addr: E->getPtr());
858
859	if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
860	E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
861	LValue lvalue = MakeAddrLValue(Addr: Ptr, T: AtomicTy);
862	EmitAtomicInit(E: E->getVal1(), lvalue);
863	return RValue::get(V: nullptr);
864	}
865
866	auto TInfo = getContext().getTypeInfoInChars(T: AtomicTy);
867	uint64_t Size = TInfo.Width.getQuantity();
868	unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
869
870	CharUnits MaxInlineWidth =
871	getContext().toCharUnitsFromBits(BitSize: MaxInlineWidthInBits);
872	DiagnosticsEngine &Diags = CGM.getDiags();
873	bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
874	bool Oversized = getContext().toBits(CharSize: TInfo.Width) > MaxInlineWidthInBits;
875	if (Misaligned) {
876	Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
877	<< (int)TInfo.Width.getQuantity()
878	<< (int)Ptr.getAlignment().getQuantity();
879	}
880	if (Oversized) {
881	Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
882	<< (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
883	}
884
885	llvm::Value *Order = EmitScalarExpr(E: E->getOrder());
886	llvm::Value *Scope =
887	E->getScopeModel() ? EmitScalarExpr(E: E->getScope()) : nullptr;
888	bool ShouldCastToIntPtrTy = true;
889
890	switch (E->getOp()) {
891	case AtomicExpr::AO__c11_atomic_init:
892	case AtomicExpr::AO__opencl_atomic_init:
893	llvm_unreachable("Already handled above with EmitAtomicInit!");
894
895	case AtomicExpr::AO__atomic_load_n:
896	case AtomicExpr::AO__scoped_atomic_load_n:
897	case AtomicExpr::AO__c11_atomic_load:
898	case AtomicExpr::AO__opencl_atomic_load:
899	case AtomicExpr::AO__hip_atomic_load:
900	case AtomicExpr::AO__atomic_test_and_set:
901	case AtomicExpr::AO__atomic_clear:
902	break;
903
904	case AtomicExpr::AO__atomic_load:
905	case AtomicExpr::AO__scoped_atomic_load:
906	Dest = EmitPointerWithAlignment(Addr: E->getVal1());
907	break;
908
909	case AtomicExpr::AO__atomic_store:
910	case AtomicExpr::AO__scoped_atomic_store:
911	Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
912	break;
913
914	case AtomicExpr::AO__atomic_exchange:
915	case AtomicExpr::AO__scoped_atomic_exchange:
916	Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
917	Dest = EmitPointerWithAlignment(Addr: E->getVal2());
918	break;
919
920	case AtomicExpr::AO__atomic_compare_exchange:
921	case AtomicExpr::AO__atomic_compare_exchange_n:
922	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
923	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
924	case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
925	case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
926	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
927	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
928	case AtomicExpr::AO__scoped_atomic_compare_exchange:
929	case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
930	Val1 = EmitPointerWithAlignment(Addr: E->getVal1());
931	if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
932	E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
933	Val2 = EmitPointerWithAlignment(Addr: E->getVal2());
934	else
935	Val2 = EmitValToTemp(CGF&: *this, E: E->getVal2());
936	OrderFail = EmitScalarExpr(E: E->getOrderFail());
937	if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
938	E->getOp() == AtomicExpr::AO__atomic_compare_exchange ||
939	E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n ||
940	E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange)
941	IsWeak = EmitScalarExpr(E: E->getWeak());
942	break;
943
944	case AtomicExpr::AO__c11_atomic_fetch_add:
945	case AtomicExpr::AO__c11_atomic_fetch_sub:
946	case AtomicExpr::AO__hip_atomic_fetch_add:
947	case AtomicExpr::AO__hip_atomic_fetch_sub:
948	case AtomicExpr::AO__opencl_atomic_fetch_add:
949	case AtomicExpr::AO__opencl_atomic_fetch_sub:
950	if (MemTy->isPointerType()) {
951	// For pointer arithmetic, we're required to do a bit of math:
952	// adding 1 to an int* is not the same as adding 1 to a uintptr_t.
953	// ... but only for the C11 builtins. The GNU builtins expect the
954	// user to multiply by sizeof(T).
955	QualType Val1Ty = E->getVal1()->getType();
956	llvm::Value *Val1Scalar = EmitScalarExpr(E: E->getVal1());
957	CharUnits PointeeIncAmt =
958	getContext().getTypeSizeInChars(T: MemTy->getPointeeType());
959	Val1Scalar = Builder.CreateMul(LHS: Val1Scalar, RHS: CGM.getSize(numChars: PointeeIncAmt));
960	auto Temp = CreateMemTemp(T: Val1Ty, Name: ".atomictmp");
961	Val1 = Temp;
962	EmitStoreOfScalar(value: Val1Scalar, lvalue: MakeAddrLValue(Addr: Temp, T: Val1Ty));
963	break;
964	}
965	[[fallthrough]];
966	case AtomicExpr::AO__atomic_fetch_add:
967	case AtomicExpr::AO__atomic_fetch_max:
968	case AtomicExpr::AO__atomic_fetch_min:
969	case AtomicExpr::AO__atomic_fetch_sub:
970	case AtomicExpr::AO__atomic_add_fetch:
971	case AtomicExpr::AO__atomic_max_fetch:
972	case AtomicExpr::AO__atomic_min_fetch:
973	case AtomicExpr::AO__atomic_sub_fetch:
974	case AtomicExpr::AO__c11_atomic_fetch_max:
975	case AtomicExpr::AO__c11_atomic_fetch_min:
976	case AtomicExpr::AO__opencl_atomic_fetch_max:
977	case AtomicExpr::AO__opencl_atomic_fetch_min:
978	case AtomicExpr::AO__hip_atomic_fetch_max:
979	case AtomicExpr::AO__hip_atomic_fetch_min:
980	case AtomicExpr::AO__scoped_atomic_fetch_add:
981	case AtomicExpr::AO__scoped_atomic_fetch_max:
982	case AtomicExpr::AO__scoped_atomic_fetch_min:
983	case AtomicExpr::AO__scoped_atomic_fetch_sub:
984	case AtomicExpr::AO__scoped_atomic_add_fetch:
985	case AtomicExpr::AO__scoped_atomic_max_fetch:
986	case AtomicExpr::AO__scoped_atomic_min_fetch:
987	case AtomicExpr::AO__scoped_atomic_sub_fetch:
988	ShouldCastToIntPtrTy = !MemTy->isFloatingType();
989	[[fallthrough]];
990
991	case AtomicExpr::AO__atomic_fetch_and:
992	case AtomicExpr::AO__atomic_fetch_nand:
993	case AtomicExpr::AO__atomic_fetch_or:
994	case AtomicExpr::AO__atomic_fetch_xor:
995	case AtomicExpr::AO__atomic_and_fetch:
996	case AtomicExpr::AO__atomic_nand_fetch:
997	case AtomicExpr::AO__atomic_or_fetch:
998	case AtomicExpr::AO__atomic_xor_fetch:
999	case AtomicExpr::AO__atomic_store_n:
1000	case AtomicExpr::AO__atomic_exchange_n:
1001	case AtomicExpr::AO__c11_atomic_fetch_and:
1002	case AtomicExpr::AO__c11_atomic_fetch_nand:
1003	case AtomicExpr::AO__c11_atomic_fetch_or:
1004	case AtomicExpr::AO__c11_atomic_fetch_xor:
1005	case AtomicExpr::AO__c11_atomic_store:
1006	case AtomicExpr::AO__c11_atomic_exchange:
1007	case AtomicExpr::AO__hip_atomic_fetch_and:
1008	case AtomicExpr::AO__hip_atomic_fetch_or:
1009	case AtomicExpr::AO__hip_atomic_fetch_xor:
1010	case AtomicExpr::AO__hip_atomic_store:
1011	case AtomicExpr::AO__hip_atomic_exchange:
1012	case AtomicExpr::AO__opencl_atomic_fetch_and:
1013	case AtomicExpr::AO__opencl_atomic_fetch_or:
1014	case AtomicExpr::AO__opencl_atomic_fetch_xor:
1015	case AtomicExpr::AO__opencl_atomic_store:
1016	case AtomicExpr::AO__opencl_atomic_exchange:
1017	case AtomicExpr::AO__scoped_atomic_fetch_and:
1018	case AtomicExpr::AO__scoped_atomic_fetch_nand:
1019	case AtomicExpr::AO__scoped_atomic_fetch_or:
1020	case AtomicExpr::AO__scoped_atomic_fetch_xor:
1021	case AtomicExpr::AO__scoped_atomic_and_fetch:
1022	case AtomicExpr::AO__scoped_atomic_nand_fetch:
1023	case AtomicExpr::AO__scoped_atomic_or_fetch:
1024	case AtomicExpr::AO__scoped_atomic_xor_fetch:
1025	case AtomicExpr::AO__scoped_atomic_store_n:
1026	case AtomicExpr::AO__scoped_atomic_exchange_n:
1027	Val1 = EmitValToTemp(CGF&: *this, E: E->getVal1());
1028	break;
1029	}
1030
1031	QualType RValTy = E->getType().getUnqualifiedType();
1032
1033	// The inlined atomics only function on iN types, where N is a power of 2. We
1034	// need to make sure (via temporaries if necessary) that all incoming values
1035	// are compatible.
1036	LValue AtomicVal = MakeAddrLValue(Addr: Ptr, T: AtomicTy);
1037	AtomicInfo Atomics(*this, AtomicVal);
1038
1039	if (ShouldCastToIntPtrTy) {
1040	Ptr = Atomics.castToAtomicIntPointer(Addr: Ptr);
1041	if (Val1.isValid())
1042	Val1 = Atomics.convertToAtomicIntPointer(Addr: Val1);
1043	if (Val2.isValid())
1044	Val2 = Atomics.convertToAtomicIntPointer(Addr: Val2);
1045	}
1046	if (Dest.isValid()) {
1047	if (ShouldCastToIntPtrTy)
1048	Dest = Atomics.castToAtomicIntPointer(Addr: Dest);
1049	} else if (E->isCmpXChg())
1050	Dest = CreateMemTemp(T: RValTy, Name: "cmpxchg.bool");
1051	else if (!RValTy->isVoidType()) {
1052	Dest = Atomics.CreateTempAlloca();
1053	if (ShouldCastToIntPtrTy)
1054	Dest = Atomics.castToAtomicIntPointer(Addr: Dest);
1055	}
1056
1057	bool PowerOf2Size = (Size & (Size - 1)) == 0;
1058	bool UseLibcall = !PowerOf2Size || (Size > 16);
1059
1060	// For atomics larger than 16 bytes, emit a libcall from the frontend. This
1061	// avoids the overhead of dealing with excessively-large value types in IR.
1062	// Non-power-of-2 values also lower to libcall here, as they are not currently
1063	// permitted in IR instructions (although that constraint could be relaxed in
1064	// the future). For other cases where a libcall is required on a given
1065	// platform, we let the backend handle it (this includes handling for all of
1066	// the size-optimized libcall variants, which are only valid up to 16 bytes.)
1067	//
1068	// See: https://llvm.org/docs/Atomics.html#libcalls-atomic
1069	if (UseLibcall) {
1070	CallArgList Args;
1071	// For non-optimized library calls, the size is the first parameter.
1072	Args.add(rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: SizeTy, V: Size)),
1073	type: getContext().getSizeType());
1074
1075	// The atomic address is the second parameter.
1076	// The OpenCL atomic library functions only accept pointer arguments to
1077	// generic address space.
1078	auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1079	if (!E->isOpenCL())
1080	return V;
1081	auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1082	if (AS == LangAS::opencl_generic)
1083	return V;
1084	auto DestAS = getContext().getTargetAddressSpace(AS: LangAS::opencl_generic);
1085	auto *DestType = llvm::PointerType::get(C&: getLLVMContext(), AddressSpace: DestAS);
1086
1087	return getTargetHooks().performAddrSpaceCast(CGF&: *this, V, SrcAddr: AS, DestTy: DestType,
1088	IsNonNull: false);
1089	};
1090
1091	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Ptr.emitRawPointer(CGF&: *this),
1092	E->getPtr()->getType())),
1093	type: getContext().VoidPtrTy);
1094
1095	// The next 1-3 parameters are op-dependent.
1096	std::string LibCallName;
1097	QualType RetTy;
1098	bool HaveRetTy = false;
1099	switch (E->getOp()) {
1100	case AtomicExpr::AO__c11_atomic_init:
1101	case AtomicExpr::AO__opencl_atomic_init:
1102	llvm_unreachable("Already handled!");
1103
1104	// There is only one libcall for compare an exchange, because there is no
1105	// optimisation benefit possible from a libcall version of a weak compare
1106	// and exchange.
1107	// bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1108	// void *desired, int success, int failure)
1109	case AtomicExpr::AO__atomic_compare_exchange:
1110	case AtomicExpr::AO__atomic_compare_exchange_n:
1111	case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1112	case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1113	case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1114	case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1115	case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1116	case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1117	case AtomicExpr::AO__scoped_atomic_compare_exchange:
1118	case AtomicExpr::AO__scoped_atomic_compare_exchange_n:
1119	LibCallName = "__atomic_compare_exchange";
1120	RetTy = getContext().BoolTy;
1121	HaveRetTy = true;
1122	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1123	E->getVal1()->getType())),
1124	type: getContext().VoidPtrTy);
1125	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val2.emitRawPointer(CGF&: *this),
1126	E->getVal2()->getType())),
1127	type: getContext().VoidPtrTy);
1128	Args.add(rvalue: RValue::get(V: Order), type: getContext().IntTy);
1129	Order = OrderFail;
1130	break;
1131	// void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1132	// int order)
1133	case AtomicExpr::AO__atomic_exchange:
1134	case AtomicExpr::AO__atomic_exchange_n:
1135	case AtomicExpr::AO__c11_atomic_exchange:
1136	case AtomicExpr::AO__hip_atomic_exchange:
1137	case AtomicExpr::AO__opencl_atomic_exchange:
1138	case AtomicExpr::AO__scoped_atomic_exchange:
1139	case AtomicExpr::AO__scoped_atomic_exchange_n:
1140	LibCallName = "__atomic_exchange";
1141	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1142	E->getVal1()->getType())),
1143	type: getContext().VoidPtrTy);
1144	break;
1145	// void __atomic_store(size_t size, void *mem, void *val, int order)
1146	case AtomicExpr::AO__atomic_store:
1147	case AtomicExpr::AO__atomic_store_n:
1148	case AtomicExpr::AO__c11_atomic_store:
1149	case AtomicExpr::AO__hip_atomic_store:
1150	case AtomicExpr::AO__opencl_atomic_store:
1151	case AtomicExpr::AO__scoped_atomic_store:
1152	case AtomicExpr::AO__scoped_atomic_store_n:
1153	LibCallName = "__atomic_store";
1154	RetTy = getContext().VoidTy;
1155	HaveRetTy = true;
1156	Args.add(rvalue: RValue::get(V: CastToGenericAddrSpace(Val1.emitRawPointer(CGF&: *this),
1157	E->getVal1()->getType())),
1158	type: getContext().VoidPtrTy);
1159	break;
1160	// void __atomic_load(size_t size, void *mem, void *return, int order)
1161	case AtomicExpr::AO__atomic_load:
1162	case AtomicExpr::AO__atomic_load_n:
1163	case AtomicExpr::AO__c11_atomic_load:
1164	case AtomicExpr::AO__hip_atomic_load:
1165	case AtomicExpr::AO__opencl_atomic_load:
1166	case AtomicExpr::AO__scoped_atomic_load:
1167	case AtomicExpr::AO__scoped_atomic_load_n:
1168	LibCallName = "__atomic_load";
1169	break;
1170	case AtomicExpr::AO__atomic_add_fetch:
1171	case AtomicExpr::AO__scoped_atomic_add_fetch:
1172	case AtomicExpr::AO__atomic_fetch_add:
1173	case AtomicExpr::AO__c11_atomic_fetch_add:
1174	case AtomicExpr::AO__hip_atomic_fetch_add:
1175	case AtomicExpr::AO__opencl_atomic_fetch_add:
1176	case AtomicExpr::AO__scoped_atomic_fetch_add:
1177	case AtomicExpr::AO__atomic_and_fetch:
1178	case AtomicExpr::AO__scoped_atomic_and_fetch:
1179	case AtomicExpr::AO__atomic_fetch_and:
1180	case AtomicExpr::AO__c11_atomic_fetch_and:
1181	case AtomicExpr::AO__hip_atomic_fetch_and:
1182	case AtomicExpr::AO__opencl_atomic_fetch_and:
1183	case AtomicExpr::AO__scoped_atomic_fetch_and:
1184	case AtomicExpr::AO__atomic_or_fetch:
1185	case AtomicExpr::AO__scoped_atomic_or_fetch:
1186	case AtomicExpr::AO__atomic_fetch_or:
1187	case AtomicExpr::AO__c11_atomic_fetch_or:
1188	case AtomicExpr::AO__hip_atomic_fetch_or:
1189	case AtomicExpr::AO__opencl_atomic_fetch_or:
1190	case AtomicExpr::AO__scoped_atomic_fetch_or:
1191	case AtomicExpr::AO__atomic_sub_fetch:
1192	case AtomicExpr::AO__scoped_atomic_sub_fetch:
1193	case AtomicExpr::AO__atomic_fetch_sub:
1194	case AtomicExpr::AO__c11_atomic_fetch_sub:
1195	case AtomicExpr::AO__hip_atomic_fetch_sub:
1196	case AtomicExpr::AO__opencl_atomic_fetch_sub:
1197	case AtomicExpr::AO__scoped_atomic_fetch_sub:
1198	case AtomicExpr::AO__atomic_xor_fetch:
1199	case AtomicExpr::AO__scoped_atomic_xor_fetch:
1200	case AtomicExpr::AO__atomic_fetch_xor:
1201	case AtomicExpr::AO__c11_atomic_fetch_xor:
1202	case AtomicExpr::AO__hip_atomic_fetch_xor:
1203	case AtomicExpr::AO__opencl_atomic_fetch_xor:
1204	case AtomicExpr::AO__scoped_atomic_fetch_xor:
1205	case AtomicExpr::AO__atomic_nand_fetch:
1206	case AtomicExpr::AO__atomic_fetch_nand:
1207	case AtomicExpr::AO__c11_atomic_fetch_nand:
1208	case AtomicExpr::AO__scoped_atomic_fetch_nand:
1209	case AtomicExpr::AO__scoped_atomic_nand_fetch:
1210	case AtomicExpr::AO__atomic_min_fetch:
1211	case AtomicExpr::AO__atomic_fetch_min:
1212	case AtomicExpr::AO__c11_atomic_fetch_min:
1213	case AtomicExpr::AO__hip_atomic_fetch_min:
1214	case AtomicExpr::AO__opencl_atomic_fetch_min:
1215	case AtomicExpr::AO__scoped_atomic_fetch_min:
1216	case AtomicExpr::AO__scoped_atomic_min_fetch:
1217	case AtomicExpr::AO__atomic_max_fetch:
1218	case AtomicExpr::AO__atomic_fetch_max:
1219	case AtomicExpr::AO__c11_atomic_fetch_max:
1220	case AtomicExpr::AO__hip_atomic_fetch_max:
1221	case AtomicExpr::AO__opencl_atomic_fetch_max:
1222	case AtomicExpr::AO__scoped_atomic_fetch_max:
1223	case AtomicExpr::AO__scoped_atomic_max_fetch:
1224	case AtomicExpr::AO__atomic_test_and_set:
1225	case AtomicExpr::AO__atomic_clear:
1226	llvm_unreachable("Integral atomic operations always become atomicrmw!");
1227	}
1228
1229	if (E->isOpenCL()) {
1230	LibCallName =
1231	std::string("__opencl") + StringRef(LibCallName).drop_front(N: 1).str();
1232	}
1233	// By default, assume we return a value of the atomic type.
1234	if (!HaveRetTy) {
1235	// Value is returned through parameter before the order.
1236	RetTy = getContext().VoidTy;
1237	Args.add(rvalue: RValue::get(
1238	V: CastToGenericAddrSpace(Dest.emitRawPointer(CGF&: *this), RetTy)),
1239	type: getContext().VoidPtrTy);
1240	}
1241	// Order is always the last parameter.
1242	Args.add(rvalue: RValue::get(V: Order),
1243	type: getContext().IntTy);
1244	if (E->isOpenCL())
1245	Args.add(rvalue: RValue::get(V: Scope), type: getContext().IntTy);
1246
1247	RValue Res = emitAtomicLibcall(CGF&: *this, fnName: LibCallName, resultType: RetTy, args&: Args);
1248	// The value is returned directly from the libcall.
1249	if (E->isCmpXChg())
1250	return Res;
1251
1252	if (RValTy->isVoidType())
1253	return RValue::get(V: nullptr);
1254
1255	return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1256	type: RValTy, Loc: E->getExprLoc());
1257	}
1258
1259	bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1260	E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1261	E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1262	E->getOp() == AtomicExpr::AO__atomic_store ||
1263	E->getOp() == AtomicExpr::AO__atomic_store_n ||
1264	E->getOp() == AtomicExpr::AO__scoped_atomic_store ||
1265	E->getOp() == AtomicExpr::AO__scoped_atomic_store_n ||
1266	E->getOp() == AtomicExpr::AO__atomic_clear;
1267	bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1268	E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1269	E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1270	E->getOp() == AtomicExpr::AO__atomic_load ||
1271	E->getOp() == AtomicExpr::AO__atomic_load_n ||
1272	E->getOp() == AtomicExpr::AO__scoped_atomic_load ||
1273	E->getOp() == AtomicExpr::AO__scoped_atomic_load_n;
1274
1275	if (isa<llvm::ConstantInt>(Val: Order)) {
1276	auto ord = cast<llvm::ConstantInt>(Val: Order)->getZExtValue();
1277	// We should not ever get to a case where the ordering isn't a valid C ABI
1278	// value, but it's hard to enforce that in general.
1279	if (llvm::isValidAtomicOrderingCABI(I: ord))
1280	switch ((llvm::AtomicOrderingCABI)ord) {
1281	case llvm::AtomicOrderingCABI::relaxed:
1282	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1283	Order: llvm::AtomicOrdering::Monotonic, Scope);
1284	break;
1285	case llvm::AtomicOrderingCABI::consume:
1286	case llvm::AtomicOrderingCABI::acquire:
1287	if (IsStore)
1288	break; // Avoid crashing on code with undefined behavior
1289	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1290	Order: llvm::AtomicOrdering::Acquire, Scope);
1291	break;
1292	case llvm::AtomicOrderingCABI::release:
1293	if (IsLoad)
1294	break; // Avoid crashing on code with undefined behavior
1295	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1296	Order: llvm::AtomicOrdering::Release, Scope);
1297	break;
1298	case llvm::AtomicOrderingCABI::acq_rel:
1299	if (IsLoad || IsStore)
1300	break; // Avoid crashing on code with undefined behavior
1301	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1302	Order: llvm::AtomicOrdering::AcquireRelease, Scope);
1303	break;
1304	case llvm::AtomicOrderingCABI::seq_cst:
1305	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1306	Order: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1307	break;
1308	}
1309	if (RValTy->isVoidType())
1310	return RValue::get(V: nullptr);
1311
1312	return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1313	type: RValTy, Loc: E->getExprLoc());
1314	}
1315
1316	// Long case, when Order isn't obviously constant.
1317
1318	// Create all the relevant BB's
1319	llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1320	*ReleaseBB = nullptr, *AcqRelBB = nullptr,
1321	*SeqCstBB = nullptr;
1322	MonotonicBB = createBasicBlock(name: "monotonic", parent: CurFn);
1323	if (!IsStore)
1324	AcquireBB = createBasicBlock(name: "acquire", parent: CurFn);
1325	if (!IsLoad)
1326	ReleaseBB = createBasicBlock(name: "release", parent: CurFn);
1327	if (!IsLoad && !IsStore)
1328	AcqRelBB = createBasicBlock(name: "acqrel", parent: CurFn);
1329	SeqCstBB = createBasicBlock(name: "seqcst", parent: CurFn);
1330	llvm::BasicBlock *ContBB = createBasicBlock(name: "atomic.continue", parent: CurFn);
1331
1332	// Create the switch for the split
1333	// MonotonicBB is arbitrarily chosen as the default case; in practice, this
1334	// doesn't matter unless someone is crazy enough to use something that
1335	// doesn't fold to a constant for the ordering.
1336	Order = Builder.CreateIntCast(V: Order, DestTy: Builder.getInt32Ty(), isSigned: false);
1337	llvm::SwitchInst *SI = Builder.CreateSwitch(V: Order, Dest: MonotonicBB);
1338
1339	// Emit all the different atomics
1340	Builder.SetInsertPoint(MonotonicBB);
1341	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1342	Order: llvm::AtomicOrdering::Monotonic, Scope);
1343	Builder.CreateBr(Dest: ContBB);
1344	if (!IsStore) {
1345	Builder.SetInsertPoint(AcquireBB);
1346	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1347	Order: llvm::AtomicOrdering::Acquire, Scope);
1348	Builder.CreateBr(Dest: ContBB);
1349	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::consume),
1350	Dest: AcquireBB);
1351	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acquire),
1352	Dest: AcquireBB);
1353	}
1354	if (!IsLoad) {
1355	Builder.SetInsertPoint(ReleaseBB);
1356	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1357	Order: llvm::AtomicOrdering::Release, Scope);
1358	Builder.CreateBr(Dest: ContBB);
1359	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::release),
1360	Dest: ReleaseBB);
1361	}
1362	if (!IsLoad && !IsStore) {
1363	Builder.SetInsertPoint(AcqRelBB);
1364	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1365	Order: llvm::AtomicOrdering::AcquireRelease, Scope);
1366	Builder.CreateBr(Dest: ContBB);
1367	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::acq_rel),
1368	Dest: AcqRelBB);
1369	}
1370	Builder.SetInsertPoint(SeqCstBB);
1371	EmitAtomicOp(CGF&: *this, Expr: E, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder: OrderFail, Size,
1372	Order: llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1373	Builder.CreateBr(Dest: ContBB);
1374	SI->addCase(OnVal: Builder.getInt32(C: (int)llvm::AtomicOrderingCABI::seq_cst),
1375	Dest: SeqCstBB);
1376
1377	// Cleanup and return
1378	Builder.SetInsertPoint(ContBB);
1379	if (RValTy->isVoidType())
1380	return RValue::get(V: nullptr);
1381
1382	assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1383	return convertTempToRValue(addr: Dest.withElementType(ElemTy: ConvertTypeForMem(T: RValTy)),
1384	type: RValTy, Loc: E->getExprLoc());
1385	}
1386
1387	Address AtomicInfo::castToAtomicIntPointer(Address addr) const {
1388	llvm::IntegerType *ty =
1389	llvm::IntegerType::get(C&: CGF.getLLVMContext(), NumBits: AtomicSizeInBits);
1390	return addr.withElementType(ElemTy: ty);
1391	}
1392
1393	Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1394	llvm::Type *Ty = Addr.getElementType();
1395	uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1396	if (SourceSizeInBits != AtomicSizeInBits) {
1397	Address Tmp = CreateTempAlloca();
1398	CGF.Builder.CreateMemCpy(Dest: Tmp, Src: Addr,
1399	Size: std::min(a: AtomicSizeInBits, b: SourceSizeInBits) / 8);
1400	Addr = Tmp;
1401	}
1402
1403	return castToAtomicIntPointer(addr: Addr);
1404	}
1405
1406	RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1407	AggValueSlot resultSlot,
1408	SourceLocation loc,
1409	bool asValue) const {
1410	if (LVal.isSimple()) {
1411	if (EvaluationKind == TEK_Aggregate)
1412	return resultSlot.asRValue();
1413
1414	// Drill into the padding structure if we have one.
1415	if (hasPadding())
1416	addr = CGF.Builder.CreateStructGEP(Addr: addr, Index: 0);
1417
1418	// Otherwise, just convert the temporary to an r-value using the
1419	// normal conversion routine.
1420	return CGF.convertTempToRValue(addr, type: getValueType(), Loc: loc);
1421	}
1422	if (!asValue)
1423	// Get RValue from temp memory as atomic for non-simple lvalues
1424	return RValue::get(V: CGF.Builder.CreateLoad(Addr: addr));
1425	if (LVal.isBitField())
1426	return CGF.EmitLoadOfBitfieldLValue(
1427	LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1428	LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1429	if (LVal.isVectorElt())
1430	return CGF.EmitLoadOfLValue(
1431	LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1432	LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1433	assert(LVal.isExtVectorElt());
1434	return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1435	addr, LVal.getExtVectorElts(), LVal.getType(),
1436	LVal.getBaseInfo(), TBAAAccessInfo()));
1437	}
1438
1439	/// Return true if \param ValTy is a type that should be casted to integer
1440	/// around the atomic memory operation. If \param CmpXchg is true, then the
1441	/// cast of a floating point type is made as that instruction can not have
1442	/// floating point operands. TODO: Allow compare-and-exchange and FP - see
1443	/// comment in AtomicExpandPass.cpp.
1444	static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) {
1445	if (ValTy->isFloatingPointTy())
1446	return ValTy->isX86_FP80Ty() || CmpXchg;
1447	return !ValTy->isIntegerTy() && !ValTy->isPointerTy();
1448	}
1449
1450	RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val,
1451	AggValueSlot ResultSlot,
1452	SourceLocation Loc, bool AsValue,
1453	bool CmpXchg) const {
1454	// Try not to in some easy cases.
1455	assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() ||
1456	Val->getType()->isIEEELikeFPTy()) &&
1457	"Expected integer, pointer or floating point value when converting "
1458	"result.");
1459	if (getEvaluationKind() == TEK_Scalar &&
1460	(((!LVal.isBitField() ||
1461	LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1462	!hasPadding()) ||
1463	!AsValue)) {
1464	auto *ValTy = AsValue
1465	? CGF.ConvertTypeForMem(ValueTy)
1466	: getAtomicAddress().getElementType();
1467	if (!shouldCastToInt(ValTy, CmpXchg)) {
1468	assert((!ValTy->isIntegerTy() || Val->getType() == ValTy) &&
1469	"Different integer types.");
1470	return RValue::get(CGF.EmitFromMemory(Val, ValueTy));
1471	}
1472	if (llvm::CastInst::isBitCastable(SrcTy: Val->getType(), DestTy: ValTy))
1473	return RValue::get(CGF.Builder.CreateBitCast(V: Val, DestTy: ValTy));
1474	}
1475
1476	// Create a temporary. This needs to be big enough to hold the
1477	// atomic integer.
1478	Address Temp = Address::invalid();
1479	bool TempIsVolatile = false;
1480	if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1481	assert(!ResultSlot.isIgnored());
1482	Temp = ResultSlot.getAddress();
1483	TempIsVolatile = ResultSlot.isVolatile();
1484	} else {
1485	Temp = CreateTempAlloca();
1486	}
1487
1488	// Slam the integer into the temporary.
1489	Address CastTemp = castToAtomicIntPointer(addr: Temp);
1490	CGF.Builder.CreateStore(Val, Addr: CastTemp)->setVolatile(TempIsVolatile);
1491
1492	return convertAtomicTempToRValue(addr: Temp, resultSlot: ResultSlot, loc: Loc, asValue: AsValue);
1493	}
1494
1495	void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1496	llvm::AtomicOrdering AO, bool) {
1497	// void __atomic_load(size_t size, void *mem, void *return, int order);
1498	CallArgList Args;
1499	Args.add(rvalue: RValue::get(V: getAtomicSizeValue()), type: CGF.getContext().getSizeType());
1500	Args.add(rvalue: RValue::get(V: getAtomicPointer()), type: CGF.getContext().VoidPtrTy);
1501	Args.add(rvalue: RValue::get(V: AddForLoaded), type: CGF.getContext().VoidPtrTy);
1502	Args.add(
1503	rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO))),
1504	type: CGF.getContext().IntTy);
1505	emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1506	}
1507
1508	llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1509	bool IsVolatile, bool CmpXchg) {
1510	// Okay, we're doing this natively.
1511	Address Addr = getAtomicAddress();
1512	if (shouldCastToInt(ValTy: Addr.getElementType(), CmpXchg))
1513	Addr = castToAtomicIntPointer(addr: Addr);
1514	llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, Name: "atomic-load");
1515	Load->setAtomic(Ordering: AO);
1516
1517	// Other decoration.
1518	if (IsVolatile)
1519	Load->setVolatile(true);
1520	CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1521	return Load;
1522	}
1523
1524	/// An LValue is a candidate for having its loads and stores be made atomic if
1525	/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1526	/// performing such an operation can be performed without a libcall.
1527	bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1528	if (!CGM.getLangOpts().MSVolatile) return false;
1529	AtomicInfo AI(*this, LV);
1530	bool IsVolatile = LV.isVolatile() || hasVolatileMember(T: LV.getType());
1531	// An atomic is inline if we don't need to use a libcall.
1532	bool AtomicIsInline = !AI.shouldUseLibcall();
1533	// MSVC doesn't seem to do this for types wider than a pointer.
1534	if (getContext().getTypeSize(T: LV.getType()) >
1535	getContext().getTypeSize(T: getContext().getIntPtrType()))
1536	return false;
1537	return IsVolatile && AtomicIsInline;
1538	}
1539
1540	RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1541	AggValueSlot Slot) {
1542	llvm::AtomicOrdering AO;
1543	bool IsVolatile = LV.isVolatileQualified();
1544	if (LV.getType()->isAtomicType()) {
1545	AO = llvm::AtomicOrdering::SequentiallyConsistent;
1546	} else {
1547	AO = llvm::AtomicOrdering::Acquire;
1548	IsVolatile = true;
1549	}
1550	return EmitAtomicLoad(lvalue: LV, loc: SL, AO, IsVolatile, slot: Slot);
1551	}
1552
1553	RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1554	bool AsValue, llvm::AtomicOrdering AO,
1555	bool IsVolatile) {
1556	// Check whether we should use a library call.
1557	if (shouldUseLibcall()) {
1558	Address TempAddr = Address::invalid();
1559	if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1560	assert(getEvaluationKind() == TEK_Aggregate);
1561	TempAddr = ResultSlot.getAddress();
1562	} else
1563	TempAddr = CreateTempAlloca();
1564
1565	EmitAtomicLoadLibcall(AddForLoaded: TempAddr.emitRawPointer(CGF), AO, IsVolatile);
1566
1567	// Okay, turn that back into the original value or whole atomic (for
1568	// non-simple lvalues) type.
1569	return convertAtomicTempToRValue(addr: TempAddr, resultSlot: ResultSlot, loc: Loc, asValue: AsValue);
1570	}
1571
1572	// Okay, we're doing this natively.
1573	auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1574
1575	// If we're ignoring an aggregate return, don't do anything.
1576	if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1577	return RValue::getAggregate(addr: Address::invalid(), isVolatile: false);
1578
1579	// Okay, turn that back into the original value or atomic (for non-simple
1580	// lvalues) type.
1581	return ConvertToValueOrAtomic(Val: Load, ResultSlot, Loc, AsValue);
1582	}
1583
1584	/// Emit a load from an l-value of atomic type. Note that the r-value
1585	/// we produce is an r-value of the atomic *value* type.
1586	RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1587	llvm::AtomicOrdering AO, bool IsVolatile,
1588	AggValueSlot resultSlot) {
1589	AtomicInfo Atomics(*this, src);
1590	return Atomics.EmitAtomicLoad(ResultSlot: resultSlot, Loc: loc, /*AsValue=*/true, AO,
1591	IsVolatile);
1592	}
1593
1594	/// Copy an r-value into memory as part of storing to an atomic type.
1595	/// This needs to create a bit-pattern suitable for atomic operations.
1596	void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1597	assert(LVal.isSimple());
1598	// If we have an r-value, the rvalue should be of the atomic type,
1599	// which means that the caller is responsible for having zeroed
1600	// any padding. Just do an aggregate copy of that type.
1601	if (rvalue.isAggregate()) {
1602	LValue Dest = CGF.MakeAddrLValue(Addr: getAtomicAddress(), T: getAtomicType());
1603	LValue Src = CGF.MakeAddrLValue(Addr: rvalue.getAggregateAddress(),
1604	T: getAtomicType());
1605	bool IsVolatile = rvalue.isVolatileQualified() ||
1606	LVal.isVolatileQualified();
1607	CGF.EmitAggregateCopy(Dest, Src, EltTy: getAtomicType(),
1608	MayOverlap: AggValueSlot::DoesNotOverlap, isVolatile: IsVolatile);
1609	return;
1610	}
1611
1612	// Okay, otherwise we're copying stuff.
1613
1614	// Zero out the buffer if necessary.
1615	emitMemSetZeroIfNecessary();
1616
1617	// Drill past the padding if present.
1618	LValue TempLVal = projectValue();
1619
1620	// Okay, store the rvalue in.
1621	if (rvalue.isScalar()) {
1622	CGF.EmitStoreOfScalar(value: rvalue.getScalarVal(), lvalue: TempLVal, /*init*/ isInit: true);
1623	} else {
1624	CGF.EmitStoreOfComplex(V: rvalue.getComplexVal(), dest: TempLVal, /*init*/ isInit: true);
1625	}
1626	}
1627
1628
1629	/// Materialize an r-value into memory for the purposes of storing it
1630	/// to an atomic type.
1631	Address AtomicInfo::materializeRValue(RValue rvalue) const {
1632	// Aggregate r-values are already in memory, and EmitAtomicStore
1633	// requires them to be values of the atomic type.
1634	if (rvalue.isAggregate())
1635	return rvalue.getAggregateAddress();
1636
1637	// Otherwise, make a temporary and materialize into it.
1638	LValue TempLV = CGF.MakeAddrLValue(Addr: CreateTempAlloca(), T: getAtomicType());
1639	AtomicInfo Atomics(CGF, TempLV);
1640	Atomics.emitCopyIntoMemory(rvalue);
1641	return TempLV.getAddress();
1642	}
1643
1644	llvm::Value *AtomicInfo::getScalarRValValueOrNull(RValue RVal) const {
1645	if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple()))
1646	return RVal.getScalarVal();
1647	return nullptr;
1648	}
1649
1650	llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal, bool CmpXchg) const {
1651	// If we've got a scalar value of the right size, try to avoid going
1652	// through memory. Floats get casted if needed by AtomicExpandPass.
1653	if (llvm::Value *Value = getScalarRValValueOrNull(RVal)) {
1654	if (!shouldCastToInt(ValTy: Value->getType(), CmpXchg))
1655	return CGF.EmitToMemory(Value, ValueTy);
1656	else {
1657	llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1658	CGF.getLLVMContext(),
1659	LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1660	if (llvm::BitCastInst::isBitCastable(SrcTy: Value->getType(), DestTy: InputIntTy))
1661	return CGF.Builder.CreateBitCast(V: Value, DestTy: InputIntTy);
1662	}
1663	}
1664	// Otherwise, we need to go through memory.
1665	// Put the r-value in memory.
1666	Address Addr = materializeRValue(rvalue: RVal);
1667
1668	// Cast the temporary to the atomic int type and pull a value out.
1669	Addr = castToAtomicIntPointer(addr: Addr);
1670	return CGF.Builder.CreateLoad(Addr);
1671	}
1672
1673	std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1674	llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1675	llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1676	// Do the atomic store.
1677	Address Addr = getAtomicAddressAsAtomicIntPointer();
1678	auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr, Cmp: ExpectedVal, New: DesiredVal,
1679	SuccessOrdering: Success, FailureOrdering: Failure);
1680	// Other decoration.
1681	Inst->setVolatile(LVal.isVolatileQualified());
1682	Inst->setWeak(IsWeak);
1683
1684	// Okay, turn that back into the original value type.
1685	auto *PreviousVal = CGF.Builder.CreateExtractValue(Agg: Inst, /*Idxs=*/0);
1686	auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Agg: Inst, /*Idxs=*/1);
1687	return std::make_pair(x&: PreviousVal, y&: SuccessFailureVal);
1688	}
1689
1690	llvm::Value *
1691	AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1692	llvm::Value *DesiredAddr,
1693	llvm::AtomicOrdering Success,
1694	llvm::AtomicOrdering Failure) {
1695	// bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1696	// void *desired, int success, int failure);
1697	CallArgList Args;
1698	Args.add(rvalue: RValue::get(V: getAtomicSizeValue()), type: CGF.getContext().getSizeType());
1699	Args.add(rvalue: RValue::get(V: getAtomicPointer()), type: CGF.getContext().VoidPtrTy);
1700	Args.add(rvalue: RValue::get(V: ExpectedAddr), type: CGF.getContext().VoidPtrTy);
1701	Args.add(rvalue: RValue::get(V: DesiredAddr), type: CGF.getContext().VoidPtrTy);
1702	Args.add(rvalue: RValue::get(
1703	V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO: Success))),
1704	type: CGF.getContext().IntTy);
1705	Args.add(rvalue: RValue::get(
1706	V: llvm::ConstantInt::get(Ty: CGF.IntTy, V: (int)llvm::toCABI(AO: Failure))),
1707	type: CGF.getContext().IntTy);
1708	auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1709	CGF.getContext().BoolTy, Args);
1710
1711	return SuccessFailureRVal.getScalarVal();
1712	}
1713
1714	std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1715	RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1716	llvm::AtomicOrdering Failure, bool IsWeak) {
1717	// Check whether we should use a library call.
1718	if (shouldUseLibcall()) {
1719	// Produce a source address.
1720	Address ExpectedAddr = materializeRValue(rvalue: Expected);
1721	llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1722	llvm::Value *DesiredPtr = materializeRValue(rvalue: Desired).emitRawPointer(CGF);
1723	auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr,
1724	Success, Failure);
1725	return std::make_pair(
1726	x: convertAtomicTempToRValue(addr: ExpectedAddr, resultSlot: AggValueSlot::ignored(),
1727	loc: SourceLocation(), /*AsValue=*/asValue: false),
1728	y&: Res);
1729	}
1730
1731	// If we've got a scalar value of the right size, try to avoid going
1732	// through memory.
1733	auto *ExpectedVal = convertRValueToInt(RVal: Expected, /*CmpXchg=*/true);
1734	auto *DesiredVal = convertRValueToInt(RVal: Desired, /*CmpXchg=*/true);
1735	auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1736	Failure, IsWeak);
1737	return std::make_pair(
1738	x: ConvertToValueOrAtomic(Val: Res.first, ResultSlot: AggValueSlot::ignored(),
1739	Loc: SourceLocation(), /*AsValue=*/false,
1740	/*CmpXchg=*/true),
1741	y&: Res.second);
1742	}
1743
1744	static void
1745	EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1746	const llvm::function_ref<RValue(RValue)> &UpdateOp,
1747	Address DesiredAddr) {
1748	RValue UpRVal;
1749	LValue AtomicLVal = Atomics.getAtomicLValue();
1750	LValue DesiredLVal;
1751	if (AtomicLVal.isSimple()) {
1752	UpRVal = OldRVal;
1753	DesiredLVal = CGF.MakeAddrLValue(Addr: DesiredAddr, T: AtomicLVal.getType());
1754	} else {
1755	// Build new lvalue for temp address.
1756	Address Ptr = Atomics.materializeRValue(rvalue: OldRVal);
1757	LValue UpdateLVal;
1758	if (AtomicLVal.isBitField()) {
1759	UpdateLVal =
1760	LValue::MakeBitfield(Addr: Ptr, Info: AtomicLVal.getBitFieldInfo(),
1761	type: AtomicLVal.getType(),
1762	BaseInfo: AtomicLVal.getBaseInfo(),
1763	TBAAInfo: AtomicLVal.getTBAAInfo());
1764	DesiredLVal =
1765	LValue::MakeBitfield(Addr: DesiredAddr, Info: AtomicLVal.getBitFieldInfo(),
1766	type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1767	TBAAInfo: AtomicLVal.getTBAAInfo());
1768	} else if (AtomicLVal.isVectorElt()) {
1769	UpdateLVal = LValue::MakeVectorElt(vecAddress: Ptr, Idx: AtomicLVal.getVectorIdx(),
1770	type: AtomicLVal.getType(),
1771	BaseInfo: AtomicLVal.getBaseInfo(),
1772	TBAAInfo: AtomicLVal.getTBAAInfo());
1773	DesiredLVal = LValue::MakeVectorElt(
1774	vecAddress: DesiredAddr, Idx: AtomicLVal.getVectorIdx(), type: AtomicLVal.getType(),
1775	BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1776	} else {
1777	assert(AtomicLVal.isExtVectorElt());
1778	UpdateLVal = LValue::MakeExtVectorElt(Addr: Ptr, Elts: AtomicLVal.getExtVectorElts(),
1779	type: AtomicLVal.getType(),
1780	BaseInfo: AtomicLVal.getBaseInfo(),
1781	TBAAInfo: AtomicLVal.getTBAAInfo());
1782	DesiredLVal = LValue::MakeExtVectorElt(
1783	Addr: DesiredAddr, Elts: AtomicLVal.getExtVectorElts(), type: AtomicLVal.getType(),
1784	BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1785	}
1786	UpRVal = CGF.EmitLoadOfLValue(V: UpdateLVal, Loc: SourceLocation());
1787	}
1788	// Store new value in the corresponding memory area.
1789	RValue NewRVal = UpdateOp(UpRVal);
1790	if (NewRVal.isScalar()) {
1791	CGF.EmitStoreThroughLValue(Src: NewRVal, Dst: DesiredLVal);
1792	} else {
1793	assert(NewRVal.isComplex());
1794	CGF.EmitStoreOfComplex(V: NewRVal.getComplexVal(), dest: DesiredLVal,
1795	/*isInit=*/false);
1796	}
1797	}
1798
1799	void AtomicInfo::EmitAtomicUpdateLibcall(
1800	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1801	bool IsVolatile) {
1802	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1803
1804	Address ExpectedAddr = CreateTempAlloca();
1805
1806	EmitAtomicLoadLibcall(AddForLoaded: ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1807	auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1808	auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1809	CGF.EmitBlock(BB: ContBB);
1810	Address DesiredAddr = CreateTempAlloca();
1811	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1812	requiresMemSetZero(getAtomicAddress().getElementType())) {
1813	auto *OldVal = CGF.Builder.CreateLoad(Addr: ExpectedAddr);
1814	CGF.Builder.CreateStore(Val: OldVal, Addr: DesiredAddr);
1815	}
1816	auto OldRVal = convertAtomicTempToRValue(addr: ExpectedAddr,
1817	resultSlot: AggValueSlot::ignored(),
1818	loc: SourceLocation(), /*AsValue=*/asValue: false);
1819	EmitAtomicUpdateValue(CGF, Atomics&: *this, OldRVal, UpdateOp, DesiredAddr);
1820	llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1821	llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1822	auto *Res =
1823	EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr, Success: AO, Failure);
1824	CGF.Builder.CreateCondBr(Cond: Res, True: ExitBB, False: ContBB);
1825	CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1826	}
1827
1828	void AtomicInfo::EmitAtomicUpdateOp(
1829	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1830	bool IsVolatile) {
1831	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1832
1833	// Do the atomic load.
1834	auto *OldVal = EmitAtomicLoadOp(AO: Failure, IsVolatile, /*CmpXchg=*/true);
1835	// For non-simple lvalues perform compare-and-swap procedure.
1836	auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1837	auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1838	auto *CurBB = CGF.Builder.GetInsertBlock();
1839	CGF.EmitBlock(BB: ContBB);
1840	llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: OldVal->getType(),
1841	/*NumReservedValues=*/2);
1842	PHI->addIncoming(V: OldVal, BB: CurBB);
1843	Address NewAtomicAddr = CreateTempAlloca();
1844	Address NewAtomicIntAddr =
1845	shouldCastToInt(ValTy: NewAtomicAddr.getElementType(), /*CmpXchg=*/true)
1846	? castToAtomicIntPointer(addr: NewAtomicAddr)
1847	: NewAtomicAddr;
1848
1849	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1850	requiresMemSetZero(getAtomicAddress().getElementType())) {
1851	CGF.Builder.CreateStore(Val: PHI, Addr: NewAtomicIntAddr);
1852	}
1853	auto OldRVal = ConvertToValueOrAtomic(Val: PHI, ResultSlot: AggValueSlot::ignored(),
1854	Loc: SourceLocation(), /*AsValue=*/false,
1855	/*CmpXchg=*/true);
1856	EmitAtomicUpdateValue(CGF, Atomics&: *this, OldRVal, UpdateOp, DesiredAddr: NewAtomicAddr);
1857	auto *DesiredVal = CGF.Builder.CreateLoad(Addr: NewAtomicIntAddr);
1858	// Try to write new value using cmpxchg operation.
1859	auto Res = EmitAtomicCompareExchangeOp(ExpectedVal: PHI, DesiredVal, Success: AO, Failure);
1860	PHI->addIncoming(V: Res.first, BB: CGF.Builder.GetInsertBlock());
1861	CGF.Builder.CreateCondBr(Cond: Res.second, True: ExitBB, False: ContBB);
1862	CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1863	}
1864
1865	static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1866	RValue UpdateRVal, Address DesiredAddr) {
1867	LValue AtomicLVal = Atomics.getAtomicLValue();
1868	LValue DesiredLVal;
1869	// Build new lvalue for temp address.
1870	if (AtomicLVal.isBitField()) {
1871	DesiredLVal =
1872	LValue::MakeBitfield(Addr: DesiredAddr, Info: AtomicLVal.getBitFieldInfo(),
1873	type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1874	TBAAInfo: AtomicLVal.getTBAAInfo());
1875	} else if (AtomicLVal.isVectorElt()) {
1876	DesiredLVal =
1877	LValue::MakeVectorElt(vecAddress: DesiredAddr, Idx: AtomicLVal.getVectorIdx(),
1878	type: AtomicLVal.getType(), BaseInfo: AtomicLVal.getBaseInfo(),
1879	TBAAInfo: AtomicLVal.getTBAAInfo());
1880	} else {
1881	assert(AtomicLVal.isExtVectorElt());
1882	DesiredLVal = LValue::MakeExtVectorElt(
1883	Addr: DesiredAddr, Elts: AtomicLVal.getExtVectorElts(), type: AtomicLVal.getType(),
1884	BaseInfo: AtomicLVal.getBaseInfo(), TBAAInfo: AtomicLVal.getTBAAInfo());
1885	}
1886	// Store new value in the corresponding memory area.
1887	assert(UpdateRVal.isScalar());
1888	CGF.EmitStoreThroughLValue(Src: UpdateRVal, Dst: DesiredLVal);
1889	}
1890
1891	void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1892	RValue UpdateRVal, bool IsVolatile) {
1893	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1894
1895	Address ExpectedAddr = CreateTempAlloca();
1896
1897	EmitAtomicLoadLibcall(AddForLoaded: ExpectedAddr.emitRawPointer(CGF), AO, IsVolatile);
1898	auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1899	auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1900	CGF.EmitBlock(BB: ContBB);
1901	Address DesiredAddr = CreateTempAlloca();
1902	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1903	requiresMemSetZero(getAtomicAddress().getElementType())) {
1904	auto *OldVal = CGF.Builder.CreateLoad(Addr: ExpectedAddr);
1905	CGF.Builder.CreateStore(Val: OldVal, Addr: DesiredAddr);
1906	}
1907	EmitAtomicUpdateValue(CGF, Atomics&: *this, UpdateRVal, DesiredAddr);
1908	llvm::Value *ExpectedPtr = ExpectedAddr.emitRawPointer(CGF);
1909	llvm::Value *DesiredPtr = DesiredAddr.emitRawPointer(CGF);
1910	auto *Res =
1911	EmitAtomicCompareExchangeLibcall(ExpectedAddr: ExpectedPtr, DesiredAddr: DesiredPtr, Success: AO, Failure);
1912	CGF.Builder.CreateCondBr(Cond: Res, True: ExitBB, False: ContBB);
1913	CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1914	}
1915
1916	void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1917	bool IsVolatile) {
1918	auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrdering: AO);
1919
1920	// Do the atomic load.
1921	auto *OldVal = EmitAtomicLoadOp(AO: Failure, IsVolatile, /*CmpXchg=*/true);
1922	// For non-simple lvalues perform compare-and-swap procedure.
1923	auto *ContBB = CGF.createBasicBlock(name: "atomic_cont");
1924	auto *ExitBB = CGF.createBasicBlock(name: "atomic_exit");
1925	auto *CurBB = CGF.Builder.GetInsertBlock();
1926	CGF.EmitBlock(BB: ContBB);
1927	llvm::PHINode *PHI = CGF.Builder.CreatePHI(Ty: OldVal->getType(),
1928	/*NumReservedValues=*/2);
1929	PHI->addIncoming(V: OldVal, BB: CurBB);
1930	Address NewAtomicAddr = CreateTempAlloca();
1931	Address NewAtomicIntAddr = castToAtomicIntPointer(addr: NewAtomicAddr);
1932	if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1933	requiresMemSetZero(getAtomicAddress().getElementType())) {
1934	CGF.Builder.CreateStore(Val: PHI, Addr: NewAtomicIntAddr);
1935	}
1936	EmitAtomicUpdateValue(CGF, Atomics&: *this, UpdateRVal, DesiredAddr: NewAtomicAddr);
1937	auto *DesiredVal = CGF.Builder.CreateLoad(Addr: NewAtomicIntAddr);
1938	// Try to write new value using cmpxchg operation.
1939	auto Res = EmitAtomicCompareExchangeOp(ExpectedVal: PHI, DesiredVal, Success: AO, Failure);
1940	PHI->addIncoming(V: Res.first, BB: CGF.Builder.GetInsertBlock());
1941	CGF.Builder.CreateCondBr(Cond: Res.second, True: ExitBB, False: ContBB);
1942	CGF.EmitBlock(BB: ExitBB, /*IsFinished=*/true);
1943	}
1944
1945	void AtomicInfo::EmitAtomicUpdate(
1946	llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1947	bool IsVolatile) {
1948	if (shouldUseLibcall()) {
1949	EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1950	} else {
1951	EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1952	}
1953	}
1954
1955	void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1956	bool IsVolatile) {
1957	if (shouldUseLibcall()) {
1958	EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1959	} else {
1960	EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1961	}
1962	}
1963
1964	void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1965	bool isInit) {
1966	bool IsVolatile = lvalue.isVolatileQualified();
1967	llvm::AtomicOrdering AO;
1968	if (lvalue.getType()->isAtomicType()) {
1969	AO = llvm::AtomicOrdering::SequentiallyConsistent;
1970	} else {
1971	AO = llvm::AtomicOrdering::Release;
1972	IsVolatile = true;
1973	}
1974	return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1975	}
1976
1977	/// Emit a store to an l-value of atomic type.
1978	///
1979	/// Note that the r-value is expected to be an r-value *of the atomic
1980	/// type*; this means that for aggregate r-values, it should include
1981	/// storage for any padding that was necessary.
1982	void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1983	llvm::AtomicOrdering AO, bool IsVolatile,
1984	bool isInit) {
1985	// If this is an aggregate r-value, it should agree in type except
1986	// maybe for address-space qualification.
1987	assert(!rvalue.isAggregate() ||
1988	rvalue.getAggregateAddress().getElementType() ==
1989	dest.getAddress().getElementType());
1990
1991	AtomicInfo atomics(*this, dest);
1992	LValue LVal = atomics.getAtomicLValue();
1993
1994	// If this is an initialization, just put the value there normally.
1995	if (LVal.isSimple()) {
1996	if (isInit) {
1997	atomics.emitCopyIntoMemory(rvalue);
1998	return;
1999	}
2000
2001	// Check whether we should use a library call.
2002	if (atomics.shouldUseLibcall()) {
2003	// Produce a source address.
2004	Address srcAddr = atomics.materializeRValue(rvalue);
2005
2006	// void __atomic_store(size_t size, void *mem, void *val, int order)
2007	CallArgList args;
2008	args.add(rvalue: RValue::get(V: atomics.getAtomicSizeValue()),
2009	type: getContext().getSizeType());
2010	args.add(rvalue: RValue::get(V: atomics.getAtomicPointer()), type: getContext().VoidPtrTy);
2011	args.add(rvalue: RValue::get(V: srcAddr.emitRawPointer(CGF&: *this)),
2012	type: getContext().VoidPtrTy);
2013	args.add(
2014	rvalue: RValue::get(V: llvm::ConstantInt::get(Ty: IntTy, V: (int)llvm::toCABI(AO))),
2015	type: getContext().IntTy);
2016	emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2017	return;
2018	}
2019
2020	// Okay, we're doing this natively.
2021	llvm::Value *ValToStore = atomics.convertRValueToInt(RVal: rvalue);
2022
2023	// Do the atomic store.
2024	Address Addr = atomics.getAtomicAddress();
2025	if (llvm::Value *Value = atomics.getScalarRValValueOrNull(RVal: rvalue))
2026	if (shouldCastToInt(ValTy: Value->getType(), /*CmpXchg=*/false)) {
2027	Addr = atomics.castToAtomicIntPointer(addr: Addr);
2028	ValToStore = Builder.CreateIntCast(V: ValToStore, DestTy: Addr.getElementType(),
2029	/*isSigned=*/false);
2030	}
2031	llvm::StoreInst *store = Builder.CreateStore(Val: ValToStore, Addr);
2032
2033	if (AO == llvm::AtomicOrdering::Acquire)
2034	AO = llvm::AtomicOrdering::Monotonic;
2035	else if (AO == llvm::AtomicOrdering::AcquireRelease)
2036	AO = llvm::AtomicOrdering::Release;
2037	// Initializations don't need to be atomic.
2038	if (!isInit)
2039	store->setAtomic(Ordering: AO);
2040
2041	// Other decoration.
2042	if (IsVolatile)
2043	store->setVolatile(true);
2044	CGM.DecorateInstructionWithTBAA(Inst: store, TBAAInfo: dest.getTBAAInfo());
2045	return;
2046	}
2047
2048	// Emit simple atomic update operation.
2049	atomics.EmitAtomicUpdate(AO, UpdateRVal: rvalue, IsVolatile);
2050	}
2051
2052	/// Emit a compare-and-exchange op for atomic type.
2053	///
2054	std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2055	LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2056	llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2057	AggValueSlot Slot) {
2058	// If this is an aggregate r-value, it should agree in type except
2059	// maybe for address-space qualification.
2060	assert(!Expected.isAggregate() ||
2061	Expected.getAggregateAddress().getElementType() ==
2062	Obj.getAddress().getElementType());
2063	assert(!Desired.isAggregate() ||
2064	Desired.getAggregateAddress().getElementType() ==
2065	Obj.getAddress().getElementType());
2066	AtomicInfo Atomics(*this, Obj);
2067
2068	return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2069	IsWeak);
2070	}
2071
2072	llvm::AtomicRMWInst *
2073	CodeGenFunction::emitAtomicRMWInst(llvm::AtomicRMWInst::BinOp Op, Address Addr,
2074	llvm::Value *Val, llvm::AtomicOrdering Order,
2075	llvm::SyncScope::ID SSID,
2076	const AtomicExpr *AE) {
2077	llvm::AtomicRMWInst *RMW =
2078	Builder.CreateAtomicRMW(Op, Addr, Val, Ordering: Order, SSID);
2079	getTargetHooks().setTargetAtomicMetadata(CGF&: *this, AtomicInst&: *RMW, Expr: AE);
2080	return RMW;
2081	}
2082
2083	void CodeGenFunction::EmitAtomicUpdate(
2084	LValue LVal, llvm::AtomicOrdering AO,
2085	const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2086	AtomicInfo Atomics(*this, LVal);
2087	Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2088	}
2089
2090	void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2091	AtomicInfo atomics(*this, dest);
2092
2093	switch (atomics.getEvaluationKind()) {
2094	case TEK_Scalar: {
2095	llvm::Value *value = EmitScalarExpr(E: init);
2096	atomics.emitCopyIntoMemory(rvalue: RValue::get(V: value));
2097	return;
2098	}
2099
2100	case TEK_Complex: {
2101	ComplexPairTy value = EmitComplexExpr(E: init);
2102	atomics.emitCopyIntoMemory(rvalue: RValue::getComplex(C: value));
2103	return;
2104	}
2105
2106	case TEK_Aggregate: {
2107	// Fix up the destination if the initializer isn't an expression
2108	// of atomic type.
2109	bool Zeroed = false;
2110	if (!init->getType()->isAtomicType()) {
2111	Zeroed = atomics.emitMemSetZeroIfNecessary();
2112	dest = atomics.projectValue();
2113	}
2114
2115	// Evaluate the expression directly into the destination.
2116	AggValueSlot slot = AggValueSlot::forLValue(
2117	LV: dest, isDestructed: AggValueSlot::IsNotDestructed,
2118	needsGC: AggValueSlot::DoesNotNeedGCBarriers, isAliased: AggValueSlot::IsNotAliased,
2119	mayOverlap: AggValueSlot::DoesNotOverlap,
2120	isZeroed: Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2121
2122	EmitAggExpr(E: init, AS: slot);
2123	return;
2124	}
2125	}
2126	llvm_unreachable("bad evaluation kind");
2127	}
2128