1 | //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains the code for emitting atomic operations. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #include "CGCall.h" |
14 | #include "CGRecordLayout.h" |
15 | #include "CodeGenFunction.h" |
16 | #include "CodeGenModule.h" |
17 | #include "TargetInfo.h" |
18 | #include "clang/AST/ASTContext.h" |
19 | #include "clang/CodeGen/CGFunctionInfo.h" |
20 | #include "clang/Frontend/FrontendDiagnostic.h" |
21 | #include "llvm/ADT/DenseMap.h" |
22 | #include "llvm/IR/DataLayout.h" |
23 | #include "llvm/IR/Intrinsics.h" |
24 | #include "llvm/IR/Operator.h" |
25 | |
26 | using namespace clang; |
27 | using namespace CodeGen; |
28 | |
29 | namespace { |
30 | class AtomicInfo { |
31 | CodeGenFunction &CGF; |
32 | QualType AtomicTy; |
33 | QualType ValueTy; |
34 | uint64_t AtomicSizeInBits; |
35 | uint64_t ValueSizeInBits; |
36 | CharUnits AtomicAlign; |
37 | CharUnits ValueAlign; |
38 | TypeEvaluationKind EvaluationKind; |
39 | bool UseLibcall; |
40 | LValue LVal; |
41 | CGBitFieldInfo BFI; |
42 | public: |
43 | AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) |
44 | : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), |
45 | EvaluationKind(TEK_Scalar), UseLibcall(true) { |
46 | assert(!lvalue.isGlobalReg()); |
47 | ASTContext &C = CGF.getContext(); |
48 | if (lvalue.isSimple()) { |
49 | AtomicTy = lvalue.getType(); |
50 | if (auto *ATy = AtomicTy->getAs<AtomicType>()) |
51 | ValueTy = ATy->getValueType(); |
52 | else |
53 | ValueTy = AtomicTy; |
54 | EvaluationKind = CGF.getEvaluationKind(ValueTy); |
55 | |
56 | uint64_t ValueAlignInBits; |
57 | uint64_t AtomicAlignInBits; |
58 | TypeInfo ValueTI = C.getTypeInfo(ValueTy); |
59 | ValueSizeInBits = ValueTI.Width; |
60 | ValueAlignInBits = ValueTI.Align; |
61 | |
62 | TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); |
63 | AtomicSizeInBits = AtomicTI.Width; |
64 | AtomicAlignInBits = AtomicTI.Align; |
65 | |
66 | assert(ValueSizeInBits <= AtomicSizeInBits); |
67 | assert(ValueAlignInBits <= AtomicAlignInBits); |
68 | |
69 | AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); |
70 | ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); |
71 | if (lvalue.getAlignment().isZero()) |
72 | lvalue.setAlignment(AtomicAlign); |
73 | |
74 | LVal = lvalue; |
75 | } else if (lvalue.isBitField()) { |
76 | ValueTy = lvalue.getType(); |
77 | ValueSizeInBits = C.getTypeSize(ValueTy); |
78 | auto &OrigBFI = lvalue.getBitFieldInfo(); |
79 | auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); |
80 | AtomicSizeInBits = C.toBits( |
81 | C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) |
82 | .alignTo(lvalue.getAlignment())); |
83 | auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); |
84 | auto OffsetInChars = |
85 | (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * |
86 | lvalue.getAlignment(); |
87 | VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( |
88 | CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity()); |
89 | llvm::Type *IntTy = CGF.Builder.getIntNTy(AtomicSizeInBits); |
90 | auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( |
91 | VoidPtrAddr, IntTy->getPointerTo(), "atomic_bitfield_base" ); |
92 | BFI = OrigBFI; |
93 | BFI.Offset = Offset; |
94 | BFI.StorageSize = AtomicSizeInBits; |
95 | BFI.StorageOffset += OffsetInChars; |
96 | LVal = LValue::MakeBitfield(Address(Addr, IntTy, lvalue.getAlignment()), |
97 | BFI, lvalue.getType(), lvalue.getBaseInfo(), |
98 | lvalue.getTBAAInfo()); |
99 | AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); |
100 | if (AtomicTy.isNull()) { |
101 | llvm::APInt Size( |
102 | /*numBits=*/32, |
103 | C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); |
104 | AtomicTy = |
105 | C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal, |
106 | /*IndexTypeQuals=*/0); |
107 | } |
108 | AtomicAlign = ValueAlign = lvalue.getAlignment(); |
109 | } else if (lvalue.isVectorElt()) { |
110 | ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType(); |
111 | ValueSizeInBits = C.getTypeSize(ValueTy); |
112 | AtomicTy = lvalue.getType(); |
113 | AtomicSizeInBits = C.getTypeSize(AtomicTy); |
114 | AtomicAlign = ValueAlign = lvalue.getAlignment(); |
115 | LVal = lvalue; |
116 | } else { |
117 | assert(lvalue.isExtVectorElt()); |
118 | ValueTy = lvalue.getType(); |
119 | ValueSizeInBits = C.getTypeSize(ValueTy); |
120 | AtomicTy = ValueTy = CGF.getContext().getExtVectorType( |
121 | lvalue.getType(), cast<llvm::FixedVectorType>( |
122 | lvalue.getExtVectorAddress().getElementType()) |
123 | ->getNumElements()); |
124 | AtomicSizeInBits = C.getTypeSize(AtomicTy); |
125 | AtomicAlign = ValueAlign = lvalue.getAlignment(); |
126 | LVal = lvalue; |
127 | } |
128 | UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( |
129 | AtomicSizeInBits, C.toBits(lvalue.getAlignment())); |
130 | } |
131 | |
132 | QualType getAtomicType() const { return AtomicTy; } |
133 | QualType getValueType() const { return ValueTy; } |
134 | CharUnits getAtomicAlignment() const { return AtomicAlign; } |
135 | uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } |
136 | uint64_t getValueSizeInBits() const { return ValueSizeInBits; } |
137 | TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } |
138 | bool shouldUseLibcall() const { return UseLibcall; } |
139 | const LValue &getAtomicLValue() const { return LVal; } |
140 | llvm::Value *getAtomicPointer() const { |
141 | if (LVal.isSimple()) |
142 | return LVal.getPointer(CGF); |
143 | else if (LVal.isBitField()) |
144 | return LVal.getBitFieldPointer(); |
145 | else if (LVal.isVectorElt()) |
146 | return LVal.getVectorPointer(); |
147 | assert(LVal.isExtVectorElt()); |
148 | return LVal.getExtVectorPointer(); |
149 | } |
150 | Address getAtomicAddress() const { |
151 | llvm::Type *ElTy; |
152 | if (LVal.isSimple()) |
153 | ElTy = LVal.getAddress(CGF).getElementType(); |
154 | else if (LVal.isBitField()) |
155 | ElTy = LVal.getBitFieldAddress().getElementType(); |
156 | else if (LVal.isVectorElt()) |
157 | ElTy = LVal.getVectorAddress().getElementType(); |
158 | else |
159 | ElTy = LVal.getExtVectorAddress().getElementType(); |
160 | return Address(getAtomicPointer(), ElTy, getAtomicAlignment()); |
161 | } |
162 | |
163 | Address getAtomicAddressAsAtomicIntPointer() const { |
164 | return emitCastToAtomicIntPointer(getAtomicAddress()); |
165 | } |
166 | |
167 | /// Is the atomic size larger than the underlying value type? |
168 | /// |
169 | /// Note that the absence of padding does not mean that atomic |
170 | /// objects are completely interchangeable with non-atomic |
171 | /// objects: we might have promoted the alignment of a type |
172 | /// without making it bigger. |
173 | bool hasPadding() const { |
174 | return (ValueSizeInBits != AtomicSizeInBits); |
175 | } |
176 | |
177 | bool emitMemSetZeroIfNecessary() const; |
178 | |
179 | llvm::Value *getAtomicSizeValue() const { |
180 | CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); |
181 | return CGF.CGM.getSize(size); |
182 | } |
183 | |
184 | /// Cast the given pointer to an integer pointer suitable for atomic |
185 | /// operations if the source. |
186 | Address emitCastToAtomicIntPointer(Address Addr) const; |
187 | |
188 | /// If Addr is compatible with the iN that will be used for an atomic |
189 | /// operation, bitcast it. Otherwise, create a temporary that is suitable |
190 | /// and copy the value across. |
191 | Address convertToAtomicIntPointer(Address Addr) const; |
192 | |
193 | /// Turn an atomic-layout object into an r-value. |
194 | RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, |
195 | SourceLocation loc, bool AsValue) const; |
196 | |
197 | /// Converts a rvalue to integer value. |
198 | llvm::Value *convertRValueToInt(RValue RVal) const; |
199 | |
200 | RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, |
201 | AggValueSlot ResultSlot, |
202 | SourceLocation Loc, bool AsValue) const; |
203 | |
204 | /// Copy an atomic r-value into atomic-layout memory. |
205 | void emitCopyIntoMemory(RValue rvalue) const; |
206 | |
207 | /// Project an l-value down to the value field. |
208 | LValue projectValue() const { |
209 | assert(LVal.isSimple()); |
210 | Address addr = getAtomicAddress(); |
211 | if (hasPadding()) |
212 | addr = CGF.Builder.CreateStructGEP(addr, 0); |
213 | |
214 | return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), |
215 | LVal.getBaseInfo(), LVal.getTBAAInfo()); |
216 | } |
217 | |
218 | /// Emits atomic load. |
219 | /// \returns Loaded value. |
220 | RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, |
221 | bool AsValue, llvm::AtomicOrdering AO, |
222 | bool IsVolatile); |
223 | |
224 | /// Emits atomic compare-and-exchange sequence. |
225 | /// \param Expected Expected value. |
226 | /// \param Desired Desired value. |
227 | /// \param Success Atomic ordering for success operation. |
228 | /// \param Failure Atomic ordering for failed operation. |
229 | /// \param IsWeak true if atomic operation is weak, false otherwise. |
230 | /// \returns Pair of values: previous value from storage (value type) and |
231 | /// boolean flag (i1 type) with true if success and false otherwise. |
232 | std::pair<RValue, llvm::Value *> |
233 | EmitAtomicCompareExchange(RValue Expected, RValue Desired, |
234 | llvm::AtomicOrdering Success = |
235 | llvm::AtomicOrdering::SequentiallyConsistent, |
236 | llvm::AtomicOrdering Failure = |
237 | llvm::AtomicOrdering::SequentiallyConsistent, |
238 | bool IsWeak = false); |
239 | |
240 | /// Emits atomic update. |
241 | /// \param AO Atomic ordering. |
242 | /// \param UpdateOp Update operation for the current lvalue. |
243 | void EmitAtomicUpdate(llvm::AtomicOrdering AO, |
244 | const llvm::function_ref<RValue(RValue)> &UpdateOp, |
245 | bool IsVolatile); |
246 | /// Emits atomic update. |
247 | /// \param AO Atomic ordering. |
248 | void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, |
249 | bool IsVolatile); |
250 | |
251 | /// Materialize an atomic r-value in atomic-layout memory. |
252 | Address materializeRValue(RValue rvalue) const; |
253 | |
254 | /// Creates temp alloca for intermediate operations on atomic value. |
255 | Address CreateTempAlloca() const; |
256 | private: |
257 | bool requiresMemSetZero(llvm::Type *type) const; |
258 | |
259 | |
260 | /// Emits atomic load as a libcall. |
261 | void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, |
262 | llvm::AtomicOrdering AO, bool IsVolatile); |
263 | /// Emits atomic load as LLVM instruction. |
264 | llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); |
265 | /// Emits atomic compare-and-exchange op as a libcall. |
266 | llvm::Value *EmitAtomicCompareExchangeLibcall( |
267 | llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, |
268 | llvm::AtomicOrdering Success = |
269 | llvm::AtomicOrdering::SequentiallyConsistent, |
270 | llvm::AtomicOrdering Failure = |
271 | llvm::AtomicOrdering::SequentiallyConsistent); |
272 | /// Emits atomic compare-and-exchange op as LLVM instruction. |
273 | std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( |
274 | llvm::Value *ExpectedVal, llvm::Value *DesiredVal, |
275 | llvm::AtomicOrdering Success = |
276 | llvm::AtomicOrdering::SequentiallyConsistent, |
277 | llvm::AtomicOrdering Failure = |
278 | llvm::AtomicOrdering::SequentiallyConsistent, |
279 | bool IsWeak = false); |
280 | /// Emit atomic update as libcalls. |
281 | void |
282 | EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, |
283 | const llvm::function_ref<RValue(RValue)> &UpdateOp, |
284 | bool IsVolatile); |
285 | /// Emit atomic update as LLVM instructions. |
286 | void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, |
287 | const llvm::function_ref<RValue(RValue)> &UpdateOp, |
288 | bool IsVolatile); |
289 | /// Emit atomic update as libcalls. |
290 | void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, |
291 | bool IsVolatile); |
292 | /// Emit atomic update as LLVM instructions. |
293 | void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, |
294 | bool IsVolatile); |
295 | }; |
296 | } |
297 | |
298 | Address AtomicInfo::CreateTempAlloca() const { |
299 | Address TempAlloca = CGF.CreateMemTemp( |
300 | (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy |
301 | : AtomicTy, |
302 | getAtomicAlignment(), |
303 | "atomic-temp" ); |
304 | // Cast to pointer to value type for bitfields. |
305 | if (LVal.isBitField()) |
306 | return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( |
307 | TempAlloca, getAtomicAddress().getType(), |
308 | getAtomicAddress().getElementType()); |
309 | return TempAlloca; |
310 | } |
311 | |
312 | static RValue emitAtomicLibcall(CodeGenFunction &CGF, |
313 | StringRef fnName, |
314 | QualType resultType, |
315 | CallArgList &args) { |
316 | const CGFunctionInfo &fnInfo = |
317 | CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); |
318 | llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); |
319 | llvm::AttrBuilder fnAttrB(CGF.getLLVMContext()); |
320 | fnAttrB.addAttribute(llvm::Attribute::NoUnwind); |
321 | fnAttrB.addAttribute(llvm::Attribute::WillReturn); |
322 | llvm::AttributeList fnAttrs = llvm::AttributeList::get( |
323 | CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB); |
324 | |
325 | llvm::FunctionCallee fn = |
326 | CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs); |
327 | auto callee = CGCallee::forDirect(fn); |
328 | return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); |
329 | } |
330 | |
331 | /// Does a store of the given IR type modify the full expected width? |
332 | static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, |
333 | uint64_t expectedSize) { |
334 | return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); |
335 | } |
336 | |
337 | /// Does the atomic type require memsetting to zero before initialization? |
338 | /// |
339 | /// The IR type is provided as a way of making certain queries faster. |
340 | bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { |
341 | // If the atomic type has size padding, we definitely need a memset. |
342 | if (hasPadding()) return true; |
343 | |
344 | // Otherwise, do some simple heuristics to try to avoid it: |
345 | switch (getEvaluationKind()) { |
346 | // For scalars and complexes, check whether the store size of the |
347 | // type uses the full size. |
348 | case TEK_Scalar: |
349 | return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); |
350 | case TEK_Complex: |
351 | return !isFullSizeType(CGF.CGM, type->getStructElementType(0), |
352 | AtomicSizeInBits / 2); |
353 | |
354 | // Padding in structs has an undefined bit pattern. User beware. |
355 | case TEK_Aggregate: |
356 | return false; |
357 | } |
358 | llvm_unreachable("bad evaluation kind" ); |
359 | } |
360 | |
361 | bool AtomicInfo::emitMemSetZeroIfNecessary() const { |
362 | assert(LVal.isSimple()); |
363 | Address addr = LVal.getAddress(CGF); |
364 | if (!requiresMemSetZero(addr.getElementType())) |
365 | return false; |
366 | |
367 | CGF.Builder.CreateMemSet( |
368 | addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0), |
369 | CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), |
370 | LVal.getAlignment().getAsAlign()); |
371 | return true; |
372 | } |
373 | |
374 | static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, |
375 | Address Dest, Address Ptr, |
376 | Address Val1, Address Val2, |
377 | uint64_t Size, |
378 | llvm::AtomicOrdering SuccessOrder, |
379 | llvm::AtomicOrdering FailureOrder, |
380 | llvm::SyncScope::ID Scope) { |
381 | // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. |
382 | llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); |
383 | llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); |
384 | |
385 | llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( |
386 | Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, |
387 | Scope); |
388 | Pair->setVolatile(E->isVolatile()); |
389 | Pair->setWeak(IsWeak); |
390 | |
391 | // Cmp holds the result of the compare-exchange operation: true on success, |
392 | // false on failure. |
393 | llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); |
394 | llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); |
395 | |
396 | // This basic block is used to hold the store instruction if the operation |
397 | // failed. |
398 | llvm::BasicBlock *StoreExpectedBB = |
399 | CGF.createBasicBlock("cmpxchg.store_expected" , CGF.CurFn); |
400 | |
401 | // This basic block is the exit point of the operation, we should end up |
402 | // here regardless of whether or not the operation succeeded. |
403 | llvm::BasicBlock *ContinueBB = |
404 | CGF.createBasicBlock("cmpxchg.continue" , CGF.CurFn); |
405 | |
406 | // Update Expected if Expected isn't equal to Old, otherwise branch to the |
407 | // exit point. |
408 | CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); |
409 | |
410 | CGF.Builder.SetInsertPoint(StoreExpectedBB); |
411 | // Update the memory at Expected with Old's value. |
412 | CGF.Builder.CreateStore(Old, Val1); |
413 | // Finally, branch to the exit point. |
414 | CGF.Builder.CreateBr(ContinueBB); |
415 | |
416 | CGF.Builder.SetInsertPoint(ContinueBB); |
417 | // Update the memory at Dest with Cmp's value. |
418 | CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); |
419 | } |
420 | |
421 | /// Given an ordering required on success, emit all possible cmpxchg |
422 | /// instructions to cope with the provided (but possibly only dynamically known) |
423 | /// FailureOrder. |
424 | static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, |
425 | bool IsWeak, Address Dest, Address Ptr, |
426 | Address Val1, Address Val2, |
427 | llvm::Value *FailureOrderVal, |
428 | uint64_t Size, |
429 | llvm::AtomicOrdering SuccessOrder, |
430 | llvm::SyncScope::ID Scope) { |
431 | llvm::AtomicOrdering FailureOrder; |
432 | if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { |
433 | auto FOS = FO->getSExtValue(); |
434 | if (!llvm::isValidAtomicOrderingCABI(FOS)) |
435 | FailureOrder = llvm::AtomicOrdering::Monotonic; |
436 | else |
437 | switch ((llvm::AtomicOrderingCABI)FOS) { |
438 | case llvm::AtomicOrderingCABI::relaxed: |
439 | // 31.7.2.18: "The failure argument shall not be memory_order_release |
440 | // nor memory_order_acq_rel". Fallback to monotonic. |
441 | case llvm::AtomicOrderingCABI::release: |
442 | case llvm::AtomicOrderingCABI::acq_rel: |
443 | FailureOrder = llvm::AtomicOrdering::Monotonic; |
444 | break; |
445 | case llvm::AtomicOrderingCABI::consume: |
446 | case llvm::AtomicOrderingCABI::acquire: |
447 | FailureOrder = llvm::AtomicOrdering::Acquire; |
448 | break; |
449 | case llvm::AtomicOrderingCABI::seq_cst: |
450 | FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; |
451 | break; |
452 | } |
453 | // Prior to c++17, "the failure argument shall be no stronger than the |
454 | // success argument". This condition has been lifted and the only |
455 | // precondition is 31.7.2.18. Effectively treat this as a DR and skip |
456 | // language version checks. |
457 | emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, |
458 | FailureOrder, Scope); |
459 | return; |
460 | } |
461 | |
462 | // Create all the relevant BB's |
463 | auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail" , CGF.CurFn); |
464 | auto *AcquireBB = CGF.createBasicBlock("acquire_fail" , CGF.CurFn); |
465 | auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail" , CGF.CurFn); |
466 | auto *ContBB = CGF.createBasicBlock("atomic.continue" , CGF.CurFn); |
467 | |
468 | // MonotonicBB is arbitrarily chosen as the default case; in practice, this |
469 | // doesn't matter unless someone is crazy enough to use something that |
470 | // doesn't fold to a constant for the ordering. |
471 | llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); |
472 | // Implemented as acquire, since it's the closest in LLVM. |
473 | SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), |
474 | AcquireBB); |
475 | SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), |
476 | AcquireBB); |
477 | SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), |
478 | SeqCstBB); |
479 | |
480 | // Emit all the different atomics |
481 | CGF.Builder.SetInsertPoint(MonotonicBB); |
482 | emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, |
483 | Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); |
484 | CGF.Builder.CreateBr(ContBB); |
485 | |
486 | CGF.Builder.SetInsertPoint(AcquireBB); |
487 | emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, |
488 | llvm::AtomicOrdering::Acquire, Scope); |
489 | CGF.Builder.CreateBr(ContBB); |
490 | |
491 | CGF.Builder.SetInsertPoint(SeqCstBB); |
492 | emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, |
493 | llvm::AtomicOrdering::SequentiallyConsistent, Scope); |
494 | CGF.Builder.CreateBr(ContBB); |
495 | |
496 | CGF.Builder.SetInsertPoint(ContBB); |
497 | } |
498 | |
499 | /// Duplicate the atomic min/max operation in conventional IR for the builtin |
500 | /// variants that return the new rather than the original value. |
501 | static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, |
502 | AtomicExpr::AtomicOp Op, |
503 | bool IsSigned, |
504 | llvm::Value *OldVal, |
505 | llvm::Value *RHS) { |
506 | llvm::CmpInst::Predicate Pred; |
507 | switch (Op) { |
508 | default: |
509 | llvm_unreachable("Unexpected min/max operation" ); |
510 | case AtomicExpr::AO__atomic_max_fetch: |
511 | Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; |
512 | break; |
513 | case AtomicExpr::AO__atomic_min_fetch: |
514 | Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; |
515 | break; |
516 | } |
517 | llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst" ); |
518 | return Builder.CreateSelect(Cmp, OldVal, RHS, "newval" ); |
519 | } |
520 | |
521 | static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, |
522 | Address Ptr, Address Val1, Address Val2, |
523 | llvm::Value *IsWeak, llvm::Value *FailureOrder, |
524 | uint64_t Size, llvm::AtomicOrdering Order, |
525 | llvm::SyncScope::ID Scope) { |
526 | llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; |
527 | bool PostOpMinMax = false; |
528 | unsigned PostOp = 0; |
529 | |
530 | switch (E->getOp()) { |
531 | case AtomicExpr::AO__c11_atomic_init: |
532 | case AtomicExpr::AO__opencl_atomic_init: |
533 | llvm_unreachable("Already handled!" ); |
534 | |
535 | case AtomicExpr::AO__c11_atomic_compare_exchange_strong: |
536 | case AtomicExpr::AO__hip_atomic_compare_exchange_strong: |
537 | case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: |
538 | emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, |
539 | FailureOrder, Size, Order, Scope); |
540 | return; |
541 | case AtomicExpr::AO__c11_atomic_compare_exchange_weak: |
542 | case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: |
543 | case AtomicExpr::AO__hip_atomic_compare_exchange_weak: |
544 | emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, |
545 | FailureOrder, Size, Order, Scope); |
546 | return; |
547 | case AtomicExpr::AO__atomic_compare_exchange: |
548 | case AtomicExpr::AO__atomic_compare_exchange_n: { |
549 | if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { |
550 | emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, |
551 | Val1, Val2, FailureOrder, Size, Order, Scope); |
552 | } else { |
553 | // Create all the relevant BB's |
554 | llvm::BasicBlock *StrongBB = |
555 | CGF.createBasicBlock("cmpxchg.strong" , CGF.CurFn); |
556 | llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak" , CGF.CurFn); |
557 | llvm::BasicBlock *ContBB = |
558 | CGF.createBasicBlock("cmpxchg.continue" , CGF.CurFn); |
559 | |
560 | llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); |
561 | SI->addCase(CGF.Builder.getInt1(false), StrongBB); |
562 | |
563 | CGF.Builder.SetInsertPoint(StrongBB); |
564 | emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, |
565 | FailureOrder, Size, Order, Scope); |
566 | CGF.Builder.CreateBr(ContBB); |
567 | |
568 | CGF.Builder.SetInsertPoint(WeakBB); |
569 | emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, |
570 | FailureOrder, Size, Order, Scope); |
571 | CGF.Builder.CreateBr(ContBB); |
572 | |
573 | CGF.Builder.SetInsertPoint(ContBB); |
574 | } |
575 | return; |
576 | } |
577 | case AtomicExpr::AO__c11_atomic_load: |
578 | case AtomicExpr::AO__opencl_atomic_load: |
579 | case AtomicExpr::AO__hip_atomic_load: |
580 | case AtomicExpr::AO__atomic_load_n: |
581 | case AtomicExpr::AO__atomic_load: { |
582 | llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); |
583 | Load->setAtomic(Order, Scope); |
584 | Load->setVolatile(E->isVolatile()); |
585 | CGF.Builder.CreateStore(Load, Dest); |
586 | return; |
587 | } |
588 | |
589 | case AtomicExpr::AO__c11_atomic_store: |
590 | case AtomicExpr::AO__opencl_atomic_store: |
591 | case AtomicExpr::AO__hip_atomic_store: |
592 | case AtomicExpr::AO__atomic_store: |
593 | case AtomicExpr::AO__atomic_store_n: { |
594 | llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); |
595 | llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); |
596 | Store->setAtomic(Order, Scope); |
597 | Store->setVolatile(E->isVolatile()); |
598 | return; |
599 | } |
600 | |
601 | case AtomicExpr::AO__c11_atomic_exchange: |
602 | case AtomicExpr::AO__hip_atomic_exchange: |
603 | case AtomicExpr::AO__opencl_atomic_exchange: |
604 | case AtomicExpr::AO__atomic_exchange_n: |
605 | case AtomicExpr::AO__atomic_exchange: |
606 | Op = llvm::AtomicRMWInst::Xchg; |
607 | break; |
608 | |
609 | case AtomicExpr::AO__atomic_add_fetch: |
610 | PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd |
611 | : llvm::Instruction::Add; |
612 | [[fallthrough]]; |
613 | case AtomicExpr::AO__c11_atomic_fetch_add: |
614 | case AtomicExpr::AO__hip_atomic_fetch_add: |
615 | case AtomicExpr::AO__opencl_atomic_fetch_add: |
616 | case AtomicExpr::AO__atomic_fetch_add: |
617 | Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd |
618 | : llvm::AtomicRMWInst::Add; |
619 | break; |
620 | |
621 | case AtomicExpr::AO__atomic_sub_fetch: |
622 | PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub |
623 | : llvm::Instruction::Sub; |
624 | [[fallthrough]]; |
625 | case AtomicExpr::AO__c11_atomic_fetch_sub: |
626 | case AtomicExpr::AO__opencl_atomic_fetch_sub: |
627 | case AtomicExpr::AO__atomic_fetch_sub: |
628 | Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub |
629 | : llvm::AtomicRMWInst::Sub; |
630 | break; |
631 | |
632 | case AtomicExpr::AO__atomic_min_fetch: |
633 | PostOpMinMax = true; |
634 | [[fallthrough]]; |
635 | case AtomicExpr::AO__c11_atomic_fetch_min: |
636 | case AtomicExpr::AO__hip_atomic_fetch_min: |
637 | case AtomicExpr::AO__opencl_atomic_fetch_min: |
638 | case AtomicExpr::AO__atomic_fetch_min: |
639 | Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min |
640 | : llvm::AtomicRMWInst::UMin; |
641 | break; |
642 | |
643 | case AtomicExpr::AO__atomic_max_fetch: |
644 | PostOpMinMax = true; |
645 | [[fallthrough]]; |
646 | case AtomicExpr::AO__c11_atomic_fetch_max: |
647 | case AtomicExpr::AO__hip_atomic_fetch_max: |
648 | case AtomicExpr::AO__opencl_atomic_fetch_max: |
649 | case AtomicExpr::AO__atomic_fetch_max: |
650 | Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max |
651 | : llvm::AtomicRMWInst::UMax; |
652 | break; |
653 | |
654 | case AtomicExpr::AO__atomic_and_fetch: |
655 | PostOp = llvm::Instruction::And; |
656 | [[fallthrough]]; |
657 | case AtomicExpr::AO__c11_atomic_fetch_and: |
658 | case AtomicExpr::AO__hip_atomic_fetch_and: |
659 | case AtomicExpr::AO__opencl_atomic_fetch_and: |
660 | case AtomicExpr::AO__atomic_fetch_and: |
661 | Op = llvm::AtomicRMWInst::And; |
662 | break; |
663 | |
664 | case AtomicExpr::AO__atomic_or_fetch: |
665 | PostOp = llvm::Instruction::Or; |
666 | [[fallthrough]]; |
667 | case AtomicExpr::AO__c11_atomic_fetch_or: |
668 | case AtomicExpr::AO__hip_atomic_fetch_or: |
669 | case AtomicExpr::AO__opencl_atomic_fetch_or: |
670 | case AtomicExpr::AO__atomic_fetch_or: |
671 | Op = llvm::AtomicRMWInst::Or; |
672 | break; |
673 | |
674 | case AtomicExpr::AO__atomic_xor_fetch: |
675 | PostOp = llvm::Instruction::Xor; |
676 | [[fallthrough]]; |
677 | case AtomicExpr::AO__c11_atomic_fetch_xor: |
678 | case AtomicExpr::AO__hip_atomic_fetch_xor: |
679 | case AtomicExpr::AO__opencl_atomic_fetch_xor: |
680 | case AtomicExpr::AO__atomic_fetch_xor: |
681 | Op = llvm::AtomicRMWInst::Xor; |
682 | break; |
683 | |
684 | case AtomicExpr::AO__atomic_nand_fetch: |
685 | PostOp = llvm::Instruction::And; // the NOT is special cased below |
686 | [[fallthrough]]; |
687 | case AtomicExpr::AO__c11_atomic_fetch_nand: |
688 | case AtomicExpr::AO__atomic_fetch_nand: |
689 | Op = llvm::AtomicRMWInst::Nand; |
690 | break; |
691 | } |
692 | |
693 | llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); |
694 | llvm::AtomicRMWInst *RMWI = |
695 | CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); |
696 | RMWI->setVolatile(E->isVolatile()); |
697 | |
698 | // For __atomic_*_fetch operations, perform the operation again to |
699 | // determine the value which was written. |
700 | llvm::Value *Result = RMWI; |
701 | if (PostOpMinMax) |
702 | Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), |
703 | E->getValueType()->isSignedIntegerType(), |
704 | RMWI, LoadVal1); |
705 | else if (PostOp) |
706 | Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, |
707 | LoadVal1); |
708 | if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) |
709 | Result = CGF.Builder.CreateNot(Result); |
710 | CGF.Builder.CreateStore(Result, Dest); |
711 | } |
712 | |
713 | // This function emits any expression (scalar, complex, or aggregate) |
714 | // into a temporary alloca. |
715 | static Address |
716 | EmitValToTemp(CodeGenFunction &CGF, Expr *E) { |
717 | Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp" ); |
718 | CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), |
719 | /*Init*/ true); |
720 | return DeclPtr; |
721 | } |
722 | |
723 | static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, |
724 | Address Ptr, Address Val1, Address Val2, |
725 | llvm::Value *IsWeak, llvm::Value *FailureOrder, |
726 | uint64_t Size, llvm::AtomicOrdering Order, |
727 | llvm::Value *Scope) { |
728 | auto ScopeModel = Expr->getScopeModel(); |
729 | |
730 | // LLVM atomic instructions always have synch scope. If clang atomic |
731 | // expression has no scope operand, use default LLVM synch scope. |
732 | if (!ScopeModel) { |
733 | EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, |
734 | Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("" )); |
735 | return; |
736 | } |
737 | |
738 | // Handle constant scope. |
739 | if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { |
740 | auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( |
741 | CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), |
742 | Order, CGF.CGM.getLLVMContext()); |
743 | EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, |
744 | Order, SCID); |
745 | return; |
746 | } |
747 | |
748 | // Handle non-constant scope. |
749 | auto &Builder = CGF.Builder; |
750 | auto Scopes = ScopeModel->getRuntimeValues(); |
751 | llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; |
752 | for (auto S : Scopes) |
753 | BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); |
754 | |
755 | llvm::BasicBlock *ContBB = |
756 | CGF.createBasicBlock("atomic.scope.continue" , CGF.CurFn); |
757 | |
758 | auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); |
759 | // If unsupported synch scope is encountered at run time, assume a fallback |
760 | // synch scope value. |
761 | auto FallBack = ScopeModel->getFallBackValue(); |
762 | llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); |
763 | for (auto S : Scopes) { |
764 | auto *B = BB[S]; |
765 | if (S != FallBack) |
766 | SI->addCase(Builder.getInt32(S), B); |
767 | |
768 | Builder.SetInsertPoint(B); |
769 | EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, |
770 | Order, |
771 | CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), |
772 | ScopeModel->map(S), |
773 | Order, |
774 | CGF.getLLVMContext())); |
775 | Builder.CreateBr(ContBB); |
776 | } |
777 | |
778 | Builder.SetInsertPoint(ContBB); |
779 | } |
780 | |
781 | static void |
782 | AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, |
783 | bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, |
784 | SourceLocation Loc, CharUnits SizeInChars) { |
785 | if (UseOptimizedLibcall) { |
786 | // Load value and pass it to the function directly. |
787 | CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); |
788 | int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); |
789 | ValTy = |
790 | CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); |
791 | llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits); |
792 | Address Ptr = Address(CGF.Builder.CreateBitCast(Val, ITy->getPointerTo()), |
793 | ITy, Align); |
794 | Val = CGF.EmitLoadOfScalar(Ptr, false, |
795 | CGF.getContext().getPointerType(ValTy), |
796 | Loc); |
797 | // Coerce the value into an appropriately sized integer type. |
798 | Args.add(RValue::get(Val), ValTy); |
799 | } else { |
800 | // Non-optimized functions always take a reference. |
801 | Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), |
802 | CGF.getContext().VoidPtrTy); |
803 | } |
804 | } |
805 | |
806 | RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { |
807 | QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); |
808 | QualType MemTy = AtomicTy; |
809 | if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) |
810 | MemTy = AT->getValueType(); |
811 | llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; |
812 | |
813 | Address Val1 = Address::invalid(); |
814 | Address Val2 = Address::invalid(); |
815 | Address Dest = Address::invalid(); |
816 | Address Ptr = EmitPointerWithAlignment(E->getPtr()); |
817 | |
818 | if (E->getOp() == AtomicExpr::AO__c11_atomic_init || |
819 | E->getOp() == AtomicExpr::AO__opencl_atomic_init) { |
820 | LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); |
821 | EmitAtomicInit(E->getVal1(), lvalue); |
822 | return RValue::get(nullptr); |
823 | } |
824 | |
825 | auto TInfo = getContext().getTypeInfoInChars(AtomicTy); |
826 | uint64_t Size = TInfo.Width.getQuantity(); |
827 | unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); |
828 | |
829 | bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits; |
830 | bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0; |
831 | bool UseLibcall = Misaligned | Oversized; |
832 | bool ShouldCastToIntPtrTy = true; |
833 | |
834 | CharUnits MaxInlineWidth = |
835 | getContext().toCharUnitsFromBits(MaxInlineWidthInBits); |
836 | |
837 | DiagnosticsEngine &Diags = CGM.getDiags(); |
838 | |
839 | if (Misaligned) { |
840 | Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) |
841 | << (int)TInfo.Width.getQuantity() |
842 | << (int)Ptr.getAlignment().getQuantity(); |
843 | } |
844 | |
845 | if (Oversized) { |
846 | Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized) |
847 | << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity(); |
848 | } |
849 | |
850 | llvm::Value *Order = EmitScalarExpr(E->getOrder()); |
851 | llvm::Value *Scope = |
852 | E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; |
853 | |
854 | switch (E->getOp()) { |
855 | case AtomicExpr::AO__c11_atomic_init: |
856 | case AtomicExpr::AO__opencl_atomic_init: |
857 | llvm_unreachable("Already handled above with EmitAtomicInit!" ); |
858 | |
859 | case AtomicExpr::AO__c11_atomic_load: |
860 | case AtomicExpr::AO__opencl_atomic_load: |
861 | case AtomicExpr::AO__hip_atomic_load: |
862 | case AtomicExpr::AO__atomic_load_n: |
863 | break; |
864 | |
865 | case AtomicExpr::AO__atomic_load: |
866 | Dest = EmitPointerWithAlignment(E->getVal1()); |
867 | break; |
868 | |
869 | case AtomicExpr::AO__atomic_store: |
870 | Val1 = EmitPointerWithAlignment(E->getVal1()); |
871 | break; |
872 | |
873 | case AtomicExpr::AO__atomic_exchange: |
874 | Val1 = EmitPointerWithAlignment(E->getVal1()); |
875 | Dest = EmitPointerWithAlignment(E->getVal2()); |
876 | break; |
877 | |
878 | case AtomicExpr::AO__c11_atomic_compare_exchange_strong: |
879 | case AtomicExpr::AO__c11_atomic_compare_exchange_weak: |
880 | case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: |
881 | case AtomicExpr::AO__hip_atomic_compare_exchange_strong: |
882 | case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: |
883 | case AtomicExpr::AO__hip_atomic_compare_exchange_weak: |
884 | case AtomicExpr::AO__atomic_compare_exchange_n: |
885 | case AtomicExpr::AO__atomic_compare_exchange: |
886 | Val1 = EmitPointerWithAlignment(E->getVal1()); |
887 | if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) |
888 | Val2 = EmitPointerWithAlignment(E->getVal2()); |
889 | else |
890 | Val2 = EmitValToTemp(*this, E->getVal2()); |
891 | OrderFail = EmitScalarExpr(E->getOrderFail()); |
892 | if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || |
893 | E->getOp() == AtomicExpr::AO__atomic_compare_exchange) |
894 | IsWeak = EmitScalarExpr(E->getWeak()); |
895 | break; |
896 | |
897 | case AtomicExpr::AO__c11_atomic_fetch_add: |
898 | case AtomicExpr::AO__c11_atomic_fetch_sub: |
899 | case AtomicExpr::AO__hip_atomic_fetch_add: |
900 | case AtomicExpr::AO__opencl_atomic_fetch_add: |
901 | case AtomicExpr::AO__opencl_atomic_fetch_sub: |
902 | if (MemTy->isPointerType()) { |
903 | // For pointer arithmetic, we're required to do a bit of math: |
904 | // adding 1 to an int* is not the same as adding 1 to a uintptr_t. |
905 | // ... but only for the C11 builtins. The GNU builtins expect the |
906 | // user to multiply by sizeof(T). |
907 | QualType Val1Ty = E->getVal1()->getType(); |
908 | llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); |
909 | CharUnits PointeeIncAmt = |
910 | getContext().getTypeSizeInChars(MemTy->getPointeeType()); |
911 | Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); |
912 | auto Temp = CreateMemTemp(Val1Ty, ".atomictmp" ); |
913 | Val1 = Temp; |
914 | EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); |
915 | break; |
916 | } |
917 | [[fallthrough]]; |
918 | case AtomicExpr::AO__atomic_fetch_add: |
919 | case AtomicExpr::AO__atomic_fetch_sub: |
920 | case AtomicExpr::AO__atomic_add_fetch: |
921 | case AtomicExpr::AO__atomic_sub_fetch: |
922 | ShouldCastToIntPtrTy = !MemTy->isFloatingType(); |
923 | [[fallthrough]]; |
924 | |
925 | case AtomicExpr::AO__c11_atomic_store: |
926 | case AtomicExpr::AO__c11_atomic_exchange: |
927 | case AtomicExpr::AO__opencl_atomic_store: |
928 | case AtomicExpr::AO__hip_atomic_store: |
929 | case AtomicExpr::AO__opencl_atomic_exchange: |
930 | case AtomicExpr::AO__hip_atomic_exchange: |
931 | case AtomicExpr::AO__atomic_store_n: |
932 | case AtomicExpr::AO__atomic_exchange_n: |
933 | case AtomicExpr::AO__c11_atomic_fetch_and: |
934 | case AtomicExpr::AO__c11_atomic_fetch_or: |
935 | case AtomicExpr::AO__c11_atomic_fetch_xor: |
936 | case AtomicExpr::AO__c11_atomic_fetch_nand: |
937 | case AtomicExpr::AO__c11_atomic_fetch_max: |
938 | case AtomicExpr::AO__c11_atomic_fetch_min: |
939 | case AtomicExpr::AO__opencl_atomic_fetch_and: |
940 | case AtomicExpr::AO__opencl_atomic_fetch_or: |
941 | case AtomicExpr::AO__opencl_atomic_fetch_xor: |
942 | case AtomicExpr::AO__opencl_atomic_fetch_min: |
943 | case AtomicExpr::AO__opencl_atomic_fetch_max: |
944 | case AtomicExpr::AO__atomic_fetch_and: |
945 | case AtomicExpr::AO__hip_atomic_fetch_and: |
946 | case AtomicExpr::AO__atomic_fetch_or: |
947 | case AtomicExpr::AO__hip_atomic_fetch_or: |
948 | case AtomicExpr::AO__atomic_fetch_xor: |
949 | case AtomicExpr::AO__hip_atomic_fetch_xor: |
950 | case AtomicExpr::AO__atomic_fetch_nand: |
951 | case AtomicExpr::AO__atomic_and_fetch: |
952 | case AtomicExpr::AO__atomic_or_fetch: |
953 | case AtomicExpr::AO__atomic_xor_fetch: |
954 | case AtomicExpr::AO__atomic_nand_fetch: |
955 | case AtomicExpr::AO__atomic_max_fetch: |
956 | case AtomicExpr::AO__atomic_min_fetch: |
957 | case AtomicExpr::AO__atomic_fetch_max: |
958 | case AtomicExpr::AO__hip_atomic_fetch_max: |
959 | case AtomicExpr::AO__atomic_fetch_min: |
960 | case AtomicExpr::AO__hip_atomic_fetch_min: |
961 | Val1 = EmitValToTemp(*this, E->getVal1()); |
962 | break; |
963 | } |
964 | |
965 | QualType RValTy = E->getType().getUnqualifiedType(); |
966 | |
967 | // The inlined atomics only function on iN types, where N is a power of 2. We |
968 | // need to make sure (via temporaries if necessary) that all incoming values |
969 | // are compatible. |
970 | LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); |
971 | AtomicInfo Atomics(*this, AtomicVal); |
972 | |
973 | if (ShouldCastToIntPtrTy) { |
974 | Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); |
975 | if (Val1.isValid()) |
976 | Val1 = Atomics.convertToAtomicIntPointer(Val1); |
977 | if (Val2.isValid()) |
978 | Val2 = Atomics.convertToAtomicIntPointer(Val2); |
979 | } |
980 | if (Dest.isValid()) { |
981 | if (ShouldCastToIntPtrTy) |
982 | Dest = Atomics.emitCastToAtomicIntPointer(Dest); |
983 | } else if (E->isCmpXChg()) |
984 | Dest = CreateMemTemp(RValTy, "cmpxchg.bool" ); |
985 | else if (!RValTy->isVoidType()) { |
986 | Dest = Atomics.CreateTempAlloca(); |
987 | if (ShouldCastToIntPtrTy) |
988 | Dest = Atomics.emitCastToAtomicIntPointer(Dest); |
989 | } |
990 | |
991 | // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . |
992 | if (UseLibcall) { |
993 | bool UseOptimizedLibcall = false; |
994 | switch (E->getOp()) { |
995 | case AtomicExpr::AO__c11_atomic_init: |
996 | case AtomicExpr::AO__opencl_atomic_init: |
997 | llvm_unreachable("Already handled above with EmitAtomicInit!" ); |
998 | |
999 | case AtomicExpr::AO__c11_atomic_fetch_add: |
1000 | case AtomicExpr::AO__opencl_atomic_fetch_add: |
1001 | case AtomicExpr::AO__atomic_fetch_add: |
1002 | case AtomicExpr::AO__hip_atomic_fetch_add: |
1003 | case AtomicExpr::AO__c11_atomic_fetch_and: |
1004 | case AtomicExpr::AO__opencl_atomic_fetch_and: |
1005 | case AtomicExpr::AO__hip_atomic_fetch_and: |
1006 | case AtomicExpr::AO__atomic_fetch_and: |
1007 | case AtomicExpr::AO__c11_atomic_fetch_or: |
1008 | case AtomicExpr::AO__opencl_atomic_fetch_or: |
1009 | case AtomicExpr::AO__hip_atomic_fetch_or: |
1010 | case AtomicExpr::AO__atomic_fetch_or: |
1011 | case AtomicExpr::AO__c11_atomic_fetch_nand: |
1012 | case AtomicExpr::AO__atomic_fetch_nand: |
1013 | case AtomicExpr::AO__c11_atomic_fetch_sub: |
1014 | case AtomicExpr::AO__opencl_atomic_fetch_sub: |
1015 | case AtomicExpr::AO__atomic_fetch_sub: |
1016 | case AtomicExpr::AO__c11_atomic_fetch_xor: |
1017 | case AtomicExpr::AO__opencl_atomic_fetch_xor: |
1018 | case AtomicExpr::AO__opencl_atomic_fetch_min: |
1019 | case AtomicExpr::AO__opencl_atomic_fetch_max: |
1020 | case AtomicExpr::AO__atomic_fetch_xor: |
1021 | case AtomicExpr::AO__hip_atomic_fetch_xor: |
1022 | case AtomicExpr::AO__c11_atomic_fetch_max: |
1023 | case AtomicExpr::AO__c11_atomic_fetch_min: |
1024 | case AtomicExpr::AO__atomic_add_fetch: |
1025 | case AtomicExpr::AO__atomic_and_fetch: |
1026 | case AtomicExpr::AO__atomic_nand_fetch: |
1027 | case AtomicExpr::AO__atomic_or_fetch: |
1028 | case AtomicExpr::AO__atomic_sub_fetch: |
1029 | case AtomicExpr::AO__atomic_xor_fetch: |
1030 | case AtomicExpr::AO__atomic_fetch_max: |
1031 | case AtomicExpr::AO__hip_atomic_fetch_max: |
1032 | case AtomicExpr::AO__atomic_fetch_min: |
1033 | case AtomicExpr::AO__hip_atomic_fetch_min: |
1034 | case AtomicExpr::AO__atomic_max_fetch: |
1035 | case AtomicExpr::AO__atomic_min_fetch: |
1036 | // For these, only library calls for certain sizes exist. |
1037 | UseOptimizedLibcall = true; |
1038 | break; |
1039 | |
1040 | case AtomicExpr::AO__atomic_load: |
1041 | case AtomicExpr::AO__atomic_store: |
1042 | case AtomicExpr::AO__atomic_exchange: |
1043 | case AtomicExpr::AO__atomic_compare_exchange: |
1044 | // Use the generic version if we don't know that the operand will be |
1045 | // suitably aligned for the optimized version. |
1046 | if (Misaligned) |
1047 | break; |
1048 | [[fallthrough]]; |
1049 | case AtomicExpr::AO__c11_atomic_load: |
1050 | case AtomicExpr::AO__c11_atomic_store: |
1051 | case AtomicExpr::AO__c11_atomic_exchange: |
1052 | case AtomicExpr::AO__c11_atomic_compare_exchange_weak: |
1053 | case AtomicExpr::AO__c11_atomic_compare_exchange_strong: |
1054 | case AtomicExpr::AO__hip_atomic_compare_exchange_strong: |
1055 | case AtomicExpr::AO__opencl_atomic_load: |
1056 | case AtomicExpr::AO__hip_atomic_load: |
1057 | case AtomicExpr::AO__opencl_atomic_store: |
1058 | case AtomicExpr::AO__hip_atomic_store: |
1059 | case AtomicExpr::AO__opencl_atomic_exchange: |
1060 | case AtomicExpr::AO__hip_atomic_exchange: |
1061 | case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: |
1062 | case AtomicExpr::AO__hip_atomic_compare_exchange_weak: |
1063 | case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: |
1064 | case AtomicExpr::AO__atomic_load_n: |
1065 | case AtomicExpr::AO__atomic_store_n: |
1066 | case AtomicExpr::AO__atomic_exchange_n: |
1067 | case AtomicExpr::AO__atomic_compare_exchange_n: |
1068 | // Only use optimized library calls for sizes for which they exist. |
1069 | // FIXME: Size == 16 optimized library functions exist too. |
1070 | if (Size == 1 || Size == 2 || Size == 4 || Size == 8) |
1071 | UseOptimizedLibcall = true; |
1072 | break; |
1073 | } |
1074 | |
1075 | CallArgList Args; |
1076 | if (!UseOptimizedLibcall) { |
1077 | // For non-optimized library calls, the size is the first parameter |
1078 | Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), |
1079 | getContext().getSizeType()); |
1080 | } |
1081 | // Atomic address is the first or second parameter |
1082 | // The OpenCL atomic library functions only accept pointer arguments to |
1083 | // generic address space. |
1084 | auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { |
1085 | if (!E->isOpenCL()) |
1086 | return V; |
1087 | auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace(); |
1088 | if (AS == LangAS::opencl_generic) |
1089 | return V; |
1090 | auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); |
1091 | auto T = llvm::cast<llvm::PointerType>(V->getType()); |
1092 | auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS); |
1093 | |
1094 | return getTargetHooks().performAddrSpaceCast( |
1095 | *this, V, AS, LangAS::opencl_generic, DestType, false); |
1096 | }; |
1097 | |
1098 | Args.add(RValue::get(CastToGenericAddrSpace( |
1099 | EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), |
1100 | getContext().VoidPtrTy); |
1101 | |
1102 | std::string LibCallName; |
1103 | QualType LoweredMemTy = |
1104 | MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; |
1105 | QualType RetTy; |
1106 | bool HaveRetTy = false; |
1107 | llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; |
1108 | bool PostOpMinMax = false; |
1109 | switch (E->getOp()) { |
1110 | case AtomicExpr::AO__c11_atomic_init: |
1111 | case AtomicExpr::AO__opencl_atomic_init: |
1112 | llvm_unreachable("Already handled!" ); |
1113 | |
1114 | // There is only one libcall for compare an exchange, because there is no |
1115 | // optimisation benefit possible from a libcall version of a weak compare |
1116 | // and exchange. |
1117 | // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, |
1118 | // void *desired, int success, int failure) |
1119 | // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, |
1120 | // int success, int failure) |
1121 | case AtomicExpr::AO__c11_atomic_compare_exchange_weak: |
1122 | case AtomicExpr::AO__c11_atomic_compare_exchange_strong: |
1123 | case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: |
1124 | case AtomicExpr::AO__hip_atomic_compare_exchange_weak: |
1125 | case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: |
1126 | case AtomicExpr::AO__hip_atomic_compare_exchange_strong: |
1127 | case AtomicExpr::AO__atomic_compare_exchange: |
1128 | case AtomicExpr::AO__atomic_compare_exchange_n: |
1129 | LibCallName = "__atomic_compare_exchange" ; |
1130 | RetTy = getContext().BoolTy; |
1131 | HaveRetTy = true; |
1132 | Args.add( |
1133 | RValue::get(CastToGenericAddrSpace( |
1134 | EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), |
1135 | getContext().VoidPtrTy); |
1136 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), |
1137 | MemTy, E->getExprLoc(), TInfo.Width); |
1138 | Args.add(RValue::get(Order), getContext().IntTy); |
1139 | Order = OrderFail; |
1140 | break; |
1141 | // void __atomic_exchange(size_t size, void *mem, void *val, void *return, |
1142 | // int order) |
1143 | // T __atomic_exchange_N(T *mem, T val, int order) |
1144 | case AtomicExpr::AO__c11_atomic_exchange: |
1145 | case AtomicExpr::AO__opencl_atomic_exchange: |
1146 | case AtomicExpr::AO__atomic_exchange_n: |
1147 | case AtomicExpr::AO__atomic_exchange: |
1148 | case AtomicExpr::AO__hip_atomic_exchange: |
1149 | LibCallName = "__atomic_exchange" ; |
1150 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1151 | MemTy, E->getExprLoc(), TInfo.Width); |
1152 | break; |
1153 | // void __atomic_store(size_t size, void *mem, void *val, int order) |
1154 | // void __atomic_store_N(T *mem, T val, int order) |
1155 | case AtomicExpr::AO__c11_atomic_store: |
1156 | case AtomicExpr::AO__opencl_atomic_store: |
1157 | case AtomicExpr::AO__hip_atomic_store: |
1158 | case AtomicExpr::AO__atomic_store: |
1159 | case AtomicExpr::AO__atomic_store_n: |
1160 | LibCallName = "__atomic_store" ; |
1161 | RetTy = getContext().VoidTy; |
1162 | HaveRetTy = true; |
1163 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1164 | MemTy, E->getExprLoc(), TInfo.Width); |
1165 | break; |
1166 | // void __atomic_load(size_t size, void *mem, void *return, int order) |
1167 | // T __atomic_load_N(T *mem, int order) |
1168 | case AtomicExpr::AO__c11_atomic_load: |
1169 | case AtomicExpr::AO__opencl_atomic_load: |
1170 | case AtomicExpr::AO__hip_atomic_load: |
1171 | case AtomicExpr::AO__atomic_load: |
1172 | case AtomicExpr::AO__atomic_load_n: |
1173 | LibCallName = "__atomic_load" ; |
1174 | break; |
1175 | // T __atomic_add_fetch_N(T *mem, T val, int order) |
1176 | // T __atomic_fetch_add_N(T *mem, T val, int order) |
1177 | case AtomicExpr::AO__atomic_add_fetch: |
1178 | PostOp = llvm::Instruction::Add; |
1179 | [[fallthrough]]; |
1180 | case AtomicExpr::AO__c11_atomic_fetch_add: |
1181 | case AtomicExpr::AO__opencl_atomic_fetch_add: |
1182 | case AtomicExpr::AO__atomic_fetch_add: |
1183 | case AtomicExpr::AO__hip_atomic_fetch_add: |
1184 | LibCallName = "__atomic_fetch_add" ; |
1185 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1186 | LoweredMemTy, E->getExprLoc(), TInfo.Width); |
1187 | break; |
1188 | // T __atomic_and_fetch_N(T *mem, T val, int order) |
1189 | // T __atomic_fetch_and_N(T *mem, T val, int order) |
1190 | case AtomicExpr::AO__atomic_and_fetch: |
1191 | PostOp = llvm::Instruction::And; |
1192 | [[fallthrough]]; |
1193 | case AtomicExpr::AO__c11_atomic_fetch_and: |
1194 | case AtomicExpr::AO__opencl_atomic_fetch_and: |
1195 | case AtomicExpr::AO__hip_atomic_fetch_and: |
1196 | case AtomicExpr::AO__atomic_fetch_and: |
1197 | LibCallName = "__atomic_fetch_and" ; |
1198 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1199 | MemTy, E->getExprLoc(), TInfo.Width); |
1200 | break; |
1201 | // T __atomic_or_fetch_N(T *mem, T val, int order) |
1202 | // T __atomic_fetch_or_N(T *mem, T val, int order) |
1203 | case AtomicExpr::AO__atomic_or_fetch: |
1204 | PostOp = llvm::Instruction::Or; |
1205 | [[fallthrough]]; |
1206 | case AtomicExpr::AO__c11_atomic_fetch_or: |
1207 | case AtomicExpr::AO__opencl_atomic_fetch_or: |
1208 | case AtomicExpr::AO__hip_atomic_fetch_or: |
1209 | case AtomicExpr::AO__atomic_fetch_or: |
1210 | LibCallName = "__atomic_fetch_or" ; |
1211 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1212 | MemTy, E->getExprLoc(), TInfo.Width); |
1213 | break; |
1214 | // T __atomic_sub_fetch_N(T *mem, T val, int order) |
1215 | // T __atomic_fetch_sub_N(T *mem, T val, int order) |
1216 | case AtomicExpr::AO__atomic_sub_fetch: |
1217 | PostOp = llvm::Instruction::Sub; |
1218 | [[fallthrough]]; |
1219 | case AtomicExpr::AO__c11_atomic_fetch_sub: |
1220 | case AtomicExpr::AO__opencl_atomic_fetch_sub: |
1221 | case AtomicExpr::AO__atomic_fetch_sub: |
1222 | LibCallName = "__atomic_fetch_sub" ; |
1223 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1224 | LoweredMemTy, E->getExprLoc(), TInfo.Width); |
1225 | break; |
1226 | // T __atomic_xor_fetch_N(T *mem, T val, int order) |
1227 | // T __atomic_fetch_xor_N(T *mem, T val, int order) |
1228 | case AtomicExpr::AO__atomic_xor_fetch: |
1229 | PostOp = llvm::Instruction::Xor; |
1230 | [[fallthrough]]; |
1231 | case AtomicExpr::AO__c11_atomic_fetch_xor: |
1232 | case AtomicExpr::AO__opencl_atomic_fetch_xor: |
1233 | case AtomicExpr::AO__hip_atomic_fetch_xor: |
1234 | case AtomicExpr::AO__atomic_fetch_xor: |
1235 | LibCallName = "__atomic_fetch_xor" ; |
1236 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1237 | MemTy, E->getExprLoc(), TInfo.Width); |
1238 | break; |
1239 | case AtomicExpr::AO__atomic_min_fetch: |
1240 | PostOpMinMax = true; |
1241 | [[fallthrough]]; |
1242 | case AtomicExpr::AO__c11_atomic_fetch_min: |
1243 | case AtomicExpr::AO__atomic_fetch_min: |
1244 | case AtomicExpr::AO__hip_atomic_fetch_min: |
1245 | case AtomicExpr::AO__opencl_atomic_fetch_min: |
1246 | LibCallName = E->getValueType()->isSignedIntegerType() |
1247 | ? "__atomic_fetch_min" |
1248 | : "__atomic_fetch_umin" ; |
1249 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1250 | LoweredMemTy, E->getExprLoc(), TInfo.Width); |
1251 | break; |
1252 | case AtomicExpr::AO__atomic_max_fetch: |
1253 | PostOpMinMax = true; |
1254 | [[fallthrough]]; |
1255 | case AtomicExpr::AO__c11_atomic_fetch_max: |
1256 | case AtomicExpr::AO__atomic_fetch_max: |
1257 | case AtomicExpr::AO__hip_atomic_fetch_max: |
1258 | case AtomicExpr::AO__opencl_atomic_fetch_max: |
1259 | LibCallName = E->getValueType()->isSignedIntegerType() |
1260 | ? "__atomic_fetch_max" |
1261 | : "__atomic_fetch_umax" ; |
1262 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1263 | LoweredMemTy, E->getExprLoc(), TInfo.Width); |
1264 | break; |
1265 | // T __atomic_nand_fetch_N(T *mem, T val, int order) |
1266 | // T __atomic_fetch_nand_N(T *mem, T val, int order) |
1267 | case AtomicExpr::AO__atomic_nand_fetch: |
1268 | PostOp = llvm::Instruction::And; // the NOT is special cased below |
1269 | [[fallthrough]]; |
1270 | case AtomicExpr::AO__c11_atomic_fetch_nand: |
1271 | case AtomicExpr::AO__atomic_fetch_nand: |
1272 | LibCallName = "__atomic_fetch_nand" ; |
1273 | AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), |
1274 | MemTy, E->getExprLoc(), TInfo.Width); |
1275 | break; |
1276 | } |
1277 | |
1278 | if (E->isOpenCL()) { |
1279 | LibCallName = std::string("__opencl" ) + |
1280 | StringRef(LibCallName).drop_front(1).str(); |
1281 | |
1282 | } |
1283 | // Optimized functions have the size in their name. |
1284 | if (UseOptimizedLibcall) |
1285 | LibCallName += "_" + llvm::utostr(Size); |
1286 | // By default, assume we return a value of the atomic type. |
1287 | if (!HaveRetTy) { |
1288 | if (UseOptimizedLibcall) { |
1289 | // Value is returned directly. |
1290 | // The function returns an appropriately sized integer type. |
1291 | RetTy = getContext().getIntTypeForBitwidth( |
1292 | getContext().toBits(TInfo.Width), /*Signed=*/false); |
1293 | } else { |
1294 | // Value is returned through parameter before the order. |
1295 | RetTy = getContext().VoidTy; |
1296 | Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), |
1297 | getContext().VoidPtrTy); |
1298 | } |
1299 | } |
1300 | // order is always the last parameter |
1301 | Args.add(RValue::get(Order), |
1302 | getContext().IntTy); |
1303 | if (E->isOpenCL()) |
1304 | Args.add(RValue::get(Scope), getContext().IntTy); |
1305 | |
1306 | // PostOp is only needed for the atomic_*_fetch operations, and |
1307 | // thus is only needed for and implemented in the |
1308 | // UseOptimizedLibcall codepath. |
1309 | assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax)); |
1310 | |
1311 | RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); |
1312 | // The value is returned directly from the libcall. |
1313 | if (E->isCmpXChg()) |
1314 | return Res; |
1315 | |
1316 | // The value is returned directly for optimized libcalls but the expr |
1317 | // provided an out-param. |
1318 | if (UseOptimizedLibcall && Res.getScalarVal()) { |
1319 | llvm::Value *ResVal = Res.getScalarVal(); |
1320 | if (PostOpMinMax) { |
1321 | llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); |
1322 | ResVal = EmitPostAtomicMinMax(Builder, E->getOp(), |
1323 | E->getValueType()->isSignedIntegerType(), |
1324 | ResVal, LoadVal1); |
1325 | } else if (PostOp) { |
1326 | llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); |
1327 | ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); |
1328 | } |
1329 | if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) |
1330 | ResVal = Builder.CreateNot(ResVal); |
1331 | |
1332 | Builder.CreateStore( |
1333 | ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType())); |
1334 | } |
1335 | |
1336 | if (RValTy->isVoidType()) |
1337 | return RValue::get(nullptr); |
1338 | |
1339 | return convertTempToRValue( |
1340 | Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), |
1341 | RValTy, E->getExprLoc()); |
1342 | } |
1343 | |
1344 | bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || |
1345 | E->getOp() == AtomicExpr::AO__opencl_atomic_store || |
1346 | E->getOp() == AtomicExpr::AO__hip_atomic_store || |
1347 | E->getOp() == AtomicExpr::AO__atomic_store || |
1348 | E->getOp() == AtomicExpr::AO__atomic_store_n; |
1349 | bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || |
1350 | E->getOp() == AtomicExpr::AO__opencl_atomic_load || |
1351 | E->getOp() == AtomicExpr::AO__hip_atomic_load || |
1352 | E->getOp() == AtomicExpr::AO__atomic_load || |
1353 | E->getOp() == AtomicExpr::AO__atomic_load_n; |
1354 | |
1355 | if (isa<llvm::ConstantInt>(Order)) { |
1356 | auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); |
1357 | // We should not ever get to a case where the ordering isn't a valid C ABI |
1358 | // value, but it's hard to enforce that in general. |
1359 | if (llvm::isValidAtomicOrderingCABI(ord)) |
1360 | switch ((llvm::AtomicOrderingCABI)ord) { |
1361 | case llvm::AtomicOrderingCABI::relaxed: |
1362 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1363 | llvm::AtomicOrdering::Monotonic, Scope); |
1364 | break; |
1365 | case llvm::AtomicOrderingCABI::consume: |
1366 | case llvm::AtomicOrderingCABI::acquire: |
1367 | if (IsStore) |
1368 | break; // Avoid crashing on code with undefined behavior |
1369 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1370 | llvm::AtomicOrdering::Acquire, Scope); |
1371 | break; |
1372 | case llvm::AtomicOrderingCABI::release: |
1373 | if (IsLoad) |
1374 | break; // Avoid crashing on code with undefined behavior |
1375 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1376 | llvm::AtomicOrdering::Release, Scope); |
1377 | break; |
1378 | case llvm::AtomicOrderingCABI::acq_rel: |
1379 | if (IsLoad || IsStore) |
1380 | break; // Avoid crashing on code with undefined behavior |
1381 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1382 | llvm::AtomicOrdering::AcquireRelease, Scope); |
1383 | break; |
1384 | case llvm::AtomicOrderingCABI::seq_cst: |
1385 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1386 | llvm::AtomicOrdering::SequentiallyConsistent, Scope); |
1387 | break; |
1388 | } |
1389 | if (RValTy->isVoidType()) |
1390 | return RValue::get(nullptr); |
1391 | |
1392 | return convertTempToRValue( |
1393 | Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), |
1394 | RValTy, E->getExprLoc()); |
1395 | } |
1396 | |
1397 | // Long case, when Order isn't obviously constant. |
1398 | |
1399 | // Create all the relevant BB's |
1400 | llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, |
1401 | *ReleaseBB = nullptr, *AcqRelBB = nullptr, |
1402 | *SeqCstBB = nullptr; |
1403 | MonotonicBB = createBasicBlock("monotonic" , CurFn); |
1404 | if (!IsStore) |
1405 | AcquireBB = createBasicBlock("acquire" , CurFn); |
1406 | if (!IsLoad) |
1407 | ReleaseBB = createBasicBlock("release" , CurFn); |
1408 | if (!IsLoad && !IsStore) |
1409 | AcqRelBB = createBasicBlock("acqrel" , CurFn); |
1410 | SeqCstBB = createBasicBlock("seqcst" , CurFn); |
1411 | llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue" , CurFn); |
1412 | |
1413 | // Create the switch for the split |
1414 | // MonotonicBB is arbitrarily chosen as the default case; in practice, this |
1415 | // doesn't matter unless someone is crazy enough to use something that |
1416 | // doesn't fold to a constant for the ordering. |
1417 | Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); |
1418 | llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); |
1419 | |
1420 | // Emit all the different atomics |
1421 | Builder.SetInsertPoint(MonotonicBB); |
1422 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1423 | llvm::AtomicOrdering::Monotonic, Scope); |
1424 | Builder.CreateBr(ContBB); |
1425 | if (!IsStore) { |
1426 | Builder.SetInsertPoint(AcquireBB); |
1427 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1428 | llvm::AtomicOrdering::Acquire, Scope); |
1429 | Builder.CreateBr(ContBB); |
1430 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), |
1431 | AcquireBB); |
1432 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), |
1433 | AcquireBB); |
1434 | } |
1435 | if (!IsLoad) { |
1436 | Builder.SetInsertPoint(ReleaseBB); |
1437 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1438 | llvm::AtomicOrdering::Release, Scope); |
1439 | Builder.CreateBr(ContBB); |
1440 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), |
1441 | ReleaseBB); |
1442 | } |
1443 | if (!IsLoad && !IsStore) { |
1444 | Builder.SetInsertPoint(AcqRelBB); |
1445 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1446 | llvm::AtomicOrdering::AcquireRelease, Scope); |
1447 | Builder.CreateBr(ContBB); |
1448 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), |
1449 | AcqRelBB); |
1450 | } |
1451 | Builder.SetInsertPoint(SeqCstBB); |
1452 | EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, |
1453 | llvm::AtomicOrdering::SequentiallyConsistent, Scope); |
1454 | Builder.CreateBr(ContBB); |
1455 | SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), |
1456 | SeqCstBB); |
1457 | |
1458 | // Cleanup and return |
1459 | Builder.SetInsertPoint(ContBB); |
1460 | if (RValTy->isVoidType()) |
1461 | return RValue::get(nullptr); |
1462 | |
1463 | assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); |
1464 | return convertTempToRValue( |
1465 | Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)), |
1466 | RValTy, E->getExprLoc()); |
1467 | } |
1468 | |
1469 | Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { |
1470 | llvm::IntegerType *ty = |
1471 | llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); |
1472 | return CGF.Builder.CreateElementBitCast(addr, ty); |
1473 | } |
1474 | |
1475 | Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { |
1476 | llvm::Type *Ty = Addr.getElementType(); |
1477 | uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); |
1478 | if (SourceSizeInBits != AtomicSizeInBits) { |
1479 | Address Tmp = CreateTempAlloca(); |
1480 | CGF.Builder.CreateMemCpy(Tmp, Addr, |
1481 | std::min(AtomicSizeInBits, SourceSizeInBits) / 8); |
1482 | Addr = Tmp; |
1483 | } |
1484 | |
1485 | return emitCastToAtomicIntPointer(Addr); |
1486 | } |
1487 | |
1488 | RValue AtomicInfo::convertAtomicTempToRValue(Address addr, |
1489 | AggValueSlot resultSlot, |
1490 | SourceLocation loc, |
1491 | bool asValue) const { |
1492 | if (LVal.isSimple()) { |
1493 | if (EvaluationKind == TEK_Aggregate) |
1494 | return resultSlot.asRValue(); |
1495 | |
1496 | // Drill into the padding structure if we have one. |
1497 | if (hasPadding()) |
1498 | addr = CGF.Builder.CreateStructGEP(addr, 0); |
1499 | |
1500 | // Otherwise, just convert the temporary to an r-value using the |
1501 | // normal conversion routine. |
1502 | return CGF.convertTempToRValue(addr, getValueType(), loc); |
1503 | } |
1504 | if (!asValue) |
1505 | // Get RValue from temp memory as atomic for non-simple lvalues |
1506 | return RValue::get(CGF.Builder.CreateLoad(addr)); |
1507 | if (LVal.isBitField()) |
1508 | return CGF.EmitLoadOfBitfieldLValue( |
1509 | LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), |
1510 | LVal.getBaseInfo(), TBAAAccessInfo()), loc); |
1511 | if (LVal.isVectorElt()) |
1512 | return CGF.EmitLoadOfLValue( |
1513 | LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), |
1514 | LVal.getBaseInfo(), TBAAAccessInfo()), loc); |
1515 | assert(LVal.isExtVectorElt()); |
1516 | return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( |
1517 | addr, LVal.getExtVectorElts(), LVal.getType(), |
1518 | LVal.getBaseInfo(), TBAAAccessInfo())); |
1519 | } |
1520 | |
1521 | RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, |
1522 | AggValueSlot ResultSlot, |
1523 | SourceLocation Loc, |
1524 | bool AsValue) const { |
1525 | // Try not to in some easy cases. |
1526 | assert(IntVal->getType()->isIntegerTy() && "Expected integer value" ); |
1527 | if (getEvaluationKind() == TEK_Scalar && |
1528 | (((!LVal.isBitField() || |
1529 | LVal.getBitFieldInfo().Size == ValueSizeInBits) && |
1530 | !hasPadding()) || |
1531 | !AsValue)) { |
1532 | auto *ValTy = AsValue |
1533 | ? CGF.ConvertTypeForMem(ValueTy) |
1534 | : getAtomicAddress().getElementType(); |
1535 | if (ValTy->isIntegerTy()) { |
1536 | assert(IntVal->getType() == ValTy && "Different integer types." ); |
1537 | return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); |
1538 | } else if (ValTy->isPointerTy()) |
1539 | return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); |
1540 | else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) |
1541 | return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); |
1542 | } |
1543 | |
1544 | // Create a temporary. This needs to be big enough to hold the |
1545 | // atomic integer. |
1546 | Address Temp = Address::invalid(); |
1547 | bool TempIsVolatile = false; |
1548 | if (AsValue && getEvaluationKind() == TEK_Aggregate) { |
1549 | assert(!ResultSlot.isIgnored()); |
1550 | Temp = ResultSlot.getAddress(); |
1551 | TempIsVolatile = ResultSlot.isVolatile(); |
1552 | } else { |
1553 | Temp = CreateTempAlloca(); |
1554 | } |
1555 | |
1556 | // Slam the integer into the temporary. |
1557 | Address CastTemp = emitCastToAtomicIntPointer(Temp); |
1558 | CGF.Builder.CreateStore(IntVal, CastTemp) |
1559 | ->setVolatile(TempIsVolatile); |
1560 | |
1561 | return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); |
1562 | } |
1563 | |
1564 | void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, |
1565 | llvm::AtomicOrdering AO, bool) { |
1566 | // void __atomic_load(size_t size, void *mem, void *return, int order); |
1567 | CallArgList Args; |
1568 | Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); |
1569 | Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), |
1570 | CGF.getContext().VoidPtrTy); |
1571 | Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), |
1572 | CGF.getContext().VoidPtrTy); |
1573 | Args.add( |
1574 | RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), |
1575 | CGF.getContext().IntTy); |
1576 | emitAtomicLibcall(CGF, "__atomic_load" , CGF.getContext().VoidTy, Args); |
1577 | } |
1578 | |
1579 | llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, |
1580 | bool IsVolatile) { |
1581 | // Okay, we're doing this natively. |
1582 | Address Addr = getAtomicAddressAsAtomicIntPointer(); |
1583 | llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load" ); |
1584 | Load->setAtomic(AO); |
1585 | |
1586 | // Other decoration. |
1587 | if (IsVolatile) |
1588 | Load->setVolatile(true); |
1589 | CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); |
1590 | return Load; |
1591 | } |
1592 | |
1593 | /// An LValue is a candidate for having its loads and stores be made atomic if |
1594 | /// we are operating under /volatile:ms *and* the LValue itself is volatile and |
1595 | /// performing such an operation can be performed without a libcall. |
1596 | bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { |
1597 | if (!CGM.getCodeGenOpts().MSVolatile) return false; |
1598 | AtomicInfo AI(*this, LV); |
1599 | bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); |
1600 | // An atomic is inline if we don't need to use a libcall. |
1601 | bool AtomicIsInline = !AI.shouldUseLibcall(); |
1602 | // MSVC doesn't seem to do this for types wider than a pointer. |
1603 | if (getContext().getTypeSize(LV.getType()) > |
1604 | getContext().getTypeSize(getContext().getIntPtrType())) |
1605 | return false; |
1606 | return IsVolatile && AtomicIsInline; |
1607 | } |
1608 | |
1609 | RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, |
1610 | AggValueSlot Slot) { |
1611 | llvm::AtomicOrdering AO; |
1612 | bool IsVolatile = LV.isVolatileQualified(); |
1613 | if (LV.getType()->isAtomicType()) { |
1614 | AO = llvm::AtomicOrdering::SequentiallyConsistent; |
1615 | } else { |
1616 | AO = llvm::AtomicOrdering::Acquire; |
1617 | IsVolatile = true; |
1618 | } |
1619 | return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); |
1620 | } |
1621 | |
1622 | RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, |
1623 | bool AsValue, llvm::AtomicOrdering AO, |
1624 | bool IsVolatile) { |
1625 | // Check whether we should use a library call. |
1626 | if (shouldUseLibcall()) { |
1627 | Address TempAddr = Address::invalid(); |
1628 | if (LVal.isSimple() && !ResultSlot.isIgnored()) { |
1629 | assert(getEvaluationKind() == TEK_Aggregate); |
1630 | TempAddr = ResultSlot.getAddress(); |
1631 | } else |
1632 | TempAddr = CreateTempAlloca(); |
1633 | |
1634 | EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); |
1635 | |
1636 | // Okay, turn that back into the original value or whole atomic (for |
1637 | // non-simple lvalues) type. |
1638 | return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); |
1639 | } |
1640 | |
1641 | // Okay, we're doing this natively. |
1642 | auto *Load = EmitAtomicLoadOp(AO, IsVolatile); |
1643 | |
1644 | // If we're ignoring an aggregate return, don't do anything. |
1645 | if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) |
1646 | return RValue::getAggregate(Address::invalid(), false); |
1647 | |
1648 | // Okay, turn that back into the original value or atomic (for non-simple |
1649 | // lvalues) type. |
1650 | return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); |
1651 | } |
1652 | |
1653 | /// Emit a load from an l-value of atomic type. Note that the r-value |
1654 | /// we produce is an r-value of the atomic *value* type. |
1655 | RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, |
1656 | llvm::AtomicOrdering AO, bool IsVolatile, |
1657 | AggValueSlot resultSlot) { |
1658 | AtomicInfo Atomics(*this, src); |
1659 | return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, |
1660 | IsVolatile); |
1661 | } |
1662 | |
1663 | /// Copy an r-value into memory as part of storing to an atomic type. |
1664 | /// This needs to create a bit-pattern suitable for atomic operations. |
1665 | void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { |
1666 | assert(LVal.isSimple()); |
1667 | // If we have an r-value, the rvalue should be of the atomic type, |
1668 | // which means that the caller is responsible for having zeroed |
1669 | // any padding. Just do an aggregate copy of that type. |
1670 | if (rvalue.isAggregate()) { |
1671 | LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); |
1672 | LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), |
1673 | getAtomicType()); |
1674 | bool IsVolatile = rvalue.isVolatileQualified() || |
1675 | LVal.isVolatileQualified(); |
1676 | CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), |
1677 | AggValueSlot::DoesNotOverlap, IsVolatile); |
1678 | return; |
1679 | } |
1680 | |
1681 | // Okay, otherwise we're copying stuff. |
1682 | |
1683 | // Zero out the buffer if necessary. |
1684 | emitMemSetZeroIfNecessary(); |
1685 | |
1686 | // Drill past the padding if present. |
1687 | LValue TempLVal = projectValue(); |
1688 | |
1689 | // Okay, store the rvalue in. |
1690 | if (rvalue.isScalar()) { |
1691 | CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); |
1692 | } else { |
1693 | CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); |
1694 | } |
1695 | } |
1696 | |
1697 | |
1698 | /// Materialize an r-value into memory for the purposes of storing it |
1699 | /// to an atomic type. |
1700 | Address AtomicInfo::materializeRValue(RValue rvalue) const { |
1701 | // Aggregate r-values are already in memory, and EmitAtomicStore |
1702 | // requires them to be values of the atomic type. |
1703 | if (rvalue.isAggregate()) |
1704 | return rvalue.getAggregateAddress(); |
1705 | |
1706 | // Otherwise, make a temporary and materialize into it. |
1707 | LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); |
1708 | AtomicInfo Atomics(CGF, TempLV); |
1709 | Atomics.emitCopyIntoMemory(rvalue); |
1710 | return TempLV.getAddress(CGF); |
1711 | } |
1712 | |
1713 | llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { |
1714 | // If we've got a scalar value of the right size, try to avoid going |
1715 | // through memory. |
1716 | if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { |
1717 | llvm::Value *Value = RVal.getScalarVal(); |
1718 | if (isa<llvm::IntegerType>(Value->getType())) |
1719 | return CGF.EmitToMemory(Value, ValueTy); |
1720 | else { |
1721 | llvm::IntegerType *InputIntTy = llvm::IntegerType::get( |
1722 | CGF.getLLVMContext(), |
1723 | LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); |
1724 | if (isa<llvm::PointerType>(Value->getType())) |
1725 | return CGF.Builder.CreatePtrToInt(Value, InputIntTy); |
1726 | else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) |
1727 | return CGF.Builder.CreateBitCast(Value, InputIntTy); |
1728 | } |
1729 | } |
1730 | // Otherwise, we need to go through memory. |
1731 | // Put the r-value in memory. |
1732 | Address Addr = materializeRValue(RVal); |
1733 | |
1734 | // Cast the temporary to the atomic int type and pull a value out. |
1735 | Addr = emitCastToAtomicIntPointer(Addr); |
1736 | return CGF.Builder.CreateLoad(Addr); |
1737 | } |
1738 | |
1739 | std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( |
1740 | llvm::Value *ExpectedVal, llvm::Value *DesiredVal, |
1741 | llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { |
1742 | // Do the atomic store. |
1743 | Address Addr = getAtomicAddressAsAtomicIntPointer(); |
1744 | auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), |
1745 | ExpectedVal, DesiredVal, |
1746 | Success, Failure); |
1747 | // Other decoration. |
1748 | Inst->setVolatile(LVal.isVolatileQualified()); |
1749 | Inst->setWeak(IsWeak); |
1750 | |
1751 | // Okay, turn that back into the original value type. |
1752 | auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); |
1753 | auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); |
1754 | return std::make_pair(PreviousVal, SuccessFailureVal); |
1755 | } |
1756 | |
1757 | llvm::Value * |
1758 | AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, |
1759 | llvm::Value *DesiredAddr, |
1760 | llvm::AtomicOrdering Success, |
1761 | llvm::AtomicOrdering Failure) { |
1762 | // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, |
1763 | // void *desired, int success, int failure); |
1764 | CallArgList Args; |
1765 | Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); |
1766 | Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), |
1767 | CGF.getContext().VoidPtrTy); |
1768 | Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), |
1769 | CGF.getContext().VoidPtrTy); |
1770 | Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), |
1771 | CGF.getContext().VoidPtrTy); |
1772 | Args.add(RValue::get( |
1773 | llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), |
1774 | CGF.getContext().IntTy); |
1775 | Args.add(RValue::get( |
1776 | llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), |
1777 | CGF.getContext().IntTy); |
1778 | auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange" , |
1779 | CGF.getContext().BoolTy, Args); |
1780 | |
1781 | return SuccessFailureRVal.getScalarVal(); |
1782 | } |
1783 | |
1784 | std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( |
1785 | RValue Expected, RValue Desired, llvm::AtomicOrdering Success, |
1786 | llvm::AtomicOrdering Failure, bool IsWeak) { |
1787 | // Check whether we should use a library call. |
1788 | if (shouldUseLibcall()) { |
1789 | // Produce a source address. |
1790 | Address ExpectedAddr = materializeRValue(Expected); |
1791 | Address DesiredAddr = materializeRValue(Desired); |
1792 | auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), |
1793 | DesiredAddr.getPointer(), |
1794 | Success, Failure); |
1795 | return std::make_pair( |
1796 | convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), |
1797 | SourceLocation(), /*AsValue=*/false), |
1798 | Res); |
1799 | } |
1800 | |
1801 | // If we've got a scalar value of the right size, try to avoid going |
1802 | // through memory. |
1803 | auto *ExpectedVal = convertRValueToInt(Expected); |
1804 | auto *DesiredVal = convertRValueToInt(Desired); |
1805 | auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, |
1806 | Failure, IsWeak); |
1807 | return std::make_pair( |
1808 | ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), |
1809 | SourceLocation(), /*AsValue=*/false), |
1810 | Res.second); |
1811 | } |
1812 | |
1813 | static void |
1814 | EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, |
1815 | const llvm::function_ref<RValue(RValue)> &UpdateOp, |
1816 | Address DesiredAddr) { |
1817 | RValue UpRVal; |
1818 | LValue AtomicLVal = Atomics.getAtomicLValue(); |
1819 | LValue DesiredLVal; |
1820 | if (AtomicLVal.isSimple()) { |
1821 | UpRVal = OldRVal; |
1822 | DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); |
1823 | } else { |
1824 | // Build new lvalue for temp address. |
1825 | Address Ptr = Atomics.materializeRValue(OldRVal); |
1826 | LValue UpdateLVal; |
1827 | if (AtomicLVal.isBitField()) { |
1828 | UpdateLVal = |
1829 | LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), |
1830 | AtomicLVal.getType(), |
1831 | AtomicLVal.getBaseInfo(), |
1832 | AtomicLVal.getTBAAInfo()); |
1833 | DesiredLVal = |
1834 | LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), |
1835 | AtomicLVal.getType(), AtomicLVal.getBaseInfo(), |
1836 | AtomicLVal.getTBAAInfo()); |
1837 | } else if (AtomicLVal.isVectorElt()) { |
1838 | UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), |
1839 | AtomicLVal.getType(), |
1840 | AtomicLVal.getBaseInfo(), |
1841 | AtomicLVal.getTBAAInfo()); |
1842 | DesiredLVal = LValue::MakeVectorElt( |
1843 | DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), |
1844 | AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); |
1845 | } else { |
1846 | assert(AtomicLVal.isExtVectorElt()); |
1847 | UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), |
1848 | AtomicLVal.getType(), |
1849 | AtomicLVal.getBaseInfo(), |
1850 | AtomicLVal.getTBAAInfo()); |
1851 | DesiredLVal = LValue::MakeExtVectorElt( |
1852 | DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), |
1853 | AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); |
1854 | } |
1855 | UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); |
1856 | } |
1857 | // Store new value in the corresponding memory area. |
1858 | RValue NewRVal = UpdateOp(UpRVal); |
1859 | if (NewRVal.isScalar()) { |
1860 | CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); |
1861 | } else { |
1862 | assert(NewRVal.isComplex()); |
1863 | CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, |
1864 | /*isInit=*/false); |
1865 | } |
1866 | } |
1867 | |
1868 | void AtomicInfo::EmitAtomicUpdateLibcall( |
1869 | llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, |
1870 | bool IsVolatile) { |
1871 | auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); |
1872 | |
1873 | Address ExpectedAddr = CreateTempAlloca(); |
1874 | |
1875 | EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); |
1876 | auto *ContBB = CGF.createBasicBlock("atomic_cont" ); |
1877 | auto *ExitBB = CGF.createBasicBlock("atomic_exit" ); |
1878 | CGF.EmitBlock(ContBB); |
1879 | Address DesiredAddr = CreateTempAlloca(); |
1880 | if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || |
1881 | requiresMemSetZero(getAtomicAddress().getElementType())) { |
1882 | auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); |
1883 | CGF.Builder.CreateStore(OldVal, DesiredAddr); |
1884 | } |
1885 | auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, |
1886 | AggValueSlot::ignored(), |
1887 | SourceLocation(), /*AsValue=*/false); |
1888 | EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); |
1889 | auto *Res = |
1890 | EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), |
1891 | DesiredAddr.getPointer(), |
1892 | AO, Failure); |
1893 | CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); |
1894 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
1895 | } |
1896 | |
1897 | void AtomicInfo::EmitAtomicUpdateOp( |
1898 | llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, |
1899 | bool IsVolatile) { |
1900 | auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); |
1901 | |
1902 | // Do the atomic load. |
1903 | auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); |
1904 | // For non-simple lvalues perform compare-and-swap procedure. |
1905 | auto *ContBB = CGF.createBasicBlock("atomic_cont" ); |
1906 | auto *ExitBB = CGF.createBasicBlock("atomic_exit" ); |
1907 | auto *CurBB = CGF.Builder.GetInsertBlock(); |
1908 | CGF.EmitBlock(ContBB); |
1909 | llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), |
1910 | /*NumReservedValues=*/2); |
1911 | PHI->addIncoming(OldVal, CurBB); |
1912 | Address NewAtomicAddr = CreateTempAlloca(); |
1913 | Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); |
1914 | if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || |
1915 | requiresMemSetZero(getAtomicAddress().getElementType())) { |
1916 | CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); |
1917 | } |
1918 | auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), |
1919 | SourceLocation(), /*AsValue=*/false); |
1920 | EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); |
1921 | auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); |
1922 | // Try to write new value using cmpxchg operation. |
1923 | auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); |
1924 | PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); |
1925 | CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); |
1926 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
1927 | } |
1928 | |
1929 | static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, |
1930 | RValue UpdateRVal, Address DesiredAddr) { |
1931 | LValue AtomicLVal = Atomics.getAtomicLValue(); |
1932 | LValue DesiredLVal; |
1933 | // Build new lvalue for temp address. |
1934 | if (AtomicLVal.isBitField()) { |
1935 | DesiredLVal = |
1936 | LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), |
1937 | AtomicLVal.getType(), AtomicLVal.getBaseInfo(), |
1938 | AtomicLVal.getTBAAInfo()); |
1939 | } else if (AtomicLVal.isVectorElt()) { |
1940 | DesiredLVal = |
1941 | LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), |
1942 | AtomicLVal.getType(), AtomicLVal.getBaseInfo(), |
1943 | AtomicLVal.getTBAAInfo()); |
1944 | } else { |
1945 | assert(AtomicLVal.isExtVectorElt()); |
1946 | DesiredLVal = LValue::MakeExtVectorElt( |
1947 | DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), |
1948 | AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); |
1949 | } |
1950 | // Store new value in the corresponding memory area. |
1951 | assert(UpdateRVal.isScalar()); |
1952 | CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); |
1953 | } |
1954 | |
1955 | void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, |
1956 | RValue UpdateRVal, bool IsVolatile) { |
1957 | auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); |
1958 | |
1959 | Address ExpectedAddr = CreateTempAlloca(); |
1960 | |
1961 | EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); |
1962 | auto *ContBB = CGF.createBasicBlock("atomic_cont" ); |
1963 | auto *ExitBB = CGF.createBasicBlock("atomic_exit" ); |
1964 | CGF.EmitBlock(ContBB); |
1965 | Address DesiredAddr = CreateTempAlloca(); |
1966 | if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || |
1967 | requiresMemSetZero(getAtomicAddress().getElementType())) { |
1968 | auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); |
1969 | CGF.Builder.CreateStore(OldVal, DesiredAddr); |
1970 | } |
1971 | EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); |
1972 | auto *Res = |
1973 | EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), |
1974 | DesiredAddr.getPointer(), |
1975 | AO, Failure); |
1976 | CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); |
1977 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
1978 | } |
1979 | |
1980 | void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, |
1981 | bool IsVolatile) { |
1982 | auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); |
1983 | |
1984 | // Do the atomic load. |
1985 | auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile); |
1986 | // For non-simple lvalues perform compare-and-swap procedure. |
1987 | auto *ContBB = CGF.createBasicBlock("atomic_cont" ); |
1988 | auto *ExitBB = CGF.createBasicBlock("atomic_exit" ); |
1989 | auto *CurBB = CGF.Builder.GetInsertBlock(); |
1990 | CGF.EmitBlock(ContBB); |
1991 | llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), |
1992 | /*NumReservedValues=*/2); |
1993 | PHI->addIncoming(OldVal, CurBB); |
1994 | Address NewAtomicAddr = CreateTempAlloca(); |
1995 | Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); |
1996 | if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || |
1997 | requiresMemSetZero(getAtomicAddress().getElementType())) { |
1998 | CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); |
1999 | } |
2000 | EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); |
2001 | auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); |
2002 | // Try to write new value using cmpxchg operation. |
2003 | auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); |
2004 | PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); |
2005 | CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); |
2006 | CGF.EmitBlock(ExitBB, /*IsFinished=*/true); |
2007 | } |
2008 | |
2009 | void AtomicInfo::EmitAtomicUpdate( |
2010 | llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, |
2011 | bool IsVolatile) { |
2012 | if (shouldUseLibcall()) { |
2013 | EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); |
2014 | } else { |
2015 | EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); |
2016 | } |
2017 | } |
2018 | |
2019 | void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, |
2020 | bool IsVolatile) { |
2021 | if (shouldUseLibcall()) { |
2022 | EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); |
2023 | } else { |
2024 | EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); |
2025 | } |
2026 | } |
2027 | |
2028 | void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, |
2029 | bool isInit) { |
2030 | bool IsVolatile = lvalue.isVolatileQualified(); |
2031 | llvm::AtomicOrdering AO; |
2032 | if (lvalue.getType()->isAtomicType()) { |
2033 | AO = llvm::AtomicOrdering::SequentiallyConsistent; |
2034 | } else { |
2035 | AO = llvm::AtomicOrdering::Release; |
2036 | IsVolatile = true; |
2037 | } |
2038 | return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); |
2039 | } |
2040 | |
2041 | /// Emit a store to an l-value of atomic type. |
2042 | /// |
2043 | /// Note that the r-value is expected to be an r-value *of the atomic |
2044 | /// type*; this means that for aggregate r-values, it should include |
2045 | /// storage for any padding that was necessary. |
2046 | void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, |
2047 | llvm::AtomicOrdering AO, bool IsVolatile, |
2048 | bool isInit) { |
2049 | // If this is an aggregate r-value, it should agree in type except |
2050 | // maybe for address-space qualification. |
2051 | assert(!rvalue.isAggregate() || |
2052 | rvalue.getAggregateAddress().getElementType() == |
2053 | dest.getAddress(*this).getElementType()); |
2054 | |
2055 | AtomicInfo atomics(*this, dest); |
2056 | LValue LVal = atomics.getAtomicLValue(); |
2057 | |
2058 | // If this is an initialization, just put the value there normally. |
2059 | if (LVal.isSimple()) { |
2060 | if (isInit) { |
2061 | atomics.emitCopyIntoMemory(rvalue); |
2062 | return; |
2063 | } |
2064 | |
2065 | // Check whether we should use a library call. |
2066 | if (atomics.shouldUseLibcall()) { |
2067 | // Produce a source address. |
2068 | Address srcAddr = atomics.materializeRValue(rvalue); |
2069 | |
2070 | // void __atomic_store(size_t size, void *mem, void *val, int order) |
2071 | CallArgList args; |
2072 | args.add(RValue::get(atomics.getAtomicSizeValue()), |
2073 | getContext().getSizeType()); |
2074 | args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), |
2075 | getContext().VoidPtrTy); |
2076 | args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), |
2077 | getContext().VoidPtrTy); |
2078 | args.add( |
2079 | RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), |
2080 | getContext().IntTy); |
2081 | emitAtomicLibcall(*this, "__atomic_store" , getContext().VoidTy, args); |
2082 | return; |
2083 | } |
2084 | |
2085 | // Okay, we're doing this natively. |
2086 | llvm::Value *intValue = atomics.convertRValueToInt(rvalue); |
2087 | |
2088 | // Do the atomic store. |
2089 | Address addr = |
2090 | atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); |
2091 | intValue = Builder.CreateIntCast( |
2092 | intValue, addr.getElementType(), /*isSigned=*/false); |
2093 | llvm::StoreInst *store = Builder.CreateStore(intValue, addr); |
2094 | |
2095 | if (AO == llvm::AtomicOrdering::Acquire) |
2096 | AO = llvm::AtomicOrdering::Monotonic; |
2097 | else if (AO == llvm::AtomicOrdering::AcquireRelease) |
2098 | AO = llvm::AtomicOrdering::Release; |
2099 | // Initializations don't need to be atomic. |
2100 | if (!isInit) |
2101 | store->setAtomic(AO); |
2102 | |
2103 | // Other decoration. |
2104 | if (IsVolatile) |
2105 | store->setVolatile(true); |
2106 | CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); |
2107 | return; |
2108 | } |
2109 | |
2110 | // Emit simple atomic update operation. |
2111 | atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); |
2112 | } |
2113 | |
2114 | /// Emit a compare-and-exchange op for atomic type. |
2115 | /// |
2116 | std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( |
2117 | LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, |
2118 | llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, |
2119 | AggValueSlot Slot) { |
2120 | // If this is an aggregate r-value, it should agree in type except |
2121 | // maybe for address-space qualification. |
2122 | assert(!Expected.isAggregate() || |
2123 | Expected.getAggregateAddress().getElementType() == |
2124 | Obj.getAddress(*this).getElementType()); |
2125 | assert(!Desired.isAggregate() || |
2126 | Desired.getAggregateAddress().getElementType() == |
2127 | Obj.getAddress(*this).getElementType()); |
2128 | AtomicInfo Atomics(*this, Obj); |
2129 | |
2130 | return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, |
2131 | IsWeak); |
2132 | } |
2133 | |
2134 | void CodeGenFunction::EmitAtomicUpdate( |
2135 | LValue LVal, llvm::AtomicOrdering AO, |
2136 | const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { |
2137 | AtomicInfo Atomics(*this, LVal); |
2138 | Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); |
2139 | } |
2140 | |
2141 | void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { |
2142 | AtomicInfo atomics(*this, dest); |
2143 | |
2144 | switch (atomics.getEvaluationKind()) { |
2145 | case TEK_Scalar: { |
2146 | llvm::Value *value = EmitScalarExpr(init); |
2147 | atomics.emitCopyIntoMemory(RValue::get(value)); |
2148 | return; |
2149 | } |
2150 | |
2151 | case TEK_Complex: { |
2152 | ComplexPairTy value = EmitComplexExpr(init); |
2153 | atomics.emitCopyIntoMemory(RValue::getComplex(value)); |
2154 | return; |
2155 | } |
2156 | |
2157 | case TEK_Aggregate: { |
2158 | // Fix up the destination if the initializer isn't an expression |
2159 | // of atomic type. |
2160 | bool Zeroed = false; |
2161 | if (!init->getType()->isAtomicType()) { |
2162 | Zeroed = atomics.emitMemSetZeroIfNecessary(); |
2163 | dest = atomics.projectValue(); |
2164 | } |
2165 | |
2166 | // Evaluate the expression directly into the destination. |
2167 | AggValueSlot slot = AggValueSlot::forLValue( |
2168 | dest, *this, AggValueSlot::IsNotDestructed, |
2169 | AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, |
2170 | AggValueSlot::DoesNotOverlap, |
2171 | Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); |
2172 | |
2173 | EmitAggExpr(init, slot); |
2174 | return; |
2175 | } |
2176 | } |
2177 | llvm_unreachable("bad evaluation kind" ); |
2178 | } |
2179 | |