1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the code for emitting atomic operations.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "CGCall.h"
14 #include "CGRecordLayout.h"
15 #include "CodeGenFunction.h"
16 #include "CodeGenModule.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/ASTContext.h"
19 #include "clang/CodeGen/CGFunctionInfo.h"
20 #include "clang/Frontend/FrontendDiagnostic.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/IR/DataLayout.h"
23 #include "llvm/IR/Intrinsics.h"
24 #include "llvm/IR/Operator.h"
25
26 using namespace clang;
27 using namespace CodeGen;
28
29 namespace {
30 class AtomicInfo {
31 CodeGenFunction &CGF;
32 QualType AtomicTy;
33 QualType ValueTy;
34 uint64_t AtomicSizeInBits;
35 uint64_t ValueSizeInBits;
36 CharUnits AtomicAlign;
37 CharUnits ValueAlign;
38 TypeEvaluationKind EvaluationKind;
39 bool UseLibcall;
40 LValue LVal;
41 CGBitFieldInfo BFI;
42 public:
AtomicInfo(CodeGenFunction & CGF,LValue & lvalue)43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45 EvaluationKind(TEK_Scalar), UseLibcall(true) {
46 assert(!lvalue.isGlobalReg());
47 ASTContext &C = CGF.getContext();
48 if (lvalue.isSimple()) {
49 AtomicTy = lvalue.getType();
50 if (auto *ATy = AtomicTy->getAs<AtomicType>())
51 ValueTy = ATy->getValueType();
52 else
53 ValueTy = AtomicTy;
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56 uint64_t ValueAlignInBits;
57 uint64_t AtomicAlignInBits;
58 TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59 ValueSizeInBits = ValueTI.Width;
60 ValueAlignInBits = ValueTI.Align;
61
62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63 AtomicSizeInBits = AtomicTI.Width;
64 AtomicAlignInBits = AtomicTI.Align;
65
66 assert(ValueSizeInBits <= AtomicSizeInBits);
67 assert(ValueAlignInBits <= AtomicAlignInBits);
68
69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71 if (lvalue.getAlignment().isZero())
72 lvalue.setAlignment(AtomicAlign);
73
74 LVal = lvalue;
75 } else if (lvalue.isBitField()) {
76 ValueTy = lvalue.getType();
77 ValueSizeInBits = C.getTypeSize(ValueTy);
78 auto &OrigBFI = lvalue.getBitFieldInfo();
79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80 AtomicSizeInBits = C.toBits(
81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82 .alignTo(lvalue.getAlignment()));
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84 auto OffsetInChars =
85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86 lvalue.getAlignment();
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88 VoidPtrAddr, OffsetInChars.getQuantity());
89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90 VoidPtrAddr,
91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92 "atomic_bitfield_base");
93 BFI = OrigBFI;
94 BFI.Offset = Offset;
95 BFI.StorageSize = AtomicSizeInBits;
96 BFI.StorageOffset += OffsetInChars;
97 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98 BFI, lvalue.getType(), lvalue.getBaseInfo(),
99 lvalue.getTBAAInfo());
100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
101 if (AtomicTy.isNull()) {
102 llvm::APInt Size(
103 /*numBits=*/32,
104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
105 AtomicTy =
106 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
107 /*IndexTypeQuals=*/0);
108 }
109 AtomicAlign = ValueAlign = lvalue.getAlignment();
110 } else if (lvalue.isVectorElt()) {
111 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
112 ValueSizeInBits = C.getTypeSize(ValueTy);
113 AtomicTy = lvalue.getType();
114 AtomicSizeInBits = C.getTypeSize(AtomicTy);
115 AtomicAlign = ValueAlign = lvalue.getAlignment();
116 LVal = lvalue;
117 } else {
118 assert(lvalue.isExtVectorElt());
119 ValueTy = lvalue.getType();
120 ValueSizeInBits = C.getTypeSize(ValueTy);
121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122 lvalue.getType(), cast<llvm::VectorType>(
123 lvalue.getExtVectorAddress().getElementType())
124 ->getNumElements());
125 AtomicSizeInBits = C.getTypeSize(AtomicTy);
126 AtomicAlign = ValueAlign = lvalue.getAlignment();
127 LVal = lvalue;
128 }
129 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
130 AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
131 }
132
getAtomicType() const133 QualType getAtomicType() const { return AtomicTy; }
getValueType() const134 QualType getValueType() const { return ValueTy; }
getAtomicAlignment() const135 CharUnits getAtomicAlignment() const { return AtomicAlign; }
getAtomicSizeInBits() const136 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
getValueSizeInBits() const137 uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
getEvaluationKind() const138 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
shouldUseLibcall() const139 bool shouldUseLibcall() const { return UseLibcall; }
getAtomicLValue() const140 const LValue &getAtomicLValue() const { return LVal; }
getAtomicPointer() const141 llvm::Value *getAtomicPointer() const {
142 if (LVal.isSimple())
143 return LVal.getPointer(CGF);
144 else if (LVal.isBitField())
145 return LVal.getBitFieldPointer();
146 else if (LVal.isVectorElt())
147 return LVal.getVectorPointer();
148 assert(LVal.isExtVectorElt());
149 return LVal.getExtVectorPointer();
150 }
getAtomicAddress() const151 Address getAtomicAddress() const {
152 return Address(getAtomicPointer(), getAtomicAlignment());
153 }
154
getAtomicAddressAsAtomicIntPointer() const155 Address getAtomicAddressAsAtomicIntPointer() const {
156 return emitCastToAtomicIntPointer(getAtomicAddress());
157 }
158
159 /// Is the atomic size larger than the underlying value type?
160 ///
161 /// Note that the absence of padding does not mean that atomic
162 /// objects are completely interchangeable with non-atomic
163 /// objects: we might have promoted the alignment of a type
164 /// without making it bigger.
hasPadding() const165 bool hasPadding() const {
166 return (ValueSizeInBits != AtomicSizeInBits);
167 }
168
169 bool emitMemSetZeroIfNecessary() const;
170
getAtomicSizeValue() const171 llvm::Value *getAtomicSizeValue() const {
172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173 return CGF.CGM.getSize(size);
174 }
175
176 /// Cast the given pointer to an integer pointer suitable for atomic
177 /// operations if the source.
178 Address emitCastToAtomicIntPointer(Address Addr) const;
179
180 /// If Addr is compatible with the iN that will be used for an atomic
181 /// operation, bitcast it. Otherwise, create a temporary that is suitable
182 /// and copy the value across.
183 Address convertToAtomicIntPointer(Address Addr) const;
184
185 /// Turn an atomic-layout object into an r-value.
186 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187 SourceLocation loc, bool AsValue) const;
188
189 /// Converts a rvalue to integer value.
190 llvm::Value *convertRValueToInt(RValue RVal) const;
191
192 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193 AggValueSlot ResultSlot,
194 SourceLocation Loc, bool AsValue) const;
195
196 /// Copy an atomic r-value into atomic-layout memory.
197 void emitCopyIntoMemory(RValue rvalue) const;
198
199 /// Project an l-value down to the value field.
projectValue() const200 LValue projectValue() const {
201 assert(LVal.isSimple());
202 Address addr = getAtomicAddress();
203 if (hasPadding())
204 addr = CGF.Builder.CreateStructGEP(addr, 0);
205
206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207 LVal.getBaseInfo(), LVal.getTBAAInfo());
208 }
209
210 /// Emits atomic load.
211 /// \returns Loaded value.
212 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213 bool AsValue, llvm::AtomicOrdering AO,
214 bool IsVolatile);
215
216 /// Emits atomic compare-and-exchange sequence.
217 /// \param Expected Expected value.
218 /// \param Desired Desired value.
219 /// \param Success Atomic ordering for success operation.
220 /// \param Failure Atomic ordering for failed operation.
221 /// \param IsWeak true if atomic operation is weak, false otherwise.
222 /// \returns Pair of values: previous value from storage (value type) and
223 /// boolean flag (i1 type) with true if success and false otherwise.
224 std::pair<RValue, llvm::Value *>
225 EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226 llvm::AtomicOrdering Success =
227 llvm::AtomicOrdering::SequentiallyConsistent,
228 llvm::AtomicOrdering Failure =
229 llvm::AtomicOrdering::SequentiallyConsistent,
230 bool IsWeak = false);
231
232 /// Emits atomic update.
233 /// \param AO Atomic ordering.
234 /// \param UpdateOp Update operation for the current lvalue.
235 void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236 const llvm::function_ref<RValue(RValue)> &UpdateOp,
237 bool IsVolatile);
238 /// Emits atomic update.
239 /// \param AO Atomic ordering.
240 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241 bool IsVolatile);
242
243 /// Materialize an atomic r-value in atomic-layout memory.
244 Address materializeRValue(RValue rvalue) const;
245
246 /// Creates temp alloca for intermediate operations on atomic value.
247 Address CreateTempAlloca() const;
248 private:
249 bool requiresMemSetZero(llvm::Type *type) const;
250
251
252 /// Emits atomic load as a libcall.
253 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254 llvm::AtomicOrdering AO, bool IsVolatile);
255 /// Emits atomic load as LLVM instruction.
256 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257 /// Emits atomic compare-and-exchange op as a libcall.
258 llvm::Value *EmitAtomicCompareExchangeLibcall(
259 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260 llvm::AtomicOrdering Success =
261 llvm::AtomicOrdering::SequentiallyConsistent,
262 llvm::AtomicOrdering Failure =
263 llvm::AtomicOrdering::SequentiallyConsistent);
264 /// Emits atomic compare-and-exchange op as LLVM instruction.
265 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267 llvm::AtomicOrdering Success =
268 llvm::AtomicOrdering::SequentiallyConsistent,
269 llvm::AtomicOrdering Failure =
270 llvm::AtomicOrdering::SequentiallyConsistent,
271 bool IsWeak = false);
272 /// Emit atomic update as libcalls.
273 void
274 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275 const llvm::function_ref<RValue(RValue)> &UpdateOp,
276 bool IsVolatile);
277 /// Emit atomic update as LLVM instructions.
278 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279 const llvm::function_ref<RValue(RValue)> &UpdateOp,
280 bool IsVolatile);
281 /// Emit atomic update as libcalls.
282 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283 bool IsVolatile);
284 /// Emit atomic update as LLVM instructions.
285 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286 bool IsVolatile);
287 };
288 }
289
CreateTempAlloca() const290 Address AtomicInfo::CreateTempAlloca() const {
291 Address TempAlloca = CGF.CreateMemTemp(
292 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293 : AtomicTy,
294 getAtomicAlignment(),
295 "atomic-temp");
296 // Cast to pointer to value type for bitfields.
297 if (LVal.isBitField())
298 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
299 TempAlloca, getAtomicAddress().getType());
300 return TempAlloca;
301 }
302
emitAtomicLibcall(CodeGenFunction & CGF,StringRef fnName,QualType resultType,CallArgList & args)303 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
304 StringRef fnName,
305 QualType resultType,
306 CallArgList &args) {
307 const CGFunctionInfo &fnInfo =
308 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310 llvm::FunctionCallee fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
311 auto callee = CGCallee::forDirect(fn);
312 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
313 }
314
315 /// Does a store of the given IR type modify the full expected width?
isFullSizeType(CodeGenModule & CGM,llvm::Type * type,uint64_t expectedSize)316 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
317 uint64_t expectedSize) {
318 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
319 }
320
321 /// Does the atomic type require memsetting to zero before initialization?
322 ///
323 /// The IR type is provided as a way of making certain queries faster.
requiresMemSetZero(llvm::Type * type) const324 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
325 // If the atomic type has size padding, we definitely need a memset.
326 if (hasPadding()) return true;
327
328 // Otherwise, do some simple heuristics to try to avoid it:
329 switch (getEvaluationKind()) {
330 // For scalars and complexes, check whether the store size of the
331 // type uses the full size.
332 case TEK_Scalar:
333 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
334 case TEK_Complex:
335 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
336 AtomicSizeInBits / 2);
337
338 // Padding in structs has an undefined bit pattern. User beware.
339 case TEK_Aggregate:
340 return false;
341 }
342 llvm_unreachable("bad evaluation kind");
343 }
344
emitMemSetZeroIfNecessary() const345 bool AtomicInfo::emitMemSetZeroIfNecessary() const {
346 assert(LVal.isSimple());
347 llvm::Value *addr = LVal.getPointer(CGF);
348 if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
349 return false;
350
351 CGF.Builder.CreateMemSet(
352 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
353 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
354 LVal.getAlignment().getAsAlign());
355 return true;
356 }
357
emitAtomicCmpXchg(CodeGenFunction & CGF,AtomicExpr * E,bool IsWeak,Address Dest,Address Ptr,Address Val1,Address Val2,uint64_t Size,llvm::AtomicOrdering SuccessOrder,llvm::AtomicOrdering FailureOrder,llvm::SyncScope::ID Scope)358 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
359 Address Dest, Address Ptr,
360 Address Val1, Address Val2,
361 uint64_t Size,
362 llvm::AtomicOrdering SuccessOrder,
363 llvm::AtomicOrdering FailureOrder,
364 llvm::SyncScope::ID Scope) {
365 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
366 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
367 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
368
369 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
370 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
371 Scope);
372 Pair->setVolatile(E->isVolatile());
373 Pair->setWeak(IsWeak);
374
375 // Cmp holds the result of the compare-exchange operation: true on success,
376 // false on failure.
377 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
378 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
379
380 // This basic block is used to hold the store instruction if the operation
381 // failed.
382 llvm::BasicBlock *StoreExpectedBB =
383 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
384
385 // This basic block is the exit point of the operation, we should end up
386 // here regardless of whether or not the operation succeeded.
387 llvm::BasicBlock *ContinueBB =
388 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
389
390 // Update Expected if Expected isn't equal to Old, otherwise branch to the
391 // exit point.
392 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
393
394 CGF.Builder.SetInsertPoint(StoreExpectedBB);
395 // Update the memory at Expected with Old's value.
396 CGF.Builder.CreateStore(Old, Val1);
397 // Finally, branch to the exit point.
398 CGF.Builder.CreateBr(ContinueBB);
399
400 CGF.Builder.SetInsertPoint(ContinueBB);
401 // Update the memory at Dest with Cmp's value.
402 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
403 }
404
405 /// Given an ordering required on success, emit all possible cmpxchg
406 /// instructions to cope with the provided (but possibly only dynamically known)
407 /// FailureOrder.
emitAtomicCmpXchgFailureSet(CodeGenFunction & CGF,AtomicExpr * E,bool IsWeak,Address Dest,Address Ptr,Address Val1,Address Val2,llvm::Value * FailureOrderVal,uint64_t Size,llvm::AtomicOrdering SuccessOrder,llvm::SyncScope::ID Scope)408 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
409 bool IsWeak, Address Dest, Address Ptr,
410 Address Val1, Address Val2,
411 llvm::Value *FailureOrderVal,
412 uint64_t Size,
413 llvm::AtomicOrdering SuccessOrder,
414 llvm::SyncScope::ID Scope) {
415 llvm::AtomicOrdering FailureOrder;
416 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
417 auto FOS = FO->getSExtValue();
418 if (!llvm::isValidAtomicOrderingCABI(FOS))
419 FailureOrder = llvm::AtomicOrdering::Monotonic;
420 else
421 switch ((llvm::AtomicOrderingCABI)FOS) {
422 case llvm::AtomicOrderingCABI::relaxed:
423 case llvm::AtomicOrderingCABI::release:
424 case llvm::AtomicOrderingCABI::acq_rel:
425 FailureOrder = llvm::AtomicOrdering::Monotonic;
426 break;
427 case llvm::AtomicOrderingCABI::consume:
428 case llvm::AtomicOrderingCABI::acquire:
429 FailureOrder = llvm::AtomicOrdering::Acquire;
430 break;
431 case llvm::AtomicOrderingCABI::seq_cst:
432 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
433 break;
434 }
435 if (isStrongerThan(FailureOrder, SuccessOrder)) {
436 // Don't assert on undefined behavior "failure argument shall be no
437 // stronger than the success argument".
438 FailureOrder =
439 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
440 }
441 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
442 FailureOrder, Scope);
443 return;
444 }
445
446 // Create all the relevant BB's
447 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
448 *SeqCstBB = nullptr;
449 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
450 if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
451 SuccessOrder != llvm::AtomicOrdering::Release)
452 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
453 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
454 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
455
456 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
457
458 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
459
460 // Emit all the different atomics
461
462 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
463 // doesn't matter unless someone is crazy enough to use something that
464 // doesn't fold to a constant for the ordering.
465 CGF.Builder.SetInsertPoint(MonotonicBB);
466 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
467 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
468 CGF.Builder.CreateBr(ContBB);
469
470 if (AcquireBB) {
471 CGF.Builder.SetInsertPoint(AcquireBB);
472 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
473 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
474 CGF.Builder.CreateBr(ContBB);
475 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
476 AcquireBB);
477 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
478 AcquireBB);
479 }
480 if (SeqCstBB) {
481 CGF.Builder.SetInsertPoint(SeqCstBB);
482 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
483 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
484 CGF.Builder.CreateBr(ContBB);
485 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
486 SeqCstBB);
487 }
488
489 CGF.Builder.SetInsertPoint(ContBB);
490 }
491
492 /// Duplicate the atomic min/max operation in conventional IR for the builtin
493 /// variants that return the new rather than the original value.
EmitPostAtomicMinMax(CGBuilderTy & Builder,AtomicExpr::AtomicOp Op,bool IsSigned,llvm::Value * OldVal,llvm::Value * RHS)494 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
495 AtomicExpr::AtomicOp Op,
496 bool IsSigned,
497 llvm::Value *OldVal,
498 llvm::Value *RHS) {
499 llvm::CmpInst::Predicate Pred;
500 switch (Op) {
501 default:
502 llvm_unreachable("Unexpected min/max operation");
503 case AtomicExpr::AO__atomic_max_fetch:
504 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
505 break;
506 case AtomicExpr::AO__atomic_min_fetch:
507 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
508 break;
509 }
510 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
511 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
512 }
513
EmitAtomicOp(CodeGenFunction & CGF,AtomicExpr * E,Address Dest,Address Ptr,Address Val1,Address Val2,llvm::Value * IsWeak,llvm::Value * FailureOrder,uint64_t Size,llvm::AtomicOrdering Order,llvm::SyncScope::ID Scope)514 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
515 Address Ptr, Address Val1, Address Val2,
516 llvm::Value *IsWeak, llvm::Value *FailureOrder,
517 uint64_t Size, llvm::AtomicOrdering Order,
518 llvm::SyncScope::ID Scope) {
519 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
520 bool PostOpMinMax = false;
521 unsigned PostOp = 0;
522
523 switch (E->getOp()) {
524 case AtomicExpr::AO__c11_atomic_init:
525 case AtomicExpr::AO__opencl_atomic_init:
526 llvm_unreachable("Already handled!");
527
528 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
529 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
530 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
531 FailureOrder, Size, Order, Scope);
532 return;
533 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
534 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
535 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
536 FailureOrder, Size, Order, Scope);
537 return;
538 case AtomicExpr::AO__atomic_compare_exchange:
539 case AtomicExpr::AO__atomic_compare_exchange_n: {
540 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
541 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
542 Val1, Val2, FailureOrder, Size, Order, Scope);
543 } else {
544 // Create all the relevant BB's
545 llvm::BasicBlock *StrongBB =
546 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
547 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
548 llvm::BasicBlock *ContBB =
549 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
550
551 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
552 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
553
554 CGF.Builder.SetInsertPoint(StrongBB);
555 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
556 FailureOrder, Size, Order, Scope);
557 CGF.Builder.CreateBr(ContBB);
558
559 CGF.Builder.SetInsertPoint(WeakBB);
560 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
561 FailureOrder, Size, Order, Scope);
562 CGF.Builder.CreateBr(ContBB);
563
564 CGF.Builder.SetInsertPoint(ContBB);
565 }
566 return;
567 }
568 case AtomicExpr::AO__c11_atomic_load:
569 case AtomicExpr::AO__opencl_atomic_load:
570 case AtomicExpr::AO__atomic_load_n:
571 case AtomicExpr::AO__atomic_load: {
572 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
573 Load->setAtomic(Order, Scope);
574 Load->setVolatile(E->isVolatile());
575 CGF.Builder.CreateStore(Load, Dest);
576 return;
577 }
578
579 case AtomicExpr::AO__c11_atomic_store:
580 case AtomicExpr::AO__opencl_atomic_store:
581 case AtomicExpr::AO__atomic_store:
582 case AtomicExpr::AO__atomic_store_n: {
583 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
584 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
585 Store->setAtomic(Order, Scope);
586 Store->setVolatile(E->isVolatile());
587 return;
588 }
589
590 case AtomicExpr::AO__c11_atomic_exchange:
591 case AtomicExpr::AO__opencl_atomic_exchange:
592 case AtomicExpr::AO__atomic_exchange_n:
593 case AtomicExpr::AO__atomic_exchange:
594 Op = llvm::AtomicRMWInst::Xchg;
595 break;
596
597 case AtomicExpr::AO__atomic_add_fetch:
598 PostOp = llvm::Instruction::Add;
599 LLVM_FALLTHROUGH;
600 case AtomicExpr::AO__c11_atomic_fetch_add:
601 case AtomicExpr::AO__opencl_atomic_fetch_add:
602 case AtomicExpr::AO__atomic_fetch_add:
603 Op = llvm::AtomicRMWInst::Add;
604 break;
605
606 case AtomicExpr::AO__atomic_sub_fetch:
607 PostOp = llvm::Instruction::Sub;
608 LLVM_FALLTHROUGH;
609 case AtomicExpr::AO__c11_atomic_fetch_sub:
610 case AtomicExpr::AO__opencl_atomic_fetch_sub:
611 case AtomicExpr::AO__atomic_fetch_sub:
612 Op = llvm::AtomicRMWInst::Sub;
613 break;
614
615 case AtomicExpr::AO__atomic_min_fetch:
616 PostOpMinMax = true;
617 LLVM_FALLTHROUGH;
618 case AtomicExpr::AO__c11_atomic_fetch_min:
619 case AtomicExpr::AO__opencl_atomic_fetch_min:
620 case AtomicExpr::AO__atomic_fetch_min:
621 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
622 : llvm::AtomicRMWInst::UMin;
623 break;
624
625 case AtomicExpr::AO__atomic_max_fetch:
626 PostOpMinMax = true;
627 LLVM_FALLTHROUGH;
628 case AtomicExpr::AO__c11_atomic_fetch_max:
629 case AtomicExpr::AO__opencl_atomic_fetch_max:
630 case AtomicExpr::AO__atomic_fetch_max:
631 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
632 : llvm::AtomicRMWInst::UMax;
633 break;
634
635 case AtomicExpr::AO__atomic_and_fetch:
636 PostOp = llvm::Instruction::And;
637 LLVM_FALLTHROUGH;
638 case AtomicExpr::AO__c11_atomic_fetch_and:
639 case AtomicExpr::AO__opencl_atomic_fetch_and:
640 case AtomicExpr::AO__atomic_fetch_and:
641 Op = llvm::AtomicRMWInst::And;
642 break;
643
644 case AtomicExpr::AO__atomic_or_fetch:
645 PostOp = llvm::Instruction::Or;
646 LLVM_FALLTHROUGH;
647 case AtomicExpr::AO__c11_atomic_fetch_or:
648 case AtomicExpr::AO__opencl_atomic_fetch_or:
649 case AtomicExpr::AO__atomic_fetch_or:
650 Op = llvm::AtomicRMWInst::Or;
651 break;
652
653 case AtomicExpr::AO__atomic_xor_fetch:
654 PostOp = llvm::Instruction::Xor;
655 LLVM_FALLTHROUGH;
656 case AtomicExpr::AO__c11_atomic_fetch_xor:
657 case AtomicExpr::AO__opencl_atomic_fetch_xor:
658 case AtomicExpr::AO__atomic_fetch_xor:
659 Op = llvm::AtomicRMWInst::Xor;
660 break;
661
662 case AtomicExpr::AO__atomic_nand_fetch:
663 PostOp = llvm::Instruction::And; // the NOT is special cased below
664 LLVM_FALLTHROUGH;
665 case AtomicExpr::AO__atomic_fetch_nand:
666 Op = llvm::AtomicRMWInst::Nand;
667 break;
668 }
669
670 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
671 llvm::AtomicRMWInst *RMWI =
672 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
673 RMWI->setVolatile(E->isVolatile());
674
675 // For __atomic_*_fetch operations, perform the operation again to
676 // determine the value which was written.
677 llvm::Value *Result = RMWI;
678 if (PostOpMinMax)
679 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
680 E->getValueType()->isSignedIntegerType(),
681 RMWI, LoadVal1);
682 else if (PostOp)
683 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
684 LoadVal1);
685 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
686 Result = CGF.Builder.CreateNot(Result);
687 CGF.Builder.CreateStore(Result, Dest);
688 }
689
690 // This function emits any expression (scalar, complex, or aggregate)
691 // into a temporary alloca.
692 static Address
EmitValToTemp(CodeGenFunction & CGF,Expr * E)693 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
694 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
695 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
696 /*Init*/ true);
697 return DeclPtr;
698 }
699
EmitAtomicOp(CodeGenFunction & CGF,AtomicExpr * Expr,Address Dest,Address Ptr,Address Val1,Address Val2,llvm::Value * IsWeak,llvm::Value * FailureOrder,uint64_t Size,llvm::AtomicOrdering Order,llvm::Value * Scope)700 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
701 Address Ptr, Address Val1, Address Val2,
702 llvm::Value *IsWeak, llvm::Value *FailureOrder,
703 uint64_t Size, llvm::AtomicOrdering Order,
704 llvm::Value *Scope) {
705 auto ScopeModel = Expr->getScopeModel();
706
707 // LLVM atomic instructions always have synch scope. If clang atomic
708 // expression has no scope operand, use default LLVM synch scope.
709 if (!ScopeModel) {
710 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
711 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
712 return;
713 }
714
715 // Handle constant scope.
716 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
717 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
718 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
719 Order, CGF.CGM.getLLVMContext());
720 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
721 Order, SCID);
722 return;
723 }
724
725 // Handle non-constant scope.
726 auto &Builder = CGF.Builder;
727 auto Scopes = ScopeModel->getRuntimeValues();
728 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
729 for (auto S : Scopes)
730 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
731
732 llvm::BasicBlock *ContBB =
733 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
734
735 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
736 // If unsupported synch scope is encountered at run time, assume a fallback
737 // synch scope value.
738 auto FallBack = ScopeModel->getFallBackValue();
739 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
740 for (auto S : Scopes) {
741 auto *B = BB[S];
742 if (S != FallBack)
743 SI->addCase(Builder.getInt32(S), B);
744
745 Builder.SetInsertPoint(B);
746 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
747 Order,
748 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
749 ScopeModel->map(S),
750 Order,
751 CGF.getLLVMContext()));
752 Builder.CreateBr(ContBB);
753 }
754
755 Builder.SetInsertPoint(ContBB);
756 }
757
758 static void
AddDirectArgument(CodeGenFunction & CGF,CallArgList & Args,bool UseOptimizedLibcall,llvm::Value * Val,QualType ValTy,SourceLocation Loc,CharUnits SizeInChars)759 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
760 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
761 SourceLocation Loc, CharUnits SizeInChars) {
762 if (UseOptimizedLibcall) {
763 // Load value and pass it to the function directly.
764 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
765 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
766 ValTy =
767 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
768 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
769 SizeInBits)->getPointerTo();
770 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
771 Val = CGF.EmitLoadOfScalar(Ptr, false,
772 CGF.getContext().getPointerType(ValTy),
773 Loc);
774 // Coerce the value into an appropriately sized integer type.
775 Args.add(RValue::get(Val), ValTy);
776 } else {
777 // Non-optimized functions always take a reference.
778 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
779 CGF.getContext().VoidPtrTy);
780 }
781 }
782
EmitAtomicExpr(AtomicExpr * E)783 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
784 QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
785 QualType MemTy = AtomicTy;
786 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
787 MemTy = AT->getValueType();
788 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
789
790 Address Val1 = Address::invalid();
791 Address Val2 = Address::invalid();
792 Address Dest = Address::invalid();
793 Address Ptr = EmitPointerWithAlignment(E->getPtr());
794
795 if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
796 E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
797 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
798 EmitAtomicInit(E->getVal1(), lvalue);
799 return RValue::get(nullptr);
800 }
801
802 CharUnits sizeChars, alignChars;
803 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
804 uint64_t Size = sizeChars.getQuantity();
805 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
806
807 bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits;
808 bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0;
809 bool UseLibcall = Misaligned | Oversized;
810
811 if (UseLibcall) {
812 CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
813 << !Oversized;
814 }
815
816 llvm::Value *Order = EmitScalarExpr(E->getOrder());
817 llvm::Value *Scope =
818 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
819
820 switch (E->getOp()) {
821 case AtomicExpr::AO__c11_atomic_init:
822 case AtomicExpr::AO__opencl_atomic_init:
823 llvm_unreachable("Already handled above with EmitAtomicInit!");
824
825 case AtomicExpr::AO__c11_atomic_load:
826 case AtomicExpr::AO__opencl_atomic_load:
827 case AtomicExpr::AO__atomic_load_n:
828 break;
829
830 case AtomicExpr::AO__atomic_load:
831 Dest = EmitPointerWithAlignment(E->getVal1());
832 break;
833
834 case AtomicExpr::AO__atomic_store:
835 Val1 = EmitPointerWithAlignment(E->getVal1());
836 break;
837
838 case AtomicExpr::AO__atomic_exchange:
839 Val1 = EmitPointerWithAlignment(E->getVal1());
840 Dest = EmitPointerWithAlignment(E->getVal2());
841 break;
842
843 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
844 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
845 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
846 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
847 case AtomicExpr::AO__atomic_compare_exchange_n:
848 case AtomicExpr::AO__atomic_compare_exchange:
849 Val1 = EmitPointerWithAlignment(E->getVal1());
850 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
851 Val2 = EmitPointerWithAlignment(E->getVal2());
852 else
853 Val2 = EmitValToTemp(*this, E->getVal2());
854 OrderFail = EmitScalarExpr(E->getOrderFail());
855 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
856 E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
857 IsWeak = EmitScalarExpr(E->getWeak());
858 break;
859
860 case AtomicExpr::AO__c11_atomic_fetch_add:
861 case AtomicExpr::AO__c11_atomic_fetch_sub:
862 case AtomicExpr::AO__opencl_atomic_fetch_add:
863 case AtomicExpr::AO__opencl_atomic_fetch_sub:
864 if (MemTy->isPointerType()) {
865 // For pointer arithmetic, we're required to do a bit of math:
866 // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
867 // ... but only for the C11 builtins. The GNU builtins expect the
868 // user to multiply by sizeof(T).
869 QualType Val1Ty = E->getVal1()->getType();
870 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
871 CharUnits PointeeIncAmt =
872 getContext().getTypeSizeInChars(MemTy->getPointeeType());
873 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
874 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
875 Val1 = Temp;
876 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
877 break;
878 }
879 LLVM_FALLTHROUGH;
880 case AtomicExpr::AO__atomic_fetch_add:
881 case AtomicExpr::AO__atomic_fetch_sub:
882 case AtomicExpr::AO__atomic_add_fetch:
883 case AtomicExpr::AO__atomic_sub_fetch:
884 case AtomicExpr::AO__c11_atomic_store:
885 case AtomicExpr::AO__c11_atomic_exchange:
886 case AtomicExpr::AO__opencl_atomic_store:
887 case AtomicExpr::AO__opencl_atomic_exchange:
888 case AtomicExpr::AO__atomic_store_n:
889 case AtomicExpr::AO__atomic_exchange_n:
890 case AtomicExpr::AO__c11_atomic_fetch_and:
891 case AtomicExpr::AO__c11_atomic_fetch_or:
892 case AtomicExpr::AO__c11_atomic_fetch_xor:
893 case AtomicExpr::AO__c11_atomic_fetch_max:
894 case AtomicExpr::AO__c11_atomic_fetch_min:
895 case AtomicExpr::AO__opencl_atomic_fetch_and:
896 case AtomicExpr::AO__opencl_atomic_fetch_or:
897 case AtomicExpr::AO__opencl_atomic_fetch_xor:
898 case AtomicExpr::AO__opencl_atomic_fetch_min:
899 case AtomicExpr::AO__opencl_atomic_fetch_max:
900 case AtomicExpr::AO__atomic_fetch_and:
901 case AtomicExpr::AO__atomic_fetch_or:
902 case AtomicExpr::AO__atomic_fetch_xor:
903 case AtomicExpr::AO__atomic_fetch_nand:
904 case AtomicExpr::AO__atomic_and_fetch:
905 case AtomicExpr::AO__atomic_or_fetch:
906 case AtomicExpr::AO__atomic_xor_fetch:
907 case AtomicExpr::AO__atomic_nand_fetch:
908 case AtomicExpr::AO__atomic_max_fetch:
909 case AtomicExpr::AO__atomic_min_fetch:
910 case AtomicExpr::AO__atomic_fetch_max:
911 case AtomicExpr::AO__atomic_fetch_min:
912 Val1 = EmitValToTemp(*this, E->getVal1());
913 break;
914 }
915
916 QualType RValTy = E->getType().getUnqualifiedType();
917
918 // The inlined atomics only function on iN types, where N is a power of 2. We
919 // need to make sure (via temporaries if necessary) that all incoming values
920 // are compatible.
921 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
922 AtomicInfo Atomics(*this, AtomicVal);
923
924 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
925 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
926 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
927 if (Dest.isValid())
928 Dest = Atomics.emitCastToAtomicIntPointer(Dest);
929 else if (E->isCmpXChg())
930 Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
931 else if (!RValTy->isVoidType())
932 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
933
934 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
935 if (UseLibcall) {
936 bool UseOptimizedLibcall = false;
937 switch (E->getOp()) {
938 case AtomicExpr::AO__c11_atomic_init:
939 case AtomicExpr::AO__opencl_atomic_init:
940 llvm_unreachable("Already handled above with EmitAtomicInit!");
941
942 case AtomicExpr::AO__c11_atomic_fetch_add:
943 case AtomicExpr::AO__opencl_atomic_fetch_add:
944 case AtomicExpr::AO__atomic_fetch_add:
945 case AtomicExpr::AO__c11_atomic_fetch_and:
946 case AtomicExpr::AO__opencl_atomic_fetch_and:
947 case AtomicExpr::AO__atomic_fetch_and:
948 case AtomicExpr::AO__c11_atomic_fetch_or:
949 case AtomicExpr::AO__opencl_atomic_fetch_or:
950 case AtomicExpr::AO__atomic_fetch_or:
951 case AtomicExpr::AO__atomic_fetch_nand:
952 case AtomicExpr::AO__c11_atomic_fetch_sub:
953 case AtomicExpr::AO__opencl_atomic_fetch_sub:
954 case AtomicExpr::AO__atomic_fetch_sub:
955 case AtomicExpr::AO__c11_atomic_fetch_xor:
956 case AtomicExpr::AO__opencl_atomic_fetch_xor:
957 case AtomicExpr::AO__opencl_atomic_fetch_min:
958 case AtomicExpr::AO__opencl_atomic_fetch_max:
959 case AtomicExpr::AO__atomic_fetch_xor:
960 case AtomicExpr::AO__c11_atomic_fetch_max:
961 case AtomicExpr::AO__c11_atomic_fetch_min:
962 case AtomicExpr::AO__atomic_add_fetch:
963 case AtomicExpr::AO__atomic_and_fetch:
964 case AtomicExpr::AO__atomic_nand_fetch:
965 case AtomicExpr::AO__atomic_or_fetch:
966 case AtomicExpr::AO__atomic_sub_fetch:
967 case AtomicExpr::AO__atomic_xor_fetch:
968 case AtomicExpr::AO__atomic_fetch_max:
969 case AtomicExpr::AO__atomic_fetch_min:
970 case AtomicExpr::AO__atomic_max_fetch:
971 case AtomicExpr::AO__atomic_min_fetch:
972 // For these, only library calls for certain sizes exist.
973 UseOptimizedLibcall = true;
974 break;
975
976 case AtomicExpr::AO__atomic_load:
977 case AtomicExpr::AO__atomic_store:
978 case AtomicExpr::AO__atomic_exchange:
979 case AtomicExpr::AO__atomic_compare_exchange:
980 // Use the generic version if we don't know that the operand will be
981 // suitably aligned for the optimized version.
982 if (Misaligned)
983 break;
984 LLVM_FALLTHROUGH;
985 case AtomicExpr::AO__c11_atomic_load:
986 case AtomicExpr::AO__c11_atomic_store:
987 case AtomicExpr::AO__c11_atomic_exchange:
988 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
989 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
990 case AtomicExpr::AO__opencl_atomic_load:
991 case AtomicExpr::AO__opencl_atomic_store:
992 case AtomicExpr::AO__opencl_atomic_exchange:
993 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
994 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
995 case AtomicExpr::AO__atomic_load_n:
996 case AtomicExpr::AO__atomic_store_n:
997 case AtomicExpr::AO__atomic_exchange_n:
998 case AtomicExpr::AO__atomic_compare_exchange_n:
999 // Only use optimized library calls for sizes for which they exist.
1000 // FIXME: Size == 16 optimized library functions exist too.
1001 if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1002 UseOptimizedLibcall = true;
1003 break;
1004 }
1005
1006 CallArgList Args;
1007 if (!UseOptimizedLibcall) {
1008 // For non-optimized library calls, the size is the first parameter
1009 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1010 getContext().getSizeType());
1011 }
1012 // Atomic address is the first or second parameter
1013 // The OpenCL atomic library functions only accept pointer arguments to
1014 // generic address space.
1015 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1016 if (!E->isOpenCL())
1017 return V;
1018 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1019 if (AS == LangAS::opencl_generic)
1020 return V;
1021 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1022 auto T = V->getType();
1023 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS);
1024
1025 return getTargetHooks().performAddrSpaceCast(
1026 *this, V, AS, LangAS::opencl_generic, DestType, false);
1027 };
1028
1029 Args.add(RValue::get(CastToGenericAddrSpace(
1030 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1031 getContext().VoidPtrTy);
1032
1033 std::string LibCallName;
1034 QualType LoweredMemTy =
1035 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1036 QualType RetTy;
1037 bool HaveRetTy = false;
1038 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1039 bool PostOpMinMax = false;
1040 switch (E->getOp()) {
1041 case AtomicExpr::AO__c11_atomic_init:
1042 case AtomicExpr::AO__opencl_atomic_init:
1043 llvm_unreachable("Already handled!");
1044
1045 // There is only one libcall for compare an exchange, because there is no
1046 // optimisation benefit possible from a libcall version of a weak compare
1047 // and exchange.
1048 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1049 // void *desired, int success, int failure)
1050 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1051 // int success, int failure)
1052 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1053 case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1054 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1055 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1056 case AtomicExpr::AO__atomic_compare_exchange:
1057 case AtomicExpr::AO__atomic_compare_exchange_n:
1058 LibCallName = "__atomic_compare_exchange";
1059 RetTy = getContext().BoolTy;
1060 HaveRetTy = true;
1061 Args.add(
1062 RValue::get(CastToGenericAddrSpace(
1063 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1064 getContext().VoidPtrTy);
1065 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1066 MemTy, E->getExprLoc(), sizeChars);
1067 Args.add(RValue::get(Order), getContext().IntTy);
1068 Order = OrderFail;
1069 break;
1070 // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1071 // int order)
1072 // T __atomic_exchange_N(T *mem, T val, int order)
1073 case AtomicExpr::AO__c11_atomic_exchange:
1074 case AtomicExpr::AO__opencl_atomic_exchange:
1075 case AtomicExpr::AO__atomic_exchange_n:
1076 case AtomicExpr::AO__atomic_exchange:
1077 LibCallName = "__atomic_exchange";
1078 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1079 MemTy, E->getExprLoc(), sizeChars);
1080 break;
1081 // void __atomic_store(size_t size, void *mem, void *val, int order)
1082 // void __atomic_store_N(T *mem, T val, int order)
1083 case AtomicExpr::AO__c11_atomic_store:
1084 case AtomicExpr::AO__opencl_atomic_store:
1085 case AtomicExpr::AO__atomic_store:
1086 case AtomicExpr::AO__atomic_store_n:
1087 LibCallName = "__atomic_store";
1088 RetTy = getContext().VoidTy;
1089 HaveRetTy = true;
1090 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1091 MemTy, E->getExprLoc(), sizeChars);
1092 break;
1093 // void __atomic_load(size_t size, void *mem, void *return, int order)
1094 // T __atomic_load_N(T *mem, int order)
1095 case AtomicExpr::AO__c11_atomic_load:
1096 case AtomicExpr::AO__opencl_atomic_load:
1097 case AtomicExpr::AO__atomic_load:
1098 case AtomicExpr::AO__atomic_load_n:
1099 LibCallName = "__atomic_load";
1100 break;
1101 // T __atomic_add_fetch_N(T *mem, T val, int order)
1102 // T __atomic_fetch_add_N(T *mem, T val, int order)
1103 case AtomicExpr::AO__atomic_add_fetch:
1104 PostOp = llvm::Instruction::Add;
1105 LLVM_FALLTHROUGH;
1106 case AtomicExpr::AO__c11_atomic_fetch_add:
1107 case AtomicExpr::AO__opencl_atomic_fetch_add:
1108 case AtomicExpr::AO__atomic_fetch_add:
1109 LibCallName = "__atomic_fetch_add";
1110 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1111 LoweredMemTy, E->getExprLoc(), sizeChars);
1112 break;
1113 // T __atomic_and_fetch_N(T *mem, T val, int order)
1114 // T __atomic_fetch_and_N(T *mem, T val, int order)
1115 case AtomicExpr::AO__atomic_and_fetch:
1116 PostOp = llvm::Instruction::And;
1117 LLVM_FALLTHROUGH;
1118 case AtomicExpr::AO__c11_atomic_fetch_and:
1119 case AtomicExpr::AO__opencl_atomic_fetch_and:
1120 case AtomicExpr::AO__atomic_fetch_and:
1121 LibCallName = "__atomic_fetch_and";
1122 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1123 MemTy, E->getExprLoc(), sizeChars);
1124 break;
1125 // T __atomic_or_fetch_N(T *mem, T val, int order)
1126 // T __atomic_fetch_or_N(T *mem, T val, int order)
1127 case AtomicExpr::AO__atomic_or_fetch:
1128 PostOp = llvm::Instruction::Or;
1129 LLVM_FALLTHROUGH;
1130 case AtomicExpr::AO__c11_atomic_fetch_or:
1131 case AtomicExpr::AO__opencl_atomic_fetch_or:
1132 case AtomicExpr::AO__atomic_fetch_or:
1133 LibCallName = "__atomic_fetch_or";
1134 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1135 MemTy, E->getExprLoc(), sizeChars);
1136 break;
1137 // T __atomic_sub_fetch_N(T *mem, T val, int order)
1138 // T __atomic_fetch_sub_N(T *mem, T val, int order)
1139 case AtomicExpr::AO__atomic_sub_fetch:
1140 PostOp = llvm::Instruction::Sub;
1141 LLVM_FALLTHROUGH;
1142 case AtomicExpr::AO__c11_atomic_fetch_sub:
1143 case AtomicExpr::AO__opencl_atomic_fetch_sub:
1144 case AtomicExpr::AO__atomic_fetch_sub:
1145 LibCallName = "__atomic_fetch_sub";
1146 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1147 LoweredMemTy, E->getExprLoc(), sizeChars);
1148 break;
1149 // T __atomic_xor_fetch_N(T *mem, T val, int order)
1150 // T __atomic_fetch_xor_N(T *mem, T val, int order)
1151 case AtomicExpr::AO__atomic_xor_fetch:
1152 PostOp = llvm::Instruction::Xor;
1153 LLVM_FALLTHROUGH;
1154 case AtomicExpr::AO__c11_atomic_fetch_xor:
1155 case AtomicExpr::AO__opencl_atomic_fetch_xor:
1156 case AtomicExpr::AO__atomic_fetch_xor:
1157 LibCallName = "__atomic_fetch_xor";
1158 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1159 MemTy, E->getExprLoc(), sizeChars);
1160 break;
1161 case AtomicExpr::AO__atomic_min_fetch:
1162 PostOpMinMax = true;
1163 LLVM_FALLTHROUGH;
1164 case AtomicExpr::AO__c11_atomic_fetch_min:
1165 case AtomicExpr::AO__atomic_fetch_min:
1166 case AtomicExpr::AO__opencl_atomic_fetch_min:
1167 LibCallName = E->getValueType()->isSignedIntegerType()
1168 ? "__atomic_fetch_min"
1169 : "__atomic_fetch_umin";
1170 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1171 LoweredMemTy, E->getExprLoc(), sizeChars);
1172 break;
1173 case AtomicExpr::AO__atomic_max_fetch:
1174 PostOpMinMax = true;
1175 LLVM_FALLTHROUGH;
1176 case AtomicExpr::AO__c11_atomic_fetch_max:
1177 case AtomicExpr::AO__atomic_fetch_max:
1178 case AtomicExpr::AO__opencl_atomic_fetch_max:
1179 LibCallName = E->getValueType()->isSignedIntegerType()
1180 ? "__atomic_fetch_max"
1181 : "__atomic_fetch_umax";
1182 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1183 LoweredMemTy, E->getExprLoc(), sizeChars);
1184 break;
1185 // T __atomic_nand_fetch_N(T *mem, T val, int order)
1186 // T __atomic_fetch_nand_N(T *mem, T val, int order)
1187 case AtomicExpr::AO__atomic_nand_fetch:
1188 PostOp = llvm::Instruction::And; // the NOT is special cased below
1189 LLVM_FALLTHROUGH;
1190 case AtomicExpr::AO__atomic_fetch_nand:
1191 LibCallName = "__atomic_fetch_nand";
1192 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1193 MemTy, E->getExprLoc(), sizeChars);
1194 break;
1195 }
1196
1197 if (E->isOpenCL()) {
1198 LibCallName = std::string("__opencl") +
1199 StringRef(LibCallName).drop_front(1).str();
1200
1201 }
1202 // Optimized functions have the size in their name.
1203 if (UseOptimizedLibcall)
1204 LibCallName += "_" + llvm::utostr(Size);
1205 // By default, assume we return a value of the atomic type.
1206 if (!HaveRetTy) {
1207 if (UseOptimizedLibcall) {
1208 // Value is returned directly.
1209 // The function returns an appropriately sized integer type.
1210 RetTy = getContext().getIntTypeForBitwidth(
1211 getContext().toBits(sizeChars), /*Signed=*/false);
1212 } else {
1213 // Value is returned through parameter before the order.
1214 RetTy = getContext().VoidTy;
1215 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1216 getContext().VoidPtrTy);
1217 }
1218 }
1219 // order is always the last parameter
1220 Args.add(RValue::get(Order),
1221 getContext().IntTy);
1222 if (E->isOpenCL())
1223 Args.add(RValue::get(Scope), getContext().IntTy);
1224
1225 // PostOp is only needed for the atomic_*_fetch operations, and
1226 // thus is only needed for and implemented in the
1227 // UseOptimizedLibcall codepath.
1228 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1229
1230 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1231 // The value is returned directly from the libcall.
1232 if (E->isCmpXChg())
1233 return Res;
1234
1235 // The value is returned directly for optimized libcalls but the expr
1236 // provided an out-param.
1237 if (UseOptimizedLibcall && Res.getScalarVal()) {
1238 llvm::Value *ResVal = Res.getScalarVal();
1239 if (PostOpMinMax) {
1240 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1241 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1242 E->getValueType()->isSignedIntegerType(),
1243 ResVal, LoadVal1);
1244 } else if (PostOp) {
1245 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1246 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1247 }
1248 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1249 ResVal = Builder.CreateNot(ResVal);
1250
1251 Builder.CreateStore(
1252 ResVal,
1253 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1254 }
1255
1256 if (RValTy->isVoidType())
1257 return RValue::get(nullptr);
1258
1259 return convertTempToRValue(
1260 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1261 RValTy, E->getExprLoc());
1262 }
1263
1264 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1265 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1266 E->getOp() == AtomicExpr::AO__atomic_store ||
1267 E->getOp() == AtomicExpr::AO__atomic_store_n;
1268 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1269 E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1270 E->getOp() == AtomicExpr::AO__atomic_load ||
1271 E->getOp() == AtomicExpr::AO__atomic_load_n;
1272
1273 if (isa<llvm::ConstantInt>(Order)) {
1274 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1275 // We should not ever get to a case where the ordering isn't a valid C ABI
1276 // value, but it's hard to enforce that in general.
1277 if (llvm::isValidAtomicOrderingCABI(ord))
1278 switch ((llvm::AtomicOrderingCABI)ord) {
1279 case llvm::AtomicOrderingCABI::relaxed:
1280 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1281 llvm::AtomicOrdering::Monotonic, Scope);
1282 break;
1283 case llvm::AtomicOrderingCABI::consume:
1284 case llvm::AtomicOrderingCABI::acquire:
1285 if (IsStore)
1286 break; // Avoid crashing on code with undefined behavior
1287 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1288 llvm::AtomicOrdering::Acquire, Scope);
1289 break;
1290 case llvm::AtomicOrderingCABI::release:
1291 if (IsLoad)
1292 break; // Avoid crashing on code with undefined behavior
1293 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1294 llvm::AtomicOrdering::Release, Scope);
1295 break;
1296 case llvm::AtomicOrderingCABI::acq_rel:
1297 if (IsLoad || IsStore)
1298 break; // Avoid crashing on code with undefined behavior
1299 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1300 llvm::AtomicOrdering::AcquireRelease, Scope);
1301 break;
1302 case llvm::AtomicOrderingCABI::seq_cst:
1303 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1304 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1305 break;
1306 }
1307 if (RValTy->isVoidType())
1308 return RValue::get(nullptr);
1309
1310 return convertTempToRValue(
1311 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1312 Dest.getAddressSpace())),
1313 RValTy, E->getExprLoc());
1314 }
1315
1316 // Long case, when Order isn't obviously constant.
1317
1318 // Create all the relevant BB's
1319 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1320 *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1321 *SeqCstBB = nullptr;
1322 MonotonicBB = createBasicBlock("monotonic", CurFn);
1323 if (!IsStore)
1324 AcquireBB = createBasicBlock("acquire", CurFn);
1325 if (!IsLoad)
1326 ReleaseBB = createBasicBlock("release", CurFn);
1327 if (!IsLoad && !IsStore)
1328 AcqRelBB = createBasicBlock("acqrel", CurFn);
1329 SeqCstBB = createBasicBlock("seqcst", CurFn);
1330 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1331
1332 // Create the switch for the split
1333 // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1334 // doesn't matter unless someone is crazy enough to use something that
1335 // doesn't fold to a constant for the ordering.
1336 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1337 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1338
1339 // Emit all the different atomics
1340 Builder.SetInsertPoint(MonotonicBB);
1341 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1342 llvm::AtomicOrdering::Monotonic, Scope);
1343 Builder.CreateBr(ContBB);
1344 if (!IsStore) {
1345 Builder.SetInsertPoint(AcquireBB);
1346 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1347 llvm::AtomicOrdering::Acquire, Scope);
1348 Builder.CreateBr(ContBB);
1349 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1350 AcquireBB);
1351 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1352 AcquireBB);
1353 }
1354 if (!IsLoad) {
1355 Builder.SetInsertPoint(ReleaseBB);
1356 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1357 llvm::AtomicOrdering::Release, Scope);
1358 Builder.CreateBr(ContBB);
1359 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1360 ReleaseBB);
1361 }
1362 if (!IsLoad && !IsStore) {
1363 Builder.SetInsertPoint(AcqRelBB);
1364 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1365 llvm::AtomicOrdering::AcquireRelease, Scope);
1366 Builder.CreateBr(ContBB);
1367 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1368 AcqRelBB);
1369 }
1370 Builder.SetInsertPoint(SeqCstBB);
1371 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1372 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1373 Builder.CreateBr(ContBB);
1374 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1375 SeqCstBB);
1376
1377 // Cleanup and return
1378 Builder.SetInsertPoint(ContBB);
1379 if (RValTy->isVoidType())
1380 return RValue::get(nullptr);
1381
1382 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1383 return convertTempToRValue(
1384 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo(
1385 Dest.getAddressSpace())),
1386 RValTy, E->getExprLoc());
1387 }
1388
emitCastToAtomicIntPointer(Address addr) const1389 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1390 unsigned addrspace =
1391 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1392 llvm::IntegerType *ty =
1393 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1394 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1395 }
1396
convertToAtomicIntPointer(Address Addr) const1397 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1398 llvm::Type *Ty = Addr.getElementType();
1399 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1400 if (SourceSizeInBits != AtomicSizeInBits) {
1401 Address Tmp = CreateTempAlloca();
1402 CGF.Builder.CreateMemCpy(Tmp, Addr,
1403 std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1404 Addr = Tmp;
1405 }
1406
1407 return emitCastToAtomicIntPointer(Addr);
1408 }
1409
convertAtomicTempToRValue(Address addr,AggValueSlot resultSlot,SourceLocation loc,bool asValue) const1410 RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1411 AggValueSlot resultSlot,
1412 SourceLocation loc,
1413 bool asValue) const {
1414 if (LVal.isSimple()) {
1415 if (EvaluationKind == TEK_Aggregate)
1416 return resultSlot.asRValue();
1417
1418 // Drill into the padding structure if we have one.
1419 if (hasPadding())
1420 addr = CGF.Builder.CreateStructGEP(addr, 0);
1421
1422 // Otherwise, just convert the temporary to an r-value using the
1423 // normal conversion routine.
1424 return CGF.convertTempToRValue(addr, getValueType(), loc);
1425 }
1426 if (!asValue)
1427 // Get RValue from temp memory as atomic for non-simple lvalues
1428 return RValue::get(CGF.Builder.CreateLoad(addr));
1429 if (LVal.isBitField())
1430 return CGF.EmitLoadOfBitfieldLValue(
1431 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1432 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1433 if (LVal.isVectorElt())
1434 return CGF.EmitLoadOfLValue(
1435 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1436 LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1437 assert(LVal.isExtVectorElt());
1438 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1439 addr, LVal.getExtVectorElts(), LVal.getType(),
1440 LVal.getBaseInfo(), TBAAAccessInfo()));
1441 }
1442
ConvertIntToValueOrAtomic(llvm::Value * IntVal,AggValueSlot ResultSlot,SourceLocation Loc,bool AsValue) const1443 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1444 AggValueSlot ResultSlot,
1445 SourceLocation Loc,
1446 bool AsValue) const {
1447 // Try not to in some easy cases.
1448 assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1449 if (getEvaluationKind() == TEK_Scalar &&
1450 (((!LVal.isBitField() ||
1451 LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1452 !hasPadding()) ||
1453 !AsValue)) {
1454 auto *ValTy = AsValue
1455 ? CGF.ConvertTypeForMem(ValueTy)
1456 : getAtomicAddress().getType()->getPointerElementType();
1457 if (ValTy->isIntegerTy()) {
1458 assert(IntVal->getType() == ValTy && "Different integer types.");
1459 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1460 } else if (ValTy->isPointerTy())
1461 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1462 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1463 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1464 }
1465
1466 // Create a temporary. This needs to be big enough to hold the
1467 // atomic integer.
1468 Address Temp = Address::invalid();
1469 bool TempIsVolatile = false;
1470 if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1471 assert(!ResultSlot.isIgnored());
1472 Temp = ResultSlot.getAddress();
1473 TempIsVolatile = ResultSlot.isVolatile();
1474 } else {
1475 Temp = CreateTempAlloca();
1476 }
1477
1478 // Slam the integer into the temporary.
1479 Address CastTemp = emitCastToAtomicIntPointer(Temp);
1480 CGF.Builder.CreateStore(IntVal, CastTemp)
1481 ->setVolatile(TempIsVolatile);
1482
1483 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1484 }
1485
EmitAtomicLoadLibcall(llvm::Value * AddForLoaded,llvm::AtomicOrdering AO,bool)1486 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1487 llvm::AtomicOrdering AO, bool) {
1488 // void __atomic_load(size_t size, void *mem, void *return, int order);
1489 CallArgList Args;
1490 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1491 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1492 CGF.getContext().VoidPtrTy);
1493 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1494 CGF.getContext().VoidPtrTy);
1495 Args.add(
1496 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1497 CGF.getContext().IntTy);
1498 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1499 }
1500
EmitAtomicLoadOp(llvm::AtomicOrdering AO,bool IsVolatile)1501 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1502 bool IsVolatile) {
1503 // Okay, we're doing this natively.
1504 Address Addr = getAtomicAddressAsAtomicIntPointer();
1505 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1506 Load->setAtomic(AO);
1507
1508 // Other decoration.
1509 if (IsVolatile)
1510 Load->setVolatile(true);
1511 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1512 return Load;
1513 }
1514
1515 /// An LValue is a candidate for having its loads and stores be made atomic if
1516 /// we are operating under /volatile:ms *and* the LValue itself is volatile and
1517 /// performing such an operation can be performed without a libcall.
LValueIsSuitableForInlineAtomic(LValue LV)1518 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1519 if (!CGM.getCodeGenOpts().MSVolatile) return false;
1520 AtomicInfo AI(*this, LV);
1521 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1522 // An atomic is inline if we don't need to use a libcall.
1523 bool AtomicIsInline = !AI.shouldUseLibcall();
1524 // MSVC doesn't seem to do this for types wider than a pointer.
1525 if (getContext().getTypeSize(LV.getType()) >
1526 getContext().getTypeSize(getContext().getIntPtrType()))
1527 return false;
1528 return IsVolatile && AtomicIsInline;
1529 }
1530
EmitAtomicLoad(LValue LV,SourceLocation SL,AggValueSlot Slot)1531 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1532 AggValueSlot Slot) {
1533 llvm::AtomicOrdering AO;
1534 bool IsVolatile = LV.isVolatileQualified();
1535 if (LV.getType()->isAtomicType()) {
1536 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1537 } else {
1538 AO = llvm::AtomicOrdering::Acquire;
1539 IsVolatile = true;
1540 }
1541 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1542 }
1543
EmitAtomicLoad(AggValueSlot ResultSlot,SourceLocation Loc,bool AsValue,llvm::AtomicOrdering AO,bool IsVolatile)1544 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1545 bool AsValue, llvm::AtomicOrdering AO,
1546 bool IsVolatile) {
1547 // Check whether we should use a library call.
1548 if (shouldUseLibcall()) {
1549 Address TempAddr = Address::invalid();
1550 if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1551 assert(getEvaluationKind() == TEK_Aggregate);
1552 TempAddr = ResultSlot.getAddress();
1553 } else
1554 TempAddr = CreateTempAlloca();
1555
1556 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1557
1558 // Okay, turn that back into the original value or whole atomic (for
1559 // non-simple lvalues) type.
1560 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1561 }
1562
1563 // Okay, we're doing this natively.
1564 auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1565
1566 // If we're ignoring an aggregate return, don't do anything.
1567 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1568 return RValue::getAggregate(Address::invalid(), false);
1569
1570 // Okay, turn that back into the original value or atomic (for non-simple
1571 // lvalues) type.
1572 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1573 }
1574
1575 /// Emit a load from an l-value of atomic type. Note that the r-value
1576 /// we produce is an r-value of the atomic *value* type.
EmitAtomicLoad(LValue src,SourceLocation loc,llvm::AtomicOrdering AO,bool IsVolatile,AggValueSlot resultSlot)1577 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1578 llvm::AtomicOrdering AO, bool IsVolatile,
1579 AggValueSlot resultSlot) {
1580 AtomicInfo Atomics(*this, src);
1581 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1582 IsVolatile);
1583 }
1584
1585 /// Copy an r-value into memory as part of storing to an atomic type.
1586 /// This needs to create a bit-pattern suitable for atomic operations.
emitCopyIntoMemory(RValue rvalue) const1587 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1588 assert(LVal.isSimple());
1589 // If we have an r-value, the rvalue should be of the atomic type,
1590 // which means that the caller is responsible for having zeroed
1591 // any padding. Just do an aggregate copy of that type.
1592 if (rvalue.isAggregate()) {
1593 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1594 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1595 getAtomicType());
1596 bool IsVolatile = rvalue.isVolatileQualified() ||
1597 LVal.isVolatileQualified();
1598 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1599 AggValueSlot::DoesNotOverlap, IsVolatile);
1600 return;
1601 }
1602
1603 // Okay, otherwise we're copying stuff.
1604
1605 // Zero out the buffer if necessary.
1606 emitMemSetZeroIfNecessary();
1607
1608 // Drill past the padding if present.
1609 LValue TempLVal = projectValue();
1610
1611 // Okay, store the rvalue in.
1612 if (rvalue.isScalar()) {
1613 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1614 } else {
1615 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1616 }
1617 }
1618
1619
1620 /// Materialize an r-value into memory for the purposes of storing it
1621 /// to an atomic type.
materializeRValue(RValue rvalue) const1622 Address AtomicInfo::materializeRValue(RValue rvalue) const {
1623 // Aggregate r-values are already in memory, and EmitAtomicStore
1624 // requires them to be values of the atomic type.
1625 if (rvalue.isAggregate())
1626 return rvalue.getAggregateAddress();
1627
1628 // Otherwise, make a temporary and materialize into it.
1629 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1630 AtomicInfo Atomics(CGF, TempLV);
1631 Atomics.emitCopyIntoMemory(rvalue);
1632 return TempLV.getAddress(CGF);
1633 }
1634
convertRValueToInt(RValue RVal) const1635 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1636 // If we've got a scalar value of the right size, try to avoid going
1637 // through memory.
1638 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1639 llvm::Value *Value = RVal.getScalarVal();
1640 if (isa<llvm::IntegerType>(Value->getType()))
1641 return CGF.EmitToMemory(Value, ValueTy);
1642 else {
1643 llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1644 CGF.getLLVMContext(),
1645 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1646 if (isa<llvm::PointerType>(Value->getType()))
1647 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1648 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1649 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1650 }
1651 }
1652 // Otherwise, we need to go through memory.
1653 // Put the r-value in memory.
1654 Address Addr = materializeRValue(RVal);
1655
1656 // Cast the temporary to the atomic int type and pull a value out.
1657 Addr = emitCastToAtomicIntPointer(Addr);
1658 return CGF.Builder.CreateLoad(Addr);
1659 }
1660
EmitAtomicCompareExchangeOp(llvm::Value * ExpectedVal,llvm::Value * DesiredVal,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak)1661 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1662 llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1663 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1664 // Do the atomic store.
1665 Address Addr = getAtomicAddressAsAtomicIntPointer();
1666 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1667 ExpectedVal, DesiredVal,
1668 Success, Failure);
1669 // Other decoration.
1670 Inst->setVolatile(LVal.isVolatileQualified());
1671 Inst->setWeak(IsWeak);
1672
1673 // Okay, turn that back into the original value type.
1674 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1675 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1676 return std::make_pair(PreviousVal, SuccessFailureVal);
1677 }
1678
1679 llvm::Value *
EmitAtomicCompareExchangeLibcall(llvm::Value * ExpectedAddr,llvm::Value * DesiredAddr,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure)1680 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1681 llvm::Value *DesiredAddr,
1682 llvm::AtomicOrdering Success,
1683 llvm::AtomicOrdering Failure) {
1684 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1685 // void *desired, int success, int failure);
1686 CallArgList Args;
1687 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1688 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1689 CGF.getContext().VoidPtrTy);
1690 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1691 CGF.getContext().VoidPtrTy);
1692 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1693 CGF.getContext().VoidPtrTy);
1694 Args.add(RValue::get(
1695 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1696 CGF.getContext().IntTy);
1697 Args.add(RValue::get(
1698 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1699 CGF.getContext().IntTy);
1700 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1701 CGF.getContext().BoolTy, Args);
1702
1703 return SuccessFailureRVal.getScalarVal();
1704 }
1705
EmitAtomicCompareExchange(RValue Expected,RValue Desired,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak)1706 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1707 RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1708 llvm::AtomicOrdering Failure, bool IsWeak) {
1709 if (isStrongerThan(Failure, Success))
1710 // Don't assert on undefined behavior "failure argument shall be no stronger
1711 // than the success argument".
1712 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1713
1714 // Check whether we should use a library call.
1715 if (shouldUseLibcall()) {
1716 // Produce a source address.
1717 Address ExpectedAddr = materializeRValue(Expected);
1718 Address DesiredAddr = materializeRValue(Desired);
1719 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1720 DesiredAddr.getPointer(),
1721 Success, Failure);
1722 return std::make_pair(
1723 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1724 SourceLocation(), /*AsValue=*/false),
1725 Res);
1726 }
1727
1728 // If we've got a scalar value of the right size, try to avoid going
1729 // through memory.
1730 auto *ExpectedVal = convertRValueToInt(Expected);
1731 auto *DesiredVal = convertRValueToInt(Desired);
1732 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1733 Failure, IsWeak);
1734 return std::make_pair(
1735 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1736 SourceLocation(), /*AsValue=*/false),
1737 Res.second);
1738 }
1739
1740 static void
EmitAtomicUpdateValue(CodeGenFunction & CGF,AtomicInfo & Atomics,RValue OldRVal,const llvm::function_ref<RValue (RValue)> & UpdateOp,Address DesiredAddr)1741 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1742 const llvm::function_ref<RValue(RValue)> &UpdateOp,
1743 Address DesiredAddr) {
1744 RValue UpRVal;
1745 LValue AtomicLVal = Atomics.getAtomicLValue();
1746 LValue DesiredLVal;
1747 if (AtomicLVal.isSimple()) {
1748 UpRVal = OldRVal;
1749 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1750 } else {
1751 // Build new lvalue for temp address.
1752 Address Ptr = Atomics.materializeRValue(OldRVal);
1753 LValue UpdateLVal;
1754 if (AtomicLVal.isBitField()) {
1755 UpdateLVal =
1756 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1757 AtomicLVal.getType(),
1758 AtomicLVal.getBaseInfo(),
1759 AtomicLVal.getTBAAInfo());
1760 DesiredLVal =
1761 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1762 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1763 AtomicLVal.getTBAAInfo());
1764 } else if (AtomicLVal.isVectorElt()) {
1765 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1766 AtomicLVal.getType(),
1767 AtomicLVal.getBaseInfo(),
1768 AtomicLVal.getTBAAInfo());
1769 DesiredLVal = LValue::MakeVectorElt(
1770 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1771 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1772 } else {
1773 assert(AtomicLVal.isExtVectorElt());
1774 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1775 AtomicLVal.getType(),
1776 AtomicLVal.getBaseInfo(),
1777 AtomicLVal.getTBAAInfo());
1778 DesiredLVal = LValue::MakeExtVectorElt(
1779 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1780 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1781 }
1782 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1783 }
1784 // Store new value in the corresponding memory area.
1785 RValue NewRVal = UpdateOp(UpRVal);
1786 if (NewRVal.isScalar()) {
1787 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1788 } else {
1789 assert(NewRVal.isComplex());
1790 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1791 /*isInit=*/false);
1792 }
1793 }
1794
EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1795 void AtomicInfo::EmitAtomicUpdateLibcall(
1796 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1797 bool IsVolatile) {
1798 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1799
1800 Address ExpectedAddr = CreateTempAlloca();
1801
1802 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1803 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1804 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1805 CGF.EmitBlock(ContBB);
1806 Address DesiredAddr = CreateTempAlloca();
1807 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1808 requiresMemSetZero(getAtomicAddress().getElementType())) {
1809 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1810 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1811 }
1812 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1813 AggValueSlot::ignored(),
1814 SourceLocation(), /*AsValue=*/false);
1815 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1816 auto *Res =
1817 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1818 DesiredAddr.getPointer(),
1819 AO, Failure);
1820 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1821 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1822 }
1823
EmitAtomicUpdateOp(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1824 void AtomicInfo::EmitAtomicUpdateOp(
1825 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1826 bool IsVolatile) {
1827 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1828
1829 // Do the atomic load.
1830 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1831 // For non-simple lvalues perform compare-and-swap procedure.
1832 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1833 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1834 auto *CurBB = CGF.Builder.GetInsertBlock();
1835 CGF.EmitBlock(ContBB);
1836 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1837 /*NumReservedValues=*/2);
1838 PHI->addIncoming(OldVal, CurBB);
1839 Address NewAtomicAddr = CreateTempAlloca();
1840 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1841 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1842 requiresMemSetZero(getAtomicAddress().getElementType())) {
1843 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1844 }
1845 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1846 SourceLocation(), /*AsValue=*/false);
1847 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1848 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1849 // Try to write new value using cmpxchg operation.
1850 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1851 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1852 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1853 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1854 }
1855
EmitAtomicUpdateValue(CodeGenFunction & CGF,AtomicInfo & Atomics,RValue UpdateRVal,Address DesiredAddr)1856 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1857 RValue UpdateRVal, Address DesiredAddr) {
1858 LValue AtomicLVal = Atomics.getAtomicLValue();
1859 LValue DesiredLVal;
1860 // Build new lvalue for temp address.
1861 if (AtomicLVal.isBitField()) {
1862 DesiredLVal =
1863 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1864 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1865 AtomicLVal.getTBAAInfo());
1866 } else if (AtomicLVal.isVectorElt()) {
1867 DesiredLVal =
1868 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1869 AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1870 AtomicLVal.getTBAAInfo());
1871 } else {
1872 assert(AtomicLVal.isExtVectorElt());
1873 DesiredLVal = LValue::MakeExtVectorElt(
1874 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1875 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1876 }
1877 // Store new value in the corresponding memory area.
1878 assert(UpdateRVal.isScalar());
1879 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1880 }
1881
EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1882 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1883 RValue UpdateRVal, bool IsVolatile) {
1884 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1885
1886 Address ExpectedAddr = CreateTempAlloca();
1887
1888 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1889 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1890 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1891 CGF.EmitBlock(ContBB);
1892 Address DesiredAddr = CreateTempAlloca();
1893 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1894 requiresMemSetZero(getAtomicAddress().getElementType())) {
1895 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1896 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1897 }
1898 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1899 auto *Res =
1900 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1901 DesiredAddr.getPointer(),
1902 AO, Failure);
1903 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1904 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1905 }
1906
EmitAtomicUpdateOp(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1907 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1908 bool IsVolatile) {
1909 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1910
1911 // Do the atomic load.
1912 auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1913 // For non-simple lvalues perform compare-and-swap procedure.
1914 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1915 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1916 auto *CurBB = CGF.Builder.GetInsertBlock();
1917 CGF.EmitBlock(ContBB);
1918 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1919 /*NumReservedValues=*/2);
1920 PHI->addIncoming(OldVal, CurBB);
1921 Address NewAtomicAddr = CreateTempAlloca();
1922 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1923 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1924 requiresMemSetZero(getAtomicAddress().getElementType())) {
1925 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1926 }
1927 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1928 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1929 // Try to write new value using cmpxchg operation.
1930 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1931 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1932 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1933 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1934 }
1935
EmitAtomicUpdate(llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)1936 void AtomicInfo::EmitAtomicUpdate(
1937 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1938 bool IsVolatile) {
1939 if (shouldUseLibcall()) {
1940 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1941 } else {
1942 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1943 }
1944 }
1945
EmitAtomicUpdate(llvm::AtomicOrdering AO,RValue UpdateRVal,bool IsVolatile)1946 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1947 bool IsVolatile) {
1948 if (shouldUseLibcall()) {
1949 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1950 } else {
1951 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1952 }
1953 }
1954
EmitAtomicStore(RValue rvalue,LValue lvalue,bool isInit)1955 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1956 bool isInit) {
1957 bool IsVolatile = lvalue.isVolatileQualified();
1958 llvm::AtomicOrdering AO;
1959 if (lvalue.getType()->isAtomicType()) {
1960 AO = llvm::AtomicOrdering::SequentiallyConsistent;
1961 } else {
1962 AO = llvm::AtomicOrdering::Release;
1963 IsVolatile = true;
1964 }
1965 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1966 }
1967
1968 /// Emit a store to an l-value of atomic type.
1969 ///
1970 /// Note that the r-value is expected to be an r-value *of the atomic
1971 /// type*; this means that for aggregate r-values, it should include
1972 /// storage for any padding that was necessary.
EmitAtomicStore(RValue rvalue,LValue dest,llvm::AtomicOrdering AO,bool IsVolatile,bool isInit)1973 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1974 llvm::AtomicOrdering AO, bool IsVolatile,
1975 bool isInit) {
1976 // If this is an aggregate r-value, it should agree in type except
1977 // maybe for address-space qualification.
1978 assert(!rvalue.isAggregate() ||
1979 rvalue.getAggregateAddress().getElementType() ==
1980 dest.getAddress(*this).getElementType());
1981
1982 AtomicInfo atomics(*this, dest);
1983 LValue LVal = atomics.getAtomicLValue();
1984
1985 // If this is an initialization, just put the value there normally.
1986 if (LVal.isSimple()) {
1987 if (isInit) {
1988 atomics.emitCopyIntoMemory(rvalue);
1989 return;
1990 }
1991
1992 // Check whether we should use a library call.
1993 if (atomics.shouldUseLibcall()) {
1994 // Produce a source address.
1995 Address srcAddr = atomics.materializeRValue(rvalue);
1996
1997 // void __atomic_store(size_t size, void *mem, void *val, int order)
1998 CallArgList args;
1999 args.add(RValue::get(atomics.getAtomicSizeValue()),
2000 getContext().getSizeType());
2001 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2002 getContext().VoidPtrTy);
2003 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2004 getContext().VoidPtrTy);
2005 args.add(
2006 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2007 getContext().IntTy);
2008 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2009 return;
2010 }
2011
2012 // Okay, we're doing this natively.
2013 llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2014
2015 // Do the atomic store.
2016 Address addr =
2017 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2018 intValue = Builder.CreateIntCast(
2019 intValue, addr.getElementType(), /*isSigned=*/false);
2020 llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2021
2022 if (AO == llvm::AtomicOrdering::Acquire)
2023 AO = llvm::AtomicOrdering::Monotonic;
2024 else if (AO == llvm::AtomicOrdering::AcquireRelease)
2025 AO = llvm::AtomicOrdering::Release;
2026 // Initializations don't need to be atomic.
2027 if (!isInit)
2028 store->setAtomic(AO);
2029
2030 // Other decoration.
2031 if (IsVolatile)
2032 store->setVolatile(true);
2033 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2034 return;
2035 }
2036
2037 // Emit simple atomic update operation.
2038 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2039 }
2040
2041 /// Emit a compare-and-exchange op for atomic type.
2042 ///
EmitAtomicCompareExchange(LValue Obj,RValue Expected,RValue Desired,SourceLocation Loc,llvm::AtomicOrdering Success,llvm::AtomicOrdering Failure,bool IsWeak,AggValueSlot Slot)2043 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2044 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2045 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2046 AggValueSlot Slot) {
2047 // If this is an aggregate r-value, it should agree in type except
2048 // maybe for address-space qualification.
2049 assert(!Expected.isAggregate() ||
2050 Expected.getAggregateAddress().getElementType() ==
2051 Obj.getAddress(*this).getElementType());
2052 assert(!Desired.isAggregate() ||
2053 Desired.getAggregateAddress().getElementType() ==
2054 Obj.getAddress(*this).getElementType());
2055 AtomicInfo Atomics(*this, Obj);
2056
2057 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2058 IsWeak);
2059 }
2060
EmitAtomicUpdate(LValue LVal,llvm::AtomicOrdering AO,const llvm::function_ref<RValue (RValue)> & UpdateOp,bool IsVolatile)2061 void CodeGenFunction::EmitAtomicUpdate(
2062 LValue LVal, llvm::AtomicOrdering AO,
2063 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2064 AtomicInfo Atomics(*this, LVal);
2065 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2066 }
2067
EmitAtomicInit(Expr * init,LValue dest)2068 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2069 AtomicInfo atomics(*this, dest);
2070
2071 switch (atomics.getEvaluationKind()) {
2072 case TEK_Scalar: {
2073 llvm::Value *value = EmitScalarExpr(init);
2074 atomics.emitCopyIntoMemory(RValue::get(value));
2075 return;
2076 }
2077
2078 case TEK_Complex: {
2079 ComplexPairTy value = EmitComplexExpr(init);
2080 atomics.emitCopyIntoMemory(RValue::getComplex(value));
2081 return;
2082 }
2083
2084 case TEK_Aggregate: {
2085 // Fix up the destination if the initializer isn't an expression
2086 // of atomic type.
2087 bool Zeroed = false;
2088 if (!init->getType()->isAtomicType()) {
2089 Zeroed = atomics.emitMemSetZeroIfNecessary();
2090 dest = atomics.projectValue();
2091 }
2092
2093 // Evaluate the expression directly into the destination.
2094 AggValueSlot slot = AggValueSlot::forLValue(
2095 dest, *this, AggValueSlot::IsNotDestructed,
2096 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2097 AggValueSlot::DoesNotOverlap,
2098 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2099
2100 EmitAggExpr(init, slot);
2101 return;
2102 }
2103 }
2104 llvm_unreachable("bad evaluation kind");
2105 }
2106