1 //===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file contains the code for emitting atomic operations. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGCall.h" 14 #include "CGRecordLayout.h" 15 #include "CodeGenFunction.h" 16 #include "CodeGenModule.h" 17 #include "TargetInfo.h" 18 #include "clang/AST/ASTContext.h" 19 #include "clang/CodeGen/CGFunctionInfo.h" 20 #include "clang/Frontend/FrontendDiagnostic.h" 21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/IR/DataLayout.h" 23 #include "llvm/IR/Intrinsics.h" 24 #include "llvm/IR/Operator.h" 25 26 using namespace clang; 27 using namespace CodeGen; 28 29 namespace { 30 class AtomicInfo { 31 CodeGenFunction &CGF; 32 QualType AtomicTy; 33 QualType ValueTy; 34 uint64_t AtomicSizeInBits; 35 uint64_t ValueSizeInBits; 36 CharUnits AtomicAlign; 37 CharUnits ValueAlign; 38 TypeEvaluationKind EvaluationKind; 39 bool UseLibcall; 40 LValue LVal; 41 CGBitFieldInfo BFI; 42 public: 43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 45 EvaluationKind(TEK_Scalar), UseLibcall(true) { 46 assert(!lvalue.isGlobalReg()); 47 ASTContext &C = CGF.getContext(); 48 if (lvalue.isSimple()) { 49 AtomicTy = lvalue.getType(); 50 if (auto *ATy = AtomicTy->getAs<AtomicType>()) 51 ValueTy = ATy->getValueType(); 52 else 53 ValueTy = AtomicTy; 54 EvaluationKind = CGF.getEvaluationKind(ValueTy); 55 56 uint64_t ValueAlignInBits; 57 uint64_t AtomicAlignInBits; 58 TypeInfo ValueTI = C.getTypeInfo(ValueTy); 59 ValueSizeInBits = ValueTI.Width; 60 ValueAlignInBits = ValueTI.Align; 61 62 TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 63 AtomicSizeInBits = AtomicTI.Width; 64 AtomicAlignInBits = AtomicTI.Align; 65 66 assert(ValueSizeInBits <= AtomicSizeInBits); 67 assert(ValueAlignInBits <= AtomicAlignInBits); 68 69 AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 70 ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 71 if (lvalue.getAlignment().isZero()) 72 lvalue.setAlignment(AtomicAlign); 73 74 LVal = lvalue; 75 } else if (lvalue.isBitField()) { 76 ValueTy = lvalue.getType(); 77 ValueSizeInBits = C.getTypeSize(ValueTy); 78 auto &OrigBFI = lvalue.getBitFieldInfo(); 79 auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 80 AtomicSizeInBits = C.toBits( 81 C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 82 .alignTo(lvalue.getAlignment())); 83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 84 auto OffsetInChars = 85 (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 86 lvalue.getAlignment(); 87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 88 VoidPtrAddr, OffsetInChars.getQuantity()); 89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 90 VoidPtrAddr, 91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 92 "atomic_bitfield_base"); 93 BFI = OrigBFI; 94 BFI.Offset = Offset; 95 BFI.StorageSize = AtomicSizeInBits; 96 BFI.StorageOffset += OffsetInChars; 97 LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 98 BFI, lvalue.getType(), lvalue.getBaseInfo(), 99 lvalue.getTBAAInfo()); 100 AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 101 if (AtomicTy.isNull()) { 102 llvm::APInt Size( 103 /*numBits=*/32, 104 C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 105 AtomicTy = 106 C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal, 107 /*IndexTypeQuals=*/0); 108 } 109 AtomicAlign = ValueAlign = lvalue.getAlignment(); 110 } else if (lvalue.isVectorElt()) { 111 ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType(); 112 ValueSizeInBits = C.getTypeSize(ValueTy); 113 AtomicTy = lvalue.getType(); 114 AtomicSizeInBits = C.getTypeSize(AtomicTy); 115 AtomicAlign = ValueAlign = lvalue.getAlignment(); 116 LVal = lvalue; 117 } else { 118 assert(lvalue.isExtVectorElt()); 119 ValueTy = lvalue.getType(); 120 ValueSizeInBits = C.getTypeSize(ValueTy); 121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 122 lvalue.getType(), lvalue.getExtVectorAddress() 123 .getElementType()->getVectorNumElements()); 124 AtomicSizeInBits = C.getTypeSize(AtomicTy); 125 AtomicAlign = ValueAlign = lvalue.getAlignment(); 126 LVal = lvalue; 127 } 128 UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 129 AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 130 } 131 132 QualType getAtomicType() const { return AtomicTy; } 133 QualType getValueType() const { return ValueTy; } 134 CharUnits getAtomicAlignment() const { return AtomicAlign; } 135 uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 136 uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 137 TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 138 bool shouldUseLibcall() const { return UseLibcall; } 139 const LValue &getAtomicLValue() const { return LVal; } 140 llvm::Value *getAtomicPointer() const { 141 if (LVal.isSimple()) 142 return LVal.getPointer(CGF); 143 else if (LVal.isBitField()) 144 return LVal.getBitFieldPointer(); 145 else if (LVal.isVectorElt()) 146 return LVal.getVectorPointer(); 147 assert(LVal.isExtVectorElt()); 148 return LVal.getExtVectorPointer(); 149 } 150 Address getAtomicAddress() const { 151 return Address(getAtomicPointer(), getAtomicAlignment()); 152 } 153 154 Address getAtomicAddressAsAtomicIntPointer() const { 155 return emitCastToAtomicIntPointer(getAtomicAddress()); 156 } 157 158 /// Is the atomic size larger than the underlying value type? 159 /// 160 /// Note that the absence of padding does not mean that atomic 161 /// objects are completely interchangeable with non-atomic 162 /// objects: we might have promoted the alignment of a type 163 /// without making it bigger. 164 bool hasPadding() const { 165 return (ValueSizeInBits != AtomicSizeInBits); 166 } 167 168 bool emitMemSetZeroIfNecessary() const; 169 170 llvm::Value *getAtomicSizeValue() const { 171 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 172 return CGF.CGM.getSize(size); 173 } 174 175 /// Cast the given pointer to an integer pointer suitable for atomic 176 /// operations if the source. 177 Address emitCastToAtomicIntPointer(Address Addr) const; 178 179 /// If Addr is compatible with the iN that will be used for an atomic 180 /// operation, bitcast it. Otherwise, create a temporary that is suitable 181 /// and copy the value across. 182 Address convertToAtomicIntPointer(Address Addr) const; 183 184 /// Turn an atomic-layout object into an r-value. 185 RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 186 SourceLocation loc, bool AsValue) const; 187 188 /// Converts a rvalue to integer value. 189 llvm::Value *convertRValueToInt(RValue RVal) const; 190 191 RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 192 AggValueSlot ResultSlot, 193 SourceLocation Loc, bool AsValue) const; 194 195 /// Copy an atomic r-value into atomic-layout memory. 196 void emitCopyIntoMemory(RValue rvalue) const; 197 198 /// Project an l-value down to the value field. 199 LValue projectValue() const { 200 assert(LVal.isSimple()); 201 Address addr = getAtomicAddress(); 202 if (hasPadding()) 203 addr = CGF.Builder.CreateStructGEP(addr, 0); 204 205 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 206 LVal.getBaseInfo(), LVal.getTBAAInfo()); 207 } 208 209 /// Emits atomic load. 210 /// \returns Loaded value. 211 RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 212 bool AsValue, llvm::AtomicOrdering AO, 213 bool IsVolatile); 214 215 /// Emits atomic compare-and-exchange sequence. 216 /// \param Expected Expected value. 217 /// \param Desired Desired value. 218 /// \param Success Atomic ordering for success operation. 219 /// \param Failure Atomic ordering for failed operation. 220 /// \param IsWeak true if atomic operation is weak, false otherwise. 221 /// \returns Pair of values: previous value from storage (value type) and 222 /// boolean flag (i1 type) with true if success and false otherwise. 223 std::pair<RValue, llvm::Value *> 224 EmitAtomicCompareExchange(RValue Expected, RValue Desired, 225 llvm::AtomicOrdering Success = 226 llvm::AtomicOrdering::SequentiallyConsistent, 227 llvm::AtomicOrdering Failure = 228 llvm::AtomicOrdering::SequentiallyConsistent, 229 bool IsWeak = false); 230 231 /// Emits atomic update. 232 /// \param AO Atomic ordering. 233 /// \param UpdateOp Update operation for the current lvalue. 234 void EmitAtomicUpdate(llvm::AtomicOrdering AO, 235 const llvm::function_ref<RValue(RValue)> &UpdateOp, 236 bool IsVolatile); 237 /// Emits atomic update. 238 /// \param AO Atomic ordering. 239 void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 240 bool IsVolatile); 241 242 /// Materialize an atomic r-value in atomic-layout memory. 243 Address materializeRValue(RValue rvalue) const; 244 245 /// Creates temp alloca for intermediate operations on atomic value. 246 Address CreateTempAlloca() const; 247 private: 248 bool requiresMemSetZero(llvm::Type *type) const; 249 250 251 /// Emits atomic load as a libcall. 252 void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 253 llvm::AtomicOrdering AO, bool IsVolatile); 254 /// Emits atomic load as LLVM instruction. 255 llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 256 /// Emits atomic compare-and-exchange op as a libcall. 257 llvm::Value *EmitAtomicCompareExchangeLibcall( 258 llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 259 llvm::AtomicOrdering Success = 260 llvm::AtomicOrdering::SequentiallyConsistent, 261 llvm::AtomicOrdering Failure = 262 llvm::AtomicOrdering::SequentiallyConsistent); 263 /// Emits atomic compare-and-exchange op as LLVM instruction. 264 std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 265 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 266 llvm::AtomicOrdering Success = 267 llvm::AtomicOrdering::SequentiallyConsistent, 268 llvm::AtomicOrdering Failure = 269 llvm::AtomicOrdering::SequentiallyConsistent, 270 bool IsWeak = false); 271 /// Emit atomic update as libcalls. 272 void 273 EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 274 const llvm::function_ref<RValue(RValue)> &UpdateOp, 275 bool IsVolatile); 276 /// Emit atomic update as LLVM instructions. 277 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 278 const llvm::function_ref<RValue(RValue)> &UpdateOp, 279 bool IsVolatile); 280 /// Emit atomic update as libcalls. 281 void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 282 bool IsVolatile); 283 /// Emit atomic update as LLVM instructions. 284 void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 285 bool IsVolatile); 286 }; 287 } 288 289 Address AtomicInfo::CreateTempAlloca() const { 290 Address TempAlloca = CGF.CreateMemTemp( 291 (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 292 : AtomicTy, 293 getAtomicAlignment(), 294 "atomic-temp"); 295 // Cast to pointer to value type for bitfields. 296 if (LVal.isBitField()) 297 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 298 TempAlloca, getAtomicAddress().getType()); 299 return TempAlloca; 300 } 301 302 static RValue emitAtomicLibcall(CodeGenFunction &CGF, 303 StringRef fnName, 304 QualType resultType, 305 CallArgList &args) { 306 const CGFunctionInfo &fnInfo = 307 CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 308 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 309 llvm::FunctionCallee fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 310 auto callee = CGCallee::forDirect(fn); 311 return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args); 312 } 313 314 /// Does a store of the given IR type modify the full expected width? 315 static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 316 uint64_t expectedSize) { 317 return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 318 } 319 320 /// Does the atomic type require memsetting to zero before initialization? 321 /// 322 /// The IR type is provided as a way of making certain queries faster. 323 bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 324 // If the atomic type has size padding, we definitely need a memset. 325 if (hasPadding()) return true; 326 327 // Otherwise, do some simple heuristics to try to avoid it: 328 switch (getEvaluationKind()) { 329 // For scalars and complexes, check whether the store size of the 330 // type uses the full size. 331 case TEK_Scalar: 332 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 333 case TEK_Complex: 334 return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 335 AtomicSizeInBits / 2); 336 337 // Padding in structs has an undefined bit pattern. User beware. 338 case TEK_Aggregate: 339 return false; 340 } 341 llvm_unreachable("bad evaluation kind"); 342 } 343 344 bool AtomicInfo::emitMemSetZeroIfNecessary() const { 345 assert(LVal.isSimple()); 346 llvm::Value *addr = LVal.getPointer(CGF); 347 if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 348 return false; 349 350 CGF.Builder.CreateMemSet( 351 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 352 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 353 LVal.getAlignment().getAsAlign()); 354 return true; 355 } 356 357 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 358 Address Dest, Address Ptr, 359 Address Val1, Address Val2, 360 uint64_t Size, 361 llvm::AtomicOrdering SuccessOrder, 362 llvm::AtomicOrdering FailureOrder, 363 llvm::SyncScope::ID Scope) { 364 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 365 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 366 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 367 368 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 369 Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder, 370 Scope); 371 Pair->setVolatile(E->isVolatile()); 372 Pair->setWeak(IsWeak); 373 374 // Cmp holds the result of the compare-exchange operation: true on success, 375 // false on failure. 376 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 377 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 378 379 // This basic block is used to hold the store instruction if the operation 380 // failed. 381 llvm::BasicBlock *StoreExpectedBB = 382 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 383 384 // This basic block is the exit point of the operation, we should end up 385 // here regardless of whether or not the operation succeeded. 386 llvm::BasicBlock *ContinueBB = 387 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 388 389 // Update Expected if Expected isn't equal to Old, otherwise branch to the 390 // exit point. 391 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 392 393 CGF.Builder.SetInsertPoint(StoreExpectedBB); 394 // Update the memory at Expected with Old's value. 395 CGF.Builder.CreateStore(Old, Val1); 396 // Finally, branch to the exit point. 397 CGF.Builder.CreateBr(ContinueBB); 398 399 CGF.Builder.SetInsertPoint(ContinueBB); 400 // Update the memory at Dest with Cmp's value. 401 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 402 } 403 404 /// Given an ordering required on success, emit all possible cmpxchg 405 /// instructions to cope with the provided (but possibly only dynamically known) 406 /// FailureOrder. 407 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 408 bool IsWeak, Address Dest, Address Ptr, 409 Address Val1, Address Val2, 410 llvm::Value *FailureOrderVal, 411 uint64_t Size, 412 llvm::AtomicOrdering SuccessOrder, 413 llvm::SyncScope::ID Scope) { 414 llvm::AtomicOrdering FailureOrder; 415 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 416 auto FOS = FO->getSExtValue(); 417 if (!llvm::isValidAtomicOrderingCABI(FOS)) 418 FailureOrder = llvm::AtomicOrdering::Monotonic; 419 else 420 switch ((llvm::AtomicOrderingCABI)FOS) { 421 case llvm::AtomicOrderingCABI::relaxed: 422 case llvm::AtomicOrderingCABI::release: 423 case llvm::AtomicOrderingCABI::acq_rel: 424 FailureOrder = llvm::AtomicOrdering::Monotonic; 425 break; 426 case llvm::AtomicOrderingCABI::consume: 427 case llvm::AtomicOrderingCABI::acquire: 428 FailureOrder = llvm::AtomicOrdering::Acquire; 429 break; 430 case llvm::AtomicOrderingCABI::seq_cst: 431 FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 432 break; 433 } 434 if (isStrongerThan(FailureOrder, SuccessOrder)) { 435 // Don't assert on undefined behavior "failure argument shall be no 436 // stronger than the success argument". 437 FailureOrder = 438 llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 439 } 440 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 441 FailureOrder, Scope); 442 return; 443 } 444 445 // Create all the relevant BB's 446 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 447 *SeqCstBB = nullptr; 448 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 449 if (SuccessOrder != llvm::AtomicOrdering::Monotonic && 450 SuccessOrder != llvm::AtomicOrdering::Release) 451 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 452 if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 453 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 454 455 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 456 457 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 458 459 // Emit all the different atomics 460 461 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 462 // doesn't matter unless someone is crazy enough to use something that 463 // doesn't fold to a constant for the ordering. 464 CGF.Builder.SetInsertPoint(MonotonicBB); 465 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 466 Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope); 467 CGF.Builder.CreateBr(ContBB); 468 469 if (AcquireBB) { 470 CGF.Builder.SetInsertPoint(AcquireBB); 471 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 472 Size, SuccessOrder, llvm::AtomicOrdering::Acquire, Scope); 473 CGF.Builder.CreateBr(ContBB); 474 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 475 AcquireBB); 476 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 477 AcquireBB); 478 } 479 if (SeqCstBB) { 480 CGF.Builder.SetInsertPoint(SeqCstBB); 481 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 482 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 483 CGF.Builder.CreateBr(ContBB); 484 SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 485 SeqCstBB); 486 } 487 488 CGF.Builder.SetInsertPoint(ContBB); 489 } 490 491 /// Duplicate the atomic min/max operation in conventional IR for the builtin 492 /// variants that return the new rather than the original value. 493 static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, 494 AtomicExpr::AtomicOp Op, 495 bool IsSigned, 496 llvm::Value *OldVal, 497 llvm::Value *RHS) { 498 llvm::CmpInst::Predicate Pred; 499 switch (Op) { 500 default: 501 llvm_unreachable("Unexpected min/max operation"); 502 case AtomicExpr::AO__atomic_max_fetch: 503 Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; 504 break; 505 case AtomicExpr::AO__atomic_min_fetch: 506 Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; 507 break; 508 } 509 llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst"); 510 return Builder.CreateSelect(Cmp, OldVal, RHS, "newval"); 511 } 512 513 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 514 Address Ptr, Address Val1, Address Val2, 515 llvm::Value *IsWeak, llvm::Value *FailureOrder, 516 uint64_t Size, llvm::AtomicOrdering Order, 517 llvm::SyncScope::ID Scope) { 518 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 519 bool PostOpMinMax = false; 520 unsigned PostOp = 0; 521 522 switch (E->getOp()) { 523 case AtomicExpr::AO__c11_atomic_init: 524 case AtomicExpr::AO__opencl_atomic_init: 525 llvm_unreachable("Already handled!"); 526 527 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 528 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 529 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 530 FailureOrder, Size, Order, Scope); 531 return; 532 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 533 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 534 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 535 FailureOrder, Size, Order, Scope); 536 return; 537 case AtomicExpr::AO__atomic_compare_exchange: 538 case AtomicExpr::AO__atomic_compare_exchange_n: { 539 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 540 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 541 Val1, Val2, FailureOrder, Size, Order, Scope); 542 } else { 543 // Create all the relevant BB's 544 llvm::BasicBlock *StrongBB = 545 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 546 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 547 llvm::BasicBlock *ContBB = 548 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 549 550 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 551 SI->addCase(CGF.Builder.getInt1(false), StrongBB); 552 553 CGF.Builder.SetInsertPoint(StrongBB); 554 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 555 FailureOrder, Size, Order, Scope); 556 CGF.Builder.CreateBr(ContBB); 557 558 CGF.Builder.SetInsertPoint(WeakBB); 559 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 560 FailureOrder, Size, Order, Scope); 561 CGF.Builder.CreateBr(ContBB); 562 563 CGF.Builder.SetInsertPoint(ContBB); 564 } 565 return; 566 } 567 case AtomicExpr::AO__c11_atomic_load: 568 case AtomicExpr::AO__opencl_atomic_load: 569 case AtomicExpr::AO__atomic_load_n: 570 case AtomicExpr::AO__atomic_load: { 571 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 572 Load->setAtomic(Order, Scope); 573 Load->setVolatile(E->isVolatile()); 574 CGF.Builder.CreateStore(Load, Dest); 575 return; 576 } 577 578 case AtomicExpr::AO__c11_atomic_store: 579 case AtomicExpr::AO__opencl_atomic_store: 580 case AtomicExpr::AO__atomic_store: 581 case AtomicExpr::AO__atomic_store_n: { 582 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 583 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 584 Store->setAtomic(Order, Scope); 585 Store->setVolatile(E->isVolatile()); 586 return; 587 } 588 589 case AtomicExpr::AO__c11_atomic_exchange: 590 case AtomicExpr::AO__opencl_atomic_exchange: 591 case AtomicExpr::AO__atomic_exchange_n: 592 case AtomicExpr::AO__atomic_exchange: 593 Op = llvm::AtomicRMWInst::Xchg; 594 break; 595 596 case AtomicExpr::AO__atomic_add_fetch: 597 PostOp = llvm::Instruction::Add; 598 LLVM_FALLTHROUGH; 599 case AtomicExpr::AO__c11_atomic_fetch_add: 600 case AtomicExpr::AO__opencl_atomic_fetch_add: 601 case AtomicExpr::AO__atomic_fetch_add: 602 Op = llvm::AtomicRMWInst::Add; 603 break; 604 605 case AtomicExpr::AO__atomic_sub_fetch: 606 PostOp = llvm::Instruction::Sub; 607 LLVM_FALLTHROUGH; 608 case AtomicExpr::AO__c11_atomic_fetch_sub: 609 case AtomicExpr::AO__opencl_atomic_fetch_sub: 610 case AtomicExpr::AO__atomic_fetch_sub: 611 Op = llvm::AtomicRMWInst::Sub; 612 break; 613 614 case AtomicExpr::AO__atomic_min_fetch: 615 PostOpMinMax = true; 616 LLVM_FALLTHROUGH; 617 case AtomicExpr::AO__c11_atomic_fetch_min: 618 case AtomicExpr::AO__opencl_atomic_fetch_min: 619 case AtomicExpr::AO__atomic_fetch_min: 620 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min 621 : llvm::AtomicRMWInst::UMin; 622 break; 623 624 case AtomicExpr::AO__atomic_max_fetch: 625 PostOpMinMax = true; 626 LLVM_FALLTHROUGH; 627 case AtomicExpr::AO__c11_atomic_fetch_max: 628 case AtomicExpr::AO__opencl_atomic_fetch_max: 629 case AtomicExpr::AO__atomic_fetch_max: 630 Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max 631 : llvm::AtomicRMWInst::UMax; 632 break; 633 634 case AtomicExpr::AO__atomic_and_fetch: 635 PostOp = llvm::Instruction::And; 636 LLVM_FALLTHROUGH; 637 case AtomicExpr::AO__c11_atomic_fetch_and: 638 case AtomicExpr::AO__opencl_atomic_fetch_and: 639 case AtomicExpr::AO__atomic_fetch_and: 640 Op = llvm::AtomicRMWInst::And; 641 break; 642 643 case AtomicExpr::AO__atomic_or_fetch: 644 PostOp = llvm::Instruction::Or; 645 LLVM_FALLTHROUGH; 646 case AtomicExpr::AO__c11_atomic_fetch_or: 647 case AtomicExpr::AO__opencl_atomic_fetch_or: 648 case AtomicExpr::AO__atomic_fetch_or: 649 Op = llvm::AtomicRMWInst::Or; 650 break; 651 652 case AtomicExpr::AO__atomic_xor_fetch: 653 PostOp = llvm::Instruction::Xor; 654 LLVM_FALLTHROUGH; 655 case AtomicExpr::AO__c11_atomic_fetch_xor: 656 case AtomicExpr::AO__opencl_atomic_fetch_xor: 657 case AtomicExpr::AO__atomic_fetch_xor: 658 Op = llvm::AtomicRMWInst::Xor; 659 break; 660 661 case AtomicExpr::AO__atomic_nand_fetch: 662 PostOp = llvm::Instruction::And; // the NOT is special cased below 663 LLVM_FALLTHROUGH; 664 case AtomicExpr::AO__atomic_fetch_nand: 665 Op = llvm::AtomicRMWInst::Nand; 666 break; 667 } 668 669 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 670 llvm::AtomicRMWInst *RMWI = 671 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope); 672 RMWI->setVolatile(E->isVolatile()); 673 674 // For __atomic_*_fetch operations, perform the operation again to 675 // determine the value which was written. 676 llvm::Value *Result = RMWI; 677 if (PostOpMinMax) 678 Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), 679 E->getValueType()->isSignedIntegerType(), 680 RMWI, LoadVal1); 681 else if (PostOp) 682 Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, 683 LoadVal1); 684 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 685 Result = CGF.Builder.CreateNot(Result); 686 CGF.Builder.CreateStore(Result, Dest); 687 } 688 689 // This function emits any expression (scalar, complex, or aggregate) 690 // into a temporary alloca. 691 static Address 692 EmitValToTemp(CodeGenFunction &CGF, Expr *E) { 693 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 694 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 695 /*Init*/ true); 696 return DeclPtr; 697 } 698 699 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest, 700 Address Ptr, Address Val1, Address Val2, 701 llvm::Value *IsWeak, llvm::Value *FailureOrder, 702 uint64_t Size, llvm::AtomicOrdering Order, 703 llvm::Value *Scope) { 704 auto ScopeModel = Expr->getScopeModel(); 705 706 // LLVM atomic instructions always have synch scope. If clang atomic 707 // expression has no scope operand, use default LLVM synch scope. 708 if (!ScopeModel) { 709 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 710 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID("")); 711 return; 712 } 713 714 // Handle constant scope. 715 if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) { 716 auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID( 717 CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()), 718 Order, CGF.CGM.getLLVMContext()); 719 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 720 Order, SCID); 721 return; 722 } 723 724 // Handle non-constant scope. 725 auto &Builder = CGF.Builder; 726 auto Scopes = ScopeModel->getRuntimeValues(); 727 llvm::DenseMap<unsigned, llvm::BasicBlock *> BB; 728 for (auto S : Scopes) 729 BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn); 730 731 llvm::BasicBlock *ContBB = 732 CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn); 733 734 auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false); 735 // If unsupported synch scope is encountered at run time, assume a fallback 736 // synch scope value. 737 auto FallBack = ScopeModel->getFallBackValue(); 738 llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]); 739 for (auto S : Scopes) { 740 auto *B = BB[S]; 741 if (S != FallBack) 742 SI->addCase(Builder.getInt32(S), B); 743 744 Builder.SetInsertPoint(B); 745 EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size, 746 Order, 747 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(), 748 ScopeModel->map(S), 749 Order, 750 CGF.getLLVMContext())); 751 Builder.CreateBr(ContBB); 752 } 753 754 Builder.SetInsertPoint(ContBB); 755 } 756 757 static void 758 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 759 bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 760 SourceLocation Loc, CharUnits SizeInChars) { 761 if (UseOptimizedLibcall) { 762 // Load value and pass it to the function directly. 763 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 764 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 765 ValTy = 766 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 767 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 768 SizeInBits)->getPointerTo(); 769 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 770 Val = CGF.EmitLoadOfScalar(Ptr, false, 771 CGF.getContext().getPointerType(ValTy), 772 Loc); 773 // Coerce the value into an appropriately sized integer type. 774 Args.add(RValue::get(Val), ValTy); 775 } else { 776 // Non-optimized functions always take a reference. 777 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 778 CGF.getContext().VoidPtrTy); 779 } 780 } 781 782 RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 783 QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 784 QualType MemTy = AtomicTy; 785 if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 786 MemTy = AT->getValueType(); 787 llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 788 789 Address Val1 = Address::invalid(); 790 Address Val2 = Address::invalid(); 791 Address Dest = Address::invalid(); 792 Address Ptr = EmitPointerWithAlignment(E->getPtr()); 793 794 if (E->getOp() == AtomicExpr::AO__c11_atomic_init || 795 E->getOp() == AtomicExpr::AO__opencl_atomic_init) { 796 LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 797 EmitAtomicInit(E->getVal1(), lvalue); 798 return RValue::get(nullptr); 799 } 800 801 CharUnits sizeChars, alignChars; 802 std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); 803 uint64_t Size = sizeChars.getQuantity(); 804 unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 805 806 bool Oversized = getContext().toBits(sizeChars) > MaxInlineWidthInBits; 807 bool Misaligned = (Ptr.getAlignment() % sizeChars) != 0; 808 bool UseLibcall = Misaligned | Oversized; 809 810 if (UseLibcall) { 811 CGM.getDiags().Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned) 812 << !Oversized; 813 } 814 815 llvm::Value *Order = EmitScalarExpr(E->getOrder()); 816 llvm::Value *Scope = 817 E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr; 818 819 switch (E->getOp()) { 820 case AtomicExpr::AO__c11_atomic_init: 821 case AtomicExpr::AO__opencl_atomic_init: 822 llvm_unreachable("Already handled above with EmitAtomicInit!"); 823 824 case AtomicExpr::AO__c11_atomic_load: 825 case AtomicExpr::AO__opencl_atomic_load: 826 case AtomicExpr::AO__atomic_load_n: 827 break; 828 829 case AtomicExpr::AO__atomic_load: 830 Dest = EmitPointerWithAlignment(E->getVal1()); 831 break; 832 833 case AtomicExpr::AO__atomic_store: 834 Val1 = EmitPointerWithAlignment(E->getVal1()); 835 break; 836 837 case AtomicExpr::AO__atomic_exchange: 838 Val1 = EmitPointerWithAlignment(E->getVal1()); 839 Dest = EmitPointerWithAlignment(E->getVal2()); 840 break; 841 842 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 843 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 844 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 845 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 846 case AtomicExpr::AO__atomic_compare_exchange_n: 847 case AtomicExpr::AO__atomic_compare_exchange: 848 Val1 = EmitPointerWithAlignment(E->getVal1()); 849 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 850 Val2 = EmitPointerWithAlignment(E->getVal2()); 851 else 852 Val2 = EmitValToTemp(*this, E->getVal2()); 853 OrderFail = EmitScalarExpr(E->getOrderFail()); 854 if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || 855 E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 856 IsWeak = EmitScalarExpr(E->getWeak()); 857 break; 858 859 case AtomicExpr::AO__c11_atomic_fetch_add: 860 case AtomicExpr::AO__c11_atomic_fetch_sub: 861 case AtomicExpr::AO__opencl_atomic_fetch_add: 862 case AtomicExpr::AO__opencl_atomic_fetch_sub: 863 if (MemTy->isPointerType()) { 864 // For pointer arithmetic, we're required to do a bit of math: 865 // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 866 // ... but only for the C11 builtins. The GNU builtins expect the 867 // user to multiply by sizeof(T). 868 QualType Val1Ty = E->getVal1()->getType(); 869 llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 870 CharUnits PointeeIncAmt = 871 getContext().getTypeSizeInChars(MemTy->getPointeeType()); 872 Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 873 auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 874 Val1 = Temp; 875 EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 876 break; 877 } 878 LLVM_FALLTHROUGH; 879 case AtomicExpr::AO__atomic_fetch_add: 880 case AtomicExpr::AO__atomic_fetch_sub: 881 case AtomicExpr::AO__atomic_add_fetch: 882 case AtomicExpr::AO__atomic_sub_fetch: 883 case AtomicExpr::AO__c11_atomic_store: 884 case AtomicExpr::AO__c11_atomic_exchange: 885 case AtomicExpr::AO__opencl_atomic_store: 886 case AtomicExpr::AO__opencl_atomic_exchange: 887 case AtomicExpr::AO__atomic_store_n: 888 case AtomicExpr::AO__atomic_exchange_n: 889 case AtomicExpr::AO__c11_atomic_fetch_and: 890 case AtomicExpr::AO__c11_atomic_fetch_or: 891 case AtomicExpr::AO__c11_atomic_fetch_xor: 892 case AtomicExpr::AO__c11_atomic_fetch_max: 893 case AtomicExpr::AO__c11_atomic_fetch_min: 894 case AtomicExpr::AO__opencl_atomic_fetch_and: 895 case AtomicExpr::AO__opencl_atomic_fetch_or: 896 case AtomicExpr::AO__opencl_atomic_fetch_xor: 897 case AtomicExpr::AO__opencl_atomic_fetch_min: 898 case AtomicExpr::AO__opencl_atomic_fetch_max: 899 case AtomicExpr::AO__atomic_fetch_and: 900 case AtomicExpr::AO__atomic_fetch_or: 901 case AtomicExpr::AO__atomic_fetch_xor: 902 case AtomicExpr::AO__atomic_fetch_nand: 903 case AtomicExpr::AO__atomic_and_fetch: 904 case AtomicExpr::AO__atomic_or_fetch: 905 case AtomicExpr::AO__atomic_xor_fetch: 906 case AtomicExpr::AO__atomic_nand_fetch: 907 case AtomicExpr::AO__atomic_max_fetch: 908 case AtomicExpr::AO__atomic_min_fetch: 909 case AtomicExpr::AO__atomic_fetch_max: 910 case AtomicExpr::AO__atomic_fetch_min: 911 Val1 = EmitValToTemp(*this, E->getVal1()); 912 break; 913 } 914 915 QualType RValTy = E->getType().getUnqualifiedType(); 916 917 // The inlined atomics only function on iN types, where N is a power of 2. We 918 // need to make sure (via temporaries if necessary) that all incoming values 919 // are compatible. 920 LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 921 AtomicInfo Atomics(*this, AtomicVal); 922 923 Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 924 if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); 925 if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); 926 if (Dest.isValid()) 927 Dest = Atomics.emitCastToAtomicIntPointer(Dest); 928 else if (E->isCmpXChg()) 929 Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 930 else if (!RValTy->isVoidType()) 931 Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); 932 933 // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 934 if (UseLibcall) { 935 bool UseOptimizedLibcall = false; 936 switch (E->getOp()) { 937 case AtomicExpr::AO__c11_atomic_init: 938 case AtomicExpr::AO__opencl_atomic_init: 939 llvm_unreachable("Already handled above with EmitAtomicInit!"); 940 941 case AtomicExpr::AO__c11_atomic_fetch_add: 942 case AtomicExpr::AO__opencl_atomic_fetch_add: 943 case AtomicExpr::AO__atomic_fetch_add: 944 case AtomicExpr::AO__c11_atomic_fetch_and: 945 case AtomicExpr::AO__opencl_atomic_fetch_and: 946 case AtomicExpr::AO__atomic_fetch_and: 947 case AtomicExpr::AO__c11_atomic_fetch_or: 948 case AtomicExpr::AO__opencl_atomic_fetch_or: 949 case AtomicExpr::AO__atomic_fetch_or: 950 case AtomicExpr::AO__atomic_fetch_nand: 951 case AtomicExpr::AO__c11_atomic_fetch_sub: 952 case AtomicExpr::AO__opencl_atomic_fetch_sub: 953 case AtomicExpr::AO__atomic_fetch_sub: 954 case AtomicExpr::AO__c11_atomic_fetch_xor: 955 case AtomicExpr::AO__opencl_atomic_fetch_xor: 956 case AtomicExpr::AO__opencl_atomic_fetch_min: 957 case AtomicExpr::AO__opencl_atomic_fetch_max: 958 case AtomicExpr::AO__atomic_fetch_xor: 959 case AtomicExpr::AO__c11_atomic_fetch_max: 960 case AtomicExpr::AO__c11_atomic_fetch_min: 961 case AtomicExpr::AO__atomic_add_fetch: 962 case AtomicExpr::AO__atomic_and_fetch: 963 case AtomicExpr::AO__atomic_nand_fetch: 964 case AtomicExpr::AO__atomic_or_fetch: 965 case AtomicExpr::AO__atomic_sub_fetch: 966 case AtomicExpr::AO__atomic_xor_fetch: 967 case AtomicExpr::AO__atomic_fetch_max: 968 case AtomicExpr::AO__atomic_fetch_min: 969 case AtomicExpr::AO__atomic_max_fetch: 970 case AtomicExpr::AO__atomic_min_fetch: 971 // For these, only library calls for certain sizes exist. 972 UseOptimizedLibcall = true; 973 break; 974 975 case AtomicExpr::AO__atomic_load: 976 case AtomicExpr::AO__atomic_store: 977 case AtomicExpr::AO__atomic_exchange: 978 case AtomicExpr::AO__atomic_compare_exchange: 979 // Use the generic version if we don't know that the operand will be 980 // suitably aligned for the optimized version. 981 if (Misaligned) 982 break; 983 LLVM_FALLTHROUGH; 984 case AtomicExpr::AO__c11_atomic_load: 985 case AtomicExpr::AO__c11_atomic_store: 986 case AtomicExpr::AO__c11_atomic_exchange: 987 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 988 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 989 case AtomicExpr::AO__opencl_atomic_load: 990 case AtomicExpr::AO__opencl_atomic_store: 991 case AtomicExpr::AO__opencl_atomic_exchange: 992 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 993 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 994 case AtomicExpr::AO__atomic_load_n: 995 case AtomicExpr::AO__atomic_store_n: 996 case AtomicExpr::AO__atomic_exchange_n: 997 case AtomicExpr::AO__atomic_compare_exchange_n: 998 // Only use optimized library calls for sizes for which they exist. 999 // FIXME: Size == 16 optimized library functions exist too. 1000 if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 1001 UseOptimizedLibcall = true; 1002 break; 1003 } 1004 1005 CallArgList Args; 1006 if (!UseOptimizedLibcall) { 1007 // For non-optimized library calls, the size is the first parameter 1008 Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 1009 getContext().getSizeType()); 1010 } 1011 // Atomic address is the first or second parameter 1012 // The OpenCL atomic library functions only accept pointer arguments to 1013 // generic address space. 1014 auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) { 1015 if (!E->isOpenCL()) 1016 return V; 1017 auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace(); 1018 if (AS == LangAS::opencl_generic) 1019 return V; 1020 auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic); 1021 auto T = V->getType(); 1022 auto *DestType = T->getPointerElementType()->getPointerTo(DestAS); 1023 1024 return getTargetHooks().performAddrSpaceCast( 1025 *this, V, AS, LangAS::opencl_generic, DestType, false); 1026 }; 1027 1028 Args.add(RValue::get(CastToGenericAddrSpace( 1029 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())), 1030 getContext().VoidPtrTy); 1031 1032 std::string LibCallName; 1033 QualType LoweredMemTy = 1034 MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 1035 QualType RetTy; 1036 bool HaveRetTy = false; 1037 llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 1038 bool PostOpMinMax = false; 1039 switch (E->getOp()) { 1040 case AtomicExpr::AO__c11_atomic_init: 1041 case AtomicExpr::AO__opencl_atomic_init: 1042 llvm_unreachable("Already handled!"); 1043 1044 // There is only one libcall for compare an exchange, because there is no 1045 // optimisation benefit possible from a libcall version of a weak compare 1046 // and exchange. 1047 // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 1048 // void *desired, int success, int failure) 1049 // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 1050 // int success, int failure) 1051 case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 1052 case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 1053 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: 1054 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: 1055 case AtomicExpr::AO__atomic_compare_exchange: 1056 case AtomicExpr::AO__atomic_compare_exchange_n: 1057 LibCallName = "__atomic_compare_exchange"; 1058 RetTy = getContext().BoolTy; 1059 HaveRetTy = true; 1060 Args.add( 1061 RValue::get(CastToGenericAddrSpace( 1062 EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())), 1063 getContext().VoidPtrTy); 1064 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 1065 MemTy, E->getExprLoc(), sizeChars); 1066 Args.add(RValue::get(Order), getContext().IntTy); 1067 Order = OrderFail; 1068 break; 1069 // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 1070 // int order) 1071 // T __atomic_exchange_N(T *mem, T val, int order) 1072 case AtomicExpr::AO__c11_atomic_exchange: 1073 case AtomicExpr::AO__opencl_atomic_exchange: 1074 case AtomicExpr::AO__atomic_exchange_n: 1075 case AtomicExpr::AO__atomic_exchange: 1076 LibCallName = "__atomic_exchange"; 1077 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1078 MemTy, E->getExprLoc(), sizeChars); 1079 break; 1080 // void __atomic_store(size_t size, void *mem, void *val, int order) 1081 // void __atomic_store_N(T *mem, T val, int order) 1082 case AtomicExpr::AO__c11_atomic_store: 1083 case AtomicExpr::AO__opencl_atomic_store: 1084 case AtomicExpr::AO__atomic_store: 1085 case AtomicExpr::AO__atomic_store_n: 1086 LibCallName = "__atomic_store"; 1087 RetTy = getContext().VoidTy; 1088 HaveRetTy = true; 1089 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1090 MemTy, E->getExprLoc(), sizeChars); 1091 break; 1092 // void __atomic_load(size_t size, void *mem, void *return, int order) 1093 // T __atomic_load_N(T *mem, int order) 1094 case AtomicExpr::AO__c11_atomic_load: 1095 case AtomicExpr::AO__opencl_atomic_load: 1096 case AtomicExpr::AO__atomic_load: 1097 case AtomicExpr::AO__atomic_load_n: 1098 LibCallName = "__atomic_load"; 1099 break; 1100 // T __atomic_add_fetch_N(T *mem, T val, int order) 1101 // T __atomic_fetch_add_N(T *mem, T val, int order) 1102 case AtomicExpr::AO__atomic_add_fetch: 1103 PostOp = llvm::Instruction::Add; 1104 LLVM_FALLTHROUGH; 1105 case AtomicExpr::AO__c11_atomic_fetch_add: 1106 case AtomicExpr::AO__opencl_atomic_fetch_add: 1107 case AtomicExpr::AO__atomic_fetch_add: 1108 LibCallName = "__atomic_fetch_add"; 1109 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1110 LoweredMemTy, E->getExprLoc(), sizeChars); 1111 break; 1112 // T __atomic_and_fetch_N(T *mem, T val, int order) 1113 // T __atomic_fetch_and_N(T *mem, T val, int order) 1114 case AtomicExpr::AO__atomic_and_fetch: 1115 PostOp = llvm::Instruction::And; 1116 LLVM_FALLTHROUGH; 1117 case AtomicExpr::AO__c11_atomic_fetch_and: 1118 case AtomicExpr::AO__opencl_atomic_fetch_and: 1119 case AtomicExpr::AO__atomic_fetch_and: 1120 LibCallName = "__atomic_fetch_and"; 1121 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1122 MemTy, E->getExprLoc(), sizeChars); 1123 break; 1124 // T __atomic_or_fetch_N(T *mem, T val, int order) 1125 // T __atomic_fetch_or_N(T *mem, T val, int order) 1126 case AtomicExpr::AO__atomic_or_fetch: 1127 PostOp = llvm::Instruction::Or; 1128 LLVM_FALLTHROUGH; 1129 case AtomicExpr::AO__c11_atomic_fetch_or: 1130 case AtomicExpr::AO__opencl_atomic_fetch_or: 1131 case AtomicExpr::AO__atomic_fetch_or: 1132 LibCallName = "__atomic_fetch_or"; 1133 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1134 MemTy, E->getExprLoc(), sizeChars); 1135 break; 1136 // T __atomic_sub_fetch_N(T *mem, T val, int order) 1137 // T __atomic_fetch_sub_N(T *mem, T val, int order) 1138 case AtomicExpr::AO__atomic_sub_fetch: 1139 PostOp = llvm::Instruction::Sub; 1140 LLVM_FALLTHROUGH; 1141 case AtomicExpr::AO__c11_atomic_fetch_sub: 1142 case AtomicExpr::AO__opencl_atomic_fetch_sub: 1143 case AtomicExpr::AO__atomic_fetch_sub: 1144 LibCallName = "__atomic_fetch_sub"; 1145 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1146 LoweredMemTy, E->getExprLoc(), sizeChars); 1147 break; 1148 // T __atomic_xor_fetch_N(T *mem, T val, int order) 1149 // T __atomic_fetch_xor_N(T *mem, T val, int order) 1150 case AtomicExpr::AO__atomic_xor_fetch: 1151 PostOp = llvm::Instruction::Xor; 1152 LLVM_FALLTHROUGH; 1153 case AtomicExpr::AO__c11_atomic_fetch_xor: 1154 case AtomicExpr::AO__opencl_atomic_fetch_xor: 1155 case AtomicExpr::AO__atomic_fetch_xor: 1156 LibCallName = "__atomic_fetch_xor"; 1157 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1158 MemTy, E->getExprLoc(), sizeChars); 1159 break; 1160 case AtomicExpr::AO__atomic_min_fetch: 1161 PostOpMinMax = true; 1162 LLVM_FALLTHROUGH; 1163 case AtomicExpr::AO__c11_atomic_fetch_min: 1164 case AtomicExpr::AO__atomic_fetch_min: 1165 case AtomicExpr::AO__opencl_atomic_fetch_min: 1166 LibCallName = E->getValueType()->isSignedIntegerType() 1167 ? "__atomic_fetch_min" 1168 : "__atomic_fetch_umin"; 1169 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1170 LoweredMemTy, E->getExprLoc(), sizeChars); 1171 break; 1172 case AtomicExpr::AO__atomic_max_fetch: 1173 PostOpMinMax = true; 1174 LLVM_FALLTHROUGH; 1175 case AtomicExpr::AO__c11_atomic_fetch_max: 1176 case AtomicExpr::AO__atomic_fetch_max: 1177 case AtomicExpr::AO__opencl_atomic_fetch_max: 1178 LibCallName = E->getValueType()->isSignedIntegerType() 1179 ? "__atomic_fetch_max" 1180 : "__atomic_fetch_umax"; 1181 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1182 LoweredMemTy, E->getExprLoc(), sizeChars); 1183 break; 1184 // T __atomic_nand_fetch_N(T *mem, T val, int order) 1185 // T __atomic_fetch_nand_N(T *mem, T val, int order) 1186 case AtomicExpr::AO__atomic_nand_fetch: 1187 PostOp = llvm::Instruction::And; // the NOT is special cased below 1188 LLVM_FALLTHROUGH; 1189 case AtomicExpr::AO__atomic_fetch_nand: 1190 LibCallName = "__atomic_fetch_nand"; 1191 AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 1192 MemTy, E->getExprLoc(), sizeChars); 1193 break; 1194 } 1195 1196 if (E->isOpenCL()) { 1197 LibCallName = std::string("__opencl") + 1198 StringRef(LibCallName).drop_front(1).str(); 1199 1200 } 1201 // Optimized functions have the size in their name. 1202 if (UseOptimizedLibcall) 1203 LibCallName += "_" + llvm::utostr(Size); 1204 // By default, assume we return a value of the atomic type. 1205 if (!HaveRetTy) { 1206 if (UseOptimizedLibcall) { 1207 // Value is returned directly. 1208 // The function returns an appropriately sized integer type. 1209 RetTy = getContext().getIntTypeForBitwidth( 1210 getContext().toBits(sizeChars), /*Signed=*/false); 1211 } else { 1212 // Value is returned through parameter before the order. 1213 RetTy = getContext().VoidTy; 1214 Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 1215 getContext().VoidPtrTy); 1216 } 1217 } 1218 // order is always the last parameter 1219 Args.add(RValue::get(Order), 1220 getContext().IntTy); 1221 if (E->isOpenCL()) 1222 Args.add(RValue::get(Scope), getContext().IntTy); 1223 1224 // PostOp is only needed for the atomic_*_fetch operations, and 1225 // thus is only needed for and implemented in the 1226 // UseOptimizedLibcall codepath. 1227 assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax)); 1228 1229 RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 1230 // The value is returned directly from the libcall. 1231 if (E->isCmpXChg()) 1232 return Res; 1233 1234 // The value is returned directly for optimized libcalls but the expr 1235 // provided an out-param. 1236 if (UseOptimizedLibcall && Res.getScalarVal()) { 1237 llvm::Value *ResVal = Res.getScalarVal(); 1238 if (PostOpMinMax) { 1239 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1240 ResVal = EmitPostAtomicMinMax(Builder, E->getOp(), 1241 E->getValueType()->isSignedIntegerType(), 1242 ResVal, LoadVal1); 1243 } else if (PostOp) { 1244 llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); 1245 ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 1246 } 1247 if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 1248 ResVal = Builder.CreateNot(ResVal); 1249 1250 Builder.CreateStore( 1251 ResVal, 1252 Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 1253 } 1254 1255 if (RValTy->isVoidType()) 1256 return RValue::get(nullptr); 1257 1258 return convertTempToRValue( 1259 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 1260 RValTy, E->getExprLoc()); 1261 } 1262 1263 bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1264 E->getOp() == AtomicExpr::AO__opencl_atomic_store || 1265 E->getOp() == AtomicExpr::AO__atomic_store || 1266 E->getOp() == AtomicExpr::AO__atomic_store_n; 1267 bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1268 E->getOp() == AtomicExpr::AO__opencl_atomic_load || 1269 E->getOp() == AtomicExpr::AO__atomic_load || 1270 E->getOp() == AtomicExpr::AO__atomic_load_n; 1271 1272 if (isa<llvm::ConstantInt>(Order)) { 1273 auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 1274 // We should not ever get to a case where the ordering isn't a valid C ABI 1275 // value, but it's hard to enforce that in general. 1276 if (llvm::isValidAtomicOrderingCABI(ord)) 1277 switch ((llvm::AtomicOrderingCABI)ord) { 1278 case llvm::AtomicOrderingCABI::relaxed: 1279 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1280 llvm::AtomicOrdering::Monotonic, Scope); 1281 break; 1282 case llvm::AtomicOrderingCABI::consume: 1283 case llvm::AtomicOrderingCABI::acquire: 1284 if (IsStore) 1285 break; // Avoid crashing on code with undefined behavior 1286 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1287 llvm::AtomicOrdering::Acquire, Scope); 1288 break; 1289 case llvm::AtomicOrderingCABI::release: 1290 if (IsLoad) 1291 break; // Avoid crashing on code with undefined behavior 1292 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1293 llvm::AtomicOrdering::Release, Scope); 1294 break; 1295 case llvm::AtomicOrderingCABI::acq_rel: 1296 if (IsLoad || IsStore) 1297 break; // Avoid crashing on code with undefined behavior 1298 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1299 llvm::AtomicOrdering::AcquireRelease, Scope); 1300 break; 1301 case llvm::AtomicOrderingCABI::seq_cst: 1302 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1303 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1304 break; 1305 } 1306 if (RValTy->isVoidType()) 1307 return RValue::get(nullptr); 1308 1309 return convertTempToRValue( 1310 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1311 Dest.getAddressSpace())), 1312 RValTy, E->getExprLoc()); 1313 } 1314 1315 // Long case, when Order isn't obviously constant. 1316 1317 // Create all the relevant BB's 1318 llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 1319 *ReleaseBB = nullptr, *AcqRelBB = nullptr, 1320 *SeqCstBB = nullptr; 1321 MonotonicBB = createBasicBlock("monotonic", CurFn); 1322 if (!IsStore) 1323 AcquireBB = createBasicBlock("acquire", CurFn); 1324 if (!IsLoad) 1325 ReleaseBB = createBasicBlock("release", CurFn); 1326 if (!IsLoad && !IsStore) 1327 AcqRelBB = createBasicBlock("acqrel", CurFn); 1328 SeqCstBB = createBasicBlock("seqcst", CurFn); 1329 llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1330 1331 // Create the switch for the split 1332 // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1333 // doesn't matter unless someone is crazy enough to use something that 1334 // doesn't fold to a constant for the ordering. 1335 Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1336 llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1337 1338 // Emit all the different atomics 1339 Builder.SetInsertPoint(MonotonicBB); 1340 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1341 llvm::AtomicOrdering::Monotonic, Scope); 1342 Builder.CreateBr(ContBB); 1343 if (!IsStore) { 1344 Builder.SetInsertPoint(AcquireBB); 1345 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1346 llvm::AtomicOrdering::Acquire, Scope); 1347 Builder.CreateBr(ContBB); 1348 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1349 AcquireBB); 1350 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1351 AcquireBB); 1352 } 1353 if (!IsLoad) { 1354 Builder.SetInsertPoint(ReleaseBB); 1355 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1356 llvm::AtomicOrdering::Release, Scope); 1357 Builder.CreateBr(ContBB); 1358 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1359 ReleaseBB); 1360 } 1361 if (!IsLoad && !IsStore) { 1362 Builder.SetInsertPoint(AcqRelBB); 1363 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1364 llvm::AtomicOrdering::AcquireRelease, Scope); 1365 Builder.CreateBr(ContBB); 1366 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1367 AcqRelBB); 1368 } 1369 Builder.SetInsertPoint(SeqCstBB); 1370 EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 1371 llvm::AtomicOrdering::SequentiallyConsistent, Scope); 1372 Builder.CreateBr(ContBB); 1373 SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1374 SeqCstBB); 1375 1376 // Cleanup and return 1377 Builder.SetInsertPoint(ContBB); 1378 if (RValTy->isVoidType()) 1379 return RValue::get(nullptr); 1380 1381 assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 1382 return convertTempToRValue( 1383 Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo( 1384 Dest.getAddressSpace())), 1385 RValTy, E->getExprLoc()); 1386 } 1387 1388 Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 1389 unsigned addrspace = 1390 cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 1391 llvm::IntegerType *ty = 1392 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 1393 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 1394 } 1395 1396 Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 1397 llvm::Type *Ty = Addr.getElementType(); 1398 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 1399 if (SourceSizeInBits != AtomicSizeInBits) { 1400 Address Tmp = CreateTempAlloca(); 1401 CGF.Builder.CreateMemCpy(Tmp, Addr, 1402 std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 1403 Addr = Tmp; 1404 } 1405 1406 return emitCastToAtomicIntPointer(Addr); 1407 } 1408 1409 RValue AtomicInfo::convertAtomicTempToRValue(Address addr, 1410 AggValueSlot resultSlot, 1411 SourceLocation loc, 1412 bool asValue) const { 1413 if (LVal.isSimple()) { 1414 if (EvaluationKind == TEK_Aggregate) 1415 return resultSlot.asRValue(); 1416 1417 // Drill into the padding structure if we have one. 1418 if (hasPadding()) 1419 addr = CGF.Builder.CreateStructGEP(addr, 0); 1420 1421 // Otherwise, just convert the temporary to an r-value using the 1422 // normal conversion routine. 1423 return CGF.convertTempToRValue(addr, getValueType(), loc); 1424 } 1425 if (!asValue) 1426 // Get RValue from temp memory as atomic for non-simple lvalues 1427 return RValue::get(CGF.Builder.CreateLoad(addr)); 1428 if (LVal.isBitField()) 1429 return CGF.EmitLoadOfBitfieldLValue( 1430 LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 1431 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1432 if (LVal.isVectorElt()) 1433 return CGF.EmitLoadOfLValue( 1434 LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 1435 LVal.getBaseInfo(), TBAAAccessInfo()), loc); 1436 assert(LVal.isExtVectorElt()); 1437 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 1438 addr, LVal.getExtVectorElts(), LVal.getType(), 1439 LVal.getBaseInfo(), TBAAAccessInfo())); 1440 } 1441 1442 RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1443 AggValueSlot ResultSlot, 1444 SourceLocation Loc, 1445 bool AsValue) const { 1446 // Try not to in some easy cases. 1447 assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 1448 if (getEvaluationKind() == TEK_Scalar && 1449 (((!LVal.isBitField() || 1450 LVal.getBitFieldInfo().Size == ValueSizeInBits) && 1451 !hasPadding()) || 1452 !AsValue)) { 1453 auto *ValTy = AsValue 1454 ? CGF.ConvertTypeForMem(ValueTy) 1455 : getAtomicAddress().getType()->getPointerElementType(); 1456 if (ValTy->isIntegerTy()) { 1457 assert(IntVal->getType() == ValTy && "Different integer types."); 1458 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 1459 } else if (ValTy->isPointerTy()) 1460 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 1461 else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 1462 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 1463 } 1464 1465 // Create a temporary. This needs to be big enough to hold the 1466 // atomic integer. 1467 Address Temp = Address::invalid(); 1468 bool TempIsVolatile = false; 1469 if (AsValue && getEvaluationKind() == TEK_Aggregate) { 1470 assert(!ResultSlot.isIgnored()); 1471 Temp = ResultSlot.getAddress(); 1472 TempIsVolatile = ResultSlot.isVolatile(); 1473 } else { 1474 Temp = CreateTempAlloca(); 1475 } 1476 1477 // Slam the integer into the temporary. 1478 Address CastTemp = emitCastToAtomicIntPointer(Temp); 1479 CGF.Builder.CreateStore(IntVal, CastTemp) 1480 ->setVolatile(TempIsVolatile); 1481 1482 return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 1483 } 1484 1485 void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 1486 llvm::AtomicOrdering AO, bool) { 1487 // void __atomic_load(size_t size, void *mem, void *return, int order); 1488 CallArgList Args; 1489 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1490 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1491 CGF.getContext().VoidPtrTy); 1492 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 1493 CGF.getContext().VoidPtrTy); 1494 Args.add( 1495 RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 1496 CGF.getContext().IntTy); 1497 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 1498 } 1499 1500 llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 1501 bool IsVolatile) { 1502 // Okay, we're doing this natively. 1503 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1504 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 1505 Load->setAtomic(AO); 1506 1507 // Other decoration. 1508 if (IsVolatile) 1509 Load->setVolatile(true); 1510 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 1511 return Load; 1512 } 1513 1514 /// An LValue is a candidate for having its loads and stores be made atomic if 1515 /// we are operating under /volatile:ms *and* the LValue itself is volatile and 1516 /// performing such an operation can be performed without a libcall. 1517 bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 1518 if (!CGM.getCodeGenOpts().MSVolatile) return false; 1519 AtomicInfo AI(*this, LV); 1520 bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 1521 // An atomic is inline if we don't need to use a libcall. 1522 bool AtomicIsInline = !AI.shouldUseLibcall(); 1523 // MSVC doesn't seem to do this for types wider than a pointer. 1524 if (getContext().getTypeSize(LV.getType()) > 1525 getContext().getTypeSize(getContext().getIntPtrType())) 1526 return false; 1527 return IsVolatile && AtomicIsInline; 1528 } 1529 1530 RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 1531 AggValueSlot Slot) { 1532 llvm::AtomicOrdering AO; 1533 bool IsVolatile = LV.isVolatileQualified(); 1534 if (LV.getType()->isAtomicType()) { 1535 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1536 } else { 1537 AO = llvm::AtomicOrdering::Acquire; 1538 IsVolatile = true; 1539 } 1540 return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 1541 } 1542 1543 RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 1544 bool AsValue, llvm::AtomicOrdering AO, 1545 bool IsVolatile) { 1546 // Check whether we should use a library call. 1547 if (shouldUseLibcall()) { 1548 Address TempAddr = Address::invalid(); 1549 if (LVal.isSimple() && !ResultSlot.isIgnored()) { 1550 assert(getEvaluationKind() == TEK_Aggregate); 1551 TempAddr = ResultSlot.getAddress(); 1552 } else 1553 TempAddr = CreateTempAlloca(); 1554 1555 EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 1556 1557 // Okay, turn that back into the original value or whole atomic (for 1558 // non-simple lvalues) type. 1559 return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 1560 } 1561 1562 // Okay, we're doing this natively. 1563 auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 1564 1565 // If we're ignoring an aggregate return, don't do anything. 1566 if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 1567 return RValue::getAggregate(Address::invalid(), false); 1568 1569 // Okay, turn that back into the original value or atomic (for non-simple 1570 // lvalues) type. 1571 return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 1572 } 1573 1574 /// Emit a load from an l-value of atomic type. Note that the r-value 1575 /// we produce is an r-value of the atomic *value* type. 1576 RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 1577 llvm::AtomicOrdering AO, bool IsVolatile, 1578 AggValueSlot resultSlot) { 1579 AtomicInfo Atomics(*this, src); 1580 return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 1581 IsVolatile); 1582 } 1583 1584 /// Copy an r-value into memory as part of storing to an atomic type. 1585 /// This needs to create a bit-pattern suitable for atomic operations. 1586 void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 1587 assert(LVal.isSimple()); 1588 // If we have an r-value, the rvalue should be of the atomic type, 1589 // which means that the caller is responsible for having zeroed 1590 // any padding. Just do an aggregate copy of that type. 1591 if (rvalue.isAggregate()) { 1592 LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType()); 1593 LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(), 1594 getAtomicType()); 1595 bool IsVolatile = rvalue.isVolatileQualified() || 1596 LVal.isVolatileQualified(); 1597 CGF.EmitAggregateCopy(Dest, Src, getAtomicType(), 1598 AggValueSlot::DoesNotOverlap, IsVolatile); 1599 return; 1600 } 1601 1602 // Okay, otherwise we're copying stuff. 1603 1604 // Zero out the buffer if necessary. 1605 emitMemSetZeroIfNecessary(); 1606 1607 // Drill past the padding if present. 1608 LValue TempLVal = projectValue(); 1609 1610 // Okay, store the rvalue in. 1611 if (rvalue.isScalar()) { 1612 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 1613 } else { 1614 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 1615 } 1616 } 1617 1618 1619 /// Materialize an r-value into memory for the purposes of storing it 1620 /// to an atomic type. 1621 Address AtomicInfo::materializeRValue(RValue rvalue) const { 1622 // Aggregate r-values are already in memory, and EmitAtomicStore 1623 // requires them to be values of the atomic type. 1624 if (rvalue.isAggregate()) 1625 return rvalue.getAggregateAddress(); 1626 1627 // Otherwise, make a temporary and materialize into it. 1628 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 1629 AtomicInfo Atomics(CGF, TempLV); 1630 Atomics.emitCopyIntoMemory(rvalue); 1631 return TempLV.getAddress(CGF); 1632 } 1633 1634 llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 1635 // If we've got a scalar value of the right size, try to avoid going 1636 // through memory. 1637 if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 1638 llvm::Value *Value = RVal.getScalarVal(); 1639 if (isa<llvm::IntegerType>(Value->getType())) 1640 return CGF.EmitToMemory(Value, ValueTy); 1641 else { 1642 llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 1643 CGF.getLLVMContext(), 1644 LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 1645 if (isa<llvm::PointerType>(Value->getType())) 1646 return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 1647 else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 1648 return CGF.Builder.CreateBitCast(Value, InputIntTy); 1649 } 1650 } 1651 // Otherwise, we need to go through memory. 1652 // Put the r-value in memory. 1653 Address Addr = materializeRValue(RVal); 1654 1655 // Cast the temporary to the atomic int type and pull a value out. 1656 Addr = emitCastToAtomicIntPointer(Addr); 1657 return CGF.Builder.CreateLoad(Addr); 1658 } 1659 1660 std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1661 llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1662 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 1663 // Do the atomic store. 1664 Address Addr = getAtomicAddressAsAtomicIntPointer(); 1665 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 1666 ExpectedVal, DesiredVal, 1667 Success, Failure); 1668 // Other decoration. 1669 Inst->setVolatile(LVal.isVolatileQualified()); 1670 Inst->setWeak(IsWeak); 1671 1672 // Okay, turn that back into the original value type. 1673 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 1674 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1675 return std::make_pair(PreviousVal, SuccessFailureVal); 1676 } 1677 1678 llvm::Value * 1679 AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1680 llvm::Value *DesiredAddr, 1681 llvm::AtomicOrdering Success, 1682 llvm::AtomicOrdering Failure) { 1683 // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 1684 // void *desired, int success, int failure); 1685 CallArgList Args; 1686 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 1687 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 1688 CGF.getContext().VoidPtrTy); 1689 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 1690 CGF.getContext().VoidPtrTy); 1691 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 1692 CGF.getContext().VoidPtrTy); 1693 Args.add(RValue::get( 1694 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 1695 CGF.getContext().IntTy); 1696 Args.add(RValue::get( 1697 llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 1698 CGF.getContext().IntTy); 1699 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 1700 CGF.getContext().BoolTy, Args); 1701 1702 return SuccessFailureRVal.getScalarVal(); 1703 } 1704 1705 std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 1706 RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 1707 llvm::AtomicOrdering Failure, bool IsWeak) { 1708 if (isStrongerThan(Failure, Success)) 1709 // Don't assert on undefined behavior "failure argument shall be no stronger 1710 // than the success argument". 1711 Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 1712 1713 // Check whether we should use a library call. 1714 if (shouldUseLibcall()) { 1715 // Produce a source address. 1716 Address ExpectedAddr = materializeRValue(Expected); 1717 Address DesiredAddr = materializeRValue(Desired); 1718 auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1719 DesiredAddr.getPointer(), 1720 Success, Failure); 1721 return std::make_pair( 1722 convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 1723 SourceLocation(), /*AsValue=*/false), 1724 Res); 1725 } 1726 1727 // If we've got a scalar value of the right size, try to avoid going 1728 // through memory. 1729 auto *ExpectedVal = convertRValueToInt(Expected); 1730 auto *DesiredVal = convertRValueToInt(Desired); 1731 auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1732 Failure, IsWeak); 1733 return std::make_pair( 1734 ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1735 SourceLocation(), /*AsValue=*/false), 1736 Res.second); 1737 } 1738 1739 static void 1740 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1741 const llvm::function_ref<RValue(RValue)> &UpdateOp, 1742 Address DesiredAddr) { 1743 RValue UpRVal; 1744 LValue AtomicLVal = Atomics.getAtomicLValue(); 1745 LValue DesiredLVal; 1746 if (AtomicLVal.isSimple()) { 1747 UpRVal = OldRVal; 1748 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1749 } else { 1750 // Build new lvalue for temp address. 1751 Address Ptr = Atomics.materializeRValue(OldRVal); 1752 LValue UpdateLVal; 1753 if (AtomicLVal.isBitField()) { 1754 UpdateLVal = 1755 LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 1756 AtomicLVal.getType(), 1757 AtomicLVal.getBaseInfo(), 1758 AtomicLVal.getTBAAInfo()); 1759 DesiredLVal = 1760 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1761 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1762 AtomicLVal.getTBAAInfo()); 1763 } else if (AtomicLVal.isVectorElt()) { 1764 UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1765 AtomicLVal.getType(), 1766 AtomicLVal.getBaseInfo(), 1767 AtomicLVal.getTBAAInfo()); 1768 DesiredLVal = LValue::MakeVectorElt( 1769 DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 1770 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1771 } else { 1772 assert(AtomicLVal.isExtVectorElt()); 1773 UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1774 AtomicLVal.getType(), 1775 AtomicLVal.getBaseInfo(), 1776 AtomicLVal.getTBAAInfo()); 1777 DesiredLVal = LValue::MakeExtVectorElt( 1778 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1779 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1780 } 1781 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1782 } 1783 // Store new value in the corresponding memory area. 1784 RValue NewRVal = UpdateOp(UpRVal); 1785 if (NewRVal.isScalar()) { 1786 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1787 } else { 1788 assert(NewRVal.isComplex()); 1789 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1790 /*isInit=*/false); 1791 } 1792 } 1793 1794 void AtomicInfo::EmitAtomicUpdateLibcall( 1795 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1796 bool IsVolatile) { 1797 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1798 1799 Address ExpectedAddr = CreateTempAlloca(); 1800 1801 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1802 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1803 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1804 CGF.EmitBlock(ContBB); 1805 Address DesiredAddr = CreateTempAlloca(); 1806 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1807 requiresMemSetZero(getAtomicAddress().getElementType())) { 1808 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1809 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1810 } 1811 auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 1812 AggValueSlot::ignored(), 1813 SourceLocation(), /*AsValue=*/false); 1814 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1815 auto *Res = 1816 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1817 DesiredAddr.getPointer(), 1818 AO, Failure); 1819 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1820 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1821 } 1822 1823 void AtomicInfo::EmitAtomicUpdateOp( 1824 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1825 bool IsVolatile) { 1826 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1827 1828 // Do the atomic load. 1829 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1830 // For non-simple lvalues perform compare-and-swap procedure. 1831 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1832 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1833 auto *CurBB = CGF.Builder.GetInsertBlock(); 1834 CGF.EmitBlock(ContBB); 1835 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1836 /*NumReservedValues=*/2); 1837 PHI->addIncoming(OldVal, CurBB); 1838 Address NewAtomicAddr = CreateTempAlloca(); 1839 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1840 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1841 requiresMemSetZero(getAtomicAddress().getElementType())) { 1842 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1843 } 1844 auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1845 SourceLocation(), /*AsValue=*/false); 1846 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 1847 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1848 // Try to write new value using cmpxchg operation. 1849 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1850 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1851 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1852 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1853 } 1854 1855 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 1856 RValue UpdateRVal, Address DesiredAddr) { 1857 LValue AtomicLVal = Atomics.getAtomicLValue(); 1858 LValue DesiredLVal; 1859 // Build new lvalue for temp address. 1860 if (AtomicLVal.isBitField()) { 1861 DesiredLVal = 1862 LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 1863 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1864 AtomicLVal.getTBAAInfo()); 1865 } else if (AtomicLVal.isVectorElt()) { 1866 DesiredLVal = 1867 LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 1868 AtomicLVal.getType(), AtomicLVal.getBaseInfo(), 1869 AtomicLVal.getTBAAInfo()); 1870 } else { 1871 assert(AtomicLVal.isExtVectorElt()); 1872 DesiredLVal = LValue::MakeExtVectorElt( 1873 DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 1874 AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo()); 1875 } 1876 // Store new value in the corresponding memory area. 1877 assert(UpdateRVal.isScalar()); 1878 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1879 } 1880 1881 void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1882 RValue UpdateRVal, bool IsVolatile) { 1883 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1884 1885 Address ExpectedAddr = CreateTempAlloca(); 1886 1887 EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1888 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1889 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1890 CGF.EmitBlock(ContBB); 1891 Address DesiredAddr = CreateTempAlloca(); 1892 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1893 requiresMemSetZero(getAtomicAddress().getElementType())) { 1894 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 1895 CGF.Builder.CreateStore(OldVal, DesiredAddr); 1896 } 1897 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1898 auto *Res = 1899 EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 1900 DesiredAddr.getPointer(), 1901 AO, Failure); 1902 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1903 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1904 } 1905 1906 void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1907 bool IsVolatile) { 1908 auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1909 1910 // Do the atomic load. 1911 auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1912 // For non-simple lvalues perform compare-and-swap procedure. 1913 auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1914 auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1915 auto *CurBB = CGF.Builder.GetInsertBlock(); 1916 CGF.EmitBlock(ContBB); 1917 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1918 /*NumReservedValues=*/2); 1919 PHI->addIncoming(OldVal, CurBB); 1920 Address NewAtomicAddr = CreateTempAlloca(); 1921 Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1922 if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 1923 requiresMemSetZero(getAtomicAddress().getElementType())) { 1924 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1925 } 1926 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 1927 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1928 // Try to write new value using cmpxchg operation. 1929 auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1930 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1931 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1932 CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1933 } 1934 1935 void AtomicInfo::EmitAtomicUpdate( 1936 llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1937 bool IsVolatile) { 1938 if (shouldUseLibcall()) { 1939 EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1940 } else { 1941 EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1942 } 1943 } 1944 1945 void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1946 bool IsVolatile) { 1947 if (shouldUseLibcall()) { 1948 EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1949 } else { 1950 EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1951 } 1952 } 1953 1954 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 1955 bool isInit) { 1956 bool IsVolatile = lvalue.isVolatileQualified(); 1957 llvm::AtomicOrdering AO; 1958 if (lvalue.getType()->isAtomicType()) { 1959 AO = llvm::AtomicOrdering::SequentiallyConsistent; 1960 } else { 1961 AO = llvm::AtomicOrdering::Release; 1962 IsVolatile = true; 1963 } 1964 return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 1965 } 1966 1967 /// Emit a store to an l-value of atomic type. 1968 /// 1969 /// Note that the r-value is expected to be an r-value *of the atomic 1970 /// type*; this means that for aggregate r-values, it should include 1971 /// storage for any padding that was necessary. 1972 void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 1973 llvm::AtomicOrdering AO, bool IsVolatile, 1974 bool isInit) { 1975 // If this is an aggregate r-value, it should agree in type except 1976 // maybe for address-space qualification. 1977 assert(!rvalue.isAggregate() || 1978 rvalue.getAggregateAddress().getElementType() == 1979 dest.getAddress(*this).getElementType()); 1980 1981 AtomicInfo atomics(*this, dest); 1982 LValue LVal = atomics.getAtomicLValue(); 1983 1984 // If this is an initialization, just put the value there normally. 1985 if (LVal.isSimple()) { 1986 if (isInit) { 1987 atomics.emitCopyIntoMemory(rvalue); 1988 return; 1989 } 1990 1991 // Check whether we should use a library call. 1992 if (atomics.shouldUseLibcall()) { 1993 // Produce a source address. 1994 Address srcAddr = atomics.materializeRValue(rvalue); 1995 1996 // void __atomic_store(size_t size, void *mem, void *val, int order) 1997 CallArgList args; 1998 args.add(RValue::get(atomics.getAtomicSizeValue()), 1999 getContext().getSizeType()); 2000 args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 2001 getContext().VoidPtrTy); 2002 args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 2003 getContext().VoidPtrTy); 2004 args.add( 2005 RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 2006 getContext().IntTy); 2007 emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 2008 return; 2009 } 2010 2011 // Okay, we're doing this natively. 2012 llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 2013 2014 // Do the atomic store. 2015 Address addr = 2016 atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 2017 intValue = Builder.CreateIntCast( 2018 intValue, addr.getElementType(), /*isSigned=*/false); 2019 llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 2020 2021 // Initializations don't need to be atomic. 2022 if (!isInit) 2023 store->setAtomic(AO); 2024 2025 // Other decoration. 2026 if (IsVolatile) 2027 store->setVolatile(true); 2028 CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 2029 return; 2030 } 2031 2032 // Emit simple atomic update operation. 2033 atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 2034 } 2035 2036 /// Emit a compare-and-exchange op for atomic type. 2037 /// 2038 std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 2039 LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 2040 llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 2041 AggValueSlot Slot) { 2042 // If this is an aggregate r-value, it should agree in type except 2043 // maybe for address-space qualification. 2044 assert(!Expected.isAggregate() || 2045 Expected.getAggregateAddress().getElementType() == 2046 Obj.getAddress(*this).getElementType()); 2047 assert(!Desired.isAggregate() || 2048 Desired.getAggregateAddress().getElementType() == 2049 Obj.getAddress(*this).getElementType()); 2050 AtomicInfo Atomics(*this, Obj); 2051 2052 return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 2053 IsWeak); 2054 } 2055 2056 void CodeGenFunction::EmitAtomicUpdate( 2057 LValue LVal, llvm::AtomicOrdering AO, 2058 const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 2059 AtomicInfo Atomics(*this, LVal); 2060 Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 2061 } 2062 2063 void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 2064 AtomicInfo atomics(*this, dest); 2065 2066 switch (atomics.getEvaluationKind()) { 2067 case TEK_Scalar: { 2068 llvm::Value *value = EmitScalarExpr(init); 2069 atomics.emitCopyIntoMemory(RValue::get(value)); 2070 return; 2071 } 2072 2073 case TEK_Complex: { 2074 ComplexPairTy value = EmitComplexExpr(init); 2075 atomics.emitCopyIntoMemory(RValue::getComplex(value)); 2076 return; 2077 } 2078 2079 case TEK_Aggregate: { 2080 // Fix up the destination if the initializer isn't an expression 2081 // of atomic type. 2082 bool Zeroed = false; 2083 if (!init->getType()->isAtomicType()) { 2084 Zeroed = atomics.emitMemSetZeroIfNecessary(); 2085 dest = atomics.projectValue(); 2086 } 2087 2088 // Evaluate the expression directly into the destination. 2089 AggValueSlot slot = AggValueSlot::forLValue( 2090 dest, *this, AggValueSlot::IsNotDestructed, 2091 AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased, 2092 AggValueSlot::DoesNotOverlap, 2093 Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed); 2094 2095 EmitAggExpr(init, slot); 2096 return; 2097 } 2098 } 2099 llvm_unreachable("bad evaluation kind"); 2100 } 2101