1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the visit functions for atomic rmw instructions.
10 //
11 //===----------------------------------------------------------------------===//
12 #include "InstCombineInternal.h"
13 #include "llvm/IR/Instructions.h"
14 
15 using namespace llvm;
16 
17 namespace {
18 /// Return true if and only if the given instruction does not modify the memory
19 /// location referenced.  Note that an idemptent atomicrmw may still have
20 /// ordering effects on nearby instructions, or be volatile.
21 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
22 /// Idemptotent is confusing in this context.
23 bool isIdempotentRMW(AtomicRMWInst& RMWI) {
24   if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
25     switch(RMWI.getOperation()) {
26     case AtomicRMWInst::FAdd: // -0.0
27       return CF->isZero() && CF->isNegative();
28     case AtomicRMWInst::FSub: // +0.0
29       return CF->isZero() && !CF->isNegative();
30     default:
31       return false;
32     };
33 
34   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
35   if(!C)
36     return false;
37 
38   switch(RMWI.getOperation()) {
39     case AtomicRMWInst::Add:
40     case AtomicRMWInst::Sub:
41     case AtomicRMWInst::Or:
42     case AtomicRMWInst::Xor:
43       return C->isZero();
44     case AtomicRMWInst::And:
45       return C->isMinusOne();
46     case AtomicRMWInst::Min:
47       return C->isMaxValue(true);
48     case AtomicRMWInst::Max:
49       return C->isMinValue(true);
50     case AtomicRMWInst::UMin:
51       return C->isMaxValue(false);
52     case AtomicRMWInst::UMax:
53       return C->isMinValue(false);
54     default:
55       return false;
56   }
57 }
58 
59 /// Return true if the given instruction always produces a value in memory
60 /// equivalent to its value operand.
61 bool isSaturating(AtomicRMWInst& RMWI) {
62   if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
63     switch(RMWI.getOperation()) {
64     case AtomicRMWInst::FAdd:
65     case AtomicRMWInst::FSub:
66       return CF->isNaN();
67     default:
68       return false;
69     };
70 
71   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
72   if(!C)
73     return false;
74 
75   switch(RMWI.getOperation()) {
76   default:
77     return false;
78   case AtomicRMWInst::Xchg:
79     return true;
80   case AtomicRMWInst::Or:
81     return C->isAllOnesValue();
82   case AtomicRMWInst::And:
83     return C->isZero();
84   case AtomicRMWInst::Min:
85     return C->isMinValue(true);
86   case AtomicRMWInst::Max:
87     return C->isMaxValue(true);
88   case AtomicRMWInst::UMin:
89     return C->isMinValue(false);
90   case AtomicRMWInst::UMax:
91     return C->isMaxValue(false);
92   };
93 }
94 }
95 
96 Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
97 
98   // Volatile RMWs perform a load and a store, we cannot replace this by just a
99   // load or just a store. We chose not to canonicalize out of general paranoia
100   // about user expectations around volatile.
101   if (RMWI.isVolatile())
102     return nullptr;
103 
104   // Any atomicrmw op which produces a known result in memory can be
105   // replaced w/an atomicrmw xchg.
106   if (isSaturating(RMWI) &&
107       RMWI.getOperation() != AtomicRMWInst::Xchg) {
108     RMWI.setOperation(AtomicRMWInst::Xchg);
109     return &RMWI;
110   }
111 
112   AtomicOrdering Ordering = RMWI.getOrdering();
113   assert(Ordering != AtomicOrdering::NotAtomic &&
114          Ordering != AtomicOrdering::Unordered &&
115          "AtomicRMWs don't make sense with Unordered or NotAtomic");
116 
117   // Any atomicrmw xchg with no uses can be converted to a atomic store if the
118   // ordering is compatible.
119   if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
120       RMWI.use_empty()) {
121     if (Ordering != AtomicOrdering::Release &&
122         Ordering != AtomicOrdering::Monotonic)
123       return nullptr;
124     auto *SI = new StoreInst(RMWI.getValOperand(),
125                              RMWI.getPointerOperand(), &RMWI);
126     SI->setAtomic(Ordering, RMWI.getSyncScopeID());
127     SI->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
128     return eraseInstFromFunction(RMWI);
129   }
130 
131   if (!isIdempotentRMW(RMWI))
132     return nullptr;
133 
134   // We chose to canonicalize all idempotent operations to an single
135   // operation code and constant.  This makes it easier for the rest of the
136   // optimizer to match easily.  The choices of or w/0 and fadd w/-0.0 are
137   // arbitrary.
138   if (RMWI.getType()->isIntegerTy() &&
139       RMWI.getOperation() != AtomicRMWInst::Or) {
140     RMWI.setOperation(AtomicRMWInst::Or);
141     RMWI.setOperand(1, ConstantInt::get(RMWI.getType(), 0));
142     return &RMWI;
143   } else if (RMWI.getType()->isFloatingPointTy() &&
144              RMWI.getOperation() != AtomicRMWInst::FAdd) {
145     RMWI.setOperation(AtomicRMWInst::FAdd);
146     RMWI.setOperand(1, ConstantFP::getNegativeZero(RMWI.getType()));
147     return &RMWI;
148   }
149 
150   // Check if the required ordering is compatible with an atomic load.
151   if (Ordering != AtomicOrdering::Acquire &&
152       Ordering != AtomicOrdering::Monotonic)
153     return nullptr;
154 
155   LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand());
156   Load->setAtomic(Ordering, RMWI.getSyncScopeID());
157   Load->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
158   return Load;
159 }
160