1 //===- InstCombineAtomicRMW.cpp -------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the visit functions for atomic rmw instructions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "InstCombineInternal.h"
14 #include "llvm/IR/Instructions.h"
15 
16 using namespace llvm;
17 
18 namespace {
19 /// Return true if and only if the given instruction does not modify the memory
20 /// location referenced.  Note that an idemptent atomicrmw may still have
21 /// ordering effects on nearby instructions, or be volatile.
22 /// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
23 /// Idemptotent is confusing in this context.
24 bool isIdempotentRMW(AtomicRMWInst& RMWI) {
25   if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
26     switch(RMWI.getOperation()) {
27     case AtomicRMWInst::FAdd: // -0.0
28       return CF->isZero() && CF->isNegative();
29     case AtomicRMWInst::FSub: // +0.0
30       return CF->isZero() && !CF->isNegative();
31     default:
32       return false;
33     };
34 
35   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
36   if(!C)
37     return false;
38 
39   switch(RMWI.getOperation()) {
40     case AtomicRMWInst::Add:
41     case AtomicRMWInst::Sub:
42     case AtomicRMWInst::Or:
43     case AtomicRMWInst::Xor:
44       return C->isZero();
45     case AtomicRMWInst::And:
46       return C->isMinusOne();
47     case AtomicRMWInst::Min:
48       return C->isMaxValue(true);
49     case AtomicRMWInst::Max:
50       return C->isMinValue(true);
51     case AtomicRMWInst::UMin:
52       return C->isMaxValue(false);
53     case AtomicRMWInst::UMax:
54       return C->isMinValue(false);
55     default:
56       return false;
57   }
58 }
59 
60 /// Return true if the given instruction always produces a value in memory
61 /// equivalent to its value operand.
62 bool isSaturating(AtomicRMWInst& RMWI) {
63   if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
64     switch (RMWI.getOperation()) {
65     case AtomicRMWInst::FMax:
66       // maxnum(x, +inf) -> +inf
67       return !CF->isNegative() && CF->isInfinity();
68     case AtomicRMWInst::FMin:
69       // minnum(x, -inf) -> +inf
70       return CF->isNegative() && CF->isInfinity();
71     case AtomicRMWInst::FAdd:
72     case AtomicRMWInst::FSub:
73       return CF->isNaN();
74     default:
75       return false;
76     };
77 
78   auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
79   if(!C)
80     return false;
81 
82   switch(RMWI.getOperation()) {
83   default:
84     return false;
85   case AtomicRMWInst::Xchg:
86     return true;
87   case AtomicRMWInst::Or:
88     return C->isAllOnesValue();
89   case AtomicRMWInst::And:
90     return C->isZero();
91   case AtomicRMWInst::Min:
92     return C->isMinValue(true);
93   case AtomicRMWInst::Max:
94     return C->isMaxValue(true);
95   case AtomicRMWInst::UMin:
96     return C->isMinValue(false);
97   case AtomicRMWInst::UMax:
98     return C->isMaxValue(false);
99   };
100 }
101 } // namespace
102 
103 Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
104 
105   // Volatile RMWs perform a load and a store, we cannot replace this by just a
106   // load or just a store. We chose not to canonicalize out of general paranoia
107   // about user expectations around volatile.
108   if (RMWI.isVolatile())
109     return nullptr;
110 
111   // Any atomicrmw op which produces a known result in memory can be
112   // replaced w/an atomicrmw xchg.
113   if (isSaturating(RMWI) &&
114       RMWI.getOperation() != AtomicRMWInst::Xchg) {
115     RMWI.setOperation(AtomicRMWInst::Xchg);
116     return &RMWI;
117   }
118 
119   AtomicOrdering Ordering = RMWI.getOrdering();
120   assert(Ordering != AtomicOrdering::NotAtomic &&
121          Ordering != AtomicOrdering::Unordered &&
122          "AtomicRMWs don't make sense with Unordered or NotAtomic");
123 
124   // Any atomicrmw xchg with no uses can be converted to a atomic store if the
125   // ordering is compatible.
126   if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
127       RMWI.use_empty()) {
128     if (Ordering != AtomicOrdering::Release &&
129         Ordering != AtomicOrdering::Monotonic)
130       return nullptr;
131     auto *SI = new StoreInst(RMWI.getValOperand(),
132                              RMWI.getPointerOperand(), &RMWI);
133     SI->setAtomic(Ordering, RMWI.getSyncScopeID());
134     SI->setAlignment(DL.getABITypeAlign(RMWI.getType()));
135     return eraseInstFromFunction(RMWI);
136   }
137 
138   if (!isIdempotentRMW(RMWI))
139     return nullptr;
140 
141   // We chose to canonicalize all idempotent operations to an single
142   // operation code and constant.  This makes it easier for the rest of the
143   // optimizer to match easily.  The choices of or w/0 and fadd w/-0.0 are
144   // arbitrary.
145   if (RMWI.getType()->isIntegerTy() &&
146       RMWI.getOperation() != AtomicRMWInst::Or) {
147     RMWI.setOperation(AtomicRMWInst::Or);
148     return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0));
149   } else if (RMWI.getType()->isFloatingPointTy() &&
150              RMWI.getOperation() != AtomicRMWInst::FAdd) {
151     RMWI.setOperation(AtomicRMWInst::FAdd);
152     return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType()));
153   }
154 
155   // Check if the required ordering is compatible with an atomic load.
156   if (Ordering != AtomicOrdering::Acquire &&
157       Ordering != AtomicOrdering::Monotonic)
158     return nullptr;
159 
160   LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand(), "",
161                                 false, DL.getABITypeAlign(RMWI.getType()),
162                                 Ordering, RMWI.getSyncScopeID());
163   return Load;
164 }
165