1 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the auto-upgrade helper functions.
10 // This is where deprecated IR intrinsics and other IR features are updated to
11 // current specifications.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/IR/AutoUpgrade.h"
16 #include "llvm/ADT/StringSwitch.h"
17 #include "llvm/IR/Constants.h"
18 #include "llvm/IR/DIBuilder.h"
19 #include "llvm/IR/DebugInfo.h"
20 #include "llvm/IR/DiagnosticInfo.h"
21 #include "llvm/IR/Function.h"
22 #include "llvm/IR/IRBuilder.h"
23 #include "llvm/IR/Instruction.h"
24 #include "llvm/IR/InstVisitor.h"
25 #include "llvm/IR/IntrinsicInst.h"
26 #include "llvm/IR/Intrinsics.h"
27 #include "llvm/IR/IntrinsicsAArch64.h"
28 #include "llvm/IR/IntrinsicsARM.h"
29 #include "llvm/IR/IntrinsicsX86.h"
30 #include "llvm/IR/LLVMContext.h"
31 #include "llvm/IR/Module.h"
32 #include "llvm/IR/Verifier.h"
33 #include "llvm/Support/ErrorHandling.h"
34 #include "llvm/Support/Regex.h"
35 #include <cstring>
36 using namespace llvm;
37
rename(GlobalValue * GV)38 static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); }
39
40 // Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have
41 // changed their type from v4f32 to v2i64.
UpgradePTESTIntrinsic(Function * F,Intrinsic::ID IID,Function * & NewFn)42 static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID,
43 Function *&NewFn) {
44 // Check whether this is an old version of the function, which received
45 // v4f32 arguments.
46 Type *Arg0Type = F->getFunctionType()->getParamType(0);
47 if (Arg0Type != FixedVectorType::get(Type::getFloatTy(F->getContext()), 4))
48 return false;
49
50 // Yes, it's old, replace it with new version.
51 rename(F);
52 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
53 return true;
54 }
55
56 // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask
57 // arguments have changed their type from i32 to i8.
UpgradeX86IntrinsicsWith8BitMask(Function * F,Intrinsic::ID IID,Function * & NewFn)58 static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID,
59 Function *&NewFn) {
60 // Check that the last argument is an i32.
61 Type *LastArgType = F->getFunctionType()->getParamType(
62 F->getFunctionType()->getNumParams() - 1);
63 if (!LastArgType->isIntegerTy(32))
64 return false;
65
66 // Move this function aside and map down.
67 rename(F);
68 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
69 return true;
70 }
71
72 // Upgrade the declaration of fp compare intrinsics that change return type
73 // from scalar to vXi1 mask.
UpgradeX86MaskedFPCompare(Function * F,Intrinsic::ID IID,Function * & NewFn)74 static bool UpgradeX86MaskedFPCompare(Function *F, Intrinsic::ID IID,
75 Function *&NewFn) {
76 // Check if the return type is a vector.
77 if (F->getReturnType()->isVectorTy())
78 return false;
79
80 rename(F);
81 NewFn = Intrinsic::getDeclaration(F->getParent(), IID);
82 return true;
83 }
84
ShouldUpgradeX86Intrinsic(Function * F,StringRef Name)85 static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
86 // All of the intrinsics matches below should be marked with which llvm
87 // version started autoupgrading them. At some point in the future we would
88 // like to use this information to remove upgrade code for some older
89 // intrinsics. It is currently undecided how we will determine that future
90 // point.
91 if (Name == "addcarryx.u32" || // Added in 8.0
92 Name == "addcarryx.u64" || // Added in 8.0
93 Name == "addcarry.u32" || // Added in 8.0
94 Name == "addcarry.u64" || // Added in 8.0
95 Name == "subborrow.u32" || // Added in 8.0
96 Name == "subborrow.u64" || // Added in 8.0
97 Name.startswith("sse2.padds.") || // Added in 8.0
98 Name.startswith("sse2.psubs.") || // Added in 8.0
99 Name.startswith("sse2.paddus.") || // Added in 8.0
100 Name.startswith("sse2.psubus.") || // Added in 8.0
101 Name.startswith("avx2.padds.") || // Added in 8.0
102 Name.startswith("avx2.psubs.") || // Added in 8.0
103 Name.startswith("avx2.paddus.") || // Added in 8.0
104 Name.startswith("avx2.psubus.") || // Added in 8.0
105 Name.startswith("avx512.padds.") || // Added in 8.0
106 Name.startswith("avx512.psubs.") || // Added in 8.0
107 Name.startswith("avx512.mask.padds.") || // Added in 8.0
108 Name.startswith("avx512.mask.psubs.") || // Added in 8.0
109 Name.startswith("avx512.mask.paddus.") || // Added in 8.0
110 Name.startswith("avx512.mask.psubus.") || // Added in 8.0
111 Name=="ssse3.pabs.b.128" || // Added in 6.0
112 Name=="ssse3.pabs.w.128" || // Added in 6.0
113 Name=="ssse3.pabs.d.128" || // Added in 6.0
114 Name.startswith("fma4.vfmadd.s") || // Added in 7.0
115 Name.startswith("fma.vfmadd.") || // Added in 7.0
116 Name.startswith("fma.vfmsub.") || // Added in 7.0
117 Name.startswith("fma.vfmsubadd.") || // Added in 7.0
118 Name.startswith("fma.vfnmadd.") || // Added in 7.0
119 Name.startswith("fma.vfnmsub.") || // Added in 7.0
120 Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0
121 Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0
122 Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0
123 Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0
124 Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0
125 Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0
126 Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0
127 Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0
128 Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0
129 Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0
130 Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0
131 Name.startswith("avx512.mask.shuf.i") || // Added in 6.0
132 Name.startswith("avx512.mask.shuf.f") || // Added in 6.0
133 Name.startswith("avx512.kunpck") || //added in 6.0
134 Name.startswith("avx2.pabs.") || // Added in 6.0
135 Name.startswith("avx512.mask.pabs.") || // Added in 6.0
136 Name.startswith("avx512.broadcastm") || // Added in 6.0
137 Name == "sse.sqrt.ss" || // Added in 7.0
138 Name == "sse2.sqrt.sd" || // Added in 7.0
139 Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0
140 Name.startswith("avx.sqrt.p") || // Added in 7.0
141 Name.startswith("sse2.sqrt.p") || // Added in 7.0
142 Name.startswith("sse.sqrt.p") || // Added in 7.0
143 Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0
144 Name.startswith("sse2.pcmpeq.") || // Added in 3.1
145 Name.startswith("sse2.pcmpgt.") || // Added in 3.1
146 Name.startswith("avx2.pcmpeq.") || // Added in 3.1
147 Name.startswith("avx2.pcmpgt.") || // Added in 3.1
148 Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9
149 Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9
150 Name.startswith("avx.vperm2f128.") || // Added in 6.0
151 Name == "avx2.vperm2i128" || // Added in 6.0
152 Name == "sse.add.ss" || // Added in 4.0
153 Name == "sse2.add.sd" || // Added in 4.0
154 Name == "sse.sub.ss" || // Added in 4.0
155 Name == "sse2.sub.sd" || // Added in 4.0
156 Name == "sse.mul.ss" || // Added in 4.0
157 Name == "sse2.mul.sd" || // Added in 4.0
158 Name == "sse.div.ss" || // Added in 4.0
159 Name == "sse2.div.sd" || // Added in 4.0
160 Name == "sse41.pmaxsb" || // Added in 3.9
161 Name == "sse2.pmaxs.w" || // Added in 3.9
162 Name == "sse41.pmaxsd" || // Added in 3.9
163 Name == "sse2.pmaxu.b" || // Added in 3.9
164 Name == "sse41.pmaxuw" || // Added in 3.9
165 Name == "sse41.pmaxud" || // Added in 3.9
166 Name == "sse41.pminsb" || // Added in 3.9
167 Name == "sse2.pmins.w" || // Added in 3.9
168 Name == "sse41.pminsd" || // Added in 3.9
169 Name == "sse2.pminu.b" || // Added in 3.9
170 Name == "sse41.pminuw" || // Added in 3.9
171 Name == "sse41.pminud" || // Added in 3.9
172 Name == "avx512.kand.w" || // Added in 7.0
173 Name == "avx512.kandn.w" || // Added in 7.0
174 Name == "avx512.knot.w" || // Added in 7.0
175 Name == "avx512.kor.w" || // Added in 7.0
176 Name == "avx512.kxor.w" || // Added in 7.0
177 Name == "avx512.kxnor.w" || // Added in 7.0
178 Name == "avx512.kortestc.w" || // Added in 7.0
179 Name == "avx512.kortestz.w" || // Added in 7.0
180 Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0
181 Name.startswith("avx2.pmax") || // Added in 3.9
182 Name.startswith("avx2.pmin") || // Added in 3.9
183 Name.startswith("avx512.mask.pmax") || // Added in 4.0
184 Name.startswith("avx512.mask.pmin") || // Added in 4.0
185 Name.startswith("avx2.vbroadcast") || // Added in 3.8
186 Name.startswith("avx2.pbroadcast") || // Added in 3.8
187 Name.startswith("avx.vpermil.") || // Added in 3.1
188 Name.startswith("sse2.pshuf") || // Added in 3.9
189 Name.startswith("avx512.pbroadcast") || // Added in 3.9
190 Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9
191 Name.startswith("avx512.mask.movddup") || // Added in 3.9
192 Name.startswith("avx512.mask.movshdup") || // Added in 3.9
193 Name.startswith("avx512.mask.movsldup") || // Added in 3.9
194 Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9
195 Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9
196 Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9
197 Name.startswith("avx512.mask.shuf.p") || // Added in 4.0
198 Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9
199 Name.startswith("avx512.mask.perm.df.") || // Added in 3.9
200 Name.startswith("avx512.mask.perm.di.") || // Added in 3.9
201 Name.startswith("avx512.mask.punpckl") || // Added in 3.9
202 Name.startswith("avx512.mask.punpckh") || // Added in 3.9
203 Name.startswith("avx512.mask.unpckl.") || // Added in 3.9
204 Name.startswith("avx512.mask.unpckh.") || // Added in 3.9
205 Name.startswith("avx512.mask.pand.") || // Added in 3.9
206 Name.startswith("avx512.mask.pandn.") || // Added in 3.9
207 Name.startswith("avx512.mask.por.") || // Added in 3.9
208 Name.startswith("avx512.mask.pxor.") || // Added in 3.9
209 Name.startswith("avx512.mask.and.") || // Added in 3.9
210 Name.startswith("avx512.mask.andn.") || // Added in 3.9
211 Name.startswith("avx512.mask.or.") || // Added in 3.9
212 Name.startswith("avx512.mask.xor.") || // Added in 3.9
213 Name.startswith("avx512.mask.padd.") || // Added in 4.0
214 Name.startswith("avx512.mask.psub.") || // Added in 4.0
215 Name.startswith("avx512.mask.pmull.") || // Added in 4.0
216 Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
217 Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
218 Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
219 Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
220 Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
221 Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
222 Name == "avx512.mask.vcvtph2ps.128" || // Added in 11.0
223 Name == "avx512.mask.vcvtph2ps.256" || // Added in 11.0
224 Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
225 Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
226 Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
227 Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
228 Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
229 Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
230 Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
231 Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0
232 Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0
233 Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0
234 Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0
235 Name == "avx512.cvtusi2sd" || // Added in 7.0
236 Name.startswith("avx512.mask.permvar.") || // Added in 7.0
237 Name == "sse2.pmulu.dq" || // Added in 7.0
238 Name == "sse41.pmuldq" || // Added in 7.0
239 Name == "avx2.pmulu.dq" || // Added in 7.0
240 Name == "avx2.pmul.dq" || // Added in 7.0
241 Name == "avx512.pmulu.dq.512" || // Added in 7.0
242 Name == "avx512.pmul.dq.512" || // Added in 7.0
243 Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0
244 Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0
245 Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0
246 Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0
247 Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0
248 Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0
249 Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0
250 Name.startswith("avx512.mask.packsswb.") || // Added in 5.0
251 Name.startswith("avx512.mask.packssdw.") || // Added in 5.0
252 Name.startswith("avx512.mask.packuswb.") || // Added in 5.0
253 Name.startswith("avx512.mask.packusdw.") || // Added in 5.0
254 Name.startswith("avx512.mask.cmp.b") || // Added in 5.0
255 Name.startswith("avx512.mask.cmp.d") || // Added in 5.0
256 Name.startswith("avx512.mask.cmp.q") || // Added in 5.0
257 Name.startswith("avx512.mask.cmp.w") || // Added in 5.0
258 Name.startswith("avx512.cmp.p") || // Added in 12.0
259 Name.startswith("avx512.mask.ucmp.") || // Added in 5.0
260 Name.startswith("avx512.cvtb2mask.") || // Added in 7.0
261 Name.startswith("avx512.cvtw2mask.") || // Added in 7.0
262 Name.startswith("avx512.cvtd2mask.") || // Added in 7.0
263 Name.startswith("avx512.cvtq2mask.") || // Added in 7.0
264 Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0
265 Name.startswith("avx512.mask.psll.d") || // Added in 4.0
266 Name.startswith("avx512.mask.psll.q") || // Added in 4.0
267 Name.startswith("avx512.mask.psll.w") || // Added in 4.0
268 Name.startswith("avx512.mask.psra.d") || // Added in 4.0
269 Name.startswith("avx512.mask.psra.q") || // Added in 4.0
270 Name.startswith("avx512.mask.psra.w") || // Added in 4.0
271 Name.startswith("avx512.mask.psrl.d") || // Added in 4.0
272 Name.startswith("avx512.mask.psrl.q") || // Added in 4.0
273 Name.startswith("avx512.mask.psrl.w") || // Added in 4.0
274 Name.startswith("avx512.mask.pslli") || // Added in 4.0
275 Name.startswith("avx512.mask.psrai") || // Added in 4.0
276 Name.startswith("avx512.mask.psrli") || // Added in 4.0
277 Name.startswith("avx512.mask.psllv") || // Added in 4.0
278 Name.startswith("avx512.mask.psrav") || // Added in 4.0
279 Name.startswith("avx512.mask.psrlv") || // Added in 4.0
280 Name.startswith("sse41.pmovsx") || // Added in 3.8
281 Name.startswith("sse41.pmovzx") || // Added in 3.9
282 Name.startswith("avx2.pmovsx") || // Added in 3.9
283 Name.startswith("avx2.pmovzx") || // Added in 3.9
284 Name.startswith("avx512.mask.pmovsx") || // Added in 4.0
285 Name.startswith("avx512.mask.pmovzx") || // Added in 4.0
286 Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0
287 Name.startswith("avx512.mask.pternlog.") || // Added in 7.0
288 Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0
289 Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0
290 Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0
291 Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0
292 Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0
293 Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0
294 Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0
295 Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0
296 Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0
297 Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0
298 Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0
299 Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0
300 Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0
301 Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0
302 Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0
303 Name.startswith("avx512.mask.vpshld.") || // Added in 7.0
304 Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0
305 Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0
306 Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0
307 Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0
308 Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0
309 Name.startswith("avx512.vpshld.") || // Added in 8.0
310 Name.startswith("avx512.vpshrd.") || // Added in 8.0
311 Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0
312 Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0
313 Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0
314 Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0
315 Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0
316 Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0
317 Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
318 Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
319 Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
320 Name.startswith("avx512.mask.conflict.") || // Added in 9.0
321 Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
322 Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
323 Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
324 Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
325 Name == "sse.cvtsi2ss" || // Added in 7.0
326 Name == "sse.cvtsi642ss" || // Added in 7.0
327 Name == "sse2.cvtsi2sd" || // Added in 7.0
328 Name == "sse2.cvtsi642sd" || // Added in 7.0
329 Name == "sse2.cvtss2sd" || // Added in 7.0
330 Name == "sse2.cvtdq2pd" || // Added in 3.9
331 Name == "sse2.cvtdq2ps" || // Added in 7.0
332 Name == "sse2.cvtps2pd" || // Added in 3.9
333 Name == "avx.cvtdq2.pd.256" || // Added in 3.9
334 Name == "avx.cvtdq2.ps.256" || // Added in 7.0
335 Name == "avx.cvt.ps2.pd.256" || // Added in 3.9
336 Name.startswith("vcvtph2ps.") || // Added in 11.0
337 Name.startswith("avx.vinsertf128.") || // Added in 3.7
338 Name == "avx2.vinserti128" || // Added in 3.7
339 Name.startswith("avx512.mask.insert") || // Added in 4.0
340 Name.startswith("avx.vextractf128.") || // Added in 3.7
341 Name == "avx2.vextracti128" || // Added in 3.7
342 Name.startswith("avx512.mask.vextract") || // Added in 4.0
343 Name.startswith("sse4a.movnt.") || // Added in 3.9
344 Name.startswith("avx.movnt.") || // Added in 3.2
345 Name.startswith("avx512.storent.") || // Added in 3.9
346 Name == "sse41.movntdqa" || // Added in 5.0
347 Name == "avx2.movntdqa" || // Added in 5.0
348 Name == "avx512.movntdqa" || // Added in 5.0
349 Name == "sse2.storel.dq" || // Added in 3.9
350 Name.startswith("sse.storeu.") || // Added in 3.9
351 Name.startswith("sse2.storeu.") || // Added in 3.9
352 Name.startswith("avx.storeu.") || // Added in 3.9
353 Name.startswith("avx512.mask.storeu.") || // Added in 3.9
354 Name.startswith("avx512.mask.store.p") || // Added in 3.9
355 Name.startswith("avx512.mask.store.b.") || // Added in 3.9
356 Name.startswith("avx512.mask.store.w.") || // Added in 3.9
357 Name.startswith("avx512.mask.store.d.") || // Added in 3.9
358 Name.startswith("avx512.mask.store.q.") || // Added in 3.9
359 Name == "avx512.mask.store.ss" || // Added in 7.0
360 Name.startswith("avx512.mask.loadu.") || // Added in 3.9
361 Name.startswith("avx512.mask.load.") || // Added in 3.9
362 Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
363 Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
364 Name.startswith("avx512.mask.expand.b") || // Added in 9.0
365 Name.startswith("avx512.mask.expand.w") || // Added in 9.0
366 Name.startswith("avx512.mask.expand.d") || // Added in 9.0
367 Name.startswith("avx512.mask.expand.q") || // Added in 9.0
368 Name.startswith("avx512.mask.expand.p") || // Added in 9.0
369 Name.startswith("avx512.mask.compress.b") || // Added in 9.0
370 Name.startswith("avx512.mask.compress.w") || // Added in 9.0
371 Name.startswith("avx512.mask.compress.d") || // Added in 9.0
372 Name.startswith("avx512.mask.compress.q") || // Added in 9.0
373 Name.startswith("avx512.mask.compress.p") || // Added in 9.0
374 Name == "sse42.crc32.64.8" || // Added in 3.4
375 Name.startswith("avx.vbroadcast.s") || // Added in 3.5
376 Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
377 Name.startswith("avx512.mask.palignr.") || // Added in 3.9
378 Name.startswith("avx512.mask.valign.") || // Added in 4.0
379 Name.startswith("sse2.psll.dq") || // Added in 3.7
380 Name.startswith("sse2.psrl.dq") || // Added in 3.7
381 Name.startswith("avx2.psll.dq") || // Added in 3.7
382 Name.startswith("avx2.psrl.dq") || // Added in 3.7
383 Name.startswith("avx512.psll.dq") || // Added in 3.9
384 Name.startswith("avx512.psrl.dq") || // Added in 3.9
385 Name == "sse41.pblendw" || // Added in 3.7
386 Name.startswith("sse41.blendp") || // Added in 3.7
387 Name.startswith("avx.blend.p") || // Added in 3.7
388 Name == "avx2.pblendw" || // Added in 3.7
389 Name.startswith("avx2.pblendd.") || // Added in 3.7
390 Name.startswith("avx.vbroadcastf128") || // Added in 4.0
391 Name == "avx2.vbroadcasti128" || // Added in 3.7
392 Name.startswith("avx512.mask.broadcastf32x4.") || // Added in 6.0
393 Name.startswith("avx512.mask.broadcastf64x2.") || // Added in 6.0
394 Name.startswith("avx512.mask.broadcastf32x8.") || // Added in 6.0
395 Name.startswith("avx512.mask.broadcastf64x4.") || // Added in 6.0
396 Name.startswith("avx512.mask.broadcasti32x4.") || // Added in 6.0
397 Name.startswith("avx512.mask.broadcasti64x2.") || // Added in 6.0
398 Name.startswith("avx512.mask.broadcasti32x8.") || // Added in 6.0
399 Name.startswith("avx512.mask.broadcasti64x4.") || // Added in 6.0
400 Name == "xop.vpcmov" || // Added in 3.8
401 Name == "xop.vpcmov.256" || // Added in 5.0
402 Name.startswith("avx512.mask.move.s") || // Added in 4.0
403 Name.startswith("avx512.cvtmask2") || // Added in 5.0
404 Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
405 Name.startswith("xop.vprot") || // Added in 8.0
406 Name.startswith("avx512.prol") || // Added in 8.0
407 Name.startswith("avx512.pror") || // Added in 8.0
408 Name.startswith("avx512.mask.prorv.") || // Added in 8.0
409 Name.startswith("avx512.mask.pror.") || // Added in 8.0
410 Name.startswith("avx512.mask.prolv.") || // Added in 8.0
411 Name.startswith("avx512.mask.prol.") || // Added in 8.0
412 Name.startswith("avx512.ptestm") || //Added in 6.0
413 Name.startswith("avx512.ptestnm") || //Added in 6.0
414 Name.startswith("avx512.mask.pavg")) // Added in 6.0
415 return true;
416
417 return false;
418 }
419
UpgradeX86IntrinsicFunction(Function * F,StringRef Name,Function * & NewFn)420 static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
421 Function *&NewFn) {
422 // Only handle intrinsics that start with "x86.".
423 if (!Name.startswith("x86."))
424 return false;
425 // Remove "x86." prefix.
426 Name = Name.substr(4);
427
428 if (ShouldUpgradeX86Intrinsic(F, Name)) {
429 NewFn = nullptr;
430 return true;
431 }
432
433 if (Name == "rdtscp") { // Added in 8.0
434 // If this intrinsic has 0 operands, it's the new version.
435 if (F->getFunctionType()->getNumParams() == 0)
436 return false;
437
438 rename(F);
439 NewFn = Intrinsic::getDeclaration(F->getParent(),
440 Intrinsic::x86_rdtscp);
441 return true;
442 }
443
444 // SSE4.1 ptest functions may have an old signature.
445 if (Name.startswith("sse41.ptest")) { // Added in 3.2
446 if (Name.substr(11) == "c")
447 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn);
448 if (Name.substr(11) == "z")
449 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn);
450 if (Name.substr(11) == "nzc")
451 return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn);
452 }
453 // Several blend and other instructions with masks used the wrong number of
454 // bits.
455 if (Name == "sse41.insertps") // Added in 3.6
456 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps,
457 NewFn);
458 if (Name == "sse41.dppd") // Added in 3.6
459 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd,
460 NewFn);
461 if (Name == "sse41.dpps") // Added in 3.6
462 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps,
463 NewFn);
464 if (Name == "sse41.mpsadbw") // Added in 3.6
465 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw,
466 NewFn);
467 if (Name == "avx.dp.ps.256") // Added in 3.6
468 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256,
469 NewFn);
470 if (Name == "avx2.mpsadbw") // Added in 3.6
471 return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw,
472 NewFn);
473 if (Name == "avx512.mask.cmp.pd.128") // Added in 7.0
474 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_128,
475 NewFn);
476 if (Name == "avx512.mask.cmp.pd.256") // Added in 7.0
477 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_256,
478 NewFn);
479 if (Name == "avx512.mask.cmp.pd.512") // Added in 7.0
480 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_pd_512,
481 NewFn);
482 if (Name == "avx512.mask.cmp.ps.128") // Added in 7.0
483 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_128,
484 NewFn);
485 if (Name == "avx512.mask.cmp.ps.256") // Added in 7.0
486 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_256,
487 NewFn);
488 if (Name == "avx512.mask.cmp.ps.512") // Added in 7.0
489 return UpgradeX86MaskedFPCompare(F, Intrinsic::x86_avx512_mask_cmp_ps_512,
490 NewFn);
491
492 // frcz.ss/sd may need to have an argument dropped. Added in 3.2
493 if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) {
494 rename(F);
495 NewFn = Intrinsic::getDeclaration(F->getParent(),
496 Intrinsic::x86_xop_vfrcz_ss);
497 return true;
498 }
499 if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) {
500 rename(F);
501 NewFn = Intrinsic::getDeclaration(F->getParent(),
502 Intrinsic::x86_xop_vfrcz_sd);
503 return true;
504 }
505 // Upgrade any XOP PERMIL2 index operand still using a float/double vector.
506 if (Name.startswith("xop.vpermil2")) { // Added in 3.9
507 auto Idx = F->getFunctionType()->getParamType(2);
508 if (Idx->isFPOrFPVectorTy()) {
509 rename(F);
510 unsigned IdxSize = Idx->getPrimitiveSizeInBits();
511 unsigned EltSize = Idx->getScalarSizeInBits();
512 Intrinsic::ID Permil2ID;
513 if (EltSize == 64 && IdxSize == 128)
514 Permil2ID = Intrinsic::x86_xop_vpermil2pd;
515 else if (EltSize == 32 && IdxSize == 128)
516 Permil2ID = Intrinsic::x86_xop_vpermil2ps;
517 else if (EltSize == 64 && IdxSize == 256)
518 Permil2ID = Intrinsic::x86_xop_vpermil2pd_256;
519 else
520 Permil2ID = Intrinsic::x86_xop_vpermil2ps_256;
521 NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID);
522 return true;
523 }
524 }
525
526 if (Name == "seh.recoverfp") {
527 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
528 return true;
529 }
530
531 return false;
532 }
533
UpgradeIntrinsicFunction1(Function * F,Function * & NewFn)534 static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
535 assert(F && "Illegal to upgrade a non-existent Function.");
536
537 // Quickly eliminate it, if it's not a candidate.
538 StringRef Name = F->getName();
539 if (Name.size() <= 8 || !Name.startswith("llvm."))
540 return false;
541 Name = Name.substr(5); // Strip off "llvm."
542
543 switch (Name[0]) {
544 default: break;
545 case 'a': {
546 if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) {
547 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse,
548 F->arg_begin()->getType());
549 return true;
550 }
551 if (Name.startswith("arm.neon.vclz")) {
552 Type* args[2] = {
553 F->arg_begin()->getType(),
554 Type::getInt1Ty(F->getContext())
555 };
556 // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to
557 // the end of the name. Change name from llvm.arm.neon.vclz.* to
558 // llvm.ctlz.*
559 FunctionType* fType = FunctionType::get(F->getReturnType(), args, false);
560 NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
561 "llvm.ctlz." + Name.substr(14), F->getParent());
562 return true;
563 }
564 if (Name.startswith("arm.neon.vcnt")) {
565 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
566 F->arg_begin()->getType());
567 return true;
568 }
569 static const Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");
570 if (vldRegex.match(Name)) {
571 auto fArgs = F->getFunctionType()->params();
572 SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());
573 // Can't use Intrinsic::getDeclaration here as the return types might
574 // then only be structurally equal.
575 FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false);
576 NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(),
577 "llvm." + Name + ".p0i8", F->getParent());
578 return true;
579 }
580 static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
581 if (vstRegex.match(Name)) {
582 static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
583 Intrinsic::arm_neon_vst2,
584 Intrinsic::arm_neon_vst3,
585 Intrinsic::arm_neon_vst4};
586
587 static const Intrinsic::ID StoreLaneInts[] = {
588 Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane,
589 Intrinsic::arm_neon_vst4lane
590 };
591
592 auto fArgs = F->getFunctionType()->params();
593 Type *Tys[] = {fArgs[0], fArgs[1]};
594 if (Name.find("lane") == StringRef::npos)
595 NewFn = Intrinsic::getDeclaration(F->getParent(),
596 StoreInts[fArgs.size() - 3], Tys);
597 else
598 NewFn = Intrinsic::getDeclaration(F->getParent(),
599 StoreLaneInts[fArgs.size() - 5], Tys);
600 return true;
601 }
602 if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") {
603 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
604 return true;
605 }
606 if (Name.startswith("arm.neon.vqadds.")) {
607 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::sadd_sat,
608 F->arg_begin()->getType());
609 return true;
610 }
611 if (Name.startswith("arm.neon.vqaddu.")) {
612 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::uadd_sat,
613 F->arg_begin()->getType());
614 return true;
615 }
616 if (Name.startswith("arm.neon.vqsubs.")) {
617 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ssub_sat,
618 F->arg_begin()->getType());
619 return true;
620 }
621 if (Name.startswith("arm.neon.vqsubu.")) {
622 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::usub_sat,
623 F->arg_begin()->getType());
624 return true;
625 }
626 if (Name.startswith("aarch64.neon.addp")) {
627 if (F->arg_size() != 2)
628 break; // Invalid IR.
629 VectorType *Ty = dyn_cast<VectorType>(F->getReturnType());
630 if (Ty && Ty->getElementType()->isFloatingPointTy()) {
631 NewFn = Intrinsic::getDeclaration(F->getParent(),
632 Intrinsic::aarch64_neon_faddp, Ty);
633 return true;
634 }
635 }
636
637 // Changed in 12.0: bfdot accept v4bf16 and v8bf16 instead of v8i8 and v16i8
638 // respectively
639 if ((Name.startswith("arm.neon.bfdot.") ||
640 Name.startswith("aarch64.neon.bfdot.")) &&
641 Name.endswith("i8")) {
642 Intrinsic::ID IID =
643 StringSwitch<Intrinsic::ID>(Name)
644 .Cases("arm.neon.bfdot.v2f32.v8i8",
645 "arm.neon.bfdot.v4f32.v16i8",
646 Intrinsic::arm_neon_bfdot)
647 .Cases("aarch64.neon.bfdot.v2f32.v8i8",
648 "aarch64.neon.bfdot.v4f32.v16i8",
649 Intrinsic::aarch64_neon_bfdot)
650 .Default(Intrinsic::not_intrinsic);
651 if (IID == Intrinsic::not_intrinsic)
652 break;
653
654 size_t OperandWidth = F->getReturnType()->getPrimitiveSizeInBits();
655 assert((OperandWidth == 64 || OperandWidth == 128) &&
656 "Unexpected operand width");
657 LLVMContext &Ctx = F->getParent()->getContext();
658 std::array<Type *, 2> Tys {{
659 F->getReturnType(),
660 FixedVectorType::get(Type::getBFloatTy(Ctx), OperandWidth / 16)
661 }};
662 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
663 return true;
664 }
665
666 // Changed in 12.0: bfmmla, bfmlalb and bfmlalt are not polymorphic anymore
667 // and accept v8bf16 instead of v16i8
668 if ((Name.startswith("arm.neon.bfm") ||
669 Name.startswith("aarch64.neon.bfm")) &&
670 Name.endswith(".v4f32.v16i8")) {
671 Intrinsic::ID IID =
672 StringSwitch<Intrinsic::ID>(Name)
673 .Case("arm.neon.bfmmla.v4f32.v16i8",
674 Intrinsic::arm_neon_bfmmla)
675 .Case("arm.neon.bfmlalb.v4f32.v16i8",
676 Intrinsic::arm_neon_bfmlalb)
677 .Case("arm.neon.bfmlalt.v4f32.v16i8",
678 Intrinsic::arm_neon_bfmlalt)
679 .Case("aarch64.neon.bfmmla.v4f32.v16i8",
680 Intrinsic::aarch64_neon_bfmmla)
681 .Case("aarch64.neon.bfmlalb.v4f32.v16i8",
682 Intrinsic::aarch64_neon_bfmlalb)
683 .Case("aarch64.neon.bfmlalt.v4f32.v16i8",
684 Intrinsic::aarch64_neon_bfmlalt)
685 .Default(Intrinsic::not_intrinsic);
686 if (IID == Intrinsic::not_intrinsic)
687 break;
688
689 std::array<Type *, 0> Tys;
690 NewFn = Intrinsic::getDeclaration(F->getParent(), IID, Tys);
691 return true;
692 }
693 break;
694 }
695
696 case 'c': {
697 if (Name.startswith("ctlz.") && F->arg_size() == 1) {
698 rename(F);
699 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
700 F->arg_begin()->getType());
701 return true;
702 }
703 if (Name.startswith("cttz.") && F->arg_size() == 1) {
704 rename(F);
705 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz,
706 F->arg_begin()->getType());
707 return true;
708 }
709 break;
710 }
711 case 'd': {
712 if (Name == "dbg.value" && F->arg_size() == 4) {
713 rename(F);
714 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value);
715 return true;
716 }
717 break;
718 }
719 case 'e': {
720 SmallVector<StringRef, 2> Groups;
721 static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[a-z][0-9]+");
722 if (R.match(Name, &Groups)) {
723 Intrinsic::ID ID;
724 ID = StringSwitch<Intrinsic::ID>(Groups[1])
725 .Case("add", Intrinsic::vector_reduce_add)
726 .Case("mul", Intrinsic::vector_reduce_mul)
727 .Case("and", Intrinsic::vector_reduce_and)
728 .Case("or", Intrinsic::vector_reduce_or)
729 .Case("xor", Intrinsic::vector_reduce_xor)
730 .Case("smax", Intrinsic::vector_reduce_smax)
731 .Case("smin", Intrinsic::vector_reduce_smin)
732 .Case("umax", Intrinsic::vector_reduce_umax)
733 .Case("umin", Intrinsic::vector_reduce_umin)
734 .Case("fmax", Intrinsic::vector_reduce_fmax)
735 .Case("fmin", Intrinsic::vector_reduce_fmin)
736 .Default(Intrinsic::not_intrinsic);
737 if (ID != Intrinsic::not_intrinsic) {
738 rename(F);
739 auto Args = F->getFunctionType()->params();
740 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, {Args[0]});
741 return true;
742 }
743 }
744 static const Regex R2(
745 "^experimental.vector.reduce.v2.([a-z]+)\\.[fi][0-9]+");
746 Groups.clear();
747 if (R2.match(Name, &Groups)) {
748 Intrinsic::ID ID = Intrinsic::not_intrinsic;
749 if (Groups[1] == "fadd")
750 ID = Intrinsic::vector_reduce_fadd;
751 if (Groups[1] == "fmul")
752 ID = Intrinsic::vector_reduce_fmul;
753 if (ID != Intrinsic::not_intrinsic) {
754 rename(F);
755 auto Args = F->getFunctionType()->params();
756 Type *Tys[] = {Args[1]};
757 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
758 return true;
759 }
760 }
761 break;
762 }
763 case 'i':
764 case 'l': {
765 bool IsLifetimeStart = Name.startswith("lifetime.start");
766 if (IsLifetimeStart || Name.startswith("invariant.start")) {
767 Intrinsic::ID ID = IsLifetimeStart ?
768 Intrinsic::lifetime_start : Intrinsic::invariant_start;
769 auto Args = F->getFunctionType()->params();
770 Type* ObjectPtr[1] = {Args[1]};
771 if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
772 rename(F);
773 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
774 return true;
775 }
776 }
777
778 bool IsLifetimeEnd = Name.startswith("lifetime.end");
779 if (IsLifetimeEnd || Name.startswith("invariant.end")) {
780 Intrinsic::ID ID = IsLifetimeEnd ?
781 Intrinsic::lifetime_end : Intrinsic::invariant_end;
782
783 auto Args = F->getFunctionType()->params();
784 Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]};
785 if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) {
786 rename(F);
787 NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr);
788 return true;
789 }
790 }
791 if (Name.startswith("invariant.group.barrier")) {
792 // Rename invariant.group.barrier to launder.invariant.group
793 auto Args = F->getFunctionType()->params();
794 Type* ObjectPtr[1] = {Args[0]};
795 rename(F);
796 NewFn = Intrinsic::getDeclaration(F->getParent(),
797 Intrinsic::launder_invariant_group, ObjectPtr);
798 return true;
799
800 }
801
802 break;
803 }
804 case 'm': {
805 if (Name.startswith("masked.load.")) {
806 Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() };
807 if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) {
808 rename(F);
809 NewFn = Intrinsic::getDeclaration(F->getParent(),
810 Intrinsic::masked_load,
811 Tys);
812 return true;
813 }
814 }
815 if (Name.startswith("masked.store.")) {
816 auto Args = F->getFunctionType()->params();
817 Type *Tys[] = { Args[0], Args[1] };
818 if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) {
819 rename(F);
820 NewFn = Intrinsic::getDeclaration(F->getParent(),
821 Intrinsic::masked_store,
822 Tys);
823 return true;
824 }
825 }
826 // Renaming gather/scatter intrinsics with no address space overloading
827 // to the new overload which includes an address space
828 if (Name.startswith("masked.gather.")) {
829 Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()};
830 if (F->getName() != Intrinsic::getName(Intrinsic::masked_gather, Tys)) {
831 rename(F);
832 NewFn = Intrinsic::getDeclaration(F->getParent(),
833 Intrinsic::masked_gather, Tys);
834 return true;
835 }
836 }
837 if (Name.startswith("masked.scatter.")) {
838 auto Args = F->getFunctionType()->params();
839 Type *Tys[] = {Args[0], Args[1]};
840 if (F->getName() != Intrinsic::getName(Intrinsic::masked_scatter, Tys)) {
841 rename(F);
842 NewFn = Intrinsic::getDeclaration(F->getParent(),
843 Intrinsic::masked_scatter, Tys);
844 return true;
845 }
846 }
847 // Updating the memory intrinsics (memcpy/memmove/memset) that have an
848 // alignment parameter to embedding the alignment as an attribute of
849 // the pointer args.
850 if (Name.startswith("memcpy.") && F->arg_size() == 5) {
851 rename(F);
852 // Get the types of dest, src, and len
853 ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
854 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy,
855 ParamTypes);
856 return true;
857 }
858 if (Name.startswith("memmove.") && F->arg_size() == 5) {
859 rename(F);
860 // Get the types of dest, src, and len
861 ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3);
862 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove,
863 ParamTypes);
864 return true;
865 }
866 if (Name.startswith("memset.") && F->arg_size() == 5) {
867 rename(F);
868 // Get the types of dest, and len
869 const auto *FT = F->getFunctionType();
870 Type *ParamTypes[2] = {
871 FT->getParamType(0), // Dest
872 FT->getParamType(2) // len
873 };
874 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset,
875 ParamTypes);
876 return true;
877 }
878 break;
879 }
880 case 'n': {
881 if (Name.startswith("nvvm.")) {
882 Name = Name.substr(5);
883
884 // The following nvvm intrinsics correspond exactly to an LLVM intrinsic.
885 Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name)
886 .Cases("brev32", "brev64", Intrinsic::bitreverse)
887 .Case("clz.i", Intrinsic::ctlz)
888 .Case("popc.i", Intrinsic::ctpop)
889 .Default(Intrinsic::not_intrinsic);
890 if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) {
891 NewFn = Intrinsic::getDeclaration(F->getParent(), IID,
892 {F->getReturnType()});
893 return true;
894 }
895
896 // The following nvvm intrinsics correspond exactly to an LLVM idiom, but
897 // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall.
898 //
899 // TODO: We could add lohi.i2d.
900 bool Expand = StringSwitch<bool>(Name)
901 .Cases("abs.i", "abs.ll", true)
902 .Cases("clz.ll", "popc.ll", "h2f", true)
903 .Cases("max.i", "max.ll", "max.ui", "max.ull", true)
904 .Cases("min.i", "min.ll", "min.ui", "min.ull", true)
905 .StartsWith("atomic.load.add.f32.p", true)
906 .StartsWith("atomic.load.add.f64.p", true)
907 .Default(false);
908 if (Expand) {
909 NewFn = nullptr;
910 return true;
911 }
912 }
913 break;
914 }
915 case 'o':
916 // We only need to change the name to match the mangling including the
917 // address space.
918 if (Name.startswith("objectsize.")) {
919 Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
920 if (F->arg_size() == 2 || F->arg_size() == 3 ||
921 F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
922 rename(F);
923 NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
924 Tys);
925 return true;
926 }
927 }
928 break;
929
930 case 'p':
931 if (Name == "prefetch") {
932 // Handle address space overloading.
933 Type *Tys[] = {F->arg_begin()->getType()};
934 if (F->getName() != Intrinsic::getName(Intrinsic::prefetch, Tys)) {
935 rename(F);
936 NewFn =
937 Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys);
938 return true;
939 }
940 } else if (Name.startswith("ptr.annotation.") && F->arg_size() == 4) {
941 rename(F);
942 NewFn = Intrinsic::getDeclaration(F->getParent(),
943 Intrinsic::ptr_annotation,
944 F->arg_begin()->getType());
945 return true;
946 }
947 break;
948
949 case 's':
950 if (Name == "stackprotectorcheck") {
951 NewFn = nullptr;
952 return true;
953 }
954 break;
955
956 case 'v': {
957 if (Name == "var.annotation" && F->arg_size() == 4) {
958 rename(F);
959 NewFn = Intrinsic::getDeclaration(F->getParent(),
960 Intrinsic::var_annotation);
961 return true;
962 }
963 break;
964 }
965
966 case 'x':
967 if (UpgradeX86IntrinsicFunction(F, Name, NewFn))
968 return true;
969 }
970 // Remangle our intrinsic since we upgrade the mangling
971 auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F);
972 if (Result != None) {
973 NewFn = Result.getValue();
974 return true;
975 }
976
977 // This may not belong here. This function is effectively being overloaded
978 // to both detect an intrinsic which needs upgrading, and to provide the
979 // upgraded form of the intrinsic. We should perhaps have two separate
980 // functions for this.
981 return false;
982 }
983
UpgradeIntrinsicFunction(Function * F,Function * & NewFn)984 bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
985 NewFn = nullptr;
986 bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn);
987 assert(F != NewFn && "Intrinsic function upgraded to the same function");
988
989 // Upgrade intrinsic attributes. This does not change the function.
990 if (NewFn)
991 F = NewFn;
992 if (Intrinsic::ID id = F->getIntrinsicID())
993 F->setAttributes(Intrinsic::getAttributes(F->getContext(), id));
994 return Upgraded;
995 }
996
UpgradeGlobalVariable(GlobalVariable * GV)997 GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
998 if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
999 GV->getName() == "llvm.global_dtors")) ||
1000 !GV->hasInitializer())
1001 return nullptr;
1002 ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
1003 if (!ATy)
1004 return nullptr;
1005 StructType *STy = dyn_cast<StructType>(ATy->getElementType());
1006 if (!STy || STy->getNumElements() != 2)
1007 return nullptr;
1008
1009 LLVMContext &C = GV->getContext();
1010 IRBuilder<> IRB(C);
1011 auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
1012 IRB.getInt8PtrTy());
1013 Constant *Init = GV->getInitializer();
1014 unsigned N = Init->getNumOperands();
1015 std::vector<Constant *> NewCtors(N);
1016 for (unsigned i = 0; i != N; ++i) {
1017 auto Ctor = cast<Constant>(Init->getOperand(i));
1018 NewCtors[i] = ConstantStruct::get(
1019 EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
1020 Constant::getNullValue(IRB.getInt8PtrTy()));
1021 }
1022 Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
1023
1024 return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
1025 NewInit, GV->getName());
1026 }
1027
1028 // Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
1029 // to byte shuffles.
UpgradeX86PSLLDQIntrinsics(IRBuilder<> & Builder,Value * Op,unsigned Shift)1030 static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder,
1031 Value *Op, unsigned Shift) {
1032 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1033 unsigned NumElts = ResultTy->getNumElements() * 8;
1034
1035 // Bitcast from a 64-bit element type to a byte element type.
1036 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1037 Op = Builder.CreateBitCast(Op, VecTy, "cast");
1038
1039 // We'll be shuffling in zeroes.
1040 Value *Res = Constant::getNullValue(VecTy);
1041
1042 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1043 // we'll just return the zero vector.
1044 if (Shift < 16) {
1045 int Idxs[64];
1046 // 256/512-bit version is split into 2/4 16-byte lanes.
1047 for (unsigned l = 0; l != NumElts; l += 16)
1048 for (unsigned i = 0; i != 16; ++i) {
1049 unsigned Idx = NumElts + i - Shift;
1050 if (Idx < NumElts)
1051 Idx -= NumElts - 16; // end of lane, switch operand.
1052 Idxs[l + i] = Idx + l;
1053 }
1054
1055 Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts));
1056 }
1057
1058 // Bitcast back to a 64-bit element type.
1059 return Builder.CreateBitCast(Res, ResultTy, "cast");
1060 }
1061
1062 // Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them
1063 // to byte shuffles.
UpgradeX86PSRLDQIntrinsics(IRBuilder<> & Builder,Value * Op,unsigned Shift)1064 static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op,
1065 unsigned Shift) {
1066 auto *ResultTy = cast<FixedVectorType>(Op->getType());
1067 unsigned NumElts = ResultTy->getNumElements() * 8;
1068
1069 // Bitcast from a 64-bit element type to a byte element type.
1070 Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), NumElts);
1071 Op = Builder.CreateBitCast(Op, VecTy, "cast");
1072
1073 // We'll be shuffling in zeroes.
1074 Value *Res = Constant::getNullValue(VecTy);
1075
1076 // If shift is less than 16, emit a shuffle to move the bytes. Otherwise,
1077 // we'll just return the zero vector.
1078 if (Shift < 16) {
1079 int Idxs[64];
1080 // 256/512-bit version is split into 2/4 16-byte lanes.
1081 for (unsigned l = 0; l != NumElts; l += 16)
1082 for (unsigned i = 0; i != 16; ++i) {
1083 unsigned Idx = i + Shift;
1084 if (Idx >= 16)
1085 Idx += NumElts - 16; // end of lane, switch operand.
1086 Idxs[l + i] = Idx + l;
1087 }
1088
1089 Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts));
1090 }
1091
1092 // Bitcast back to a 64-bit element type.
1093 return Builder.CreateBitCast(Res, ResultTy, "cast");
1094 }
1095
getX86MaskVec(IRBuilder<> & Builder,Value * Mask,unsigned NumElts)1096 static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask,
1097 unsigned NumElts) {
1098 assert(isPowerOf2_32(NumElts) && "Expected power-of-2 mask elements");
1099 llvm::VectorType *MaskTy = FixedVectorType::get(
1100 Builder.getInt1Ty(), cast<IntegerType>(Mask->getType())->getBitWidth());
1101 Mask = Builder.CreateBitCast(Mask, MaskTy);
1102
1103 // If we have less than 8 elements (1, 2 or 4), then the starting mask was an
1104 // i8 and we need to extract down to the right number of elements.
1105 if (NumElts <= 4) {
1106 int Indices[4];
1107 for (unsigned i = 0; i != NumElts; ++i)
1108 Indices[i] = i;
1109 Mask = Builder.CreateShuffleVector(
1110 Mask, Mask, makeArrayRef(Indices, NumElts), "extract");
1111 }
1112
1113 return Mask;
1114 }
1115
EmitX86Select(IRBuilder<> & Builder,Value * Mask,Value * Op0,Value * Op1)1116 static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask,
1117 Value *Op0, Value *Op1) {
1118 // If the mask is all ones just emit the first operation.
1119 if (const auto *C = dyn_cast<Constant>(Mask))
1120 if (C->isAllOnesValue())
1121 return Op0;
1122
1123 Mask = getX86MaskVec(Builder, Mask,
1124 cast<FixedVectorType>(Op0->getType())->getNumElements());
1125 return Builder.CreateSelect(Mask, Op0, Op1);
1126 }
1127
EmitX86ScalarSelect(IRBuilder<> & Builder,Value * Mask,Value * Op0,Value * Op1)1128 static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask,
1129 Value *Op0, Value *Op1) {
1130 // If the mask is all ones just emit the first operation.
1131 if (const auto *C = dyn_cast<Constant>(Mask))
1132 if (C->isAllOnesValue())
1133 return Op0;
1134
1135 auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(),
1136 Mask->getType()->getIntegerBitWidth());
1137 Mask = Builder.CreateBitCast(Mask, MaskTy);
1138 Mask = Builder.CreateExtractElement(Mask, (uint64_t)0);
1139 return Builder.CreateSelect(Mask, Op0, Op1);
1140 }
1141
1142 // Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics.
1143 // PALIGNR handles large immediates by shifting while VALIGN masks the immediate
1144 // so we need to handle both cases. VALIGN also doesn't have 128-bit lanes.
UpgradeX86ALIGNIntrinsics(IRBuilder<> & Builder,Value * Op0,Value * Op1,Value * Shift,Value * Passthru,Value * Mask,bool IsVALIGN)1145 static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
1146 Value *Op1, Value *Shift,
1147 Value *Passthru, Value *Mask,
1148 bool IsVALIGN) {
1149 unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue();
1150
1151 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1152 assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!");
1153 assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!");
1154 assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!");
1155
1156 // Mask the immediate for VALIGN.
1157 if (IsVALIGN)
1158 ShiftVal &= (NumElts - 1);
1159
1160 // If palignr is shifting the pair of vectors more than the size of two
1161 // lanes, emit zero.
1162 if (ShiftVal >= 32)
1163 return llvm::Constant::getNullValue(Op0->getType());
1164
1165 // If palignr is shifting the pair of input vectors more than one lane,
1166 // but less than two lanes, convert to shifting in zeroes.
1167 if (ShiftVal > 16) {
1168 ShiftVal -= 16;
1169 Op1 = Op0;
1170 Op0 = llvm::Constant::getNullValue(Op0->getType());
1171 }
1172
1173 int Indices[64];
1174 // 256-bit palignr operates on 128-bit lanes so we need to handle that
1175 for (unsigned l = 0; l < NumElts; l += 16) {
1176 for (unsigned i = 0; i != 16; ++i) {
1177 unsigned Idx = ShiftVal + i;
1178 if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN.
1179 Idx += NumElts - 16; // End of lane, switch operand.
1180 Indices[l + i] = Idx + l;
1181 }
1182 }
1183
1184 Value *Align = Builder.CreateShuffleVector(Op1, Op0,
1185 makeArrayRef(Indices, NumElts),
1186 "palignr");
1187
1188 return EmitX86Select(Builder, Mask, Align, Passthru);
1189 }
1190
UpgradeX86VPERMT2Intrinsics(IRBuilder<> & Builder,CallInst & CI,bool ZeroMask,bool IndexForm)1191 static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,
1192 bool ZeroMask, bool IndexForm) {
1193 Type *Ty = CI.getType();
1194 unsigned VecWidth = Ty->getPrimitiveSizeInBits();
1195 unsigned EltWidth = Ty->getScalarSizeInBits();
1196 bool IsFloat = Ty->isFPOrFPVectorTy();
1197 Intrinsic::ID IID;
1198 if (VecWidth == 128 && EltWidth == 32 && IsFloat)
1199 IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
1200 else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
1201 IID = Intrinsic::x86_avx512_vpermi2var_d_128;
1202 else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
1203 IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
1204 else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
1205 IID = Intrinsic::x86_avx512_vpermi2var_q_128;
1206 else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1207 IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
1208 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1209 IID = Intrinsic::x86_avx512_vpermi2var_d_256;
1210 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1211 IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
1212 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1213 IID = Intrinsic::x86_avx512_vpermi2var_q_256;
1214 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1215 IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
1216 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1217 IID = Intrinsic::x86_avx512_vpermi2var_d_512;
1218 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1219 IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
1220 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1221 IID = Intrinsic::x86_avx512_vpermi2var_q_512;
1222 else if (VecWidth == 128 && EltWidth == 16)
1223 IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
1224 else if (VecWidth == 256 && EltWidth == 16)
1225 IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
1226 else if (VecWidth == 512 && EltWidth == 16)
1227 IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
1228 else if (VecWidth == 128 && EltWidth == 8)
1229 IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
1230 else if (VecWidth == 256 && EltWidth == 8)
1231 IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
1232 else if (VecWidth == 512 && EltWidth == 8)
1233 IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
1234 else
1235 llvm_unreachable("Unexpected intrinsic");
1236
1237 Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
1238 CI.getArgOperand(2) };
1239
1240 // If this isn't index form we need to swap operand 0 and 1.
1241 if (!IndexForm)
1242 std::swap(Args[0], Args[1]);
1243
1244 Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1245 Args);
1246 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
1247 : Builder.CreateBitCast(CI.getArgOperand(1),
1248 Ty);
1249 return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
1250 }
1251
UpgradeX86BinaryIntrinsics(IRBuilder<> & Builder,CallInst & CI,Intrinsic::ID IID)1252 static Value *UpgradeX86BinaryIntrinsics(IRBuilder<> &Builder, CallInst &CI,
1253 Intrinsic::ID IID) {
1254 Type *Ty = CI.getType();
1255 Value *Op0 = CI.getOperand(0);
1256 Value *Op1 = CI.getOperand(1);
1257 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1258 Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
1259
1260 if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
1261 Value *VecSrc = CI.getOperand(2);
1262 Value *Mask = CI.getOperand(3);
1263 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1264 }
1265 return Res;
1266 }
1267
upgradeX86Rotate(IRBuilder<> & Builder,CallInst & CI,bool IsRotateRight)1268 static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
1269 bool IsRotateRight) {
1270 Type *Ty = CI.getType();
1271 Value *Src = CI.getArgOperand(0);
1272 Value *Amt = CI.getArgOperand(1);
1273
1274 // Amount may be scalar immediate, in which case create a splat vector.
1275 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1276 // we only care about the lowest log2 bits anyway.
1277 if (Amt->getType() != Ty) {
1278 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1279 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1280 Amt = Builder.CreateVectorSplat(NumElts, Amt);
1281 }
1282
1283 Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
1284 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1285 Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
1286
1287 if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
1288 Value *VecSrc = CI.getOperand(2);
1289 Value *Mask = CI.getOperand(3);
1290 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1291 }
1292 return Res;
1293 }
1294
upgradeX86vpcom(IRBuilder<> & Builder,CallInst & CI,unsigned Imm,bool IsSigned)1295 static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,
1296 bool IsSigned) {
1297 Type *Ty = CI.getType();
1298 Value *LHS = CI.getArgOperand(0);
1299 Value *RHS = CI.getArgOperand(1);
1300
1301 CmpInst::Predicate Pred;
1302 switch (Imm) {
1303 case 0x0:
1304 Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
1305 break;
1306 case 0x1:
1307 Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
1308 break;
1309 case 0x2:
1310 Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
1311 break;
1312 case 0x3:
1313 Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
1314 break;
1315 case 0x4:
1316 Pred = ICmpInst::ICMP_EQ;
1317 break;
1318 case 0x5:
1319 Pred = ICmpInst::ICMP_NE;
1320 break;
1321 case 0x6:
1322 return Constant::getNullValue(Ty); // FALSE
1323 case 0x7:
1324 return Constant::getAllOnesValue(Ty); // TRUE
1325 default:
1326 llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
1327 }
1328
1329 Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
1330 Value *Ext = Builder.CreateSExt(Cmp, Ty);
1331 return Ext;
1332 }
1333
upgradeX86ConcatShift(IRBuilder<> & Builder,CallInst & CI,bool IsShiftRight,bool ZeroMask)1334 static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
1335 bool IsShiftRight, bool ZeroMask) {
1336 Type *Ty = CI.getType();
1337 Value *Op0 = CI.getArgOperand(0);
1338 Value *Op1 = CI.getArgOperand(1);
1339 Value *Amt = CI.getArgOperand(2);
1340
1341 if (IsShiftRight)
1342 std::swap(Op0, Op1);
1343
1344 // Amount may be scalar immediate, in which case create a splat vector.
1345 // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
1346 // we only care about the lowest log2 bits anyway.
1347 if (Amt->getType() != Ty) {
1348 unsigned NumElts = cast<FixedVectorType>(Ty)->getNumElements();
1349 Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
1350 Amt = Builder.CreateVectorSplat(NumElts, Amt);
1351 }
1352
1353 Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
1354 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
1355 Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
1356
1357 unsigned NumArgs = CI.getNumArgOperands();
1358 if (NumArgs >= 4) { // For masked intrinsics.
1359 Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
1360 ZeroMask ? ConstantAggregateZero::get(CI.getType()) :
1361 CI.getArgOperand(0);
1362 Value *Mask = CI.getOperand(NumArgs - 1);
1363 Res = EmitX86Select(Builder, Mask, Res, VecSrc);
1364 }
1365 return Res;
1366 }
1367
UpgradeMaskedStore(IRBuilder<> & Builder,Value * Ptr,Value * Data,Value * Mask,bool Aligned)1368 static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
1369 Value *Ptr, Value *Data, Value *Mask,
1370 bool Aligned) {
1371 // Cast the pointer to the right type.
1372 Ptr = Builder.CreateBitCast(Ptr,
1373 llvm::PointerType::getUnqual(Data->getType()));
1374 const Align Alignment =
1375 Aligned
1376 ? Align(Data->getType()->getPrimitiveSizeInBits().getFixedSize() / 8)
1377 : Align(1);
1378
1379 // If the mask is all ones just emit a regular store.
1380 if (const auto *C = dyn_cast<Constant>(Mask))
1381 if (C->isAllOnesValue())
1382 return Builder.CreateAlignedStore(Data, Ptr, Alignment);
1383
1384 // Convert the mask from an integer type to a vector of i1.
1385 unsigned NumElts = cast<FixedVectorType>(Data->getType())->getNumElements();
1386 Mask = getX86MaskVec(Builder, Mask, NumElts);
1387 return Builder.CreateMaskedStore(Data, Ptr, Alignment, Mask);
1388 }
1389
UpgradeMaskedLoad(IRBuilder<> & Builder,Value * Ptr,Value * Passthru,Value * Mask,bool Aligned)1390 static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
1391 Value *Ptr, Value *Passthru, Value *Mask,
1392 bool Aligned) {
1393 Type *ValTy = Passthru->getType();
1394 // Cast the pointer to the right type.
1395 Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
1396 const Align Alignment =
1397 Aligned
1398 ? Align(Passthru->getType()->getPrimitiveSizeInBits().getFixedSize() /
1399 8)
1400 : Align(1);
1401
1402 // If the mask is all ones just emit a regular store.
1403 if (const auto *C = dyn_cast<Constant>(Mask))
1404 if (C->isAllOnesValue())
1405 return Builder.CreateAlignedLoad(ValTy, Ptr, Alignment);
1406
1407 // Convert the mask from an integer type to a vector of i1.
1408 unsigned NumElts =
1409 cast<FixedVectorType>(Passthru->getType())->getNumElements();
1410 Mask = getX86MaskVec(Builder, Mask, NumElts);
1411 return Builder.CreateMaskedLoad(Ptr, Alignment, Mask, Passthru);
1412 }
1413
upgradeAbs(IRBuilder<> & Builder,CallInst & CI)1414 static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) {
1415 Type *Ty = CI.getType();
1416 Value *Op0 = CI.getArgOperand(0);
1417 Function *F = Intrinsic::getDeclaration(CI.getModule(), Intrinsic::abs, Ty);
1418 Value *Res = Builder.CreateCall(F, {Op0, Builder.getInt1(false)});
1419 if (CI.getNumArgOperands() == 3)
1420 Res = EmitX86Select(Builder, CI.getArgOperand(2), Res, CI.getArgOperand(1));
1421 return Res;
1422 }
1423
upgradePMULDQ(IRBuilder<> & Builder,CallInst & CI,bool IsSigned)1424 static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) {
1425 Type *Ty = CI.getType();
1426
1427 // Arguments have a vXi32 type so cast to vXi64.
1428 Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty);
1429 Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty);
1430
1431 if (IsSigned) {
1432 // Shift left then arithmetic shift right.
1433 Constant *ShiftAmt = ConstantInt::get(Ty, 32);
1434 LHS = Builder.CreateShl(LHS, ShiftAmt);
1435 LHS = Builder.CreateAShr(LHS, ShiftAmt);
1436 RHS = Builder.CreateShl(RHS, ShiftAmt);
1437 RHS = Builder.CreateAShr(RHS, ShiftAmt);
1438 } else {
1439 // Clear the upper bits.
1440 Constant *Mask = ConstantInt::get(Ty, 0xffffffff);
1441 LHS = Builder.CreateAnd(LHS, Mask);
1442 RHS = Builder.CreateAnd(RHS, Mask);
1443 }
1444
1445 Value *Res = Builder.CreateMul(LHS, RHS);
1446
1447 if (CI.getNumArgOperands() == 4)
1448 Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2));
1449
1450 return Res;
1451 }
1452
1453 // Applying mask on vector of i1's and make sure result is at least 8 bits wide.
ApplyX86MaskOn1BitsVec(IRBuilder<> & Builder,Value * Vec,Value * Mask)1454 static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec,
1455 Value *Mask) {
1456 unsigned NumElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
1457 if (Mask) {
1458 const auto *C = dyn_cast<Constant>(Mask);
1459 if (!C || !C->isAllOnesValue())
1460 Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts));
1461 }
1462
1463 if (NumElts < 8) {
1464 int Indices[8];
1465 for (unsigned i = 0; i != NumElts; ++i)
1466 Indices[i] = i;
1467 for (unsigned i = NumElts; i != 8; ++i)
1468 Indices[i] = NumElts + i % NumElts;
1469 Vec = Builder.CreateShuffleVector(Vec,
1470 Constant::getNullValue(Vec->getType()),
1471 Indices);
1472 }
1473 return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U)));
1474 }
1475
upgradeMaskedCompare(IRBuilder<> & Builder,CallInst & CI,unsigned CC,bool Signed)1476 static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI,
1477 unsigned CC, bool Signed) {
1478 Value *Op0 = CI.getArgOperand(0);
1479 unsigned NumElts = cast<FixedVectorType>(Op0->getType())->getNumElements();
1480
1481 Value *Cmp;
1482 if (CC == 3) {
1483 Cmp = Constant::getNullValue(
1484 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1485 } else if (CC == 7) {
1486 Cmp = Constant::getAllOnesValue(
1487 FixedVectorType::get(Builder.getInt1Ty(), NumElts));
1488 } else {
1489 ICmpInst::Predicate Pred;
1490 switch (CC) {
1491 default: llvm_unreachable("Unknown condition code");
1492 case 0: Pred = ICmpInst::ICMP_EQ; break;
1493 case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break;
1494 case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break;
1495 case 4: Pred = ICmpInst::ICMP_NE; break;
1496 case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break;
1497 case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break;
1498 }
1499 Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1));
1500 }
1501
1502 Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1);
1503
1504 return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask);
1505 }
1506
1507 // Replace a masked intrinsic with an older unmasked intrinsic.
UpgradeX86MaskedShift(IRBuilder<> & Builder,CallInst & CI,Intrinsic::ID IID)1508 static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI,
1509 Intrinsic::ID IID) {
1510 Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID);
1511 Value *Rep = Builder.CreateCall(Intrin,
1512 { CI.getArgOperand(0), CI.getArgOperand(1) });
1513 return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2));
1514 }
1515
upgradeMaskedMove(IRBuilder<> & Builder,CallInst & CI)1516 static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) {
1517 Value* A = CI.getArgOperand(0);
1518 Value* B = CI.getArgOperand(1);
1519 Value* Src = CI.getArgOperand(2);
1520 Value* Mask = CI.getArgOperand(3);
1521
1522 Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1));
1523 Value* Cmp = Builder.CreateIsNotNull(AndNode);
1524 Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0);
1525 Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0);
1526 Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2);
1527 return Builder.CreateInsertElement(A, Select, (uint64_t)0);
1528 }
1529
1530
UpgradeMaskToInt(IRBuilder<> & Builder,CallInst & CI)1531 static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) {
1532 Value* Op = CI.getArgOperand(0);
1533 Type* ReturnOp = CI.getType();
1534 unsigned NumElts = cast<FixedVectorType>(CI.getType())->getNumElements();
1535 Value *Mask = getX86MaskVec(Builder, Op, NumElts);
1536 return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2");
1537 }
1538
1539 // Replace intrinsic with unmasked version and a select.
upgradeAVX512MaskToSelect(StringRef Name,IRBuilder<> & Builder,CallInst & CI,Value * & Rep)1540 static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
1541 CallInst &CI, Value *&Rep) {
1542 Name = Name.substr(12); // Remove avx512.mask.
1543
1544 unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits();
1545 unsigned EltWidth = CI.getType()->getScalarSizeInBits();
1546 Intrinsic::ID IID;
1547 if (Name.startswith("max.p")) {
1548 if (VecWidth == 128 && EltWidth == 32)
1549 IID = Intrinsic::x86_sse_max_ps;
1550 else if (VecWidth == 128 && EltWidth == 64)
1551 IID = Intrinsic::x86_sse2_max_pd;
1552 else if (VecWidth == 256 && EltWidth == 32)
1553 IID = Intrinsic::x86_avx_max_ps_256;
1554 else if (VecWidth == 256 && EltWidth == 64)
1555 IID = Intrinsic::x86_avx_max_pd_256;
1556 else
1557 llvm_unreachable("Unexpected intrinsic");
1558 } else if (Name.startswith("min.p")) {
1559 if (VecWidth == 128 && EltWidth == 32)
1560 IID = Intrinsic::x86_sse_min_ps;
1561 else if (VecWidth == 128 && EltWidth == 64)
1562 IID = Intrinsic::x86_sse2_min_pd;
1563 else if (VecWidth == 256 && EltWidth == 32)
1564 IID = Intrinsic::x86_avx_min_ps_256;
1565 else if (VecWidth == 256 && EltWidth == 64)
1566 IID = Intrinsic::x86_avx_min_pd_256;
1567 else
1568 llvm_unreachable("Unexpected intrinsic");
1569 } else if (Name.startswith("pshuf.b.")) {
1570 if (VecWidth == 128)
1571 IID = Intrinsic::x86_ssse3_pshuf_b_128;
1572 else if (VecWidth == 256)
1573 IID = Intrinsic::x86_avx2_pshuf_b;
1574 else if (VecWidth == 512)
1575 IID = Intrinsic::x86_avx512_pshuf_b_512;
1576 else
1577 llvm_unreachable("Unexpected intrinsic");
1578 } else if (Name.startswith("pmul.hr.sw.")) {
1579 if (VecWidth == 128)
1580 IID = Intrinsic::x86_ssse3_pmul_hr_sw_128;
1581 else if (VecWidth == 256)
1582 IID = Intrinsic::x86_avx2_pmul_hr_sw;
1583 else if (VecWidth == 512)
1584 IID = Intrinsic::x86_avx512_pmul_hr_sw_512;
1585 else
1586 llvm_unreachable("Unexpected intrinsic");
1587 } else if (Name.startswith("pmulh.w.")) {
1588 if (VecWidth == 128)
1589 IID = Intrinsic::x86_sse2_pmulh_w;
1590 else if (VecWidth == 256)
1591 IID = Intrinsic::x86_avx2_pmulh_w;
1592 else if (VecWidth == 512)
1593 IID = Intrinsic::x86_avx512_pmulh_w_512;
1594 else
1595 llvm_unreachable("Unexpected intrinsic");
1596 } else if (Name.startswith("pmulhu.w.")) {
1597 if (VecWidth == 128)
1598 IID = Intrinsic::x86_sse2_pmulhu_w;
1599 else if (VecWidth == 256)
1600 IID = Intrinsic::x86_avx2_pmulhu_w;
1601 else if (VecWidth == 512)
1602 IID = Intrinsic::x86_avx512_pmulhu_w_512;
1603 else
1604 llvm_unreachable("Unexpected intrinsic");
1605 } else if (Name.startswith("pmaddw.d.")) {
1606 if (VecWidth == 128)
1607 IID = Intrinsic::x86_sse2_pmadd_wd;
1608 else if (VecWidth == 256)
1609 IID = Intrinsic::x86_avx2_pmadd_wd;
1610 else if (VecWidth == 512)
1611 IID = Intrinsic::x86_avx512_pmaddw_d_512;
1612 else
1613 llvm_unreachable("Unexpected intrinsic");
1614 } else if (Name.startswith("pmaddubs.w.")) {
1615 if (VecWidth == 128)
1616 IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128;
1617 else if (VecWidth == 256)
1618 IID = Intrinsic::x86_avx2_pmadd_ub_sw;
1619 else if (VecWidth == 512)
1620 IID = Intrinsic::x86_avx512_pmaddubs_w_512;
1621 else
1622 llvm_unreachable("Unexpected intrinsic");
1623 } else if (Name.startswith("packsswb.")) {
1624 if (VecWidth == 128)
1625 IID = Intrinsic::x86_sse2_packsswb_128;
1626 else if (VecWidth == 256)
1627 IID = Intrinsic::x86_avx2_packsswb;
1628 else if (VecWidth == 512)
1629 IID = Intrinsic::x86_avx512_packsswb_512;
1630 else
1631 llvm_unreachable("Unexpected intrinsic");
1632 } else if (Name.startswith("packssdw.")) {
1633 if (VecWidth == 128)
1634 IID = Intrinsic::x86_sse2_packssdw_128;
1635 else if (VecWidth == 256)
1636 IID = Intrinsic::x86_avx2_packssdw;
1637 else if (VecWidth == 512)
1638 IID = Intrinsic::x86_avx512_packssdw_512;
1639 else
1640 llvm_unreachable("Unexpected intrinsic");
1641 } else if (Name.startswith("packuswb.")) {
1642 if (VecWidth == 128)
1643 IID = Intrinsic::x86_sse2_packuswb_128;
1644 else if (VecWidth == 256)
1645 IID = Intrinsic::x86_avx2_packuswb;
1646 else if (VecWidth == 512)
1647 IID = Intrinsic::x86_avx512_packuswb_512;
1648 else
1649 llvm_unreachable("Unexpected intrinsic");
1650 } else if (Name.startswith("packusdw.")) {
1651 if (VecWidth == 128)
1652 IID = Intrinsic::x86_sse41_packusdw;
1653 else if (VecWidth == 256)
1654 IID = Intrinsic::x86_avx2_packusdw;
1655 else if (VecWidth == 512)
1656 IID = Intrinsic::x86_avx512_packusdw_512;
1657 else
1658 llvm_unreachable("Unexpected intrinsic");
1659 } else if (Name.startswith("vpermilvar.")) {
1660 if (VecWidth == 128 && EltWidth == 32)
1661 IID = Intrinsic::x86_avx_vpermilvar_ps;
1662 else if (VecWidth == 128 && EltWidth == 64)
1663 IID = Intrinsic::x86_avx_vpermilvar_pd;
1664 else if (VecWidth == 256 && EltWidth == 32)
1665 IID = Intrinsic::x86_avx_vpermilvar_ps_256;
1666 else if (VecWidth == 256 && EltWidth == 64)
1667 IID = Intrinsic::x86_avx_vpermilvar_pd_256;
1668 else if (VecWidth == 512 && EltWidth == 32)
1669 IID = Intrinsic::x86_avx512_vpermilvar_ps_512;
1670 else if (VecWidth == 512 && EltWidth == 64)
1671 IID = Intrinsic::x86_avx512_vpermilvar_pd_512;
1672 else
1673 llvm_unreachable("Unexpected intrinsic");
1674 } else if (Name == "cvtpd2dq.256") {
1675 IID = Intrinsic::x86_avx_cvt_pd2dq_256;
1676 } else if (Name == "cvtpd2ps.256") {
1677 IID = Intrinsic::x86_avx_cvt_pd2_ps_256;
1678 } else if (Name == "cvttpd2dq.256") {
1679 IID = Intrinsic::x86_avx_cvtt_pd2dq_256;
1680 } else if (Name == "cvttps2dq.128") {
1681 IID = Intrinsic::x86_sse2_cvttps2dq;
1682 } else if (Name == "cvttps2dq.256") {
1683 IID = Intrinsic::x86_avx_cvtt_ps2dq_256;
1684 } else if (Name.startswith("permvar.")) {
1685 bool IsFloat = CI.getType()->isFPOrFPVectorTy();
1686 if (VecWidth == 256 && EltWidth == 32 && IsFloat)
1687 IID = Intrinsic::x86_avx2_permps;
1688 else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
1689 IID = Intrinsic::x86_avx2_permd;
1690 else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
1691 IID = Intrinsic::x86_avx512_permvar_df_256;
1692 else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
1693 IID = Intrinsic::x86_avx512_permvar_di_256;
1694 else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
1695 IID = Intrinsic::x86_avx512_permvar_sf_512;
1696 else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
1697 IID = Intrinsic::x86_avx512_permvar_si_512;
1698 else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
1699 IID = Intrinsic::x86_avx512_permvar_df_512;
1700 else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
1701 IID = Intrinsic::x86_avx512_permvar_di_512;
1702 else if (VecWidth == 128 && EltWidth == 16)
1703 IID = Intrinsic::x86_avx512_permvar_hi_128;
1704 else if (VecWidth == 256 && EltWidth == 16)
1705 IID = Intrinsic::x86_avx512_permvar_hi_256;
1706 else if (VecWidth == 512 && EltWidth == 16)
1707 IID = Intrinsic::x86_avx512_permvar_hi_512;
1708 else if (VecWidth == 128 && EltWidth == 8)
1709 IID = Intrinsic::x86_avx512_permvar_qi_128;
1710 else if (VecWidth == 256 && EltWidth == 8)
1711 IID = Intrinsic::x86_avx512_permvar_qi_256;
1712 else if (VecWidth == 512 && EltWidth == 8)
1713 IID = Intrinsic::x86_avx512_permvar_qi_512;
1714 else
1715 llvm_unreachable("Unexpected intrinsic");
1716 } else if (Name.startswith("dbpsadbw.")) {
1717 if (VecWidth == 128)
1718 IID = Intrinsic::x86_avx512_dbpsadbw_128;
1719 else if (VecWidth == 256)
1720 IID = Intrinsic::x86_avx512_dbpsadbw_256;
1721 else if (VecWidth == 512)
1722 IID = Intrinsic::x86_avx512_dbpsadbw_512;
1723 else
1724 llvm_unreachable("Unexpected intrinsic");
1725 } else if (Name.startswith("pmultishift.qb.")) {
1726 if (VecWidth == 128)
1727 IID = Intrinsic::x86_avx512_pmultishift_qb_128;
1728 else if (VecWidth == 256)
1729 IID = Intrinsic::x86_avx512_pmultishift_qb_256;
1730 else if (VecWidth == 512)
1731 IID = Intrinsic::x86_avx512_pmultishift_qb_512;
1732 else
1733 llvm_unreachable("Unexpected intrinsic");
1734 } else if (Name.startswith("conflict.")) {
1735 if (Name[9] == 'd' && VecWidth == 128)
1736 IID = Intrinsic::x86_avx512_conflict_d_128;
1737 else if (Name[9] == 'd' && VecWidth == 256)
1738 IID = Intrinsic::x86_avx512_conflict_d_256;
1739 else if (Name[9] == 'd' && VecWidth == 512)
1740 IID = Intrinsic::x86_avx512_conflict_d_512;
1741 else if (Name[9] == 'q' && VecWidth == 128)
1742 IID = Intrinsic::x86_avx512_conflict_q_128;
1743 else if (Name[9] == 'q' && VecWidth == 256)
1744 IID = Intrinsic::x86_avx512_conflict_q_256;
1745 else if (Name[9] == 'q' && VecWidth == 512)
1746 IID = Intrinsic::x86_avx512_conflict_q_512;
1747 else
1748 llvm_unreachable("Unexpected intrinsic");
1749 } else if (Name.startswith("pavg.")) {
1750 if (Name[5] == 'b' && VecWidth == 128)
1751 IID = Intrinsic::x86_sse2_pavg_b;
1752 else if (Name[5] == 'b' && VecWidth == 256)
1753 IID = Intrinsic::x86_avx2_pavg_b;
1754 else if (Name[5] == 'b' && VecWidth == 512)
1755 IID = Intrinsic::x86_avx512_pavg_b_512;
1756 else if (Name[5] == 'w' && VecWidth == 128)
1757 IID = Intrinsic::x86_sse2_pavg_w;
1758 else if (Name[5] == 'w' && VecWidth == 256)
1759 IID = Intrinsic::x86_avx2_pavg_w;
1760 else if (Name[5] == 'w' && VecWidth == 512)
1761 IID = Intrinsic::x86_avx512_pavg_w_512;
1762 else
1763 llvm_unreachable("Unexpected intrinsic");
1764 } else
1765 return false;
1766
1767 SmallVector<Value *, 4> Args(CI.arg_operands().begin(),
1768 CI.arg_operands().end());
1769 Args.pop_back();
1770 Args.pop_back();
1771 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
1772 Args);
1773 unsigned NumArgs = CI.getNumArgOperands();
1774 Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep,
1775 CI.getArgOperand(NumArgs - 2));
1776 return true;
1777 }
1778
1779 /// Upgrade comment in call to inline asm that represents an objc retain release
1780 /// marker.
UpgradeInlineAsmString(std::string * AsmStr)1781 void llvm::UpgradeInlineAsmString(std::string *AsmStr) {
1782 size_t Pos;
1783 if (AsmStr->find("mov\tfp") == 0 &&
1784 AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos &&
1785 (Pos = AsmStr->find("# marker")) != std::string::npos) {
1786 AsmStr->replace(Pos, 1, ";");
1787 }
1788 }
1789
1790 /// Upgrade a call to an old intrinsic. All argument and return casting must be
1791 /// provided to seamlessly integrate with existing context.
UpgradeIntrinsicCall(CallInst * CI,Function * NewFn)1792 void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
1793 Function *F = CI->getCalledFunction();
1794 LLVMContext &C = CI->getContext();
1795 IRBuilder<> Builder(C);
1796 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
1797
1798 assert(F && "Intrinsic call is not direct?");
1799
1800 if (!NewFn) {
1801 // Get the Function's name.
1802 StringRef Name = F->getName();
1803
1804 assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");
1805 Name = Name.substr(5);
1806
1807 bool IsX86 = Name.startswith("x86.");
1808 if (IsX86)
1809 Name = Name.substr(4);
1810 bool IsNVVM = Name.startswith("nvvm.");
1811 if (IsNVVM)
1812 Name = Name.substr(5);
1813
1814 if (IsX86 && Name.startswith("sse4a.movnt.")) {
1815 Module *M = F->getParent();
1816 SmallVector<Metadata *, 1> Elts;
1817 Elts.push_back(
1818 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
1819 MDNode *Node = MDNode::get(C, Elts);
1820
1821 Value *Arg0 = CI->getArgOperand(0);
1822 Value *Arg1 = CI->getArgOperand(1);
1823
1824 // Nontemporal (unaligned) store of the 0'th element of the float/double
1825 // vector.
1826 Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType();
1827 PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy);
1828 Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast");
1829 Value *Extract =
1830 Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement");
1831
1832 StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, Align(1));
1833 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
1834
1835 // Remove intrinsic.
1836 CI->eraseFromParent();
1837 return;
1838 }
1839
1840 if (IsX86 && (Name.startswith("avx.movnt.") ||
1841 Name.startswith("avx512.storent."))) {
1842 Module *M = F->getParent();
1843 SmallVector<Metadata *, 1> Elts;
1844 Elts.push_back(
1845 ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
1846 MDNode *Node = MDNode::get(C, Elts);
1847
1848 Value *Arg0 = CI->getArgOperand(0);
1849 Value *Arg1 = CI->getArgOperand(1);
1850
1851 // Convert the type of the pointer to a pointer to the stored type.
1852 Value *BC = Builder.CreateBitCast(Arg0,
1853 PointerType::getUnqual(Arg1->getType()),
1854 "cast");
1855 StoreInst *SI = Builder.CreateAlignedStore(
1856 Arg1, BC,
1857 Align(Arg1->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
1858 SI->setMetadata(M->getMDKindID("nontemporal"), Node);
1859
1860 // Remove intrinsic.
1861 CI->eraseFromParent();
1862 return;
1863 }
1864
1865 if (IsX86 && Name == "sse2.storel.dq") {
1866 Value *Arg0 = CI->getArgOperand(0);
1867 Value *Arg1 = CI->getArgOperand(1);
1868
1869 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
1870 Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
1871 Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0);
1872 Value *BC = Builder.CreateBitCast(Arg0,
1873 PointerType::getUnqual(Elt->getType()),
1874 "cast");
1875 Builder.CreateAlignedStore(Elt, BC, Align(1));
1876
1877 // Remove intrinsic.
1878 CI->eraseFromParent();
1879 return;
1880 }
1881
1882 if (IsX86 && (Name.startswith("sse.storeu.") ||
1883 Name.startswith("sse2.storeu.") ||
1884 Name.startswith("avx.storeu."))) {
1885 Value *Arg0 = CI->getArgOperand(0);
1886 Value *Arg1 = CI->getArgOperand(1);
1887
1888 Arg0 = Builder.CreateBitCast(Arg0,
1889 PointerType::getUnqual(Arg1->getType()),
1890 "cast");
1891 Builder.CreateAlignedStore(Arg1, Arg0, Align(1));
1892
1893 // Remove intrinsic.
1894 CI->eraseFromParent();
1895 return;
1896 }
1897
1898 if (IsX86 && Name == "avx512.mask.store.ss") {
1899 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
1900 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1901 Mask, false);
1902
1903 // Remove intrinsic.
1904 CI->eraseFromParent();
1905 return;
1906 }
1907
1908 if (IsX86 && (Name.startswith("avx512.mask.store"))) {
1909 // "avx512.mask.storeu." or "avx512.mask.store."
1910 bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu".
1911 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1912 CI->getArgOperand(2), Aligned);
1913
1914 // Remove intrinsic.
1915 CI->eraseFromParent();
1916 return;
1917 }
1918
1919 Value *Rep;
1920 // Upgrade packed integer vector compare intrinsics to compare instructions.
1921 if (IsX86 && (Name.startswith("sse2.pcmp") ||
1922 Name.startswith("avx2.pcmp"))) {
1923 // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt."
1924 bool CmpEq = Name[9] == 'e';
1925 Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT,
1926 CI->getArgOperand(0), CI->getArgOperand(1));
1927 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
1928 } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) {
1929 Type *ExtTy = Type::getInt32Ty(C);
1930 if (CI->getOperand(0)->getType()->isIntegerTy(8))
1931 ExtTy = Type::getInt64Ty(C);
1932 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
1933 ExtTy->getPrimitiveSizeInBits();
1934 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
1935 Rep = Builder.CreateVectorSplat(NumElts, Rep);
1936 } else if (IsX86 && (Name == "sse.sqrt.ss" ||
1937 Name == "sse2.sqrt.sd")) {
1938 Value *Vec = CI->getArgOperand(0);
1939 Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0);
1940 Function *Intr = Intrinsic::getDeclaration(F->getParent(),
1941 Intrinsic::sqrt, Elt0->getType());
1942 Elt0 = Builder.CreateCall(Intr, Elt0);
1943 Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0);
1944 } else if (IsX86 && (Name.startswith("avx.sqrt.p") ||
1945 Name.startswith("sse2.sqrt.p") ||
1946 Name.startswith("sse.sqrt.p"))) {
1947 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1948 Intrinsic::sqrt,
1949 CI->getType()),
1950 {CI->getArgOperand(0)});
1951 } else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) {
1952 if (CI->getNumArgOperands() == 4 &&
1953 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
1954 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
1955 Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512
1956 : Intrinsic::x86_avx512_sqrt_pd_512;
1957
1958 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
1959 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
1960 IID), Args);
1961 } else {
1962 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
1963 Intrinsic::sqrt,
1964 CI->getType()),
1965 {CI->getArgOperand(0)});
1966 }
1967 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1968 CI->getArgOperand(1));
1969 } else if (IsX86 && (Name.startswith("avx512.ptestm") ||
1970 Name.startswith("avx512.ptestnm"))) {
1971 Value *Op0 = CI->getArgOperand(0);
1972 Value *Op1 = CI->getArgOperand(1);
1973 Value *Mask = CI->getArgOperand(2);
1974 Rep = Builder.CreateAnd(Op0, Op1);
1975 llvm::Type *Ty = Op0->getType();
1976 Value *Zero = llvm::Constant::getNullValue(Ty);
1977 ICmpInst::Predicate Pred =
1978 Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
1979 Rep = Builder.CreateICmp(Pred, Rep, Zero);
1980 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask);
1981 } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){
1982 unsigned NumElts = cast<FixedVectorType>(CI->getArgOperand(1)->getType())
1983 ->getNumElements();
1984 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
1985 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1986 CI->getArgOperand(1));
1987 } else if (IsX86 && (Name.startswith("avx512.kunpck"))) {
1988 unsigned NumElts = CI->getType()->getScalarSizeInBits();
1989 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
1990 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
1991 int Indices[64];
1992 for (unsigned i = 0; i != NumElts; ++i)
1993 Indices[i] = i;
1994
1995 // First extract half of each vector. This gives better codegen than
1996 // doing it in a single shuffle.
1997 LHS = Builder.CreateShuffleVector(LHS, LHS,
1998 makeArrayRef(Indices, NumElts / 2));
1999 RHS = Builder.CreateShuffleVector(RHS, RHS,
2000 makeArrayRef(Indices, NumElts / 2));
2001 // Concat the vectors.
2002 // NOTE: Operands have to be swapped to match intrinsic definition.
2003 Rep = Builder.CreateShuffleVector(RHS, LHS,
2004 makeArrayRef(Indices, NumElts));
2005 Rep = Builder.CreateBitCast(Rep, CI->getType());
2006 } else if (IsX86 && Name == "avx512.kand.w") {
2007 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2008 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2009 Rep = Builder.CreateAnd(LHS, RHS);
2010 Rep = Builder.CreateBitCast(Rep, CI->getType());
2011 } else if (IsX86 && Name == "avx512.kandn.w") {
2012 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2013 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2014 LHS = Builder.CreateNot(LHS);
2015 Rep = Builder.CreateAnd(LHS, RHS);
2016 Rep = Builder.CreateBitCast(Rep, CI->getType());
2017 } else if (IsX86 && Name == "avx512.kor.w") {
2018 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2019 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2020 Rep = Builder.CreateOr(LHS, RHS);
2021 Rep = Builder.CreateBitCast(Rep, CI->getType());
2022 } else if (IsX86 && Name == "avx512.kxor.w") {
2023 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2024 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2025 Rep = Builder.CreateXor(LHS, RHS);
2026 Rep = Builder.CreateBitCast(Rep, CI->getType());
2027 } else if (IsX86 && Name == "avx512.kxnor.w") {
2028 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2029 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2030 LHS = Builder.CreateNot(LHS);
2031 Rep = Builder.CreateXor(LHS, RHS);
2032 Rep = Builder.CreateBitCast(Rep, CI->getType());
2033 } else if (IsX86 && Name == "avx512.knot.w") {
2034 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2035 Rep = Builder.CreateNot(Rep);
2036 Rep = Builder.CreateBitCast(Rep, CI->getType());
2037 } else if (IsX86 &&
2038 (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) {
2039 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
2040 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
2041 Rep = Builder.CreateOr(LHS, RHS);
2042 Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty());
2043 Value *C;
2044 if (Name[14] == 'c')
2045 C = ConstantInt::getAllOnesValue(Builder.getInt16Ty());
2046 else
2047 C = ConstantInt::getNullValue(Builder.getInt16Ty());
2048 Rep = Builder.CreateICmpEQ(Rep, C);
2049 Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty());
2050 } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" ||
2051 Name == "sse.sub.ss" || Name == "sse2.sub.sd" ||
2052 Name == "sse.mul.ss" || Name == "sse2.mul.sd" ||
2053 Name == "sse.div.ss" || Name == "sse2.div.sd")) {
2054 Type *I32Ty = Type::getInt32Ty(C);
2055 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
2056 ConstantInt::get(I32Ty, 0));
2057 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
2058 ConstantInt::get(I32Ty, 0));
2059 Value *EltOp;
2060 if (Name.contains(".add."))
2061 EltOp = Builder.CreateFAdd(Elt0, Elt1);
2062 else if (Name.contains(".sub."))
2063 EltOp = Builder.CreateFSub(Elt0, Elt1);
2064 else if (Name.contains(".mul."))
2065 EltOp = Builder.CreateFMul(Elt0, Elt1);
2066 else
2067 EltOp = Builder.CreateFDiv(Elt0, Elt1);
2068 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
2069 ConstantInt::get(I32Ty, 0));
2070 } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) {
2071 // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
2072 bool CmpEq = Name[16] == 'e';
2073 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
2074 } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {
2075 Type *OpTy = CI->getArgOperand(0)->getType();
2076 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2077 Intrinsic::ID IID;
2078 switch (VecWidth) {
2079 default: llvm_unreachable("Unexpected intrinsic");
2080 case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
2081 case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
2082 case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
2083 }
2084
2085 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2086 { CI->getOperand(0), CI->getArgOperand(1) });
2087 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2088 } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {
2089 Type *OpTy = CI->getArgOperand(0)->getType();
2090 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2091 unsigned EltWidth = OpTy->getScalarSizeInBits();
2092 Intrinsic::ID IID;
2093 if (VecWidth == 128 && EltWidth == 32)
2094 IID = Intrinsic::x86_avx512_fpclass_ps_128;
2095 else if (VecWidth == 256 && EltWidth == 32)
2096 IID = Intrinsic::x86_avx512_fpclass_ps_256;
2097 else if (VecWidth == 512 && EltWidth == 32)
2098 IID = Intrinsic::x86_avx512_fpclass_ps_512;
2099 else if (VecWidth == 128 && EltWidth == 64)
2100 IID = Intrinsic::x86_avx512_fpclass_pd_128;
2101 else if (VecWidth == 256 && EltWidth == 64)
2102 IID = Intrinsic::x86_avx512_fpclass_pd_256;
2103 else if (VecWidth == 512 && EltWidth == 64)
2104 IID = Intrinsic::x86_avx512_fpclass_pd_512;
2105 else
2106 llvm_unreachable("Unexpected intrinsic");
2107
2108 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2109 { CI->getOperand(0), CI->getArgOperand(1) });
2110 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
2111 } else if (IsX86 && Name.startswith("avx512.cmp.p")) {
2112 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
2113 CI->arg_operands().end());
2114 Type *OpTy = Args[0]->getType();
2115 unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
2116 unsigned EltWidth = OpTy->getScalarSizeInBits();
2117 Intrinsic::ID IID;
2118 if (VecWidth == 128 && EltWidth == 32)
2119 IID = Intrinsic::x86_avx512_mask_cmp_ps_128;
2120 else if (VecWidth == 256 && EltWidth == 32)
2121 IID = Intrinsic::x86_avx512_mask_cmp_ps_256;
2122 else if (VecWidth == 512 && EltWidth == 32)
2123 IID = Intrinsic::x86_avx512_mask_cmp_ps_512;
2124 else if (VecWidth == 128 && EltWidth == 64)
2125 IID = Intrinsic::x86_avx512_mask_cmp_pd_128;
2126 else if (VecWidth == 256 && EltWidth == 64)
2127 IID = Intrinsic::x86_avx512_mask_cmp_pd_256;
2128 else if (VecWidth == 512 && EltWidth == 64)
2129 IID = Intrinsic::x86_avx512_mask_cmp_pd_512;
2130 else
2131 llvm_unreachable("Unexpected intrinsic");
2132
2133 Value *Mask = Constant::getAllOnesValue(CI->getType());
2134 if (VecWidth == 512)
2135 std::swap(Mask, Args.back());
2136 Args.push_back(Mask);
2137
2138 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2139 Args);
2140 } else if (IsX86 && Name.startswith("avx512.mask.cmp.")) {
2141 // Integer compare intrinsics.
2142 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2143 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
2144 } else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) {
2145 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2146 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
2147 } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") ||
2148 Name.startswith("avx512.cvtw2mask.") ||
2149 Name.startswith("avx512.cvtd2mask.") ||
2150 Name.startswith("avx512.cvtq2mask."))) {
2151 Value *Op = CI->getArgOperand(0);
2152 Value *Zero = llvm::Constant::getNullValue(Op->getType());
2153 Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero);
2154 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr);
2155 } else if(IsX86 && (Name == "ssse3.pabs.b.128" ||
2156 Name == "ssse3.pabs.w.128" ||
2157 Name == "ssse3.pabs.d.128" ||
2158 Name.startswith("avx2.pabs") ||
2159 Name.startswith("avx512.mask.pabs"))) {
2160 Rep = upgradeAbs(Builder, *CI);
2161 } else if (IsX86 && (Name == "sse41.pmaxsb" ||
2162 Name == "sse2.pmaxs.w" ||
2163 Name == "sse41.pmaxsd" ||
2164 Name.startswith("avx2.pmaxs") ||
2165 Name.startswith("avx512.mask.pmaxs"))) {
2166 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smax);
2167 } else if (IsX86 && (Name == "sse2.pmaxu.b" ||
2168 Name == "sse41.pmaxuw" ||
2169 Name == "sse41.pmaxud" ||
2170 Name.startswith("avx2.pmaxu") ||
2171 Name.startswith("avx512.mask.pmaxu"))) {
2172 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umax);
2173 } else if (IsX86 && (Name == "sse41.pminsb" ||
2174 Name == "sse2.pmins.w" ||
2175 Name == "sse41.pminsd" ||
2176 Name.startswith("avx2.pmins") ||
2177 Name.startswith("avx512.mask.pmins"))) {
2178 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::smin);
2179 } else if (IsX86 && (Name == "sse2.pminu.b" ||
2180 Name == "sse41.pminuw" ||
2181 Name == "sse41.pminud" ||
2182 Name.startswith("avx2.pminu") ||
2183 Name.startswith("avx512.mask.pminu"))) {
2184 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::umin);
2185 } else if (IsX86 && (Name == "sse2.pmulu.dq" ||
2186 Name == "avx2.pmulu.dq" ||
2187 Name == "avx512.pmulu.dq.512" ||
2188 Name.startswith("avx512.mask.pmulu.dq."))) {
2189 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
2190 } else if (IsX86 && (Name == "sse41.pmuldq" ||
2191 Name == "avx2.pmul.dq" ||
2192 Name == "avx512.pmul.dq.512" ||
2193 Name.startswith("avx512.mask.pmul.dq."))) {
2194 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
2195 } else if (IsX86 && (Name == "sse.cvtsi2ss" ||
2196 Name == "sse2.cvtsi2sd" ||
2197 Name == "sse.cvtsi642ss" ||
2198 Name == "sse2.cvtsi642sd")) {
2199 Rep = Builder.CreateSIToFP(
2200 CI->getArgOperand(1),
2201 cast<VectorType>(CI->getType())->getElementType());
2202 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2203 } else if (IsX86 && Name == "avx512.cvtusi2sd") {
2204 Rep = Builder.CreateUIToFP(
2205 CI->getArgOperand(1),
2206 cast<VectorType>(CI->getType())->getElementType());
2207 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2208 } else if (IsX86 && Name == "sse2.cvtss2sd") {
2209 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
2210 Rep = Builder.CreateFPExt(
2211 Rep, cast<VectorType>(CI->getType())->getElementType());
2212 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2213 } else if (IsX86 && (Name == "sse2.cvtdq2pd" ||
2214 Name == "sse2.cvtdq2ps" ||
2215 Name == "avx.cvtdq2.pd.256" ||
2216 Name == "avx.cvtdq2.ps.256" ||
2217 Name.startswith("avx512.mask.cvtdq2pd.") ||
2218 Name.startswith("avx512.mask.cvtudq2pd.") ||
2219 Name.startswith("avx512.mask.cvtdq2ps.") ||
2220 Name.startswith("avx512.mask.cvtudq2ps.") ||
2221 Name.startswith("avx512.mask.cvtqq2pd.") ||
2222 Name.startswith("avx512.mask.cvtuqq2pd.") ||
2223 Name == "avx512.mask.cvtqq2ps.256" ||
2224 Name == "avx512.mask.cvtqq2ps.512" ||
2225 Name == "avx512.mask.cvtuqq2ps.256" ||
2226 Name == "avx512.mask.cvtuqq2ps.512" ||
2227 Name == "sse2.cvtps2pd" ||
2228 Name == "avx.cvt.ps2.pd.256" ||
2229 Name == "avx512.mask.cvtps2pd.128" ||
2230 Name == "avx512.mask.cvtps2pd.256")) {
2231 auto *DstTy = cast<FixedVectorType>(CI->getType());
2232 Rep = CI->getArgOperand(0);
2233 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2234
2235 unsigned NumDstElts = DstTy->getNumElements();
2236 if (NumDstElts < SrcTy->getNumElements()) {
2237 assert(NumDstElts == 2 && "Unexpected vector size");
2238 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1});
2239 }
2240
2241 bool IsPS2PD = SrcTy->getElementType()->isFloatTy();
2242 bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
2243 if (IsPS2PD)
2244 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
2245 else if (CI->getNumArgOperands() == 4 &&
2246 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2247 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2248 Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
2249 : Intrinsic::x86_avx512_sitofp_round;
2250 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
2251 { DstTy, SrcTy });
2252 Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
2253 } else {
2254 Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
2255 : Builder.CreateSIToFP(Rep, DstTy, "cvt");
2256 }
2257
2258 if (CI->getNumArgOperands() >= 3)
2259 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2260 CI->getArgOperand(1));
2261 } else if (IsX86 && (Name.startswith("avx512.mask.vcvtph2ps.") ||
2262 Name.startswith("vcvtph2ps."))) {
2263 auto *DstTy = cast<FixedVectorType>(CI->getType());
2264 Rep = CI->getArgOperand(0);
2265 auto *SrcTy = cast<FixedVectorType>(Rep->getType());
2266 unsigned NumDstElts = DstTy->getNumElements();
2267 if (NumDstElts != SrcTy->getNumElements()) {
2268 assert(NumDstElts == 4 && "Unexpected vector size");
2269 Rep = Builder.CreateShuffleVector(Rep, Rep, ArrayRef<int>{0, 1, 2, 3});
2270 }
2271 Rep = Builder.CreateBitCast(
2272 Rep, FixedVectorType::get(Type::getHalfTy(C), NumDstElts));
2273 Rep = Builder.CreateFPExt(Rep, DstTy, "cvtph2ps");
2274 if (CI->getNumArgOperands() >= 3)
2275 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2276 CI->getArgOperand(1));
2277 } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
2278 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2279 CI->getArgOperand(1), CI->getArgOperand(2),
2280 /*Aligned*/false);
2281 } else if (IsX86 && (Name.startswith("avx512.mask.load."))) {
2282 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2283 CI->getArgOperand(1),CI->getArgOperand(2),
2284 /*Aligned*/true);
2285 } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) {
2286 auto *ResultTy = cast<FixedVectorType>(CI->getType());
2287 Type *PtrTy = ResultTy->getElementType();
2288
2289 // Cast the pointer to element type.
2290 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2291 llvm::PointerType::getUnqual(PtrTy));
2292
2293 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2294 ResultTy->getNumElements());
2295
2296 Function *ELd = Intrinsic::getDeclaration(F->getParent(),
2297 Intrinsic::masked_expandload,
2298 ResultTy);
2299 Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
2300 } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) {
2301 auto *ResultTy = cast<VectorType>(CI->getArgOperand(1)->getType());
2302 Type *PtrTy = ResultTy->getElementType();
2303
2304 // Cast the pointer to element type.
2305 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2306 llvm::PointerType::getUnqual(PtrTy));
2307
2308 Value *MaskVec =
2309 getX86MaskVec(Builder, CI->getArgOperand(2),
2310 cast<FixedVectorType>(ResultTy)->getNumElements());
2311
2312 Function *CSt = Intrinsic::getDeclaration(F->getParent(),
2313 Intrinsic::masked_compressstore,
2314 ResultTy);
2315 Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
2316 } else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
2317 Name.startswith("avx512.mask.expand."))) {
2318 auto *ResultTy = cast<FixedVectorType>(CI->getType());
2319
2320 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2321 ResultTy->getNumElements());
2322
2323 bool IsCompress = Name[12] == 'c';
2324 Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
2325 : Intrinsic::x86_avx512_mask_expand;
2326 Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
2327 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
2328 MaskVec });
2329 } else if (IsX86 && Name.startswith("xop.vpcom")) {
2330 bool IsSigned;
2331 if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
2332 Name.endswith("uq"))
2333 IsSigned = false;
2334 else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
2335 Name.endswith("q"))
2336 IsSigned = true;
2337 else
2338 llvm_unreachable("Unknown suffix");
2339
2340 unsigned Imm;
2341 if (CI->getNumArgOperands() == 3) {
2342 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2343 } else {
2344 Name = Name.substr(9); // strip off "xop.vpcom"
2345 if (Name.startswith("lt"))
2346 Imm = 0;
2347 else if (Name.startswith("le"))
2348 Imm = 1;
2349 else if (Name.startswith("gt"))
2350 Imm = 2;
2351 else if (Name.startswith("ge"))
2352 Imm = 3;
2353 else if (Name.startswith("eq"))
2354 Imm = 4;
2355 else if (Name.startswith("ne"))
2356 Imm = 5;
2357 else if (Name.startswith("false"))
2358 Imm = 6;
2359 else if (Name.startswith("true"))
2360 Imm = 7;
2361 else
2362 llvm_unreachable("Unknown condition");
2363 }
2364
2365 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
2366 } else if (IsX86 && Name.startswith("xop.vpcmov")) {
2367 Value *Sel = CI->getArgOperand(2);
2368 Value *NotSel = Builder.CreateNot(Sel);
2369 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
2370 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
2371 Rep = Builder.CreateOr(Sel0, Sel1);
2372 } else if (IsX86 && (Name.startswith("xop.vprot") ||
2373 Name.startswith("avx512.prol") ||
2374 Name.startswith("avx512.mask.prol"))) {
2375 Rep = upgradeX86Rotate(Builder, *CI, false);
2376 } else if (IsX86 && (Name.startswith("avx512.pror") ||
2377 Name.startswith("avx512.mask.pror"))) {
2378 Rep = upgradeX86Rotate(Builder, *CI, true);
2379 } else if (IsX86 && (Name.startswith("avx512.vpshld.") ||
2380 Name.startswith("avx512.mask.vpshld") ||
2381 Name.startswith("avx512.maskz.vpshld"))) {
2382 bool ZeroMask = Name[11] == 'z';
2383 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
2384 } else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||
2385 Name.startswith("avx512.mask.vpshrd") ||
2386 Name.startswith("avx512.maskz.vpshrd"))) {
2387 bool ZeroMask = Name[11] == 'z';
2388 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
2389 } else if (IsX86 && Name == "sse42.crc32.64.8") {
2390 Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
2391 Intrinsic::x86_sse42_crc32_32_8);
2392 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
2393 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
2394 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
2395 } else if (IsX86 && (Name.startswith("avx.vbroadcast.s") ||
2396 Name.startswith("avx512.vbroadcast.s"))) {
2397 // Replace broadcasts with a series of insertelements.
2398 auto *VecTy = cast<FixedVectorType>(CI->getType());
2399 Type *EltTy = VecTy->getElementType();
2400 unsigned EltNum = VecTy->getNumElements();
2401 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
2402 EltTy->getPointerTo());
2403 Value *Load = Builder.CreateLoad(EltTy, Cast);
2404 Type *I32Ty = Type::getInt32Ty(C);
2405 Rep = UndefValue::get(VecTy);
2406 for (unsigned I = 0; I < EltNum; ++I)
2407 Rep = Builder.CreateInsertElement(Rep, Load,
2408 ConstantInt::get(I32Ty, I));
2409 } else if (IsX86 && (Name.startswith("sse41.pmovsx") ||
2410 Name.startswith("sse41.pmovzx") ||
2411 Name.startswith("avx2.pmovsx") ||
2412 Name.startswith("avx2.pmovzx") ||
2413 Name.startswith("avx512.mask.pmovsx") ||
2414 Name.startswith("avx512.mask.pmovzx"))) {
2415 auto *DstTy = cast<FixedVectorType>(CI->getType());
2416 unsigned NumDstElts = DstTy->getNumElements();
2417
2418 // Extract a subvector of the first NumDstElts lanes and sign/zero extend.
2419 SmallVector<int, 8> ShuffleMask(NumDstElts);
2420 for (unsigned i = 0; i != NumDstElts; ++i)
2421 ShuffleMask[i] = i;
2422
2423 Value *SV =
2424 Builder.CreateShuffleVector(CI->getArgOperand(0), ShuffleMask);
2425
2426 bool DoSext = (StringRef::npos != Name.find("pmovsx"));
2427 Rep = DoSext ? Builder.CreateSExt(SV, DstTy)
2428 : Builder.CreateZExt(SV, DstTy);
2429 // If there are 3 arguments, it's a masked intrinsic so we need a select.
2430 if (CI->getNumArgOperands() == 3)
2431 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2432 CI->getArgOperand(1));
2433 } else if (Name == "avx512.mask.pmov.qd.256" ||
2434 Name == "avx512.mask.pmov.qd.512" ||
2435 Name == "avx512.mask.pmov.wb.256" ||
2436 Name == "avx512.mask.pmov.wb.512") {
2437 Type *Ty = CI->getArgOperand(1)->getType();
2438 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
2439 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2440 CI->getArgOperand(1));
2441 } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
2442 Name == "avx2.vbroadcasti128")) {
2443 // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
2444 Type *EltTy = cast<VectorType>(CI->getType())->getElementType();
2445 unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits();
2446 auto *VT = FixedVectorType::get(EltTy, NumSrcElts);
2447 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
2448 PointerType::getUnqual(VT));
2449 Value *Load = Builder.CreateAlignedLoad(VT, Op, Align(1));
2450 if (NumSrcElts == 2)
2451 Rep = Builder.CreateShuffleVector(Load, ArrayRef<int>{0, 1, 0, 1});
2452 else
2453 Rep = Builder.CreateShuffleVector(
2454 Load, ArrayRef<int>{0, 1, 2, 3, 0, 1, 2, 3});
2455 } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") ||
2456 Name.startswith("avx512.mask.shuf.f"))) {
2457 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2458 Type *VT = CI->getType();
2459 unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128;
2460 unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits();
2461 unsigned ControlBitsMask = NumLanes - 1;
2462 unsigned NumControlBits = NumLanes / 2;
2463 SmallVector<int, 8> ShuffleMask(0);
2464
2465 for (unsigned l = 0; l != NumLanes; ++l) {
2466 unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask;
2467 // We actually need the other source.
2468 if (l >= NumLanes / 2)
2469 LaneMask += NumLanes;
2470 for (unsigned i = 0; i != NumElementsInLane; ++i)
2471 ShuffleMask.push_back(LaneMask * NumElementsInLane + i);
2472 }
2473 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2474 CI->getArgOperand(1), ShuffleMask);
2475 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2476 CI->getArgOperand(3));
2477 }else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") ||
2478 Name.startswith("avx512.mask.broadcasti"))) {
2479 unsigned NumSrcElts =
2480 cast<FixedVectorType>(CI->getArgOperand(0)->getType())
2481 ->getNumElements();
2482 unsigned NumDstElts =
2483 cast<FixedVectorType>(CI->getType())->getNumElements();
2484
2485 SmallVector<int, 8> ShuffleMask(NumDstElts);
2486 for (unsigned i = 0; i != NumDstElts; ++i)
2487 ShuffleMask[i] = i % NumSrcElts;
2488
2489 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2490 CI->getArgOperand(0),
2491 ShuffleMask);
2492 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2493 CI->getArgOperand(1));
2494 } else if (IsX86 && (Name.startswith("avx2.pbroadcast") ||
2495 Name.startswith("avx2.vbroadcast") ||
2496 Name.startswith("avx512.pbroadcast") ||
2497 Name.startswith("avx512.mask.broadcast.s"))) {
2498 // Replace vp?broadcasts with a vector shuffle.
2499 Value *Op = CI->getArgOperand(0);
2500 ElementCount EC = cast<VectorType>(CI->getType())->getElementCount();
2501 Type *MaskTy = VectorType::get(Type::getInt32Ty(C), EC);
2502 SmallVector<int, 8> M;
2503 ShuffleVectorInst::getShuffleMask(Constant::getNullValue(MaskTy), M);
2504 Rep = Builder.CreateShuffleVector(Op, M);
2505
2506 if (CI->getNumArgOperands() == 3)
2507 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2508 CI->getArgOperand(1));
2509 } else if (IsX86 && (Name.startswith("sse2.padds.") ||
2510 Name.startswith("avx2.padds.") ||
2511 Name.startswith("avx512.padds.") ||
2512 Name.startswith("avx512.mask.padds."))) {
2513 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::sadd_sat);
2514 } else if (IsX86 && (Name.startswith("sse2.psubs.") ||
2515 Name.startswith("avx2.psubs.") ||
2516 Name.startswith("avx512.psubs.") ||
2517 Name.startswith("avx512.mask.psubs."))) {
2518 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::ssub_sat);
2519 } else if (IsX86 && (Name.startswith("sse2.paddus.") ||
2520 Name.startswith("avx2.paddus.") ||
2521 Name.startswith("avx512.mask.paddus."))) {
2522 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::uadd_sat);
2523 } else if (IsX86 && (Name.startswith("sse2.psubus.") ||
2524 Name.startswith("avx2.psubus.") ||
2525 Name.startswith("avx512.mask.psubus."))) {
2526 Rep = UpgradeX86BinaryIntrinsics(Builder, *CI, Intrinsic::usub_sat);
2527 } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
2528 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2529 CI->getArgOperand(1),
2530 CI->getArgOperand(2),
2531 CI->getArgOperand(3),
2532 CI->getArgOperand(4),
2533 false);
2534 } else if (IsX86 && Name.startswith("avx512.mask.valign.")) {
2535 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2536 CI->getArgOperand(1),
2537 CI->getArgOperand(2),
2538 CI->getArgOperand(3),
2539 CI->getArgOperand(4),
2540 true);
2541 } else if (IsX86 && (Name == "sse2.psll.dq" ||
2542 Name == "avx2.psll.dq")) {
2543 // 128/256-bit shift left specified in bits.
2544 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2545 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
2546 Shift / 8); // Shift is in bits.
2547 } else if (IsX86 && (Name == "sse2.psrl.dq" ||
2548 Name == "avx2.psrl.dq")) {
2549 // 128/256-bit shift right specified in bits.
2550 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2551 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
2552 Shift / 8); // Shift is in bits.
2553 } else if (IsX86 && (Name == "sse2.psll.dq.bs" ||
2554 Name == "avx2.psll.dq.bs" ||
2555 Name == "avx512.psll.dq.512")) {
2556 // 128/256/512-bit shift left specified in bytes.
2557 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2558 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2559 } else if (IsX86 && (Name == "sse2.psrl.dq.bs" ||
2560 Name == "avx2.psrl.dq.bs" ||
2561 Name == "avx512.psrl.dq.512")) {
2562 // 128/256/512-bit shift right specified in bytes.
2563 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2564 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2565 } else if (IsX86 && (Name == "sse41.pblendw" ||
2566 Name.startswith("sse41.blendp") ||
2567 Name.startswith("avx.blend.p") ||
2568 Name == "avx2.pblendw" ||
2569 Name.startswith("avx2.pblendd."))) {
2570 Value *Op0 = CI->getArgOperand(0);
2571 Value *Op1 = CI->getArgOperand(1);
2572 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2573 auto *VecTy = cast<FixedVectorType>(CI->getType());
2574 unsigned NumElts = VecTy->getNumElements();
2575
2576 SmallVector<int, 16> Idxs(NumElts);
2577 for (unsigned i = 0; i != NumElts; ++i)
2578 Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i;
2579
2580 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2581 } else if (IsX86 && (Name.startswith("avx.vinsertf128.") ||
2582 Name == "avx2.vinserti128" ||
2583 Name.startswith("avx512.mask.insert"))) {
2584 Value *Op0 = CI->getArgOperand(0);
2585 Value *Op1 = CI->getArgOperand(1);
2586 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2587 unsigned DstNumElts =
2588 cast<FixedVectorType>(CI->getType())->getNumElements();
2589 unsigned SrcNumElts =
2590 cast<FixedVectorType>(Op1->getType())->getNumElements();
2591 unsigned Scale = DstNumElts / SrcNumElts;
2592
2593 // Mask off the high bits of the immediate value; hardware ignores those.
2594 Imm = Imm % Scale;
2595
2596 // Extend the second operand into a vector the size of the destination.
2597 SmallVector<int, 8> Idxs(DstNumElts);
2598 for (unsigned i = 0; i != SrcNumElts; ++i)
2599 Idxs[i] = i;
2600 for (unsigned i = SrcNumElts; i != DstNumElts; ++i)
2601 Idxs[i] = SrcNumElts;
2602 Rep = Builder.CreateShuffleVector(Op1, Idxs);
2603
2604 // Insert the second operand into the first operand.
2605
2606 // Note that there is no guarantee that instruction lowering will actually
2607 // produce a vinsertf128 instruction for the created shuffles. In
2608 // particular, the 0 immediate case involves no lane changes, so it can
2609 // be handled as a blend.
2610
2611 // Example of shuffle mask for 32-bit elements:
2612 // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
2613 // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 >
2614
2615 // First fill with identify mask.
2616 for (unsigned i = 0; i != DstNumElts; ++i)
2617 Idxs[i] = i;
2618 // Then replace the elements where we need to insert.
2619 for (unsigned i = 0; i != SrcNumElts; ++i)
2620 Idxs[i + Imm * SrcNumElts] = i + DstNumElts;
2621 Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs);
2622
2623 // If the intrinsic has a mask operand, handle that.
2624 if (CI->getNumArgOperands() == 5)
2625 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2626 CI->getArgOperand(3));
2627 } else if (IsX86 && (Name.startswith("avx.vextractf128.") ||
2628 Name == "avx2.vextracti128" ||
2629 Name.startswith("avx512.mask.vextract"))) {
2630 Value *Op0 = CI->getArgOperand(0);
2631 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2632 unsigned DstNumElts =
2633 cast<FixedVectorType>(CI->getType())->getNumElements();
2634 unsigned SrcNumElts =
2635 cast<FixedVectorType>(Op0->getType())->getNumElements();
2636 unsigned Scale = SrcNumElts / DstNumElts;
2637
2638 // Mask off the high bits of the immediate value; hardware ignores those.
2639 Imm = Imm % Scale;
2640
2641 // Get indexes for the subvector of the input vector.
2642 SmallVector<int, 8> Idxs(DstNumElts);
2643 for (unsigned i = 0; i != DstNumElts; ++i) {
2644 Idxs[i] = i + (Imm * DstNumElts);
2645 }
2646 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2647
2648 // If the intrinsic has a mask operand, handle that.
2649 if (CI->getNumArgOperands() == 4)
2650 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2651 CI->getArgOperand(2));
2652 } else if (!IsX86 && Name == "stackprotectorcheck") {
2653 Rep = nullptr;
2654 } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") ||
2655 Name.startswith("avx512.mask.perm.di."))) {
2656 Value *Op0 = CI->getArgOperand(0);
2657 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2658 auto *VecTy = cast<FixedVectorType>(CI->getType());
2659 unsigned NumElts = VecTy->getNumElements();
2660
2661 SmallVector<int, 8> Idxs(NumElts);
2662 for (unsigned i = 0; i != NumElts; ++i)
2663 Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3);
2664
2665 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2666
2667 if (CI->getNumArgOperands() == 4)
2668 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2669 CI->getArgOperand(2));
2670 } else if (IsX86 && (Name.startswith("avx.vperm2f128.") ||
2671 Name == "avx2.vperm2i128")) {
2672 // The immediate permute control byte looks like this:
2673 // [1:0] - select 128 bits from sources for low half of destination
2674 // [2] - ignore
2675 // [3] - zero low half of destination
2676 // [5:4] - select 128 bits from sources for high half of destination
2677 // [6] - ignore
2678 // [7] - zero high half of destination
2679
2680 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2681
2682 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2683 unsigned HalfSize = NumElts / 2;
2684 SmallVector<int, 8> ShuffleMask(NumElts);
2685
2686 // Determine which operand(s) are actually in use for this instruction.
2687 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2688 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2689
2690 // If needed, replace operands based on zero mask.
2691 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
2692 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
2693
2694 // Permute low half of result.
2695 unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0;
2696 for (unsigned i = 0; i < HalfSize; ++i)
2697 ShuffleMask[i] = StartIndex + i;
2698
2699 // Permute high half of result.
2700 StartIndex = (Imm & 0x10) ? HalfSize : 0;
2701 for (unsigned i = 0; i < HalfSize; ++i)
2702 ShuffleMask[i + HalfSize] = NumElts + StartIndex + i;
2703
2704 Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask);
2705
2706 } else if (IsX86 && (Name.startswith("avx.vpermil.") ||
2707 Name == "sse2.pshuf.d" ||
2708 Name.startswith("avx512.mask.vpermil.p") ||
2709 Name.startswith("avx512.mask.pshuf.d."))) {
2710 Value *Op0 = CI->getArgOperand(0);
2711 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2712 auto *VecTy = cast<FixedVectorType>(CI->getType());
2713 unsigned NumElts = VecTy->getNumElements();
2714 // Calculate the size of each index in the immediate.
2715 unsigned IdxSize = 64 / VecTy->getScalarSizeInBits();
2716 unsigned IdxMask = ((1 << IdxSize) - 1);
2717
2718 SmallVector<int, 8> Idxs(NumElts);
2719 // Lookup the bits for this element, wrapping around the immediate every
2720 // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need
2721 // to offset by the first index of each group.
2722 for (unsigned i = 0; i != NumElts; ++i)
2723 Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask);
2724
2725 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2726
2727 if (CI->getNumArgOperands() == 4)
2728 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2729 CI->getArgOperand(2));
2730 } else if (IsX86 && (Name == "sse2.pshufl.w" ||
2731 Name.startswith("avx512.mask.pshufl.w."))) {
2732 Value *Op0 = CI->getArgOperand(0);
2733 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2734 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2735
2736 SmallVector<int, 16> Idxs(NumElts);
2737 for (unsigned l = 0; l != NumElts; l += 8) {
2738 for (unsigned i = 0; i != 4; ++i)
2739 Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l;
2740 for (unsigned i = 4; i != 8; ++i)
2741 Idxs[i + l] = i + l;
2742 }
2743
2744 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2745
2746 if (CI->getNumArgOperands() == 4)
2747 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2748 CI->getArgOperand(2));
2749 } else if (IsX86 && (Name == "sse2.pshufh.w" ||
2750 Name.startswith("avx512.mask.pshufh.w."))) {
2751 Value *Op0 = CI->getArgOperand(0);
2752 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2753 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2754
2755 SmallVector<int, 16> Idxs(NumElts);
2756 for (unsigned l = 0; l != NumElts; l += 8) {
2757 for (unsigned i = 0; i != 4; ++i)
2758 Idxs[i + l] = i + l;
2759 for (unsigned i = 0; i != 4; ++i)
2760 Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l;
2761 }
2762
2763 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2764
2765 if (CI->getNumArgOperands() == 4)
2766 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2767 CI->getArgOperand(2));
2768 } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) {
2769 Value *Op0 = CI->getArgOperand(0);
2770 Value *Op1 = CI->getArgOperand(1);
2771 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2772 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2773
2774 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2775 unsigned HalfLaneElts = NumLaneElts / 2;
2776
2777 SmallVector<int, 16> Idxs(NumElts);
2778 for (unsigned i = 0; i != NumElts; ++i) {
2779 // Base index is the starting element of the lane.
2780 Idxs[i] = i - (i % NumLaneElts);
2781 // If we are half way through the lane switch to the other source.
2782 if ((i % NumLaneElts) >= HalfLaneElts)
2783 Idxs[i] += NumElts;
2784 // Now select the specific element. By adding HalfLaneElts bits from
2785 // the immediate. Wrapping around the immediate every 8-bits.
2786 Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1);
2787 }
2788
2789 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2790
2791 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2792 CI->getArgOperand(3));
2793 } else if (IsX86 && (Name.startswith("avx512.mask.movddup") ||
2794 Name.startswith("avx512.mask.movshdup") ||
2795 Name.startswith("avx512.mask.movsldup"))) {
2796 Value *Op0 = CI->getArgOperand(0);
2797 unsigned NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2798 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2799
2800 unsigned Offset = 0;
2801 if (Name.startswith("avx512.mask.movshdup."))
2802 Offset = 1;
2803
2804 SmallVector<int, 16> Idxs(NumElts);
2805 for (unsigned l = 0; l != NumElts; l += NumLaneElts)
2806 for (unsigned i = 0; i != NumLaneElts; i += 2) {
2807 Idxs[i + l + 0] = i + l + Offset;
2808 Idxs[i + l + 1] = i + l + Offset;
2809 }
2810
2811 Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs);
2812
2813 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2814 CI->getArgOperand(1));
2815 } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") ||
2816 Name.startswith("avx512.mask.unpckl."))) {
2817 Value *Op0 = CI->getArgOperand(0);
2818 Value *Op1 = CI->getArgOperand(1);
2819 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2820 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2821
2822 SmallVector<int, 64> Idxs(NumElts);
2823 for (int l = 0; l != NumElts; l += NumLaneElts)
2824 for (int i = 0; i != NumLaneElts; ++i)
2825 Idxs[i + l] = l + (i / 2) + NumElts * (i % 2);
2826
2827 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2828
2829 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2830 CI->getArgOperand(2));
2831 } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") ||
2832 Name.startswith("avx512.mask.unpckh."))) {
2833 Value *Op0 = CI->getArgOperand(0);
2834 Value *Op1 = CI->getArgOperand(1);
2835 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
2836 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2837
2838 SmallVector<int, 64> Idxs(NumElts);
2839 for (int l = 0; l != NumElts; l += NumLaneElts)
2840 for (int i = 0; i != NumLaneElts; ++i)
2841 Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2);
2842
2843 Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs);
2844
2845 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2846 CI->getArgOperand(2));
2847 } else if (IsX86 && (Name.startswith("avx512.mask.and.") ||
2848 Name.startswith("avx512.mask.pand."))) {
2849 VectorType *FTy = cast<VectorType>(CI->getType());
2850 VectorType *ITy = VectorType::getInteger(FTy);
2851 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2852 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2853 Rep = Builder.CreateBitCast(Rep, FTy);
2854 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2855 CI->getArgOperand(2));
2856 } else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||
2857 Name.startswith("avx512.mask.pandn."))) {
2858 VectorType *FTy = cast<VectorType>(CI->getType());
2859 VectorType *ITy = VectorType::getInteger(FTy);
2860 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
2861 Rep = Builder.CreateAnd(Rep,
2862 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2863 Rep = Builder.CreateBitCast(Rep, FTy);
2864 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2865 CI->getArgOperand(2));
2866 } else if (IsX86 && (Name.startswith("avx512.mask.or.") ||
2867 Name.startswith("avx512.mask.por."))) {
2868 VectorType *FTy = cast<VectorType>(CI->getType());
2869 VectorType *ITy = VectorType::getInteger(FTy);
2870 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2871 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2872 Rep = Builder.CreateBitCast(Rep, FTy);
2873 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2874 CI->getArgOperand(2));
2875 } else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||
2876 Name.startswith("avx512.mask.pxor."))) {
2877 VectorType *FTy = cast<VectorType>(CI->getType());
2878 VectorType *ITy = VectorType::getInteger(FTy);
2879 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2880 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2881 Rep = Builder.CreateBitCast(Rep, FTy);
2882 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2883 CI->getArgOperand(2));
2884 } else if (IsX86 && Name.startswith("avx512.mask.padd.")) {
2885 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2886 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2887 CI->getArgOperand(2));
2888 } else if (IsX86 && Name.startswith("avx512.mask.psub.")) {
2889 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
2890 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2891 CI->getArgOperand(2));
2892 } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) {
2893 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
2894 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2895 CI->getArgOperand(2));
2896 } else if (IsX86 && Name.startswith("avx512.mask.add.p")) {
2897 if (Name.endswith(".512")) {
2898 Intrinsic::ID IID;
2899 if (Name[17] == 's')
2900 IID = Intrinsic::x86_avx512_add_ps_512;
2901 else
2902 IID = Intrinsic::x86_avx512_add_pd_512;
2903
2904 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2905 { CI->getArgOperand(0), CI->getArgOperand(1),
2906 CI->getArgOperand(4) });
2907 } else {
2908 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2909 }
2910 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2911 CI->getArgOperand(2));
2912 } else if (IsX86 && Name.startswith("avx512.mask.div.p")) {
2913 if (Name.endswith(".512")) {
2914 Intrinsic::ID IID;
2915 if (Name[17] == 's')
2916 IID = Intrinsic::x86_avx512_div_ps_512;
2917 else
2918 IID = Intrinsic::x86_avx512_div_pd_512;
2919
2920 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2921 { CI->getArgOperand(0), CI->getArgOperand(1),
2922 CI->getArgOperand(4) });
2923 } else {
2924 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
2925 }
2926 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2927 CI->getArgOperand(2));
2928 } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) {
2929 if (Name.endswith(".512")) {
2930 Intrinsic::ID IID;
2931 if (Name[17] == 's')
2932 IID = Intrinsic::x86_avx512_mul_ps_512;
2933 else
2934 IID = Intrinsic::x86_avx512_mul_pd_512;
2935
2936 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2937 { CI->getArgOperand(0), CI->getArgOperand(1),
2938 CI->getArgOperand(4) });
2939 } else {
2940 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
2941 }
2942 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2943 CI->getArgOperand(2));
2944 } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) {
2945 if (Name.endswith(".512")) {
2946 Intrinsic::ID IID;
2947 if (Name[17] == 's')
2948 IID = Intrinsic::x86_avx512_sub_ps_512;
2949 else
2950 IID = Intrinsic::x86_avx512_sub_pd_512;
2951
2952 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2953 { CI->getArgOperand(0), CI->getArgOperand(1),
2954 CI->getArgOperand(4) });
2955 } else {
2956 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
2957 }
2958 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2959 CI->getArgOperand(2));
2960 } else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
2961 Name.startswith("avx512.mask.min.p")) &&
2962 Name.drop_front(18) == ".512") {
2963 bool IsDouble = Name[17] == 'd';
2964 bool IsMin = Name[13] == 'i';
2965 static const Intrinsic::ID MinMaxTbl[2][2] = {
2966 { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
2967 { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
2968 };
2969 Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
2970
2971 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
2972 { CI->getArgOperand(0), CI->getArgOperand(1),
2973 CI->getArgOperand(4) });
2974 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2975 CI->getArgOperand(2));
2976 } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) {
2977 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(),
2978 Intrinsic::ctlz,
2979 CI->getType()),
2980 { CI->getArgOperand(0), Builder.getInt1(false) });
2981 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2982 CI->getArgOperand(1));
2983 } else if (IsX86 && Name.startswith("avx512.mask.psll")) {
2984 bool IsImmediate = Name[16] == 'i' ||
2985 (Name.size() > 18 && Name[18] == 'i');
2986 bool IsVariable = Name[16] == 'v';
2987 char Size = Name[16] == '.' ? Name[17] :
2988 Name[17] == '.' ? Name[18] :
2989 Name[18] == '.' ? Name[19] :
2990 Name[20];
2991
2992 Intrinsic::ID IID;
2993 if (IsVariable && Name[17] != '.') {
2994 if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di
2995 IID = Intrinsic::x86_avx2_psllv_q;
2996 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di
2997 IID = Intrinsic::x86_avx2_psllv_q_256;
2998 else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si
2999 IID = Intrinsic::x86_avx2_psllv_d;
3000 else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si
3001 IID = Intrinsic::x86_avx2_psllv_d_256;
3002 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi
3003 IID = Intrinsic::x86_avx512_psllv_w_128;
3004 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi
3005 IID = Intrinsic::x86_avx512_psllv_w_256;
3006 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi
3007 IID = Intrinsic::x86_avx512_psllv_w_512;
3008 else
3009 llvm_unreachable("Unexpected size");
3010 } else if (Name.endswith(".128")) {
3011 if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128
3012 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d
3013 : Intrinsic::x86_sse2_psll_d;
3014 else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128
3015 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q
3016 : Intrinsic::x86_sse2_psll_q;
3017 else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128
3018 IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w
3019 : Intrinsic::x86_sse2_psll_w;
3020 else
3021 llvm_unreachable("Unexpected size");
3022 } else if (Name.endswith(".256")) {
3023 if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256
3024 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d
3025 : Intrinsic::x86_avx2_psll_d;
3026 else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256
3027 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q
3028 : Intrinsic::x86_avx2_psll_q;
3029 else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256
3030 IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w
3031 : Intrinsic::x86_avx2_psll_w;
3032 else
3033 llvm_unreachable("Unexpected size");
3034 } else {
3035 if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512
3036 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 :
3037 IsVariable ? Intrinsic::x86_avx512_psllv_d_512 :
3038 Intrinsic::x86_avx512_psll_d_512;
3039 else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512
3040 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 :
3041 IsVariable ? Intrinsic::x86_avx512_psllv_q_512 :
3042 Intrinsic::x86_avx512_psll_q_512;
3043 else if (Size == 'w') // psll.wi.512, pslli.w, psll.w
3044 IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512
3045 : Intrinsic::x86_avx512_psll_w_512;
3046 else
3047 llvm_unreachable("Unexpected size");
3048 }
3049
3050 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3051 } else if (IsX86 && Name.startswith("avx512.mask.psrl")) {
3052 bool IsImmediate = Name[16] == 'i' ||
3053 (Name.size() > 18 && Name[18] == 'i');
3054 bool IsVariable = Name[16] == 'v';
3055 char Size = Name[16] == '.' ? Name[17] :
3056 Name[17] == '.' ? Name[18] :
3057 Name[18] == '.' ? Name[19] :
3058 Name[20];
3059
3060 Intrinsic::ID IID;
3061 if (IsVariable && Name[17] != '.') {
3062 if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di
3063 IID = Intrinsic::x86_avx2_psrlv_q;
3064 else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di
3065 IID = Intrinsic::x86_avx2_psrlv_q_256;
3066 else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si
3067 IID = Intrinsic::x86_avx2_psrlv_d;
3068 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si
3069 IID = Intrinsic::x86_avx2_psrlv_d_256;
3070 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi
3071 IID = Intrinsic::x86_avx512_psrlv_w_128;
3072 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi
3073 IID = Intrinsic::x86_avx512_psrlv_w_256;
3074 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi
3075 IID = Intrinsic::x86_avx512_psrlv_w_512;
3076 else
3077 llvm_unreachable("Unexpected size");
3078 } else if (Name.endswith(".128")) {
3079 if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128
3080 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d
3081 : Intrinsic::x86_sse2_psrl_d;
3082 else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128
3083 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q
3084 : Intrinsic::x86_sse2_psrl_q;
3085 else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128
3086 IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w
3087 : Intrinsic::x86_sse2_psrl_w;
3088 else
3089 llvm_unreachable("Unexpected size");
3090 } else if (Name.endswith(".256")) {
3091 if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256
3092 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d
3093 : Intrinsic::x86_avx2_psrl_d;
3094 else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256
3095 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q
3096 : Intrinsic::x86_avx2_psrl_q;
3097 else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256
3098 IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w
3099 : Intrinsic::x86_avx2_psrl_w;
3100 else
3101 llvm_unreachable("Unexpected size");
3102 } else {
3103 if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512
3104 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 :
3105 IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 :
3106 Intrinsic::x86_avx512_psrl_d_512;
3107 else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512
3108 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 :
3109 IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 :
3110 Intrinsic::x86_avx512_psrl_q_512;
3111 else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w)
3112 IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512
3113 : Intrinsic::x86_avx512_psrl_w_512;
3114 else
3115 llvm_unreachable("Unexpected size");
3116 }
3117
3118 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3119 } else if (IsX86 && Name.startswith("avx512.mask.psra")) {
3120 bool IsImmediate = Name[16] == 'i' ||
3121 (Name.size() > 18 && Name[18] == 'i');
3122 bool IsVariable = Name[16] == 'v';
3123 char Size = Name[16] == '.' ? Name[17] :
3124 Name[17] == '.' ? Name[18] :
3125 Name[18] == '.' ? Name[19] :
3126 Name[20];
3127
3128 Intrinsic::ID IID;
3129 if (IsVariable && Name[17] != '.') {
3130 if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si
3131 IID = Intrinsic::x86_avx2_psrav_d;
3132 else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si
3133 IID = Intrinsic::x86_avx2_psrav_d_256;
3134 else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi
3135 IID = Intrinsic::x86_avx512_psrav_w_128;
3136 else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi
3137 IID = Intrinsic::x86_avx512_psrav_w_256;
3138 else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi
3139 IID = Intrinsic::x86_avx512_psrav_w_512;
3140 else
3141 llvm_unreachable("Unexpected size");
3142 } else if (Name.endswith(".128")) {
3143 if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128
3144 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d
3145 : Intrinsic::x86_sse2_psra_d;
3146 else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128
3147 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 :
3148 IsVariable ? Intrinsic::x86_avx512_psrav_q_128 :
3149 Intrinsic::x86_avx512_psra_q_128;
3150 else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128
3151 IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w
3152 : Intrinsic::x86_sse2_psra_w;
3153 else
3154 llvm_unreachable("Unexpected size");
3155 } else if (Name.endswith(".256")) {
3156 if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256
3157 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d
3158 : Intrinsic::x86_avx2_psra_d;
3159 else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256
3160 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 :
3161 IsVariable ? Intrinsic::x86_avx512_psrav_q_256 :
3162 Intrinsic::x86_avx512_psra_q_256;
3163 else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256
3164 IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w
3165 : Intrinsic::x86_avx2_psra_w;
3166 else
3167 llvm_unreachable("Unexpected size");
3168 } else {
3169 if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512
3170 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 :
3171 IsVariable ? Intrinsic::x86_avx512_psrav_d_512 :
3172 Intrinsic::x86_avx512_psra_d_512;
3173 else if (Size == 'q') // psra.qi.512, psrai.q, psra.q
3174 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 :
3175 IsVariable ? Intrinsic::x86_avx512_psrav_q_512 :
3176 Intrinsic::x86_avx512_psra_q_512;
3177 else if (Size == 'w') // psra.wi.512, psrai.w, psra.w
3178 IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512
3179 : Intrinsic::x86_avx512_psra_w_512;
3180 else
3181 llvm_unreachable("Unexpected size");
3182 }
3183
3184 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3185 } else if (IsX86 && Name.startswith("avx512.mask.move.s")) {
3186 Rep = upgradeMaskedMove(Builder, *CI);
3187 } else if (IsX86 && Name.startswith("avx512.cvtmask2")) {
3188 Rep = UpgradeMaskToInt(Builder, *CI);
3189 } else if (IsX86 && Name.endswith(".movntdqa")) {
3190 Module *M = F->getParent();
3191 MDNode *Node = MDNode::get(
3192 C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1)));
3193
3194 Value *Ptr = CI->getArgOperand(0);
3195
3196 // Convert the type of the pointer to a pointer to the stored type.
3197 Value *BC = Builder.CreateBitCast(
3198 Ptr, PointerType::getUnqual(CI->getType()), "cast");
3199 LoadInst *LI = Builder.CreateAlignedLoad(
3200 CI->getType(), BC,
3201 Align(CI->getType()->getPrimitiveSizeInBits().getFixedSize() / 8));
3202 LI->setMetadata(M->getMDKindID("nontemporal"), Node);
3203 Rep = LI;
3204 } else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
3205 Name.startswith("fma.vfmsub.") ||
3206 Name.startswith("fma.vfnmadd.") ||
3207 Name.startswith("fma.vfnmsub."))) {
3208 bool NegMul = Name[6] == 'n';
3209 bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's';
3210 bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's';
3211
3212 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3213 CI->getArgOperand(2) };
3214
3215 if (IsScalar) {
3216 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3217 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3218 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3219 }
3220
3221 if (NegMul && !IsScalar)
3222 Ops[0] = Builder.CreateFNeg(Ops[0]);
3223 if (NegMul && IsScalar)
3224 Ops[1] = Builder.CreateFNeg(Ops[1]);
3225 if (NegAcc)
3226 Ops[2] = Builder.CreateFNeg(Ops[2]);
3227
3228 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3229 Intrinsic::fma,
3230 Ops[0]->getType()),
3231 Ops);
3232
3233 if (IsScalar)
3234 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
3235 (uint64_t)0);
3236 } else if (IsX86 && Name.startswith("fma4.vfmadd.s")) {
3237 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3238 CI->getArgOperand(2) };
3239
3240 Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0);
3241 Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0);
3242 Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0);
3243
3244 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3245 Intrinsic::fma,
3246 Ops[0]->getType()),
3247 Ops);
3248
3249 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
3250 Rep, (uint64_t)0);
3251 } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") ||
3252 Name.startswith("avx512.maskz.vfmadd.s") ||
3253 Name.startswith("avx512.mask3.vfmadd.s") ||
3254 Name.startswith("avx512.mask3.vfmsub.s") ||
3255 Name.startswith("avx512.mask3.vfnmsub.s"))) {
3256 bool IsMask3 = Name[11] == '3';
3257 bool IsMaskZ = Name[11] == 'z';
3258 // Drop the "avx512.mask." to make it easier.
3259 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3260 bool NegMul = Name[2] == 'n';
3261 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3262
3263 Value *A = CI->getArgOperand(0);
3264 Value *B = CI->getArgOperand(1);
3265 Value *C = CI->getArgOperand(2);
3266
3267 if (NegMul && (IsMask3 || IsMaskZ))
3268 A = Builder.CreateFNeg(A);
3269 if (NegMul && !(IsMask3 || IsMaskZ))
3270 B = Builder.CreateFNeg(B);
3271 if (NegAcc)
3272 C = Builder.CreateFNeg(C);
3273
3274 A = Builder.CreateExtractElement(A, (uint64_t)0);
3275 B = Builder.CreateExtractElement(B, (uint64_t)0);
3276 C = Builder.CreateExtractElement(C, (uint64_t)0);
3277
3278 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3279 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
3280 Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
3281
3282 Intrinsic::ID IID;
3283 if (Name.back() == 'd')
3284 IID = Intrinsic::x86_avx512_vfmadd_f64;
3285 else
3286 IID = Intrinsic::x86_avx512_vfmadd_f32;
3287 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
3288 Rep = Builder.CreateCall(FMA, Ops);
3289 } else {
3290 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3291 Intrinsic::fma,
3292 A->getType());
3293 Rep = Builder.CreateCall(FMA, { A, B, C });
3294 }
3295
3296 Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) :
3297 IsMask3 ? C : A;
3298
3299 // For Mask3 with NegAcc, we need to create a new extractelement that
3300 // avoids the negation above.
3301 if (NegAcc && IsMask3)
3302 PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
3303 (uint64_t)0);
3304
3305 Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
3306 Rep, PassThru);
3307 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
3308 Rep, (uint64_t)0);
3309 } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") ||
3310 Name.startswith("avx512.mask.vfnmadd.p") ||
3311 Name.startswith("avx512.mask.vfnmsub.p") ||
3312 Name.startswith("avx512.mask3.vfmadd.p") ||
3313 Name.startswith("avx512.mask3.vfmsub.p") ||
3314 Name.startswith("avx512.mask3.vfnmsub.p") ||
3315 Name.startswith("avx512.maskz.vfmadd.p"))) {
3316 bool IsMask3 = Name[11] == '3';
3317 bool IsMaskZ = Name[11] == 'z';
3318 // Drop the "avx512.mask." to make it easier.
3319 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3320 bool NegMul = Name[2] == 'n';
3321 bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's';
3322
3323 Value *A = CI->getArgOperand(0);
3324 Value *B = CI->getArgOperand(1);
3325 Value *C = CI->getArgOperand(2);
3326
3327 if (NegMul && (IsMask3 || IsMaskZ))
3328 A = Builder.CreateFNeg(A);
3329 if (NegMul && !(IsMask3 || IsMaskZ))
3330 B = Builder.CreateFNeg(B);
3331 if (NegAcc)
3332 C = Builder.CreateFNeg(C);
3333
3334 if (CI->getNumArgOperands() == 5 &&
3335 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3336 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
3337 Intrinsic::ID IID;
3338 // Check the character before ".512" in string.
3339 if (Name[Name.size()-5] == 's')
3340 IID = Intrinsic::x86_avx512_vfmadd_ps_512;
3341 else
3342 IID = Intrinsic::x86_avx512_vfmadd_pd_512;
3343
3344 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3345 { A, B, C, CI->getArgOperand(4) });
3346 } else {
3347 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3348 Intrinsic::fma,
3349 A->getType());
3350 Rep = Builder.CreateCall(FMA, { A, B, C });
3351 }
3352
3353 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3354 IsMask3 ? CI->getArgOperand(2) :
3355 CI->getArgOperand(0);
3356
3357 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3358 } else if (IsX86 && Name.startswith("fma.vfmsubadd.p")) {
3359 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3360 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3361 Intrinsic::ID IID;
3362 if (VecWidth == 128 && EltWidth == 32)
3363 IID = Intrinsic::x86_fma_vfmaddsub_ps;
3364 else if (VecWidth == 256 && EltWidth == 32)
3365 IID = Intrinsic::x86_fma_vfmaddsub_ps_256;
3366 else if (VecWidth == 128 && EltWidth == 64)
3367 IID = Intrinsic::x86_fma_vfmaddsub_pd;
3368 else if (VecWidth == 256 && EltWidth == 64)
3369 IID = Intrinsic::x86_fma_vfmaddsub_pd_256;
3370 else
3371 llvm_unreachable("Unexpected intrinsic");
3372
3373 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3374 CI->getArgOperand(2) };
3375 Ops[2] = Builder.CreateFNeg(Ops[2]);
3376 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3377 Ops);
3378 } else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") ||
3379 Name.startswith("avx512.mask3.vfmaddsub.p") ||
3380 Name.startswith("avx512.maskz.vfmaddsub.p") ||
3381 Name.startswith("avx512.mask3.vfmsubadd.p"))) {
3382 bool IsMask3 = Name[11] == '3';
3383 bool IsMaskZ = Name[11] == 'z';
3384 // Drop the "avx512.mask." to make it easier.
3385 Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12);
3386 bool IsSubAdd = Name[3] == 's';
3387 if (CI->getNumArgOperands() == 5) {
3388 Intrinsic::ID IID;
3389 // Check the character before ".512" in string.
3390 if (Name[Name.size()-5] == 's')
3391 IID = Intrinsic::x86_avx512_vfmaddsub_ps_512;
3392 else
3393 IID = Intrinsic::x86_avx512_vfmaddsub_pd_512;
3394
3395 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3396 CI->getArgOperand(2), CI->getArgOperand(4) };
3397 if (IsSubAdd)
3398 Ops[2] = Builder.CreateFNeg(Ops[2]);
3399
3400 Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
3401 Ops);
3402 } else {
3403 int NumElts = cast<FixedVectorType>(CI->getType())->getNumElements();
3404
3405 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3406 CI->getArgOperand(2) };
3407
3408 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3409 Ops[0]->getType());
3410 Value *Odd = Builder.CreateCall(FMA, Ops);
3411 Ops[2] = Builder.CreateFNeg(Ops[2]);
3412 Value *Even = Builder.CreateCall(FMA, Ops);
3413
3414 if (IsSubAdd)
3415 std::swap(Even, Odd);
3416
3417 SmallVector<int, 32> Idxs(NumElts);
3418 for (int i = 0; i != NumElts; ++i)
3419 Idxs[i] = i + (i % 2) * NumElts;
3420
3421 Rep = Builder.CreateShuffleVector(Even, Odd, Idxs);
3422 }
3423
3424 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3425 IsMask3 ? CI->getArgOperand(2) :
3426 CI->getArgOperand(0);
3427
3428 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3429 } else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") ||
3430 Name.startswith("avx512.maskz.pternlog."))) {
3431 bool ZeroMask = Name[11] == 'z';
3432 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3433 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3434 Intrinsic::ID IID;
3435 if (VecWidth == 128 && EltWidth == 32)
3436 IID = Intrinsic::x86_avx512_pternlog_d_128;
3437 else if (VecWidth == 256 && EltWidth == 32)
3438 IID = Intrinsic::x86_avx512_pternlog_d_256;
3439 else if (VecWidth == 512 && EltWidth == 32)
3440 IID = Intrinsic::x86_avx512_pternlog_d_512;
3441 else if (VecWidth == 128 && EltWidth == 64)
3442 IID = Intrinsic::x86_avx512_pternlog_q_128;
3443 else if (VecWidth == 256 && EltWidth == 64)
3444 IID = Intrinsic::x86_avx512_pternlog_q_256;
3445 else if (VecWidth == 512 && EltWidth == 64)
3446 IID = Intrinsic::x86_avx512_pternlog_q_512;
3447 else
3448 llvm_unreachable("Unexpected intrinsic");
3449
3450 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3451 CI->getArgOperand(2), CI->getArgOperand(3) };
3452 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3453 Args);
3454 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3455 : CI->getArgOperand(0);
3456 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
3457 } else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") ||
3458 Name.startswith("avx512.maskz.vpmadd52"))) {
3459 bool ZeroMask = Name[11] == 'z';
3460 bool High = Name[20] == 'h' || Name[21] == 'h';
3461 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3462 Intrinsic::ID IID;
3463 if (VecWidth == 128 && !High)
3464 IID = Intrinsic::x86_avx512_vpmadd52l_uq_128;
3465 else if (VecWidth == 256 && !High)
3466 IID = Intrinsic::x86_avx512_vpmadd52l_uq_256;
3467 else if (VecWidth == 512 && !High)
3468 IID = Intrinsic::x86_avx512_vpmadd52l_uq_512;
3469 else if (VecWidth == 128 && High)
3470 IID = Intrinsic::x86_avx512_vpmadd52h_uq_128;
3471 else if (VecWidth == 256 && High)
3472 IID = Intrinsic::x86_avx512_vpmadd52h_uq_256;
3473 else if (VecWidth == 512 && High)
3474 IID = Intrinsic::x86_avx512_vpmadd52h_uq_512;
3475 else
3476 llvm_unreachable("Unexpected intrinsic");
3477
3478 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3479 CI->getArgOperand(2) };
3480 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3481 Args);
3482 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3483 : CI->getArgOperand(0);
3484 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3485 } else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") ||
3486 Name.startswith("avx512.mask.vpermt2var.") ||
3487 Name.startswith("avx512.maskz.vpermt2var."))) {
3488 bool ZeroMask = Name[11] == 'z';
3489 bool IndexForm = Name[17] == 'i';
3490 Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
3491 } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||
3492 Name.startswith("avx512.maskz.vpdpbusd.") ||
3493 Name.startswith("avx512.mask.vpdpbusds.") ||
3494 Name.startswith("avx512.maskz.vpdpbusds."))) {
3495 bool ZeroMask = Name[11] == 'z';
3496 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3497 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3498 Intrinsic::ID IID;
3499 if (VecWidth == 128 && !IsSaturating)
3500 IID = Intrinsic::x86_avx512_vpdpbusd_128;
3501 else if (VecWidth == 256 && !IsSaturating)
3502 IID = Intrinsic::x86_avx512_vpdpbusd_256;
3503 else if (VecWidth == 512 && !IsSaturating)
3504 IID = Intrinsic::x86_avx512_vpdpbusd_512;
3505 else if (VecWidth == 128 && IsSaturating)
3506 IID = Intrinsic::x86_avx512_vpdpbusds_128;
3507 else if (VecWidth == 256 && IsSaturating)
3508 IID = Intrinsic::x86_avx512_vpdpbusds_256;
3509 else if (VecWidth == 512 && IsSaturating)
3510 IID = Intrinsic::x86_avx512_vpdpbusds_512;
3511 else
3512 llvm_unreachable("Unexpected intrinsic");
3513
3514 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3515 CI->getArgOperand(2) };
3516 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3517 Args);
3518 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3519 : CI->getArgOperand(0);
3520 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3521 } else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") ||
3522 Name.startswith("avx512.maskz.vpdpwssd.") ||
3523 Name.startswith("avx512.mask.vpdpwssds.") ||
3524 Name.startswith("avx512.maskz.vpdpwssds."))) {
3525 bool ZeroMask = Name[11] == 'z';
3526 bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's';
3527 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3528 Intrinsic::ID IID;
3529 if (VecWidth == 128 && !IsSaturating)
3530 IID = Intrinsic::x86_avx512_vpdpwssd_128;
3531 else if (VecWidth == 256 && !IsSaturating)
3532 IID = Intrinsic::x86_avx512_vpdpwssd_256;
3533 else if (VecWidth == 512 && !IsSaturating)
3534 IID = Intrinsic::x86_avx512_vpdpwssd_512;
3535 else if (VecWidth == 128 && IsSaturating)
3536 IID = Intrinsic::x86_avx512_vpdpwssds_128;
3537 else if (VecWidth == 256 && IsSaturating)
3538 IID = Intrinsic::x86_avx512_vpdpwssds_256;
3539 else if (VecWidth == 512 && IsSaturating)
3540 IID = Intrinsic::x86_avx512_vpdpwssds_512;
3541 else
3542 llvm_unreachable("Unexpected intrinsic");
3543
3544 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3545 CI->getArgOperand(2) };
3546 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3547 Args);
3548 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3549 : CI->getArgOperand(0);
3550 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3551 } else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" ||
3552 Name == "addcarry.u32" || Name == "addcarry.u64" ||
3553 Name == "subborrow.u32" || Name == "subborrow.u64")) {
3554 Intrinsic::ID IID;
3555 if (Name[0] == 'a' && Name.back() == '2')
3556 IID = Intrinsic::x86_addcarry_32;
3557 else if (Name[0] == 'a' && Name.back() == '4')
3558 IID = Intrinsic::x86_addcarry_64;
3559 else if (Name[0] == 's' && Name.back() == '2')
3560 IID = Intrinsic::x86_subborrow_32;
3561 else if (Name[0] == 's' && Name.back() == '4')
3562 IID = Intrinsic::x86_subborrow_64;
3563 else
3564 llvm_unreachable("Unexpected intrinsic");
3565
3566 // Make a call with 3 operands.
3567 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3568 CI->getArgOperand(2)};
3569 Value *NewCall = Builder.CreateCall(
3570 Intrinsic::getDeclaration(CI->getModule(), IID),
3571 Args);
3572
3573 // Extract the second result and store it.
3574 Value *Data = Builder.CreateExtractValue(NewCall, 1);
3575 // Cast the pointer to the right type.
3576 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
3577 llvm::PointerType::getUnqual(Data->getType()));
3578 Builder.CreateAlignedStore(Data, Ptr, Align(1));
3579 // Replace the original call result with the first result of the new call.
3580 Value *CF = Builder.CreateExtractValue(NewCall, 0);
3581
3582 CI->replaceAllUsesWith(CF);
3583 Rep = nullptr;
3584 } else if (IsX86 && Name.startswith("avx512.mask.") &&
3585 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
3586 // Rep will be updated by the call in the condition.
3587 } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) {
3588 Value *Arg = CI->getArgOperand(0);
3589 Value *Neg = Builder.CreateNeg(Arg, "neg");
3590 Value *Cmp = Builder.CreateICmpSGE(
3591 Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
3592 Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
3593 } else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||
3594 Name.startswith("atomic.load.add.f64.p"))) {
3595 Value *Ptr = CI->getArgOperand(0);
3596 Value *Val = CI->getArgOperand(1);
3597 Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val,
3598 AtomicOrdering::SequentiallyConsistent);
3599 } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
3600 Name == "max.ui" || Name == "max.ull")) {
3601 Value *Arg0 = CI->getArgOperand(0);
3602 Value *Arg1 = CI->getArgOperand(1);
3603 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3604 ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond")
3605 : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond");
3606 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max");
3607 } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" ||
3608 Name == "min.ui" || Name == "min.ull")) {
3609 Value *Arg0 = CI->getArgOperand(0);
3610 Value *Arg1 = CI->getArgOperand(1);
3611 Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull")
3612 ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond")
3613 : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond");
3614 Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min");
3615 } else if (IsNVVM && Name == "clz.ll") {
3616 // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64.
3617 Value *Arg = CI->getArgOperand(0);
3618 Value *Ctlz = Builder.CreateCall(
3619 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
3620 {Arg->getType()}),
3621 {Arg, Builder.getFalse()}, "ctlz");
3622 Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc");
3623 } else if (IsNVVM && Name == "popc.ll") {
3624 // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an
3625 // i64.
3626 Value *Arg = CI->getArgOperand(0);
3627 Value *Popc = Builder.CreateCall(
3628 Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop,
3629 {Arg->getType()}),
3630 Arg, "ctpop");
3631 Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc");
3632 } else if (IsNVVM && Name == "h2f") {
3633 Rep = Builder.CreateCall(Intrinsic::getDeclaration(
3634 F->getParent(), Intrinsic::convert_from_fp16,
3635 {Builder.getFloatTy()}),
3636 CI->getArgOperand(0), "h2f");
3637 } else {
3638 llvm_unreachable("Unknown function for CallInst upgrade.");
3639 }
3640
3641 if (Rep)
3642 CI->replaceAllUsesWith(Rep);
3643 CI->eraseFromParent();
3644 return;
3645 }
3646
3647 const auto &DefaultCase = [&NewFn, &CI]() -> void {
3648 // Handle generic mangling change, but nothing else
3649 assert(
3650 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
3651 "Unknown function for CallInst upgrade and isn't just a name change");
3652 CI->setCalledFunction(NewFn);
3653 };
3654 CallInst *NewCall = nullptr;
3655 switch (NewFn->getIntrinsicID()) {
3656 default: {
3657 DefaultCase();
3658 return;
3659 }
3660 case Intrinsic::arm_neon_vld1:
3661 case Intrinsic::arm_neon_vld2:
3662 case Intrinsic::arm_neon_vld3:
3663 case Intrinsic::arm_neon_vld4:
3664 case Intrinsic::arm_neon_vld2lane:
3665 case Intrinsic::arm_neon_vld3lane:
3666 case Intrinsic::arm_neon_vld4lane:
3667 case Intrinsic::arm_neon_vst1:
3668 case Intrinsic::arm_neon_vst2:
3669 case Intrinsic::arm_neon_vst3:
3670 case Intrinsic::arm_neon_vst4:
3671 case Intrinsic::arm_neon_vst2lane:
3672 case Intrinsic::arm_neon_vst3lane:
3673 case Intrinsic::arm_neon_vst4lane: {
3674 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3675 CI->arg_operands().end());
3676 NewCall = Builder.CreateCall(NewFn, Args);
3677 break;
3678 }
3679
3680 case Intrinsic::arm_neon_bfdot:
3681 case Intrinsic::arm_neon_bfmmla:
3682 case Intrinsic::arm_neon_bfmlalb:
3683 case Intrinsic::arm_neon_bfmlalt:
3684 case Intrinsic::aarch64_neon_bfdot:
3685 case Intrinsic::aarch64_neon_bfmmla:
3686 case Intrinsic::aarch64_neon_bfmlalb:
3687 case Intrinsic::aarch64_neon_bfmlalt: {
3688 SmallVector<Value *, 3> Args;
3689 assert(CI->getNumArgOperands() == 3 &&
3690 "Mismatch between function args and call args");
3691 size_t OperandWidth =
3692 CI->getArgOperand(1)->getType()->getPrimitiveSizeInBits();
3693 assert((OperandWidth == 64 || OperandWidth == 128) &&
3694 "Unexpected operand width");
3695 Type *NewTy = FixedVectorType::get(Type::getBFloatTy(C), OperandWidth / 16);
3696 auto Iter = CI->arg_operands().begin();
3697 Args.push_back(*Iter++);
3698 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
3699 Args.push_back(Builder.CreateBitCast(*Iter++, NewTy));
3700 NewCall = Builder.CreateCall(NewFn, Args);
3701 break;
3702 }
3703
3704 case Intrinsic::bitreverse:
3705 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3706 break;
3707
3708 case Intrinsic::ctlz:
3709 case Intrinsic::cttz:
3710 assert(CI->getNumArgOperands() == 1 &&
3711 "Mismatch between function args and call args");
3712 NewCall =
3713 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
3714 break;
3715
3716 case Intrinsic::objectsize: {
3717 Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
3718 ? Builder.getFalse()
3719 : CI->getArgOperand(2);
3720 Value *Dynamic =
3721 CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
3722 NewCall = Builder.CreateCall(
3723 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
3724 break;
3725 }
3726
3727 case Intrinsic::ctpop:
3728 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3729 break;
3730
3731 case Intrinsic::convert_from_fp16:
3732 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3733 break;
3734
3735 case Intrinsic::dbg_value:
3736 // Upgrade from the old version that had an extra offset argument.
3737 assert(CI->getNumArgOperands() == 4);
3738 // Drop nonzero offsets instead of attempting to upgrade them.
3739 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
3740 if (Offset->isZeroValue()) {
3741 NewCall = Builder.CreateCall(
3742 NewFn,
3743 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
3744 break;
3745 }
3746 CI->eraseFromParent();
3747 return;
3748
3749 case Intrinsic::ptr_annotation:
3750 // Upgrade from versions that lacked the annotation attribute argument.
3751 assert(CI->getNumArgOperands() == 4 &&
3752 "Before LLVM 12.0 this intrinsic took four arguments");
3753 // Create a new call with an added null annotation attribute argument.
3754 NewCall = Builder.CreateCall(
3755 NewFn,
3756 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
3757 CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
3758 NewCall->takeName(CI);
3759 CI->replaceAllUsesWith(NewCall);
3760 CI->eraseFromParent();
3761 return;
3762
3763 case Intrinsic::var_annotation:
3764 // Upgrade from versions that lacked the annotation attribute argument.
3765 assert(CI->getNumArgOperands() == 4 &&
3766 "Before LLVM 12.0 this intrinsic took four arguments");
3767 // Create a new call with an added null annotation attribute argument.
3768 NewCall = Builder.CreateCall(
3769 NewFn,
3770 {CI->getArgOperand(0), CI->getArgOperand(1), CI->getArgOperand(2),
3771 CI->getArgOperand(3), Constant::getNullValue(Builder.getInt8PtrTy())});
3772 CI->eraseFromParent();
3773 return;
3774
3775 case Intrinsic::x86_xop_vfrcz_ss:
3776 case Intrinsic::x86_xop_vfrcz_sd:
3777 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
3778 break;
3779
3780 case Intrinsic::x86_xop_vpermil2pd:
3781 case Intrinsic::x86_xop_vpermil2ps:
3782 case Intrinsic::x86_xop_vpermil2pd_256:
3783 case Intrinsic::x86_xop_vpermil2ps_256: {
3784 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3785 CI->arg_operands().end());
3786 VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType());
3787 VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy);
3788 Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy);
3789 NewCall = Builder.CreateCall(NewFn, Args);
3790 break;
3791 }
3792
3793 case Intrinsic::x86_sse41_ptestc:
3794 case Intrinsic::x86_sse41_ptestz:
3795 case Intrinsic::x86_sse41_ptestnzc: {
3796 // The arguments for these intrinsics used to be v4f32, and changed
3797 // to v2i64. This is purely a nop, since those are bitwise intrinsics.
3798 // So, the only thing required is a bitcast for both arguments.
3799 // First, check the arguments have the old type.
3800 Value *Arg0 = CI->getArgOperand(0);
3801 if (Arg0->getType() != FixedVectorType::get(Type::getFloatTy(C), 4))
3802 return;
3803
3804 // Old intrinsic, add bitcasts
3805 Value *Arg1 = CI->getArgOperand(1);
3806
3807 auto *NewVecTy = FixedVectorType::get(Type::getInt64Ty(C), 2);
3808
3809 Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast");
3810 Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast");
3811
3812 NewCall = Builder.CreateCall(NewFn, {BC0, BC1});
3813 break;
3814 }
3815
3816 case Intrinsic::x86_rdtscp: {
3817 // This used to take 1 arguments. If we have no arguments, it is already
3818 // upgraded.
3819 if (CI->getNumOperands() == 0)
3820 return;
3821
3822 NewCall = Builder.CreateCall(NewFn);
3823 // Extract the second result and store it.
3824 Value *Data = Builder.CreateExtractValue(NewCall, 1);
3825 // Cast the pointer to the right type.
3826 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
3827 llvm::PointerType::getUnqual(Data->getType()));
3828 Builder.CreateAlignedStore(Data, Ptr, Align(1));
3829 // Replace the original call result with the first result of the new call.
3830 Value *TSC = Builder.CreateExtractValue(NewCall, 0);
3831
3832 NewCall->takeName(CI);
3833 CI->replaceAllUsesWith(TSC);
3834 CI->eraseFromParent();
3835 return;
3836 }
3837
3838 case Intrinsic::x86_sse41_insertps:
3839 case Intrinsic::x86_sse41_dppd:
3840 case Intrinsic::x86_sse41_dpps:
3841 case Intrinsic::x86_sse41_mpsadbw:
3842 case Intrinsic::x86_avx_dp_ps_256:
3843 case Intrinsic::x86_avx2_mpsadbw: {
3844 // Need to truncate the last argument from i32 to i8 -- this argument models
3845 // an inherently 8-bit immediate operand to these x86 instructions.
3846 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3847 CI->arg_operands().end());
3848
3849 // Replace the last argument with a trunc.
3850 Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc");
3851 NewCall = Builder.CreateCall(NewFn, Args);
3852 break;
3853 }
3854
3855 case Intrinsic::x86_avx512_mask_cmp_pd_128:
3856 case Intrinsic::x86_avx512_mask_cmp_pd_256:
3857 case Intrinsic::x86_avx512_mask_cmp_pd_512:
3858 case Intrinsic::x86_avx512_mask_cmp_ps_128:
3859 case Intrinsic::x86_avx512_mask_cmp_ps_256:
3860 case Intrinsic::x86_avx512_mask_cmp_ps_512: {
3861 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3862 CI->arg_operands().end());
3863 unsigned NumElts =
3864 cast<FixedVectorType>(Args[0]->getType())->getNumElements();
3865 Args[3] = getX86MaskVec(Builder, Args[3], NumElts);
3866
3867 NewCall = Builder.CreateCall(NewFn, Args);
3868 Value *Res = ApplyX86MaskOn1BitsVec(Builder, NewCall, nullptr);
3869
3870 NewCall->takeName(CI);
3871 CI->replaceAllUsesWith(Res);
3872 CI->eraseFromParent();
3873 return;
3874 }
3875
3876 case Intrinsic::thread_pointer: {
3877 NewCall = Builder.CreateCall(NewFn, {});
3878 break;
3879 }
3880
3881 case Intrinsic::invariant_start:
3882 case Intrinsic::invariant_end:
3883 case Intrinsic::masked_load:
3884 case Intrinsic::masked_store:
3885 case Intrinsic::masked_gather:
3886 case Intrinsic::masked_scatter: {
3887 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3888 CI->arg_operands().end());
3889 NewCall = Builder.CreateCall(NewFn, Args);
3890 break;
3891 }
3892
3893 case Intrinsic::memcpy:
3894 case Intrinsic::memmove:
3895 case Intrinsic::memset: {
3896 // We have to make sure that the call signature is what we're expecting.
3897 // We only want to change the old signatures by removing the alignment arg:
3898 // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1)
3899 // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1)
3900 // @llvm.memset...(i8*, i8, i[32|64], i32, i1)
3901 // -> @llvm.memset...(i8*, i8, i[32|64], i1)
3902 // Note: i8*'s in the above can be any pointer type
3903 if (CI->getNumArgOperands() != 5) {
3904 DefaultCase();
3905 return;
3906 }
3907 // Remove alignment argument (3), and add alignment attributes to the
3908 // dest/src pointers.
3909 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
3910 CI->getArgOperand(2), CI->getArgOperand(4)};
3911 NewCall = Builder.CreateCall(NewFn, Args);
3912 auto *MemCI = cast<MemIntrinsic>(NewCall);
3913 // All mem intrinsics support dest alignment.
3914 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
3915 MemCI->setDestAlignment(Align->getMaybeAlignValue());
3916 // Memcpy/Memmove also support source alignment.
3917 if (auto *MTI = dyn_cast<MemTransferInst>(MemCI))
3918 MTI->setSourceAlignment(Align->getMaybeAlignValue());
3919 break;
3920 }
3921 }
3922 assert(NewCall && "Should have either set this variable or returned through "
3923 "the default case");
3924 NewCall->takeName(CI);
3925 CI->replaceAllUsesWith(NewCall);
3926 CI->eraseFromParent();
3927 }
3928
UpgradeCallsToIntrinsic(Function * F)3929 void llvm::UpgradeCallsToIntrinsic(Function *F) {
3930 assert(F && "Illegal attempt to upgrade a non-existent intrinsic.");
3931
3932 // Check if this function should be upgraded and get the replacement function
3933 // if there is one.
3934 Function *NewFn;
3935 if (UpgradeIntrinsicFunction(F, NewFn)) {
3936 // Replace all users of the old function with the new function or new
3937 // instructions. This is not a range loop because the call is deleted.
3938 for (User *U : make_early_inc_range(F->users()))
3939 if (CallInst *CI = dyn_cast<CallInst>(U))
3940 UpgradeIntrinsicCall(CI, NewFn);
3941
3942 // Remove old function, no longer used, from the module.
3943 F->eraseFromParent();
3944 }
3945 }
3946
UpgradeTBAANode(MDNode & MD)3947 MDNode *llvm::UpgradeTBAANode(MDNode &MD) {
3948 // Check if the tag uses struct-path aware TBAA format.
3949 if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3)
3950 return &MD;
3951
3952 auto &Context = MD.getContext();
3953 if (MD.getNumOperands() == 3) {
3954 Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)};
3955 MDNode *ScalarType = MDNode::get(Context, Elts);
3956 // Create a MDNode <ScalarType, ScalarType, offset 0, const>
3957 Metadata *Elts2[] = {ScalarType, ScalarType,
3958 ConstantAsMetadata::get(
3959 Constant::getNullValue(Type::getInt64Ty(Context))),
3960 MD.getOperand(2)};
3961 return MDNode::get(Context, Elts2);
3962 }
3963 // Create a MDNode <MD, MD, offset 0>
3964 Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue(
3965 Type::getInt64Ty(Context)))};
3966 return MDNode::get(Context, Elts);
3967 }
3968
UpgradeBitCastInst(unsigned Opc,Value * V,Type * DestTy,Instruction * & Temp)3969 Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy,
3970 Instruction *&Temp) {
3971 if (Opc != Instruction::BitCast)
3972 return nullptr;
3973
3974 Temp = nullptr;
3975 Type *SrcTy = V->getType();
3976 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
3977 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
3978 LLVMContext &Context = V->getContext();
3979
3980 // We have no information about target data layout, so we assume that
3981 // the maximum pointer size is 64bit.
3982 Type *MidTy = Type::getInt64Ty(Context);
3983 Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy);
3984
3985 return CastInst::Create(Instruction::IntToPtr, Temp, DestTy);
3986 }
3987
3988 return nullptr;
3989 }
3990
UpgradeBitCastExpr(unsigned Opc,Constant * C,Type * DestTy)3991 Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) {
3992 if (Opc != Instruction::BitCast)
3993 return nullptr;
3994
3995 Type *SrcTy = C->getType();
3996 if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() &&
3997 SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) {
3998 LLVMContext &Context = C->getContext();
3999
4000 // We have no information about target data layout, so we assume that
4001 // the maximum pointer size is 64bit.
4002 Type *MidTy = Type::getInt64Ty(Context);
4003
4004 return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy),
4005 DestTy);
4006 }
4007
4008 return nullptr;
4009 }
4010
4011 /// Check the debug info version number, if it is out-dated, drop the debug
4012 /// info. Return true if module is modified.
UpgradeDebugInfo(Module & M)4013 bool llvm::UpgradeDebugInfo(Module &M) {
4014 unsigned Version = getDebugMetadataVersionFromModule(M);
4015 if (Version == DEBUG_METADATA_VERSION) {
4016 bool BrokenDebugInfo = false;
4017 if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo))
4018 report_fatal_error("Broken module found, compilation aborted!");
4019 if (!BrokenDebugInfo)
4020 // Everything is ok.
4021 return false;
4022 else {
4023 // Diagnose malformed debug info.
4024 DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M);
4025 M.getContext().diagnose(Diag);
4026 }
4027 }
4028 bool Modified = StripDebugInfo(M);
4029 if (Modified && Version != DEBUG_METADATA_VERSION) {
4030 // Diagnose a version mismatch.
4031 DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version);
4032 M.getContext().diagnose(DiagVersion);
4033 }
4034 return Modified;
4035 }
4036
4037 /// This checks for objc retain release marker which should be upgraded. It
4038 /// returns true if module is modified.
UpgradeRetainReleaseMarker(Module & M)4039 static bool UpgradeRetainReleaseMarker(Module &M) {
4040 bool Changed = false;
4041 const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
4042 NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
4043 if (ModRetainReleaseMarker) {
4044 MDNode *Op = ModRetainReleaseMarker->getOperand(0);
4045 if (Op) {
4046 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0));
4047 if (ID) {
4048 SmallVector<StringRef, 4> ValueComp;
4049 ID->getString().split(ValueComp, "#");
4050 if (ValueComp.size() == 2) {
4051 std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
4052 ID = MDString::get(M.getContext(), NewValue);
4053 }
4054 M.addModuleFlag(Module::Error, MarkerKey, ID);
4055 M.eraseNamedMetadata(ModRetainReleaseMarker);
4056 Changed = true;
4057 }
4058 }
4059 }
4060 return Changed;
4061 }
4062
UpgradeARCRuntime(Module & M)4063 void llvm::UpgradeARCRuntime(Module &M) {
4064 // This lambda converts normal function calls to ARC runtime functions to
4065 // intrinsic calls.
4066 auto UpgradeToIntrinsic = [&](const char *OldFunc,
4067 llvm::Intrinsic::ID IntrinsicFunc) {
4068 Function *Fn = M.getFunction(OldFunc);
4069
4070 if (!Fn)
4071 return;
4072
4073 Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc);
4074
4075 for (User *U : make_early_inc_range(Fn->users())) {
4076 CallInst *CI = dyn_cast<CallInst>(U);
4077 if (!CI || CI->getCalledFunction() != Fn)
4078 continue;
4079
4080 IRBuilder<> Builder(CI->getParent(), CI->getIterator());
4081 FunctionType *NewFuncTy = NewFn->getFunctionType();
4082 SmallVector<Value *, 2> Args;
4083
4084 // Don't upgrade the intrinsic if it's not valid to bitcast the return
4085 // value to the return type of the old function.
4086 if (NewFuncTy->getReturnType() != CI->getType() &&
4087 !CastInst::castIsValid(Instruction::BitCast, CI,
4088 NewFuncTy->getReturnType()))
4089 continue;
4090
4091 bool InvalidCast = false;
4092
4093 for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) {
4094 Value *Arg = CI->getArgOperand(I);
4095
4096 // Bitcast argument to the parameter type of the new function if it's
4097 // not a variadic argument.
4098 if (I < NewFuncTy->getNumParams()) {
4099 // Don't upgrade the intrinsic if it's not valid to bitcast the argument
4100 // to the parameter type of the new function.
4101 if (!CastInst::castIsValid(Instruction::BitCast, Arg,
4102 NewFuncTy->getParamType(I))) {
4103 InvalidCast = true;
4104 break;
4105 }
4106 Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I));
4107 }
4108 Args.push_back(Arg);
4109 }
4110
4111 if (InvalidCast)
4112 continue;
4113
4114 // Create a call instruction that calls the new function.
4115 CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args);
4116 NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind());
4117 NewCall->takeName(CI);
4118
4119 // Bitcast the return value back to the type of the old call.
4120 Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType());
4121
4122 if (!CI->use_empty())
4123 CI->replaceAllUsesWith(NewRetVal);
4124 CI->eraseFromParent();
4125 }
4126
4127 if (Fn->use_empty())
4128 Fn->eraseFromParent();
4129 };
4130
4131 // Unconditionally convert a call to "clang.arc.use" to a call to
4132 // "llvm.objc.clang.arc.use".
4133 UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use);
4134
4135 // Upgrade the retain release marker. If there is no need to upgrade
4136 // the marker, that means either the module is already new enough to contain
4137 // new intrinsics or it is not ARC. There is no need to upgrade runtime call.
4138 if (!UpgradeRetainReleaseMarker(M))
4139 return;
4140
4141 std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = {
4142 {"objc_autorelease", llvm::Intrinsic::objc_autorelease},
4143 {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop},
4144 {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush},
4145 {"objc_autoreleaseReturnValue",
4146 llvm::Intrinsic::objc_autoreleaseReturnValue},
4147 {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak},
4148 {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak},
4149 {"objc_initWeak", llvm::Intrinsic::objc_initWeak},
4150 {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak},
4151 {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained},
4152 {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak},
4153 {"objc_release", llvm::Intrinsic::objc_release},
4154 {"objc_retain", llvm::Intrinsic::objc_retain},
4155 {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease},
4156 {"objc_retainAutoreleaseReturnValue",
4157 llvm::Intrinsic::objc_retainAutoreleaseReturnValue},
4158 {"objc_retainAutoreleasedReturnValue",
4159 llvm::Intrinsic::objc_retainAutoreleasedReturnValue},
4160 {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock},
4161 {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong},
4162 {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak},
4163 {"objc_unsafeClaimAutoreleasedReturnValue",
4164 llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue},
4165 {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject},
4166 {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject},
4167 {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer},
4168 {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease},
4169 {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter},
4170 {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit},
4171 {"objc_arc_annotation_topdown_bbstart",
4172 llvm::Intrinsic::objc_arc_annotation_topdown_bbstart},
4173 {"objc_arc_annotation_topdown_bbend",
4174 llvm::Intrinsic::objc_arc_annotation_topdown_bbend},
4175 {"objc_arc_annotation_bottomup_bbstart",
4176 llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart},
4177 {"objc_arc_annotation_bottomup_bbend",
4178 llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}};
4179
4180 for (auto &I : RuntimeFuncs)
4181 UpgradeToIntrinsic(I.first, I.second);
4182 }
4183
UpgradeModuleFlags(Module & M)4184 bool llvm::UpgradeModuleFlags(Module &M) {
4185 NamedMDNode *ModFlags = M.getModuleFlagsMetadata();
4186 if (!ModFlags)
4187 return false;
4188
4189 bool HasObjCFlag = false, HasClassProperties = false, Changed = false;
4190 bool HasSwiftVersionFlag = false;
4191 uint8_t SwiftMajorVersion, SwiftMinorVersion;
4192 uint32_t SwiftABIVersion;
4193 auto Int8Ty = Type::getInt8Ty(M.getContext());
4194 auto Int32Ty = Type::getInt32Ty(M.getContext());
4195
4196 for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) {
4197 MDNode *Op = ModFlags->getOperand(I);
4198 if (Op->getNumOperands() != 3)
4199 continue;
4200 MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1));
4201 if (!ID)
4202 continue;
4203 if (ID->getString() == "Objective-C Image Info Version")
4204 HasObjCFlag = true;
4205 if (ID->getString() == "Objective-C Class Properties")
4206 HasClassProperties = true;
4207 // Upgrade PIC/PIE Module Flags. The module flag behavior for these two
4208 // field was Error and now they are Max.
4209 if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") {
4210 if (auto *Behavior =
4211 mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) {
4212 if (Behavior->getLimitedValue() == Module::Error) {
4213 Type *Int32Ty = Type::getInt32Ty(M.getContext());
4214 Metadata *Ops[3] = {
4215 ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)),
4216 MDString::get(M.getContext(), ID->getString()),
4217 Op->getOperand(2)};
4218 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4219 Changed = true;
4220 }
4221 }
4222 }
4223 // Upgrade Objective-C Image Info Section. Removed the whitespce in the
4224 // section name so that llvm-lto will not complain about mismatching
4225 // module flags that is functionally the same.
4226 if (ID->getString() == "Objective-C Image Info Section") {
4227 if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) {
4228 SmallVector<StringRef, 4> ValueComp;
4229 Value->getString().split(ValueComp, " ");
4230 if (ValueComp.size() != 1) {
4231 std::string NewValue;
4232 for (auto &S : ValueComp)
4233 NewValue += S.str();
4234 Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1),
4235 MDString::get(M.getContext(), NewValue)};
4236 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4237 Changed = true;
4238 }
4239 }
4240 }
4241
4242 // IRUpgrader turns a i32 type "Objective-C Garbage Collection" into i8 value.
4243 // If the higher bits are set, it adds new module flag for swift info.
4244 if (ID->getString() == "Objective-C Garbage Collection") {
4245 auto Md = dyn_cast<ConstantAsMetadata>(Op->getOperand(2));
4246 if (Md) {
4247 assert(Md->getValue() && "Expected non-empty metadata");
4248 auto Type = Md->getValue()->getType();
4249 if (Type == Int8Ty)
4250 continue;
4251 unsigned Val = Md->getValue()->getUniqueInteger().getZExtValue();
4252 if ((Val & 0xff) != Val) {
4253 HasSwiftVersionFlag = true;
4254 SwiftABIVersion = (Val & 0xff00) >> 8;
4255 SwiftMajorVersion = (Val & 0xff000000) >> 24;
4256 SwiftMinorVersion = (Val & 0xff0000) >> 16;
4257 }
4258 Metadata *Ops[3] = {
4259 ConstantAsMetadata::get(ConstantInt::get(Int32Ty,Module::Error)),
4260 Op->getOperand(1),
4261 ConstantAsMetadata::get(ConstantInt::get(Int8Ty,Val & 0xff))};
4262 ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops));
4263 Changed = true;
4264 }
4265 }
4266 }
4267
4268 // "Objective-C Class Properties" is recently added for Objective-C. We
4269 // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module
4270 // flag of value 0, so we can correclty downgrade this flag when trying to
4271 // link an ObjC bitcode without this module flag with an ObjC bitcode with
4272 // this module flag.
4273 if (HasObjCFlag && !HasClassProperties) {
4274 M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties",
4275 (uint32_t)0);
4276 Changed = true;
4277 }
4278
4279 if (HasSwiftVersionFlag) {
4280 M.addModuleFlag(Module::Error, "Swift ABI Version",
4281 SwiftABIVersion);
4282 M.addModuleFlag(Module::Error, "Swift Major Version",
4283 ConstantInt::get(Int8Ty, SwiftMajorVersion));
4284 M.addModuleFlag(Module::Error, "Swift Minor Version",
4285 ConstantInt::get(Int8Ty, SwiftMinorVersion));
4286 Changed = true;
4287 }
4288
4289 return Changed;
4290 }
4291
UpgradeSectionAttributes(Module & M)4292 void llvm::UpgradeSectionAttributes(Module &M) {
4293 auto TrimSpaces = [](StringRef Section) -> std::string {
4294 SmallVector<StringRef, 5> Components;
4295 Section.split(Components, ',');
4296
4297 SmallString<32> Buffer;
4298 raw_svector_ostream OS(Buffer);
4299
4300 for (auto Component : Components)
4301 OS << ',' << Component.trim();
4302
4303 return std::string(OS.str().substr(1));
4304 };
4305
4306 for (auto &GV : M.globals()) {
4307 if (!GV.hasSection())
4308 continue;
4309
4310 StringRef Section = GV.getSection();
4311
4312 if (!Section.startswith("__DATA, __objc_catlist"))
4313 continue;
4314
4315 // __DATA, __objc_catlist, regular, no_dead_strip
4316 // __DATA,__objc_catlist,regular,no_dead_strip
4317 GV.setSection(TrimSpaces(Section));
4318 }
4319 }
4320
4321 namespace {
4322 // Prior to LLVM 10.0, the strictfp attribute could be used on individual
4323 // callsites within a function that did not also have the strictfp attribute.
4324 // Since 10.0, if strict FP semantics are needed within a function, the
4325 // function must have the strictfp attribute and all calls within the function
4326 // must also have the strictfp attribute. This latter restriction is
4327 // necessary to prevent unwanted libcall simplification when a function is
4328 // being cloned (such as for inlining).
4329 //
4330 // The "dangling" strictfp attribute usage was only used to prevent constant
4331 // folding and other libcall simplification. The nobuiltin attribute on the
4332 // callsite has the same effect.
4333 struct StrictFPUpgradeVisitor : public InstVisitor<StrictFPUpgradeVisitor> {
StrictFPUpgradeVisitor__anon9ec957d30411::StrictFPUpgradeVisitor4334 StrictFPUpgradeVisitor() {}
4335
visitCallBase__anon9ec957d30411::StrictFPUpgradeVisitor4336 void visitCallBase(CallBase &Call) {
4337 if (!Call.isStrictFP())
4338 return;
4339 if (isa<ConstrainedFPIntrinsic>(&Call))
4340 return;
4341 // If we get here, the caller doesn't have the strictfp attribute
4342 // but this callsite does. Replace the strictfp attribute with nobuiltin.
4343 Call.removeAttribute(AttributeList::FunctionIndex, Attribute::StrictFP);
4344 Call.addAttribute(AttributeList::FunctionIndex, Attribute::NoBuiltin);
4345 }
4346 };
4347 } // namespace
4348
UpgradeFunctionAttributes(Function & F)4349 void llvm::UpgradeFunctionAttributes(Function &F) {
4350 // If a function definition doesn't have the strictfp attribute,
4351 // convert any callsite strictfp attributes to nobuiltin.
4352 if (!F.isDeclaration() && !F.hasFnAttribute(Attribute::StrictFP)) {
4353 StrictFPUpgradeVisitor SFPV;
4354 SFPV.visit(F);
4355 }
4356
4357 if (F.getCallingConv() == CallingConv::X86_INTR &&
4358 !F.arg_empty() && !F.hasParamAttribute(0, Attribute::ByVal)) {
4359 Type *ByValTy = cast<PointerType>(F.getArg(0)->getType())->getElementType();
4360 Attribute NewAttr = Attribute::getWithByValType(F.getContext(), ByValTy);
4361 F.addParamAttr(0, NewAttr);
4362 }
4363 }
4364
isOldLoopArgument(Metadata * MD)4365 static bool isOldLoopArgument(Metadata *MD) {
4366 auto *T = dyn_cast_or_null<MDTuple>(MD);
4367 if (!T)
4368 return false;
4369 if (T->getNumOperands() < 1)
4370 return false;
4371 auto *S = dyn_cast_or_null<MDString>(T->getOperand(0));
4372 if (!S)
4373 return false;
4374 return S->getString().startswith("llvm.vectorizer.");
4375 }
4376
upgradeLoopTag(LLVMContext & C,StringRef OldTag)4377 static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) {
4378 StringRef OldPrefix = "llvm.vectorizer.";
4379 assert(OldTag.startswith(OldPrefix) && "Expected old prefix");
4380
4381 if (OldTag == "llvm.vectorizer.unroll")
4382 return MDString::get(C, "llvm.loop.interleave.count");
4383
4384 return MDString::get(
4385 C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size()))
4386 .str());
4387 }
4388
upgradeLoopArgument(Metadata * MD)4389 static Metadata *upgradeLoopArgument(Metadata *MD) {
4390 auto *T = dyn_cast_or_null<MDTuple>(MD);
4391 if (!T)
4392 return MD;
4393 if (T->getNumOperands() < 1)
4394 return MD;
4395 auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0));
4396 if (!OldTag)
4397 return MD;
4398 if (!OldTag->getString().startswith("llvm.vectorizer."))
4399 return MD;
4400
4401 // This has an old tag. Upgrade it.
4402 SmallVector<Metadata *, 8> Ops;
4403 Ops.reserve(T->getNumOperands());
4404 Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString()));
4405 for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I)
4406 Ops.push_back(T->getOperand(I));
4407
4408 return MDTuple::get(T->getContext(), Ops);
4409 }
4410
upgradeInstructionLoopAttachment(MDNode & N)4411 MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
4412 auto *T = dyn_cast<MDTuple>(&N);
4413 if (!T)
4414 return &N;
4415
4416 if (none_of(T->operands(), isOldLoopArgument))
4417 return &N;
4418
4419 SmallVector<Metadata *, 8> Ops;
4420 Ops.reserve(T->getNumOperands());
4421 for (Metadata *MD : T->operands())
4422 Ops.push_back(upgradeLoopArgument(MD));
4423
4424 return MDTuple::get(T->getContext(), Ops);
4425 }
4426
UpgradeDataLayoutString(StringRef DL,StringRef TT)4427 std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
4428 Triple T(TT);
4429 // For AMDGPU we uprgrade older DataLayouts to include the default globals
4430 // address space of 1.
4431 if (T.isAMDGPU() && !DL.contains("-G") && !DL.startswith("G")) {
4432 return DL.empty() ? std::string("G1") : (DL + "-G1").str();
4433 }
4434
4435 std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
4436 // If X86, and the datalayout matches the expected format, add pointer size
4437 // address spaces to the datalayout.
4438 if (!T.isX86() || DL.contains(AddrSpaces))
4439 return std::string(DL);
4440
4441 SmallVector<StringRef, 4> Groups;
4442 Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
4443 if (!R.match(DL, &Groups))
4444 return std::string(DL);
4445
4446 return (Groups[1] + AddrSpaces + Groups[3]).str();
4447 }
4448
UpgradeAttributes(AttrBuilder & B)4449 void llvm::UpgradeAttributes(AttrBuilder &B) {
4450 StringRef FramePointer;
4451 if (B.contains("no-frame-pointer-elim")) {
4452 // The value can be "true" or "false".
4453 for (const auto &I : B.td_attrs())
4454 if (I.first == "no-frame-pointer-elim")
4455 FramePointer = I.second == "true" ? "all" : "none";
4456 B.removeAttribute("no-frame-pointer-elim");
4457 }
4458 if (B.contains("no-frame-pointer-elim-non-leaf")) {
4459 // The value is ignored. "no-frame-pointer-elim"="true" takes priority.
4460 if (FramePointer != "all")
4461 FramePointer = "non-leaf";
4462 B.removeAttribute("no-frame-pointer-elim-non-leaf");
4463 }
4464 if (!FramePointer.empty())
4465 B.addAttribute("frame-pointer", FramePointer);
4466
4467 if (B.contains("null-pointer-is-valid")) {
4468 // The value can be "true" or "false".
4469 bool NullPointerIsValid = false;
4470 for (const auto &I : B.td_attrs())
4471 if (I.first == "null-pointer-is-valid")
4472 NullPointerIsValid = I.second == "true";
4473 B.removeAttribute("null-pointer-is-valid");
4474 if (NullPointerIsValid)
4475 B.addAttribute(Attribute::NullPointerIsValid);
4476 }
4477 }
4478