10b57cec5SDimitry Andric //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // This file implements the auto-upgrade helper functions. 100b57cec5SDimitry Andric // This is where deprecated IR intrinsics and other IR features are updated to 110b57cec5SDimitry Andric // current specifications. 120b57cec5SDimitry Andric // 130b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 140b57cec5SDimitry Andric 150b57cec5SDimitry Andric #include "llvm/IR/AutoUpgrade.h" 160b57cec5SDimitry Andric #include "llvm/ADT/StringSwitch.h" 170b57cec5SDimitry Andric #include "llvm/IR/Constants.h" 180b57cec5SDimitry Andric #include "llvm/IR/DIBuilder.h" 190b57cec5SDimitry Andric #include "llvm/IR/DebugInfo.h" 200b57cec5SDimitry Andric #include "llvm/IR/DiagnosticInfo.h" 210b57cec5SDimitry Andric #include "llvm/IR/Function.h" 220b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h" 230b57cec5SDimitry Andric #include "llvm/IR/Instruction.h" 240b57cec5SDimitry Andric #include "llvm/IR/IntrinsicInst.h" 250b57cec5SDimitry Andric #include "llvm/IR/LLVMContext.h" 260b57cec5SDimitry Andric #include "llvm/IR/Module.h" 270b57cec5SDimitry Andric #include "llvm/IR/Verifier.h" 280b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h" 290b57cec5SDimitry Andric #include "llvm/Support/Regex.h" 300b57cec5SDimitry Andric #include <cstring> 310b57cec5SDimitry Andric using namespace llvm; 320b57cec5SDimitry Andric 330b57cec5SDimitry Andric static void rename(GlobalValue *GV) { GV->setName(GV->getName() + ".old"); } 340b57cec5SDimitry Andric 350b57cec5SDimitry Andric // Upgrade the declarations of the SSE4.1 ptest intrinsics whose arguments have 360b57cec5SDimitry Andric // changed their type from v4f32 to v2i64. 370b57cec5SDimitry Andric static bool UpgradePTESTIntrinsic(Function* F, Intrinsic::ID IID, 380b57cec5SDimitry Andric Function *&NewFn) { 390b57cec5SDimitry Andric // Check whether this is an old version of the function, which received 400b57cec5SDimitry Andric // v4f32 arguments. 410b57cec5SDimitry Andric Type *Arg0Type = F->getFunctionType()->getParamType(0); 420b57cec5SDimitry Andric if (Arg0Type != VectorType::get(Type::getFloatTy(F->getContext()), 4)) 430b57cec5SDimitry Andric return false; 440b57cec5SDimitry Andric 450b57cec5SDimitry Andric // Yes, it's old, replace it with new version. 460b57cec5SDimitry Andric rename(F); 470b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 480b57cec5SDimitry Andric return true; 490b57cec5SDimitry Andric } 500b57cec5SDimitry Andric 510b57cec5SDimitry Andric // Upgrade the declarations of intrinsic functions whose 8-bit immediate mask 520b57cec5SDimitry Andric // arguments have changed their type from i32 to i8. 530b57cec5SDimitry Andric static bool UpgradeX86IntrinsicsWith8BitMask(Function *F, Intrinsic::ID IID, 540b57cec5SDimitry Andric Function *&NewFn) { 550b57cec5SDimitry Andric // Check that the last argument is an i32. 560b57cec5SDimitry Andric Type *LastArgType = F->getFunctionType()->getParamType( 570b57cec5SDimitry Andric F->getFunctionType()->getNumParams() - 1); 580b57cec5SDimitry Andric if (!LastArgType->isIntegerTy(32)) 590b57cec5SDimitry Andric return false; 600b57cec5SDimitry Andric 610b57cec5SDimitry Andric // Move this function aside and map down. 620b57cec5SDimitry Andric rename(F); 630b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), IID); 640b57cec5SDimitry Andric return true; 650b57cec5SDimitry Andric } 660b57cec5SDimitry Andric 670b57cec5SDimitry Andric static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) { 680b57cec5SDimitry Andric // All of the intrinsics matches below should be marked with which llvm 690b57cec5SDimitry Andric // version started autoupgrading them. At some point in the future we would 700b57cec5SDimitry Andric // like to use this information to remove upgrade code for some older 710b57cec5SDimitry Andric // intrinsics. It is currently undecided how we will determine that future 720b57cec5SDimitry Andric // point. 730b57cec5SDimitry Andric if (Name == "addcarryx.u32" || // Added in 8.0 740b57cec5SDimitry Andric Name == "addcarryx.u64" || // Added in 8.0 750b57cec5SDimitry Andric Name == "addcarry.u32" || // Added in 8.0 760b57cec5SDimitry Andric Name == "addcarry.u64" || // Added in 8.0 770b57cec5SDimitry Andric Name == "subborrow.u32" || // Added in 8.0 780b57cec5SDimitry Andric Name == "subborrow.u64" || // Added in 8.0 790b57cec5SDimitry Andric Name.startswith("sse2.padds.") || // Added in 8.0 800b57cec5SDimitry Andric Name.startswith("sse2.psubs.") || // Added in 8.0 810b57cec5SDimitry Andric Name.startswith("sse2.paddus.") || // Added in 8.0 820b57cec5SDimitry Andric Name.startswith("sse2.psubus.") || // Added in 8.0 830b57cec5SDimitry Andric Name.startswith("avx2.padds.") || // Added in 8.0 840b57cec5SDimitry Andric Name.startswith("avx2.psubs.") || // Added in 8.0 850b57cec5SDimitry Andric Name.startswith("avx2.paddus.") || // Added in 8.0 860b57cec5SDimitry Andric Name.startswith("avx2.psubus.") || // Added in 8.0 870b57cec5SDimitry Andric Name.startswith("avx512.padds.") || // Added in 8.0 880b57cec5SDimitry Andric Name.startswith("avx512.psubs.") || // Added in 8.0 890b57cec5SDimitry Andric Name.startswith("avx512.mask.padds.") || // Added in 8.0 900b57cec5SDimitry Andric Name.startswith("avx512.mask.psubs.") || // Added in 8.0 910b57cec5SDimitry Andric Name.startswith("avx512.mask.paddus.") || // Added in 8.0 920b57cec5SDimitry Andric Name.startswith("avx512.mask.psubus.") || // Added in 8.0 930b57cec5SDimitry Andric Name=="ssse3.pabs.b.128" || // Added in 6.0 940b57cec5SDimitry Andric Name=="ssse3.pabs.w.128" || // Added in 6.0 950b57cec5SDimitry Andric Name=="ssse3.pabs.d.128" || // Added in 6.0 960b57cec5SDimitry Andric Name.startswith("fma4.vfmadd.s") || // Added in 7.0 970b57cec5SDimitry Andric Name.startswith("fma.vfmadd.") || // Added in 7.0 980b57cec5SDimitry Andric Name.startswith("fma.vfmsub.") || // Added in 7.0 990b57cec5SDimitry Andric Name.startswith("fma.vfmaddsub.") || // Added in 7.0 1000b57cec5SDimitry Andric Name.startswith("fma.vfmsubadd.") || // Added in 7.0 1010b57cec5SDimitry Andric Name.startswith("fma.vfnmadd.") || // Added in 7.0 1020b57cec5SDimitry Andric Name.startswith("fma.vfnmsub.") || // Added in 7.0 1030b57cec5SDimitry Andric Name.startswith("avx512.mask.vfmadd.") || // Added in 7.0 1040b57cec5SDimitry Andric Name.startswith("avx512.mask.vfnmadd.") || // Added in 7.0 1050b57cec5SDimitry Andric Name.startswith("avx512.mask.vfnmsub.") || // Added in 7.0 1060b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmadd.") || // Added in 7.0 1070b57cec5SDimitry Andric Name.startswith("avx512.maskz.vfmadd.") || // Added in 7.0 1080b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmsub.") || // Added in 7.0 1090b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfnmsub.") || // Added in 7.0 1100b57cec5SDimitry Andric Name.startswith("avx512.mask.vfmaddsub.") || // Added in 7.0 1110b57cec5SDimitry Andric Name.startswith("avx512.maskz.vfmaddsub.") || // Added in 7.0 1120b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmaddsub.") || // Added in 7.0 1130b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmsubadd.") || // Added in 7.0 1140b57cec5SDimitry Andric Name.startswith("avx512.mask.shuf.i") || // Added in 6.0 1150b57cec5SDimitry Andric Name.startswith("avx512.mask.shuf.f") || // Added in 6.0 1160b57cec5SDimitry Andric Name.startswith("avx512.kunpck") || //added in 6.0 1170b57cec5SDimitry Andric Name.startswith("avx2.pabs.") || // Added in 6.0 1180b57cec5SDimitry Andric Name.startswith("avx512.mask.pabs.") || // Added in 6.0 1190b57cec5SDimitry Andric Name.startswith("avx512.broadcastm") || // Added in 6.0 1200b57cec5SDimitry Andric Name == "sse.sqrt.ss" || // Added in 7.0 1210b57cec5SDimitry Andric Name == "sse2.sqrt.sd" || // Added in 7.0 1220b57cec5SDimitry Andric Name.startswith("avx512.mask.sqrt.p") || // Added in 7.0 1230b57cec5SDimitry Andric Name.startswith("avx.sqrt.p") || // Added in 7.0 1240b57cec5SDimitry Andric Name.startswith("sse2.sqrt.p") || // Added in 7.0 1250b57cec5SDimitry Andric Name.startswith("sse.sqrt.p") || // Added in 7.0 1260b57cec5SDimitry Andric Name.startswith("avx512.mask.pbroadcast") || // Added in 6.0 1270b57cec5SDimitry Andric Name.startswith("sse2.pcmpeq.") || // Added in 3.1 1280b57cec5SDimitry Andric Name.startswith("sse2.pcmpgt.") || // Added in 3.1 1290b57cec5SDimitry Andric Name.startswith("avx2.pcmpeq.") || // Added in 3.1 1300b57cec5SDimitry Andric Name.startswith("avx2.pcmpgt.") || // Added in 3.1 1310b57cec5SDimitry Andric Name.startswith("avx512.mask.pcmpeq.") || // Added in 3.9 1320b57cec5SDimitry Andric Name.startswith("avx512.mask.pcmpgt.") || // Added in 3.9 1330b57cec5SDimitry Andric Name.startswith("avx.vperm2f128.") || // Added in 6.0 1340b57cec5SDimitry Andric Name == "avx2.vperm2i128" || // Added in 6.0 1350b57cec5SDimitry Andric Name == "sse.add.ss" || // Added in 4.0 1360b57cec5SDimitry Andric Name == "sse2.add.sd" || // Added in 4.0 1370b57cec5SDimitry Andric Name == "sse.sub.ss" || // Added in 4.0 1380b57cec5SDimitry Andric Name == "sse2.sub.sd" || // Added in 4.0 1390b57cec5SDimitry Andric Name == "sse.mul.ss" || // Added in 4.0 1400b57cec5SDimitry Andric Name == "sse2.mul.sd" || // Added in 4.0 1410b57cec5SDimitry Andric Name == "sse.div.ss" || // Added in 4.0 1420b57cec5SDimitry Andric Name == "sse2.div.sd" || // Added in 4.0 1430b57cec5SDimitry Andric Name == "sse41.pmaxsb" || // Added in 3.9 1440b57cec5SDimitry Andric Name == "sse2.pmaxs.w" || // Added in 3.9 1450b57cec5SDimitry Andric Name == "sse41.pmaxsd" || // Added in 3.9 1460b57cec5SDimitry Andric Name == "sse2.pmaxu.b" || // Added in 3.9 1470b57cec5SDimitry Andric Name == "sse41.pmaxuw" || // Added in 3.9 1480b57cec5SDimitry Andric Name == "sse41.pmaxud" || // Added in 3.9 1490b57cec5SDimitry Andric Name == "sse41.pminsb" || // Added in 3.9 1500b57cec5SDimitry Andric Name == "sse2.pmins.w" || // Added in 3.9 1510b57cec5SDimitry Andric Name == "sse41.pminsd" || // Added in 3.9 1520b57cec5SDimitry Andric Name == "sse2.pminu.b" || // Added in 3.9 1530b57cec5SDimitry Andric Name == "sse41.pminuw" || // Added in 3.9 1540b57cec5SDimitry Andric Name == "sse41.pminud" || // Added in 3.9 1550b57cec5SDimitry Andric Name == "avx512.kand.w" || // Added in 7.0 1560b57cec5SDimitry Andric Name == "avx512.kandn.w" || // Added in 7.0 1570b57cec5SDimitry Andric Name == "avx512.knot.w" || // Added in 7.0 1580b57cec5SDimitry Andric Name == "avx512.kor.w" || // Added in 7.0 1590b57cec5SDimitry Andric Name == "avx512.kxor.w" || // Added in 7.0 1600b57cec5SDimitry Andric Name == "avx512.kxnor.w" || // Added in 7.0 1610b57cec5SDimitry Andric Name == "avx512.kortestc.w" || // Added in 7.0 1620b57cec5SDimitry Andric Name == "avx512.kortestz.w" || // Added in 7.0 1630b57cec5SDimitry Andric Name.startswith("avx512.mask.pshuf.b.") || // Added in 4.0 1640b57cec5SDimitry Andric Name.startswith("avx2.pmax") || // Added in 3.9 1650b57cec5SDimitry Andric Name.startswith("avx2.pmin") || // Added in 3.9 1660b57cec5SDimitry Andric Name.startswith("avx512.mask.pmax") || // Added in 4.0 1670b57cec5SDimitry Andric Name.startswith("avx512.mask.pmin") || // Added in 4.0 1680b57cec5SDimitry Andric Name.startswith("avx2.vbroadcast") || // Added in 3.8 1690b57cec5SDimitry Andric Name.startswith("avx2.pbroadcast") || // Added in 3.8 1700b57cec5SDimitry Andric Name.startswith("avx.vpermil.") || // Added in 3.1 1710b57cec5SDimitry Andric Name.startswith("sse2.pshuf") || // Added in 3.9 1720b57cec5SDimitry Andric Name.startswith("avx512.pbroadcast") || // Added in 3.9 1730b57cec5SDimitry Andric Name.startswith("avx512.mask.broadcast.s") || // Added in 3.9 1740b57cec5SDimitry Andric Name.startswith("avx512.mask.movddup") || // Added in 3.9 1750b57cec5SDimitry Andric Name.startswith("avx512.mask.movshdup") || // Added in 3.9 1760b57cec5SDimitry Andric Name.startswith("avx512.mask.movsldup") || // Added in 3.9 1770b57cec5SDimitry Andric Name.startswith("avx512.mask.pshuf.d.") || // Added in 3.9 1780b57cec5SDimitry Andric Name.startswith("avx512.mask.pshufl.w.") || // Added in 3.9 1790b57cec5SDimitry Andric Name.startswith("avx512.mask.pshufh.w.") || // Added in 3.9 1800b57cec5SDimitry Andric Name.startswith("avx512.mask.shuf.p") || // Added in 4.0 1810b57cec5SDimitry Andric Name.startswith("avx512.mask.vpermil.p") || // Added in 3.9 1820b57cec5SDimitry Andric Name.startswith("avx512.mask.perm.df.") || // Added in 3.9 1830b57cec5SDimitry Andric Name.startswith("avx512.mask.perm.di.") || // Added in 3.9 1840b57cec5SDimitry Andric Name.startswith("avx512.mask.punpckl") || // Added in 3.9 1850b57cec5SDimitry Andric Name.startswith("avx512.mask.punpckh") || // Added in 3.9 1860b57cec5SDimitry Andric Name.startswith("avx512.mask.unpckl.") || // Added in 3.9 1870b57cec5SDimitry Andric Name.startswith("avx512.mask.unpckh.") || // Added in 3.9 1880b57cec5SDimitry Andric Name.startswith("avx512.mask.pand.") || // Added in 3.9 1890b57cec5SDimitry Andric Name.startswith("avx512.mask.pandn.") || // Added in 3.9 1900b57cec5SDimitry Andric Name.startswith("avx512.mask.por.") || // Added in 3.9 1910b57cec5SDimitry Andric Name.startswith("avx512.mask.pxor.") || // Added in 3.9 1920b57cec5SDimitry Andric Name.startswith("avx512.mask.and.") || // Added in 3.9 1930b57cec5SDimitry Andric Name.startswith("avx512.mask.andn.") || // Added in 3.9 1940b57cec5SDimitry Andric Name.startswith("avx512.mask.or.") || // Added in 3.9 1950b57cec5SDimitry Andric Name.startswith("avx512.mask.xor.") || // Added in 3.9 1960b57cec5SDimitry Andric Name.startswith("avx512.mask.padd.") || // Added in 4.0 1970b57cec5SDimitry Andric Name.startswith("avx512.mask.psub.") || // Added in 4.0 1980b57cec5SDimitry Andric Name.startswith("avx512.mask.pmull.") || // Added in 4.0 1990b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0 2000b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0 2010b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0 2020b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0 2030b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0 2040b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0 2050b57cec5SDimitry Andric Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0 2060b57cec5SDimitry Andric Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0 2070b57cec5SDimitry Andric Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0 2080b57cec5SDimitry Andric Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0 2090b57cec5SDimitry Andric Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0 2100b57cec5SDimitry Andric Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0 2110b57cec5SDimitry Andric Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0 2120b57cec5SDimitry Andric Name == "avx512.mask.cvttps2dq.128" || // Added in 7.0 2130b57cec5SDimitry Andric Name == "avx512.mask.cvttps2dq.256" || // Added in 7.0 2140b57cec5SDimitry Andric Name == "avx512.mask.cvtps2pd.128" || // Added in 7.0 2150b57cec5SDimitry Andric Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0 2160b57cec5SDimitry Andric Name == "avx512.cvtusi2sd" || // Added in 7.0 2170b57cec5SDimitry Andric Name.startswith("avx512.mask.permvar.") || // Added in 7.0 2180b57cec5SDimitry Andric Name == "sse2.pmulu.dq" || // Added in 7.0 2190b57cec5SDimitry Andric Name == "sse41.pmuldq" || // Added in 7.0 2200b57cec5SDimitry Andric Name == "avx2.pmulu.dq" || // Added in 7.0 2210b57cec5SDimitry Andric Name == "avx2.pmul.dq" || // Added in 7.0 2220b57cec5SDimitry Andric Name == "avx512.pmulu.dq.512" || // Added in 7.0 2230b57cec5SDimitry Andric Name == "avx512.pmul.dq.512" || // Added in 7.0 2240b57cec5SDimitry Andric Name.startswith("avx512.mask.pmul.dq.") || // Added in 4.0 2250b57cec5SDimitry Andric Name.startswith("avx512.mask.pmulu.dq.") || // Added in 4.0 2260b57cec5SDimitry Andric Name.startswith("avx512.mask.pmul.hr.sw.") || // Added in 7.0 2270b57cec5SDimitry Andric Name.startswith("avx512.mask.pmulh.w.") || // Added in 7.0 2280b57cec5SDimitry Andric Name.startswith("avx512.mask.pmulhu.w.") || // Added in 7.0 2290b57cec5SDimitry Andric Name.startswith("avx512.mask.pmaddw.d.") || // Added in 7.0 2300b57cec5SDimitry Andric Name.startswith("avx512.mask.pmaddubs.w.") || // Added in 7.0 2310b57cec5SDimitry Andric Name.startswith("avx512.mask.packsswb.") || // Added in 5.0 2320b57cec5SDimitry Andric Name.startswith("avx512.mask.packssdw.") || // Added in 5.0 2330b57cec5SDimitry Andric Name.startswith("avx512.mask.packuswb.") || // Added in 5.0 2340b57cec5SDimitry Andric Name.startswith("avx512.mask.packusdw.") || // Added in 5.0 2350b57cec5SDimitry Andric Name.startswith("avx512.mask.cmp.b") || // Added in 5.0 2360b57cec5SDimitry Andric Name.startswith("avx512.mask.cmp.d") || // Added in 5.0 2370b57cec5SDimitry Andric Name.startswith("avx512.mask.cmp.q") || // Added in 5.0 2380b57cec5SDimitry Andric Name.startswith("avx512.mask.cmp.w") || // Added in 5.0 2390b57cec5SDimitry Andric Name.startswith("avx512.mask.cmp.p") || // Added in 7.0 2400b57cec5SDimitry Andric Name.startswith("avx512.mask.ucmp.") || // Added in 5.0 2410b57cec5SDimitry Andric Name.startswith("avx512.cvtb2mask.") || // Added in 7.0 2420b57cec5SDimitry Andric Name.startswith("avx512.cvtw2mask.") || // Added in 7.0 2430b57cec5SDimitry Andric Name.startswith("avx512.cvtd2mask.") || // Added in 7.0 2440b57cec5SDimitry Andric Name.startswith("avx512.cvtq2mask.") || // Added in 7.0 2450b57cec5SDimitry Andric Name.startswith("avx512.mask.vpermilvar.") || // Added in 4.0 2460b57cec5SDimitry Andric Name.startswith("avx512.mask.psll.d") || // Added in 4.0 2470b57cec5SDimitry Andric Name.startswith("avx512.mask.psll.q") || // Added in 4.0 2480b57cec5SDimitry Andric Name.startswith("avx512.mask.psll.w") || // Added in 4.0 2490b57cec5SDimitry Andric Name.startswith("avx512.mask.psra.d") || // Added in 4.0 2500b57cec5SDimitry Andric Name.startswith("avx512.mask.psra.q") || // Added in 4.0 2510b57cec5SDimitry Andric Name.startswith("avx512.mask.psra.w") || // Added in 4.0 2520b57cec5SDimitry Andric Name.startswith("avx512.mask.psrl.d") || // Added in 4.0 2530b57cec5SDimitry Andric Name.startswith("avx512.mask.psrl.q") || // Added in 4.0 2540b57cec5SDimitry Andric Name.startswith("avx512.mask.psrl.w") || // Added in 4.0 2550b57cec5SDimitry Andric Name.startswith("avx512.mask.pslli") || // Added in 4.0 2560b57cec5SDimitry Andric Name.startswith("avx512.mask.psrai") || // Added in 4.0 2570b57cec5SDimitry Andric Name.startswith("avx512.mask.psrli") || // Added in 4.0 2580b57cec5SDimitry Andric Name.startswith("avx512.mask.psllv") || // Added in 4.0 2590b57cec5SDimitry Andric Name.startswith("avx512.mask.psrav") || // Added in 4.0 2600b57cec5SDimitry Andric Name.startswith("avx512.mask.psrlv") || // Added in 4.0 2610b57cec5SDimitry Andric Name.startswith("sse41.pmovsx") || // Added in 3.8 2620b57cec5SDimitry Andric Name.startswith("sse41.pmovzx") || // Added in 3.9 2630b57cec5SDimitry Andric Name.startswith("avx2.pmovsx") || // Added in 3.9 2640b57cec5SDimitry Andric Name.startswith("avx2.pmovzx") || // Added in 3.9 2650b57cec5SDimitry Andric Name.startswith("avx512.mask.pmovsx") || // Added in 4.0 2660b57cec5SDimitry Andric Name.startswith("avx512.mask.pmovzx") || // Added in 4.0 2670b57cec5SDimitry Andric Name.startswith("avx512.mask.lzcnt.") || // Added in 5.0 2680b57cec5SDimitry Andric Name.startswith("avx512.mask.pternlog.") || // Added in 7.0 2690b57cec5SDimitry Andric Name.startswith("avx512.maskz.pternlog.") || // Added in 7.0 2700b57cec5SDimitry Andric Name.startswith("avx512.mask.vpmadd52") || // Added in 7.0 2710b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpmadd52") || // Added in 7.0 2720b57cec5SDimitry Andric Name.startswith("avx512.mask.vpermi2var.") || // Added in 7.0 2730b57cec5SDimitry Andric Name.startswith("avx512.mask.vpermt2var.") || // Added in 7.0 2740b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpermt2var.") || // Added in 7.0 2750b57cec5SDimitry Andric Name.startswith("avx512.mask.vpdpbusd.") || // Added in 7.0 2760b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpdpbusd.") || // Added in 7.0 2770b57cec5SDimitry Andric Name.startswith("avx512.mask.vpdpbusds.") || // Added in 7.0 2780b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpdpbusds.") || // Added in 7.0 2790b57cec5SDimitry Andric Name.startswith("avx512.mask.vpdpwssd.") || // Added in 7.0 2800b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpdpwssd.") || // Added in 7.0 2810b57cec5SDimitry Andric Name.startswith("avx512.mask.vpdpwssds.") || // Added in 7.0 2820b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpdpwssds.") || // Added in 7.0 2830b57cec5SDimitry Andric Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0 2840b57cec5SDimitry Andric Name.startswith("avx512.mask.vpshld.") || // Added in 7.0 2850b57cec5SDimitry Andric Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0 2860b57cec5SDimitry Andric Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0 2870b57cec5SDimitry Andric Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0 2880b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0 2890b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0 2900b57cec5SDimitry Andric Name.startswith("avx512.vpshld.") || // Added in 8.0 2910b57cec5SDimitry Andric Name.startswith("avx512.vpshrd.") || // Added in 8.0 2920b57cec5SDimitry Andric Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0 2930b57cec5SDimitry Andric Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0 2940b57cec5SDimitry Andric Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0 2950b57cec5SDimitry Andric Name.startswith("avx512.mask.div.p") || // Added in 7.0. 128/256 in 4.0 2960b57cec5SDimitry Andric Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0 2970b57cec5SDimitry Andric Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0 2980b57cec5SDimitry Andric Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0 2990b57cec5SDimitry Andric Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0 3000b57cec5SDimitry Andric Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0 3010b57cec5SDimitry Andric Name.startswith("avx512.mask.conflict.") || // Added in 9.0 3020b57cec5SDimitry Andric Name == "avx512.mask.pmov.qd.256" || // Added in 9.0 3030b57cec5SDimitry Andric Name == "avx512.mask.pmov.qd.512" || // Added in 9.0 3040b57cec5SDimitry Andric Name == "avx512.mask.pmov.wb.256" || // Added in 9.0 3050b57cec5SDimitry Andric Name == "avx512.mask.pmov.wb.512" || // Added in 9.0 3060b57cec5SDimitry Andric Name == "sse.cvtsi2ss" || // Added in 7.0 3070b57cec5SDimitry Andric Name == "sse.cvtsi642ss" || // Added in 7.0 3080b57cec5SDimitry Andric Name == "sse2.cvtsi2sd" || // Added in 7.0 3090b57cec5SDimitry Andric Name == "sse2.cvtsi642sd" || // Added in 7.0 3100b57cec5SDimitry Andric Name == "sse2.cvtss2sd" || // Added in 7.0 3110b57cec5SDimitry Andric Name == "sse2.cvtdq2pd" || // Added in 3.9 3120b57cec5SDimitry Andric Name == "sse2.cvtdq2ps" || // Added in 7.0 3130b57cec5SDimitry Andric Name == "sse2.cvtps2pd" || // Added in 3.9 3140b57cec5SDimitry Andric Name == "avx.cvtdq2.pd.256" || // Added in 3.9 3150b57cec5SDimitry Andric Name == "avx.cvtdq2.ps.256" || // Added in 7.0 3160b57cec5SDimitry Andric Name == "avx.cvt.ps2.pd.256" || // Added in 3.9 3170b57cec5SDimitry Andric Name.startswith("avx.vinsertf128.") || // Added in 3.7 3180b57cec5SDimitry Andric Name == "avx2.vinserti128" || // Added in 3.7 3190b57cec5SDimitry Andric Name.startswith("avx512.mask.insert") || // Added in 4.0 3200b57cec5SDimitry Andric Name.startswith("avx.vextractf128.") || // Added in 3.7 3210b57cec5SDimitry Andric Name == "avx2.vextracti128" || // Added in 3.7 3220b57cec5SDimitry Andric Name.startswith("avx512.mask.vextract") || // Added in 4.0 3230b57cec5SDimitry Andric Name.startswith("sse4a.movnt.") || // Added in 3.9 3240b57cec5SDimitry Andric Name.startswith("avx.movnt.") || // Added in 3.2 3250b57cec5SDimitry Andric Name.startswith("avx512.storent.") || // Added in 3.9 3260b57cec5SDimitry Andric Name == "sse41.movntdqa" || // Added in 5.0 3270b57cec5SDimitry Andric Name == "avx2.movntdqa" || // Added in 5.0 3280b57cec5SDimitry Andric Name == "avx512.movntdqa" || // Added in 5.0 3290b57cec5SDimitry Andric Name == "sse2.storel.dq" || // Added in 3.9 3300b57cec5SDimitry Andric Name.startswith("sse.storeu.") || // Added in 3.9 3310b57cec5SDimitry Andric Name.startswith("sse2.storeu.") || // Added in 3.9 3320b57cec5SDimitry Andric Name.startswith("avx.storeu.") || // Added in 3.9 3330b57cec5SDimitry Andric Name.startswith("avx512.mask.storeu.") || // Added in 3.9 3340b57cec5SDimitry Andric Name.startswith("avx512.mask.store.p") || // Added in 3.9 3350b57cec5SDimitry Andric Name.startswith("avx512.mask.store.b.") || // Added in 3.9 3360b57cec5SDimitry Andric Name.startswith("avx512.mask.store.w.") || // Added in 3.9 3370b57cec5SDimitry Andric Name.startswith("avx512.mask.store.d.") || // Added in 3.9 3380b57cec5SDimitry Andric Name.startswith("avx512.mask.store.q.") || // Added in 3.9 3390b57cec5SDimitry Andric Name == "avx512.mask.store.ss" || // Added in 7.0 3400b57cec5SDimitry Andric Name.startswith("avx512.mask.loadu.") || // Added in 3.9 3410b57cec5SDimitry Andric Name.startswith("avx512.mask.load.") || // Added in 3.9 3420b57cec5SDimitry Andric Name.startswith("avx512.mask.expand.load.") || // Added in 7.0 3430b57cec5SDimitry Andric Name.startswith("avx512.mask.compress.store.") || // Added in 7.0 3440b57cec5SDimitry Andric Name.startswith("avx512.mask.expand.b") || // Added in 9.0 3450b57cec5SDimitry Andric Name.startswith("avx512.mask.expand.w") || // Added in 9.0 3460b57cec5SDimitry Andric Name.startswith("avx512.mask.expand.d") || // Added in 9.0 3470b57cec5SDimitry Andric Name.startswith("avx512.mask.expand.q") || // Added in 9.0 3480b57cec5SDimitry Andric Name.startswith("avx512.mask.expand.p") || // Added in 9.0 3490b57cec5SDimitry Andric Name.startswith("avx512.mask.compress.b") || // Added in 9.0 3500b57cec5SDimitry Andric Name.startswith("avx512.mask.compress.w") || // Added in 9.0 3510b57cec5SDimitry Andric Name.startswith("avx512.mask.compress.d") || // Added in 9.0 3520b57cec5SDimitry Andric Name.startswith("avx512.mask.compress.q") || // Added in 9.0 3530b57cec5SDimitry Andric Name.startswith("avx512.mask.compress.p") || // Added in 9.0 3540b57cec5SDimitry Andric Name == "sse42.crc32.64.8" || // Added in 3.4 3550b57cec5SDimitry Andric Name.startswith("avx.vbroadcast.s") || // Added in 3.5 3560b57cec5SDimitry Andric Name.startswith("avx512.vbroadcast.s") || // Added in 7.0 3570b57cec5SDimitry Andric Name.startswith("avx512.mask.palignr.") || // Added in 3.9 3580b57cec5SDimitry Andric Name.startswith("avx512.mask.valign.") || // Added in 4.0 3590b57cec5SDimitry Andric Name.startswith("sse2.psll.dq") || // Added in 3.7 3600b57cec5SDimitry Andric Name.startswith("sse2.psrl.dq") || // Added in 3.7 3610b57cec5SDimitry Andric Name.startswith("avx2.psll.dq") || // Added in 3.7 3620b57cec5SDimitry Andric Name.startswith("avx2.psrl.dq") || // Added in 3.7 3630b57cec5SDimitry Andric Name.startswith("avx512.psll.dq") || // Added in 3.9 3640b57cec5SDimitry Andric Name.startswith("avx512.psrl.dq") || // Added in 3.9 3650b57cec5SDimitry Andric Name == "sse41.pblendw" || // Added in 3.7 3660b57cec5SDimitry Andric Name.startswith("sse41.blendp") || // Added in 3.7 3670b57cec5SDimitry Andric Name.startswith("avx.blend.p") || // Added in 3.7 3680b57cec5SDimitry Andric Name == "avx2.pblendw" || // Added in 3.7 3690b57cec5SDimitry Andric Name.startswith("avx2.pblendd.") || // Added in 3.7 3700b57cec5SDimitry Andric Name.startswith("avx.vbroadcastf128") || // Added in 4.0 3710b57cec5SDimitry Andric Name == "avx2.vbroadcasti128" || // Added in 3.7 3720b57cec5SDimitry Andric Name.startswith("avx512.mask.broadcastf") || // Added in 6.0 3730b57cec5SDimitry Andric Name.startswith("avx512.mask.broadcasti") || // Added in 6.0 3740b57cec5SDimitry Andric Name == "xop.vpcmov" || // Added in 3.8 3750b57cec5SDimitry Andric Name == "xop.vpcmov.256" || // Added in 5.0 3760b57cec5SDimitry Andric Name.startswith("avx512.mask.move.s") || // Added in 4.0 3770b57cec5SDimitry Andric Name.startswith("avx512.cvtmask2") || // Added in 5.0 3780b57cec5SDimitry Andric Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0 3790b57cec5SDimitry Andric Name.startswith("xop.vprot") || // Added in 8.0 3800b57cec5SDimitry Andric Name.startswith("avx512.prol") || // Added in 8.0 3810b57cec5SDimitry Andric Name.startswith("avx512.pror") || // Added in 8.0 3820b57cec5SDimitry Andric Name.startswith("avx512.mask.prorv.") || // Added in 8.0 3830b57cec5SDimitry Andric Name.startswith("avx512.mask.pror.") || // Added in 8.0 3840b57cec5SDimitry Andric Name.startswith("avx512.mask.prolv.") || // Added in 8.0 3850b57cec5SDimitry Andric Name.startswith("avx512.mask.prol.") || // Added in 8.0 3860b57cec5SDimitry Andric Name.startswith("avx512.ptestm") || //Added in 6.0 3870b57cec5SDimitry Andric Name.startswith("avx512.ptestnm") || //Added in 6.0 3880b57cec5SDimitry Andric Name.startswith("avx512.mask.pavg")) // Added in 6.0 3890b57cec5SDimitry Andric return true; 3900b57cec5SDimitry Andric 3910b57cec5SDimitry Andric return false; 3920b57cec5SDimitry Andric } 3930b57cec5SDimitry Andric 3940b57cec5SDimitry Andric static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name, 3950b57cec5SDimitry Andric Function *&NewFn) { 3960b57cec5SDimitry Andric // Only handle intrinsics that start with "x86.". 3970b57cec5SDimitry Andric if (!Name.startswith("x86.")) 3980b57cec5SDimitry Andric return false; 3990b57cec5SDimitry Andric // Remove "x86." prefix. 4000b57cec5SDimitry Andric Name = Name.substr(4); 4010b57cec5SDimitry Andric 4020b57cec5SDimitry Andric if (ShouldUpgradeX86Intrinsic(F, Name)) { 4030b57cec5SDimitry Andric NewFn = nullptr; 4040b57cec5SDimitry Andric return true; 4050b57cec5SDimitry Andric } 4060b57cec5SDimitry Andric 4070b57cec5SDimitry Andric if (Name == "rdtscp") { // Added in 8.0 4080b57cec5SDimitry Andric // If this intrinsic has 0 operands, it's the new version. 4090b57cec5SDimitry Andric if (F->getFunctionType()->getNumParams() == 0) 4100b57cec5SDimitry Andric return false; 4110b57cec5SDimitry Andric 4120b57cec5SDimitry Andric rename(F); 4130b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 4140b57cec5SDimitry Andric Intrinsic::x86_rdtscp); 4150b57cec5SDimitry Andric return true; 4160b57cec5SDimitry Andric } 4170b57cec5SDimitry Andric 4180b57cec5SDimitry Andric // SSE4.1 ptest functions may have an old signature. 4190b57cec5SDimitry Andric if (Name.startswith("sse41.ptest")) { // Added in 3.2 4200b57cec5SDimitry Andric if (Name.substr(11) == "c") 4210b57cec5SDimitry Andric return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestc, NewFn); 4220b57cec5SDimitry Andric if (Name.substr(11) == "z") 4230b57cec5SDimitry Andric return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestz, NewFn); 4240b57cec5SDimitry Andric if (Name.substr(11) == "nzc") 4250b57cec5SDimitry Andric return UpgradePTESTIntrinsic(F, Intrinsic::x86_sse41_ptestnzc, NewFn); 4260b57cec5SDimitry Andric } 4270b57cec5SDimitry Andric // Several blend and other instructions with masks used the wrong number of 4280b57cec5SDimitry Andric // bits. 4290b57cec5SDimitry Andric if (Name == "sse41.insertps") // Added in 3.6 4300b57cec5SDimitry Andric return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_insertps, 4310b57cec5SDimitry Andric NewFn); 4320b57cec5SDimitry Andric if (Name == "sse41.dppd") // Added in 3.6 4330b57cec5SDimitry Andric return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dppd, 4340b57cec5SDimitry Andric NewFn); 4350b57cec5SDimitry Andric if (Name == "sse41.dpps") // Added in 3.6 4360b57cec5SDimitry Andric return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_dpps, 4370b57cec5SDimitry Andric NewFn); 4380b57cec5SDimitry Andric if (Name == "sse41.mpsadbw") // Added in 3.6 4390b57cec5SDimitry Andric return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_sse41_mpsadbw, 4400b57cec5SDimitry Andric NewFn); 4410b57cec5SDimitry Andric if (Name == "avx.dp.ps.256") // Added in 3.6 4420b57cec5SDimitry Andric return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx_dp_ps_256, 4430b57cec5SDimitry Andric NewFn); 4440b57cec5SDimitry Andric if (Name == "avx2.mpsadbw") // Added in 3.6 4450b57cec5SDimitry Andric return UpgradeX86IntrinsicsWith8BitMask(F, Intrinsic::x86_avx2_mpsadbw, 4460b57cec5SDimitry Andric NewFn); 4470b57cec5SDimitry Andric 4480b57cec5SDimitry Andric // frcz.ss/sd may need to have an argument dropped. Added in 3.2 4490b57cec5SDimitry Andric if (Name.startswith("xop.vfrcz.ss") && F->arg_size() == 2) { 4500b57cec5SDimitry Andric rename(F); 4510b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 4520b57cec5SDimitry Andric Intrinsic::x86_xop_vfrcz_ss); 4530b57cec5SDimitry Andric return true; 4540b57cec5SDimitry Andric } 4550b57cec5SDimitry Andric if (Name.startswith("xop.vfrcz.sd") && F->arg_size() == 2) { 4560b57cec5SDimitry Andric rename(F); 4570b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 4580b57cec5SDimitry Andric Intrinsic::x86_xop_vfrcz_sd); 4590b57cec5SDimitry Andric return true; 4600b57cec5SDimitry Andric } 4610b57cec5SDimitry Andric // Upgrade any XOP PERMIL2 index operand still using a float/double vector. 4620b57cec5SDimitry Andric if (Name.startswith("xop.vpermil2")) { // Added in 3.9 4630b57cec5SDimitry Andric auto Idx = F->getFunctionType()->getParamType(2); 4640b57cec5SDimitry Andric if (Idx->isFPOrFPVectorTy()) { 4650b57cec5SDimitry Andric rename(F); 4660b57cec5SDimitry Andric unsigned IdxSize = Idx->getPrimitiveSizeInBits(); 4670b57cec5SDimitry Andric unsigned EltSize = Idx->getScalarSizeInBits(); 4680b57cec5SDimitry Andric Intrinsic::ID Permil2ID; 4690b57cec5SDimitry Andric if (EltSize == 64 && IdxSize == 128) 4700b57cec5SDimitry Andric Permil2ID = Intrinsic::x86_xop_vpermil2pd; 4710b57cec5SDimitry Andric else if (EltSize == 32 && IdxSize == 128) 4720b57cec5SDimitry Andric Permil2ID = Intrinsic::x86_xop_vpermil2ps; 4730b57cec5SDimitry Andric else if (EltSize == 64 && IdxSize == 256) 4740b57cec5SDimitry Andric Permil2ID = Intrinsic::x86_xop_vpermil2pd_256; 4750b57cec5SDimitry Andric else 4760b57cec5SDimitry Andric Permil2ID = Intrinsic::x86_xop_vpermil2ps_256; 4770b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Permil2ID); 4780b57cec5SDimitry Andric return true; 4790b57cec5SDimitry Andric } 4800b57cec5SDimitry Andric } 4810b57cec5SDimitry Andric 4820b57cec5SDimitry Andric if (Name == "seh.recoverfp") { 4830b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp); 4840b57cec5SDimitry Andric return true; 4850b57cec5SDimitry Andric } 4860b57cec5SDimitry Andric 4870b57cec5SDimitry Andric return false; 4880b57cec5SDimitry Andric } 4890b57cec5SDimitry Andric 4900b57cec5SDimitry Andric static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { 4910b57cec5SDimitry Andric assert(F && "Illegal to upgrade a non-existent Function."); 4920b57cec5SDimitry Andric 4930b57cec5SDimitry Andric // Quickly eliminate it, if it's not a candidate. 4940b57cec5SDimitry Andric StringRef Name = F->getName(); 4950b57cec5SDimitry Andric if (Name.size() <= 8 || !Name.startswith("llvm.")) 4960b57cec5SDimitry Andric return false; 4970b57cec5SDimitry Andric Name = Name.substr(5); // Strip off "llvm." 4980b57cec5SDimitry Andric 4990b57cec5SDimitry Andric switch (Name[0]) { 5000b57cec5SDimitry Andric default: break; 5010b57cec5SDimitry Andric case 'a': { 5020b57cec5SDimitry Andric if (Name.startswith("arm.rbit") || Name.startswith("aarch64.rbit")) { 5030b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::bitreverse, 5040b57cec5SDimitry Andric F->arg_begin()->getType()); 5050b57cec5SDimitry Andric return true; 5060b57cec5SDimitry Andric } 5070b57cec5SDimitry Andric if (Name.startswith("arm.neon.vclz")) { 5080b57cec5SDimitry Andric Type* args[2] = { 5090b57cec5SDimitry Andric F->arg_begin()->getType(), 5100b57cec5SDimitry Andric Type::getInt1Ty(F->getContext()) 5110b57cec5SDimitry Andric }; 5120b57cec5SDimitry Andric // Can't use Intrinsic::getDeclaration here as it adds a ".i1" to 5130b57cec5SDimitry Andric // the end of the name. Change name from llvm.arm.neon.vclz.* to 5140b57cec5SDimitry Andric // llvm.ctlz.* 5150b57cec5SDimitry Andric FunctionType* fType = FunctionType::get(F->getReturnType(), args, false); 5160b57cec5SDimitry Andric NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(), 5170b57cec5SDimitry Andric "llvm.ctlz." + Name.substr(14), F->getParent()); 5180b57cec5SDimitry Andric return true; 5190b57cec5SDimitry Andric } 5200b57cec5SDimitry Andric if (Name.startswith("arm.neon.vcnt")) { 5210b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 5220b57cec5SDimitry Andric F->arg_begin()->getType()); 5230b57cec5SDimitry Andric return true; 5240b57cec5SDimitry Andric } 5258bcb0991SDimitry Andric static const Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$"); 5260b57cec5SDimitry Andric if (vldRegex.match(Name)) { 5270b57cec5SDimitry Andric auto fArgs = F->getFunctionType()->params(); 5280b57cec5SDimitry Andric SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end()); 5290b57cec5SDimitry Andric // Can't use Intrinsic::getDeclaration here as the return types might 5300b57cec5SDimitry Andric // then only be structurally equal. 5310b57cec5SDimitry Andric FunctionType* fType = FunctionType::get(F->getReturnType(), Tys, false); 5320b57cec5SDimitry Andric NewFn = Function::Create(fType, F->getLinkage(), F->getAddressSpace(), 5330b57cec5SDimitry Andric "llvm." + Name + ".p0i8", F->getParent()); 5340b57cec5SDimitry Andric return true; 5350b57cec5SDimitry Andric } 5368bcb0991SDimitry Andric static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$"); 5370b57cec5SDimitry Andric if (vstRegex.match(Name)) { 5380b57cec5SDimitry Andric static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1, 5390b57cec5SDimitry Andric Intrinsic::arm_neon_vst2, 5400b57cec5SDimitry Andric Intrinsic::arm_neon_vst3, 5410b57cec5SDimitry Andric Intrinsic::arm_neon_vst4}; 5420b57cec5SDimitry Andric 5430b57cec5SDimitry Andric static const Intrinsic::ID StoreLaneInts[] = { 5440b57cec5SDimitry Andric Intrinsic::arm_neon_vst2lane, Intrinsic::arm_neon_vst3lane, 5450b57cec5SDimitry Andric Intrinsic::arm_neon_vst4lane 5460b57cec5SDimitry Andric }; 5470b57cec5SDimitry Andric 5480b57cec5SDimitry Andric auto fArgs = F->getFunctionType()->params(); 5490b57cec5SDimitry Andric Type *Tys[] = {fArgs[0], fArgs[1]}; 5500b57cec5SDimitry Andric if (Name.find("lane") == StringRef::npos) 5510b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 5520b57cec5SDimitry Andric StoreInts[fArgs.size() - 3], Tys); 5530b57cec5SDimitry Andric else 5540b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 5550b57cec5SDimitry Andric StoreLaneInts[fArgs.size() - 5], Tys); 5560b57cec5SDimitry Andric return true; 5570b57cec5SDimitry Andric } 5580b57cec5SDimitry Andric if (Name == "aarch64.thread.pointer" || Name == "arm.thread.pointer") { 5590b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer); 5600b57cec5SDimitry Andric return true; 5610b57cec5SDimitry Andric } 5620b57cec5SDimitry Andric if (Name.startswith("aarch64.neon.addp")) { 5630b57cec5SDimitry Andric if (F->arg_size() != 2) 5640b57cec5SDimitry Andric break; // Invalid IR. 5650b57cec5SDimitry Andric auto fArgs = F->getFunctionType()->params(); 5660b57cec5SDimitry Andric VectorType *ArgTy = dyn_cast<VectorType>(fArgs[0]); 5670b57cec5SDimitry Andric if (ArgTy && ArgTy->getElementType()->isFloatingPointTy()) { 5680b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 5690b57cec5SDimitry Andric Intrinsic::aarch64_neon_faddp, fArgs); 5700b57cec5SDimitry Andric return true; 5710b57cec5SDimitry Andric } 5720b57cec5SDimitry Andric } 5730b57cec5SDimitry Andric break; 5740b57cec5SDimitry Andric } 5750b57cec5SDimitry Andric 5760b57cec5SDimitry Andric case 'c': { 5770b57cec5SDimitry Andric if (Name.startswith("ctlz.") && F->arg_size() == 1) { 5780b57cec5SDimitry Andric rename(F); 5790b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 5800b57cec5SDimitry Andric F->arg_begin()->getType()); 5810b57cec5SDimitry Andric return true; 5820b57cec5SDimitry Andric } 5830b57cec5SDimitry Andric if (Name.startswith("cttz.") && F->arg_size() == 1) { 5840b57cec5SDimitry Andric rename(F); 5850b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::cttz, 5860b57cec5SDimitry Andric F->arg_begin()->getType()); 5870b57cec5SDimitry Andric return true; 5880b57cec5SDimitry Andric } 5890b57cec5SDimitry Andric break; 5900b57cec5SDimitry Andric } 5910b57cec5SDimitry Andric case 'd': { 5920b57cec5SDimitry Andric if (Name == "dbg.value" && F->arg_size() == 4) { 5930b57cec5SDimitry Andric rename(F); 5940b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::dbg_value); 5950b57cec5SDimitry Andric return true; 5960b57cec5SDimitry Andric } 5970b57cec5SDimitry Andric break; 5980b57cec5SDimitry Andric } 5990b57cec5SDimitry Andric case 'e': { 6000b57cec5SDimitry Andric SmallVector<StringRef, 2> Groups; 6018bcb0991SDimitry Andric static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[fi][0-9]+"); 6020b57cec5SDimitry Andric if (R.match(Name, &Groups)) { 6030b57cec5SDimitry Andric Intrinsic::ID ID = Intrinsic::not_intrinsic; 6040b57cec5SDimitry Andric if (Groups[1] == "fadd") 6050b57cec5SDimitry Andric ID = Intrinsic::experimental_vector_reduce_v2_fadd; 6060b57cec5SDimitry Andric if (Groups[1] == "fmul") 6070b57cec5SDimitry Andric ID = Intrinsic::experimental_vector_reduce_v2_fmul; 6080b57cec5SDimitry Andric 6090b57cec5SDimitry Andric if (ID != Intrinsic::not_intrinsic) { 6100b57cec5SDimitry Andric rename(F); 6110b57cec5SDimitry Andric auto Args = F->getFunctionType()->params(); 6120b57cec5SDimitry Andric Type *Tys[] = {F->getFunctionType()->getReturnType(), Args[1]}; 6130b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys); 6140b57cec5SDimitry Andric return true; 6150b57cec5SDimitry Andric } 6160b57cec5SDimitry Andric } 6170b57cec5SDimitry Andric break; 6180b57cec5SDimitry Andric } 6190b57cec5SDimitry Andric case 'i': 6200b57cec5SDimitry Andric case 'l': { 6210b57cec5SDimitry Andric bool IsLifetimeStart = Name.startswith("lifetime.start"); 6220b57cec5SDimitry Andric if (IsLifetimeStart || Name.startswith("invariant.start")) { 6230b57cec5SDimitry Andric Intrinsic::ID ID = IsLifetimeStart ? 6240b57cec5SDimitry Andric Intrinsic::lifetime_start : Intrinsic::invariant_start; 6250b57cec5SDimitry Andric auto Args = F->getFunctionType()->params(); 6260b57cec5SDimitry Andric Type* ObjectPtr[1] = {Args[1]}; 6270b57cec5SDimitry Andric if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) { 6280b57cec5SDimitry Andric rename(F); 6290b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr); 6300b57cec5SDimitry Andric return true; 6310b57cec5SDimitry Andric } 6320b57cec5SDimitry Andric } 6330b57cec5SDimitry Andric 6340b57cec5SDimitry Andric bool IsLifetimeEnd = Name.startswith("lifetime.end"); 6350b57cec5SDimitry Andric if (IsLifetimeEnd || Name.startswith("invariant.end")) { 6360b57cec5SDimitry Andric Intrinsic::ID ID = IsLifetimeEnd ? 6370b57cec5SDimitry Andric Intrinsic::lifetime_end : Intrinsic::invariant_end; 6380b57cec5SDimitry Andric 6390b57cec5SDimitry Andric auto Args = F->getFunctionType()->params(); 6400b57cec5SDimitry Andric Type* ObjectPtr[1] = {Args[IsLifetimeEnd ? 1 : 2]}; 6410b57cec5SDimitry Andric if (F->getName() != Intrinsic::getName(ID, ObjectPtr)) { 6420b57cec5SDimitry Andric rename(F); 6430b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), ID, ObjectPtr); 6440b57cec5SDimitry Andric return true; 6450b57cec5SDimitry Andric } 6460b57cec5SDimitry Andric } 6470b57cec5SDimitry Andric if (Name.startswith("invariant.group.barrier")) { 6480b57cec5SDimitry Andric // Rename invariant.group.barrier to launder.invariant.group 6490b57cec5SDimitry Andric auto Args = F->getFunctionType()->params(); 6500b57cec5SDimitry Andric Type* ObjectPtr[1] = {Args[0]}; 6510b57cec5SDimitry Andric rename(F); 6520b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 6530b57cec5SDimitry Andric Intrinsic::launder_invariant_group, ObjectPtr); 6540b57cec5SDimitry Andric return true; 6550b57cec5SDimitry Andric 6560b57cec5SDimitry Andric } 6570b57cec5SDimitry Andric 6580b57cec5SDimitry Andric break; 6590b57cec5SDimitry Andric } 6600b57cec5SDimitry Andric case 'm': { 6610b57cec5SDimitry Andric if (Name.startswith("masked.load.")) { 6620b57cec5SDimitry Andric Type *Tys[] = { F->getReturnType(), F->arg_begin()->getType() }; 6630b57cec5SDimitry Andric if (F->getName() != Intrinsic::getName(Intrinsic::masked_load, Tys)) { 6640b57cec5SDimitry Andric rename(F); 6650b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 6660b57cec5SDimitry Andric Intrinsic::masked_load, 6670b57cec5SDimitry Andric Tys); 6680b57cec5SDimitry Andric return true; 6690b57cec5SDimitry Andric } 6700b57cec5SDimitry Andric } 6710b57cec5SDimitry Andric if (Name.startswith("masked.store.")) { 6720b57cec5SDimitry Andric auto Args = F->getFunctionType()->params(); 6730b57cec5SDimitry Andric Type *Tys[] = { Args[0], Args[1] }; 6740b57cec5SDimitry Andric if (F->getName() != Intrinsic::getName(Intrinsic::masked_store, Tys)) { 6750b57cec5SDimitry Andric rename(F); 6760b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 6770b57cec5SDimitry Andric Intrinsic::masked_store, 6780b57cec5SDimitry Andric Tys); 6790b57cec5SDimitry Andric return true; 6800b57cec5SDimitry Andric } 6810b57cec5SDimitry Andric } 6820b57cec5SDimitry Andric // Renaming gather/scatter intrinsics with no address space overloading 6830b57cec5SDimitry Andric // to the new overload which includes an address space 6840b57cec5SDimitry Andric if (Name.startswith("masked.gather.")) { 6850b57cec5SDimitry Andric Type *Tys[] = {F->getReturnType(), F->arg_begin()->getType()}; 6860b57cec5SDimitry Andric if (F->getName() != Intrinsic::getName(Intrinsic::masked_gather, Tys)) { 6870b57cec5SDimitry Andric rename(F); 6880b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 6890b57cec5SDimitry Andric Intrinsic::masked_gather, Tys); 6900b57cec5SDimitry Andric return true; 6910b57cec5SDimitry Andric } 6920b57cec5SDimitry Andric } 6930b57cec5SDimitry Andric if (Name.startswith("masked.scatter.")) { 6940b57cec5SDimitry Andric auto Args = F->getFunctionType()->params(); 6950b57cec5SDimitry Andric Type *Tys[] = {Args[0], Args[1]}; 6960b57cec5SDimitry Andric if (F->getName() != Intrinsic::getName(Intrinsic::masked_scatter, Tys)) { 6970b57cec5SDimitry Andric rename(F); 6980b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), 6990b57cec5SDimitry Andric Intrinsic::masked_scatter, Tys); 7000b57cec5SDimitry Andric return true; 7010b57cec5SDimitry Andric } 7020b57cec5SDimitry Andric } 7030b57cec5SDimitry Andric // Updating the memory intrinsics (memcpy/memmove/memset) that have an 7040b57cec5SDimitry Andric // alignment parameter to embedding the alignment as an attribute of 7050b57cec5SDimitry Andric // the pointer args. 7060b57cec5SDimitry Andric if (Name.startswith("memcpy.") && F->arg_size() == 5) { 7070b57cec5SDimitry Andric rename(F); 7080b57cec5SDimitry Andric // Get the types of dest, src, and len 7090b57cec5SDimitry Andric ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3); 7100b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memcpy, 7110b57cec5SDimitry Andric ParamTypes); 7120b57cec5SDimitry Andric return true; 7130b57cec5SDimitry Andric } 7140b57cec5SDimitry Andric if (Name.startswith("memmove.") && F->arg_size() == 5) { 7150b57cec5SDimitry Andric rename(F); 7160b57cec5SDimitry Andric // Get the types of dest, src, and len 7170b57cec5SDimitry Andric ArrayRef<Type *> ParamTypes = F->getFunctionType()->params().slice(0, 3); 7180b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memmove, 7190b57cec5SDimitry Andric ParamTypes); 7200b57cec5SDimitry Andric return true; 7210b57cec5SDimitry Andric } 7220b57cec5SDimitry Andric if (Name.startswith("memset.") && F->arg_size() == 5) { 7230b57cec5SDimitry Andric rename(F); 7240b57cec5SDimitry Andric // Get the types of dest, and len 7250b57cec5SDimitry Andric const auto *FT = F->getFunctionType(); 7260b57cec5SDimitry Andric Type *ParamTypes[2] = { 7270b57cec5SDimitry Andric FT->getParamType(0), // Dest 7280b57cec5SDimitry Andric FT->getParamType(2) // len 7290b57cec5SDimitry Andric }; 7300b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::memset, 7310b57cec5SDimitry Andric ParamTypes); 7320b57cec5SDimitry Andric return true; 7330b57cec5SDimitry Andric } 7340b57cec5SDimitry Andric break; 7350b57cec5SDimitry Andric } 7360b57cec5SDimitry Andric case 'n': { 7370b57cec5SDimitry Andric if (Name.startswith("nvvm.")) { 7380b57cec5SDimitry Andric Name = Name.substr(5); 7390b57cec5SDimitry Andric 7400b57cec5SDimitry Andric // The following nvvm intrinsics correspond exactly to an LLVM intrinsic. 7410b57cec5SDimitry Andric Intrinsic::ID IID = StringSwitch<Intrinsic::ID>(Name) 7420b57cec5SDimitry Andric .Cases("brev32", "brev64", Intrinsic::bitreverse) 7430b57cec5SDimitry Andric .Case("clz.i", Intrinsic::ctlz) 7440b57cec5SDimitry Andric .Case("popc.i", Intrinsic::ctpop) 7450b57cec5SDimitry Andric .Default(Intrinsic::not_intrinsic); 7460b57cec5SDimitry Andric if (IID != Intrinsic::not_intrinsic && F->arg_size() == 1) { 7470b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), IID, 7480b57cec5SDimitry Andric {F->getReturnType()}); 7490b57cec5SDimitry Andric return true; 7500b57cec5SDimitry Andric } 7510b57cec5SDimitry Andric 7520b57cec5SDimitry Andric // The following nvvm intrinsics correspond exactly to an LLVM idiom, but 7530b57cec5SDimitry Andric // not to an intrinsic alone. We expand them in UpgradeIntrinsicCall. 7540b57cec5SDimitry Andric // 7550b57cec5SDimitry Andric // TODO: We could add lohi.i2d. 7560b57cec5SDimitry Andric bool Expand = StringSwitch<bool>(Name) 7570b57cec5SDimitry Andric .Cases("abs.i", "abs.ll", true) 7580b57cec5SDimitry Andric .Cases("clz.ll", "popc.ll", "h2f", true) 7590b57cec5SDimitry Andric .Cases("max.i", "max.ll", "max.ui", "max.ull", true) 7600b57cec5SDimitry Andric .Cases("min.i", "min.ll", "min.ui", "min.ull", true) 7610b57cec5SDimitry Andric .StartsWith("atomic.load.add.f32.p", true) 7620b57cec5SDimitry Andric .StartsWith("atomic.load.add.f64.p", true) 7630b57cec5SDimitry Andric .Default(false); 7640b57cec5SDimitry Andric if (Expand) { 7650b57cec5SDimitry Andric NewFn = nullptr; 7660b57cec5SDimitry Andric return true; 7670b57cec5SDimitry Andric } 7680b57cec5SDimitry Andric } 7690b57cec5SDimitry Andric break; 7700b57cec5SDimitry Andric } 7710b57cec5SDimitry Andric case 'o': 7720b57cec5SDimitry Andric // We only need to change the name to match the mangling including the 7730b57cec5SDimitry Andric // address space. 7740b57cec5SDimitry Andric if (Name.startswith("objectsize.")) { 7750b57cec5SDimitry Andric Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() }; 7760b57cec5SDimitry Andric if (F->arg_size() == 2 || F->arg_size() == 3 || 7770b57cec5SDimitry Andric F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) { 7780b57cec5SDimitry Andric rename(F); 7790b57cec5SDimitry Andric NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize, 7800b57cec5SDimitry Andric Tys); 7810b57cec5SDimitry Andric return true; 7820b57cec5SDimitry Andric } 7830b57cec5SDimitry Andric } 7840b57cec5SDimitry Andric break; 7850b57cec5SDimitry Andric 7868bcb0991SDimitry Andric case 'p': 7878bcb0991SDimitry Andric if (Name == "prefetch") { 7888bcb0991SDimitry Andric // Handle address space overloading. 7898bcb0991SDimitry Andric Type *Tys[] = {F->arg_begin()->getType()}; 7908bcb0991SDimitry Andric if (F->getName() != Intrinsic::getName(Intrinsic::prefetch, Tys)) { 7918bcb0991SDimitry Andric rename(F); 7928bcb0991SDimitry Andric NewFn = 7938bcb0991SDimitry Andric Intrinsic::getDeclaration(F->getParent(), Intrinsic::prefetch, Tys); 7948bcb0991SDimitry Andric return true; 7958bcb0991SDimitry Andric } 7968bcb0991SDimitry Andric } 7978bcb0991SDimitry Andric break; 7988bcb0991SDimitry Andric 7990b57cec5SDimitry Andric case 's': 8000b57cec5SDimitry Andric if (Name == "stackprotectorcheck") { 8010b57cec5SDimitry Andric NewFn = nullptr; 8020b57cec5SDimitry Andric return true; 8030b57cec5SDimitry Andric } 8040b57cec5SDimitry Andric break; 8050b57cec5SDimitry Andric 8060b57cec5SDimitry Andric case 'x': 8070b57cec5SDimitry Andric if (UpgradeX86IntrinsicFunction(F, Name, NewFn)) 8080b57cec5SDimitry Andric return true; 8090b57cec5SDimitry Andric } 8100b57cec5SDimitry Andric // Remangle our intrinsic since we upgrade the mangling 8110b57cec5SDimitry Andric auto Result = llvm::Intrinsic::remangleIntrinsicFunction(F); 8120b57cec5SDimitry Andric if (Result != None) { 8130b57cec5SDimitry Andric NewFn = Result.getValue(); 8140b57cec5SDimitry Andric return true; 8150b57cec5SDimitry Andric } 8160b57cec5SDimitry Andric 8170b57cec5SDimitry Andric // This may not belong here. This function is effectively being overloaded 8180b57cec5SDimitry Andric // to both detect an intrinsic which needs upgrading, and to provide the 8190b57cec5SDimitry Andric // upgraded form of the intrinsic. We should perhaps have two separate 8200b57cec5SDimitry Andric // functions for this. 8210b57cec5SDimitry Andric return false; 8220b57cec5SDimitry Andric } 8230b57cec5SDimitry Andric 8240b57cec5SDimitry Andric bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) { 8250b57cec5SDimitry Andric NewFn = nullptr; 8260b57cec5SDimitry Andric bool Upgraded = UpgradeIntrinsicFunction1(F, NewFn); 8270b57cec5SDimitry Andric assert(F != NewFn && "Intrinsic function upgraded to the same function"); 8280b57cec5SDimitry Andric 8290b57cec5SDimitry Andric // Upgrade intrinsic attributes. This does not change the function. 8300b57cec5SDimitry Andric if (NewFn) 8310b57cec5SDimitry Andric F = NewFn; 8320b57cec5SDimitry Andric if (Intrinsic::ID id = F->getIntrinsicID()) 8330b57cec5SDimitry Andric F->setAttributes(Intrinsic::getAttributes(F->getContext(), id)); 8340b57cec5SDimitry Andric return Upgraded; 8350b57cec5SDimitry Andric } 8360b57cec5SDimitry Andric 8370b57cec5SDimitry Andric GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) { 8380b57cec5SDimitry Andric if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" || 8390b57cec5SDimitry Andric GV->getName() == "llvm.global_dtors")) || 8400b57cec5SDimitry Andric !GV->hasInitializer()) 8410b57cec5SDimitry Andric return nullptr; 8420b57cec5SDimitry Andric ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType()); 8430b57cec5SDimitry Andric if (!ATy) 8440b57cec5SDimitry Andric return nullptr; 8450b57cec5SDimitry Andric StructType *STy = dyn_cast<StructType>(ATy->getElementType()); 8460b57cec5SDimitry Andric if (!STy || STy->getNumElements() != 2) 8470b57cec5SDimitry Andric return nullptr; 8480b57cec5SDimitry Andric 8490b57cec5SDimitry Andric LLVMContext &C = GV->getContext(); 8500b57cec5SDimitry Andric IRBuilder<> IRB(C); 8510b57cec5SDimitry Andric auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1), 8520b57cec5SDimitry Andric IRB.getInt8PtrTy()); 8530b57cec5SDimitry Andric Constant *Init = GV->getInitializer(); 8540b57cec5SDimitry Andric unsigned N = Init->getNumOperands(); 8550b57cec5SDimitry Andric std::vector<Constant *> NewCtors(N); 8560b57cec5SDimitry Andric for (unsigned i = 0; i != N; ++i) { 8570b57cec5SDimitry Andric auto Ctor = cast<Constant>(Init->getOperand(i)); 8580b57cec5SDimitry Andric NewCtors[i] = ConstantStruct::get( 8590b57cec5SDimitry Andric EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1), 8600b57cec5SDimitry Andric Constant::getNullValue(IRB.getInt8PtrTy())); 8610b57cec5SDimitry Andric } 8620b57cec5SDimitry Andric Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors); 8630b57cec5SDimitry Andric 8640b57cec5SDimitry Andric return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(), 8650b57cec5SDimitry Andric NewInit, GV->getName()); 8660b57cec5SDimitry Andric } 8670b57cec5SDimitry Andric 8680b57cec5SDimitry Andric // Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them 8690b57cec5SDimitry Andric // to byte shuffles. 8700b57cec5SDimitry Andric static Value *UpgradeX86PSLLDQIntrinsics(IRBuilder<> &Builder, 8710b57cec5SDimitry Andric Value *Op, unsigned Shift) { 8720b57cec5SDimitry Andric Type *ResultTy = Op->getType(); 8730b57cec5SDimitry Andric unsigned NumElts = ResultTy->getVectorNumElements() * 8; 8740b57cec5SDimitry Andric 8750b57cec5SDimitry Andric // Bitcast from a 64-bit element type to a byte element type. 8760b57cec5SDimitry Andric Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); 8770b57cec5SDimitry Andric Op = Builder.CreateBitCast(Op, VecTy, "cast"); 8780b57cec5SDimitry Andric 8790b57cec5SDimitry Andric // We'll be shuffling in zeroes. 8800b57cec5SDimitry Andric Value *Res = Constant::getNullValue(VecTy); 8810b57cec5SDimitry Andric 8820b57cec5SDimitry Andric // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, 8830b57cec5SDimitry Andric // we'll just return the zero vector. 8840b57cec5SDimitry Andric if (Shift < 16) { 8850b57cec5SDimitry Andric uint32_t Idxs[64]; 8860b57cec5SDimitry Andric // 256/512-bit version is split into 2/4 16-byte lanes. 8870b57cec5SDimitry Andric for (unsigned l = 0; l != NumElts; l += 16) 8880b57cec5SDimitry Andric for (unsigned i = 0; i != 16; ++i) { 8890b57cec5SDimitry Andric unsigned Idx = NumElts + i - Shift; 8900b57cec5SDimitry Andric if (Idx < NumElts) 8910b57cec5SDimitry Andric Idx -= NumElts - 16; // end of lane, switch operand. 8920b57cec5SDimitry Andric Idxs[l + i] = Idx + l; 8930b57cec5SDimitry Andric } 8940b57cec5SDimitry Andric 8950b57cec5SDimitry Andric Res = Builder.CreateShuffleVector(Res, Op, makeArrayRef(Idxs, NumElts)); 8960b57cec5SDimitry Andric } 8970b57cec5SDimitry Andric 8980b57cec5SDimitry Andric // Bitcast back to a 64-bit element type. 8990b57cec5SDimitry Andric return Builder.CreateBitCast(Res, ResultTy, "cast"); 9000b57cec5SDimitry Andric } 9010b57cec5SDimitry Andric 9020b57cec5SDimitry Andric // Handles upgrading SSE2/AVX2/AVX512BW PSRLDQ intrinsics by converting them 9030b57cec5SDimitry Andric // to byte shuffles. 9040b57cec5SDimitry Andric static Value *UpgradeX86PSRLDQIntrinsics(IRBuilder<> &Builder, Value *Op, 9050b57cec5SDimitry Andric unsigned Shift) { 9060b57cec5SDimitry Andric Type *ResultTy = Op->getType(); 9070b57cec5SDimitry Andric unsigned NumElts = ResultTy->getVectorNumElements() * 8; 9080b57cec5SDimitry Andric 9090b57cec5SDimitry Andric // Bitcast from a 64-bit element type to a byte element type. 9100b57cec5SDimitry Andric Type *VecTy = VectorType::get(Builder.getInt8Ty(), NumElts); 9110b57cec5SDimitry Andric Op = Builder.CreateBitCast(Op, VecTy, "cast"); 9120b57cec5SDimitry Andric 9130b57cec5SDimitry Andric // We'll be shuffling in zeroes. 9140b57cec5SDimitry Andric Value *Res = Constant::getNullValue(VecTy); 9150b57cec5SDimitry Andric 9160b57cec5SDimitry Andric // If shift is less than 16, emit a shuffle to move the bytes. Otherwise, 9170b57cec5SDimitry Andric // we'll just return the zero vector. 9180b57cec5SDimitry Andric if (Shift < 16) { 9190b57cec5SDimitry Andric uint32_t Idxs[64]; 9200b57cec5SDimitry Andric // 256/512-bit version is split into 2/4 16-byte lanes. 9210b57cec5SDimitry Andric for (unsigned l = 0; l != NumElts; l += 16) 9220b57cec5SDimitry Andric for (unsigned i = 0; i != 16; ++i) { 9230b57cec5SDimitry Andric unsigned Idx = i + Shift; 9240b57cec5SDimitry Andric if (Idx >= 16) 9250b57cec5SDimitry Andric Idx += NumElts - 16; // end of lane, switch operand. 9260b57cec5SDimitry Andric Idxs[l + i] = Idx + l; 9270b57cec5SDimitry Andric } 9280b57cec5SDimitry Andric 9290b57cec5SDimitry Andric Res = Builder.CreateShuffleVector(Op, Res, makeArrayRef(Idxs, NumElts)); 9300b57cec5SDimitry Andric } 9310b57cec5SDimitry Andric 9320b57cec5SDimitry Andric // Bitcast back to a 64-bit element type. 9330b57cec5SDimitry Andric return Builder.CreateBitCast(Res, ResultTy, "cast"); 9340b57cec5SDimitry Andric } 9350b57cec5SDimitry Andric 9360b57cec5SDimitry Andric static Value *getX86MaskVec(IRBuilder<> &Builder, Value *Mask, 9370b57cec5SDimitry Andric unsigned NumElts) { 9380b57cec5SDimitry Andric llvm::VectorType *MaskTy = llvm::VectorType::get(Builder.getInt1Ty(), 9390b57cec5SDimitry Andric cast<IntegerType>(Mask->getType())->getBitWidth()); 9400b57cec5SDimitry Andric Mask = Builder.CreateBitCast(Mask, MaskTy); 9410b57cec5SDimitry Andric 9420b57cec5SDimitry Andric // If we have less than 8 elements, then the starting mask was an i8 and 9430b57cec5SDimitry Andric // we need to extract down to the right number of elements. 9440b57cec5SDimitry Andric if (NumElts < 8) { 9450b57cec5SDimitry Andric uint32_t Indices[4]; 9460b57cec5SDimitry Andric for (unsigned i = 0; i != NumElts; ++i) 9470b57cec5SDimitry Andric Indices[i] = i; 9480b57cec5SDimitry Andric Mask = Builder.CreateShuffleVector(Mask, Mask, 9490b57cec5SDimitry Andric makeArrayRef(Indices, NumElts), 9500b57cec5SDimitry Andric "extract"); 9510b57cec5SDimitry Andric } 9520b57cec5SDimitry Andric 9530b57cec5SDimitry Andric return Mask; 9540b57cec5SDimitry Andric } 9550b57cec5SDimitry Andric 9560b57cec5SDimitry Andric static Value *EmitX86Select(IRBuilder<> &Builder, Value *Mask, 9570b57cec5SDimitry Andric Value *Op0, Value *Op1) { 9580b57cec5SDimitry Andric // If the mask is all ones just emit the first operation. 9590b57cec5SDimitry Andric if (const auto *C = dyn_cast<Constant>(Mask)) 9600b57cec5SDimitry Andric if (C->isAllOnesValue()) 9610b57cec5SDimitry Andric return Op0; 9620b57cec5SDimitry Andric 9630b57cec5SDimitry Andric Mask = getX86MaskVec(Builder, Mask, Op0->getType()->getVectorNumElements()); 9640b57cec5SDimitry Andric return Builder.CreateSelect(Mask, Op0, Op1); 9650b57cec5SDimitry Andric } 9660b57cec5SDimitry Andric 9670b57cec5SDimitry Andric static Value *EmitX86ScalarSelect(IRBuilder<> &Builder, Value *Mask, 9680b57cec5SDimitry Andric Value *Op0, Value *Op1) { 9690b57cec5SDimitry Andric // If the mask is all ones just emit the first operation. 9700b57cec5SDimitry Andric if (const auto *C = dyn_cast<Constant>(Mask)) 9710b57cec5SDimitry Andric if (C->isAllOnesValue()) 9720b57cec5SDimitry Andric return Op0; 9730b57cec5SDimitry Andric 9740b57cec5SDimitry Andric llvm::VectorType *MaskTy = 9750b57cec5SDimitry Andric llvm::VectorType::get(Builder.getInt1Ty(), 9760b57cec5SDimitry Andric Mask->getType()->getIntegerBitWidth()); 9770b57cec5SDimitry Andric Mask = Builder.CreateBitCast(Mask, MaskTy); 9780b57cec5SDimitry Andric Mask = Builder.CreateExtractElement(Mask, (uint64_t)0); 9790b57cec5SDimitry Andric return Builder.CreateSelect(Mask, Op0, Op1); 9800b57cec5SDimitry Andric } 9810b57cec5SDimitry Andric 9820b57cec5SDimitry Andric // Handle autoupgrade for masked PALIGNR and VALIGND/Q intrinsics. 9830b57cec5SDimitry Andric // PALIGNR handles large immediates by shifting while VALIGN masks the immediate 9840b57cec5SDimitry Andric // so we need to handle both cases. VALIGN also doesn't have 128-bit lanes. 9850b57cec5SDimitry Andric static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0, 9860b57cec5SDimitry Andric Value *Op1, Value *Shift, 9870b57cec5SDimitry Andric Value *Passthru, Value *Mask, 9880b57cec5SDimitry Andric bool IsVALIGN) { 9890b57cec5SDimitry Andric unsigned ShiftVal = cast<llvm::ConstantInt>(Shift)->getZExtValue(); 9900b57cec5SDimitry Andric 9910b57cec5SDimitry Andric unsigned NumElts = Op0->getType()->getVectorNumElements(); 9920b57cec5SDimitry Andric assert((IsVALIGN || NumElts % 16 == 0) && "Illegal NumElts for PALIGNR!"); 9930b57cec5SDimitry Andric assert((!IsVALIGN || NumElts <= 16) && "NumElts too large for VALIGN!"); 9940b57cec5SDimitry Andric assert(isPowerOf2_32(NumElts) && "NumElts not a power of 2!"); 9950b57cec5SDimitry Andric 9960b57cec5SDimitry Andric // Mask the immediate for VALIGN. 9970b57cec5SDimitry Andric if (IsVALIGN) 9980b57cec5SDimitry Andric ShiftVal &= (NumElts - 1); 9990b57cec5SDimitry Andric 10000b57cec5SDimitry Andric // If palignr is shifting the pair of vectors more than the size of two 10010b57cec5SDimitry Andric // lanes, emit zero. 10020b57cec5SDimitry Andric if (ShiftVal >= 32) 10030b57cec5SDimitry Andric return llvm::Constant::getNullValue(Op0->getType()); 10040b57cec5SDimitry Andric 10050b57cec5SDimitry Andric // If palignr is shifting the pair of input vectors more than one lane, 10060b57cec5SDimitry Andric // but less than two lanes, convert to shifting in zeroes. 10070b57cec5SDimitry Andric if (ShiftVal > 16) { 10080b57cec5SDimitry Andric ShiftVal -= 16; 10090b57cec5SDimitry Andric Op1 = Op0; 10100b57cec5SDimitry Andric Op0 = llvm::Constant::getNullValue(Op0->getType()); 10110b57cec5SDimitry Andric } 10120b57cec5SDimitry Andric 10130b57cec5SDimitry Andric uint32_t Indices[64]; 10140b57cec5SDimitry Andric // 256-bit palignr operates on 128-bit lanes so we need to handle that 10150b57cec5SDimitry Andric for (unsigned l = 0; l < NumElts; l += 16) { 10160b57cec5SDimitry Andric for (unsigned i = 0; i != 16; ++i) { 10170b57cec5SDimitry Andric unsigned Idx = ShiftVal + i; 10180b57cec5SDimitry Andric if (!IsVALIGN && Idx >= 16) // Disable wrap for VALIGN. 10190b57cec5SDimitry Andric Idx += NumElts - 16; // End of lane, switch operand. 10200b57cec5SDimitry Andric Indices[l + i] = Idx + l; 10210b57cec5SDimitry Andric } 10220b57cec5SDimitry Andric } 10230b57cec5SDimitry Andric 10240b57cec5SDimitry Andric Value *Align = Builder.CreateShuffleVector(Op1, Op0, 10250b57cec5SDimitry Andric makeArrayRef(Indices, NumElts), 10260b57cec5SDimitry Andric "palignr"); 10270b57cec5SDimitry Andric 10280b57cec5SDimitry Andric return EmitX86Select(Builder, Mask, Align, Passthru); 10290b57cec5SDimitry Andric } 10300b57cec5SDimitry Andric 10310b57cec5SDimitry Andric static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI, 10320b57cec5SDimitry Andric bool ZeroMask, bool IndexForm) { 10330b57cec5SDimitry Andric Type *Ty = CI.getType(); 10340b57cec5SDimitry Andric unsigned VecWidth = Ty->getPrimitiveSizeInBits(); 10350b57cec5SDimitry Andric unsigned EltWidth = Ty->getScalarSizeInBits(); 10360b57cec5SDimitry Andric bool IsFloat = Ty->isFPOrFPVectorTy(); 10370b57cec5SDimitry Andric Intrinsic::ID IID; 10380b57cec5SDimitry Andric if (VecWidth == 128 && EltWidth == 32 && IsFloat) 10390b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_ps_128; 10400b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 32 && !IsFloat) 10410b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_d_128; 10420b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 64 && IsFloat) 10430b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_pd_128; 10440b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 64 && !IsFloat) 10450b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_q_128; 10460b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32 && IsFloat) 10470b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_ps_256; 10480b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32 && !IsFloat) 10490b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_d_256; 10500b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64 && IsFloat) 10510b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_pd_256; 10520b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64 && !IsFloat) 10530b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_q_256; 10540b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 32 && IsFloat) 10550b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_ps_512; 10560b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 32 && !IsFloat) 10570b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_d_512; 10580b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 64 && IsFloat) 10590b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_pd_512; 10600b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 64 && !IsFloat) 10610b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_q_512; 10620b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 16) 10630b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_hi_128; 10640b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 16) 10650b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_hi_256; 10660b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 16) 10670b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_hi_512; 10680b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 8) 10690b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_qi_128; 10700b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 8) 10710b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_qi_256; 10720b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 8) 10730b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermi2var_qi_512; 10740b57cec5SDimitry Andric else 10750b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 10760b57cec5SDimitry Andric 10770b57cec5SDimitry Andric Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1), 10780b57cec5SDimitry Andric CI.getArgOperand(2) }; 10790b57cec5SDimitry Andric 10800b57cec5SDimitry Andric // If this isn't index form we need to swap operand 0 and 1. 10810b57cec5SDimitry Andric if (!IndexForm) 10820b57cec5SDimitry Andric std::swap(Args[0], Args[1]); 10830b57cec5SDimitry Andric 10840b57cec5SDimitry Andric Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), 10850b57cec5SDimitry Andric Args); 10860b57cec5SDimitry Andric Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty) 10870b57cec5SDimitry Andric : Builder.CreateBitCast(CI.getArgOperand(1), 10880b57cec5SDimitry Andric Ty); 10890b57cec5SDimitry Andric return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru); 10900b57cec5SDimitry Andric } 10910b57cec5SDimitry Andric 10920b57cec5SDimitry Andric static Value *UpgradeX86AddSubSatIntrinsics(IRBuilder<> &Builder, CallInst &CI, 10930b57cec5SDimitry Andric bool IsSigned, bool IsAddition) { 10940b57cec5SDimitry Andric Type *Ty = CI.getType(); 10950b57cec5SDimitry Andric Value *Op0 = CI.getOperand(0); 10960b57cec5SDimitry Andric Value *Op1 = CI.getOperand(1); 10970b57cec5SDimitry Andric 10980b57cec5SDimitry Andric Intrinsic::ID IID = 10990b57cec5SDimitry Andric IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat) 11000b57cec5SDimitry Andric : (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat); 11010b57cec5SDimitry Andric Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); 11020b57cec5SDimitry Andric Value *Res = Builder.CreateCall(Intrin, {Op0, Op1}); 11030b57cec5SDimitry Andric 11040b57cec5SDimitry Andric if (CI.getNumArgOperands() == 4) { // For masked intrinsics. 11050b57cec5SDimitry Andric Value *VecSrc = CI.getOperand(2); 11060b57cec5SDimitry Andric Value *Mask = CI.getOperand(3); 11070b57cec5SDimitry Andric Res = EmitX86Select(Builder, Mask, Res, VecSrc); 11080b57cec5SDimitry Andric } 11090b57cec5SDimitry Andric return Res; 11100b57cec5SDimitry Andric } 11110b57cec5SDimitry Andric 11120b57cec5SDimitry Andric static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI, 11130b57cec5SDimitry Andric bool IsRotateRight) { 11140b57cec5SDimitry Andric Type *Ty = CI.getType(); 11150b57cec5SDimitry Andric Value *Src = CI.getArgOperand(0); 11160b57cec5SDimitry Andric Value *Amt = CI.getArgOperand(1); 11170b57cec5SDimitry Andric 11180b57cec5SDimitry Andric // Amount may be scalar immediate, in which case create a splat vector. 11190b57cec5SDimitry Andric // Funnel shifts amounts are treated as modulo and types are all power-of-2 so 11200b57cec5SDimitry Andric // we only care about the lowest log2 bits anyway. 11210b57cec5SDimitry Andric if (Amt->getType() != Ty) { 11220b57cec5SDimitry Andric unsigned NumElts = Ty->getVectorNumElements(); 11230b57cec5SDimitry Andric Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); 11240b57cec5SDimitry Andric Amt = Builder.CreateVectorSplat(NumElts, Amt); 11250b57cec5SDimitry Andric } 11260b57cec5SDimitry Andric 11270b57cec5SDimitry Andric Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl; 11280b57cec5SDimitry Andric Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); 11290b57cec5SDimitry Andric Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt}); 11300b57cec5SDimitry Andric 11310b57cec5SDimitry Andric if (CI.getNumArgOperands() == 4) { // For masked intrinsics. 11320b57cec5SDimitry Andric Value *VecSrc = CI.getOperand(2); 11330b57cec5SDimitry Andric Value *Mask = CI.getOperand(3); 11340b57cec5SDimitry Andric Res = EmitX86Select(Builder, Mask, Res, VecSrc); 11350b57cec5SDimitry Andric } 11360b57cec5SDimitry Andric return Res; 11370b57cec5SDimitry Andric } 11380b57cec5SDimitry Andric 11390b57cec5SDimitry Andric static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm, 11400b57cec5SDimitry Andric bool IsSigned) { 11410b57cec5SDimitry Andric Type *Ty = CI.getType(); 11420b57cec5SDimitry Andric Value *LHS = CI.getArgOperand(0); 11430b57cec5SDimitry Andric Value *RHS = CI.getArgOperand(1); 11440b57cec5SDimitry Andric 11450b57cec5SDimitry Andric CmpInst::Predicate Pred; 11460b57cec5SDimitry Andric switch (Imm) { 11470b57cec5SDimitry Andric case 0x0: 11480b57cec5SDimitry Andric Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; 11490b57cec5SDimitry Andric break; 11500b57cec5SDimitry Andric case 0x1: 11510b57cec5SDimitry Andric Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; 11520b57cec5SDimitry Andric break; 11530b57cec5SDimitry Andric case 0x2: 11540b57cec5SDimitry Andric Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; 11550b57cec5SDimitry Andric break; 11560b57cec5SDimitry Andric case 0x3: 11570b57cec5SDimitry Andric Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; 11580b57cec5SDimitry Andric break; 11590b57cec5SDimitry Andric case 0x4: 11600b57cec5SDimitry Andric Pred = ICmpInst::ICMP_EQ; 11610b57cec5SDimitry Andric break; 11620b57cec5SDimitry Andric case 0x5: 11630b57cec5SDimitry Andric Pred = ICmpInst::ICMP_NE; 11640b57cec5SDimitry Andric break; 11650b57cec5SDimitry Andric case 0x6: 11660b57cec5SDimitry Andric return Constant::getNullValue(Ty); // FALSE 11670b57cec5SDimitry Andric case 0x7: 11680b57cec5SDimitry Andric return Constant::getAllOnesValue(Ty); // TRUE 11690b57cec5SDimitry Andric default: 11700b57cec5SDimitry Andric llvm_unreachable("Unknown XOP vpcom/vpcomu predicate"); 11710b57cec5SDimitry Andric } 11720b57cec5SDimitry Andric 11730b57cec5SDimitry Andric Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS); 11740b57cec5SDimitry Andric Value *Ext = Builder.CreateSExt(Cmp, Ty); 11750b57cec5SDimitry Andric return Ext; 11760b57cec5SDimitry Andric } 11770b57cec5SDimitry Andric 11780b57cec5SDimitry Andric static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI, 11790b57cec5SDimitry Andric bool IsShiftRight, bool ZeroMask) { 11800b57cec5SDimitry Andric Type *Ty = CI.getType(); 11810b57cec5SDimitry Andric Value *Op0 = CI.getArgOperand(0); 11820b57cec5SDimitry Andric Value *Op1 = CI.getArgOperand(1); 11830b57cec5SDimitry Andric Value *Amt = CI.getArgOperand(2); 11840b57cec5SDimitry Andric 11850b57cec5SDimitry Andric if (IsShiftRight) 11860b57cec5SDimitry Andric std::swap(Op0, Op1); 11870b57cec5SDimitry Andric 11880b57cec5SDimitry Andric // Amount may be scalar immediate, in which case create a splat vector. 11890b57cec5SDimitry Andric // Funnel shifts amounts are treated as modulo and types are all power-of-2 so 11900b57cec5SDimitry Andric // we only care about the lowest log2 bits anyway. 11910b57cec5SDimitry Andric if (Amt->getType() != Ty) { 11920b57cec5SDimitry Andric unsigned NumElts = Ty->getVectorNumElements(); 11930b57cec5SDimitry Andric Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false); 11940b57cec5SDimitry Andric Amt = Builder.CreateVectorSplat(NumElts, Amt); 11950b57cec5SDimitry Andric } 11960b57cec5SDimitry Andric 11970b57cec5SDimitry Andric Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl; 11980b57cec5SDimitry Andric Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty); 11990b57cec5SDimitry Andric Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt}); 12000b57cec5SDimitry Andric 12010b57cec5SDimitry Andric unsigned NumArgs = CI.getNumArgOperands(); 12020b57cec5SDimitry Andric if (NumArgs >= 4) { // For masked intrinsics. 12030b57cec5SDimitry Andric Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) : 12040b57cec5SDimitry Andric ZeroMask ? ConstantAggregateZero::get(CI.getType()) : 12050b57cec5SDimitry Andric CI.getArgOperand(0); 12060b57cec5SDimitry Andric Value *Mask = CI.getOperand(NumArgs - 1); 12070b57cec5SDimitry Andric Res = EmitX86Select(Builder, Mask, Res, VecSrc); 12080b57cec5SDimitry Andric } 12090b57cec5SDimitry Andric return Res; 12100b57cec5SDimitry Andric } 12110b57cec5SDimitry Andric 12120b57cec5SDimitry Andric static Value *UpgradeMaskedStore(IRBuilder<> &Builder, 12130b57cec5SDimitry Andric Value *Ptr, Value *Data, Value *Mask, 12140b57cec5SDimitry Andric bool Aligned) { 12150b57cec5SDimitry Andric // Cast the pointer to the right type. 12160b57cec5SDimitry Andric Ptr = Builder.CreateBitCast(Ptr, 12170b57cec5SDimitry Andric llvm::PointerType::getUnqual(Data->getType())); 12180b57cec5SDimitry Andric unsigned Align = 12190b57cec5SDimitry Andric Aligned ? cast<VectorType>(Data->getType())->getBitWidth() / 8 : 1; 12200b57cec5SDimitry Andric 12210b57cec5SDimitry Andric // If the mask is all ones just emit a regular store. 12220b57cec5SDimitry Andric if (const auto *C = dyn_cast<Constant>(Mask)) 12230b57cec5SDimitry Andric if (C->isAllOnesValue()) 12240b57cec5SDimitry Andric return Builder.CreateAlignedStore(Data, Ptr, Align); 12250b57cec5SDimitry Andric 12260b57cec5SDimitry Andric // Convert the mask from an integer type to a vector of i1. 12270b57cec5SDimitry Andric unsigned NumElts = Data->getType()->getVectorNumElements(); 12280b57cec5SDimitry Andric Mask = getX86MaskVec(Builder, Mask, NumElts); 12290b57cec5SDimitry Andric return Builder.CreateMaskedStore(Data, Ptr, Align, Mask); 12300b57cec5SDimitry Andric } 12310b57cec5SDimitry Andric 12320b57cec5SDimitry Andric static Value *UpgradeMaskedLoad(IRBuilder<> &Builder, 12330b57cec5SDimitry Andric Value *Ptr, Value *Passthru, Value *Mask, 12340b57cec5SDimitry Andric bool Aligned) { 12350b57cec5SDimitry Andric Type *ValTy = Passthru->getType(); 12360b57cec5SDimitry Andric // Cast the pointer to the right type. 12370b57cec5SDimitry Andric Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy)); 12380b57cec5SDimitry Andric unsigned Align = 12390b57cec5SDimitry Andric Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1; 12400b57cec5SDimitry Andric 12410b57cec5SDimitry Andric // If the mask is all ones just emit a regular store. 12420b57cec5SDimitry Andric if (const auto *C = dyn_cast<Constant>(Mask)) 12430b57cec5SDimitry Andric if (C->isAllOnesValue()) 12440b57cec5SDimitry Andric return Builder.CreateAlignedLoad(ValTy, Ptr, Align); 12450b57cec5SDimitry Andric 12460b57cec5SDimitry Andric // Convert the mask from an integer type to a vector of i1. 12470b57cec5SDimitry Andric unsigned NumElts = Passthru->getType()->getVectorNumElements(); 12480b57cec5SDimitry Andric Mask = getX86MaskVec(Builder, Mask, NumElts); 12490b57cec5SDimitry Andric return Builder.CreateMaskedLoad(Ptr, Align, Mask, Passthru); 12500b57cec5SDimitry Andric } 12510b57cec5SDimitry Andric 12520b57cec5SDimitry Andric static Value *upgradeAbs(IRBuilder<> &Builder, CallInst &CI) { 12530b57cec5SDimitry Andric Value *Op0 = CI.getArgOperand(0); 12540b57cec5SDimitry Andric llvm::Type *Ty = Op0->getType(); 12550b57cec5SDimitry Andric Value *Zero = llvm::Constant::getNullValue(Ty); 12560b57cec5SDimitry Andric Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_SGT, Op0, Zero); 12570b57cec5SDimitry Andric Value *Neg = Builder.CreateNeg(Op0); 12580b57cec5SDimitry Andric Value *Res = Builder.CreateSelect(Cmp, Op0, Neg); 12590b57cec5SDimitry Andric 12600b57cec5SDimitry Andric if (CI.getNumArgOperands() == 3) 12610b57cec5SDimitry Andric Res = EmitX86Select(Builder,CI.getArgOperand(2), Res, CI.getArgOperand(1)); 12620b57cec5SDimitry Andric 12630b57cec5SDimitry Andric return Res; 12640b57cec5SDimitry Andric } 12650b57cec5SDimitry Andric 12660b57cec5SDimitry Andric static Value *upgradeIntMinMax(IRBuilder<> &Builder, CallInst &CI, 12670b57cec5SDimitry Andric ICmpInst::Predicate Pred) { 12680b57cec5SDimitry Andric Value *Op0 = CI.getArgOperand(0); 12690b57cec5SDimitry Andric Value *Op1 = CI.getArgOperand(1); 12700b57cec5SDimitry Andric Value *Cmp = Builder.CreateICmp(Pred, Op0, Op1); 12710b57cec5SDimitry Andric Value *Res = Builder.CreateSelect(Cmp, Op0, Op1); 12720b57cec5SDimitry Andric 12730b57cec5SDimitry Andric if (CI.getNumArgOperands() == 4) 12740b57cec5SDimitry Andric Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); 12750b57cec5SDimitry Andric 12760b57cec5SDimitry Andric return Res; 12770b57cec5SDimitry Andric } 12780b57cec5SDimitry Andric 12790b57cec5SDimitry Andric static Value *upgradePMULDQ(IRBuilder<> &Builder, CallInst &CI, bool IsSigned) { 12800b57cec5SDimitry Andric Type *Ty = CI.getType(); 12810b57cec5SDimitry Andric 12820b57cec5SDimitry Andric // Arguments have a vXi32 type so cast to vXi64. 12830b57cec5SDimitry Andric Value *LHS = Builder.CreateBitCast(CI.getArgOperand(0), Ty); 12840b57cec5SDimitry Andric Value *RHS = Builder.CreateBitCast(CI.getArgOperand(1), Ty); 12850b57cec5SDimitry Andric 12860b57cec5SDimitry Andric if (IsSigned) { 12870b57cec5SDimitry Andric // Shift left then arithmetic shift right. 12880b57cec5SDimitry Andric Constant *ShiftAmt = ConstantInt::get(Ty, 32); 12890b57cec5SDimitry Andric LHS = Builder.CreateShl(LHS, ShiftAmt); 12900b57cec5SDimitry Andric LHS = Builder.CreateAShr(LHS, ShiftAmt); 12910b57cec5SDimitry Andric RHS = Builder.CreateShl(RHS, ShiftAmt); 12920b57cec5SDimitry Andric RHS = Builder.CreateAShr(RHS, ShiftAmt); 12930b57cec5SDimitry Andric } else { 12940b57cec5SDimitry Andric // Clear the upper bits. 12950b57cec5SDimitry Andric Constant *Mask = ConstantInt::get(Ty, 0xffffffff); 12960b57cec5SDimitry Andric LHS = Builder.CreateAnd(LHS, Mask); 12970b57cec5SDimitry Andric RHS = Builder.CreateAnd(RHS, Mask); 12980b57cec5SDimitry Andric } 12990b57cec5SDimitry Andric 13000b57cec5SDimitry Andric Value *Res = Builder.CreateMul(LHS, RHS); 13010b57cec5SDimitry Andric 13020b57cec5SDimitry Andric if (CI.getNumArgOperands() == 4) 13030b57cec5SDimitry Andric Res = EmitX86Select(Builder, CI.getArgOperand(3), Res, CI.getArgOperand(2)); 13040b57cec5SDimitry Andric 13050b57cec5SDimitry Andric return Res; 13060b57cec5SDimitry Andric } 13070b57cec5SDimitry Andric 13080b57cec5SDimitry Andric // Applying mask on vector of i1's and make sure result is at least 8 bits wide. 13090b57cec5SDimitry Andric static Value *ApplyX86MaskOn1BitsVec(IRBuilder<> &Builder, Value *Vec, 13100b57cec5SDimitry Andric Value *Mask) { 13110b57cec5SDimitry Andric unsigned NumElts = Vec->getType()->getVectorNumElements(); 13120b57cec5SDimitry Andric if (Mask) { 13130b57cec5SDimitry Andric const auto *C = dyn_cast<Constant>(Mask); 13140b57cec5SDimitry Andric if (!C || !C->isAllOnesValue()) 13150b57cec5SDimitry Andric Vec = Builder.CreateAnd(Vec, getX86MaskVec(Builder, Mask, NumElts)); 13160b57cec5SDimitry Andric } 13170b57cec5SDimitry Andric 13180b57cec5SDimitry Andric if (NumElts < 8) { 13190b57cec5SDimitry Andric uint32_t Indices[8]; 13200b57cec5SDimitry Andric for (unsigned i = 0; i != NumElts; ++i) 13210b57cec5SDimitry Andric Indices[i] = i; 13220b57cec5SDimitry Andric for (unsigned i = NumElts; i != 8; ++i) 13230b57cec5SDimitry Andric Indices[i] = NumElts + i % NumElts; 13240b57cec5SDimitry Andric Vec = Builder.CreateShuffleVector(Vec, 13250b57cec5SDimitry Andric Constant::getNullValue(Vec->getType()), 13260b57cec5SDimitry Andric Indices); 13270b57cec5SDimitry Andric } 13280b57cec5SDimitry Andric return Builder.CreateBitCast(Vec, Builder.getIntNTy(std::max(NumElts, 8U))); 13290b57cec5SDimitry Andric } 13300b57cec5SDimitry Andric 13310b57cec5SDimitry Andric static Value *upgradeMaskedCompare(IRBuilder<> &Builder, CallInst &CI, 13320b57cec5SDimitry Andric unsigned CC, bool Signed) { 13330b57cec5SDimitry Andric Value *Op0 = CI.getArgOperand(0); 13340b57cec5SDimitry Andric unsigned NumElts = Op0->getType()->getVectorNumElements(); 13350b57cec5SDimitry Andric 13360b57cec5SDimitry Andric Value *Cmp; 13370b57cec5SDimitry Andric if (CC == 3) { 13380b57cec5SDimitry Andric Cmp = Constant::getNullValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts)); 13390b57cec5SDimitry Andric } else if (CC == 7) { 13400b57cec5SDimitry Andric Cmp = Constant::getAllOnesValue(llvm::VectorType::get(Builder.getInt1Ty(), NumElts)); 13410b57cec5SDimitry Andric } else { 13420b57cec5SDimitry Andric ICmpInst::Predicate Pred; 13430b57cec5SDimitry Andric switch (CC) { 13440b57cec5SDimitry Andric default: llvm_unreachable("Unknown condition code"); 13450b57cec5SDimitry Andric case 0: Pred = ICmpInst::ICMP_EQ; break; 13460b57cec5SDimitry Andric case 1: Pred = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT; break; 13470b57cec5SDimitry Andric case 2: Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; break; 13480b57cec5SDimitry Andric case 4: Pred = ICmpInst::ICMP_NE; break; 13490b57cec5SDimitry Andric case 5: Pred = Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; break; 13500b57cec5SDimitry Andric case 6: Pred = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT; break; 13510b57cec5SDimitry Andric } 13520b57cec5SDimitry Andric Cmp = Builder.CreateICmp(Pred, Op0, CI.getArgOperand(1)); 13530b57cec5SDimitry Andric } 13540b57cec5SDimitry Andric 13550b57cec5SDimitry Andric Value *Mask = CI.getArgOperand(CI.getNumArgOperands() - 1); 13560b57cec5SDimitry Andric 13570b57cec5SDimitry Andric return ApplyX86MaskOn1BitsVec(Builder, Cmp, Mask); 13580b57cec5SDimitry Andric } 13590b57cec5SDimitry Andric 13600b57cec5SDimitry Andric // Replace a masked intrinsic with an older unmasked intrinsic. 13610b57cec5SDimitry Andric static Value *UpgradeX86MaskedShift(IRBuilder<> &Builder, CallInst &CI, 13620b57cec5SDimitry Andric Intrinsic::ID IID) { 13630b57cec5SDimitry Andric Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID); 13640b57cec5SDimitry Andric Value *Rep = Builder.CreateCall(Intrin, 13650b57cec5SDimitry Andric { CI.getArgOperand(0), CI.getArgOperand(1) }); 13660b57cec5SDimitry Andric return EmitX86Select(Builder, CI.getArgOperand(3), Rep, CI.getArgOperand(2)); 13670b57cec5SDimitry Andric } 13680b57cec5SDimitry Andric 13690b57cec5SDimitry Andric static Value* upgradeMaskedMove(IRBuilder<> &Builder, CallInst &CI) { 13700b57cec5SDimitry Andric Value* A = CI.getArgOperand(0); 13710b57cec5SDimitry Andric Value* B = CI.getArgOperand(1); 13720b57cec5SDimitry Andric Value* Src = CI.getArgOperand(2); 13730b57cec5SDimitry Andric Value* Mask = CI.getArgOperand(3); 13740b57cec5SDimitry Andric 13750b57cec5SDimitry Andric Value* AndNode = Builder.CreateAnd(Mask, APInt(8, 1)); 13760b57cec5SDimitry Andric Value* Cmp = Builder.CreateIsNotNull(AndNode); 13770b57cec5SDimitry Andric Value* Extract1 = Builder.CreateExtractElement(B, (uint64_t)0); 13780b57cec5SDimitry Andric Value* Extract2 = Builder.CreateExtractElement(Src, (uint64_t)0); 13790b57cec5SDimitry Andric Value* Select = Builder.CreateSelect(Cmp, Extract1, Extract2); 13800b57cec5SDimitry Andric return Builder.CreateInsertElement(A, Select, (uint64_t)0); 13810b57cec5SDimitry Andric } 13820b57cec5SDimitry Andric 13830b57cec5SDimitry Andric 13840b57cec5SDimitry Andric static Value* UpgradeMaskToInt(IRBuilder<> &Builder, CallInst &CI) { 13850b57cec5SDimitry Andric Value* Op = CI.getArgOperand(0); 13860b57cec5SDimitry Andric Type* ReturnOp = CI.getType(); 13870b57cec5SDimitry Andric unsigned NumElts = CI.getType()->getVectorNumElements(); 13880b57cec5SDimitry Andric Value *Mask = getX86MaskVec(Builder, Op, NumElts); 13890b57cec5SDimitry Andric return Builder.CreateSExt(Mask, ReturnOp, "vpmovm2"); 13900b57cec5SDimitry Andric } 13910b57cec5SDimitry Andric 13920b57cec5SDimitry Andric // Replace intrinsic with unmasked version and a select. 13930b57cec5SDimitry Andric static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder, 13940b57cec5SDimitry Andric CallInst &CI, Value *&Rep) { 13950b57cec5SDimitry Andric Name = Name.substr(12); // Remove avx512.mask. 13960b57cec5SDimitry Andric 13970b57cec5SDimitry Andric unsigned VecWidth = CI.getType()->getPrimitiveSizeInBits(); 13980b57cec5SDimitry Andric unsigned EltWidth = CI.getType()->getScalarSizeInBits(); 13990b57cec5SDimitry Andric Intrinsic::ID IID; 14000b57cec5SDimitry Andric if (Name.startswith("max.p")) { 14010b57cec5SDimitry Andric if (VecWidth == 128 && EltWidth == 32) 14020b57cec5SDimitry Andric IID = Intrinsic::x86_sse_max_ps; 14030b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 64) 14040b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_max_pd; 14050b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32) 14060b57cec5SDimitry Andric IID = Intrinsic::x86_avx_max_ps_256; 14070b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64) 14080b57cec5SDimitry Andric IID = Intrinsic::x86_avx_max_pd_256; 14090b57cec5SDimitry Andric else 14100b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14110b57cec5SDimitry Andric } else if (Name.startswith("min.p")) { 14120b57cec5SDimitry Andric if (VecWidth == 128 && EltWidth == 32) 14130b57cec5SDimitry Andric IID = Intrinsic::x86_sse_min_ps; 14140b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 64) 14150b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_min_pd; 14160b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32) 14170b57cec5SDimitry Andric IID = Intrinsic::x86_avx_min_ps_256; 14180b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64) 14190b57cec5SDimitry Andric IID = Intrinsic::x86_avx_min_pd_256; 14200b57cec5SDimitry Andric else 14210b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14220b57cec5SDimitry Andric } else if (Name.startswith("pshuf.b.")) { 14230b57cec5SDimitry Andric if (VecWidth == 128) 14240b57cec5SDimitry Andric IID = Intrinsic::x86_ssse3_pshuf_b_128; 14250b57cec5SDimitry Andric else if (VecWidth == 256) 14260b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_pshuf_b; 14270b57cec5SDimitry Andric else if (VecWidth == 512) 14280b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pshuf_b_512; 14290b57cec5SDimitry Andric else 14300b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14310b57cec5SDimitry Andric } else if (Name.startswith("pmul.hr.sw.")) { 14320b57cec5SDimitry Andric if (VecWidth == 128) 14330b57cec5SDimitry Andric IID = Intrinsic::x86_ssse3_pmul_hr_sw_128; 14340b57cec5SDimitry Andric else if (VecWidth == 256) 14350b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_pmul_hr_sw; 14360b57cec5SDimitry Andric else if (VecWidth == 512) 14370b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pmul_hr_sw_512; 14380b57cec5SDimitry Andric else 14390b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14400b57cec5SDimitry Andric } else if (Name.startswith("pmulh.w.")) { 14410b57cec5SDimitry Andric if (VecWidth == 128) 14420b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_pmulh_w; 14430b57cec5SDimitry Andric else if (VecWidth == 256) 14440b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_pmulh_w; 14450b57cec5SDimitry Andric else if (VecWidth == 512) 14460b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pmulh_w_512; 14470b57cec5SDimitry Andric else 14480b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14490b57cec5SDimitry Andric } else if (Name.startswith("pmulhu.w.")) { 14500b57cec5SDimitry Andric if (VecWidth == 128) 14510b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_pmulhu_w; 14520b57cec5SDimitry Andric else if (VecWidth == 256) 14530b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_pmulhu_w; 14540b57cec5SDimitry Andric else if (VecWidth == 512) 14550b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pmulhu_w_512; 14560b57cec5SDimitry Andric else 14570b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14580b57cec5SDimitry Andric } else if (Name.startswith("pmaddw.d.")) { 14590b57cec5SDimitry Andric if (VecWidth == 128) 14600b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_pmadd_wd; 14610b57cec5SDimitry Andric else if (VecWidth == 256) 14620b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_pmadd_wd; 14630b57cec5SDimitry Andric else if (VecWidth == 512) 14640b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pmaddw_d_512; 14650b57cec5SDimitry Andric else 14660b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14670b57cec5SDimitry Andric } else if (Name.startswith("pmaddubs.w.")) { 14680b57cec5SDimitry Andric if (VecWidth == 128) 14690b57cec5SDimitry Andric IID = Intrinsic::x86_ssse3_pmadd_ub_sw_128; 14700b57cec5SDimitry Andric else if (VecWidth == 256) 14710b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_pmadd_ub_sw; 14720b57cec5SDimitry Andric else if (VecWidth == 512) 14730b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pmaddubs_w_512; 14740b57cec5SDimitry Andric else 14750b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14760b57cec5SDimitry Andric } else if (Name.startswith("packsswb.")) { 14770b57cec5SDimitry Andric if (VecWidth == 128) 14780b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_packsswb_128; 14790b57cec5SDimitry Andric else if (VecWidth == 256) 14800b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_packsswb; 14810b57cec5SDimitry Andric else if (VecWidth == 512) 14820b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_packsswb_512; 14830b57cec5SDimitry Andric else 14840b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14850b57cec5SDimitry Andric } else if (Name.startswith("packssdw.")) { 14860b57cec5SDimitry Andric if (VecWidth == 128) 14870b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_packssdw_128; 14880b57cec5SDimitry Andric else if (VecWidth == 256) 14890b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_packssdw; 14900b57cec5SDimitry Andric else if (VecWidth == 512) 14910b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_packssdw_512; 14920b57cec5SDimitry Andric else 14930b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 14940b57cec5SDimitry Andric } else if (Name.startswith("packuswb.")) { 14950b57cec5SDimitry Andric if (VecWidth == 128) 14960b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_packuswb_128; 14970b57cec5SDimitry Andric else if (VecWidth == 256) 14980b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_packuswb; 14990b57cec5SDimitry Andric else if (VecWidth == 512) 15000b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_packuswb_512; 15010b57cec5SDimitry Andric else 15020b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 15030b57cec5SDimitry Andric } else if (Name.startswith("packusdw.")) { 15040b57cec5SDimitry Andric if (VecWidth == 128) 15050b57cec5SDimitry Andric IID = Intrinsic::x86_sse41_packusdw; 15060b57cec5SDimitry Andric else if (VecWidth == 256) 15070b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_packusdw; 15080b57cec5SDimitry Andric else if (VecWidth == 512) 15090b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_packusdw_512; 15100b57cec5SDimitry Andric else 15110b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 15120b57cec5SDimitry Andric } else if (Name.startswith("vpermilvar.")) { 15130b57cec5SDimitry Andric if (VecWidth == 128 && EltWidth == 32) 15140b57cec5SDimitry Andric IID = Intrinsic::x86_avx_vpermilvar_ps; 15150b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 64) 15160b57cec5SDimitry Andric IID = Intrinsic::x86_avx_vpermilvar_pd; 15170b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32) 15180b57cec5SDimitry Andric IID = Intrinsic::x86_avx_vpermilvar_ps_256; 15190b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64) 15200b57cec5SDimitry Andric IID = Intrinsic::x86_avx_vpermilvar_pd_256; 15210b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 32) 15220b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermilvar_ps_512; 15230b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 64) 15240b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpermilvar_pd_512; 15250b57cec5SDimitry Andric else 15260b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 15270b57cec5SDimitry Andric } else if (Name == "cvtpd2dq.256") { 15280b57cec5SDimitry Andric IID = Intrinsic::x86_avx_cvt_pd2dq_256; 15290b57cec5SDimitry Andric } else if (Name == "cvtpd2ps.256") { 15300b57cec5SDimitry Andric IID = Intrinsic::x86_avx_cvt_pd2_ps_256; 15310b57cec5SDimitry Andric } else if (Name == "cvttpd2dq.256") { 15320b57cec5SDimitry Andric IID = Intrinsic::x86_avx_cvtt_pd2dq_256; 15330b57cec5SDimitry Andric } else if (Name == "cvttps2dq.128") { 15340b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_cvttps2dq; 15350b57cec5SDimitry Andric } else if (Name == "cvttps2dq.256") { 15360b57cec5SDimitry Andric IID = Intrinsic::x86_avx_cvtt_ps2dq_256; 15370b57cec5SDimitry Andric } else if (Name.startswith("permvar.")) { 15380b57cec5SDimitry Andric bool IsFloat = CI.getType()->isFPOrFPVectorTy(); 15390b57cec5SDimitry Andric if (VecWidth == 256 && EltWidth == 32 && IsFloat) 15400b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_permps; 15410b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32 && !IsFloat) 15420b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_permd; 15430b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64 && IsFloat) 15440b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_df_256; 15450b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64 && !IsFloat) 15460b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_di_256; 15470b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 32 && IsFloat) 15480b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_sf_512; 15490b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 32 && !IsFloat) 15500b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_si_512; 15510b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 64 && IsFloat) 15520b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_df_512; 15530b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 64 && !IsFloat) 15540b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_di_512; 15550b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 16) 15560b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_hi_128; 15570b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 16) 15580b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_hi_256; 15590b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 16) 15600b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_hi_512; 15610b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 8) 15620b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_qi_128; 15630b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 8) 15640b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_qi_256; 15650b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 8) 15660b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_permvar_qi_512; 15670b57cec5SDimitry Andric else 15680b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 15690b57cec5SDimitry Andric } else if (Name.startswith("dbpsadbw.")) { 15700b57cec5SDimitry Andric if (VecWidth == 128) 15710b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_dbpsadbw_128; 15720b57cec5SDimitry Andric else if (VecWidth == 256) 15730b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_dbpsadbw_256; 15740b57cec5SDimitry Andric else if (VecWidth == 512) 15750b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_dbpsadbw_512; 15760b57cec5SDimitry Andric else 15770b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 15780b57cec5SDimitry Andric } else if (Name.startswith("pmultishift.qb.")) { 15790b57cec5SDimitry Andric if (VecWidth == 128) 15800b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pmultishift_qb_128; 15810b57cec5SDimitry Andric else if (VecWidth == 256) 15820b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pmultishift_qb_256; 15830b57cec5SDimitry Andric else if (VecWidth == 512) 15840b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pmultishift_qb_512; 15850b57cec5SDimitry Andric else 15860b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 15870b57cec5SDimitry Andric } else if (Name.startswith("conflict.")) { 15880b57cec5SDimitry Andric if (Name[9] == 'd' && VecWidth == 128) 15890b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_conflict_d_128; 15900b57cec5SDimitry Andric else if (Name[9] == 'd' && VecWidth == 256) 15910b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_conflict_d_256; 15920b57cec5SDimitry Andric else if (Name[9] == 'd' && VecWidth == 512) 15930b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_conflict_d_512; 15940b57cec5SDimitry Andric else if (Name[9] == 'q' && VecWidth == 128) 15950b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_conflict_q_128; 15960b57cec5SDimitry Andric else if (Name[9] == 'q' && VecWidth == 256) 15970b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_conflict_q_256; 15980b57cec5SDimitry Andric else if (Name[9] == 'q' && VecWidth == 512) 15990b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_conflict_q_512; 16000b57cec5SDimitry Andric else 16010b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 16020b57cec5SDimitry Andric } else if (Name.startswith("pavg.")) { 16030b57cec5SDimitry Andric if (Name[5] == 'b' && VecWidth == 128) 16040b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_pavg_b; 16050b57cec5SDimitry Andric else if (Name[5] == 'b' && VecWidth == 256) 16060b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_pavg_b; 16070b57cec5SDimitry Andric else if (Name[5] == 'b' && VecWidth == 512) 16080b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pavg_b_512; 16090b57cec5SDimitry Andric else if (Name[5] == 'w' && VecWidth == 128) 16100b57cec5SDimitry Andric IID = Intrinsic::x86_sse2_pavg_w; 16110b57cec5SDimitry Andric else if (Name[5] == 'w' && VecWidth == 256) 16120b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_pavg_w; 16130b57cec5SDimitry Andric else if (Name[5] == 'w' && VecWidth == 512) 16140b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pavg_w_512; 16150b57cec5SDimitry Andric else 16160b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 16170b57cec5SDimitry Andric } else 16180b57cec5SDimitry Andric return false; 16190b57cec5SDimitry Andric 16200b57cec5SDimitry Andric SmallVector<Value *, 4> Args(CI.arg_operands().begin(), 16210b57cec5SDimitry Andric CI.arg_operands().end()); 16220b57cec5SDimitry Andric Args.pop_back(); 16230b57cec5SDimitry Andric Args.pop_back(); 16240b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID), 16250b57cec5SDimitry Andric Args); 16260b57cec5SDimitry Andric unsigned NumArgs = CI.getNumArgOperands(); 16270b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI.getArgOperand(NumArgs - 1), Rep, 16280b57cec5SDimitry Andric CI.getArgOperand(NumArgs - 2)); 16290b57cec5SDimitry Andric return true; 16300b57cec5SDimitry Andric } 16310b57cec5SDimitry Andric 16320b57cec5SDimitry Andric /// Upgrade comment in call to inline asm that represents an objc retain release 16330b57cec5SDimitry Andric /// marker. 16340b57cec5SDimitry Andric void llvm::UpgradeInlineAsmString(std::string *AsmStr) { 16350b57cec5SDimitry Andric size_t Pos; 16360b57cec5SDimitry Andric if (AsmStr->find("mov\tfp") == 0 && 16370b57cec5SDimitry Andric AsmStr->find("objc_retainAutoreleaseReturnValue") != std::string::npos && 16380b57cec5SDimitry Andric (Pos = AsmStr->find("# marker")) != std::string::npos) { 16390b57cec5SDimitry Andric AsmStr->replace(Pos, 1, ";"); 16400b57cec5SDimitry Andric } 16410b57cec5SDimitry Andric return; 16420b57cec5SDimitry Andric } 16430b57cec5SDimitry Andric 16440b57cec5SDimitry Andric /// Upgrade a call to an old intrinsic. All argument and return casting must be 16450b57cec5SDimitry Andric /// provided to seamlessly integrate with existing context. 16460b57cec5SDimitry Andric void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) { 16470b57cec5SDimitry Andric Function *F = CI->getCalledFunction(); 16480b57cec5SDimitry Andric LLVMContext &C = CI->getContext(); 16490b57cec5SDimitry Andric IRBuilder<> Builder(C); 16500b57cec5SDimitry Andric Builder.SetInsertPoint(CI->getParent(), CI->getIterator()); 16510b57cec5SDimitry Andric 16520b57cec5SDimitry Andric assert(F && "Intrinsic call is not direct?"); 16530b57cec5SDimitry Andric 16540b57cec5SDimitry Andric if (!NewFn) { 16550b57cec5SDimitry Andric // Get the Function's name. 16560b57cec5SDimitry Andric StringRef Name = F->getName(); 16570b57cec5SDimitry Andric 16580b57cec5SDimitry Andric assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'"); 16590b57cec5SDimitry Andric Name = Name.substr(5); 16600b57cec5SDimitry Andric 16610b57cec5SDimitry Andric bool IsX86 = Name.startswith("x86."); 16620b57cec5SDimitry Andric if (IsX86) 16630b57cec5SDimitry Andric Name = Name.substr(4); 16640b57cec5SDimitry Andric bool IsNVVM = Name.startswith("nvvm."); 16650b57cec5SDimitry Andric if (IsNVVM) 16660b57cec5SDimitry Andric Name = Name.substr(5); 16670b57cec5SDimitry Andric 16680b57cec5SDimitry Andric if (IsX86 && Name.startswith("sse4a.movnt.")) { 16690b57cec5SDimitry Andric Module *M = F->getParent(); 16700b57cec5SDimitry Andric SmallVector<Metadata *, 1> Elts; 16710b57cec5SDimitry Andric Elts.push_back( 16720b57cec5SDimitry Andric ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); 16730b57cec5SDimitry Andric MDNode *Node = MDNode::get(C, Elts); 16740b57cec5SDimitry Andric 16750b57cec5SDimitry Andric Value *Arg0 = CI->getArgOperand(0); 16760b57cec5SDimitry Andric Value *Arg1 = CI->getArgOperand(1); 16770b57cec5SDimitry Andric 16780b57cec5SDimitry Andric // Nontemporal (unaligned) store of the 0'th element of the float/double 16790b57cec5SDimitry Andric // vector. 16800b57cec5SDimitry Andric Type *SrcEltTy = cast<VectorType>(Arg1->getType())->getElementType(); 16810b57cec5SDimitry Andric PointerType *EltPtrTy = PointerType::getUnqual(SrcEltTy); 16820b57cec5SDimitry Andric Value *Addr = Builder.CreateBitCast(Arg0, EltPtrTy, "cast"); 16830b57cec5SDimitry Andric Value *Extract = 16840b57cec5SDimitry Andric Builder.CreateExtractElement(Arg1, (uint64_t)0, "extractelement"); 16850b57cec5SDimitry Andric 16860b57cec5SDimitry Andric StoreInst *SI = Builder.CreateAlignedStore(Extract, Addr, 1); 16870b57cec5SDimitry Andric SI->setMetadata(M->getMDKindID("nontemporal"), Node); 16880b57cec5SDimitry Andric 16890b57cec5SDimitry Andric // Remove intrinsic. 16900b57cec5SDimitry Andric CI->eraseFromParent(); 16910b57cec5SDimitry Andric return; 16920b57cec5SDimitry Andric } 16930b57cec5SDimitry Andric 16940b57cec5SDimitry Andric if (IsX86 && (Name.startswith("avx.movnt.") || 16950b57cec5SDimitry Andric Name.startswith("avx512.storent."))) { 16960b57cec5SDimitry Andric Module *M = F->getParent(); 16970b57cec5SDimitry Andric SmallVector<Metadata *, 1> Elts; 16980b57cec5SDimitry Andric Elts.push_back( 16990b57cec5SDimitry Andric ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); 17000b57cec5SDimitry Andric MDNode *Node = MDNode::get(C, Elts); 17010b57cec5SDimitry Andric 17020b57cec5SDimitry Andric Value *Arg0 = CI->getArgOperand(0); 17030b57cec5SDimitry Andric Value *Arg1 = CI->getArgOperand(1); 17040b57cec5SDimitry Andric 17050b57cec5SDimitry Andric // Convert the type of the pointer to a pointer to the stored type. 17060b57cec5SDimitry Andric Value *BC = Builder.CreateBitCast(Arg0, 17070b57cec5SDimitry Andric PointerType::getUnqual(Arg1->getType()), 17080b57cec5SDimitry Andric "cast"); 17090b57cec5SDimitry Andric VectorType *VTy = cast<VectorType>(Arg1->getType()); 17100b57cec5SDimitry Andric StoreInst *SI = Builder.CreateAlignedStore(Arg1, BC, 17110b57cec5SDimitry Andric VTy->getBitWidth() / 8); 17120b57cec5SDimitry Andric SI->setMetadata(M->getMDKindID("nontemporal"), Node); 17130b57cec5SDimitry Andric 17140b57cec5SDimitry Andric // Remove intrinsic. 17150b57cec5SDimitry Andric CI->eraseFromParent(); 17160b57cec5SDimitry Andric return; 17170b57cec5SDimitry Andric } 17180b57cec5SDimitry Andric 17190b57cec5SDimitry Andric if (IsX86 && Name == "sse2.storel.dq") { 17200b57cec5SDimitry Andric Value *Arg0 = CI->getArgOperand(0); 17210b57cec5SDimitry Andric Value *Arg1 = CI->getArgOperand(1); 17220b57cec5SDimitry Andric 17230b57cec5SDimitry Andric Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2); 17240b57cec5SDimitry Andric Value *BC0 = Builder.CreateBitCast(Arg1, NewVecTy, "cast"); 17250b57cec5SDimitry Andric Value *Elt = Builder.CreateExtractElement(BC0, (uint64_t)0); 17260b57cec5SDimitry Andric Value *BC = Builder.CreateBitCast(Arg0, 17270b57cec5SDimitry Andric PointerType::getUnqual(Elt->getType()), 17280b57cec5SDimitry Andric "cast"); 17290b57cec5SDimitry Andric Builder.CreateAlignedStore(Elt, BC, 1); 17300b57cec5SDimitry Andric 17310b57cec5SDimitry Andric // Remove intrinsic. 17320b57cec5SDimitry Andric CI->eraseFromParent(); 17330b57cec5SDimitry Andric return; 17340b57cec5SDimitry Andric } 17350b57cec5SDimitry Andric 17360b57cec5SDimitry Andric if (IsX86 && (Name.startswith("sse.storeu.") || 17370b57cec5SDimitry Andric Name.startswith("sse2.storeu.") || 17380b57cec5SDimitry Andric Name.startswith("avx.storeu."))) { 17390b57cec5SDimitry Andric Value *Arg0 = CI->getArgOperand(0); 17400b57cec5SDimitry Andric Value *Arg1 = CI->getArgOperand(1); 17410b57cec5SDimitry Andric 17420b57cec5SDimitry Andric Arg0 = Builder.CreateBitCast(Arg0, 17430b57cec5SDimitry Andric PointerType::getUnqual(Arg1->getType()), 17440b57cec5SDimitry Andric "cast"); 17450b57cec5SDimitry Andric Builder.CreateAlignedStore(Arg1, Arg0, 1); 17460b57cec5SDimitry Andric 17470b57cec5SDimitry Andric // Remove intrinsic. 17480b57cec5SDimitry Andric CI->eraseFromParent(); 17490b57cec5SDimitry Andric return; 17500b57cec5SDimitry Andric } 17510b57cec5SDimitry Andric 17520b57cec5SDimitry Andric if (IsX86 && Name == "avx512.mask.store.ss") { 17530b57cec5SDimitry Andric Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1)); 17540b57cec5SDimitry Andric UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), 17550b57cec5SDimitry Andric Mask, false); 17560b57cec5SDimitry Andric 17570b57cec5SDimitry Andric // Remove intrinsic. 17580b57cec5SDimitry Andric CI->eraseFromParent(); 17590b57cec5SDimitry Andric return; 17600b57cec5SDimitry Andric } 17610b57cec5SDimitry Andric 17620b57cec5SDimitry Andric if (IsX86 && (Name.startswith("avx512.mask.store"))) { 17630b57cec5SDimitry Andric // "avx512.mask.storeu." or "avx512.mask.store." 17640b57cec5SDimitry Andric bool Aligned = Name[17] != 'u'; // "avx512.mask.storeu". 17650b57cec5SDimitry Andric UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1), 17660b57cec5SDimitry Andric CI->getArgOperand(2), Aligned); 17670b57cec5SDimitry Andric 17680b57cec5SDimitry Andric // Remove intrinsic. 17690b57cec5SDimitry Andric CI->eraseFromParent(); 17700b57cec5SDimitry Andric return; 17710b57cec5SDimitry Andric } 17720b57cec5SDimitry Andric 17730b57cec5SDimitry Andric Value *Rep; 17740b57cec5SDimitry Andric // Upgrade packed integer vector compare intrinsics to compare instructions. 17750b57cec5SDimitry Andric if (IsX86 && (Name.startswith("sse2.pcmp") || 17760b57cec5SDimitry Andric Name.startswith("avx2.pcmp"))) { 17770b57cec5SDimitry Andric // "sse2.pcpmpeq." "sse2.pcmpgt." "avx2.pcmpeq." or "avx2.pcmpgt." 17780b57cec5SDimitry Andric bool CmpEq = Name[9] == 'e'; 17790b57cec5SDimitry Andric Rep = Builder.CreateICmp(CmpEq ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_SGT, 17800b57cec5SDimitry Andric CI->getArgOperand(0), CI->getArgOperand(1)); 17810b57cec5SDimitry Andric Rep = Builder.CreateSExt(Rep, CI->getType(), ""); 17820b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.broadcastm"))) { 17830b57cec5SDimitry Andric Type *ExtTy = Type::getInt32Ty(C); 17840b57cec5SDimitry Andric if (CI->getOperand(0)->getType()->isIntegerTy(8)) 17850b57cec5SDimitry Andric ExtTy = Type::getInt64Ty(C); 17860b57cec5SDimitry Andric unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() / 17870b57cec5SDimitry Andric ExtTy->getPrimitiveSizeInBits(); 17880b57cec5SDimitry Andric Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy); 17890b57cec5SDimitry Andric Rep = Builder.CreateVectorSplat(NumElts, Rep); 17900b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse.sqrt.ss" || 17910b57cec5SDimitry Andric Name == "sse2.sqrt.sd")) { 17920b57cec5SDimitry Andric Value *Vec = CI->getArgOperand(0); 17930b57cec5SDimitry Andric Value *Elt0 = Builder.CreateExtractElement(Vec, (uint64_t)0); 17940b57cec5SDimitry Andric Function *Intr = Intrinsic::getDeclaration(F->getParent(), 17950b57cec5SDimitry Andric Intrinsic::sqrt, Elt0->getType()); 17960b57cec5SDimitry Andric Elt0 = Builder.CreateCall(Intr, Elt0); 17970b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(Vec, Elt0, (uint64_t)0); 17980b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx.sqrt.p") || 17990b57cec5SDimitry Andric Name.startswith("sse2.sqrt.p") || 18000b57cec5SDimitry Andric Name.startswith("sse.sqrt.p"))) { 18010b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), 18020b57cec5SDimitry Andric Intrinsic::sqrt, 18030b57cec5SDimitry Andric CI->getType()), 18040b57cec5SDimitry Andric {CI->getArgOperand(0)}); 18050b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.sqrt.p"))) { 18060b57cec5SDimitry Andric if (CI->getNumArgOperands() == 4 && 18070b57cec5SDimitry Andric (!isa<ConstantInt>(CI->getArgOperand(3)) || 18080b57cec5SDimitry Andric cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { 18090b57cec5SDimitry Andric Intrinsic::ID IID = Name[18] == 's' ? Intrinsic::x86_avx512_sqrt_ps_512 18100b57cec5SDimitry Andric : Intrinsic::x86_avx512_sqrt_pd_512; 18110b57cec5SDimitry Andric 18120b57cec5SDimitry Andric Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) }; 18130b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), 18140b57cec5SDimitry Andric IID), Args); 18150b57cec5SDimitry Andric } else { 18160b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), 18170b57cec5SDimitry Andric Intrinsic::sqrt, 18180b57cec5SDimitry Andric CI->getType()), 18190b57cec5SDimitry Andric {CI->getArgOperand(0)}); 18200b57cec5SDimitry Andric } 18210b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 18220b57cec5SDimitry Andric CI->getArgOperand(1)); 18230b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.ptestm") || 18240b57cec5SDimitry Andric Name.startswith("avx512.ptestnm"))) { 18250b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 18260b57cec5SDimitry Andric Value *Op1 = CI->getArgOperand(1); 18270b57cec5SDimitry Andric Value *Mask = CI->getArgOperand(2); 18280b57cec5SDimitry Andric Rep = Builder.CreateAnd(Op0, Op1); 18290b57cec5SDimitry Andric llvm::Type *Ty = Op0->getType(); 18300b57cec5SDimitry Andric Value *Zero = llvm::Constant::getNullValue(Ty); 18310b57cec5SDimitry Andric ICmpInst::Predicate Pred = 18320b57cec5SDimitry Andric Name.startswith("avx512.ptestm") ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ; 18330b57cec5SDimitry Andric Rep = Builder.CreateICmp(Pred, Rep, Zero); 18340b57cec5SDimitry Andric Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, Mask); 18350b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.pbroadcast"))){ 18360b57cec5SDimitry Andric unsigned NumElts = 18370b57cec5SDimitry Andric CI->getArgOperand(1)->getType()->getVectorNumElements(); 18380b57cec5SDimitry Andric Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0)); 18390b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 18400b57cec5SDimitry Andric CI->getArgOperand(1)); 18410b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.kunpck"))) { 18420b57cec5SDimitry Andric unsigned NumElts = CI->getType()->getScalarSizeInBits(); 18430b57cec5SDimitry Andric Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts); 18440b57cec5SDimitry Andric Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts); 18450b57cec5SDimitry Andric uint32_t Indices[64]; 18460b57cec5SDimitry Andric for (unsigned i = 0; i != NumElts; ++i) 18470b57cec5SDimitry Andric Indices[i] = i; 18480b57cec5SDimitry Andric 18490b57cec5SDimitry Andric // First extract half of each vector. This gives better codegen than 18500b57cec5SDimitry Andric // doing it in a single shuffle. 18510b57cec5SDimitry Andric LHS = Builder.CreateShuffleVector(LHS, LHS, 18520b57cec5SDimitry Andric makeArrayRef(Indices, NumElts / 2)); 18530b57cec5SDimitry Andric RHS = Builder.CreateShuffleVector(RHS, RHS, 18540b57cec5SDimitry Andric makeArrayRef(Indices, NumElts / 2)); 18550b57cec5SDimitry Andric // Concat the vectors. 18560b57cec5SDimitry Andric // NOTE: Operands have to be swapped to match intrinsic definition. 18570b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(RHS, LHS, 18580b57cec5SDimitry Andric makeArrayRef(Indices, NumElts)); 18590b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, CI->getType()); 18600b57cec5SDimitry Andric } else if (IsX86 && Name == "avx512.kand.w") { 18610b57cec5SDimitry Andric Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); 18620b57cec5SDimitry Andric Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); 18630b57cec5SDimitry Andric Rep = Builder.CreateAnd(LHS, RHS); 18640b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, CI->getType()); 18650b57cec5SDimitry Andric } else if (IsX86 && Name == "avx512.kandn.w") { 18660b57cec5SDimitry Andric Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); 18670b57cec5SDimitry Andric Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); 18680b57cec5SDimitry Andric LHS = Builder.CreateNot(LHS); 18690b57cec5SDimitry Andric Rep = Builder.CreateAnd(LHS, RHS); 18700b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, CI->getType()); 18710b57cec5SDimitry Andric } else if (IsX86 && Name == "avx512.kor.w") { 18720b57cec5SDimitry Andric Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); 18730b57cec5SDimitry Andric Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); 18740b57cec5SDimitry Andric Rep = Builder.CreateOr(LHS, RHS); 18750b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, CI->getType()); 18760b57cec5SDimitry Andric } else if (IsX86 && Name == "avx512.kxor.w") { 18770b57cec5SDimitry Andric Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); 18780b57cec5SDimitry Andric Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); 18790b57cec5SDimitry Andric Rep = Builder.CreateXor(LHS, RHS); 18800b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, CI->getType()); 18810b57cec5SDimitry Andric } else if (IsX86 && Name == "avx512.kxnor.w") { 18820b57cec5SDimitry Andric Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); 18830b57cec5SDimitry Andric Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); 18840b57cec5SDimitry Andric LHS = Builder.CreateNot(LHS); 18850b57cec5SDimitry Andric Rep = Builder.CreateXor(LHS, RHS); 18860b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, CI->getType()); 18870b57cec5SDimitry Andric } else if (IsX86 && Name == "avx512.knot.w") { 18880b57cec5SDimitry Andric Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16); 18890b57cec5SDimitry Andric Rep = Builder.CreateNot(Rep); 18900b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, CI->getType()); 18910b57cec5SDimitry Andric } else if (IsX86 && 18920b57cec5SDimitry Andric (Name == "avx512.kortestz.w" || Name == "avx512.kortestc.w")) { 18930b57cec5SDimitry Andric Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16); 18940b57cec5SDimitry Andric Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16); 18950b57cec5SDimitry Andric Rep = Builder.CreateOr(LHS, RHS); 18960b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, Builder.getInt16Ty()); 18970b57cec5SDimitry Andric Value *C; 18980b57cec5SDimitry Andric if (Name[14] == 'c') 18990b57cec5SDimitry Andric C = ConstantInt::getAllOnesValue(Builder.getInt16Ty()); 19000b57cec5SDimitry Andric else 19010b57cec5SDimitry Andric C = ConstantInt::getNullValue(Builder.getInt16Ty()); 19020b57cec5SDimitry Andric Rep = Builder.CreateICmpEQ(Rep, C); 19030b57cec5SDimitry Andric Rep = Builder.CreateZExt(Rep, Builder.getInt32Ty()); 19040b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse.add.ss" || Name == "sse2.add.sd" || 19050b57cec5SDimitry Andric Name == "sse.sub.ss" || Name == "sse2.sub.sd" || 19060b57cec5SDimitry Andric Name == "sse.mul.ss" || Name == "sse2.mul.sd" || 19070b57cec5SDimitry Andric Name == "sse.div.ss" || Name == "sse2.div.sd")) { 19080b57cec5SDimitry Andric Type *I32Ty = Type::getInt32Ty(C); 19090b57cec5SDimitry Andric Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0), 19100b57cec5SDimitry Andric ConstantInt::get(I32Ty, 0)); 19110b57cec5SDimitry Andric Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1), 19120b57cec5SDimitry Andric ConstantInt::get(I32Ty, 0)); 19130b57cec5SDimitry Andric Value *EltOp; 19140b57cec5SDimitry Andric if (Name.contains(".add.")) 19150b57cec5SDimitry Andric EltOp = Builder.CreateFAdd(Elt0, Elt1); 19160b57cec5SDimitry Andric else if (Name.contains(".sub.")) 19170b57cec5SDimitry Andric EltOp = Builder.CreateFSub(Elt0, Elt1); 19180b57cec5SDimitry Andric else if (Name.contains(".mul.")) 19190b57cec5SDimitry Andric EltOp = Builder.CreateFMul(Elt0, Elt1); 19200b57cec5SDimitry Andric else 19210b57cec5SDimitry Andric EltOp = Builder.CreateFDiv(Elt0, Elt1); 19220b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp, 19230b57cec5SDimitry Andric ConstantInt::get(I32Ty, 0)); 19240b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.pcmp")) { 19250b57cec5SDimitry Andric // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt." 19260b57cec5SDimitry Andric bool CmpEq = Name[16] == 'e'; 19270b57cec5SDimitry Andric Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true); 19280b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) { 19290b57cec5SDimitry Andric Type *OpTy = CI->getArgOperand(0)->getType(); 19300b57cec5SDimitry Andric unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); 19310b57cec5SDimitry Andric Intrinsic::ID IID; 19320b57cec5SDimitry Andric switch (VecWidth) { 19330b57cec5SDimitry Andric default: llvm_unreachable("Unexpected intrinsic"); 19340b57cec5SDimitry Andric case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break; 19350b57cec5SDimitry Andric case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break; 19360b57cec5SDimitry Andric case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break; 19370b57cec5SDimitry Andric } 19380b57cec5SDimitry Andric 19390b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 19400b57cec5SDimitry Andric { CI->getOperand(0), CI->getArgOperand(1) }); 19410b57cec5SDimitry Andric Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); 19420b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) { 19430b57cec5SDimitry Andric Type *OpTy = CI->getArgOperand(0)->getType(); 19440b57cec5SDimitry Andric unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); 19450b57cec5SDimitry Andric unsigned EltWidth = OpTy->getScalarSizeInBits(); 19460b57cec5SDimitry Andric Intrinsic::ID IID; 19470b57cec5SDimitry Andric if (VecWidth == 128 && EltWidth == 32) 19480b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_fpclass_ps_128; 19490b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32) 19500b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_fpclass_ps_256; 19510b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 32) 19520b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_fpclass_ps_512; 19530b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 64) 19540b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_fpclass_pd_128; 19550b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64) 19560b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_fpclass_pd_256; 19570b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 64) 19580b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_fpclass_pd_512; 19590b57cec5SDimitry Andric else 19600b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 19610b57cec5SDimitry Andric 19620b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 19630b57cec5SDimitry Andric { CI->getOperand(0), CI->getArgOperand(1) }); 19640b57cec5SDimitry Andric Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2)); 19650b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.cmp.p")) { 19660b57cec5SDimitry Andric Type *OpTy = CI->getArgOperand(0)->getType(); 19670b57cec5SDimitry Andric unsigned VecWidth = OpTy->getPrimitiveSizeInBits(); 19680b57cec5SDimitry Andric unsigned EltWidth = OpTy->getScalarSizeInBits(); 19690b57cec5SDimitry Andric Intrinsic::ID IID; 19700b57cec5SDimitry Andric if (VecWidth == 128 && EltWidth == 32) 19710b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_cmp_ps_128; 19720b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32) 19730b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_cmp_ps_256; 19740b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 32) 19750b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_cmp_ps_512; 19760b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 64) 19770b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_cmp_pd_128; 19780b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64) 19790b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_cmp_pd_256; 19800b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 64) 19810b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_cmp_pd_512; 19820b57cec5SDimitry Andric else 19830b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 19840b57cec5SDimitry Andric 19850b57cec5SDimitry Andric SmallVector<Value *, 4> Args; 19860b57cec5SDimitry Andric Args.push_back(CI->getArgOperand(0)); 19870b57cec5SDimitry Andric Args.push_back(CI->getArgOperand(1)); 19880b57cec5SDimitry Andric Args.push_back(CI->getArgOperand(2)); 19890b57cec5SDimitry Andric if (CI->getNumArgOperands() == 5) 19900b57cec5SDimitry Andric Args.push_back(CI->getArgOperand(4)); 19910b57cec5SDimitry Andric 19920b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 19930b57cec5SDimitry Andric Args); 19940b57cec5SDimitry Andric Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(3)); 19950b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.cmp.") && 19960b57cec5SDimitry Andric Name[16] != 'p') { 19970b57cec5SDimitry Andric // Integer compare intrinsics. 19980b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 19990b57cec5SDimitry Andric Rep = upgradeMaskedCompare(Builder, *CI, Imm, true); 20000b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.ucmp.")) { 20010b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 20020b57cec5SDimitry Andric Rep = upgradeMaskedCompare(Builder, *CI, Imm, false); 20030b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.cvtb2mask.") || 20040b57cec5SDimitry Andric Name.startswith("avx512.cvtw2mask.") || 20050b57cec5SDimitry Andric Name.startswith("avx512.cvtd2mask.") || 20060b57cec5SDimitry Andric Name.startswith("avx512.cvtq2mask."))) { 20070b57cec5SDimitry Andric Value *Op = CI->getArgOperand(0); 20080b57cec5SDimitry Andric Value *Zero = llvm::Constant::getNullValue(Op->getType()); 20090b57cec5SDimitry Andric Rep = Builder.CreateICmp(ICmpInst::ICMP_SLT, Op, Zero); 20100b57cec5SDimitry Andric Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, nullptr); 20110b57cec5SDimitry Andric } else if(IsX86 && (Name == "ssse3.pabs.b.128" || 20120b57cec5SDimitry Andric Name == "ssse3.pabs.w.128" || 20130b57cec5SDimitry Andric Name == "ssse3.pabs.d.128" || 20140b57cec5SDimitry Andric Name.startswith("avx2.pabs") || 20150b57cec5SDimitry Andric Name.startswith("avx512.mask.pabs"))) { 20160b57cec5SDimitry Andric Rep = upgradeAbs(Builder, *CI); 20170b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse41.pmaxsb" || 20180b57cec5SDimitry Andric Name == "sse2.pmaxs.w" || 20190b57cec5SDimitry Andric Name == "sse41.pmaxsd" || 20200b57cec5SDimitry Andric Name.startswith("avx2.pmaxs") || 20210b57cec5SDimitry Andric Name.startswith("avx512.mask.pmaxs"))) { 20220b57cec5SDimitry Andric Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT); 20230b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.pmaxu.b" || 20240b57cec5SDimitry Andric Name == "sse41.pmaxuw" || 20250b57cec5SDimitry Andric Name == "sse41.pmaxud" || 20260b57cec5SDimitry Andric Name.startswith("avx2.pmaxu") || 20270b57cec5SDimitry Andric Name.startswith("avx512.mask.pmaxu"))) { 20280b57cec5SDimitry Andric Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT); 20290b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse41.pminsb" || 20300b57cec5SDimitry Andric Name == "sse2.pmins.w" || 20310b57cec5SDimitry Andric Name == "sse41.pminsd" || 20320b57cec5SDimitry Andric Name.startswith("avx2.pmins") || 20330b57cec5SDimitry Andric Name.startswith("avx512.mask.pmins"))) { 20340b57cec5SDimitry Andric Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT); 20350b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.pminu.b" || 20360b57cec5SDimitry Andric Name == "sse41.pminuw" || 20370b57cec5SDimitry Andric Name == "sse41.pminud" || 20380b57cec5SDimitry Andric Name.startswith("avx2.pminu") || 20390b57cec5SDimitry Andric Name.startswith("avx512.mask.pminu"))) { 20400b57cec5SDimitry Andric Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT); 20410b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.pmulu.dq" || 20420b57cec5SDimitry Andric Name == "avx2.pmulu.dq" || 20430b57cec5SDimitry Andric Name == "avx512.pmulu.dq.512" || 20440b57cec5SDimitry Andric Name.startswith("avx512.mask.pmulu.dq."))) { 20450b57cec5SDimitry Andric Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false); 20460b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse41.pmuldq" || 20470b57cec5SDimitry Andric Name == "avx2.pmul.dq" || 20480b57cec5SDimitry Andric Name == "avx512.pmul.dq.512" || 20490b57cec5SDimitry Andric Name.startswith("avx512.mask.pmul.dq."))) { 20500b57cec5SDimitry Andric Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true); 20510b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse.cvtsi2ss" || 20520b57cec5SDimitry Andric Name == "sse2.cvtsi2sd" || 20530b57cec5SDimitry Andric Name == "sse.cvtsi642ss" || 20540b57cec5SDimitry Andric Name == "sse2.cvtsi642sd")) { 20550b57cec5SDimitry Andric Rep = Builder.CreateSIToFP(CI->getArgOperand(1), 20560b57cec5SDimitry Andric CI->getType()->getVectorElementType()); 20570b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); 20580b57cec5SDimitry Andric } else if (IsX86 && Name == "avx512.cvtusi2sd") { 20590b57cec5SDimitry Andric Rep = Builder.CreateUIToFP(CI->getArgOperand(1), 20600b57cec5SDimitry Andric CI->getType()->getVectorElementType()); 20610b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); 20620b57cec5SDimitry Andric } else if (IsX86 && Name == "sse2.cvtss2sd") { 20630b57cec5SDimitry Andric Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0); 20640b57cec5SDimitry Andric Rep = Builder.CreateFPExt(Rep, CI->getType()->getVectorElementType()); 20650b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0); 20660b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.cvtdq2pd" || 20670b57cec5SDimitry Andric Name == "sse2.cvtdq2ps" || 20680b57cec5SDimitry Andric Name == "avx.cvtdq2.pd.256" || 20690b57cec5SDimitry Andric Name == "avx.cvtdq2.ps.256" || 20700b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtdq2pd.") || 20710b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtudq2pd.") || 20720b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtdq2ps.") || 20730b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtudq2ps.") || 20740b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtqq2pd.") || 20750b57cec5SDimitry Andric Name.startswith("avx512.mask.cvtuqq2pd.") || 20760b57cec5SDimitry Andric Name == "avx512.mask.cvtqq2ps.256" || 20770b57cec5SDimitry Andric Name == "avx512.mask.cvtqq2ps.512" || 20780b57cec5SDimitry Andric Name == "avx512.mask.cvtuqq2ps.256" || 20790b57cec5SDimitry Andric Name == "avx512.mask.cvtuqq2ps.512" || 20800b57cec5SDimitry Andric Name == "sse2.cvtps2pd" || 20810b57cec5SDimitry Andric Name == "avx.cvt.ps2.pd.256" || 20820b57cec5SDimitry Andric Name == "avx512.mask.cvtps2pd.128" || 20830b57cec5SDimitry Andric Name == "avx512.mask.cvtps2pd.256")) { 20840b57cec5SDimitry Andric Type *DstTy = CI->getType(); 20850b57cec5SDimitry Andric Rep = CI->getArgOperand(0); 20860b57cec5SDimitry Andric Type *SrcTy = Rep->getType(); 20870b57cec5SDimitry Andric 20880b57cec5SDimitry Andric unsigned NumDstElts = DstTy->getVectorNumElements(); 20890b57cec5SDimitry Andric if (NumDstElts < SrcTy->getVectorNumElements()) { 20900b57cec5SDimitry Andric assert(NumDstElts == 2 && "Unexpected vector size"); 20910b57cec5SDimitry Andric uint32_t ShuffleMask[2] = { 0, 1 }; 20920b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Rep, Rep, ShuffleMask); 20930b57cec5SDimitry Andric } 20940b57cec5SDimitry Andric 20950b57cec5SDimitry Andric bool IsPS2PD = SrcTy->getVectorElementType()->isFloatTy(); 20960b57cec5SDimitry Andric bool IsUnsigned = (StringRef::npos != Name.find("cvtu")); 20970b57cec5SDimitry Andric if (IsPS2PD) 20980b57cec5SDimitry Andric Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd"); 20990b57cec5SDimitry Andric else if (CI->getNumArgOperands() == 4 && 21000b57cec5SDimitry Andric (!isa<ConstantInt>(CI->getArgOperand(3)) || 21010b57cec5SDimitry Andric cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) { 21020b57cec5SDimitry Andric Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round 21030b57cec5SDimitry Andric : Intrinsic::x86_avx512_sitofp_round; 21040b57cec5SDimitry Andric Function *F = Intrinsic::getDeclaration(CI->getModule(), IID, 21050b57cec5SDimitry Andric { DstTy, SrcTy }); 21060b57cec5SDimitry Andric Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) }); 21070b57cec5SDimitry Andric } else { 21080b57cec5SDimitry Andric Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt") 21090b57cec5SDimitry Andric : Builder.CreateSIToFP(Rep, DstTy, "cvt"); 21100b57cec5SDimitry Andric } 21110b57cec5SDimitry Andric 21120b57cec5SDimitry Andric if (CI->getNumArgOperands() >= 3) 21130b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 21140b57cec5SDimitry Andric CI->getArgOperand(1)); 21150b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) { 21160b57cec5SDimitry Andric Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), 21170b57cec5SDimitry Andric CI->getArgOperand(1), CI->getArgOperand(2), 21180b57cec5SDimitry Andric /*Aligned*/false); 21190b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.load."))) { 21200b57cec5SDimitry Andric Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0), 21210b57cec5SDimitry Andric CI->getArgOperand(1),CI->getArgOperand(2), 21220b57cec5SDimitry Andric /*Aligned*/true); 21230b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.expand.load.")) { 21240b57cec5SDimitry Andric Type *ResultTy = CI->getType(); 21250b57cec5SDimitry Andric Type *PtrTy = ResultTy->getVectorElementType(); 21260b57cec5SDimitry Andric 21270b57cec5SDimitry Andric // Cast the pointer to element type. 21280b57cec5SDimitry Andric Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), 21290b57cec5SDimitry Andric llvm::PointerType::getUnqual(PtrTy)); 21300b57cec5SDimitry Andric 21310b57cec5SDimitry Andric Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), 21320b57cec5SDimitry Andric ResultTy->getVectorNumElements()); 21330b57cec5SDimitry Andric 21340b57cec5SDimitry Andric Function *ELd = Intrinsic::getDeclaration(F->getParent(), 21350b57cec5SDimitry Andric Intrinsic::masked_expandload, 21360b57cec5SDimitry Andric ResultTy); 21370b57cec5SDimitry Andric Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) }); 21380b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.compress.store.")) { 21390b57cec5SDimitry Andric Type *ResultTy = CI->getArgOperand(1)->getType(); 21400b57cec5SDimitry Andric Type *PtrTy = ResultTy->getVectorElementType(); 21410b57cec5SDimitry Andric 21420b57cec5SDimitry Andric // Cast the pointer to element type. 21430b57cec5SDimitry Andric Value *Ptr = Builder.CreateBitCast(CI->getOperand(0), 21440b57cec5SDimitry Andric llvm::PointerType::getUnqual(PtrTy)); 21450b57cec5SDimitry Andric 21460b57cec5SDimitry Andric Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), 21470b57cec5SDimitry Andric ResultTy->getVectorNumElements()); 21480b57cec5SDimitry Andric 21490b57cec5SDimitry Andric Function *CSt = Intrinsic::getDeclaration(F->getParent(), 21500b57cec5SDimitry Andric Intrinsic::masked_compressstore, 21510b57cec5SDimitry Andric ResultTy); 21520b57cec5SDimitry Andric Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec }); 21530b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.compress.") || 21540b57cec5SDimitry Andric Name.startswith("avx512.mask.expand."))) { 21550b57cec5SDimitry Andric Type *ResultTy = CI->getType(); 21560b57cec5SDimitry Andric 21570b57cec5SDimitry Andric Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2), 21580b57cec5SDimitry Andric ResultTy->getVectorNumElements()); 21590b57cec5SDimitry Andric 21600b57cec5SDimitry Andric bool IsCompress = Name[12] == 'c'; 21610b57cec5SDimitry Andric Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress 21620b57cec5SDimitry Andric : Intrinsic::x86_avx512_mask_expand; 21630b57cec5SDimitry Andric Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy); 21640b57cec5SDimitry Andric Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1), 21650b57cec5SDimitry Andric MaskVec }); 21660b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("xop.vpcom")) { 21670b57cec5SDimitry Andric bool IsSigned; 21680b57cec5SDimitry Andric if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") || 21690b57cec5SDimitry Andric Name.endswith("uq")) 21700b57cec5SDimitry Andric IsSigned = false; 21710b57cec5SDimitry Andric else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") || 21720b57cec5SDimitry Andric Name.endswith("q")) 21730b57cec5SDimitry Andric IsSigned = true; 21740b57cec5SDimitry Andric else 21750b57cec5SDimitry Andric llvm_unreachable("Unknown suffix"); 21760b57cec5SDimitry Andric 21770b57cec5SDimitry Andric unsigned Imm; 21780b57cec5SDimitry Andric if (CI->getNumArgOperands() == 3) { 21790b57cec5SDimitry Andric Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 21800b57cec5SDimitry Andric } else { 21810b57cec5SDimitry Andric Name = Name.substr(9); // strip off "xop.vpcom" 21820b57cec5SDimitry Andric if (Name.startswith("lt")) 21830b57cec5SDimitry Andric Imm = 0; 21840b57cec5SDimitry Andric else if (Name.startswith("le")) 21850b57cec5SDimitry Andric Imm = 1; 21860b57cec5SDimitry Andric else if (Name.startswith("gt")) 21870b57cec5SDimitry Andric Imm = 2; 21880b57cec5SDimitry Andric else if (Name.startswith("ge")) 21890b57cec5SDimitry Andric Imm = 3; 21900b57cec5SDimitry Andric else if (Name.startswith("eq")) 21910b57cec5SDimitry Andric Imm = 4; 21920b57cec5SDimitry Andric else if (Name.startswith("ne")) 21930b57cec5SDimitry Andric Imm = 5; 21940b57cec5SDimitry Andric else if (Name.startswith("false")) 21950b57cec5SDimitry Andric Imm = 6; 21960b57cec5SDimitry Andric else if (Name.startswith("true")) 21970b57cec5SDimitry Andric Imm = 7; 21980b57cec5SDimitry Andric else 21990b57cec5SDimitry Andric llvm_unreachable("Unknown condition"); 22000b57cec5SDimitry Andric } 22010b57cec5SDimitry Andric 22020b57cec5SDimitry Andric Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned); 22030b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("xop.vpcmov")) { 22040b57cec5SDimitry Andric Value *Sel = CI->getArgOperand(2); 22050b57cec5SDimitry Andric Value *NotSel = Builder.CreateNot(Sel); 22060b57cec5SDimitry Andric Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel); 22070b57cec5SDimitry Andric Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel); 22080b57cec5SDimitry Andric Rep = Builder.CreateOr(Sel0, Sel1); 22090b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("xop.vprot") || 22100b57cec5SDimitry Andric Name.startswith("avx512.prol") || 22110b57cec5SDimitry Andric Name.startswith("avx512.mask.prol"))) { 22120b57cec5SDimitry Andric Rep = upgradeX86Rotate(Builder, *CI, false); 22130b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.pror") || 22140b57cec5SDimitry Andric Name.startswith("avx512.mask.pror"))) { 22150b57cec5SDimitry Andric Rep = upgradeX86Rotate(Builder, *CI, true); 22160b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.vpshld.") || 22170b57cec5SDimitry Andric Name.startswith("avx512.mask.vpshld") || 22180b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpshld"))) { 22190b57cec5SDimitry Andric bool ZeroMask = Name[11] == 'z'; 22200b57cec5SDimitry Andric Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask); 22210b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.vpshrd.") || 22220b57cec5SDimitry Andric Name.startswith("avx512.mask.vpshrd") || 22230b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpshrd"))) { 22240b57cec5SDimitry Andric bool ZeroMask = Name[11] == 'z'; 22250b57cec5SDimitry Andric Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask); 22260b57cec5SDimitry Andric } else if (IsX86 && Name == "sse42.crc32.64.8") { 22270b57cec5SDimitry Andric Function *CRC32 = Intrinsic::getDeclaration(F->getParent(), 22280b57cec5SDimitry Andric Intrinsic::x86_sse42_crc32_32_8); 22290b57cec5SDimitry Andric Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C)); 22300b57cec5SDimitry Andric Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)}); 22310b57cec5SDimitry Andric Rep = Builder.CreateZExt(Rep, CI->getType(), ""); 22320b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx.vbroadcast.s") || 22330b57cec5SDimitry Andric Name.startswith("avx512.vbroadcast.s"))) { 22340b57cec5SDimitry Andric // Replace broadcasts with a series of insertelements. 22350b57cec5SDimitry Andric Type *VecTy = CI->getType(); 22360b57cec5SDimitry Andric Type *EltTy = VecTy->getVectorElementType(); 22370b57cec5SDimitry Andric unsigned EltNum = VecTy->getVectorNumElements(); 22380b57cec5SDimitry Andric Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0), 22390b57cec5SDimitry Andric EltTy->getPointerTo()); 22400b57cec5SDimitry Andric Value *Load = Builder.CreateLoad(EltTy, Cast); 22410b57cec5SDimitry Andric Type *I32Ty = Type::getInt32Ty(C); 22420b57cec5SDimitry Andric Rep = UndefValue::get(VecTy); 22430b57cec5SDimitry Andric for (unsigned I = 0; I < EltNum; ++I) 22440b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(Rep, Load, 22450b57cec5SDimitry Andric ConstantInt::get(I32Ty, I)); 22460b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("sse41.pmovsx") || 22470b57cec5SDimitry Andric Name.startswith("sse41.pmovzx") || 22480b57cec5SDimitry Andric Name.startswith("avx2.pmovsx") || 22490b57cec5SDimitry Andric Name.startswith("avx2.pmovzx") || 22500b57cec5SDimitry Andric Name.startswith("avx512.mask.pmovsx") || 22510b57cec5SDimitry Andric Name.startswith("avx512.mask.pmovzx"))) { 22520b57cec5SDimitry Andric VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType()); 22530b57cec5SDimitry Andric VectorType *DstTy = cast<VectorType>(CI->getType()); 22540b57cec5SDimitry Andric unsigned NumDstElts = DstTy->getNumElements(); 22550b57cec5SDimitry Andric 22560b57cec5SDimitry Andric // Extract a subvector of the first NumDstElts lanes and sign/zero extend. 22570b57cec5SDimitry Andric SmallVector<uint32_t, 8> ShuffleMask(NumDstElts); 22580b57cec5SDimitry Andric for (unsigned i = 0; i != NumDstElts; ++i) 22590b57cec5SDimitry Andric ShuffleMask[i] = i; 22600b57cec5SDimitry Andric 22610b57cec5SDimitry Andric Value *SV = Builder.CreateShuffleVector( 22620b57cec5SDimitry Andric CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask); 22630b57cec5SDimitry Andric 22640b57cec5SDimitry Andric bool DoSext = (StringRef::npos != Name.find("pmovsx")); 22650b57cec5SDimitry Andric Rep = DoSext ? Builder.CreateSExt(SV, DstTy) 22660b57cec5SDimitry Andric : Builder.CreateZExt(SV, DstTy); 22670b57cec5SDimitry Andric // If there are 3 arguments, it's a masked intrinsic so we need a select. 22680b57cec5SDimitry Andric if (CI->getNumArgOperands() == 3) 22690b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 22700b57cec5SDimitry Andric CI->getArgOperand(1)); 22710b57cec5SDimitry Andric } else if (Name == "avx512.mask.pmov.qd.256" || 22720b57cec5SDimitry Andric Name == "avx512.mask.pmov.qd.512" || 22730b57cec5SDimitry Andric Name == "avx512.mask.pmov.wb.256" || 22740b57cec5SDimitry Andric Name == "avx512.mask.pmov.wb.512") { 22750b57cec5SDimitry Andric Type *Ty = CI->getArgOperand(1)->getType(); 22760b57cec5SDimitry Andric Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty); 22770b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 22780b57cec5SDimitry Andric CI->getArgOperand(1)); 22790b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") || 22800b57cec5SDimitry Andric Name == "avx2.vbroadcasti128")) { 22810b57cec5SDimitry Andric // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle. 22820b57cec5SDimitry Andric Type *EltTy = CI->getType()->getVectorElementType(); 22830b57cec5SDimitry Andric unsigned NumSrcElts = 128 / EltTy->getPrimitiveSizeInBits(); 22840b57cec5SDimitry Andric Type *VT = VectorType::get(EltTy, NumSrcElts); 22850b57cec5SDimitry Andric Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0), 22860b57cec5SDimitry Andric PointerType::getUnqual(VT)); 22870b57cec5SDimitry Andric Value *Load = Builder.CreateAlignedLoad(VT, Op, 1); 22880b57cec5SDimitry Andric if (NumSrcElts == 2) 22890b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), 22900b57cec5SDimitry Andric { 0, 1, 0, 1 }); 22910b57cec5SDimitry Andric else 22920b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()), 22930b57cec5SDimitry Andric { 0, 1, 2, 3, 0, 1, 2, 3 }); 22940b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.shuf.i") || 22950b57cec5SDimitry Andric Name.startswith("avx512.mask.shuf.f"))) { 22960b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 22970b57cec5SDimitry Andric Type *VT = CI->getType(); 22980b57cec5SDimitry Andric unsigned NumLanes = VT->getPrimitiveSizeInBits() / 128; 22990b57cec5SDimitry Andric unsigned NumElementsInLane = 128 / VT->getScalarSizeInBits(); 23000b57cec5SDimitry Andric unsigned ControlBitsMask = NumLanes - 1; 23010b57cec5SDimitry Andric unsigned NumControlBits = NumLanes / 2; 23020b57cec5SDimitry Andric SmallVector<uint32_t, 8> ShuffleMask(0); 23030b57cec5SDimitry Andric 23040b57cec5SDimitry Andric for (unsigned l = 0; l != NumLanes; ++l) { 23050b57cec5SDimitry Andric unsigned LaneMask = (Imm >> (l * NumControlBits)) & ControlBitsMask; 23060b57cec5SDimitry Andric // We actually need the other source. 23070b57cec5SDimitry Andric if (l >= NumLanes / 2) 23080b57cec5SDimitry Andric LaneMask += NumLanes; 23090b57cec5SDimitry Andric for (unsigned i = 0; i != NumElementsInLane; ++i) 23100b57cec5SDimitry Andric ShuffleMask.push_back(LaneMask * NumElementsInLane + i); 23110b57cec5SDimitry Andric } 23120b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), 23130b57cec5SDimitry Andric CI->getArgOperand(1), ShuffleMask); 23140b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, 23150b57cec5SDimitry Andric CI->getArgOperand(3)); 23160b57cec5SDimitry Andric }else if (IsX86 && (Name.startswith("avx512.mask.broadcastf") || 23170b57cec5SDimitry Andric Name.startswith("avx512.mask.broadcasti"))) { 23180b57cec5SDimitry Andric unsigned NumSrcElts = 23190b57cec5SDimitry Andric CI->getArgOperand(0)->getType()->getVectorNumElements(); 23200b57cec5SDimitry Andric unsigned NumDstElts = CI->getType()->getVectorNumElements(); 23210b57cec5SDimitry Andric 23220b57cec5SDimitry Andric SmallVector<uint32_t, 8> ShuffleMask(NumDstElts); 23230b57cec5SDimitry Andric for (unsigned i = 0; i != NumDstElts; ++i) 23240b57cec5SDimitry Andric ShuffleMask[i] = i % NumSrcElts; 23250b57cec5SDimitry Andric 23260b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(CI->getArgOperand(0), 23270b57cec5SDimitry Andric CI->getArgOperand(0), 23280b57cec5SDimitry Andric ShuffleMask); 23290b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 23300b57cec5SDimitry Andric CI->getArgOperand(1)); 23310b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx2.pbroadcast") || 23320b57cec5SDimitry Andric Name.startswith("avx2.vbroadcast") || 23330b57cec5SDimitry Andric Name.startswith("avx512.pbroadcast") || 23340b57cec5SDimitry Andric Name.startswith("avx512.mask.broadcast.s"))) { 23350b57cec5SDimitry Andric // Replace vp?broadcasts with a vector shuffle. 23360b57cec5SDimitry Andric Value *Op = CI->getArgOperand(0); 23370b57cec5SDimitry Andric unsigned NumElts = CI->getType()->getVectorNumElements(); 23380b57cec5SDimitry Andric Type *MaskTy = VectorType::get(Type::getInt32Ty(C), NumElts); 23390b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op, UndefValue::get(Op->getType()), 23400b57cec5SDimitry Andric Constant::getNullValue(MaskTy)); 23410b57cec5SDimitry Andric 23420b57cec5SDimitry Andric if (CI->getNumArgOperands() == 3) 23430b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 23440b57cec5SDimitry Andric CI->getArgOperand(1)); 23450b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("sse2.padds.") || 23460b57cec5SDimitry Andric Name.startswith("sse2.psubs.") || 23470b57cec5SDimitry Andric Name.startswith("avx2.padds.") || 23480b57cec5SDimitry Andric Name.startswith("avx2.psubs.") || 23490b57cec5SDimitry Andric Name.startswith("avx512.padds.") || 23500b57cec5SDimitry Andric Name.startswith("avx512.psubs.") || 23510b57cec5SDimitry Andric Name.startswith("avx512.mask.padds.") || 23520b57cec5SDimitry Andric Name.startswith("avx512.mask.psubs."))) { 23530b57cec5SDimitry Andric bool IsAdd = Name.contains(".padds"); 23540b57cec5SDimitry Andric Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, true, IsAdd); 23550b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("sse2.paddus.") || 23560b57cec5SDimitry Andric Name.startswith("sse2.psubus.") || 23570b57cec5SDimitry Andric Name.startswith("avx2.paddus.") || 23580b57cec5SDimitry Andric Name.startswith("avx2.psubus.") || 23590b57cec5SDimitry Andric Name.startswith("avx512.mask.paddus.") || 23600b57cec5SDimitry Andric Name.startswith("avx512.mask.psubus."))) { 23610b57cec5SDimitry Andric bool IsAdd = Name.contains(".paddus"); 23620b57cec5SDimitry Andric Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, false, IsAdd); 23630b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) { 23640b57cec5SDimitry Andric Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), 23650b57cec5SDimitry Andric CI->getArgOperand(1), 23660b57cec5SDimitry Andric CI->getArgOperand(2), 23670b57cec5SDimitry Andric CI->getArgOperand(3), 23680b57cec5SDimitry Andric CI->getArgOperand(4), 23690b57cec5SDimitry Andric false); 23700b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.valign.")) { 23710b57cec5SDimitry Andric Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0), 23720b57cec5SDimitry Andric CI->getArgOperand(1), 23730b57cec5SDimitry Andric CI->getArgOperand(2), 23740b57cec5SDimitry Andric CI->getArgOperand(3), 23750b57cec5SDimitry Andric CI->getArgOperand(4), 23760b57cec5SDimitry Andric true); 23770b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.psll.dq" || 23780b57cec5SDimitry Andric Name == "avx2.psll.dq")) { 23790b57cec5SDimitry Andric // 128/256-bit shift left specified in bits. 23800b57cec5SDimitry Andric unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 23810b57cec5SDimitry Andric Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), 23820b57cec5SDimitry Andric Shift / 8); // Shift is in bits. 23830b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.psrl.dq" || 23840b57cec5SDimitry Andric Name == "avx2.psrl.dq")) { 23850b57cec5SDimitry Andric // 128/256-bit shift right specified in bits. 23860b57cec5SDimitry Andric unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 23870b57cec5SDimitry Andric Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), 23880b57cec5SDimitry Andric Shift / 8); // Shift is in bits. 23890b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.psll.dq.bs" || 23900b57cec5SDimitry Andric Name == "avx2.psll.dq.bs" || 23910b57cec5SDimitry Andric Name == "avx512.psll.dq.512")) { 23920b57cec5SDimitry Andric // 128/256/512-bit shift left specified in bytes. 23930b57cec5SDimitry Andric unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 23940b57cec5SDimitry Andric Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); 23950b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.psrl.dq.bs" || 23960b57cec5SDimitry Andric Name == "avx2.psrl.dq.bs" || 23970b57cec5SDimitry Andric Name == "avx512.psrl.dq.512")) { 23980b57cec5SDimitry Andric // 128/256/512-bit shift right specified in bytes. 23990b57cec5SDimitry Andric unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 24000b57cec5SDimitry Andric Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift); 24010b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse41.pblendw" || 24020b57cec5SDimitry Andric Name.startswith("sse41.blendp") || 24030b57cec5SDimitry Andric Name.startswith("avx.blend.p") || 24040b57cec5SDimitry Andric Name == "avx2.pblendw" || 24050b57cec5SDimitry Andric Name.startswith("avx2.pblendd."))) { 24060b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 24070b57cec5SDimitry Andric Value *Op1 = CI->getArgOperand(1); 24080b57cec5SDimitry Andric unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 24090b57cec5SDimitry Andric VectorType *VecTy = cast<VectorType>(CI->getType()); 24100b57cec5SDimitry Andric unsigned NumElts = VecTy->getNumElements(); 24110b57cec5SDimitry Andric 24120b57cec5SDimitry Andric SmallVector<uint32_t, 16> Idxs(NumElts); 24130b57cec5SDimitry Andric for (unsigned i = 0; i != NumElts; ++i) 24140b57cec5SDimitry Andric Idxs[i] = ((Imm >> (i%8)) & 1) ? i + NumElts : i; 24150b57cec5SDimitry Andric 24160b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); 24170b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx.vinsertf128.") || 24180b57cec5SDimitry Andric Name == "avx2.vinserti128" || 24190b57cec5SDimitry Andric Name.startswith("avx512.mask.insert"))) { 24200b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 24210b57cec5SDimitry Andric Value *Op1 = CI->getArgOperand(1); 24220b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 24230b57cec5SDimitry Andric unsigned DstNumElts = CI->getType()->getVectorNumElements(); 24240b57cec5SDimitry Andric unsigned SrcNumElts = Op1->getType()->getVectorNumElements(); 24250b57cec5SDimitry Andric unsigned Scale = DstNumElts / SrcNumElts; 24260b57cec5SDimitry Andric 24270b57cec5SDimitry Andric // Mask off the high bits of the immediate value; hardware ignores those. 24280b57cec5SDimitry Andric Imm = Imm % Scale; 24290b57cec5SDimitry Andric 24300b57cec5SDimitry Andric // Extend the second operand into a vector the size of the destination. 24310b57cec5SDimitry Andric Value *UndefV = UndefValue::get(Op1->getType()); 24320b57cec5SDimitry Andric SmallVector<uint32_t, 8> Idxs(DstNumElts); 24330b57cec5SDimitry Andric for (unsigned i = 0; i != SrcNumElts; ++i) 24340b57cec5SDimitry Andric Idxs[i] = i; 24350b57cec5SDimitry Andric for (unsigned i = SrcNumElts; i != DstNumElts; ++i) 24360b57cec5SDimitry Andric Idxs[i] = SrcNumElts; 24370b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op1, UndefV, Idxs); 24380b57cec5SDimitry Andric 24390b57cec5SDimitry Andric // Insert the second operand into the first operand. 24400b57cec5SDimitry Andric 24410b57cec5SDimitry Andric // Note that there is no guarantee that instruction lowering will actually 24420b57cec5SDimitry Andric // produce a vinsertf128 instruction for the created shuffles. In 24430b57cec5SDimitry Andric // particular, the 0 immediate case involves no lane changes, so it can 24440b57cec5SDimitry Andric // be handled as a blend. 24450b57cec5SDimitry Andric 24460b57cec5SDimitry Andric // Example of shuffle mask for 32-bit elements: 24470b57cec5SDimitry Andric // Imm = 1 <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11> 24480b57cec5SDimitry Andric // Imm = 0 <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7 > 24490b57cec5SDimitry Andric 24500b57cec5SDimitry Andric // First fill with identify mask. 24510b57cec5SDimitry Andric for (unsigned i = 0; i != DstNumElts; ++i) 24520b57cec5SDimitry Andric Idxs[i] = i; 24530b57cec5SDimitry Andric // Then replace the elements where we need to insert. 24540b57cec5SDimitry Andric for (unsigned i = 0; i != SrcNumElts; ++i) 24550b57cec5SDimitry Andric Idxs[i + Imm * SrcNumElts] = i + DstNumElts; 24560b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Rep, Idxs); 24570b57cec5SDimitry Andric 24580b57cec5SDimitry Andric // If the intrinsic has a mask operand, handle that. 24590b57cec5SDimitry Andric if (CI->getNumArgOperands() == 5) 24600b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, 24610b57cec5SDimitry Andric CI->getArgOperand(3)); 24620b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx.vextractf128.") || 24630b57cec5SDimitry Andric Name == "avx2.vextracti128" || 24640b57cec5SDimitry Andric Name.startswith("avx512.mask.vextract"))) { 24650b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 24660b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 24670b57cec5SDimitry Andric unsigned DstNumElts = CI->getType()->getVectorNumElements(); 24680b57cec5SDimitry Andric unsigned SrcNumElts = Op0->getType()->getVectorNumElements(); 24690b57cec5SDimitry Andric unsigned Scale = SrcNumElts / DstNumElts; 24700b57cec5SDimitry Andric 24710b57cec5SDimitry Andric // Mask off the high bits of the immediate value; hardware ignores those. 24720b57cec5SDimitry Andric Imm = Imm % Scale; 24730b57cec5SDimitry Andric 24740b57cec5SDimitry Andric // Get indexes for the subvector of the input vector. 24750b57cec5SDimitry Andric SmallVector<uint32_t, 8> Idxs(DstNumElts); 24760b57cec5SDimitry Andric for (unsigned i = 0; i != DstNumElts; ++i) { 24770b57cec5SDimitry Andric Idxs[i] = i + (Imm * DstNumElts); 24780b57cec5SDimitry Andric } 24790b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 24800b57cec5SDimitry Andric 24810b57cec5SDimitry Andric // If the intrinsic has a mask operand, handle that. 24820b57cec5SDimitry Andric if (CI->getNumArgOperands() == 4) 24830b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 24840b57cec5SDimitry Andric CI->getArgOperand(2)); 24850b57cec5SDimitry Andric } else if (!IsX86 && Name == "stackprotectorcheck") { 24860b57cec5SDimitry Andric Rep = nullptr; 24870b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.perm.df.") || 24880b57cec5SDimitry Andric Name.startswith("avx512.mask.perm.di."))) { 24890b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 24900b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 24910b57cec5SDimitry Andric VectorType *VecTy = cast<VectorType>(CI->getType()); 24920b57cec5SDimitry Andric unsigned NumElts = VecTy->getNumElements(); 24930b57cec5SDimitry Andric 24940b57cec5SDimitry Andric SmallVector<uint32_t, 8> Idxs(NumElts); 24950b57cec5SDimitry Andric for (unsigned i = 0; i != NumElts; ++i) 24960b57cec5SDimitry Andric Idxs[i] = (i & ~0x3) + ((Imm >> (2 * (i & 0x3))) & 3); 24970b57cec5SDimitry Andric 24980b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 24990b57cec5SDimitry Andric 25000b57cec5SDimitry Andric if (CI->getNumArgOperands() == 4) 25010b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 25020b57cec5SDimitry Andric CI->getArgOperand(2)); 25030b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx.vperm2f128.") || 25040b57cec5SDimitry Andric Name == "avx2.vperm2i128")) { 25050b57cec5SDimitry Andric // The immediate permute control byte looks like this: 25060b57cec5SDimitry Andric // [1:0] - select 128 bits from sources for low half of destination 25070b57cec5SDimitry Andric // [2] - ignore 25080b57cec5SDimitry Andric // [3] - zero low half of destination 25090b57cec5SDimitry Andric // [5:4] - select 128 bits from sources for high half of destination 25100b57cec5SDimitry Andric // [6] - ignore 25110b57cec5SDimitry Andric // [7] - zero high half of destination 25120b57cec5SDimitry Andric 25130b57cec5SDimitry Andric uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 25140b57cec5SDimitry Andric 25150b57cec5SDimitry Andric unsigned NumElts = CI->getType()->getVectorNumElements(); 25160b57cec5SDimitry Andric unsigned HalfSize = NumElts / 2; 25170b57cec5SDimitry Andric SmallVector<uint32_t, 8> ShuffleMask(NumElts); 25180b57cec5SDimitry Andric 25190b57cec5SDimitry Andric // Determine which operand(s) are actually in use for this instruction. 25200b57cec5SDimitry Andric Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0); 25210b57cec5SDimitry Andric Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0); 25220b57cec5SDimitry Andric 25230b57cec5SDimitry Andric // If needed, replace operands based on zero mask. 25240b57cec5SDimitry Andric V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0; 25250b57cec5SDimitry Andric V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1; 25260b57cec5SDimitry Andric 25270b57cec5SDimitry Andric // Permute low half of result. 25280b57cec5SDimitry Andric unsigned StartIndex = (Imm & 0x01) ? HalfSize : 0; 25290b57cec5SDimitry Andric for (unsigned i = 0; i < HalfSize; ++i) 25300b57cec5SDimitry Andric ShuffleMask[i] = StartIndex + i; 25310b57cec5SDimitry Andric 25320b57cec5SDimitry Andric // Permute high half of result. 25330b57cec5SDimitry Andric StartIndex = (Imm & 0x10) ? HalfSize : 0; 25340b57cec5SDimitry Andric for (unsigned i = 0; i < HalfSize; ++i) 25350b57cec5SDimitry Andric ShuffleMask[i + HalfSize] = NumElts + StartIndex + i; 25360b57cec5SDimitry Andric 25370b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(V0, V1, ShuffleMask); 25380b57cec5SDimitry Andric 25390b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx.vpermil.") || 25400b57cec5SDimitry Andric Name == "sse2.pshuf.d" || 25410b57cec5SDimitry Andric Name.startswith("avx512.mask.vpermil.p") || 25420b57cec5SDimitry Andric Name.startswith("avx512.mask.pshuf.d."))) { 25430b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 25440b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 25450b57cec5SDimitry Andric VectorType *VecTy = cast<VectorType>(CI->getType()); 25460b57cec5SDimitry Andric unsigned NumElts = VecTy->getNumElements(); 25470b57cec5SDimitry Andric // Calculate the size of each index in the immediate. 25480b57cec5SDimitry Andric unsigned IdxSize = 64 / VecTy->getScalarSizeInBits(); 25490b57cec5SDimitry Andric unsigned IdxMask = ((1 << IdxSize) - 1); 25500b57cec5SDimitry Andric 25510b57cec5SDimitry Andric SmallVector<uint32_t, 8> Idxs(NumElts); 25520b57cec5SDimitry Andric // Lookup the bits for this element, wrapping around the immediate every 25530b57cec5SDimitry Andric // 8-bits. Elements are grouped into sets of 2 or 4 elements so we need 25540b57cec5SDimitry Andric // to offset by the first index of each group. 25550b57cec5SDimitry Andric for (unsigned i = 0; i != NumElts; ++i) 25560b57cec5SDimitry Andric Idxs[i] = ((Imm >> ((i * IdxSize) % 8)) & IdxMask) | (i & ~IdxMask); 25570b57cec5SDimitry Andric 25580b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 25590b57cec5SDimitry Andric 25600b57cec5SDimitry Andric if (CI->getNumArgOperands() == 4) 25610b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 25620b57cec5SDimitry Andric CI->getArgOperand(2)); 25630b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.pshufl.w" || 25640b57cec5SDimitry Andric Name.startswith("avx512.mask.pshufl.w."))) { 25650b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 25660b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 25670b57cec5SDimitry Andric unsigned NumElts = CI->getType()->getVectorNumElements(); 25680b57cec5SDimitry Andric 25690b57cec5SDimitry Andric SmallVector<uint32_t, 16> Idxs(NumElts); 25700b57cec5SDimitry Andric for (unsigned l = 0; l != NumElts; l += 8) { 25710b57cec5SDimitry Andric for (unsigned i = 0; i != 4; ++i) 25720b57cec5SDimitry Andric Idxs[i + l] = ((Imm >> (2 * i)) & 0x3) + l; 25730b57cec5SDimitry Andric for (unsigned i = 4; i != 8; ++i) 25740b57cec5SDimitry Andric Idxs[i + l] = i + l; 25750b57cec5SDimitry Andric } 25760b57cec5SDimitry Andric 25770b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 25780b57cec5SDimitry Andric 25790b57cec5SDimitry Andric if (CI->getNumArgOperands() == 4) 25800b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 25810b57cec5SDimitry Andric CI->getArgOperand(2)); 25820b57cec5SDimitry Andric } else if (IsX86 && (Name == "sse2.pshufh.w" || 25830b57cec5SDimitry Andric Name.startswith("avx512.mask.pshufh.w."))) { 25840b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 25850b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue(); 25860b57cec5SDimitry Andric unsigned NumElts = CI->getType()->getVectorNumElements(); 25870b57cec5SDimitry Andric 25880b57cec5SDimitry Andric SmallVector<uint32_t, 16> Idxs(NumElts); 25890b57cec5SDimitry Andric for (unsigned l = 0; l != NumElts; l += 8) { 25900b57cec5SDimitry Andric for (unsigned i = 0; i != 4; ++i) 25910b57cec5SDimitry Andric Idxs[i + l] = i + l; 25920b57cec5SDimitry Andric for (unsigned i = 0; i != 4; ++i) 25930b57cec5SDimitry Andric Idxs[i + l + 4] = ((Imm >> (2 * i)) & 0x3) + 4 + l; 25940b57cec5SDimitry Andric } 25950b57cec5SDimitry Andric 25960b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 25970b57cec5SDimitry Andric 25980b57cec5SDimitry Andric if (CI->getNumArgOperands() == 4) 25990b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 26000b57cec5SDimitry Andric CI->getArgOperand(2)); 26010b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.shuf.p")) { 26020b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 26030b57cec5SDimitry Andric Value *Op1 = CI->getArgOperand(1); 26040b57cec5SDimitry Andric unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue(); 26050b57cec5SDimitry Andric unsigned NumElts = CI->getType()->getVectorNumElements(); 26060b57cec5SDimitry Andric 26070b57cec5SDimitry Andric unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); 26080b57cec5SDimitry Andric unsigned HalfLaneElts = NumLaneElts / 2; 26090b57cec5SDimitry Andric 26100b57cec5SDimitry Andric SmallVector<uint32_t, 16> Idxs(NumElts); 26110b57cec5SDimitry Andric for (unsigned i = 0; i != NumElts; ++i) { 26120b57cec5SDimitry Andric // Base index is the starting element of the lane. 26130b57cec5SDimitry Andric Idxs[i] = i - (i % NumLaneElts); 26140b57cec5SDimitry Andric // If we are half way through the lane switch to the other source. 26150b57cec5SDimitry Andric if ((i % NumLaneElts) >= HalfLaneElts) 26160b57cec5SDimitry Andric Idxs[i] += NumElts; 26170b57cec5SDimitry Andric // Now select the specific element. By adding HalfLaneElts bits from 26180b57cec5SDimitry Andric // the immediate. Wrapping around the immediate every 8-bits. 26190b57cec5SDimitry Andric Idxs[i] += (Imm >> ((i * HalfLaneElts) % 8)) & ((1 << HalfLaneElts) - 1); 26200b57cec5SDimitry Andric } 26210b57cec5SDimitry Andric 26220b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); 26230b57cec5SDimitry Andric 26240b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, 26250b57cec5SDimitry Andric CI->getArgOperand(3)); 26260b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.movddup") || 26270b57cec5SDimitry Andric Name.startswith("avx512.mask.movshdup") || 26280b57cec5SDimitry Andric Name.startswith("avx512.mask.movsldup"))) { 26290b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 26300b57cec5SDimitry Andric unsigned NumElts = CI->getType()->getVectorNumElements(); 26310b57cec5SDimitry Andric unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); 26320b57cec5SDimitry Andric 26330b57cec5SDimitry Andric unsigned Offset = 0; 26340b57cec5SDimitry Andric if (Name.startswith("avx512.mask.movshdup.")) 26350b57cec5SDimitry Andric Offset = 1; 26360b57cec5SDimitry Andric 26370b57cec5SDimitry Andric SmallVector<uint32_t, 16> Idxs(NumElts); 26380b57cec5SDimitry Andric for (unsigned l = 0; l != NumElts; l += NumLaneElts) 26390b57cec5SDimitry Andric for (unsigned i = 0; i != NumLaneElts; i += 2) { 26400b57cec5SDimitry Andric Idxs[i + l + 0] = i + l + Offset; 26410b57cec5SDimitry Andric Idxs[i + l + 1] = i + l + Offset; 26420b57cec5SDimitry Andric } 26430b57cec5SDimitry Andric 26440b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op0, Idxs); 26450b57cec5SDimitry Andric 26460b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 26470b57cec5SDimitry Andric CI->getArgOperand(1)); 26480b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.punpckl") || 26490b57cec5SDimitry Andric Name.startswith("avx512.mask.unpckl."))) { 26500b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 26510b57cec5SDimitry Andric Value *Op1 = CI->getArgOperand(1); 26520b57cec5SDimitry Andric int NumElts = CI->getType()->getVectorNumElements(); 26530b57cec5SDimitry Andric int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); 26540b57cec5SDimitry Andric 26550b57cec5SDimitry Andric SmallVector<uint32_t, 64> Idxs(NumElts); 26560b57cec5SDimitry Andric for (int l = 0; l != NumElts; l += NumLaneElts) 26570b57cec5SDimitry Andric for (int i = 0; i != NumLaneElts; ++i) 26580b57cec5SDimitry Andric Idxs[i + l] = l + (i / 2) + NumElts * (i % 2); 26590b57cec5SDimitry Andric 26600b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); 26610b57cec5SDimitry Andric 26620b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 26630b57cec5SDimitry Andric CI->getArgOperand(2)); 26640b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.punpckh") || 26650b57cec5SDimitry Andric Name.startswith("avx512.mask.unpckh."))) { 26660b57cec5SDimitry Andric Value *Op0 = CI->getArgOperand(0); 26670b57cec5SDimitry Andric Value *Op1 = CI->getArgOperand(1); 26680b57cec5SDimitry Andric int NumElts = CI->getType()->getVectorNumElements(); 26690b57cec5SDimitry Andric int NumLaneElts = 128/CI->getType()->getScalarSizeInBits(); 26700b57cec5SDimitry Andric 26710b57cec5SDimitry Andric SmallVector<uint32_t, 64> Idxs(NumElts); 26720b57cec5SDimitry Andric for (int l = 0; l != NumElts; l += NumLaneElts) 26730b57cec5SDimitry Andric for (int i = 0; i != NumLaneElts; ++i) 26740b57cec5SDimitry Andric Idxs[i + l] = (NumLaneElts / 2) + l + (i / 2) + NumElts * (i % 2); 26750b57cec5SDimitry Andric 26760b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Op0, Op1, Idxs); 26770b57cec5SDimitry Andric 26780b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 26790b57cec5SDimitry Andric CI->getArgOperand(2)); 26800b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.and.") || 26810b57cec5SDimitry Andric Name.startswith("avx512.mask.pand."))) { 26820b57cec5SDimitry Andric VectorType *FTy = cast<VectorType>(CI->getType()); 26830b57cec5SDimitry Andric VectorType *ITy = VectorType::getInteger(FTy); 26840b57cec5SDimitry Andric Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy), 26850b57cec5SDimitry Andric Builder.CreateBitCast(CI->getArgOperand(1), ITy)); 26860b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, FTy); 26870b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 26880b57cec5SDimitry Andric CI->getArgOperand(2)); 26890b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.andn.") || 26900b57cec5SDimitry Andric Name.startswith("avx512.mask.pandn."))) { 26910b57cec5SDimitry Andric VectorType *FTy = cast<VectorType>(CI->getType()); 26920b57cec5SDimitry Andric VectorType *ITy = VectorType::getInteger(FTy); 26930b57cec5SDimitry Andric Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy)); 26940b57cec5SDimitry Andric Rep = Builder.CreateAnd(Rep, 26950b57cec5SDimitry Andric Builder.CreateBitCast(CI->getArgOperand(1), ITy)); 26960b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, FTy); 26970b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 26980b57cec5SDimitry Andric CI->getArgOperand(2)); 26990b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.or.") || 27000b57cec5SDimitry Andric Name.startswith("avx512.mask.por."))) { 27010b57cec5SDimitry Andric VectorType *FTy = cast<VectorType>(CI->getType()); 27020b57cec5SDimitry Andric VectorType *ITy = VectorType::getInteger(FTy); 27030b57cec5SDimitry Andric Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy), 27040b57cec5SDimitry Andric Builder.CreateBitCast(CI->getArgOperand(1), ITy)); 27050b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, FTy); 27060b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27070b57cec5SDimitry Andric CI->getArgOperand(2)); 27080b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.xor.") || 27090b57cec5SDimitry Andric Name.startswith("avx512.mask.pxor."))) { 27100b57cec5SDimitry Andric VectorType *FTy = cast<VectorType>(CI->getType()); 27110b57cec5SDimitry Andric VectorType *ITy = VectorType::getInteger(FTy); 27120b57cec5SDimitry Andric Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy), 27130b57cec5SDimitry Andric Builder.CreateBitCast(CI->getArgOperand(1), ITy)); 27140b57cec5SDimitry Andric Rep = Builder.CreateBitCast(Rep, FTy); 27150b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27160b57cec5SDimitry Andric CI->getArgOperand(2)); 27170b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.padd.")) { 27180b57cec5SDimitry Andric Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1)); 27190b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27200b57cec5SDimitry Andric CI->getArgOperand(2)); 27210b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.psub.")) { 27220b57cec5SDimitry Andric Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1)); 27230b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27240b57cec5SDimitry Andric CI->getArgOperand(2)); 27250b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.pmull.")) { 27260b57cec5SDimitry Andric Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1)); 27270b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27280b57cec5SDimitry Andric CI->getArgOperand(2)); 27290b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.add.p")) { 27300b57cec5SDimitry Andric if (Name.endswith(".512")) { 27310b57cec5SDimitry Andric Intrinsic::ID IID; 27320b57cec5SDimitry Andric if (Name[17] == 's') 27330b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_add_ps_512; 27340b57cec5SDimitry Andric else 27350b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_add_pd_512; 27360b57cec5SDimitry Andric 27370b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 27380b57cec5SDimitry Andric { CI->getArgOperand(0), CI->getArgOperand(1), 27390b57cec5SDimitry Andric CI->getArgOperand(4) }); 27400b57cec5SDimitry Andric } else { 27410b57cec5SDimitry Andric Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1)); 27420b57cec5SDimitry Andric } 27430b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27440b57cec5SDimitry Andric CI->getArgOperand(2)); 27450b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.div.p")) { 27460b57cec5SDimitry Andric if (Name.endswith(".512")) { 27470b57cec5SDimitry Andric Intrinsic::ID IID; 27480b57cec5SDimitry Andric if (Name[17] == 's') 27490b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_div_ps_512; 27500b57cec5SDimitry Andric else 27510b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_div_pd_512; 27520b57cec5SDimitry Andric 27530b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 27540b57cec5SDimitry Andric { CI->getArgOperand(0), CI->getArgOperand(1), 27550b57cec5SDimitry Andric CI->getArgOperand(4) }); 27560b57cec5SDimitry Andric } else { 27570b57cec5SDimitry Andric Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1)); 27580b57cec5SDimitry Andric } 27590b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27600b57cec5SDimitry Andric CI->getArgOperand(2)); 27610b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.mul.p")) { 27620b57cec5SDimitry Andric if (Name.endswith(".512")) { 27630b57cec5SDimitry Andric Intrinsic::ID IID; 27640b57cec5SDimitry Andric if (Name[17] == 's') 27650b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_mul_ps_512; 27660b57cec5SDimitry Andric else 27670b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_mul_pd_512; 27680b57cec5SDimitry Andric 27690b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 27700b57cec5SDimitry Andric { CI->getArgOperand(0), CI->getArgOperand(1), 27710b57cec5SDimitry Andric CI->getArgOperand(4) }); 27720b57cec5SDimitry Andric } else { 27730b57cec5SDimitry Andric Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1)); 27740b57cec5SDimitry Andric } 27750b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27760b57cec5SDimitry Andric CI->getArgOperand(2)); 27770b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.sub.p")) { 27780b57cec5SDimitry Andric if (Name.endswith(".512")) { 27790b57cec5SDimitry Andric Intrinsic::ID IID; 27800b57cec5SDimitry Andric if (Name[17] == 's') 27810b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_sub_ps_512; 27820b57cec5SDimitry Andric else 27830b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_sub_pd_512; 27840b57cec5SDimitry Andric 27850b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 27860b57cec5SDimitry Andric { CI->getArgOperand(0), CI->getArgOperand(1), 27870b57cec5SDimitry Andric CI->getArgOperand(4) }); 27880b57cec5SDimitry Andric } else { 27890b57cec5SDimitry Andric Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1)); 27900b57cec5SDimitry Andric } 27910b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 27920b57cec5SDimitry Andric CI->getArgOperand(2)); 27930b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.max.p") || 27940b57cec5SDimitry Andric Name.startswith("avx512.mask.min.p")) && 27950b57cec5SDimitry Andric Name.drop_front(18) == ".512") { 27960b57cec5SDimitry Andric bool IsDouble = Name[17] == 'd'; 27970b57cec5SDimitry Andric bool IsMin = Name[13] == 'i'; 27980b57cec5SDimitry Andric static const Intrinsic::ID MinMaxTbl[2][2] = { 27990b57cec5SDimitry Andric { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 }, 28000b57cec5SDimitry Andric { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 } 28010b57cec5SDimitry Andric }; 28020b57cec5SDimitry Andric Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble]; 28030b57cec5SDimitry Andric 28040b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 28050b57cec5SDimitry Andric { CI->getArgOperand(0), CI->getArgOperand(1), 28060b57cec5SDimitry Andric CI->getArgOperand(4) }); 28070b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, 28080b57cec5SDimitry Andric CI->getArgOperand(2)); 28090b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.lzcnt.")) { 28100b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), 28110b57cec5SDimitry Andric Intrinsic::ctlz, 28120b57cec5SDimitry Andric CI->getType()), 28130b57cec5SDimitry Andric { CI->getArgOperand(0), Builder.getInt1(false) }); 28140b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep, 28150b57cec5SDimitry Andric CI->getArgOperand(1)); 28160b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.psll")) { 28170b57cec5SDimitry Andric bool IsImmediate = Name[16] == 'i' || 28180b57cec5SDimitry Andric (Name.size() > 18 && Name[18] == 'i'); 28190b57cec5SDimitry Andric bool IsVariable = Name[16] == 'v'; 28200b57cec5SDimitry Andric char Size = Name[16] == '.' ? Name[17] : 28210b57cec5SDimitry Andric Name[17] == '.' ? Name[18] : 28220b57cec5SDimitry Andric Name[18] == '.' ? Name[19] : 28230b57cec5SDimitry Andric Name[20]; 28240b57cec5SDimitry Andric 28250b57cec5SDimitry Andric Intrinsic::ID IID; 28260b57cec5SDimitry Andric if (IsVariable && Name[17] != '.') { 28270b57cec5SDimitry Andric if (Size == 'd' && Name[17] == '2') // avx512.mask.psllv2.di 28280b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psllv_q; 28290b57cec5SDimitry Andric else if (Size == 'd' && Name[17] == '4') // avx512.mask.psllv4.di 28300b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psllv_q_256; 28310b57cec5SDimitry Andric else if (Size == 's' && Name[17] == '4') // avx512.mask.psllv4.si 28320b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psllv_d; 28330b57cec5SDimitry Andric else if (Size == 's' && Name[17] == '8') // avx512.mask.psllv8.si 28340b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psllv_d_256; 28350b57cec5SDimitry Andric else if (Size == 'h' && Name[17] == '8') // avx512.mask.psllv8.hi 28360b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psllv_w_128; 28370b57cec5SDimitry Andric else if (Size == 'h' && Name[17] == '1') // avx512.mask.psllv16.hi 28380b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psllv_w_256; 28390b57cec5SDimitry Andric else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psllv32hi 28400b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psllv_w_512; 28410b57cec5SDimitry Andric else 28420b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 28430b57cec5SDimitry Andric } else if (Name.endswith(".128")) { 28440b57cec5SDimitry Andric if (Size == 'd') // avx512.mask.psll.d.128, avx512.mask.psll.di.128 28450b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_sse2_pslli_d 28460b57cec5SDimitry Andric : Intrinsic::x86_sse2_psll_d; 28470b57cec5SDimitry Andric else if (Size == 'q') // avx512.mask.psll.q.128, avx512.mask.psll.qi.128 28480b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_sse2_pslli_q 28490b57cec5SDimitry Andric : Intrinsic::x86_sse2_psll_q; 28500b57cec5SDimitry Andric else if (Size == 'w') // avx512.mask.psll.w.128, avx512.mask.psll.wi.128 28510b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_sse2_pslli_w 28520b57cec5SDimitry Andric : Intrinsic::x86_sse2_psll_w; 28530b57cec5SDimitry Andric else 28540b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 28550b57cec5SDimitry Andric } else if (Name.endswith(".256")) { 28560b57cec5SDimitry Andric if (Size == 'd') // avx512.mask.psll.d.256, avx512.mask.psll.di.256 28570b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx2_pslli_d 28580b57cec5SDimitry Andric : Intrinsic::x86_avx2_psll_d; 28590b57cec5SDimitry Andric else if (Size == 'q') // avx512.mask.psll.q.256, avx512.mask.psll.qi.256 28600b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx2_pslli_q 28610b57cec5SDimitry Andric : Intrinsic::x86_avx2_psll_q; 28620b57cec5SDimitry Andric else if (Size == 'w') // avx512.mask.psll.w.256, avx512.mask.psll.wi.256 28630b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx2_pslli_w 28640b57cec5SDimitry Andric : Intrinsic::x86_avx2_psll_w; 28650b57cec5SDimitry Andric else 28660b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 28670b57cec5SDimitry Andric } else { 28680b57cec5SDimitry Andric if (Size == 'd') // psll.di.512, pslli.d, psll.d, psllv.d.512 28690b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_pslli_d_512 : 28700b57cec5SDimitry Andric IsVariable ? Intrinsic::x86_avx512_psllv_d_512 : 28710b57cec5SDimitry Andric Intrinsic::x86_avx512_psll_d_512; 28720b57cec5SDimitry Andric else if (Size == 'q') // psll.qi.512, pslli.q, psll.q, psllv.q.512 28730b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_pslli_q_512 : 28740b57cec5SDimitry Andric IsVariable ? Intrinsic::x86_avx512_psllv_q_512 : 28750b57cec5SDimitry Andric Intrinsic::x86_avx512_psll_q_512; 28760b57cec5SDimitry Andric else if (Size == 'w') // psll.wi.512, pslli.w, psll.w 28770b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_pslli_w_512 28780b57cec5SDimitry Andric : Intrinsic::x86_avx512_psll_w_512; 28790b57cec5SDimitry Andric else 28800b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 28810b57cec5SDimitry Andric } 28820b57cec5SDimitry Andric 28830b57cec5SDimitry Andric Rep = UpgradeX86MaskedShift(Builder, *CI, IID); 28840b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.psrl")) { 28850b57cec5SDimitry Andric bool IsImmediate = Name[16] == 'i' || 28860b57cec5SDimitry Andric (Name.size() > 18 && Name[18] == 'i'); 28870b57cec5SDimitry Andric bool IsVariable = Name[16] == 'v'; 28880b57cec5SDimitry Andric char Size = Name[16] == '.' ? Name[17] : 28890b57cec5SDimitry Andric Name[17] == '.' ? Name[18] : 28900b57cec5SDimitry Andric Name[18] == '.' ? Name[19] : 28910b57cec5SDimitry Andric Name[20]; 28920b57cec5SDimitry Andric 28930b57cec5SDimitry Andric Intrinsic::ID IID; 28940b57cec5SDimitry Andric if (IsVariable && Name[17] != '.') { 28950b57cec5SDimitry Andric if (Size == 'd' && Name[17] == '2') // avx512.mask.psrlv2.di 28960b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psrlv_q; 28970b57cec5SDimitry Andric else if (Size == 'd' && Name[17] == '4') // avx512.mask.psrlv4.di 28980b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psrlv_q_256; 28990b57cec5SDimitry Andric else if (Size == 's' && Name[17] == '4') // avx512.mask.psrlv4.si 29000b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psrlv_d; 29010b57cec5SDimitry Andric else if (Size == 's' && Name[17] == '8') // avx512.mask.psrlv8.si 29020b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psrlv_d_256; 29030b57cec5SDimitry Andric else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrlv8.hi 29040b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psrlv_w_128; 29050b57cec5SDimitry Andric else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrlv16.hi 29060b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psrlv_w_256; 29070b57cec5SDimitry Andric else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrlv32hi 29080b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psrlv_w_512; 29090b57cec5SDimitry Andric else 29100b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 29110b57cec5SDimitry Andric } else if (Name.endswith(".128")) { 29120b57cec5SDimitry Andric if (Size == 'd') // avx512.mask.psrl.d.128, avx512.mask.psrl.di.128 29130b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_sse2_psrli_d 29140b57cec5SDimitry Andric : Intrinsic::x86_sse2_psrl_d; 29150b57cec5SDimitry Andric else if (Size == 'q') // avx512.mask.psrl.q.128, avx512.mask.psrl.qi.128 29160b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_sse2_psrli_q 29170b57cec5SDimitry Andric : Intrinsic::x86_sse2_psrl_q; 29180b57cec5SDimitry Andric else if (Size == 'w') // avx512.mask.psrl.w.128, avx512.mask.psrl.wi.128 29190b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_sse2_psrli_w 29200b57cec5SDimitry Andric : Intrinsic::x86_sse2_psrl_w; 29210b57cec5SDimitry Andric else 29220b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 29230b57cec5SDimitry Andric } else if (Name.endswith(".256")) { 29240b57cec5SDimitry Andric if (Size == 'd') // avx512.mask.psrl.d.256, avx512.mask.psrl.di.256 29250b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx2_psrli_d 29260b57cec5SDimitry Andric : Intrinsic::x86_avx2_psrl_d; 29270b57cec5SDimitry Andric else if (Size == 'q') // avx512.mask.psrl.q.256, avx512.mask.psrl.qi.256 29280b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx2_psrli_q 29290b57cec5SDimitry Andric : Intrinsic::x86_avx2_psrl_q; 29300b57cec5SDimitry Andric else if (Size == 'w') // avx512.mask.psrl.w.256, avx512.mask.psrl.wi.256 29310b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx2_psrli_w 29320b57cec5SDimitry Andric : Intrinsic::x86_avx2_psrl_w; 29330b57cec5SDimitry Andric else 29340b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 29350b57cec5SDimitry Andric } else { 29360b57cec5SDimitry Andric if (Size == 'd') // psrl.di.512, psrli.d, psrl.d, psrl.d.512 29370b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_psrli_d_512 : 29380b57cec5SDimitry Andric IsVariable ? Intrinsic::x86_avx512_psrlv_d_512 : 29390b57cec5SDimitry Andric Intrinsic::x86_avx512_psrl_d_512; 29400b57cec5SDimitry Andric else if (Size == 'q') // psrl.qi.512, psrli.q, psrl.q, psrl.q.512 29410b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_psrli_q_512 : 29420b57cec5SDimitry Andric IsVariable ? Intrinsic::x86_avx512_psrlv_q_512 : 29430b57cec5SDimitry Andric Intrinsic::x86_avx512_psrl_q_512; 29440b57cec5SDimitry Andric else if (Size == 'w') // psrl.wi.512, psrli.w, psrl.w) 29450b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_psrli_w_512 29460b57cec5SDimitry Andric : Intrinsic::x86_avx512_psrl_w_512; 29470b57cec5SDimitry Andric else 29480b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 29490b57cec5SDimitry Andric } 29500b57cec5SDimitry Andric 29510b57cec5SDimitry Andric Rep = UpgradeX86MaskedShift(Builder, *CI, IID); 29520b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.psra")) { 29530b57cec5SDimitry Andric bool IsImmediate = Name[16] == 'i' || 29540b57cec5SDimitry Andric (Name.size() > 18 && Name[18] == 'i'); 29550b57cec5SDimitry Andric bool IsVariable = Name[16] == 'v'; 29560b57cec5SDimitry Andric char Size = Name[16] == '.' ? Name[17] : 29570b57cec5SDimitry Andric Name[17] == '.' ? Name[18] : 29580b57cec5SDimitry Andric Name[18] == '.' ? Name[19] : 29590b57cec5SDimitry Andric Name[20]; 29600b57cec5SDimitry Andric 29610b57cec5SDimitry Andric Intrinsic::ID IID; 29620b57cec5SDimitry Andric if (IsVariable && Name[17] != '.') { 29630b57cec5SDimitry Andric if (Size == 's' && Name[17] == '4') // avx512.mask.psrav4.si 29640b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psrav_d; 29650b57cec5SDimitry Andric else if (Size == 's' && Name[17] == '8') // avx512.mask.psrav8.si 29660b57cec5SDimitry Andric IID = Intrinsic::x86_avx2_psrav_d_256; 29670b57cec5SDimitry Andric else if (Size == 'h' && Name[17] == '8') // avx512.mask.psrav8.hi 29680b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psrav_w_128; 29690b57cec5SDimitry Andric else if (Size == 'h' && Name[17] == '1') // avx512.mask.psrav16.hi 29700b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psrav_w_256; 29710b57cec5SDimitry Andric else if (Name[17] == '3' && Name[18] == '2') // avx512.mask.psrav32hi 29720b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_psrav_w_512; 29730b57cec5SDimitry Andric else 29740b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 29750b57cec5SDimitry Andric } else if (Name.endswith(".128")) { 29760b57cec5SDimitry Andric if (Size == 'd') // avx512.mask.psra.d.128, avx512.mask.psra.di.128 29770b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_sse2_psrai_d 29780b57cec5SDimitry Andric : Intrinsic::x86_sse2_psra_d; 29790b57cec5SDimitry Andric else if (Size == 'q') // avx512.mask.psra.q.128, avx512.mask.psra.qi.128 29800b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_128 : 29810b57cec5SDimitry Andric IsVariable ? Intrinsic::x86_avx512_psrav_q_128 : 29820b57cec5SDimitry Andric Intrinsic::x86_avx512_psra_q_128; 29830b57cec5SDimitry Andric else if (Size == 'w') // avx512.mask.psra.w.128, avx512.mask.psra.wi.128 29840b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_sse2_psrai_w 29850b57cec5SDimitry Andric : Intrinsic::x86_sse2_psra_w; 29860b57cec5SDimitry Andric else 29870b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 29880b57cec5SDimitry Andric } else if (Name.endswith(".256")) { 29890b57cec5SDimitry Andric if (Size == 'd') // avx512.mask.psra.d.256, avx512.mask.psra.di.256 29900b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx2_psrai_d 29910b57cec5SDimitry Andric : Intrinsic::x86_avx2_psra_d; 29920b57cec5SDimitry Andric else if (Size == 'q') // avx512.mask.psra.q.256, avx512.mask.psra.qi.256 29930b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_256 : 29940b57cec5SDimitry Andric IsVariable ? Intrinsic::x86_avx512_psrav_q_256 : 29950b57cec5SDimitry Andric Intrinsic::x86_avx512_psra_q_256; 29960b57cec5SDimitry Andric else if (Size == 'w') // avx512.mask.psra.w.256, avx512.mask.psra.wi.256 29970b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx2_psrai_w 29980b57cec5SDimitry Andric : Intrinsic::x86_avx2_psra_w; 29990b57cec5SDimitry Andric else 30000b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 30010b57cec5SDimitry Andric } else { 30020b57cec5SDimitry Andric if (Size == 'd') // psra.di.512, psrai.d, psra.d, psrav.d.512 30030b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_psrai_d_512 : 30040b57cec5SDimitry Andric IsVariable ? Intrinsic::x86_avx512_psrav_d_512 : 30050b57cec5SDimitry Andric Intrinsic::x86_avx512_psra_d_512; 30060b57cec5SDimitry Andric else if (Size == 'q') // psra.qi.512, psrai.q, psra.q 30070b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_psrai_q_512 : 30080b57cec5SDimitry Andric IsVariable ? Intrinsic::x86_avx512_psrav_q_512 : 30090b57cec5SDimitry Andric Intrinsic::x86_avx512_psra_q_512; 30100b57cec5SDimitry Andric else if (Size == 'w') // psra.wi.512, psrai.w, psra.w 30110b57cec5SDimitry Andric IID = IsImmediate ? Intrinsic::x86_avx512_psrai_w_512 30120b57cec5SDimitry Andric : Intrinsic::x86_avx512_psra_w_512; 30130b57cec5SDimitry Andric else 30140b57cec5SDimitry Andric llvm_unreachable("Unexpected size"); 30150b57cec5SDimitry Andric } 30160b57cec5SDimitry Andric 30170b57cec5SDimitry Andric Rep = UpgradeX86MaskedShift(Builder, *CI, IID); 30180b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.move.s")) { 30190b57cec5SDimitry Andric Rep = upgradeMaskedMove(Builder, *CI); 30200b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.cvtmask2")) { 30210b57cec5SDimitry Andric Rep = UpgradeMaskToInt(Builder, *CI); 30220b57cec5SDimitry Andric } else if (IsX86 && Name.endswith(".movntdqa")) { 30230b57cec5SDimitry Andric Module *M = F->getParent(); 30240b57cec5SDimitry Andric MDNode *Node = MDNode::get( 30250b57cec5SDimitry Andric C, ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(C), 1))); 30260b57cec5SDimitry Andric 30270b57cec5SDimitry Andric Value *Ptr = CI->getArgOperand(0); 30280b57cec5SDimitry Andric VectorType *VTy = cast<VectorType>(CI->getType()); 30290b57cec5SDimitry Andric 30300b57cec5SDimitry Andric // Convert the type of the pointer to a pointer to the stored type. 30310b57cec5SDimitry Andric Value *BC = 30320b57cec5SDimitry Andric Builder.CreateBitCast(Ptr, PointerType::getUnqual(VTy), "cast"); 30330b57cec5SDimitry Andric LoadInst *LI = Builder.CreateAlignedLoad(VTy, BC, VTy->getBitWidth() / 8); 30340b57cec5SDimitry Andric LI->setMetadata(M->getMDKindID("nontemporal"), Node); 30350b57cec5SDimitry Andric Rep = LI; 30360b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("fma.vfmadd.") || 30370b57cec5SDimitry Andric Name.startswith("fma.vfmsub.") || 30380b57cec5SDimitry Andric Name.startswith("fma.vfnmadd.") || 30390b57cec5SDimitry Andric Name.startswith("fma.vfnmsub."))) { 30400b57cec5SDimitry Andric bool NegMul = Name[6] == 'n'; 30410b57cec5SDimitry Andric bool NegAcc = NegMul ? Name[8] == 's' : Name[7] == 's'; 30420b57cec5SDimitry Andric bool IsScalar = NegMul ? Name[12] == 's' : Name[11] == 's'; 30430b57cec5SDimitry Andric 30440b57cec5SDimitry Andric Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), 30450b57cec5SDimitry Andric CI->getArgOperand(2) }; 30460b57cec5SDimitry Andric 30470b57cec5SDimitry Andric if (IsScalar) { 30480b57cec5SDimitry Andric Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0); 30490b57cec5SDimitry Andric Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0); 30500b57cec5SDimitry Andric Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0); 30510b57cec5SDimitry Andric } 30520b57cec5SDimitry Andric 30530b57cec5SDimitry Andric if (NegMul && !IsScalar) 30540b57cec5SDimitry Andric Ops[0] = Builder.CreateFNeg(Ops[0]); 30550b57cec5SDimitry Andric if (NegMul && IsScalar) 30560b57cec5SDimitry Andric Ops[1] = Builder.CreateFNeg(Ops[1]); 30570b57cec5SDimitry Andric if (NegAcc) 30580b57cec5SDimitry Andric Ops[2] = Builder.CreateFNeg(Ops[2]); 30590b57cec5SDimitry Andric 30600b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), 30610b57cec5SDimitry Andric Intrinsic::fma, 30620b57cec5SDimitry Andric Ops[0]->getType()), 30630b57cec5SDimitry Andric Ops); 30640b57cec5SDimitry Andric 30650b57cec5SDimitry Andric if (IsScalar) 30660b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, 30670b57cec5SDimitry Andric (uint64_t)0); 30680b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("fma4.vfmadd.s")) { 30690b57cec5SDimitry Andric Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), 30700b57cec5SDimitry Andric CI->getArgOperand(2) }; 30710b57cec5SDimitry Andric 30720b57cec5SDimitry Andric Ops[0] = Builder.CreateExtractElement(Ops[0], (uint64_t)0); 30730b57cec5SDimitry Andric Ops[1] = Builder.CreateExtractElement(Ops[1], (uint64_t)0); 30740b57cec5SDimitry Andric Ops[2] = Builder.CreateExtractElement(Ops[2], (uint64_t)0); 30750b57cec5SDimitry Andric 30760b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), 30770b57cec5SDimitry Andric Intrinsic::fma, 30780b57cec5SDimitry Andric Ops[0]->getType()), 30790b57cec5SDimitry Andric Ops); 30800b57cec5SDimitry Andric 30810b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()), 30820b57cec5SDimitry Andric Rep, (uint64_t)0); 30830b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.s") || 30840b57cec5SDimitry Andric Name.startswith("avx512.maskz.vfmadd.s") || 30850b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmadd.s") || 30860b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmsub.s") || 30870b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfnmsub.s"))) { 30880b57cec5SDimitry Andric bool IsMask3 = Name[11] == '3'; 30890b57cec5SDimitry Andric bool IsMaskZ = Name[11] == 'z'; 30900b57cec5SDimitry Andric // Drop the "avx512.mask." to make it easier. 30910b57cec5SDimitry Andric Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12); 30920b57cec5SDimitry Andric bool NegMul = Name[2] == 'n'; 30930b57cec5SDimitry Andric bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's'; 30940b57cec5SDimitry Andric 30950b57cec5SDimitry Andric Value *A = CI->getArgOperand(0); 30960b57cec5SDimitry Andric Value *B = CI->getArgOperand(1); 30970b57cec5SDimitry Andric Value *C = CI->getArgOperand(2); 30980b57cec5SDimitry Andric 30990b57cec5SDimitry Andric if (NegMul && (IsMask3 || IsMaskZ)) 31000b57cec5SDimitry Andric A = Builder.CreateFNeg(A); 31010b57cec5SDimitry Andric if (NegMul && !(IsMask3 || IsMaskZ)) 31020b57cec5SDimitry Andric B = Builder.CreateFNeg(B); 31030b57cec5SDimitry Andric if (NegAcc) 31040b57cec5SDimitry Andric C = Builder.CreateFNeg(C); 31050b57cec5SDimitry Andric 31060b57cec5SDimitry Andric A = Builder.CreateExtractElement(A, (uint64_t)0); 31070b57cec5SDimitry Andric B = Builder.CreateExtractElement(B, (uint64_t)0); 31080b57cec5SDimitry Andric C = Builder.CreateExtractElement(C, (uint64_t)0); 31090b57cec5SDimitry Andric 31100b57cec5SDimitry Andric if (!isa<ConstantInt>(CI->getArgOperand(4)) || 31110b57cec5SDimitry Andric cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) { 31120b57cec5SDimitry Andric Value *Ops[] = { A, B, C, CI->getArgOperand(4) }; 31130b57cec5SDimitry Andric 31140b57cec5SDimitry Andric Intrinsic::ID IID; 31150b57cec5SDimitry Andric if (Name.back() == 'd') 31160b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vfmadd_f64; 31170b57cec5SDimitry Andric else 31180b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vfmadd_f32; 31190b57cec5SDimitry Andric Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID); 31200b57cec5SDimitry Andric Rep = Builder.CreateCall(FMA, Ops); 31210b57cec5SDimitry Andric } else { 31220b57cec5SDimitry Andric Function *FMA = Intrinsic::getDeclaration(CI->getModule(), 31230b57cec5SDimitry Andric Intrinsic::fma, 31240b57cec5SDimitry Andric A->getType()); 31250b57cec5SDimitry Andric Rep = Builder.CreateCall(FMA, { A, B, C }); 31260b57cec5SDimitry Andric } 31270b57cec5SDimitry Andric 31280b57cec5SDimitry Andric Value *PassThru = IsMaskZ ? Constant::getNullValue(Rep->getType()) : 31290b57cec5SDimitry Andric IsMask3 ? C : A; 31300b57cec5SDimitry Andric 31310b57cec5SDimitry Andric // For Mask3 with NegAcc, we need to create a new extractelement that 31320b57cec5SDimitry Andric // avoids the negation above. 31330b57cec5SDimitry Andric if (NegAcc && IsMask3) 31340b57cec5SDimitry Andric PassThru = Builder.CreateExtractElement(CI->getArgOperand(2), 31350b57cec5SDimitry Andric (uint64_t)0); 31360b57cec5SDimitry Andric 31370b57cec5SDimitry Andric Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3), 31380b57cec5SDimitry Andric Rep, PassThru); 31390b57cec5SDimitry Andric Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0), 31400b57cec5SDimitry Andric Rep, (uint64_t)0); 31410b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.vfmadd.p") || 31420b57cec5SDimitry Andric Name.startswith("avx512.mask.vfnmadd.p") || 31430b57cec5SDimitry Andric Name.startswith("avx512.mask.vfnmsub.p") || 31440b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmadd.p") || 31450b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmsub.p") || 31460b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfnmsub.p") || 31470b57cec5SDimitry Andric Name.startswith("avx512.maskz.vfmadd.p"))) { 31480b57cec5SDimitry Andric bool IsMask3 = Name[11] == '3'; 31490b57cec5SDimitry Andric bool IsMaskZ = Name[11] == 'z'; 31500b57cec5SDimitry Andric // Drop the "avx512.mask." to make it easier. 31510b57cec5SDimitry Andric Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12); 31520b57cec5SDimitry Andric bool NegMul = Name[2] == 'n'; 31530b57cec5SDimitry Andric bool NegAcc = NegMul ? Name[4] == 's' : Name[3] == 's'; 31540b57cec5SDimitry Andric 31550b57cec5SDimitry Andric Value *A = CI->getArgOperand(0); 31560b57cec5SDimitry Andric Value *B = CI->getArgOperand(1); 31570b57cec5SDimitry Andric Value *C = CI->getArgOperand(2); 31580b57cec5SDimitry Andric 31590b57cec5SDimitry Andric if (NegMul && (IsMask3 || IsMaskZ)) 31600b57cec5SDimitry Andric A = Builder.CreateFNeg(A); 31610b57cec5SDimitry Andric if (NegMul && !(IsMask3 || IsMaskZ)) 31620b57cec5SDimitry Andric B = Builder.CreateFNeg(B); 31630b57cec5SDimitry Andric if (NegAcc) 31640b57cec5SDimitry Andric C = Builder.CreateFNeg(C); 31650b57cec5SDimitry Andric 31660b57cec5SDimitry Andric if (CI->getNumArgOperands() == 5 && 31670b57cec5SDimitry Andric (!isa<ConstantInt>(CI->getArgOperand(4)) || 31680b57cec5SDimitry Andric cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) { 31690b57cec5SDimitry Andric Intrinsic::ID IID; 31700b57cec5SDimitry Andric // Check the character before ".512" in string. 31710b57cec5SDimitry Andric if (Name[Name.size()-5] == 's') 31720b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vfmadd_ps_512; 31730b57cec5SDimitry Andric else 31740b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vfmadd_pd_512; 31750b57cec5SDimitry Andric 31760b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 31770b57cec5SDimitry Andric { A, B, C, CI->getArgOperand(4) }); 31780b57cec5SDimitry Andric } else { 31790b57cec5SDimitry Andric Function *FMA = Intrinsic::getDeclaration(CI->getModule(), 31800b57cec5SDimitry Andric Intrinsic::fma, 31810b57cec5SDimitry Andric A->getType()); 31820b57cec5SDimitry Andric Rep = Builder.CreateCall(FMA, { A, B, C }); 31830b57cec5SDimitry Andric } 31840b57cec5SDimitry Andric 31850b57cec5SDimitry Andric Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) : 31860b57cec5SDimitry Andric IsMask3 ? CI->getArgOperand(2) : 31870b57cec5SDimitry Andric CI->getArgOperand(0); 31880b57cec5SDimitry Andric 31890b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); 31900b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("fma.vfmaddsub.p") || 31910b57cec5SDimitry Andric Name.startswith("fma.vfmsubadd.p"))) { 31920b57cec5SDimitry Andric bool IsSubAdd = Name[7] == 's'; 31930b57cec5SDimitry Andric int NumElts = CI->getType()->getVectorNumElements(); 31940b57cec5SDimitry Andric 31950b57cec5SDimitry Andric Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), 31960b57cec5SDimitry Andric CI->getArgOperand(2) }; 31970b57cec5SDimitry Andric 31980b57cec5SDimitry Andric Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, 31990b57cec5SDimitry Andric Ops[0]->getType()); 32000b57cec5SDimitry Andric Value *Odd = Builder.CreateCall(FMA, Ops); 32010b57cec5SDimitry Andric Ops[2] = Builder.CreateFNeg(Ops[2]); 32020b57cec5SDimitry Andric Value *Even = Builder.CreateCall(FMA, Ops); 32030b57cec5SDimitry Andric 32040b57cec5SDimitry Andric if (IsSubAdd) 32050b57cec5SDimitry Andric std::swap(Even, Odd); 32060b57cec5SDimitry Andric 32070b57cec5SDimitry Andric SmallVector<uint32_t, 32> Idxs(NumElts); 32080b57cec5SDimitry Andric for (int i = 0; i != NumElts; ++i) 32090b57cec5SDimitry Andric Idxs[i] = i + (i % 2) * NumElts; 32100b57cec5SDimitry Andric 32110b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Even, Odd, Idxs); 32120b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.vfmaddsub.p") || 32130b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmaddsub.p") || 32140b57cec5SDimitry Andric Name.startswith("avx512.maskz.vfmaddsub.p") || 32150b57cec5SDimitry Andric Name.startswith("avx512.mask3.vfmsubadd.p"))) { 32160b57cec5SDimitry Andric bool IsMask3 = Name[11] == '3'; 32170b57cec5SDimitry Andric bool IsMaskZ = Name[11] == 'z'; 32180b57cec5SDimitry Andric // Drop the "avx512.mask." to make it easier. 32190b57cec5SDimitry Andric Name = Name.drop_front(IsMask3 || IsMaskZ ? 13 : 12); 32200b57cec5SDimitry Andric bool IsSubAdd = Name[3] == 's'; 32210b57cec5SDimitry Andric if (CI->getNumArgOperands() == 5 && 32220b57cec5SDimitry Andric (!isa<ConstantInt>(CI->getArgOperand(4)) || 32230b57cec5SDimitry Andric cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) { 32240b57cec5SDimitry Andric Intrinsic::ID IID; 32250b57cec5SDimitry Andric // Check the character before ".512" in string. 32260b57cec5SDimitry Andric if (Name[Name.size()-5] == 's') 32270b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vfmaddsub_ps_512; 32280b57cec5SDimitry Andric else 32290b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vfmaddsub_pd_512; 32300b57cec5SDimitry Andric 32310b57cec5SDimitry Andric Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), 32320b57cec5SDimitry Andric CI->getArgOperand(2), CI->getArgOperand(4) }; 32330b57cec5SDimitry Andric if (IsSubAdd) 32340b57cec5SDimitry Andric Ops[2] = Builder.CreateFNeg(Ops[2]); 32350b57cec5SDimitry Andric 32360b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID), 32370b57cec5SDimitry Andric {CI->getArgOperand(0), CI->getArgOperand(1), 32380b57cec5SDimitry Andric CI->getArgOperand(2), CI->getArgOperand(4)}); 32390b57cec5SDimitry Andric } else { 32400b57cec5SDimitry Andric int NumElts = CI->getType()->getVectorNumElements(); 32410b57cec5SDimitry Andric 32420b57cec5SDimitry Andric Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1), 32430b57cec5SDimitry Andric CI->getArgOperand(2) }; 32440b57cec5SDimitry Andric 32450b57cec5SDimitry Andric Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma, 32460b57cec5SDimitry Andric Ops[0]->getType()); 32470b57cec5SDimitry Andric Value *Odd = Builder.CreateCall(FMA, Ops); 32480b57cec5SDimitry Andric Ops[2] = Builder.CreateFNeg(Ops[2]); 32490b57cec5SDimitry Andric Value *Even = Builder.CreateCall(FMA, Ops); 32500b57cec5SDimitry Andric 32510b57cec5SDimitry Andric if (IsSubAdd) 32520b57cec5SDimitry Andric std::swap(Even, Odd); 32530b57cec5SDimitry Andric 32540b57cec5SDimitry Andric SmallVector<uint32_t, 32> Idxs(NumElts); 32550b57cec5SDimitry Andric for (int i = 0; i != NumElts; ++i) 32560b57cec5SDimitry Andric Idxs[i] = i + (i % 2) * NumElts; 32570b57cec5SDimitry Andric 32580b57cec5SDimitry Andric Rep = Builder.CreateShuffleVector(Even, Odd, Idxs); 32590b57cec5SDimitry Andric } 32600b57cec5SDimitry Andric 32610b57cec5SDimitry Andric Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) : 32620b57cec5SDimitry Andric IsMask3 ? CI->getArgOperand(2) : 32630b57cec5SDimitry Andric CI->getArgOperand(0); 32640b57cec5SDimitry Andric 32650b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); 32660b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.pternlog.") || 32670b57cec5SDimitry Andric Name.startswith("avx512.maskz.pternlog."))) { 32680b57cec5SDimitry Andric bool ZeroMask = Name[11] == 'z'; 32690b57cec5SDimitry Andric unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); 32700b57cec5SDimitry Andric unsigned EltWidth = CI->getType()->getScalarSizeInBits(); 32710b57cec5SDimitry Andric Intrinsic::ID IID; 32720b57cec5SDimitry Andric if (VecWidth == 128 && EltWidth == 32) 32730b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pternlog_d_128; 32740b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 32) 32750b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pternlog_d_256; 32760b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 32) 32770b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pternlog_d_512; 32780b57cec5SDimitry Andric else if (VecWidth == 128 && EltWidth == 64) 32790b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pternlog_q_128; 32800b57cec5SDimitry Andric else if (VecWidth == 256 && EltWidth == 64) 32810b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pternlog_q_256; 32820b57cec5SDimitry Andric else if (VecWidth == 512 && EltWidth == 64) 32830b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_pternlog_q_512; 32840b57cec5SDimitry Andric else 32850b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 32860b57cec5SDimitry Andric 32870b57cec5SDimitry Andric Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), 32880b57cec5SDimitry Andric CI->getArgOperand(2), CI->getArgOperand(3) }; 32890b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), 32900b57cec5SDimitry Andric Args); 32910b57cec5SDimitry Andric Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) 32920b57cec5SDimitry Andric : CI->getArgOperand(0); 32930b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru); 32940b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.vpmadd52") || 32950b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpmadd52"))) { 32960b57cec5SDimitry Andric bool ZeroMask = Name[11] == 'z'; 32970b57cec5SDimitry Andric bool High = Name[20] == 'h' || Name[21] == 'h'; 32980b57cec5SDimitry Andric unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); 32990b57cec5SDimitry Andric Intrinsic::ID IID; 33000b57cec5SDimitry Andric if (VecWidth == 128 && !High) 33010b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpmadd52l_uq_128; 33020b57cec5SDimitry Andric else if (VecWidth == 256 && !High) 33030b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpmadd52l_uq_256; 33040b57cec5SDimitry Andric else if (VecWidth == 512 && !High) 33050b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpmadd52l_uq_512; 33060b57cec5SDimitry Andric else if (VecWidth == 128 && High) 33070b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpmadd52h_uq_128; 33080b57cec5SDimitry Andric else if (VecWidth == 256 && High) 33090b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpmadd52h_uq_256; 33100b57cec5SDimitry Andric else if (VecWidth == 512 && High) 33110b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpmadd52h_uq_512; 33120b57cec5SDimitry Andric else 33130b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 33140b57cec5SDimitry Andric 33150b57cec5SDimitry Andric Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1), 33160b57cec5SDimitry Andric CI->getArgOperand(2) }; 33170b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), 33180b57cec5SDimitry Andric Args); 33190b57cec5SDimitry Andric Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) 33200b57cec5SDimitry Andric : CI->getArgOperand(0); 33210b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); 33220b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.vpermi2var.") || 33230b57cec5SDimitry Andric Name.startswith("avx512.mask.vpermt2var.") || 33240b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpermt2var."))) { 33250b57cec5SDimitry Andric bool ZeroMask = Name[11] == 'z'; 33260b57cec5SDimitry Andric bool IndexForm = Name[17] == 'i'; 33270b57cec5SDimitry Andric Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm); 33280b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") || 33290b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpdpbusd.") || 33300b57cec5SDimitry Andric Name.startswith("avx512.mask.vpdpbusds.") || 33310b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpdpbusds."))) { 33320b57cec5SDimitry Andric bool ZeroMask = Name[11] == 'z'; 33330b57cec5SDimitry Andric bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's'; 33340b57cec5SDimitry Andric unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); 33350b57cec5SDimitry Andric Intrinsic::ID IID; 33360b57cec5SDimitry Andric if (VecWidth == 128 && !IsSaturating) 33370b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpbusd_128; 33380b57cec5SDimitry Andric else if (VecWidth == 256 && !IsSaturating) 33390b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpbusd_256; 33400b57cec5SDimitry Andric else if (VecWidth == 512 && !IsSaturating) 33410b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpbusd_512; 33420b57cec5SDimitry Andric else if (VecWidth == 128 && IsSaturating) 33430b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpbusds_128; 33440b57cec5SDimitry Andric else if (VecWidth == 256 && IsSaturating) 33450b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpbusds_256; 33460b57cec5SDimitry Andric else if (VecWidth == 512 && IsSaturating) 33470b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpbusds_512; 33480b57cec5SDimitry Andric else 33490b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 33500b57cec5SDimitry Andric 33510b57cec5SDimitry Andric Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), 33520b57cec5SDimitry Andric CI->getArgOperand(2) }; 33530b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), 33540b57cec5SDimitry Andric Args); 33550b57cec5SDimitry Andric Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) 33560b57cec5SDimitry Andric : CI->getArgOperand(0); 33570b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); 33580b57cec5SDimitry Andric } else if (IsX86 && (Name.startswith("avx512.mask.vpdpwssd.") || 33590b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpdpwssd.") || 33600b57cec5SDimitry Andric Name.startswith("avx512.mask.vpdpwssds.") || 33610b57cec5SDimitry Andric Name.startswith("avx512.maskz.vpdpwssds."))) { 33620b57cec5SDimitry Andric bool ZeroMask = Name[11] == 'z'; 33630b57cec5SDimitry Andric bool IsSaturating = Name[ZeroMask ? 21 : 20] == 's'; 33640b57cec5SDimitry Andric unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits(); 33650b57cec5SDimitry Andric Intrinsic::ID IID; 33660b57cec5SDimitry Andric if (VecWidth == 128 && !IsSaturating) 33670b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpwssd_128; 33680b57cec5SDimitry Andric else if (VecWidth == 256 && !IsSaturating) 33690b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpwssd_256; 33700b57cec5SDimitry Andric else if (VecWidth == 512 && !IsSaturating) 33710b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpwssd_512; 33720b57cec5SDimitry Andric else if (VecWidth == 128 && IsSaturating) 33730b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpwssds_128; 33740b57cec5SDimitry Andric else if (VecWidth == 256 && IsSaturating) 33750b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpwssds_256; 33760b57cec5SDimitry Andric else if (VecWidth == 512 && IsSaturating) 33770b57cec5SDimitry Andric IID = Intrinsic::x86_avx512_vpdpwssds_512; 33780b57cec5SDimitry Andric else 33790b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 33800b57cec5SDimitry Andric 33810b57cec5SDimitry Andric Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), 33820b57cec5SDimitry Andric CI->getArgOperand(2) }; 33830b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID), 33840b57cec5SDimitry Andric Args); 33850b57cec5SDimitry Andric Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType()) 33860b57cec5SDimitry Andric : CI->getArgOperand(0); 33870b57cec5SDimitry Andric Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru); 33880b57cec5SDimitry Andric } else if (IsX86 && (Name == "addcarryx.u32" || Name == "addcarryx.u64" || 33890b57cec5SDimitry Andric Name == "addcarry.u32" || Name == "addcarry.u64" || 33900b57cec5SDimitry Andric Name == "subborrow.u32" || Name == "subborrow.u64")) { 33910b57cec5SDimitry Andric Intrinsic::ID IID; 33920b57cec5SDimitry Andric if (Name[0] == 'a' && Name.back() == '2') 33930b57cec5SDimitry Andric IID = Intrinsic::x86_addcarry_32; 33940b57cec5SDimitry Andric else if (Name[0] == 'a' && Name.back() == '4') 33950b57cec5SDimitry Andric IID = Intrinsic::x86_addcarry_64; 33960b57cec5SDimitry Andric else if (Name[0] == 's' && Name.back() == '2') 33970b57cec5SDimitry Andric IID = Intrinsic::x86_subborrow_32; 33980b57cec5SDimitry Andric else if (Name[0] == 's' && Name.back() == '4') 33990b57cec5SDimitry Andric IID = Intrinsic::x86_subborrow_64; 34000b57cec5SDimitry Andric else 34010b57cec5SDimitry Andric llvm_unreachable("Unexpected intrinsic"); 34020b57cec5SDimitry Andric 34030b57cec5SDimitry Andric // Make a call with 3 operands. 34040b57cec5SDimitry Andric Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1), 34050b57cec5SDimitry Andric CI->getArgOperand(2)}; 34060b57cec5SDimitry Andric Value *NewCall = Builder.CreateCall( 34070b57cec5SDimitry Andric Intrinsic::getDeclaration(CI->getModule(), IID), 34080b57cec5SDimitry Andric Args); 34090b57cec5SDimitry Andric 34100b57cec5SDimitry Andric // Extract the second result and store it. 34110b57cec5SDimitry Andric Value *Data = Builder.CreateExtractValue(NewCall, 1); 34120b57cec5SDimitry Andric // Cast the pointer to the right type. 34130b57cec5SDimitry Andric Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3), 34140b57cec5SDimitry Andric llvm::PointerType::getUnqual(Data->getType())); 34150b57cec5SDimitry Andric Builder.CreateAlignedStore(Data, Ptr, 1); 34160b57cec5SDimitry Andric // Replace the original call result with the first result of the new call. 34170b57cec5SDimitry Andric Value *CF = Builder.CreateExtractValue(NewCall, 0); 34180b57cec5SDimitry Andric 34190b57cec5SDimitry Andric CI->replaceAllUsesWith(CF); 34200b57cec5SDimitry Andric Rep = nullptr; 34210b57cec5SDimitry Andric } else if (IsX86 && Name.startswith("avx512.mask.") && 34220b57cec5SDimitry Andric upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) { 34230b57cec5SDimitry Andric // Rep will be updated by the call in the condition. 34240b57cec5SDimitry Andric } else if (IsNVVM && (Name == "abs.i" || Name == "abs.ll")) { 34250b57cec5SDimitry Andric Value *Arg = CI->getArgOperand(0); 34260b57cec5SDimitry Andric Value *Neg = Builder.CreateNeg(Arg, "neg"); 34270b57cec5SDimitry Andric Value *Cmp = Builder.CreateICmpSGE( 34280b57cec5SDimitry Andric Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond"); 34290b57cec5SDimitry Andric Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs"); 34300b57cec5SDimitry Andric } else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") || 34310b57cec5SDimitry Andric Name.startswith("atomic.load.add.f64.p"))) { 34320b57cec5SDimitry Andric Value *Ptr = CI->getArgOperand(0); 34330b57cec5SDimitry Andric Value *Val = CI->getArgOperand(1); 34340b57cec5SDimitry Andric Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val, 34350b57cec5SDimitry Andric AtomicOrdering::SequentiallyConsistent); 34360b57cec5SDimitry Andric } else if (IsNVVM && (Name == "max.i" || Name == "max.ll" || 34370b57cec5SDimitry Andric Name == "max.ui" || Name == "max.ull")) { 34380b57cec5SDimitry Andric Value *Arg0 = CI->getArgOperand(0); 34390b57cec5SDimitry Andric Value *Arg1 = CI->getArgOperand(1); 34400b57cec5SDimitry Andric Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull") 34410b57cec5SDimitry Andric ? Builder.CreateICmpUGE(Arg0, Arg1, "max.cond") 34420b57cec5SDimitry Andric : Builder.CreateICmpSGE(Arg0, Arg1, "max.cond"); 34430b57cec5SDimitry Andric Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "max"); 34440b57cec5SDimitry Andric } else if (IsNVVM && (Name == "min.i" || Name == "min.ll" || 34450b57cec5SDimitry Andric Name == "min.ui" || Name == "min.ull")) { 34460b57cec5SDimitry Andric Value *Arg0 = CI->getArgOperand(0); 34470b57cec5SDimitry Andric Value *Arg1 = CI->getArgOperand(1); 34480b57cec5SDimitry Andric Value *Cmp = Name.endswith(".ui") || Name.endswith(".ull") 34490b57cec5SDimitry Andric ? Builder.CreateICmpULE(Arg0, Arg1, "min.cond") 34500b57cec5SDimitry Andric : Builder.CreateICmpSLE(Arg0, Arg1, "min.cond"); 34510b57cec5SDimitry Andric Rep = Builder.CreateSelect(Cmp, Arg0, Arg1, "min"); 34520b57cec5SDimitry Andric } else if (IsNVVM && Name == "clz.ll") { 34530b57cec5SDimitry Andric // llvm.nvvm.clz.ll returns an i32, but llvm.ctlz.i64 and returns an i64. 34540b57cec5SDimitry Andric Value *Arg = CI->getArgOperand(0); 34550b57cec5SDimitry Andric Value *Ctlz = Builder.CreateCall( 34560b57cec5SDimitry Andric Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz, 34570b57cec5SDimitry Andric {Arg->getType()}), 34580b57cec5SDimitry Andric {Arg, Builder.getFalse()}, "ctlz"); 34590b57cec5SDimitry Andric Rep = Builder.CreateTrunc(Ctlz, Builder.getInt32Ty(), "ctlz.trunc"); 34600b57cec5SDimitry Andric } else if (IsNVVM && Name == "popc.ll") { 34610b57cec5SDimitry Andric // llvm.nvvm.popc.ll returns an i32, but llvm.ctpop.i64 and returns an 34620b57cec5SDimitry Andric // i64. 34630b57cec5SDimitry Andric Value *Arg = CI->getArgOperand(0); 34640b57cec5SDimitry Andric Value *Popc = Builder.CreateCall( 34650b57cec5SDimitry Andric Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctpop, 34660b57cec5SDimitry Andric {Arg->getType()}), 34670b57cec5SDimitry Andric Arg, "ctpop"); 34680b57cec5SDimitry Andric Rep = Builder.CreateTrunc(Popc, Builder.getInt32Ty(), "ctpop.trunc"); 34690b57cec5SDimitry Andric } else if (IsNVVM && Name == "h2f") { 34700b57cec5SDimitry Andric Rep = Builder.CreateCall(Intrinsic::getDeclaration( 34710b57cec5SDimitry Andric F->getParent(), Intrinsic::convert_from_fp16, 34720b57cec5SDimitry Andric {Builder.getFloatTy()}), 34730b57cec5SDimitry Andric CI->getArgOperand(0), "h2f"); 34740b57cec5SDimitry Andric } else { 34750b57cec5SDimitry Andric llvm_unreachable("Unknown function for CallInst upgrade."); 34760b57cec5SDimitry Andric } 34770b57cec5SDimitry Andric 34780b57cec5SDimitry Andric if (Rep) 34790b57cec5SDimitry Andric CI->replaceAllUsesWith(Rep); 34800b57cec5SDimitry Andric CI->eraseFromParent(); 34810b57cec5SDimitry Andric return; 34820b57cec5SDimitry Andric } 34830b57cec5SDimitry Andric 34840b57cec5SDimitry Andric const auto &DefaultCase = [&NewFn, &CI]() -> void { 34850b57cec5SDimitry Andric // Handle generic mangling change, but nothing else 34860b57cec5SDimitry Andric assert( 34870b57cec5SDimitry Andric (CI->getCalledFunction()->getName() != NewFn->getName()) && 34880b57cec5SDimitry Andric "Unknown function for CallInst upgrade and isn't just a name change"); 34890b57cec5SDimitry Andric CI->setCalledFunction(NewFn); 34900b57cec5SDimitry Andric }; 34910b57cec5SDimitry Andric CallInst *NewCall = nullptr; 34920b57cec5SDimitry Andric switch (NewFn->getIntrinsicID()) { 34930b57cec5SDimitry Andric default: { 34940b57cec5SDimitry Andric DefaultCase(); 34950b57cec5SDimitry Andric return; 34960b57cec5SDimitry Andric } 34970b57cec5SDimitry Andric case Intrinsic::experimental_vector_reduce_v2_fmul: { 34980b57cec5SDimitry Andric SmallVector<Value *, 2> Args; 34990b57cec5SDimitry Andric if (CI->isFast()) 35000b57cec5SDimitry Andric Args.push_back(ConstantFP::get(CI->getOperand(0)->getType(), 1.0)); 35010b57cec5SDimitry Andric else 35020b57cec5SDimitry Andric Args.push_back(CI->getOperand(0)); 35030b57cec5SDimitry Andric Args.push_back(CI->getOperand(1)); 35040b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, Args); 35050b57cec5SDimitry Andric cast<Instruction>(NewCall)->copyFastMathFlags(CI); 35060b57cec5SDimitry Andric break; 35070b57cec5SDimitry Andric } 35080b57cec5SDimitry Andric case Intrinsic::experimental_vector_reduce_v2_fadd: { 35090b57cec5SDimitry Andric SmallVector<Value *, 2> Args; 35100b57cec5SDimitry Andric if (CI->isFast()) 35110b57cec5SDimitry Andric Args.push_back(Constant::getNullValue(CI->getOperand(0)->getType())); 35120b57cec5SDimitry Andric else 35130b57cec5SDimitry Andric Args.push_back(CI->getOperand(0)); 35140b57cec5SDimitry Andric Args.push_back(CI->getOperand(1)); 35150b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, Args); 35160b57cec5SDimitry Andric cast<Instruction>(NewCall)->copyFastMathFlags(CI); 35170b57cec5SDimitry Andric break; 35180b57cec5SDimitry Andric } 35190b57cec5SDimitry Andric case Intrinsic::arm_neon_vld1: 35200b57cec5SDimitry Andric case Intrinsic::arm_neon_vld2: 35210b57cec5SDimitry Andric case Intrinsic::arm_neon_vld3: 35220b57cec5SDimitry Andric case Intrinsic::arm_neon_vld4: 35230b57cec5SDimitry Andric case Intrinsic::arm_neon_vld2lane: 35240b57cec5SDimitry Andric case Intrinsic::arm_neon_vld3lane: 35250b57cec5SDimitry Andric case Intrinsic::arm_neon_vld4lane: 35260b57cec5SDimitry Andric case Intrinsic::arm_neon_vst1: 35270b57cec5SDimitry Andric case Intrinsic::arm_neon_vst2: 35280b57cec5SDimitry Andric case Intrinsic::arm_neon_vst3: 35290b57cec5SDimitry Andric case Intrinsic::arm_neon_vst4: 35300b57cec5SDimitry Andric case Intrinsic::arm_neon_vst2lane: 35310b57cec5SDimitry Andric case Intrinsic::arm_neon_vst3lane: 35320b57cec5SDimitry Andric case Intrinsic::arm_neon_vst4lane: { 35330b57cec5SDimitry Andric SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 35340b57cec5SDimitry Andric CI->arg_operands().end()); 35350b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, Args); 35360b57cec5SDimitry Andric break; 35370b57cec5SDimitry Andric } 35380b57cec5SDimitry Andric 35390b57cec5SDimitry Andric case Intrinsic::bitreverse: 35400b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); 35410b57cec5SDimitry Andric break; 35420b57cec5SDimitry Andric 35430b57cec5SDimitry Andric case Intrinsic::ctlz: 35440b57cec5SDimitry Andric case Intrinsic::cttz: 35450b57cec5SDimitry Andric assert(CI->getNumArgOperands() == 1 && 35460b57cec5SDimitry Andric "Mismatch between function args and call args"); 35470b57cec5SDimitry Andric NewCall = 35480b57cec5SDimitry Andric Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()}); 35490b57cec5SDimitry Andric break; 35500b57cec5SDimitry Andric 35510b57cec5SDimitry Andric case Intrinsic::objectsize: { 35520b57cec5SDimitry Andric Value *NullIsUnknownSize = CI->getNumArgOperands() == 2 35530b57cec5SDimitry Andric ? Builder.getFalse() 35540b57cec5SDimitry Andric : CI->getArgOperand(2); 35550b57cec5SDimitry Andric Value *Dynamic = 35560b57cec5SDimitry Andric CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3); 35570b57cec5SDimitry Andric NewCall = Builder.CreateCall( 35580b57cec5SDimitry Andric NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic}); 35590b57cec5SDimitry Andric break; 35600b57cec5SDimitry Andric } 35610b57cec5SDimitry Andric 35620b57cec5SDimitry Andric case Intrinsic::ctpop: 35630b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); 35640b57cec5SDimitry Andric break; 35650b57cec5SDimitry Andric 35660b57cec5SDimitry Andric case Intrinsic::convert_from_fp16: 35670b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)}); 35680b57cec5SDimitry Andric break; 35690b57cec5SDimitry Andric 35700b57cec5SDimitry Andric case Intrinsic::dbg_value: 35710b57cec5SDimitry Andric // Upgrade from the old version that had an extra offset argument. 35720b57cec5SDimitry Andric assert(CI->getNumArgOperands() == 4); 35730b57cec5SDimitry Andric // Drop nonzero offsets instead of attempting to upgrade them. 35740b57cec5SDimitry Andric if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1))) 35750b57cec5SDimitry Andric if (Offset->isZeroValue()) { 35760b57cec5SDimitry Andric NewCall = Builder.CreateCall( 35770b57cec5SDimitry Andric NewFn, 35780b57cec5SDimitry Andric {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)}); 35790b57cec5SDimitry Andric break; 35800b57cec5SDimitry Andric } 35810b57cec5SDimitry Andric CI->eraseFromParent(); 35820b57cec5SDimitry Andric return; 35830b57cec5SDimitry Andric 35840b57cec5SDimitry Andric case Intrinsic::x86_xop_vfrcz_ss: 35850b57cec5SDimitry Andric case Intrinsic::x86_xop_vfrcz_sd: 35860b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)}); 35870b57cec5SDimitry Andric break; 35880b57cec5SDimitry Andric 35890b57cec5SDimitry Andric case Intrinsic::x86_xop_vpermil2pd: 35900b57cec5SDimitry Andric case Intrinsic::x86_xop_vpermil2ps: 35910b57cec5SDimitry Andric case Intrinsic::x86_xop_vpermil2pd_256: 35920b57cec5SDimitry Andric case Intrinsic::x86_xop_vpermil2ps_256: { 35930b57cec5SDimitry Andric SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 35940b57cec5SDimitry Andric CI->arg_operands().end()); 35950b57cec5SDimitry Andric VectorType *FltIdxTy = cast<VectorType>(Args[2]->getType()); 35960b57cec5SDimitry Andric VectorType *IntIdxTy = VectorType::getInteger(FltIdxTy); 35970b57cec5SDimitry Andric Args[2] = Builder.CreateBitCast(Args[2], IntIdxTy); 35980b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, Args); 35990b57cec5SDimitry Andric break; 36000b57cec5SDimitry Andric } 36010b57cec5SDimitry Andric 36020b57cec5SDimitry Andric case Intrinsic::x86_sse41_ptestc: 36030b57cec5SDimitry Andric case Intrinsic::x86_sse41_ptestz: 36040b57cec5SDimitry Andric case Intrinsic::x86_sse41_ptestnzc: { 36050b57cec5SDimitry Andric // The arguments for these intrinsics used to be v4f32, and changed 36060b57cec5SDimitry Andric // to v2i64. This is purely a nop, since those are bitwise intrinsics. 36070b57cec5SDimitry Andric // So, the only thing required is a bitcast for both arguments. 36080b57cec5SDimitry Andric // First, check the arguments have the old type. 36090b57cec5SDimitry Andric Value *Arg0 = CI->getArgOperand(0); 36100b57cec5SDimitry Andric if (Arg0->getType() != VectorType::get(Type::getFloatTy(C), 4)) 36110b57cec5SDimitry Andric return; 36120b57cec5SDimitry Andric 36130b57cec5SDimitry Andric // Old intrinsic, add bitcasts 36140b57cec5SDimitry Andric Value *Arg1 = CI->getArgOperand(1); 36150b57cec5SDimitry Andric 36160b57cec5SDimitry Andric Type *NewVecTy = VectorType::get(Type::getInt64Ty(C), 2); 36170b57cec5SDimitry Andric 36180b57cec5SDimitry Andric Value *BC0 = Builder.CreateBitCast(Arg0, NewVecTy, "cast"); 36190b57cec5SDimitry Andric Value *BC1 = Builder.CreateBitCast(Arg1, NewVecTy, "cast"); 36200b57cec5SDimitry Andric 36210b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, {BC0, BC1}); 36220b57cec5SDimitry Andric break; 36230b57cec5SDimitry Andric } 36240b57cec5SDimitry Andric 36250b57cec5SDimitry Andric case Intrinsic::x86_rdtscp: { 36260b57cec5SDimitry Andric // This used to take 1 arguments. If we have no arguments, it is already 36270b57cec5SDimitry Andric // upgraded. 36280b57cec5SDimitry Andric if (CI->getNumOperands() == 0) 36290b57cec5SDimitry Andric return; 36300b57cec5SDimitry Andric 36310b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn); 36320b57cec5SDimitry Andric // Extract the second result and store it. 36330b57cec5SDimitry Andric Value *Data = Builder.CreateExtractValue(NewCall, 1); 36340b57cec5SDimitry Andric // Cast the pointer to the right type. 36350b57cec5SDimitry Andric Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0), 36360b57cec5SDimitry Andric llvm::PointerType::getUnqual(Data->getType())); 36370b57cec5SDimitry Andric Builder.CreateAlignedStore(Data, Ptr, 1); 36380b57cec5SDimitry Andric // Replace the original call result with the first result of the new call. 36390b57cec5SDimitry Andric Value *TSC = Builder.CreateExtractValue(NewCall, 0); 36400b57cec5SDimitry Andric 36410b57cec5SDimitry Andric std::string Name = CI->getName(); 36420b57cec5SDimitry Andric if (!Name.empty()) { 36430b57cec5SDimitry Andric CI->setName(Name + ".old"); 36440b57cec5SDimitry Andric NewCall->setName(Name); 36450b57cec5SDimitry Andric } 36460b57cec5SDimitry Andric CI->replaceAllUsesWith(TSC); 36470b57cec5SDimitry Andric CI->eraseFromParent(); 36480b57cec5SDimitry Andric return; 36490b57cec5SDimitry Andric } 36500b57cec5SDimitry Andric 36510b57cec5SDimitry Andric case Intrinsic::x86_sse41_insertps: 36520b57cec5SDimitry Andric case Intrinsic::x86_sse41_dppd: 36530b57cec5SDimitry Andric case Intrinsic::x86_sse41_dpps: 36540b57cec5SDimitry Andric case Intrinsic::x86_sse41_mpsadbw: 36550b57cec5SDimitry Andric case Intrinsic::x86_avx_dp_ps_256: 36560b57cec5SDimitry Andric case Intrinsic::x86_avx2_mpsadbw: { 36570b57cec5SDimitry Andric // Need to truncate the last argument from i32 to i8 -- this argument models 36580b57cec5SDimitry Andric // an inherently 8-bit immediate operand to these x86 instructions. 36590b57cec5SDimitry Andric SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 36600b57cec5SDimitry Andric CI->arg_operands().end()); 36610b57cec5SDimitry Andric 36620b57cec5SDimitry Andric // Replace the last argument with a trunc. 36630b57cec5SDimitry Andric Args.back() = Builder.CreateTrunc(Args.back(), Type::getInt8Ty(C), "trunc"); 36640b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, Args); 36650b57cec5SDimitry Andric break; 36660b57cec5SDimitry Andric } 36670b57cec5SDimitry Andric 36680b57cec5SDimitry Andric case Intrinsic::thread_pointer: { 36690b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, {}); 36700b57cec5SDimitry Andric break; 36710b57cec5SDimitry Andric } 36720b57cec5SDimitry Andric 36730b57cec5SDimitry Andric case Intrinsic::invariant_start: 36740b57cec5SDimitry Andric case Intrinsic::invariant_end: 36750b57cec5SDimitry Andric case Intrinsic::masked_load: 36760b57cec5SDimitry Andric case Intrinsic::masked_store: 36770b57cec5SDimitry Andric case Intrinsic::masked_gather: 36780b57cec5SDimitry Andric case Intrinsic::masked_scatter: { 36790b57cec5SDimitry Andric SmallVector<Value *, 4> Args(CI->arg_operands().begin(), 36800b57cec5SDimitry Andric CI->arg_operands().end()); 36810b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, Args); 36820b57cec5SDimitry Andric break; 36830b57cec5SDimitry Andric } 36840b57cec5SDimitry Andric 36850b57cec5SDimitry Andric case Intrinsic::memcpy: 36860b57cec5SDimitry Andric case Intrinsic::memmove: 36870b57cec5SDimitry Andric case Intrinsic::memset: { 36880b57cec5SDimitry Andric // We have to make sure that the call signature is what we're expecting. 36890b57cec5SDimitry Andric // We only want to change the old signatures by removing the alignment arg: 36900b57cec5SDimitry Andric // @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i32, i1) 36910b57cec5SDimitry Andric // -> @llvm.mem[cpy|move]...(i8*, i8*, i[32|i64], i1) 36920b57cec5SDimitry Andric // @llvm.memset...(i8*, i8, i[32|64], i32, i1) 36930b57cec5SDimitry Andric // -> @llvm.memset...(i8*, i8, i[32|64], i1) 36940b57cec5SDimitry Andric // Note: i8*'s in the above can be any pointer type 36950b57cec5SDimitry Andric if (CI->getNumArgOperands() != 5) { 36960b57cec5SDimitry Andric DefaultCase(); 36970b57cec5SDimitry Andric return; 36980b57cec5SDimitry Andric } 36990b57cec5SDimitry Andric // Remove alignment argument (3), and add alignment attributes to the 37000b57cec5SDimitry Andric // dest/src pointers. 37010b57cec5SDimitry Andric Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1), 37020b57cec5SDimitry Andric CI->getArgOperand(2), CI->getArgOperand(4)}; 37030b57cec5SDimitry Andric NewCall = Builder.CreateCall(NewFn, Args); 37040b57cec5SDimitry Andric auto *MemCI = cast<MemIntrinsic>(NewCall); 37050b57cec5SDimitry Andric // All mem intrinsics support dest alignment. 37060b57cec5SDimitry Andric const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3)); 37070b57cec5SDimitry Andric MemCI->setDestAlignment(Align->getZExtValue()); 37080b57cec5SDimitry Andric // Memcpy/Memmove also support source alignment. 37090b57cec5SDimitry Andric if (auto *MTI = dyn_cast<MemTransferInst>(MemCI)) 37100b57cec5SDimitry Andric MTI->setSourceAlignment(Align->getZExtValue()); 37110b57cec5SDimitry Andric break; 37120b57cec5SDimitry Andric } 37130b57cec5SDimitry Andric } 37140b57cec5SDimitry Andric assert(NewCall && "Should have either set this variable or returned through " 37150b57cec5SDimitry Andric "the default case"); 37160b57cec5SDimitry Andric std::string Name = CI->getName(); 37170b57cec5SDimitry Andric if (!Name.empty()) { 37180b57cec5SDimitry Andric CI->setName(Name + ".old"); 37190b57cec5SDimitry Andric NewCall->setName(Name); 37200b57cec5SDimitry Andric } 37210b57cec5SDimitry Andric CI->replaceAllUsesWith(NewCall); 37220b57cec5SDimitry Andric CI->eraseFromParent(); 37230b57cec5SDimitry Andric } 37240b57cec5SDimitry Andric 37250b57cec5SDimitry Andric void llvm::UpgradeCallsToIntrinsic(Function *F) { 37260b57cec5SDimitry Andric assert(F && "Illegal attempt to upgrade a non-existent intrinsic."); 37270b57cec5SDimitry Andric 37280b57cec5SDimitry Andric // Check if this function should be upgraded and get the replacement function 37290b57cec5SDimitry Andric // if there is one. 37300b57cec5SDimitry Andric Function *NewFn; 37310b57cec5SDimitry Andric if (UpgradeIntrinsicFunction(F, NewFn)) { 37320b57cec5SDimitry Andric // Replace all users of the old function with the new function or new 37330b57cec5SDimitry Andric // instructions. This is not a range loop because the call is deleted. 37340b57cec5SDimitry Andric for (auto UI = F->user_begin(), UE = F->user_end(); UI != UE; ) 37350b57cec5SDimitry Andric if (CallInst *CI = dyn_cast<CallInst>(*UI++)) 37360b57cec5SDimitry Andric UpgradeIntrinsicCall(CI, NewFn); 37370b57cec5SDimitry Andric 37380b57cec5SDimitry Andric // Remove old function, no longer used, from the module. 37390b57cec5SDimitry Andric F->eraseFromParent(); 37400b57cec5SDimitry Andric } 37410b57cec5SDimitry Andric } 37420b57cec5SDimitry Andric 37430b57cec5SDimitry Andric MDNode *llvm::UpgradeTBAANode(MDNode &MD) { 37440b57cec5SDimitry Andric // Check if the tag uses struct-path aware TBAA format. 37450b57cec5SDimitry Andric if (isa<MDNode>(MD.getOperand(0)) && MD.getNumOperands() >= 3) 37460b57cec5SDimitry Andric return &MD; 37470b57cec5SDimitry Andric 37480b57cec5SDimitry Andric auto &Context = MD.getContext(); 37490b57cec5SDimitry Andric if (MD.getNumOperands() == 3) { 37500b57cec5SDimitry Andric Metadata *Elts[] = {MD.getOperand(0), MD.getOperand(1)}; 37510b57cec5SDimitry Andric MDNode *ScalarType = MDNode::get(Context, Elts); 37520b57cec5SDimitry Andric // Create a MDNode <ScalarType, ScalarType, offset 0, const> 37530b57cec5SDimitry Andric Metadata *Elts2[] = {ScalarType, ScalarType, 37540b57cec5SDimitry Andric ConstantAsMetadata::get( 37550b57cec5SDimitry Andric Constant::getNullValue(Type::getInt64Ty(Context))), 37560b57cec5SDimitry Andric MD.getOperand(2)}; 37570b57cec5SDimitry Andric return MDNode::get(Context, Elts2); 37580b57cec5SDimitry Andric } 37590b57cec5SDimitry Andric // Create a MDNode <MD, MD, offset 0> 37600b57cec5SDimitry Andric Metadata *Elts[] = {&MD, &MD, ConstantAsMetadata::get(Constant::getNullValue( 37610b57cec5SDimitry Andric Type::getInt64Ty(Context)))}; 37620b57cec5SDimitry Andric return MDNode::get(Context, Elts); 37630b57cec5SDimitry Andric } 37640b57cec5SDimitry Andric 37650b57cec5SDimitry Andric Instruction *llvm::UpgradeBitCastInst(unsigned Opc, Value *V, Type *DestTy, 37660b57cec5SDimitry Andric Instruction *&Temp) { 37670b57cec5SDimitry Andric if (Opc != Instruction::BitCast) 37680b57cec5SDimitry Andric return nullptr; 37690b57cec5SDimitry Andric 37700b57cec5SDimitry Andric Temp = nullptr; 37710b57cec5SDimitry Andric Type *SrcTy = V->getType(); 37720b57cec5SDimitry Andric if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() && 37730b57cec5SDimitry Andric SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) { 37740b57cec5SDimitry Andric LLVMContext &Context = V->getContext(); 37750b57cec5SDimitry Andric 37760b57cec5SDimitry Andric // We have no information about target data layout, so we assume that 37770b57cec5SDimitry Andric // the maximum pointer size is 64bit. 37780b57cec5SDimitry Andric Type *MidTy = Type::getInt64Ty(Context); 37790b57cec5SDimitry Andric Temp = CastInst::Create(Instruction::PtrToInt, V, MidTy); 37800b57cec5SDimitry Andric 37810b57cec5SDimitry Andric return CastInst::Create(Instruction::IntToPtr, Temp, DestTy); 37820b57cec5SDimitry Andric } 37830b57cec5SDimitry Andric 37840b57cec5SDimitry Andric return nullptr; 37850b57cec5SDimitry Andric } 37860b57cec5SDimitry Andric 37870b57cec5SDimitry Andric Value *llvm::UpgradeBitCastExpr(unsigned Opc, Constant *C, Type *DestTy) { 37880b57cec5SDimitry Andric if (Opc != Instruction::BitCast) 37890b57cec5SDimitry Andric return nullptr; 37900b57cec5SDimitry Andric 37910b57cec5SDimitry Andric Type *SrcTy = C->getType(); 37920b57cec5SDimitry Andric if (SrcTy->isPtrOrPtrVectorTy() && DestTy->isPtrOrPtrVectorTy() && 37930b57cec5SDimitry Andric SrcTy->getPointerAddressSpace() != DestTy->getPointerAddressSpace()) { 37940b57cec5SDimitry Andric LLVMContext &Context = C->getContext(); 37950b57cec5SDimitry Andric 37960b57cec5SDimitry Andric // We have no information about target data layout, so we assume that 37970b57cec5SDimitry Andric // the maximum pointer size is 64bit. 37980b57cec5SDimitry Andric Type *MidTy = Type::getInt64Ty(Context); 37990b57cec5SDimitry Andric 38000b57cec5SDimitry Andric return ConstantExpr::getIntToPtr(ConstantExpr::getPtrToInt(C, MidTy), 38010b57cec5SDimitry Andric DestTy); 38020b57cec5SDimitry Andric } 38030b57cec5SDimitry Andric 38040b57cec5SDimitry Andric return nullptr; 38050b57cec5SDimitry Andric } 38060b57cec5SDimitry Andric 38070b57cec5SDimitry Andric /// Check the debug info version number, if it is out-dated, drop the debug 38080b57cec5SDimitry Andric /// info. Return true if module is modified. 38090b57cec5SDimitry Andric bool llvm::UpgradeDebugInfo(Module &M) { 38100b57cec5SDimitry Andric unsigned Version = getDebugMetadataVersionFromModule(M); 38110b57cec5SDimitry Andric if (Version == DEBUG_METADATA_VERSION) { 38120b57cec5SDimitry Andric bool BrokenDebugInfo = false; 38130b57cec5SDimitry Andric if (verifyModule(M, &llvm::errs(), &BrokenDebugInfo)) 38140b57cec5SDimitry Andric report_fatal_error("Broken module found, compilation aborted!"); 38150b57cec5SDimitry Andric if (!BrokenDebugInfo) 38160b57cec5SDimitry Andric // Everything is ok. 38170b57cec5SDimitry Andric return false; 38180b57cec5SDimitry Andric else { 38190b57cec5SDimitry Andric // Diagnose malformed debug info. 38200b57cec5SDimitry Andric DiagnosticInfoIgnoringInvalidDebugMetadata Diag(M); 38210b57cec5SDimitry Andric M.getContext().diagnose(Diag); 38220b57cec5SDimitry Andric } 38230b57cec5SDimitry Andric } 38240b57cec5SDimitry Andric bool Modified = StripDebugInfo(M); 38250b57cec5SDimitry Andric if (Modified && Version != DEBUG_METADATA_VERSION) { 38260b57cec5SDimitry Andric // Diagnose a version mismatch. 38270b57cec5SDimitry Andric DiagnosticInfoDebugMetadataVersion DiagVersion(M, Version); 38280b57cec5SDimitry Andric M.getContext().diagnose(DiagVersion); 38290b57cec5SDimitry Andric } 38300b57cec5SDimitry Andric return Modified; 38310b57cec5SDimitry Andric } 38320b57cec5SDimitry Andric 38338bcb0991SDimitry Andric /// This checks for objc retain release marker which should be upgraded. It 38348bcb0991SDimitry Andric /// returns true if module is modified. 38358bcb0991SDimitry Andric static bool UpgradeRetainReleaseMarker(Module &M) { 38360b57cec5SDimitry Andric bool Changed = false; 38370b57cec5SDimitry Andric const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker"; 38380b57cec5SDimitry Andric NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey); 38390b57cec5SDimitry Andric if (ModRetainReleaseMarker) { 38400b57cec5SDimitry Andric MDNode *Op = ModRetainReleaseMarker->getOperand(0); 38410b57cec5SDimitry Andric if (Op) { 38420b57cec5SDimitry Andric MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(0)); 38430b57cec5SDimitry Andric if (ID) { 38440b57cec5SDimitry Andric SmallVector<StringRef, 4> ValueComp; 38450b57cec5SDimitry Andric ID->getString().split(ValueComp, "#"); 38460b57cec5SDimitry Andric if (ValueComp.size() == 2) { 38470b57cec5SDimitry Andric std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str(); 38480b57cec5SDimitry Andric ID = MDString::get(M.getContext(), NewValue); 38490b57cec5SDimitry Andric } 38500b57cec5SDimitry Andric M.addModuleFlag(Module::Error, MarkerKey, ID); 38510b57cec5SDimitry Andric M.eraseNamedMetadata(ModRetainReleaseMarker); 38520b57cec5SDimitry Andric Changed = true; 38530b57cec5SDimitry Andric } 38540b57cec5SDimitry Andric } 38550b57cec5SDimitry Andric } 38560b57cec5SDimitry Andric return Changed; 38570b57cec5SDimitry Andric } 38580b57cec5SDimitry Andric 38598bcb0991SDimitry Andric void llvm::UpgradeARCRuntime(Module &M) { 38608bcb0991SDimitry Andric // This lambda converts normal function calls to ARC runtime functions to 38618bcb0991SDimitry Andric // intrinsic calls. 38628bcb0991SDimitry Andric auto UpgradeToIntrinsic = [&](const char *OldFunc, 38638bcb0991SDimitry Andric llvm::Intrinsic::ID IntrinsicFunc) { 38648bcb0991SDimitry Andric Function *Fn = M.getFunction(OldFunc); 38658bcb0991SDimitry Andric 38668bcb0991SDimitry Andric if (!Fn) 38678bcb0991SDimitry Andric return; 38688bcb0991SDimitry Andric 38698bcb0991SDimitry Andric Function *NewFn = llvm::Intrinsic::getDeclaration(&M, IntrinsicFunc); 38708bcb0991SDimitry Andric 38718bcb0991SDimitry Andric for (auto I = Fn->user_begin(), E = Fn->user_end(); I != E;) { 38728bcb0991SDimitry Andric CallInst *CI = dyn_cast<CallInst>(*I++); 38738bcb0991SDimitry Andric if (!CI || CI->getCalledFunction() != Fn) 38748bcb0991SDimitry Andric continue; 38758bcb0991SDimitry Andric 38768bcb0991SDimitry Andric IRBuilder<> Builder(CI->getParent(), CI->getIterator()); 38778bcb0991SDimitry Andric FunctionType *NewFuncTy = NewFn->getFunctionType(); 38788bcb0991SDimitry Andric SmallVector<Value *, 2> Args; 38798bcb0991SDimitry Andric 38808bcb0991SDimitry Andric for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) { 38818bcb0991SDimitry Andric Value *Arg = CI->getArgOperand(I); 38828bcb0991SDimitry Andric // Bitcast argument to the parameter type of the new function if it's 38838bcb0991SDimitry Andric // not a variadic argument. 38848bcb0991SDimitry Andric if (I < NewFuncTy->getNumParams()) 38858bcb0991SDimitry Andric Arg = Builder.CreateBitCast(Arg, NewFuncTy->getParamType(I)); 38868bcb0991SDimitry Andric Args.push_back(Arg); 38878bcb0991SDimitry Andric } 38888bcb0991SDimitry Andric 38898bcb0991SDimitry Andric // Create a call instruction that calls the new function. 38908bcb0991SDimitry Andric CallInst *NewCall = Builder.CreateCall(NewFuncTy, NewFn, Args); 38918bcb0991SDimitry Andric NewCall->setTailCallKind(cast<CallInst>(CI)->getTailCallKind()); 38928bcb0991SDimitry Andric NewCall->setName(CI->getName()); 38938bcb0991SDimitry Andric 38948bcb0991SDimitry Andric // Bitcast the return value back to the type of the old call. 38958bcb0991SDimitry Andric Value *NewRetVal = Builder.CreateBitCast(NewCall, CI->getType()); 38968bcb0991SDimitry Andric 38978bcb0991SDimitry Andric if (!CI->use_empty()) 38988bcb0991SDimitry Andric CI->replaceAllUsesWith(NewRetVal); 38998bcb0991SDimitry Andric CI->eraseFromParent(); 39008bcb0991SDimitry Andric } 39018bcb0991SDimitry Andric 39028bcb0991SDimitry Andric if (Fn->use_empty()) 39038bcb0991SDimitry Andric Fn->eraseFromParent(); 39048bcb0991SDimitry Andric }; 39058bcb0991SDimitry Andric 39068bcb0991SDimitry Andric // Unconditionally convert a call to "clang.arc.use" to a call to 39078bcb0991SDimitry Andric // "llvm.objc.clang.arc.use". 39088bcb0991SDimitry Andric UpgradeToIntrinsic("clang.arc.use", llvm::Intrinsic::objc_clang_arc_use); 39098bcb0991SDimitry Andric 39108bcb0991SDimitry Andric // Upgrade the retain release marker. If there is no need to upgrade 39118bcb0991SDimitry Andric // the marker, that means either the module is already new enough to contain 39128bcb0991SDimitry Andric // new intrinsics or it is not ARC. There is no need to upgrade runtime call. 39138bcb0991SDimitry Andric if (!UpgradeRetainReleaseMarker(M)) 39148bcb0991SDimitry Andric return; 39158bcb0991SDimitry Andric 39168bcb0991SDimitry Andric std::pair<const char *, llvm::Intrinsic::ID> RuntimeFuncs[] = { 39178bcb0991SDimitry Andric {"objc_autorelease", llvm::Intrinsic::objc_autorelease}, 39188bcb0991SDimitry Andric {"objc_autoreleasePoolPop", llvm::Intrinsic::objc_autoreleasePoolPop}, 39198bcb0991SDimitry Andric {"objc_autoreleasePoolPush", llvm::Intrinsic::objc_autoreleasePoolPush}, 39208bcb0991SDimitry Andric {"objc_autoreleaseReturnValue", 39218bcb0991SDimitry Andric llvm::Intrinsic::objc_autoreleaseReturnValue}, 39228bcb0991SDimitry Andric {"objc_copyWeak", llvm::Intrinsic::objc_copyWeak}, 39238bcb0991SDimitry Andric {"objc_destroyWeak", llvm::Intrinsic::objc_destroyWeak}, 39248bcb0991SDimitry Andric {"objc_initWeak", llvm::Intrinsic::objc_initWeak}, 39258bcb0991SDimitry Andric {"objc_loadWeak", llvm::Intrinsic::objc_loadWeak}, 39268bcb0991SDimitry Andric {"objc_loadWeakRetained", llvm::Intrinsic::objc_loadWeakRetained}, 39278bcb0991SDimitry Andric {"objc_moveWeak", llvm::Intrinsic::objc_moveWeak}, 39288bcb0991SDimitry Andric {"objc_release", llvm::Intrinsic::objc_release}, 39298bcb0991SDimitry Andric {"objc_retain", llvm::Intrinsic::objc_retain}, 39308bcb0991SDimitry Andric {"objc_retainAutorelease", llvm::Intrinsic::objc_retainAutorelease}, 39318bcb0991SDimitry Andric {"objc_retainAutoreleaseReturnValue", 39328bcb0991SDimitry Andric llvm::Intrinsic::objc_retainAutoreleaseReturnValue}, 39338bcb0991SDimitry Andric {"objc_retainAutoreleasedReturnValue", 39348bcb0991SDimitry Andric llvm::Intrinsic::objc_retainAutoreleasedReturnValue}, 39358bcb0991SDimitry Andric {"objc_retainBlock", llvm::Intrinsic::objc_retainBlock}, 39368bcb0991SDimitry Andric {"objc_storeStrong", llvm::Intrinsic::objc_storeStrong}, 39378bcb0991SDimitry Andric {"objc_storeWeak", llvm::Intrinsic::objc_storeWeak}, 39388bcb0991SDimitry Andric {"objc_unsafeClaimAutoreleasedReturnValue", 39398bcb0991SDimitry Andric llvm::Intrinsic::objc_unsafeClaimAutoreleasedReturnValue}, 39408bcb0991SDimitry Andric {"objc_retainedObject", llvm::Intrinsic::objc_retainedObject}, 39418bcb0991SDimitry Andric {"objc_unretainedObject", llvm::Intrinsic::objc_unretainedObject}, 39428bcb0991SDimitry Andric {"objc_unretainedPointer", llvm::Intrinsic::objc_unretainedPointer}, 39438bcb0991SDimitry Andric {"objc_retain_autorelease", llvm::Intrinsic::objc_retain_autorelease}, 39448bcb0991SDimitry Andric {"objc_sync_enter", llvm::Intrinsic::objc_sync_enter}, 39458bcb0991SDimitry Andric {"objc_sync_exit", llvm::Intrinsic::objc_sync_exit}, 39468bcb0991SDimitry Andric {"objc_arc_annotation_topdown_bbstart", 39478bcb0991SDimitry Andric llvm::Intrinsic::objc_arc_annotation_topdown_bbstart}, 39488bcb0991SDimitry Andric {"objc_arc_annotation_topdown_bbend", 39498bcb0991SDimitry Andric llvm::Intrinsic::objc_arc_annotation_topdown_bbend}, 39508bcb0991SDimitry Andric {"objc_arc_annotation_bottomup_bbstart", 39518bcb0991SDimitry Andric llvm::Intrinsic::objc_arc_annotation_bottomup_bbstart}, 39528bcb0991SDimitry Andric {"objc_arc_annotation_bottomup_bbend", 39538bcb0991SDimitry Andric llvm::Intrinsic::objc_arc_annotation_bottomup_bbend}}; 39548bcb0991SDimitry Andric 39558bcb0991SDimitry Andric for (auto &I : RuntimeFuncs) 39568bcb0991SDimitry Andric UpgradeToIntrinsic(I.first, I.second); 39578bcb0991SDimitry Andric } 39588bcb0991SDimitry Andric 39590b57cec5SDimitry Andric bool llvm::UpgradeModuleFlags(Module &M) { 39600b57cec5SDimitry Andric NamedMDNode *ModFlags = M.getModuleFlagsMetadata(); 39610b57cec5SDimitry Andric if (!ModFlags) 39620b57cec5SDimitry Andric return false; 39630b57cec5SDimitry Andric 39640b57cec5SDimitry Andric bool HasObjCFlag = false, HasClassProperties = false, Changed = false; 39650b57cec5SDimitry Andric for (unsigned I = 0, E = ModFlags->getNumOperands(); I != E; ++I) { 39660b57cec5SDimitry Andric MDNode *Op = ModFlags->getOperand(I); 39670b57cec5SDimitry Andric if (Op->getNumOperands() != 3) 39680b57cec5SDimitry Andric continue; 39690b57cec5SDimitry Andric MDString *ID = dyn_cast_or_null<MDString>(Op->getOperand(1)); 39700b57cec5SDimitry Andric if (!ID) 39710b57cec5SDimitry Andric continue; 39720b57cec5SDimitry Andric if (ID->getString() == "Objective-C Image Info Version") 39730b57cec5SDimitry Andric HasObjCFlag = true; 39740b57cec5SDimitry Andric if (ID->getString() == "Objective-C Class Properties") 39750b57cec5SDimitry Andric HasClassProperties = true; 39760b57cec5SDimitry Andric // Upgrade PIC/PIE Module Flags. The module flag behavior for these two 39770b57cec5SDimitry Andric // field was Error and now they are Max. 39780b57cec5SDimitry Andric if (ID->getString() == "PIC Level" || ID->getString() == "PIE Level") { 39790b57cec5SDimitry Andric if (auto *Behavior = 39800b57cec5SDimitry Andric mdconst::dyn_extract_or_null<ConstantInt>(Op->getOperand(0))) { 39810b57cec5SDimitry Andric if (Behavior->getLimitedValue() == Module::Error) { 39820b57cec5SDimitry Andric Type *Int32Ty = Type::getInt32Ty(M.getContext()); 39830b57cec5SDimitry Andric Metadata *Ops[3] = { 39840b57cec5SDimitry Andric ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Module::Max)), 39850b57cec5SDimitry Andric MDString::get(M.getContext(), ID->getString()), 39860b57cec5SDimitry Andric Op->getOperand(2)}; 39870b57cec5SDimitry Andric ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); 39880b57cec5SDimitry Andric Changed = true; 39890b57cec5SDimitry Andric } 39900b57cec5SDimitry Andric } 39910b57cec5SDimitry Andric } 39920b57cec5SDimitry Andric // Upgrade Objective-C Image Info Section. Removed the whitespce in the 39930b57cec5SDimitry Andric // section name so that llvm-lto will not complain about mismatching 39940b57cec5SDimitry Andric // module flags that is functionally the same. 39950b57cec5SDimitry Andric if (ID->getString() == "Objective-C Image Info Section") { 39960b57cec5SDimitry Andric if (auto *Value = dyn_cast_or_null<MDString>(Op->getOperand(2))) { 39970b57cec5SDimitry Andric SmallVector<StringRef, 4> ValueComp; 39980b57cec5SDimitry Andric Value->getString().split(ValueComp, " "); 39990b57cec5SDimitry Andric if (ValueComp.size() != 1) { 40000b57cec5SDimitry Andric std::string NewValue; 40010b57cec5SDimitry Andric for (auto &S : ValueComp) 40020b57cec5SDimitry Andric NewValue += S.str(); 40030b57cec5SDimitry Andric Metadata *Ops[3] = {Op->getOperand(0), Op->getOperand(1), 40040b57cec5SDimitry Andric MDString::get(M.getContext(), NewValue)}; 40050b57cec5SDimitry Andric ModFlags->setOperand(I, MDNode::get(M.getContext(), Ops)); 40060b57cec5SDimitry Andric Changed = true; 40070b57cec5SDimitry Andric } 40080b57cec5SDimitry Andric } 40090b57cec5SDimitry Andric } 40100b57cec5SDimitry Andric } 40110b57cec5SDimitry Andric 40120b57cec5SDimitry Andric // "Objective-C Class Properties" is recently added for Objective-C. We 40130b57cec5SDimitry Andric // upgrade ObjC bitcodes to contain a "Objective-C Class Properties" module 40140b57cec5SDimitry Andric // flag of value 0, so we can correclty downgrade this flag when trying to 40150b57cec5SDimitry Andric // link an ObjC bitcode without this module flag with an ObjC bitcode with 40160b57cec5SDimitry Andric // this module flag. 40170b57cec5SDimitry Andric if (HasObjCFlag && !HasClassProperties) { 40180b57cec5SDimitry Andric M.addModuleFlag(llvm::Module::Override, "Objective-C Class Properties", 40190b57cec5SDimitry Andric (uint32_t)0); 40200b57cec5SDimitry Andric Changed = true; 40210b57cec5SDimitry Andric } 40220b57cec5SDimitry Andric 40230b57cec5SDimitry Andric return Changed; 40240b57cec5SDimitry Andric } 40250b57cec5SDimitry Andric 40260b57cec5SDimitry Andric void llvm::UpgradeSectionAttributes(Module &M) { 40270b57cec5SDimitry Andric auto TrimSpaces = [](StringRef Section) -> std::string { 40280b57cec5SDimitry Andric SmallVector<StringRef, 5> Components; 40290b57cec5SDimitry Andric Section.split(Components, ','); 40300b57cec5SDimitry Andric 40310b57cec5SDimitry Andric SmallString<32> Buffer; 40320b57cec5SDimitry Andric raw_svector_ostream OS(Buffer); 40330b57cec5SDimitry Andric 40340b57cec5SDimitry Andric for (auto Component : Components) 40350b57cec5SDimitry Andric OS << ',' << Component.trim(); 40360b57cec5SDimitry Andric 40370b57cec5SDimitry Andric return OS.str().substr(1); 40380b57cec5SDimitry Andric }; 40390b57cec5SDimitry Andric 40400b57cec5SDimitry Andric for (auto &GV : M.globals()) { 40410b57cec5SDimitry Andric if (!GV.hasSection()) 40420b57cec5SDimitry Andric continue; 40430b57cec5SDimitry Andric 40440b57cec5SDimitry Andric StringRef Section = GV.getSection(); 40450b57cec5SDimitry Andric 40460b57cec5SDimitry Andric if (!Section.startswith("__DATA, __objc_catlist")) 40470b57cec5SDimitry Andric continue; 40480b57cec5SDimitry Andric 40490b57cec5SDimitry Andric // __DATA, __objc_catlist, regular, no_dead_strip 40500b57cec5SDimitry Andric // __DATA,__objc_catlist,regular,no_dead_strip 40510b57cec5SDimitry Andric GV.setSection(TrimSpaces(Section)); 40520b57cec5SDimitry Andric } 40530b57cec5SDimitry Andric } 40540b57cec5SDimitry Andric 40550b57cec5SDimitry Andric static bool isOldLoopArgument(Metadata *MD) { 40560b57cec5SDimitry Andric auto *T = dyn_cast_or_null<MDTuple>(MD); 40570b57cec5SDimitry Andric if (!T) 40580b57cec5SDimitry Andric return false; 40590b57cec5SDimitry Andric if (T->getNumOperands() < 1) 40600b57cec5SDimitry Andric return false; 40610b57cec5SDimitry Andric auto *S = dyn_cast_or_null<MDString>(T->getOperand(0)); 40620b57cec5SDimitry Andric if (!S) 40630b57cec5SDimitry Andric return false; 40640b57cec5SDimitry Andric return S->getString().startswith("llvm.vectorizer."); 40650b57cec5SDimitry Andric } 40660b57cec5SDimitry Andric 40670b57cec5SDimitry Andric static MDString *upgradeLoopTag(LLVMContext &C, StringRef OldTag) { 40680b57cec5SDimitry Andric StringRef OldPrefix = "llvm.vectorizer."; 40690b57cec5SDimitry Andric assert(OldTag.startswith(OldPrefix) && "Expected old prefix"); 40700b57cec5SDimitry Andric 40710b57cec5SDimitry Andric if (OldTag == "llvm.vectorizer.unroll") 40720b57cec5SDimitry Andric return MDString::get(C, "llvm.loop.interleave.count"); 40730b57cec5SDimitry Andric 40740b57cec5SDimitry Andric return MDString::get( 40750b57cec5SDimitry Andric C, (Twine("llvm.loop.vectorize.") + OldTag.drop_front(OldPrefix.size())) 40760b57cec5SDimitry Andric .str()); 40770b57cec5SDimitry Andric } 40780b57cec5SDimitry Andric 40790b57cec5SDimitry Andric static Metadata *upgradeLoopArgument(Metadata *MD) { 40800b57cec5SDimitry Andric auto *T = dyn_cast_or_null<MDTuple>(MD); 40810b57cec5SDimitry Andric if (!T) 40820b57cec5SDimitry Andric return MD; 40830b57cec5SDimitry Andric if (T->getNumOperands() < 1) 40840b57cec5SDimitry Andric return MD; 40850b57cec5SDimitry Andric auto *OldTag = dyn_cast_or_null<MDString>(T->getOperand(0)); 40860b57cec5SDimitry Andric if (!OldTag) 40870b57cec5SDimitry Andric return MD; 40880b57cec5SDimitry Andric if (!OldTag->getString().startswith("llvm.vectorizer.")) 40890b57cec5SDimitry Andric return MD; 40900b57cec5SDimitry Andric 40910b57cec5SDimitry Andric // This has an old tag. Upgrade it. 40920b57cec5SDimitry Andric SmallVector<Metadata *, 8> Ops; 40930b57cec5SDimitry Andric Ops.reserve(T->getNumOperands()); 40940b57cec5SDimitry Andric Ops.push_back(upgradeLoopTag(T->getContext(), OldTag->getString())); 40950b57cec5SDimitry Andric for (unsigned I = 1, E = T->getNumOperands(); I != E; ++I) 40960b57cec5SDimitry Andric Ops.push_back(T->getOperand(I)); 40970b57cec5SDimitry Andric 40980b57cec5SDimitry Andric return MDTuple::get(T->getContext(), Ops); 40990b57cec5SDimitry Andric } 41000b57cec5SDimitry Andric 41010b57cec5SDimitry Andric MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) { 41020b57cec5SDimitry Andric auto *T = dyn_cast<MDTuple>(&N); 41030b57cec5SDimitry Andric if (!T) 41040b57cec5SDimitry Andric return &N; 41050b57cec5SDimitry Andric 41060b57cec5SDimitry Andric if (none_of(T->operands(), isOldLoopArgument)) 41070b57cec5SDimitry Andric return &N; 41080b57cec5SDimitry Andric 41090b57cec5SDimitry Andric SmallVector<Metadata *, 8> Ops; 41100b57cec5SDimitry Andric Ops.reserve(T->getNumOperands()); 41110b57cec5SDimitry Andric for (Metadata *MD : T->operands()) 41120b57cec5SDimitry Andric Ops.push_back(upgradeLoopArgument(MD)); 41130b57cec5SDimitry Andric 41140b57cec5SDimitry Andric return MDTuple::get(T->getContext(), Ops); 41150b57cec5SDimitry Andric } 41168bcb0991SDimitry Andric 41178bcb0991SDimitry Andric std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) { 41188bcb0991SDimitry Andric std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64"; 41198bcb0991SDimitry Andric 41208bcb0991SDimitry Andric // If X86, and the datalayout matches the expected format, add pointer size 41218bcb0991SDimitry Andric // address spaces to the datalayout. 41228bcb0991SDimitry Andric Triple::ArchType Arch = Triple(TT).getArch(); 41238bcb0991SDimitry Andric if ((Arch != llvm::Triple::x86 && Arch != llvm::Triple::x86_64) || 41248bcb0991SDimitry Andric DL.contains(AddrSpaces)) 41258bcb0991SDimitry Andric return DL; 41268bcb0991SDimitry Andric 41278bcb0991SDimitry Andric SmallVector<StringRef, 4> Groups; 41288bcb0991SDimitry Andric Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)"); 41298bcb0991SDimitry Andric if (!R.match(DL, &Groups)) 41308bcb0991SDimitry Andric return DL; 41318bcb0991SDimitry Andric 41328bcb0991SDimitry Andric SmallString<1024> Buf; 41338bcb0991SDimitry Andric std::string Res = (Groups[1] + AddrSpaces + Groups[3]).toStringRef(Buf).str(); 41348bcb0991SDimitry Andric return Res; 41358bcb0991SDimitry Andric } 4136