10b57cec5SDimitry Andric //===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//
20b57cec5SDimitry Andric //
30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
60b57cec5SDimitry Andric //
70b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
80b57cec5SDimitry Andric //
90b57cec5SDimitry Andric // This implements the TargetLoweringBase class.
100b57cec5SDimitry Andric //
110b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
120b57cec5SDimitry Andric
130b57cec5SDimitry Andric #include "llvm/ADT/BitVector.h"
140b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h"
150b57cec5SDimitry Andric #include "llvm/ADT/SmallVector.h"
160b57cec5SDimitry Andric #include "llvm/ADT/StringExtras.h"
170b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h"
180b57cec5SDimitry Andric #include "llvm/ADT/Twine.h"
195ffd83dbSDimitry Andric #include "llvm/Analysis/Loads.h"
205ffd83dbSDimitry Andric #include "llvm/Analysis/TargetTransformInfo.h"
210b57cec5SDimitry Andric #include "llvm/CodeGen/Analysis.h"
220b57cec5SDimitry Andric #include "llvm/CodeGen/ISDOpcodes.h"
230b57cec5SDimitry Andric #include "llvm/CodeGen/MachineBasicBlock.h"
240b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFrameInfo.h"
250b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunction.h"
260b57cec5SDimitry Andric #include "llvm/CodeGen/MachineInstr.h"
270b57cec5SDimitry Andric #include "llvm/CodeGen/MachineInstrBuilder.h"
280b57cec5SDimitry Andric #include "llvm/CodeGen/MachineMemOperand.h"
290b57cec5SDimitry Andric #include "llvm/CodeGen/MachineOperand.h"
300b57cec5SDimitry Andric #include "llvm/CodeGen/MachineRegisterInfo.h"
3106c3fb27SDimitry Andric #include "llvm/CodeGen/MachineValueType.h"
320b57cec5SDimitry Andric #include "llvm/CodeGen/RuntimeLibcalls.h"
330b57cec5SDimitry Andric #include "llvm/CodeGen/StackMaps.h"
340b57cec5SDimitry Andric #include "llvm/CodeGen/TargetLowering.h"
350b57cec5SDimitry Andric #include "llvm/CodeGen/TargetOpcodes.h"
360b57cec5SDimitry Andric #include "llvm/CodeGen/TargetRegisterInfo.h"
370b57cec5SDimitry Andric #include "llvm/CodeGen/ValueTypes.h"
380b57cec5SDimitry Andric #include "llvm/IR/Attributes.h"
390b57cec5SDimitry Andric #include "llvm/IR/CallingConv.h"
400b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h"
410b57cec5SDimitry Andric #include "llvm/IR/DerivedTypes.h"
420b57cec5SDimitry Andric #include "llvm/IR/Function.h"
430b57cec5SDimitry Andric #include "llvm/IR/GlobalValue.h"
440b57cec5SDimitry Andric #include "llvm/IR/GlobalVariable.h"
450b57cec5SDimitry Andric #include "llvm/IR/IRBuilder.h"
460b57cec5SDimitry Andric #include "llvm/IR/Module.h"
470b57cec5SDimitry Andric #include "llvm/IR/Type.h"
480b57cec5SDimitry Andric #include "llvm/Support/Casting.h"
490b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h"
500b57cec5SDimitry Andric #include "llvm/Support/Compiler.h"
510b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h"
520b57cec5SDimitry Andric #include "llvm/Support/MathExtras.h"
530b57cec5SDimitry Andric #include "llvm/Target/TargetMachine.h"
54349cc55cSDimitry Andric #include "llvm/Target/TargetOptions.h"
5506c3fb27SDimitry Andric #include "llvm/TargetParser/Triple.h"
565ffd83dbSDimitry Andric #include "llvm/Transforms/Utils/SizeOpts.h"
570b57cec5SDimitry Andric #include <algorithm>
580b57cec5SDimitry Andric #include <cassert>
590b57cec5SDimitry Andric #include <cstdint>
600b57cec5SDimitry Andric #include <cstring>
610b57cec5SDimitry Andric #include <iterator>
620b57cec5SDimitry Andric #include <string>
630b57cec5SDimitry Andric #include <tuple>
640b57cec5SDimitry Andric #include <utility>
650b57cec5SDimitry Andric
660b57cec5SDimitry Andric using namespace llvm;
670b57cec5SDimitry Andric
680b57cec5SDimitry Andric static cl::opt<bool> JumpIsExpensiveOverride(
690b57cec5SDimitry Andric "jump-is-expensive", cl::init(false),
700b57cec5SDimitry Andric cl::desc("Do not create extra branches to split comparison logic."),
710b57cec5SDimitry Andric cl::Hidden);
720b57cec5SDimitry Andric
730b57cec5SDimitry Andric static cl::opt<unsigned> MinimumJumpTableEntries
740b57cec5SDimitry Andric ("min-jump-table-entries", cl::init(4), cl::Hidden,
750b57cec5SDimitry Andric cl::desc("Set minimum number of entries to use a jump table."));
760b57cec5SDimitry Andric
770b57cec5SDimitry Andric static cl::opt<unsigned> MaximumJumpTableSize
780b57cec5SDimitry Andric ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden,
790b57cec5SDimitry Andric cl::desc("Set maximum size of jump tables."));
800b57cec5SDimitry Andric
810b57cec5SDimitry Andric /// Minimum jump table density for normal functions.
820b57cec5SDimitry Andric static cl::opt<unsigned>
830b57cec5SDimitry Andric JumpTableDensity("jump-table-density", cl::init(10), cl::Hidden,
840b57cec5SDimitry Andric cl::desc("Minimum density for building a jump table in "
850b57cec5SDimitry Andric "a normal function"));
860b57cec5SDimitry Andric
870b57cec5SDimitry Andric /// Minimum jump table density for -Os or -Oz functions.
880b57cec5SDimitry Andric static cl::opt<unsigned> OptsizeJumpTableDensity(
890b57cec5SDimitry Andric "optsize-jump-table-density", cl::init(40), cl::Hidden,
900b57cec5SDimitry Andric cl::desc("Minimum density for building a jump table in "
910b57cec5SDimitry Andric "an optsize function"));
920b57cec5SDimitry Andric
93480093f4SDimitry Andric // FIXME: This option is only to test if the strict fp operation processed
94480093f4SDimitry Andric // correctly by preventing mutating strict fp operation to normal fp operation
95480093f4SDimitry Andric // during development. When the backend supports strict float operation, this
96480093f4SDimitry Andric // option will be meaningless.
97480093f4SDimitry Andric static cl::opt<bool> DisableStrictNodeMutation("disable-strictnode-mutation",
98480093f4SDimitry Andric cl::desc("Don't mutate strict-float node to a legalize node"),
99480093f4SDimitry Andric cl::init(false), cl::Hidden);
100480093f4SDimitry Andric
darwinHasSinCos(const Triple & TT)1010b57cec5SDimitry Andric static bool darwinHasSinCos(const Triple &TT) {
1020b57cec5SDimitry Andric assert(TT.isOSDarwin() && "should be called with darwin triple");
1030b57cec5SDimitry Andric // Don't bother with 32 bit x86.
1040b57cec5SDimitry Andric if (TT.getArch() == Triple::x86)
1050b57cec5SDimitry Andric return false;
1060b57cec5SDimitry Andric // Macos < 10.9 has no sincos_stret.
1070b57cec5SDimitry Andric if (TT.isMacOSX())
1080b57cec5SDimitry Andric return !TT.isMacOSXVersionLT(10, 9) && TT.isArch64Bit();
1090b57cec5SDimitry Andric // iOS < 7.0 has no sincos_stret.
1100b57cec5SDimitry Andric if (TT.isiOS())
1110b57cec5SDimitry Andric return !TT.isOSVersionLT(7, 0);
1120b57cec5SDimitry Andric // Any other darwin such as WatchOS/TvOS is new enough.
1130b57cec5SDimitry Andric return true;
1140b57cec5SDimitry Andric }
1150b57cec5SDimitry Andric
InitLibcalls(const Triple & TT)1160b57cec5SDimitry Andric void TargetLoweringBase::InitLibcalls(const Triple &TT) {
1170b57cec5SDimitry Andric #define HANDLE_LIBCALL(code, name) \
1180b57cec5SDimitry Andric setLibcallName(RTLIB::code, name);
1190b57cec5SDimitry Andric #include "llvm/IR/RuntimeLibcalls.def"
1200b57cec5SDimitry Andric #undef HANDLE_LIBCALL
1210b57cec5SDimitry Andric // Initialize calling conventions to their default.
1220b57cec5SDimitry Andric for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC)
1230b57cec5SDimitry Andric setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C);
1240b57cec5SDimitry Andric
1250b57cec5SDimitry Andric // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
126e8d8bef9SDimitry Andric if (TT.isPPC()) {
1270b57cec5SDimitry Andric setLibcallName(RTLIB::ADD_F128, "__addkf3");
1280b57cec5SDimitry Andric setLibcallName(RTLIB::SUB_F128, "__subkf3");
1290b57cec5SDimitry Andric setLibcallName(RTLIB::MUL_F128, "__mulkf3");
1300b57cec5SDimitry Andric setLibcallName(RTLIB::DIV_F128, "__divkf3");
131e8d8bef9SDimitry Andric setLibcallName(RTLIB::POWI_F128, "__powikf2");
1320b57cec5SDimitry Andric setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2");
1330b57cec5SDimitry Andric setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2");
1340b57cec5SDimitry Andric setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2");
1350b57cec5SDimitry Andric setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2");
1360b57cec5SDimitry Andric setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi");
1370b57cec5SDimitry Andric setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi");
138e8d8bef9SDimitry Andric setLibcallName(RTLIB::FPTOSINT_F128_I128, "__fixkfti");
1390b57cec5SDimitry Andric setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi");
1400b57cec5SDimitry Andric setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi");
141e8d8bef9SDimitry Andric setLibcallName(RTLIB::FPTOUINT_F128_I128, "__fixunskfti");
1420b57cec5SDimitry Andric setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf");
1430b57cec5SDimitry Andric setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf");
144e8d8bef9SDimitry Andric setLibcallName(RTLIB::SINTTOFP_I128_F128, "__floattikf");
1450b57cec5SDimitry Andric setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf");
1460b57cec5SDimitry Andric setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf");
147e8d8bef9SDimitry Andric setLibcallName(RTLIB::UINTTOFP_I128_F128, "__floatuntikf");
1480b57cec5SDimitry Andric setLibcallName(RTLIB::OEQ_F128, "__eqkf2");
1490b57cec5SDimitry Andric setLibcallName(RTLIB::UNE_F128, "__nekf2");
1500b57cec5SDimitry Andric setLibcallName(RTLIB::OGE_F128, "__gekf2");
1510b57cec5SDimitry Andric setLibcallName(RTLIB::OLT_F128, "__ltkf2");
1520b57cec5SDimitry Andric setLibcallName(RTLIB::OLE_F128, "__lekf2");
1530b57cec5SDimitry Andric setLibcallName(RTLIB::OGT_F128, "__gtkf2");
1540b57cec5SDimitry Andric setLibcallName(RTLIB::UO_F128, "__unordkf2");
1550b57cec5SDimitry Andric }
1560b57cec5SDimitry Andric
1570b57cec5SDimitry Andric // A few names are different on particular architectures or environments.
1580b57cec5SDimitry Andric if (TT.isOSDarwin()) {
1590b57cec5SDimitry Andric // For f16/f32 conversions, Darwin uses the standard naming scheme, instead
1600b57cec5SDimitry Andric // of the gnueabi-style __gnu_*_ieee.
1610b57cec5SDimitry Andric // FIXME: What about other targets?
1620b57cec5SDimitry Andric setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
1630b57cec5SDimitry Andric setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
1640b57cec5SDimitry Andric
1650b57cec5SDimitry Andric // Some darwins have an optimized __bzero/bzero function.
1660b57cec5SDimitry Andric switch (TT.getArch()) {
1670b57cec5SDimitry Andric case Triple::x86:
1680b57cec5SDimitry Andric case Triple::x86_64:
1690b57cec5SDimitry Andric if (TT.isMacOSX() && !TT.isMacOSXVersionLT(10, 6))
1700b57cec5SDimitry Andric setLibcallName(RTLIB::BZERO, "__bzero");
1710b57cec5SDimitry Andric break;
1720b57cec5SDimitry Andric case Triple::aarch64:
1738bcb0991SDimitry Andric case Triple::aarch64_32:
1740b57cec5SDimitry Andric setLibcallName(RTLIB::BZERO, "bzero");
1750b57cec5SDimitry Andric break;
1760b57cec5SDimitry Andric default:
1770b57cec5SDimitry Andric break;
1780b57cec5SDimitry Andric }
1790b57cec5SDimitry Andric
1800b57cec5SDimitry Andric if (darwinHasSinCos(TT)) {
1810b57cec5SDimitry Andric setLibcallName(RTLIB::SINCOS_STRET_F32, "__sincosf_stret");
1820b57cec5SDimitry Andric setLibcallName(RTLIB::SINCOS_STRET_F64, "__sincos_stret");
1830b57cec5SDimitry Andric if (TT.isWatchABI()) {
1840b57cec5SDimitry Andric setLibcallCallingConv(RTLIB::SINCOS_STRET_F32,
1850b57cec5SDimitry Andric CallingConv::ARM_AAPCS_VFP);
1860b57cec5SDimitry Andric setLibcallCallingConv(RTLIB::SINCOS_STRET_F64,
1870b57cec5SDimitry Andric CallingConv::ARM_AAPCS_VFP);
1880b57cec5SDimitry Andric }
1890b57cec5SDimitry Andric }
1900b57cec5SDimitry Andric } else {
1910b57cec5SDimitry Andric setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
1920b57cec5SDimitry Andric setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
1930b57cec5SDimitry Andric }
1940b57cec5SDimitry Andric
1950b57cec5SDimitry Andric if (TT.isGNUEnvironment() || TT.isOSFuchsia() ||
1960b57cec5SDimitry Andric (TT.isAndroid() && !TT.isAndroidVersionLT(9))) {
1970b57cec5SDimitry Andric setLibcallName(RTLIB::SINCOS_F32, "sincosf");
1980b57cec5SDimitry Andric setLibcallName(RTLIB::SINCOS_F64, "sincos");
1990b57cec5SDimitry Andric setLibcallName(RTLIB::SINCOS_F80, "sincosl");
2000b57cec5SDimitry Andric setLibcallName(RTLIB::SINCOS_F128, "sincosl");
2010b57cec5SDimitry Andric setLibcallName(RTLIB::SINCOS_PPCF128, "sincosl");
2020b57cec5SDimitry Andric }
2030b57cec5SDimitry Andric
20481ad6265SDimitry Andric if (TT.isPS()) {
2058bcb0991SDimitry Andric setLibcallName(RTLIB::SINCOS_F32, "sincosf");
2068bcb0991SDimitry Andric setLibcallName(RTLIB::SINCOS_F64, "sincos");
2078bcb0991SDimitry Andric }
2088bcb0991SDimitry Andric
2090b57cec5SDimitry Andric if (TT.isOSOpenBSD()) {
2100b57cec5SDimitry Andric setLibcallName(RTLIB::STACKPROTECTOR_CHECK_FAIL, nullptr);
2110b57cec5SDimitry Andric }
21206c3fb27SDimitry Andric
21306c3fb27SDimitry Andric if (TT.isOSWindows() && !TT.isOSCygMing()) {
21406c3fb27SDimitry Andric setLibcallName(RTLIB::LDEXP_F32, nullptr);
21506c3fb27SDimitry Andric setLibcallName(RTLIB::LDEXP_F80, nullptr);
21606c3fb27SDimitry Andric setLibcallName(RTLIB::LDEXP_F128, nullptr);
21706c3fb27SDimitry Andric setLibcallName(RTLIB::LDEXP_PPCF128, nullptr);
21806c3fb27SDimitry Andric
21906c3fb27SDimitry Andric setLibcallName(RTLIB::FREXP_F32, nullptr);
22006c3fb27SDimitry Andric setLibcallName(RTLIB::FREXP_F80, nullptr);
22106c3fb27SDimitry Andric setLibcallName(RTLIB::FREXP_F128, nullptr);
22206c3fb27SDimitry Andric setLibcallName(RTLIB::FREXP_PPCF128, nullptr);
22306c3fb27SDimitry Andric }
2240b57cec5SDimitry Andric }
2250b57cec5SDimitry Andric
226fe6060f1SDimitry Andric /// GetFPLibCall - Helper to return the right libcall for the given floating
227fe6060f1SDimitry Andric /// point type, or UNKNOWN_LIBCALL if there is none.
getFPLibCall(EVT VT,RTLIB::Libcall Call_F32,RTLIB::Libcall Call_F64,RTLIB::Libcall Call_F80,RTLIB::Libcall Call_F128,RTLIB::Libcall Call_PPCF128)228fe6060f1SDimitry Andric RTLIB::Libcall RTLIB::getFPLibCall(EVT VT,
229fe6060f1SDimitry Andric RTLIB::Libcall Call_F32,
230fe6060f1SDimitry Andric RTLIB::Libcall Call_F64,
231fe6060f1SDimitry Andric RTLIB::Libcall Call_F80,
232fe6060f1SDimitry Andric RTLIB::Libcall Call_F128,
233fe6060f1SDimitry Andric RTLIB::Libcall Call_PPCF128) {
234fe6060f1SDimitry Andric return
235fe6060f1SDimitry Andric VT == MVT::f32 ? Call_F32 :
236fe6060f1SDimitry Andric VT == MVT::f64 ? Call_F64 :
237fe6060f1SDimitry Andric VT == MVT::f80 ? Call_F80 :
238fe6060f1SDimitry Andric VT == MVT::f128 ? Call_F128 :
239fe6060f1SDimitry Andric VT == MVT::ppcf128 ? Call_PPCF128 :
240fe6060f1SDimitry Andric RTLIB::UNKNOWN_LIBCALL;
241fe6060f1SDimitry Andric }
242fe6060f1SDimitry Andric
2430b57cec5SDimitry Andric /// getFPEXT - Return the FPEXT_*_* value for the given types, or
2440b57cec5SDimitry Andric /// UNKNOWN_LIBCALL if there is none.
getFPEXT(EVT OpVT,EVT RetVT)2450b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getFPEXT(EVT OpVT, EVT RetVT) {
2460b57cec5SDimitry Andric if (OpVT == MVT::f16) {
2470b57cec5SDimitry Andric if (RetVT == MVT::f32)
2480b57cec5SDimitry Andric return FPEXT_F16_F32;
249e8d8bef9SDimitry Andric if (RetVT == MVT::f64)
250e8d8bef9SDimitry Andric return FPEXT_F16_F64;
251349cc55cSDimitry Andric if (RetVT == MVT::f80)
252349cc55cSDimitry Andric return FPEXT_F16_F80;
253e8d8bef9SDimitry Andric if (RetVT == MVT::f128)
254e8d8bef9SDimitry Andric return FPEXT_F16_F128;
2550b57cec5SDimitry Andric } else if (OpVT == MVT::f32) {
2560b57cec5SDimitry Andric if (RetVT == MVT::f64)
2570b57cec5SDimitry Andric return FPEXT_F32_F64;
2580b57cec5SDimitry Andric if (RetVT == MVT::f128)
2590b57cec5SDimitry Andric return FPEXT_F32_F128;
2600b57cec5SDimitry Andric if (RetVT == MVT::ppcf128)
2610b57cec5SDimitry Andric return FPEXT_F32_PPCF128;
2620b57cec5SDimitry Andric } else if (OpVT == MVT::f64) {
2630b57cec5SDimitry Andric if (RetVT == MVT::f128)
2640b57cec5SDimitry Andric return FPEXT_F64_F128;
2650b57cec5SDimitry Andric else if (RetVT == MVT::ppcf128)
2660b57cec5SDimitry Andric return FPEXT_F64_PPCF128;
2670b57cec5SDimitry Andric } else if (OpVT == MVT::f80) {
2680b57cec5SDimitry Andric if (RetVT == MVT::f128)
2690b57cec5SDimitry Andric return FPEXT_F80_F128;
2700b57cec5SDimitry Andric }
2710b57cec5SDimitry Andric
2720b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
2730b57cec5SDimitry Andric }
2740b57cec5SDimitry Andric
2750b57cec5SDimitry Andric /// getFPROUND - Return the FPROUND_*_* value for the given types, or
2760b57cec5SDimitry Andric /// UNKNOWN_LIBCALL if there is none.
getFPROUND(EVT OpVT,EVT RetVT)2770b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getFPROUND(EVT OpVT, EVT RetVT) {
2780b57cec5SDimitry Andric if (RetVT == MVT::f16) {
2790b57cec5SDimitry Andric if (OpVT == MVT::f32)
2800b57cec5SDimitry Andric return FPROUND_F32_F16;
2810b57cec5SDimitry Andric if (OpVT == MVT::f64)
2820b57cec5SDimitry Andric return FPROUND_F64_F16;
2830b57cec5SDimitry Andric if (OpVT == MVT::f80)
2840b57cec5SDimitry Andric return FPROUND_F80_F16;
2850b57cec5SDimitry Andric if (OpVT == MVT::f128)
2860b57cec5SDimitry Andric return FPROUND_F128_F16;
2870b57cec5SDimitry Andric if (OpVT == MVT::ppcf128)
2880b57cec5SDimitry Andric return FPROUND_PPCF128_F16;
28981ad6265SDimitry Andric } else if (RetVT == MVT::bf16) {
29081ad6265SDimitry Andric if (OpVT == MVT::f32)
29181ad6265SDimitry Andric return FPROUND_F32_BF16;
29281ad6265SDimitry Andric if (OpVT == MVT::f64)
29381ad6265SDimitry Andric return FPROUND_F64_BF16;
2940b57cec5SDimitry Andric } else if (RetVT == MVT::f32) {
2950b57cec5SDimitry Andric if (OpVT == MVT::f64)
2960b57cec5SDimitry Andric return FPROUND_F64_F32;
2970b57cec5SDimitry Andric if (OpVT == MVT::f80)
2980b57cec5SDimitry Andric return FPROUND_F80_F32;
2990b57cec5SDimitry Andric if (OpVT == MVT::f128)
3000b57cec5SDimitry Andric return FPROUND_F128_F32;
3010b57cec5SDimitry Andric if (OpVT == MVT::ppcf128)
3020b57cec5SDimitry Andric return FPROUND_PPCF128_F32;
3030b57cec5SDimitry Andric } else if (RetVT == MVT::f64) {
3040b57cec5SDimitry Andric if (OpVT == MVT::f80)
3050b57cec5SDimitry Andric return FPROUND_F80_F64;
3060b57cec5SDimitry Andric if (OpVT == MVT::f128)
3070b57cec5SDimitry Andric return FPROUND_F128_F64;
3080b57cec5SDimitry Andric if (OpVT == MVT::ppcf128)
3090b57cec5SDimitry Andric return FPROUND_PPCF128_F64;
3100b57cec5SDimitry Andric } else if (RetVT == MVT::f80) {
3110b57cec5SDimitry Andric if (OpVT == MVT::f128)
3120b57cec5SDimitry Andric return FPROUND_F128_F80;
3130b57cec5SDimitry Andric }
3140b57cec5SDimitry Andric
3150b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
3160b57cec5SDimitry Andric }
3170b57cec5SDimitry Andric
3180b57cec5SDimitry Andric /// getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or
3190b57cec5SDimitry Andric /// UNKNOWN_LIBCALL if there is none.
getFPTOSINT(EVT OpVT,EVT RetVT)3200b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getFPTOSINT(EVT OpVT, EVT RetVT) {
321e8d8bef9SDimitry Andric if (OpVT == MVT::f16) {
322e8d8bef9SDimitry Andric if (RetVT == MVT::i32)
323e8d8bef9SDimitry Andric return FPTOSINT_F16_I32;
324e8d8bef9SDimitry Andric if (RetVT == MVT::i64)
325e8d8bef9SDimitry Andric return FPTOSINT_F16_I64;
326e8d8bef9SDimitry Andric if (RetVT == MVT::i128)
327e8d8bef9SDimitry Andric return FPTOSINT_F16_I128;
328e8d8bef9SDimitry Andric } else if (OpVT == MVT::f32) {
3290b57cec5SDimitry Andric if (RetVT == MVT::i32)
3300b57cec5SDimitry Andric return FPTOSINT_F32_I32;
3310b57cec5SDimitry Andric if (RetVT == MVT::i64)
3320b57cec5SDimitry Andric return FPTOSINT_F32_I64;
3330b57cec5SDimitry Andric if (RetVT == MVT::i128)
3340b57cec5SDimitry Andric return FPTOSINT_F32_I128;
3350b57cec5SDimitry Andric } else if (OpVT == MVT::f64) {
3360b57cec5SDimitry Andric if (RetVT == MVT::i32)
3370b57cec5SDimitry Andric return FPTOSINT_F64_I32;
3380b57cec5SDimitry Andric if (RetVT == MVT::i64)
3390b57cec5SDimitry Andric return FPTOSINT_F64_I64;
3400b57cec5SDimitry Andric if (RetVT == MVT::i128)
3410b57cec5SDimitry Andric return FPTOSINT_F64_I128;
3420b57cec5SDimitry Andric } else if (OpVT == MVT::f80) {
3430b57cec5SDimitry Andric if (RetVT == MVT::i32)
3440b57cec5SDimitry Andric return FPTOSINT_F80_I32;
3450b57cec5SDimitry Andric if (RetVT == MVT::i64)
3460b57cec5SDimitry Andric return FPTOSINT_F80_I64;
3470b57cec5SDimitry Andric if (RetVT == MVT::i128)
3480b57cec5SDimitry Andric return FPTOSINT_F80_I128;
3490b57cec5SDimitry Andric } else if (OpVT == MVT::f128) {
3500b57cec5SDimitry Andric if (RetVT == MVT::i32)
3510b57cec5SDimitry Andric return FPTOSINT_F128_I32;
3520b57cec5SDimitry Andric if (RetVT == MVT::i64)
3530b57cec5SDimitry Andric return FPTOSINT_F128_I64;
3540b57cec5SDimitry Andric if (RetVT == MVT::i128)
3550b57cec5SDimitry Andric return FPTOSINT_F128_I128;
3560b57cec5SDimitry Andric } else if (OpVT == MVT::ppcf128) {
3570b57cec5SDimitry Andric if (RetVT == MVT::i32)
3580b57cec5SDimitry Andric return FPTOSINT_PPCF128_I32;
3590b57cec5SDimitry Andric if (RetVT == MVT::i64)
3600b57cec5SDimitry Andric return FPTOSINT_PPCF128_I64;
3610b57cec5SDimitry Andric if (RetVT == MVT::i128)
3620b57cec5SDimitry Andric return FPTOSINT_PPCF128_I128;
3630b57cec5SDimitry Andric }
3640b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
3650b57cec5SDimitry Andric }
3660b57cec5SDimitry Andric
3670b57cec5SDimitry Andric /// getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or
3680b57cec5SDimitry Andric /// UNKNOWN_LIBCALL if there is none.
getFPTOUINT(EVT OpVT,EVT RetVT)3690b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getFPTOUINT(EVT OpVT, EVT RetVT) {
370e8d8bef9SDimitry Andric if (OpVT == MVT::f16) {
371e8d8bef9SDimitry Andric if (RetVT == MVT::i32)
372e8d8bef9SDimitry Andric return FPTOUINT_F16_I32;
373e8d8bef9SDimitry Andric if (RetVT == MVT::i64)
374e8d8bef9SDimitry Andric return FPTOUINT_F16_I64;
375e8d8bef9SDimitry Andric if (RetVT == MVT::i128)
376e8d8bef9SDimitry Andric return FPTOUINT_F16_I128;
377e8d8bef9SDimitry Andric } else if (OpVT == MVT::f32) {
3780b57cec5SDimitry Andric if (RetVT == MVT::i32)
3790b57cec5SDimitry Andric return FPTOUINT_F32_I32;
3800b57cec5SDimitry Andric if (RetVT == MVT::i64)
3810b57cec5SDimitry Andric return FPTOUINT_F32_I64;
3820b57cec5SDimitry Andric if (RetVT == MVT::i128)
3830b57cec5SDimitry Andric return FPTOUINT_F32_I128;
3840b57cec5SDimitry Andric } else if (OpVT == MVT::f64) {
3850b57cec5SDimitry Andric if (RetVT == MVT::i32)
3860b57cec5SDimitry Andric return FPTOUINT_F64_I32;
3870b57cec5SDimitry Andric if (RetVT == MVT::i64)
3880b57cec5SDimitry Andric return FPTOUINT_F64_I64;
3890b57cec5SDimitry Andric if (RetVT == MVT::i128)
3900b57cec5SDimitry Andric return FPTOUINT_F64_I128;
3910b57cec5SDimitry Andric } else if (OpVT == MVT::f80) {
3920b57cec5SDimitry Andric if (RetVT == MVT::i32)
3930b57cec5SDimitry Andric return FPTOUINT_F80_I32;
3940b57cec5SDimitry Andric if (RetVT == MVT::i64)
3950b57cec5SDimitry Andric return FPTOUINT_F80_I64;
3960b57cec5SDimitry Andric if (RetVT == MVT::i128)
3970b57cec5SDimitry Andric return FPTOUINT_F80_I128;
3980b57cec5SDimitry Andric } else if (OpVT == MVT::f128) {
3990b57cec5SDimitry Andric if (RetVT == MVT::i32)
4000b57cec5SDimitry Andric return FPTOUINT_F128_I32;
4010b57cec5SDimitry Andric if (RetVT == MVT::i64)
4020b57cec5SDimitry Andric return FPTOUINT_F128_I64;
4030b57cec5SDimitry Andric if (RetVT == MVT::i128)
4040b57cec5SDimitry Andric return FPTOUINT_F128_I128;
4050b57cec5SDimitry Andric } else if (OpVT == MVT::ppcf128) {
4060b57cec5SDimitry Andric if (RetVT == MVT::i32)
4070b57cec5SDimitry Andric return FPTOUINT_PPCF128_I32;
4080b57cec5SDimitry Andric if (RetVT == MVT::i64)
4090b57cec5SDimitry Andric return FPTOUINT_PPCF128_I64;
4100b57cec5SDimitry Andric if (RetVT == MVT::i128)
4110b57cec5SDimitry Andric return FPTOUINT_PPCF128_I128;
4120b57cec5SDimitry Andric }
4130b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
4140b57cec5SDimitry Andric }
4150b57cec5SDimitry Andric
4160b57cec5SDimitry Andric /// getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or
4170b57cec5SDimitry Andric /// UNKNOWN_LIBCALL if there is none.
getSINTTOFP(EVT OpVT,EVT RetVT)4180b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getSINTTOFP(EVT OpVT, EVT RetVT) {
4190b57cec5SDimitry Andric if (OpVT == MVT::i32) {
420e8d8bef9SDimitry Andric if (RetVT == MVT::f16)
421e8d8bef9SDimitry Andric return SINTTOFP_I32_F16;
4220b57cec5SDimitry Andric if (RetVT == MVT::f32)
4230b57cec5SDimitry Andric return SINTTOFP_I32_F32;
4240b57cec5SDimitry Andric if (RetVT == MVT::f64)
4250b57cec5SDimitry Andric return SINTTOFP_I32_F64;
4260b57cec5SDimitry Andric if (RetVT == MVT::f80)
4270b57cec5SDimitry Andric return SINTTOFP_I32_F80;
4280b57cec5SDimitry Andric if (RetVT == MVT::f128)
4290b57cec5SDimitry Andric return SINTTOFP_I32_F128;
4300b57cec5SDimitry Andric if (RetVT == MVT::ppcf128)
4310b57cec5SDimitry Andric return SINTTOFP_I32_PPCF128;
4320b57cec5SDimitry Andric } else if (OpVT == MVT::i64) {
433e8d8bef9SDimitry Andric if (RetVT == MVT::f16)
434e8d8bef9SDimitry Andric return SINTTOFP_I64_F16;
4350b57cec5SDimitry Andric if (RetVT == MVT::f32)
4360b57cec5SDimitry Andric return SINTTOFP_I64_F32;
4370b57cec5SDimitry Andric if (RetVT == MVT::f64)
4380b57cec5SDimitry Andric return SINTTOFP_I64_F64;
4390b57cec5SDimitry Andric if (RetVT == MVT::f80)
4400b57cec5SDimitry Andric return SINTTOFP_I64_F80;
4410b57cec5SDimitry Andric if (RetVT == MVT::f128)
4420b57cec5SDimitry Andric return SINTTOFP_I64_F128;
4430b57cec5SDimitry Andric if (RetVT == MVT::ppcf128)
4440b57cec5SDimitry Andric return SINTTOFP_I64_PPCF128;
4450b57cec5SDimitry Andric } else if (OpVT == MVT::i128) {
446e8d8bef9SDimitry Andric if (RetVT == MVT::f16)
447e8d8bef9SDimitry Andric return SINTTOFP_I128_F16;
4480b57cec5SDimitry Andric if (RetVT == MVT::f32)
4490b57cec5SDimitry Andric return SINTTOFP_I128_F32;
4500b57cec5SDimitry Andric if (RetVT == MVT::f64)
4510b57cec5SDimitry Andric return SINTTOFP_I128_F64;
4520b57cec5SDimitry Andric if (RetVT == MVT::f80)
4530b57cec5SDimitry Andric return SINTTOFP_I128_F80;
4540b57cec5SDimitry Andric if (RetVT == MVT::f128)
4550b57cec5SDimitry Andric return SINTTOFP_I128_F128;
4560b57cec5SDimitry Andric if (RetVT == MVT::ppcf128)
4570b57cec5SDimitry Andric return SINTTOFP_I128_PPCF128;
4580b57cec5SDimitry Andric }
4590b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
4600b57cec5SDimitry Andric }
4610b57cec5SDimitry Andric
4620b57cec5SDimitry Andric /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
4630b57cec5SDimitry Andric /// UNKNOWN_LIBCALL if there is none.
getUINTTOFP(EVT OpVT,EVT RetVT)4640b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getUINTTOFP(EVT OpVT, EVT RetVT) {
4650b57cec5SDimitry Andric if (OpVT == MVT::i32) {
466e8d8bef9SDimitry Andric if (RetVT == MVT::f16)
467e8d8bef9SDimitry Andric return UINTTOFP_I32_F16;
4680b57cec5SDimitry Andric if (RetVT == MVT::f32)
4690b57cec5SDimitry Andric return UINTTOFP_I32_F32;
4700b57cec5SDimitry Andric if (RetVT == MVT::f64)
4710b57cec5SDimitry Andric return UINTTOFP_I32_F64;
4720b57cec5SDimitry Andric if (RetVT == MVT::f80)
4730b57cec5SDimitry Andric return UINTTOFP_I32_F80;
4740b57cec5SDimitry Andric if (RetVT == MVT::f128)
4750b57cec5SDimitry Andric return UINTTOFP_I32_F128;
4760b57cec5SDimitry Andric if (RetVT == MVT::ppcf128)
4770b57cec5SDimitry Andric return UINTTOFP_I32_PPCF128;
4780b57cec5SDimitry Andric } else if (OpVT == MVT::i64) {
479e8d8bef9SDimitry Andric if (RetVT == MVT::f16)
480e8d8bef9SDimitry Andric return UINTTOFP_I64_F16;
4810b57cec5SDimitry Andric if (RetVT == MVT::f32)
4820b57cec5SDimitry Andric return UINTTOFP_I64_F32;
4830b57cec5SDimitry Andric if (RetVT == MVT::f64)
4840b57cec5SDimitry Andric return UINTTOFP_I64_F64;
4850b57cec5SDimitry Andric if (RetVT == MVT::f80)
4860b57cec5SDimitry Andric return UINTTOFP_I64_F80;
4870b57cec5SDimitry Andric if (RetVT == MVT::f128)
4880b57cec5SDimitry Andric return UINTTOFP_I64_F128;
4890b57cec5SDimitry Andric if (RetVT == MVT::ppcf128)
4900b57cec5SDimitry Andric return UINTTOFP_I64_PPCF128;
4910b57cec5SDimitry Andric } else if (OpVT == MVT::i128) {
492e8d8bef9SDimitry Andric if (RetVT == MVT::f16)
493e8d8bef9SDimitry Andric return UINTTOFP_I128_F16;
4940b57cec5SDimitry Andric if (RetVT == MVT::f32)
4950b57cec5SDimitry Andric return UINTTOFP_I128_F32;
4960b57cec5SDimitry Andric if (RetVT == MVT::f64)
4970b57cec5SDimitry Andric return UINTTOFP_I128_F64;
4980b57cec5SDimitry Andric if (RetVT == MVT::f80)
4990b57cec5SDimitry Andric return UINTTOFP_I128_F80;
5000b57cec5SDimitry Andric if (RetVT == MVT::f128)
5010b57cec5SDimitry Andric return UINTTOFP_I128_F128;
5020b57cec5SDimitry Andric if (RetVT == MVT::ppcf128)
5030b57cec5SDimitry Andric return UINTTOFP_I128_PPCF128;
5040b57cec5SDimitry Andric }
5050b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
5060b57cec5SDimitry Andric }
5070b57cec5SDimitry Andric
getPOWI(EVT RetVT)508fe6060f1SDimitry Andric RTLIB::Libcall RTLIB::getPOWI(EVT RetVT) {
509fe6060f1SDimitry Andric return getFPLibCall(RetVT, POWI_F32, POWI_F64, POWI_F80, POWI_F128,
510fe6060f1SDimitry Andric POWI_PPCF128);
511fe6060f1SDimitry Andric }
512fe6060f1SDimitry Andric
getLDEXP(EVT RetVT)51306c3fb27SDimitry Andric RTLIB::Libcall RTLIB::getLDEXP(EVT RetVT) {
51406c3fb27SDimitry Andric return getFPLibCall(RetVT, LDEXP_F32, LDEXP_F64, LDEXP_F80, LDEXP_F128,
51506c3fb27SDimitry Andric LDEXP_PPCF128);
51606c3fb27SDimitry Andric }
51706c3fb27SDimitry Andric
getFREXP(EVT RetVT)51806c3fb27SDimitry Andric RTLIB::Libcall RTLIB::getFREXP(EVT RetVT) {
51906c3fb27SDimitry Andric return getFPLibCall(RetVT, FREXP_F32, FREXP_F64, FREXP_F80, FREXP_F128,
52006c3fb27SDimitry Andric FREXP_PPCF128);
52106c3fb27SDimitry Andric }
52206c3fb27SDimitry Andric
getOutlineAtomicHelper(const Libcall (& LC)[5][4],AtomicOrdering Order,uint64_t MemSize)5231db9f3b2SDimitry Andric RTLIB::Libcall RTLIB::getOutlineAtomicHelper(const Libcall (&LC)[5][4],
5241db9f3b2SDimitry Andric AtomicOrdering Order,
5251db9f3b2SDimitry Andric uint64_t MemSize) {
526e8d8bef9SDimitry Andric unsigned ModeN, ModelN;
5271db9f3b2SDimitry Andric switch (MemSize) {
5281db9f3b2SDimitry Andric case 1:
529e8d8bef9SDimitry Andric ModeN = 0;
530e8d8bef9SDimitry Andric break;
5311db9f3b2SDimitry Andric case 2:
532e8d8bef9SDimitry Andric ModeN = 1;
533e8d8bef9SDimitry Andric break;
5341db9f3b2SDimitry Andric case 4:
535e8d8bef9SDimitry Andric ModeN = 2;
536e8d8bef9SDimitry Andric break;
5371db9f3b2SDimitry Andric case 8:
538e8d8bef9SDimitry Andric ModeN = 3;
539e8d8bef9SDimitry Andric break;
5401db9f3b2SDimitry Andric case 16:
541e8d8bef9SDimitry Andric ModeN = 4;
542e8d8bef9SDimitry Andric break;
543e8d8bef9SDimitry Andric default:
5441db9f3b2SDimitry Andric return RTLIB::UNKNOWN_LIBCALL;
545e8d8bef9SDimitry Andric }
546e8d8bef9SDimitry Andric
547e8d8bef9SDimitry Andric switch (Order) {
548e8d8bef9SDimitry Andric case AtomicOrdering::Monotonic:
549e8d8bef9SDimitry Andric ModelN = 0;
550e8d8bef9SDimitry Andric break;
551e8d8bef9SDimitry Andric case AtomicOrdering::Acquire:
552e8d8bef9SDimitry Andric ModelN = 1;
553e8d8bef9SDimitry Andric break;
554e8d8bef9SDimitry Andric case AtomicOrdering::Release:
555e8d8bef9SDimitry Andric ModelN = 2;
556e8d8bef9SDimitry Andric break;
557e8d8bef9SDimitry Andric case AtomicOrdering::AcquireRelease:
558e8d8bef9SDimitry Andric case AtomicOrdering::SequentiallyConsistent:
559e8d8bef9SDimitry Andric ModelN = 3;
560e8d8bef9SDimitry Andric break;
561e8d8bef9SDimitry Andric default:
562e8d8bef9SDimitry Andric return UNKNOWN_LIBCALL;
563e8d8bef9SDimitry Andric }
564e8d8bef9SDimitry Andric
5651db9f3b2SDimitry Andric return LC[ModeN][ModelN];
5661db9f3b2SDimitry Andric }
5671db9f3b2SDimitry Andric
getOUTLINE_ATOMIC(unsigned Opc,AtomicOrdering Order,MVT VT)5681db9f3b2SDimitry Andric RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,
5691db9f3b2SDimitry Andric MVT VT) {
5701db9f3b2SDimitry Andric if (!VT.isScalarInteger())
5711db9f3b2SDimitry Andric return UNKNOWN_LIBCALL;
5721db9f3b2SDimitry Andric uint64_t MemSize = VT.getScalarSizeInBits() / 8;
5731db9f3b2SDimitry Andric
574e8d8bef9SDimitry Andric #define LCALLS(A, B) \
575e8d8bef9SDimitry Andric { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
576e8d8bef9SDimitry Andric #define LCALL5(A) \
577e8d8bef9SDimitry Andric LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
578e8d8bef9SDimitry Andric switch (Opc) {
579e8d8bef9SDimitry Andric case ISD::ATOMIC_CMP_SWAP: {
580e8d8bef9SDimitry Andric const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};
5811db9f3b2SDimitry Andric return getOutlineAtomicHelper(LC, Order, MemSize);
582e8d8bef9SDimitry Andric }
583e8d8bef9SDimitry Andric case ISD::ATOMIC_SWAP: {
584e8d8bef9SDimitry Andric const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};
5851db9f3b2SDimitry Andric return getOutlineAtomicHelper(LC, Order, MemSize);
586e8d8bef9SDimitry Andric }
587e8d8bef9SDimitry Andric case ISD::ATOMIC_LOAD_ADD: {
588e8d8bef9SDimitry Andric const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};
5891db9f3b2SDimitry Andric return getOutlineAtomicHelper(LC, Order, MemSize);
590e8d8bef9SDimitry Andric }
591e8d8bef9SDimitry Andric case ISD::ATOMIC_LOAD_OR: {
592e8d8bef9SDimitry Andric const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};
5931db9f3b2SDimitry Andric return getOutlineAtomicHelper(LC, Order, MemSize);
594e8d8bef9SDimitry Andric }
595e8d8bef9SDimitry Andric case ISD::ATOMIC_LOAD_CLR: {
596e8d8bef9SDimitry Andric const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};
5971db9f3b2SDimitry Andric return getOutlineAtomicHelper(LC, Order, MemSize);
598e8d8bef9SDimitry Andric }
599e8d8bef9SDimitry Andric case ISD::ATOMIC_LOAD_XOR: {
600e8d8bef9SDimitry Andric const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};
6011db9f3b2SDimitry Andric return getOutlineAtomicHelper(LC, Order, MemSize);
602e8d8bef9SDimitry Andric }
603e8d8bef9SDimitry Andric default:
604e8d8bef9SDimitry Andric return UNKNOWN_LIBCALL;
605e8d8bef9SDimitry Andric }
606e8d8bef9SDimitry Andric #undef LCALLS
607e8d8bef9SDimitry Andric #undef LCALL5
608e8d8bef9SDimitry Andric }
609e8d8bef9SDimitry Andric
getSYNC(unsigned Opc,MVT VT)6100b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getSYNC(unsigned Opc, MVT VT) {
6110b57cec5SDimitry Andric #define OP_TO_LIBCALL(Name, Enum) \
6120b57cec5SDimitry Andric case Name: \
6130b57cec5SDimitry Andric switch (VT.SimpleTy) { \
6140b57cec5SDimitry Andric default: \
6150b57cec5SDimitry Andric return UNKNOWN_LIBCALL; \
6160b57cec5SDimitry Andric case MVT::i8: \
6170b57cec5SDimitry Andric return Enum##_1; \
6180b57cec5SDimitry Andric case MVT::i16: \
6190b57cec5SDimitry Andric return Enum##_2; \
6200b57cec5SDimitry Andric case MVT::i32: \
6210b57cec5SDimitry Andric return Enum##_4; \
6220b57cec5SDimitry Andric case MVT::i64: \
6230b57cec5SDimitry Andric return Enum##_8; \
6240b57cec5SDimitry Andric case MVT::i128: \
6250b57cec5SDimitry Andric return Enum##_16; \
6260b57cec5SDimitry Andric }
6270b57cec5SDimitry Andric
6280b57cec5SDimitry Andric switch (Opc) {
6290b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_SWAP, SYNC_LOCK_TEST_AND_SET)
6300b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_CMP_SWAP, SYNC_VAL_COMPARE_AND_SWAP)
6310b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_ADD, SYNC_FETCH_AND_ADD)
6320b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_SUB, SYNC_FETCH_AND_SUB)
6330b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_AND, SYNC_FETCH_AND_AND)
6340b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_OR, SYNC_FETCH_AND_OR)
6350b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_XOR, SYNC_FETCH_AND_XOR)
6360b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_NAND, SYNC_FETCH_AND_NAND)
6370b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MAX, SYNC_FETCH_AND_MAX)
6380b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMAX, SYNC_FETCH_AND_UMAX)
6390b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_MIN, SYNC_FETCH_AND_MIN)
6400b57cec5SDimitry Andric OP_TO_LIBCALL(ISD::ATOMIC_LOAD_UMIN, SYNC_FETCH_AND_UMIN)
6410b57cec5SDimitry Andric }
6420b57cec5SDimitry Andric
6430b57cec5SDimitry Andric #undef OP_TO_LIBCALL
6440b57cec5SDimitry Andric
6450b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
6460b57cec5SDimitry Andric }
6470b57cec5SDimitry Andric
getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)6480b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
6490b57cec5SDimitry Andric switch (ElementSize) {
6500b57cec5SDimitry Andric case 1:
6510b57cec5SDimitry Andric return MEMCPY_ELEMENT_UNORDERED_ATOMIC_1;
6520b57cec5SDimitry Andric case 2:
6530b57cec5SDimitry Andric return MEMCPY_ELEMENT_UNORDERED_ATOMIC_2;
6540b57cec5SDimitry Andric case 4:
6550b57cec5SDimitry Andric return MEMCPY_ELEMENT_UNORDERED_ATOMIC_4;
6560b57cec5SDimitry Andric case 8:
6570b57cec5SDimitry Andric return MEMCPY_ELEMENT_UNORDERED_ATOMIC_8;
6580b57cec5SDimitry Andric case 16:
6590b57cec5SDimitry Andric return MEMCPY_ELEMENT_UNORDERED_ATOMIC_16;
6600b57cec5SDimitry Andric default:
6610b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
6620b57cec5SDimitry Andric }
6630b57cec5SDimitry Andric }
6640b57cec5SDimitry Andric
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)6650b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
6660b57cec5SDimitry Andric switch (ElementSize) {
6670b57cec5SDimitry Andric case 1:
6680b57cec5SDimitry Andric return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_1;
6690b57cec5SDimitry Andric case 2:
6700b57cec5SDimitry Andric return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_2;
6710b57cec5SDimitry Andric case 4:
6720b57cec5SDimitry Andric return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_4;
6730b57cec5SDimitry Andric case 8:
6740b57cec5SDimitry Andric return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_8;
6750b57cec5SDimitry Andric case 16:
6760b57cec5SDimitry Andric return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_16;
6770b57cec5SDimitry Andric default:
6780b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
6790b57cec5SDimitry Andric }
6800b57cec5SDimitry Andric }
6810b57cec5SDimitry Andric
getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)6820b57cec5SDimitry Andric RTLIB::Libcall RTLIB::getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize) {
6830b57cec5SDimitry Andric switch (ElementSize) {
6840b57cec5SDimitry Andric case 1:
6850b57cec5SDimitry Andric return MEMSET_ELEMENT_UNORDERED_ATOMIC_1;
6860b57cec5SDimitry Andric case 2:
6870b57cec5SDimitry Andric return MEMSET_ELEMENT_UNORDERED_ATOMIC_2;
6880b57cec5SDimitry Andric case 4:
6890b57cec5SDimitry Andric return MEMSET_ELEMENT_UNORDERED_ATOMIC_4;
6900b57cec5SDimitry Andric case 8:
6910b57cec5SDimitry Andric return MEMSET_ELEMENT_UNORDERED_ATOMIC_8;
6920b57cec5SDimitry Andric case 16:
6930b57cec5SDimitry Andric return MEMSET_ELEMENT_UNORDERED_ATOMIC_16;
6940b57cec5SDimitry Andric default:
6950b57cec5SDimitry Andric return UNKNOWN_LIBCALL;
6960b57cec5SDimitry Andric }
6970b57cec5SDimitry Andric }
6980b57cec5SDimitry Andric
6990b57cec5SDimitry Andric /// InitCmpLibcallCCs - Set default comparison libcall CC.
InitCmpLibcallCCs(ISD::CondCode * CCs)7000b57cec5SDimitry Andric static void InitCmpLibcallCCs(ISD::CondCode *CCs) {
701349cc55cSDimitry Andric std::fill(CCs, CCs + RTLIB::UNKNOWN_LIBCALL, ISD::SETCC_INVALID);
7020b57cec5SDimitry Andric CCs[RTLIB::OEQ_F32] = ISD::SETEQ;
7030b57cec5SDimitry Andric CCs[RTLIB::OEQ_F64] = ISD::SETEQ;
7040b57cec5SDimitry Andric CCs[RTLIB::OEQ_F128] = ISD::SETEQ;
7050b57cec5SDimitry Andric CCs[RTLIB::OEQ_PPCF128] = ISD::SETEQ;
7060b57cec5SDimitry Andric CCs[RTLIB::UNE_F32] = ISD::SETNE;
7070b57cec5SDimitry Andric CCs[RTLIB::UNE_F64] = ISD::SETNE;
7080b57cec5SDimitry Andric CCs[RTLIB::UNE_F128] = ISD::SETNE;
7090b57cec5SDimitry Andric CCs[RTLIB::UNE_PPCF128] = ISD::SETNE;
7100b57cec5SDimitry Andric CCs[RTLIB::OGE_F32] = ISD::SETGE;
7110b57cec5SDimitry Andric CCs[RTLIB::OGE_F64] = ISD::SETGE;
7120b57cec5SDimitry Andric CCs[RTLIB::OGE_F128] = ISD::SETGE;
7130b57cec5SDimitry Andric CCs[RTLIB::OGE_PPCF128] = ISD::SETGE;
7140b57cec5SDimitry Andric CCs[RTLIB::OLT_F32] = ISD::SETLT;
7150b57cec5SDimitry Andric CCs[RTLIB::OLT_F64] = ISD::SETLT;
7160b57cec5SDimitry Andric CCs[RTLIB::OLT_F128] = ISD::SETLT;
7170b57cec5SDimitry Andric CCs[RTLIB::OLT_PPCF128] = ISD::SETLT;
7180b57cec5SDimitry Andric CCs[RTLIB::OLE_F32] = ISD::SETLE;
7190b57cec5SDimitry Andric CCs[RTLIB::OLE_F64] = ISD::SETLE;
7200b57cec5SDimitry Andric CCs[RTLIB::OLE_F128] = ISD::SETLE;
7210b57cec5SDimitry Andric CCs[RTLIB::OLE_PPCF128] = ISD::SETLE;
7220b57cec5SDimitry Andric CCs[RTLIB::OGT_F32] = ISD::SETGT;
7230b57cec5SDimitry Andric CCs[RTLIB::OGT_F64] = ISD::SETGT;
7240b57cec5SDimitry Andric CCs[RTLIB::OGT_F128] = ISD::SETGT;
7250b57cec5SDimitry Andric CCs[RTLIB::OGT_PPCF128] = ISD::SETGT;
7260b57cec5SDimitry Andric CCs[RTLIB::UO_F32] = ISD::SETNE;
7270b57cec5SDimitry Andric CCs[RTLIB::UO_F64] = ISD::SETNE;
7280b57cec5SDimitry Andric CCs[RTLIB::UO_F128] = ISD::SETNE;
7290b57cec5SDimitry Andric CCs[RTLIB::UO_PPCF128] = ISD::SETNE;
7300b57cec5SDimitry Andric }
7310b57cec5SDimitry Andric
7320b57cec5SDimitry Andric /// NOTE: The TargetMachine owns TLOF.
TargetLoweringBase(const TargetMachine & tm)7330b57cec5SDimitry Andric TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) {
7340b57cec5SDimitry Andric initActions();
7350b57cec5SDimitry Andric
7360b57cec5SDimitry Andric // Perform these initializations only once.
7370b57cec5SDimitry Andric MaxStoresPerMemset = MaxStoresPerMemcpy = MaxStoresPerMemmove =
7380b57cec5SDimitry Andric MaxLoadsPerMemcmp = 8;
7390b57cec5SDimitry Andric MaxGluedStoresPerMemcpy = 0;
7400b57cec5SDimitry Andric MaxStoresPerMemsetOptSize = MaxStoresPerMemcpyOptSize =
7410b57cec5SDimitry Andric MaxStoresPerMemmoveOptSize = MaxLoadsPerMemcmpOptSize = 4;
7420b57cec5SDimitry Andric HasMultipleConditionRegisters = false;
7430b57cec5SDimitry Andric HasExtractBitsInsn = false;
7440b57cec5SDimitry Andric JumpIsExpensive = JumpIsExpensiveOverride;
7450b57cec5SDimitry Andric PredictableSelectIsExpensive = false;
7460b57cec5SDimitry Andric EnableExtLdPromotion = false;
7470b57cec5SDimitry Andric StackPointerRegisterToSaveRestore = 0;
7480b57cec5SDimitry Andric BooleanContents = UndefinedBooleanContent;
7490b57cec5SDimitry Andric BooleanFloatContents = UndefinedBooleanContent;
7500b57cec5SDimitry Andric BooleanVectorContents = UndefinedBooleanContent;
7510b57cec5SDimitry Andric SchedPreferenceInfo = Sched::ILP;
7520b57cec5SDimitry Andric GatherAllAliasesMaxDepth = 18;
753480093f4SDimitry Andric IsStrictFPEnabled = DisableStrictNodeMutation;
75404eeddc0SDimitry Andric MaxBytesForAlignment = 0;
7557a6dacacSDimitry Andric MaxAtomicSizeInBitsSupported = 0;
7560b57cec5SDimitry Andric
7571ac55f4cSDimitry Andric // Assume that even with libcalls, no target supports wider than 128 bit
7581ac55f4cSDimitry Andric // division.
7591ac55f4cSDimitry Andric MaxDivRemBitWidthSupported = 128;
760bdd1243dSDimitry Andric
761bdd1243dSDimitry Andric MaxLargeFPConvertBitWidthSupported = llvm::IntegerType::MAX_INT_BITS;
762bdd1243dSDimitry Andric
7630b57cec5SDimitry Andric MinCmpXchgSizeInBits = 0;
7640b57cec5SDimitry Andric SupportsUnalignedAtomics = false;
7650b57cec5SDimitry Andric
7660b57cec5SDimitry Andric std::fill(std::begin(LibcallRoutineNames), std::end(LibcallRoutineNames), nullptr);
7670b57cec5SDimitry Andric
7680b57cec5SDimitry Andric InitLibcalls(TM.getTargetTriple());
7690b57cec5SDimitry Andric InitCmpLibcallCCs(CmpLibcallCCs);
7700b57cec5SDimitry Andric }
7710b57cec5SDimitry Andric
initActions()7720b57cec5SDimitry Andric void TargetLoweringBase::initActions() {
7730b57cec5SDimitry Andric // All operations default to being supported.
7740b57cec5SDimitry Andric memset(OpActions, 0, sizeof(OpActions));
7750b57cec5SDimitry Andric memset(LoadExtActions, 0, sizeof(LoadExtActions));
7760b57cec5SDimitry Andric memset(TruncStoreActions, 0, sizeof(TruncStoreActions));
7770b57cec5SDimitry Andric memset(IndexedModeActions, 0, sizeof(IndexedModeActions));
7780b57cec5SDimitry Andric memset(CondCodeActions, 0, sizeof(CondCodeActions));
7790b57cec5SDimitry Andric std::fill(std::begin(RegClassForVT), std::end(RegClassForVT), nullptr);
7800b57cec5SDimitry Andric std::fill(std::begin(TargetDAGCombineArray),
7810b57cec5SDimitry Andric std::end(TargetDAGCombineArray), 0);
7820b57cec5SDimitry Andric
78381ad6265SDimitry Andric // We're somewhat special casing MVT::i2 and MVT::i4. Ideally we want to
78481ad6265SDimitry Andric // remove this and targets should individually set these types if not legal.
78581ad6265SDimitry Andric for (ISD::NodeType NT : enum_seq(ISD::DELETED_NODE, ISD::BUILTIN_OP_END,
78681ad6265SDimitry Andric force_iteration_on_noniterable_enum)) {
78781ad6265SDimitry Andric for (MVT VT : {MVT::i2, MVT::i4})
78881ad6265SDimitry Andric OpActions[(unsigned)VT.SimpleTy][NT] = Expand;
78981ad6265SDimitry Andric }
79081ad6265SDimitry Andric for (MVT AVT : MVT::all_valuetypes()) {
79181ad6265SDimitry Andric for (MVT VT : {MVT::i2, MVT::i4, MVT::v128i2, MVT::v64i4}) {
79281ad6265SDimitry Andric setTruncStoreAction(AVT, VT, Expand);
79381ad6265SDimitry Andric setLoadExtAction(ISD::EXTLOAD, AVT, VT, Expand);
79481ad6265SDimitry Andric setLoadExtAction(ISD::ZEXTLOAD, AVT, VT, Expand);
79581ad6265SDimitry Andric }
79681ad6265SDimitry Andric }
79781ad6265SDimitry Andric for (unsigned IM = (unsigned)ISD::PRE_INC;
79881ad6265SDimitry Andric IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
79981ad6265SDimitry Andric for (MVT VT : {MVT::i2, MVT::i4}) {
80081ad6265SDimitry Andric setIndexedLoadAction(IM, VT, Expand);
80181ad6265SDimitry Andric setIndexedStoreAction(IM, VT, Expand);
80281ad6265SDimitry Andric setIndexedMaskedLoadAction(IM, VT, Expand);
80381ad6265SDimitry Andric setIndexedMaskedStoreAction(IM, VT, Expand);
80481ad6265SDimitry Andric }
80581ad6265SDimitry Andric }
80681ad6265SDimitry Andric
8070b57cec5SDimitry Andric for (MVT VT : MVT::fp_valuetypes()) {
808e8d8bef9SDimitry Andric MVT IntVT = MVT::getIntegerVT(VT.getFixedSizeInBits());
8090b57cec5SDimitry Andric if (IntVT.isValid()) {
8100b57cec5SDimitry Andric setOperationAction(ISD::ATOMIC_SWAP, VT, Promote);
8110b57cec5SDimitry Andric AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT);
8120b57cec5SDimitry Andric }
8130b57cec5SDimitry Andric }
8140b57cec5SDimitry Andric
8150b57cec5SDimitry Andric // Set default actions for various operations.
8160b57cec5SDimitry Andric for (MVT VT : MVT::all_valuetypes()) {
8170b57cec5SDimitry Andric // Default all indexed load / store to expand.
8180b57cec5SDimitry Andric for (unsigned IM = (unsigned)ISD::PRE_INC;
8190b57cec5SDimitry Andric IM != (unsigned)ISD::LAST_INDEXED_MODE; ++IM) {
8200b57cec5SDimitry Andric setIndexedLoadAction(IM, VT, Expand);
8210b57cec5SDimitry Andric setIndexedStoreAction(IM, VT, Expand);
822480093f4SDimitry Andric setIndexedMaskedLoadAction(IM, VT, Expand);
823480093f4SDimitry Andric setIndexedMaskedStoreAction(IM, VT, Expand);
8240b57cec5SDimitry Andric }
8250b57cec5SDimitry Andric
8260b57cec5SDimitry Andric // Most backends expect to see the node which just returns the value loaded.
8270b57cec5SDimitry Andric setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Expand);
8280b57cec5SDimitry Andric
8290b57cec5SDimitry Andric // These operations default to expand.
83081ad6265SDimitry Andric setOperationAction({ISD::FGETSIGN, ISD::CONCAT_VECTORS,
83181ad6265SDimitry Andric ISD::FMINNUM, ISD::FMAXNUM,
83281ad6265SDimitry Andric ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE,
83381ad6265SDimitry Andric ISD::FMINIMUM, ISD::FMAXIMUM,
83481ad6265SDimitry Andric ISD::FMAD, ISD::SMIN,
83581ad6265SDimitry Andric ISD::SMAX, ISD::UMIN,
83681ad6265SDimitry Andric ISD::UMAX, ISD::ABS,
83781ad6265SDimitry Andric ISD::FSHL, ISD::FSHR,
83881ad6265SDimitry Andric ISD::SADDSAT, ISD::UADDSAT,
83981ad6265SDimitry Andric ISD::SSUBSAT, ISD::USUBSAT,
84081ad6265SDimitry Andric ISD::SSHLSAT, ISD::USHLSAT,
84181ad6265SDimitry Andric ISD::SMULFIX, ISD::SMULFIXSAT,
84281ad6265SDimitry Andric ISD::UMULFIX, ISD::UMULFIXSAT,
84381ad6265SDimitry Andric ISD::SDIVFIX, ISD::SDIVFIXSAT,
84481ad6265SDimitry Andric ISD::UDIVFIX, ISD::UDIVFIXSAT,
84581ad6265SDimitry Andric ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
84681ad6265SDimitry Andric ISD::IS_FPCLASS},
84781ad6265SDimitry Andric VT, Expand);
8480b57cec5SDimitry Andric
8490b57cec5SDimitry Andric // Overflow operations default to expand
85081ad6265SDimitry Andric setOperationAction({ISD::SADDO, ISD::SSUBO, ISD::UADDO, ISD::USUBO,
85181ad6265SDimitry Andric ISD::SMULO, ISD::UMULO},
85281ad6265SDimitry Andric VT, Expand);
8530b57cec5SDimitry Andric
85406c3fb27SDimitry Andric // Carry-using overflow operations default to expand.
85506c3fb27SDimitry Andric setOperationAction({ISD::UADDO_CARRY, ISD::USUBO_CARRY, ISD::SETCCCARRY,
85681ad6265SDimitry Andric ISD::SADDO_CARRY, ISD::SSUBO_CARRY},
85781ad6265SDimitry Andric VT, Expand);
8580b57cec5SDimitry Andric
8590b57cec5SDimitry Andric // ADDC/ADDE/SUBC/SUBE default to expand.
86081ad6265SDimitry Andric setOperationAction({ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}, VT,
86181ad6265SDimitry Andric Expand);
86281ad6265SDimitry Andric
86381ad6265SDimitry Andric // Halving adds
86481ad6265SDimitry Andric setOperationAction(
86581ad6265SDimitry Andric {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS, ISD::AVGCEILU}, VT,
86681ad6265SDimitry Andric Expand);
8670b57cec5SDimitry Andric
868fe6060f1SDimitry Andric // Absolute difference
86981ad6265SDimitry Andric setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Expand);
870fe6060f1SDimitry Andric
8710b57cec5SDimitry Andric // These default to Expand so they will be expanded to CTLZ/CTTZ by default.
87281ad6265SDimitry Andric setOperationAction({ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT,
87381ad6265SDimitry Andric Expand);
8740b57cec5SDimitry Andric
87581ad6265SDimitry Andric setOperationAction({ISD::BITREVERSE, ISD::PARITY}, VT, Expand);
8760b57cec5SDimitry Andric
8770b57cec5SDimitry Andric // These library functions default to expand.
8785f757f3fSDimitry Andric setOperationAction({ISD::FROUND, ISD::FPOWI, ISD::FLDEXP, ISD::FFREXP}, VT,
8795f757f3fSDimitry Andric Expand);
8800b57cec5SDimitry Andric
8810b57cec5SDimitry Andric // These operations default to expand for vector types.
88281ad6265SDimitry Andric if (VT.isVector())
8835f757f3fSDimitry Andric setOperationAction(
8845f757f3fSDimitry Andric {ISD::FCOPYSIGN, ISD::SIGN_EXTEND_INREG, ISD::ANY_EXTEND_VECTOR_INREG,
8855f757f3fSDimitry Andric ISD::SIGN_EXTEND_VECTOR_INREG, ISD::ZERO_EXTEND_VECTOR_INREG,
8865f757f3fSDimitry Andric ISD::SPLAT_VECTOR, ISD::LRINT, ISD::LLRINT},
88781ad6265SDimitry Andric VT, Expand);
8880b57cec5SDimitry Andric
8890b57cec5SDimitry Andric // Constrained floating-point operations default to expand.
8905ffd83dbSDimitry Andric #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
891480093f4SDimitry Andric setOperationAction(ISD::STRICT_##DAGN, VT, Expand);
892480093f4SDimitry Andric #include "llvm/IR/ConstrainedOps.def"
8930b57cec5SDimitry Andric
8940b57cec5SDimitry Andric // For most targets @llvm.get.dynamic.area.offset just returns 0.
8950b57cec5SDimitry Andric setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, VT, Expand);
8960b57cec5SDimitry Andric
8970b57cec5SDimitry Andric // Vector reduction default to expand.
89881ad6265SDimitry Andric setOperationAction(
89981ad6265SDimitry Andric {ISD::VECREDUCE_FADD, ISD::VECREDUCE_FMUL, ISD::VECREDUCE_ADD,
90081ad6265SDimitry Andric ISD::VECREDUCE_MUL, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
90181ad6265SDimitry Andric ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
90281ad6265SDimitry Andric ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN, ISD::VECREDUCE_FMAX,
90306c3fb27SDimitry Andric ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAXIMUM, ISD::VECREDUCE_FMINIMUM,
90406c3fb27SDimitry Andric ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_SEQ_FMUL},
90581ad6265SDimitry Andric VT, Expand);
906fe6060f1SDimitry Andric
907fe6060f1SDimitry Andric // Named vector shuffles default to expand.
908fe6060f1SDimitry Andric setOperationAction(ISD::VECTOR_SPLICE, VT, Expand);
909bdd1243dSDimitry Andric
91006c3fb27SDimitry Andric // VP operations default to expand.
91106c3fb27SDimitry Andric #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) \
91206c3fb27SDimitry Andric setOperationAction(ISD::SDOPC, VT, Expand);
91306c3fb27SDimitry Andric #include "llvm/IR/VPIntrinsics.def"
91406c3fb27SDimitry Andric
91506c3fb27SDimitry Andric // FP environment operations default to expand.
91606c3fb27SDimitry Andric setOperationAction(ISD::GET_FPENV, VT, Expand);
91706c3fb27SDimitry Andric setOperationAction(ISD::SET_FPENV, VT, Expand);
91806c3fb27SDimitry Andric setOperationAction(ISD::RESET_FPENV, VT, Expand);
9190b57cec5SDimitry Andric }
9200b57cec5SDimitry Andric
9210b57cec5SDimitry Andric // Most targets ignore the @llvm.prefetch intrinsic.
9220b57cec5SDimitry Andric setOperationAction(ISD::PREFETCH, MVT::Other, Expand);
9230b57cec5SDimitry Andric
9240b57cec5SDimitry Andric // Most targets also ignore the @llvm.readcyclecounter intrinsic.
9250b57cec5SDimitry Andric setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Expand);
9260b57cec5SDimitry Andric
9270b57cec5SDimitry Andric // ConstantFP nodes default to expand. Targets can either change this to
9280b57cec5SDimitry Andric // Legal, in which case all fp constants are legal, or use isFPImmLegal()
9290b57cec5SDimitry Andric // to optimize expansions for certain constants.
93081ad6265SDimitry Andric setOperationAction(ISD::ConstantFP,
9318a4dda33SDimitry Andric {MVT::bf16, MVT::f16, MVT::f32, MVT::f64, MVT::f80, MVT::f128},
93281ad6265SDimitry Andric Expand);
9330b57cec5SDimitry Andric
9340b57cec5SDimitry Andric // These library functions default to expand.
93581ad6265SDimitry Andric setOperationAction({ISD::FCBRT, ISD::FLOG, ISD::FLOG2, ISD::FLOG10, ISD::FEXP,
9365f757f3fSDimitry Andric ISD::FEXP2, ISD::FEXP10, ISD::FFLOOR, ISD::FNEARBYINT,
9375f757f3fSDimitry Andric ISD::FCEIL, ISD::FRINT, ISD::FTRUNC, ISD::LROUND,
9385f757f3fSDimitry Andric ISD::LLROUND, ISD::LRINT, ISD::LLRINT, ISD::FROUNDEVEN},
93981ad6265SDimitry Andric {MVT::f32, MVT::f64, MVT::f128}, Expand);
9400b57cec5SDimitry Andric
9410b57cec5SDimitry Andric // Default ISD::TRAP to expand (which turns it into abort).
9420b57cec5SDimitry Andric setOperationAction(ISD::TRAP, MVT::Other, Expand);
9430b57cec5SDimitry Andric
9440b57cec5SDimitry Andric // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
9450b57cec5SDimitry Andric // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
9460b57cec5SDimitry Andric setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand);
947e8d8bef9SDimitry Andric
948e8d8bef9SDimitry Andric setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);
94906c3fb27SDimitry Andric
95006c3fb27SDimitry Andric setOperationAction(ISD::GET_FPENV_MEM, MVT::Other, Expand);
95106c3fb27SDimitry Andric setOperationAction(ISD::SET_FPENV_MEM, MVT::Other, Expand);
9525f757f3fSDimitry Andric
9535f757f3fSDimitry Andric for (MVT VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
9545f757f3fSDimitry Andric setOperationAction(ISD::GET_FPMODE, VT, Expand);
9555f757f3fSDimitry Andric setOperationAction(ISD::SET_FPMODE, VT, Expand);
9565f757f3fSDimitry Andric }
9575f757f3fSDimitry Andric setOperationAction(ISD::RESET_FPMODE, MVT::Other, Expand);
9580b57cec5SDimitry Andric }
9590b57cec5SDimitry Andric
getScalarShiftAmountTy(const DataLayout & DL,EVT) const9600b57cec5SDimitry Andric MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL,
9610b57cec5SDimitry Andric EVT) const {
9620b57cec5SDimitry Andric return MVT::getIntegerVT(DL.getPointerSizeInBits(0));
9630b57cec5SDimitry Andric }
9640b57cec5SDimitry Andric
getShiftAmountTy(EVT LHSTy,const DataLayout & DL,bool LegalTypes) const9650b57cec5SDimitry Andric EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy, const DataLayout &DL,
9660b57cec5SDimitry Andric bool LegalTypes) const {
9670b57cec5SDimitry Andric assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
9680b57cec5SDimitry Andric if (LHSTy.isVector())
9690b57cec5SDimitry Andric return LHSTy;
970349cc55cSDimitry Andric MVT ShiftVT =
971349cc55cSDimitry Andric LegalTypes ? getScalarShiftAmountTy(DL, LHSTy) : getPointerTy(DL);
972349cc55cSDimitry Andric // If any possible shift value won't fit in the prefered type, just use
973349cc55cSDimitry Andric // something safe. Assume it will be legalized when the shift is expanded.
974349cc55cSDimitry Andric if (ShiftVT.getSizeInBits() < Log2_32_Ceil(LHSTy.getSizeInBits()))
975349cc55cSDimitry Andric ShiftVT = MVT::i32;
976349cc55cSDimitry Andric assert(ShiftVT.getSizeInBits() >= Log2_32_Ceil(LHSTy.getSizeInBits()) &&
977349cc55cSDimitry Andric "ShiftVT is still too small!");
978349cc55cSDimitry Andric return ShiftVT;
9790b57cec5SDimitry Andric }
9800b57cec5SDimitry Andric
canOpTrap(unsigned Op,EVT VT) const9810b57cec5SDimitry Andric bool TargetLoweringBase::canOpTrap(unsigned Op, EVT VT) const {
9820b57cec5SDimitry Andric assert(isTypeLegal(VT));
9830b57cec5SDimitry Andric switch (Op) {
9840b57cec5SDimitry Andric default:
9850b57cec5SDimitry Andric return false;
9860b57cec5SDimitry Andric case ISD::SDIV:
9870b57cec5SDimitry Andric case ISD::UDIV:
9880b57cec5SDimitry Andric case ISD::SREM:
9890b57cec5SDimitry Andric case ISD::UREM:
9900b57cec5SDimitry Andric return true;
9910b57cec5SDimitry Andric }
9920b57cec5SDimitry Andric }
9930b57cec5SDimitry Andric
isFreeAddrSpaceCast(unsigned SrcAS,unsigned DestAS) const994e8d8bef9SDimitry Andric bool TargetLoweringBase::isFreeAddrSpaceCast(unsigned SrcAS,
995e8d8bef9SDimitry Andric unsigned DestAS) const {
996e8d8bef9SDimitry Andric return TM.isNoopAddrSpaceCast(SrcAS, DestAS);
997e8d8bef9SDimitry Andric }
998e8d8bef9SDimitry Andric
setJumpIsExpensive(bool isExpensive)9990b57cec5SDimitry Andric void TargetLoweringBase::setJumpIsExpensive(bool isExpensive) {
10000b57cec5SDimitry Andric // If the command-line option was specified, ignore this request.
10010b57cec5SDimitry Andric if (!JumpIsExpensiveOverride.getNumOccurrences())
10020b57cec5SDimitry Andric JumpIsExpensive = isExpensive;
10030b57cec5SDimitry Andric }
10040b57cec5SDimitry Andric
10050b57cec5SDimitry Andric TargetLoweringBase::LegalizeKind
getTypeConversion(LLVMContext & Context,EVT VT) const10060b57cec5SDimitry Andric TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const {
10070b57cec5SDimitry Andric // If this is a simple type, use the ComputeRegisterProp mechanism.
10080b57cec5SDimitry Andric if (VT.isSimple()) {
10090b57cec5SDimitry Andric MVT SVT = VT.getSimpleVT();
1010bdd1243dSDimitry Andric assert((unsigned)SVT.SimpleTy < std::size(TransformToType));
10110b57cec5SDimitry Andric MVT NVT = TransformToType[SVT.SimpleTy];
10120b57cec5SDimitry Andric LegalizeTypeAction LA = ValueTypeActions.getTypeAction(SVT);
10130b57cec5SDimitry Andric
10140b57cec5SDimitry Andric assert((LA == TypeLegal || LA == TypeSoftenFloat ||
10155ffd83dbSDimitry Andric LA == TypeSoftPromoteHalf ||
10168bcb0991SDimitry Andric (NVT.isVector() ||
10178bcb0991SDimitry Andric ValueTypeActions.getTypeAction(NVT) != TypePromoteInteger)) &&
10180b57cec5SDimitry Andric "Promote may not follow Expand or Promote");
10190b57cec5SDimitry Andric
10200b57cec5SDimitry Andric if (LA == TypeSplitVector)
1021e8d8bef9SDimitry Andric return LegalizeKind(LA, EVT(SVT).getHalfNumVectorElementsVT(Context));
10220b57cec5SDimitry Andric if (LA == TypeScalarizeVector)
10230b57cec5SDimitry Andric return LegalizeKind(LA, SVT.getVectorElementType());
10240b57cec5SDimitry Andric return LegalizeKind(LA, NVT);
10250b57cec5SDimitry Andric }
10260b57cec5SDimitry Andric
10270b57cec5SDimitry Andric // Handle Extended Scalar Types.
10280b57cec5SDimitry Andric if (!VT.isVector()) {
10290b57cec5SDimitry Andric assert(VT.isInteger() && "Float types must be simple");
10300b57cec5SDimitry Andric unsigned BitSize = VT.getSizeInBits();
10310b57cec5SDimitry Andric // First promote to a power-of-two size, then expand if necessary.
10320b57cec5SDimitry Andric if (BitSize < 8 || !isPowerOf2_32(BitSize)) {
10330b57cec5SDimitry Andric EVT NVT = VT.getRoundIntegerType(Context);
10340b57cec5SDimitry Andric assert(NVT != VT && "Unable to round integer VT");
10350b57cec5SDimitry Andric LegalizeKind NextStep = getTypeConversion(Context, NVT);
10360b57cec5SDimitry Andric // Avoid multi-step promotion.
10370b57cec5SDimitry Andric if (NextStep.first == TypePromoteInteger)
10380b57cec5SDimitry Andric return NextStep;
10390b57cec5SDimitry Andric // Return rounded integer type.
10400b57cec5SDimitry Andric return LegalizeKind(TypePromoteInteger, NVT);
10410b57cec5SDimitry Andric }
10420b57cec5SDimitry Andric
10430b57cec5SDimitry Andric return LegalizeKind(TypeExpandInteger,
10440b57cec5SDimitry Andric EVT::getIntegerVT(Context, VT.getSizeInBits() / 2));
10450b57cec5SDimitry Andric }
10460b57cec5SDimitry Andric
10470b57cec5SDimitry Andric // Handle vector types.
10485ffd83dbSDimitry Andric ElementCount NumElts = VT.getVectorElementCount();
10490b57cec5SDimitry Andric EVT EltVT = VT.getVectorElementType();
10500b57cec5SDimitry Andric
10510b57cec5SDimitry Andric // Vectors with only one element are always scalarized.
1052e8d8bef9SDimitry Andric if (NumElts.isScalar())
10530b57cec5SDimitry Andric return LegalizeKind(TypeScalarizeVector, EltVT);
10540b57cec5SDimitry Andric
10550b57cec5SDimitry Andric // Try to widen vector elements until the element type is a power of two and
10560b57cec5SDimitry Andric // promote it to a legal type later on, for example:
10570b57cec5SDimitry Andric // <3 x i8> -> <4 x i8> -> <4 x i32>
10580b57cec5SDimitry Andric if (EltVT.isInteger()) {
10590b57cec5SDimitry Andric // Vectors with a number of elements that is not a power of two are always
10600b57cec5SDimitry Andric // widened, for example <3 x i8> -> <4 x i8>.
10610b57cec5SDimitry Andric if (!VT.isPow2VectorType()) {
1062e8d8bef9SDimitry Andric NumElts = NumElts.coefficientNextPowerOf2();
10630b57cec5SDimitry Andric EVT NVT = EVT::getVectorVT(Context, EltVT, NumElts);
10640b57cec5SDimitry Andric return LegalizeKind(TypeWidenVector, NVT);
10650b57cec5SDimitry Andric }
10660b57cec5SDimitry Andric
10670b57cec5SDimitry Andric // Examine the element type.
10680b57cec5SDimitry Andric LegalizeKind LK = getTypeConversion(Context, EltVT);
10690b57cec5SDimitry Andric
10700b57cec5SDimitry Andric // If type is to be expanded, split the vector.
10710b57cec5SDimitry Andric // <4 x i140> -> <2 x i140>
1072fe6060f1SDimitry Andric if (LK.first == TypeExpandInteger) {
1073fe6060f1SDimitry Andric if (VT.getVectorElementCount().isScalable())
1074fe6060f1SDimitry Andric return LegalizeKind(TypeScalarizeScalableVector, EltVT);
10750b57cec5SDimitry Andric return LegalizeKind(TypeSplitVector,
1076e8d8bef9SDimitry Andric VT.getHalfNumVectorElementsVT(Context));
1077fe6060f1SDimitry Andric }
10780b57cec5SDimitry Andric
10790b57cec5SDimitry Andric // Promote the integer element types until a legal vector type is found
10800b57cec5SDimitry Andric // or until the element integer type is too big. If a legal type was not
10810b57cec5SDimitry Andric // found, fallback to the usual mechanism of widening/splitting the
10820b57cec5SDimitry Andric // vector.
10830b57cec5SDimitry Andric EVT OldEltVT = EltVT;
10840b57cec5SDimitry Andric while (true) {
10850b57cec5SDimitry Andric // Increase the bitwidth of the element to the next pow-of-two
10860b57cec5SDimitry Andric // (which is greater than 8 bits).
10870b57cec5SDimitry Andric EltVT = EVT::getIntegerVT(Context, 1 + EltVT.getSizeInBits())
10880b57cec5SDimitry Andric .getRoundIntegerType(Context);
10890b57cec5SDimitry Andric
10900b57cec5SDimitry Andric // Stop trying when getting a non-simple element type.
10910b57cec5SDimitry Andric // Note that vector elements may be greater than legal vector element
10920b57cec5SDimitry Andric // types. Example: X86 XMM registers hold 64bit element on 32bit
10930b57cec5SDimitry Andric // systems.
10940b57cec5SDimitry Andric if (!EltVT.isSimple())
10950b57cec5SDimitry Andric break;
10960b57cec5SDimitry Andric
10970b57cec5SDimitry Andric // Build a new vector type and check if it is legal.
10980b57cec5SDimitry Andric MVT NVT = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
10990b57cec5SDimitry Andric // Found a legal promoted vector type.
11000b57cec5SDimitry Andric if (NVT != MVT() && ValueTypeActions.getTypeAction(NVT) == TypeLegal)
11010b57cec5SDimitry Andric return LegalizeKind(TypePromoteInteger,
11020b57cec5SDimitry Andric EVT::getVectorVT(Context, EltVT, NumElts));
11030b57cec5SDimitry Andric }
11040b57cec5SDimitry Andric
11050b57cec5SDimitry Andric // Reset the type to the unexpanded type if we did not find a legal vector
11060b57cec5SDimitry Andric // type with a promoted vector element type.
11070b57cec5SDimitry Andric EltVT = OldEltVT;
11080b57cec5SDimitry Andric }
11090b57cec5SDimitry Andric
11100b57cec5SDimitry Andric // Try to widen the vector until a legal type is found.
11110b57cec5SDimitry Andric // If there is no wider legal type, split the vector.
11120b57cec5SDimitry Andric while (true) {
11130b57cec5SDimitry Andric // Round up to the next power of 2.
1114e8d8bef9SDimitry Andric NumElts = NumElts.coefficientNextPowerOf2();
11150b57cec5SDimitry Andric
11160b57cec5SDimitry Andric // If there is no simple vector type with this many elements then there
11170b57cec5SDimitry Andric // cannot be a larger legal vector type. Note that this assumes that
11180b57cec5SDimitry Andric // there are no skipped intermediate vector types in the simple types.
11190b57cec5SDimitry Andric if (!EltVT.isSimple())
11200b57cec5SDimitry Andric break;
11210b57cec5SDimitry Andric MVT LargerVector = MVT::getVectorVT(EltVT.getSimpleVT(), NumElts);
11220b57cec5SDimitry Andric if (LargerVector == MVT())
11230b57cec5SDimitry Andric break;
11240b57cec5SDimitry Andric
11250b57cec5SDimitry Andric // If this type is legal then widen the vector.
11260b57cec5SDimitry Andric if (ValueTypeActions.getTypeAction(LargerVector) == TypeLegal)
11270b57cec5SDimitry Andric return LegalizeKind(TypeWidenVector, LargerVector);
11280b57cec5SDimitry Andric }
11290b57cec5SDimitry Andric
11300b57cec5SDimitry Andric // Widen odd vectors to next power of two.
11310b57cec5SDimitry Andric if (!VT.isPow2VectorType()) {
11320b57cec5SDimitry Andric EVT NVT = VT.getPow2VectorType(Context);
11330b57cec5SDimitry Andric return LegalizeKind(TypeWidenVector, NVT);
11340b57cec5SDimitry Andric }
11350b57cec5SDimitry Andric
1136fe6060f1SDimitry Andric if (VT.getVectorElementCount() == ElementCount::getScalable(1))
1137fe6060f1SDimitry Andric return LegalizeKind(TypeScalarizeScalableVector, EltVT);
1138fe6060f1SDimitry Andric
11390b57cec5SDimitry Andric // Vectors with illegal element types are expanded.
1140e8d8bef9SDimitry Andric EVT NVT = EVT::getVectorVT(Context, EltVT,
1141e8d8bef9SDimitry Andric VT.getVectorElementCount().divideCoefficientBy(2));
11420b57cec5SDimitry Andric return LegalizeKind(TypeSplitVector, NVT);
11430b57cec5SDimitry Andric }
11440b57cec5SDimitry Andric
getVectorTypeBreakdownMVT(MVT VT,MVT & IntermediateVT,unsigned & NumIntermediates,MVT & RegisterVT,TargetLoweringBase * TLI)11450b57cec5SDimitry Andric static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
11460b57cec5SDimitry Andric unsigned &NumIntermediates,
11470b57cec5SDimitry Andric MVT &RegisterVT,
11480b57cec5SDimitry Andric TargetLoweringBase *TLI) {
11490b57cec5SDimitry Andric // Figure out the right, legal destination reg to copy into.
11505ffd83dbSDimitry Andric ElementCount EC = VT.getVectorElementCount();
11510b57cec5SDimitry Andric MVT EltTy = VT.getVectorElementType();
11520b57cec5SDimitry Andric
11530b57cec5SDimitry Andric unsigned NumVectorRegs = 1;
11540b57cec5SDimitry Andric
11555ffd83dbSDimitry Andric // Scalable vectors cannot be scalarized, so splitting or widening is
11565ffd83dbSDimitry Andric // required.
1157e8d8bef9SDimitry Andric if (VT.isScalableVector() && !isPowerOf2_32(EC.getKnownMinValue()))
11585ffd83dbSDimitry Andric llvm_unreachable(
11595ffd83dbSDimitry Andric "Splitting or widening of non-power-of-2 MVTs is not implemented.");
11605ffd83dbSDimitry Andric
11615ffd83dbSDimitry Andric // FIXME: We don't support non-power-of-2-sized vectors for now.
11625ffd83dbSDimitry Andric // Ideally we could break down into LHS/RHS like LegalizeDAG does.
1163e8d8bef9SDimitry Andric if (!isPowerOf2_32(EC.getKnownMinValue())) {
11645ffd83dbSDimitry Andric // Split EC to unit size (scalable property is preserved).
1165e8d8bef9SDimitry Andric NumVectorRegs = EC.getKnownMinValue();
1166e8d8bef9SDimitry Andric EC = ElementCount::getFixed(1);
11670b57cec5SDimitry Andric }
11680b57cec5SDimitry Andric
11695ffd83dbSDimitry Andric // Divide the input until we get to a supported size. This will
11705ffd83dbSDimitry Andric // always end up with an EC that represent a scalar or a scalable
11715ffd83dbSDimitry Andric // scalar.
1172e8d8bef9SDimitry Andric while (EC.getKnownMinValue() > 1 &&
1173e8d8bef9SDimitry Andric !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) {
1174e8d8bef9SDimitry Andric EC = EC.divideCoefficientBy(2);
11750b57cec5SDimitry Andric NumVectorRegs <<= 1;
11760b57cec5SDimitry Andric }
11770b57cec5SDimitry Andric
11780b57cec5SDimitry Andric NumIntermediates = NumVectorRegs;
11790b57cec5SDimitry Andric
11805ffd83dbSDimitry Andric MVT NewVT = MVT::getVectorVT(EltTy, EC);
11810b57cec5SDimitry Andric if (!TLI->isTypeLegal(NewVT))
11820b57cec5SDimitry Andric NewVT = EltTy;
11830b57cec5SDimitry Andric IntermediateVT = NewVT;
11840b57cec5SDimitry Andric
1185e8d8bef9SDimitry Andric unsigned LaneSizeInBits = NewVT.getScalarSizeInBits();
11860b57cec5SDimitry Andric
11870b57cec5SDimitry Andric // Convert sizes such as i33 to i64.
118806c3fb27SDimitry Andric LaneSizeInBits = llvm::bit_ceil(LaneSizeInBits);
11890b57cec5SDimitry Andric
11900b57cec5SDimitry Andric MVT DestVT = TLI->getRegisterType(NewVT);
11910b57cec5SDimitry Andric RegisterVT = DestVT;
11920b57cec5SDimitry Andric if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
1193e8d8bef9SDimitry Andric return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits());
11940b57cec5SDimitry Andric
11950b57cec5SDimitry Andric // Otherwise, promotion or legal types use the same number of registers as
11960b57cec5SDimitry Andric // the vector decimated to the appropriate level.
11970b57cec5SDimitry Andric return NumVectorRegs;
11980b57cec5SDimitry Andric }
11990b57cec5SDimitry Andric
12000b57cec5SDimitry Andric /// isLegalRC - Return true if the value types that can be represented by the
12010b57cec5SDimitry Andric /// specified register class are all legal.
isLegalRC(const TargetRegisterInfo & TRI,const TargetRegisterClass & RC) const12020b57cec5SDimitry Andric bool TargetLoweringBase::isLegalRC(const TargetRegisterInfo &TRI,
12030b57cec5SDimitry Andric const TargetRegisterClass &RC) const {
1204fcaf7f86SDimitry Andric for (const auto *I = TRI.legalclasstypes_begin(RC); *I != MVT::Other; ++I)
12050b57cec5SDimitry Andric if (isTypeLegal(*I))
12060b57cec5SDimitry Andric return true;
12070b57cec5SDimitry Andric return false;
12080b57cec5SDimitry Andric }
12090b57cec5SDimitry Andric
12100b57cec5SDimitry Andric /// Replace/modify any TargetFrameIndex operands with a targte-dependent
12110b57cec5SDimitry Andric /// sequence of memory operands that is recognized by PrologEpilogInserter.
12120b57cec5SDimitry Andric MachineBasicBlock *
emitPatchPoint(MachineInstr & InitialMI,MachineBasicBlock * MBB) const12130b57cec5SDimitry Andric TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI,
12140b57cec5SDimitry Andric MachineBasicBlock *MBB) const {
12150b57cec5SDimitry Andric MachineInstr *MI = &InitialMI;
12160b57cec5SDimitry Andric MachineFunction &MF = *MI->getMF();
12170b57cec5SDimitry Andric MachineFrameInfo &MFI = MF.getFrameInfo();
12180b57cec5SDimitry Andric
12190b57cec5SDimitry Andric // We're handling multiple types of operands here:
12200b57cec5SDimitry Andric // PATCHPOINT MetaArgs - live-in, read only, direct
12210b57cec5SDimitry Andric // STATEPOINT Deopt Spill - live-through, read only, indirect
12220b57cec5SDimitry Andric // STATEPOINT Deopt Alloca - live-through, read only, direct
12230b57cec5SDimitry Andric // (We're currently conservative and mark the deopt slots read/write in
12240b57cec5SDimitry Andric // practice.)
12250b57cec5SDimitry Andric // STATEPOINT GC Spill - live-through, read/write, indirect
12260b57cec5SDimitry Andric // STATEPOINT GC Alloca - live-through, read/write, direct
12270b57cec5SDimitry Andric // The live-in vs live-through is handled already (the live through ones are
12280b57cec5SDimitry Andric // all stack slots), but we need to handle the different type of stackmap
12290b57cec5SDimitry Andric // operands and memory effects here.
12300b57cec5SDimitry Andric
12310eae32dcSDimitry Andric if (llvm::none_of(MI->operands(),
12325ffd83dbSDimitry Andric [](MachineOperand &Operand) { return Operand.isFI(); }))
12335ffd83dbSDimitry Andric return MBB;
12345ffd83dbSDimitry Andric
12355ffd83dbSDimitry Andric MachineInstrBuilder MIB = BuildMI(MF, MI->getDebugLoc(), MI->getDesc());
12365ffd83dbSDimitry Andric
12375ffd83dbSDimitry Andric // Inherit previous memory operands.
12385ffd83dbSDimitry Andric MIB.cloneMemRefs(*MI);
12395ffd83dbSDimitry Andric
1240e8d8bef9SDimitry Andric for (unsigned i = 0; i < MI->getNumOperands(); ++i) {
1241e8d8bef9SDimitry Andric MachineOperand &MO = MI->getOperand(i);
12425ffd83dbSDimitry Andric if (!MO.isFI()) {
1243e8d8bef9SDimitry Andric // Index of Def operand this Use it tied to.
1244e8d8bef9SDimitry Andric // Since Defs are coming before Uses, if Use is tied, then
1245e8d8bef9SDimitry Andric // index of Def must be smaller that index of that Use.
1246e8d8bef9SDimitry Andric // Also, Defs preserve their position in new MI.
1247e8d8bef9SDimitry Andric unsigned TiedTo = i;
1248e8d8bef9SDimitry Andric if (MO.isReg() && MO.isTied())
1249e8d8bef9SDimitry Andric TiedTo = MI->findTiedOperandIdx(i);
12505ffd83dbSDimitry Andric MIB.add(MO);
1251e8d8bef9SDimitry Andric if (TiedTo < i)
1252e8d8bef9SDimitry Andric MIB->tieOperands(TiedTo, MIB->getNumOperands() - 1);
12530b57cec5SDimitry Andric continue;
12545ffd83dbSDimitry Andric }
12550b57cec5SDimitry Andric
12560b57cec5SDimitry Andric // foldMemoryOperand builds a new MI after replacing a single FI operand
12570b57cec5SDimitry Andric // with the canonical set of five x86 addressing-mode operands.
12580b57cec5SDimitry Andric int FI = MO.getIndex();
12590b57cec5SDimitry Andric
12600b57cec5SDimitry Andric // Add frame index operands recognized by stackmaps.cpp
12610b57cec5SDimitry Andric if (MFI.isStatepointSpillSlotObjectIndex(FI)) {
12620b57cec5SDimitry Andric // indirect-mem-ref tag, size, #FI, offset.
12630b57cec5SDimitry Andric // Used for spills inserted by StatepointLowering. This codepath is not
12640b57cec5SDimitry Andric // used for patchpoints/stackmaps at all, for these spilling is done via
12650b57cec5SDimitry Andric // foldMemoryOperand callback only.
12660b57cec5SDimitry Andric assert(MI->getOpcode() == TargetOpcode::STATEPOINT && "sanity");
12670b57cec5SDimitry Andric MIB.addImm(StackMaps::IndirectMemRefOp);
12680b57cec5SDimitry Andric MIB.addImm(MFI.getObjectSize(FI));
12695ffd83dbSDimitry Andric MIB.add(MO);
12700b57cec5SDimitry Andric MIB.addImm(0);
12710b57cec5SDimitry Andric } else {
12720b57cec5SDimitry Andric // direct-mem-ref tag, #FI, offset.
12730b57cec5SDimitry Andric // Used by patchpoint, and direct alloca arguments to statepoints
12740b57cec5SDimitry Andric MIB.addImm(StackMaps::DirectMemRefOp);
12755ffd83dbSDimitry Andric MIB.add(MO);
12760b57cec5SDimitry Andric MIB.addImm(0);
12770b57cec5SDimitry Andric }
12780b57cec5SDimitry Andric
12790b57cec5SDimitry Andric assert(MIB->mayLoad() && "Folded a stackmap use to a non-load!");
12800b57cec5SDimitry Andric
12810b57cec5SDimitry Andric // Add a new memory operand for this FI.
12820b57cec5SDimitry Andric assert(MFI.getObjectOffset(FI) != -1);
12830b57cec5SDimitry Andric
12840b57cec5SDimitry Andric // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and
12850b57cec5SDimitry Andric // PATCHPOINT should be updated to do the same. (TODO)
12860b57cec5SDimitry Andric if (MI->getOpcode() != TargetOpcode::STATEPOINT) {
12870b57cec5SDimitry Andric auto Flags = MachineMemOperand::MOLoad;
12880b57cec5SDimitry Andric MachineMemOperand *MMO = MF.getMachineMemOperand(
12890b57cec5SDimitry Andric MachinePointerInfo::getFixedStack(MF, FI), Flags,
12905ffd83dbSDimitry Andric MF.getDataLayout().getPointerSize(), MFI.getObjectAlign(FI));
12910b57cec5SDimitry Andric MIB->addMemOperand(MF, MMO);
12920b57cec5SDimitry Andric }
12930b57cec5SDimitry Andric }
12945ffd83dbSDimitry Andric MBB->insert(MachineBasicBlock::iterator(MI), MIB);
12955ffd83dbSDimitry Andric MI->eraseFromParent();
12960b57cec5SDimitry Andric return MBB;
12970b57cec5SDimitry Andric }
12980b57cec5SDimitry Andric
12990b57cec5SDimitry Andric /// findRepresentativeClass - Return the largest legal super-reg register class
13000b57cec5SDimitry Andric /// of the register class for the specified type and its associated "cost".
13010b57cec5SDimitry Andric // This function is in TargetLowering because it uses RegClassForVT which would
13020b57cec5SDimitry Andric // need to be moved to TargetRegisterInfo and would necessitate moving
13030b57cec5SDimitry Andric // isTypeLegal over as well - a massive change that would just require
13040b57cec5SDimitry Andric // TargetLowering having a TargetRegisterInfo class member that it would use.
13050b57cec5SDimitry Andric std::pair<const TargetRegisterClass *, uint8_t>
findRepresentativeClass(const TargetRegisterInfo * TRI,MVT VT) const13060b57cec5SDimitry Andric TargetLoweringBase::findRepresentativeClass(const TargetRegisterInfo *TRI,
13070b57cec5SDimitry Andric MVT VT) const {
13080b57cec5SDimitry Andric const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
13090b57cec5SDimitry Andric if (!RC)
13100b57cec5SDimitry Andric return std::make_pair(RC, 0);
13110b57cec5SDimitry Andric
13120b57cec5SDimitry Andric // Compute the set of all super-register classes.
13130b57cec5SDimitry Andric BitVector SuperRegRC(TRI->getNumRegClasses());
13140b57cec5SDimitry Andric for (SuperRegClassIterator RCI(RC, TRI); RCI.isValid(); ++RCI)
13150b57cec5SDimitry Andric SuperRegRC.setBitsInMask(RCI.getMask());
13160b57cec5SDimitry Andric
13170b57cec5SDimitry Andric // Find the first legal register class with the largest spill size.
13180b57cec5SDimitry Andric const TargetRegisterClass *BestRC = RC;
13190b57cec5SDimitry Andric for (unsigned i : SuperRegRC.set_bits()) {
13200b57cec5SDimitry Andric const TargetRegisterClass *SuperRC = TRI->getRegClass(i);
13210b57cec5SDimitry Andric // We want the largest possible spill size.
13220b57cec5SDimitry Andric if (TRI->getSpillSize(*SuperRC) <= TRI->getSpillSize(*BestRC))
13230b57cec5SDimitry Andric continue;
13240b57cec5SDimitry Andric if (!isLegalRC(*TRI, *SuperRC))
13250b57cec5SDimitry Andric continue;
13260b57cec5SDimitry Andric BestRC = SuperRC;
13270b57cec5SDimitry Andric }
13280b57cec5SDimitry Andric return std::make_pair(BestRC, 1);
13290b57cec5SDimitry Andric }
13300b57cec5SDimitry Andric
13310b57cec5SDimitry Andric /// computeRegisterProperties - Once all of the register classes are added,
13320b57cec5SDimitry Andric /// this allows us to compute derived properties we expose.
computeRegisterProperties(const TargetRegisterInfo * TRI)13330b57cec5SDimitry Andric void TargetLoweringBase::computeRegisterProperties(
13340b57cec5SDimitry Andric const TargetRegisterInfo *TRI) {
1335fe6060f1SDimitry Andric static_assert(MVT::VALUETYPE_SIZE <= MVT::MAX_ALLOWED_VALUETYPE,
13360b57cec5SDimitry Andric "Too many value types for ValueTypeActions to hold!");
13370b57cec5SDimitry Andric
13380b57cec5SDimitry Andric // Everything defaults to needing one register.
1339fe6060f1SDimitry Andric for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
13400b57cec5SDimitry Andric NumRegistersForVT[i] = 1;
13410b57cec5SDimitry Andric RegisterTypeForVT[i] = TransformToType[i] = (MVT::SimpleValueType)i;
13420b57cec5SDimitry Andric }
13430b57cec5SDimitry Andric // ...except isVoid, which doesn't need any registers.
13440b57cec5SDimitry Andric NumRegistersForVT[MVT::isVoid] = 0;
13450b57cec5SDimitry Andric
13460b57cec5SDimitry Andric // Find the largest integer register class.
13470b57cec5SDimitry Andric unsigned LargestIntReg = MVT::LAST_INTEGER_VALUETYPE;
13480b57cec5SDimitry Andric for (; RegClassForVT[LargestIntReg] == nullptr; --LargestIntReg)
13490b57cec5SDimitry Andric assert(LargestIntReg != MVT::i1 && "No integer registers defined!");
13500b57cec5SDimitry Andric
13510b57cec5SDimitry Andric // Every integer value type larger than this largest register takes twice as
13520b57cec5SDimitry Andric // many registers to represent as the previous ValueType.
13530b57cec5SDimitry Andric for (unsigned ExpandedReg = LargestIntReg + 1;
13540b57cec5SDimitry Andric ExpandedReg <= MVT::LAST_INTEGER_VALUETYPE; ++ExpandedReg) {
13550b57cec5SDimitry Andric NumRegistersForVT[ExpandedReg] = 2*NumRegistersForVT[ExpandedReg-1];
13560b57cec5SDimitry Andric RegisterTypeForVT[ExpandedReg] = (MVT::SimpleValueType)LargestIntReg;
13570b57cec5SDimitry Andric TransformToType[ExpandedReg] = (MVT::SimpleValueType)(ExpandedReg - 1);
13580b57cec5SDimitry Andric ValueTypeActions.setTypeAction((MVT::SimpleValueType)ExpandedReg,
13590b57cec5SDimitry Andric TypeExpandInteger);
13600b57cec5SDimitry Andric }
13610b57cec5SDimitry Andric
13620b57cec5SDimitry Andric // Inspect all of the ValueType's smaller than the largest integer
13630b57cec5SDimitry Andric // register to see which ones need promotion.
13640b57cec5SDimitry Andric unsigned LegalIntReg = LargestIntReg;
13650b57cec5SDimitry Andric for (unsigned IntReg = LargestIntReg - 1;
13660b57cec5SDimitry Andric IntReg >= (unsigned)MVT::i1; --IntReg) {
13670b57cec5SDimitry Andric MVT IVT = (MVT::SimpleValueType)IntReg;
13680b57cec5SDimitry Andric if (isTypeLegal(IVT)) {
13690b57cec5SDimitry Andric LegalIntReg = IntReg;
13700b57cec5SDimitry Andric } else {
13710b57cec5SDimitry Andric RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
13720b57cec5SDimitry Andric (MVT::SimpleValueType)LegalIntReg;
13730b57cec5SDimitry Andric ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
13740b57cec5SDimitry Andric }
13750b57cec5SDimitry Andric }
13760b57cec5SDimitry Andric
13770b57cec5SDimitry Andric // ppcf128 type is really two f64's.
13780b57cec5SDimitry Andric if (!isTypeLegal(MVT::ppcf128)) {
13790b57cec5SDimitry Andric if (isTypeLegal(MVT::f64)) {
13800b57cec5SDimitry Andric NumRegistersForVT[MVT::ppcf128] = 2*NumRegistersForVT[MVT::f64];
13810b57cec5SDimitry Andric RegisterTypeForVT[MVT::ppcf128] = MVT::f64;
13820b57cec5SDimitry Andric TransformToType[MVT::ppcf128] = MVT::f64;
13830b57cec5SDimitry Andric ValueTypeActions.setTypeAction(MVT::ppcf128, TypeExpandFloat);
13840b57cec5SDimitry Andric } else {
13850b57cec5SDimitry Andric NumRegistersForVT[MVT::ppcf128] = NumRegistersForVT[MVT::i128];
13860b57cec5SDimitry Andric RegisterTypeForVT[MVT::ppcf128] = RegisterTypeForVT[MVT::i128];
13870b57cec5SDimitry Andric TransformToType[MVT::ppcf128] = MVT::i128;
13880b57cec5SDimitry Andric ValueTypeActions.setTypeAction(MVT::ppcf128, TypeSoftenFloat);
13890b57cec5SDimitry Andric }
13900b57cec5SDimitry Andric }
13910b57cec5SDimitry Andric
13920b57cec5SDimitry Andric // Decide how to handle f128. If the target does not have native f128 support,
13930b57cec5SDimitry Andric // expand it to i128 and we will be generating soft float library calls.
13940b57cec5SDimitry Andric if (!isTypeLegal(MVT::f128)) {
13950b57cec5SDimitry Andric NumRegistersForVT[MVT::f128] = NumRegistersForVT[MVT::i128];
13960b57cec5SDimitry Andric RegisterTypeForVT[MVT::f128] = RegisterTypeForVT[MVT::i128];
13970b57cec5SDimitry Andric TransformToType[MVT::f128] = MVT::i128;
13980b57cec5SDimitry Andric ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
13990b57cec5SDimitry Andric }
14000b57cec5SDimitry Andric
1401bdd1243dSDimitry Andric // Decide how to handle f80. If the target does not have native f80 support,
1402bdd1243dSDimitry Andric // expand it to i96 and we will be generating soft float library calls.
1403bdd1243dSDimitry Andric if (!isTypeLegal(MVT::f80)) {
1404bdd1243dSDimitry Andric NumRegistersForVT[MVT::f80] = 3*NumRegistersForVT[MVT::i32];
1405bdd1243dSDimitry Andric RegisterTypeForVT[MVT::f80] = RegisterTypeForVT[MVT::i32];
1406bdd1243dSDimitry Andric TransformToType[MVT::f80] = MVT::i32;
1407bdd1243dSDimitry Andric ValueTypeActions.setTypeAction(MVT::f80, TypeSoftenFloat);
1408bdd1243dSDimitry Andric }
1409bdd1243dSDimitry Andric
14100b57cec5SDimitry Andric // Decide how to handle f64. If the target does not have native f64 support,
14110b57cec5SDimitry Andric // expand it to i64 and we will be generating soft float library calls.
14120b57cec5SDimitry Andric if (!isTypeLegal(MVT::f64)) {
14130b57cec5SDimitry Andric NumRegistersForVT[MVT::f64] = NumRegistersForVT[MVT::i64];
14140b57cec5SDimitry Andric RegisterTypeForVT[MVT::f64] = RegisterTypeForVT[MVT::i64];
14150b57cec5SDimitry Andric TransformToType[MVT::f64] = MVT::i64;
14160b57cec5SDimitry Andric ValueTypeActions.setTypeAction(MVT::f64, TypeSoftenFloat);
14170b57cec5SDimitry Andric }
14180b57cec5SDimitry Andric
14190b57cec5SDimitry Andric // Decide how to handle f32. If the target does not have native f32 support,
14200b57cec5SDimitry Andric // expand it to i32 and we will be generating soft float library calls.
14210b57cec5SDimitry Andric if (!isTypeLegal(MVT::f32)) {
14220b57cec5SDimitry Andric NumRegistersForVT[MVT::f32] = NumRegistersForVT[MVT::i32];
14230b57cec5SDimitry Andric RegisterTypeForVT[MVT::f32] = RegisterTypeForVT[MVT::i32];
14240b57cec5SDimitry Andric TransformToType[MVT::f32] = MVT::i32;
14250b57cec5SDimitry Andric ValueTypeActions.setTypeAction(MVT::f32, TypeSoftenFloat);
14260b57cec5SDimitry Andric }
14270b57cec5SDimitry Andric
14280b57cec5SDimitry Andric // Decide how to handle f16. If the target does not have native f16 support,
14290b57cec5SDimitry Andric // promote it to f32, because there are no f16 library calls (except for
14300b57cec5SDimitry Andric // conversions).
14310b57cec5SDimitry Andric if (!isTypeLegal(MVT::f16)) {
14325ffd83dbSDimitry Andric // Allow targets to control how we legalize half.
14335ffd83dbSDimitry Andric if (softPromoteHalfType()) {
14345ffd83dbSDimitry Andric NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::i16];
14355ffd83dbSDimitry Andric RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::i16];
14365ffd83dbSDimitry Andric TransformToType[MVT::f16] = MVT::f32;
14375ffd83dbSDimitry Andric ValueTypeActions.setTypeAction(MVT::f16, TypeSoftPromoteHalf);
14385ffd83dbSDimitry Andric } else {
14390b57cec5SDimitry Andric NumRegistersForVT[MVT::f16] = NumRegistersForVT[MVT::f32];
14400b57cec5SDimitry Andric RegisterTypeForVT[MVT::f16] = RegisterTypeForVT[MVT::f32];
14410b57cec5SDimitry Andric TransformToType[MVT::f16] = MVT::f32;
14420b57cec5SDimitry Andric ValueTypeActions.setTypeAction(MVT::f16, TypePromoteFloat);
14430b57cec5SDimitry Andric }
14445ffd83dbSDimitry Andric }
14450b57cec5SDimitry Andric
144681ad6265SDimitry Andric // Decide how to handle bf16. If the target does not have native bf16 support,
144781ad6265SDimitry Andric // promote it to f32, because there are no bf16 library calls (except for
144881ad6265SDimitry Andric // converting from f32 to bf16).
144981ad6265SDimitry Andric if (!isTypeLegal(MVT::bf16)) {
145081ad6265SDimitry Andric NumRegistersForVT[MVT::bf16] = NumRegistersForVT[MVT::f32];
145181ad6265SDimitry Andric RegisterTypeForVT[MVT::bf16] = RegisterTypeForVT[MVT::f32];
145281ad6265SDimitry Andric TransformToType[MVT::bf16] = MVT::f32;
1453bdd1243dSDimitry Andric ValueTypeActions.setTypeAction(MVT::bf16, TypeSoftPromoteHalf);
145481ad6265SDimitry Andric }
145581ad6265SDimitry Andric
14560b57cec5SDimitry Andric // Loop over all of the vector value types to see which need transformations.
14570b57cec5SDimitry Andric for (unsigned i = MVT::FIRST_VECTOR_VALUETYPE;
14580b57cec5SDimitry Andric i <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++i) {
14590b57cec5SDimitry Andric MVT VT = (MVT::SimpleValueType) i;
14600b57cec5SDimitry Andric if (isTypeLegal(VT))
14610b57cec5SDimitry Andric continue;
14620b57cec5SDimitry Andric
14630b57cec5SDimitry Andric MVT EltVT = VT.getVectorElementType();
14645ffd83dbSDimitry Andric ElementCount EC = VT.getVectorElementCount();
14650b57cec5SDimitry Andric bool IsLegalWiderType = false;
14668bcb0991SDimitry Andric bool IsScalable = VT.isScalableVector();
14670b57cec5SDimitry Andric LegalizeTypeAction PreferredAction = getPreferredVectorAction(VT);
14680b57cec5SDimitry Andric switch (PreferredAction) {
14698bcb0991SDimitry Andric case TypePromoteInteger: {
14708bcb0991SDimitry Andric MVT::SimpleValueType EndVT = IsScalable ?
14718bcb0991SDimitry Andric MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE :
14728bcb0991SDimitry Andric MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE;
14730b57cec5SDimitry Andric // Try to promote the elements of integer vectors. If no legal
14740b57cec5SDimitry Andric // promotion was found, fall through to the widen-vector method.
14758bcb0991SDimitry Andric for (unsigned nVT = i + 1;
14768bcb0991SDimitry Andric (MVT::SimpleValueType)nVT <= EndVT; ++nVT) {
14770b57cec5SDimitry Andric MVT SVT = (MVT::SimpleValueType) nVT;
14780b57cec5SDimitry Andric // Promote vectors of integers to vectors with the same number
14790b57cec5SDimitry Andric // of elements, with a wider element type.
1480e8d8bef9SDimitry Andric if (SVT.getScalarSizeInBits() > EltVT.getFixedSizeInBits() &&
14815ffd83dbSDimitry Andric SVT.getVectorElementCount() == EC && isTypeLegal(SVT)) {
14820b57cec5SDimitry Andric TransformToType[i] = SVT;
14830b57cec5SDimitry Andric RegisterTypeForVT[i] = SVT;
14840b57cec5SDimitry Andric NumRegistersForVT[i] = 1;
14850b57cec5SDimitry Andric ValueTypeActions.setTypeAction(VT, TypePromoteInteger);
14860b57cec5SDimitry Andric IsLegalWiderType = true;
14870b57cec5SDimitry Andric break;
14880b57cec5SDimitry Andric }
14890b57cec5SDimitry Andric }
14900b57cec5SDimitry Andric if (IsLegalWiderType)
14910b57cec5SDimitry Andric break;
1492bdd1243dSDimitry Andric [[fallthrough]];
14938bcb0991SDimitry Andric }
14940b57cec5SDimitry Andric
14950b57cec5SDimitry Andric case TypeWidenVector:
1496e8d8bef9SDimitry Andric if (isPowerOf2_32(EC.getKnownMinValue())) {
14970b57cec5SDimitry Andric // Try to widen the vector.
14980b57cec5SDimitry Andric for (unsigned nVT = i + 1; nVT <= MVT::LAST_VECTOR_VALUETYPE; ++nVT) {
14990b57cec5SDimitry Andric MVT SVT = (MVT::SimpleValueType) nVT;
15005ffd83dbSDimitry Andric if (SVT.getVectorElementType() == EltVT &&
15015ffd83dbSDimitry Andric SVT.isScalableVector() == IsScalable &&
1502e8d8bef9SDimitry Andric SVT.getVectorElementCount().getKnownMinValue() >
1503e8d8bef9SDimitry Andric EC.getKnownMinValue() &&
1504e8d8bef9SDimitry Andric isTypeLegal(SVT)) {
15050b57cec5SDimitry Andric TransformToType[i] = SVT;
15060b57cec5SDimitry Andric RegisterTypeForVT[i] = SVT;
15070b57cec5SDimitry Andric NumRegistersForVT[i] = 1;
15080b57cec5SDimitry Andric ValueTypeActions.setTypeAction(VT, TypeWidenVector);
15090b57cec5SDimitry Andric IsLegalWiderType = true;
15100b57cec5SDimitry Andric break;
15110b57cec5SDimitry Andric }
15120b57cec5SDimitry Andric }
15130b57cec5SDimitry Andric if (IsLegalWiderType)
15140b57cec5SDimitry Andric break;
15158bcb0991SDimitry Andric } else {
15168bcb0991SDimitry Andric // Only widen to the next power of 2 to keep consistency with EVT.
15178bcb0991SDimitry Andric MVT NVT = VT.getPow2VectorType();
15188bcb0991SDimitry Andric if (isTypeLegal(NVT)) {
15198bcb0991SDimitry Andric TransformToType[i] = NVT;
15208bcb0991SDimitry Andric ValueTypeActions.setTypeAction(VT, TypeWidenVector);
15218bcb0991SDimitry Andric RegisterTypeForVT[i] = NVT;
15228bcb0991SDimitry Andric NumRegistersForVT[i] = 1;
15238bcb0991SDimitry Andric break;
15248bcb0991SDimitry Andric }
15258bcb0991SDimitry Andric }
1526bdd1243dSDimitry Andric [[fallthrough]];
15270b57cec5SDimitry Andric
15280b57cec5SDimitry Andric case TypeSplitVector:
15290b57cec5SDimitry Andric case TypeScalarizeVector: {
15300b57cec5SDimitry Andric MVT IntermediateVT;
15310b57cec5SDimitry Andric MVT RegisterVT;
15320b57cec5SDimitry Andric unsigned NumIntermediates;
1533480093f4SDimitry Andric unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT,
15340b57cec5SDimitry Andric NumIntermediates, RegisterVT, this);
1535480093f4SDimitry Andric NumRegistersForVT[i] = NumRegisters;
1536480093f4SDimitry Andric assert(NumRegistersForVT[i] == NumRegisters &&
1537480093f4SDimitry Andric "NumRegistersForVT size cannot represent NumRegisters!");
15380b57cec5SDimitry Andric RegisterTypeForVT[i] = RegisterVT;
15390b57cec5SDimitry Andric
15400b57cec5SDimitry Andric MVT NVT = VT.getPow2VectorType();
15410b57cec5SDimitry Andric if (NVT == VT) {
15420b57cec5SDimitry Andric // Type is already a power of 2. The default action is to split.
15430b57cec5SDimitry Andric TransformToType[i] = MVT::Other;
15440b57cec5SDimitry Andric if (PreferredAction == TypeScalarizeVector)
15450b57cec5SDimitry Andric ValueTypeActions.setTypeAction(VT, TypeScalarizeVector);
15460b57cec5SDimitry Andric else if (PreferredAction == TypeSplitVector)
15470b57cec5SDimitry Andric ValueTypeActions.setTypeAction(VT, TypeSplitVector);
1548e8d8bef9SDimitry Andric else if (EC.getKnownMinValue() > 1)
15495ffd83dbSDimitry Andric ValueTypeActions.setTypeAction(VT, TypeSplitVector);
15500b57cec5SDimitry Andric else
1551e8d8bef9SDimitry Andric ValueTypeActions.setTypeAction(VT, EC.isScalable()
15525ffd83dbSDimitry Andric ? TypeScalarizeScalableVector
15535ffd83dbSDimitry Andric : TypeScalarizeVector);
15540b57cec5SDimitry Andric } else {
15550b57cec5SDimitry Andric TransformToType[i] = NVT;
15560b57cec5SDimitry Andric ValueTypeActions.setTypeAction(VT, TypeWidenVector);
15570b57cec5SDimitry Andric }
15580b57cec5SDimitry Andric break;
15590b57cec5SDimitry Andric }
15600b57cec5SDimitry Andric default:
15610b57cec5SDimitry Andric llvm_unreachable("Unknown vector legalization action!");
15620b57cec5SDimitry Andric }
15630b57cec5SDimitry Andric }
15640b57cec5SDimitry Andric
15650b57cec5SDimitry Andric // Determine the 'representative' register class for each value type.
15660b57cec5SDimitry Andric // An representative register class is the largest (meaning one which is
15670b57cec5SDimitry Andric // not a sub-register class / subreg register class) legal register class for
15680b57cec5SDimitry Andric // a group of value types. For example, on i386, i8, i16, and i32
15690b57cec5SDimitry Andric // representative would be GR32; while on x86_64 it's GR64.
1570fe6060f1SDimitry Andric for (unsigned i = 0; i != MVT::VALUETYPE_SIZE; ++i) {
15710b57cec5SDimitry Andric const TargetRegisterClass* RRC;
15720b57cec5SDimitry Andric uint8_t Cost;
15730b57cec5SDimitry Andric std::tie(RRC, Cost) = findRepresentativeClass(TRI, (MVT::SimpleValueType)i);
15740b57cec5SDimitry Andric RepRegClassForVT[i] = RRC;
15750b57cec5SDimitry Andric RepRegClassCostForVT[i] = Cost;
15760b57cec5SDimitry Andric }
15770b57cec5SDimitry Andric }
15780b57cec5SDimitry Andric
getSetCCResultType(const DataLayout & DL,LLVMContext &,EVT VT) const15790b57cec5SDimitry Andric EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,
15800b57cec5SDimitry Andric EVT VT) const {
15810b57cec5SDimitry Andric assert(!VT.isVector() && "No default SetCC type for vectors!");
15820b57cec5SDimitry Andric return getPointerTy(DL).SimpleTy;
15830b57cec5SDimitry Andric }
15840b57cec5SDimitry Andric
getCmpLibcallReturnType() const15850b57cec5SDimitry Andric MVT::SimpleValueType TargetLoweringBase::getCmpLibcallReturnType() const {
15860b57cec5SDimitry Andric return MVT::i32; // return the default value
15870b57cec5SDimitry Andric }
15880b57cec5SDimitry Andric
15890b57cec5SDimitry Andric /// getVectorTypeBreakdown - Vector types are broken down into some number of
15900b57cec5SDimitry Andric /// legal first class types. For example, MVT::v8f32 maps to 2 MVT::v4f32
15910b57cec5SDimitry Andric /// with Altivec or SSE1, or 8 promoted MVT::f64 values with the X86 FP stack.
15920b57cec5SDimitry Andric /// Similarly, MVT::v2i64 turns into 4 MVT::i32 values with both PPC and X86.
15930b57cec5SDimitry Andric ///
15940b57cec5SDimitry Andric /// This method returns the number of registers needed, and the VT for each
15950b57cec5SDimitry Andric /// register. It also returns the VT and quantity of the intermediate values
15960b57cec5SDimitry Andric /// before they are promoted/expanded.
getVectorTypeBreakdown(LLVMContext & Context,EVT VT,EVT & IntermediateVT,unsigned & NumIntermediates,MVT & RegisterVT) const1597fe6060f1SDimitry Andric unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
1598fe6060f1SDimitry Andric EVT VT, EVT &IntermediateVT,
15990b57cec5SDimitry Andric unsigned &NumIntermediates,
16000b57cec5SDimitry Andric MVT &RegisterVT) const {
16015ffd83dbSDimitry Andric ElementCount EltCnt = VT.getVectorElementCount();
16020b57cec5SDimitry Andric
16030b57cec5SDimitry Andric // If there is a wider vector type with the same element type as this one,
16040b57cec5SDimitry Andric // or a promoted vector type that has the same number of elements which
16050b57cec5SDimitry Andric // are wider, then we should convert to that legal vector type.
16060b57cec5SDimitry Andric // This handles things like <2 x float> -> <4 x float> and
16070b57cec5SDimitry Andric // <4 x i1> -> <4 x i32>.
16080b57cec5SDimitry Andric LegalizeTypeAction TA = getTypeAction(Context, VT);
1609fe6060f1SDimitry Andric if (!EltCnt.isScalar() &&
1610e8d8bef9SDimitry Andric (TA == TypeWidenVector || TA == TypePromoteInteger)) {
16110b57cec5SDimitry Andric EVT RegisterEVT = getTypeToTransformTo(Context, VT);
16120b57cec5SDimitry Andric if (isTypeLegal(RegisterEVT)) {
16130b57cec5SDimitry Andric IntermediateVT = RegisterEVT;
16140b57cec5SDimitry Andric RegisterVT = RegisterEVT.getSimpleVT();
16150b57cec5SDimitry Andric NumIntermediates = 1;
16160b57cec5SDimitry Andric return 1;
16170b57cec5SDimitry Andric }
16180b57cec5SDimitry Andric }
16190b57cec5SDimitry Andric
16200b57cec5SDimitry Andric // Figure out the right, legal destination reg to copy into.
16210b57cec5SDimitry Andric EVT EltTy = VT.getVectorElementType();
16220b57cec5SDimitry Andric
16230b57cec5SDimitry Andric unsigned NumVectorRegs = 1;
16240b57cec5SDimitry Andric
16255ffd83dbSDimitry Andric // Scalable vectors cannot be scalarized, so handle the legalisation of the
16265ffd83dbSDimitry Andric // types like done elsewhere in SelectionDAG.
1627349cc55cSDimitry Andric if (EltCnt.isScalable()) {
16285ffd83dbSDimitry Andric LegalizeKind LK;
16295ffd83dbSDimitry Andric EVT PartVT = VT;
16305ffd83dbSDimitry Andric do {
16315ffd83dbSDimitry Andric // Iterate until we've found a legal (part) type to hold VT.
16325ffd83dbSDimitry Andric LK = getTypeConversion(Context, PartVT);
16335ffd83dbSDimitry Andric PartVT = LK.second;
16345ffd83dbSDimitry Andric } while (LK.first != TypeLegal);
16355ffd83dbSDimitry Andric
1636349cc55cSDimitry Andric if (!PartVT.isVector()) {
1637349cc55cSDimitry Andric report_fatal_error(
1638349cc55cSDimitry Andric "Don't know how to legalize this scalable vector type");
1639349cc55cSDimitry Andric }
16405ffd83dbSDimitry Andric
1641349cc55cSDimitry Andric NumIntermediates =
1642349cc55cSDimitry Andric divideCeil(VT.getVectorElementCount().getKnownMinValue(),
1643349cc55cSDimitry Andric PartVT.getVectorElementCount().getKnownMinValue());
16445ffd83dbSDimitry Andric IntermediateVT = PartVT;
16455ffd83dbSDimitry Andric RegisterVT = getRegisterType(Context, IntermediateVT);
16465ffd83dbSDimitry Andric return NumIntermediates;
16475ffd83dbSDimitry Andric }
16485ffd83dbSDimitry Andric
16495ffd83dbSDimitry Andric // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally
16505ffd83dbSDimitry Andric // we could break down into LHS/RHS like LegalizeDAG does.
1651e8d8bef9SDimitry Andric if (!isPowerOf2_32(EltCnt.getKnownMinValue())) {
1652e8d8bef9SDimitry Andric NumVectorRegs = EltCnt.getKnownMinValue();
1653e8d8bef9SDimitry Andric EltCnt = ElementCount::getFixed(1);
16540b57cec5SDimitry Andric }
16550b57cec5SDimitry Andric
16560b57cec5SDimitry Andric // Divide the input until we get to a supported size. This will always
16570b57cec5SDimitry Andric // end with a scalar if the target doesn't support vectors.
1658e8d8bef9SDimitry Andric while (EltCnt.getKnownMinValue() > 1 &&
16595ffd83dbSDimitry Andric !isTypeLegal(EVT::getVectorVT(Context, EltTy, EltCnt))) {
1660e8d8bef9SDimitry Andric EltCnt = EltCnt.divideCoefficientBy(2);
16610b57cec5SDimitry Andric NumVectorRegs <<= 1;
16620b57cec5SDimitry Andric }
16630b57cec5SDimitry Andric
16640b57cec5SDimitry Andric NumIntermediates = NumVectorRegs;
16650b57cec5SDimitry Andric
16665ffd83dbSDimitry Andric EVT NewVT = EVT::getVectorVT(Context, EltTy, EltCnt);
16670b57cec5SDimitry Andric if (!isTypeLegal(NewVT))
16680b57cec5SDimitry Andric NewVT = EltTy;
16690b57cec5SDimitry Andric IntermediateVT = NewVT;
16700b57cec5SDimitry Andric
16710b57cec5SDimitry Andric MVT DestVT = getRegisterType(Context, NewVT);
16720b57cec5SDimitry Andric RegisterVT = DestVT;
16730b57cec5SDimitry Andric
16745ffd83dbSDimitry Andric if (EVT(DestVT).bitsLT(NewVT)) { // Value is expanded, e.g. i64 -> i16.
16755ffd83dbSDimitry Andric TypeSize NewVTSize = NewVT.getSizeInBits();
16760b57cec5SDimitry Andric // Convert sizes such as i33 to i64.
167706c3fb27SDimitry Andric if (!llvm::has_single_bit<uint32_t>(NewVTSize.getKnownMinValue()))
1678e8d8bef9SDimitry Andric NewVTSize = NewVTSize.coefficientNextPowerOf2();
16790b57cec5SDimitry Andric return NumVectorRegs*(NewVTSize/DestVT.getSizeInBits());
16805ffd83dbSDimitry Andric }
16810b57cec5SDimitry Andric
16820b57cec5SDimitry Andric // Otherwise, promotion or legal types use the same number of registers as
16830b57cec5SDimitry Andric // the vector decimated to the appropriate level.
16840b57cec5SDimitry Andric return NumVectorRegs;
16850b57cec5SDimitry Andric }
16860b57cec5SDimitry Andric
isSuitableForJumpTable(const SwitchInst * SI,uint64_t NumCases,uint64_t Range,ProfileSummaryInfo * PSI,BlockFrequencyInfo * BFI) const1687480093f4SDimitry Andric bool TargetLoweringBase::isSuitableForJumpTable(const SwitchInst *SI,
1688480093f4SDimitry Andric uint64_t NumCases,
1689480093f4SDimitry Andric uint64_t Range,
1690480093f4SDimitry Andric ProfileSummaryInfo *PSI,
1691480093f4SDimitry Andric BlockFrequencyInfo *BFI) const {
1692480093f4SDimitry Andric // FIXME: This function check the maximum table size and density, but the
1693480093f4SDimitry Andric // minimum size is not checked. It would be nice if the minimum size is
1694480093f4SDimitry Andric // also combined within this function. Currently, the minimum size check is
1695480093f4SDimitry Andric // performed in findJumpTable() in SelectionDAGBuiler and
1696480093f4SDimitry Andric // getEstimatedNumberOfCaseClusters() in BasicTTIImpl.
1697480093f4SDimitry Andric const bool OptForSize =
1698480093f4SDimitry Andric SI->getParent()->getParent()->hasOptSize() ||
1699480093f4SDimitry Andric llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI);
1700480093f4SDimitry Andric const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize);
1701480093f4SDimitry Andric const unsigned MaxJumpTableSize = getMaximumJumpTableSize();
1702480093f4SDimitry Andric
1703480093f4SDimitry Andric // Check whether the number of cases is small enough and
1704480093f4SDimitry Andric // the range is dense enough for a jump table.
1705480093f4SDimitry Andric return (OptForSize || Range <= MaxJumpTableSize) &&
1706480093f4SDimitry Andric (NumCases * 100 >= Range * MinDensity);
1707480093f4SDimitry Andric }
1708480093f4SDimitry Andric
getPreferredSwitchConditionType(LLVMContext & Context,EVT ConditionVT) const170981ad6265SDimitry Andric MVT TargetLoweringBase::getPreferredSwitchConditionType(LLVMContext &Context,
171081ad6265SDimitry Andric EVT ConditionVT) const {
171181ad6265SDimitry Andric return getRegisterType(Context, ConditionVT);
171281ad6265SDimitry Andric }
171381ad6265SDimitry Andric
17140b57cec5SDimitry Andric /// Get the EVTs and ArgFlags collections that represent the legalized return
17150b57cec5SDimitry Andric /// type of the given function. This does not require a DAG or a return value,
17160b57cec5SDimitry Andric /// and is suitable for use before any DAGs for the function are constructed.
17170b57cec5SDimitry Andric /// TODO: Move this out of TargetLowering.cpp.
GetReturnInfo(CallingConv::ID CC,Type * ReturnType,AttributeList attr,SmallVectorImpl<ISD::OutputArg> & Outs,const TargetLowering & TLI,const DataLayout & DL)17180b57cec5SDimitry Andric void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
17190b57cec5SDimitry Andric AttributeList attr,
17200b57cec5SDimitry Andric SmallVectorImpl<ISD::OutputArg> &Outs,
17210b57cec5SDimitry Andric const TargetLowering &TLI, const DataLayout &DL) {
17220b57cec5SDimitry Andric SmallVector<EVT, 4> ValueVTs;
17230b57cec5SDimitry Andric ComputeValueVTs(TLI, DL, ReturnType, ValueVTs);
17240b57cec5SDimitry Andric unsigned NumValues = ValueVTs.size();
17250b57cec5SDimitry Andric if (NumValues == 0) return;
17260b57cec5SDimitry Andric
17270b57cec5SDimitry Andric for (unsigned j = 0, f = NumValues; j != f; ++j) {
17280b57cec5SDimitry Andric EVT VT = ValueVTs[j];
17290b57cec5SDimitry Andric ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
17300b57cec5SDimitry Andric
1731349cc55cSDimitry Andric if (attr.hasRetAttr(Attribute::SExt))
17320b57cec5SDimitry Andric ExtendKind = ISD::SIGN_EXTEND;
1733349cc55cSDimitry Andric else if (attr.hasRetAttr(Attribute::ZExt))
17340b57cec5SDimitry Andric ExtendKind = ISD::ZERO_EXTEND;
17350b57cec5SDimitry Andric
17360b57cec5SDimitry Andric // FIXME: C calling convention requires the return type to be promoted to
17370b57cec5SDimitry Andric // at least 32-bit. But this is not necessary for non-C calling
17380b57cec5SDimitry Andric // conventions. The frontend should mark functions whose return values
17390b57cec5SDimitry Andric // require promoting with signext or zeroext attributes.
17400b57cec5SDimitry Andric if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) {
174106c3fb27SDimitry Andric MVT MinVT = TLI.getRegisterType(MVT::i32);
17420b57cec5SDimitry Andric if (VT.bitsLT(MinVT))
17430b57cec5SDimitry Andric VT = MinVT;
17440b57cec5SDimitry Andric }
17450b57cec5SDimitry Andric
17460b57cec5SDimitry Andric unsigned NumParts =
17470b57cec5SDimitry Andric TLI.getNumRegistersForCallingConv(ReturnType->getContext(), CC, VT);
17480b57cec5SDimitry Andric MVT PartVT =
17490b57cec5SDimitry Andric TLI.getRegisterTypeForCallingConv(ReturnType->getContext(), CC, VT);
17500b57cec5SDimitry Andric
17510b57cec5SDimitry Andric // 'inreg' on function refers to return value
17520b57cec5SDimitry Andric ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1753349cc55cSDimitry Andric if (attr.hasRetAttr(Attribute::InReg))
17540b57cec5SDimitry Andric Flags.setInReg();
17550b57cec5SDimitry Andric
17560b57cec5SDimitry Andric // Propagate extension type if any
1757349cc55cSDimitry Andric if (attr.hasRetAttr(Attribute::SExt))
17580b57cec5SDimitry Andric Flags.setSExt();
1759349cc55cSDimitry Andric else if (attr.hasRetAttr(Attribute::ZExt))
17600b57cec5SDimitry Andric Flags.setZExt();
17610b57cec5SDimitry Andric
17620b57cec5SDimitry Andric for (unsigned i = 0; i < NumParts; ++i)
17630b57cec5SDimitry Andric Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0));
17640b57cec5SDimitry Andric }
17650b57cec5SDimitry Andric }
17660b57cec5SDimitry Andric
17670b57cec5SDimitry Andric /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
17680b57cec5SDimitry Andric /// function arguments in the caller parameter area. This is the actual
17690b57cec5SDimitry Andric /// alignment, not its logarithm.
getByValTypeAlignment(Type * Ty,const DataLayout & DL) const1770349cc55cSDimitry Andric uint64_t TargetLoweringBase::getByValTypeAlignment(Type *Ty,
17710b57cec5SDimitry Andric const DataLayout &DL) const {
17725ffd83dbSDimitry Andric return DL.getABITypeAlign(Ty).value();
17730b57cec5SDimitry Andric }
17740b57cec5SDimitry Andric
allowsMemoryAccessForAlignment(LLVMContext & Context,const DataLayout & DL,EVT VT,unsigned AddrSpace,Align Alignment,MachineMemOperand::Flags Flags,unsigned * Fast) const17758bcb0991SDimitry Andric bool TargetLoweringBase::allowsMemoryAccessForAlignment(
17768bcb0991SDimitry Andric LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
1777bdd1243dSDimitry Andric Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const {
17780b57cec5SDimitry Andric // Check if the specified alignment is sufficient based on the data layout.
17790b57cec5SDimitry Andric // TODO: While using the data layout works in practice, a better solution
17800b57cec5SDimitry Andric // would be to implement this check directly (make this a virtual function).
17810b57cec5SDimitry Andric // For example, the ABI alignment may change based on software platform while
17820b57cec5SDimitry Andric // this function should only be affected by hardware implementation.
17830b57cec5SDimitry Andric Type *Ty = VT.getTypeForEVT(Context);
1784fe6060f1SDimitry Andric if (VT.isZeroSized() || Alignment >= DL.getABITypeAlign(Ty)) {
17850b57cec5SDimitry Andric // Assume that an access that meets the ABI-specified alignment is fast.
17860b57cec5SDimitry Andric if (Fast != nullptr)
1787bdd1243dSDimitry Andric *Fast = 1;
17880b57cec5SDimitry Andric return true;
17890b57cec5SDimitry Andric }
17900b57cec5SDimitry Andric
17910b57cec5SDimitry Andric // This is a misaligned access.
1792fe6060f1SDimitry Andric return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
17930b57cec5SDimitry Andric }
17940b57cec5SDimitry Andric
allowsMemoryAccessForAlignment(LLVMContext & Context,const DataLayout & DL,EVT VT,const MachineMemOperand & MMO,unsigned * Fast) const17958bcb0991SDimitry Andric bool TargetLoweringBase::allowsMemoryAccessForAlignment(
17968bcb0991SDimitry Andric LLVMContext &Context, const DataLayout &DL, EVT VT,
1797bdd1243dSDimitry Andric const MachineMemOperand &MMO, unsigned *Fast) const {
17988bcb0991SDimitry Andric return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(),
17995ffd83dbSDimitry Andric MMO.getAlign(), MMO.getFlags(), Fast);
18008bcb0991SDimitry Andric }
18018bcb0991SDimitry Andric
allowsMemoryAccess(LLVMContext & Context,const DataLayout & DL,EVT VT,unsigned AddrSpace,Align Alignment,MachineMemOperand::Flags Flags,unsigned * Fast) const18025ffd83dbSDimitry Andric bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
18035ffd83dbSDimitry Andric const DataLayout &DL, EVT VT,
18045ffd83dbSDimitry Andric unsigned AddrSpace, Align Alignment,
18055ffd83dbSDimitry Andric MachineMemOperand::Flags Flags,
1806bdd1243dSDimitry Andric unsigned *Fast) const {
18078bcb0991SDimitry Andric return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment,
18088bcb0991SDimitry Andric Flags, Fast);
18098bcb0991SDimitry Andric }
18108bcb0991SDimitry Andric
allowsMemoryAccess(LLVMContext & Context,const DataLayout & DL,EVT VT,const MachineMemOperand & MMO,unsigned * Fast) const18110b57cec5SDimitry Andric bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
18120b57cec5SDimitry Andric const DataLayout &DL, EVT VT,
18130b57cec5SDimitry Andric const MachineMemOperand &MMO,
1814bdd1243dSDimitry Andric unsigned *Fast) const {
18155ffd83dbSDimitry Andric return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
18165ffd83dbSDimitry Andric MMO.getFlags(), Fast);
18170b57cec5SDimitry Andric }
18180b57cec5SDimitry Andric
allowsMemoryAccess(LLVMContext & Context,const DataLayout & DL,LLT Ty,const MachineMemOperand & MMO,unsigned * Fast) const1819e8d8bef9SDimitry Andric bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
1820e8d8bef9SDimitry Andric const DataLayout &DL, LLT Ty,
1821e8d8bef9SDimitry Andric const MachineMemOperand &MMO,
1822bdd1243dSDimitry Andric unsigned *Fast) const {
1823349cc55cSDimitry Andric EVT VT = getApproximateEVTForLLT(Ty, DL, Context);
1824349cc55cSDimitry Andric return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), MMO.getAlign(),
1825349cc55cSDimitry Andric MMO.getFlags(), Fast);
1826e8d8bef9SDimitry Andric }
1827e8d8bef9SDimitry Andric
18280b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
18290b57cec5SDimitry Andric // TargetTransformInfo Helpers
18300b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
18310b57cec5SDimitry Andric
InstructionOpcodeToISD(unsigned Opcode) const18320b57cec5SDimitry Andric int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {
18330b57cec5SDimitry Andric enum InstructionOpcodes {
18340b57cec5SDimitry Andric #define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
18350b57cec5SDimitry Andric #define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
18360b57cec5SDimitry Andric #include "llvm/IR/Instruction.def"
18370b57cec5SDimitry Andric };
18380b57cec5SDimitry Andric switch (static_cast<InstructionOpcodes>(Opcode)) {
18390b57cec5SDimitry Andric case Ret: return 0;
18400b57cec5SDimitry Andric case Br: return 0;
18410b57cec5SDimitry Andric case Switch: return 0;
18420b57cec5SDimitry Andric case IndirectBr: return 0;
18430b57cec5SDimitry Andric case Invoke: return 0;
18440b57cec5SDimitry Andric case CallBr: return 0;
18450b57cec5SDimitry Andric case Resume: return 0;
18460b57cec5SDimitry Andric case Unreachable: return 0;
18470b57cec5SDimitry Andric case CleanupRet: return 0;
18480b57cec5SDimitry Andric case CatchRet: return 0;
18490b57cec5SDimitry Andric case CatchPad: return 0;
18500b57cec5SDimitry Andric case CatchSwitch: return 0;
18510b57cec5SDimitry Andric case CleanupPad: return 0;
18520b57cec5SDimitry Andric case FNeg: return ISD::FNEG;
18530b57cec5SDimitry Andric case Add: return ISD::ADD;
18540b57cec5SDimitry Andric case FAdd: return ISD::FADD;
18550b57cec5SDimitry Andric case Sub: return ISD::SUB;
18560b57cec5SDimitry Andric case FSub: return ISD::FSUB;
18570b57cec5SDimitry Andric case Mul: return ISD::MUL;
18580b57cec5SDimitry Andric case FMul: return ISD::FMUL;
18590b57cec5SDimitry Andric case UDiv: return ISD::UDIV;
18600b57cec5SDimitry Andric case SDiv: return ISD::SDIV;
18610b57cec5SDimitry Andric case FDiv: return ISD::FDIV;
18620b57cec5SDimitry Andric case URem: return ISD::UREM;
18630b57cec5SDimitry Andric case SRem: return ISD::SREM;
18640b57cec5SDimitry Andric case FRem: return ISD::FREM;
18650b57cec5SDimitry Andric case Shl: return ISD::SHL;
18660b57cec5SDimitry Andric case LShr: return ISD::SRL;
18670b57cec5SDimitry Andric case AShr: return ISD::SRA;
18680b57cec5SDimitry Andric case And: return ISD::AND;
18690b57cec5SDimitry Andric case Or: return ISD::OR;
18700b57cec5SDimitry Andric case Xor: return ISD::XOR;
18710b57cec5SDimitry Andric case Alloca: return 0;
18720b57cec5SDimitry Andric case Load: return ISD::LOAD;
18730b57cec5SDimitry Andric case Store: return ISD::STORE;
18740b57cec5SDimitry Andric case GetElementPtr: return 0;
18750b57cec5SDimitry Andric case Fence: return 0;
18760b57cec5SDimitry Andric case AtomicCmpXchg: return 0;
18770b57cec5SDimitry Andric case AtomicRMW: return 0;
18780b57cec5SDimitry Andric case Trunc: return ISD::TRUNCATE;
18790b57cec5SDimitry Andric case ZExt: return ISD::ZERO_EXTEND;
18800b57cec5SDimitry Andric case SExt: return ISD::SIGN_EXTEND;
18810b57cec5SDimitry Andric case FPToUI: return ISD::FP_TO_UINT;
18820b57cec5SDimitry Andric case FPToSI: return ISD::FP_TO_SINT;
18830b57cec5SDimitry Andric case UIToFP: return ISD::UINT_TO_FP;
18840b57cec5SDimitry Andric case SIToFP: return ISD::SINT_TO_FP;
18850b57cec5SDimitry Andric case FPTrunc: return ISD::FP_ROUND;
18860b57cec5SDimitry Andric case FPExt: return ISD::FP_EXTEND;
18870b57cec5SDimitry Andric case PtrToInt: return ISD::BITCAST;
18880b57cec5SDimitry Andric case IntToPtr: return ISD::BITCAST;
18890b57cec5SDimitry Andric case BitCast: return ISD::BITCAST;
18900b57cec5SDimitry Andric case AddrSpaceCast: return ISD::ADDRSPACECAST;
18910b57cec5SDimitry Andric case ICmp: return ISD::SETCC;
18920b57cec5SDimitry Andric case FCmp: return ISD::SETCC;
18930b57cec5SDimitry Andric case PHI: return 0;
18940b57cec5SDimitry Andric case Call: return 0;
18950b57cec5SDimitry Andric case Select: return ISD::SELECT;
18960b57cec5SDimitry Andric case UserOp1: return 0;
18970b57cec5SDimitry Andric case UserOp2: return 0;
18980b57cec5SDimitry Andric case VAArg: return 0;
18990b57cec5SDimitry Andric case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
19000b57cec5SDimitry Andric case InsertElement: return ISD::INSERT_VECTOR_ELT;
19010b57cec5SDimitry Andric case ShuffleVector: return ISD::VECTOR_SHUFFLE;
19020b57cec5SDimitry Andric case ExtractValue: return ISD::MERGE_VALUES;
19030b57cec5SDimitry Andric case InsertValue: return ISD::MERGE_VALUES;
19040b57cec5SDimitry Andric case LandingPad: return 0;
19055ffd83dbSDimitry Andric case Freeze: return ISD::FREEZE;
19060b57cec5SDimitry Andric }
19070b57cec5SDimitry Andric
19080b57cec5SDimitry Andric llvm_unreachable("Unknown instruction type encountered!");
19090b57cec5SDimitry Andric }
19100b57cec5SDimitry Andric
1911fe6060f1SDimitry Andric Value *
getDefaultSafeStackPointerLocation(IRBuilderBase & IRB,bool UseTLS) const1912fe6060f1SDimitry Andric TargetLoweringBase::getDefaultSafeStackPointerLocation(IRBuilderBase &IRB,
19130b57cec5SDimitry Andric bool UseTLS) const {
19140b57cec5SDimitry Andric // compiler-rt provides a variable with a magic name. Targets that do not
19150b57cec5SDimitry Andric // link with compiler-rt may also provide such a variable.
19160b57cec5SDimitry Andric Module *M = IRB.GetInsertBlock()->getParent()->getParent();
19170b57cec5SDimitry Andric const char *UnsafeStackPtrVar = "__safestack_unsafe_stack_ptr";
19180b57cec5SDimitry Andric auto UnsafeStackPtr =
19190b57cec5SDimitry Andric dyn_cast_or_null<GlobalVariable>(M->getNamedValue(UnsafeStackPtrVar));
19200b57cec5SDimitry Andric
19215f757f3fSDimitry Andric Type *StackPtrTy = PointerType::getUnqual(M->getContext());
19220b57cec5SDimitry Andric
19230b57cec5SDimitry Andric if (!UnsafeStackPtr) {
19240b57cec5SDimitry Andric auto TLSModel = UseTLS ?
19250b57cec5SDimitry Andric GlobalValue::InitialExecTLSModel :
19260b57cec5SDimitry Andric GlobalValue::NotThreadLocal;
19270b57cec5SDimitry Andric // The global variable is not defined yet, define it ourselves.
19280b57cec5SDimitry Andric // We use the initial-exec TLS model because we do not support the
19290b57cec5SDimitry Andric // variable living anywhere other than in the main executable.
19300b57cec5SDimitry Andric UnsafeStackPtr = new GlobalVariable(
19310b57cec5SDimitry Andric *M, StackPtrTy, false, GlobalValue::ExternalLinkage, nullptr,
19320b57cec5SDimitry Andric UnsafeStackPtrVar, nullptr, TLSModel);
19330b57cec5SDimitry Andric } else {
19340b57cec5SDimitry Andric // The variable exists, check its type and attributes.
19350b57cec5SDimitry Andric if (UnsafeStackPtr->getValueType() != StackPtrTy)
19360b57cec5SDimitry Andric report_fatal_error(Twine(UnsafeStackPtrVar) + " must have void* type");
19370b57cec5SDimitry Andric if (UseTLS != UnsafeStackPtr->isThreadLocal())
19380b57cec5SDimitry Andric report_fatal_error(Twine(UnsafeStackPtrVar) + " must " +
19390b57cec5SDimitry Andric (UseTLS ? "" : "not ") + "be thread-local");
19400b57cec5SDimitry Andric }
19410b57cec5SDimitry Andric return UnsafeStackPtr;
19420b57cec5SDimitry Andric }
19430b57cec5SDimitry Andric
1944fe6060f1SDimitry Andric Value *
getSafeStackPointerLocation(IRBuilderBase & IRB) const1945fe6060f1SDimitry Andric TargetLoweringBase::getSafeStackPointerLocation(IRBuilderBase &IRB) const {
19460b57cec5SDimitry Andric if (!TM.getTargetTriple().isAndroid())
19470b57cec5SDimitry Andric return getDefaultSafeStackPointerLocation(IRB, true);
19480b57cec5SDimitry Andric
19490b57cec5SDimitry Andric // Android provides a libc function to retrieve the address of the current
19500b57cec5SDimitry Andric // thread's unsafe stack pointer.
19510b57cec5SDimitry Andric Module *M = IRB.GetInsertBlock()->getParent()->getParent();
19525f757f3fSDimitry Andric auto *PtrTy = PointerType::getUnqual(M->getContext());
19535f757f3fSDimitry Andric FunctionCallee Fn =
19545f757f3fSDimitry Andric M->getOrInsertFunction("__safestack_pointer_address", PtrTy);
19550b57cec5SDimitry Andric return IRB.CreateCall(Fn);
19560b57cec5SDimitry Andric }
19570b57cec5SDimitry Andric
19580b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
19590b57cec5SDimitry Andric // Loop Strength Reduction hooks
19600b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
19610b57cec5SDimitry Andric
19620b57cec5SDimitry Andric /// isLegalAddressingMode - Return true if the addressing mode represented
19630b57cec5SDimitry Andric /// by AM is legal for this target, for a load/store of the specified type.
isLegalAddressingMode(const DataLayout & DL,const AddrMode & AM,Type * Ty,unsigned AS,Instruction * I) const19640b57cec5SDimitry Andric bool TargetLoweringBase::isLegalAddressingMode(const DataLayout &DL,
19650b57cec5SDimitry Andric const AddrMode &AM, Type *Ty,
19660b57cec5SDimitry Andric unsigned AS, Instruction *I) const {
19670b57cec5SDimitry Andric // The default implementation of this implements a conservative RISCy, r+r and
19680b57cec5SDimitry Andric // r+i addr mode.
19690b57cec5SDimitry Andric
19700b57cec5SDimitry Andric // Allows a sign-extended 16-bit immediate field.
19710b57cec5SDimitry Andric if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
19720b57cec5SDimitry Andric return false;
19730b57cec5SDimitry Andric
19740b57cec5SDimitry Andric // No global is ever allowed as a base.
19750b57cec5SDimitry Andric if (AM.BaseGV)
19760b57cec5SDimitry Andric return false;
19770b57cec5SDimitry Andric
19780b57cec5SDimitry Andric // Only support r+r,
19790b57cec5SDimitry Andric switch (AM.Scale) {
19800b57cec5SDimitry Andric case 0: // "r+i" or just "i", depending on HasBaseReg.
19810b57cec5SDimitry Andric break;
19820b57cec5SDimitry Andric case 1:
19830b57cec5SDimitry Andric if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
19840b57cec5SDimitry Andric return false;
19850b57cec5SDimitry Andric // Otherwise we have r+r or r+i.
19860b57cec5SDimitry Andric break;
19870b57cec5SDimitry Andric case 2:
19880b57cec5SDimitry Andric if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
19890b57cec5SDimitry Andric return false;
19900b57cec5SDimitry Andric // Allow 2*r as r+r.
19910b57cec5SDimitry Andric break;
19920b57cec5SDimitry Andric default: // Don't allow n * r
19930b57cec5SDimitry Andric return false;
19940b57cec5SDimitry Andric }
19950b57cec5SDimitry Andric
19960b57cec5SDimitry Andric return true;
19970b57cec5SDimitry Andric }
19980b57cec5SDimitry Andric
19990b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
20000b57cec5SDimitry Andric // Stack Protector
20010b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
20020b57cec5SDimitry Andric
20030b57cec5SDimitry Andric // For OpenBSD return its special guard variable. Otherwise return nullptr,
20040b57cec5SDimitry Andric // so that SelectionDAG handle SSP.
getIRStackGuard(IRBuilderBase & IRB) const2005fe6060f1SDimitry Andric Value *TargetLoweringBase::getIRStackGuard(IRBuilderBase &IRB) const {
20060b57cec5SDimitry Andric if (getTargetMachine().getTargetTriple().isOSOpenBSD()) {
20070b57cec5SDimitry Andric Module &M = *IRB.GetInsertBlock()->getParent()->getParent();
20085f757f3fSDimitry Andric PointerType *PtrTy = PointerType::getUnqual(M.getContext());
200916d6b3b3SDimitry Andric Constant *C = M.getOrInsertGlobal("__guard_local", PtrTy);
201016d6b3b3SDimitry Andric if (GlobalVariable *G = dyn_cast_or_null<GlobalVariable>(C))
201116d6b3b3SDimitry Andric G->setVisibility(GlobalValue::HiddenVisibility);
201216d6b3b3SDimitry Andric return C;
20130b57cec5SDimitry Andric }
20140b57cec5SDimitry Andric return nullptr;
20150b57cec5SDimitry Andric }
20160b57cec5SDimitry Andric
20170b57cec5SDimitry Andric // Currently only support "standard" __stack_chk_guard.
20180b57cec5SDimitry Andric // TODO: add LOAD_STACK_GUARD support.
insertSSPDeclarations(Module & M) const20190b57cec5SDimitry Andric void TargetLoweringBase::insertSSPDeclarations(Module &M) const {
2020e8d8bef9SDimitry Andric if (!M.getNamedValue("__stack_chk_guard")) {
20215f757f3fSDimitry Andric auto *GV = new GlobalVariable(M, PointerType::getUnqual(M.getContext()),
20225f757f3fSDimitry Andric false, GlobalVariable::ExternalLinkage,
20235f757f3fSDimitry Andric nullptr, "__stack_chk_guard");
2024349cc55cSDimitry Andric
2025349cc55cSDimitry Andric // FreeBSD has "__stack_chk_guard" defined externally on libc.so
202606c3fb27SDimitry Andric if (M.getDirectAccessExternalData() &&
20279a4d48a6SAlfredo Dal'Ava Junior !TM.getTargetTriple().isWindowsGNUEnvironment() &&
202806c3fb27SDimitry Andric !(TM.getTargetTriple().isPPC64() && TM.getTargetTriple().isOSFreeBSD()) &&
20295f757f3fSDimitry Andric (!TM.getTargetTriple().isOSDarwin() ||
20305f757f3fSDimitry Andric TM.getRelocationModel() == Reloc::Static))
2031e8d8bef9SDimitry Andric GV->setDSOLocal(true);
2032e8d8bef9SDimitry Andric }
20330b57cec5SDimitry Andric }
20340b57cec5SDimitry Andric
20350b57cec5SDimitry Andric // Currently only support "standard" __stack_chk_guard.
20360b57cec5SDimitry Andric // TODO: add LOAD_STACK_GUARD support.
getSDagStackGuard(const Module & M) const20370b57cec5SDimitry Andric Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const {
20380b57cec5SDimitry Andric return M.getNamedValue("__stack_chk_guard");
20390b57cec5SDimitry Andric }
20400b57cec5SDimitry Andric
getSSPStackGuardCheck(const Module & M) const20410b57cec5SDimitry Andric Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const {
20420b57cec5SDimitry Andric return nullptr;
20430b57cec5SDimitry Andric }
20440b57cec5SDimitry Andric
getMinimumJumpTableEntries() const20450b57cec5SDimitry Andric unsigned TargetLoweringBase::getMinimumJumpTableEntries() const {
20460b57cec5SDimitry Andric return MinimumJumpTableEntries;
20470b57cec5SDimitry Andric }
20480b57cec5SDimitry Andric
setMinimumJumpTableEntries(unsigned Val)20490b57cec5SDimitry Andric void TargetLoweringBase::setMinimumJumpTableEntries(unsigned Val) {
20500b57cec5SDimitry Andric MinimumJumpTableEntries = Val;
20510b57cec5SDimitry Andric }
20520b57cec5SDimitry Andric
getMinimumJumpTableDensity(bool OptForSize) const20530b57cec5SDimitry Andric unsigned TargetLoweringBase::getMinimumJumpTableDensity(bool OptForSize) const {
20540b57cec5SDimitry Andric return OptForSize ? OptsizeJumpTableDensity : JumpTableDensity;
20550b57cec5SDimitry Andric }
20560b57cec5SDimitry Andric
getMaximumJumpTableSize() const20570b57cec5SDimitry Andric unsigned TargetLoweringBase::getMaximumJumpTableSize() const {
20580b57cec5SDimitry Andric return MaximumJumpTableSize;
20590b57cec5SDimitry Andric }
20600b57cec5SDimitry Andric
setMaximumJumpTableSize(unsigned Val)20610b57cec5SDimitry Andric void TargetLoweringBase::setMaximumJumpTableSize(unsigned Val) {
20620b57cec5SDimitry Andric MaximumJumpTableSize = Val;
20630b57cec5SDimitry Andric }
20640b57cec5SDimitry Andric
isJumpTableRelative() const20655ffd83dbSDimitry Andric bool TargetLoweringBase::isJumpTableRelative() const {
20665ffd83dbSDimitry Andric return getTargetMachine().isPositionIndependent();
20675ffd83dbSDimitry Andric }
20685ffd83dbSDimitry Andric
getPrefLoopAlignment(MachineLoop * ML) const2069349cc55cSDimitry Andric Align TargetLoweringBase::getPrefLoopAlignment(MachineLoop *ML) const {
2070349cc55cSDimitry Andric if (TM.Options.LoopAlignment)
2071349cc55cSDimitry Andric return Align(TM.Options.LoopAlignment);
2072349cc55cSDimitry Andric return PrefLoopAlignment;
2073349cc55cSDimitry Andric }
2074349cc55cSDimitry Andric
getMaxPermittedBytesForAlignment(MachineBasicBlock * MBB) const207504eeddc0SDimitry Andric unsigned TargetLoweringBase::getMaxPermittedBytesForAlignment(
207604eeddc0SDimitry Andric MachineBasicBlock *MBB) const {
207704eeddc0SDimitry Andric return MaxBytesForAlignment;
207804eeddc0SDimitry Andric }
207904eeddc0SDimitry Andric
20800b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
20810b57cec5SDimitry Andric // Reciprocal Estimates
20820b57cec5SDimitry Andric //===----------------------------------------------------------------------===//
20830b57cec5SDimitry Andric
20840b57cec5SDimitry Andric /// Get the reciprocal estimate attribute string for a function that will
20850b57cec5SDimitry Andric /// override the target defaults.
getRecipEstimateForFunc(MachineFunction & MF)20860b57cec5SDimitry Andric static StringRef getRecipEstimateForFunc(MachineFunction &MF) {
20870b57cec5SDimitry Andric const Function &F = MF.getFunction();
20880b57cec5SDimitry Andric return F.getFnAttribute("reciprocal-estimates").getValueAsString();
20890b57cec5SDimitry Andric }
20900b57cec5SDimitry Andric
20910b57cec5SDimitry Andric /// Construct a string for the given reciprocal operation of the given type.
20920b57cec5SDimitry Andric /// This string should match the corresponding option to the front-end's
20930b57cec5SDimitry Andric /// "-mrecip" flag assuming those strings have been passed through in an
20940b57cec5SDimitry Andric /// attribute string. For example, "vec-divf" for a division of a vXf32.
getReciprocalOpName(bool IsSqrt,EVT VT)20950b57cec5SDimitry Andric static std::string getReciprocalOpName(bool IsSqrt, EVT VT) {
20960b57cec5SDimitry Andric std::string Name = VT.isVector() ? "vec-" : "";
20970b57cec5SDimitry Andric
20980b57cec5SDimitry Andric Name += IsSqrt ? "sqrt" : "div";
20990b57cec5SDimitry Andric
210081ad6265SDimitry Andric // TODO: Handle other float types?
21010b57cec5SDimitry Andric if (VT.getScalarType() == MVT::f64) {
21020b57cec5SDimitry Andric Name += "d";
210381ad6265SDimitry Andric } else if (VT.getScalarType() == MVT::f16) {
210481ad6265SDimitry Andric Name += "h";
21050b57cec5SDimitry Andric } else {
21060b57cec5SDimitry Andric assert(VT.getScalarType() == MVT::f32 &&
21070b57cec5SDimitry Andric "Unexpected FP type for reciprocal estimate");
21080b57cec5SDimitry Andric Name += "f";
21090b57cec5SDimitry Andric }
21100b57cec5SDimitry Andric
21110b57cec5SDimitry Andric return Name;
21120b57cec5SDimitry Andric }
21130b57cec5SDimitry Andric
21140b57cec5SDimitry Andric /// Return the character position and value (a single numeric character) of a
21150b57cec5SDimitry Andric /// customized refinement operation in the input string if it exists. Return
21160b57cec5SDimitry Andric /// false if there is no customized refinement step count.
parseRefinementStep(StringRef In,size_t & Position,uint8_t & Value)21170b57cec5SDimitry Andric static bool parseRefinementStep(StringRef In, size_t &Position,
21180b57cec5SDimitry Andric uint8_t &Value) {
21190b57cec5SDimitry Andric const char RefStepToken = ':';
21200b57cec5SDimitry Andric Position = In.find(RefStepToken);
21210b57cec5SDimitry Andric if (Position == StringRef::npos)
21220b57cec5SDimitry Andric return false;
21230b57cec5SDimitry Andric
21240b57cec5SDimitry Andric StringRef RefStepString = In.substr(Position + 1);
21250b57cec5SDimitry Andric // Allow exactly one numeric character for the additional refinement
21260b57cec5SDimitry Andric // step parameter.
21270b57cec5SDimitry Andric if (RefStepString.size() == 1) {
21280b57cec5SDimitry Andric char RefStepChar = RefStepString[0];
2129e8d8bef9SDimitry Andric if (isDigit(RefStepChar)) {
21300b57cec5SDimitry Andric Value = RefStepChar - '0';
21310b57cec5SDimitry Andric return true;
21320b57cec5SDimitry Andric }
21330b57cec5SDimitry Andric }
21340b57cec5SDimitry Andric report_fatal_error("Invalid refinement step for -recip.");
21350b57cec5SDimitry Andric }
21360b57cec5SDimitry Andric
21370b57cec5SDimitry Andric /// For the input attribute string, return one of the ReciprocalEstimate enum
21380b57cec5SDimitry Andric /// status values (enabled, disabled, or not specified) for this operation on
21390b57cec5SDimitry Andric /// the specified data type.
getOpEnabled(bool IsSqrt,EVT VT,StringRef Override)21400b57cec5SDimitry Andric static int getOpEnabled(bool IsSqrt, EVT VT, StringRef Override) {
21410b57cec5SDimitry Andric if (Override.empty())
21420b57cec5SDimitry Andric return TargetLoweringBase::ReciprocalEstimate::Unspecified;
21430b57cec5SDimitry Andric
21440b57cec5SDimitry Andric SmallVector<StringRef, 4> OverrideVector;
21450b57cec5SDimitry Andric Override.split(OverrideVector, ',');
21460b57cec5SDimitry Andric unsigned NumArgs = OverrideVector.size();
21470b57cec5SDimitry Andric
21480b57cec5SDimitry Andric // Check if "all", "none", or "default" was specified.
21490b57cec5SDimitry Andric if (NumArgs == 1) {
21500b57cec5SDimitry Andric // Look for an optional setting of the number of refinement steps needed
21510b57cec5SDimitry Andric // for this type of reciprocal operation.
21520b57cec5SDimitry Andric size_t RefPos;
21530b57cec5SDimitry Andric uint8_t RefSteps;
21540b57cec5SDimitry Andric if (parseRefinementStep(Override, RefPos, RefSteps)) {
21550b57cec5SDimitry Andric // Split the string for further processing.
21560b57cec5SDimitry Andric Override = Override.substr(0, RefPos);
21570b57cec5SDimitry Andric }
21580b57cec5SDimitry Andric
21590b57cec5SDimitry Andric // All reciprocal types are enabled.
21600b57cec5SDimitry Andric if (Override == "all")
21610b57cec5SDimitry Andric return TargetLoweringBase::ReciprocalEstimate::Enabled;
21620b57cec5SDimitry Andric
21630b57cec5SDimitry Andric // All reciprocal types are disabled.
21640b57cec5SDimitry Andric if (Override == "none")
21650b57cec5SDimitry Andric return TargetLoweringBase::ReciprocalEstimate::Disabled;
21660b57cec5SDimitry Andric
21670b57cec5SDimitry Andric // Target defaults for enablement are used.
21680b57cec5SDimitry Andric if (Override == "default")
21690b57cec5SDimitry Andric return TargetLoweringBase::ReciprocalEstimate::Unspecified;
21700b57cec5SDimitry Andric }
21710b57cec5SDimitry Andric
21720b57cec5SDimitry Andric // The attribute string may omit the size suffix ('f'/'d').
21730b57cec5SDimitry Andric std::string VTName = getReciprocalOpName(IsSqrt, VT);
21740b57cec5SDimitry Andric std::string VTNameNoSize = VTName;
21750b57cec5SDimitry Andric VTNameNoSize.pop_back();
21760b57cec5SDimitry Andric static const char DisabledPrefix = '!';
21770b57cec5SDimitry Andric
21780b57cec5SDimitry Andric for (StringRef RecipType : OverrideVector) {
21790b57cec5SDimitry Andric size_t RefPos;
21800b57cec5SDimitry Andric uint8_t RefSteps;
21810b57cec5SDimitry Andric if (parseRefinementStep(RecipType, RefPos, RefSteps))
21820b57cec5SDimitry Andric RecipType = RecipType.substr(0, RefPos);
21830b57cec5SDimitry Andric
21840b57cec5SDimitry Andric // Ignore the disablement token for string matching.
21850b57cec5SDimitry Andric bool IsDisabled = RecipType[0] == DisabledPrefix;
21860b57cec5SDimitry Andric if (IsDisabled)
21870b57cec5SDimitry Andric RecipType = RecipType.substr(1);
21880b57cec5SDimitry Andric
21890b57cec5SDimitry Andric if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
21900b57cec5SDimitry Andric return IsDisabled ? TargetLoweringBase::ReciprocalEstimate::Disabled
21910b57cec5SDimitry Andric : TargetLoweringBase::ReciprocalEstimate::Enabled;
21920b57cec5SDimitry Andric }
21930b57cec5SDimitry Andric
21940b57cec5SDimitry Andric return TargetLoweringBase::ReciprocalEstimate::Unspecified;
21950b57cec5SDimitry Andric }
21960b57cec5SDimitry Andric
21970b57cec5SDimitry Andric /// For the input attribute string, return the customized refinement step count
21980b57cec5SDimitry Andric /// for this operation on the specified data type. If the step count does not
21990b57cec5SDimitry Andric /// exist, return the ReciprocalEstimate enum value for unspecified.
getOpRefinementSteps(bool IsSqrt,EVT VT,StringRef Override)22000b57cec5SDimitry Andric static int getOpRefinementSteps(bool IsSqrt, EVT VT, StringRef Override) {
22010b57cec5SDimitry Andric if (Override.empty())
22020b57cec5SDimitry Andric return TargetLoweringBase::ReciprocalEstimate::Unspecified;
22030b57cec5SDimitry Andric
22040b57cec5SDimitry Andric SmallVector<StringRef, 4> OverrideVector;
22050b57cec5SDimitry Andric Override.split(OverrideVector, ',');
22060b57cec5SDimitry Andric unsigned NumArgs = OverrideVector.size();
22070b57cec5SDimitry Andric
22080b57cec5SDimitry Andric // Check if "all", "default", or "none" was specified.
22090b57cec5SDimitry Andric if (NumArgs == 1) {
22100b57cec5SDimitry Andric // Look for an optional setting of the number of refinement steps needed
22110b57cec5SDimitry Andric // for this type of reciprocal operation.
22120b57cec5SDimitry Andric size_t RefPos;
22130b57cec5SDimitry Andric uint8_t RefSteps;
22140b57cec5SDimitry Andric if (!parseRefinementStep(Override, RefPos, RefSteps))
22150b57cec5SDimitry Andric return TargetLoweringBase::ReciprocalEstimate::Unspecified;
22160b57cec5SDimitry Andric
22170b57cec5SDimitry Andric // Split the string for further processing.
22180b57cec5SDimitry Andric Override = Override.substr(0, RefPos);
22190b57cec5SDimitry Andric assert(Override != "none" &&
22200b57cec5SDimitry Andric "Disabled reciprocals, but specifed refinement steps?");
22210b57cec5SDimitry Andric
22220b57cec5SDimitry Andric // If this is a general override, return the specified number of steps.
22230b57cec5SDimitry Andric if (Override == "all" || Override == "default")
22240b57cec5SDimitry Andric return RefSteps;
22250b57cec5SDimitry Andric }
22260b57cec5SDimitry Andric
22270b57cec5SDimitry Andric // The attribute string may omit the size suffix ('f'/'d').
22280b57cec5SDimitry Andric std::string VTName = getReciprocalOpName(IsSqrt, VT);
22290b57cec5SDimitry Andric std::string VTNameNoSize = VTName;
22300b57cec5SDimitry Andric VTNameNoSize.pop_back();
22310b57cec5SDimitry Andric
22320b57cec5SDimitry Andric for (StringRef RecipType : OverrideVector) {
22330b57cec5SDimitry Andric size_t RefPos;
22340b57cec5SDimitry Andric uint8_t RefSteps;
22350b57cec5SDimitry Andric if (!parseRefinementStep(RecipType, RefPos, RefSteps))
22360b57cec5SDimitry Andric continue;
22370b57cec5SDimitry Andric
22380b57cec5SDimitry Andric RecipType = RecipType.substr(0, RefPos);
22390b57cec5SDimitry Andric if (RecipType.equals(VTName) || RecipType.equals(VTNameNoSize))
22400b57cec5SDimitry Andric return RefSteps;
22410b57cec5SDimitry Andric }
22420b57cec5SDimitry Andric
22430b57cec5SDimitry Andric return TargetLoweringBase::ReciprocalEstimate::Unspecified;
22440b57cec5SDimitry Andric }
22450b57cec5SDimitry Andric
getRecipEstimateSqrtEnabled(EVT VT,MachineFunction & MF) const22460b57cec5SDimitry Andric int TargetLoweringBase::getRecipEstimateSqrtEnabled(EVT VT,
22470b57cec5SDimitry Andric MachineFunction &MF) const {
22480b57cec5SDimitry Andric return getOpEnabled(true, VT, getRecipEstimateForFunc(MF));
22490b57cec5SDimitry Andric }
22500b57cec5SDimitry Andric
getRecipEstimateDivEnabled(EVT VT,MachineFunction & MF) const22510b57cec5SDimitry Andric int TargetLoweringBase::getRecipEstimateDivEnabled(EVT VT,
22520b57cec5SDimitry Andric MachineFunction &MF) const {
22530b57cec5SDimitry Andric return getOpEnabled(false, VT, getRecipEstimateForFunc(MF));
22540b57cec5SDimitry Andric }
22550b57cec5SDimitry Andric
getSqrtRefinementSteps(EVT VT,MachineFunction & MF) const22560b57cec5SDimitry Andric int TargetLoweringBase::getSqrtRefinementSteps(EVT VT,
22570b57cec5SDimitry Andric MachineFunction &MF) const {
22580b57cec5SDimitry Andric return getOpRefinementSteps(true, VT, getRecipEstimateForFunc(MF));
22590b57cec5SDimitry Andric }
22600b57cec5SDimitry Andric
getDivRefinementSteps(EVT VT,MachineFunction & MF) const22610b57cec5SDimitry Andric int TargetLoweringBase::getDivRefinementSteps(EVT VT,
22620b57cec5SDimitry Andric MachineFunction &MF) const {
22630b57cec5SDimitry Andric return getOpRefinementSteps(false, VT, getRecipEstimateForFunc(MF));
22640b57cec5SDimitry Andric }
22650b57cec5SDimitry Andric
isLoadBitCastBeneficial(EVT LoadVT,EVT BitcastVT,const SelectionDAG & DAG,const MachineMemOperand & MMO) const2266bdd1243dSDimitry Andric bool TargetLoweringBase::isLoadBitCastBeneficial(
2267bdd1243dSDimitry Andric EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG,
2268bdd1243dSDimitry Andric const MachineMemOperand &MMO) const {
2269bdd1243dSDimitry Andric // Single-element vectors are scalarized, so we should generally avoid having
2270bdd1243dSDimitry Andric // any memory operations on such types, as they would get scalarized too.
2271bdd1243dSDimitry Andric if (LoadVT.isFixedLengthVector() && BitcastVT.isFixedLengthVector() &&
2272bdd1243dSDimitry Andric BitcastVT.getVectorNumElements() == 1)
2273bdd1243dSDimitry Andric return false;
2274bdd1243dSDimitry Andric
2275bdd1243dSDimitry Andric // Don't do if we could do an indexed load on the original type, but not on
2276bdd1243dSDimitry Andric // the new one.
2277bdd1243dSDimitry Andric if (!LoadVT.isSimple() || !BitcastVT.isSimple())
2278bdd1243dSDimitry Andric return true;
2279bdd1243dSDimitry Andric
2280bdd1243dSDimitry Andric MVT LoadMVT = LoadVT.getSimpleVT();
2281bdd1243dSDimitry Andric
2282bdd1243dSDimitry Andric // Don't bother doing this if it's just going to be promoted again later, as
2283bdd1243dSDimitry Andric // doing so might interfere with other combines.
2284bdd1243dSDimitry Andric if (getOperationAction(ISD::LOAD, LoadMVT) == Promote &&
2285bdd1243dSDimitry Andric getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())
2286bdd1243dSDimitry Andric return false;
2287bdd1243dSDimitry Andric
2288bdd1243dSDimitry Andric unsigned Fast = 0;
2289bdd1243dSDimitry Andric return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,
2290bdd1243dSDimitry Andric MMO, &Fast) &&
2291bdd1243dSDimitry Andric Fast;
2292bdd1243dSDimitry Andric }
2293bdd1243dSDimitry Andric
finalizeLowering(MachineFunction & MF) const22940b57cec5SDimitry Andric void TargetLoweringBase::finalizeLowering(MachineFunction &MF) const {
22950b57cec5SDimitry Andric MF.getRegInfo().freezeReservedRegs(MF);
22960b57cec5SDimitry Andric }
22975ffd83dbSDimitry Andric
getLoadMemOperandFlags(const LoadInst & LI,const DataLayout & DL,AssumptionCache * AC,const TargetLibraryInfo * LibInfo) const2298bdd1243dSDimitry Andric MachineMemOperand::Flags TargetLoweringBase::getLoadMemOperandFlags(
2299bdd1243dSDimitry Andric const LoadInst &LI, const DataLayout &DL, AssumptionCache *AC,
2300bdd1243dSDimitry Andric const TargetLibraryInfo *LibInfo) const {
23015ffd83dbSDimitry Andric MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
23025ffd83dbSDimitry Andric if (LI.isVolatile())
23035ffd83dbSDimitry Andric Flags |= MachineMemOperand::MOVolatile;
23045ffd83dbSDimitry Andric
23055ffd83dbSDimitry Andric if (LI.hasMetadata(LLVMContext::MD_nontemporal))
23065ffd83dbSDimitry Andric Flags |= MachineMemOperand::MONonTemporal;
23075ffd83dbSDimitry Andric
23085ffd83dbSDimitry Andric if (LI.hasMetadata(LLVMContext::MD_invariant_load))
23095ffd83dbSDimitry Andric Flags |= MachineMemOperand::MOInvariant;
23105ffd83dbSDimitry Andric
2311bdd1243dSDimitry Andric if (isDereferenceableAndAlignedPointer(LI.getPointerOperand(), LI.getType(),
2312bdd1243dSDimitry Andric LI.getAlign(), DL, &LI, AC,
2313bdd1243dSDimitry Andric /*DT=*/nullptr, LibInfo))
23145ffd83dbSDimitry Andric Flags |= MachineMemOperand::MODereferenceable;
23155ffd83dbSDimitry Andric
23165ffd83dbSDimitry Andric Flags |= getTargetMMOFlags(LI);
23175ffd83dbSDimitry Andric return Flags;
23185ffd83dbSDimitry Andric }
23195ffd83dbSDimitry Andric
23205ffd83dbSDimitry Andric MachineMemOperand::Flags
getStoreMemOperandFlags(const StoreInst & SI,const DataLayout & DL) const23215ffd83dbSDimitry Andric TargetLoweringBase::getStoreMemOperandFlags(const StoreInst &SI,
23225ffd83dbSDimitry Andric const DataLayout &DL) const {
23235ffd83dbSDimitry Andric MachineMemOperand::Flags Flags = MachineMemOperand::MOStore;
23245ffd83dbSDimitry Andric
23255ffd83dbSDimitry Andric if (SI.isVolatile())
23265ffd83dbSDimitry Andric Flags |= MachineMemOperand::MOVolatile;
23275ffd83dbSDimitry Andric
23285ffd83dbSDimitry Andric if (SI.hasMetadata(LLVMContext::MD_nontemporal))
23295ffd83dbSDimitry Andric Flags |= MachineMemOperand::MONonTemporal;
23305ffd83dbSDimitry Andric
23315ffd83dbSDimitry Andric // FIXME: Not preserving dereferenceable
23325ffd83dbSDimitry Andric Flags |= getTargetMMOFlags(SI);
23335ffd83dbSDimitry Andric return Flags;
23345ffd83dbSDimitry Andric }
23355ffd83dbSDimitry Andric
23365ffd83dbSDimitry Andric MachineMemOperand::Flags
getAtomicMemOperandFlags(const Instruction & AI,const DataLayout & DL) const23375ffd83dbSDimitry Andric TargetLoweringBase::getAtomicMemOperandFlags(const Instruction &AI,
23385ffd83dbSDimitry Andric const DataLayout &DL) const {
23395ffd83dbSDimitry Andric auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
23405ffd83dbSDimitry Andric
23415ffd83dbSDimitry Andric if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(&AI)) {
23425ffd83dbSDimitry Andric if (RMW->isVolatile())
23435ffd83dbSDimitry Andric Flags |= MachineMemOperand::MOVolatile;
23445ffd83dbSDimitry Andric } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(&AI)) {
23455ffd83dbSDimitry Andric if (CmpX->isVolatile())
23465ffd83dbSDimitry Andric Flags |= MachineMemOperand::MOVolatile;
23475ffd83dbSDimitry Andric } else
23485ffd83dbSDimitry Andric llvm_unreachable("not an atomic instruction");
23495ffd83dbSDimitry Andric
23505ffd83dbSDimitry Andric // FIXME: Not preserving dereferenceable
23515ffd83dbSDimitry Andric Flags |= getTargetMMOFlags(AI);
23525ffd83dbSDimitry Andric return Flags;
23535ffd83dbSDimitry Andric }
23545ffd83dbSDimitry Andric
emitLeadingFence(IRBuilderBase & Builder,Instruction * Inst,AtomicOrdering Ord) const2355fe6060f1SDimitry Andric Instruction *TargetLoweringBase::emitLeadingFence(IRBuilderBase &Builder,
2356fe6060f1SDimitry Andric Instruction *Inst,
2357fe6060f1SDimitry Andric AtomicOrdering Ord) const {
2358fe6060f1SDimitry Andric if (isReleaseOrStronger(Ord) && Inst->hasAtomicStore())
2359fe6060f1SDimitry Andric return Builder.CreateFence(Ord);
2360fe6060f1SDimitry Andric else
2361fe6060f1SDimitry Andric return nullptr;
2362fe6060f1SDimitry Andric }
2363fe6060f1SDimitry Andric
emitTrailingFence(IRBuilderBase & Builder,Instruction * Inst,AtomicOrdering Ord) const2364fe6060f1SDimitry Andric Instruction *TargetLoweringBase::emitTrailingFence(IRBuilderBase &Builder,
2365fe6060f1SDimitry Andric Instruction *Inst,
2366fe6060f1SDimitry Andric AtomicOrdering Ord) const {
2367fe6060f1SDimitry Andric if (isAcquireOrStronger(Ord))
2368fe6060f1SDimitry Andric return Builder.CreateFence(Ord);
2369fe6060f1SDimitry Andric else
2370fe6060f1SDimitry Andric return nullptr;
2371fe6060f1SDimitry Andric }
2372fe6060f1SDimitry Andric
23735ffd83dbSDimitry Andric //===----------------------------------------------------------------------===//
23745ffd83dbSDimitry Andric // GlobalISel Hooks
23755ffd83dbSDimitry Andric //===----------------------------------------------------------------------===//
23765ffd83dbSDimitry Andric
shouldLocalize(const MachineInstr & MI,const TargetTransformInfo * TTI) const23775ffd83dbSDimitry Andric bool TargetLoweringBase::shouldLocalize(const MachineInstr &MI,
23785ffd83dbSDimitry Andric const TargetTransformInfo *TTI) const {
23795ffd83dbSDimitry Andric auto &MF = *MI.getMF();
23805ffd83dbSDimitry Andric auto &MRI = MF.getRegInfo();
23815ffd83dbSDimitry Andric // Assuming a spill and reload of a value has a cost of 1 instruction each,
23825ffd83dbSDimitry Andric // this helper function computes the maximum number of uses we should consider
23835ffd83dbSDimitry Andric // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We
23845ffd83dbSDimitry Andric // break even in terms of code size when the original MI has 2 users vs
23855ffd83dbSDimitry Andric // choosing to potentially spill. Any more than 2 users we we have a net code
23865ffd83dbSDimitry Andric // size increase. This doesn't take into account register pressure though.
23875ffd83dbSDimitry Andric auto maxUses = [](unsigned RematCost) {
23885ffd83dbSDimitry Andric // A cost of 1 means remats are basically free.
23895ffd83dbSDimitry Andric if (RematCost == 1)
2390bdd1243dSDimitry Andric return std::numeric_limits<unsigned>::max();
23915ffd83dbSDimitry Andric if (RematCost == 2)
23925ffd83dbSDimitry Andric return 2U;
23935ffd83dbSDimitry Andric
23945ffd83dbSDimitry Andric // Remat is too expensive, only sink if there's one user.
23955ffd83dbSDimitry Andric if (RematCost > 2)
23965ffd83dbSDimitry Andric return 1U;
23975ffd83dbSDimitry Andric llvm_unreachable("Unexpected remat cost");
23985ffd83dbSDimitry Andric };
23995ffd83dbSDimitry Andric
24005ffd83dbSDimitry Andric switch (MI.getOpcode()) {
24015ffd83dbSDimitry Andric default:
24025ffd83dbSDimitry Andric return false;
24035ffd83dbSDimitry Andric // Constants-like instructions should be close to their users.
24045ffd83dbSDimitry Andric // We don't want long live-ranges for them.
24055ffd83dbSDimitry Andric case TargetOpcode::G_CONSTANT:
24065ffd83dbSDimitry Andric case TargetOpcode::G_FCONSTANT:
24075ffd83dbSDimitry Andric case TargetOpcode::G_FRAME_INDEX:
24085ffd83dbSDimitry Andric case TargetOpcode::G_INTTOPTR:
24095ffd83dbSDimitry Andric return true;
24105ffd83dbSDimitry Andric case TargetOpcode::G_GLOBAL_VALUE: {
24115ffd83dbSDimitry Andric unsigned RematCost = TTI->getGISelRematGlobalCost();
24125ffd83dbSDimitry Andric Register Reg = MI.getOperand(0).getReg();
24135ffd83dbSDimitry Andric unsigned MaxUses = maxUses(RematCost);
24145ffd83dbSDimitry Andric if (MaxUses == UINT_MAX)
24155ffd83dbSDimitry Andric return true; // Remats are "free" so always localize.
2416bdd1243dSDimitry Andric return MRI.hasAtMostUserInstrs(Reg, MaxUses);
24175ffd83dbSDimitry Andric }
24185ffd83dbSDimitry Andric }
24195ffd83dbSDimitry Andric }
2420