1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64SelectionDAGInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "AArch64TargetMachine.h"
14 using namespace llvm;
15
16 #define DEBUG_TYPE "aarch64-selectiondag-info"
17
EmitTargetCodeForMemset(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVolatile,MachinePointerInfo DstPtrInfo) const18 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
19 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
20 SDValue Size, Align Alignment, bool isVolatile,
21 MachinePointerInfo DstPtrInfo) const {
22 // Check to see if there is a specialized entry-point for memory zeroing.
23 ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
24 ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size);
25 const AArch64Subtarget &STI =
26 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
27 const char *bzeroName = (V && V->isNullValue())
28 ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO) : nullptr;
29 // For small size (< 256), it is not beneficial to use bzero
30 // instead of memset.
31 if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) {
32 const AArch64TargetLowering &TLI = *STI.getTargetLowering();
33
34 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
35 Type *IntPtrTy = Type::getInt8PtrTy(*DAG.getContext());
36 TargetLowering::ArgListTy Args;
37 TargetLowering::ArgListEntry Entry;
38 Entry.Node = Dst;
39 Entry.Ty = IntPtrTy;
40 Args.push_back(Entry);
41 Entry.Node = Size;
42 Args.push_back(Entry);
43 TargetLowering::CallLoweringInfo CLI(DAG);
44 CLI.setDebugLoc(dl)
45 .setChain(Chain)
46 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
47 DAG.getExternalSymbol(bzeroName, IntPtr),
48 std::move(Args))
49 .setDiscardResult();
50 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
51 return CallResult.second;
52 }
53 return SDValue();
54 }
generateFMAsInMachineCombiner(CodeGenOpt::Level OptLevel) const55 bool AArch64SelectionDAGInfo::generateFMAsInMachineCombiner(
56 CodeGenOpt::Level OptLevel) const {
57 return OptLevel >= CodeGenOpt::Aggressive;
58 }
59
60 static const int kSetTagLoopThreshold = 176;
61
EmitUnrolledSetTag(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Ptr,uint64_t ObjSize,const MachineMemOperand * BaseMemOperand,bool ZeroData)62 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
63 SDValue Chain, SDValue Ptr, uint64_t ObjSize,
64 const MachineMemOperand *BaseMemOperand,
65 bool ZeroData) {
66 MachineFunction &MF = DAG.getMachineFunction();
67 unsigned ObjSizeScaled = ObjSize / 16;
68
69 SDValue TagSrc = Ptr;
70 if (Ptr.getOpcode() == ISD::FrameIndex) {
71 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
72 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
73 // A frame index operand may end up as [SP + offset] => it is fine to use SP
74 // register as the tag source.
75 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
76 }
77
78 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
79 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
80
81 SmallVector<SDValue, 8> OutChains;
82 unsigned OffsetScaled = 0;
83 while (OffsetScaled < ObjSizeScaled) {
84 if (ObjSizeScaled - OffsetScaled >= 2) {
85 SDValue AddrNode =
86 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
87 SDValue St = DAG.getMemIntrinsicNode(
88 OpCode2, dl, DAG.getVTList(MVT::Other),
89 {Chain, TagSrc, AddrNode},
90 MVT::v4i64,
91 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
92 OffsetScaled += 2;
93 OutChains.push_back(St);
94 continue;
95 }
96
97 if (ObjSizeScaled - OffsetScaled > 0) {
98 SDValue AddrNode =
99 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
100 SDValue St = DAG.getMemIntrinsicNode(
101 OpCode1, dl, DAG.getVTList(MVT::Other),
102 {Chain, TagSrc, AddrNode},
103 MVT::v2i64,
104 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
105 OffsetScaled += 1;
106 OutChains.push_back(St);
107 }
108 }
109
110 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
111 return Res;
112 }
113
EmitTargetCodeForSetTag(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Addr,SDValue Size,MachinePointerInfo DstPtrInfo,bool ZeroData) const114 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
115 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
116 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
117 uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
118 assert(ObjSize % 16 == 0);
119
120 MachineFunction &MF = DAG.getMachineFunction();
121 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
122 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
123
124 bool UseSetTagRangeLoop =
125 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
126 if (!UseSetTagRangeLoop)
127 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
128 ZeroData);
129
130 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
131
132 unsigned Opcode;
133 if (Addr.getOpcode() == ISD::FrameIndex) {
134 int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
135 Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
136 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
137 } else {
138 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
139 }
140 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
141 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
142
143 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
144 return SDValue(St, 2);
145 }
146