1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64SelectionDAGInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64TargetMachine.h"
14 using namespace llvm;
15 
16 #define DEBUG_TYPE "aarch64-selectiondag-info"
17 
18 SDValue AArch64SelectionDAGInfo::EmitMOPS(AArch64ISD::NodeType SDOpcode,
19                                           SelectionDAG &DAG, const SDLoc &DL,
20                                           SDValue Chain, SDValue Dst,
21                                           SDValue SrcOrValue, SDValue Size,
22                                           Align Alignment, bool isVolatile,
23                                           MachinePointerInfo DstPtrInfo,
24                                           MachinePointerInfo SrcPtrInfo) const {
25 
26   // Get the constant size of the copy/set.
27   uint64_t ConstSize = 0;
28   if (auto *C = dyn_cast<ConstantSDNode>(Size))
29     ConstSize = C->getZExtValue();
30 
31   const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET ||
32                      SDOpcode == AArch64ISD::MOPS_MEMSET_TAGGING;
33 
34   const auto MachineOpcode = [&]() {
35     switch (SDOpcode) {
36     case AArch64ISD::MOPS_MEMSET:
37       return AArch64::MOPSMemorySetPseudo;
38     case AArch64ISD::MOPS_MEMSET_TAGGING:
39       return AArch64::MOPSMemorySetTaggingPseudo;
40     case AArch64ISD::MOPS_MEMCOPY:
41       return AArch64::MOPSMemoryCopyPseudo;
42     case AArch64ISD::MOPS_MEMMOVE:
43       return AArch64::MOPSMemoryMovePseudo;
44     default:
45       llvm_unreachable("Unhandled MOPS ISD Opcode");
46     }
47   }();
48 
49   MachineMemOperand::Flags Flags = MachineMemOperand::MOStore;
50   if (isVolatile)
51     Flags |= MachineMemOperand::MOVolatile;
52   if (!IsSet)
53     Flags |= MachineMemOperand::MOLoad;
54 
55   MachineFunction &MF = DAG.getMachineFunction();
56 
57   auto *DstOp =
58       MF.getMachineMemOperand(DstPtrInfo, Flags, ConstSize, Alignment);
59   auto *SrcOp =
60       MF.getMachineMemOperand(SrcPtrInfo, Flags, ConstSize, Alignment);
61 
62   if (IsSet) {
63     // Extend value to i64 if required
64     if (SrcOrValue.getValueType() != MVT::i64)
65       SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue);
66     SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};
67     const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
68     MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
69     DAG.setNodeMemRefs(Node, {DstOp});
70     return SDValue(Node, 2);
71   } else {
72     SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};
73     const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
74     MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
75     DAG.setNodeMemRefs(Node, {DstOp, SrcOp});
76     return SDValue(Node, 3);
77   }
78 }
79 
80 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(
81     SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
82     SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
83     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
84   const AArch64Subtarget &STI =
85       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
86   if (STI.hasMOPS())
87     return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size,
88                     Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
89   return SDValue();
90 }
91 
92 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
93     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
94     SDValue Size, Align Alignment, bool isVolatile,
95     MachinePointerInfo DstPtrInfo) const {
96   const AArch64Subtarget &STI =
97       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
98 
99   if (STI.hasMOPS()) {
100     return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size,
101                     Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{});
102   }
103 
104   // Check to see if there is a specialized entry-point for memory zeroing.
105   ConstantSDNode *V = dyn_cast<ConstantSDNode>(Src);
106   ConstantSDNode *SizeValue = dyn_cast<ConstantSDNode>(Size);
107   const char *bzeroName =
108       (V && V->isZero())
109           ? DAG.getTargetLoweringInfo().getLibcallName(RTLIB::BZERO)
110           : nullptr;
111   // For small size (< 256), it is not beneficial to use bzero
112   // instead of memset.
113   if (bzeroName && (!SizeValue || SizeValue->getZExtValue() > 256)) {
114     const AArch64TargetLowering &TLI = *STI.getTargetLowering();
115 
116     EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout());
117     Type *IntPtrTy = Type::getInt8PtrTy(*DAG.getContext());
118     TargetLowering::ArgListTy Args;
119     TargetLowering::ArgListEntry Entry;
120     Entry.Node = Dst;
121     Entry.Ty = IntPtrTy;
122     Args.push_back(Entry);
123     Entry.Node = Size;
124     Args.push_back(Entry);
125     TargetLowering::CallLoweringInfo CLI(DAG);
126     CLI.setDebugLoc(dl)
127         .setChain(Chain)
128         .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
129                       DAG.getExternalSymbol(bzeroName, IntPtr),
130                       std::move(Args))
131         .setDiscardResult();
132     std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
133     return CallResult.second;
134   }
135   return SDValue();
136 }
137 
138 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(
139     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
140     SDValue Size, Align Alignment, bool isVolatile,
141     MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
142   const AArch64Subtarget &STI =
143       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
144   if (STI.hasMOPS()) {
145     return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size,
146                     Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
147   }
148   return SDValue();
149 }
150 
151 static const int kSetTagLoopThreshold = 176;
152 
153 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
154                                   SDValue Chain, SDValue Ptr, uint64_t ObjSize,
155                                   const MachineMemOperand *BaseMemOperand,
156                                   bool ZeroData) {
157   MachineFunction &MF = DAG.getMachineFunction();
158   unsigned ObjSizeScaled = ObjSize / 16;
159 
160   SDValue TagSrc = Ptr;
161   if (Ptr.getOpcode() == ISD::FrameIndex) {
162     int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
163     Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
164     // A frame index operand may end up as [SP + offset] => it is fine to use SP
165     // register as the tag source.
166     TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
167   }
168 
169   const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
170   const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
171 
172   SmallVector<SDValue, 8> OutChains;
173   unsigned OffsetScaled = 0;
174   while (OffsetScaled < ObjSizeScaled) {
175     if (ObjSizeScaled - OffsetScaled >= 2) {
176       SDValue AddrNode =
177           DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
178       SDValue St = DAG.getMemIntrinsicNode(
179           OpCode2, dl, DAG.getVTList(MVT::Other),
180           {Chain, TagSrc, AddrNode},
181           MVT::v4i64,
182           MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
183       OffsetScaled += 2;
184       OutChains.push_back(St);
185       continue;
186     }
187 
188     if (ObjSizeScaled - OffsetScaled > 0) {
189       SDValue AddrNode =
190           DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
191       SDValue St = DAG.getMemIntrinsicNode(
192           OpCode1, dl, DAG.getVTList(MVT::Other),
193           {Chain, TagSrc, AddrNode},
194           MVT::v2i64,
195           MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
196       OffsetScaled += 1;
197       OutChains.push_back(St);
198     }
199   }
200 
201   SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
202   return Res;
203 }
204 
205 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
206     SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
207     SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
208   uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
209   assert(ObjSize % 16 == 0);
210 
211   MachineFunction &MF = DAG.getMachineFunction();
212   MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
213       DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
214 
215   bool UseSetTagRangeLoop =
216       kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
217   if (!UseSetTagRangeLoop)
218     return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
219                               ZeroData);
220 
221   const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
222 
223   unsigned Opcode;
224   if (Addr.getOpcode() == ISD::FrameIndex) {
225     int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
226     Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
227     Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
228   } else {
229     Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
230   }
231   SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
232   SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
233 
234   DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
235   return SDValue(St, 2);
236 }
237