1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "llvm/CodeGen/MachineFrameInfo.h"
17 #include "llvm/IR/IntrinsicsRISCV.h"
18 #include "llvm/Support/Alignment.h"
19 #include "llvm/Support/Debug.h"
20 #include "llvm/Support/MathExtras.h"
21 #include "llvm/Support/raw_ostream.h"
22
23 using namespace llvm;
24
25 #define DEBUG_TYPE "riscv-isel"
26
PostprocessISelDAG()27 void RISCVDAGToDAGISel::PostprocessISelDAG() {
28 doPeepholeLoadStoreADDI();
29 }
30
selectImm(SelectionDAG * CurDAG,const SDLoc & DL,int64_t Imm,MVT XLenVT)31 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
32 MVT XLenVT) {
33 RISCVMatInt::InstSeq Seq;
34 RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64, Seq);
35
36 SDNode *Result = nullptr;
37 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
38 for (RISCVMatInt::Inst &Inst : Seq) {
39 SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
40 if (Inst.Opc == RISCV::LUI)
41 Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
42 else
43 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
44
45 // Only the first instruction has X0 as its source.
46 SrcReg = SDValue(Result, 0);
47 }
48
49 return Result;
50 }
51
getLMUL(EVT VT)52 static RISCVVLMUL getLMUL(EVT VT) {
53 switch (VT.getSizeInBits().getKnownMinValue() / 8) {
54 default:
55 llvm_unreachable("Invalid LMUL.");
56 case 1:
57 return RISCVVLMUL::LMUL_F8;
58 case 2:
59 return RISCVVLMUL::LMUL_F4;
60 case 4:
61 return RISCVVLMUL::LMUL_F2;
62 case 8:
63 return RISCVVLMUL::LMUL_1;
64 case 16:
65 return RISCVVLMUL::LMUL_2;
66 case 32:
67 return RISCVVLMUL::LMUL_4;
68 case 64:
69 return RISCVVLMUL::LMUL_8;
70 }
71 }
72
getSubregIndexByEVT(EVT VT,unsigned Index)73 static unsigned getSubregIndexByEVT(EVT VT, unsigned Index) {
74 RISCVVLMUL LMUL = getLMUL(VT);
75 if (LMUL == RISCVVLMUL::LMUL_F8 || LMUL == RISCVVLMUL::LMUL_F4 ||
76 LMUL == RISCVVLMUL::LMUL_F2 || LMUL == RISCVVLMUL::LMUL_1) {
77 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
78 "Unexpected subreg numbering");
79 return RISCV::sub_vrm1_0 + Index;
80 } else if (LMUL == RISCVVLMUL::LMUL_2) {
81 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
82 "Unexpected subreg numbering");
83 return RISCV::sub_vrm2_0 + Index;
84 } else if (LMUL == RISCVVLMUL::LMUL_4) {
85 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
86 "Unexpected subreg numbering");
87 return RISCV::sub_vrm4_0 + Index;
88 }
89 llvm_unreachable("Invalid vector type.");
90 }
91
createTupleImpl(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned RegClassID,unsigned SubReg0)92 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
93 unsigned RegClassID, unsigned SubReg0) {
94 assert(Regs.size() >= 2 && Regs.size() <= 8);
95
96 SDLoc DL(Regs[0]);
97 SmallVector<SDValue, 8> Ops;
98
99 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
100
101 for (unsigned I = 0; I < Regs.size(); ++I) {
102 Ops.push_back(Regs[I]);
103 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
104 }
105 SDNode *N =
106 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
107 return SDValue(N, 0);
108 }
109
createM1Tuple(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned NF)110 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
111 unsigned NF) {
112 static const unsigned RegClassIDs[] = {
113 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
114 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
115 RISCV::VRN8M1RegClassID};
116
117 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
118 }
119
createM2Tuple(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned NF)120 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
121 unsigned NF) {
122 static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
123 RISCV::VRN3M2RegClassID,
124 RISCV::VRN4M2RegClassID};
125
126 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
127 }
128
createM4Tuple(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned NF)129 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
130 unsigned NF) {
131 return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
132 RISCV::sub_vrm4_0);
133 }
134
createTuple(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned NF,RISCVVLMUL LMUL)135 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
136 unsigned NF, RISCVVLMUL LMUL) {
137 switch (LMUL) {
138 default:
139 llvm_unreachable("Invalid LMUL.");
140 case RISCVVLMUL::LMUL_F8:
141 case RISCVVLMUL::LMUL_F4:
142 case RISCVVLMUL::LMUL_F2:
143 case RISCVVLMUL::LMUL_1:
144 return createM1Tuple(CurDAG, Regs, NF);
145 case RISCVVLMUL::LMUL_2:
146 return createM2Tuple(CurDAG, Regs, NF);
147 case RISCVVLMUL::LMUL_4:
148 return createM4Tuple(CurDAG, Regs, NF);
149 }
150 }
151
selectVLSEG(SDNode * Node,unsigned IntNo,bool IsStrided)152 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned IntNo,
153 bool IsStrided) {
154 SDLoc DL(Node);
155 unsigned NF = Node->getNumValues() - 1;
156 EVT VT = Node->getValueType(0);
157 unsigned ScalarSize = VT.getScalarSizeInBits();
158 MVT XLenVT = Subtarget->getXLenVT();
159 RISCVVLMUL LMUL = getLMUL(VT);
160 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
161 SmallVector<SDValue, 5> Operands;
162 Operands.push_back(Node->getOperand(2)); // Base pointer.
163 if (IsStrided) {
164 Operands.push_back(Node->getOperand(3)); // Stride.
165 Operands.push_back(Node->getOperand(4)); // VL.
166 } else {
167 Operands.push_back(Node->getOperand(3)); // VL.
168 }
169 Operands.push_back(SEW);
170 Operands.push_back(Node->getOperand(0)); // Chain.
171 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
172 IntNo, ScalarSize, static_cast<unsigned>(LMUL),
173 static_cast<unsigned>(RISCVVLMUL::LMUL_1));
174 SDNode *Load =
175 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
176 SDValue SuperReg = SDValue(Load, 0);
177 for (unsigned I = 0; I < NF; ++I)
178 ReplaceUses(SDValue(Node, I),
179 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
180 VT, SuperReg));
181
182 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
183 CurDAG->RemoveDeadNode(Node);
184 }
185
selectVLSEGMask(SDNode * Node,unsigned IntNo,bool IsStrided)186 void RISCVDAGToDAGISel::selectVLSEGMask(SDNode *Node, unsigned IntNo,
187 bool IsStrided) {
188 SDLoc DL(Node);
189 unsigned NF = Node->getNumValues() - 1;
190 EVT VT = Node->getValueType(0);
191 unsigned ScalarSize = VT.getScalarSizeInBits();
192 MVT XLenVT = Subtarget->getXLenVT();
193 RISCVVLMUL LMUL = getLMUL(VT);
194 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
195 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
196 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
197 SmallVector<SDValue, 7> Operands;
198 Operands.push_back(MaskedOff);
199 Operands.push_back(Node->getOperand(NF + 2)); // Base pointer.
200 if (IsStrided) {
201 Operands.push_back(Node->getOperand(NF + 3)); // Stride.
202 Operands.push_back(Node->getOperand(NF + 4)); // Mask.
203 Operands.push_back(Node->getOperand(NF + 5)); // VL.
204 } else {
205 Operands.push_back(Node->getOperand(NF + 3)); // Mask.
206 Operands.push_back(Node->getOperand(NF + 4)); // VL.
207 }
208 Operands.push_back(SEW);
209 Operands.push_back(Node->getOperand(0)); /// Chain.
210 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
211 IntNo, ScalarSize, static_cast<unsigned>(LMUL),
212 static_cast<unsigned>(RISCVVLMUL::LMUL_1));
213 SDNode *Load =
214 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
215 SDValue SuperReg = SDValue(Load, 0);
216 for (unsigned I = 0; I < NF; ++I)
217 ReplaceUses(SDValue(Node, I),
218 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
219 VT, SuperReg));
220
221 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
222 CurDAG->RemoveDeadNode(Node);
223 }
224
selectVLSEGFF(SDNode * Node)225 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node) {
226 SDLoc DL(Node);
227 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
228 unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
229 EVT VT = Node->getValueType(0);
230 unsigned ScalarSize = VT.getScalarSizeInBits();
231 MVT XLenVT = Subtarget->getXLenVT();
232 RISCVVLMUL LMUL = getLMUL(VT);
233 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
234 SmallVector<SDValue, 5> Operands;
235 Operands.push_back(Node->getOperand(2)); // Base pointer.
236 Operands.push_back(Node->getOperand(3)); // VL.
237 Operands.push_back(SEW);
238 Operands.push_back(Node->getOperand(0)); // Chain.
239 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
240 IntNo, ScalarSize, static_cast<unsigned>(LMUL),
241 static_cast<unsigned>(RISCVVLMUL::LMUL_1));
242 SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
243 MVT::Glue, Operands);
244 SDValue SuperReg = SDValue(Load, 0);
245 for (unsigned I = 0; I < NF; ++I)
246 ReplaceUses(SDValue(Node, I),
247 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
248 VT, SuperReg));
249
250 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // Chain.
251 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
252 CurDAG->RemoveDeadNode(Node);
253 }
254
selectVLSEGFFMask(SDNode * Node)255 void RISCVDAGToDAGISel::selectVLSEGFFMask(SDNode *Node) {
256 SDLoc DL(Node);
257 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
258 unsigned NF = Node->getNumValues() - 2; // Do not count Chain and Glue.
259 EVT VT = Node->getValueType(0);
260 unsigned ScalarSize = VT.getScalarSizeInBits();
261 MVT XLenVT = Subtarget->getXLenVT();
262 RISCVVLMUL LMUL = getLMUL(VT);
263 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
264 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
265 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
266 SmallVector<SDValue, 7> Operands;
267 Operands.push_back(MaskedOff);
268 Operands.push_back(Node->getOperand(NF + 2)); // Base pointer.
269 Operands.push_back(Node->getOperand(NF + 3)); // Mask.
270 Operands.push_back(Node->getOperand(NF + 4)); // VL.
271 Operands.push_back(SEW);
272 Operands.push_back(Node->getOperand(0)); /// Chain.
273 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
274 IntNo, ScalarSize, static_cast<unsigned>(LMUL),
275 static_cast<unsigned>(RISCVVLMUL::LMUL_1));
276 SDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other,
277 MVT::Glue, Operands);
278 SDValue SuperReg = SDValue(Load, 0);
279 for (unsigned I = 0; I < NF; ++I)
280 ReplaceUses(SDValue(Node, I),
281 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
282 VT, SuperReg));
283
284 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // Chain.
285 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Glue.
286 CurDAG->RemoveDeadNode(Node);
287 }
288
selectVLXSEG(SDNode * Node,unsigned IntNo)289 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned IntNo) {
290 SDLoc DL(Node);
291 unsigned NF = Node->getNumValues() - 1;
292 EVT VT = Node->getValueType(0);
293 unsigned ScalarSize = VT.getScalarSizeInBits();
294 MVT XLenVT = Subtarget->getXLenVT();
295 RISCVVLMUL LMUL = getLMUL(VT);
296 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
297 SDValue Operands[] = {
298 Node->getOperand(2), // Base pointer.
299 Node->getOperand(3), // Index.
300 Node->getOperand(4), // VL.
301 SEW, Node->getOperand(0) // Chain.
302 };
303
304 EVT IndexVT = Node->getOperand(3)->getValueType(0);
305 RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
306 unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
307 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
308 IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
309 static_cast<unsigned>(IndexLMUL));
310 SDNode *Load =
311 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
312 SDValue SuperReg = SDValue(Load, 0);
313 for (unsigned I = 0; I < NF; ++I)
314 ReplaceUses(SDValue(Node, I),
315 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
316 VT, SuperReg));
317
318 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
319 CurDAG->RemoveDeadNode(Node);
320 }
321
selectVLXSEGMask(SDNode * Node,unsigned IntNo)322 void RISCVDAGToDAGISel::selectVLXSEGMask(SDNode *Node, unsigned IntNo) {
323 SDLoc DL(Node);
324 unsigned NF = Node->getNumValues() - 1;
325 EVT VT = Node->getValueType(0);
326 unsigned ScalarSize = VT.getScalarSizeInBits();
327 MVT XLenVT = Subtarget->getXLenVT();
328 RISCVVLMUL LMUL = getLMUL(VT);
329 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
330 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
331 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
332 SDValue Operands[] = {
333 MaskedOff,
334 Node->getOperand(NF + 2), // Base pointer.
335 Node->getOperand(NF + 3), // Index.
336 Node->getOperand(NF + 4), // Mask.
337 Node->getOperand(NF + 5), // VL.
338 SEW,
339 Node->getOperand(0) // Chain.
340 };
341
342 EVT IndexVT = Node->getOperand(NF + 3)->getValueType(0);
343 RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
344 unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
345 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
346 IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
347 static_cast<unsigned>(IndexLMUL));
348 SDNode *Load =
349 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
350 SDValue SuperReg = SDValue(Load, 0);
351 for (unsigned I = 0; I < NF; ++I)
352 ReplaceUses(SDValue(Node, I),
353 CurDAG->getTargetExtractSubreg(getSubregIndexByEVT(VT, I), DL,
354 VT, SuperReg));
355
356 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
357 CurDAG->RemoveDeadNode(Node);
358 }
359
selectVSSEG(SDNode * Node,unsigned IntNo,bool IsStrided)360 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned IntNo,
361 bool IsStrided) {
362 SDLoc DL(Node);
363 unsigned NF = Node->getNumOperands() - 4;
364 if (IsStrided)
365 NF--;
366 EVT VT = Node->getOperand(2)->getValueType(0);
367 unsigned ScalarSize = VT.getScalarSizeInBits();
368 MVT XLenVT = Subtarget->getXLenVT();
369 RISCVVLMUL LMUL = getLMUL(VT);
370 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
371 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
372 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
373 SmallVector<SDValue, 6> Operands;
374 Operands.push_back(StoreVal);
375 Operands.push_back(Node->getOperand(2 + NF)); // Base pointer.
376 if (IsStrided) {
377 Operands.push_back(Node->getOperand(3 + NF)); // Stride.
378 Operands.push_back(Node->getOperand(4 + NF)); // VL.
379 } else {
380 Operands.push_back(Node->getOperand(3 + NF)); // VL.
381 }
382 Operands.push_back(SEW);
383 Operands.push_back(Node->getOperand(0)); // Chain.
384 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
385 IntNo, ScalarSize, static_cast<unsigned>(LMUL),
386 static_cast<unsigned>(RISCVVLMUL::LMUL_1));
387 SDNode *Store =
388 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
389 ReplaceNode(Node, Store);
390 }
391
selectVSSEGMask(SDNode * Node,unsigned IntNo,bool IsStrided)392 void RISCVDAGToDAGISel::selectVSSEGMask(SDNode *Node, unsigned IntNo,
393 bool IsStrided) {
394 SDLoc DL(Node);
395 unsigned NF = Node->getNumOperands() - 5;
396 if (IsStrided)
397 NF--;
398 EVT VT = Node->getOperand(2)->getValueType(0);
399 unsigned ScalarSize = VT.getScalarSizeInBits();
400 MVT XLenVT = Subtarget->getXLenVT();
401 RISCVVLMUL LMUL = getLMUL(VT);
402 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
403 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
404 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
405 SmallVector<SDValue, 7> Operands;
406 Operands.push_back(StoreVal);
407 Operands.push_back(Node->getOperand(2 + NF)); // Base pointer.
408 if (IsStrided) {
409 Operands.push_back(Node->getOperand(3 + NF)); // Stride.
410 Operands.push_back(Node->getOperand(4 + NF)); // Mask.
411 Operands.push_back(Node->getOperand(5 + NF)); // VL.
412 } else {
413 Operands.push_back(Node->getOperand(3 + NF)); // Mask.
414 Operands.push_back(Node->getOperand(4 + NF)); // VL.
415 }
416 Operands.push_back(SEW);
417 Operands.push_back(Node->getOperand(0)); // Chain.
418 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
419 IntNo, ScalarSize, static_cast<unsigned>(LMUL),
420 static_cast<unsigned>(RISCVVLMUL::LMUL_1));
421 SDNode *Store =
422 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
423 ReplaceNode(Node, Store);
424 }
425
selectVSXSEG(SDNode * Node,unsigned IntNo)426 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned IntNo) {
427 SDLoc DL(Node);
428 unsigned NF = Node->getNumOperands() - 5;
429 EVT VT = Node->getOperand(2)->getValueType(0);
430 unsigned ScalarSize = VT.getScalarSizeInBits();
431 MVT XLenVT = Subtarget->getXLenVT();
432 RISCVVLMUL LMUL = getLMUL(VT);
433 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
434 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
435 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
436 SDValue Operands[] = {
437 StoreVal,
438 Node->getOperand(2 + NF), // Base pointer.
439 Node->getOperand(3 + NF), // Index.
440 Node->getOperand(4 + NF), // VL.
441 SEW,
442 Node->getOperand(0) // Chain.
443 };
444
445 EVT IndexVT = Node->getOperand(3 + NF)->getValueType(0);
446 RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
447 unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
448 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
449 IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
450 static_cast<unsigned>(IndexLMUL));
451 SDNode *Store =
452 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
453 ReplaceNode(Node, Store);
454 }
455
selectVSXSEGMask(SDNode * Node,unsigned IntNo)456 void RISCVDAGToDAGISel::selectVSXSEGMask(SDNode *Node, unsigned IntNo) {
457 SDLoc DL(Node);
458 unsigned NF = Node->getNumOperands() - 6;
459 EVT VT = Node->getOperand(2)->getValueType(0);
460 unsigned ScalarSize = VT.getScalarSizeInBits();
461 MVT XLenVT = Subtarget->getXLenVT();
462 RISCVVLMUL LMUL = getLMUL(VT);
463 SDValue SEW = CurDAG->getTargetConstant(ScalarSize, DL, XLenVT);
464 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
465 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
466 SDValue Operands[] = {
467 StoreVal,
468 Node->getOperand(2 + NF), // Base pointer.
469 Node->getOperand(3 + NF), // Index.
470 Node->getOperand(4 + NF), // Mask.
471 Node->getOperand(5 + NF), // VL.
472 SEW,
473 Node->getOperand(0) // Chain.
474 };
475
476 EVT IndexVT = Node->getOperand(3 + NF)->getValueType(0);
477 RISCVVLMUL IndexLMUL = getLMUL(IndexVT);
478 unsigned IndexScalarSize = IndexVT.getScalarSizeInBits();
479 const RISCVZvlssegTable::RISCVZvlsseg *P = RISCVZvlssegTable::getPseudo(
480 IntNo, IndexScalarSize, static_cast<unsigned>(LMUL),
481 static_cast<unsigned>(IndexLMUL));
482 SDNode *Store =
483 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
484 ReplaceNode(Node, Store);
485 }
486
Select(SDNode * Node)487 void RISCVDAGToDAGISel::Select(SDNode *Node) {
488 // If we have a custom node, we have already selected.
489 if (Node->isMachineOpcode()) {
490 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
491 Node->setNodeId(-1);
492 return;
493 }
494
495 // Instruction Selection not handled by the auto-generated tablegen selection
496 // should be handled here.
497 unsigned Opcode = Node->getOpcode();
498 MVT XLenVT = Subtarget->getXLenVT();
499 SDLoc DL(Node);
500 EVT VT = Node->getValueType(0);
501
502 switch (Opcode) {
503 case ISD::ADD: {
504 // Optimize (add r, imm) to (addi (addi r, imm0) imm1) if applicable. The
505 // immediate must be in specific ranges and have a single use.
506 if (auto *ConstOp = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
507 if (!(ConstOp->hasOneUse()))
508 break;
509 // The imm must be in range [-4096,-2049] or [2048,4094].
510 int64_t Imm = ConstOp->getSExtValue();
511 if (!(-4096 <= Imm && Imm <= -2049) && !(2048 <= Imm && Imm <= 4094))
512 break;
513 // Break the imm to imm0+imm1.
514 EVT VT = Node->getValueType(0);
515 const SDValue ImmOp0 = CurDAG->getTargetConstant(Imm - Imm / 2, DL, VT);
516 const SDValue ImmOp1 = CurDAG->getTargetConstant(Imm / 2, DL, VT);
517 auto *NodeAddi0 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
518 Node->getOperand(0), ImmOp0);
519 auto *NodeAddi1 = CurDAG->getMachineNode(RISCV::ADDI, DL, VT,
520 SDValue(NodeAddi0, 0), ImmOp1);
521 ReplaceNode(Node, NodeAddi1);
522 return;
523 }
524 break;
525 }
526 case ISD::Constant: {
527 auto ConstNode = cast<ConstantSDNode>(Node);
528 if (VT == XLenVT && ConstNode->isNullValue()) {
529 SDValue New =
530 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
531 ReplaceNode(Node, New.getNode());
532 return;
533 }
534 int64_t Imm = ConstNode->getSExtValue();
535 if (XLenVT == MVT::i64) {
536 ReplaceNode(Node, selectImm(CurDAG, DL, Imm, XLenVT));
537 return;
538 }
539 break;
540 }
541 case ISD::FrameIndex: {
542 SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
543 int FI = cast<FrameIndexSDNode>(Node)->getIndex();
544 SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
545 ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
546 return;
547 }
548 case ISD::INTRINSIC_W_CHAIN: {
549 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
550 switch (IntNo) {
551 // By default we do not custom select any intrinsic.
552 default:
553 break;
554
555 case Intrinsic::riscv_vsetvli: {
556 if (!Subtarget->hasStdExtV())
557 break;
558
559 assert(Node->getNumOperands() == 5);
560
561 RISCVVSEW VSEW =
562 static_cast<RISCVVSEW>(Node->getConstantOperandVal(3) & 0x7);
563 RISCVVLMUL VLMul =
564 static_cast<RISCVVLMUL>(Node->getConstantOperandVal(4) & 0x7);
565
566 unsigned VTypeI = RISCVVType::encodeVTYPE(
567 VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
568 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
569
570 SDValue VLOperand = Node->getOperand(2);
571 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
572 uint64_t AVL = C->getZExtValue();
573 if (isUInt<5>(AVL)) {
574 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
575 ReplaceNode(Node,
576 CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
577 MVT::Other, VLImm, VTypeIOp,
578 /* Chain */ Node->getOperand(0)));
579 return;
580 }
581 }
582
583 ReplaceNode(Node,
584 CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
585 MVT::Other, VLOperand, VTypeIOp,
586 /* Chain */ Node->getOperand(0)));
587 return;
588 }
589 case Intrinsic::riscv_vsetvlimax: {
590 if (!Subtarget->hasStdExtV())
591 break;
592
593 assert(Node->getNumOperands() == 4);
594
595 RISCVVSEW VSEW =
596 static_cast<RISCVVSEW>(Node->getConstantOperandVal(2) & 0x7);
597 RISCVVLMUL VLMul =
598 static_cast<RISCVVLMUL>(Node->getConstantOperandVal(3) & 0x7);
599
600 unsigned VTypeI = RISCVVType::encodeVTYPE(
601 VLMul, VSEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
602 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
603
604 SDValue VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
605 ReplaceNode(Node,
606 CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
607 MVT::Other, VLOperand, VTypeIOp,
608 /* Chain */ Node->getOperand(0)));
609 return;
610 }
611 case Intrinsic::riscv_vlseg2:
612 case Intrinsic::riscv_vlseg3:
613 case Intrinsic::riscv_vlseg4:
614 case Intrinsic::riscv_vlseg5:
615 case Intrinsic::riscv_vlseg6:
616 case Intrinsic::riscv_vlseg7:
617 case Intrinsic::riscv_vlseg8: {
618 selectVLSEG(Node, IntNo, /*IsStrided=*/false);
619 return;
620 }
621 case Intrinsic::riscv_vlseg2_mask:
622 case Intrinsic::riscv_vlseg3_mask:
623 case Intrinsic::riscv_vlseg4_mask:
624 case Intrinsic::riscv_vlseg5_mask:
625 case Intrinsic::riscv_vlseg6_mask:
626 case Intrinsic::riscv_vlseg7_mask:
627 case Intrinsic::riscv_vlseg8_mask: {
628 selectVLSEGMask(Node, IntNo, /*IsStrided=*/false);
629 return;
630 }
631 case Intrinsic::riscv_vlsseg2:
632 case Intrinsic::riscv_vlsseg3:
633 case Intrinsic::riscv_vlsseg4:
634 case Intrinsic::riscv_vlsseg5:
635 case Intrinsic::riscv_vlsseg6:
636 case Intrinsic::riscv_vlsseg7:
637 case Intrinsic::riscv_vlsseg8: {
638 selectVLSEG(Node, IntNo, /*IsStrided=*/true);
639 return;
640 }
641 case Intrinsic::riscv_vlsseg2_mask:
642 case Intrinsic::riscv_vlsseg3_mask:
643 case Intrinsic::riscv_vlsseg4_mask:
644 case Intrinsic::riscv_vlsseg5_mask:
645 case Intrinsic::riscv_vlsseg6_mask:
646 case Intrinsic::riscv_vlsseg7_mask:
647 case Intrinsic::riscv_vlsseg8_mask: {
648 selectVLSEGMask(Node, IntNo, /*IsStrided=*/true);
649 return;
650 }
651 case Intrinsic::riscv_vloxseg2:
652 case Intrinsic::riscv_vloxseg3:
653 case Intrinsic::riscv_vloxseg4:
654 case Intrinsic::riscv_vloxseg5:
655 case Intrinsic::riscv_vloxseg6:
656 case Intrinsic::riscv_vloxseg7:
657 case Intrinsic::riscv_vloxseg8:
658 case Intrinsic::riscv_vluxseg2:
659 case Intrinsic::riscv_vluxseg3:
660 case Intrinsic::riscv_vluxseg4:
661 case Intrinsic::riscv_vluxseg5:
662 case Intrinsic::riscv_vluxseg6:
663 case Intrinsic::riscv_vluxseg7:
664 case Intrinsic::riscv_vluxseg8: {
665 selectVLXSEG(Node, IntNo);
666 return;
667 }
668 case Intrinsic::riscv_vloxseg2_mask:
669 case Intrinsic::riscv_vloxseg3_mask:
670 case Intrinsic::riscv_vloxseg4_mask:
671 case Intrinsic::riscv_vloxseg5_mask:
672 case Intrinsic::riscv_vloxseg6_mask:
673 case Intrinsic::riscv_vloxseg7_mask:
674 case Intrinsic::riscv_vloxseg8_mask:
675 case Intrinsic::riscv_vluxseg2_mask:
676 case Intrinsic::riscv_vluxseg3_mask:
677 case Intrinsic::riscv_vluxseg4_mask:
678 case Intrinsic::riscv_vluxseg5_mask:
679 case Intrinsic::riscv_vluxseg6_mask:
680 case Intrinsic::riscv_vluxseg7_mask:
681 case Intrinsic::riscv_vluxseg8_mask: {
682 selectVLXSEGMask(Node, IntNo);
683 return;
684 }
685 }
686 break;
687 }
688 case ISD::INTRINSIC_VOID: {
689 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
690 switch (IntNo) {
691 case Intrinsic::riscv_vsseg2:
692 case Intrinsic::riscv_vsseg3:
693 case Intrinsic::riscv_vsseg4:
694 case Intrinsic::riscv_vsseg5:
695 case Intrinsic::riscv_vsseg6:
696 case Intrinsic::riscv_vsseg7:
697 case Intrinsic::riscv_vsseg8: {
698 selectVSSEG(Node, IntNo, /*IsStrided=*/false);
699 return;
700 }
701 case Intrinsic::riscv_vsseg2_mask:
702 case Intrinsic::riscv_vsseg3_mask:
703 case Intrinsic::riscv_vsseg4_mask:
704 case Intrinsic::riscv_vsseg5_mask:
705 case Intrinsic::riscv_vsseg6_mask:
706 case Intrinsic::riscv_vsseg7_mask:
707 case Intrinsic::riscv_vsseg8_mask: {
708 selectVSSEGMask(Node, IntNo, /*IsStrided=*/false);
709 return;
710 }
711 case Intrinsic::riscv_vssseg2:
712 case Intrinsic::riscv_vssseg3:
713 case Intrinsic::riscv_vssseg4:
714 case Intrinsic::riscv_vssseg5:
715 case Intrinsic::riscv_vssseg6:
716 case Intrinsic::riscv_vssseg7:
717 case Intrinsic::riscv_vssseg8: {
718 selectVSSEG(Node, IntNo, /*IsStrided=*/true);
719 return;
720 }
721 case Intrinsic::riscv_vssseg2_mask:
722 case Intrinsic::riscv_vssseg3_mask:
723 case Intrinsic::riscv_vssseg4_mask:
724 case Intrinsic::riscv_vssseg5_mask:
725 case Intrinsic::riscv_vssseg6_mask:
726 case Intrinsic::riscv_vssseg7_mask:
727 case Intrinsic::riscv_vssseg8_mask: {
728 selectVSSEGMask(Node, IntNo, /*IsStrided=*/true);
729 return;
730 }
731 case Intrinsic::riscv_vsoxseg2:
732 case Intrinsic::riscv_vsoxseg3:
733 case Intrinsic::riscv_vsoxseg4:
734 case Intrinsic::riscv_vsoxseg5:
735 case Intrinsic::riscv_vsoxseg6:
736 case Intrinsic::riscv_vsoxseg7:
737 case Intrinsic::riscv_vsoxseg8:
738 case Intrinsic::riscv_vsuxseg2:
739 case Intrinsic::riscv_vsuxseg3:
740 case Intrinsic::riscv_vsuxseg4:
741 case Intrinsic::riscv_vsuxseg5:
742 case Intrinsic::riscv_vsuxseg6:
743 case Intrinsic::riscv_vsuxseg7:
744 case Intrinsic::riscv_vsuxseg8: {
745 selectVSXSEG(Node, IntNo);
746 return;
747 }
748 case Intrinsic::riscv_vsoxseg2_mask:
749 case Intrinsic::riscv_vsoxseg3_mask:
750 case Intrinsic::riscv_vsoxseg4_mask:
751 case Intrinsic::riscv_vsoxseg5_mask:
752 case Intrinsic::riscv_vsoxseg6_mask:
753 case Intrinsic::riscv_vsoxseg7_mask:
754 case Intrinsic::riscv_vsoxseg8_mask:
755 case Intrinsic::riscv_vsuxseg2_mask:
756 case Intrinsic::riscv_vsuxseg3_mask:
757 case Intrinsic::riscv_vsuxseg4_mask:
758 case Intrinsic::riscv_vsuxseg5_mask:
759 case Intrinsic::riscv_vsuxseg6_mask:
760 case Intrinsic::riscv_vsuxseg7_mask:
761 case Intrinsic::riscv_vsuxseg8_mask: {
762 selectVSXSEGMask(Node, IntNo);
763 return;
764 }
765 }
766 break;
767 }
768 case RISCVISD::VLSEGFF: {
769 selectVLSEGFF(Node);
770 return;
771 }
772 case RISCVISD::VLSEGFF_MASK: {
773 selectVLSEGFFMask(Node);
774 return;
775 }
776 }
777
778 // Select the default instruction.
779 SelectCode(Node);
780 }
781
SelectInlineAsmMemoryOperand(const SDValue & Op,unsigned ConstraintID,std::vector<SDValue> & OutOps)782 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
783 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
784 switch (ConstraintID) {
785 case InlineAsm::Constraint_m:
786 // We just support simple memory operands that have a single address
787 // operand and need no special handling.
788 OutOps.push_back(Op);
789 return false;
790 case InlineAsm::Constraint_A:
791 OutOps.push_back(Op);
792 return false;
793 default:
794 break;
795 }
796
797 return true;
798 }
799
SelectAddrFI(SDValue Addr,SDValue & Base)800 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
801 if (auto FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
802 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
803 return true;
804 }
805 return false;
806 }
807
808 // Match (srl (and val, mask), imm) where the result would be a
809 // zero-extended 32-bit integer. i.e. the mask is 0xffffffff or the result
810 // is equivalent to this (SimplifyDemandedBits may have removed lower bits
811 // from the mask that aren't necessary due to the right-shifting).
MatchSRLIW(SDNode * N) const812 bool RISCVDAGToDAGISel::MatchSRLIW(SDNode *N) const {
813 assert(N->getOpcode() == ISD::SRL);
814 assert(N->getOperand(0).getOpcode() == ISD::AND);
815 assert(isa<ConstantSDNode>(N->getOperand(1)));
816 assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
817
818 // The IsRV64 predicate is checked after PatFrag predicates so we can get
819 // here even on RV32.
820 if (!Subtarget->is64Bit())
821 return false;
822
823 SDValue And = N->getOperand(0);
824 uint64_t ShAmt = N->getConstantOperandVal(1);
825 uint64_t Mask = And.getConstantOperandVal(1);
826 return (Mask | maskTrailingOnes<uint64_t>(ShAmt)) == 0xffffffff;
827 }
828
829 // Check that it is a SLLIUW (Shift Logical Left Immediate Unsigned i32
830 // on RV64).
831 // SLLIUW is the same as SLLI except for the fact that it clears the bits
832 // XLEN-1:32 of the input RS1 before shifting.
833 // A PatFrag has already checked that it has the right structure:
834 //
835 // (AND (SHL RS1, VC2), VC1)
836 //
837 // We check that VC2, the shamt is less than 32, otherwise the pattern is
838 // exactly the same as SLLI and we give priority to that.
839 // Eventually we check that VC1, the mask used to clear the upper 32 bits
840 // of RS1, is correct:
841 //
842 // VC1 == (0xFFFFFFFF << VC2)
843 //
MatchSLLIUW(SDNode * N) const844 bool RISCVDAGToDAGISel::MatchSLLIUW(SDNode *N) const {
845 assert(N->getOpcode() == ISD::AND);
846 assert(N->getOperand(0).getOpcode() == ISD::SHL);
847 assert(isa<ConstantSDNode>(N->getOperand(1)));
848 assert(isa<ConstantSDNode>(N->getOperand(0).getOperand(1)));
849
850 // The IsRV64 predicate is checked after PatFrag predicates so we can get
851 // here even on RV32.
852 if (!Subtarget->is64Bit())
853 return false;
854
855 SDValue Shl = N->getOperand(0);
856 uint64_t VC1 = N->getConstantOperandVal(1);
857 uint64_t VC2 = Shl.getConstantOperandVal(1);
858
859 // Immediate range should be enforced by uimm5 predicate.
860 assert(VC2 < 32 && "Unexpected immediate");
861 return (VC1 >> VC2) == UINT64_C(0xFFFFFFFF);
862 }
863
864 // X0 has special meaning for vsetvl/vsetvli.
865 // rd | rs1 | AVL value | Effect on vl
866 //--------------------------------------------------------------
867 // !X0 | X0 | VLMAX | Set vl to VLMAX
868 // X0 | X0 | Value in vl | Keep current vl, just change vtype.
selectVLOp(SDValue N,SDValue & VL)869 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
870 // If the VL value is a constant 0, manually select it to an ADDI with 0
871 // immediate to prevent the default selection path from matching it to X0.
872 auto *C = dyn_cast<ConstantSDNode>(N);
873 if (C && C->isNullValue())
874 VL = SDValue(selectImm(CurDAG, SDLoc(N), 0, Subtarget->getXLenVT()), 0);
875 else
876 VL = N;
877
878 return true;
879 }
880
selectVSplat(SDValue N,SDValue & SplatVal)881 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
882 if (N.getOpcode() != ISD::SPLAT_VECTOR &&
883 N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64)
884 return false;
885 SplatVal = N.getOperand(0);
886 return true;
887 }
888
selectVSplatSimm5(SDValue N,SDValue & SplatVal)889 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
890 if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
891 N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64) ||
892 !isa<ConstantSDNode>(N.getOperand(0)))
893 return false;
894
895 int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
896
897 // Both ISD::SPLAT_VECTOR and RISCVISD::SPLAT_VECTOR_I64 share semantics when
898 // the operand type is wider than the resulting vector element type: an
899 // implicit truncation first takes place. Therefore, perform a manual
900 // truncation/sign-extension in order to ignore any truncated bits and catch
901 // any zero-extended immediate.
902 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
903 // sign-extending to (XLenVT -1).
904 auto XLenVT = Subtarget->getXLenVT();
905 assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
906 "Unexpected splat operand type");
907 auto EltVT = N.getValueType().getVectorElementType();
908 if (EltVT.bitsLT(XLenVT)) {
909 SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
910 }
911
912 if (!isInt<5>(SplatImm))
913 return false;
914
915 SplatVal = CurDAG->getTargetConstant(SplatImm, SDLoc(N), XLenVT);
916 return true;
917 }
918
selectVSplatUimm5(SDValue N,SDValue & SplatVal)919 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
920 if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
921 N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64) ||
922 !isa<ConstantSDNode>(N.getOperand(0)))
923 return false;
924
925 int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
926
927 if (!isUInt<5>(SplatImm))
928 return false;
929
930 SplatVal =
931 CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
932
933 return true;
934 }
935
936 // Merge an ADDI into the offset of a load/store instruction where possible.
937 // (load (addi base, off1), off2) -> (load base, off1+off2)
938 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
939 // This is possible when off1+off2 fits a 12-bit immediate.
doPeepholeLoadStoreADDI()940 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
941 SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
942 ++Position;
943
944 while (Position != CurDAG->allnodes_begin()) {
945 SDNode *N = &*--Position;
946 // Skip dead nodes and any non-machine opcodes.
947 if (N->use_empty() || !N->isMachineOpcode())
948 continue;
949
950 int OffsetOpIdx;
951 int BaseOpIdx;
952
953 // Only attempt this optimisation for I-type loads and S-type stores.
954 switch (N->getMachineOpcode()) {
955 default:
956 continue;
957 case RISCV::LB:
958 case RISCV::LH:
959 case RISCV::LW:
960 case RISCV::LBU:
961 case RISCV::LHU:
962 case RISCV::LWU:
963 case RISCV::LD:
964 case RISCV::FLH:
965 case RISCV::FLW:
966 case RISCV::FLD:
967 BaseOpIdx = 0;
968 OffsetOpIdx = 1;
969 break;
970 case RISCV::SB:
971 case RISCV::SH:
972 case RISCV::SW:
973 case RISCV::SD:
974 case RISCV::FSH:
975 case RISCV::FSW:
976 case RISCV::FSD:
977 BaseOpIdx = 1;
978 OffsetOpIdx = 2;
979 break;
980 }
981
982 if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
983 continue;
984
985 SDValue Base = N->getOperand(BaseOpIdx);
986
987 // If the base is an ADDI, we can merge it in to the load/store.
988 if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
989 continue;
990
991 SDValue ImmOperand = Base.getOperand(1);
992 uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
993
994 if (auto Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
995 int64_t Offset1 = Const->getSExtValue();
996 int64_t CombinedOffset = Offset1 + Offset2;
997 if (!isInt<12>(CombinedOffset))
998 continue;
999 ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1000 ImmOperand.getValueType());
1001 } else if (auto GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1002 // If the off1 in (addi base, off1) is a global variable's address (its
1003 // low part, really), then we can rely on the alignment of that variable
1004 // to provide a margin of safety before off1 can overflow the 12 bits.
1005 // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1006 const DataLayout &DL = CurDAG->getDataLayout();
1007 Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1008 if (Offset2 != 0 && Alignment <= Offset2)
1009 continue;
1010 int64_t Offset1 = GA->getOffset();
1011 int64_t CombinedOffset = Offset1 + Offset2;
1012 ImmOperand = CurDAG->getTargetGlobalAddress(
1013 GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1014 CombinedOffset, GA->getTargetFlags());
1015 } else if (auto CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1016 // Ditto.
1017 Align Alignment = CP->getAlign();
1018 if (Offset2 != 0 && Alignment <= Offset2)
1019 continue;
1020 int64_t Offset1 = CP->getOffset();
1021 int64_t CombinedOffset = Offset1 + Offset2;
1022 ImmOperand = CurDAG->getTargetConstantPool(
1023 CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1024 CombinedOffset, CP->getTargetFlags());
1025 } else {
1026 continue;
1027 }
1028
1029 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: ");
1030 LLVM_DEBUG(Base->dump(CurDAG));
1031 LLVM_DEBUG(dbgs() << "\nN: ");
1032 LLVM_DEBUG(N->dump(CurDAG));
1033 LLVM_DEBUG(dbgs() << "\n");
1034
1035 // Modify the offset operand of the load/store.
1036 if (BaseOpIdx == 0) // Load
1037 CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1038 N->getOperand(2));
1039 else // Store
1040 CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1041 ImmOperand, N->getOperand(3));
1042
1043 // The add-immediate may now be dead, in which case remove it.
1044 if (Base.getNode()->use_empty())
1045 CurDAG->RemoveDeadNode(Base.getNode());
1046 }
1047 }
1048
1049 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1050 // for instruction scheduling.
createRISCVISelDag(RISCVTargetMachine & TM)1051 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1052 return new RISCVDAGToDAGISel(TM);
1053 }
1054