1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "RISCVISelDAGToDAG.h"
14 #include "MCTargetDesc/RISCVMCTargetDesc.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCVISelLowering.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "llvm/CodeGen/MachineFrameInfo.h"
19 #include "llvm/IR/IntrinsicsRISCV.h"
20 #include "llvm/Support/Alignment.h"
21 #include "llvm/Support/Debug.h"
22 #include "llvm/Support/KnownBits.h"
23 #include "llvm/Support/MathExtras.h"
24 #include "llvm/Support/raw_ostream.h"
25
26 using namespace llvm;
27
28 #define DEBUG_TYPE "riscv-isel"
29
30 namespace llvm {
31 namespace RISCV {
32 #define GET_RISCVVSSEGTable_IMPL
33 #define GET_RISCVVLSEGTable_IMPL
34 #define GET_RISCVVLXSEGTable_IMPL
35 #define GET_RISCVVSXSEGTable_IMPL
36 #define GET_RISCVVLETable_IMPL
37 #define GET_RISCVVSETable_IMPL
38 #define GET_RISCVVLXTable_IMPL
39 #define GET_RISCVVSXTable_IMPL
40 #include "RISCVGenSearchableTables.inc"
41 } // namespace RISCV
42 } // namespace llvm
43
PreprocessISelDAG()44 void RISCVDAGToDAGISel::PreprocessISelDAG() {
45 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
46 E = CurDAG->allnodes_end();
47 I != E;) {
48 SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
49
50 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
51 // load. Done after lowering and combining so that we have a chance to
52 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
53 if (N->getOpcode() != RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL)
54 continue;
55
56 assert(N->getNumOperands() == 3 && "Unexpected number of operands");
57 MVT VT = N->getSimpleValueType(0);
58 SDValue Lo = N->getOperand(0);
59 SDValue Hi = N->getOperand(1);
60 SDValue VL = N->getOperand(2);
61 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
62 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
63 "Unexpected VTs!");
64 MachineFunction &MF = CurDAG->getMachineFunction();
65 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
66 SDLoc DL(N);
67
68 // We use the same frame index we use for moving two i32s into 64-bit FPR.
69 // This is an analogous operation.
70 int FI = FuncInfo->getMoveF64FrameIndex(MF);
71 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
72 const TargetLowering &TLI = CurDAG->getTargetLoweringInfo();
73 SDValue StackSlot =
74 CurDAG->getFrameIndex(FI, TLI.getPointerTy(CurDAG->getDataLayout()));
75
76 SDValue Chain = CurDAG->getEntryNode();
77 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8));
78
79 SDValue OffsetSlot =
80 CurDAG->getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), DL);
81 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4),
82 Align(8));
83
84 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
85
86 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other});
87 SDValue IntID =
88 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64);
89 SDValue Ops[] = {Chain, IntID, StackSlot,
90 CurDAG->getRegister(RISCV::X0, MVT::i64), VL};
91
92 SDValue Result = CurDAG->getMemIntrinsicNode(
93 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MVT::i64, MPI, Align(8),
94 MachineMemOperand::MOLoad);
95
96 // We're about to replace all uses of the SPLAT_VECTOR_SPLIT_I64 with the
97 // vlse we created. This will cause general havok on the dag because
98 // anything below the conversion could be folded into other existing nodes.
99 // To avoid invalidating 'I', back it up to the convert node.
100 --I;
101 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
102
103 // Now that we did that, the node is dead. Increment the iterator to the
104 // next node to process, then delete N.
105 ++I;
106 CurDAG->DeleteNode(N);
107 }
108 }
109
PostprocessISelDAG()110 void RISCVDAGToDAGISel::PostprocessISelDAG() {
111 doPeepholeLoadStoreADDI();
112 }
113
selectImm(SelectionDAG * CurDAG,const SDLoc & DL,int64_t Imm,const RISCVSubtarget & Subtarget)114 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
115 const RISCVSubtarget &Subtarget) {
116 MVT XLenVT = Subtarget.getXLenVT();
117 RISCVMatInt::InstSeq Seq =
118 RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
119
120 SDNode *Result = nullptr;
121 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
122 for (RISCVMatInt::Inst &Inst : Seq) {
123 SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
124 if (Inst.Opc == RISCV::LUI)
125 Result = CurDAG->getMachineNode(RISCV::LUI, DL, XLenVT, SDImm);
126 else if (Inst.Opc == RISCV::ADDUW)
127 Result = CurDAG->getMachineNode(RISCV::ADDUW, DL, XLenVT, SrcReg,
128 CurDAG->getRegister(RISCV::X0, XLenVT));
129 else
130 Result = CurDAG->getMachineNode(Inst.Opc, DL, XLenVT, SrcReg, SDImm);
131
132 // Only the first instruction has X0 as its source.
133 SrcReg = SDValue(Result, 0);
134 }
135
136 return Result;
137 }
138
createTupleImpl(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned RegClassID,unsigned SubReg0)139 static SDValue createTupleImpl(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
140 unsigned RegClassID, unsigned SubReg0) {
141 assert(Regs.size() >= 2 && Regs.size() <= 8);
142
143 SDLoc DL(Regs[0]);
144 SmallVector<SDValue, 8> Ops;
145
146 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
147
148 for (unsigned I = 0; I < Regs.size(); ++I) {
149 Ops.push_back(Regs[I]);
150 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
151 }
152 SDNode *N =
153 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
154 return SDValue(N, 0);
155 }
156
createM1Tuple(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned NF)157 static SDValue createM1Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
158 unsigned NF) {
159 static const unsigned RegClassIDs[] = {
160 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
161 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
162 RISCV::VRN8M1RegClassID};
163
164 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm1_0);
165 }
166
createM2Tuple(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned NF)167 static SDValue createM2Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
168 unsigned NF) {
169 static const unsigned RegClassIDs[] = {RISCV::VRN2M2RegClassID,
170 RISCV::VRN3M2RegClassID,
171 RISCV::VRN4M2RegClassID};
172
173 return createTupleImpl(CurDAG, Regs, RegClassIDs[NF - 2], RISCV::sub_vrm2_0);
174 }
175
createM4Tuple(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned NF)176 static SDValue createM4Tuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
177 unsigned NF) {
178 return createTupleImpl(CurDAG, Regs, RISCV::VRN2M4RegClassID,
179 RISCV::sub_vrm4_0);
180 }
181
createTuple(SelectionDAG & CurDAG,ArrayRef<SDValue> Regs,unsigned NF,RISCVII::VLMUL LMUL)182 static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef<SDValue> Regs,
183 unsigned NF, RISCVII::VLMUL LMUL) {
184 switch (LMUL) {
185 default:
186 llvm_unreachable("Invalid LMUL.");
187 case RISCVII::VLMUL::LMUL_F8:
188 case RISCVII::VLMUL::LMUL_F4:
189 case RISCVII::VLMUL::LMUL_F2:
190 case RISCVII::VLMUL::LMUL_1:
191 return createM1Tuple(CurDAG, Regs, NF);
192 case RISCVII::VLMUL::LMUL_2:
193 return createM2Tuple(CurDAG, Regs, NF);
194 case RISCVII::VLMUL::LMUL_4:
195 return createM4Tuple(CurDAG, Regs, NF);
196 }
197 }
198
addVectorLoadStoreOperands(SDNode * Node,unsigned Log2SEW,const SDLoc & DL,unsigned CurOp,bool IsMasked,bool IsStridedOrIndexed,SmallVectorImpl<SDValue> & Operands,MVT * IndexVT)199 void RISCVDAGToDAGISel::addVectorLoadStoreOperands(
200 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
201 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
202 MVT *IndexVT) {
203 SDValue Chain = Node->getOperand(0);
204 SDValue Glue;
205
206 SDValue Base;
207 SelectBaseAddr(Node->getOperand(CurOp++), Base);
208 Operands.push_back(Base); // Base pointer.
209
210 if (IsStridedOrIndexed) {
211 Operands.push_back(Node->getOperand(CurOp++)); // Index.
212 if (IndexVT)
213 *IndexVT = Operands.back()->getSimpleValueType(0);
214 }
215
216 if (IsMasked) {
217 // Mask needs to be copied to V0.
218 SDValue Mask = Node->getOperand(CurOp++);
219 Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
220 Glue = Chain.getValue(1);
221 Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
222 }
223 SDValue VL;
224 selectVLOp(Node->getOperand(CurOp++), VL);
225 Operands.push_back(VL);
226
227 MVT XLenVT = Subtarget->getXLenVT();
228 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
229 Operands.push_back(SEWOp);
230
231 Operands.push_back(Chain); // Chain.
232 if (Glue)
233 Operands.push_back(Glue);
234 }
235
selectVLSEG(SDNode * Node,bool IsMasked,bool IsStrided)236 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
237 bool IsStrided) {
238 SDLoc DL(Node);
239 unsigned NF = Node->getNumValues() - 1;
240 MVT VT = Node->getSimpleValueType(0);
241 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
242 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
243
244 unsigned CurOp = 2;
245 SmallVector<SDValue, 8> Operands;
246 if (IsMasked) {
247 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
248 Node->op_begin() + CurOp + NF);
249 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
250 Operands.push_back(MaskedOff);
251 CurOp += NF;
252 }
253
254 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
255 Operands);
256
257 const RISCV::VLSEGPseudo *P =
258 RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, Log2SEW,
259 static_cast<unsigned>(LMUL));
260 MachineSDNode *Load =
261 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
262
263 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
264 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
265
266 SDValue SuperReg = SDValue(Load, 0);
267 for (unsigned I = 0; I < NF; ++I) {
268 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
269 ReplaceUses(SDValue(Node, I),
270 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
271 }
272
273 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
274 CurDAG->RemoveDeadNode(Node);
275 }
276
selectVLSEGFF(SDNode * Node,bool IsMasked)277 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
278 SDLoc DL(Node);
279 unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
280 MVT VT = Node->getSimpleValueType(0);
281 MVT XLenVT = Subtarget->getXLenVT();
282 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
283 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
284
285 unsigned CurOp = 2;
286 SmallVector<SDValue, 7> Operands;
287 if (IsMasked) {
288 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
289 Node->op_begin() + CurOp + NF);
290 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
291 Operands.push_back(MaskedOff);
292 CurOp += NF;
293 }
294
295 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
296 /*IsStridedOrIndexed*/ false, Operands);
297
298 const RISCV::VLSEGPseudo *P =
299 RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
300 Log2SEW, static_cast<unsigned>(LMUL));
301 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
302 MVT::Other, MVT::Glue, Operands);
303 SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
304 /*Glue*/ SDValue(Load, 2));
305
306 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
307 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
308
309 SDValue SuperReg = SDValue(Load, 0);
310 for (unsigned I = 0; I < NF; ++I) {
311 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
312 ReplaceUses(SDValue(Node, I),
313 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
314 }
315
316 ReplaceUses(SDValue(Node, NF), SDValue(ReadVL, 0)); // VL
317 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 1)); // Chain
318 CurDAG->RemoveDeadNode(Node);
319 }
320
selectVLXSEG(SDNode * Node,bool IsMasked,bool IsOrdered)321 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
322 bool IsOrdered) {
323 SDLoc DL(Node);
324 unsigned NF = Node->getNumValues() - 1;
325 MVT VT = Node->getSimpleValueType(0);
326 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
327 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
328
329 unsigned CurOp = 2;
330 SmallVector<SDValue, 8> Operands;
331 if (IsMasked) {
332 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
333 Node->op_begin() + CurOp + NF);
334 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
335 Operands.push_back(MaskedOff);
336 CurOp += NF;
337 }
338
339 MVT IndexVT;
340 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
341 /*IsStridedOrIndexed*/ true, Operands, &IndexVT);
342
343 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
344 "Element count mismatch");
345
346 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
347 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
348 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
349 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
350 static_cast<unsigned>(IndexLMUL));
351 MachineSDNode *Load =
352 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
353
354 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
355 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
356
357 SDValue SuperReg = SDValue(Load, 0);
358 for (unsigned I = 0; I < NF; ++I) {
359 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
360 ReplaceUses(SDValue(Node, I),
361 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
362 }
363
364 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
365 CurDAG->RemoveDeadNode(Node);
366 }
367
selectVSSEG(SDNode * Node,bool IsMasked,bool IsStrided)368 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
369 bool IsStrided) {
370 SDLoc DL(Node);
371 unsigned NF = Node->getNumOperands() - 4;
372 if (IsStrided)
373 NF--;
374 if (IsMasked)
375 NF--;
376 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
377 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
378 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
379 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
380 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
381
382 SmallVector<SDValue, 8> Operands;
383 Operands.push_back(StoreVal);
384 unsigned CurOp = 2 + NF;
385
386 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
387 Operands);
388
389 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
390 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
391 MachineSDNode *Store =
392 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
393
394 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
395 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
396
397 ReplaceNode(Node, Store);
398 }
399
selectVSXSEG(SDNode * Node,bool IsMasked,bool IsOrdered)400 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
401 bool IsOrdered) {
402 SDLoc DL(Node);
403 unsigned NF = Node->getNumOperands() - 5;
404 if (IsMasked)
405 --NF;
406 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
407 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
408 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
409 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
410 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
411
412 SmallVector<SDValue, 8> Operands;
413 Operands.push_back(StoreVal);
414 unsigned CurOp = 2 + NF;
415
416 MVT IndexVT;
417 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
418 /*IsStridedOrIndexed*/ true, Operands, &IndexVT);
419
420 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
421 "Element count mismatch");
422
423 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
424 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
425 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
426 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
427 static_cast<unsigned>(IndexLMUL));
428 MachineSDNode *Store =
429 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
430
431 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
432 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
433
434 ReplaceNode(Node, Store);
435 }
436
437
Select(SDNode * Node)438 void RISCVDAGToDAGISel::Select(SDNode *Node) {
439 // If we have a custom node, we have already selected.
440 if (Node->isMachineOpcode()) {
441 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
442 Node->setNodeId(-1);
443 return;
444 }
445
446 // Instruction Selection not handled by the auto-generated tablegen selection
447 // should be handled here.
448 unsigned Opcode = Node->getOpcode();
449 MVT XLenVT = Subtarget->getXLenVT();
450 SDLoc DL(Node);
451 MVT VT = Node->getSimpleValueType(0);
452
453 switch (Opcode) {
454 case ISD::Constant: {
455 auto *ConstNode = cast<ConstantSDNode>(Node);
456 if (VT == XLenVT && ConstNode->isNullValue()) {
457 SDValue New =
458 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
459 ReplaceNode(Node, New.getNode());
460 return;
461 }
462 ReplaceNode(Node,
463 selectImm(CurDAG, DL, ConstNode->getSExtValue(), *Subtarget));
464 return;
465 }
466 case ISD::FrameIndex: {
467 SDValue Imm = CurDAG->getTargetConstant(0, DL, XLenVT);
468 int FI = cast<FrameIndexSDNode>(Node)->getIndex();
469 SDValue TFI = CurDAG->getTargetFrameIndex(FI, VT);
470 ReplaceNode(Node, CurDAG->getMachineNode(RISCV::ADDI, DL, VT, TFI, Imm));
471 return;
472 }
473 case ISD::SRL: {
474 // We don't need this transform if zext.h is supported.
475 if (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp())
476 break;
477 // Optimize (srl (and X, 0xffff), C) ->
478 // (srli (slli X, (XLen-16), (XLen-16) + C)
479 // Taking into account that the 0xffff may have had lower bits unset by
480 // SimplifyDemandedBits. This avoids materializing the 0xffff immediate.
481 // This pattern occurs when type legalizing i16 right shifts.
482 // FIXME: This could be extended to other AND masks.
483 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
484 if (N1C) {
485 uint64_t ShAmt = N1C->getZExtValue();
486 SDValue N0 = Node->getOperand(0);
487 if (ShAmt < 16 && N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
488 isa<ConstantSDNode>(N0.getOperand(1))) {
489 uint64_t Mask = N0.getConstantOperandVal(1);
490 Mask |= maskTrailingOnes<uint64_t>(ShAmt);
491 if (Mask == 0xffff) {
492 unsigned LShAmt = Subtarget->getXLen() - 16;
493 SDNode *SLLI =
494 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
495 CurDAG->getTargetConstant(LShAmt, DL, VT));
496 SDNode *SRLI = CurDAG->getMachineNode(
497 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
498 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
499 ReplaceNode(Node, SRLI);
500 return;
501 }
502 }
503 }
504
505 break;
506 }
507 case ISD::AND: {
508 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
509 if (!N1C)
510 break;
511
512 SDValue N0 = Node->getOperand(0);
513
514 bool LeftShift = N0.getOpcode() == ISD::SHL;
515 if (!LeftShift && N0.getOpcode() != ISD::SRL)
516 break;
517
518 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
519 if (!C)
520 break;
521 uint64_t C2 = C->getZExtValue();
522 unsigned XLen = Subtarget->getXLen();
523 if (!C2 || C2 >= XLen)
524 break;
525
526 uint64_t C1 = N1C->getZExtValue();
527
528 // Keep track of whether this is a andi, zext.h, or zext.w.
529 bool ZExtOrANDI = isInt<12>(N1C->getSExtValue());
530 if (C1 == UINT64_C(0xFFFF) &&
531 (Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbp()))
532 ZExtOrANDI = true;
533 if (C1 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba())
534 ZExtOrANDI = true;
535
536 // Clear irrelevant bits in the mask.
537 if (LeftShift)
538 C1 &= maskTrailingZeros<uint64_t>(C2);
539 else
540 C1 &= maskTrailingOnes<uint64_t>(XLen - C2);
541
542 // Some transforms should only be done if the shift has a single use or
543 // the AND would become (srli (slli X, 32), 32)
544 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
545
546 SDValue X = N0.getOperand(0);
547
548 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
549 // with c3 leading zeros.
550 if (!LeftShift && isMask_64(C1)) {
551 uint64_t C3 = XLen - (64 - countLeadingZeros(C1));
552 if (C2 < C3) {
553 // If the number of leading zeros is C2+32 this can be SRLIW.
554 if (C2 + 32 == C3) {
555 SDNode *SRLIW =
556 CurDAG->getMachineNode(RISCV::SRLIW, DL, XLenVT, X,
557 CurDAG->getTargetConstant(C2, DL, XLenVT));
558 ReplaceNode(Node, SRLIW);
559 return;
560 }
561
562 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32) if
563 // c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
564 //
565 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
566 // legalized and goes through DAG combine.
567 SDValue Y;
568 if (C2 >= 32 && (C3 - C2) == 1 && N0.hasOneUse() &&
569 selectSExti32(X, Y)) {
570 SDNode *SRAIW =
571 CurDAG->getMachineNode(RISCV::SRAIW, DL, XLenVT, Y,
572 CurDAG->getTargetConstant(31, DL, XLenVT));
573 SDNode *SRLIW = CurDAG->getMachineNode(
574 RISCV::SRLIW, DL, XLenVT, SDValue(SRAIW, 0),
575 CurDAG->getTargetConstant(C3 - 32, DL, XLenVT));
576 ReplaceNode(Node, SRLIW);
577 return;
578 }
579
580 // (srli (slli x, c3-c2), c3).
581 if (OneUseOrZExtW && !ZExtOrANDI) {
582 SDNode *SLLI = CurDAG->getMachineNode(
583 RISCV::SLLI, DL, XLenVT, X,
584 CurDAG->getTargetConstant(C3 - C2, DL, XLenVT));
585 SDNode *SRLI =
586 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0),
587 CurDAG->getTargetConstant(C3, DL, XLenVT));
588 ReplaceNode(Node, SRLI);
589 return;
590 }
591 }
592 }
593
594 // Turn (and (shl x, c2) c1) -> (srli (slli c2+c3), c3) if c1 is a mask
595 // shifted by c2 bits with c3 leading zeros.
596 if (LeftShift && isShiftedMask_64(C1)) {
597 uint64_t C3 = XLen - (64 - countLeadingZeros(C1));
598
599 if (C2 + C3 < XLen &&
600 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + C3)) << C2)) {
601 // Use slli.uw when possible.
602 if ((XLen - (C2 + C3)) == 32 && Subtarget->hasStdExtZba()) {
603 SDNode *SLLIUW =
604 CurDAG->getMachineNode(RISCV::SLLIUW, DL, XLenVT, X,
605 CurDAG->getTargetConstant(C2, DL, XLenVT));
606 ReplaceNode(Node, SLLIUW);
607 return;
608 }
609
610 // (srli (slli c2+c3), c3)
611 if (OneUseOrZExtW && !ZExtOrANDI) {
612 SDNode *SLLI = CurDAG->getMachineNode(
613 RISCV::SLLI, DL, XLenVT, X,
614 CurDAG->getTargetConstant(C2 + C3, DL, XLenVT));
615 SDNode *SRLI =
616 CurDAG->getMachineNode(RISCV::SRLI, DL, XLenVT, SDValue(SLLI, 0),
617 CurDAG->getTargetConstant(C3, DL, XLenVT));
618 ReplaceNode(Node, SRLI);
619 return;
620 }
621 }
622 }
623
624 break;
625 }
626 case ISD::INTRINSIC_WO_CHAIN: {
627 unsigned IntNo = Node->getConstantOperandVal(0);
628 switch (IntNo) {
629 // By default we do not custom select any intrinsic.
630 default:
631 break;
632 case Intrinsic::riscv_vmsgeu:
633 case Intrinsic::riscv_vmsge: {
634 SDValue Src1 = Node->getOperand(1);
635 SDValue Src2 = Node->getOperand(2);
636 // Only custom select scalar second operand.
637 if (Src2.getValueType() != XLenVT)
638 break;
639 // Small constants are handled with patterns.
640 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
641 int64_t CVal = C->getSExtValue();
642 if (CVal >= -15 && CVal <= 16)
643 break;
644 }
645 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
646 MVT Src1VT = Src1.getSimpleValueType();
647 unsigned VMSLTOpcode, VMNANDOpcode;
648 switch (RISCVTargetLowering::getLMUL(Src1VT)) {
649 default:
650 llvm_unreachable("Unexpected LMUL!");
651 case RISCVII::VLMUL::LMUL_F8:
652 VMSLTOpcode =
653 IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF8 : RISCV::PseudoVMSLT_VX_MF8;
654 VMNANDOpcode = RISCV::PseudoVMNAND_MM_MF8;
655 break;
656 case RISCVII::VLMUL::LMUL_F4:
657 VMSLTOpcode =
658 IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF4 : RISCV::PseudoVMSLT_VX_MF4;
659 VMNANDOpcode = RISCV::PseudoVMNAND_MM_MF4;
660 break;
661 case RISCVII::VLMUL::LMUL_F2:
662 VMSLTOpcode =
663 IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF2 : RISCV::PseudoVMSLT_VX_MF2;
664 VMNANDOpcode = RISCV::PseudoVMNAND_MM_MF2;
665 break;
666 case RISCVII::VLMUL::LMUL_1:
667 VMSLTOpcode =
668 IsUnsigned ? RISCV::PseudoVMSLTU_VX_M1 : RISCV::PseudoVMSLT_VX_M1;
669 VMNANDOpcode = RISCV::PseudoVMNAND_MM_M1;
670 break;
671 case RISCVII::VLMUL::LMUL_2:
672 VMSLTOpcode =
673 IsUnsigned ? RISCV::PseudoVMSLTU_VX_M2 : RISCV::PseudoVMSLT_VX_M2;
674 VMNANDOpcode = RISCV::PseudoVMNAND_MM_M2;
675 break;
676 case RISCVII::VLMUL::LMUL_4:
677 VMSLTOpcode =
678 IsUnsigned ? RISCV::PseudoVMSLTU_VX_M4 : RISCV::PseudoVMSLT_VX_M4;
679 VMNANDOpcode = RISCV::PseudoVMNAND_MM_M4;
680 break;
681 case RISCVII::VLMUL::LMUL_8:
682 VMSLTOpcode =
683 IsUnsigned ? RISCV::PseudoVMSLTU_VX_M8 : RISCV::PseudoVMSLT_VX_M8;
684 VMNANDOpcode = RISCV::PseudoVMNAND_MM_M8;
685 break;
686 }
687 SDValue SEW = CurDAG->getTargetConstant(
688 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
689 SDValue VL;
690 selectVLOp(Node->getOperand(3), VL);
691
692 // Expand to
693 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
694 SDValue Cmp = SDValue(
695 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
696 0);
697 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT,
698 {Cmp, Cmp, VL, SEW}));
699 return;
700 }
701 case Intrinsic::riscv_vmsgeu_mask:
702 case Intrinsic::riscv_vmsge_mask: {
703 SDValue Src1 = Node->getOperand(2);
704 SDValue Src2 = Node->getOperand(3);
705 // Only custom select scalar second operand.
706 if (Src2.getValueType() != XLenVT)
707 break;
708 // Small constants are handled with patterns.
709 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
710 int64_t CVal = C->getSExtValue();
711 if (CVal >= -15 && CVal <= 16)
712 break;
713 }
714 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
715 MVT Src1VT = Src1.getSimpleValueType();
716 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOTOpcode;
717 switch (RISCVTargetLowering::getLMUL(Src1VT)) {
718 default:
719 llvm_unreachable("Unexpected LMUL!");
720 case RISCVII::VLMUL::LMUL_F8:
721 VMSLTOpcode =
722 IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF8 : RISCV::PseudoVMSLT_VX_MF8;
723 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF8_MASK
724 : RISCV::PseudoVMSLT_VX_MF8_MASK;
725 break;
726 case RISCVII::VLMUL::LMUL_F4:
727 VMSLTOpcode =
728 IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF4 : RISCV::PseudoVMSLT_VX_MF4;
729 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF4_MASK
730 : RISCV::PseudoVMSLT_VX_MF4_MASK;
731 break;
732 case RISCVII::VLMUL::LMUL_F2:
733 VMSLTOpcode =
734 IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF2 : RISCV::PseudoVMSLT_VX_MF2;
735 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_MF2_MASK
736 : RISCV::PseudoVMSLT_VX_MF2_MASK;
737 break;
738 case RISCVII::VLMUL::LMUL_1:
739 VMSLTOpcode =
740 IsUnsigned ? RISCV::PseudoVMSLTU_VX_M1 : RISCV::PseudoVMSLT_VX_M1;
741 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_M1_MASK
742 : RISCV::PseudoVMSLT_VX_M1_MASK;
743 break;
744 case RISCVII::VLMUL::LMUL_2:
745 VMSLTOpcode =
746 IsUnsigned ? RISCV::PseudoVMSLTU_VX_M2 : RISCV::PseudoVMSLT_VX_M2;
747 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_M2_MASK
748 : RISCV::PseudoVMSLT_VX_M2_MASK;
749 break;
750 case RISCVII::VLMUL::LMUL_4:
751 VMSLTOpcode =
752 IsUnsigned ? RISCV::PseudoVMSLTU_VX_M4 : RISCV::PseudoVMSLT_VX_M4;
753 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_M4_MASK
754 : RISCV::PseudoVMSLT_VX_M4_MASK;
755 break;
756 case RISCVII::VLMUL::LMUL_8:
757 VMSLTOpcode =
758 IsUnsigned ? RISCV::PseudoVMSLTU_VX_M8 : RISCV::PseudoVMSLT_VX_M8;
759 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_M8_MASK
760 : RISCV::PseudoVMSLT_VX_M8_MASK;
761 break;
762 }
763 // Mask operations use the LMUL from the mask type.
764 switch (RISCVTargetLowering::getLMUL(VT)) {
765 default:
766 llvm_unreachable("Unexpected LMUL!");
767 case RISCVII::VLMUL::LMUL_F8:
768 VMXOROpcode = RISCV::PseudoVMXOR_MM_MF8;
769 VMANDNOTOpcode = RISCV::PseudoVMANDNOT_MM_MF8;
770 break;
771 case RISCVII::VLMUL::LMUL_F4:
772 VMXOROpcode = RISCV::PseudoVMXOR_MM_MF4;
773 VMANDNOTOpcode = RISCV::PseudoVMANDNOT_MM_MF4;
774 break;
775 case RISCVII::VLMUL::LMUL_F2:
776 VMXOROpcode = RISCV::PseudoVMXOR_MM_MF2;
777 VMANDNOTOpcode = RISCV::PseudoVMANDNOT_MM_MF2;
778 break;
779 case RISCVII::VLMUL::LMUL_1:
780 VMXOROpcode = RISCV::PseudoVMXOR_MM_M1;
781 VMANDNOTOpcode = RISCV::PseudoVMANDNOT_MM_M1;
782 break;
783 case RISCVII::VLMUL::LMUL_2:
784 VMXOROpcode = RISCV::PseudoVMXOR_MM_M2;
785 VMANDNOTOpcode = RISCV::PseudoVMANDNOT_MM_M2;
786 break;
787 case RISCVII::VLMUL::LMUL_4:
788 VMXOROpcode = RISCV::PseudoVMXOR_MM_M4;
789 VMANDNOTOpcode = RISCV::PseudoVMANDNOT_MM_M4;
790 break;
791 case RISCVII::VLMUL::LMUL_8:
792 VMXOROpcode = RISCV::PseudoVMXOR_MM_M8;
793 VMANDNOTOpcode = RISCV::PseudoVMANDNOT_MM_M8;
794 break;
795 }
796 SDValue SEW = CurDAG->getTargetConstant(
797 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
798 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT);
799 SDValue VL;
800 selectVLOp(Node->getOperand(5), VL);
801 SDValue MaskedOff = Node->getOperand(1);
802 SDValue Mask = Node->getOperand(4);
803 // If the MaskedOff value and the Mask are the same value use
804 // vmslt{u}.vx vt, va, x; vmandnot.mm vd, vd, vt
805 // This avoids needing to copy v0 to vd before starting the next sequence.
806 if (Mask == MaskedOff) {
807 SDValue Cmp = SDValue(
808 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
809 0);
810 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOTOpcode, DL, VT,
811 {Mask, Cmp, VL, MaskSEW}));
812 return;
813 }
814
815 // Mask needs to be copied to V0.
816 SDValue Chain = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
817 RISCV::V0, Mask, SDValue());
818 SDValue Glue = Chain.getValue(1);
819 SDValue V0 = CurDAG->getRegister(RISCV::V0, VT);
820
821 // Otherwise use
822 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
823 SDValue Cmp = SDValue(
824 CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT,
825 {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}),
826 0);
827 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT,
828 {Cmp, Mask, VL, MaskSEW}));
829 return;
830 }
831 }
832 break;
833 }
834 case ISD::INTRINSIC_W_CHAIN: {
835 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
836 switch (IntNo) {
837 // By default we do not custom select any intrinsic.
838 default:
839 break;
840
841 case Intrinsic::riscv_vsetvli:
842 case Intrinsic::riscv_vsetvlimax: {
843 if (!Subtarget->hasStdExtV())
844 break;
845
846 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
847 unsigned Offset = VLMax ? 2 : 3;
848
849 assert(Node->getNumOperands() == Offset + 2 &&
850 "Unexpected number of operands");
851
852 unsigned SEW =
853 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7);
854 RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>(
855 Node->getConstantOperandVal(Offset + 1) & 0x7);
856
857 unsigned VTypeI = RISCVVType::encodeVTYPE(
858 VLMul, SEW, /*TailAgnostic*/ true, /*MaskAgnostic*/ false);
859 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
860
861 SDValue VLOperand;
862 if (VLMax) {
863 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
864 } else {
865 VLOperand = Node->getOperand(2);
866
867 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
868 uint64_t AVL = C->getZExtValue();
869 if (isUInt<5>(AVL)) {
870 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
871 ReplaceNode(
872 Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, XLenVT,
873 MVT::Other, VLImm, VTypeIOp,
874 /* Chain */ Node->getOperand(0)));
875 return;
876 }
877 }
878 }
879
880 ReplaceNode(Node,
881 CurDAG->getMachineNode(RISCV::PseudoVSETVLI, DL, XLenVT,
882 MVT::Other, VLOperand, VTypeIOp,
883 /* Chain */ Node->getOperand(0)));
884 return;
885 }
886 case Intrinsic::riscv_vlseg2:
887 case Intrinsic::riscv_vlseg3:
888 case Intrinsic::riscv_vlseg4:
889 case Intrinsic::riscv_vlseg5:
890 case Intrinsic::riscv_vlseg6:
891 case Intrinsic::riscv_vlseg7:
892 case Intrinsic::riscv_vlseg8: {
893 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
894 return;
895 }
896 case Intrinsic::riscv_vlseg2_mask:
897 case Intrinsic::riscv_vlseg3_mask:
898 case Intrinsic::riscv_vlseg4_mask:
899 case Intrinsic::riscv_vlseg5_mask:
900 case Intrinsic::riscv_vlseg6_mask:
901 case Intrinsic::riscv_vlseg7_mask:
902 case Intrinsic::riscv_vlseg8_mask: {
903 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
904 return;
905 }
906 case Intrinsic::riscv_vlsseg2:
907 case Intrinsic::riscv_vlsseg3:
908 case Intrinsic::riscv_vlsseg4:
909 case Intrinsic::riscv_vlsseg5:
910 case Intrinsic::riscv_vlsseg6:
911 case Intrinsic::riscv_vlsseg7:
912 case Intrinsic::riscv_vlsseg8: {
913 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
914 return;
915 }
916 case Intrinsic::riscv_vlsseg2_mask:
917 case Intrinsic::riscv_vlsseg3_mask:
918 case Intrinsic::riscv_vlsseg4_mask:
919 case Intrinsic::riscv_vlsseg5_mask:
920 case Intrinsic::riscv_vlsseg6_mask:
921 case Intrinsic::riscv_vlsseg7_mask:
922 case Intrinsic::riscv_vlsseg8_mask: {
923 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
924 return;
925 }
926 case Intrinsic::riscv_vloxseg2:
927 case Intrinsic::riscv_vloxseg3:
928 case Intrinsic::riscv_vloxseg4:
929 case Intrinsic::riscv_vloxseg5:
930 case Intrinsic::riscv_vloxseg6:
931 case Intrinsic::riscv_vloxseg7:
932 case Intrinsic::riscv_vloxseg8:
933 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
934 return;
935 case Intrinsic::riscv_vluxseg2:
936 case Intrinsic::riscv_vluxseg3:
937 case Intrinsic::riscv_vluxseg4:
938 case Intrinsic::riscv_vluxseg5:
939 case Intrinsic::riscv_vluxseg6:
940 case Intrinsic::riscv_vluxseg7:
941 case Intrinsic::riscv_vluxseg8:
942 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
943 return;
944 case Intrinsic::riscv_vloxseg2_mask:
945 case Intrinsic::riscv_vloxseg3_mask:
946 case Intrinsic::riscv_vloxseg4_mask:
947 case Intrinsic::riscv_vloxseg5_mask:
948 case Intrinsic::riscv_vloxseg6_mask:
949 case Intrinsic::riscv_vloxseg7_mask:
950 case Intrinsic::riscv_vloxseg8_mask:
951 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
952 return;
953 case Intrinsic::riscv_vluxseg2_mask:
954 case Intrinsic::riscv_vluxseg3_mask:
955 case Intrinsic::riscv_vluxseg4_mask:
956 case Intrinsic::riscv_vluxseg5_mask:
957 case Intrinsic::riscv_vluxseg6_mask:
958 case Intrinsic::riscv_vluxseg7_mask:
959 case Intrinsic::riscv_vluxseg8_mask:
960 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
961 return;
962 case Intrinsic::riscv_vlseg8ff:
963 case Intrinsic::riscv_vlseg7ff:
964 case Intrinsic::riscv_vlseg6ff:
965 case Intrinsic::riscv_vlseg5ff:
966 case Intrinsic::riscv_vlseg4ff:
967 case Intrinsic::riscv_vlseg3ff:
968 case Intrinsic::riscv_vlseg2ff: {
969 selectVLSEGFF(Node, /*IsMasked*/ false);
970 return;
971 }
972 case Intrinsic::riscv_vlseg8ff_mask:
973 case Intrinsic::riscv_vlseg7ff_mask:
974 case Intrinsic::riscv_vlseg6ff_mask:
975 case Intrinsic::riscv_vlseg5ff_mask:
976 case Intrinsic::riscv_vlseg4ff_mask:
977 case Intrinsic::riscv_vlseg3ff_mask:
978 case Intrinsic::riscv_vlseg2ff_mask: {
979 selectVLSEGFF(Node, /*IsMasked*/ true);
980 return;
981 }
982 case Intrinsic::riscv_vloxei:
983 case Intrinsic::riscv_vloxei_mask:
984 case Intrinsic::riscv_vluxei:
985 case Intrinsic::riscv_vluxei_mask: {
986 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
987 IntNo == Intrinsic::riscv_vluxei_mask;
988 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
989 IntNo == Intrinsic::riscv_vloxei_mask;
990
991 MVT VT = Node->getSimpleValueType(0);
992 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
993
994 unsigned CurOp = 2;
995 SmallVector<SDValue, 8> Operands;
996 if (IsMasked)
997 Operands.push_back(Node->getOperand(CurOp++));
998
999 MVT IndexVT;
1000 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1001 /*IsStridedOrIndexed*/ true, Operands,
1002 &IndexVT);
1003
1004 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
1005 "Element count mismatch");
1006
1007 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1008 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1009 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1010 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
1011 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
1012 static_cast<unsigned>(IndexLMUL));
1013 MachineSDNode *Load =
1014 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1015
1016 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1017 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1018
1019 ReplaceNode(Node, Load);
1020 return;
1021 }
1022 case Intrinsic::riscv_vle1:
1023 case Intrinsic::riscv_vle:
1024 case Intrinsic::riscv_vle_mask:
1025 case Intrinsic::riscv_vlse:
1026 case Intrinsic::riscv_vlse_mask: {
1027 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
1028 IntNo == Intrinsic::riscv_vlse_mask;
1029 bool IsStrided =
1030 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
1031
1032 MVT VT = Node->getSimpleValueType(0);
1033 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1034
1035 unsigned CurOp = 2;
1036 SmallVector<SDValue, 8> Operands;
1037 if (IsMasked)
1038 Operands.push_back(Node->getOperand(CurOp++));
1039
1040 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1041 Operands);
1042
1043 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1044 const RISCV::VLEPseudo *P =
1045 RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, Log2SEW,
1046 static_cast<unsigned>(LMUL));
1047 MachineSDNode *Load =
1048 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1049
1050 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1051 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1052
1053 ReplaceNode(Node, Load);
1054 return;
1055 }
1056 case Intrinsic::riscv_vleff:
1057 case Intrinsic::riscv_vleff_mask: {
1058 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
1059
1060 MVT VT = Node->getSimpleValueType(0);
1061 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1062
1063 unsigned CurOp = 2;
1064 SmallVector<SDValue, 7> Operands;
1065 if (IsMasked)
1066 Operands.push_back(Node->getOperand(CurOp++));
1067
1068 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1069 /*IsStridedOrIndexed*/ false, Operands);
1070
1071 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1072 const RISCV::VLEPseudo *P =
1073 RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true, Log2SEW,
1074 static_cast<unsigned>(LMUL));
1075 MachineSDNode *Load =
1076 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0),
1077 MVT::Other, MVT::Glue, Operands);
1078 SDNode *ReadVL = CurDAG->getMachineNode(RISCV::PseudoReadVL, DL, XLenVT,
1079 /*Glue*/ SDValue(Load, 2));
1080
1081 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1082 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1083
1084 ReplaceUses(SDValue(Node, 0), SDValue(Load, 0));
1085 ReplaceUses(SDValue(Node, 1), SDValue(ReadVL, 0)); // VL
1086 ReplaceUses(SDValue(Node, 2), SDValue(Load, 1)); // Chain
1087 CurDAG->RemoveDeadNode(Node);
1088 return;
1089 }
1090 }
1091 break;
1092 }
1093 case ISD::INTRINSIC_VOID: {
1094 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1095 switch (IntNo) {
1096 case Intrinsic::riscv_vsseg2:
1097 case Intrinsic::riscv_vsseg3:
1098 case Intrinsic::riscv_vsseg4:
1099 case Intrinsic::riscv_vsseg5:
1100 case Intrinsic::riscv_vsseg6:
1101 case Intrinsic::riscv_vsseg7:
1102 case Intrinsic::riscv_vsseg8: {
1103 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1104 return;
1105 }
1106 case Intrinsic::riscv_vsseg2_mask:
1107 case Intrinsic::riscv_vsseg3_mask:
1108 case Intrinsic::riscv_vsseg4_mask:
1109 case Intrinsic::riscv_vsseg5_mask:
1110 case Intrinsic::riscv_vsseg6_mask:
1111 case Intrinsic::riscv_vsseg7_mask:
1112 case Intrinsic::riscv_vsseg8_mask: {
1113 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1114 return;
1115 }
1116 case Intrinsic::riscv_vssseg2:
1117 case Intrinsic::riscv_vssseg3:
1118 case Intrinsic::riscv_vssseg4:
1119 case Intrinsic::riscv_vssseg5:
1120 case Intrinsic::riscv_vssseg6:
1121 case Intrinsic::riscv_vssseg7:
1122 case Intrinsic::riscv_vssseg8: {
1123 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1124 return;
1125 }
1126 case Intrinsic::riscv_vssseg2_mask:
1127 case Intrinsic::riscv_vssseg3_mask:
1128 case Intrinsic::riscv_vssseg4_mask:
1129 case Intrinsic::riscv_vssseg5_mask:
1130 case Intrinsic::riscv_vssseg6_mask:
1131 case Intrinsic::riscv_vssseg7_mask:
1132 case Intrinsic::riscv_vssseg8_mask: {
1133 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1134 return;
1135 }
1136 case Intrinsic::riscv_vsoxseg2:
1137 case Intrinsic::riscv_vsoxseg3:
1138 case Intrinsic::riscv_vsoxseg4:
1139 case Intrinsic::riscv_vsoxseg5:
1140 case Intrinsic::riscv_vsoxseg6:
1141 case Intrinsic::riscv_vsoxseg7:
1142 case Intrinsic::riscv_vsoxseg8:
1143 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1144 return;
1145 case Intrinsic::riscv_vsuxseg2:
1146 case Intrinsic::riscv_vsuxseg3:
1147 case Intrinsic::riscv_vsuxseg4:
1148 case Intrinsic::riscv_vsuxseg5:
1149 case Intrinsic::riscv_vsuxseg6:
1150 case Intrinsic::riscv_vsuxseg7:
1151 case Intrinsic::riscv_vsuxseg8:
1152 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1153 return;
1154 case Intrinsic::riscv_vsoxseg2_mask:
1155 case Intrinsic::riscv_vsoxseg3_mask:
1156 case Intrinsic::riscv_vsoxseg4_mask:
1157 case Intrinsic::riscv_vsoxseg5_mask:
1158 case Intrinsic::riscv_vsoxseg6_mask:
1159 case Intrinsic::riscv_vsoxseg7_mask:
1160 case Intrinsic::riscv_vsoxseg8_mask:
1161 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1162 return;
1163 case Intrinsic::riscv_vsuxseg2_mask:
1164 case Intrinsic::riscv_vsuxseg3_mask:
1165 case Intrinsic::riscv_vsuxseg4_mask:
1166 case Intrinsic::riscv_vsuxseg5_mask:
1167 case Intrinsic::riscv_vsuxseg6_mask:
1168 case Intrinsic::riscv_vsuxseg7_mask:
1169 case Intrinsic::riscv_vsuxseg8_mask:
1170 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1171 return;
1172 case Intrinsic::riscv_vsoxei:
1173 case Intrinsic::riscv_vsoxei_mask:
1174 case Intrinsic::riscv_vsuxei:
1175 case Intrinsic::riscv_vsuxei_mask: {
1176 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
1177 IntNo == Intrinsic::riscv_vsuxei_mask;
1178 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
1179 IntNo == Intrinsic::riscv_vsoxei_mask;
1180
1181 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1182 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1183
1184 unsigned CurOp = 2;
1185 SmallVector<SDValue, 8> Operands;
1186 Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1187
1188 MVT IndexVT;
1189 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1190 /*IsStridedOrIndexed*/ true, Operands,
1191 &IndexVT);
1192
1193 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
1194 "Element count mismatch");
1195
1196 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1197 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1198 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1199 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1200 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
1201 static_cast<unsigned>(IndexLMUL));
1202 MachineSDNode *Store =
1203 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1204
1205 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1206 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1207
1208 ReplaceNode(Node, Store);
1209 return;
1210 }
1211 case Intrinsic::riscv_vse1:
1212 case Intrinsic::riscv_vse:
1213 case Intrinsic::riscv_vse_mask:
1214 case Intrinsic::riscv_vsse:
1215 case Intrinsic::riscv_vsse_mask: {
1216 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
1217 IntNo == Intrinsic::riscv_vsse_mask;
1218 bool IsStrided =
1219 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
1220
1221 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1222 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1223
1224 unsigned CurOp = 2;
1225 SmallVector<SDValue, 8> Operands;
1226 Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1227
1228 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1229 Operands);
1230
1231 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1232 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1233 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
1234 MachineSDNode *Store =
1235 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1236 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1237 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1238
1239 ReplaceNode(Node, Store);
1240 return;
1241 }
1242 }
1243 break;
1244 }
1245 case ISD::BITCAST: {
1246 MVT SrcVT = Node->getOperand(0).getSimpleValueType();
1247 // Just drop bitcasts between vectors if both are fixed or both are
1248 // scalable.
1249 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
1250 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
1251 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
1252 CurDAG->RemoveDeadNode(Node);
1253 return;
1254 }
1255 break;
1256 }
1257 case ISD::INSERT_SUBVECTOR: {
1258 SDValue V = Node->getOperand(0);
1259 SDValue SubV = Node->getOperand(1);
1260 SDLoc DL(SubV);
1261 auto Idx = Node->getConstantOperandVal(2);
1262 MVT SubVecVT = SubV.getSimpleValueType();
1263
1264 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
1265 MVT SubVecContainerVT = SubVecVT;
1266 // Establish the correct scalable-vector types for any fixed-length type.
1267 if (SubVecVT.isFixedLengthVector())
1268 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT);
1269 if (VT.isFixedLengthVector())
1270 VT = TLI.getContainerForFixedLengthVector(VT);
1271
1272 const auto *TRI = Subtarget->getRegisterInfo();
1273 unsigned SubRegIdx;
1274 std::tie(SubRegIdx, Idx) =
1275 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1276 VT, SubVecContainerVT, Idx, TRI);
1277
1278 // If the Idx hasn't been completely eliminated then this is a subvector
1279 // insert which doesn't naturally align to a vector register. These must
1280 // be handled using instructions to manipulate the vector registers.
1281 if (Idx != 0)
1282 break;
1283
1284 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
1285 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
1286 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
1287 SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
1288 (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
1289 assert((!IsSubVecPartReg || V.isUndef()) &&
1290 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
1291 "the subvector is smaller than a full-sized register");
1292
1293 // If we haven't set a SubRegIdx, then we must be going between
1294 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1295 if (SubRegIdx == RISCV::NoSubRegister) {
1296 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
1297 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1298 InRegClassID &&
1299 "Unexpected subvector extraction");
1300 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1301 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1302 DL, VT, SubV, RC);
1303 ReplaceNode(Node, NewNode);
1304 return;
1305 }
1306
1307 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
1308 ReplaceNode(Node, Insert.getNode());
1309 return;
1310 }
1311 case ISD::EXTRACT_SUBVECTOR: {
1312 SDValue V = Node->getOperand(0);
1313 auto Idx = Node->getConstantOperandVal(1);
1314 MVT InVT = V.getSimpleValueType();
1315 SDLoc DL(V);
1316
1317 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
1318 MVT SubVecContainerVT = VT;
1319 // Establish the correct scalable-vector types for any fixed-length type.
1320 if (VT.isFixedLengthVector())
1321 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
1322 if (InVT.isFixedLengthVector())
1323 InVT = TLI.getContainerForFixedLengthVector(InVT);
1324
1325 const auto *TRI = Subtarget->getRegisterInfo();
1326 unsigned SubRegIdx;
1327 std::tie(SubRegIdx, Idx) =
1328 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1329 InVT, SubVecContainerVT, Idx, TRI);
1330
1331 // If the Idx hasn't been completely eliminated then this is a subvector
1332 // extract which doesn't naturally align to a vector register. These must
1333 // be handled using instructions to manipulate the vector registers.
1334 if (Idx != 0)
1335 break;
1336
1337 // If we haven't set a SubRegIdx, then we must be going between
1338 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
1339 if (SubRegIdx == RISCV::NoSubRegister) {
1340 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1341 assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecContainerVT) ==
1342 InRegClassID &&
1343 "Unexpected subvector extraction");
1344 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1345 SDNode *NewNode =
1346 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
1347 ReplaceNode(Node, NewNode);
1348 return;
1349 }
1350
1351 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
1352 ReplaceNode(Node, Extract.getNode());
1353 return;
1354 }
1355 case RISCVISD::VMV_V_X_VL:
1356 case RISCVISD::VFMV_V_F_VL: {
1357 // Try to match splat of a scalar load to a strided load with stride of x0.
1358 SDValue Src = Node->getOperand(0);
1359 auto *Ld = dyn_cast<LoadSDNode>(Src);
1360 if (!Ld)
1361 break;
1362 EVT MemVT = Ld->getMemoryVT();
1363 // The memory VT should be the same size as the element type.
1364 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
1365 break;
1366 if (!IsProfitableToFold(Src, Node, Node) ||
1367 !IsLegalToFold(Src, Node, Node, TM.getOptLevel()))
1368 break;
1369
1370 SDValue VL;
1371 selectVLOp(Node->getOperand(1), VL);
1372
1373 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1374 SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
1375
1376 SDValue Operands[] = {Ld->getBasePtr(),
1377 CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW,
1378 Ld->getChain()};
1379
1380 RISCVII::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1381 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
1382 /*IsMasked*/ false, /*IsStrided*/ true, /*FF*/ false, Log2SEW,
1383 static_cast<unsigned>(LMUL));
1384 MachineSDNode *Load =
1385 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1386
1387 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1388 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1389
1390 ReplaceNode(Node, Load);
1391 return;
1392 }
1393 }
1394
1395 // Select the default instruction.
1396 SelectCode(Node);
1397 }
1398
SelectInlineAsmMemoryOperand(const SDValue & Op,unsigned ConstraintID,std::vector<SDValue> & OutOps)1399 bool RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand(
1400 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
1401 switch (ConstraintID) {
1402 case InlineAsm::Constraint_m:
1403 // We just support simple memory operands that have a single address
1404 // operand and need no special handling.
1405 OutOps.push_back(Op);
1406 return false;
1407 case InlineAsm::Constraint_A:
1408 OutOps.push_back(Op);
1409 return false;
1410 default:
1411 break;
1412 }
1413
1414 return true;
1415 }
1416
SelectAddrFI(SDValue Addr,SDValue & Base)1417 bool RISCVDAGToDAGISel::SelectAddrFI(SDValue Addr, SDValue &Base) {
1418 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1419 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1420 return true;
1421 }
1422 return false;
1423 }
1424
SelectBaseAddr(SDValue Addr,SDValue & Base)1425 bool RISCVDAGToDAGISel::SelectBaseAddr(SDValue Addr, SDValue &Base) {
1426 // If this is FrameIndex, select it directly. Otherwise just let it get
1427 // selected to a register independently.
1428 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr))
1429 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1430 else
1431 Base = Addr;
1432 return true;
1433 }
1434
selectShiftMask(SDValue N,unsigned ShiftWidth,SDValue & ShAmt)1435 bool RISCVDAGToDAGISel::selectShiftMask(SDValue N, unsigned ShiftWidth,
1436 SDValue &ShAmt) {
1437 // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
1438 // amount. If there is an AND on the shift amount, we can bypass it if it
1439 // doesn't affect any of those bits.
1440 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
1441 const APInt &AndMask = N->getConstantOperandAPInt(1);
1442
1443 // Since the max shift amount is a power of 2 we can subtract 1 to make a
1444 // mask that covers the bits needed to represent all shift amounts.
1445 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
1446 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
1447
1448 if (ShMask.isSubsetOf(AndMask)) {
1449 ShAmt = N.getOperand(0);
1450 return true;
1451 }
1452
1453 // SimplifyDemandedBits may have optimized the mask so try restoring any
1454 // bits that are known zero.
1455 KnownBits Known = CurDAG->computeKnownBits(N->getOperand(0));
1456 if (ShMask.isSubsetOf(AndMask | Known.Zero)) {
1457 ShAmt = N.getOperand(0);
1458 return true;
1459 }
1460 }
1461
1462 ShAmt = N;
1463 return true;
1464 }
1465
selectSExti32(SDValue N,SDValue & Val)1466 bool RISCVDAGToDAGISel::selectSExti32(SDValue N, SDValue &Val) {
1467 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1468 cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) {
1469 Val = N.getOperand(0);
1470 return true;
1471 }
1472 MVT VT = N.getSimpleValueType();
1473 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - 32)) {
1474 Val = N;
1475 return true;
1476 }
1477
1478 return false;
1479 }
1480
selectZExti32(SDValue N,SDValue & Val)1481 bool RISCVDAGToDAGISel::selectZExti32(SDValue N, SDValue &Val) {
1482 if (N.getOpcode() == ISD::AND) {
1483 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
1484 if (C && C->getZExtValue() == UINT64_C(0xFFFFFFFF)) {
1485 Val = N.getOperand(0);
1486 return true;
1487 }
1488 }
1489 MVT VT = N.getSimpleValueType();
1490 APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(), 32);
1491 if (CurDAG->MaskedValueIsZero(N, Mask)) {
1492 Val = N;
1493 return true;
1494 }
1495
1496 return false;
1497 }
1498
1499 // Select VL as a 5 bit immediate or a value that will become a register. This
1500 // allows us to choose betwen VSETIVLI or VSETVLI later.
selectVLOp(SDValue N,SDValue & VL)1501 bool RISCVDAGToDAGISel::selectVLOp(SDValue N, SDValue &VL) {
1502 auto *C = dyn_cast<ConstantSDNode>(N);
1503 if (C && isUInt<5>(C->getZExtValue()))
1504 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N),
1505 N->getValueType(0));
1506 else
1507 VL = N;
1508
1509 return true;
1510 }
1511
selectVSplat(SDValue N,SDValue & SplatVal)1512 bool RISCVDAGToDAGISel::selectVSplat(SDValue N, SDValue &SplatVal) {
1513 if (N.getOpcode() != ISD::SPLAT_VECTOR &&
1514 N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1515 N.getOpcode() != RISCVISD::VMV_V_X_VL)
1516 return false;
1517 SplatVal = N.getOperand(0);
1518 return true;
1519 }
1520
1521 using ValidateFn = bool (*)(int64_t);
1522
selectVSplatSimmHelper(SDValue N,SDValue & SplatVal,SelectionDAG & DAG,const RISCVSubtarget & Subtarget,ValidateFn ValidateImm)1523 static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal,
1524 SelectionDAG &DAG,
1525 const RISCVSubtarget &Subtarget,
1526 ValidateFn ValidateImm) {
1527 if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1528 N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1529 N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1530 !isa<ConstantSDNode>(N.getOperand(0)))
1531 return false;
1532
1533 int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1534
1535 // ISD::SPLAT_VECTOR, RISCVISD::SPLAT_VECTOR_I64 and RISCVISD::VMV_V_X_VL
1536 // share semantics when the operand type is wider than the resulting vector
1537 // element type: an implicit truncation first takes place. Therefore, perform
1538 // a manual truncation/sign-extension in order to ignore any truncated bits
1539 // and catch any zero-extended immediate.
1540 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
1541 // sign-extending to (XLenVT -1).
1542 MVT XLenVT = Subtarget.getXLenVT();
1543 assert(XLenVT == N.getOperand(0).getSimpleValueType() &&
1544 "Unexpected splat operand type");
1545 MVT EltVT = N.getSimpleValueType().getVectorElementType();
1546 if (EltVT.bitsLT(XLenVT))
1547 SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
1548
1549 if (!ValidateImm(SplatImm))
1550 return false;
1551
1552 SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), XLenVT);
1553 return true;
1554 }
1555
selectVSplatSimm5(SDValue N,SDValue & SplatVal)1556 bool RISCVDAGToDAGISel::selectVSplatSimm5(SDValue N, SDValue &SplatVal) {
1557 return selectVSplatSimmHelper(N, SplatVal, *CurDAG, *Subtarget,
1558 [](int64_t Imm) { return isInt<5>(Imm); });
1559 }
1560
selectVSplatSimm5Plus1(SDValue N,SDValue & SplatVal)1561 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal) {
1562 return selectVSplatSimmHelper(
1563 N, SplatVal, *CurDAG, *Subtarget,
1564 [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; });
1565 }
1566
selectVSplatSimm5Plus1NonZero(SDValue N,SDValue & SplatVal)1567 bool RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero(SDValue N,
1568 SDValue &SplatVal) {
1569 return selectVSplatSimmHelper(
1570 N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) {
1571 return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16);
1572 });
1573 }
1574
selectVSplatUimm5(SDValue N,SDValue & SplatVal)1575 bool RISCVDAGToDAGISel::selectVSplatUimm5(SDValue N, SDValue &SplatVal) {
1576 if ((N.getOpcode() != ISD::SPLAT_VECTOR &&
1577 N.getOpcode() != RISCVISD::SPLAT_VECTOR_I64 &&
1578 N.getOpcode() != RISCVISD::VMV_V_X_VL) ||
1579 !isa<ConstantSDNode>(N.getOperand(0)))
1580 return false;
1581
1582 int64_t SplatImm = cast<ConstantSDNode>(N.getOperand(0))->getSExtValue();
1583
1584 if (!isUInt<5>(SplatImm))
1585 return false;
1586
1587 SplatVal =
1588 CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
1589
1590 return true;
1591 }
1592
selectRVVSimm5(SDValue N,unsigned Width,SDValue & Imm)1593 bool RISCVDAGToDAGISel::selectRVVSimm5(SDValue N, unsigned Width,
1594 SDValue &Imm) {
1595 if (auto *C = dyn_cast<ConstantSDNode>(N)) {
1596 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
1597
1598 if (!isInt<5>(ImmVal))
1599 return false;
1600
1601 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
1602 return true;
1603 }
1604
1605 return false;
1606 }
1607
1608 // Merge an ADDI into the offset of a load/store instruction where possible.
1609 // (load (addi base, off1), off2) -> (load base, off1+off2)
1610 // (store val, (addi base, off1), off2) -> (store val, base, off1+off2)
1611 // This is possible when off1+off2 fits a 12-bit immediate.
doPeepholeLoadStoreADDI()1612 void RISCVDAGToDAGISel::doPeepholeLoadStoreADDI() {
1613 SelectionDAG::allnodes_iterator Position(CurDAG->getRoot().getNode());
1614 ++Position;
1615
1616 while (Position != CurDAG->allnodes_begin()) {
1617 SDNode *N = &*--Position;
1618 // Skip dead nodes and any non-machine opcodes.
1619 if (N->use_empty() || !N->isMachineOpcode())
1620 continue;
1621
1622 int OffsetOpIdx;
1623 int BaseOpIdx;
1624
1625 // Only attempt this optimisation for I-type loads and S-type stores.
1626 switch (N->getMachineOpcode()) {
1627 default:
1628 continue;
1629 case RISCV::LB:
1630 case RISCV::LH:
1631 case RISCV::LW:
1632 case RISCV::LBU:
1633 case RISCV::LHU:
1634 case RISCV::LWU:
1635 case RISCV::LD:
1636 case RISCV::FLH:
1637 case RISCV::FLW:
1638 case RISCV::FLD:
1639 BaseOpIdx = 0;
1640 OffsetOpIdx = 1;
1641 break;
1642 case RISCV::SB:
1643 case RISCV::SH:
1644 case RISCV::SW:
1645 case RISCV::SD:
1646 case RISCV::FSH:
1647 case RISCV::FSW:
1648 case RISCV::FSD:
1649 BaseOpIdx = 1;
1650 OffsetOpIdx = 2;
1651 break;
1652 }
1653
1654 if (!isa<ConstantSDNode>(N->getOperand(OffsetOpIdx)))
1655 continue;
1656
1657 SDValue Base = N->getOperand(BaseOpIdx);
1658
1659 // If the base is an ADDI, we can merge it in to the load/store.
1660 if (!Base.isMachineOpcode() || Base.getMachineOpcode() != RISCV::ADDI)
1661 continue;
1662
1663 SDValue ImmOperand = Base.getOperand(1);
1664 uint64_t Offset2 = N->getConstantOperandVal(OffsetOpIdx);
1665
1666 if (auto *Const = dyn_cast<ConstantSDNode>(ImmOperand)) {
1667 int64_t Offset1 = Const->getSExtValue();
1668 int64_t CombinedOffset = Offset1 + Offset2;
1669 if (!isInt<12>(CombinedOffset))
1670 continue;
1671 ImmOperand = CurDAG->getTargetConstant(CombinedOffset, SDLoc(ImmOperand),
1672 ImmOperand.getValueType());
1673 } else if (auto *GA = dyn_cast<GlobalAddressSDNode>(ImmOperand)) {
1674 // If the off1 in (addi base, off1) is a global variable's address (its
1675 // low part, really), then we can rely on the alignment of that variable
1676 // to provide a margin of safety before off1 can overflow the 12 bits.
1677 // Check if off2 falls within that margin; if so off1+off2 can't overflow.
1678 const DataLayout &DL = CurDAG->getDataLayout();
1679 Align Alignment = GA->getGlobal()->getPointerAlignment(DL);
1680 if (Offset2 != 0 && Alignment <= Offset2)
1681 continue;
1682 int64_t Offset1 = GA->getOffset();
1683 int64_t CombinedOffset = Offset1 + Offset2;
1684 ImmOperand = CurDAG->getTargetGlobalAddress(
1685 GA->getGlobal(), SDLoc(ImmOperand), ImmOperand.getValueType(),
1686 CombinedOffset, GA->getTargetFlags());
1687 } else if (auto *CP = dyn_cast<ConstantPoolSDNode>(ImmOperand)) {
1688 // Ditto.
1689 Align Alignment = CP->getAlign();
1690 if (Offset2 != 0 && Alignment <= Offset2)
1691 continue;
1692 int64_t Offset1 = CP->getOffset();
1693 int64_t CombinedOffset = Offset1 + Offset2;
1694 ImmOperand = CurDAG->getTargetConstantPool(
1695 CP->getConstVal(), ImmOperand.getValueType(), CP->getAlign(),
1696 CombinedOffset, CP->getTargetFlags());
1697 } else {
1698 continue;
1699 }
1700
1701 LLVM_DEBUG(dbgs() << "Folding add-immediate into mem-op:\nBase: ");
1702 LLVM_DEBUG(Base->dump(CurDAG));
1703 LLVM_DEBUG(dbgs() << "\nN: ");
1704 LLVM_DEBUG(N->dump(CurDAG));
1705 LLVM_DEBUG(dbgs() << "\n");
1706
1707 // Modify the offset operand of the load/store.
1708 if (BaseOpIdx == 0) // Load
1709 CurDAG->UpdateNodeOperands(N, Base.getOperand(0), ImmOperand,
1710 N->getOperand(2));
1711 else // Store
1712 CurDAG->UpdateNodeOperands(N, N->getOperand(0), Base.getOperand(0),
1713 ImmOperand, N->getOperand(3));
1714
1715 // The add-immediate may now be dead, in which case remove it.
1716 if (Base.getNode()->use_empty())
1717 CurDAG->RemoveDeadNode(Base.getNode());
1718 }
1719 }
1720
1721 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
1722 // for instruction scheduling.
createRISCVISelDag(RISCVTargetMachine & TM)1723 FunctionPass *llvm::createRISCVISelDag(RISCVTargetMachine &TM) {
1724 return new RISCVDAGToDAGISel(TM);
1725 }
1726