1 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "HexagonISelLowering.h"
10 #include "HexagonRegisterInfo.h"
11 #include "HexagonSubtarget.h"
12 #include "llvm/IR/IntrinsicsHexagon.h"
13 #include "llvm/Support/CommandLine.h"
14
15 using namespace llvm;
16
17 static const MVT LegalV64[] = { MVT::v64i8, MVT::v32i16, MVT::v16i32 };
18 static const MVT LegalW64[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
19 static const MVT LegalV128[] = { MVT::v128i8, MVT::v64i16, MVT::v32i32 };
20 static const MVT LegalW128[] = { MVT::v256i8, MVT::v128i16, MVT::v64i32 };
21
22
23 void
initializeHVXLowering()24 HexagonTargetLowering::initializeHVXLowering() {
25 if (Subtarget.useHVX64BOps()) {
26 addRegisterClass(MVT::v64i8, &Hexagon::HvxVRRegClass);
27 addRegisterClass(MVT::v32i16, &Hexagon::HvxVRRegClass);
28 addRegisterClass(MVT::v16i32, &Hexagon::HvxVRRegClass);
29 addRegisterClass(MVT::v128i8, &Hexagon::HvxWRRegClass);
30 addRegisterClass(MVT::v64i16, &Hexagon::HvxWRRegClass);
31 addRegisterClass(MVT::v32i32, &Hexagon::HvxWRRegClass);
32 // These "short" boolean vector types should be legal because
33 // they will appear as results of vector compares. If they were
34 // not legal, type legalization would try to make them legal
35 // and that would require using operations that do not use or
36 // produce such types. That, in turn, would imply using custom
37 // nodes, which would be unoptimizable by the DAG combiner.
38 // The idea is to rely on target-independent operations as much
39 // as possible.
40 addRegisterClass(MVT::v16i1, &Hexagon::HvxQRRegClass);
41 addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
42 addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
43 } else if (Subtarget.useHVX128BOps()) {
44 addRegisterClass(MVT::v128i8, &Hexagon::HvxVRRegClass);
45 addRegisterClass(MVT::v64i16, &Hexagon::HvxVRRegClass);
46 addRegisterClass(MVT::v32i32, &Hexagon::HvxVRRegClass);
47 addRegisterClass(MVT::v256i8, &Hexagon::HvxWRRegClass);
48 addRegisterClass(MVT::v128i16, &Hexagon::HvxWRRegClass);
49 addRegisterClass(MVT::v64i32, &Hexagon::HvxWRRegClass);
50 addRegisterClass(MVT::v32i1, &Hexagon::HvxQRRegClass);
51 addRegisterClass(MVT::v64i1, &Hexagon::HvxQRRegClass);
52 addRegisterClass(MVT::v128i1, &Hexagon::HvxQRRegClass);
53 }
54
55 // Set up operation actions.
56
57 bool Use64b = Subtarget.useHVX64BOps();
58 ArrayRef<MVT> LegalV = Use64b ? LegalV64 : LegalV128;
59 ArrayRef<MVT> LegalW = Use64b ? LegalW64 : LegalW128;
60 MVT ByteV = Use64b ? MVT::v64i8 : MVT::v128i8;
61 MVT ByteW = Use64b ? MVT::v128i8 : MVT::v256i8;
62
63 auto setPromoteTo = [this] (unsigned Opc, MVT FromTy, MVT ToTy) {
64 setOperationAction(Opc, FromTy, Promote);
65 AddPromotedToType(Opc, FromTy, ToTy);
66 };
67
68 // Handle bitcasts of vector predicates to scalars (e.g. v32i1 to i32).
69 // Note: v16i1 -> i16 is handled in type legalization instead of op
70 // legalization.
71 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
72 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
73 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
74 setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
75 setOperationAction(ISD::BITCAST, MVT::v128i1, Custom);
76 setOperationAction(ISD::BITCAST, MVT::i128, Custom);
77 setOperationAction(ISD::VECTOR_SHUFFLE, ByteV, Legal);
78 setOperationAction(ISD::VECTOR_SHUFFLE, ByteW, Legal);
79 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
80
81 for (MVT T : LegalV) {
82 setIndexedLoadAction(ISD::POST_INC, T, Legal);
83 setIndexedStoreAction(ISD::POST_INC, T, Legal);
84
85 setOperationAction(ISD::AND, T, Legal);
86 setOperationAction(ISD::OR, T, Legal);
87 setOperationAction(ISD::XOR, T, Legal);
88 setOperationAction(ISD::ADD, T, Legal);
89 setOperationAction(ISD::SUB, T, Legal);
90 setOperationAction(ISD::CTPOP, T, Legal);
91 setOperationAction(ISD::CTLZ, T, Legal);
92 if (T != ByteV) {
93 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal);
94 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal);
95 setOperationAction(ISD::BSWAP, T, Legal);
96 }
97
98 setOperationAction(ISD::CTTZ, T, Custom);
99 setOperationAction(ISD::LOAD, T, Custom);
100 setOperationAction(ISD::MUL, T, Custom);
101 setOperationAction(ISD::MULHS, T, Custom);
102 setOperationAction(ISD::MULHU, T, Custom);
103 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
104 // Make concat-vectors custom to handle concats of more than 2 vectors.
105 setOperationAction(ISD::CONCAT_VECTORS, T, Custom);
106 setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom);
107 setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom);
108 setOperationAction(ISD::EXTRACT_SUBVECTOR, T, Custom);
109 setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom);
110 setOperationAction(ISD::ANY_EXTEND, T, Custom);
111 setOperationAction(ISD::SIGN_EXTEND, T, Custom);
112 setOperationAction(ISD::ZERO_EXTEND, T, Custom);
113 if (T != ByteV) {
114 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);
115 // HVX only has shifts of words and halfwords.
116 setOperationAction(ISD::SRA, T, Custom);
117 setOperationAction(ISD::SHL, T, Custom);
118 setOperationAction(ISD::SRL, T, Custom);
119
120 // Promote all shuffles to operate on vectors of bytes.
121 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteV);
122 }
123
124 setCondCodeAction(ISD::SETNE, T, Expand);
125 setCondCodeAction(ISD::SETLE, T, Expand);
126 setCondCodeAction(ISD::SETGE, T, Expand);
127 setCondCodeAction(ISD::SETLT, T, Expand);
128 setCondCodeAction(ISD::SETULE, T, Expand);
129 setCondCodeAction(ISD::SETUGE, T, Expand);
130 setCondCodeAction(ISD::SETULT, T, Expand);
131 }
132
133 for (MVT T : LegalW) {
134 // Custom-lower BUILD_VECTOR for vector pairs. The standard (target-
135 // independent) handling of it would convert it to a load, which is
136 // not always the optimal choice.
137 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
138 // Make concat-vectors custom to handle concats of more than 2 vectors.
139 setOperationAction(ISD::CONCAT_VECTORS, T, Custom);
140
141 // Custom-lower these operations for pairs. Expand them into a concat
142 // of the corresponding operations on individual vectors.
143 setOperationAction(ISD::ANY_EXTEND, T, Custom);
144 setOperationAction(ISD::SIGN_EXTEND, T, Custom);
145 setOperationAction(ISD::ZERO_EXTEND, T, Custom);
146 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Custom);
147 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, T, Custom);
148 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, T, Legal);
149 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, T, Legal);
150
151 setOperationAction(ISD::LOAD, T, Custom);
152 setOperationAction(ISD::STORE, T, Custom);
153 setOperationAction(ISD::CTLZ, T, Custom);
154 setOperationAction(ISD::CTTZ, T, Custom);
155 setOperationAction(ISD::CTPOP, T, Custom);
156
157 setOperationAction(ISD::ADD, T, Legal);
158 setOperationAction(ISD::SUB, T, Legal);
159 setOperationAction(ISD::MUL, T, Custom);
160 setOperationAction(ISD::MULHS, T, Custom);
161 setOperationAction(ISD::MULHU, T, Custom);
162 setOperationAction(ISD::AND, T, Custom);
163 setOperationAction(ISD::OR, T, Custom);
164 setOperationAction(ISD::XOR, T, Custom);
165 setOperationAction(ISD::SETCC, T, Custom);
166 setOperationAction(ISD::VSELECT, T, Custom);
167 if (T != ByteW) {
168 setOperationAction(ISD::SRA, T, Custom);
169 setOperationAction(ISD::SHL, T, Custom);
170 setOperationAction(ISD::SRL, T, Custom);
171
172 // Promote all shuffles to operate on vectors of bytes.
173 setPromoteTo(ISD::VECTOR_SHUFFLE, T, ByteW);
174 }
175 }
176
177 // Boolean vectors.
178
179 for (MVT T : LegalW) {
180 // Boolean types for vector pairs will overlap with the boolean
181 // types for single vectors, e.g.
182 // v64i8 -> v64i1 (single)
183 // v64i16 -> v64i1 (pair)
184 // Set these actions first, and allow the single actions to overwrite
185 // any duplicates.
186 MVT BoolW = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
187 setOperationAction(ISD::SETCC, BoolW, Custom);
188 setOperationAction(ISD::AND, BoolW, Custom);
189 setOperationAction(ISD::OR, BoolW, Custom);
190 setOperationAction(ISD::XOR, BoolW, Custom);
191 }
192
193 for (MVT T : LegalV) {
194 MVT BoolV = MVT::getVectorVT(MVT::i1, T.getVectorNumElements());
195 setOperationAction(ISD::BUILD_VECTOR, BoolV, Custom);
196 setOperationAction(ISD::CONCAT_VECTORS, BoolV, Custom);
197 setOperationAction(ISD::INSERT_SUBVECTOR, BoolV, Custom);
198 setOperationAction(ISD::INSERT_VECTOR_ELT, BoolV, Custom);
199 setOperationAction(ISD::EXTRACT_SUBVECTOR, BoolV, Custom);
200 setOperationAction(ISD::EXTRACT_VECTOR_ELT, BoolV, Custom);
201 setOperationAction(ISD::AND, BoolV, Legal);
202 setOperationAction(ISD::OR, BoolV, Legal);
203 setOperationAction(ISD::XOR, BoolV, Legal);
204 }
205
206 if (Use64b) {
207 for (MVT T: {MVT::v32i8, MVT::v32i16, MVT::v16i8, MVT::v16i16, MVT::v16i32})
208 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal);
209 } else {
210 for (MVT T: {MVT::v64i8, MVT::v64i16, MVT::v32i8, MVT::v32i16, MVT::v32i32})
211 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Legal);
212 }
213
214 setTargetDAGCombine(ISD::VSELECT);
215 }
216
217 SDValue
getInt(unsigned IntId,MVT ResTy,ArrayRef<SDValue> Ops,const SDLoc & dl,SelectionDAG & DAG) const218 HexagonTargetLowering::getInt(unsigned IntId, MVT ResTy, ArrayRef<SDValue> Ops,
219 const SDLoc &dl, SelectionDAG &DAG) const {
220 SmallVector<SDValue,4> IntOps;
221 IntOps.push_back(DAG.getConstant(IntId, dl, MVT::i32));
222 for (const SDValue &Op : Ops)
223 IntOps.push_back(Op);
224 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, ResTy, IntOps);
225 }
226
227 MVT
typeJoin(const TypePair & Tys) const228 HexagonTargetLowering::typeJoin(const TypePair &Tys) const {
229 assert(Tys.first.getVectorElementType() == Tys.second.getVectorElementType());
230
231 MVT ElemTy = Tys.first.getVectorElementType();
232 return MVT::getVectorVT(ElemTy, Tys.first.getVectorNumElements() +
233 Tys.second.getVectorNumElements());
234 }
235
236 HexagonTargetLowering::TypePair
typeSplit(MVT VecTy) const237 HexagonTargetLowering::typeSplit(MVT VecTy) const {
238 assert(VecTy.isVector());
239 unsigned NumElem = VecTy.getVectorNumElements();
240 assert((NumElem % 2) == 0 && "Expecting even-sized vector type");
241 MVT HalfTy = MVT::getVectorVT(VecTy.getVectorElementType(), NumElem/2);
242 return { HalfTy, HalfTy };
243 }
244
245 MVT
typeExtElem(MVT VecTy,unsigned Factor) const246 HexagonTargetLowering::typeExtElem(MVT VecTy, unsigned Factor) const {
247 MVT ElemTy = VecTy.getVectorElementType();
248 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() * Factor);
249 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
250 }
251
252 MVT
typeTruncElem(MVT VecTy,unsigned Factor) const253 HexagonTargetLowering::typeTruncElem(MVT VecTy, unsigned Factor) const {
254 MVT ElemTy = VecTy.getVectorElementType();
255 MVT NewElemTy = MVT::getIntegerVT(ElemTy.getSizeInBits() / Factor);
256 return MVT::getVectorVT(NewElemTy, VecTy.getVectorNumElements());
257 }
258
259 SDValue
opCastElem(SDValue Vec,MVT ElemTy,SelectionDAG & DAG) const260 HexagonTargetLowering::opCastElem(SDValue Vec, MVT ElemTy,
261 SelectionDAG &DAG) const {
262 if (ty(Vec).getVectorElementType() == ElemTy)
263 return Vec;
264 MVT CastTy = tyVector(Vec.getValueType().getSimpleVT(), ElemTy);
265 return DAG.getBitcast(CastTy, Vec);
266 }
267
268 SDValue
opJoin(const VectorPair & Ops,const SDLoc & dl,SelectionDAG & DAG) const269 HexagonTargetLowering::opJoin(const VectorPair &Ops, const SDLoc &dl,
270 SelectionDAG &DAG) const {
271 return DAG.getNode(ISD::CONCAT_VECTORS, dl, typeJoin(ty(Ops)),
272 Ops.second, Ops.first);
273 }
274
275 HexagonTargetLowering::VectorPair
opSplit(SDValue Vec,const SDLoc & dl,SelectionDAG & DAG) const276 HexagonTargetLowering::opSplit(SDValue Vec, const SDLoc &dl,
277 SelectionDAG &DAG) const {
278 TypePair Tys = typeSplit(ty(Vec));
279 if (Vec.getOpcode() == HexagonISD::QCAT)
280 return VectorPair(Vec.getOperand(0), Vec.getOperand(1));
281 return DAG.SplitVector(Vec, dl, Tys.first, Tys.second);
282 }
283
284 bool
isHvxSingleTy(MVT Ty) const285 HexagonTargetLowering::isHvxSingleTy(MVT Ty) const {
286 return Subtarget.isHVXVectorType(Ty) &&
287 Ty.getSizeInBits() == 8 * Subtarget.getVectorLength();
288 }
289
290 bool
isHvxPairTy(MVT Ty) const291 HexagonTargetLowering::isHvxPairTy(MVT Ty) const {
292 return Subtarget.isHVXVectorType(Ty) &&
293 Ty.getSizeInBits() == 16 * Subtarget.getVectorLength();
294 }
295
296 bool
isHvxBoolTy(MVT Ty) const297 HexagonTargetLowering::isHvxBoolTy(MVT Ty) const {
298 return Subtarget.isHVXVectorType(Ty, true) &&
299 Ty.getVectorElementType() == MVT::i1;
300 }
301
allowsHvxMemoryAccess(MVT VecTy,MachineMemOperand::Flags Flags,bool * Fast) const302 bool HexagonTargetLowering::allowsHvxMemoryAccess(
303 MVT VecTy, MachineMemOperand::Flags Flags, bool *Fast) const {
304 // Bool vectors are excluded by default, but make it explicit to
305 // emphasize that bool vectors cannot be loaded or stored.
306 // Also, disallow double vector stores (to prevent unnecessary
307 // store widening in DAG combiner).
308 if (VecTy.getSizeInBits() > 8*Subtarget.getVectorLength())
309 return false;
310 if (!Subtarget.isHVXVectorType(VecTy, /*IncludeBool=*/false))
311 return false;
312 if (Fast)
313 *Fast = true;
314 return true;
315 }
316
allowsHvxMisalignedMemoryAccesses(MVT VecTy,MachineMemOperand::Flags Flags,bool * Fast) const317 bool HexagonTargetLowering::allowsHvxMisalignedMemoryAccesses(
318 MVT VecTy, MachineMemOperand::Flags Flags, bool *Fast) const {
319 if (!Subtarget.isHVXVectorType(VecTy))
320 return false;
321 // XXX Should this be false? vmemu are a bit slower than vmem.
322 if (Fast)
323 *Fast = true;
324 return true;
325 }
326
327 SDValue
convertToByteIndex(SDValue ElemIdx,MVT ElemTy,SelectionDAG & DAG) const328 HexagonTargetLowering::convertToByteIndex(SDValue ElemIdx, MVT ElemTy,
329 SelectionDAG &DAG) const {
330 if (ElemIdx.getValueType().getSimpleVT() != MVT::i32)
331 ElemIdx = DAG.getBitcast(MVT::i32, ElemIdx);
332
333 unsigned ElemWidth = ElemTy.getSizeInBits();
334 if (ElemWidth == 8)
335 return ElemIdx;
336
337 unsigned L = Log2_32(ElemWidth/8);
338 const SDLoc &dl(ElemIdx);
339 return DAG.getNode(ISD::SHL, dl, MVT::i32,
340 {ElemIdx, DAG.getConstant(L, dl, MVT::i32)});
341 }
342
343 SDValue
getIndexInWord32(SDValue Idx,MVT ElemTy,SelectionDAG & DAG) const344 HexagonTargetLowering::getIndexInWord32(SDValue Idx, MVT ElemTy,
345 SelectionDAG &DAG) const {
346 unsigned ElemWidth = ElemTy.getSizeInBits();
347 assert(ElemWidth >= 8 && ElemWidth <= 32);
348 if (ElemWidth == 32)
349 return Idx;
350
351 if (ty(Idx) != MVT::i32)
352 Idx = DAG.getBitcast(MVT::i32, Idx);
353 const SDLoc &dl(Idx);
354 SDValue Mask = DAG.getConstant(32/ElemWidth - 1, dl, MVT::i32);
355 SDValue SubIdx = DAG.getNode(ISD::AND, dl, MVT::i32, {Idx, Mask});
356 return SubIdx;
357 }
358
359 SDValue
getByteShuffle(const SDLoc & dl,SDValue Op0,SDValue Op1,ArrayRef<int> Mask,SelectionDAG & DAG) const360 HexagonTargetLowering::getByteShuffle(const SDLoc &dl, SDValue Op0,
361 SDValue Op1, ArrayRef<int> Mask,
362 SelectionDAG &DAG) const {
363 MVT OpTy = ty(Op0);
364 assert(OpTy == ty(Op1));
365
366 MVT ElemTy = OpTy.getVectorElementType();
367 if (ElemTy == MVT::i8)
368 return DAG.getVectorShuffle(OpTy, dl, Op0, Op1, Mask);
369 assert(ElemTy.getSizeInBits() >= 8);
370
371 MVT ResTy = tyVector(OpTy, MVT::i8);
372 unsigned ElemSize = ElemTy.getSizeInBits() / 8;
373
374 SmallVector<int,128> ByteMask;
375 for (int M : Mask) {
376 if (M < 0) {
377 for (unsigned I = 0; I != ElemSize; ++I)
378 ByteMask.push_back(-1);
379 } else {
380 int NewM = M*ElemSize;
381 for (unsigned I = 0; I != ElemSize; ++I)
382 ByteMask.push_back(NewM+I);
383 }
384 }
385 assert(ResTy.getVectorNumElements() == ByteMask.size());
386 return DAG.getVectorShuffle(ResTy, dl, opCastElem(Op0, MVT::i8, DAG),
387 opCastElem(Op1, MVT::i8, DAG), ByteMask);
388 }
389
390 SDValue
buildHvxVectorReg(ArrayRef<SDValue> Values,const SDLoc & dl,MVT VecTy,SelectionDAG & DAG) const391 HexagonTargetLowering::buildHvxVectorReg(ArrayRef<SDValue> Values,
392 const SDLoc &dl, MVT VecTy,
393 SelectionDAG &DAG) const {
394 unsigned VecLen = Values.size();
395 MachineFunction &MF = DAG.getMachineFunction();
396 MVT ElemTy = VecTy.getVectorElementType();
397 unsigned ElemWidth = ElemTy.getSizeInBits();
398 unsigned HwLen = Subtarget.getVectorLength();
399
400 unsigned ElemSize = ElemWidth / 8;
401 assert(ElemSize*VecLen == HwLen);
402 SmallVector<SDValue,32> Words;
403
404 if (VecTy.getVectorElementType() != MVT::i32) {
405 assert((ElemSize == 1 || ElemSize == 2) && "Invalid element size");
406 unsigned OpsPerWord = (ElemSize == 1) ? 4 : 2;
407 MVT PartVT = MVT::getVectorVT(VecTy.getVectorElementType(), OpsPerWord);
408 for (unsigned i = 0; i != VecLen; i += OpsPerWord) {
409 SDValue W = buildVector32(Values.slice(i, OpsPerWord), dl, PartVT, DAG);
410 Words.push_back(DAG.getBitcast(MVT::i32, W));
411 }
412 } else {
413 Words.assign(Values.begin(), Values.end());
414 }
415
416 unsigned NumWords = Words.size();
417 bool IsSplat = true, IsUndef = true;
418 SDValue SplatV;
419 for (unsigned i = 0; i != NumWords && IsSplat; ++i) {
420 if (isUndef(Words[i]))
421 continue;
422 IsUndef = false;
423 if (!SplatV.getNode())
424 SplatV = Words[i];
425 else if (SplatV != Words[i])
426 IsSplat = false;
427 }
428 if (IsUndef)
429 return DAG.getUNDEF(VecTy);
430 if (IsSplat) {
431 assert(SplatV.getNode());
432 auto *IdxN = dyn_cast<ConstantSDNode>(SplatV.getNode());
433 if (IdxN && IdxN->isNullValue())
434 return getZero(dl, VecTy, DAG);
435 return DAG.getNode(HexagonISD::VSPLATW, dl, VecTy, SplatV);
436 }
437
438 // Delay recognizing constant vectors until here, so that we can generate
439 // a vsplat.
440 SmallVector<ConstantInt*, 128> Consts(VecLen);
441 bool AllConst = getBuildVectorConstInts(Values, VecTy, DAG, Consts);
442 if (AllConst) {
443 ArrayRef<Constant*> Tmp((Constant**)Consts.begin(),
444 (Constant**)Consts.end());
445 Constant *CV = ConstantVector::get(Tmp);
446 Align Alignment(HwLen);
447 SDValue CP =
448 LowerConstantPool(DAG.getConstantPool(CV, VecTy, Alignment), DAG);
449 return DAG.getLoad(VecTy, dl, DAG.getEntryNode(), CP,
450 MachinePointerInfo::getConstantPool(MF), Alignment);
451 }
452
453 // A special case is a situation where the vector is built entirely from
454 // elements extracted from another vector. This could be done via a shuffle
455 // more efficiently, but typically, the size of the source vector will not
456 // match the size of the vector being built (which precludes the use of a
457 // shuffle directly).
458 // This only handles a single source vector, and the vector being built
459 // should be of a sub-vector type of the source vector type.
460 auto IsBuildFromExtracts = [this,&Values] (SDValue &SrcVec,
461 SmallVectorImpl<int> &SrcIdx) {
462 SDValue Vec;
463 for (SDValue V : Values) {
464 if (isUndef(V)) {
465 SrcIdx.push_back(-1);
466 continue;
467 }
468 if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
469 return false;
470 // All extracts should come from the same vector.
471 SDValue T = V.getOperand(0);
472 if (Vec.getNode() != nullptr && T.getNode() != Vec.getNode())
473 return false;
474 Vec = T;
475 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V.getOperand(1));
476 if (C == nullptr)
477 return false;
478 int I = C->getSExtValue();
479 assert(I >= 0 && "Negative element index");
480 SrcIdx.push_back(I);
481 }
482 SrcVec = Vec;
483 return true;
484 };
485
486 SmallVector<int,128> ExtIdx;
487 SDValue ExtVec;
488 if (IsBuildFromExtracts(ExtVec, ExtIdx)) {
489 MVT ExtTy = ty(ExtVec);
490 unsigned ExtLen = ExtTy.getVectorNumElements();
491 if (ExtLen == VecLen || ExtLen == 2*VecLen) {
492 // Construct a new shuffle mask that will produce a vector with the same
493 // number of elements as the input vector, and such that the vector we
494 // want will be the initial subvector of it.
495 SmallVector<int,128> Mask;
496 BitVector Used(ExtLen);
497
498 for (int M : ExtIdx) {
499 Mask.push_back(M);
500 if (M >= 0)
501 Used.set(M);
502 }
503 // Fill the rest of the mask with the unused elements of ExtVec in hopes
504 // that it will result in a permutation of ExtVec's elements. It's still
505 // fine if it doesn't (e.g. if undefs are present, or elements are
506 // repeated), but permutations can always be done efficiently via vdelta
507 // and vrdelta.
508 for (unsigned I = 0; I != ExtLen; ++I) {
509 if (Mask.size() == ExtLen)
510 break;
511 if (!Used.test(I))
512 Mask.push_back(I);
513 }
514
515 SDValue S = DAG.getVectorShuffle(ExtTy, dl, ExtVec,
516 DAG.getUNDEF(ExtTy), Mask);
517 if (ExtLen == VecLen)
518 return S;
519 return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, VecTy, S);
520 }
521 }
522
523 // Construct two halves in parallel, then or them together.
524 assert(4*Words.size() == Subtarget.getVectorLength());
525 SDValue HalfV0 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
526 SDValue HalfV1 = getInstr(Hexagon::V6_vd0, dl, VecTy, {}, DAG);
527 SDValue S = DAG.getConstant(4, dl, MVT::i32);
528 for (unsigned i = 0; i != NumWords/2; ++i) {
529 SDValue N = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
530 {HalfV0, Words[i]});
531 SDValue M = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy,
532 {HalfV1, Words[i+NumWords/2]});
533 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {N, S});
534 HalfV1 = DAG.getNode(HexagonISD::VROR, dl, VecTy, {M, S});
535 }
536
537 HalfV0 = DAG.getNode(HexagonISD::VROR, dl, VecTy,
538 {HalfV0, DAG.getConstant(HwLen/2, dl, MVT::i32)});
539 SDValue DstV = DAG.getNode(ISD::OR, dl, VecTy, {HalfV0, HalfV1});
540 return DstV;
541 }
542
543 SDValue
createHvxPrefixPred(SDValue PredV,const SDLoc & dl,unsigned BitBytes,bool ZeroFill,SelectionDAG & DAG) const544 HexagonTargetLowering::createHvxPrefixPred(SDValue PredV, const SDLoc &dl,
545 unsigned BitBytes, bool ZeroFill, SelectionDAG &DAG) const {
546 MVT PredTy = ty(PredV);
547 unsigned HwLen = Subtarget.getVectorLength();
548 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
549
550 if (Subtarget.isHVXVectorType(PredTy, true)) {
551 // Move the vector predicate SubV to a vector register, and scale it
552 // down to match the representation (bytes per type element) that VecV
553 // uses. The scaling down will pick every 2nd or 4th (every Scale-th
554 // in general) element and put them at the front of the resulting
555 // vector. This subvector will then be inserted into the Q2V of VecV.
556 // To avoid having an operation that generates an illegal type (short
557 // vector), generate a full size vector.
558 //
559 SDValue T = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, PredV);
560 SmallVector<int,128> Mask(HwLen);
561 // Scale = BitBytes(PredV) / Given BitBytes.
562 unsigned Scale = HwLen / (PredTy.getVectorNumElements() * BitBytes);
563 unsigned BlockLen = PredTy.getVectorNumElements() * BitBytes;
564
565 for (unsigned i = 0; i != HwLen; ++i) {
566 unsigned Num = i % Scale;
567 unsigned Off = i / Scale;
568 Mask[BlockLen*Num + Off] = i;
569 }
570 SDValue S = DAG.getVectorShuffle(ByteTy, dl, T, DAG.getUNDEF(ByteTy), Mask);
571 if (!ZeroFill)
572 return S;
573 // Fill the bytes beyond BlockLen with 0s.
574 MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
575 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
576 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
577 SDValue M = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, Q);
578 return DAG.getNode(ISD::AND, dl, ByteTy, S, M);
579 }
580
581 // Make sure that this is a valid scalar predicate.
582 assert(PredTy == MVT::v2i1 || PredTy == MVT::v4i1 || PredTy == MVT::v8i1);
583
584 unsigned Bytes = 8 / PredTy.getVectorNumElements();
585 SmallVector<SDValue,4> Words[2];
586 unsigned IdxW = 0;
587
588 auto Lo32 = [&DAG, &dl] (SDValue P) {
589 return DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, P);
590 };
591 auto Hi32 = [&DAG, &dl] (SDValue P) {
592 return DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, P);
593 };
594
595 SDValue W0 = isUndef(PredV)
596 ? DAG.getUNDEF(MVT::i64)
597 : DAG.getNode(HexagonISD::P2D, dl, MVT::i64, PredV);
598 Words[IdxW].push_back(Hi32(W0));
599 Words[IdxW].push_back(Lo32(W0));
600
601 while (Bytes < BitBytes) {
602 IdxW ^= 1;
603 Words[IdxW].clear();
604
605 if (Bytes < 4) {
606 for (const SDValue &W : Words[IdxW ^ 1]) {
607 SDValue T = expandPredicate(W, dl, DAG);
608 Words[IdxW].push_back(Hi32(T));
609 Words[IdxW].push_back(Lo32(T));
610 }
611 } else {
612 for (const SDValue &W : Words[IdxW ^ 1]) {
613 Words[IdxW].push_back(W);
614 Words[IdxW].push_back(W);
615 }
616 }
617 Bytes *= 2;
618 }
619
620 assert(Bytes == BitBytes);
621
622 SDValue Vec = ZeroFill ? getZero(dl, ByteTy, DAG) : DAG.getUNDEF(ByteTy);
623 SDValue S4 = DAG.getConstant(HwLen-4, dl, MVT::i32);
624 for (const SDValue &W : Words[IdxW]) {
625 Vec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Vec, S4);
626 Vec = DAG.getNode(HexagonISD::VINSERTW0, dl, ByteTy, Vec, W);
627 }
628
629 return Vec;
630 }
631
632 SDValue
buildHvxVectorPred(ArrayRef<SDValue> Values,const SDLoc & dl,MVT VecTy,SelectionDAG & DAG) const633 HexagonTargetLowering::buildHvxVectorPred(ArrayRef<SDValue> Values,
634 const SDLoc &dl, MVT VecTy,
635 SelectionDAG &DAG) const {
636 // Construct a vector V of bytes, such that a comparison V >u 0 would
637 // produce the required vector predicate.
638 unsigned VecLen = Values.size();
639 unsigned HwLen = Subtarget.getVectorLength();
640 assert(VecLen <= HwLen || VecLen == 8*HwLen);
641 SmallVector<SDValue,128> Bytes;
642 bool AllT = true, AllF = true;
643
644 auto IsTrue = [] (SDValue V) {
645 if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
646 return !N->isNullValue();
647 return false;
648 };
649 auto IsFalse = [] (SDValue V) {
650 if (const auto *N = dyn_cast<ConstantSDNode>(V.getNode()))
651 return N->isNullValue();
652 return false;
653 };
654
655 if (VecLen <= HwLen) {
656 // In the hardware, each bit of a vector predicate corresponds to a byte
657 // of a vector register. Calculate how many bytes does a bit of VecTy
658 // correspond to.
659 assert(HwLen % VecLen == 0);
660 unsigned BitBytes = HwLen / VecLen;
661 for (SDValue V : Values) {
662 AllT &= IsTrue(V);
663 AllF &= IsFalse(V);
664
665 SDValue Ext = !V.isUndef() ? DAG.getZExtOrTrunc(V, dl, MVT::i8)
666 : DAG.getUNDEF(MVT::i8);
667 for (unsigned B = 0; B != BitBytes; ++B)
668 Bytes.push_back(Ext);
669 }
670 } else {
671 // There are as many i1 values, as there are bits in a vector register.
672 // Divide the values into groups of 8 and check that each group consists
673 // of the same value (ignoring undefs).
674 for (unsigned I = 0; I != VecLen; I += 8) {
675 unsigned B = 0;
676 // Find the first non-undef value in this group.
677 for (; B != 8; ++B) {
678 if (!Values[I+B].isUndef())
679 break;
680 }
681 SDValue F = Values[I+B];
682 AllT &= IsTrue(F);
683 AllF &= IsFalse(F);
684
685 SDValue Ext = (B < 8) ? DAG.getZExtOrTrunc(F, dl, MVT::i8)
686 : DAG.getUNDEF(MVT::i8);
687 Bytes.push_back(Ext);
688 // Verify that the rest of values in the group are the same as the
689 // first.
690 for (; B != 8; ++B)
691 assert(Values[I+B].isUndef() || Values[I+B] == F);
692 }
693 }
694
695 if (AllT)
696 return DAG.getNode(HexagonISD::QTRUE, dl, VecTy);
697 if (AllF)
698 return DAG.getNode(HexagonISD::QFALSE, dl, VecTy);
699
700 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
701 SDValue ByteVec = buildHvxVectorReg(Bytes, dl, ByteTy, DAG);
702 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
703 }
704
705 SDValue
extractHvxElementReg(SDValue VecV,SDValue IdxV,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const706 HexagonTargetLowering::extractHvxElementReg(SDValue VecV, SDValue IdxV,
707 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
708 MVT ElemTy = ty(VecV).getVectorElementType();
709
710 unsigned ElemWidth = ElemTy.getSizeInBits();
711 assert(ElemWidth >= 8 && ElemWidth <= 32);
712 (void)ElemWidth;
713
714 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
715 SDValue ExWord = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
716 {VecV, ByteIdx});
717 if (ElemTy == MVT::i32)
718 return ExWord;
719
720 // Have an extracted word, need to extract the smaller element out of it.
721 // 1. Extract the bits of (the original) IdxV that correspond to the index
722 // of the desired element in the 32-bit word.
723 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
724 // 2. Extract the element from the word.
725 SDValue ExVec = DAG.getBitcast(tyVector(ty(ExWord), ElemTy), ExWord);
726 return extractVector(ExVec, SubIdx, dl, ElemTy, MVT::i32, DAG);
727 }
728
729 SDValue
extractHvxElementPred(SDValue VecV,SDValue IdxV,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const730 HexagonTargetLowering::extractHvxElementPred(SDValue VecV, SDValue IdxV,
731 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
732 // Implement other return types if necessary.
733 assert(ResTy == MVT::i1);
734
735 unsigned HwLen = Subtarget.getVectorLength();
736 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
737 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
738
739 unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
740 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
741 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
742
743 SDValue ExtB = extractHvxElementReg(ByteVec, IdxV, dl, MVT::i32, DAG);
744 SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
745 return getInstr(Hexagon::C2_cmpgtui, dl, MVT::i1, {ExtB, Zero}, DAG);
746 }
747
748 SDValue
insertHvxElementReg(SDValue VecV,SDValue IdxV,SDValue ValV,const SDLoc & dl,SelectionDAG & DAG) const749 HexagonTargetLowering::insertHvxElementReg(SDValue VecV, SDValue IdxV,
750 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
751 MVT ElemTy = ty(VecV).getVectorElementType();
752
753 unsigned ElemWidth = ElemTy.getSizeInBits();
754 assert(ElemWidth >= 8 && ElemWidth <= 32);
755 (void)ElemWidth;
756
757 auto InsertWord = [&DAG,&dl,this] (SDValue VecV, SDValue ValV,
758 SDValue ByteIdxV) {
759 MVT VecTy = ty(VecV);
760 unsigned HwLen = Subtarget.getVectorLength();
761 SDValue MaskV = DAG.getNode(ISD::AND, dl, MVT::i32,
762 {ByteIdxV, DAG.getConstant(-4, dl, MVT::i32)});
763 SDValue RotV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {VecV, MaskV});
764 SDValue InsV = DAG.getNode(HexagonISD::VINSERTW0, dl, VecTy, {RotV, ValV});
765 SDValue SubV = DAG.getNode(ISD::SUB, dl, MVT::i32,
766 {DAG.getConstant(HwLen, dl, MVT::i32), MaskV});
767 SDValue TorV = DAG.getNode(HexagonISD::VROR, dl, VecTy, {InsV, SubV});
768 return TorV;
769 };
770
771 SDValue ByteIdx = convertToByteIndex(IdxV, ElemTy, DAG);
772 if (ElemTy == MVT::i32)
773 return InsertWord(VecV, ValV, ByteIdx);
774
775 // If this is not inserting a 32-bit word, convert it into such a thing.
776 // 1. Extract the existing word from the target vector.
777 SDValue WordIdx = DAG.getNode(ISD::SRL, dl, MVT::i32,
778 {ByteIdx, DAG.getConstant(2, dl, MVT::i32)});
779 SDValue Ext = extractHvxElementReg(opCastElem(VecV, MVT::i32, DAG), WordIdx,
780 dl, MVT::i32, DAG);
781
782 // 2. Treating the extracted word as a 32-bit vector, insert the given
783 // value into it.
784 SDValue SubIdx = getIndexInWord32(IdxV, ElemTy, DAG);
785 MVT SubVecTy = tyVector(ty(Ext), ElemTy);
786 SDValue Ins = insertVector(DAG.getBitcast(SubVecTy, Ext),
787 ValV, SubIdx, dl, ElemTy, DAG);
788
789 // 3. Insert the 32-bit word back into the original vector.
790 return InsertWord(VecV, Ins, ByteIdx);
791 }
792
793 SDValue
insertHvxElementPred(SDValue VecV,SDValue IdxV,SDValue ValV,const SDLoc & dl,SelectionDAG & DAG) const794 HexagonTargetLowering::insertHvxElementPred(SDValue VecV, SDValue IdxV,
795 SDValue ValV, const SDLoc &dl, SelectionDAG &DAG) const {
796 unsigned HwLen = Subtarget.getVectorLength();
797 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
798 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
799
800 unsigned Scale = HwLen / ty(VecV).getVectorNumElements();
801 SDValue ScV = DAG.getConstant(Scale, dl, MVT::i32);
802 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV, ScV);
803 ValV = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, ValV);
804
805 SDValue InsV = insertHvxElementReg(ByteVec, IdxV, ValV, dl, DAG);
806 return DAG.getNode(HexagonISD::V2Q, dl, ty(VecV), InsV);
807 }
808
809 SDValue
extractHvxSubvectorReg(SDValue VecV,SDValue IdxV,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const810 HexagonTargetLowering::extractHvxSubvectorReg(SDValue VecV, SDValue IdxV,
811 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
812 MVT VecTy = ty(VecV);
813 unsigned HwLen = Subtarget.getVectorLength();
814 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
815 MVT ElemTy = VecTy.getVectorElementType();
816 unsigned ElemWidth = ElemTy.getSizeInBits();
817
818 // If the source vector is a vector pair, get the single vector containing
819 // the subvector of interest. The subvector will never overlap two single
820 // vectors.
821 if (isHvxPairTy(VecTy)) {
822 unsigned SubIdx;
823 if (Idx * ElemWidth >= 8*HwLen) {
824 SubIdx = Hexagon::vsub_hi;
825 Idx -= VecTy.getVectorNumElements() / 2;
826 } else {
827 SubIdx = Hexagon::vsub_lo;
828 }
829 VecTy = typeSplit(VecTy).first;
830 VecV = DAG.getTargetExtractSubreg(SubIdx, dl, VecTy, VecV);
831 if (VecTy == ResTy)
832 return VecV;
833 }
834
835 // The only meaningful subvectors of a single HVX vector are those that
836 // fit in a scalar register.
837 assert(ResTy.getSizeInBits() == 32 || ResTy.getSizeInBits() == 64);
838
839 MVT WordTy = tyVector(VecTy, MVT::i32);
840 SDValue WordVec = DAG.getBitcast(WordTy, VecV);
841 unsigned WordIdx = (Idx*ElemWidth) / 32;
842
843 SDValue W0Idx = DAG.getConstant(WordIdx, dl, MVT::i32);
844 SDValue W0 = extractHvxElementReg(WordVec, W0Idx, dl, MVT::i32, DAG);
845 if (ResTy.getSizeInBits() == 32)
846 return DAG.getBitcast(ResTy, W0);
847
848 SDValue W1Idx = DAG.getConstant(WordIdx+1, dl, MVT::i32);
849 SDValue W1 = extractHvxElementReg(WordVec, W1Idx, dl, MVT::i32, DAG);
850 SDValue WW = DAG.getNode(HexagonISD::COMBINE, dl, MVT::i64, {W1, W0});
851 return DAG.getBitcast(ResTy, WW);
852 }
853
854 SDValue
extractHvxSubvectorPred(SDValue VecV,SDValue IdxV,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const855 HexagonTargetLowering::extractHvxSubvectorPred(SDValue VecV, SDValue IdxV,
856 const SDLoc &dl, MVT ResTy, SelectionDAG &DAG) const {
857 MVT VecTy = ty(VecV);
858 unsigned HwLen = Subtarget.getVectorLength();
859 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
860 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
861 // IdxV is required to be a constant.
862 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
863
864 unsigned ResLen = ResTy.getVectorNumElements();
865 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
866 unsigned Offset = Idx * BitBytes;
867 SDValue Undef = DAG.getUNDEF(ByteTy);
868 SmallVector<int,128> Mask;
869
870 if (Subtarget.isHVXVectorType(ResTy, true)) {
871 // Converting between two vector predicates. Since the result is shorter
872 // than the source, it will correspond to a vector predicate with the
873 // relevant bits replicated. The replication count is the ratio of the
874 // source and target vector lengths.
875 unsigned Rep = VecTy.getVectorNumElements() / ResLen;
876 assert(isPowerOf2_32(Rep) && HwLen % Rep == 0);
877 for (unsigned i = 0; i != HwLen/Rep; ++i) {
878 for (unsigned j = 0; j != Rep; ++j)
879 Mask.push_back(i + Offset);
880 }
881 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
882 return DAG.getNode(HexagonISD::V2Q, dl, ResTy, ShuffV);
883 }
884
885 // Converting between a vector predicate and a scalar predicate. In the
886 // vector predicate, a group of BitBytes bits will correspond to a single
887 // i1 element of the source vector type. Those bits will all have the same
888 // value. The same will be true for ByteVec, where each byte corresponds
889 // to a bit in the vector predicate.
890 // The algorithm is to traverse the ByteVec, going over the i1 values from
891 // the source vector, and generate the corresponding representation in an
892 // 8-byte vector. To avoid repeated extracts from ByteVec, shuffle the
893 // elements so that the interesting 8 bytes will be in the low end of the
894 // vector.
895 unsigned Rep = 8 / ResLen;
896 // Make sure the output fill the entire vector register, so repeat the
897 // 8-byte groups as many times as necessary.
898 for (unsigned r = 0; r != HwLen/ResLen; ++r) {
899 // This will generate the indexes of the 8 interesting bytes.
900 for (unsigned i = 0; i != ResLen; ++i) {
901 for (unsigned j = 0; j != Rep; ++j)
902 Mask.push_back(Offset + i*BitBytes);
903 }
904 }
905
906 SDValue Zero = getZero(dl, MVT::i32, DAG);
907 SDValue ShuffV = DAG.getVectorShuffle(ByteTy, dl, ByteVec, Undef, Mask);
908 // Combine the two low words from ShuffV into a v8i8, and byte-compare
909 // them against 0.
910 SDValue W0 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32, {ShuffV, Zero});
911 SDValue W1 = DAG.getNode(HexagonISD::VEXTRACTW, dl, MVT::i32,
912 {ShuffV, DAG.getConstant(4, dl, MVT::i32)});
913 SDValue Vec64 = DAG.getNode(HexagonISD::COMBINE, dl, MVT::v8i8, {W1, W0});
914 return getInstr(Hexagon::A4_vcmpbgtui, dl, ResTy,
915 {Vec64, DAG.getTargetConstant(0, dl, MVT::i32)}, DAG);
916 }
917
918 SDValue
insertHvxSubvectorReg(SDValue VecV,SDValue SubV,SDValue IdxV,const SDLoc & dl,SelectionDAG & DAG) const919 HexagonTargetLowering::insertHvxSubvectorReg(SDValue VecV, SDValue SubV,
920 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
921 MVT VecTy = ty(VecV);
922 MVT SubTy = ty(SubV);
923 unsigned HwLen = Subtarget.getVectorLength();
924 MVT ElemTy = VecTy.getVectorElementType();
925 unsigned ElemWidth = ElemTy.getSizeInBits();
926
927 bool IsPair = isHvxPairTy(VecTy);
928 MVT SingleTy = MVT::getVectorVT(ElemTy, (8*HwLen)/ElemWidth);
929 // The two single vectors that VecV consists of, if it's a pair.
930 SDValue V0, V1;
931 SDValue SingleV = VecV;
932 SDValue PickHi;
933
934 if (IsPair) {
935 V0 = DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, SingleTy, VecV);
936 V1 = DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, SingleTy, VecV);
937
938 SDValue HalfV = DAG.getConstant(SingleTy.getVectorNumElements(),
939 dl, MVT::i32);
940 PickHi = DAG.getSetCC(dl, MVT::i1, IdxV, HalfV, ISD::SETUGT);
941 if (isHvxSingleTy(SubTy)) {
942 if (const auto *CN = dyn_cast<const ConstantSDNode>(IdxV.getNode())) {
943 unsigned Idx = CN->getZExtValue();
944 assert(Idx == 0 || Idx == VecTy.getVectorNumElements()/2);
945 unsigned SubIdx = (Idx == 0) ? Hexagon::vsub_lo : Hexagon::vsub_hi;
946 return DAG.getTargetInsertSubreg(SubIdx, dl, VecTy, VecV, SubV);
947 }
948 // If IdxV is not a constant, generate the two variants: with the
949 // SubV as the high and as the low subregister, and select the right
950 // pair based on the IdxV.
951 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SubV, V1});
952 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SubV});
953 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
954 }
955 // The subvector being inserted must be entirely contained in one of
956 // the vectors V0 or V1. Set SingleV to the correct one, and update
957 // IdxV to be the index relative to the beginning of that vector.
958 SDValue S = DAG.getNode(ISD::SUB, dl, MVT::i32, IdxV, HalfV);
959 IdxV = DAG.getNode(ISD::SELECT, dl, MVT::i32, PickHi, S, IdxV);
960 SingleV = DAG.getNode(ISD::SELECT, dl, SingleTy, PickHi, V1, V0);
961 }
962
963 // The only meaningful subvectors of a single HVX vector are those that
964 // fit in a scalar register.
965 assert(SubTy.getSizeInBits() == 32 || SubTy.getSizeInBits() == 64);
966 // Convert IdxV to be index in bytes.
967 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
968 if (!IdxN || !IdxN->isNullValue()) {
969 IdxV = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
970 DAG.getConstant(ElemWidth/8, dl, MVT::i32));
971 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, IdxV);
972 }
973 // When inserting a single word, the rotation back to the original position
974 // would be by HwLen-Idx, but if two words are inserted, it will need to be
975 // by (HwLen-4)-Idx.
976 unsigned RolBase = HwLen;
977 if (VecTy.getSizeInBits() == 32) {
978 SDValue V = DAG.getBitcast(MVT::i32, SubV);
979 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, V);
980 } else {
981 SDValue V = DAG.getBitcast(MVT::i64, SubV);
982 SDValue R0 = DAG.getTargetExtractSubreg(Hexagon::isub_lo, dl, MVT::i32, V);
983 SDValue R1 = DAG.getTargetExtractSubreg(Hexagon::isub_hi, dl, MVT::i32, V);
984 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R0);
985 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV,
986 DAG.getConstant(4, dl, MVT::i32));
987 SingleV = DAG.getNode(HexagonISD::VINSERTW0, dl, SingleTy, SingleV, R1);
988 RolBase = HwLen-4;
989 }
990 // If the vector wasn't ror'ed, don't ror it back.
991 if (RolBase != 4 || !IdxN || !IdxN->isNullValue()) {
992 SDValue RolV = DAG.getNode(ISD::SUB, dl, MVT::i32,
993 DAG.getConstant(RolBase, dl, MVT::i32), IdxV);
994 SingleV = DAG.getNode(HexagonISD::VROR, dl, SingleTy, SingleV, RolV);
995 }
996
997 if (IsPair) {
998 SDValue InLo = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {SingleV, V1});
999 SDValue InHi = DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, {V0, SingleV});
1000 return DAG.getNode(ISD::SELECT, dl, VecTy, PickHi, InHi, InLo);
1001 }
1002 return SingleV;
1003 }
1004
1005 SDValue
insertHvxSubvectorPred(SDValue VecV,SDValue SubV,SDValue IdxV,const SDLoc & dl,SelectionDAG & DAG) const1006 HexagonTargetLowering::insertHvxSubvectorPred(SDValue VecV, SDValue SubV,
1007 SDValue IdxV, const SDLoc &dl, SelectionDAG &DAG) const {
1008 MVT VecTy = ty(VecV);
1009 MVT SubTy = ty(SubV);
1010 assert(Subtarget.isHVXVectorType(VecTy, true));
1011 // VecV is an HVX vector predicate. SubV may be either an HVX vector
1012 // predicate as well, or it can be a scalar predicate.
1013
1014 unsigned VecLen = VecTy.getVectorNumElements();
1015 unsigned HwLen = Subtarget.getVectorLength();
1016 assert(HwLen % VecLen == 0 && "Unexpected vector type");
1017
1018 unsigned Scale = VecLen / SubTy.getVectorNumElements();
1019 unsigned BitBytes = HwLen / VecLen;
1020 unsigned BlockLen = HwLen / Scale;
1021
1022 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1023 SDValue ByteVec = DAG.getNode(HexagonISD::Q2V, dl, ByteTy, VecV);
1024 SDValue ByteSub = createHvxPrefixPred(SubV, dl, BitBytes, false, DAG);
1025 SDValue ByteIdx;
1026
1027 auto *IdxN = dyn_cast<ConstantSDNode>(IdxV.getNode());
1028 if (!IdxN || !IdxN->isNullValue()) {
1029 ByteIdx = DAG.getNode(ISD::MUL, dl, MVT::i32, IdxV,
1030 DAG.getConstant(BitBytes, dl, MVT::i32));
1031 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteIdx);
1032 }
1033
1034 // ByteVec is the target vector VecV rotated in such a way that the
1035 // subvector should be inserted at index 0. Generate a predicate mask
1036 // and use vmux to do the insertion.
1037 MVT BoolTy = MVT::getVectorVT(MVT::i1, HwLen);
1038 SDValue Q = getInstr(Hexagon::V6_pred_scalar2, dl, BoolTy,
1039 {DAG.getConstant(BlockLen, dl, MVT::i32)}, DAG);
1040 ByteVec = getInstr(Hexagon::V6_vmux, dl, ByteTy, {Q, ByteSub, ByteVec}, DAG);
1041 // Rotate ByteVec back, and convert to a vector predicate.
1042 if (!IdxN || !IdxN->isNullValue()) {
1043 SDValue HwLenV = DAG.getConstant(HwLen, dl, MVT::i32);
1044 SDValue ByteXdi = DAG.getNode(ISD::SUB, dl, MVT::i32, HwLenV, ByteIdx);
1045 ByteVec = DAG.getNode(HexagonISD::VROR, dl, ByteTy, ByteVec, ByteXdi);
1046 }
1047 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, ByteVec);
1048 }
1049
1050 SDValue
extendHvxVectorPred(SDValue VecV,const SDLoc & dl,MVT ResTy,bool ZeroExt,SelectionDAG & DAG) const1051 HexagonTargetLowering::extendHvxVectorPred(SDValue VecV, const SDLoc &dl,
1052 MVT ResTy, bool ZeroExt, SelectionDAG &DAG) const {
1053 // Sign- and any-extending of a vector predicate to a vector register is
1054 // equivalent to Q2V. For zero-extensions, generate a vmux between 0 and
1055 // a vector of 1s (where the 1s are of type matching the vector type).
1056 assert(Subtarget.isHVXVectorType(ResTy));
1057 if (!ZeroExt)
1058 return DAG.getNode(HexagonISD::Q2V, dl, ResTy, VecV);
1059
1060 assert(ty(VecV).getVectorNumElements() == ResTy.getVectorNumElements());
1061 SDValue True = DAG.getNode(HexagonISD::VSPLAT, dl, ResTy,
1062 DAG.getConstant(1, dl, MVT::i32));
1063 SDValue False = getZero(dl, ResTy, DAG);
1064 return DAG.getSelect(dl, ResTy, VecV, True, False);
1065 }
1066
1067 SDValue
compressHvxPred(SDValue VecQ,const SDLoc & dl,MVT ResTy,SelectionDAG & DAG) const1068 HexagonTargetLowering::compressHvxPred(SDValue VecQ, const SDLoc &dl,
1069 MVT ResTy, SelectionDAG &DAG) const {
1070 // Given a predicate register VecQ, transfer bits VecQ[0..HwLen-1]
1071 // (i.e. the entire predicate register) to bits [0..HwLen-1] of a
1072 // vector register. The remaining bits of the vector register are
1073 // unspecified.
1074
1075 MachineFunction &MF = DAG.getMachineFunction();
1076 unsigned HwLen = Subtarget.getVectorLength();
1077 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1078 MVT PredTy = ty(VecQ);
1079 unsigned PredLen = PredTy.getVectorNumElements();
1080 assert(HwLen % PredLen == 0);
1081 MVT VecTy = MVT::getVectorVT(MVT::getIntegerVT(8*HwLen/PredLen), PredLen);
1082
1083 Type *Int8Ty = Type::getInt8Ty(*DAG.getContext());
1084 SmallVector<Constant*, 128> Tmp;
1085 // Create an array of bytes (hex): 01,02,04,08,10,20,40,80, 01,02,04,08,...
1086 // These are bytes with the LSB rotated left with respect to their index.
1087 for (unsigned i = 0; i != HwLen/8; ++i) {
1088 for (unsigned j = 0; j != 8; ++j)
1089 Tmp.push_back(ConstantInt::get(Int8Ty, 1ull << j));
1090 }
1091 Constant *CV = ConstantVector::get(Tmp);
1092 Align Alignment(HwLen);
1093 SDValue CP =
1094 LowerConstantPool(DAG.getConstantPool(CV, ByteTy, Alignment), DAG);
1095 SDValue Bytes =
1096 DAG.getLoad(ByteTy, dl, DAG.getEntryNode(), CP,
1097 MachinePointerInfo::getConstantPool(MF), Alignment);
1098
1099 // Select the bytes that correspond to true bits in the vector predicate.
1100 SDValue Sel = DAG.getSelect(dl, VecTy, VecQ, DAG.getBitcast(VecTy, Bytes),
1101 getZero(dl, VecTy, DAG));
1102 // Calculate the OR of all bytes in each group of 8. That will compress
1103 // all the individual bits into a single byte.
1104 // First, OR groups of 4, via vrmpy with 0x01010101.
1105 SDValue All1 =
1106 DAG.getSplatBuildVector(MVT::v4i8, dl, DAG.getConstant(1, dl, MVT::i32));
1107 SDValue Vrmpy = getInstr(Hexagon::V6_vrmpyub, dl, ByteTy, {Sel, All1}, DAG);
1108 // Then rotate the accumulated vector by 4 bytes, and do the final OR.
1109 SDValue Rot = getInstr(Hexagon::V6_valignbi, dl, ByteTy,
1110 {Vrmpy, Vrmpy, DAG.getTargetConstant(4, dl, MVT::i32)}, DAG);
1111 SDValue Vor = DAG.getNode(ISD::OR, dl, ByteTy, {Vrmpy, Rot});
1112
1113 // Pick every 8th byte and coalesce them at the beginning of the output.
1114 // For symmetry, coalesce every 1+8th byte after that, then every 2+8th
1115 // byte and so on.
1116 SmallVector<int,128> Mask;
1117 for (unsigned i = 0; i != HwLen; ++i)
1118 Mask.push_back((8*i) % HwLen + i/(HwLen/8));
1119 SDValue Collect =
1120 DAG.getVectorShuffle(ByteTy, dl, Vor, DAG.getUNDEF(ByteTy), Mask);
1121 return DAG.getBitcast(ResTy, Collect);
1122 }
1123
1124 SDValue
LowerHvxBuildVector(SDValue Op,SelectionDAG & DAG) const1125 HexagonTargetLowering::LowerHvxBuildVector(SDValue Op, SelectionDAG &DAG)
1126 const {
1127 const SDLoc &dl(Op);
1128 MVT VecTy = ty(Op);
1129
1130 unsigned Size = Op.getNumOperands();
1131 SmallVector<SDValue,128> Ops;
1132 for (unsigned i = 0; i != Size; ++i)
1133 Ops.push_back(Op.getOperand(i));
1134
1135 if (VecTy.getVectorElementType() == MVT::i1)
1136 return buildHvxVectorPred(Ops, dl, VecTy, DAG);
1137
1138 if (VecTy.getSizeInBits() == 16*Subtarget.getVectorLength()) {
1139 ArrayRef<SDValue> A(Ops);
1140 MVT SingleTy = typeSplit(VecTy).first;
1141 SDValue V0 = buildHvxVectorReg(A.take_front(Size/2), dl, SingleTy, DAG);
1142 SDValue V1 = buildHvxVectorReg(A.drop_front(Size/2), dl, SingleTy, DAG);
1143 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VecTy, V0, V1);
1144 }
1145
1146 return buildHvxVectorReg(Ops, dl, VecTy, DAG);
1147 }
1148
1149 SDValue
LowerHvxConcatVectors(SDValue Op,SelectionDAG & DAG) const1150 HexagonTargetLowering::LowerHvxConcatVectors(SDValue Op, SelectionDAG &DAG)
1151 const {
1152 // Vector concatenation of two integer (non-bool) vectors does not need
1153 // special lowering. Custom-lower concats of bool vectors and expand
1154 // concats of more than 2 vectors.
1155 MVT VecTy = ty(Op);
1156 const SDLoc &dl(Op);
1157 unsigned NumOp = Op.getNumOperands();
1158 if (VecTy.getVectorElementType() != MVT::i1) {
1159 if (NumOp == 2)
1160 return Op;
1161 // Expand the other cases into a build-vector.
1162 SmallVector<SDValue,8> Elems;
1163 for (SDValue V : Op.getNode()->ops())
1164 DAG.ExtractVectorElements(V, Elems);
1165 // A vector of i16 will be broken up into a build_vector of i16's.
1166 // This is a problem, since at the time of operation legalization,
1167 // all operations are expected to be type-legalized, and i16 is not
1168 // a legal type. If any of the extracted elements is not of a valid
1169 // type, sign-extend it to a valid one.
1170 for (unsigned i = 0, e = Elems.size(); i != e; ++i) {
1171 SDValue V = Elems[i];
1172 MVT Ty = ty(V);
1173 if (!isTypeLegal(Ty)) {
1174 EVT NTy = getTypeToTransformTo(*DAG.getContext(), Ty);
1175 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1176 Elems[i] = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NTy,
1177 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NTy,
1178 V.getOperand(0), V.getOperand(1)),
1179 DAG.getValueType(Ty));
1180 continue;
1181 }
1182 // A few less complicated cases.
1183 if (V.getOpcode() == ISD::Constant)
1184 Elems[i] = DAG.getSExtOrTrunc(V, dl, NTy);
1185 else if (V.isUndef())
1186 Elems[i] = DAG.getUNDEF(NTy);
1187 else
1188 llvm_unreachable("Unexpected vector element");
1189 }
1190 }
1191 return DAG.getBuildVector(VecTy, dl, Elems);
1192 }
1193
1194 assert(VecTy.getVectorElementType() == MVT::i1);
1195 unsigned HwLen = Subtarget.getVectorLength();
1196 assert(isPowerOf2_32(NumOp) && HwLen % NumOp == 0);
1197
1198 SDValue Op0 = Op.getOperand(0);
1199
1200 // If the operands are HVX types (i.e. not scalar predicates), then
1201 // defer the concatenation, and create QCAT instead.
1202 if (Subtarget.isHVXVectorType(ty(Op0), true)) {
1203 if (NumOp == 2)
1204 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, Op0, Op.getOperand(1));
1205
1206 ArrayRef<SDUse> U(Op.getNode()->ops());
1207 SmallVector<SDValue,4> SV(U.begin(), U.end());
1208 ArrayRef<SDValue> Ops(SV);
1209
1210 MVT HalfTy = typeSplit(VecTy).first;
1211 SDValue V0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1212 Ops.take_front(NumOp/2));
1213 SDValue V1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfTy,
1214 Ops.take_back(NumOp/2));
1215 return DAG.getNode(HexagonISD::QCAT, dl, VecTy, V0, V1);
1216 }
1217
1218 // Count how many bytes (in a vector register) each bit in VecTy
1219 // corresponds to.
1220 unsigned BitBytes = HwLen / VecTy.getVectorNumElements();
1221
1222 SmallVector<SDValue,8> Prefixes;
1223 for (SDValue V : Op.getNode()->op_values()) {
1224 SDValue P = createHvxPrefixPred(V, dl, BitBytes, true, DAG);
1225 Prefixes.push_back(P);
1226 }
1227
1228 unsigned InpLen = ty(Op.getOperand(0)).getVectorNumElements();
1229 MVT ByteTy = MVT::getVectorVT(MVT::i8, HwLen);
1230 SDValue S = DAG.getConstant(InpLen*BitBytes, dl, MVT::i32);
1231 SDValue Res = getZero(dl, ByteTy, DAG);
1232 for (unsigned i = 0, e = Prefixes.size(); i != e; ++i) {
1233 Res = DAG.getNode(HexagonISD::VROR, dl, ByteTy, Res, S);
1234 Res = DAG.getNode(ISD::OR, dl, ByteTy, Res, Prefixes[e-i-1]);
1235 }
1236 return DAG.getNode(HexagonISD::V2Q, dl, VecTy, Res);
1237 }
1238
1239 SDValue
LowerHvxExtractElement(SDValue Op,SelectionDAG & DAG) const1240 HexagonTargetLowering::LowerHvxExtractElement(SDValue Op, SelectionDAG &DAG)
1241 const {
1242 // Change the type of the extracted element to i32.
1243 SDValue VecV = Op.getOperand(0);
1244 MVT ElemTy = ty(VecV).getVectorElementType();
1245 const SDLoc &dl(Op);
1246 SDValue IdxV = Op.getOperand(1);
1247 if (ElemTy == MVT::i1)
1248 return extractHvxElementPred(VecV, IdxV, dl, ty(Op), DAG);
1249
1250 return extractHvxElementReg(VecV, IdxV, dl, ty(Op), DAG);
1251 }
1252
1253 SDValue
LowerHvxInsertElement(SDValue Op,SelectionDAG & DAG) const1254 HexagonTargetLowering::LowerHvxInsertElement(SDValue Op, SelectionDAG &DAG)
1255 const {
1256 const SDLoc &dl(Op);
1257 SDValue VecV = Op.getOperand(0);
1258 SDValue ValV = Op.getOperand(1);
1259 SDValue IdxV = Op.getOperand(2);
1260 MVT ElemTy = ty(VecV).getVectorElementType();
1261 if (ElemTy == MVT::i1)
1262 return insertHvxElementPred(VecV, IdxV, ValV, dl, DAG);
1263
1264 return insertHvxElementReg(VecV, IdxV, ValV, dl, DAG);
1265 }
1266
1267 SDValue
LowerHvxExtractSubvector(SDValue Op,SelectionDAG & DAG) const1268 HexagonTargetLowering::LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG)
1269 const {
1270 SDValue SrcV = Op.getOperand(0);
1271 MVT SrcTy = ty(SrcV);
1272 MVT DstTy = ty(Op);
1273 SDValue IdxV = Op.getOperand(1);
1274 unsigned Idx = cast<ConstantSDNode>(IdxV.getNode())->getZExtValue();
1275 assert(Idx % DstTy.getVectorNumElements() == 0);
1276 (void)Idx;
1277 const SDLoc &dl(Op);
1278
1279 MVT ElemTy = SrcTy.getVectorElementType();
1280 if (ElemTy == MVT::i1)
1281 return extractHvxSubvectorPred(SrcV, IdxV, dl, DstTy, DAG);
1282
1283 return extractHvxSubvectorReg(SrcV, IdxV, dl, DstTy, DAG);
1284 }
1285
1286 SDValue
LowerHvxInsertSubvector(SDValue Op,SelectionDAG & DAG) const1287 HexagonTargetLowering::LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG)
1288 const {
1289 // Idx does not need to be a constant.
1290 SDValue VecV = Op.getOperand(0);
1291 SDValue ValV = Op.getOperand(1);
1292 SDValue IdxV = Op.getOperand(2);
1293
1294 const SDLoc &dl(Op);
1295 MVT VecTy = ty(VecV);
1296 MVT ElemTy = VecTy.getVectorElementType();
1297 if (ElemTy == MVT::i1)
1298 return insertHvxSubvectorPred(VecV, ValV, IdxV, dl, DAG);
1299
1300 return insertHvxSubvectorReg(VecV, ValV, IdxV, dl, DAG);
1301 }
1302
1303 SDValue
LowerHvxAnyExt(SDValue Op,SelectionDAG & DAG) const1304 HexagonTargetLowering::LowerHvxAnyExt(SDValue Op, SelectionDAG &DAG) const {
1305 // Lower any-extends of boolean vectors to sign-extends, since they
1306 // translate directly to Q2V. Zero-extending could also be done equally
1307 // fast, but Q2V is used/recognized in more places.
1308 // For all other vectors, use zero-extend.
1309 MVT ResTy = ty(Op);
1310 SDValue InpV = Op.getOperand(0);
1311 MVT ElemTy = ty(InpV).getVectorElementType();
1312 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1313 return LowerHvxSignExt(Op, DAG);
1314 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), ResTy, InpV);
1315 }
1316
1317 SDValue
LowerHvxSignExt(SDValue Op,SelectionDAG & DAG) const1318 HexagonTargetLowering::LowerHvxSignExt(SDValue Op, SelectionDAG &DAG) const {
1319 MVT ResTy = ty(Op);
1320 SDValue InpV = Op.getOperand(0);
1321 MVT ElemTy = ty(InpV).getVectorElementType();
1322 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1323 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), false, DAG);
1324 return Op;
1325 }
1326
1327 SDValue
LowerHvxZeroExt(SDValue Op,SelectionDAG & DAG) const1328 HexagonTargetLowering::LowerHvxZeroExt(SDValue Op, SelectionDAG &DAG) const {
1329 MVT ResTy = ty(Op);
1330 SDValue InpV = Op.getOperand(0);
1331 MVT ElemTy = ty(InpV).getVectorElementType();
1332 if (ElemTy == MVT::i1 && Subtarget.isHVXVectorType(ResTy))
1333 return extendHvxVectorPred(InpV, SDLoc(Op), ty(Op), true, DAG);
1334 return Op;
1335 }
1336
1337 SDValue
LowerHvxCttz(SDValue Op,SelectionDAG & DAG) const1338 HexagonTargetLowering::LowerHvxCttz(SDValue Op, SelectionDAG &DAG) const {
1339 // Lower vector CTTZ into a computation using CTLZ (Hacker's Delight):
1340 // cttz(x) = bitwidth(x) - ctlz(~x & (x-1))
1341 const SDLoc &dl(Op);
1342 MVT ResTy = ty(Op);
1343 SDValue InpV = Op.getOperand(0);
1344 assert(ResTy == ty(InpV));
1345
1346 // Calculate the vectors of 1 and bitwidth(x).
1347 MVT ElemTy = ty(InpV).getVectorElementType();
1348 unsigned ElemWidth = ElemTy.getSizeInBits();
1349 // Using uint64_t because a shift by 32 can happen.
1350 uint64_t Splat1 = 0, SplatW = 0;
1351 assert(isPowerOf2_32(ElemWidth) && ElemWidth <= 32);
1352 for (unsigned i = 0; i != 32/ElemWidth; ++i) {
1353 Splat1 = (Splat1 << ElemWidth) | 1;
1354 SplatW = (SplatW << ElemWidth) | ElemWidth;
1355 }
1356 SDValue Vec1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1357 DAG.getConstant(uint32_t(Splat1), dl, MVT::i32));
1358 SDValue VecW = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1359 DAG.getConstant(uint32_t(SplatW), dl, MVT::i32));
1360 SDValue VecN1 = DAG.getNode(HexagonISD::VSPLATW, dl, ResTy,
1361 DAG.getConstant(-1, dl, MVT::i32));
1362 // Do not use DAG.getNOT, because that would create BUILD_VECTOR with
1363 // a BITCAST. Here we can skip the BITCAST (so we don't have to handle
1364 // it separately in custom combine or selection).
1365 SDValue A = DAG.getNode(ISD::AND, dl, ResTy,
1366 {DAG.getNode(ISD::XOR, dl, ResTy, {InpV, VecN1}),
1367 DAG.getNode(ISD::SUB, dl, ResTy, {InpV, Vec1})});
1368 return DAG.getNode(ISD::SUB, dl, ResTy,
1369 {VecW, DAG.getNode(ISD::CTLZ, dl, ResTy, A)});
1370 }
1371
1372 SDValue
LowerHvxMul(SDValue Op,SelectionDAG & DAG) const1373 HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
1374 MVT ResTy = ty(Op);
1375 assert(ResTy.isVector() && isHvxSingleTy(ResTy));
1376 const SDLoc &dl(Op);
1377 SmallVector<int,256> ShuffMask;
1378
1379 MVT ElemTy = ResTy.getVectorElementType();
1380 unsigned VecLen = ResTy.getVectorNumElements();
1381 SDValue Vs = Op.getOperand(0);
1382 SDValue Vt = Op.getOperand(1);
1383
1384 switch (ElemTy.SimpleTy) {
1385 case MVT::i8: {
1386 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1387 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1388 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1389 MVT ExtTy = typeExtElem(ResTy, 2);
1390 unsigned MpyOpc = ElemTy == MVT::i8 ? Hexagon::V6_vmpybv
1391 : Hexagon::V6_vmpyhv;
1392 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1393
1394 // Discard high halves of the resulting values, collect the low halves.
1395 for (unsigned I = 0; I < VecLen; I += 2) {
1396 ShuffMask.push_back(I); // Pick even element.
1397 ShuffMask.push_back(I+VecLen); // Pick odd element.
1398 }
1399 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1400 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1401 return DAG.getBitcast(ResTy, BS);
1402 }
1403 case MVT::i16:
1404 // For i16 there is V6_vmpyih, which acts exactly like the MUL opcode.
1405 // (There is also V6_vmpyhv, which behaves in an analogous way to
1406 // V6_vmpybv.)
1407 return getInstr(Hexagon::V6_vmpyih, dl, ResTy, {Vs, Vt}, DAG);
1408 case MVT::i32: {
1409 // Use the following sequence for signed word multiply:
1410 // T0 = V6_vmpyiowh Vs, Vt
1411 // T1 = V6_vaslw T0, 16
1412 // T2 = V6_vmpyiewuh_acc T1, Vs, Vt
1413 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1414 SDValue T0 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {Vs, Vt}, DAG);
1415 SDValue T1 = getInstr(Hexagon::V6_vaslw, dl, ResTy, {T0, S16}, DAG);
1416 SDValue T2 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1417 {T1, Vs, Vt}, DAG);
1418 return T2;
1419 }
1420 default:
1421 break;
1422 }
1423 return SDValue();
1424 }
1425
1426 SDValue
LowerHvxMulh(SDValue Op,SelectionDAG & DAG) const1427 HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
1428 MVT ResTy = ty(Op);
1429 assert(ResTy.isVector());
1430 const SDLoc &dl(Op);
1431 SmallVector<int,256> ShuffMask;
1432
1433 MVT ElemTy = ResTy.getVectorElementType();
1434 unsigned VecLen = ResTy.getVectorNumElements();
1435 SDValue Vs = Op.getOperand(0);
1436 SDValue Vt = Op.getOperand(1);
1437 bool IsSigned = Op.getOpcode() == ISD::MULHS;
1438
1439 if (ElemTy == MVT::i8 || ElemTy == MVT::i16) {
1440 // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
1441 // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
1442 // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
1443 // For i16, use V6_vmpyhv, which behaves in an analogous way to
1444 // V6_vmpybv: results Lo and Hi are products of even/odd elements
1445 // respectively.
1446 MVT ExtTy = typeExtElem(ResTy, 2);
1447 unsigned MpyOpc = ElemTy == MVT::i8
1448 ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv)
1449 : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv);
1450 SDValue M = getInstr(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
1451
1452 // Discard low halves of the resulting values, collect the high halves.
1453 for (unsigned I = 0; I < VecLen; I += 2) {
1454 ShuffMask.push_back(I+1); // Pick even element.
1455 ShuffMask.push_back(I+VecLen+1); // Pick odd element.
1456 }
1457 VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
1458 SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
1459 return DAG.getBitcast(ResTy, BS);
1460 }
1461
1462 assert(ElemTy == MVT::i32);
1463 SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
1464
1465 if (IsSigned) {
1466 // mulhs(Vs,Vt) =
1467 // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1468 // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16
1469 // + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32
1470 // = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16
1471 // + Lo(Vs) *us Vt] >> 32
1472 // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to
1473 // anything, so it cannot produce any carry over to higher bits),
1474 // so everything in [] can be shifted by 16 without loss of precision.
1475 // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16
1476 // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16
1477 // Denote Hi(Vs) = Vs':
1478 // = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16
1479 // = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16
1480 SDValue T0 = getInstr(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
1481 // Get Vs':
1482 SDValue S0 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
1483 SDValue T1 = getInstr(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
1484 {T0, S0, Vt}, DAG);
1485 // Shift by 16:
1486 SDValue S2 = getInstr(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG);
1487 // Get Vs'*Hi(Vt):
1488 SDValue T2 = getInstr(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
1489 // Add:
1490 SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2});
1491 return T3;
1492 }
1493
1494 // Unsigned mulhw. (Would expansion using signed mulhw be better?)
1495
1496 auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) {
1497 return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair);
1498 };
1499 auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) {
1500 return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair);
1501 };
1502
1503 MVT PairTy = typeJoin({ResTy, ResTy});
1504 SDValue P = getInstr(Hexagon::V6_lvsplatw, dl, ResTy,
1505 {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
1506 // Multiply-unsigned halfwords:
1507 // LoVec = Vs.uh[2i] * Vt.uh[2i],
1508 // HiVec = Vs.uh[2i+1] * Vt.uh[2i+1]
1509 SDValue T0 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG);
1510 // The low halves in the LoVec of the pair can be discarded. They are
1511 // not added to anything (in the full-precision product), so they cannot
1512 // produce a carry into the higher bits.
1513 SDValue T1 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
1514 // Swap low and high halves in Vt, and do the halfword multiplication
1515 // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i].
1516 SDValue D0 = getInstr(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG);
1517 SDValue T2 = getInstr(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG);
1518 // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs).
1519 // These products are words, but cannot be added directly because the
1520 // sums could overflow. Add these products, by halfwords, where each sum
1521 // of a pair of halfwords gives a word.
1522 SDValue T3 = getInstr(Hexagon::V6_vadduhw, dl, PairTy,
1523 {LoVec(T2), HiVec(T2)}, DAG);
1524 // Add the high halfwords from the products of the low halfwords.
1525 SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)});
1526 SDValue T5 = getInstr(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
1527 SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)});
1528 SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6});
1529 return T7;
1530 }
1531
1532 SDValue
LowerHvxBitcast(SDValue Op,SelectionDAG & DAG) const1533 HexagonTargetLowering::LowerHvxBitcast(SDValue Op, SelectionDAG &DAG) const {
1534 SDValue ValQ = Op.getOperand(0);
1535 MVT ResTy = ty(Op);
1536 MVT VecTy = ty(ValQ);
1537 const SDLoc &dl(Op);
1538
1539 if (isHvxBoolTy(VecTy) && ResTy.isScalarInteger()) {
1540 unsigned HwLen = Subtarget.getVectorLength();
1541 MVT WordTy = MVT::getVectorVT(MVT::i32, HwLen/4);
1542 SDValue VQ = compressHvxPred(ValQ, dl, WordTy, DAG);
1543 unsigned BitWidth = ResTy.getSizeInBits();
1544
1545 if (BitWidth < 64) {
1546 SDValue W0 = extractHvxElementReg(VQ, DAG.getConstant(0, dl, MVT::i32),
1547 dl, MVT::i32, DAG);
1548 if (BitWidth == 32)
1549 return W0;
1550 assert(BitWidth < 32u);
1551 return DAG.getZExtOrTrunc(W0, dl, ResTy);
1552 }
1553
1554 // The result is >= 64 bits. The only options are 64 or 128.
1555 assert(BitWidth == 64 || BitWidth == 128);
1556 SmallVector<SDValue,4> Words;
1557 for (unsigned i = 0; i != BitWidth/32; ++i) {
1558 SDValue W = extractHvxElementReg(
1559 VQ, DAG.getConstant(i, dl, MVT::i32), dl, MVT::i32, DAG);
1560 Words.push_back(W);
1561 }
1562 SmallVector<SDValue,2> Combines;
1563 assert(Words.size() % 2 == 0);
1564 for (unsigned i = 0, e = Words.size(); i < e; i += 2) {
1565 SDValue C = DAG.getNode(
1566 HexagonISD::COMBINE, dl, MVT::i64, {Words[i+1], Words[i]});
1567 Combines.push_back(C);
1568 }
1569
1570 if (BitWidth == 64)
1571 return Combines[0];
1572
1573 return DAG.getNode(ISD::BUILD_PAIR, dl, ResTy, Combines);
1574 }
1575
1576 return Op;
1577 }
1578
1579 SDValue
LowerHvxExtend(SDValue Op,SelectionDAG & DAG) const1580 HexagonTargetLowering::LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const {
1581 // Sign- and zero-extends are legal.
1582 assert(Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG);
1583 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), ty(Op),
1584 Op.getOperand(0));
1585 }
1586
1587 SDValue
LowerHvxShift(SDValue Op,SelectionDAG & DAG) const1588 HexagonTargetLowering::LowerHvxShift(SDValue Op, SelectionDAG &DAG) const {
1589 if (SDValue S = getVectorShiftByInt(Op, DAG))
1590 return S;
1591 return Op;
1592 }
1593
1594 SDValue
LowerHvxIntrinsic(SDValue Op,SelectionDAG & DAG) const1595 HexagonTargetLowering::LowerHvxIntrinsic(SDValue Op, SelectionDAG &DAG) const {
1596 const SDLoc &dl(Op);
1597 MVT ResTy = ty(Op);
1598
1599 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1600 bool Use64b = Subtarget.useHVX64BOps();
1601 unsigned IntPredCast = Use64b ? Intrinsic::hexagon_V6_pred_typecast
1602 : Intrinsic::hexagon_V6_pred_typecast_128B;
1603 if (IntNo == IntPredCast) {
1604 SDValue Vs = Op.getOperand(1);
1605 MVT OpTy = ty(Vs);
1606 if (isHvxBoolTy(ResTy) && isHvxBoolTy(OpTy)) {
1607 if (ResTy == OpTy)
1608 return Vs;
1609 return DAG.getNode(HexagonISD::TYPECAST, dl, ResTy, Vs);
1610 }
1611 }
1612
1613 return Op;
1614 }
1615
1616 SDValue
SplitHvxPairOp(SDValue Op,SelectionDAG & DAG) const1617 HexagonTargetLowering::SplitHvxPairOp(SDValue Op, SelectionDAG &DAG) const {
1618 assert(!Op.isMachineOpcode());
1619 SmallVector<SDValue,2> OpsL, OpsH;
1620 const SDLoc &dl(Op);
1621
1622 auto SplitVTNode = [&DAG,this] (const VTSDNode *N) {
1623 MVT Ty = typeSplit(N->getVT().getSimpleVT()).first;
1624 SDValue TV = DAG.getValueType(Ty);
1625 return std::make_pair(TV, TV);
1626 };
1627
1628 for (SDValue A : Op.getNode()->ops()) {
1629 VectorPair P = Subtarget.isHVXVectorType(ty(A), true)
1630 ? opSplit(A, dl, DAG)
1631 : std::make_pair(A, A);
1632 // Special case for type operand.
1633 if (Op.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1634 if (const auto *N = dyn_cast<const VTSDNode>(A.getNode()))
1635 P = SplitVTNode(N);
1636 }
1637 OpsL.push_back(P.first);
1638 OpsH.push_back(P.second);
1639 }
1640
1641 MVT ResTy = ty(Op);
1642 MVT HalfTy = typeSplit(ResTy).first;
1643 SDValue L = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsL);
1644 SDValue H = DAG.getNode(Op.getOpcode(), dl, HalfTy, OpsH);
1645 SDValue S = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResTy, L, H);
1646 return S;
1647 }
1648
1649 SDValue
SplitHvxMemOp(SDValue Op,SelectionDAG & DAG) const1650 HexagonTargetLowering::SplitHvxMemOp(SDValue Op, SelectionDAG &DAG) const {
1651 LSBaseSDNode *BN = cast<LSBaseSDNode>(Op.getNode());
1652 assert(BN->isUnindexed());
1653 MVT MemTy = BN->getMemoryVT().getSimpleVT();
1654 if (!isHvxPairTy(MemTy))
1655 return Op;
1656
1657 const SDLoc &dl(Op);
1658 unsigned HwLen = Subtarget.getVectorLength();
1659 MVT SingleTy = typeSplit(MemTy).first;
1660 SDValue Chain = BN->getChain();
1661 SDValue Base0 = BN->getBasePtr();
1662 SDValue Base1 = DAG.getMemBasePlusOffset(Base0, HwLen, dl);
1663
1664 MachineMemOperand *MOp0 = nullptr, *MOp1 = nullptr;
1665 if (MachineMemOperand *MMO = BN->getMemOperand()) {
1666 MachineFunction &MF = DAG.getMachineFunction();
1667 MOp0 = MF.getMachineMemOperand(MMO, 0, HwLen);
1668 MOp1 = MF.getMachineMemOperand(MMO, HwLen, HwLen);
1669 }
1670
1671 unsigned MemOpc = BN->getOpcode();
1672 SDValue NewOp;
1673
1674 if (MemOpc == ISD::LOAD) {
1675 SDValue Load0 = DAG.getLoad(SingleTy, dl, Chain, Base0, MOp0);
1676 SDValue Load1 = DAG.getLoad(SingleTy, dl, Chain, Base1, MOp1);
1677 NewOp = DAG.getMergeValues(
1678 { DAG.getNode(ISD::CONCAT_VECTORS, dl, MemTy, Load0, Load1),
1679 DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1680 Load0.getValue(1), Load1.getValue(1)) }, dl);
1681 } else {
1682 assert(MemOpc == ISD::STORE);
1683 VectorPair Vals = opSplit(cast<StoreSDNode>(Op)->getValue(), dl, DAG);
1684 SDValue Store0 = DAG.getStore(Chain, dl, Vals.first, Base0, MOp0);
1685 SDValue Store1 = DAG.getStore(Chain, dl, Vals.second, Base1, MOp1);
1686 NewOp = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store0, Store1);
1687 }
1688
1689 return NewOp;
1690 }
1691
1692 SDValue
LowerHvxOperation(SDValue Op,SelectionDAG & DAG) const1693 HexagonTargetLowering::LowerHvxOperation(SDValue Op, SelectionDAG &DAG) const {
1694 unsigned Opc = Op.getOpcode();
1695 bool IsPairOp = isHvxPairTy(ty(Op)) ||
1696 llvm::any_of(Op.getNode()->ops(), [this] (SDValue V) {
1697 return isHvxPairTy(ty(V));
1698 });
1699
1700 if (IsPairOp) {
1701 switch (Opc) {
1702 default:
1703 break;
1704 case ISD::LOAD:
1705 case ISD::STORE:
1706 return SplitHvxMemOp(Op, DAG);
1707 case ISD::CTPOP:
1708 case ISD::CTLZ:
1709 case ISD::CTTZ:
1710 case ISD::MUL:
1711 case ISD::MULHS:
1712 case ISD::MULHU:
1713 case ISD::AND:
1714 case ISD::OR:
1715 case ISD::XOR:
1716 case ISD::SRA:
1717 case ISD::SHL:
1718 case ISD::SRL:
1719 case ISD::SETCC:
1720 case ISD::VSELECT:
1721 case ISD::SIGN_EXTEND:
1722 case ISD::ZERO_EXTEND:
1723 case ISD::SIGN_EXTEND_INREG:
1724 return SplitHvxPairOp(Op, DAG);
1725 }
1726 }
1727
1728 switch (Opc) {
1729 default:
1730 break;
1731 case ISD::BUILD_VECTOR: return LowerHvxBuildVector(Op, DAG);
1732 case ISD::CONCAT_VECTORS: return LowerHvxConcatVectors(Op, DAG);
1733 case ISD::INSERT_SUBVECTOR: return LowerHvxInsertSubvector(Op, DAG);
1734 case ISD::INSERT_VECTOR_ELT: return LowerHvxInsertElement(Op, DAG);
1735 case ISD::EXTRACT_SUBVECTOR: return LowerHvxExtractSubvector(Op, DAG);
1736 case ISD::EXTRACT_VECTOR_ELT: return LowerHvxExtractElement(Op, DAG);
1737 case ISD::BITCAST: return LowerHvxBitcast(Op, DAG);
1738 case ISD::ANY_EXTEND: return LowerHvxAnyExt(Op, DAG);
1739 case ISD::SIGN_EXTEND: return LowerHvxSignExt(Op, DAG);
1740 case ISD::ZERO_EXTEND: return LowerHvxZeroExt(Op, DAG);
1741 case ISD::CTTZ: return LowerHvxCttz(Op, DAG);
1742 case ISD::SRA:
1743 case ISD::SHL:
1744 case ISD::SRL: return LowerHvxShift(Op, DAG);
1745 case ISD::MUL: return LowerHvxMul(Op, DAG);
1746 case ISD::MULHS:
1747 case ISD::MULHU: return LowerHvxMulh(Op, DAG);
1748 case ISD::ANY_EXTEND_VECTOR_INREG: return LowerHvxExtend(Op, DAG);
1749 case ISD::SETCC:
1750 case ISD::INTRINSIC_VOID: return Op;
1751 case ISD::INTRINSIC_WO_CHAIN: return LowerHvxIntrinsic(Op, DAG);
1752 // Unaligned loads will be handled by the default lowering.
1753 case ISD::LOAD: return SDValue();
1754 }
1755 #ifndef NDEBUG
1756 Op.dumpr(&DAG);
1757 #endif
1758 llvm_unreachable("Unhandled HVX operation");
1759 }
1760
1761 void
LowerHvxOperationWrapper(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG) const1762 HexagonTargetLowering::LowerHvxOperationWrapper(SDNode *N,
1763 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
1764 }
1765
1766 void
ReplaceHvxNodeResults(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG) const1767 HexagonTargetLowering::ReplaceHvxNodeResults(SDNode *N,
1768 SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
1769 unsigned Opc = N->getOpcode();
1770 switch (Opc) {
1771 case ISD::BITCAST:
1772 if (isHvxBoolTy(ty(N->getOperand(0)))) {
1773 SDValue Op(N, 0);
1774 SDValue C = LowerHvxBitcast(Op, DAG);
1775 Results.push_back(C);
1776 }
1777 break;
1778 default:
1779 break;
1780 }
1781 }
1782
1783 SDValue
PerformHvxDAGCombine(SDNode * N,DAGCombinerInfo & DCI) const1784 HexagonTargetLowering::PerformHvxDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
1785 const {
1786 const SDLoc &dl(N);
1787 SDValue Op(N, 0);
1788
1789 unsigned Opc = Op.getOpcode();
1790 if (Opc == ISD::VSELECT) {
1791 // (vselect (xor x, qtrue), v0, v1) -> (vselect x, v1, v0)
1792 SDValue Cond = Op.getOperand(0);
1793 if (Cond->getOpcode() == ISD::XOR) {
1794 SDValue C0 = Cond.getOperand(0), C1 = Cond.getOperand(1);
1795 if (C1->getOpcode() == HexagonISD::QTRUE) {
1796 SDValue VSel = DCI.DAG.getNode(ISD::VSELECT, dl, ty(Op), C0,
1797 Op.getOperand(2), Op.getOperand(1));
1798 return VSel;
1799 }
1800 }
1801 }
1802 return SDValue();
1803 }
1804
1805 bool
isHvxOperation(SDValue Op) const1806 HexagonTargetLowering::isHvxOperation(SDValue Op) const {
1807 // If the type of the result, or any operand type are HVX vector types,
1808 // this is an HVX operation.
1809 return Subtarget.isHVXVectorType(ty(Op), true) ||
1810 llvm::any_of(Op.getNode()->ops(),
1811 [this] (SDValue V) {
1812 return Subtarget.isHVXVectorType(ty(V), true);
1813 });
1814 }
1815
1816 bool
isHvxOperation(SDNode * N) const1817 HexagonTargetLowering::isHvxOperation(SDNode *N) const {
1818 // If the type of any result, or any operand type are HVX vector types,
1819 // this is an HVX operation.
1820 auto IsHvxTy = [this] (EVT Ty) {
1821 return Ty.isSimple() && Subtarget.isHVXVectorType(Ty.getSimpleVT(), true);
1822 };
1823 auto IsHvxOp = [this] (SDValue Op) {
1824 return Subtarget.isHVXVectorType(ty(Op), true);
1825 };
1826 return llvm::any_of(N->values(), IsHvxTy) || llvm::any_of(N->ops(), IsHvxOp);
1827 }
1828