1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Define several functions to decode x86 specific shuffle semantics using
10 // constants from the constant pool.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "Utils/X86ShuffleDecode.h"
15 #include "llvm/ADT/APInt.h"
16 #include "llvm/IR/Constants.h"
17 
18 //===----------------------------------------------------------------------===//
19 //  Vector Mask Decoding
20 //===----------------------------------------------------------------------===//
21 
22 namespace llvm {
23 
extractConstantMask(const Constant * C,unsigned MaskEltSizeInBits,APInt & UndefElts,SmallVectorImpl<uint64_t> & RawMask)24 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
25                                 APInt &UndefElts,
26                                 SmallVectorImpl<uint64_t> &RawMask) {
27   // It is not an error for shuffle masks to not be a vector of
28   // MaskEltSizeInBits because the constant pool uniques constants by their
29   // bit representation.
30   // e.g. the following take up the same space in the constant pool:
31   //   i128 -170141183420855150465331762880109871104
32   //
33   //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
34   //
35   //   <4 x i32> <i32 -2147483648, i32 -2147483648,
36   //              i32 -2147483648, i32 -2147483648>
37   Type *CstTy = C->getType();
38   if (!CstTy->isVectorTy())
39     return false;
40 
41   Type *CstEltTy = CstTy->getVectorElementType();
42   if (!CstEltTy->isIntegerTy())
43     return false;
44 
45   unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
46   unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
47   unsigned NumCstElts = CstTy->getVectorNumElements();
48 
49   assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
50          "Unaligned shuffle mask size");
51 
52   unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
53   UndefElts = APInt(NumMaskElts, 0);
54   RawMask.resize(NumMaskElts, 0);
55 
56   // Fast path - if the constants match the mask size then copy direct.
57   if (MaskEltSizeInBits == CstEltSizeInBits) {
58     assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
59     for (unsigned i = 0; i != NumMaskElts; ++i) {
60       Constant *COp = C->getAggregateElement(i);
61       if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
62         return false;
63 
64       if (isa<UndefValue>(COp)) {
65         UndefElts.setBit(i);
66         RawMask[i] = 0;
67         continue;
68       }
69 
70       auto *Elt = cast<ConstantInt>(COp);
71       RawMask[i] = Elt->getValue().getZExtValue();
72     }
73     return true;
74   }
75 
76   // Extract all the undef/constant element data and pack into single bitsets.
77   APInt UndefBits(CstSizeInBits, 0);
78   APInt MaskBits(CstSizeInBits, 0);
79   for (unsigned i = 0; i != NumCstElts; ++i) {
80     Constant *COp = C->getAggregateElement(i);
81     if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
82       return false;
83 
84     unsigned BitOffset = i * CstEltSizeInBits;
85 
86     if (isa<UndefValue>(COp)) {
87       UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
88       continue;
89     }
90 
91     MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
92   }
93 
94   // Now extract the undef/constant bit data into the raw shuffle masks.
95   for (unsigned i = 0; i != NumMaskElts; ++i) {
96     unsigned BitOffset = i * MaskEltSizeInBits;
97     APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
98 
99     // Only treat the element as UNDEF if all bits are UNDEF, otherwise
100     // treat it as zero.
101     if (EltUndef.isAllOnesValue()) {
102       UndefElts.setBit(i);
103       RawMask[i] = 0;
104       continue;
105     }
106 
107     APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
108     RawMask[i] = EltBits.getZExtValue();
109   }
110 
111   return true;
112 }
113 
DecodePSHUFBMask(const Constant * C,unsigned Width,SmallVectorImpl<int> & ShuffleMask)114 void DecodePSHUFBMask(const Constant *C, unsigned Width,
115                       SmallVectorImpl<int> &ShuffleMask) {
116   assert((Width == 128 || Width == 256 || Width == 512) &&
117          C->getType()->getPrimitiveSizeInBits() >= Width &&
118          "Unexpected vector size.");
119 
120   // The shuffle mask requires a byte vector.
121   APInt UndefElts;
122   SmallVector<uint64_t, 64> RawMask;
123   if (!extractConstantMask(C, 8, UndefElts, RawMask))
124     return;
125 
126   unsigned NumElts = Width / 8;
127   assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
128          "Unexpected number of vector elements.");
129 
130   for (unsigned i = 0; i != NumElts; ++i) {
131     if (UndefElts[i]) {
132       ShuffleMask.push_back(SM_SentinelUndef);
133       continue;
134     }
135 
136     uint64_t Element = RawMask[i];
137     // If the high bit (7) of the byte is set, the element is zeroed.
138     if (Element & (1 << 7))
139       ShuffleMask.push_back(SM_SentinelZero);
140     else {
141       // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
142       // lane of the vector we're inside.
143       unsigned Base = i & ~0xf;
144 
145       // Only the least significant 4 bits of the byte are used.
146       int Index = Base + (Element & 0xf);
147       ShuffleMask.push_back(Index);
148     }
149   }
150 }
151 
DecodeVPERMILPMask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)152 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,
153                         SmallVectorImpl<int> &ShuffleMask) {
154   assert((Width == 128 || Width == 256 || Width == 512) &&
155          C->getType()->getPrimitiveSizeInBits() >= Width &&
156          "Unexpected vector size.");
157   assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
158 
159   // The shuffle mask requires elements the same size as the target.
160   APInt UndefElts;
161   SmallVector<uint64_t, 16> RawMask;
162   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
163     return;
164 
165   unsigned NumElts = Width / ElSize;
166   unsigned NumEltsPerLane = 128 / ElSize;
167   assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
168          "Unexpected number of vector elements.");
169 
170   for (unsigned i = 0; i != NumElts; ++i) {
171     if (UndefElts[i]) {
172       ShuffleMask.push_back(SM_SentinelUndef);
173       continue;
174     }
175 
176     int Index = i & ~(NumEltsPerLane - 1);
177     uint64_t Element = RawMask[i];
178     if (ElSize == 64)
179       Index += (Element >> 1) & 0x1;
180     else
181       Index += Element & 0x3;
182 
183     ShuffleMask.push_back(Index);
184   }
185 }
186 
DecodeVPERMIL2PMask(const Constant * C,unsigned M2Z,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)187 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
188                          unsigned Width,
189                          SmallVectorImpl<int> &ShuffleMask) {
190   Type *MaskTy = C->getType();
191   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
192   (void)MaskTySize;
193   assert((MaskTySize == 128 || MaskTySize == 256) &&
194          Width >= MaskTySize && "Unexpected vector size.");
195 
196   // The shuffle mask requires elements the same size as the target.
197   APInt UndefElts;
198   SmallVector<uint64_t, 8> RawMask;
199   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
200     return;
201 
202   unsigned NumElts = Width / ElSize;
203   unsigned NumEltsPerLane = 128 / ElSize;
204   assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
205          "Unexpected number of vector elements.");
206 
207   for (unsigned i = 0; i != NumElts; ++i) {
208     if (UndefElts[i]) {
209       ShuffleMask.push_back(SM_SentinelUndef);
210       continue;
211     }
212 
213     // VPERMIL2 Operation.
214     // Bits[3] - Match Bit.
215     // Bits[2:1] - (Per Lane) PD Shuffle Mask.
216     // Bits[2:0] - (Per Lane) PS Shuffle Mask.
217     uint64_t Selector = RawMask[i];
218     unsigned MatchBit = (Selector >> 3) & 0x1;
219 
220     // M2Z[0:1]     MatchBit
221     //   0Xb           X        Source selected by Selector index.
222     //   10b           0        Source selected by Selector index.
223     //   10b           1        Zero.
224     //   11b           0        Zero.
225     //   11b           1        Source selected by Selector index.
226     if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
227       ShuffleMask.push_back(SM_SentinelZero);
228       continue;
229     }
230 
231     int Index = i & ~(NumEltsPerLane - 1);
232     if (ElSize == 64)
233       Index += (Selector >> 1) & 0x1;
234     else
235       Index += Selector & 0x3;
236 
237     int Src = (Selector >> 2) & 0x1;
238     Index += Src * NumElts;
239     ShuffleMask.push_back(Index);
240   }
241 }
242 
DecodeVPPERMMask(const Constant * C,unsigned Width,SmallVectorImpl<int> & ShuffleMask)243 void DecodeVPPERMMask(const Constant *C, unsigned Width,
244                       SmallVectorImpl<int> &ShuffleMask) {
245   Type *MaskTy = C->getType();
246   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
247   (void)MaskTySize;
248   assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");
249 
250   // The shuffle mask requires a byte vector.
251   APInt UndefElts;
252   SmallVector<uint64_t, 16> RawMask;
253   if (!extractConstantMask(C, 8, UndefElts, RawMask))
254     return;
255 
256   unsigned NumElts = Width / 8;
257   assert(NumElts == 16 && "Unexpected number of vector elements.");
258 
259   for (unsigned i = 0; i != NumElts; ++i) {
260     if (UndefElts[i]) {
261       ShuffleMask.push_back(SM_SentinelUndef);
262       continue;
263     }
264 
265     // VPPERM Operation
266     // Bits[4:0] - Byte Index (0 - 31)
267     // Bits[7:5] - Permute Operation
268     //
269     // Permute Operation:
270     // 0 - Source byte (no logical operation).
271     // 1 - Invert source byte.
272     // 2 - Bit reverse of source byte.
273     // 3 - Bit reverse of inverted source byte.
274     // 4 - 00h (zero - fill).
275     // 5 - FFh (ones - fill).
276     // 6 - Most significant bit of source byte replicated in all bit positions.
277     // 7 - Invert most significant bit of source byte and replicate in all bit
278     // positions.
279     uint64_t Element = RawMask[i];
280     uint64_t Index = Element & 0x1F;
281     uint64_t PermuteOp = (Element >> 5) & 0x7;
282 
283     if (PermuteOp == 4) {
284       ShuffleMask.push_back(SM_SentinelZero);
285       continue;
286     }
287     if (PermuteOp != 0) {
288       ShuffleMask.clear();
289       return;
290     }
291     ShuffleMask.push_back((int)Index);
292   }
293 }
294 
DecodeVPERMVMask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)295 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
296                       SmallVectorImpl<int> &ShuffleMask) {
297   assert((Width == 128 || Width == 256 || Width == 512) &&
298          C->getType()->getPrimitiveSizeInBits() >= Width &&
299          "Unexpected vector size.");
300   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
301          "Unexpected vector element size.");
302 
303   // The shuffle mask requires elements the same size as the target.
304   APInt UndefElts;
305   SmallVector<uint64_t, 64> RawMask;
306   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
307     return;
308 
309   unsigned NumElts = Width / ElSize;
310 
311   for (unsigned i = 0; i != NumElts; ++i) {
312     if (UndefElts[i]) {
313       ShuffleMask.push_back(SM_SentinelUndef);
314       continue;
315     }
316     int Index = RawMask[i] & (NumElts - 1);
317     ShuffleMask.push_back(Index);
318   }
319 }
320 
DecodeVPERMV3Mask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)321 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
322                        SmallVectorImpl<int> &ShuffleMask) {
323   assert((Width == 128 || Width == 256 || Width == 512) &&
324          C->getType()->getPrimitiveSizeInBits() >= Width &&
325          "Unexpected vector size.");
326   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
327          "Unexpected vector element size.");
328 
329   // The shuffle mask requires elements the same size as the target.
330   APInt UndefElts;
331   SmallVector<uint64_t, 64> RawMask;
332   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
333     return;
334 
335   unsigned NumElts = Width / ElSize;
336 
337   for (unsigned i = 0; i != NumElts; ++i) {
338     if (UndefElts[i]) {
339       ShuffleMask.push_back(SM_SentinelUndef);
340       continue;
341     }
342     int Index = RawMask[i] & (NumElts*2 - 1);
343     ShuffleMask.push_back(Index);
344   }
345 }
346 } // llvm namespace
347