1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Define several functions to decode x86 specific shuffle semantics using
11 // constants from the constant pool.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "Utils/X86ShuffleDecode.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/IR/Constants.h"
18 
19 //===----------------------------------------------------------------------===//
20 //  Vector Mask Decoding
21 //===----------------------------------------------------------------------===//
22 
23 namespace llvm {
24 
extractConstantMask(const Constant * C,unsigned MaskEltSizeInBits,APInt & UndefElts,SmallVectorImpl<uint64_t> & RawMask)25 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
26                                 APInt &UndefElts,
27                                 SmallVectorImpl<uint64_t> &RawMask) {
28   // It is not an error for shuffle masks to not be a vector of
29   // MaskEltSizeInBits because the constant pool uniques constants by their
30   // bit representation.
31   // e.g. the following take up the same space in the constant pool:
32   //   i128 -170141183420855150465331762880109871104
33   //
34   //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
35   //
36   //   <4 x i32> <i32 -2147483648, i32 -2147483648,
37   //              i32 -2147483648, i32 -2147483648>
38   Type *CstTy = C->getType();
39   if (!CstTy->isVectorTy())
40     return false;
41 
42   Type *CstEltTy = CstTy->getVectorElementType();
43   if (!CstEltTy->isIntegerTy())
44     return false;
45 
46   unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
47   unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
48   unsigned NumCstElts = CstTy->getVectorNumElements();
49 
50   assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
51          "Unaligned shuffle mask size");
52 
53   unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
54   UndefElts = APInt(NumMaskElts, 0);
55   RawMask.resize(NumMaskElts, 0);
56 
57   // Fast path - if the constants match the mask size then copy direct.
58   if (MaskEltSizeInBits == CstEltSizeInBits) {
59     assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
60     for (unsigned i = 0; i != NumMaskElts; ++i) {
61       Constant *COp = C->getAggregateElement(i);
62       if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
63         return false;
64 
65       if (isa<UndefValue>(COp)) {
66         UndefElts.setBit(i);
67         RawMask[i] = 0;
68         continue;
69       }
70 
71       auto *Elt = cast<ConstantInt>(COp);
72       RawMask[i] = Elt->getValue().getZExtValue();
73     }
74     return true;
75   }
76 
77   // Extract all the undef/constant element data and pack into single bitsets.
78   APInt UndefBits(CstSizeInBits, 0);
79   APInt MaskBits(CstSizeInBits, 0);
80   for (unsigned i = 0; i != NumCstElts; ++i) {
81     Constant *COp = C->getAggregateElement(i);
82     if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
83       return false;
84 
85     unsigned BitOffset = i * CstEltSizeInBits;
86 
87     if (isa<UndefValue>(COp)) {
88       UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
89       continue;
90     }
91 
92     MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
93   }
94 
95   // Now extract the undef/constant bit data into the raw shuffle masks.
96   for (unsigned i = 0; i != NumMaskElts; ++i) {
97     unsigned BitOffset = i * MaskEltSizeInBits;
98     APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
99 
100     // Only treat the element as UNDEF if all bits are UNDEF, otherwise
101     // treat it as zero.
102     if (EltUndef.isAllOnesValue()) {
103       UndefElts.setBit(i);
104       RawMask[i] = 0;
105       continue;
106     }
107 
108     APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
109     RawMask[i] = EltBits.getZExtValue();
110   }
111 
112   return true;
113 }
114 
DecodePSHUFBMask(const Constant * C,unsigned Width,SmallVectorImpl<int> & ShuffleMask)115 void DecodePSHUFBMask(const Constant *C, unsigned Width,
116                       SmallVectorImpl<int> &ShuffleMask) {
117   assert((Width == 128 || Width == 256 || Width == 512) &&
118          C->getType()->getPrimitiveSizeInBits() >= Width &&
119          "Unexpected vector size.");
120 
121   // The shuffle mask requires a byte vector.
122   APInt UndefElts;
123   SmallVector<uint64_t, 64> RawMask;
124   if (!extractConstantMask(C, 8, UndefElts, RawMask))
125     return;
126 
127   unsigned NumElts = Width / 8;
128   assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
129          "Unexpected number of vector elements.");
130 
131   for (unsigned i = 0; i != NumElts; ++i) {
132     if (UndefElts[i]) {
133       ShuffleMask.push_back(SM_SentinelUndef);
134       continue;
135     }
136 
137     uint64_t Element = RawMask[i];
138     // If the high bit (7) of the byte is set, the element is zeroed.
139     if (Element & (1 << 7))
140       ShuffleMask.push_back(SM_SentinelZero);
141     else {
142       // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
143       // lane of the vector we're inside.
144       unsigned Base = i & ~0xf;
145 
146       // Only the least significant 4 bits of the byte are used.
147       int Index = Base + (Element & 0xf);
148       ShuffleMask.push_back(Index);
149     }
150   }
151 }
152 
DecodeVPERMILPMask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)153 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,
154                         SmallVectorImpl<int> &ShuffleMask) {
155   assert((Width == 128 || Width == 256 || Width == 512) &&
156          C->getType()->getPrimitiveSizeInBits() >= Width &&
157          "Unexpected vector size.");
158   assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
159 
160   // The shuffle mask requires elements the same size as the target.
161   APInt UndefElts;
162   SmallVector<uint64_t, 16> RawMask;
163   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
164     return;
165 
166   unsigned NumElts = Width / ElSize;
167   unsigned NumEltsPerLane = 128 / ElSize;
168   assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
169          "Unexpected number of vector elements.");
170 
171   for (unsigned i = 0; i != NumElts; ++i) {
172     if (UndefElts[i]) {
173       ShuffleMask.push_back(SM_SentinelUndef);
174       continue;
175     }
176 
177     int Index = i & ~(NumEltsPerLane - 1);
178     uint64_t Element = RawMask[i];
179     if (ElSize == 64)
180       Index += (Element >> 1) & 0x1;
181     else
182       Index += Element & 0x3;
183 
184     ShuffleMask.push_back(Index);
185   }
186 }
187 
DecodeVPERMIL2PMask(const Constant * C,unsigned M2Z,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)188 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
189                          unsigned Width,
190                          SmallVectorImpl<int> &ShuffleMask) {
191   Type *MaskTy = C->getType();
192   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
193   (void)MaskTySize;
194   assert((MaskTySize == 128 || MaskTySize == 256) &&
195          Width >= MaskTySize && "Unexpected vector size.");
196 
197   // The shuffle mask requires elements the same size as the target.
198   APInt UndefElts;
199   SmallVector<uint64_t, 8> RawMask;
200   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
201     return;
202 
203   unsigned NumElts = Width / ElSize;
204   unsigned NumEltsPerLane = 128 / ElSize;
205   assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
206          "Unexpected number of vector elements.");
207 
208   for (unsigned i = 0; i != NumElts; ++i) {
209     if (UndefElts[i]) {
210       ShuffleMask.push_back(SM_SentinelUndef);
211       continue;
212     }
213 
214     // VPERMIL2 Operation.
215     // Bits[3] - Match Bit.
216     // Bits[2:1] - (Per Lane) PD Shuffle Mask.
217     // Bits[2:0] - (Per Lane) PS Shuffle Mask.
218     uint64_t Selector = RawMask[i];
219     unsigned MatchBit = (Selector >> 3) & 0x1;
220 
221     // M2Z[0:1]     MatchBit
222     //   0Xb           X        Source selected by Selector index.
223     //   10b           0        Source selected by Selector index.
224     //   10b           1        Zero.
225     //   11b           0        Zero.
226     //   11b           1        Source selected by Selector index.
227     if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
228       ShuffleMask.push_back(SM_SentinelZero);
229       continue;
230     }
231 
232     int Index = i & ~(NumEltsPerLane - 1);
233     if (ElSize == 64)
234       Index += (Selector >> 1) & 0x1;
235     else
236       Index += Selector & 0x3;
237 
238     int Src = (Selector >> 2) & 0x1;
239     Index += Src * NumElts;
240     ShuffleMask.push_back(Index);
241   }
242 }
243 
DecodeVPPERMMask(const Constant * C,unsigned Width,SmallVectorImpl<int> & ShuffleMask)244 void DecodeVPPERMMask(const Constant *C, unsigned Width,
245                       SmallVectorImpl<int> &ShuffleMask) {
246   Type *MaskTy = C->getType();
247   unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
248   (void)MaskTySize;
249   assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");
250 
251   // The shuffle mask requires a byte vector.
252   APInt UndefElts;
253   SmallVector<uint64_t, 16> RawMask;
254   if (!extractConstantMask(C, 8, UndefElts, RawMask))
255     return;
256 
257   unsigned NumElts = Width / 8;
258   assert(NumElts == 16 && "Unexpected number of vector elements.");
259 
260   for (unsigned i = 0; i != NumElts; ++i) {
261     if (UndefElts[i]) {
262       ShuffleMask.push_back(SM_SentinelUndef);
263       continue;
264     }
265 
266     // VPPERM Operation
267     // Bits[4:0] - Byte Index (0 - 31)
268     // Bits[7:5] - Permute Operation
269     //
270     // Permute Operation:
271     // 0 - Source byte (no logical operation).
272     // 1 - Invert source byte.
273     // 2 - Bit reverse of source byte.
274     // 3 - Bit reverse of inverted source byte.
275     // 4 - 00h (zero - fill).
276     // 5 - FFh (ones - fill).
277     // 6 - Most significant bit of source byte replicated in all bit positions.
278     // 7 - Invert most significant bit of source byte and replicate in all bit
279     // positions.
280     uint64_t Element = RawMask[i];
281     uint64_t Index = Element & 0x1F;
282     uint64_t PermuteOp = (Element >> 5) & 0x7;
283 
284     if (PermuteOp == 4) {
285       ShuffleMask.push_back(SM_SentinelZero);
286       continue;
287     }
288     if (PermuteOp != 0) {
289       ShuffleMask.clear();
290       return;
291     }
292     ShuffleMask.push_back((int)Index);
293   }
294 }
295 
DecodeVPERMVMask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)296 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
297                       SmallVectorImpl<int> &ShuffleMask) {
298   assert((Width == 128 || Width == 256 || Width == 512) &&
299          C->getType()->getPrimitiveSizeInBits() >= Width &&
300          "Unexpected vector size.");
301   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
302          "Unexpected vector element size.");
303 
304   // The shuffle mask requires elements the same size as the target.
305   APInt UndefElts;
306   SmallVector<uint64_t, 64> RawMask;
307   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
308     return;
309 
310   unsigned NumElts = Width / ElSize;
311 
312   for (unsigned i = 0; i != NumElts; ++i) {
313     if (UndefElts[i]) {
314       ShuffleMask.push_back(SM_SentinelUndef);
315       continue;
316     }
317     int Index = RawMask[i] & (NumElts - 1);
318     ShuffleMask.push_back(Index);
319   }
320 }
321 
DecodeVPERMV3Mask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)322 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
323                        SmallVectorImpl<int> &ShuffleMask) {
324   assert((Width == 128 || Width == 256 || Width == 512) &&
325          C->getType()->getPrimitiveSizeInBits() >= Width &&
326          "Unexpected vector size.");
327   assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
328          "Unexpected vector element size.");
329 
330   // The shuffle mask requires elements the same size as the target.
331   APInt UndefElts;
332   SmallVector<uint64_t, 64> RawMask;
333   if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
334     return;
335 
336   unsigned NumElts = Width / ElSize;
337 
338   for (unsigned i = 0; i != NumElts; ++i) {
339     if (UndefElts[i]) {
340       ShuffleMask.push_back(SM_SentinelUndef);
341       continue;
342     }
343     int Index = RawMask[i] & (NumElts*2 - 1);
344     ShuffleMask.push_back(Index);
345   }
346 }
347 } // llvm namespace
348