1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Define several functions to decode x86 specific shuffle semantics using
11 // constants from the constant pool.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "Utils/X86ShuffleDecode.h"
16 #include "llvm/ADT/APInt.h"
17 #include "llvm/IR/Constants.h"
18
19 //===----------------------------------------------------------------------===//
20 // Vector Mask Decoding
21 //===----------------------------------------------------------------------===//
22
23 namespace llvm {
24
extractConstantMask(const Constant * C,unsigned MaskEltSizeInBits,APInt & UndefElts,SmallVectorImpl<uint64_t> & RawMask)25 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
26 APInt &UndefElts,
27 SmallVectorImpl<uint64_t> &RawMask) {
28 // It is not an error for shuffle masks to not be a vector of
29 // MaskEltSizeInBits because the constant pool uniques constants by their
30 // bit representation.
31 // e.g. the following take up the same space in the constant pool:
32 // i128 -170141183420855150465331762880109871104
33 //
34 // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
35 //
36 // <4 x i32> <i32 -2147483648, i32 -2147483648,
37 // i32 -2147483648, i32 -2147483648>
38 Type *CstTy = C->getType();
39 if (!CstTy->isVectorTy())
40 return false;
41
42 Type *CstEltTy = CstTy->getVectorElementType();
43 if (!CstEltTy->isIntegerTy())
44 return false;
45
46 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
47 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
48 unsigned NumCstElts = CstTy->getVectorNumElements();
49
50 assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
51 "Unaligned shuffle mask size");
52
53 unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
54 UndefElts = APInt(NumMaskElts, 0);
55 RawMask.resize(NumMaskElts, 0);
56
57 // Fast path - if the constants match the mask size then copy direct.
58 if (MaskEltSizeInBits == CstEltSizeInBits) {
59 assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
60 for (unsigned i = 0; i != NumMaskElts; ++i) {
61 Constant *COp = C->getAggregateElement(i);
62 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
63 return false;
64
65 if (isa<UndefValue>(COp)) {
66 UndefElts.setBit(i);
67 RawMask[i] = 0;
68 continue;
69 }
70
71 auto *Elt = cast<ConstantInt>(COp);
72 RawMask[i] = Elt->getValue().getZExtValue();
73 }
74 return true;
75 }
76
77 // Extract all the undef/constant element data and pack into single bitsets.
78 APInt UndefBits(CstSizeInBits, 0);
79 APInt MaskBits(CstSizeInBits, 0);
80 for (unsigned i = 0; i != NumCstElts; ++i) {
81 Constant *COp = C->getAggregateElement(i);
82 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
83 return false;
84
85 unsigned BitOffset = i * CstEltSizeInBits;
86
87 if (isa<UndefValue>(COp)) {
88 UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
89 continue;
90 }
91
92 MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
93 }
94
95 // Now extract the undef/constant bit data into the raw shuffle masks.
96 for (unsigned i = 0; i != NumMaskElts; ++i) {
97 unsigned BitOffset = i * MaskEltSizeInBits;
98 APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
99
100 // Only treat the element as UNDEF if all bits are UNDEF, otherwise
101 // treat it as zero.
102 if (EltUndef.isAllOnesValue()) {
103 UndefElts.setBit(i);
104 RawMask[i] = 0;
105 continue;
106 }
107
108 APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
109 RawMask[i] = EltBits.getZExtValue();
110 }
111
112 return true;
113 }
114
DecodePSHUFBMask(const Constant * C,unsigned Width,SmallVectorImpl<int> & ShuffleMask)115 void DecodePSHUFBMask(const Constant *C, unsigned Width,
116 SmallVectorImpl<int> &ShuffleMask) {
117 assert((Width == 128 || Width == 256 || Width == 512) &&
118 C->getType()->getPrimitiveSizeInBits() >= Width &&
119 "Unexpected vector size.");
120
121 // The shuffle mask requires a byte vector.
122 APInt UndefElts;
123 SmallVector<uint64_t, 64> RawMask;
124 if (!extractConstantMask(C, 8, UndefElts, RawMask))
125 return;
126
127 unsigned NumElts = Width / 8;
128 assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
129 "Unexpected number of vector elements.");
130
131 for (unsigned i = 0; i != NumElts; ++i) {
132 if (UndefElts[i]) {
133 ShuffleMask.push_back(SM_SentinelUndef);
134 continue;
135 }
136
137 uint64_t Element = RawMask[i];
138 // If the high bit (7) of the byte is set, the element is zeroed.
139 if (Element & (1 << 7))
140 ShuffleMask.push_back(SM_SentinelZero);
141 else {
142 // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
143 // lane of the vector we're inside.
144 unsigned Base = i & ~0xf;
145
146 // Only the least significant 4 bits of the byte are used.
147 int Index = Base + (Element & 0xf);
148 ShuffleMask.push_back(Index);
149 }
150 }
151 }
152
DecodeVPERMILPMask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)153 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,
154 SmallVectorImpl<int> &ShuffleMask) {
155 assert((Width == 128 || Width == 256 || Width == 512) &&
156 C->getType()->getPrimitiveSizeInBits() >= Width &&
157 "Unexpected vector size.");
158 assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
159
160 // The shuffle mask requires elements the same size as the target.
161 APInt UndefElts;
162 SmallVector<uint64_t, 16> RawMask;
163 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
164 return;
165
166 unsigned NumElts = Width / ElSize;
167 unsigned NumEltsPerLane = 128 / ElSize;
168 assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
169 "Unexpected number of vector elements.");
170
171 for (unsigned i = 0; i != NumElts; ++i) {
172 if (UndefElts[i]) {
173 ShuffleMask.push_back(SM_SentinelUndef);
174 continue;
175 }
176
177 int Index = i & ~(NumEltsPerLane - 1);
178 uint64_t Element = RawMask[i];
179 if (ElSize == 64)
180 Index += (Element >> 1) & 0x1;
181 else
182 Index += Element & 0x3;
183
184 ShuffleMask.push_back(Index);
185 }
186 }
187
DecodeVPERMIL2PMask(const Constant * C,unsigned M2Z,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)188 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
189 unsigned Width,
190 SmallVectorImpl<int> &ShuffleMask) {
191 Type *MaskTy = C->getType();
192 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
193 (void)MaskTySize;
194 assert((MaskTySize == 128 || MaskTySize == 256) &&
195 Width >= MaskTySize && "Unexpected vector size.");
196
197 // The shuffle mask requires elements the same size as the target.
198 APInt UndefElts;
199 SmallVector<uint64_t, 8> RawMask;
200 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
201 return;
202
203 unsigned NumElts = Width / ElSize;
204 unsigned NumEltsPerLane = 128 / ElSize;
205 assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
206 "Unexpected number of vector elements.");
207
208 for (unsigned i = 0; i != NumElts; ++i) {
209 if (UndefElts[i]) {
210 ShuffleMask.push_back(SM_SentinelUndef);
211 continue;
212 }
213
214 // VPERMIL2 Operation.
215 // Bits[3] - Match Bit.
216 // Bits[2:1] - (Per Lane) PD Shuffle Mask.
217 // Bits[2:0] - (Per Lane) PS Shuffle Mask.
218 uint64_t Selector = RawMask[i];
219 unsigned MatchBit = (Selector >> 3) & 0x1;
220
221 // M2Z[0:1] MatchBit
222 // 0Xb X Source selected by Selector index.
223 // 10b 0 Source selected by Selector index.
224 // 10b 1 Zero.
225 // 11b 0 Zero.
226 // 11b 1 Source selected by Selector index.
227 if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
228 ShuffleMask.push_back(SM_SentinelZero);
229 continue;
230 }
231
232 int Index = i & ~(NumEltsPerLane - 1);
233 if (ElSize == 64)
234 Index += (Selector >> 1) & 0x1;
235 else
236 Index += Selector & 0x3;
237
238 int Src = (Selector >> 2) & 0x1;
239 Index += Src * NumElts;
240 ShuffleMask.push_back(Index);
241 }
242 }
243
DecodeVPPERMMask(const Constant * C,unsigned Width,SmallVectorImpl<int> & ShuffleMask)244 void DecodeVPPERMMask(const Constant *C, unsigned Width,
245 SmallVectorImpl<int> &ShuffleMask) {
246 Type *MaskTy = C->getType();
247 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
248 (void)MaskTySize;
249 assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");
250
251 // The shuffle mask requires a byte vector.
252 APInt UndefElts;
253 SmallVector<uint64_t, 16> RawMask;
254 if (!extractConstantMask(C, 8, UndefElts, RawMask))
255 return;
256
257 unsigned NumElts = Width / 8;
258 assert(NumElts == 16 && "Unexpected number of vector elements.");
259
260 for (unsigned i = 0; i != NumElts; ++i) {
261 if (UndefElts[i]) {
262 ShuffleMask.push_back(SM_SentinelUndef);
263 continue;
264 }
265
266 // VPPERM Operation
267 // Bits[4:0] - Byte Index (0 - 31)
268 // Bits[7:5] - Permute Operation
269 //
270 // Permute Operation:
271 // 0 - Source byte (no logical operation).
272 // 1 - Invert source byte.
273 // 2 - Bit reverse of source byte.
274 // 3 - Bit reverse of inverted source byte.
275 // 4 - 00h (zero - fill).
276 // 5 - FFh (ones - fill).
277 // 6 - Most significant bit of source byte replicated in all bit positions.
278 // 7 - Invert most significant bit of source byte and replicate in all bit
279 // positions.
280 uint64_t Element = RawMask[i];
281 uint64_t Index = Element & 0x1F;
282 uint64_t PermuteOp = (Element >> 5) & 0x7;
283
284 if (PermuteOp == 4) {
285 ShuffleMask.push_back(SM_SentinelZero);
286 continue;
287 }
288 if (PermuteOp != 0) {
289 ShuffleMask.clear();
290 return;
291 }
292 ShuffleMask.push_back((int)Index);
293 }
294 }
295
DecodeVPERMVMask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)296 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
297 SmallVectorImpl<int> &ShuffleMask) {
298 assert((Width == 128 || Width == 256 || Width == 512) &&
299 C->getType()->getPrimitiveSizeInBits() >= Width &&
300 "Unexpected vector size.");
301 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
302 "Unexpected vector element size.");
303
304 // The shuffle mask requires elements the same size as the target.
305 APInt UndefElts;
306 SmallVector<uint64_t, 64> RawMask;
307 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
308 return;
309
310 unsigned NumElts = Width / ElSize;
311
312 for (unsigned i = 0; i != NumElts; ++i) {
313 if (UndefElts[i]) {
314 ShuffleMask.push_back(SM_SentinelUndef);
315 continue;
316 }
317 int Index = RawMask[i] & (NumElts - 1);
318 ShuffleMask.push_back(Index);
319 }
320 }
321
DecodeVPERMV3Mask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)322 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
323 SmallVectorImpl<int> &ShuffleMask) {
324 assert((Width == 128 || Width == 256 || Width == 512) &&
325 C->getType()->getPrimitiveSizeInBits() >= Width &&
326 "Unexpected vector size.");
327 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
328 "Unexpected vector element size.");
329
330 // The shuffle mask requires elements the same size as the target.
331 APInt UndefElts;
332 SmallVector<uint64_t, 64> RawMask;
333 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
334 return;
335
336 unsigned NumElts = Width / ElSize;
337
338 for (unsigned i = 0; i != NumElts; ++i) {
339 if (UndefElts[i]) {
340 ShuffleMask.push_back(SM_SentinelUndef);
341 continue;
342 }
343 int Index = RawMask[i] & (NumElts*2 - 1);
344 ShuffleMask.push_back(Index);
345 }
346 }
347 } // llvm namespace
348