1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Define several functions to decode x86 specific shuffle semantics using
10 // constants from the constant pool.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "Utils/X86ShuffleDecode.h"
15 #include "llvm/ADT/APInt.h"
16 #include "llvm/IR/Constants.h"
17
18 //===----------------------------------------------------------------------===//
19 // Vector Mask Decoding
20 //===----------------------------------------------------------------------===//
21
22 namespace llvm {
23
extractConstantMask(const Constant * C,unsigned MaskEltSizeInBits,APInt & UndefElts,SmallVectorImpl<uint64_t> & RawMask)24 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
25 APInt &UndefElts,
26 SmallVectorImpl<uint64_t> &RawMask) {
27 // It is not an error for shuffle masks to not be a vector of
28 // MaskEltSizeInBits because the constant pool uniques constants by their
29 // bit representation.
30 // e.g. the following take up the same space in the constant pool:
31 // i128 -170141183420855150465331762880109871104
32 //
33 // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
34 //
35 // <4 x i32> <i32 -2147483648, i32 -2147483648,
36 // i32 -2147483648, i32 -2147483648>
37 Type *CstTy = C->getType();
38 if (!CstTy->isVectorTy())
39 return false;
40
41 Type *CstEltTy = CstTy->getVectorElementType();
42 if (!CstEltTy->isIntegerTy())
43 return false;
44
45 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
46 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
47 unsigned NumCstElts = CstTy->getVectorNumElements();
48
49 assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
50 "Unaligned shuffle mask size");
51
52 unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
53 UndefElts = APInt(NumMaskElts, 0);
54 RawMask.resize(NumMaskElts, 0);
55
56 // Fast path - if the constants match the mask size then copy direct.
57 if (MaskEltSizeInBits == CstEltSizeInBits) {
58 assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
59 for (unsigned i = 0; i != NumMaskElts; ++i) {
60 Constant *COp = C->getAggregateElement(i);
61 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
62 return false;
63
64 if (isa<UndefValue>(COp)) {
65 UndefElts.setBit(i);
66 RawMask[i] = 0;
67 continue;
68 }
69
70 auto *Elt = cast<ConstantInt>(COp);
71 RawMask[i] = Elt->getValue().getZExtValue();
72 }
73 return true;
74 }
75
76 // Extract all the undef/constant element data and pack into single bitsets.
77 APInt UndefBits(CstSizeInBits, 0);
78 APInt MaskBits(CstSizeInBits, 0);
79 for (unsigned i = 0; i != NumCstElts; ++i) {
80 Constant *COp = C->getAggregateElement(i);
81 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
82 return false;
83
84 unsigned BitOffset = i * CstEltSizeInBits;
85
86 if (isa<UndefValue>(COp)) {
87 UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
88 continue;
89 }
90
91 MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
92 }
93
94 // Now extract the undef/constant bit data into the raw shuffle masks.
95 for (unsigned i = 0; i != NumMaskElts; ++i) {
96 unsigned BitOffset = i * MaskEltSizeInBits;
97 APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);
98
99 // Only treat the element as UNDEF if all bits are UNDEF, otherwise
100 // treat it as zero.
101 if (EltUndef.isAllOnesValue()) {
102 UndefElts.setBit(i);
103 RawMask[i] = 0;
104 continue;
105 }
106
107 APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
108 RawMask[i] = EltBits.getZExtValue();
109 }
110
111 return true;
112 }
113
DecodePSHUFBMask(const Constant * C,unsigned Width,SmallVectorImpl<int> & ShuffleMask)114 void DecodePSHUFBMask(const Constant *C, unsigned Width,
115 SmallVectorImpl<int> &ShuffleMask) {
116 assert((Width == 128 || Width == 256 || Width == 512) &&
117 C->getType()->getPrimitiveSizeInBits() >= Width &&
118 "Unexpected vector size.");
119
120 // The shuffle mask requires a byte vector.
121 APInt UndefElts;
122 SmallVector<uint64_t, 64> RawMask;
123 if (!extractConstantMask(C, 8, UndefElts, RawMask))
124 return;
125
126 unsigned NumElts = Width / 8;
127 assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
128 "Unexpected number of vector elements.");
129
130 for (unsigned i = 0; i != NumElts; ++i) {
131 if (UndefElts[i]) {
132 ShuffleMask.push_back(SM_SentinelUndef);
133 continue;
134 }
135
136 uint64_t Element = RawMask[i];
137 // If the high bit (7) of the byte is set, the element is zeroed.
138 if (Element & (1 << 7))
139 ShuffleMask.push_back(SM_SentinelZero);
140 else {
141 // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
142 // lane of the vector we're inside.
143 unsigned Base = i & ~0xf;
144
145 // Only the least significant 4 bits of the byte are used.
146 int Index = Base + (Element & 0xf);
147 ShuffleMask.push_back(Index);
148 }
149 }
150 }
151
DecodeVPERMILPMask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)152 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,
153 SmallVectorImpl<int> &ShuffleMask) {
154 assert((Width == 128 || Width == 256 || Width == 512) &&
155 C->getType()->getPrimitiveSizeInBits() >= Width &&
156 "Unexpected vector size.");
157 assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");
158
159 // The shuffle mask requires elements the same size as the target.
160 APInt UndefElts;
161 SmallVector<uint64_t, 16> RawMask;
162 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
163 return;
164
165 unsigned NumElts = Width / ElSize;
166 unsigned NumEltsPerLane = 128 / ElSize;
167 assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
168 "Unexpected number of vector elements.");
169
170 for (unsigned i = 0; i != NumElts; ++i) {
171 if (UndefElts[i]) {
172 ShuffleMask.push_back(SM_SentinelUndef);
173 continue;
174 }
175
176 int Index = i & ~(NumEltsPerLane - 1);
177 uint64_t Element = RawMask[i];
178 if (ElSize == 64)
179 Index += (Element >> 1) & 0x1;
180 else
181 Index += Element & 0x3;
182
183 ShuffleMask.push_back(Index);
184 }
185 }
186
DecodeVPERMIL2PMask(const Constant * C,unsigned M2Z,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)187 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
188 unsigned Width,
189 SmallVectorImpl<int> &ShuffleMask) {
190 Type *MaskTy = C->getType();
191 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
192 (void)MaskTySize;
193 assert((MaskTySize == 128 || MaskTySize == 256) &&
194 Width >= MaskTySize && "Unexpected vector size.");
195
196 // The shuffle mask requires elements the same size as the target.
197 APInt UndefElts;
198 SmallVector<uint64_t, 8> RawMask;
199 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
200 return;
201
202 unsigned NumElts = Width / ElSize;
203 unsigned NumEltsPerLane = 128 / ElSize;
204 assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
205 "Unexpected number of vector elements.");
206
207 for (unsigned i = 0; i != NumElts; ++i) {
208 if (UndefElts[i]) {
209 ShuffleMask.push_back(SM_SentinelUndef);
210 continue;
211 }
212
213 // VPERMIL2 Operation.
214 // Bits[3] - Match Bit.
215 // Bits[2:1] - (Per Lane) PD Shuffle Mask.
216 // Bits[2:0] - (Per Lane) PS Shuffle Mask.
217 uint64_t Selector = RawMask[i];
218 unsigned MatchBit = (Selector >> 3) & 0x1;
219
220 // M2Z[0:1] MatchBit
221 // 0Xb X Source selected by Selector index.
222 // 10b 0 Source selected by Selector index.
223 // 10b 1 Zero.
224 // 11b 0 Zero.
225 // 11b 1 Source selected by Selector index.
226 if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
227 ShuffleMask.push_back(SM_SentinelZero);
228 continue;
229 }
230
231 int Index = i & ~(NumEltsPerLane - 1);
232 if (ElSize == 64)
233 Index += (Selector >> 1) & 0x1;
234 else
235 Index += Selector & 0x3;
236
237 int Src = (Selector >> 2) & 0x1;
238 Index += Src * NumElts;
239 ShuffleMask.push_back(Index);
240 }
241 }
242
DecodeVPPERMMask(const Constant * C,unsigned Width,SmallVectorImpl<int> & ShuffleMask)243 void DecodeVPPERMMask(const Constant *C, unsigned Width,
244 SmallVectorImpl<int> &ShuffleMask) {
245 Type *MaskTy = C->getType();
246 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
247 (void)MaskTySize;
248 assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");
249
250 // The shuffle mask requires a byte vector.
251 APInt UndefElts;
252 SmallVector<uint64_t, 16> RawMask;
253 if (!extractConstantMask(C, 8, UndefElts, RawMask))
254 return;
255
256 unsigned NumElts = Width / 8;
257 assert(NumElts == 16 && "Unexpected number of vector elements.");
258
259 for (unsigned i = 0; i != NumElts; ++i) {
260 if (UndefElts[i]) {
261 ShuffleMask.push_back(SM_SentinelUndef);
262 continue;
263 }
264
265 // VPPERM Operation
266 // Bits[4:0] - Byte Index (0 - 31)
267 // Bits[7:5] - Permute Operation
268 //
269 // Permute Operation:
270 // 0 - Source byte (no logical operation).
271 // 1 - Invert source byte.
272 // 2 - Bit reverse of source byte.
273 // 3 - Bit reverse of inverted source byte.
274 // 4 - 00h (zero - fill).
275 // 5 - FFh (ones - fill).
276 // 6 - Most significant bit of source byte replicated in all bit positions.
277 // 7 - Invert most significant bit of source byte and replicate in all bit
278 // positions.
279 uint64_t Element = RawMask[i];
280 uint64_t Index = Element & 0x1F;
281 uint64_t PermuteOp = (Element >> 5) & 0x7;
282
283 if (PermuteOp == 4) {
284 ShuffleMask.push_back(SM_SentinelZero);
285 continue;
286 }
287 if (PermuteOp != 0) {
288 ShuffleMask.clear();
289 return;
290 }
291 ShuffleMask.push_back((int)Index);
292 }
293 }
294
DecodeVPERMVMask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)295 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
296 SmallVectorImpl<int> &ShuffleMask) {
297 assert((Width == 128 || Width == 256 || Width == 512) &&
298 C->getType()->getPrimitiveSizeInBits() >= Width &&
299 "Unexpected vector size.");
300 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
301 "Unexpected vector element size.");
302
303 // The shuffle mask requires elements the same size as the target.
304 APInt UndefElts;
305 SmallVector<uint64_t, 64> RawMask;
306 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
307 return;
308
309 unsigned NumElts = Width / ElSize;
310
311 for (unsigned i = 0; i != NumElts; ++i) {
312 if (UndefElts[i]) {
313 ShuffleMask.push_back(SM_SentinelUndef);
314 continue;
315 }
316 int Index = RawMask[i] & (NumElts - 1);
317 ShuffleMask.push_back(Index);
318 }
319 }
320
DecodeVPERMV3Mask(const Constant * C,unsigned ElSize,unsigned Width,SmallVectorImpl<int> & ShuffleMask)321 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
322 SmallVectorImpl<int> &ShuffleMask) {
323 assert((Width == 128 || Width == 256 || Width == 512) &&
324 C->getType()->getPrimitiveSizeInBits() >= Width &&
325 "Unexpected vector size.");
326 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
327 "Unexpected vector element size.");
328
329 // The shuffle mask requires elements the same size as the target.
330 APInt UndefElts;
331 SmallVector<uint64_t, 64> RawMask;
332 if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
333 return;
334
335 unsigned NumElts = Width / ElSize;
336
337 for (unsigned i = 0; i != NumElts; ++i) {
338 if (UndefElts[i]) {
339 ShuffleMask.push_back(SM_SentinelUndef);
340 continue;
341 }
342 int Index = RawMask[i] & (NumElts*2 - 1);
343 ShuffleMask.push_back(Index);
344 }
345 }
346 } // llvm namespace
347