1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements the LegalizerHelper class to legalize
10 /// individual instructions and the LegalizeMachineIR wrapper pass for the
11 /// primary legalization.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
16 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
17 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
18 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
19 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
20 #include "llvm/CodeGen/MachineRegisterInfo.h"
21 #include "llvm/CodeGen/TargetFrameLowering.h"
22 #include "llvm/CodeGen/TargetInstrInfo.h"
23 #include "llvm/CodeGen/TargetLowering.h"
24 #include "llvm/CodeGen/TargetSubtargetInfo.h"
25 #include "llvm/Support/Debug.h"
26 #include "llvm/Support/MathExtras.h"
27 #include "llvm/Support/raw_ostream.h"
28
29 #define DEBUG_TYPE "legalizer"
30
31 using namespace llvm;
32 using namespace LegalizeActions;
33 using namespace MIPatternMatch;
34
35 /// Try to break down \p OrigTy into \p NarrowTy sized pieces.
36 ///
37 /// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
38 /// with any leftover piece as type \p LeftoverTy
39 ///
40 /// Returns -1 in the first element of the pair if the breakdown is not
41 /// satisfiable.
42 static std::pair<int, int>
getNarrowTypeBreakDown(LLT OrigTy,LLT NarrowTy,LLT & LeftoverTy)43 getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
44 assert(!LeftoverTy.isValid() && "this is an out argument");
45
46 unsigned Size = OrigTy.getSizeInBits();
47 unsigned NarrowSize = NarrowTy.getSizeInBits();
48 unsigned NumParts = Size / NarrowSize;
49 unsigned LeftoverSize = Size - NumParts * NarrowSize;
50 assert(Size > NarrowSize);
51
52 if (LeftoverSize == 0)
53 return {NumParts, 0};
54
55 if (NarrowTy.isVector()) {
56 unsigned EltSize = OrigTy.getScalarSizeInBits();
57 if (LeftoverSize % EltSize != 0)
58 return {-1, -1};
59 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
60 } else {
61 LeftoverTy = LLT::scalar(LeftoverSize);
62 }
63
64 int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
65 return std::make_pair(NumParts, NumLeftover);
66 }
67
getFloatTypeForLLT(LLVMContext & Ctx,LLT Ty)68 static Type *getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty) {
69
70 if (!Ty.isScalar())
71 return nullptr;
72
73 switch (Ty.getSizeInBits()) {
74 case 16:
75 return Type::getHalfTy(Ctx);
76 case 32:
77 return Type::getFloatTy(Ctx);
78 case 64:
79 return Type::getDoubleTy(Ctx);
80 case 80:
81 return Type::getX86_FP80Ty(Ctx);
82 case 128:
83 return Type::getFP128Ty(Ctx);
84 default:
85 return nullptr;
86 }
87 }
88
LegalizerHelper(MachineFunction & MF,GISelChangeObserver & Observer,MachineIRBuilder & Builder)89 LegalizerHelper::LegalizerHelper(MachineFunction &MF,
90 GISelChangeObserver &Observer,
91 MachineIRBuilder &Builder)
92 : MIRBuilder(Builder), Observer(Observer), MRI(MF.getRegInfo()),
93 LI(*MF.getSubtarget().getLegalizerInfo()),
94 TLI(*MF.getSubtarget().getTargetLowering()) {
95 MIRBuilder.setChangeObserver(Observer);
96 }
97
LegalizerHelper(MachineFunction & MF,const LegalizerInfo & LI,GISelChangeObserver & Observer,MachineIRBuilder & B)98 LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
99 GISelChangeObserver &Observer,
100 MachineIRBuilder &B)
101 : MIRBuilder(B), Observer(Observer), MRI(MF.getRegInfo()), LI(LI),
102 TLI(*MF.getSubtarget().getTargetLowering()) {
103 MIRBuilder.setChangeObserver(Observer);
104 }
105 LegalizerHelper::LegalizeResult
legalizeInstrStep(MachineInstr & MI)106 LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
107 LLVM_DEBUG(dbgs() << "Legalizing: " << MI);
108
109 MIRBuilder.setInstrAndDebugLoc(MI);
110
111 if (MI.getOpcode() == TargetOpcode::G_INTRINSIC ||
112 MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
113 return LI.legalizeIntrinsic(*this, MI) ? Legalized : UnableToLegalize;
114 auto Step = LI.getAction(MI, MRI);
115 switch (Step.Action) {
116 case Legal:
117 LLVM_DEBUG(dbgs() << ".. Already legal\n");
118 return AlreadyLegal;
119 case Libcall:
120 LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
121 return libcall(MI);
122 case NarrowScalar:
123 LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
124 return narrowScalar(MI, Step.TypeIdx, Step.NewType);
125 case WidenScalar:
126 LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
127 return widenScalar(MI, Step.TypeIdx, Step.NewType);
128 case Bitcast:
129 LLVM_DEBUG(dbgs() << ".. Bitcast type\n");
130 return bitcast(MI, Step.TypeIdx, Step.NewType);
131 case Lower:
132 LLVM_DEBUG(dbgs() << ".. Lower\n");
133 return lower(MI, Step.TypeIdx, Step.NewType);
134 case FewerElements:
135 LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
136 return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
137 case MoreElements:
138 LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
139 return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
140 case Custom:
141 LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
142 return LI.legalizeCustom(*this, MI) ? Legalized : UnableToLegalize;
143 default:
144 LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
145 return UnableToLegalize;
146 }
147 }
148
extractParts(Register Reg,LLT Ty,int NumParts,SmallVectorImpl<Register> & VRegs)149 void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts,
150 SmallVectorImpl<Register> &VRegs) {
151 for (int i = 0; i < NumParts; ++i)
152 VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
153 MIRBuilder.buildUnmerge(VRegs, Reg);
154 }
155
extractParts(Register Reg,LLT RegTy,LLT MainTy,LLT & LeftoverTy,SmallVectorImpl<Register> & VRegs,SmallVectorImpl<Register> & LeftoverRegs)156 bool LegalizerHelper::extractParts(Register Reg, LLT RegTy,
157 LLT MainTy, LLT &LeftoverTy,
158 SmallVectorImpl<Register> &VRegs,
159 SmallVectorImpl<Register> &LeftoverRegs) {
160 assert(!LeftoverTy.isValid() && "this is an out argument");
161
162 unsigned RegSize = RegTy.getSizeInBits();
163 unsigned MainSize = MainTy.getSizeInBits();
164 unsigned NumParts = RegSize / MainSize;
165 unsigned LeftoverSize = RegSize - NumParts * MainSize;
166
167 // Use an unmerge when possible.
168 if (LeftoverSize == 0) {
169 for (unsigned I = 0; I < NumParts; ++I)
170 VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
171 MIRBuilder.buildUnmerge(VRegs, Reg);
172 return true;
173 }
174
175 if (MainTy.isVector()) {
176 unsigned EltSize = MainTy.getScalarSizeInBits();
177 if (LeftoverSize % EltSize != 0)
178 return false;
179 LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
180 } else {
181 LeftoverTy = LLT::scalar(LeftoverSize);
182 }
183
184 // For irregular sizes, extract the individual parts.
185 for (unsigned I = 0; I != NumParts; ++I) {
186 Register NewReg = MRI.createGenericVirtualRegister(MainTy);
187 VRegs.push_back(NewReg);
188 MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
189 }
190
191 for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
192 Offset += LeftoverSize) {
193 Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
194 LeftoverRegs.push_back(NewReg);
195 MIRBuilder.buildExtract(NewReg, Reg, Offset);
196 }
197
198 return true;
199 }
200
insertParts(Register DstReg,LLT ResultTy,LLT PartTy,ArrayRef<Register> PartRegs,LLT LeftoverTy,ArrayRef<Register> LeftoverRegs)201 void LegalizerHelper::insertParts(Register DstReg,
202 LLT ResultTy, LLT PartTy,
203 ArrayRef<Register> PartRegs,
204 LLT LeftoverTy,
205 ArrayRef<Register> LeftoverRegs) {
206 if (!LeftoverTy.isValid()) {
207 assert(LeftoverRegs.empty());
208
209 if (!ResultTy.isVector()) {
210 MIRBuilder.buildMerge(DstReg, PartRegs);
211 return;
212 }
213
214 if (PartTy.isVector())
215 MIRBuilder.buildConcatVectors(DstReg, PartRegs);
216 else
217 MIRBuilder.buildBuildVector(DstReg, PartRegs);
218 return;
219 }
220
221 unsigned PartSize = PartTy.getSizeInBits();
222 unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
223
224 Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
225 MIRBuilder.buildUndef(CurResultReg);
226
227 unsigned Offset = 0;
228 for (Register PartReg : PartRegs) {
229 Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
230 MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
231 CurResultReg = NewResultReg;
232 Offset += PartSize;
233 }
234
235 for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
236 // Use the original output register for the final insert to avoid a copy.
237 Register NewResultReg = (I + 1 == E) ?
238 DstReg : MRI.createGenericVirtualRegister(ResultTy);
239
240 MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
241 CurResultReg = NewResultReg;
242 Offset += LeftoverPartSize;
243 }
244 }
245
246 /// Append the result registers of G_UNMERGE_VALUES \p MI to \p Regs.
getUnmergeResults(SmallVectorImpl<Register> & Regs,const MachineInstr & MI)247 static void getUnmergeResults(SmallVectorImpl<Register> &Regs,
248 const MachineInstr &MI) {
249 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
250
251 const int StartIdx = Regs.size();
252 const int NumResults = MI.getNumOperands() - 1;
253 Regs.resize(Regs.size() + NumResults);
254 for (int I = 0; I != NumResults; ++I)
255 Regs[StartIdx + I] = MI.getOperand(I).getReg();
256 }
257
extractGCDType(SmallVectorImpl<Register> & Parts,LLT GCDTy,Register SrcReg)258 void LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts,
259 LLT GCDTy, Register SrcReg) {
260 LLT SrcTy = MRI.getType(SrcReg);
261 if (SrcTy == GCDTy) {
262 // If the source already evenly divides the result type, we don't need to do
263 // anything.
264 Parts.push_back(SrcReg);
265 } else {
266 // Need to split into common type sized pieces.
267 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
268 getUnmergeResults(Parts, *Unmerge);
269 }
270 }
271
extractGCDType(SmallVectorImpl<Register> & Parts,LLT DstTy,LLT NarrowTy,Register SrcReg)272 LLT LegalizerHelper::extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy,
273 LLT NarrowTy, Register SrcReg) {
274 LLT SrcTy = MRI.getType(SrcReg);
275 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
276 extractGCDType(Parts, GCDTy, SrcReg);
277 return GCDTy;
278 }
279
buildLCMMergePieces(LLT DstTy,LLT NarrowTy,LLT GCDTy,SmallVectorImpl<Register> & VRegs,unsigned PadStrategy)280 LLT LegalizerHelper::buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy,
281 SmallVectorImpl<Register> &VRegs,
282 unsigned PadStrategy) {
283 LLT LCMTy = getLCMType(DstTy, NarrowTy);
284
285 int NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
286 int NumSubParts = NarrowTy.getSizeInBits() / GCDTy.getSizeInBits();
287 int NumOrigSrc = VRegs.size();
288
289 Register PadReg;
290
291 // Get a value we can use to pad the source value if the sources won't evenly
292 // cover the result type.
293 if (NumOrigSrc < NumParts * NumSubParts) {
294 if (PadStrategy == TargetOpcode::G_ZEXT)
295 PadReg = MIRBuilder.buildConstant(GCDTy, 0).getReg(0);
296 else if (PadStrategy == TargetOpcode::G_ANYEXT)
297 PadReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
298 else {
299 assert(PadStrategy == TargetOpcode::G_SEXT);
300
301 // Shift the sign bit of the low register through the high register.
302 auto ShiftAmt =
303 MIRBuilder.buildConstant(LLT::scalar(64), GCDTy.getSizeInBits() - 1);
304 PadReg = MIRBuilder.buildAShr(GCDTy, VRegs.back(), ShiftAmt).getReg(0);
305 }
306 }
307
308 // Registers for the final merge to be produced.
309 SmallVector<Register, 4> Remerge(NumParts);
310
311 // Registers needed for intermediate merges, which will be merged into a
312 // source for Remerge.
313 SmallVector<Register, 4> SubMerge(NumSubParts);
314
315 // Once we've fully read off the end of the original source bits, we can reuse
316 // the same high bits for remaining padding elements.
317 Register AllPadReg;
318
319 // Build merges to the LCM type to cover the original result type.
320 for (int I = 0; I != NumParts; ++I) {
321 bool AllMergePartsArePadding = true;
322
323 // Build the requested merges to the requested type.
324 for (int J = 0; J != NumSubParts; ++J) {
325 int Idx = I * NumSubParts + J;
326 if (Idx >= NumOrigSrc) {
327 SubMerge[J] = PadReg;
328 continue;
329 }
330
331 SubMerge[J] = VRegs[Idx];
332
333 // There are meaningful bits here we can't reuse later.
334 AllMergePartsArePadding = false;
335 }
336
337 // If we've filled up a complete piece with padding bits, we can directly
338 // emit the natural sized constant if applicable, rather than a merge of
339 // smaller constants.
340 if (AllMergePartsArePadding && !AllPadReg) {
341 if (PadStrategy == TargetOpcode::G_ANYEXT)
342 AllPadReg = MIRBuilder.buildUndef(NarrowTy).getReg(0);
343 else if (PadStrategy == TargetOpcode::G_ZEXT)
344 AllPadReg = MIRBuilder.buildConstant(NarrowTy, 0).getReg(0);
345
346 // If this is a sign extension, we can't materialize a trivial constant
347 // with the right type and have to produce a merge.
348 }
349
350 if (AllPadReg) {
351 // Avoid creating additional instructions if we're just adding additional
352 // copies of padding bits.
353 Remerge[I] = AllPadReg;
354 continue;
355 }
356
357 if (NumSubParts == 1)
358 Remerge[I] = SubMerge[0];
359 else
360 Remerge[I] = MIRBuilder.buildMerge(NarrowTy, SubMerge).getReg(0);
361
362 // In the sign extend padding case, re-use the first all-signbit merge.
363 if (AllMergePartsArePadding && !AllPadReg)
364 AllPadReg = Remerge[I];
365 }
366
367 VRegs = std::move(Remerge);
368 return LCMTy;
369 }
370
buildWidenedRemergeToDst(Register DstReg,LLT LCMTy,ArrayRef<Register> RemergeRegs)371 void LegalizerHelper::buildWidenedRemergeToDst(Register DstReg, LLT LCMTy,
372 ArrayRef<Register> RemergeRegs) {
373 LLT DstTy = MRI.getType(DstReg);
374
375 // Create the merge to the widened source, and extract the relevant bits into
376 // the result.
377
378 if (DstTy == LCMTy) {
379 MIRBuilder.buildMerge(DstReg, RemergeRegs);
380 return;
381 }
382
383 auto Remerge = MIRBuilder.buildMerge(LCMTy, RemergeRegs);
384 if (DstTy.isScalar() && LCMTy.isScalar()) {
385 MIRBuilder.buildTrunc(DstReg, Remerge);
386 return;
387 }
388
389 if (LCMTy.isVector()) {
390 unsigned NumDefs = LCMTy.getSizeInBits() / DstTy.getSizeInBits();
391 SmallVector<Register, 8> UnmergeDefs(NumDefs);
392 UnmergeDefs[0] = DstReg;
393 for (unsigned I = 1; I != NumDefs; ++I)
394 UnmergeDefs[I] = MRI.createGenericVirtualRegister(DstTy);
395
396 MIRBuilder.buildUnmerge(UnmergeDefs,
397 MIRBuilder.buildMerge(LCMTy, RemergeRegs));
398 return;
399 }
400
401 llvm_unreachable("unhandled case");
402 }
403
getRTLibDesc(unsigned Opcode,unsigned Size)404 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
405 #define RTLIBCASE_INT(LibcallPrefix) \
406 do { \
407 switch (Size) { \
408 case 32: \
409 return RTLIB::LibcallPrefix##32; \
410 case 64: \
411 return RTLIB::LibcallPrefix##64; \
412 case 128: \
413 return RTLIB::LibcallPrefix##128; \
414 default: \
415 llvm_unreachable("unexpected size"); \
416 } \
417 } while (0)
418
419 #define RTLIBCASE(LibcallPrefix) \
420 do { \
421 switch (Size) { \
422 case 32: \
423 return RTLIB::LibcallPrefix##32; \
424 case 64: \
425 return RTLIB::LibcallPrefix##64; \
426 case 80: \
427 return RTLIB::LibcallPrefix##80; \
428 case 128: \
429 return RTLIB::LibcallPrefix##128; \
430 default: \
431 llvm_unreachable("unexpected size"); \
432 } \
433 } while (0)
434
435 switch (Opcode) {
436 case TargetOpcode::G_SDIV:
437 RTLIBCASE_INT(SDIV_I);
438 case TargetOpcode::G_UDIV:
439 RTLIBCASE_INT(UDIV_I);
440 case TargetOpcode::G_SREM:
441 RTLIBCASE_INT(SREM_I);
442 case TargetOpcode::G_UREM:
443 RTLIBCASE_INT(UREM_I);
444 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
445 RTLIBCASE_INT(CTLZ_I);
446 case TargetOpcode::G_FADD:
447 RTLIBCASE(ADD_F);
448 case TargetOpcode::G_FSUB:
449 RTLIBCASE(SUB_F);
450 case TargetOpcode::G_FMUL:
451 RTLIBCASE(MUL_F);
452 case TargetOpcode::G_FDIV:
453 RTLIBCASE(DIV_F);
454 case TargetOpcode::G_FEXP:
455 RTLIBCASE(EXP_F);
456 case TargetOpcode::G_FEXP2:
457 RTLIBCASE(EXP2_F);
458 case TargetOpcode::G_FREM:
459 RTLIBCASE(REM_F);
460 case TargetOpcode::G_FPOW:
461 RTLIBCASE(POW_F);
462 case TargetOpcode::G_FMA:
463 RTLIBCASE(FMA_F);
464 case TargetOpcode::G_FSIN:
465 RTLIBCASE(SIN_F);
466 case TargetOpcode::G_FCOS:
467 RTLIBCASE(COS_F);
468 case TargetOpcode::G_FLOG10:
469 RTLIBCASE(LOG10_F);
470 case TargetOpcode::G_FLOG:
471 RTLIBCASE(LOG_F);
472 case TargetOpcode::G_FLOG2:
473 RTLIBCASE(LOG2_F);
474 case TargetOpcode::G_FCEIL:
475 RTLIBCASE(CEIL_F);
476 case TargetOpcode::G_FFLOOR:
477 RTLIBCASE(FLOOR_F);
478 case TargetOpcode::G_FMINNUM:
479 RTLIBCASE(FMIN_F);
480 case TargetOpcode::G_FMAXNUM:
481 RTLIBCASE(FMAX_F);
482 case TargetOpcode::G_FSQRT:
483 RTLIBCASE(SQRT_F);
484 case TargetOpcode::G_FRINT:
485 RTLIBCASE(RINT_F);
486 case TargetOpcode::G_FNEARBYINT:
487 RTLIBCASE(NEARBYINT_F);
488 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
489 RTLIBCASE(ROUNDEVEN_F);
490 }
491 llvm_unreachable("Unknown libcall function");
492 }
493
494 /// True if an instruction is in tail position in its caller. Intended for
495 /// legalizing libcalls as tail calls when possible.
isLibCallInTailPosition(const TargetInstrInfo & TII,MachineInstr & MI)496 static bool isLibCallInTailPosition(const TargetInstrInfo &TII,
497 MachineInstr &MI) {
498 MachineBasicBlock &MBB = *MI.getParent();
499 const Function &F = MBB.getParent()->getFunction();
500
501 // Conservatively require the attributes of the call to match those of
502 // the return. Ignore NoAlias and NonNull because they don't affect the
503 // call sequence.
504 AttributeList CallerAttrs = F.getAttributes();
505 if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
506 .removeAttribute(Attribute::NoAlias)
507 .removeAttribute(Attribute::NonNull)
508 .hasAttributes())
509 return false;
510
511 // It's not safe to eliminate the sign / zero extension of the return value.
512 if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
513 CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
514 return false;
515
516 // Only tail call if the following instruction is a standard return.
517 auto Next = next_nodbg(MI.getIterator(), MBB.instr_end());
518 if (Next == MBB.instr_end() || TII.isTailCall(*Next) || !Next->isReturn())
519 return false;
520
521 return true;
522 }
523
524 LegalizerHelper::LegalizeResult
createLibcall(MachineIRBuilder & MIRBuilder,const char * Name,const CallLowering::ArgInfo & Result,ArrayRef<CallLowering::ArgInfo> Args,const CallingConv::ID CC)525 llvm::createLibcall(MachineIRBuilder &MIRBuilder, const char *Name,
526 const CallLowering::ArgInfo &Result,
527 ArrayRef<CallLowering::ArgInfo> Args,
528 const CallingConv::ID CC) {
529 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
530
531 CallLowering::CallLoweringInfo Info;
532 Info.CallConv = CC;
533 Info.Callee = MachineOperand::CreateES(Name);
534 Info.OrigRet = Result;
535 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
536 if (!CLI.lowerCall(MIRBuilder, Info))
537 return LegalizerHelper::UnableToLegalize;
538
539 return LegalizerHelper::Legalized;
540 }
541
542 LegalizerHelper::LegalizeResult
createLibcall(MachineIRBuilder & MIRBuilder,RTLIB::Libcall Libcall,const CallLowering::ArgInfo & Result,ArrayRef<CallLowering::ArgInfo> Args)543 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
544 const CallLowering::ArgInfo &Result,
545 ArrayRef<CallLowering::ArgInfo> Args) {
546 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
547 const char *Name = TLI.getLibcallName(Libcall);
548 const CallingConv::ID CC = TLI.getLibcallCallingConv(Libcall);
549 return createLibcall(MIRBuilder, Name, Result, Args, CC);
550 }
551
552 // Useful for libcalls where all operands have the same type.
553 static LegalizerHelper::LegalizeResult
simpleLibcall(MachineInstr & MI,MachineIRBuilder & MIRBuilder,unsigned Size,Type * OpType)554 simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
555 Type *OpType) {
556 auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
557
558 SmallVector<CallLowering::ArgInfo, 3> Args;
559 for (unsigned i = 1; i < MI.getNumOperands(); i++)
560 Args.push_back({MI.getOperand(i).getReg(), OpType});
561 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
562 Args);
563 }
564
565 LegalizerHelper::LegalizeResult
createMemLibcall(MachineIRBuilder & MIRBuilder,MachineRegisterInfo & MRI,MachineInstr & MI)566 llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
567 MachineInstr &MI) {
568 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
569
570 SmallVector<CallLowering::ArgInfo, 3> Args;
571 // Add all the args, except for the last which is an imm denoting 'tail'.
572 for (unsigned i = 0; i < MI.getNumOperands() - 1; ++i) {
573 Register Reg = MI.getOperand(i).getReg();
574
575 // Need derive an IR type for call lowering.
576 LLT OpLLT = MRI.getType(Reg);
577 Type *OpTy = nullptr;
578 if (OpLLT.isPointer())
579 OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
580 else
581 OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
582 Args.push_back({Reg, OpTy});
583 }
584
585 auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
586 auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
587 RTLIB::Libcall RTLibcall;
588 switch (MI.getOpcode()) {
589 case TargetOpcode::G_MEMCPY:
590 RTLibcall = RTLIB::MEMCPY;
591 break;
592 case TargetOpcode::G_MEMMOVE:
593 RTLibcall = RTLIB::MEMMOVE;
594 break;
595 case TargetOpcode::G_MEMSET:
596 RTLibcall = RTLIB::MEMSET;
597 break;
598 default:
599 return LegalizerHelper::UnableToLegalize;
600 }
601 const char *Name = TLI.getLibcallName(RTLibcall);
602
603 CallLowering::CallLoweringInfo Info;
604 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
605 Info.Callee = MachineOperand::CreateES(Name);
606 Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
607 Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() &&
608 isLibCallInTailPosition(MIRBuilder.getTII(), MI);
609
610 std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
611 if (!CLI.lowerCall(MIRBuilder, Info))
612 return LegalizerHelper::UnableToLegalize;
613
614 if (Info.LoweredTailCall) {
615 assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
616 // We must have a return following the call (or debug insts) to get past
617 // isLibCallInTailPosition.
618 do {
619 MachineInstr *Next = MI.getNextNode();
620 assert(Next && (Next->isReturn() || Next->isDebugInstr()) &&
621 "Expected instr following MI to be return or debug inst?");
622 // We lowered a tail call, so the call is now the return from the block.
623 // Delete the old return.
624 Next->eraseFromParent();
625 } while (MI.getNextNode());
626 }
627
628 return LegalizerHelper::Legalized;
629 }
630
getConvRTLibDesc(unsigned Opcode,Type * ToType,Type * FromType)631 static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
632 Type *FromType) {
633 auto ToMVT = MVT::getVT(ToType);
634 auto FromMVT = MVT::getVT(FromType);
635
636 switch (Opcode) {
637 case TargetOpcode::G_FPEXT:
638 return RTLIB::getFPEXT(FromMVT, ToMVT);
639 case TargetOpcode::G_FPTRUNC:
640 return RTLIB::getFPROUND(FromMVT, ToMVT);
641 case TargetOpcode::G_FPTOSI:
642 return RTLIB::getFPTOSINT(FromMVT, ToMVT);
643 case TargetOpcode::G_FPTOUI:
644 return RTLIB::getFPTOUINT(FromMVT, ToMVT);
645 case TargetOpcode::G_SITOFP:
646 return RTLIB::getSINTTOFP(FromMVT, ToMVT);
647 case TargetOpcode::G_UITOFP:
648 return RTLIB::getUINTTOFP(FromMVT, ToMVT);
649 }
650 llvm_unreachable("Unsupported libcall function");
651 }
652
653 static LegalizerHelper::LegalizeResult
conversionLibcall(MachineInstr & MI,MachineIRBuilder & MIRBuilder,Type * ToType,Type * FromType)654 conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType,
655 Type *FromType) {
656 RTLIB::Libcall Libcall = getConvRTLibDesc(MI.getOpcode(), ToType, FromType);
657 return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
658 {{MI.getOperand(1).getReg(), FromType}});
659 }
660
661 LegalizerHelper::LegalizeResult
libcall(MachineInstr & MI)662 LegalizerHelper::libcall(MachineInstr &MI) {
663 LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
664 unsigned Size = LLTy.getSizeInBits();
665 auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
666
667 switch (MI.getOpcode()) {
668 default:
669 return UnableToLegalize;
670 case TargetOpcode::G_SDIV:
671 case TargetOpcode::G_UDIV:
672 case TargetOpcode::G_SREM:
673 case TargetOpcode::G_UREM:
674 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
675 Type *HLTy = IntegerType::get(Ctx, Size);
676 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
677 if (Status != Legalized)
678 return Status;
679 break;
680 }
681 case TargetOpcode::G_FADD:
682 case TargetOpcode::G_FSUB:
683 case TargetOpcode::G_FMUL:
684 case TargetOpcode::G_FDIV:
685 case TargetOpcode::G_FMA:
686 case TargetOpcode::G_FPOW:
687 case TargetOpcode::G_FREM:
688 case TargetOpcode::G_FCOS:
689 case TargetOpcode::G_FSIN:
690 case TargetOpcode::G_FLOG10:
691 case TargetOpcode::G_FLOG:
692 case TargetOpcode::G_FLOG2:
693 case TargetOpcode::G_FEXP:
694 case TargetOpcode::G_FEXP2:
695 case TargetOpcode::G_FCEIL:
696 case TargetOpcode::G_FFLOOR:
697 case TargetOpcode::G_FMINNUM:
698 case TargetOpcode::G_FMAXNUM:
699 case TargetOpcode::G_FSQRT:
700 case TargetOpcode::G_FRINT:
701 case TargetOpcode::G_FNEARBYINT:
702 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
703 Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
704 if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
705 LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
706 return UnableToLegalize;
707 }
708 auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
709 if (Status != Legalized)
710 return Status;
711 break;
712 }
713 case TargetOpcode::G_FPEXT:
714 case TargetOpcode::G_FPTRUNC: {
715 Type *FromTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(1).getReg()));
716 Type *ToTy = getFloatTypeForLLT(Ctx, MRI.getType(MI.getOperand(0).getReg()));
717 if (!FromTy || !ToTy)
718 return UnableToLegalize;
719 LegalizeResult Status = conversionLibcall(MI, MIRBuilder, ToTy, FromTy );
720 if (Status != Legalized)
721 return Status;
722 break;
723 }
724 case TargetOpcode::G_FPTOSI:
725 case TargetOpcode::G_FPTOUI: {
726 // FIXME: Support other types
727 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
728 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
729 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
730 return UnableToLegalize;
731 LegalizeResult Status = conversionLibcall(
732 MI, MIRBuilder,
733 ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx),
734 FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
735 if (Status != Legalized)
736 return Status;
737 break;
738 }
739 case TargetOpcode::G_SITOFP:
740 case TargetOpcode::G_UITOFP: {
741 // FIXME: Support other types
742 unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
743 unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
744 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
745 return UnableToLegalize;
746 LegalizeResult Status = conversionLibcall(
747 MI, MIRBuilder,
748 ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
749 FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx));
750 if (Status != Legalized)
751 return Status;
752 break;
753 }
754 case TargetOpcode::G_MEMCPY:
755 case TargetOpcode::G_MEMMOVE:
756 case TargetOpcode::G_MEMSET: {
757 LegalizeResult Result = createMemLibcall(MIRBuilder, *MIRBuilder.getMRI(), MI);
758 MI.eraseFromParent();
759 return Result;
760 }
761 }
762
763 MI.eraseFromParent();
764 return Legalized;
765 }
766
narrowScalar(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)767 LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
768 unsigned TypeIdx,
769 LLT NarrowTy) {
770 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
771 uint64_t NarrowSize = NarrowTy.getSizeInBits();
772
773 switch (MI.getOpcode()) {
774 default:
775 return UnableToLegalize;
776 case TargetOpcode::G_IMPLICIT_DEF: {
777 Register DstReg = MI.getOperand(0).getReg();
778 LLT DstTy = MRI.getType(DstReg);
779
780 // If SizeOp0 is not an exact multiple of NarrowSize, emit
781 // G_ANYEXT(G_IMPLICIT_DEF). Cast result to vector if needed.
782 // FIXME: Although this would also be legal for the general case, it causes
783 // a lot of regressions in the emitted code (superfluous COPYs, artifact
784 // combines not being hit). This seems to be a problem related to the
785 // artifact combiner.
786 if (SizeOp0 % NarrowSize != 0) {
787 LLT ImplicitTy = NarrowTy;
788 if (DstTy.isVector())
789 ImplicitTy = LLT::vector(DstTy.getNumElements(), ImplicitTy);
790
791 Register ImplicitReg = MIRBuilder.buildUndef(ImplicitTy).getReg(0);
792 MIRBuilder.buildAnyExt(DstReg, ImplicitReg);
793
794 MI.eraseFromParent();
795 return Legalized;
796 }
797
798 int NumParts = SizeOp0 / NarrowSize;
799
800 SmallVector<Register, 2> DstRegs;
801 for (int i = 0; i < NumParts; ++i)
802 DstRegs.push_back(MIRBuilder.buildUndef(NarrowTy).getReg(0));
803
804 if (DstTy.isVector())
805 MIRBuilder.buildBuildVector(DstReg, DstRegs);
806 else
807 MIRBuilder.buildMerge(DstReg, DstRegs);
808 MI.eraseFromParent();
809 return Legalized;
810 }
811 case TargetOpcode::G_CONSTANT: {
812 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
813 const APInt &Val = MI.getOperand(1).getCImm()->getValue();
814 unsigned TotalSize = Ty.getSizeInBits();
815 unsigned NarrowSize = NarrowTy.getSizeInBits();
816 int NumParts = TotalSize / NarrowSize;
817
818 SmallVector<Register, 4> PartRegs;
819 for (int I = 0; I != NumParts; ++I) {
820 unsigned Offset = I * NarrowSize;
821 auto K = MIRBuilder.buildConstant(NarrowTy,
822 Val.lshr(Offset).trunc(NarrowSize));
823 PartRegs.push_back(K.getReg(0));
824 }
825
826 LLT LeftoverTy;
827 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
828 SmallVector<Register, 1> LeftoverRegs;
829 if (LeftoverBits != 0) {
830 LeftoverTy = LLT::scalar(LeftoverBits);
831 auto K = MIRBuilder.buildConstant(
832 LeftoverTy,
833 Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits));
834 LeftoverRegs.push_back(K.getReg(0));
835 }
836
837 insertParts(MI.getOperand(0).getReg(),
838 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
839
840 MI.eraseFromParent();
841 return Legalized;
842 }
843 case TargetOpcode::G_SEXT:
844 case TargetOpcode::G_ZEXT:
845 case TargetOpcode::G_ANYEXT:
846 return narrowScalarExt(MI, TypeIdx, NarrowTy);
847 case TargetOpcode::G_TRUNC: {
848 if (TypeIdx != 1)
849 return UnableToLegalize;
850
851 uint64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
852 if (NarrowTy.getSizeInBits() * 2 != SizeOp1) {
853 LLVM_DEBUG(dbgs() << "Can't narrow trunc to type " << NarrowTy << "\n");
854 return UnableToLegalize;
855 }
856
857 auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
858 MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0));
859 MI.eraseFromParent();
860 return Legalized;
861 }
862
863 case TargetOpcode::G_FREEZE:
864 return reduceOperationWidth(MI, TypeIdx, NarrowTy);
865
866 case TargetOpcode::G_ADD: {
867 // FIXME: add support for when SizeOp0 isn't an exact multiple of
868 // NarrowSize.
869 if (SizeOp0 % NarrowSize != 0)
870 return UnableToLegalize;
871 // Expand in terms of carry-setting/consuming G_ADDE instructions.
872 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
873
874 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
875 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
876 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
877
878 Register CarryIn;
879 for (int i = 0; i < NumParts; ++i) {
880 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
881 Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
882
883 if (i == 0)
884 MIRBuilder.buildUAddo(DstReg, CarryOut, Src1Regs[i], Src2Regs[i]);
885 else {
886 MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
887 Src2Regs[i], CarryIn);
888 }
889
890 DstRegs.push_back(DstReg);
891 CarryIn = CarryOut;
892 }
893 Register DstReg = MI.getOperand(0).getReg();
894 if(MRI.getType(DstReg).isVector())
895 MIRBuilder.buildBuildVector(DstReg, DstRegs);
896 else
897 MIRBuilder.buildMerge(DstReg, DstRegs);
898 MI.eraseFromParent();
899 return Legalized;
900 }
901 case TargetOpcode::G_SUB: {
902 // FIXME: add support for when SizeOp0 isn't an exact multiple of
903 // NarrowSize.
904 if (SizeOp0 % NarrowSize != 0)
905 return UnableToLegalize;
906
907 int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
908
909 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
910 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
911 extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
912
913 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
914 Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
915 MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
916 {Src1Regs[0], Src2Regs[0]});
917 DstRegs.push_back(DstReg);
918 Register BorrowIn = BorrowOut;
919 for (int i = 1; i < NumParts; ++i) {
920 DstReg = MRI.createGenericVirtualRegister(NarrowTy);
921 BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
922
923 MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
924 {Src1Regs[i], Src2Regs[i], BorrowIn});
925
926 DstRegs.push_back(DstReg);
927 BorrowIn = BorrowOut;
928 }
929 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
930 MI.eraseFromParent();
931 return Legalized;
932 }
933 case TargetOpcode::G_MUL:
934 case TargetOpcode::G_UMULH:
935 return narrowScalarMul(MI, NarrowTy);
936 case TargetOpcode::G_EXTRACT:
937 return narrowScalarExtract(MI, TypeIdx, NarrowTy);
938 case TargetOpcode::G_INSERT:
939 return narrowScalarInsert(MI, TypeIdx, NarrowTy);
940 case TargetOpcode::G_LOAD: {
941 auto &MMO = **MI.memoperands_begin();
942 Register DstReg = MI.getOperand(0).getReg();
943 LLT DstTy = MRI.getType(DstReg);
944 if (DstTy.isVector())
945 return UnableToLegalize;
946
947 if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
948 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
949 MIRBuilder.buildLoad(TmpReg, MI.getOperand(1), MMO);
950 MIRBuilder.buildAnyExt(DstReg, TmpReg);
951 MI.eraseFromParent();
952 return Legalized;
953 }
954
955 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
956 }
957 case TargetOpcode::G_ZEXTLOAD:
958 case TargetOpcode::G_SEXTLOAD: {
959 bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
960 Register DstReg = MI.getOperand(0).getReg();
961 Register PtrReg = MI.getOperand(1).getReg();
962
963 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
964 auto &MMO = **MI.memoperands_begin();
965 if (MMO.getSizeInBits() == NarrowSize) {
966 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
967 } else {
968 MIRBuilder.buildLoadInstr(MI.getOpcode(), TmpReg, PtrReg, MMO);
969 }
970
971 if (ZExt)
972 MIRBuilder.buildZExt(DstReg, TmpReg);
973 else
974 MIRBuilder.buildSExt(DstReg, TmpReg);
975
976 MI.eraseFromParent();
977 return Legalized;
978 }
979 case TargetOpcode::G_STORE: {
980 const auto &MMO = **MI.memoperands_begin();
981
982 Register SrcReg = MI.getOperand(0).getReg();
983 LLT SrcTy = MRI.getType(SrcReg);
984 if (SrcTy.isVector())
985 return UnableToLegalize;
986
987 int NumParts = SizeOp0 / NarrowSize;
988 unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
989 unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
990 if (SrcTy.isVector() && LeftoverBits != 0)
991 return UnableToLegalize;
992
993 if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
994 Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
995 auto &MMO = **MI.memoperands_begin();
996 MIRBuilder.buildTrunc(TmpReg, SrcReg);
997 MIRBuilder.buildStore(TmpReg, MI.getOperand(1), MMO);
998 MI.eraseFromParent();
999 return Legalized;
1000 }
1001
1002 return reduceLoadStoreWidth(MI, 0, NarrowTy);
1003 }
1004 case TargetOpcode::G_SELECT:
1005 return narrowScalarSelect(MI, TypeIdx, NarrowTy);
1006 case TargetOpcode::G_AND:
1007 case TargetOpcode::G_OR:
1008 case TargetOpcode::G_XOR: {
1009 // Legalize bitwise operation:
1010 // A = BinOp<Ty> B, C
1011 // into:
1012 // B1, ..., BN = G_UNMERGE_VALUES B
1013 // C1, ..., CN = G_UNMERGE_VALUES C
1014 // A1 = BinOp<Ty/N> B1, C2
1015 // ...
1016 // AN = BinOp<Ty/N> BN, CN
1017 // A = G_MERGE_VALUES A1, ..., AN
1018 return narrowScalarBasic(MI, TypeIdx, NarrowTy);
1019 }
1020 case TargetOpcode::G_SHL:
1021 case TargetOpcode::G_LSHR:
1022 case TargetOpcode::G_ASHR:
1023 return narrowScalarShift(MI, TypeIdx, NarrowTy);
1024 case TargetOpcode::G_CTLZ:
1025 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1026 case TargetOpcode::G_CTTZ:
1027 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1028 case TargetOpcode::G_CTPOP:
1029 if (TypeIdx == 1)
1030 switch (MI.getOpcode()) {
1031 case TargetOpcode::G_CTLZ:
1032 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1033 return narrowScalarCTLZ(MI, TypeIdx, NarrowTy);
1034 case TargetOpcode::G_CTTZ:
1035 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1036 return narrowScalarCTTZ(MI, TypeIdx, NarrowTy);
1037 case TargetOpcode::G_CTPOP:
1038 return narrowScalarCTPOP(MI, TypeIdx, NarrowTy);
1039 default:
1040 return UnableToLegalize;
1041 }
1042
1043 Observer.changingInstr(MI);
1044 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1045 Observer.changedInstr(MI);
1046 return Legalized;
1047 case TargetOpcode::G_INTTOPTR:
1048 if (TypeIdx != 1)
1049 return UnableToLegalize;
1050
1051 Observer.changingInstr(MI);
1052 narrowScalarSrc(MI, NarrowTy, 1);
1053 Observer.changedInstr(MI);
1054 return Legalized;
1055 case TargetOpcode::G_PTRTOINT:
1056 if (TypeIdx != 0)
1057 return UnableToLegalize;
1058
1059 Observer.changingInstr(MI);
1060 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1061 Observer.changedInstr(MI);
1062 return Legalized;
1063 case TargetOpcode::G_PHI: {
1064 unsigned NumParts = SizeOp0 / NarrowSize;
1065 SmallVector<Register, 2> DstRegs(NumParts);
1066 SmallVector<SmallVector<Register, 2>, 2> SrcRegs(MI.getNumOperands() / 2);
1067 Observer.changingInstr(MI);
1068 for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
1069 MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB();
1070 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1071 extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts,
1072 SrcRegs[i / 2]);
1073 }
1074 MachineBasicBlock &MBB = *MI.getParent();
1075 MIRBuilder.setInsertPt(MBB, MI);
1076 for (unsigned i = 0; i < NumParts; ++i) {
1077 DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy);
1078 MachineInstrBuilder MIB =
1079 MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]);
1080 for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
1081 MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
1082 }
1083 MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI());
1084 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
1085 Observer.changedInstr(MI);
1086 MI.eraseFromParent();
1087 return Legalized;
1088 }
1089 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1090 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1091 if (TypeIdx != 2)
1092 return UnableToLegalize;
1093
1094 int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1095 Observer.changingInstr(MI);
1096 narrowScalarSrc(MI, NarrowTy, OpIdx);
1097 Observer.changedInstr(MI);
1098 return Legalized;
1099 }
1100 case TargetOpcode::G_ICMP: {
1101 uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
1102 if (NarrowSize * 2 != SrcSize)
1103 return UnableToLegalize;
1104
1105 Observer.changingInstr(MI);
1106 Register LHSL = MRI.createGenericVirtualRegister(NarrowTy);
1107 Register LHSH = MRI.createGenericVirtualRegister(NarrowTy);
1108 MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2));
1109
1110 Register RHSL = MRI.createGenericVirtualRegister(NarrowTy);
1111 Register RHSH = MRI.createGenericVirtualRegister(NarrowTy);
1112 MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3));
1113
1114 CmpInst::Predicate Pred =
1115 static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
1116 LLT ResTy = MRI.getType(MI.getOperand(0).getReg());
1117
1118 if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) {
1119 MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL);
1120 MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH);
1121 MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH);
1122 MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0);
1123 MIRBuilder.buildICmp(Pred, MI.getOperand(0), Or, Zero);
1124 } else {
1125 MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, ResTy, LHSH, RHSH);
1126 MachineInstrBuilder CmpHEQ =
1127 MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, ResTy, LHSH, RHSH);
1128 MachineInstrBuilder CmpLU = MIRBuilder.buildICmp(
1129 ICmpInst::getUnsignedPredicate(Pred), ResTy, LHSL, RHSL);
1130 MIRBuilder.buildSelect(MI.getOperand(0), CmpHEQ, CmpLU, CmpH);
1131 }
1132 Observer.changedInstr(MI);
1133 MI.eraseFromParent();
1134 return Legalized;
1135 }
1136 case TargetOpcode::G_SEXT_INREG: {
1137 if (TypeIdx != 0)
1138 return UnableToLegalize;
1139
1140 int64_t SizeInBits = MI.getOperand(2).getImm();
1141
1142 // So long as the new type has more bits than the bits we're extending we
1143 // don't need to break it apart.
1144 if (NarrowTy.getScalarSizeInBits() >= SizeInBits) {
1145 Observer.changingInstr(MI);
1146 // We don't lose any non-extension bits by truncating the src and
1147 // sign-extending the dst.
1148 MachineOperand &MO1 = MI.getOperand(1);
1149 auto TruncMIB = MIRBuilder.buildTrunc(NarrowTy, MO1);
1150 MO1.setReg(TruncMIB.getReg(0));
1151
1152 MachineOperand &MO2 = MI.getOperand(0);
1153 Register DstExt = MRI.createGenericVirtualRegister(NarrowTy);
1154 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1155 MIRBuilder.buildSExt(MO2, DstExt);
1156 MO2.setReg(DstExt);
1157 Observer.changedInstr(MI);
1158 return Legalized;
1159 }
1160
1161 // Break it apart. Components below the extension point are unmodified. The
1162 // component containing the extension point becomes a narrower SEXT_INREG.
1163 // Components above it are ashr'd from the component containing the
1164 // extension point.
1165 if (SizeOp0 % NarrowSize != 0)
1166 return UnableToLegalize;
1167 int NumParts = SizeOp0 / NarrowSize;
1168
1169 // List the registers where the destination will be scattered.
1170 SmallVector<Register, 2> DstRegs;
1171 // List the registers where the source will be split.
1172 SmallVector<Register, 2> SrcRegs;
1173
1174 // Create all the temporary registers.
1175 for (int i = 0; i < NumParts; ++i) {
1176 Register SrcReg = MRI.createGenericVirtualRegister(NarrowTy);
1177
1178 SrcRegs.push_back(SrcReg);
1179 }
1180
1181 // Explode the big arguments into smaller chunks.
1182 MIRBuilder.buildUnmerge(SrcRegs, MI.getOperand(1));
1183
1184 Register AshrCstReg =
1185 MIRBuilder.buildConstant(NarrowTy, NarrowTy.getScalarSizeInBits() - 1)
1186 .getReg(0);
1187 Register FullExtensionReg = 0;
1188 Register PartialExtensionReg = 0;
1189
1190 // Do the operation on each small part.
1191 for (int i = 0; i < NumParts; ++i) {
1192 if ((i + 1) * NarrowTy.getScalarSizeInBits() < SizeInBits)
1193 DstRegs.push_back(SrcRegs[i]);
1194 else if (i * NarrowTy.getScalarSizeInBits() > SizeInBits) {
1195 assert(PartialExtensionReg &&
1196 "Expected to visit partial extension before full");
1197 if (FullExtensionReg) {
1198 DstRegs.push_back(FullExtensionReg);
1199 continue;
1200 }
1201 DstRegs.push_back(
1202 MIRBuilder.buildAShr(NarrowTy, PartialExtensionReg, AshrCstReg)
1203 .getReg(0));
1204 FullExtensionReg = DstRegs.back();
1205 } else {
1206 DstRegs.push_back(
1207 MIRBuilder
1208 .buildInstr(
1209 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1210 {SrcRegs[i], SizeInBits % NarrowTy.getScalarSizeInBits()})
1211 .getReg(0));
1212 PartialExtensionReg = DstRegs.back();
1213 }
1214 }
1215
1216 // Gather the destination registers into the final destination.
1217 Register DstReg = MI.getOperand(0).getReg();
1218 MIRBuilder.buildMerge(DstReg, DstRegs);
1219 MI.eraseFromParent();
1220 return Legalized;
1221 }
1222 case TargetOpcode::G_BSWAP:
1223 case TargetOpcode::G_BITREVERSE: {
1224 if (SizeOp0 % NarrowSize != 0)
1225 return UnableToLegalize;
1226
1227 Observer.changingInstr(MI);
1228 SmallVector<Register, 2> SrcRegs, DstRegs;
1229 unsigned NumParts = SizeOp0 / NarrowSize;
1230 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
1231
1232 for (unsigned i = 0; i < NumParts; ++i) {
1233 auto DstPart = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
1234 {SrcRegs[NumParts - 1 - i]});
1235 DstRegs.push_back(DstPart.getReg(0));
1236 }
1237
1238 MIRBuilder.buildMerge(MI.getOperand(0), DstRegs);
1239
1240 Observer.changedInstr(MI);
1241 MI.eraseFromParent();
1242 return Legalized;
1243 }
1244 case TargetOpcode::G_PTR_ADD:
1245 case TargetOpcode::G_PTRMASK: {
1246 if (TypeIdx != 1)
1247 return UnableToLegalize;
1248 Observer.changingInstr(MI);
1249 narrowScalarSrc(MI, NarrowTy, 2);
1250 Observer.changedInstr(MI);
1251 return Legalized;
1252 }
1253 case TargetOpcode::G_FPTOUI: {
1254 if (TypeIdx != 0)
1255 return UnableToLegalize;
1256 Observer.changingInstr(MI);
1257 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
1258 Observer.changedInstr(MI);
1259 return Legalized;
1260 }
1261 case TargetOpcode::G_FPTOSI: {
1262 if (TypeIdx != 0)
1263 return UnableToLegalize;
1264 Observer.changingInstr(MI);
1265 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_SEXT);
1266 Observer.changedInstr(MI);
1267 return Legalized;
1268 }
1269 case TargetOpcode::G_FPEXT:
1270 if (TypeIdx != 0)
1271 return UnableToLegalize;
1272 Observer.changingInstr(MI);
1273 narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_FPEXT);
1274 Observer.changedInstr(MI);
1275 return Legalized;
1276 }
1277 }
1278
coerceToScalar(Register Val)1279 Register LegalizerHelper::coerceToScalar(Register Val) {
1280 LLT Ty = MRI.getType(Val);
1281 if (Ty.isScalar())
1282 return Val;
1283
1284 const DataLayout &DL = MIRBuilder.getDataLayout();
1285 LLT NewTy = LLT::scalar(Ty.getSizeInBits());
1286 if (Ty.isPointer()) {
1287 if (DL.isNonIntegralAddressSpace(Ty.getAddressSpace()))
1288 return Register();
1289 return MIRBuilder.buildPtrToInt(NewTy, Val).getReg(0);
1290 }
1291
1292 Register NewVal = Val;
1293
1294 assert(Ty.isVector());
1295 LLT EltTy = Ty.getElementType();
1296 if (EltTy.isPointer())
1297 NewVal = MIRBuilder.buildPtrToInt(NewTy, NewVal).getReg(0);
1298 return MIRBuilder.buildBitcast(NewTy, NewVal).getReg(0);
1299 }
1300
widenScalarSrc(MachineInstr & MI,LLT WideTy,unsigned OpIdx,unsigned ExtOpcode)1301 void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
1302 unsigned OpIdx, unsigned ExtOpcode) {
1303 MachineOperand &MO = MI.getOperand(OpIdx);
1304 auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO});
1305 MO.setReg(ExtB.getReg(0));
1306 }
1307
narrowScalarSrc(MachineInstr & MI,LLT NarrowTy,unsigned OpIdx)1308 void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
1309 unsigned OpIdx) {
1310 MachineOperand &MO = MI.getOperand(OpIdx);
1311 auto ExtB = MIRBuilder.buildTrunc(NarrowTy, MO);
1312 MO.setReg(ExtB.getReg(0));
1313 }
1314
widenScalarDst(MachineInstr & MI,LLT WideTy,unsigned OpIdx,unsigned TruncOpcode)1315 void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
1316 unsigned OpIdx, unsigned TruncOpcode) {
1317 MachineOperand &MO = MI.getOperand(OpIdx);
1318 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1319 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1320 MIRBuilder.buildInstr(TruncOpcode, {MO}, {DstExt});
1321 MO.setReg(DstExt);
1322 }
1323
narrowScalarDst(MachineInstr & MI,LLT NarrowTy,unsigned OpIdx,unsigned ExtOpcode)1324 void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
1325 unsigned OpIdx, unsigned ExtOpcode) {
1326 MachineOperand &MO = MI.getOperand(OpIdx);
1327 Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
1328 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1329 MIRBuilder.buildInstr(ExtOpcode, {MO}, {DstTrunc});
1330 MO.setReg(DstTrunc);
1331 }
1332
moreElementsVectorDst(MachineInstr & MI,LLT WideTy,unsigned OpIdx)1333 void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
1334 unsigned OpIdx) {
1335 MachineOperand &MO = MI.getOperand(OpIdx);
1336 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1337 MO.setReg(widenWithUnmerge(WideTy, MO.getReg()));
1338 }
1339
moreElementsVectorSrc(MachineInstr & MI,LLT MoreTy,unsigned OpIdx)1340 void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
1341 unsigned OpIdx) {
1342 MachineOperand &MO = MI.getOperand(OpIdx);
1343
1344 LLT OldTy = MRI.getType(MO.getReg());
1345 unsigned OldElts = OldTy.getNumElements();
1346 unsigned NewElts = MoreTy.getNumElements();
1347
1348 unsigned NumParts = NewElts / OldElts;
1349
1350 // Use concat_vectors if the result is a multiple of the number of elements.
1351 if (NumParts * OldElts == NewElts) {
1352 SmallVector<Register, 8> Parts;
1353 Parts.push_back(MO.getReg());
1354
1355 Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
1356 for (unsigned I = 1; I != NumParts; ++I)
1357 Parts.push_back(ImpDef);
1358
1359 auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
1360 MO.setReg(Concat.getReg(0));
1361 return;
1362 }
1363
1364 Register MoreReg = MRI.createGenericVirtualRegister(MoreTy);
1365 Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
1366 MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
1367 MO.setReg(MoreReg);
1368 }
1369
bitcastSrc(MachineInstr & MI,LLT CastTy,unsigned OpIdx)1370 void LegalizerHelper::bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
1371 MachineOperand &Op = MI.getOperand(OpIdx);
1372 Op.setReg(MIRBuilder.buildBitcast(CastTy, Op).getReg(0));
1373 }
1374
bitcastDst(MachineInstr & MI,LLT CastTy,unsigned OpIdx)1375 void LegalizerHelper::bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx) {
1376 MachineOperand &MO = MI.getOperand(OpIdx);
1377 Register CastDst = MRI.createGenericVirtualRegister(CastTy);
1378 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1379 MIRBuilder.buildBitcast(MO, CastDst);
1380 MO.setReg(CastDst);
1381 }
1382
1383 LegalizerHelper::LegalizeResult
widenScalarMergeValues(MachineInstr & MI,unsigned TypeIdx,LLT WideTy)1384 LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
1385 LLT WideTy) {
1386 if (TypeIdx != 1)
1387 return UnableToLegalize;
1388
1389 Register DstReg = MI.getOperand(0).getReg();
1390 LLT DstTy = MRI.getType(DstReg);
1391 if (DstTy.isVector())
1392 return UnableToLegalize;
1393
1394 Register Src1 = MI.getOperand(1).getReg();
1395 LLT SrcTy = MRI.getType(Src1);
1396 const int DstSize = DstTy.getSizeInBits();
1397 const int SrcSize = SrcTy.getSizeInBits();
1398 const int WideSize = WideTy.getSizeInBits();
1399 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1400
1401 unsigned NumOps = MI.getNumOperands();
1402 unsigned NumSrc = MI.getNumOperands() - 1;
1403 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1404
1405 if (WideSize >= DstSize) {
1406 // Directly pack the bits in the target type.
1407 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0);
1408
1409 for (unsigned I = 2; I != NumOps; ++I) {
1410 const unsigned Offset = (I - 1) * PartSize;
1411
1412 Register SrcReg = MI.getOperand(I).getReg();
1413 assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
1414
1415 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
1416
1417 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
1418 MRI.createGenericVirtualRegister(WideTy);
1419
1420 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
1421 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
1422 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
1423 ResultReg = NextResult;
1424 }
1425
1426 if (WideSize > DstSize)
1427 MIRBuilder.buildTrunc(DstReg, ResultReg);
1428 else if (DstTy.isPointer())
1429 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
1430
1431 MI.eraseFromParent();
1432 return Legalized;
1433 }
1434
1435 // Unmerge the original values to the GCD type, and recombine to the next
1436 // multiple greater than the original type.
1437 //
1438 // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6
1439 // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0
1440 // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1
1441 // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2
1442 // %10:_(s6) = G_MERGE_VALUES %4, %5, %6
1443 // %11:_(s6) = G_MERGE_VALUES %7, %8, %9
1444 // %12:_(s12) = G_MERGE_VALUES %10, %11
1445 //
1446 // Padding with undef if necessary:
1447 //
1448 // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6
1449 // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0
1450 // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1
1451 // %7:_(s2) = G_IMPLICIT_DEF
1452 // %8:_(s6) = G_MERGE_VALUES %3, %4, %5
1453 // %9:_(s6) = G_MERGE_VALUES %6, %7, %7
1454 // %10:_(s12) = G_MERGE_VALUES %8, %9
1455
1456 const int GCD = greatestCommonDivisor(SrcSize, WideSize);
1457 LLT GCDTy = LLT::scalar(GCD);
1458
1459 SmallVector<Register, 8> Parts;
1460 SmallVector<Register, 8> NewMergeRegs;
1461 SmallVector<Register, 8> Unmerges;
1462 LLT WideDstTy = LLT::scalar(NumMerge * WideSize);
1463
1464 // Decompose the original operands if they don't evenly divide.
1465 for (int I = 1, E = MI.getNumOperands(); I != E; ++I) {
1466 Register SrcReg = MI.getOperand(I).getReg();
1467 if (GCD == SrcSize) {
1468 Unmerges.push_back(SrcReg);
1469 } else {
1470 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
1471 for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1472 Unmerges.push_back(Unmerge.getReg(J));
1473 }
1474 }
1475
1476 // Pad with undef to the next size that is a multiple of the requested size.
1477 if (static_cast<int>(Unmerges.size()) != NumMerge * WideSize) {
1478 Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0);
1479 for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I)
1480 Unmerges.push_back(UndefReg);
1481 }
1482
1483 const int PartsPerGCD = WideSize / GCD;
1484
1485 // Build merges of each piece.
1486 ArrayRef<Register> Slicer(Unmerges);
1487 for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1488 auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD));
1489 NewMergeRegs.push_back(Merge.getReg(0));
1490 }
1491
1492 // A truncate may be necessary if the requested type doesn't evenly divide the
1493 // original result type.
1494 if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) {
1495 MIRBuilder.buildMerge(DstReg, NewMergeRegs);
1496 } else {
1497 auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs);
1498 MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0));
1499 }
1500
1501 MI.eraseFromParent();
1502 return Legalized;
1503 }
1504
widenWithUnmerge(LLT WideTy,Register OrigReg)1505 Register LegalizerHelper::widenWithUnmerge(LLT WideTy, Register OrigReg) {
1506 Register WideReg = MRI.createGenericVirtualRegister(WideTy);
1507 LLT OrigTy = MRI.getType(OrigReg);
1508 LLT LCMTy = getLCMType(WideTy, OrigTy);
1509
1510 const int NumMergeParts = LCMTy.getSizeInBits() / WideTy.getSizeInBits();
1511 const int NumUnmergeParts = LCMTy.getSizeInBits() / OrigTy.getSizeInBits();
1512
1513 Register UnmergeSrc = WideReg;
1514
1515 // Create a merge to the LCM type, padding with undef
1516 // %0:_(<3 x s32>) = G_FOO => <4 x s32>
1517 // =>
1518 // %1:_(<4 x s32>) = G_FOO
1519 // %2:_(<4 x s32>) = G_IMPLICIT_DEF
1520 // %3:_(<12 x s32>) = G_CONCAT_VECTORS %1, %2, %2
1521 // %0:_(<3 x s32>), %4:_, %5:_, %6:_ = G_UNMERGE_VALUES %3
1522 if (NumMergeParts > 1) {
1523 Register Undef = MIRBuilder.buildUndef(WideTy).getReg(0);
1524 SmallVector<Register, 8> MergeParts(NumMergeParts, Undef);
1525 MergeParts[0] = WideReg;
1526 UnmergeSrc = MIRBuilder.buildMerge(LCMTy, MergeParts).getReg(0);
1527 }
1528
1529 // Unmerge to the original register and pad with dead defs.
1530 SmallVector<Register, 8> UnmergeResults(NumUnmergeParts);
1531 UnmergeResults[0] = OrigReg;
1532 for (int I = 1; I != NumUnmergeParts; ++I)
1533 UnmergeResults[I] = MRI.createGenericVirtualRegister(OrigTy);
1534
1535 MIRBuilder.buildUnmerge(UnmergeResults, UnmergeSrc);
1536 return WideReg;
1537 }
1538
1539 LegalizerHelper::LegalizeResult
widenScalarUnmergeValues(MachineInstr & MI,unsigned TypeIdx,LLT WideTy)1540 LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
1541 LLT WideTy) {
1542 if (TypeIdx != 0)
1543 return UnableToLegalize;
1544
1545 int NumDst = MI.getNumOperands() - 1;
1546 Register SrcReg = MI.getOperand(NumDst).getReg();
1547 LLT SrcTy = MRI.getType(SrcReg);
1548 if (SrcTy.isVector())
1549 return UnableToLegalize;
1550
1551 Register Dst0Reg = MI.getOperand(0).getReg();
1552 LLT DstTy = MRI.getType(Dst0Reg);
1553 if (!DstTy.isScalar())
1554 return UnableToLegalize;
1555
1556 if (WideTy.getSizeInBits() >= SrcTy.getSizeInBits()) {
1557 if (SrcTy.isPointer()) {
1558 const DataLayout &DL = MIRBuilder.getDataLayout();
1559 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) {
1560 LLVM_DEBUG(
1561 dbgs() << "Not casting non-integral address space integer\n");
1562 return UnableToLegalize;
1563 }
1564
1565 SrcTy = LLT::scalar(SrcTy.getSizeInBits());
1566 SrcReg = MIRBuilder.buildPtrToInt(SrcTy, SrcReg).getReg(0);
1567 }
1568
1569 // Widen SrcTy to WideTy. This does not affect the result, but since the
1570 // user requested this size, it is probably better handled than SrcTy and
1571 // should reduce the total number of legalization artifacts
1572 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1573 SrcTy = WideTy;
1574 SrcReg = MIRBuilder.buildAnyExt(WideTy, SrcReg).getReg(0);
1575 }
1576
1577 // Theres no unmerge type to target. Directly extract the bits from the
1578 // source type
1579 unsigned DstSize = DstTy.getSizeInBits();
1580
1581 MIRBuilder.buildTrunc(Dst0Reg, SrcReg);
1582 for (int I = 1; I != NumDst; ++I) {
1583 auto ShiftAmt = MIRBuilder.buildConstant(SrcTy, DstSize * I);
1584 auto Shr = MIRBuilder.buildLShr(SrcTy, SrcReg, ShiftAmt);
1585 MIRBuilder.buildTrunc(MI.getOperand(I), Shr);
1586 }
1587
1588 MI.eraseFromParent();
1589 return Legalized;
1590 }
1591
1592 // Extend the source to a wider type.
1593 LLT LCMTy = getLCMType(SrcTy, WideTy);
1594
1595 Register WideSrc = SrcReg;
1596 if (LCMTy.getSizeInBits() != SrcTy.getSizeInBits()) {
1597 // TODO: If this is an integral address space, cast to integer and anyext.
1598 if (SrcTy.isPointer()) {
1599 LLVM_DEBUG(dbgs() << "Widening pointer source types not implemented\n");
1600 return UnableToLegalize;
1601 }
1602
1603 WideSrc = MIRBuilder.buildAnyExt(LCMTy, WideSrc).getReg(0);
1604 }
1605
1606 auto Unmerge = MIRBuilder.buildUnmerge(WideTy, WideSrc);
1607
1608 // Create a sequence of unmerges and merges to the original results. Since we
1609 // may have widened the source, we will need to pad the results with dead defs
1610 // to cover the source register.
1611 // e.g. widen s48 to s64:
1612 // %1:_(s48), %2:_(s48) = G_UNMERGE_VALUES %0:_(s96)
1613 //
1614 // =>
1615 // %4:_(s192) = G_ANYEXT %0:_(s96)
1616 // %5:_(s64), %6, %7 = G_UNMERGE_VALUES %4 ; Requested unmerge
1617 // ; unpack to GCD type, with extra dead defs
1618 // %8:_(s16), %9, %10, %11 = G_UNMERGE_VALUES %5:_(s64)
1619 // %12:_(s16), %13, dead %14, dead %15 = G_UNMERGE_VALUES %6:_(s64)
1620 // dead %16:_(s16), dead %17, dead %18, dead %18 = G_UNMERGE_VALUES %7:_(s64)
1621 // %1:_(s48) = G_MERGE_VALUES %8:_(s16), %9, %10 ; Remerge to destination
1622 // %2:_(s48) = G_MERGE_VALUES %11:_(s16), %12, %13 ; Remerge to destination
1623 const LLT GCDTy = getGCDType(WideTy, DstTy);
1624 const int NumUnmerge = Unmerge->getNumOperands() - 1;
1625 const int PartsPerRemerge = DstTy.getSizeInBits() / GCDTy.getSizeInBits();
1626
1627 // Directly unmerge to the destination without going through a GCD type
1628 // if possible
1629 if (PartsPerRemerge == 1) {
1630 const int PartsPerUnmerge = WideTy.getSizeInBits() / DstTy.getSizeInBits();
1631
1632 for (int I = 0; I != NumUnmerge; ++I) {
1633 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
1634
1635 for (int J = 0; J != PartsPerUnmerge; ++J) {
1636 int Idx = I * PartsPerUnmerge + J;
1637 if (Idx < NumDst)
1638 MIB.addDef(MI.getOperand(Idx).getReg());
1639 else {
1640 // Create dead def for excess components.
1641 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
1642 }
1643 }
1644
1645 MIB.addUse(Unmerge.getReg(I));
1646 }
1647 } else {
1648 SmallVector<Register, 16> Parts;
1649 for (int J = 0; J != NumUnmerge; ++J)
1650 extractGCDType(Parts, GCDTy, Unmerge.getReg(J));
1651
1652 SmallVector<Register, 8> RemergeParts;
1653 for (int I = 0; I != NumDst; ++I) {
1654 for (int J = 0; J < PartsPerRemerge; ++J) {
1655 const int Idx = I * PartsPerRemerge + J;
1656 RemergeParts.emplace_back(Parts[Idx]);
1657 }
1658
1659 MIRBuilder.buildMerge(MI.getOperand(I).getReg(), RemergeParts);
1660 RemergeParts.clear();
1661 }
1662 }
1663
1664 MI.eraseFromParent();
1665 return Legalized;
1666 }
1667
1668 LegalizerHelper::LegalizeResult
widenScalarExtract(MachineInstr & MI,unsigned TypeIdx,LLT WideTy)1669 LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
1670 LLT WideTy) {
1671 Register DstReg = MI.getOperand(0).getReg();
1672 Register SrcReg = MI.getOperand(1).getReg();
1673 LLT SrcTy = MRI.getType(SrcReg);
1674
1675 LLT DstTy = MRI.getType(DstReg);
1676 unsigned Offset = MI.getOperand(2).getImm();
1677
1678 if (TypeIdx == 0) {
1679 if (SrcTy.isVector() || DstTy.isVector())
1680 return UnableToLegalize;
1681
1682 SrcOp Src(SrcReg);
1683 if (SrcTy.isPointer()) {
1684 // Extracts from pointers can be handled only if they are really just
1685 // simple integers.
1686 const DataLayout &DL = MIRBuilder.getDataLayout();
1687 if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace()))
1688 return UnableToLegalize;
1689
1690 LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
1691 Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
1692 SrcTy = SrcAsIntTy;
1693 }
1694
1695 if (DstTy.isPointer())
1696 return UnableToLegalize;
1697
1698 if (Offset == 0) {
1699 // Avoid a shift in the degenerate case.
1700 MIRBuilder.buildTrunc(DstReg,
1701 MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
1702 MI.eraseFromParent();
1703 return Legalized;
1704 }
1705
1706 // Do a shift in the source type.
1707 LLT ShiftTy = SrcTy;
1708 if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
1709 Src = MIRBuilder.buildAnyExt(WideTy, Src);
1710 ShiftTy = WideTy;
1711 }
1712
1713 auto LShr = MIRBuilder.buildLShr(
1714 ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
1715 MIRBuilder.buildTrunc(DstReg, LShr);
1716 MI.eraseFromParent();
1717 return Legalized;
1718 }
1719
1720 if (SrcTy.isScalar()) {
1721 Observer.changingInstr(MI);
1722 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1723 Observer.changedInstr(MI);
1724 return Legalized;
1725 }
1726
1727 if (!SrcTy.isVector())
1728 return UnableToLegalize;
1729
1730 if (DstTy != SrcTy.getElementType())
1731 return UnableToLegalize;
1732
1733 if (Offset % SrcTy.getScalarSizeInBits() != 0)
1734 return UnableToLegalize;
1735
1736 Observer.changingInstr(MI);
1737 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1738
1739 MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
1740 Offset);
1741 widenScalarDst(MI, WideTy.getScalarType(), 0);
1742 Observer.changedInstr(MI);
1743 return Legalized;
1744 }
1745
1746 LegalizerHelper::LegalizeResult
widenScalarInsert(MachineInstr & MI,unsigned TypeIdx,LLT WideTy)1747 LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
1748 LLT WideTy) {
1749 if (TypeIdx != 0 || WideTy.isVector())
1750 return UnableToLegalize;
1751 Observer.changingInstr(MI);
1752 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1753 widenScalarDst(MI, WideTy);
1754 Observer.changedInstr(MI);
1755 return Legalized;
1756 }
1757
1758 LegalizerHelper::LegalizeResult
widenScalarAddSubShlSat(MachineInstr & MI,unsigned TypeIdx,LLT WideTy)1759 LegalizerHelper::widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx,
1760 LLT WideTy) {
1761 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SADDSAT ||
1762 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
1763 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
1764 bool IsShift = MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
1765 MI.getOpcode() == TargetOpcode::G_USHLSAT;
1766 // We can convert this to:
1767 // 1. Any extend iN to iM
1768 // 2. SHL by M-N
1769 // 3. [US][ADD|SUB|SHL]SAT
1770 // 4. L/ASHR by M-N
1771 //
1772 // It may be more efficient to lower this to a min and a max operation in
1773 // the higher precision arithmetic if the promoted operation isn't legal,
1774 // but this decision is up to the target's lowering request.
1775 Register DstReg = MI.getOperand(0).getReg();
1776
1777 unsigned NewBits = WideTy.getScalarSizeInBits();
1778 unsigned SHLAmount = NewBits - MRI.getType(DstReg).getScalarSizeInBits();
1779
1780 // Shifts must zero-extend the RHS to preserve the unsigned quantity, and
1781 // must not left shift the RHS to preserve the shift amount.
1782 auto LHS = MIRBuilder.buildAnyExt(WideTy, MI.getOperand(1));
1783 auto RHS = IsShift ? MIRBuilder.buildZExt(WideTy, MI.getOperand(2))
1784 : MIRBuilder.buildAnyExt(WideTy, MI.getOperand(2));
1785 auto ShiftK = MIRBuilder.buildConstant(WideTy, SHLAmount);
1786 auto ShiftL = MIRBuilder.buildShl(WideTy, LHS, ShiftK);
1787 auto ShiftR = IsShift ? RHS : MIRBuilder.buildShl(WideTy, RHS, ShiftK);
1788
1789 auto WideInst = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy},
1790 {ShiftL, ShiftR}, MI.getFlags());
1791
1792 // Use a shift that will preserve the number of sign bits when the trunc is
1793 // folded away.
1794 auto Result = IsSigned ? MIRBuilder.buildAShr(WideTy, WideInst, ShiftK)
1795 : MIRBuilder.buildLShr(WideTy, WideInst, ShiftK);
1796
1797 MIRBuilder.buildTrunc(DstReg, Result);
1798 MI.eraseFromParent();
1799 return Legalized;
1800 }
1801
1802 LegalizerHelper::LegalizeResult
widenScalar(MachineInstr & MI,unsigned TypeIdx,LLT WideTy)1803 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
1804 switch (MI.getOpcode()) {
1805 default:
1806 return UnableToLegalize;
1807 case TargetOpcode::G_EXTRACT:
1808 return widenScalarExtract(MI, TypeIdx, WideTy);
1809 case TargetOpcode::G_INSERT:
1810 return widenScalarInsert(MI, TypeIdx, WideTy);
1811 case TargetOpcode::G_MERGE_VALUES:
1812 return widenScalarMergeValues(MI, TypeIdx, WideTy);
1813 case TargetOpcode::G_UNMERGE_VALUES:
1814 return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
1815 case TargetOpcode::G_UADDO:
1816 case TargetOpcode::G_USUBO: {
1817 if (TypeIdx == 1)
1818 return UnableToLegalize; // TODO
1819 auto LHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(2));
1820 auto RHSZext = MIRBuilder.buildZExt(WideTy, MI.getOperand(3));
1821 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
1822 ? TargetOpcode::G_ADD
1823 : TargetOpcode::G_SUB;
1824 // Do the arithmetic in the larger type.
1825 auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
1826 LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
1827 APInt Mask =
1828 APInt::getLowBitsSet(WideTy.getSizeInBits(), OrigTy.getSizeInBits());
1829 auto AndOp = MIRBuilder.buildAnd(
1830 WideTy, NewOp, MIRBuilder.buildConstant(WideTy, Mask));
1831 // There is no overflow if the AndOp is the same as NewOp.
1832 MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1), NewOp, AndOp);
1833 // Now trunc the NewOp to the original result.
1834 MIRBuilder.buildTrunc(MI.getOperand(0), NewOp);
1835 MI.eraseFromParent();
1836 return Legalized;
1837 }
1838 case TargetOpcode::G_SADDSAT:
1839 case TargetOpcode::G_SSUBSAT:
1840 case TargetOpcode::G_SSHLSAT:
1841 case TargetOpcode::G_UADDSAT:
1842 case TargetOpcode::G_USUBSAT:
1843 case TargetOpcode::G_USHLSAT:
1844 return widenScalarAddSubShlSat(MI, TypeIdx, WideTy);
1845 case TargetOpcode::G_CTTZ:
1846 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1847 case TargetOpcode::G_CTLZ:
1848 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1849 case TargetOpcode::G_CTPOP: {
1850 if (TypeIdx == 0) {
1851 Observer.changingInstr(MI);
1852 widenScalarDst(MI, WideTy, 0);
1853 Observer.changedInstr(MI);
1854 return Legalized;
1855 }
1856
1857 Register SrcReg = MI.getOperand(1).getReg();
1858
1859 // First ZEXT the input.
1860 auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1861 LLT CurTy = MRI.getType(SrcReg);
1862 if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1863 // The count is the same in the larger type except if the original
1864 // value was zero. This can be handled by setting the bit just off
1865 // the top of the original type.
1866 auto TopBit =
1867 APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
1868 MIBSrc = MIRBuilder.buildOr(
1869 WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
1870 }
1871
1872 // Perform the operation at the larger size.
1873 auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
1874 // This is already the correct result for CTPOP and CTTZs
1875 if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1876 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1877 // The correct result is NewOp - (Difference in widety and current ty).
1878 unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
1879 MIBNewOp = MIRBuilder.buildSub(
1880 WideTy, MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff));
1881 }
1882
1883 MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1884 MI.eraseFromParent();
1885 return Legalized;
1886 }
1887 case TargetOpcode::G_BSWAP: {
1888 Observer.changingInstr(MI);
1889 Register DstReg = MI.getOperand(0).getReg();
1890
1891 Register ShrReg = MRI.createGenericVirtualRegister(WideTy);
1892 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1893 Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
1894 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1895
1896 MI.getOperand(0).setReg(DstExt);
1897
1898 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1899
1900 LLT Ty = MRI.getType(DstReg);
1901 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1902 MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1903 MIRBuilder.buildLShr(ShrReg, DstExt, ShiftAmtReg);
1904
1905 MIRBuilder.buildTrunc(DstReg, ShrReg);
1906 Observer.changedInstr(MI);
1907 return Legalized;
1908 }
1909 case TargetOpcode::G_BITREVERSE: {
1910 Observer.changingInstr(MI);
1911
1912 Register DstReg = MI.getOperand(0).getReg();
1913 LLT Ty = MRI.getType(DstReg);
1914 unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1915
1916 Register DstExt = MRI.createGenericVirtualRegister(WideTy);
1917 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1918 MI.getOperand(0).setReg(DstExt);
1919 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1920
1921 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits);
1922 auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt);
1923 MIRBuilder.buildTrunc(DstReg, Shift);
1924 Observer.changedInstr(MI);
1925 return Legalized;
1926 }
1927 case TargetOpcode::G_FREEZE:
1928 Observer.changingInstr(MI);
1929 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1930 widenScalarDst(MI, WideTy);
1931 Observer.changedInstr(MI);
1932 return Legalized;
1933
1934 case TargetOpcode::G_ADD:
1935 case TargetOpcode::G_AND:
1936 case TargetOpcode::G_MUL:
1937 case TargetOpcode::G_OR:
1938 case TargetOpcode::G_XOR:
1939 case TargetOpcode::G_SUB:
1940 // Perform operation at larger width (any extension is fines here, high bits
1941 // don't affect the result) and then truncate the result back to the
1942 // original type.
1943 Observer.changingInstr(MI);
1944 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1945 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1946 widenScalarDst(MI, WideTy);
1947 Observer.changedInstr(MI);
1948 return Legalized;
1949
1950 case TargetOpcode::G_SHL:
1951 Observer.changingInstr(MI);
1952
1953 if (TypeIdx == 0) {
1954 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1955 widenScalarDst(MI, WideTy);
1956 } else {
1957 assert(TypeIdx == 1);
1958 // The "number of bits to shift" operand must preserve its value as an
1959 // unsigned integer:
1960 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1961 }
1962
1963 Observer.changedInstr(MI);
1964 return Legalized;
1965
1966 case TargetOpcode::G_SDIV:
1967 case TargetOpcode::G_SREM:
1968 case TargetOpcode::G_SMIN:
1969 case TargetOpcode::G_SMAX:
1970 Observer.changingInstr(MI);
1971 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1972 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1973 widenScalarDst(MI, WideTy);
1974 Observer.changedInstr(MI);
1975 return Legalized;
1976
1977 case TargetOpcode::G_ASHR:
1978 case TargetOpcode::G_LSHR:
1979 Observer.changingInstr(MI);
1980
1981 if (TypeIdx == 0) {
1982 unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1983 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1984
1985 widenScalarSrc(MI, WideTy, 1, CvtOp);
1986 widenScalarDst(MI, WideTy);
1987 } else {
1988 assert(TypeIdx == 1);
1989 // The "number of bits to shift" operand must preserve its value as an
1990 // unsigned integer:
1991 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1992 }
1993
1994 Observer.changedInstr(MI);
1995 return Legalized;
1996 case TargetOpcode::G_UDIV:
1997 case TargetOpcode::G_UREM:
1998 case TargetOpcode::G_UMIN:
1999 case TargetOpcode::G_UMAX:
2000 Observer.changingInstr(MI);
2001 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2002 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2003 widenScalarDst(MI, WideTy);
2004 Observer.changedInstr(MI);
2005 return Legalized;
2006
2007 case TargetOpcode::G_SELECT:
2008 Observer.changingInstr(MI);
2009 if (TypeIdx == 0) {
2010 // Perform operation at larger width (any extension is fine here, high
2011 // bits don't affect the result) and then truncate the result back to the
2012 // original type.
2013 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2014 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
2015 widenScalarDst(MI, WideTy);
2016 } else {
2017 bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
2018 // Explicit extension is required here since high bits affect the result.
2019 widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
2020 }
2021 Observer.changedInstr(MI);
2022 return Legalized;
2023
2024 case TargetOpcode::G_FPTOSI:
2025 case TargetOpcode::G_FPTOUI:
2026 Observer.changingInstr(MI);
2027
2028 if (TypeIdx == 0)
2029 widenScalarDst(MI, WideTy);
2030 else
2031 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
2032
2033 Observer.changedInstr(MI);
2034 return Legalized;
2035 case TargetOpcode::G_SITOFP:
2036 Observer.changingInstr(MI);
2037
2038 if (TypeIdx == 0)
2039 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2040 else
2041 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
2042
2043 Observer.changedInstr(MI);
2044 return Legalized;
2045 case TargetOpcode::G_UITOFP:
2046 Observer.changingInstr(MI);
2047
2048 if (TypeIdx == 0)
2049 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2050 else
2051 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2052
2053 Observer.changedInstr(MI);
2054 return Legalized;
2055 case TargetOpcode::G_LOAD:
2056 case TargetOpcode::G_SEXTLOAD:
2057 case TargetOpcode::G_ZEXTLOAD:
2058 Observer.changingInstr(MI);
2059 widenScalarDst(MI, WideTy);
2060 Observer.changedInstr(MI);
2061 return Legalized;
2062
2063 case TargetOpcode::G_STORE: {
2064 if (TypeIdx != 0)
2065 return UnableToLegalize;
2066
2067 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
2068 if (!Ty.isScalar())
2069 return UnableToLegalize;
2070
2071 Observer.changingInstr(MI);
2072
2073 unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
2074 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
2075 widenScalarSrc(MI, WideTy, 0, ExtType);
2076
2077 Observer.changedInstr(MI);
2078 return Legalized;
2079 }
2080 case TargetOpcode::G_CONSTANT: {
2081 MachineOperand &SrcMO = MI.getOperand(1);
2082 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
2083 unsigned ExtOpc = LI.getExtOpcodeForWideningConstant(
2084 MRI.getType(MI.getOperand(0).getReg()));
2085 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
2086 ExtOpc == TargetOpcode::G_ANYEXT) &&
2087 "Illegal Extend");
2088 const APInt &SrcVal = SrcMO.getCImm()->getValue();
2089 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
2090 ? SrcVal.sext(WideTy.getSizeInBits())
2091 : SrcVal.zext(WideTy.getSizeInBits());
2092 Observer.changingInstr(MI);
2093 SrcMO.setCImm(ConstantInt::get(Ctx, Val));
2094
2095 widenScalarDst(MI, WideTy);
2096 Observer.changedInstr(MI);
2097 return Legalized;
2098 }
2099 case TargetOpcode::G_FCONSTANT: {
2100 MachineOperand &SrcMO = MI.getOperand(1);
2101 LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
2102 APFloat Val = SrcMO.getFPImm()->getValueAPF();
2103 bool LosesInfo;
2104 switch (WideTy.getSizeInBits()) {
2105 case 32:
2106 Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
2107 &LosesInfo);
2108 break;
2109 case 64:
2110 Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
2111 &LosesInfo);
2112 break;
2113 default:
2114 return UnableToLegalize;
2115 }
2116
2117 assert(!LosesInfo && "extend should always be lossless");
2118
2119 Observer.changingInstr(MI);
2120 SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
2121
2122 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2123 Observer.changedInstr(MI);
2124 return Legalized;
2125 }
2126 case TargetOpcode::G_IMPLICIT_DEF: {
2127 Observer.changingInstr(MI);
2128 widenScalarDst(MI, WideTy);
2129 Observer.changedInstr(MI);
2130 return Legalized;
2131 }
2132 case TargetOpcode::G_BRCOND:
2133 Observer.changingInstr(MI);
2134 widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
2135 Observer.changedInstr(MI);
2136 return Legalized;
2137
2138 case TargetOpcode::G_FCMP:
2139 Observer.changingInstr(MI);
2140 if (TypeIdx == 0)
2141 widenScalarDst(MI, WideTy);
2142 else {
2143 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
2144 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
2145 }
2146 Observer.changedInstr(MI);
2147 return Legalized;
2148
2149 case TargetOpcode::G_ICMP:
2150 Observer.changingInstr(MI);
2151 if (TypeIdx == 0)
2152 widenScalarDst(MI, WideTy);
2153 else {
2154 unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
2155 MI.getOperand(1).getPredicate()))
2156 ? TargetOpcode::G_SEXT
2157 : TargetOpcode::G_ZEXT;
2158 widenScalarSrc(MI, WideTy, 2, ExtOpcode);
2159 widenScalarSrc(MI, WideTy, 3, ExtOpcode);
2160 }
2161 Observer.changedInstr(MI);
2162 return Legalized;
2163
2164 case TargetOpcode::G_PTR_ADD:
2165 assert(TypeIdx == 1 && "unable to legalize pointer of G_PTR_ADD");
2166 Observer.changingInstr(MI);
2167 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2168 Observer.changedInstr(MI);
2169 return Legalized;
2170
2171 case TargetOpcode::G_PHI: {
2172 assert(TypeIdx == 0 && "Expecting only Idx 0");
2173
2174 Observer.changingInstr(MI);
2175 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
2176 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2177 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2178 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
2179 }
2180
2181 MachineBasicBlock &MBB = *MI.getParent();
2182 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
2183 widenScalarDst(MI, WideTy);
2184 Observer.changedInstr(MI);
2185 return Legalized;
2186 }
2187 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2188 if (TypeIdx == 0) {
2189 Register VecReg = MI.getOperand(1).getReg();
2190 LLT VecTy = MRI.getType(VecReg);
2191 Observer.changingInstr(MI);
2192
2193 widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
2194 WideTy.getSizeInBits()),
2195 1, TargetOpcode::G_SEXT);
2196
2197 widenScalarDst(MI, WideTy, 0);
2198 Observer.changedInstr(MI);
2199 return Legalized;
2200 }
2201
2202 if (TypeIdx != 2)
2203 return UnableToLegalize;
2204 Observer.changingInstr(MI);
2205 // TODO: Probably should be zext
2206 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
2207 Observer.changedInstr(MI);
2208 return Legalized;
2209 }
2210 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2211 if (TypeIdx == 1) {
2212 Observer.changingInstr(MI);
2213
2214 Register VecReg = MI.getOperand(1).getReg();
2215 LLT VecTy = MRI.getType(VecReg);
2216 LLT WideVecTy = LLT::vector(VecTy.getNumElements(), WideTy);
2217
2218 widenScalarSrc(MI, WideVecTy, 1, TargetOpcode::G_ANYEXT);
2219 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
2220 widenScalarDst(MI, WideVecTy, 0);
2221 Observer.changedInstr(MI);
2222 return Legalized;
2223 }
2224
2225 if (TypeIdx == 2) {
2226 Observer.changingInstr(MI);
2227 // TODO: Probably should be zext
2228 widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_SEXT);
2229 Observer.changedInstr(MI);
2230 return Legalized;
2231 }
2232
2233 return UnableToLegalize;
2234 }
2235 case TargetOpcode::G_FADD:
2236 case TargetOpcode::G_FMUL:
2237 case TargetOpcode::G_FSUB:
2238 case TargetOpcode::G_FMA:
2239 case TargetOpcode::G_FMAD:
2240 case TargetOpcode::G_FNEG:
2241 case TargetOpcode::G_FABS:
2242 case TargetOpcode::G_FCANONICALIZE:
2243 case TargetOpcode::G_FMINNUM:
2244 case TargetOpcode::G_FMAXNUM:
2245 case TargetOpcode::G_FMINNUM_IEEE:
2246 case TargetOpcode::G_FMAXNUM_IEEE:
2247 case TargetOpcode::G_FMINIMUM:
2248 case TargetOpcode::G_FMAXIMUM:
2249 case TargetOpcode::G_FDIV:
2250 case TargetOpcode::G_FREM:
2251 case TargetOpcode::G_FCEIL:
2252 case TargetOpcode::G_FFLOOR:
2253 case TargetOpcode::G_FCOS:
2254 case TargetOpcode::G_FSIN:
2255 case TargetOpcode::G_FLOG10:
2256 case TargetOpcode::G_FLOG:
2257 case TargetOpcode::G_FLOG2:
2258 case TargetOpcode::G_FRINT:
2259 case TargetOpcode::G_FNEARBYINT:
2260 case TargetOpcode::G_FSQRT:
2261 case TargetOpcode::G_FEXP:
2262 case TargetOpcode::G_FEXP2:
2263 case TargetOpcode::G_FPOW:
2264 case TargetOpcode::G_INTRINSIC_TRUNC:
2265 case TargetOpcode::G_INTRINSIC_ROUND:
2266 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2267 assert(TypeIdx == 0);
2268 Observer.changingInstr(MI);
2269
2270 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
2271 widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
2272
2273 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2274 Observer.changedInstr(MI);
2275 return Legalized;
2276 case TargetOpcode::G_FPOWI: {
2277 if (TypeIdx != 0)
2278 return UnableToLegalize;
2279 Observer.changingInstr(MI);
2280 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
2281 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
2282 Observer.changedInstr(MI);
2283 return Legalized;
2284 }
2285 case TargetOpcode::G_INTTOPTR:
2286 if (TypeIdx != 1)
2287 return UnableToLegalize;
2288
2289 Observer.changingInstr(MI);
2290 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
2291 Observer.changedInstr(MI);
2292 return Legalized;
2293 case TargetOpcode::G_PTRTOINT:
2294 if (TypeIdx != 0)
2295 return UnableToLegalize;
2296
2297 Observer.changingInstr(MI);
2298 widenScalarDst(MI, WideTy, 0);
2299 Observer.changedInstr(MI);
2300 return Legalized;
2301 case TargetOpcode::G_BUILD_VECTOR: {
2302 Observer.changingInstr(MI);
2303
2304 const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType();
2305 for (int I = 1, E = MI.getNumOperands(); I != E; ++I)
2306 widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT);
2307
2308 // Avoid changing the result vector type if the source element type was
2309 // requested.
2310 if (TypeIdx == 1) {
2311 MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BUILD_VECTOR_TRUNC));
2312 } else {
2313 widenScalarDst(MI, WideTy, 0);
2314 }
2315
2316 Observer.changedInstr(MI);
2317 return Legalized;
2318 }
2319 case TargetOpcode::G_SEXT_INREG:
2320 if (TypeIdx != 0)
2321 return UnableToLegalize;
2322
2323 Observer.changingInstr(MI);
2324 widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
2325 widenScalarDst(MI, WideTy, 0, TargetOpcode::G_TRUNC);
2326 Observer.changedInstr(MI);
2327 return Legalized;
2328 case TargetOpcode::G_PTRMASK: {
2329 if (TypeIdx != 1)
2330 return UnableToLegalize;
2331 Observer.changingInstr(MI);
2332 widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
2333 Observer.changedInstr(MI);
2334 return Legalized;
2335 }
2336 }
2337 }
2338
getUnmergePieces(SmallVectorImpl<Register> & Pieces,MachineIRBuilder & B,Register Src,LLT Ty)2339 static void getUnmergePieces(SmallVectorImpl<Register> &Pieces,
2340 MachineIRBuilder &B, Register Src, LLT Ty) {
2341 auto Unmerge = B.buildUnmerge(Ty, Src);
2342 for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
2343 Pieces.push_back(Unmerge.getReg(I));
2344 }
2345
2346 LegalizerHelper::LegalizeResult
lowerBitcast(MachineInstr & MI)2347 LegalizerHelper::lowerBitcast(MachineInstr &MI) {
2348 Register Dst = MI.getOperand(0).getReg();
2349 Register Src = MI.getOperand(1).getReg();
2350 LLT DstTy = MRI.getType(Dst);
2351 LLT SrcTy = MRI.getType(Src);
2352
2353 if (SrcTy.isVector()) {
2354 LLT SrcEltTy = SrcTy.getElementType();
2355 SmallVector<Register, 8> SrcRegs;
2356
2357 if (DstTy.isVector()) {
2358 int NumDstElt = DstTy.getNumElements();
2359 int NumSrcElt = SrcTy.getNumElements();
2360
2361 LLT DstEltTy = DstTy.getElementType();
2362 LLT DstCastTy = DstEltTy; // Intermediate bitcast result type
2363 LLT SrcPartTy = SrcEltTy; // Original unmerge result type.
2364
2365 // If there's an element size mismatch, insert intermediate casts to match
2366 // the result element type.
2367 if (NumSrcElt < NumDstElt) { // Source element type is larger.
2368 // %1:_(<4 x s8>) = G_BITCAST %0:_(<2 x s16>)
2369 //
2370 // =>
2371 //
2372 // %2:_(s16), %3:_(s16) = G_UNMERGE_VALUES %0
2373 // %3:_(<2 x s8>) = G_BITCAST %2
2374 // %4:_(<2 x s8>) = G_BITCAST %3
2375 // %1:_(<4 x s16>) = G_CONCAT_VECTORS %3, %4
2376 DstCastTy = LLT::vector(NumDstElt / NumSrcElt, DstEltTy);
2377 SrcPartTy = SrcEltTy;
2378 } else if (NumSrcElt > NumDstElt) { // Source element type is smaller.
2379 //
2380 // %1:_(<2 x s16>) = G_BITCAST %0:_(<4 x s8>)
2381 //
2382 // =>
2383 //
2384 // %2:_(<2 x s8>), %3:_(<2 x s8>) = G_UNMERGE_VALUES %0
2385 // %3:_(s16) = G_BITCAST %2
2386 // %4:_(s16) = G_BITCAST %3
2387 // %1:_(<2 x s16>) = G_BUILD_VECTOR %3, %4
2388 SrcPartTy = LLT::vector(NumSrcElt / NumDstElt, SrcEltTy);
2389 DstCastTy = DstEltTy;
2390 }
2391
2392 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcPartTy);
2393 for (Register &SrcReg : SrcRegs)
2394 SrcReg = MIRBuilder.buildBitcast(DstCastTy, SrcReg).getReg(0);
2395 } else
2396 getUnmergePieces(SrcRegs, MIRBuilder, Src, SrcEltTy);
2397
2398 MIRBuilder.buildMerge(Dst, SrcRegs);
2399 MI.eraseFromParent();
2400 return Legalized;
2401 }
2402
2403 if (DstTy.isVector()) {
2404 SmallVector<Register, 8> SrcRegs;
2405 getUnmergePieces(SrcRegs, MIRBuilder, Src, DstTy.getElementType());
2406 MIRBuilder.buildMerge(Dst, SrcRegs);
2407 MI.eraseFromParent();
2408 return Legalized;
2409 }
2410
2411 return UnableToLegalize;
2412 }
2413
2414 /// Figure out the bit offset into a register when coercing a vector index for
2415 /// the wide element type. This is only for the case when promoting vector to
2416 /// one with larger elements.
2417 //
2418 ///
2419 /// %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
2420 /// %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
getBitcastWiderVectorElementOffset(MachineIRBuilder & B,Register Idx,unsigned NewEltSize,unsigned OldEltSize)2421 static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B,
2422 Register Idx,
2423 unsigned NewEltSize,
2424 unsigned OldEltSize) {
2425 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2426 LLT IdxTy = B.getMRI()->getType(Idx);
2427
2428 // Now figure out the amount we need to shift to get the target bits.
2429 auto OffsetMask = B.buildConstant(
2430 IdxTy, ~(APInt::getAllOnesValue(IdxTy.getSizeInBits()) << Log2EltRatio));
2431 auto OffsetIdx = B.buildAnd(IdxTy, Idx, OffsetMask);
2432 return B.buildShl(IdxTy, OffsetIdx,
2433 B.buildConstant(IdxTy, Log2_32(OldEltSize))).getReg(0);
2434 }
2435
2436 /// Perform a G_EXTRACT_VECTOR_ELT in a different sized vector element. If this
2437 /// is casting to a vector with a smaller element size, perform multiple element
2438 /// extracts and merge the results. If this is coercing to a vector with larger
2439 /// elements, index the bitcasted vector and extract the target element with bit
2440 /// operations. This is intended to force the indexing in the native register
2441 /// size for architectures that can dynamically index the register file.
2442 LegalizerHelper::LegalizeResult
bitcastExtractVectorElt(MachineInstr & MI,unsigned TypeIdx,LLT CastTy)2443 LegalizerHelper::bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx,
2444 LLT CastTy) {
2445 if (TypeIdx != 1)
2446 return UnableToLegalize;
2447
2448 Register Dst = MI.getOperand(0).getReg();
2449 Register SrcVec = MI.getOperand(1).getReg();
2450 Register Idx = MI.getOperand(2).getReg();
2451 LLT SrcVecTy = MRI.getType(SrcVec);
2452 LLT IdxTy = MRI.getType(Idx);
2453
2454 LLT SrcEltTy = SrcVecTy.getElementType();
2455 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
2456 unsigned OldNumElts = SrcVecTy.getNumElements();
2457
2458 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
2459 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
2460
2461 const unsigned NewEltSize = NewEltTy.getSizeInBits();
2462 const unsigned OldEltSize = SrcEltTy.getSizeInBits();
2463 if (NewNumElts > OldNumElts) {
2464 // Decreasing the vector element size
2465 //
2466 // e.g. i64 = extract_vector_elt x:v2i64, y:i32
2467 // =>
2468 // v4i32:castx = bitcast x:v2i64
2469 //
2470 // i64 = bitcast
2471 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
2472 // (i32 (extract_vector_elt castx, (2 * y + 1)))
2473 //
2474 if (NewNumElts % OldNumElts != 0)
2475 return UnableToLegalize;
2476
2477 // Type of the intermediate result vector.
2478 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
2479 LLT MidTy = LLT::scalarOrVector(NewEltsPerOldElt, NewEltTy);
2480
2481 auto NewEltsPerOldEltK = MIRBuilder.buildConstant(IdxTy, NewEltsPerOldElt);
2482
2483 SmallVector<Register, 8> NewOps(NewEltsPerOldElt);
2484 auto NewBaseIdx = MIRBuilder.buildMul(IdxTy, Idx, NewEltsPerOldEltK);
2485
2486 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
2487 auto IdxOffset = MIRBuilder.buildConstant(IdxTy, I);
2488 auto TmpIdx = MIRBuilder.buildAdd(IdxTy, NewBaseIdx, IdxOffset);
2489 auto Elt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec, TmpIdx);
2490 NewOps[I] = Elt.getReg(0);
2491 }
2492
2493 auto NewVec = MIRBuilder.buildBuildVector(MidTy, NewOps);
2494 MIRBuilder.buildBitcast(Dst, NewVec);
2495 MI.eraseFromParent();
2496 return Legalized;
2497 }
2498
2499 if (NewNumElts < OldNumElts) {
2500 if (NewEltSize % OldEltSize != 0)
2501 return UnableToLegalize;
2502
2503 // This only depends on powers of 2 because we use bit tricks to figure out
2504 // the bit offset we need to shift to get the target element. A general
2505 // expansion could emit division/multiply.
2506 if (!isPowerOf2_32(NewEltSize / OldEltSize))
2507 return UnableToLegalize;
2508
2509 // Increasing the vector element size.
2510 // %elt:_(small_elt) = G_EXTRACT_VECTOR_ELT %vec:_(<N x small_elt>), %idx
2511 //
2512 // =>
2513 //
2514 // %cast = G_BITCAST %vec
2515 // %scaled_idx = G_LSHR %idx, Log2(DstEltSize / SrcEltSize)
2516 // %wide_elt = G_EXTRACT_VECTOR_ELT %cast, %scaled_idx
2517 // %offset_idx = G_AND %idx, ~(-1 << Log2(DstEltSize / SrcEltSize))
2518 // %offset_bits = G_SHL %offset_idx, Log2(SrcEltSize)
2519 // %elt_bits = G_LSHR %wide_elt, %offset_bits
2520 // %elt = G_TRUNC %elt_bits
2521
2522 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2523 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
2524
2525 // Divide to get the index in the wider element type.
2526 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
2527
2528 Register WideElt = CastVec;
2529 if (CastTy.isVector()) {
2530 WideElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
2531 ScaledIdx).getReg(0);
2532 }
2533
2534 // Compute the bit offset into the register of the target element.
2535 Register OffsetBits = getBitcastWiderVectorElementOffset(
2536 MIRBuilder, Idx, NewEltSize, OldEltSize);
2537
2538 // Shift the wide element to get the target element.
2539 auto ExtractedBits = MIRBuilder.buildLShr(NewEltTy, WideElt, OffsetBits);
2540 MIRBuilder.buildTrunc(Dst, ExtractedBits);
2541 MI.eraseFromParent();
2542 return Legalized;
2543 }
2544
2545 return UnableToLegalize;
2546 }
2547
2548 /// Emit code to insert \p InsertReg into \p TargetRet at \p OffsetBits in \p
2549 /// TargetReg, while preserving other bits in \p TargetReg.
2550 ///
2551 /// (InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
buildBitFieldInsert(MachineIRBuilder & B,Register TargetReg,Register InsertReg,Register OffsetBits)2552 static Register buildBitFieldInsert(MachineIRBuilder &B,
2553 Register TargetReg, Register InsertReg,
2554 Register OffsetBits) {
2555 LLT TargetTy = B.getMRI()->getType(TargetReg);
2556 LLT InsertTy = B.getMRI()->getType(InsertReg);
2557 auto ZextVal = B.buildZExt(TargetTy, InsertReg);
2558 auto ShiftedInsertVal = B.buildShl(TargetTy, ZextVal, OffsetBits);
2559
2560 // Produce a bitmask of the value to insert
2561 auto EltMask = B.buildConstant(
2562 TargetTy, APInt::getLowBitsSet(TargetTy.getSizeInBits(),
2563 InsertTy.getSizeInBits()));
2564 // Shift it into position
2565 auto ShiftedMask = B.buildShl(TargetTy, EltMask, OffsetBits);
2566 auto InvShiftedMask = B.buildNot(TargetTy, ShiftedMask);
2567
2568 // Clear out the bits in the wide element
2569 auto MaskedOldElt = B.buildAnd(TargetTy, TargetReg, InvShiftedMask);
2570
2571 // The value to insert has all zeros already, so stick it into the masked
2572 // wide element.
2573 return B.buildOr(TargetTy, MaskedOldElt, ShiftedInsertVal).getReg(0);
2574 }
2575
2576 /// Perform a G_INSERT_VECTOR_ELT in a different sized vector element. If this
2577 /// is increasing the element size, perform the indexing in the target element
2578 /// type, and use bit operations to insert at the element position. This is
2579 /// intended for architectures that can dynamically index the register file and
2580 /// want to force indexing in the native register size.
2581 LegalizerHelper::LegalizeResult
bitcastInsertVectorElt(MachineInstr & MI,unsigned TypeIdx,LLT CastTy)2582 LegalizerHelper::bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx,
2583 LLT CastTy) {
2584 if (TypeIdx != 0)
2585 return UnableToLegalize;
2586
2587 Register Dst = MI.getOperand(0).getReg();
2588 Register SrcVec = MI.getOperand(1).getReg();
2589 Register Val = MI.getOperand(2).getReg();
2590 Register Idx = MI.getOperand(3).getReg();
2591
2592 LLT VecTy = MRI.getType(Dst);
2593 LLT IdxTy = MRI.getType(Idx);
2594
2595 LLT VecEltTy = VecTy.getElementType();
2596 LLT NewEltTy = CastTy.isVector() ? CastTy.getElementType() : CastTy;
2597 const unsigned NewEltSize = NewEltTy.getSizeInBits();
2598 const unsigned OldEltSize = VecEltTy.getSizeInBits();
2599
2600 unsigned NewNumElts = CastTy.isVector() ? CastTy.getNumElements() : 1;
2601 unsigned OldNumElts = VecTy.getNumElements();
2602
2603 Register CastVec = MIRBuilder.buildBitcast(CastTy, SrcVec).getReg(0);
2604 if (NewNumElts < OldNumElts) {
2605 if (NewEltSize % OldEltSize != 0)
2606 return UnableToLegalize;
2607
2608 // This only depends on powers of 2 because we use bit tricks to figure out
2609 // the bit offset we need to shift to get the target element. A general
2610 // expansion could emit division/multiply.
2611 if (!isPowerOf2_32(NewEltSize / OldEltSize))
2612 return UnableToLegalize;
2613
2614 const unsigned Log2EltRatio = Log2_32(NewEltSize / OldEltSize);
2615 auto Log2Ratio = MIRBuilder.buildConstant(IdxTy, Log2EltRatio);
2616
2617 // Divide to get the index in the wider element type.
2618 auto ScaledIdx = MIRBuilder.buildLShr(IdxTy, Idx, Log2Ratio);
2619
2620 Register ExtractedElt = CastVec;
2621 if (CastTy.isVector()) {
2622 ExtractedElt = MIRBuilder.buildExtractVectorElement(NewEltTy, CastVec,
2623 ScaledIdx).getReg(0);
2624 }
2625
2626 // Compute the bit offset into the register of the target element.
2627 Register OffsetBits = getBitcastWiderVectorElementOffset(
2628 MIRBuilder, Idx, NewEltSize, OldEltSize);
2629
2630 Register InsertedElt = buildBitFieldInsert(MIRBuilder, ExtractedElt,
2631 Val, OffsetBits);
2632 if (CastTy.isVector()) {
2633 InsertedElt = MIRBuilder.buildInsertVectorElement(
2634 CastTy, CastVec, InsertedElt, ScaledIdx).getReg(0);
2635 }
2636
2637 MIRBuilder.buildBitcast(Dst, InsertedElt);
2638 MI.eraseFromParent();
2639 return Legalized;
2640 }
2641
2642 return UnableToLegalize;
2643 }
2644
2645 LegalizerHelper::LegalizeResult
lowerLoad(MachineInstr & MI)2646 LegalizerHelper::lowerLoad(MachineInstr &MI) {
2647 // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
2648 Register DstReg = MI.getOperand(0).getReg();
2649 Register PtrReg = MI.getOperand(1).getReg();
2650 LLT DstTy = MRI.getType(DstReg);
2651 auto &MMO = **MI.memoperands_begin();
2652
2653 if (DstTy.getSizeInBits() == MMO.getSizeInBits()) {
2654 if (MI.getOpcode() == TargetOpcode::G_LOAD) {
2655 // This load needs splitting into power of 2 sized loads.
2656 if (DstTy.isVector())
2657 return UnableToLegalize;
2658 if (isPowerOf2_32(DstTy.getSizeInBits()))
2659 return UnableToLegalize; // Don't know what we're being asked to do.
2660
2661 // Our strategy here is to generate anyextending loads for the smaller
2662 // types up to next power-2 result type, and then combine the two larger
2663 // result values together, before truncating back down to the non-pow-2
2664 // type.
2665 // E.g. v1 = i24 load =>
2666 // v2 = i32 zextload (2 byte)
2667 // v3 = i32 load (1 byte)
2668 // v4 = i32 shl v3, 16
2669 // v5 = i32 or v4, v2
2670 // v1 = i24 trunc v5
2671 // By doing this we generate the correct truncate which should get
2672 // combined away as an artifact with a matching extend.
2673 uint64_t LargeSplitSize = PowerOf2Floor(DstTy.getSizeInBits());
2674 uint64_t SmallSplitSize = DstTy.getSizeInBits() - LargeSplitSize;
2675
2676 MachineFunction &MF = MIRBuilder.getMF();
2677 MachineMemOperand *LargeMMO =
2678 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2679 MachineMemOperand *SmallMMO = MF.getMachineMemOperand(
2680 &MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2681
2682 LLT PtrTy = MRI.getType(PtrReg);
2683 unsigned AnyExtSize = NextPowerOf2(DstTy.getSizeInBits());
2684 LLT AnyExtTy = LLT::scalar(AnyExtSize);
2685 Register LargeLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2686 Register SmallLdReg = MRI.createGenericVirtualRegister(AnyExtTy);
2687 auto LargeLoad = MIRBuilder.buildLoadInstr(
2688 TargetOpcode::G_ZEXTLOAD, LargeLdReg, PtrReg, *LargeMMO);
2689
2690 auto OffsetCst = MIRBuilder.buildConstant(
2691 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8);
2692 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
2693 auto SmallPtr =
2694 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
2695 auto SmallLoad = MIRBuilder.buildLoad(SmallLdReg, SmallPtr.getReg(0),
2696 *SmallMMO);
2697
2698 auto ShiftAmt = MIRBuilder.buildConstant(AnyExtTy, LargeSplitSize);
2699 auto Shift = MIRBuilder.buildShl(AnyExtTy, SmallLoad, ShiftAmt);
2700 auto Or = MIRBuilder.buildOr(AnyExtTy, Shift, LargeLoad);
2701 MIRBuilder.buildTrunc(DstReg, {Or.getReg(0)});
2702 MI.eraseFromParent();
2703 return Legalized;
2704 }
2705
2706 MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
2707 MI.eraseFromParent();
2708 return Legalized;
2709 }
2710
2711 if (DstTy.isScalar()) {
2712 Register TmpReg =
2713 MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits()));
2714 MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
2715 switch (MI.getOpcode()) {
2716 default:
2717 llvm_unreachable("Unexpected opcode");
2718 case TargetOpcode::G_LOAD:
2719 MIRBuilder.buildAnyExtOrTrunc(DstReg, TmpReg);
2720 break;
2721 case TargetOpcode::G_SEXTLOAD:
2722 MIRBuilder.buildSExt(DstReg, TmpReg);
2723 break;
2724 case TargetOpcode::G_ZEXTLOAD:
2725 MIRBuilder.buildZExt(DstReg, TmpReg);
2726 break;
2727 }
2728
2729 MI.eraseFromParent();
2730 return Legalized;
2731 }
2732
2733 return UnableToLegalize;
2734 }
2735
2736 LegalizerHelper::LegalizeResult
lowerStore(MachineInstr & MI)2737 LegalizerHelper::lowerStore(MachineInstr &MI) {
2738 // Lower a non-power of 2 store into multiple pow-2 stores.
2739 // E.g. split an i24 store into an i16 store + i8 store.
2740 // We do this by first extending the stored value to the next largest power
2741 // of 2 type, and then using truncating stores to store the components.
2742 // By doing this, likewise with G_LOAD, generate an extend that can be
2743 // artifact-combined away instead of leaving behind extracts.
2744 Register SrcReg = MI.getOperand(0).getReg();
2745 Register PtrReg = MI.getOperand(1).getReg();
2746 LLT SrcTy = MRI.getType(SrcReg);
2747 MachineMemOperand &MMO = **MI.memoperands_begin();
2748 if (SrcTy.getSizeInBits() != MMO.getSizeInBits())
2749 return UnableToLegalize;
2750 if (SrcTy.isVector())
2751 return UnableToLegalize;
2752 if (isPowerOf2_32(SrcTy.getSizeInBits()))
2753 return UnableToLegalize; // Don't know what we're being asked to do.
2754
2755 // Extend to the next pow-2.
2756 const LLT ExtendTy = LLT::scalar(NextPowerOf2(SrcTy.getSizeInBits()));
2757 auto ExtVal = MIRBuilder.buildAnyExt(ExtendTy, SrcReg);
2758
2759 // Obtain the smaller value by shifting away the larger value.
2760 uint64_t LargeSplitSize = PowerOf2Floor(SrcTy.getSizeInBits());
2761 uint64_t SmallSplitSize = SrcTy.getSizeInBits() - LargeSplitSize;
2762 auto ShiftAmt = MIRBuilder.buildConstant(ExtendTy, LargeSplitSize);
2763 auto SmallVal = MIRBuilder.buildLShr(ExtendTy, ExtVal, ShiftAmt);
2764
2765 // Generate the PtrAdd and truncating stores.
2766 LLT PtrTy = MRI.getType(PtrReg);
2767 auto OffsetCst = MIRBuilder.buildConstant(
2768 LLT::scalar(PtrTy.getSizeInBits()), LargeSplitSize / 8);
2769 Register PtrAddReg = MRI.createGenericVirtualRegister(PtrTy);
2770 auto SmallPtr =
2771 MIRBuilder.buildPtrAdd(PtrAddReg, PtrReg, OffsetCst.getReg(0));
2772
2773 MachineFunction &MF = MIRBuilder.getMF();
2774 MachineMemOperand *LargeMMO =
2775 MF.getMachineMemOperand(&MMO, 0, LargeSplitSize / 8);
2776 MachineMemOperand *SmallMMO =
2777 MF.getMachineMemOperand(&MMO, LargeSplitSize / 8, SmallSplitSize / 8);
2778 MIRBuilder.buildStore(ExtVal.getReg(0), PtrReg, *LargeMMO);
2779 MIRBuilder.buildStore(SmallVal.getReg(0), SmallPtr.getReg(0), *SmallMMO);
2780 MI.eraseFromParent();
2781 return Legalized;
2782 }
2783
2784 LegalizerHelper::LegalizeResult
bitcast(MachineInstr & MI,unsigned TypeIdx,LLT CastTy)2785 LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
2786 switch (MI.getOpcode()) {
2787 case TargetOpcode::G_LOAD: {
2788 if (TypeIdx != 0)
2789 return UnableToLegalize;
2790
2791 Observer.changingInstr(MI);
2792 bitcastDst(MI, CastTy, 0);
2793 Observer.changedInstr(MI);
2794 return Legalized;
2795 }
2796 case TargetOpcode::G_STORE: {
2797 if (TypeIdx != 0)
2798 return UnableToLegalize;
2799
2800 Observer.changingInstr(MI);
2801 bitcastSrc(MI, CastTy, 0);
2802 Observer.changedInstr(MI);
2803 return Legalized;
2804 }
2805 case TargetOpcode::G_SELECT: {
2806 if (TypeIdx != 0)
2807 return UnableToLegalize;
2808
2809 if (MRI.getType(MI.getOperand(1).getReg()).isVector()) {
2810 LLVM_DEBUG(
2811 dbgs() << "bitcast action not implemented for vector select\n");
2812 return UnableToLegalize;
2813 }
2814
2815 Observer.changingInstr(MI);
2816 bitcastSrc(MI, CastTy, 2);
2817 bitcastSrc(MI, CastTy, 3);
2818 bitcastDst(MI, CastTy, 0);
2819 Observer.changedInstr(MI);
2820 return Legalized;
2821 }
2822 case TargetOpcode::G_AND:
2823 case TargetOpcode::G_OR:
2824 case TargetOpcode::G_XOR: {
2825 Observer.changingInstr(MI);
2826 bitcastSrc(MI, CastTy, 1);
2827 bitcastSrc(MI, CastTy, 2);
2828 bitcastDst(MI, CastTy, 0);
2829 Observer.changedInstr(MI);
2830 return Legalized;
2831 }
2832 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
2833 return bitcastExtractVectorElt(MI, TypeIdx, CastTy);
2834 case TargetOpcode::G_INSERT_VECTOR_ELT:
2835 return bitcastInsertVectorElt(MI, TypeIdx, CastTy);
2836 default:
2837 return UnableToLegalize;
2838 }
2839 }
2840
2841 // Legalize an instruction by changing the opcode in place.
changeOpcode(MachineInstr & MI,unsigned NewOpcode)2842 void LegalizerHelper::changeOpcode(MachineInstr &MI, unsigned NewOpcode) {
2843 Observer.changingInstr(MI);
2844 MI.setDesc(MIRBuilder.getTII().get(NewOpcode));
2845 Observer.changedInstr(MI);
2846 }
2847
2848 LegalizerHelper::LegalizeResult
lower(MachineInstr & MI,unsigned TypeIdx,LLT LowerHintTy)2849 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
2850 using namespace TargetOpcode;
2851
2852 switch(MI.getOpcode()) {
2853 default:
2854 return UnableToLegalize;
2855 case TargetOpcode::G_BITCAST:
2856 return lowerBitcast(MI);
2857 case TargetOpcode::G_SREM:
2858 case TargetOpcode::G_UREM: {
2859 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
2860 auto Quot =
2861 MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV, {Ty},
2862 {MI.getOperand(1), MI.getOperand(2)});
2863
2864 auto Prod = MIRBuilder.buildMul(Ty, Quot, MI.getOperand(2));
2865 MIRBuilder.buildSub(MI.getOperand(0), MI.getOperand(1), Prod);
2866 MI.eraseFromParent();
2867 return Legalized;
2868 }
2869 case TargetOpcode::G_SADDO:
2870 case TargetOpcode::G_SSUBO:
2871 return lowerSADDO_SSUBO(MI);
2872 case TargetOpcode::G_UMULH:
2873 case TargetOpcode::G_SMULH:
2874 return lowerSMULH_UMULH(MI);
2875 case TargetOpcode::G_SMULO:
2876 case TargetOpcode::G_UMULO: {
2877 // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
2878 // result.
2879 Register Res = MI.getOperand(0).getReg();
2880 Register Overflow = MI.getOperand(1).getReg();
2881 Register LHS = MI.getOperand(2).getReg();
2882 Register RHS = MI.getOperand(3).getReg();
2883 LLT Ty = MRI.getType(Res);
2884
2885 unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
2886 ? TargetOpcode::G_SMULH
2887 : TargetOpcode::G_UMULH;
2888
2889 Observer.changingInstr(MI);
2890 const auto &TII = MIRBuilder.getTII();
2891 MI.setDesc(TII.get(TargetOpcode::G_MUL));
2892 MI.RemoveOperand(1);
2893 Observer.changedInstr(MI);
2894
2895 auto HiPart = MIRBuilder.buildInstr(Opcode, {Ty}, {LHS, RHS});
2896 auto Zero = MIRBuilder.buildConstant(Ty, 0);
2897
2898 // Move insert point forward so we can use the Res register if needed.
2899 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
2900
2901 // For *signed* multiply, overflow is detected by checking:
2902 // (hi != (lo >> bitwidth-1))
2903 if (Opcode == TargetOpcode::G_SMULH) {
2904 auto ShiftAmt = MIRBuilder.buildConstant(Ty, Ty.getSizeInBits() - 1);
2905 auto Shifted = MIRBuilder.buildAShr(Ty, Res, ShiftAmt);
2906 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
2907 } else {
2908 MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
2909 }
2910 return Legalized;
2911 }
2912 case TargetOpcode::G_FNEG: {
2913 Register Res = MI.getOperand(0).getReg();
2914 LLT Ty = MRI.getType(Res);
2915
2916 // TODO: Handle vector types once we are able to
2917 // represent them.
2918 if (Ty.isVector())
2919 return UnableToLegalize;
2920 auto SignMask =
2921 MIRBuilder.buildConstant(Ty, APInt::getSignMask(Ty.getSizeInBits()));
2922 Register SubByReg = MI.getOperand(1).getReg();
2923 MIRBuilder.buildXor(Res, SubByReg, SignMask);
2924 MI.eraseFromParent();
2925 return Legalized;
2926 }
2927 case TargetOpcode::G_FSUB: {
2928 Register Res = MI.getOperand(0).getReg();
2929 LLT Ty = MRI.getType(Res);
2930
2931 // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
2932 // First, check if G_FNEG is marked as Lower. If so, we may
2933 // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
2934 if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
2935 return UnableToLegalize;
2936 Register LHS = MI.getOperand(1).getReg();
2937 Register RHS = MI.getOperand(2).getReg();
2938 Register Neg = MRI.createGenericVirtualRegister(Ty);
2939 MIRBuilder.buildFNeg(Neg, RHS);
2940 MIRBuilder.buildFAdd(Res, LHS, Neg, MI.getFlags());
2941 MI.eraseFromParent();
2942 return Legalized;
2943 }
2944 case TargetOpcode::G_FMAD:
2945 return lowerFMad(MI);
2946 case TargetOpcode::G_FFLOOR:
2947 return lowerFFloor(MI);
2948 case TargetOpcode::G_INTRINSIC_ROUND:
2949 return lowerIntrinsicRound(MI);
2950 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
2951 // Since round even is the assumed rounding mode for unconstrained FP
2952 // operations, rint and roundeven are the same operation.
2953 changeOpcode(MI, TargetOpcode::G_FRINT);
2954 return Legalized;
2955 }
2956 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
2957 Register OldValRes = MI.getOperand(0).getReg();
2958 Register SuccessRes = MI.getOperand(1).getReg();
2959 Register Addr = MI.getOperand(2).getReg();
2960 Register CmpVal = MI.getOperand(3).getReg();
2961 Register NewVal = MI.getOperand(4).getReg();
2962 MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
2963 **MI.memoperands_begin());
2964 MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
2965 MI.eraseFromParent();
2966 return Legalized;
2967 }
2968 case TargetOpcode::G_LOAD:
2969 case TargetOpcode::G_SEXTLOAD:
2970 case TargetOpcode::G_ZEXTLOAD:
2971 return lowerLoad(MI);
2972 case TargetOpcode::G_STORE:
2973 return lowerStore(MI);
2974 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2975 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2976 case TargetOpcode::G_CTLZ:
2977 case TargetOpcode::G_CTTZ:
2978 case TargetOpcode::G_CTPOP:
2979 return lowerBitCount(MI);
2980 case G_UADDO: {
2981 Register Res = MI.getOperand(0).getReg();
2982 Register CarryOut = MI.getOperand(1).getReg();
2983 Register LHS = MI.getOperand(2).getReg();
2984 Register RHS = MI.getOperand(3).getReg();
2985
2986 MIRBuilder.buildAdd(Res, LHS, RHS);
2987 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
2988
2989 MI.eraseFromParent();
2990 return Legalized;
2991 }
2992 case G_UADDE: {
2993 Register Res = MI.getOperand(0).getReg();
2994 Register CarryOut = MI.getOperand(1).getReg();
2995 Register LHS = MI.getOperand(2).getReg();
2996 Register RHS = MI.getOperand(3).getReg();
2997 Register CarryIn = MI.getOperand(4).getReg();
2998 LLT Ty = MRI.getType(Res);
2999
3000 auto TmpRes = MIRBuilder.buildAdd(Ty, LHS, RHS);
3001 auto ZExtCarryIn = MIRBuilder.buildZExt(Ty, CarryIn);
3002 MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
3003 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
3004
3005 MI.eraseFromParent();
3006 return Legalized;
3007 }
3008 case G_USUBO: {
3009 Register Res = MI.getOperand(0).getReg();
3010 Register BorrowOut = MI.getOperand(1).getReg();
3011 Register LHS = MI.getOperand(2).getReg();
3012 Register RHS = MI.getOperand(3).getReg();
3013
3014 MIRBuilder.buildSub(Res, LHS, RHS);
3015 MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
3016
3017 MI.eraseFromParent();
3018 return Legalized;
3019 }
3020 case G_USUBE: {
3021 Register Res = MI.getOperand(0).getReg();
3022 Register BorrowOut = MI.getOperand(1).getReg();
3023 Register LHS = MI.getOperand(2).getReg();
3024 Register RHS = MI.getOperand(3).getReg();
3025 Register BorrowIn = MI.getOperand(4).getReg();
3026 const LLT CondTy = MRI.getType(BorrowOut);
3027 const LLT Ty = MRI.getType(Res);
3028
3029 auto TmpRes = MIRBuilder.buildSub(Ty, LHS, RHS);
3030 auto ZExtBorrowIn = MIRBuilder.buildZExt(Ty, BorrowIn);
3031 MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
3032
3033 auto LHS_EQ_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, CondTy, LHS, RHS);
3034 auto LHS_ULT_RHS = MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CondTy, LHS, RHS);
3035 MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
3036
3037 MI.eraseFromParent();
3038 return Legalized;
3039 }
3040 case G_UITOFP:
3041 return lowerUITOFP(MI);
3042 case G_SITOFP:
3043 return lowerSITOFP(MI);
3044 case G_FPTOUI:
3045 return lowerFPTOUI(MI);
3046 case G_FPTOSI:
3047 return lowerFPTOSI(MI);
3048 case G_FPTRUNC:
3049 return lowerFPTRUNC(MI);
3050 case G_FPOWI:
3051 return lowerFPOWI(MI);
3052 case G_SMIN:
3053 case G_SMAX:
3054 case G_UMIN:
3055 case G_UMAX:
3056 return lowerMinMax(MI);
3057 case G_FCOPYSIGN:
3058 return lowerFCopySign(MI);
3059 case G_FMINNUM:
3060 case G_FMAXNUM:
3061 return lowerFMinNumMaxNum(MI);
3062 case G_MERGE_VALUES:
3063 return lowerMergeValues(MI);
3064 case G_UNMERGE_VALUES:
3065 return lowerUnmergeValues(MI);
3066 case TargetOpcode::G_SEXT_INREG: {
3067 assert(MI.getOperand(2).isImm() && "Expected immediate");
3068 int64_t SizeInBits = MI.getOperand(2).getImm();
3069
3070 Register DstReg = MI.getOperand(0).getReg();
3071 Register SrcReg = MI.getOperand(1).getReg();
3072 LLT DstTy = MRI.getType(DstReg);
3073 Register TmpRes = MRI.createGenericVirtualRegister(DstTy);
3074
3075 auto MIBSz = MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - SizeInBits);
3076 MIRBuilder.buildShl(TmpRes, SrcReg, MIBSz->getOperand(0));
3077 MIRBuilder.buildAShr(DstReg, TmpRes, MIBSz->getOperand(0));
3078 MI.eraseFromParent();
3079 return Legalized;
3080 }
3081 case G_EXTRACT_VECTOR_ELT:
3082 case G_INSERT_VECTOR_ELT:
3083 return lowerExtractInsertVectorElt(MI);
3084 case G_SHUFFLE_VECTOR:
3085 return lowerShuffleVector(MI);
3086 case G_DYN_STACKALLOC:
3087 return lowerDynStackAlloc(MI);
3088 case G_EXTRACT:
3089 return lowerExtract(MI);
3090 case G_INSERT:
3091 return lowerInsert(MI);
3092 case G_BSWAP:
3093 return lowerBswap(MI);
3094 case G_BITREVERSE:
3095 return lowerBitreverse(MI);
3096 case G_READ_REGISTER:
3097 case G_WRITE_REGISTER:
3098 return lowerReadWriteRegister(MI);
3099 case G_UADDSAT:
3100 case G_USUBSAT: {
3101 // Try to make a reasonable guess about which lowering strategy to use. The
3102 // target can override this with custom lowering and calling the
3103 // implementation functions.
3104 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3105 if (LI.isLegalOrCustom({G_UMIN, Ty}))
3106 return lowerAddSubSatToMinMax(MI);
3107 return lowerAddSubSatToAddoSubo(MI);
3108 }
3109 case G_SADDSAT:
3110 case G_SSUBSAT: {
3111 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
3112
3113 // FIXME: It would probably make more sense to see if G_SADDO is preferred,
3114 // since it's a shorter expansion. However, we would need to figure out the
3115 // preferred boolean type for the carry out for the query.
3116 if (LI.isLegalOrCustom({G_SMIN, Ty}) && LI.isLegalOrCustom({G_SMAX, Ty}))
3117 return lowerAddSubSatToMinMax(MI);
3118 return lowerAddSubSatToAddoSubo(MI);
3119 }
3120 case G_SSHLSAT:
3121 case G_USHLSAT:
3122 return lowerShlSat(MI);
3123 case G_ABS: {
3124 // Expand %res = G_ABS %a into:
3125 // %v1 = G_ASHR %a, scalar_size-1
3126 // %v2 = G_ADD %a, %v1
3127 // %res = G_XOR %v2, %v1
3128 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3129 Register OpReg = MI.getOperand(1).getReg();
3130 auto ShiftAmt =
3131 MIRBuilder.buildConstant(DstTy, DstTy.getScalarSizeInBits() - 1);
3132 auto Shift =
3133 MIRBuilder.buildAShr(DstTy, OpReg, ShiftAmt);
3134 auto Add = MIRBuilder.buildAdd(DstTy, OpReg, Shift);
3135 MIRBuilder.buildXor(MI.getOperand(0).getReg(), Add, Shift);
3136 MI.eraseFromParent();
3137 return Legalized;
3138 }
3139 case G_SELECT:
3140 return lowerSelect(MI);
3141 }
3142 }
3143
getStackTemporaryAlignment(LLT Ty,Align MinAlign) const3144 Align LegalizerHelper::getStackTemporaryAlignment(LLT Ty,
3145 Align MinAlign) const {
3146 // FIXME: We're missing a way to go back from LLT to llvm::Type to query the
3147 // datalayout for the preferred alignment. Also there should be a target hook
3148 // for this to allow targets to reduce the alignment and ignore the
3149 // datalayout. e.g. AMDGPU should always use a 4-byte alignment, regardless of
3150 // the type.
3151 return std::max(Align(PowerOf2Ceil(Ty.getSizeInBytes())), MinAlign);
3152 }
3153
3154 MachineInstrBuilder
createStackTemporary(TypeSize Bytes,Align Alignment,MachinePointerInfo & PtrInfo)3155 LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
3156 MachinePointerInfo &PtrInfo) {
3157 MachineFunction &MF = MIRBuilder.getMF();
3158 const DataLayout &DL = MIRBuilder.getDataLayout();
3159 int FrameIdx = MF.getFrameInfo().CreateStackObject(Bytes, Alignment, false);
3160
3161 unsigned AddrSpace = DL.getAllocaAddrSpace();
3162 LLT FramePtrTy = LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace));
3163
3164 PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIdx);
3165 return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx);
3166 }
3167
clampDynamicVectorIndex(MachineIRBuilder & B,Register IdxReg,LLT VecTy)3168 static Register clampDynamicVectorIndex(MachineIRBuilder &B, Register IdxReg,
3169 LLT VecTy) {
3170 int64_t IdxVal;
3171 if (mi_match(IdxReg, *B.getMRI(), m_ICst(IdxVal)))
3172 return IdxReg;
3173
3174 LLT IdxTy = B.getMRI()->getType(IdxReg);
3175 unsigned NElts = VecTy.getNumElements();
3176 if (isPowerOf2_32(NElts)) {
3177 APInt Imm = APInt::getLowBitsSet(IdxTy.getSizeInBits(), Log2_32(NElts));
3178 return B.buildAnd(IdxTy, IdxReg, B.buildConstant(IdxTy, Imm)).getReg(0);
3179 }
3180
3181 return B.buildUMin(IdxTy, IdxReg, B.buildConstant(IdxTy, NElts - 1))
3182 .getReg(0);
3183 }
3184
getVectorElementPointer(Register VecPtr,LLT VecTy,Register Index)3185 Register LegalizerHelper::getVectorElementPointer(Register VecPtr, LLT VecTy,
3186 Register Index) {
3187 LLT EltTy = VecTy.getElementType();
3188
3189 // Calculate the element offset and add it to the pointer.
3190 unsigned EltSize = EltTy.getSizeInBits() / 8; // FIXME: should be ABI size.
3191 assert(EltSize * 8 == EltTy.getSizeInBits() &&
3192 "Converting bits to bytes lost precision");
3193
3194 Index = clampDynamicVectorIndex(MIRBuilder, Index, VecTy);
3195
3196 LLT IdxTy = MRI.getType(Index);
3197 auto Mul = MIRBuilder.buildMul(IdxTy, Index,
3198 MIRBuilder.buildConstant(IdxTy, EltSize));
3199
3200 LLT PtrTy = MRI.getType(VecPtr);
3201 return MIRBuilder.buildPtrAdd(PtrTy, VecPtr, Mul).getReg(0);
3202 }
3203
fewerElementsVectorImplicitDef(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3204 LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
3205 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
3206 Register DstReg = MI.getOperand(0).getReg();
3207 LLT DstTy = MRI.getType(DstReg);
3208 LLT LCMTy = getLCMType(DstTy, NarrowTy);
3209
3210 unsigned NumParts = LCMTy.getSizeInBits() / NarrowTy.getSizeInBits();
3211
3212 auto NewUndef = MIRBuilder.buildUndef(NarrowTy);
3213 SmallVector<Register, 8> Parts(NumParts, NewUndef.getReg(0));
3214
3215 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
3216 MI.eraseFromParent();
3217 return Legalized;
3218 }
3219
3220 // Handle splitting vector operations which need to have the same number of
3221 // elements in each type index, but each type index may have a different element
3222 // type.
3223 //
3224 // e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
3225 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3226 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3227 //
3228 // Also handles some irregular breakdown cases, e.g.
3229 // e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
3230 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
3231 // s64 = G_SHL s64, s32
3232 LegalizerHelper::LegalizeResult
fewerElementsVectorMultiEltType(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTyArg)3233 LegalizerHelper::fewerElementsVectorMultiEltType(
3234 MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
3235 if (TypeIdx != 0)
3236 return UnableToLegalize;
3237
3238 const LLT NarrowTy0 = NarrowTyArg;
3239 const unsigned NewNumElts =
3240 NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
3241
3242 const Register DstReg = MI.getOperand(0).getReg();
3243 LLT DstTy = MRI.getType(DstReg);
3244 LLT LeftoverTy0;
3245
3246 // All of the operands need to have the same number of elements, so if we can
3247 // determine a type breakdown for the result type, we can for all of the
3248 // source types.
3249 int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first;
3250 if (NumParts < 0)
3251 return UnableToLegalize;
3252
3253 SmallVector<MachineInstrBuilder, 4> NewInsts;
3254
3255 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
3256 SmallVector<Register, 4> PartRegs, LeftoverRegs;
3257
3258 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
3259 Register SrcReg = MI.getOperand(I).getReg();
3260 LLT SrcTyI = MRI.getType(SrcReg);
3261 LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
3262 LLT LeftoverTyI;
3263
3264 // Split this operand into the requested typed registers, and any leftover
3265 // required to reproduce the original type.
3266 if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
3267 LeftoverRegs))
3268 return UnableToLegalize;
3269
3270 if (I == 1) {
3271 // For the first operand, create an instruction for each part and setup
3272 // the result.
3273 for (Register PartReg : PartRegs) {
3274 Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3275 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
3276 .addDef(PartDstReg)
3277 .addUse(PartReg));
3278 DstRegs.push_back(PartDstReg);
3279 }
3280
3281 for (Register LeftoverReg : LeftoverRegs) {
3282 Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
3283 NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
3284 .addDef(PartDstReg)
3285 .addUse(LeftoverReg));
3286 LeftoverDstRegs.push_back(PartDstReg);
3287 }
3288 } else {
3289 assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size());
3290
3291 // Add the newly created operand splits to the existing instructions. The
3292 // odd-sized pieces are ordered after the requested NarrowTyArg sized
3293 // pieces.
3294 unsigned InstCount = 0;
3295 for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
3296 NewInsts[InstCount++].addUse(PartRegs[J]);
3297 for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
3298 NewInsts[InstCount++].addUse(LeftoverRegs[J]);
3299 }
3300
3301 PartRegs.clear();
3302 LeftoverRegs.clear();
3303 }
3304
3305 // Insert the newly built operations and rebuild the result register.
3306 for (auto &MIB : NewInsts)
3307 MIRBuilder.insertInstr(MIB);
3308
3309 insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
3310
3311 MI.eraseFromParent();
3312 return Legalized;
3313 }
3314
3315 LegalizerHelper::LegalizeResult
fewerElementsVectorCasts(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3316 LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
3317 LLT NarrowTy) {
3318 if (TypeIdx != 0)
3319 return UnableToLegalize;
3320
3321 Register DstReg = MI.getOperand(0).getReg();
3322 Register SrcReg = MI.getOperand(1).getReg();
3323 LLT DstTy = MRI.getType(DstReg);
3324 LLT SrcTy = MRI.getType(SrcReg);
3325
3326 LLT NarrowTy0 = NarrowTy;
3327 LLT NarrowTy1;
3328 unsigned NumParts;
3329
3330 if (NarrowTy.isVector()) {
3331 // Uneven breakdown not handled.
3332 NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
3333 if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
3334 return UnableToLegalize;
3335
3336 NarrowTy1 = LLT::vector(NarrowTy.getNumElements(), SrcTy.getElementType());
3337 } else {
3338 NumParts = DstTy.getNumElements();
3339 NarrowTy1 = SrcTy.getElementType();
3340 }
3341
3342 SmallVector<Register, 4> SrcRegs, DstRegs;
3343 extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
3344
3345 for (unsigned I = 0; I < NumParts; ++I) {
3346 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3347 MachineInstr *NewInst =
3348 MIRBuilder.buildInstr(MI.getOpcode(), {DstReg}, {SrcRegs[I]});
3349
3350 NewInst->setFlags(MI.getFlags());
3351 DstRegs.push_back(DstReg);
3352 }
3353
3354 if (NarrowTy.isVector())
3355 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3356 else
3357 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3358
3359 MI.eraseFromParent();
3360 return Legalized;
3361 }
3362
3363 LegalizerHelper::LegalizeResult
fewerElementsVectorCmp(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3364 LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
3365 LLT NarrowTy) {
3366 Register DstReg = MI.getOperand(0).getReg();
3367 Register Src0Reg = MI.getOperand(2).getReg();
3368 LLT DstTy = MRI.getType(DstReg);
3369 LLT SrcTy = MRI.getType(Src0Reg);
3370
3371 unsigned NumParts;
3372 LLT NarrowTy0, NarrowTy1;
3373
3374 if (TypeIdx == 0) {
3375 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
3376 unsigned OldElts = DstTy.getNumElements();
3377
3378 NarrowTy0 = NarrowTy;
3379 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
3380 NarrowTy1 = NarrowTy.isVector() ?
3381 LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
3382 SrcTy.getElementType();
3383
3384 } else {
3385 unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
3386 unsigned OldElts = SrcTy.getNumElements();
3387
3388 NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
3389 NarrowTy.getNumElements();
3390 NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
3391 DstTy.getScalarSizeInBits());
3392 NarrowTy1 = NarrowTy;
3393 }
3394
3395 // FIXME: Don't know how to handle the situation where the small vectors
3396 // aren't all the same size yet.
3397 if (NarrowTy1.isVector() &&
3398 NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
3399 return UnableToLegalize;
3400
3401 CmpInst::Predicate Pred
3402 = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
3403
3404 SmallVector<Register, 2> Src1Regs, Src2Regs, DstRegs;
3405 extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
3406 extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
3407
3408 for (unsigned I = 0; I < NumParts; ++I) {
3409 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3410 DstRegs.push_back(DstReg);
3411
3412 if (MI.getOpcode() == TargetOpcode::G_ICMP)
3413 MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
3414 else {
3415 MachineInstr *NewCmp
3416 = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
3417 NewCmp->setFlags(MI.getFlags());
3418 }
3419 }
3420
3421 if (NarrowTy1.isVector())
3422 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3423 else
3424 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3425
3426 MI.eraseFromParent();
3427 return Legalized;
3428 }
3429
3430 LegalizerHelper::LegalizeResult
fewerElementsVectorSelect(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3431 LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
3432 LLT NarrowTy) {
3433 Register DstReg = MI.getOperand(0).getReg();
3434 Register CondReg = MI.getOperand(1).getReg();
3435
3436 unsigned NumParts = 0;
3437 LLT NarrowTy0, NarrowTy1;
3438
3439 LLT DstTy = MRI.getType(DstReg);
3440 LLT CondTy = MRI.getType(CondReg);
3441 unsigned Size = DstTy.getSizeInBits();
3442
3443 assert(TypeIdx == 0 || CondTy.isVector());
3444
3445 if (TypeIdx == 0) {
3446 NarrowTy0 = NarrowTy;
3447 NarrowTy1 = CondTy;
3448
3449 unsigned NarrowSize = NarrowTy0.getSizeInBits();
3450 // FIXME: Don't know how to handle the situation where the small vectors
3451 // aren't all the same size yet.
3452 if (Size % NarrowSize != 0)
3453 return UnableToLegalize;
3454
3455 NumParts = Size / NarrowSize;
3456
3457 // Need to break down the condition type
3458 if (CondTy.isVector()) {
3459 if (CondTy.getNumElements() == NumParts)
3460 NarrowTy1 = CondTy.getElementType();
3461 else
3462 NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
3463 CondTy.getScalarSizeInBits());
3464 }
3465 } else {
3466 NumParts = CondTy.getNumElements();
3467 if (NarrowTy.isVector()) {
3468 // TODO: Handle uneven breakdown.
3469 if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
3470 return UnableToLegalize;
3471
3472 return UnableToLegalize;
3473 } else {
3474 NarrowTy0 = DstTy.getElementType();
3475 NarrowTy1 = NarrowTy;
3476 }
3477 }
3478
3479 SmallVector<Register, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
3480 if (CondTy.isVector())
3481 extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
3482
3483 extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
3484 extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
3485
3486 for (unsigned i = 0; i < NumParts; ++i) {
3487 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
3488 MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
3489 Src1Regs[i], Src2Regs[i]);
3490 DstRegs.push_back(DstReg);
3491 }
3492
3493 if (NarrowTy0.isVector())
3494 MIRBuilder.buildConcatVectors(DstReg, DstRegs);
3495 else
3496 MIRBuilder.buildBuildVector(DstReg, DstRegs);
3497
3498 MI.eraseFromParent();
3499 return Legalized;
3500 }
3501
3502 LegalizerHelper::LegalizeResult
fewerElementsVectorPhi(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3503 LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
3504 LLT NarrowTy) {
3505 const Register DstReg = MI.getOperand(0).getReg();
3506 LLT PhiTy = MRI.getType(DstReg);
3507 LLT LeftoverTy;
3508
3509 // All of the operands need to have the same number of elements, so if we can
3510 // determine a type breakdown for the result type, we can for all of the
3511 // source types.
3512 int NumParts, NumLeftover;
3513 std::tie(NumParts, NumLeftover)
3514 = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
3515 if (NumParts < 0)
3516 return UnableToLegalize;
3517
3518 SmallVector<Register, 4> DstRegs, LeftoverDstRegs;
3519 SmallVector<MachineInstrBuilder, 4> NewInsts;
3520
3521 const int TotalNumParts = NumParts + NumLeftover;
3522
3523 // Insert the new phis in the result block first.
3524 for (int I = 0; I != TotalNumParts; ++I) {
3525 LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
3526 Register PartDstReg = MRI.createGenericVirtualRegister(Ty);
3527 NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
3528 .addDef(PartDstReg));
3529 if (I < NumParts)
3530 DstRegs.push_back(PartDstReg);
3531 else
3532 LeftoverDstRegs.push_back(PartDstReg);
3533 }
3534
3535 MachineBasicBlock *MBB = MI.getParent();
3536 MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
3537 insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
3538
3539 SmallVector<Register, 4> PartRegs, LeftoverRegs;
3540
3541 // Insert code to extract the incoming values in each predecessor block.
3542 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3543 PartRegs.clear();
3544 LeftoverRegs.clear();
3545
3546 Register SrcReg = MI.getOperand(I).getReg();
3547 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
3548 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
3549
3550 LLT Unused;
3551 if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
3552 LeftoverRegs))
3553 return UnableToLegalize;
3554
3555 // Add the newly created operand splits to the existing instructions. The
3556 // odd-sized pieces are ordered after the requested NarrowTyArg sized
3557 // pieces.
3558 for (int J = 0; J != TotalNumParts; ++J) {
3559 MachineInstrBuilder MIB = NewInsts[J];
3560 MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
3561 MIB.addMBB(&OpMBB);
3562 }
3563 }
3564
3565 MI.eraseFromParent();
3566 return Legalized;
3567 }
3568
3569 LegalizerHelper::LegalizeResult
fewerElementsVectorUnmergeValues(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3570 LegalizerHelper::fewerElementsVectorUnmergeValues(MachineInstr &MI,
3571 unsigned TypeIdx,
3572 LLT NarrowTy) {
3573 if (TypeIdx != 1)
3574 return UnableToLegalize;
3575
3576 const int NumDst = MI.getNumOperands() - 1;
3577 const Register SrcReg = MI.getOperand(NumDst).getReg();
3578 LLT SrcTy = MRI.getType(SrcReg);
3579
3580 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
3581
3582 // TODO: Create sequence of extracts.
3583 if (DstTy == NarrowTy)
3584 return UnableToLegalize;
3585
3586 LLT GCDTy = getGCDType(SrcTy, NarrowTy);
3587 if (DstTy == GCDTy) {
3588 // This would just be a copy of the same unmerge.
3589 // TODO: Create extracts, pad with undef and create intermediate merges.
3590 return UnableToLegalize;
3591 }
3592
3593 auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg);
3594 const int NumUnmerge = Unmerge->getNumOperands() - 1;
3595 const int PartsPerUnmerge = NumDst / NumUnmerge;
3596
3597 for (int I = 0; I != NumUnmerge; ++I) {
3598 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
3599
3600 for (int J = 0; J != PartsPerUnmerge; ++J)
3601 MIB.addDef(MI.getOperand(I * PartsPerUnmerge + J).getReg());
3602 MIB.addUse(Unmerge.getReg(I));
3603 }
3604
3605 MI.eraseFromParent();
3606 return Legalized;
3607 }
3608
3609 // Handle FewerElementsVector a G_BUILD_VECTOR or G_CONCAT_VECTORS that produces
3610 // a vector
3611 //
3612 // Create a G_BUILD_VECTOR or G_CONCAT_VECTORS of NarrowTy pieces, padding with
3613 // undef as necessary.
3614 //
3615 // %3:_(<3 x s16>) = G_BUILD_VECTOR %0, %1, %2
3616 // -> <2 x s16>
3617 //
3618 // %4:_(s16) = G_IMPLICIT_DEF
3619 // %5:_(<2 x s16>) = G_BUILD_VECTOR %0, %1
3620 // %6:_(<2 x s16>) = G_BUILD_VECTOR %2, %4
3621 // %7:_(<2 x s16>) = G_IMPLICIT_DEF
3622 // %8:_(<6 x s16>) = G_CONCAT_VECTORS %5, %6, %7
3623 // %3:_(<3 x s16>), %8:_(<3 x s16>) = G_UNMERGE_VALUES %8
3624 LegalizerHelper::LegalizeResult
fewerElementsVectorMerge(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3625 LegalizerHelper::fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx,
3626 LLT NarrowTy) {
3627 Register DstReg = MI.getOperand(0).getReg();
3628 LLT DstTy = MRI.getType(DstReg);
3629 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
3630 LLT GCDTy = getGCDType(getGCDType(SrcTy, NarrowTy), DstTy);
3631
3632 // Break into a common type
3633 SmallVector<Register, 16> Parts;
3634 for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
3635 extractGCDType(Parts, GCDTy, MI.getOperand(I).getReg());
3636
3637 // Build the requested new merge, padding with undef.
3638 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts,
3639 TargetOpcode::G_ANYEXT);
3640
3641 // Pack into the original result register.
3642 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
3643
3644 MI.eraseFromParent();
3645 return Legalized;
3646 }
3647
3648 LegalizerHelper::LegalizeResult
fewerElementsVectorExtractInsertVectorElt(MachineInstr & MI,unsigned TypeIdx,LLT NarrowVecTy)3649 LegalizerHelper::fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI,
3650 unsigned TypeIdx,
3651 LLT NarrowVecTy) {
3652 Register DstReg = MI.getOperand(0).getReg();
3653 Register SrcVec = MI.getOperand(1).getReg();
3654 Register InsertVal;
3655 bool IsInsert = MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
3656
3657 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) && "not a vector type index");
3658 if (IsInsert)
3659 InsertVal = MI.getOperand(2).getReg();
3660
3661 Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg();
3662
3663 // TODO: Handle total scalarization case.
3664 if (!NarrowVecTy.isVector())
3665 return UnableToLegalize;
3666
3667 LLT VecTy = MRI.getType(SrcVec);
3668
3669 // If the index is a constant, we can really break this down as you would
3670 // expect, and index into the target size pieces.
3671 int64_t IdxVal;
3672 if (mi_match(Idx, MRI, m_ICst(IdxVal))) {
3673 // Avoid out of bounds indexing the pieces.
3674 if (IdxVal >= VecTy.getNumElements()) {
3675 MIRBuilder.buildUndef(DstReg);
3676 MI.eraseFromParent();
3677 return Legalized;
3678 }
3679
3680 SmallVector<Register, 8> VecParts;
3681 LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec);
3682
3683 // Build a sequence of NarrowTy pieces in VecParts for this operand.
3684 LLT LCMTy = buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts,
3685 TargetOpcode::G_ANYEXT);
3686
3687 unsigned NewNumElts = NarrowVecTy.getNumElements();
3688
3689 LLT IdxTy = MRI.getType(Idx);
3690 int64_t PartIdx = IdxVal / NewNumElts;
3691 auto NewIdx =
3692 MIRBuilder.buildConstant(IdxTy, IdxVal - NewNumElts * PartIdx);
3693
3694 if (IsInsert) {
3695 LLT PartTy = MRI.getType(VecParts[PartIdx]);
3696
3697 // Use the adjusted index to insert into one of the subvectors.
3698 auto InsertPart = MIRBuilder.buildInsertVectorElement(
3699 PartTy, VecParts[PartIdx], InsertVal, NewIdx);
3700 VecParts[PartIdx] = InsertPart.getReg(0);
3701
3702 // Recombine the inserted subvector with the others to reform the result
3703 // vector.
3704 buildWidenedRemergeToDst(DstReg, LCMTy, VecParts);
3705 } else {
3706 MIRBuilder.buildExtractVectorElement(DstReg, VecParts[PartIdx], NewIdx);
3707 }
3708
3709 MI.eraseFromParent();
3710 return Legalized;
3711 }
3712
3713 // With a variable index, we can't perform the operation in a smaller type, so
3714 // we're forced to expand this.
3715 //
3716 // TODO: We could emit a chain of compare/select to figure out which piece to
3717 // index.
3718 return lowerExtractInsertVectorElt(MI);
3719 }
3720
3721 LegalizerHelper::LegalizeResult
reduceLoadStoreWidth(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3722 LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
3723 LLT NarrowTy) {
3724 // FIXME: Don't know how to handle secondary types yet.
3725 if (TypeIdx != 0)
3726 return UnableToLegalize;
3727
3728 MachineMemOperand *MMO = *MI.memoperands_begin();
3729
3730 // This implementation doesn't work for atomics. Give up instead of doing
3731 // something invalid.
3732 if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
3733 MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
3734 return UnableToLegalize;
3735
3736 bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
3737 Register ValReg = MI.getOperand(0).getReg();
3738 Register AddrReg = MI.getOperand(1).getReg();
3739 LLT ValTy = MRI.getType(ValReg);
3740
3741 // FIXME: Do we need a distinct NarrowMemory legalize action?
3742 if (ValTy.getSizeInBits() != 8 * MMO->getSize()) {
3743 LLVM_DEBUG(dbgs() << "Can't narrow extload/truncstore\n");
3744 return UnableToLegalize;
3745 }
3746
3747 int NumParts = -1;
3748 int NumLeftover = -1;
3749 LLT LeftoverTy;
3750 SmallVector<Register, 8> NarrowRegs, NarrowLeftoverRegs;
3751 if (IsLoad) {
3752 std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
3753 } else {
3754 if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
3755 NarrowLeftoverRegs)) {
3756 NumParts = NarrowRegs.size();
3757 NumLeftover = NarrowLeftoverRegs.size();
3758 }
3759 }
3760
3761 if (NumParts == -1)
3762 return UnableToLegalize;
3763
3764 LLT PtrTy = MRI.getType(AddrReg);
3765 const LLT OffsetTy = LLT::scalar(PtrTy.getSizeInBits());
3766
3767 unsigned TotalSize = ValTy.getSizeInBits();
3768
3769 // Split the load/store into PartTy sized pieces starting at Offset. If this
3770 // is a load, return the new registers in ValRegs. For a store, each elements
3771 // of ValRegs should be PartTy. Returns the next offset that needs to be
3772 // handled.
3773 auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<Register> &ValRegs,
3774 unsigned Offset) -> unsigned {
3775 MachineFunction &MF = MIRBuilder.getMF();
3776 unsigned PartSize = PartTy.getSizeInBits();
3777 for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
3778 Offset += PartSize, ++Idx) {
3779 unsigned ByteSize = PartSize / 8;
3780 unsigned ByteOffset = Offset / 8;
3781 Register NewAddrReg;
3782
3783 MIRBuilder.materializePtrAdd(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
3784
3785 MachineMemOperand *NewMMO =
3786 MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
3787
3788 if (IsLoad) {
3789 Register Dst = MRI.createGenericVirtualRegister(PartTy);
3790 ValRegs.push_back(Dst);
3791 MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
3792 } else {
3793 MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
3794 }
3795 }
3796
3797 return Offset;
3798 };
3799
3800 unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
3801
3802 // Handle the rest of the register if this isn't an even type breakdown.
3803 if (LeftoverTy.isValid())
3804 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
3805
3806 if (IsLoad) {
3807 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
3808 LeftoverTy, NarrowLeftoverRegs);
3809 }
3810
3811 MI.eraseFromParent();
3812 return Legalized;
3813 }
3814
3815 LegalizerHelper::LegalizeResult
reduceOperationWidth(MachineInstr & MI,unsigned int TypeIdx,LLT NarrowTy)3816 LegalizerHelper::reduceOperationWidth(MachineInstr &MI, unsigned int TypeIdx,
3817 LLT NarrowTy) {
3818 assert(TypeIdx == 0 && "only one type index expected");
3819
3820 const unsigned Opc = MI.getOpcode();
3821 const int NumOps = MI.getNumOperands() - 1;
3822 const Register DstReg = MI.getOperand(0).getReg();
3823 const unsigned Flags = MI.getFlags();
3824 const unsigned NarrowSize = NarrowTy.getSizeInBits();
3825 const LLT NarrowScalarTy = LLT::scalar(NarrowSize);
3826
3827 assert(NumOps <= 3 && "expected instruction with 1 result and 1-3 sources");
3828
3829 // First of all check whether we are narrowing (changing the element type)
3830 // or reducing the vector elements
3831 const LLT DstTy = MRI.getType(DstReg);
3832 const bool IsNarrow = NarrowTy.getScalarType() != DstTy.getScalarType();
3833
3834 SmallVector<Register, 8> ExtractedRegs[3];
3835 SmallVector<Register, 8> Parts;
3836
3837 unsigned NarrowElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
3838
3839 // Break down all the sources into NarrowTy pieces we can operate on. This may
3840 // involve creating merges to a wider type, padded with undef.
3841 for (int I = 0; I != NumOps; ++I) {
3842 Register SrcReg = MI.getOperand(I + 1).getReg();
3843 LLT SrcTy = MRI.getType(SrcReg);
3844
3845 // The type to narrow SrcReg to. For narrowing, this is a smaller scalar.
3846 // For fewerElements, this is a smaller vector with the same element type.
3847 LLT OpNarrowTy;
3848 if (IsNarrow) {
3849 OpNarrowTy = NarrowScalarTy;
3850
3851 // In case of narrowing, we need to cast vectors to scalars for this to
3852 // work properly
3853 // FIXME: Can we do without the bitcast here if we're narrowing?
3854 if (SrcTy.isVector()) {
3855 SrcTy = LLT::scalar(SrcTy.getSizeInBits());
3856 SrcReg = MIRBuilder.buildBitcast(SrcTy, SrcReg).getReg(0);
3857 }
3858 } else {
3859 OpNarrowTy = LLT::scalarOrVector(NarrowElts, SrcTy.getScalarType());
3860 }
3861
3862 LLT GCDTy = extractGCDType(ExtractedRegs[I], SrcTy, OpNarrowTy, SrcReg);
3863
3864 // Build a sequence of NarrowTy pieces in ExtractedRegs for this operand.
3865 buildLCMMergePieces(SrcTy, OpNarrowTy, GCDTy, ExtractedRegs[I],
3866 TargetOpcode::G_ANYEXT);
3867 }
3868
3869 SmallVector<Register, 8> ResultRegs;
3870
3871 // Input operands for each sub-instruction.
3872 SmallVector<SrcOp, 4> InputRegs(NumOps, Register());
3873
3874 int NumParts = ExtractedRegs[0].size();
3875 const unsigned DstSize = DstTy.getSizeInBits();
3876 const LLT DstScalarTy = LLT::scalar(DstSize);
3877
3878 // Narrowing needs to use scalar types
3879 LLT DstLCMTy, NarrowDstTy;
3880 if (IsNarrow) {
3881 DstLCMTy = getLCMType(DstScalarTy, NarrowScalarTy);
3882 NarrowDstTy = NarrowScalarTy;
3883 } else {
3884 DstLCMTy = getLCMType(DstTy, NarrowTy);
3885 NarrowDstTy = NarrowTy;
3886 }
3887
3888 // We widened the source registers to satisfy merge/unmerge size
3889 // constraints. We'll have some extra fully undef parts.
3890 const int NumRealParts = (DstSize + NarrowSize - 1) / NarrowSize;
3891
3892 for (int I = 0; I != NumRealParts; ++I) {
3893 // Emit this instruction on each of the split pieces.
3894 for (int J = 0; J != NumOps; ++J)
3895 InputRegs[J] = ExtractedRegs[J][I];
3896
3897 auto Inst = MIRBuilder.buildInstr(Opc, {NarrowDstTy}, InputRegs, Flags);
3898 ResultRegs.push_back(Inst.getReg(0));
3899 }
3900
3901 // Fill out the widened result with undef instead of creating instructions
3902 // with undef inputs.
3903 int NumUndefParts = NumParts - NumRealParts;
3904 if (NumUndefParts != 0)
3905 ResultRegs.append(NumUndefParts,
3906 MIRBuilder.buildUndef(NarrowDstTy).getReg(0));
3907
3908 // Extract the possibly padded result. Use a scratch register if we need to do
3909 // a final bitcast, otherwise use the original result register.
3910 Register MergeDstReg;
3911 if (IsNarrow && DstTy.isVector())
3912 MergeDstReg = MRI.createGenericVirtualRegister(DstScalarTy);
3913 else
3914 MergeDstReg = DstReg;
3915
3916 buildWidenedRemergeToDst(MergeDstReg, DstLCMTy, ResultRegs);
3917
3918 // Recast to vector if we narrowed a vector
3919 if (IsNarrow && DstTy.isVector())
3920 MIRBuilder.buildBitcast(DstReg, MergeDstReg);
3921
3922 MI.eraseFromParent();
3923 return Legalized;
3924 }
3925
3926 LegalizerHelper::LegalizeResult
fewerElementsVectorSextInReg(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3927 LegalizerHelper::fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx,
3928 LLT NarrowTy) {
3929 Register DstReg = MI.getOperand(0).getReg();
3930 Register SrcReg = MI.getOperand(1).getReg();
3931 int64_t Imm = MI.getOperand(2).getImm();
3932
3933 LLT DstTy = MRI.getType(DstReg);
3934
3935 SmallVector<Register, 8> Parts;
3936 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
3937 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts);
3938
3939 for (Register &R : Parts)
3940 R = MIRBuilder.buildSExtInReg(NarrowTy, R, Imm).getReg(0);
3941
3942 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
3943
3944 MI.eraseFromParent();
3945 return Legalized;
3946 }
3947
3948 LegalizerHelper::LegalizeResult
fewerElementsVector(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)3949 LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
3950 LLT NarrowTy) {
3951 using namespace TargetOpcode;
3952
3953 switch (MI.getOpcode()) {
3954 case G_IMPLICIT_DEF:
3955 return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
3956 case G_TRUNC:
3957 case G_AND:
3958 case G_OR:
3959 case G_XOR:
3960 case G_ADD:
3961 case G_SUB:
3962 case G_MUL:
3963 case G_PTR_ADD:
3964 case G_SMULH:
3965 case G_UMULH:
3966 case G_FADD:
3967 case G_FMUL:
3968 case G_FSUB:
3969 case G_FNEG:
3970 case G_FABS:
3971 case G_FCANONICALIZE:
3972 case G_FDIV:
3973 case G_FREM:
3974 case G_FMA:
3975 case G_FMAD:
3976 case G_FPOW:
3977 case G_FEXP:
3978 case G_FEXP2:
3979 case G_FLOG:
3980 case G_FLOG2:
3981 case G_FLOG10:
3982 case G_FNEARBYINT:
3983 case G_FCEIL:
3984 case G_FFLOOR:
3985 case G_FRINT:
3986 case G_INTRINSIC_ROUND:
3987 case G_INTRINSIC_ROUNDEVEN:
3988 case G_INTRINSIC_TRUNC:
3989 case G_FCOS:
3990 case G_FSIN:
3991 case G_FSQRT:
3992 case G_BSWAP:
3993 case G_BITREVERSE:
3994 case G_SDIV:
3995 case G_UDIV:
3996 case G_SREM:
3997 case G_UREM:
3998 case G_SMIN:
3999 case G_SMAX:
4000 case G_UMIN:
4001 case G_UMAX:
4002 case G_FMINNUM:
4003 case G_FMAXNUM:
4004 case G_FMINNUM_IEEE:
4005 case G_FMAXNUM_IEEE:
4006 case G_FMINIMUM:
4007 case G_FMAXIMUM:
4008 case G_FSHL:
4009 case G_FSHR:
4010 case G_FREEZE:
4011 case G_SADDSAT:
4012 case G_SSUBSAT:
4013 case G_UADDSAT:
4014 case G_USUBSAT:
4015 return reduceOperationWidth(MI, TypeIdx, NarrowTy);
4016 case G_SHL:
4017 case G_LSHR:
4018 case G_ASHR:
4019 case G_SSHLSAT:
4020 case G_USHLSAT:
4021 case G_CTLZ:
4022 case G_CTLZ_ZERO_UNDEF:
4023 case G_CTTZ:
4024 case G_CTTZ_ZERO_UNDEF:
4025 case G_CTPOP:
4026 case G_FCOPYSIGN:
4027 return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
4028 case G_ZEXT:
4029 case G_SEXT:
4030 case G_ANYEXT:
4031 case G_FPEXT:
4032 case G_FPTRUNC:
4033 case G_SITOFP:
4034 case G_UITOFP:
4035 case G_FPTOSI:
4036 case G_FPTOUI:
4037 case G_INTTOPTR:
4038 case G_PTRTOINT:
4039 case G_ADDRSPACE_CAST:
4040 return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
4041 case G_ICMP:
4042 case G_FCMP:
4043 return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
4044 case G_SELECT:
4045 return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
4046 case G_PHI:
4047 return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
4048 case G_UNMERGE_VALUES:
4049 return fewerElementsVectorUnmergeValues(MI, TypeIdx, NarrowTy);
4050 case G_BUILD_VECTOR:
4051 assert(TypeIdx == 0 && "not a vector type index");
4052 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
4053 case G_CONCAT_VECTORS:
4054 if (TypeIdx != 1) // TODO: This probably does work as expected already.
4055 return UnableToLegalize;
4056 return fewerElementsVectorMerge(MI, TypeIdx, NarrowTy);
4057 case G_EXTRACT_VECTOR_ELT:
4058 case G_INSERT_VECTOR_ELT:
4059 return fewerElementsVectorExtractInsertVectorElt(MI, TypeIdx, NarrowTy);
4060 case G_LOAD:
4061 case G_STORE:
4062 return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
4063 case G_SEXT_INREG:
4064 return fewerElementsVectorSextInReg(MI, TypeIdx, NarrowTy);
4065 default:
4066 return UnableToLegalize;
4067 }
4068 }
4069
4070 LegalizerHelper::LegalizeResult
narrowScalarShiftByConstant(MachineInstr & MI,const APInt & Amt,const LLT HalfTy,const LLT AmtTy)4071 LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
4072 const LLT HalfTy, const LLT AmtTy) {
4073
4074 Register InL = MRI.createGenericVirtualRegister(HalfTy);
4075 Register InH = MRI.createGenericVirtualRegister(HalfTy);
4076 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
4077
4078 if (Amt.isNullValue()) {
4079 MIRBuilder.buildMerge(MI.getOperand(0), {InL, InH});
4080 MI.eraseFromParent();
4081 return Legalized;
4082 }
4083
4084 LLT NVT = HalfTy;
4085 unsigned NVTBits = HalfTy.getSizeInBits();
4086 unsigned VTBits = 2 * NVTBits;
4087
4088 SrcOp Lo(Register(0)), Hi(Register(0));
4089 if (MI.getOpcode() == TargetOpcode::G_SHL) {
4090 if (Amt.ugt(VTBits)) {
4091 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
4092 } else if (Amt.ugt(NVTBits)) {
4093 Lo = MIRBuilder.buildConstant(NVT, 0);
4094 Hi = MIRBuilder.buildShl(NVT, InL,
4095 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4096 } else if (Amt == NVTBits) {
4097 Lo = MIRBuilder.buildConstant(NVT, 0);
4098 Hi = InL;
4099 } else {
4100 Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
4101 auto OrLHS =
4102 MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
4103 auto OrRHS = MIRBuilder.buildLShr(
4104 NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4105 Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4106 }
4107 } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
4108 if (Amt.ugt(VTBits)) {
4109 Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
4110 } else if (Amt.ugt(NVTBits)) {
4111 Lo = MIRBuilder.buildLShr(NVT, InH,
4112 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4113 Hi = MIRBuilder.buildConstant(NVT, 0);
4114 } else if (Amt == NVTBits) {
4115 Lo = InH;
4116 Hi = MIRBuilder.buildConstant(NVT, 0);
4117 } else {
4118 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
4119
4120 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
4121 auto OrRHS = MIRBuilder.buildShl(
4122 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4123
4124 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4125 Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
4126 }
4127 } else {
4128 if (Amt.ugt(VTBits)) {
4129 Hi = Lo = MIRBuilder.buildAShr(
4130 NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4131 } else if (Amt.ugt(NVTBits)) {
4132 Lo = MIRBuilder.buildAShr(NVT, InH,
4133 MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
4134 Hi = MIRBuilder.buildAShr(NVT, InH,
4135 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4136 } else if (Amt == NVTBits) {
4137 Lo = InH;
4138 Hi = MIRBuilder.buildAShr(NVT, InH,
4139 MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
4140 } else {
4141 auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
4142
4143 auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
4144 auto OrRHS = MIRBuilder.buildShl(
4145 NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
4146
4147 Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
4148 Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
4149 }
4150 }
4151
4152 MIRBuilder.buildMerge(MI.getOperand(0), {Lo, Hi});
4153 MI.eraseFromParent();
4154
4155 return Legalized;
4156 }
4157
4158 // TODO: Optimize if constant shift amount.
4159 LegalizerHelper::LegalizeResult
narrowScalarShift(MachineInstr & MI,unsigned TypeIdx,LLT RequestedTy)4160 LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
4161 LLT RequestedTy) {
4162 if (TypeIdx == 1) {
4163 Observer.changingInstr(MI);
4164 narrowScalarSrc(MI, RequestedTy, 2);
4165 Observer.changedInstr(MI);
4166 return Legalized;
4167 }
4168
4169 Register DstReg = MI.getOperand(0).getReg();
4170 LLT DstTy = MRI.getType(DstReg);
4171 if (DstTy.isVector())
4172 return UnableToLegalize;
4173
4174 Register Amt = MI.getOperand(2).getReg();
4175 LLT ShiftAmtTy = MRI.getType(Amt);
4176 const unsigned DstEltSize = DstTy.getScalarSizeInBits();
4177 if (DstEltSize % 2 != 0)
4178 return UnableToLegalize;
4179
4180 // Ignore the input type. We can only go to exactly half the size of the
4181 // input. If that isn't small enough, the resulting pieces will be further
4182 // legalized.
4183 const unsigned NewBitSize = DstEltSize / 2;
4184 const LLT HalfTy = LLT::scalar(NewBitSize);
4185 const LLT CondTy = LLT::scalar(1);
4186
4187 if (const MachineInstr *KShiftAmt =
4188 getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
4189 return narrowScalarShiftByConstant(
4190 MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
4191 }
4192
4193 // TODO: Expand with known bits.
4194
4195 // Handle the fully general expansion by an unknown amount.
4196 auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
4197
4198 Register InL = MRI.createGenericVirtualRegister(HalfTy);
4199 Register InH = MRI.createGenericVirtualRegister(HalfTy);
4200 MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1));
4201
4202 auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
4203 auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
4204
4205 auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
4206 auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
4207 auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
4208
4209 Register ResultRegs[2];
4210 switch (MI.getOpcode()) {
4211 case TargetOpcode::G_SHL: {
4212 // Short: ShAmt < NewBitSize
4213 auto LoS = MIRBuilder.buildShl(HalfTy, InL, Amt);
4214
4215 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
4216 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, Amt);
4217 auto HiS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
4218
4219 // Long: ShAmt >= NewBitSize
4220 auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
4221 auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
4222
4223 auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
4224 auto Hi = MIRBuilder.buildSelect(
4225 HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
4226
4227 ResultRegs[0] = Lo.getReg(0);
4228 ResultRegs[1] = Hi.getReg(0);
4229 break;
4230 }
4231 case TargetOpcode::G_LSHR:
4232 case TargetOpcode::G_ASHR: {
4233 // Short: ShAmt < NewBitSize
4234 auto HiS = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy}, {InH, Amt});
4235
4236 auto LoOr = MIRBuilder.buildLShr(HalfTy, InL, Amt);
4237 auto HiOr = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
4238 auto LoS = MIRBuilder.buildOr(HalfTy, LoOr, HiOr);
4239
4240 // Long: ShAmt >= NewBitSize
4241 MachineInstrBuilder HiL;
4242 if (MI.getOpcode() == TargetOpcode::G_LSHR) {
4243 HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
4244 } else {
4245 auto ShiftAmt = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1);
4246 HiL = MIRBuilder.buildAShr(HalfTy, InH, ShiftAmt); // Sign of Hi part.
4247 }
4248 auto LoL = MIRBuilder.buildInstr(MI.getOpcode(), {HalfTy},
4249 {InH, AmtExcess}); // Lo from Hi part.
4250
4251 auto Lo = MIRBuilder.buildSelect(
4252 HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
4253
4254 auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
4255
4256 ResultRegs[0] = Lo.getReg(0);
4257 ResultRegs[1] = Hi.getReg(0);
4258 break;
4259 }
4260 default:
4261 llvm_unreachable("not a shift");
4262 }
4263
4264 MIRBuilder.buildMerge(DstReg, ResultRegs);
4265 MI.eraseFromParent();
4266 return Legalized;
4267 }
4268
4269 LegalizerHelper::LegalizeResult
moreElementsVectorPhi(MachineInstr & MI,unsigned TypeIdx,LLT MoreTy)4270 LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
4271 LLT MoreTy) {
4272 assert(TypeIdx == 0 && "Expecting only Idx 0");
4273
4274 Observer.changingInstr(MI);
4275 for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4276 MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
4277 MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
4278 moreElementsVectorSrc(MI, MoreTy, I);
4279 }
4280
4281 MachineBasicBlock &MBB = *MI.getParent();
4282 MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
4283 moreElementsVectorDst(MI, MoreTy, 0);
4284 Observer.changedInstr(MI);
4285 return Legalized;
4286 }
4287
4288 LegalizerHelper::LegalizeResult
moreElementsVector(MachineInstr & MI,unsigned TypeIdx,LLT MoreTy)4289 LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
4290 LLT MoreTy) {
4291 unsigned Opc = MI.getOpcode();
4292 switch (Opc) {
4293 case TargetOpcode::G_IMPLICIT_DEF:
4294 case TargetOpcode::G_LOAD: {
4295 if (TypeIdx != 0)
4296 return UnableToLegalize;
4297 Observer.changingInstr(MI);
4298 moreElementsVectorDst(MI, MoreTy, 0);
4299 Observer.changedInstr(MI);
4300 return Legalized;
4301 }
4302 case TargetOpcode::G_STORE:
4303 if (TypeIdx != 0)
4304 return UnableToLegalize;
4305 Observer.changingInstr(MI);
4306 moreElementsVectorSrc(MI, MoreTy, 0);
4307 Observer.changedInstr(MI);
4308 return Legalized;
4309 case TargetOpcode::G_AND:
4310 case TargetOpcode::G_OR:
4311 case TargetOpcode::G_XOR:
4312 case TargetOpcode::G_SMIN:
4313 case TargetOpcode::G_SMAX:
4314 case TargetOpcode::G_UMIN:
4315 case TargetOpcode::G_UMAX:
4316 case TargetOpcode::G_FMINNUM:
4317 case TargetOpcode::G_FMAXNUM:
4318 case TargetOpcode::G_FMINNUM_IEEE:
4319 case TargetOpcode::G_FMAXNUM_IEEE:
4320 case TargetOpcode::G_FMINIMUM:
4321 case TargetOpcode::G_FMAXIMUM: {
4322 Observer.changingInstr(MI);
4323 moreElementsVectorSrc(MI, MoreTy, 1);
4324 moreElementsVectorSrc(MI, MoreTy, 2);
4325 moreElementsVectorDst(MI, MoreTy, 0);
4326 Observer.changedInstr(MI);
4327 return Legalized;
4328 }
4329 case TargetOpcode::G_EXTRACT:
4330 if (TypeIdx != 1)
4331 return UnableToLegalize;
4332 Observer.changingInstr(MI);
4333 moreElementsVectorSrc(MI, MoreTy, 1);
4334 Observer.changedInstr(MI);
4335 return Legalized;
4336 case TargetOpcode::G_INSERT:
4337 case TargetOpcode::G_FREEZE:
4338 if (TypeIdx != 0)
4339 return UnableToLegalize;
4340 Observer.changingInstr(MI);
4341 moreElementsVectorSrc(MI, MoreTy, 1);
4342 moreElementsVectorDst(MI, MoreTy, 0);
4343 Observer.changedInstr(MI);
4344 return Legalized;
4345 case TargetOpcode::G_SELECT:
4346 if (TypeIdx != 0)
4347 return UnableToLegalize;
4348 if (MRI.getType(MI.getOperand(1).getReg()).isVector())
4349 return UnableToLegalize;
4350
4351 Observer.changingInstr(MI);
4352 moreElementsVectorSrc(MI, MoreTy, 2);
4353 moreElementsVectorSrc(MI, MoreTy, 3);
4354 moreElementsVectorDst(MI, MoreTy, 0);
4355 Observer.changedInstr(MI);
4356 return Legalized;
4357 case TargetOpcode::G_UNMERGE_VALUES: {
4358 if (TypeIdx != 1)
4359 return UnableToLegalize;
4360
4361 LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
4362 int NumDst = MI.getNumOperands() - 1;
4363 moreElementsVectorSrc(MI, MoreTy, NumDst);
4364
4365 auto MIB = MIRBuilder.buildInstr(TargetOpcode::G_UNMERGE_VALUES);
4366 for (int I = 0; I != NumDst; ++I)
4367 MIB.addDef(MI.getOperand(I).getReg());
4368
4369 int NewNumDst = MoreTy.getSizeInBits() / DstTy.getSizeInBits();
4370 for (int I = NumDst; I != NewNumDst; ++I)
4371 MIB.addDef(MRI.createGenericVirtualRegister(DstTy));
4372
4373 MIB.addUse(MI.getOperand(NumDst).getReg());
4374 MI.eraseFromParent();
4375 return Legalized;
4376 }
4377 case TargetOpcode::G_PHI:
4378 return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
4379 default:
4380 return UnableToLegalize;
4381 }
4382 }
4383
multiplyRegisters(SmallVectorImpl<Register> & DstRegs,ArrayRef<Register> Src1Regs,ArrayRef<Register> Src2Regs,LLT NarrowTy)4384 void LegalizerHelper::multiplyRegisters(SmallVectorImpl<Register> &DstRegs,
4385 ArrayRef<Register> Src1Regs,
4386 ArrayRef<Register> Src2Regs,
4387 LLT NarrowTy) {
4388 MachineIRBuilder &B = MIRBuilder;
4389 unsigned SrcParts = Src1Regs.size();
4390 unsigned DstParts = DstRegs.size();
4391
4392 unsigned DstIdx = 0; // Low bits of the result.
4393 Register FactorSum =
4394 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
4395 DstRegs[DstIdx] = FactorSum;
4396
4397 unsigned CarrySumPrevDstIdx;
4398 SmallVector<Register, 4> Factors;
4399
4400 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
4401 // Collect low parts of muls for DstIdx.
4402 for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
4403 i <= std::min(DstIdx, SrcParts - 1); ++i) {
4404 MachineInstrBuilder Mul =
4405 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
4406 Factors.push_back(Mul.getReg(0));
4407 }
4408 // Collect high parts of muls from previous DstIdx.
4409 for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
4410 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
4411 MachineInstrBuilder Umulh =
4412 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
4413 Factors.push_back(Umulh.getReg(0));
4414 }
4415 // Add CarrySum from additions calculated for previous DstIdx.
4416 if (DstIdx != 1) {
4417 Factors.push_back(CarrySumPrevDstIdx);
4418 }
4419
4420 Register CarrySum;
4421 // Add all factors and accumulate all carries into CarrySum.
4422 if (DstIdx != DstParts - 1) {
4423 MachineInstrBuilder Uaddo =
4424 B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
4425 FactorSum = Uaddo.getReg(0);
4426 CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
4427 for (unsigned i = 2; i < Factors.size(); ++i) {
4428 MachineInstrBuilder Uaddo =
4429 B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
4430 FactorSum = Uaddo.getReg(0);
4431 MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
4432 CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
4433 }
4434 } else {
4435 // Since value for the next index is not calculated, neither is CarrySum.
4436 FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
4437 for (unsigned i = 2; i < Factors.size(); ++i)
4438 FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
4439 }
4440
4441 CarrySumPrevDstIdx = CarrySum;
4442 DstRegs[DstIdx] = FactorSum;
4443 Factors.clear();
4444 }
4445 }
4446
4447 LegalizerHelper::LegalizeResult
narrowScalarMul(MachineInstr & MI,LLT NarrowTy)4448 LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
4449 Register DstReg = MI.getOperand(0).getReg();
4450 Register Src1 = MI.getOperand(1).getReg();
4451 Register Src2 = MI.getOperand(2).getReg();
4452
4453 LLT Ty = MRI.getType(DstReg);
4454 if (Ty.isVector())
4455 return UnableToLegalize;
4456
4457 unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
4458 unsigned DstSize = Ty.getSizeInBits();
4459 unsigned NarrowSize = NarrowTy.getSizeInBits();
4460 if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
4461 return UnableToLegalize;
4462
4463 unsigned NumDstParts = DstSize / NarrowSize;
4464 unsigned NumSrcParts = SrcSize / NarrowSize;
4465 bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
4466 unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
4467
4468 SmallVector<Register, 2> Src1Parts, Src2Parts;
4469 SmallVector<Register, 2> DstTmpRegs(DstTmpParts);
4470 extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
4471 extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
4472 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
4473
4474 // Take only high half of registers if this is high mul.
4475 ArrayRef<Register> DstRegs(
4476 IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
4477 MIRBuilder.buildMerge(DstReg, DstRegs);
4478 MI.eraseFromParent();
4479 return Legalized;
4480 }
4481
4482 LegalizerHelper::LegalizeResult
narrowScalarExtract(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)4483 LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
4484 LLT NarrowTy) {
4485 if (TypeIdx != 1)
4486 return UnableToLegalize;
4487
4488 uint64_t NarrowSize = NarrowTy.getSizeInBits();
4489
4490 int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
4491 // FIXME: add support for when SizeOp1 isn't an exact multiple of
4492 // NarrowSize.
4493 if (SizeOp1 % NarrowSize != 0)
4494 return UnableToLegalize;
4495 int NumParts = SizeOp1 / NarrowSize;
4496
4497 SmallVector<Register, 2> SrcRegs, DstRegs;
4498 SmallVector<uint64_t, 2> Indexes;
4499 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
4500
4501 Register OpReg = MI.getOperand(0).getReg();
4502 uint64_t OpStart = MI.getOperand(2).getImm();
4503 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
4504 for (int i = 0; i < NumParts; ++i) {
4505 unsigned SrcStart = i * NarrowSize;
4506
4507 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
4508 // No part of the extract uses this subregister, ignore it.
4509 continue;
4510 } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
4511 // The entire subregister is extracted, forward the value.
4512 DstRegs.push_back(SrcRegs[i]);
4513 continue;
4514 }
4515
4516 // OpSegStart is where this destination segment would start in OpReg if it
4517 // extended infinitely in both directions.
4518 int64_t ExtractOffset;
4519 uint64_t SegSize;
4520 if (OpStart < SrcStart) {
4521 ExtractOffset = 0;
4522 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
4523 } else {
4524 ExtractOffset = OpStart - SrcStart;
4525 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
4526 }
4527
4528 Register SegReg = SrcRegs[i];
4529 if (ExtractOffset != 0 || SegSize != NarrowSize) {
4530 // A genuine extract is needed.
4531 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
4532 MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
4533 }
4534
4535 DstRegs.push_back(SegReg);
4536 }
4537
4538 Register DstReg = MI.getOperand(0).getReg();
4539 if (MRI.getType(DstReg).isVector())
4540 MIRBuilder.buildBuildVector(DstReg, DstRegs);
4541 else if (DstRegs.size() > 1)
4542 MIRBuilder.buildMerge(DstReg, DstRegs);
4543 else
4544 MIRBuilder.buildCopy(DstReg, DstRegs[0]);
4545 MI.eraseFromParent();
4546 return Legalized;
4547 }
4548
4549 LegalizerHelper::LegalizeResult
narrowScalarInsert(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)4550 LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
4551 LLT NarrowTy) {
4552 // FIXME: Don't know how to handle secondary types yet.
4553 if (TypeIdx != 0)
4554 return UnableToLegalize;
4555
4556 uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
4557 uint64_t NarrowSize = NarrowTy.getSizeInBits();
4558
4559 // FIXME: add support for when SizeOp0 isn't an exact multiple of
4560 // NarrowSize.
4561 if (SizeOp0 % NarrowSize != 0)
4562 return UnableToLegalize;
4563
4564 int NumParts = SizeOp0 / NarrowSize;
4565
4566 SmallVector<Register, 2> SrcRegs, DstRegs;
4567 SmallVector<uint64_t, 2> Indexes;
4568 extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
4569
4570 Register OpReg = MI.getOperand(2).getReg();
4571 uint64_t OpStart = MI.getOperand(3).getImm();
4572 uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
4573 for (int i = 0; i < NumParts; ++i) {
4574 unsigned DstStart = i * NarrowSize;
4575
4576 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
4577 // No part of the insert affects this subregister, forward the original.
4578 DstRegs.push_back(SrcRegs[i]);
4579 continue;
4580 } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
4581 // The entire subregister is defined by this insert, forward the new
4582 // value.
4583 DstRegs.push_back(OpReg);
4584 continue;
4585 }
4586
4587 // OpSegStart is where this destination segment would start in OpReg if it
4588 // extended infinitely in both directions.
4589 int64_t ExtractOffset, InsertOffset;
4590 uint64_t SegSize;
4591 if (OpStart < DstStart) {
4592 InsertOffset = 0;
4593 ExtractOffset = DstStart - OpStart;
4594 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
4595 } else {
4596 InsertOffset = OpStart - DstStart;
4597 ExtractOffset = 0;
4598 SegSize =
4599 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
4600 }
4601
4602 Register SegReg = OpReg;
4603 if (ExtractOffset != 0 || SegSize != OpSize) {
4604 // A genuine extract is needed.
4605 SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
4606 MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
4607 }
4608
4609 Register DstReg = MRI.createGenericVirtualRegister(NarrowTy);
4610 MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
4611 DstRegs.push_back(DstReg);
4612 }
4613
4614 assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
4615 Register DstReg = MI.getOperand(0).getReg();
4616 if(MRI.getType(DstReg).isVector())
4617 MIRBuilder.buildBuildVector(DstReg, DstRegs);
4618 else
4619 MIRBuilder.buildMerge(DstReg, DstRegs);
4620 MI.eraseFromParent();
4621 return Legalized;
4622 }
4623
4624 LegalizerHelper::LegalizeResult
narrowScalarBasic(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)4625 LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx,
4626 LLT NarrowTy) {
4627 Register DstReg = MI.getOperand(0).getReg();
4628 LLT DstTy = MRI.getType(DstReg);
4629
4630 assert(MI.getNumOperands() == 3 && TypeIdx == 0);
4631
4632 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
4633 SmallVector<Register, 4> Src0Regs, Src0LeftoverRegs;
4634 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
4635 LLT LeftoverTy;
4636 if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
4637 Src0Regs, Src0LeftoverRegs))
4638 return UnableToLegalize;
4639
4640 LLT Unused;
4641 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
4642 Src1Regs, Src1LeftoverRegs))
4643 llvm_unreachable("inconsistent extractParts result");
4644
4645 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
4646 auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy},
4647 {Src0Regs[I], Src1Regs[I]});
4648 DstRegs.push_back(Inst.getReg(0));
4649 }
4650
4651 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
4652 auto Inst = MIRBuilder.buildInstr(
4653 MI.getOpcode(),
4654 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
4655 DstLeftoverRegs.push_back(Inst.getReg(0));
4656 }
4657
4658 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
4659 LeftoverTy, DstLeftoverRegs);
4660
4661 MI.eraseFromParent();
4662 return Legalized;
4663 }
4664
4665 LegalizerHelper::LegalizeResult
narrowScalarExt(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)4666 LegalizerHelper::narrowScalarExt(MachineInstr &MI, unsigned TypeIdx,
4667 LLT NarrowTy) {
4668 if (TypeIdx != 0)
4669 return UnableToLegalize;
4670
4671 Register DstReg = MI.getOperand(0).getReg();
4672 Register SrcReg = MI.getOperand(1).getReg();
4673
4674 LLT DstTy = MRI.getType(DstReg);
4675 if (DstTy.isVector())
4676 return UnableToLegalize;
4677
4678 SmallVector<Register, 8> Parts;
4679 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
4680 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts, MI.getOpcode());
4681 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
4682
4683 MI.eraseFromParent();
4684 return Legalized;
4685 }
4686
4687 LegalizerHelper::LegalizeResult
narrowScalarSelect(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)4688 LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
4689 LLT NarrowTy) {
4690 if (TypeIdx != 0)
4691 return UnableToLegalize;
4692
4693 Register CondReg = MI.getOperand(1).getReg();
4694 LLT CondTy = MRI.getType(CondReg);
4695 if (CondTy.isVector()) // TODO: Handle vselect
4696 return UnableToLegalize;
4697
4698 Register DstReg = MI.getOperand(0).getReg();
4699 LLT DstTy = MRI.getType(DstReg);
4700
4701 SmallVector<Register, 4> DstRegs, DstLeftoverRegs;
4702 SmallVector<Register, 4> Src1Regs, Src1LeftoverRegs;
4703 SmallVector<Register, 4> Src2Regs, Src2LeftoverRegs;
4704 LLT LeftoverTy;
4705 if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
4706 Src1Regs, Src1LeftoverRegs))
4707 return UnableToLegalize;
4708
4709 LLT Unused;
4710 if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
4711 Src2Regs, Src2LeftoverRegs))
4712 llvm_unreachable("inconsistent extractParts result");
4713
4714 for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
4715 auto Select = MIRBuilder.buildSelect(NarrowTy,
4716 CondReg, Src1Regs[I], Src2Regs[I]);
4717 DstRegs.push_back(Select.getReg(0));
4718 }
4719
4720 for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
4721 auto Select = MIRBuilder.buildSelect(
4722 LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
4723 DstLeftoverRegs.push_back(Select.getReg(0));
4724 }
4725
4726 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
4727 LeftoverTy, DstLeftoverRegs);
4728
4729 MI.eraseFromParent();
4730 return Legalized;
4731 }
4732
4733 LegalizerHelper::LegalizeResult
narrowScalarCTLZ(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)4734 LegalizerHelper::narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx,
4735 LLT NarrowTy) {
4736 if (TypeIdx != 1)
4737 return UnableToLegalize;
4738
4739 Register DstReg = MI.getOperand(0).getReg();
4740 Register SrcReg = MI.getOperand(1).getReg();
4741 LLT DstTy = MRI.getType(DstReg);
4742 LLT SrcTy = MRI.getType(SrcReg);
4743 unsigned NarrowSize = NarrowTy.getSizeInBits();
4744
4745 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
4746 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF;
4747
4748 MachineIRBuilder &B = MIRBuilder;
4749 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg);
4750 // ctlz(Hi:Lo) -> Hi == 0 ? (NarrowSize + ctlz(Lo)) : ctlz(Hi)
4751 auto C_0 = B.buildConstant(NarrowTy, 0);
4752 auto HiIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
4753 UnmergeSrc.getReg(1), C_0);
4754 auto LoCTLZ = IsUndef ?
4755 B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) :
4756 B.buildCTLZ(DstTy, UnmergeSrc.getReg(0));
4757 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize);
4758 auto HiIsZeroCTLZ = B.buildAdd(DstTy, LoCTLZ, C_NarrowSize);
4759 auto HiCTLZ = B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1));
4760 B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ);
4761
4762 MI.eraseFromParent();
4763 return Legalized;
4764 }
4765
4766 return UnableToLegalize;
4767 }
4768
4769 LegalizerHelper::LegalizeResult
narrowScalarCTTZ(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)4770 LegalizerHelper::narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx,
4771 LLT NarrowTy) {
4772 if (TypeIdx != 1)
4773 return UnableToLegalize;
4774
4775 Register DstReg = MI.getOperand(0).getReg();
4776 Register SrcReg = MI.getOperand(1).getReg();
4777 LLT DstTy = MRI.getType(DstReg);
4778 LLT SrcTy = MRI.getType(SrcReg);
4779 unsigned NarrowSize = NarrowTy.getSizeInBits();
4780
4781 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
4782 const bool IsUndef = MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF;
4783
4784 MachineIRBuilder &B = MIRBuilder;
4785 auto UnmergeSrc = B.buildUnmerge(NarrowTy, SrcReg);
4786 // cttz(Hi:Lo) -> Lo == 0 ? (cttz(Hi) + NarrowSize) : cttz(Lo)
4787 auto C_0 = B.buildConstant(NarrowTy, 0);
4788 auto LoIsZero = B.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
4789 UnmergeSrc.getReg(0), C_0);
4790 auto HiCTTZ = IsUndef ?
4791 B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) :
4792 B.buildCTTZ(DstTy, UnmergeSrc.getReg(1));
4793 auto C_NarrowSize = B.buildConstant(DstTy, NarrowSize);
4794 auto LoIsZeroCTTZ = B.buildAdd(DstTy, HiCTTZ, C_NarrowSize);
4795 auto LoCTTZ = B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0));
4796 B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ);
4797
4798 MI.eraseFromParent();
4799 return Legalized;
4800 }
4801
4802 return UnableToLegalize;
4803 }
4804
4805 LegalizerHelper::LegalizeResult
narrowScalarCTPOP(MachineInstr & MI,unsigned TypeIdx,LLT NarrowTy)4806 LegalizerHelper::narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx,
4807 LLT NarrowTy) {
4808 if (TypeIdx != 1)
4809 return UnableToLegalize;
4810
4811 Register DstReg = MI.getOperand(0).getReg();
4812 LLT DstTy = MRI.getType(DstReg);
4813 LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
4814 unsigned NarrowSize = NarrowTy.getSizeInBits();
4815
4816 if (SrcTy.isScalar() && SrcTy.getSizeInBits() == 2 * NarrowSize) {
4817 auto UnmergeSrc = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
4818
4819 auto LoCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(0));
4820 auto HiCTPOP = MIRBuilder.buildCTPOP(DstTy, UnmergeSrc.getReg(1));
4821 MIRBuilder.buildAdd(DstReg, HiCTPOP, LoCTPOP);
4822
4823 MI.eraseFromParent();
4824 return Legalized;
4825 }
4826
4827 return UnableToLegalize;
4828 }
4829
4830 LegalizerHelper::LegalizeResult
lowerBitCount(MachineInstr & MI)4831 LegalizerHelper::lowerBitCount(MachineInstr &MI) {
4832 unsigned Opc = MI.getOpcode();
4833 const auto &TII = MIRBuilder.getTII();
4834 auto isSupported = [this](const LegalityQuery &Q) {
4835 auto QAction = LI.getAction(Q).Action;
4836 return QAction == Legal || QAction == Libcall || QAction == Custom;
4837 };
4838 switch (Opc) {
4839 default:
4840 return UnableToLegalize;
4841 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
4842 // This trivially expands to CTLZ.
4843 Observer.changingInstr(MI);
4844 MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
4845 Observer.changedInstr(MI);
4846 return Legalized;
4847 }
4848 case TargetOpcode::G_CTLZ: {
4849 Register DstReg = MI.getOperand(0).getReg();
4850 Register SrcReg = MI.getOperand(1).getReg();
4851 LLT DstTy = MRI.getType(DstReg);
4852 LLT SrcTy = MRI.getType(SrcReg);
4853 unsigned Len = SrcTy.getSizeInBits();
4854
4855 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
4856 // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
4857 auto CtlzZU = MIRBuilder.buildCTLZ_ZERO_UNDEF(DstTy, SrcReg);
4858 auto ZeroSrc = MIRBuilder.buildConstant(SrcTy, 0);
4859 auto ICmp = MIRBuilder.buildICmp(
4860 CmpInst::ICMP_EQ, SrcTy.changeElementSize(1), SrcReg, ZeroSrc);
4861 auto LenConst = MIRBuilder.buildConstant(DstTy, Len);
4862 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CtlzZU);
4863 MI.eraseFromParent();
4864 return Legalized;
4865 }
4866 // for now, we do this:
4867 // NewLen = NextPowerOf2(Len);
4868 // x = x | (x >> 1);
4869 // x = x | (x >> 2);
4870 // ...
4871 // x = x | (x >>16);
4872 // x = x | (x >>32); // for 64-bit input
4873 // Upto NewLen/2
4874 // return Len - popcount(x);
4875 //
4876 // Ref: "Hacker's Delight" by Henry Warren
4877 Register Op = SrcReg;
4878 unsigned NewLen = PowerOf2Ceil(Len);
4879 for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
4880 auto MIBShiftAmt = MIRBuilder.buildConstant(SrcTy, 1ULL << i);
4881 auto MIBOp = MIRBuilder.buildOr(
4882 SrcTy, Op, MIRBuilder.buildLShr(SrcTy, Op, MIBShiftAmt));
4883 Op = MIBOp.getReg(0);
4884 }
4885 auto MIBPop = MIRBuilder.buildCTPOP(DstTy, Op);
4886 MIRBuilder.buildSub(MI.getOperand(0), MIRBuilder.buildConstant(DstTy, Len),
4887 MIBPop);
4888 MI.eraseFromParent();
4889 return Legalized;
4890 }
4891 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
4892 // This trivially expands to CTTZ.
4893 Observer.changingInstr(MI);
4894 MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
4895 Observer.changedInstr(MI);
4896 return Legalized;
4897 }
4898 case TargetOpcode::G_CTTZ: {
4899 Register DstReg = MI.getOperand(0).getReg();
4900 Register SrcReg = MI.getOperand(1).getReg();
4901 LLT DstTy = MRI.getType(DstReg);
4902 LLT SrcTy = MRI.getType(SrcReg);
4903
4904 unsigned Len = SrcTy.getSizeInBits();
4905 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
4906 // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
4907 // zero.
4908 auto CttzZU = MIRBuilder.buildCTTZ_ZERO_UNDEF(DstTy, SrcReg);
4909 auto Zero = MIRBuilder.buildConstant(SrcTy, 0);
4910 auto ICmp = MIRBuilder.buildICmp(
4911 CmpInst::ICMP_EQ, DstTy.changeElementSize(1), SrcReg, Zero);
4912 auto LenConst = MIRBuilder.buildConstant(DstTy, Len);
4913 MIRBuilder.buildSelect(DstReg, ICmp, LenConst, CttzZU);
4914 MI.eraseFromParent();
4915 return Legalized;
4916 }
4917 // for now, we use: { return popcount(~x & (x - 1)); }
4918 // unless the target has ctlz but not ctpop, in which case we use:
4919 // { return 32 - nlz(~x & (x-1)); }
4920 // Ref: "Hacker's Delight" by Henry Warren
4921 auto MIBCstNeg1 = MIRBuilder.buildConstant(SrcTy, -1);
4922 auto MIBNot = MIRBuilder.buildXor(SrcTy, SrcReg, MIBCstNeg1);
4923 auto MIBTmp = MIRBuilder.buildAnd(
4924 SrcTy, MIBNot, MIRBuilder.buildAdd(SrcTy, SrcReg, MIBCstNeg1));
4925 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
4926 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
4927 auto MIBCstLen = MIRBuilder.buildConstant(SrcTy, Len);
4928 MIRBuilder.buildSub(MI.getOperand(0), MIBCstLen,
4929 MIRBuilder.buildCTLZ(SrcTy, MIBTmp));
4930 MI.eraseFromParent();
4931 return Legalized;
4932 }
4933 MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
4934 MI.getOperand(1).setReg(MIBTmp.getReg(0));
4935 return Legalized;
4936 }
4937 case TargetOpcode::G_CTPOP: {
4938 Register SrcReg = MI.getOperand(1).getReg();
4939 LLT Ty = MRI.getType(SrcReg);
4940 unsigned Size = Ty.getSizeInBits();
4941 MachineIRBuilder &B = MIRBuilder;
4942
4943 // Count set bits in blocks of 2 bits. Default approach would be
4944 // B2Count = { val & 0x55555555 } + { (val >> 1) & 0x55555555 }
4945 // We use following formula instead:
4946 // B2Count = val - { (val >> 1) & 0x55555555 }
4947 // since it gives same result in blocks of 2 with one instruction less.
4948 auto C_1 = B.buildConstant(Ty, 1);
4949 auto B2Set1LoTo1Hi = B.buildLShr(Ty, SrcReg, C_1);
4950 APInt B2Mask1HiTo0 = APInt::getSplat(Size, APInt(8, 0x55));
4951 auto C_B2Mask1HiTo0 = B.buildConstant(Ty, B2Mask1HiTo0);
4952 auto B2Count1Hi = B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0);
4953 auto B2Count = B.buildSub(Ty, SrcReg, B2Count1Hi);
4954
4955 // In order to get count in blocks of 4 add values from adjacent block of 2.
4956 // B4Count = { B2Count & 0x33333333 } + { (B2Count >> 2) & 0x33333333 }
4957 auto C_2 = B.buildConstant(Ty, 2);
4958 auto B4Set2LoTo2Hi = B.buildLShr(Ty, B2Count, C_2);
4959 APInt B4Mask2HiTo0 = APInt::getSplat(Size, APInt(8, 0x33));
4960 auto C_B4Mask2HiTo0 = B.buildConstant(Ty, B4Mask2HiTo0);
4961 auto B4HiB2Count = B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0);
4962 auto B4LoB2Count = B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0);
4963 auto B4Count = B.buildAdd(Ty, B4HiB2Count, B4LoB2Count);
4964
4965 // For count in blocks of 8 bits we don't have to mask high 4 bits before
4966 // addition since count value sits in range {0,...,8} and 4 bits are enough
4967 // to hold such binary values. After addition high 4 bits still hold count
4968 // of set bits in high 4 bit block, set them to zero and get 8 bit result.
4969 // B8Count = { B4Count + (B4Count >> 4) } & 0x0F0F0F0F
4970 auto C_4 = B.buildConstant(Ty, 4);
4971 auto B8HiB4Count = B.buildLShr(Ty, B4Count, C_4);
4972 auto B8CountDirty4Hi = B.buildAdd(Ty, B8HiB4Count, B4Count);
4973 APInt B8Mask4HiTo0 = APInt::getSplat(Size, APInt(8, 0x0F));
4974 auto C_B8Mask4HiTo0 = B.buildConstant(Ty, B8Mask4HiTo0);
4975 auto B8Count = B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0);
4976
4977 assert(Size<=128 && "Scalar size is too large for CTPOP lower algorithm");
4978 // 8 bits can hold CTPOP result of 128 bit int or smaller. Mul with this
4979 // bitmask will set 8 msb in ResTmp to sum of all B8Counts in 8 bit blocks.
4980 auto MulMask = B.buildConstant(Ty, APInt::getSplat(Size, APInt(8, 0x01)));
4981 auto ResTmp = B.buildMul(Ty, B8Count, MulMask);
4982
4983 // Shift count result from 8 high bits to low bits.
4984 auto C_SizeM8 = B.buildConstant(Ty, Size - 8);
4985 B.buildLShr(MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
4986
4987 MI.eraseFromParent();
4988 return Legalized;
4989 }
4990 }
4991 }
4992
4993 // Expand s32 = G_UITOFP s64 using bit operations to an IEEE float
4994 // representation.
4995 LegalizerHelper::LegalizeResult
lowerU64ToF32BitOps(MachineInstr & MI)4996 LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) {
4997 Register Dst = MI.getOperand(0).getReg();
4998 Register Src = MI.getOperand(1).getReg();
4999 const LLT S64 = LLT::scalar(64);
5000 const LLT S32 = LLT::scalar(32);
5001 const LLT S1 = LLT::scalar(1);
5002
5003 assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32);
5004
5005 // unsigned cul2f(ulong u) {
5006 // uint lz = clz(u);
5007 // uint e = (u != 0) ? 127U + 63U - lz : 0;
5008 // u = (u << lz) & 0x7fffffffffffffffUL;
5009 // ulong t = u & 0xffffffffffUL;
5010 // uint v = (e << 23) | (uint)(u >> 40);
5011 // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U);
5012 // return as_float(v + r);
5013 // }
5014
5015 auto Zero32 = MIRBuilder.buildConstant(S32, 0);
5016 auto Zero64 = MIRBuilder.buildConstant(S64, 0);
5017
5018 auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src);
5019
5020 auto K = MIRBuilder.buildConstant(S32, 127U + 63U);
5021 auto Sub = MIRBuilder.buildSub(S32, K, LZ);
5022
5023 auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64);
5024 auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32);
5025
5026 auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1);
5027 auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ);
5028
5029 auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0);
5030
5031 auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL);
5032 auto T = MIRBuilder.buildAnd(S64, U, Mask1);
5033
5034 auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40));
5035 auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23));
5036 auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl));
5037
5038 auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL);
5039 auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C);
5040 auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C);
5041 auto One = MIRBuilder.buildConstant(S32, 1);
5042
5043 auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One);
5044 auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32);
5045 auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0);
5046 MIRBuilder.buildAdd(Dst, V, R);
5047
5048 MI.eraseFromParent();
5049 return Legalized;
5050 }
5051
lowerUITOFP(MachineInstr & MI)5052 LegalizerHelper::LegalizeResult LegalizerHelper::lowerUITOFP(MachineInstr &MI) {
5053 Register Dst = MI.getOperand(0).getReg();
5054 Register Src = MI.getOperand(1).getReg();
5055 LLT DstTy = MRI.getType(Dst);
5056 LLT SrcTy = MRI.getType(Src);
5057
5058 if (SrcTy == LLT::scalar(1)) {
5059 auto True = MIRBuilder.buildFConstant(DstTy, 1.0);
5060 auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
5061 MIRBuilder.buildSelect(Dst, Src, True, False);
5062 MI.eraseFromParent();
5063 return Legalized;
5064 }
5065
5066 if (SrcTy != LLT::scalar(64))
5067 return UnableToLegalize;
5068
5069 if (DstTy == LLT::scalar(32)) {
5070 // TODO: SelectionDAG has several alternative expansions to port which may
5071 // be more reasonble depending on the available instructions. If a target
5072 // has sitofp, does not have CTLZ, or can efficiently use f64 as an
5073 // intermediate type, this is probably worse.
5074 return lowerU64ToF32BitOps(MI);
5075 }
5076
5077 return UnableToLegalize;
5078 }
5079
lowerSITOFP(MachineInstr & MI)5080 LegalizerHelper::LegalizeResult LegalizerHelper::lowerSITOFP(MachineInstr &MI) {
5081 Register Dst = MI.getOperand(0).getReg();
5082 Register Src = MI.getOperand(1).getReg();
5083 LLT DstTy = MRI.getType(Dst);
5084 LLT SrcTy = MRI.getType(Src);
5085
5086 const LLT S64 = LLT::scalar(64);
5087 const LLT S32 = LLT::scalar(32);
5088 const LLT S1 = LLT::scalar(1);
5089
5090 if (SrcTy == S1) {
5091 auto True = MIRBuilder.buildFConstant(DstTy, -1.0);
5092 auto False = MIRBuilder.buildFConstant(DstTy, 0.0);
5093 MIRBuilder.buildSelect(Dst, Src, True, False);
5094 MI.eraseFromParent();
5095 return Legalized;
5096 }
5097
5098 if (SrcTy != S64)
5099 return UnableToLegalize;
5100
5101 if (DstTy == S32) {
5102 // signed cl2f(long l) {
5103 // long s = l >> 63;
5104 // float r = cul2f((l + s) ^ s);
5105 // return s ? -r : r;
5106 // }
5107 Register L = Src;
5108 auto SignBit = MIRBuilder.buildConstant(S64, 63);
5109 auto S = MIRBuilder.buildAShr(S64, L, SignBit);
5110
5111 auto LPlusS = MIRBuilder.buildAdd(S64, L, S);
5112 auto Xor = MIRBuilder.buildXor(S64, LPlusS, S);
5113 auto R = MIRBuilder.buildUITOFP(S32, Xor);
5114
5115 auto RNeg = MIRBuilder.buildFNeg(S32, R);
5116 auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S,
5117 MIRBuilder.buildConstant(S64, 0));
5118 MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R);
5119 MI.eraseFromParent();
5120 return Legalized;
5121 }
5122
5123 return UnableToLegalize;
5124 }
5125
lowerFPTOUI(MachineInstr & MI)5126 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOUI(MachineInstr &MI) {
5127 Register Dst = MI.getOperand(0).getReg();
5128 Register Src = MI.getOperand(1).getReg();
5129 LLT DstTy = MRI.getType(Dst);
5130 LLT SrcTy = MRI.getType(Src);
5131 const LLT S64 = LLT::scalar(64);
5132 const LLT S32 = LLT::scalar(32);
5133
5134 if (SrcTy != S64 && SrcTy != S32)
5135 return UnableToLegalize;
5136 if (DstTy != S32 && DstTy != S64)
5137 return UnableToLegalize;
5138
5139 // FPTOSI gives same result as FPTOUI for positive signed integers.
5140 // FPTOUI needs to deal with fp values that convert to unsigned integers
5141 // greater or equal to 2^31 for float or 2^63 for double. For brevity 2^Exp.
5142
5143 APInt TwoPExpInt = APInt::getSignMask(DstTy.getSizeInBits());
5144 APFloat TwoPExpFP(SrcTy.getSizeInBits() == 32 ? APFloat::IEEEsingle()
5145 : APFloat::IEEEdouble(),
5146 APInt::getNullValue(SrcTy.getSizeInBits()));
5147 TwoPExpFP.convertFromAPInt(TwoPExpInt, false, APFloat::rmNearestTiesToEven);
5148
5149 MachineInstrBuilder FPTOSI = MIRBuilder.buildFPTOSI(DstTy, Src);
5150
5151 MachineInstrBuilder Threshold = MIRBuilder.buildFConstant(SrcTy, TwoPExpFP);
5152 // For fp Value greater or equal to Threshold(2^Exp), we use FPTOSI on
5153 // (Value - 2^Exp) and add 2^Exp by setting highest bit in result to 1.
5154 MachineInstrBuilder FSub = MIRBuilder.buildFSub(SrcTy, Src, Threshold);
5155 MachineInstrBuilder ResLowBits = MIRBuilder.buildFPTOSI(DstTy, FSub);
5156 MachineInstrBuilder ResHighBit = MIRBuilder.buildConstant(DstTy, TwoPExpInt);
5157 MachineInstrBuilder Res = MIRBuilder.buildXor(DstTy, ResLowBits, ResHighBit);
5158
5159 const LLT S1 = LLT::scalar(1);
5160
5161 MachineInstrBuilder FCMP =
5162 MIRBuilder.buildFCmp(CmpInst::FCMP_ULT, S1, Src, Threshold);
5163 MIRBuilder.buildSelect(Dst, FCMP, FPTOSI, Res);
5164
5165 MI.eraseFromParent();
5166 return Legalized;
5167 }
5168
lowerFPTOSI(MachineInstr & MI)5169 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPTOSI(MachineInstr &MI) {
5170 Register Dst = MI.getOperand(0).getReg();
5171 Register Src = MI.getOperand(1).getReg();
5172 LLT DstTy = MRI.getType(Dst);
5173 LLT SrcTy = MRI.getType(Src);
5174 const LLT S64 = LLT::scalar(64);
5175 const LLT S32 = LLT::scalar(32);
5176
5177 // FIXME: Only f32 to i64 conversions are supported.
5178 if (SrcTy.getScalarType() != S32 || DstTy.getScalarType() != S64)
5179 return UnableToLegalize;
5180
5181 // Expand f32 -> i64 conversion
5182 // This algorithm comes from compiler-rt's implementation of fixsfdi:
5183 // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c
5184
5185 unsigned SrcEltBits = SrcTy.getScalarSizeInBits();
5186
5187 auto ExponentMask = MIRBuilder.buildConstant(SrcTy, 0x7F800000);
5188 auto ExponentLoBit = MIRBuilder.buildConstant(SrcTy, 23);
5189
5190 auto AndExpMask = MIRBuilder.buildAnd(SrcTy, Src, ExponentMask);
5191 auto ExponentBits = MIRBuilder.buildLShr(SrcTy, AndExpMask, ExponentLoBit);
5192
5193 auto SignMask = MIRBuilder.buildConstant(SrcTy,
5194 APInt::getSignMask(SrcEltBits));
5195 auto AndSignMask = MIRBuilder.buildAnd(SrcTy, Src, SignMask);
5196 auto SignLowBit = MIRBuilder.buildConstant(SrcTy, SrcEltBits - 1);
5197 auto Sign = MIRBuilder.buildAShr(SrcTy, AndSignMask, SignLowBit);
5198 Sign = MIRBuilder.buildSExt(DstTy, Sign);
5199
5200 auto MantissaMask = MIRBuilder.buildConstant(SrcTy, 0x007FFFFF);
5201 auto AndMantissaMask = MIRBuilder.buildAnd(SrcTy, Src, MantissaMask);
5202 auto K = MIRBuilder.buildConstant(SrcTy, 0x00800000);
5203
5204 auto R = MIRBuilder.buildOr(SrcTy, AndMantissaMask, K);
5205 R = MIRBuilder.buildZExt(DstTy, R);
5206
5207 auto Bias = MIRBuilder.buildConstant(SrcTy, 127);
5208 auto Exponent = MIRBuilder.buildSub(SrcTy, ExponentBits, Bias);
5209 auto SubExponent = MIRBuilder.buildSub(SrcTy, Exponent, ExponentLoBit);
5210 auto ExponentSub = MIRBuilder.buildSub(SrcTy, ExponentLoBit, Exponent);
5211
5212 auto Shl = MIRBuilder.buildShl(DstTy, R, SubExponent);
5213 auto Srl = MIRBuilder.buildLShr(DstTy, R, ExponentSub);
5214
5215 const LLT S1 = LLT::scalar(1);
5216 auto CmpGt = MIRBuilder.buildICmp(CmpInst::ICMP_SGT,
5217 S1, Exponent, ExponentLoBit);
5218
5219 R = MIRBuilder.buildSelect(DstTy, CmpGt, Shl, Srl);
5220
5221 auto XorSign = MIRBuilder.buildXor(DstTy, R, Sign);
5222 auto Ret = MIRBuilder.buildSub(DstTy, XorSign, Sign);
5223
5224 auto ZeroSrcTy = MIRBuilder.buildConstant(SrcTy, 0);
5225
5226 auto ExponentLt0 = MIRBuilder.buildICmp(CmpInst::ICMP_SLT,
5227 S1, Exponent, ZeroSrcTy);
5228
5229 auto ZeroDstTy = MIRBuilder.buildConstant(DstTy, 0);
5230 MIRBuilder.buildSelect(Dst, ExponentLt0, ZeroDstTy, Ret);
5231
5232 MI.eraseFromParent();
5233 return Legalized;
5234 }
5235
5236 // f64 -> f16 conversion using round-to-nearest-even rounding mode.
5237 LegalizerHelper::LegalizeResult
lowerFPTRUNC_F64_TO_F16(MachineInstr & MI)5238 LegalizerHelper::lowerFPTRUNC_F64_TO_F16(MachineInstr &MI) {
5239 Register Dst = MI.getOperand(0).getReg();
5240 Register Src = MI.getOperand(1).getReg();
5241
5242 if (MRI.getType(Src).isVector()) // TODO: Handle vectors directly.
5243 return UnableToLegalize;
5244
5245 const unsigned ExpMask = 0x7ff;
5246 const unsigned ExpBiasf64 = 1023;
5247 const unsigned ExpBiasf16 = 15;
5248 const LLT S32 = LLT::scalar(32);
5249 const LLT S1 = LLT::scalar(1);
5250
5251 auto Unmerge = MIRBuilder.buildUnmerge(S32, Src);
5252 Register U = Unmerge.getReg(0);
5253 Register UH = Unmerge.getReg(1);
5254
5255 auto E = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 20));
5256 E = MIRBuilder.buildAnd(S32, E, MIRBuilder.buildConstant(S32, ExpMask));
5257
5258 // Subtract the fp64 exponent bias (1023) to get the real exponent and
5259 // add the f16 bias (15) to get the biased exponent for the f16 format.
5260 E = MIRBuilder.buildAdd(
5261 S32, E, MIRBuilder.buildConstant(S32, -ExpBiasf64 + ExpBiasf16));
5262
5263 auto M = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 8));
5264 M = MIRBuilder.buildAnd(S32, M, MIRBuilder.buildConstant(S32, 0xffe));
5265
5266 auto MaskedSig = MIRBuilder.buildAnd(S32, UH,
5267 MIRBuilder.buildConstant(S32, 0x1ff));
5268 MaskedSig = MIRBuilder.buildOr(S32, MaskedSig, U);
5269
5270 auto Zero = MIRBuilder.buildConstant(S32, 0);
5271 auto SigCmpNE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, MaskedSig, Zero);
5272 auto Lo40Set = MIRBuilder.buildZExt(S32, SigCmpNE0);
5273 M = MIRBuilder.buildOr(S32, M, Lo40Set);
5274
5275 // (M != 0 ? 0x0200 : 0) | 0x7c00;
5276 auto Bits0x200 = MIRBuilder.buildConstant(S32, 0x0200);
5277 auto CmpM_NE0 = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, M, Zero);
5278 auto SelectCC = MIRBuilder.buildSelect(S32, CmpM_NE0, Bits0x200, Zero);
5279
5280 auto Bits0x7c00 = MIRBuilder.buildConstant(S32, 0x7c00);
5281 auto I = MIRBuilder.buildOr(S32, SelectCC, Bits0x7c00);
5282
5283 // N = M | (E << 12);
5284 auto EShl12 = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 12));
5285 auto N = MIRBuilder.buildOr(S32, M, EShl12);
5286
5287 // B = clamp(1-E, 0, 13);
5288 auto One = MIRBuilder.buildConstant(S32, 1);
5289 auto OneSubExp = MIRBuilder.buildSub(S32, One, E);
5290 auto B = MIRBuilder.buildSMax(S32, OneSubExp, Zero);
5291 B = MIRBuilder.buildSMin(S32, B, MIRBuilder.buildConstant(S32, 13));
5292
5293 auto SigSetHigh = MIRBuilder.buildOr(S32, M,
5294 MIRBuilder.buildConstant(S32, 0x1000));
5295
5296 auto D = MIRBuilder.buildLShr(S32, SigSetHigh, B);
5297 auto D0 = MIRBuilder.buildShl(S32, D, B);
5298
5299 auto D0_NE_SigSetHigh = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1,
5300 D0, SigSetHigh);
5301 auto D1 = MIRBuilder.buildZExt(S32, D0_NE_SigSetHigh);
5302 D = MIRBuilder.buildOr(S32, D, D1);
5303
5304 auto CmpELtOne = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, S1, E, One);
5305 auto V = MIRBuilder.buildSelect(S32, CmpELtOne, D, N);
5306
5307 auto VLow3 = MIRBuilder.buildAnd(S32, V, MIRBuilder.buildConstant(S32, 7));
5308 V = MIRBuilder.buildLShr(S32, V, MIRBuilder.buildConstant(S32, 2));
5309
5310 auto VLow3Eq3 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, VLow3,
5311 MIRBuilder.buildConstant(S32, 3));
5312 auto V0 = MIRBuilder.buildZExt(S32, VLow3Eq3);
5313
5314 auto VLow3Gt5 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1, VLow3,
5315 MIRBuilder.buildConstant(S32, 5));
5316 auto V1 = MIRBuilder.buildZExt(S32, VLow3Gt5);
5317
5318 V1 = MIRBuilder.buildOr(S32, V0, V1);
5319 V = MIRBuilder.buildAdd(S32, V, V1);
5320
5321 auto CmpEGt30 = MIRBuilder.buildICmp(CmpInst::ICMP_SGT, S1,
5322 E, MIRBuilder.buildConstant(S32, 30));
5323 V = MIRBuilder.buildSelect(S32, CmpEGt30,
5324 MIRBuilder.buildConstant(S32, 0x7c00), V);
5325
5326 auto CmpEGt1039 = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1,
5327 E, MIRBuilder.buildConstant(S32, 1039));
5328 V = MIRBuilder.buildSelect(S32, CmpEGt1039, I, V);
5329
5330 // Extract the sign bit.
5331 auto Sign = MIRBuilder.buildLShr(S32, UH, MIRBuilder.buildConstant(S32, 16));
5332 Sign = MIRBuilder.buildAnd(S32, Sign, MIRBuilder.buildConstant(S32, 0x8000));
5333
5334 // Insert the sign bit
5335 V = MIRBuilder.buildOr(S32, Sign, V);
5336
5337 MIRBuilder.buildTrunc(Dst, V);
5338 MI.eraseFromParent();
5339 return Legalized;
5340 }
5341
5342 LegalizerHelper::LegalizeResult
lowerFPTRUNC(MachineInstr & MI)5343 LegalizerHelper::lowerFPTRUNC(MachineInstr &MI) {
5344 Register Dst = MI.getOperand(0).getReg();
5345 Register Src = MI.getOperand(1).getReg();
5346
5347 LLT DstTy = MRI.getType(Dst);
5348 LLT SrcTy = MRI.getType(Src);
5349 const LLT S64 = LLT::scalar(64);
5350 const LLT S16 = LLT::scalar(16);
5351
5352 if (DstTy.getScalarType() == S16 && SrcTy.getScalarType() == S64)
5353 return lowerFPTRUNC_F64_TO_F16(MI);
5354
5355 return UnableToLegalize;
5356 }
5357
5358 // TODO: If RHS is a constant SelectionDAGBuilder expands this into a
5359 // multiplication tree.
lowerFPOWI(MachineInstr & MI)5360 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFPOWI(MachineInstr &MI) {
5361 Register Dst = MI.getOperand(0).getReg();
5362 Register Src0 = MI.getOperand(1).getReg();
5363 Register Src1 = MI.getOperand(2).getReg();
5364 LLT Ty = MRI.getType(Dst);
5365
5366 auto CvtSrc1 = MIRBuilder.buildSITOFP(Ty, Src1);
5367 MIRBuilder.buildFPow(Dst, Src0, CvtSrc1, MI.getFlags());
5368 MI.eraseFromParent();
5369 return Legalized;
5370 }
5371
minMaxToCompare(unsigned Opc)5372 static CmpInst::Predicate minMaxToCompare(unsigned Opc) {
5373 switch (Opc) {
5374 case TargetOpcode::G_SMIN:
5375 return CmpInst::ICMP_SLT;
5376 case TargetOpcode::G_SMAX:
5377 return CmpInst::ICMP_SGT;
5378 case TargetOpcode::G_UMIN:
5379 return CmpInst::ICMP_ULT;
5380 case TargetOpcode::G_UMAX:
5381 return CmpInst::ICMP_UGT;
5382 default:
5383 llvm_unreachable("not in integer min/max");
5384 }
5385 }
5386
lowerMinMax(MachineInstr & MI)5387 LegalizerHelper::LegalizeResult LegalizerHelper::lowerMinMax(MachineInstr &MI) {
5388 Register Dst = MI.getOperand(0).getReg();
5389 Register Src0 = MI.getOperand(1).getReg();
5390 Register Src1 = MI.getOperand(2).getReg();
5391
5392 const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode());
5393 LLT CmpType = MRI.getType(Dst).changeElementSize(1);
5394
5395 auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1);
5396 MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1);
5397
5398 MI.eraseFromParent();
5399 return Legalized;
5400 }
5401
5402 LegalizerHelper::LegalizeResult
lowerFCopySign(MachineInstr & MI)5403 LegalizerHelper::lowerFCopySign(MachineInstr &MI) {
5404 Register Dst = MI.getOperand(0).getReg();
5405 Register Src0 = MI.getOperand(1).getReg();
5406 Register Src1 = MI.getOperand(2).getReg();
5407
5408 const LLT Src0Ty = MRI.getType(Src0);
5409 const LLT Src1Ty = MRI.getType(Src1);
5410
5411 const int Src0Size = Src0Ty.getScalarSizeInBits();
5412 const int Src1Size = Src1Ty.getScalarSizeInBits();
5413
5414 auto SignBitMask = MIRBuilder.buildConstant(
5415 Src0Ty, APInt::getSignMask(Src0Size));
5416
5417 auto NotSignBitMask = MIRBuilder.buildConstant(
5418 Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1));
5419
5420 auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask);
5421 MachineInstr *Or;
5422
5423 if (Src0Ty == Src1Ty) {
5424 auto And1 = MIRBuilder.buildAnd(Src1Ty, Src1, SignBitMask);
5425 Or = MIRBuilder.buildOr(Dst, And0, And1);
5426 } else if (Src0Size > Src1Size) {
5427 auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size);
5428 auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1);
5429 auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt);
5430 auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask);
5431 Or = MIRBuilder.buildOr(Dst, And0, And1);
5432 } else {
5433 auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size);
5434 auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt);
5435 auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift);
5436 auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask);
5437 Or = MIRBuilder.buildOr(Dst, And0, And1);
5438 }
5439
5440 // Be careful about setting nsz/nnan/ninf on every instruction, since the
5441 // constants are a nan and -0.0, but the final result should preserve
5442 // everything.
5443 if (unsigned Flags = MI.getFlags())
5444 Or->setFlags(Flags);
5445
5446 MI.eraseFromParent();
5447 return Legalized;
5448 }
5449
5450 LegalizerHelper::LegalizeResult
lowerFMinNumMaxNum(MachineInstr & MI)5451 LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) {
5452 unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ?
5453 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
5454
5455 Register Dst = MI.getOperand(0).getReg();
5456 Register Src0 = MI.getOperand(1).getReg();
5457 Register Src1 = MI.getOperand(2).getReg();
5458 LLT Ty = MRI.getType(Dst);
5459
5460 if (!MI.getFlag(MachineInstr::FmNoNans)) {
5461 // Insert canonicalizes if it's possible we need to quiet to get correct
5462 // sNaN behavior.
5463
5464 // Note this must be done here, and not as an optimization combine in the
5465 // absence of a dedicate quiet-snan instruction as we're using an
5466 // omni-purpose G_FCANONICALIZE.
5467 if (!isKnownNeverSNaN(Src0, MRI))
5468 Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0);
5469
5470 if (!isKnownNeverSNaN(Src1, MRI))
5471 Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0);
5472 }
5473
5474 // If there are no nans, it's safe to simply replace this with the non-IEEE
5475 // version.
5476 MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags());
5477 MI.eraseFromParent();
5478 return Legalized;
5479 }
5480
lowerFMad(MachineInstr & MI)5481 LegalizerHelper::LegalizeResult LegalizerHelper::lowerFMad(MachineInstr &MI) {
5482 // Expand G_FMAD a, b, c -> G_FADD (G_FMUL a, b), c
5483 Register DstReg = MI.getOperand(0).getReg();
5484 LLT Ty = MRI.getType(DstReg);
5485 unsigned Flags = MI.getFlags();
5486
5487 auto Mul = MIRBuilder.buildFMul(Ty, MI.getOperand(1), MI.getOperand(2),
5488 Flags);
5489 MIRBuilder.buildFAdd(DstReg, Mul, MI.getOperand(3), Flags);
5490 MI.eraseFromParent();
5491 return Legalized;
5492 }
5493
5494 LegalizerHelper::LegalizeResult
lowerIntrinsicRound(MachineInstr & MI)5495 LegalizerHelper::lowerIntrinsicRound(MachineInstr &MI) {
5496 Register DstReg = MI.getOperand(0).getReg();
5497 Register X = MI.getOperand(1).getReg();
5498 const unsigned Flags = MI.getFlags();
5499 const LLT Ty = MRI.getType(DstReg);
5500 const LLT CondTy = Ty.changeElementSize(1);
5501
5502 // round(x) =>
5503 // t = trunc(x);
5504 // d = fabs(x - t);
5505 // o = copysign(1.0f, x);
5506 // return t + (d >= 0.5 ? o : 0.0);
5507
5508 auto T = MIRBuilder.buildIntrinsicTrunc(Ty, X, Flags);
5509
5510 auto Diff = MIRBuilder.buildFSub(Ty, X, T, Flags);
5511 auto AbsDiff = MIRBuilder.buildFAbs(Ty, Diff, Flags);
5512 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0);
5513 auto One = MIRBuilder.buildFConstant(Ty, 1.0);
5514 auto Half = MIRBuilder.buildFConstant(Ty, 0.5);
5515 auto SignOne = MIRBuilder.buildFCopysign(Ty, One, X);
5516
5517 auto Cmp = MIRBuilder.buildFCmp(CmpInst::FCMP_OGE, CondTy, AbsDiff, Half,
5518 Flags);
5519 auto Sel = MIRBuilder.buildSelect(Ty, Cmp, SignOne, Zero, Flags);
5520
5521 MIRBuilder.buildFAdd(DstReg, T, Sel, Flags);
5522
5523 MI.eraseFromParent();
5524 return Legalized;
5525 }
5526
5527 LegalizerHelper::LegalizeResult
lowerFFloor(MachineInstr & MI)5528 LegalizerHelper::lowerFFloor(MachineInstr &MI) {
5529 Register DstReg = MI.getOperand(0).getReg();
5530 Register SrcReg = MI.getOperand(1).getReg();
5531 unsigned Flags = MI.getFlags();
5532 LLT Ty = MRI.getType(DstReg);
5533 const LLT CondTy = Ty.changeElementSize(1);
5534
5535 // result = trunc(src);
5536 // if (src < 0.0 && src != result)
5537 // result += -1.0.
5538
5539 auto Trunc = MIRBuilder.buildIntrinsicTrunc(Ty, SrcReg, Flags);
5540 auto Zero = MIRBuilder.buildFConstant(Ty, 0.0);
5541
5542 auto Lt0 = MIRBuilder.buildFCmp(CmpInst::FCMP_OLT, CondTy,
5543 SrcReg, Zero, Flags);
5544 auto NeTrunc = MIRBuilder.buildFCmp(CmpInst::FCMP_ONE, CondTy,
5545 SrcReg, Trunc, Flags);
5546 auto And = MIRBuilder.buildAnd(CondTy, Lt0, NeTrunc);
5547 auto AddVal = MIRBuilder.buildSITOFP(Ty, And);
5548
5549 MIRBuilder.buildFAdd(DstReg, Trunc, AddVal, Flags);
5550 MI.eraseFromParent();
5551 return Legalized;
5552 }
5553
5554 LegalizerHelper::LegalizeResult
lowerMergeValues(MachineInstr & MI)5555 LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
5556 const unsigned NumOps = MI.getNumOperands();
5557 Register DstReg = MI.getOperand(0).getReg();
5558 Register Src0Reg = MI.getOperand(1).getReg();
5559 LLT DstTy = MRI.getType(DstReg);
5560 LLT SrcTy = MRI.getType(Src0Reg);
5561 unsigned PartSize = SrcTy.getSizeInBits();
5562
5563 LLT WideTy = LLT::scalar(DstTy.getSizeInBits());
5564 Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0);
5565
5566 for (unsigned I = 2; I != NumOps; ++I) {
5567 const unsigned Offset = (I - 1) * PartSize;
5568
5569 Register SrcReg = MI.getOperand(I).getReg();
5570 auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg);
5571
5572 Register NextResult = I + 1 == NumOps && WideTy == DstTy ? DstReg :
5573 MRI.createGenericVirtualRegister(WideTy);
5574
5575 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset);
5576 auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt);
5577 MIRBuilder.buildOr(NextResult, ResultReg, Shl);
5578 ResultReg = NextResult;
5579 }
5580
5581 if (DstTy.isPointer()) {
5582 if (MIRBuilder.getDataLayout().isNonIntegralAddressSpace(
5583 DstTy.getAddressSpace())) {
5584 LLVM_DEBUG(dbgs() << "Not casting nonintegral address space\n");
5585 return UnableToLegalize;
5586 }
5587
5588 MIRBuilder.buildIntToPtr(DstReg, ResultReg);
5589 }
5590
5591 MI.eraseFromParent();
5592 return Legalized;
5593 }
5594
5595 LegalizerHelper::LegalizeResult
lowerUnmergeValues(MachineInstr & MI)5596 LegalizerHelper::lowerUnmergeValues(MachineInstr &MI) {
5597 const unsigned NumDst = MI.getNumOperands() - 1;
5598 Register SrcReg = MI.getOperand(NumDst).getReg();
5599 Register Dst0Reg = MI.getOperand(0).getReg();
5600 LLT DstTy = MRI.getType(Dst0Reg);
5601 if (DstTy.isPointer())
5602 return UnableToLegalize; // TODO
5603
5604 SrcReg = coerceToScalar(SrcReg);
5605 if (!SrcReg)
5606 return UnableToLegalize;
5607
5608 // Expand scalarizing unmerge as bitcast to integer and shift.
5609 LLT IntTy = MRI.getType(SrcReg);
5610
5611 MIRBuilder.buildTrunc(Dst0Reg, SrcReg);
5612
5613 const unsigned DstSize = DstTy.getSizeInBits();
5614 unsigned Offset = DstSize;
5615 for (unsigned I = 1; I != NumDst; ++I, Offset += DstSize) {
5616 auto ShiftAmt = MIRBuilder.buildConstant(IntTy, Offset);
5617 auto Shift = MIRBuilder.buildLShr(IntTy, SrcReg, ShiftAmt);
5618 MIRBuilder.buildTrunc(MI.getOperand(I), Shift);
5619 }
5620
5621 MI.eraseFromParent();
5622 return Legalized;
5623 }
5624
5625 /// Lower a vector extract or insert by writing the vector to a stack temporary
5626 /// and reloading the element or vector.
5627 ///
5628 /// %dst = G_EXTRACT_VECTOR_ELT %vec, %idx
5629 /// =>
5630 /// %stack_temp = G_FRAME_INDEX
5631 /// G_STORE %vec, %stack_temp
5632 /// %idx = clamp(%idx, %vec.getNumElements())
5633 /// %element_ptr = G_PTR_ADD %stack_temp, %idx
5634 /// %dst = G_LOAD %element_ptr
5635 LegalizerHelper::LegalizeResult
lowerExtractInsertVectorElt(MachineInstr & MI)5636 LegalizerHelper::lowerExtractInsertVectorElt(MachineInstr &MI) {
5637 Register DstReg = MI.getOperand(0).getReg();
5638 Register SrcVec = MI.getOperand(1).getReg();
5639 Register InsertVal;
5640 if (MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
5641 InsertVal = MI.getOperand(2).getReg();
5642
5643 Register Idx = MI.getOperand(MI.getNumOperands() - 1).getReg();
5644
5645 LLT VecTy = MRI.getType(SrcVec);
5646 LLT EltTy = VecTy.getElementType();
5647 if (!EltTy.isByteSized()) { // Not implemented.
5648 LLVM_DEBUG(dbgs() << "Can't handle non-byte element vectors yet\n");
5649 return UnableToLegalize;
5650 }
5651
5652 unsigned EltBytes = EltTy.getSizeInBytes();
5653 Align VecAlign = getStackTemporaryAlignment(VecTy);
5654 Align EltAlign;
5655
5656 MachinePointerInfo PtrInfo;
5657 auto StackTemp = createStackTemporary(TypeSize::Fixed(VecTy.getSizeInBytes()),
5658 VecAlign, PtrInfo);
5659 MIRBuilder.buildStore(SrcVec, StackTemp, PtrInfo, VecAlign);
5660
5661 // Get the pointer to the element, and be sure not to hit undefined behavior
5662 // if the index is out of bounds.
5663 Register EltPtr = getVectorElementPointer(StackTemp.getReg(0), VecTy, Idx);
5664
5665 int64_t IdxVal;
5666 if (mi_match(Idx, MRI, m_ICst(IdxVal))) {
5667 int64_t Offset = IdxVal * EltBytes;
5668 PtrInfo = PtrInfo.getWithOffset(Offset);
5669 EltAlign = commonAlignment(VecAlign, Offset);
5670 } else {
5671 // We lose information with a variable offset.
5672 EltAlign = getStackTemporaryAlignment(EltTy);
5673 PtrInfo = MachinePointerInfo(MRI.getType(EltPtr).getAddressSpace());
5674 }
5675
5676 if (InsertVal) {
5677 // Write the inserted element
5678 MIRBuilder.buildStore(InsertVal, EltPtr, PtrInfo, EltAlign);
5679
5680 // Reload the whole vector.
5681 MIRBuilder.buildLoad(DstReg, StackTemp, PtrInfo, VecAlign);
5682 } else {
5683 MIRBuilder.buildLoad(DstReg, EltPtr, PtrInfo, EltAlign);
5684 }
5685
5686 MI.eraseFromParent();
5687 return Legalized;
5688 }
5689
5690 LegalizerHelper::LegalizeResult
lowerShuffleVector(MachineInstr & MI)5691 LegalizerHelper::lowerShuffleVector(MachineInstr &MI) {
5692 Register DstReg = MI.getOperand(0).getReg();
5693 Register Src0Reg = MI.getOperand(1).getReg();
5694 Register Src1Reg = MI.getOperand(2).getReg();
5695 LLT Src0Ty = MRI.getType(Src0Reg);
5696 LLT DstTy = MRI.getType(DstReg);
5697 LLT IdxTy = LLT::scalar(32);
5698
5699 ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask();
5700
5701 if (DstTy.isScalar()) {
5702 if (Src0Ty.isVector())
5703 return UnableToLegalize;
5704
5705 // This is just a SELECT.
5706 assert(Mask.size() == 1 && "Expected a single mask element");
5707 Register Val;
5708 if (Mask[0] < 0 || Mask[0] > 1)
5709 Val = MIRBuilder.buildUndef(DstTy).getReg(0);
5710 else
5711 Val = Mask[0] == 0 ? Src0Reg : Src1Reg;
5712 MIRBuilder.buildCopy(DstReg, Val);
5713 MI.eraseFromParent();
5714 return Legalized;
5715 }
5716
5717 Register Undef;
5718 SmallVector<Register, 32> BuildVec;
5719 LLT EltTy = DstTy.getElementType();
5720
5721 for (int Idx : Mask) {
5722 if (Idx < 0) {
5723 if (!Undef.isValid())
5724 Undef = MIRBuilder.buildUndef(EltTy).getReg(0);
5725 BuildVec.push_back(Undef);
5726 continue;
5727 }
5728
5729 if (Src0Ty.isScalar()) {
5730 BuildVec.push_back(Idx == 0 ? Src0Reg : Src1Reg);
5731 } else {
5732 int NumElts = Src0Ty.getNumElements();
5733 Register SrcVec = Idx < NumElts ? Src0Reg : Src1Reg;
5734 int ExtractIdx = Idx < NumElts ? Idx : Idx - NumElts;
5735 auto IdxK = MIRBuilder.buildConstant(IdxTy, ExtractIdx);
5736 auto Extract = MIRBuilder.buildExtractVectorElement(EltTy, SrcVec, IdxK);
5737 BuildVec.push_back(Extract.getReg(0));
5738 }
5739 }
5740
5741 MIRBuilder.buildBuildVector(DstReg, BuildVec);
5742 MI.eraseFromParent();
5743 return Legalized;
5744 }
5745
5746 LegalizerHelper::LegalizeResult
lowerDynStackAlloc(MachineInstr & MI)5747 LegalizerHelper::lowerDynStackAlloc(MachineInstr &MI) {
5748 const auto &MF = *MI.getMF();
5749 const auto &TFI = *MF.getSubtarget().getFrameLowering();
5750 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
5751 return UnableToLegalize;
5752
5753 Register Dst = MI.getOperand(0).getReg();
5754 Register AllocSize = MI.getOperand(1).getReg();
5755 Align Alignment = assumeAligned(MI.getOperand(2).getImm());
5756
5757 LLT PtrTy = MRI.getType(Dst);
5758 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
5759
5760 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
5761 auto SPTmp = MIRBuilder.buildCopy(PtrTy, SPReg);
5762 SPTmp = MIRBuilder.buildCast(IntPtrTy, SPTmp);
5763
5764 // Subtract the final alloc from the SP. We use G_PTRTOINT here so we don't
5765 // have to generate an extra instruction to negate the alloc and then use
5766 // G_PTR_ADD to add the negative offset.
5767 auto Alloc = MIRBuilder.buildSub(IntPtrTy, SPTmp, AllocSize);
5768 if (Alignment > Align(1)) {
5769 APInt AlignMask(IntPtrTy.getSizeInBits(), Alignment.value(), true);
5770 AlignMask.negate();
5771 auto AlignCst = MIRBuilder.buildConstant(IntPtrTy, AlignMask);
5772 Alloc = MIRBuilder.buildAnd(IntPtrTy, Alloc, AlignCst);
5773 }
5774
5775 SPTmp = MIRBuilder.buildCast(PtrTy, Alloc);
5776 MIRBuilder.buildCopy(SPReg, SPTmp);
5777 MIRBuilder.buildCopy(Dst, SPTmp);
5778
5779 MI.eraseFromParent();
5780 return Legalized;
5781 }
5782
5783 LegalizerHelper::LegalizeResult
lowerExtract(MachineInstr & MI)5784 LegalizerHelper::lowerExtract(MachineInstr &MI) {
5785 Register Dst = MI.getOperand(0).getReg();
5786 Register Src = MI.getOperand(1).getReg();
5787 unsigned Offset = MI.getOperand(2).getImm();
5788
5789 LLT DstTy = MRI.getType(Dst);
5790 LLT SrcTy = MRI.getType(Src);
5791
5792 if (DstTy.isScalar() &&
5793 (SrcTy.isScalar() ||
5794 (SrcTy.isVector() && DstTy == SrcTy.getElementType()))) {
5795 LLT SrcIntTy = SrcTy;
5796 if (!SrcTy.isScalar()) {
5797 SrcIntTy = LLT::scalar(SrcTy.getSizeInBits());
5798 Src = MIRBuilder.buildBitcast(SrcIntTy, Src).getReg(0);
5799 }
5800
5801 if (Offset == 0)
5802 MIRBuilder.buildTrunc(Dst, Src);
5803 else {
5804 auto ShiftAmt = MIRBuilder.buildConstant(SrcIntTy, Offset);
5805 auto Shr = MIRBuilder.buildLShr(SrcIntTy, Src, ShiftAmt);
5806 MIRBuilder.buildTrunc(Dst, Shr);
5807 }
5808
5809 MI.eraseFromParent();
5810 return Legalized;
5811 }
5812
5813 return UnableToLegalize;
5814 }
5815
lowerInsert(MachineInstr & MI)5816 LegalizerHelper::LegalizeResult LegalizerHelper::lowerInsert(MachineInstr &MI) {
5817 Register Dst = MI.getOperand(0).getReg();
5818 Register Src = MI.getOperand(1).getReg();
5819 Register InsertSrc = MI.getOperand(2).getReg();
5820 uint64_t Offset = MI.getOperand(3).getImm();
5821
5822 LLT DstTy = MRI.getType(Src);
5823 LLT InsertTy = MRI.getType(InsertSrc);
5824
5825 if (InsertTy.isVector() ||
5826 (DstTy.isVector() && DstTy.getElementType() != InsertTy))
5827 return UnableToLegalize;
5828
5829 const DataLayout &DL = MIRBuilder.getDataLayout();
5830 if ((DstTy.isPointer() &&
5831 DL.isNonIntegralAddressSpace(DstTy.getAddressSpace())) ||
5832 (InsertTy.isPointer() &&
5833 DL.isNonIntegralAddressSpace(InsertTy.getAddressSpace()))) {
5834 LLVM_DEBUG(dbgs() << "Not casting non-integral address space integer\n");
5835 return UnableToLegalize;
5836 }
5837
5838 LLT IntDstTy = DstTy;
5839
5840 if (!DstTy.isScalar()) {
5841 IntDstTy = LLT::scalar(DstTy.getSizeInBits());
5842 Src = MIRBuilder.buildCast(IntDstTy, Src).getReg(0);
5843 }
5844
5845 if (!InsertTy.isScalar()) {
5846 const LLT IntInsertTy = LLT::scalar(InsertTy.getSizeInBits());
5847 InsertSrc = MIRBuilder.buildPtrToInt(IntInsertTy, InsertSrc).getReg(0);
5848 }
5849
5850 Register ExtInsSrc = MIRBuilder.buildZExt(IntDstTy, InsertSrc).getReg(0);
5851 if (Offset != 0) {
5852 auto ShiftAmt = MIRBuilder.buildConstant(IntDstTy, Offset);
5853 ExtInsSrc = MIRBuilder.buildShl(IntDstTy, ExtInsSrc, ShiftAmt).getReg(0);
5854 }
5855
5856 APInt MaskVal = APInt::getBitsSetWithWrap(
5857 DstTy.getSizeInBits(), Offset + InsertTy.getSizeInBits(), Offset);
5858
5859 auto Mask = MIRBuilder.buildConstant(IntDstTy, MaskVal);
5860 auto MaskedSrc = MIRBuilder.buildAnd(IntDstTy, Src, Mask);
5861 auto Or = MIRBuilder.buildOr(IntDstTy, MaskedSrc, ExtInsSrc);
5862
5863 MIRBuilder.buildCast(Dst, Or);
5864 MI.eraseFromParent();
5865 return Legalized;
5866 }
5867
5868 LegalizerHelper::LegalizeResult
lowerSADDO_SSUBO(MachineInstr & MI)5869 LegalizerHelper::lowerSADDO_SSUBO(MachineInstr &MI) {
5870 Register Dst0 = MI.getOperand(0).getReg();
5871 Register Dst1 = MI.getOperand(1).getReg();
5872 Register LHS = MI.getOperand(2).getReg();
5873 Register RHS = MI.getOperand(3).getReg();
5874 const bool IsAdd = MI.getOpcode() == TargetOpcode::G_SADDO;
5875
5876 LLT Ty = MRI.getType(Dst0);
5877 LLT BoolTy = MRI.getType(Dst1);
5878
5879 if (IsAdd)
5880 MIRBuilder.buildAdd(Dst0, LHS, RHS);
5881 else
5882 MIRBuilder.buildSub(Dst0, LHS, RHS);
5883
5884 // TODO: If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
5885
5886 auto Zero = MIRBuilder.buildConstant(Ty, 0);
5887
5888 // For an addition, the result should be less than one of the operands (LHS)
5889 // if and only if the other operand (RHS) is negative, otherwise there will
5890 // be overflow.
5891 // For a subtraction, the result should be less than one of the operands
5892 // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
5893 // otherwise there will be overflow.
5894 auto ResultLowerThanLHS =
5895 MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, Dst0, LHS);
5896 auto ConditionRHS = MIRBuilder.buildICmp(
5897 IsAdd ? CmpInst::ICMP_SLT : CmpInst::ICMP_SGT, BoolTy, RHS, Zero);
5898
5899 MIRBuilder.buildXor(Dst1, ConditionRHS, ResultLowerThanLHS);
5900 MI.eraseFromParent();
5901 return Legalized;
5902 }
5903
5904 LegalizerHelper::LegalizeResult
lowerAddSubSatToMinMax(MachineInstr & MI)5905 LegalizerHelper::lowerAddSubSatToMinMax(MachineInstr &MI) {
5906 Register Res = MI.getOperand(0).getReg();
5907 Register LHS = MI.getOperand(1).getReg();
5908 Register RHS = MI.getOperand(2).getReg();
5909 LLT Ty = MRI.getType(Res);
5910 bool IsSigned;
5911 bool IsAdd;
5912 unsigned BaseOp;
5913 switch (MI.getOpcode()) {
5914 default:
5915 llvm_unreachable("unexpected addsat/subsat opcode");
5916 case TargetOpcode::G_UADDSAT:
5917 IsSigned = false;
5918 IsAdd = true;
5919 BaseOp = TargetOpcode::G_ADD;
5920 break;
5921 case TargetOpcode::G_SADDSAT:
5922 IsSigned = true;
5923 IsAdd = true;
5924 BaseOp = TargetOpcode::G_ADD;
5925 break;
5926 case TargetOpcode::G_USUBSAT:
5927 IsSigned = false;
5928 IsAdd = false;
5929 BaseOp = TargetOpcode::G_SUB;
5930 break;
5931 case TargetOpcode::G_SSUBSAT:
5932 IsSigned = true;
5933 IsAdd = false;
5934 BaseOp = TargetOpcode::G_SUB;
5935 break;
5936 }
5937
5938 if (IsSigned) {
5939 // sadd.sat(a, b) ->
5940 // hi = 0x7fffffff - smax(a, 0)
5941 // lo = 0x80000000 - smin(a, 0)
5942 // a + smin(smax(lo, b), hi)
5943 // ssub.sat(a, b) ->
5944 // lo = smax(a, -1) - 0x7fffffff
5945 // hi = smin(a, -1) - 0x80000000
5946 // a - smin(smax(lo, b), hi)
5947 // TODO: AMDGPU can use a "median of 3" instruction here:
5948 // a +/- med3(lo, b, hi)
5949 uint64_t NumBits = Ty.getScalarSizeInBits();
5950 auto MaxVal =
5951 MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(NumBits));
5952 auto MinVal =
5953 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits));
5954 MachineInstrBuilder Hi, Lo;
5955 if (IsAdd) {
5956 auto Zero = MIRBuilder.buildConstant(Ty, 0);
5957 Hi = MIRBuilder.buildSub(Ty, MaxVal, MIRBuilder.buildSMax(Ty, LHS, Zero));
5958 Lo = MIRBuilder.buildSub(Ty, MinVal, MIRBuilder.buildSMin(Ty, LHS, Zero));
5959 } else {
5960 auto NegOne = MIRBuilder.buildConstant(Ty, -1);
5961 Lo = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMax(Ty, LHS, NegOne),
5962 MaxVal);
5963 Hi = MIRBuilder.buildSub(Ty, MIRBuilder.buildSMin(Ty, LHS, NegOne),
5964 MinVal);
5965 }
5966 auto RHSClamped =
5967 MIRBuilder.buildSMin(Ty, MIRBuilder.buildSMax(Ty, Lo, RHS), Hi);
5968 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, RHSClamped});
5969 } else {
5970 // uadd.sat(a, b) -> a + umin(~a, b)
5971 // usub.sat(a, b) -> a - umin(a, b)
5972 Register Not = IsAdd ? MIRBuilder.buildNot(Ty, LHS).getReg(0) : LHS;
5973 auto Min = MIRBuilder.buildUMin(Ty, Not, RHS);
5974 MIRBuilder.buildInstr(BaseOp, {Res}, {LHS, Min});
5975 }
5976
5977 MI.eraseFromParent();
5978 return Legalized;
5979 }
5980
5981 LegalizerHelper::LegalizeResult
lowerAddSubSatToAddoSubo(MachineInstr & MI)5982 LegalizerHelper::lowerAddSubSatToAddoSubo(MachineInstr &MI) {
5983 Register Res = MI.getOperand(0).getReg();
5984 Register LHS = MI.getOperand(1).getReg();
5985 Register RHS = MI.getOperand(2).getReg();
5986 LLT Ty = MRI.getType(Res);
5987 LLT BoolTy = Ty.changeElementSize(1);
5988 bool IsSigned;
5989 bool IsAdd;
5990 unsigned OverflowOp;
5991 switch (MI.getOpcode()) {
5992 default:
5993 llvm_unreachable("unexpected addsat/subsat opcode");
5994 case TargetOpcode::G_UADDSAT:
5995 IsSigned = false;
5996 IsAdd = true;
5997 OverflowOp = TargetOpcode::G_UADDO;
5998 break;
5999 case TargetOpcode::G_SADDSAT:
6000 IsSigned = true;
6001 IsAdd = true;
6002 OverflowOp = TargetOpcode::G_SADDO;
6003 break;
6004 case TargetOpcode::G_USUBSAT:
6005 IsSigned = false;
6006 IsAdd = false;
6007 OverflowOp = TargetOpcode::G_USUBO;
6008 break;
6009 case TargetOpcode::G_SSUBSAT:
6010 IsSigned = true;
6011 IsAdd = false;
6012 OverflowOp = TargetOpcode::G_SSUBO;
6013 break;
6014 }
6015
6016 auto OverflowRes =
6017 MIRBuilder.buildInstr(OverflowOp, {Ty, BoolTy}, {LHS, RHS});
6018 Register Tmp = OverflowRes.getReg(0);
6019 Register Ov = OverflowRes.getReg(1);
6020 MachineInstrBuilder Clamp;
6021 if (IsSigned) {
6022 // sadd.sat(a, b) ->
6023 // {tmp, ov} = saddo(a, b)
6024 // ov ? (tmp >>s 31) + 0x80000000 : r
6025 // ssub.sat(a, b) ->
6026 // {tmp, ov} = ssubo(a, b)
6027 // ov ? (tmp >>s 31) + 0x80000000 : r
6028 uint64_t NumBits = Ty.getScalarSizeInBits();
6029 auto ShiftAmount = MIRBuilder.buildConstant(Ty, NumBits - 1);
6030 auto Sign = MIRBuilder.buildAShr(Ty, Tmp, ShiftAmount);
6031 auto MinVal =
6032 MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(NumBits));
6033 Clamp = MIRBuilder.buildAdd(Ty, Sign, MinVal);
6034 } else {
6035 // uadd.sat(a, b) ->
6036 // {tmp, ov} = uaddo(a, b)
6037 // ov ? 0xffffffff : tmp
6038 // usub.sat(a, b) ->
6039 // {tmp, ov} = usubo(a, b)
6040 // ov ? 0 : tmp
6041 Clamp = MIRBuilder.buildConstant(Ty, IsAdd ? -1 : 0);
6042 }
6043 MIRBuilder.buildSelect(Res, Ov, Clamp, Tmp);
6044
6045 MI.eraseFromParent();
6046 return Legalized;
6047 }
6048
6049 LegalizerHelper::LegalizeResult
lowerShlSat(MachineInstr & MI)6050 LegalizerHelper::lowerShlSat(MachineInstr &MI) {
6051 assert((MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
6052 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
6053 "Expected shlsat opcode!");
6054 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SSHLSAT;
6055 Register Res = MI.getOperand(0).getReg();
6056 Register LHS = MI.getOperand(1).getReg();
6057 Register RHS = MI.getOperand(2).getReg();
6058 LLT Ty = MRI.getType(Res);
6059 LLT BoolTy = Ty.changeElementSize(1);
6060
6061 unsigned BW = Ty.getScalarSizeInBits();
6062 auto Result = MIRBuilder.buildShl(Ty, LHS, RHS);
6063 auto Orig = IsSigned ? MIRBuilder.buildAShr(Ty, Result, RHS)
6064 : MIRBuilder.buildLShr(Ty, Result, RHS);
6065
6066 MachineInstrBuilder SatVal;
6067 if (IsSigned) {
6068 auto SatMin = MIRBuilder.buildConstant(Ty, APInt::getSignedMinValue(BW));
6069 auto SatMax = MIRBuilder.buildConstant(Ty, APInt::getSignedMaxValue(BW));
6070 auto Cmp = MIRBuilder.buildICmp(CmpInst::ICMP_SLT, BoolTy, LHS,
6071 MIRBuilder.buildConstant(Ty, 0));
6072 SatVal = MIRBuilder.buildSelect(Ty, Cmp, SatMin, SatMax);
6073 } else {
6074 SatVal = MIRBuilder.buildConstant(Ty, APInt::getMaxValue(BW));
6075 }
6076 auto Ov = MIRBuilder.buildICmp(CmpInst::ICMP_NE, Ty, LHS, Orig);
6077 MIRBuilder.buildSelect(Res, Ov, SatVal, Result);
6078
6079 MI.eraseFromParent();
6080 return Legalized;
6081 }
6082
6083 LegalizerHelper::LegalizeResult
lowerBswap(MachineInstr & MI)6084 LegalizerHelper::lowerBswap(MachineInstr &MI) {
6085 Register Dst = MI.getOperand(0).getReg();
6086 Register Src = MI.getOperand(1).getReg();
6087 const LLT Ty = MRI.getType(Src);
6088 unsigned SizeInBytes = (Ty.getScalarSizeInBits() + 7) / 8;
6089 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
6090
6091 // Swap most and least significant byte, set remaining bytes in Res to zero.
6092 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt);
6093 auto LSByteShiftedLeft = MIRBuilder.buildShl(Ty, Src, ShiftAmt);
6094 auto MSByteShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
6095 auto Res = MIRBuilder.buildOr(Ty, MSByteShiftedRight, LSByteShiftedLeft);
6096
6097 // Set i-th high/low byte in Res to i-th low/high byte from Src.
6098 for (unsigned i = 1; i < SizeInBytes / 2; ++i) {
6099 // AND with Mask leaves byte i unchanged and sets remaining bytes to 0.
6100 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8));
6101 auto Mask = MIRBuilder.buildConstant(Ty, APMask);
6102 auto ShiftAmt = MIRBuilder.buildConstant(Ty, BaseShiftAmt - 16 * i);
6103 // Low byte shifted left to place of high byte: (Src & Mask) << ShiftAmt.
6104 auto LoByte = MIRBuilder.buildAnd(Ty, Src, Mask);
6105 auto LoShiftedLeft = MIRBuilder.buildShl(Ty, LoByte, ShiftAmt);
6106 Res = MIRBuilder.buildOr(Ty, Res, LoShiftedLeft);
6107 // High byte shifted right to place of low byte: (Src >> ShiftAmt) & Mask.
6108 auto SrcShiftedRight = MIRBuilder.buildLShr(Ty, Src, ShiftAmt);
6109 auto HiShiftedRight = MIRBuilder.buildAnd(Ty, SrcShiftedRight, Mask);
6110 Res = MIRBuilder.buildOr(Ty, Res, HiShiftedRight);
6111 }
6112 Res.getInstr()->getOperand(0).setReg(Dst);
6113
6114 MI.eraseFromParent();
6115 return Legalized;
6116 }
6117
6118 //{ (Src & Mask) >> N } | { (Src << N) & Mask }
SwapN(unsigned N,DstOp Dst,MachineIRBuilder & B,MachineInstrBuilder Src,APInt Mask)6119 static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B,
6120 MachineInstrBuilder Src, APInt Mask) {
6121 const LLT Ty = Dst.getLLTTy(*B.getMRI());
6122 MachineInstrBuilder C_N = B.buildConstant(Ty, N);
6123 MachineInstrBuilder MaskLoNTo0 = B.buildConstant(Ty, Mask);
6124 auto LHS = B.buildLShr(Ty, B.buildAnd(Ty, Src, MaskLoNTo0), C_N);
6125 auto RHS = B.buildAnd(Ty, B.buildShl(Ty, Src, C_N), MaskLoNTo0);
6126 return B.buildOr(Dst, LHS, RHS);
6127 }
6128
6129 LegalizerHelper::LegalizeResult
lowerBitreverse(MachineInstr & MI)6130 LegalizerHelper::lowerBitreverse(MachineInstr &MI) {
6131 Register Dst = MI.getOperand(0).getReg();
6132 Register Src = MI.getOperand(1).getReg();
6133 const LLT Ty = MRI.getType(Src);
6134 unsigned Size = Ty.getSizeInBits();
6135
6136 MachineInstrBuilder BSWAP =
6137 MIRBuilder.buildInstr(TargetOpcode::G_BSWAP, {Ty}, {Src});
6138
6139 // swap high and low 4 bits in 8 bit blocks 7654|3210 -> 3210|7654
6140 // [(val & 0xF0F0F0F0) >> 4] | [(val & 0x0F0F0F0F) << 4]
6141 // -> [(val & 0xF0F0F0F0) >> 4] | [(val << 4) & 0xF0F0F0F0]
6142 MachineInstrBuilder Swap4 =
6143 SwapN(4, Ty, MIRBuilder, BSWAP, APInt::getSplat(Size, APInt(8, 0xF0)));
6144
6145 // swap high and low 2 bits in 4 bit blocks 32|10 76|54 -> 10|32 54|76
6146 // [(val & 0xCCCCCCCC) >> 2] & [(val & 0x33333333) << 2]
6147 // -> [(val & 0xCCCCCCCC) >> 2] & [(val << 2) & 0xCCCCCCCC]
6148 MachineInstrBuilder Swap2 =
6149 SwapN(2, Ty, MIRBuilder, Swap4, APInt::getSplat(Size, APInt(8, 0xCC)));
6150
6151 // swap high and low 1 bit in 2 bit blocks 1|0 3|2 5|4 7|6 -> 0|1 2|3 4|5 6|7
6152 // [(val & 0xAAAAAAAA) >> 1] & [(val & 0x55555555) << 1]
6153 // -> [(val & 0xAAAAAAAA) >> 1] & [(val << 1) & 0xAAAAAAAA]
6154 SwapN(1, Dst, MIRBuilder, Swap2, APInt::getSplat(Size, APInt(8, 0xAA)));
6155
6156 MI.eraseFromParent();
6157 return Legalized;
6158 }
6159
6160 LegalizerHelper::LegalizeResult
lowerReadWriteRegister(MachineInstr & MI)6161 LegalizerHelper::lowerReadWriteRegister(MachineInstr &MI) {
6162 MachineFunction &MF = MIRBuilder.getMF();
6163
6164 bool IsRead = MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
6165 int NameOpIdx = IsRead ? 1 : 0;
6166 int ValRegIndex = IsRead ? 0 : 1;
6167
6168 Register ValReg = MI.getOperand(ValRegIndex).getReg();
6169 const LLT Ty = MRI.getType(ValReg);
6170 const MDString *RegStr = cast<MDString>(
6171 cast<MDNode>(MI.getOperand(NameOpIdx).getMetadata())->getOperand(0));
6172
6173 Register PhysReg = TLI.getRegisterByName(RegStr->getString().data(), Ty, MF);
6174 if (!PhysReg.isValid())
6175 return UnableToLegalize;
6176
6177 if (IsRead)
6178 MIRBuilder.buildCopy(ValReg, PhysReg);
6179 else
6180 MIRBuilder.buildCopy(PhysReg, ValReg);
6181
6182 MI.eraseFromParent();
6183 return Legalized;
6184 }
6185
6186 LegalizerHelper::LegalizeResult
lowerSMULH_UMULH(MachineInstr & MI)6187 LegalizerHelper::lowerSMULH_UMULH(MachineInstr &MI) {
6188 bool IsSigned = MI.getOpcode() == TargetOpcode::G_SMULH;
6189 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
6190 Register Result = MI.getOperand(0).getReg();
6191 LLT OrigTy = MRI.getType(Result);
6192 auto SizeInBits = OrigTy.getScalarSizeInBits();
6193 LLT WideTy = OrigTy.changeElementSize(SizeInBits * 2);
6194
6195 auto LHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(1)});
6196 auto RHS = MIRBuilder.buildInstr(ExtOp, {WideTy}, {MI.getOperand(2)});
6197 auto Mul = MIRBuilder.buildMul(WideTy, LHS, RHS);
6198 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
6199
6200 auto ShiftAmt = MIRBuilder.buildConstant(WideTy, SizeInBits);
6201 auto Shifted = MIRBuilder.buildInstr(ShiftOp, {WideTy}, {Mul, ShiftAmt});
6202 MIRBuilder.buildTrunc(Result, Shifted);
6203
6204 MI.eraseFromParent();
6205 return Legalized;
6206 }
6207
lowerSelect(MachineInstr & MI)6208 LegalizerHelper::LegalizeResult LegalizerHelper::lowerSelect(MachineInstr &MI) {
6209 // Implement vector G_SELECT in terms of XOR, AND, OR.
6210 Register DstReg = MI.getOperand(0).getReg();
6211 Register MaskReg = MI.getOperand(1).getReg();
6212 Register Op1Reg = MI.getOperand(2).getReg();
6213 Register Op2Reg = MI.getOperand(3).getReg();
6214 LLT DstTy = MRI.getType(DstReg);
6215 LLT MaskTy = MRI.getType(MaskReg);
6216 LLT Op1Ty = MRI.getType(Op1Reg);
6217 if (!DstTy.isVector())
6218 return UnableToLegalize;
6219
6220 if (MaskTy.getSizeInBits() != Op1Ty.getSizeInBits())
6221 return UnableToLegalize;
6222
6223 auto NotMask = MIRBuilder.buildNot(MaskTy, MaskReg);
6224 auto NewOp1 = MIRBuilder.buildAnd(MaskTy, Op1Reg, MaskReg);
6225 auto NewOp2 = MIRBuilder.buildAnd(MaskTy, Op2Reg, NotMask);
6226 MIRBuilder.buildOr(DstReg, NewOp1, NewOp2);
6227 MI.eraseFromParent();
6228 return Legalized;
6229 }