1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "llvm/CodeGen/TargetLowering.h"
14 #include "llvm/ADT/STLExtras.h"
15 #include "llvm/CodeGen/CallingConvLower.h"
16 #include "llvm/CodeGen/MachineFrameInfo.h"
17 #include "llvm/CodeGen/MachineFunction.h"
18 #include "llvm/CodeGen/MachineJumpTableInfo.h"
19 #include "llvm/CodeGen/MachineRegisterInfo.h"
20 #include "llvm/CodeGen/SelectionDAG.h"
21 #include "llvm/CodeGen/TargetRegisterInfo.h"
22 #include "llvm/CodeGen/TargetSubtargetInfo.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/GlobalVariable.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCExpr.h"
29 #include "llvm/Support/ErrorHandling.h"
30 #include "llvm/Support/KnownBits.h"
31 #include "llvm/Support/MathExtras.h"
32 #include "llvm/Target/TargetLoweringObjectFile.h"
33 #include "llvm/Target/TargetMachine.h"
34 #include <cctype>
35 using namespace llvm;
36
37 /// NOTE: The TargetMachine owns TLOF.
TargetLowering(const TargetMachine & tm)38 TargetLowering::TargetLowering(const TargetMachine &tm)
39 : TargetLoweringBase(tm) {}
40
getTargetNodeName(unsigned Opcode) const41 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
42 return nullptr;
43 }
44
isPositionIndependent() const45 bool TargetLowering::isPositionIndependent() const {
46 return getTargetMachine().isPositionIndependent();
47 }
48
49 /// Check whether a given call node is in tail position within its function. If
50 /// so, it sets Chain to the input chain of the tail call.
isInTailCallPosition(SelectionDAG & DAG,SDNode * Node,SDValue & Chain) const51 bool TargetLowering::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
52 SDValue &Chain) const {
53 const Function &F = DAG.getMachineFunction().getFunction();
54
55 // First, check if tail calls have been disabled in this function.
56 if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
57 return false;
58
59 // Conservatively require the attributes of the call to match those of
60 // the return. Ignore following attributes because they don't affect the
61 // call sequence.
62 AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);
63 for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
64 Attribute::DereferenceableOrNull, Attribute::NoAlias,
65 Attribute::NonNull})
66 CallerAttrs.removeAttribute(Attr);
67
68 if (CallerAttrs.hasAttributes())
69 return false;
70
71 // It's not safe to eliminate the sign / zero extension of the return value.
72 if (CallerAttrs.contains(Attribute::ZExt) ||
73 CallerAttrs.contains(Attribute::SExt))
74 return false;
75
76 // Check if the only use is a function return node.
77 return isUsedByReturnOnly(Node, Chain);
78 }
79
parametersInCSRMatch(const MachineRegisterInfo & MRI,const uint32_t * CallerPreservedMask,const SmallVectorImpl<CCValAssign> & ArgLocs,const SmallVectorImpl<SDValue> & OutVals) const80 bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
81 const uint32_t *CallerPreservedMask,
82 const SmallVectorImpl<CCValAssign> &ArgLocs,
83 const SmallVectorImpl<SDValue> &OutVals) const {
84 for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
85 const CCValAssign &ArgLoc = ArgLocs[I];
86 if (!ArgLoc.isRegLoc())
87 continue;
88 MCRegister Reg = ArgLoc.getLocReg();
89 // Only look at callee saved registers.
90 if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
91 continue;
92 // Check that we pass the value used for the caller.
93 // (We look for a CopyFromReg reading a virtual register that is used
94 // for the function live-in value of register Reg)
95 SDValue Value = OutVals[I];
96 if (Value->getOpcode() != ISD::CopyFromReg)
97 return false;
98 Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
99 if (MRI.getLiveInPhysReg(ArgReg) != Reg)
100 return false;
101 }
102 return true;
103 }
104
105 /// Set CallLoweringInfo attribute flags based on the call instruction's
106 /// argument attributes.
setAttributes(const CallBase * Call,unsigned ArgIdx)107 void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
108 unsigned ArgIdx) {
109 auto Attrs = Call->getAttributes();
110
111 IsSExt = Attrs.hasParamAttribute(ArgIdx, Attribute::SExt);
112 IsZExt = Attrs.hasParamAttribute(ArgIdx, Attribute::ZExt);
113 IsInReg = Attrs.hasParamAttribute(ArgIdx, Attribute::InReg);
114 IsSRet = Attrs.hasParamAttribute(ArgIdx, Attribute::StructRet);
115 IsNest = Attrs.hasParamAttribute(ArgIdx, Attribute::Nest);
116 IsReturned = Attrs.hasParamAttribute(ArgIdx, Attribute::Returned);
117 IsSwiftSelf = Attrs.hasParamAttribute(ArgIdx, Attribute::SwiftSelf);
118 IsSwiftAsync = Attrs.hasParamAttribute(ArgIdx, Attribute::SwiftAsync);
119 IsSwiftError = Attrs.hasParamAttribute(ArgIdx, Attribute::SwiftError);
120 Alignment = Attrs.getParamStackAlignment(ArgIdx);
121
122 IsByVal = Attrs.hasParamAttribute(ArgIdx, Attribute::ByVal);
123 IsInAlloca = Attrs.hasParamAttribute(ArgIdx, Attribute::InAlloca);
124 IsPreallocated = Attrs.hasParamAttribute(ArgIdx, Attribute::Preallocated);
125
126 assert(IsByVal + IsInAlloca + IsPreallocated <= 1 &&
127 "can't have multiple indirect attributes");
128 IndirectType = nullptr;
129 if (IsByVal) {
130 IndirectType = Call->getParamByValType(ArgIdx);
131 assert(IndirectType && "no byval type?");
132 if (!Alignment)
133 Alignment = Call->getParamAlign(ArgIdx);
134 }
135 if (IsInAlloca) {
136 IndirectType = Call->getParamInAllocaType(ArgIdx);
137 assert(IndirectType && "no inalloca type?");
138 }
139 if (IsPreallocated) {
140 IndirectType = Call->getParamPreallocatedType(ArgIdx);
141 assert(IndirectType && "no preallocated type?");
142 }
143 }
144
145 /// Generate a libcall taking the given operands as arguments and returning a
146 /// result of type RetVT.
147 std::pair<SDValue, SDValue>
makeLibCall(SelectionDAG & DAG,RTLIB::Libcall LC,EVT RetVT,ArrayRef<SDValue> Ops,MakeLibCallOptions CallOptions,const SDLoc & dl,SDValue InChain) const148 TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT,
149 ArrayRef<SDValue> Ops,
150 MakeLibCallOptions CallOptions,
151 const SDLoc &dl,
152 SDValue InChain) const {
153 if (!InChain)
154 InChain = DAG.getEntryNode();
155
156 TargetLowering::ArgListTy Args;
157 Args.reserve(Ops.size());
158
159 TargetLowering::ArgListEntry Entry;
160 for (unsigned i = 0; i < Ops.size(); ++i) {
161 SDValue NewOp = Ops[i];
162 Entry.Node = NewOp;
163 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
164 Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
165 CallOptions.IsSExt);
166 Entry.IsZExt = !Entry.IsSExt;
167
168 if (CallOptions.IsSoften &&
169 !shouldExtendTypeInLibCall(CallOptions.OpsVTBeforeSoften[i])) {
170 Entry.IsSExt = Entry.IsZExt = false;
171 }
172 Args.push_back(Entry);
173 }
174
175 if (LC == RTLIB::UNKNOWN_LIBCALL)
176 report_fatal_error("Unsupported library call operation!");
177 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
178 getPointerTy(DAG.getDataLayout()));
179
180 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
181 TargetLowering::CallLoweringInfo CLI(DAG);
182 bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
183 bool zeroExtend = !signExtend;
184
185 if (CallOptions.IsSoften &&
186 !shouldExtendTypeInLibCall(CallOptions.RetVTBeforeSoften)) {
187 signExtend = zeroExtend = false;
188 }
189
190 CLI.setDebugLoc(dl)
191 .setChain(InChain)
192 .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
193 .setNoReturn(CallOptions.DoesNotReturn)
194 .setDiscardResult(!CallOptions.IsReturnValueUsed)
195 .setIsPostTypeLegalization(CallOptions.IsPostTypeLegalization)
196 .setSExtResult(signExtend)
197 .setZExtResult(zeroExtend);
198 return LowerCallTo(CLI);
199 }
200
findOptimalMemOpLowering(std::vector<EVT> & MemOps,unsigned Limit,const MemOp & Op,unsigned DstAS,unsigned SrcAS,const AttributeList & FuncAttributes) const201 bool TargetLowering::findOptimalMemOpLowering(
202 std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
203 unsigned SrcAS, const AttributeList &FuncAttributes) const {
204 if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
205 return false;
206
207 EVT VT = getOptimalMemOpType(Op, FuncAttributes);
208
209 if (VT == MVT::Other) {
210 // Use the largest integer type whose alignment constraints are satisfied.
211 // We only need to check DstAlign here as SrcAlign is always greater or
212 // equal to DstAlign (or zero).
213 VT = MVT::i64;
214 if (Op.isFixedDstAlign())
215 while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
216 !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
217 VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
218 assert(VT.isInteger());
219
220 // Find the largest legal integer type.
221 MVT LVT = MVT::i64;
222 while (!isTypeLegal(LVT))
223 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
224 assert(LVT.isInteger());
225
226 // If the type we've chosen is larger than the largest legal integer type
227 // then use that instead.
228 if (VT.bitsGT(LVT))
229 VT = LVT;
230 }
231
232 unsigned NumMemOps = 0;
233 uint64_t Size = Op.size();
234 while (Size) {
235 unsigned VTSize = VT.getSizeInBits() / 8;
236 while (VTSize > Size) {
237 // For now, only use non-vector load / store's for the left-over pieces.
238 EVT NewVT = VT;
239 unsigned NewVTSize;
240
241 bool Found = false;
242 if (VT.isVector() || VT.isFloatingPoint()) {
243 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
244 if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
245 isSafeMemOpType(NewVT.getSimpleVT()))
246 Found = true;
247 else if (NewVT == MVT::i64 &&
248 isOperationLegalOrCustom(ISD::STORE, MVT::f64) &&
249 isSafeMemOpType(MVT::f64)) {
250 // i64 is usually not legal on 32-bit targets, but f64 may be.
251 NewVT = MVT::f64;
252 Found = true;
253 }
254 }
255
256 if (!Found) {
257 do {
258 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
259 if (NewVT == MVT::i8)
260 break;
261 } while (!isSafeMemOpType(NewVT.getSimpleVT()));
262 }
263 NewVTSize = NewVT.getSizeInBits() / 8;
264
265 // If the new VT cannot cover all of the remaining bits, then consider
266 // issuing a (or a pair of) unaligned and overlapping load / store.
267 bool Fast;
268 if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
269 allowsMisalignedMemoryAccesses(
270 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
271 MachineMemOperand::MONone, &Fast) &&
272 Fast)
273 VTSize = Size;
274 else {
275 VT = NewVT;
276 VTSize = NewVTSize;
277 }
278 }
279
280 if (++NumMemOps > Limit)
281 return false;
282
283 MemOps.push_back(VT);
284 Size -= VTSize;
285 }
286
287 return true;
288 }
289
290 /// Soften the operands of a comparison. This code is shared among BR_CC,
291 /// SELECT_CC, and SETCC handlers.
softenSetCCOperands(SelectionDAG & DAG,EVT VT,SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl,const SDValue OldLHS,const SDValue OldRHS) const292 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
293 SDValue &NewLHS, SDValue &NewRHS,
294 ISD::CondCode &CCCode,
295 const SDLoc &dl, const SDValue OldLHS,
296 const SDValue OldRHS) const {
297 SDValue Chain;
298 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
299 OldRHS, Chain);
300 }
301
softenSetCCOperands(SelectionDAG & DAG,EVT VT,SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,const SDLoc & dl,const SDValue OldLHS,const SDValue OldRHS,SDValue & Chain,bool IsSignaling) const302 void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT,
303 SDValue &NewLHS, SDValue &NewRHS,
304 ISD::CondCode &CCCode,
305 const SDLoc &dl, const SDValue OldLHS,
306 const SDValue OldRHS,
307 SDValue &Chain,
308 bool IsSignaling) const {
309 // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
310 // not supporting it. We can update this code when libgcc provides such
311 // functions.
312
313 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
314 && "Unsupported setcc type!");
315
316 // Expand into one or more soft-fp libcall(s).
317 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
318 bool ShouldInvertCC = false;
319 switch (CCCode) {
320 case ISD::SETEQ:
321 case ISD::SETOEQ:
322 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
323 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
324 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
325 break;
326 case ISD::SETNE:
327 case ISD::SETUNE:
328 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
329 (VT == MVT::f64) ? RTLIB::UNE_F64 :
330 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
331 break;
332 case ISD::SETGE:
333 case ISD::SETOGE:
334 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
335 (VT == MVT::f64) ? RTLIB::OGE_F64 :
336 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
337 break;
338 case ISD::SETLT:
339 case ISD::SETOLT:
340 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
341 (VT == MVT::f64) ? RTLIB::OLT_F64 :
342 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
343 break;
344 case ISD::SETLE:
345 case ISD::SETOLE:
346 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
347 (VT == MVT::f64) ? RTLIB::OLE_F64 :
348 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
349 break;
350 case ISD::SETGT:
351 case ISD::SETOGT:
352 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
353 (VT == MVT::f64) ? RTLIB::OGT_F64 :
354 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
355 break;
356 case ISD::SETO:
357 ShouldInvertCC = true;
358 LLVM_FALLTHROUGH;
359 case ISD::SETUO:
360 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
361 (VT == MVT::f64) ? RTLIB::UO_F64 :
362 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
363 break;
364 case ISD::SETONE:
365 // SETONE = O && UNE
366 ShouldInvertCC = true;
367 LLVM_FALLTHROUGH;
368 case ISD::SETUEQ:
369 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
370 (VT == MVT::f64) ? RTLIB::UO_F64 :
371 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
372 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
373 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
374 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
375 break;
376 default:
377 // Invert CC for unordered comparisons
378 ShouldInvertCC = true;
379 switch (CCCode) {
380 case ISD::SETULT:
381 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
382 (VT == MVT::f64) ? RTLIB::OGE_F64 :
383 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
384 break;
385 case ISD::SETULE:
386 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
387 (VT == MVT::f64) ? RTLIB::OGT_F64 :
388 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
389 break;
390 case ISD::SETUGT:
391 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
392 (VT == MVT::f64) ? RTLIB::OLE_F64 :
393 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
394 break;
395 case ISD::SETUGE:
396 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
397 (VT == MVT::f64) ? RTLIB::OLT_F64 :
398 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
399 break;
400 default: llvm_unreachable("Do not know how to soften this setcc!");
401 }
402 }
403
404 // Use the target specific return value for comparions lib calls.
405 EVT RetVT = getCmpLibcallReturnType();
406 SDValue Ops[2] = {NewLHS, NewRHS};
407 TargetLowering::MakeLibCallOptions CallOptions;
408 EVT OpsVT[2] = { OldLHS.getValueType(),
409 OldRHS.getValueType() };
410 CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
411 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
412 NewLHS = Call.first;
413 NewRHS = DAG.getConstant(0, dl, RetVT);
414
415 CCCode = getCmpLibcallCC(LC1);
416 if (ShouldInvertCC) {
417 assert(RetVT.isInteger());
418 CCCode = getSetCCInverse(CCCode, RetVT);
419 }
420
421 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
422 // Update Chain.
423 Chain = Call.second;
424 } else {
425 EVT SetCCVT =
426 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
427 SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
428 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
429 CCCode = getCmpLibcallCC(LC2);
430 if (ShouldInvertCC)
431 CCCode = getSetCCInverse(CCCode, RetVT);
432 NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
433 if (Chain)
434 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
435 Call2.second);
436 NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
437 Tmp.getValueType(), Tmp, NewLHS);
438 NewRHS = SDValue();
439 }
440 }
441
442 /// Return the entry encoding for a jump table in the current function. The
443 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
getJumpTableEncoding() const444 unsigned TargetLowering::getJumpTableEncoding() const {
445 // In non-pic modes, just use the address of a block.
446 if (!isPositionIndependent())
447 return MachineJumpTableInfo::EK_BlockAddress;
448
449 // In PIC mode, if the target supports a GPRel32 directive, use it.
450 if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
451 return MachineJumpTableInfo::EK_GPRel32BlockAddress;
452
453 // Otherwise, use a label difference.
454 return MachineJumpTableInfo::EK_LabelDifference32;
455 }
456
getPICJumpTableRelocBase(SDValue Table,SelectionDAG & DAG) const457 SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
458 SelectionDAG &DAG) const {
459 // If our PIC model is GP relative, use the global offset table as the base.
460 unsigned JTEncoding = getJumpTableEncoding();
461
462 if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
463 (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
464 return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(DAG.getDataLayout()));
465
466 return Table;
467 }
468
469 /// This returns the relocation base for the given PIC jumptable, the same as
470 /// getPICJumpTableRelocBase, but as an MCExpr.
471 const MCExpr *
getPICJumpTableRelocBaseExpr(const MachineFunction * MF,unsigned JTI,MCContext & Ctx) const472 TargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
473 unsigned JTI,MCContext &Ctx) const{
474 // The normal PIC reloc base is the label at the start of the jump table.
475 return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
476 }
477
478 bool
isOffsetFoldingLegal(const GlobalAddressSDNode * GA) const479 TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
480 const TargetMachine &TM = getTargetMachine();
481 const GlobalValue *GV = GA->getGlobal();
482
483 // If the address is not even local to this DSO we will have to load it from
484 // a got and then add the offset.
485 if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
486 return false;
487
488 // If the code is position independent we will have to add a base register.
489 if (isPositionIndependent())
490 return false;
491
492 // Otherwise we can do it.
493 return true;
494 }
495
496 //===----------------------------------------------------------------------===//
497 // Optimization Methods
498 //===----------------------------------------------------------------------===//
499
500 /// If the specified instruction has a constant integer operand and there are
501 /// bits set in that constant that are not demanded, then clear those bits and
502 /// return true.
ShrinkDemandedConstant(SDValue Op,const APInt & DemandedBits,const APInt & DemandedElts,TargetLoweringOpt & TLO) const503 bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
504 const APInt &DemandedBits,
505 const APInt &DemandedElts,
506 TargetLoweringOpt &TLO) const {
507 SDLoc DL(Op);
508 unsigned Opcode = Op.getOpcode();
509
510 // Do target-specific constant optimization.
511 if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
512 return TLO.New.getNode();
513
514 // FIXME: ISD::SELECT, ISD::SELECT_CC
515 switch (Opcode) {
516 default:
517 break;
518 case ISD::XOR:
519 case ISD::AND:
520 case ISD::OR: {
521 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
522 if (!Op1C)
523 return false;
524
525 // If this is a 'not' op, don't touch it because that's a canonical form.
526 const APInt &C = Op1C->getAPIntValue();
527 if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
528 return false;
529
530 if (!C.isSubsetOf(DemandedBits)) {
531 EVT VT = Op.getValueType();
532 SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
533 SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
534 return TLO.CombineTo(Op, NewOp);
535 }
536
537 break;
538 }
539 }
540
541 return false;
542 }
543
ShrinkDemandedConstant(SDValue Op,const APInt & DemandedBits,TargetLoweringOpt & TLO) const544 bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
545 const APInt &DemandedBits,
546 TargetLoweringOpt &TLO) const {
547 EVT VT = Op.getValueType();
548 APInt DemandedElts = VT.isVector()
549 ? APInt::getAllOnesValue(VT.getVectorNumElements())
550 : APInt(1, 1);
551 return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
552 }
553
554 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
555 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
556 /// generalized for targets with other types of implicit widening casts.
ShrinkDemandedOp(SDValue Op,unsigned BitWidth,const APInt & Demanded,TargetLoweringOpt & TLO) const557 bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
558 const APInt &Demanded,
559 TargetLoweringOpt &TLO) const {
560 assert(Op.getNumOperands() == 2 &&
561 "ShrinkDemandedOp only supports binary operators!");
562 assert(Op.getNode()->getNumValues() == 1 &&
563 "ShrinkDemandedOp only supports nodes with one result!");
564
565 SelectionDAG &DAG = TLO.DAG;
566 SDLoc dl(Op);
567
568 // Early return, as this function cannot handle vector types.
569 if (Op.getValueType().isVector())
570 return false;
571
572 // Don't do this if the node has another user, which may require the
573 // full value.
574 if (!Op.getNode()->hasOneUse())
575 return false;
576
577 // Search for the smallest integer type with free casts to and from
578 // Op's type. For expedience, just check power-of-2 integer types.
579 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
580 unsigned DemandedSize = Demanded.getActiveBits();
581 unsigned SmallVTBits = DemandedSize;
582 if (!isPowerOf2_32(SmallVTBits))
583 SmallVTBits = NextPowerOf2(SmallVTBits);
584 for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
585 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
586 if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
587 TLI.isZExtFree(SmallVT, Op.getValueType())) {
588 // We found a type with free casts.
589 SDValue X = DAG.getNode(
590 Op.getOpcode(), dl, SmallVT,
591 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
592 DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
593 assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
594 SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
595 return TLO.CombineTo(Op, Z);
596 }
597 }
598 return false;
599 }
600
SimplifyDemandedBits(SDValue Op,const APInt & DemandedBits,DAGCombinerInfo & DCI) const601 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
602 DAGCombinerInfo &DCI) const {
603 SelectionDAG &DAG = DCI.DAG;
604 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
605 !DCI.isBeforeLegalizeOps());
606 KnownBits Known;
607
608 bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
609 if (Simplified) {
610 DCI.AddToWorklist(Op.getNode());
611 DCI.CommitTargetLoweringOpt(TLO);
612 }
613 return Simplified;
614 }
615
SimplifyDemandedBits(SDValue Op,const APInt & DemandedBits,KnownBits & Known,TargetLoweringOpt & TLO,unsigned Depth,bool AssumeSingleUse) const616 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
617 KnownBits &Known,
618 TargetLoweringOpt &TLO,
619 unsigned Depth,
620 bool AssumeSingleUse) const {
621 EVT VT = Op.getValueType();
622
623 // TODO: We can probably do more work on calculating the known bits and
624 // simplifying the operations for scalable vectors, but for now we just
625 // bail out.
626 if (VT.isScalableVector()) {
627 // Pretend we don't know anything for now.
628 Known = KnownBits(DemandedBits.getBitWidth());
629 return false;
630 }
631
632 APInt DemandedElts = VT.isVector()
633 ? APInt::getAllOnesValue(VT.getVectorNumElements())
634 : APInt(1, 1);
635 return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
636 AssumeSingleUse);
637 }
638
639 // TODO: Can we merge SelectionDAG::GetDemandedBits into this?
640 // TODO: Under what circumstances can we create nodes? Constant folding?
SimplifyMultipleUseDemandedBits(SDValue Op,const APInt & DemandedBits,const APInt & DemandedElts,SelectionDAG & DAG,unsigned Depth) const641 SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
642 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
643 SelectionDAG &DAG, unsigned Depth) const {
644 // Limit search depth.
645 if (Depth >= SelectionDAG::MaxRecursionDepth)
646 return SDValue();
647
648 // Ignore UNDEFs.
649 if (Op.isUndef())
650 return SDValue();
651
652 // Not demanding any bits/elts from Op.
653 if (DemandedBits == 0 || DemandedElts == 0)
654 return DAG.getUNDEF(Op.getValueType());
655
656 unsigned NumElts = DemandedElts.getBitWidth();
657 unsigned BitWidth = DemandedBits.getBitWidth();
658 KnownBits LHSKnown, RHSKnown;
659 switch (Op.getOpcode()) {
660 case ISD::BITCAST: {
661 SDValue Src = peekThroughBitcasts(Op.getOperand(0));
662 EVT SrcVT = Src.getValueType();
663 EVT DstVT = Op.getValueType();
664 if (SrcVT == DstVT)
665 return Src;
666
667 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
668 unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
669 if (NumSrcEltBits == NumDstEltBits)
670 if (SDValue V = SimplifyMultipleUseDemandedBits(
671 Src, DemandedBits, DemandedElts, DAG, Depth + 1))
672 return DAG.getBitcast(DstVT, V);
673
674 // TODO - bigendian once we have test coverage.
675 if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
676 DAG.getDataLayout().isLittleEndian()) {
677 unsigned Scale = NumDstEltBits / NumSrcEltBits;
678 unsigned NumSrcElts = SrcVT.getVectorNumElements();
679 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
680 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
681 for (unsigned i = 0; i != Scale; ++i) {
682 unsigned Offset = i * NumSrcEltBits;
683 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
684 if (!Sub.isNullValue()) {
685 DemandedSrcBits |= Sub;
686 for (unsigned j = 0; j != NumElts; ++j)
687 if (DemandedElts[j])
688 DemandedSrcElts.setBit((j * Scale) + i);
689 }
690 }
691
692 if (SDValue V = SimplifyMultipleUseDemandedBits(
693 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
694 return DAG.getBitcast(DstVT, V);
695 }
696
697 // TODO - bigendian once we have test coverage.
698 if ((NumSrcEltBits % NumDstEltBits) == 0 &&
699 DAG.getDataLayout().isLittleEndian()) {
700 unsigned Scale = NumSrcEltBits / NumDstEltBits;
701 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
702 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
703 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
704 for (unsigned i = 0; i != NumElts; ++i)
705 if (DemandedElts[i]) {
706 unsigned Offset = (i % Scale) * NumDstEltBits;
707 DemandedSrcBits.insertBits(DemandedBits, Offset);
708 DemandedSrcElts.setBit(i / Scale);
709 }
710
711 if (SDValue V = SimplifyMultipleUseDemandedBits(
712 Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
713 return DAG.getBitcast(DstVT, V);
714 }
715
716 break;
717 }
718 case ISD::AND: {
719 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
720 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
721
722 // If all of the demanded bits are known 1 on one side, return the other.
723 // These bits cannot contribute to the result of the 'and' in this
724 // context.
725 if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
726 return Op.getOperand(0);
727 if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
728 return Op.getOperand(1);
729 break;
730 }
731 case ISD::OR: {
732 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
733 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
734
735 // If all of the demanded bits are known zero on one side, return the
736 // other. These bits cannot contribute to the result of the 'or' in this
737 // context.
738 if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
739 return Op.getOperand(0);
740 if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
741 return Op.getOperand(1);
742 break;
743 }
744 case ISD::XOR: {
745 LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
746 RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
747
748 // If all of the demanded bits are known zero on one side, return the
749 // other.
750 if (DemandedBits.isSubsetOf(RHSKnown.Zero))
751 return Op.getOperand(0);
752 if (DemandedBits.isSubsetOf(LHSKnown.Zero))
753 return Op.getOperand(1);
754 break;
755 }
756 case ISD::SHL: {
757 // If we are only demanding sign bits then we can use the shift source
758 // directly.
759 if (const APInt *MaxSA =
760 DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
761 SDValue Op0 = Op.getOperand(0);
762 unsigned ShAmt = MaxSA->getZExtValue();
763 unsigned NumSignBits =
764 DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
765 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
766 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
767 return Op0;
768 }
769 break;
770 }
771 case ISD::SETCC: {
772 SDValue Op0 = Op.getOperand(0);
773 SDValue Op1 = Op.getOperand(1);
774 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
775 // If (1) we only need the sign-bit, (2) the setcc operands are the same
776 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
777 // -1, we may be able to bypass the setcc.
778 if (DemandedBits.isSignMask() &&
779 Op0.getScalarValueSizeInBits() == BitWidth &&
780 getBooleanContents(Op0.getValueType()) ==
781 BooleanContent::ZeroOrNegativeOneBooleanContent) {
782 // If we're testing X < 0, then this compare isn't needed - just use X!
783 // FIXME: We're limiting to integer types here, but this should also work
784 // if we don't care about FP signed-zero. The use of SETLT with FP means
785 // that we don't care about NaNs.
786 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
787 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
788 return Op0;
789 }
790 break;
791 }
792 case ISD::SIGN_EXTEND_INREG: {
793 // If none of the extended bits are demanded, eliminate the sextinreg.
794 SDValue Op0 = Op.getOperand(0);
795 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
796 unsigned ExBits = ExVT.getScalarSizeInBits();
797 if (DemandedBits.getActiveBits() <= ExBits)
798 return Op0;
799 // If the input is already sign extended, just drop the extension.
800 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
801 if (NumSignBits >= (BitWidth - ExBits + 1))
802 return Op0;
803 break;
804 }
805 case ISD::ANY_EXTEND_VECTOR_INREG:
806 case ISD::SIGN_EXTEND_VECTOR_INREG:
807 case ISD::ZERO_EXTEND_VECTOR_INREG: {
808 // If we only want the lowest element and none of extended bits, then we can
809 // return the bitcasted source vector.
810 SDValue Src = Op.getOperand(0);
811 EVT SrcVT = Src.getValueType();
812 EVT DstVT = Op.getValueType();
813 if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
814 DAG.getDataLayout().isLittleEndian() &&
815 DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
816 return DAG.getBitcast(DstVT, Src);
817 }
818 break;
819 }
820 case ISD::INSERT_VECTOR_ELT: {
821 // If we don't demand the inserted element, return the base vector.
822 SDValue Vec = Op.getOperand(0);
823 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
824 EVT VecVT = Vec.getValueType();
825 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
826 !DemandedElts[CIdx->getZExtValue()])
827 return Vec;
828 break;
829 }
830 case ISD::INSERT_SUBVECTOR: {
831 // If we don't demand the inserted subvector, return the base vector.
832 SDValue Vec = Op.getOperand(0);
833 SDValue Sub = Op.getOperand(1);
834 uint64_t Idx = Op.getConstantOperandVal(2);
835 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
836 if (DemandedElts.extractBits(NumSubElts, Idx) == 0)
837 return Vec;
838 break;
839 }
840 case ISD::VECTOR_SHUFFLE: {
841 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
842
843 // If all the demanded elts are from one operand and are inline,
844 // then we can use the operand directly.
845 bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
846 for (unsigned i = 0; i != NumElts; ++i) {
847 int M = ShuffleMask[i];
848 if (M < 0 || !DemandedElts[i])
849 continue;
850 AllUndef = false;
851 IdentityLHS &= (M == (int)i);
852 IdentityRHS &= ((M - NumElts) == i);
853 }
854
855 if (AllUndef)
856 return DAG.getUNDEF(Op.getValueType());
857 if (IdentityLHS)
858 return Op.getOperand(0);
859 if (IdentityRHS)
860 return Op.getOperand(1);
861 break;
862 }
863 default:
864 if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
865 if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
866 Op, DemandedBits, DemandedElts, DAG, Depth))
867 return V;
868 break;
869 }
870 return SDValue();
871 }
872
SimplifyMultipleUseDemandedBits(SDValue Op,const APInt & DemandedBits,SelectionDAG & DAG,unsigned Depth) const873 SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
874 SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
875 unsigned Depth) const {
876 EVT VT = Op.getValueType();
877 APInt DemandedElts = VT.isVector()
878 ? APInt::getAllOnesValue(VT.getVectorNumElements())
879 : APInt(1, 1);
880 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
881 Depth);
882 }
883
SimplifyMultipleUseDemandedVectorElts(SDValue Op,const APInt & DemandedElts,SelectionDAG & DAG,unsigned Depth) const884 SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
885 SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
886 unsigned Depth) const {
887 APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());
888 return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
889 Depth);
890 }
891
892 /// Look at Op. At this point, we know that only the OriginalDemandedBits of the
893 /// result of Op are ever used downstream. If we can use this information to
894 /// simplify Op, create a new simplified DAG node and return true, returning the
895 /// original and new nodes in Old and New. Otherwise, analyze the expression and
896 /// return a mask of Known bits for the expression (used to simplify the
897 /// caller). The Known bits may only be accurate for those bits in the
898 /// OriginalDemandedBits and OriginalDemandedElts.
SimplifyDemandedBits(SDValue Op,const APInt & OriginalDemandedBits,const APInt & OriginalDemandedElts,KnownBits & Known,TargetLoweringOpt & TLO,unsigned Depth,bool AssumeSingleUse) const899 bool TargetLowering::SimplifyDemandedBits(
900 SDValue Op, const APInt &OriginalDemandedBits,
901 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
902 unsigned Depth, bool AssumeSingleUse) const {
903 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
904 assert(Op.getScalarValueSizeInBits() == BitWidth &&
905 "Mask size mismatches value type size!");
906
907 // Don't know anything.
908 Known = KnownBits(BitWidth);
909
910 // TODO: We can probably do more work on calculating the known bits and
911 // simplifying the operations for scalable vectors, but for now we just
912 // bail out.
913 if (Op.getValueType().isScalableVector())
914 return false;
915
916 unsigned NumElts = OriginalDemandedElts.getBitWidth();
917 assert((!Op.getValueType().isVector() ||
918 NumElts == Op.getValueType().getVectorNumElements()) &&
919 "Unexpected vector size");
920
921 APInt DemandedBits = OriginalDemandedBits;
922 APInt DemandedElts = OriginalDemandedElts;
923 SDLoc dl(Op);
924 auto &DL = TLO.DAG.getDataLayout();
925
926 // Undef operand.
927 if (Op.isUndef())
928 return false;
929
930 if (Op.getOpcode() == ISD::Constant) {
931 // We know all of the bits for a constant!
932 Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
933 return false;
934 }
935
936 if (Op.getOpcode() == ISD::ConstantFP) {
937 // We know all of the bits for a floating point constant!
938 Known = KnownBits::makeConstant(
939 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
940 return false;
941 }
942
943 // Other users may use these bits.
944 EVT VT = Op.getValueType();
945 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
946 if (Depth != 0) {
947 // If not at the root, Just compute the Known bits to
948 // simplify things downstream.
949 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
950 return false;
951 }
952 // If this is the root being simplified, allow it to have multiple uses,
953 // just set the DemandedBits/Elts to all bits.
954 DemandedBits = APInt::getAllOnesValue(BitWidth);
955 DemandedElts = APInt::getAllOnesValue(NumElts);
956 } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
957 // Not demanding any bits/elts from Op.
958 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
959 } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
960 // Limit search depth.
961 return false;
962 }
963
964 KnownBits Known2;
965 switch (Op.getOpcode()) {
966 case ISD::TargetConstant:
967 llvm_unreachable("Can't simplify this node");
968 case ISD::SCALAR_TO_VECTOR: {
969 if (!DemandedElts[0])
970 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
971
972 KnownBits SrcKnown;
973 SDValue Src = Op.getOperand(0);
974 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
975 APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
976 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
977 return true;
978
979 // Upper elements are undef, so only get the knownbits if we just demand
980 // the bottom element.
981 if (DemandedElts == 1)
982 Known = SrcKnown.anyextOrTrunc(BitWidth);
983 break;
984 }
985 case ISD::BUILD_VECTOR:
986 // Collect the known bits that are shared by every demanded element.
987 // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
988 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
989 return false; // Don't fall through, will infinitely loop.
990 case ISD::LOAD: {
991 auto *LD = cast<LoadSDNode>(Op);
992 if (getTargetConstantFromLoad(LD)) {
993 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
994 return false; // Don't fall through, will infinitely loop.
995 }
996 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
997 // If this is a ZEXTLoad and we are looking at the loaded value.
998 EVT MemVT = LD->getMemoryVT();
999 unsigned MemBits = MemVT.getScalarSizeInBits();
1000 Known.Zero.setBitsFrom(MemBits);
1001 return false; // Don't fall through, will infinitely loop.
1002 }
1003 break;
1004 }
1005 case ISD::INSERT_VECTOR_ELT: {
1006 SDValue Vec = Op.getOperand(0);
1007 SDValue Scl = Op.getOperand(1);
1008 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1009 EVT VecVT = Vec.getValueType();
1010
1011 // If index isn't constant, assume we need all vector elements AND the
1012 // inserted element.
1013 APInt DemandedVecElts(DemandedElts);
1014 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1015 unsigned Idx = CIdx->getZExtValue();
1016 DemandedVecElts.clearBit(Idx);
1017
1018 // Inserted element is not required.
1019 if (!DemandedElts[Idx])
1020 return TLO.CombineTo(Op, Vec);
1021 }
1022
1023 KnownBits KnownScl;
1024 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1025 APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1026 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1027 return true;
1028
1029 Known = KnownScl.anyextOrTrunc(BitWidth);
1030
1031 KnownBits KnownVec;
1032 if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1033 Depth + 1))
1034 return true;
1035
1036 if (!!DemandedVecElts)
1037 Known = KnownBits::commonBits(Known, KnownVec);
1038
1039 return false;
1040 }
1041 case ISD::INSERT_SUBVECTOR: {
1042 // Demand any elements from the subvector and the remainder from the src its
1043 // inserted into.
1044 SDValue Src = Op.getOperand(0);
1045 SDValue Sub = Op.getOperand(1);
1046 uint64_t Idx = Op.getConstantOperandVal(2);
1047 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1048 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1049 APInt DemandedSrcElts = DemandedElts;
1050 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
1051
1052 KnownBits KnownSub, KnownSrc;
1053 if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1054 Depth + 1))
1055 return true;
1056 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1057 Depth + 1))
1058 return true;
1059
1060 Known.Zero.setAllBits();
1061 Known.One.setAllBits();
1062 if (!!DemandedSubElts)
1063 Known = KnownBits::commonBits(Known, KnownSub);
1064 if (!!DemandedSrcElts)
1065 Known = KnownBits::commonBits(Known, KnownSrc);
1066
1067 // Attempt to avoid multi-use src if we don't need anything from it.
1068 if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||
1069 !DemandedSrcElts.isAllOnesValue()) {
1070 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1071 Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1072 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1073 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1074 if (NewSub || NewSrc) {
1075 NewSub = NewSub ? NewSub : Sub;
1076 NewSrc = NewSrc ? NewSrc : Src;
1077 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1078 Op.getOperand(2));
1079 return TLO.CombineTo(Op, NewOp);
1080 }
1081 }
1082 break;
1083 }
1084 case ISD::EXTRACT_SUBVECTOR: {
1085 // Offset the demanded elts by the subvector index.
1086 SDValue Src = Op.getOperand(0);
1087 if (Src.getValueType().isScalableVector())
1088 break;
1089 uint64_t Idx = Op.getConstantOperandVal(1);
1090 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1091 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1092
1093 if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1094 Depth + 1))
1095 return true;
1096
1097 // Attempt to avoid multi-use src if we don't need anything from it.
1098 if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {
1099 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1100 Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1101 if (DemandedSrc) {
1102 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1103 Op.getOperand(1));
1104 return TLO.CombineTo(Op, NewOp);
1105 }
1106 }
1107 break;
1108 }
1109 case ISD::CONCAT_VECTORS: {
1110 Known.Zero.setAllBits();
1111 Known.One.setAllBits();
1112 EVT SubVT = Op.getOperand(0).getValueType();
1113 unsigned NumSubVecs = Op.getNumOperands();
1114 unsigned NumSubElts = SubVT.getVectorNumElements();
1115 for (unsigned i = 0; i != NumSubVecs; ++i) {
1116 APInt DemandedSubElts =
1117 DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1118 if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1119 Known2, TLO, Depth + 1))
1120 return true;
1121 // Known bits are shared by every demanded subvector element.
1122 if (!!DemandedSubElts)
1123 Known = KnownBits::commonBits(Known, Known2);
1124 }
1125 break;
1126 }
1127 case ISD::VECTOR_SHUFFLE: {
1128 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1129
1130 // Collect demanded elements from shuffle operands..
1131 APInt DemandedLHS(NumElts, 0);
1132 APInt DemandedRHS(NumElts, 0);
1133 for (unsigned i = 0; i != NumElts; ++i) {
1134 if (!DemandedElts[i])
1135 continue;
1136 int M = ShuffleMask[i];
1137 if (M < 0) {
1138 // For UNDEF elements, we don't know anything about the common state of
1139 // the shuffle result.
1140 DemandedLHS.clearAllBits();
1141 DemandedRHS.clearAllBits();
1142 break;
1143 }
1144 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1145 if (M < (int)NumElts)
1146 DemandedLHS.setBit(M);
1147 else
1148 DemandedRHS.setBit(M - NumElts);
1149 }
1150
1151 if (!!DemandedLHS || !!DemandedRHS) {
1152 SDValue Op0 = Op.getOperand(0);
1153 SDValue Op1 = Op.getOperand(1);
1154
1155 Known.Zero.setAllBits();
1156 Known.One.setAllBits();
1157 if (!!DemandedLHS) {
1158 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1159 Depth + 1))
1160 return true;
1161 Known = KnownBits::commonBits(Known, Known2);
1162 }
1163 if (!!DemandedRHS) {
1164 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1165 Depth + 1))
1166 return true;
1167 Known = KnownBits::commonBits(Known, Known2);
1168 }
1169
1170 // Attempt to avoid multi-use ops if we don't need anything from them.
1171 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1172 Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1173 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1174 Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1175 if (DemandedOp0 || DemandedOp1) {
1176 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1177 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1178 SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1179 return TLO.CombineTo(Op, NewOp);
1180 }
1181 }
1182 break;
1183 }
1184 case ISD::AND: {
1185 SDValue Op0 = Op.getOperand(0);
1186 SDValue Op1 = Op.getOperand(1);
1187
1188 // If the RHS is a constant, check to see if the LHS would be zero without
1189 // using the bits from the RHS. Below, we use knowledge about the RHS to
1190 // simplify the LHS, here we're using information from the LHS to simplify
1191 // the RHS.
1192 if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1193 // Do not increment Depth here; that can cause an infinite loop.
1194 KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1195 // If the LHS already has zeros where RHSC does, this 'and' is dead.
1196 if ((LHSKnown.Zero & DemandedBits) ==
1197 (~RHSC->getAPIntValue() & DemandedBits))
1198 return TLO.CombineTo(Op, Op0);
1199
1200 // If any of the set bits in the RHS are known zero on the LHS, shrink
1201 // the constant.
1202 if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1203 DemandedElts, TLO))
1204 return true;
1205
1206 // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1207 // constant, but if this 'and' is only clearing bits that were just set by
1208 // the xor, then this 'and' can be eliminated by shrinking the mask of
1209 // the xor. For example, for a 32-bit X:
1210 // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1211 if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1212 LHSKnown.One == ~RHSC->getAPIntValue()) {
1213 SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1214 return TLO.CombineTo(Op, Xor);
1215 }
1216 }
1217
1218 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1219 Depth + 1))
1220 return true;
1221 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1222 if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1223 Known2, TLO, Depth + 1))
1224 return true;
1225 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1226
1227 // Attempt to avoid multi-use ops if we don't need anything from them.
1228 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1229 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1230 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1231 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1232 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1233 if (DemandedOp0 || DemandedOp1) {
1234 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1235 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1236 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1237 return TLO.CombineTo(Op, NewOp);
1238 }
1239 }
1240
1241 // If all of the demanded bits are known one on one side, return the other.
1242 // These bits cannot contribute to the result of the 'and'.
1243 if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1244 return TLO.CombineTo(Op, Op0);
1245 if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1246 return TLO.CombineTo(Op, Op1);
1247 // If all of the demanded bits in the inputs are known zeros, return zero.
1248 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1249 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1250 // If the RHS is a constant, see if we can simplify it.
1251 if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1252 TLO))
1253 return true;
1254 // If the operation can be done in a smaller type, do so.
1255 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1256 return true;
1257
1258 Known &= Known2;
1259 break;
1260 }
1261 case ISD::OR: {
1262 SDValue Op0 = Op.getOperand(0);
1263 SDValue Op1 = Op.getOperand(1);
1264
1265 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1266 Depth + 1))
1267 return true;
1268 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1269 if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1270 Known2, TLO, Depth + 1))
1271 return true;
1272 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1273
1274 // Attempt to avoid multi-use ops if we don't need anything from them.
1275 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1276 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1277 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1278 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1279 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1280 if (DemandedOp0 || DemandedOp1) {
1281 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1282 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1283 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1284 return TLO.CombineTo(Op, NewOp);
1285 }
1286 }
1287
1288 // If all of the demanded bits are known zero on one side, return the other.
1289 // These bits cannot contribute to the result of the 'or'.
1290 if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1291 return TLO.CombineTo(Op, Op0);
1292 if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1293 return TLO.CombineTo(Op, Op1);
1294 // If the RHS is a constant, see if we can simplify it.
1295 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1296 return true;
1297 // If the operation can be done in a smaller type, do so.
1298 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1299 return true;
1300
1301 Known |= Known2;
1302 break;
1303 }
1304 case ISD::XOR: {
1305 SDValue Op0 = Op.getOperand(0);
1306 SDValue Op1 = Op.getOperand(1);
1307
1308 if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1309 Depth + 1))
1310 return true;
1311 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1312 if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1313 Depth + 1))
1314 return true;
1315 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1316
1317 // Attempt to avoid multi-use ops if we don't need anything from them.
1318 if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1319 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1320 Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1321 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1322 Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1323 if (DemandedOp0 || DemandedOp1) {
1324 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1325 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1326 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1327 return TLO.CombineTo(Op, NewOp);
1328 }
1329 }
1330
1331 // If all of the demanded bits are known zero on one side, return the other.
1332 // These bits cannot contribute to the result of the 'xor'.
1333 if (DemandedBits.isSubsetOf(Known.Zero))
1334 return TLO.CombineTo(Op, Op0);
1335 if (DemandedBits.isSubsetOf(Known2.Zero))
1336 return TLO.CombineTo(Op, Op1);
1337 // If the operation can be done in a smaller type, do so.
1338 if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1339 return true;
1340
1341 // If all of the unknown bits are known to be zero on one side or the other
1342 // turn this into an *inclusive* or.
1343 // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1344 if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1345 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1346
1347 ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1348 if (C) {
1349 // If one side is a constant, and all of the set bits in the constant are
1350 // also known set on the other side, turn this into an AND, as we know
1351 // the bits will be cleared.
1352 // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1353 // NB: it is okay if more bits are known than are requested
1354 if (C->getAPIntValue() == Known2.One) {
1355 SDValue ANDC =
1356 TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1357 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1358 }
1359
1360 // If the RHS is a constant, see if we can change it. Don't alter a -1
1361 // constant because that's a 'not' op, and that is better for combining
1362 // and codegen.
1363 if (!C->isAllOnesValue() &&
1364 DemandedBits.isSubsetOf(C->getAPIntValue())) {
1365 // We're flipping all demanded bits. Flip the undemanded bits too.
1366 SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1367 return TLO.CombineTo(Op, New);
1368 }
1369 }
1370
1371 // If we can't turn this into a 'not', try to shrink the constant.
1372 if (!C || !C->isAllOnesValue())
1373 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1374 return true;
1375
1376 Known ^= Known2;
1377 break;
1378 }
1379 case ISD::SELECT:
1380 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1381 Depth + 1))
1382 return true;
1383 if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1384 Depth + 1))
1385 return true;
1386 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1387 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1388
1389 // If the operands are constants, see if we can simplify them.
1390 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1391 return true;
1392
1393 // Only known if known in both the LHS and RHS.
1394 Known = KnownBits::commonBits(Known, Known2);
1395 break;
1396 case ISD::SELECT_CC:
1397 if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1398 Depth + 1))
1399 return true;
1400 if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1401 Depth + 1))
1402 return true;
1403 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1404 assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1405
1406 // If the operands are constants, see if we can simplify them.
1407 if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1408 return true;
1409
1410 // Only known if known in both the LHS and RHS.
1411 Known = KnownBits::commonBits(Known, Known2);
1412 break;
1413 case ISD::SETCC: {
1414 SDValue Op0 = Op.getOperand(0);
1415 SDValue Op1 = Op.getOperand(1);
1416 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1417 // If (1) we only need the sign-bit, (2) the setcc operands are the same
1418 // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1419 // -1, we may be able to bypass the setcc.
1420 if (DemandedBits.isSignMask() &&
1421 Op0.getScalarValueSizeInBits() == BitWidth &&
1422 getBooleanContents(Op0.getValueType()) ==
1423 BooleanContent::ZeroOrNegativeOneBooleanContent) {
1424 // If we're testing X < 0, then this compare isn't needed - just use X!
1425 // FIXME: We're limiting to integer types here, but this should also work
1426 // if we don't care about FP signed-zero. The use of SETLT with FP means
1427 // that we don't care about NaNs.
1428 if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1429 (isNullConstant(Op1) || ISD::isBuildVectorAllZeros(Op1.getNode())))
1430 return TLO.CombineTo(Op, Op0);
1431
1432 // TODO: Should we check for other forms of sign-bit comparisons?
1433 // Examples: X <= -1, X >= 0
1434 }
1435 if (getBooleanContents(Op0.getValueType()) ==
1436 TargetLowering::ZeroOrOneBooleanContent &&
1437 BitWidth > 1)
1438 Known.Zero.setBitsFrom(1);
1439 break;
1440 }
1441 case ISD::SHL: {
1442 SDValue Op0 = Op.getOperand(0);
1443 SDValue Op1 = Op.getOperand(1);
1444 EVT ShiftVT = Op1.getValueType();
1445
1446 if (const APInt *SA =
1447 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1448 unsigned ShAmt = SA->getZExtValue();
1449 if (ShAmt == 0)
1450 return TLO.CombineTo(Op, Op0);
1451
1452 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1453 // single shift. We can do this if the bottom bits (which are shifted
1454 // out) are never demanded.
1455 // TODO - support non-uniform vector amounts.
1456 if (Op0.getOpcode() == ISD::SRL) {
1457 if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1458 if (const APInt *SA2 =
1459 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1460 unsigned C1 = SA2->getZExtValue();
1461 unsigned Opc = ISD::SHL;
1462 int Diff = ShAmt - C1;
1463 if (Diff < 0) {
1464 Diff = -Diff;
1465 Opc = ISD::SRL;
1466 }
1467 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1468 return TLO.CombineTo(
1469 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1470 }
1471 }
1472 }
1473
1474 // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1475 // are not demanded. This will likely allow the anyext to be folded away.
1476 // TODO - support non-uniform vector amounts.
1477 if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1478 SDValue InnerOp = Op0.getOperand(0);
1479 EVT InnerVT = InnerOp.getValueType();
1480 unsigned InnerBits = InnerVT.getScalarSizeInBits();
1481 if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1482 isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1483 EVT ShTy = getShiftAmountTy(InnerVT, DL);
1484 if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1485 ShTy = InnerVT;
1486 SDValue NarrowShl =
1487 TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1488 TLO.DAG.getConstant(ShAmt, dl, ShTy));
1489 return TLO.CombineTo(
1490 Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1491 }
1492
1493 // Repeat the SHL optimization above in cases where an extension
1494 // intervenes: (shl (anyext (shr x, c1)), c2) to
1495 // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1496 // aren't demanded (as above) and that the shifted upper c1 bits of
1497 // x aren't demanded.
1498 // TODO - support non-uniform vector amounts.
1499 if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1500 InnerOp.hasOneUse()) {
1501 if (const APInt *SA2 =
1502 TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1503 unsigned InnerShAmt = SA2->getZExtValue();
1504 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1505 DemandedBits.getActiveBits() <=
1506 (InnerBits - InnerShAmt + ShAmt) &&
1507 DemandedBits.countTrailingZeros() >= ShAmt) {
1508 SDValue NewSA =
1509 TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1510 SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1511 InnerOp.getOperand(0));
1512 return TLO.CombineTo(
1513 Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1514 }
1515 }
1516 }
1517 }
1518
1519 APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1520 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1521 Depth + 1))
1522 return true;
1523 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1524 Known.Zero <<= ShAmt;
1525 Known.One <<= ShAmt;
1526 // low bits known zero.
1527 Known.Zero.setLowBits(ShAmt);
1528
1529 // Try shrinking the operation as long as the shift amount will still be
1530 // in range.
1531 if ((ShAmt < DemandedBits.getActiveBits()) &&
1532 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1533 return true;
1534 }
1535
1536 // If we are only demanding sign bits then we can use the shift source
1537 // directly.
1538 if (const APInt *MaxSA =
1539 TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1540 unsigned ShAmt = MaxSA->getZExtValue();
1541 unsigned NumSignBits =
1542 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1543 unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1544 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1545 return TLO.CombineTo(Op, Op0);
1546 }
1547 break;
1548 }
1549 case ISD::SRL: {
1550 SDValue Op0 = Op.getOperand(0);
1551 SDValue Op1 = Op.getOperand(1);
1552 EVT ShiftVT = Op1.getValueType();
1553
1554 if (const APInt *SA =
1555 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1556 unsigned ShAmt = SA->getZExtValue();
1557 if (ShAmt == 0)
1558 return TLO.CombineTo(Op, Op0);
1559
1560 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1561 // single shift. We can do this if the top bits (which are shifted out)
1562 // are never demanded.
1563 // TODO - support non-uniform vector amounts.
1564 if (Op0.getOpcode() == ISD::SHL) {
1565 if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1566 if (const APInt *SA2 =
1567 TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1568 unsigned C1 = SA2->getZExtValue();
1569 unsigned Opc = ISD::SRL;
1570 int Diff = ShAmt - C1;
1571 if (Diff < 0) {
1572 Diff = -Diff;
1573 Opc = ISD::SHL;
1574 }
1575 SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1576 return TLO.CombineTo(
1577 Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1578 }
1579 }
1580 }
1581
1582 APInt InDemandedMask = (DemandedBits << ShAmt);
1583
1584 // If the shift is exact, then it does demand the low bits (and knows that
1585 // they are zero).
1586 if (Op->getFlags().hasExact())
1587 InDemandedMask.setLowBits(ShAmt);
1588
1589 // Compute the new bits that are at the top now.
1590 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1591 Depth + 1))
1592 return true;
1593 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1594 Known.Zero.lshrInPlace(ShAmt);
1595 Known.One.lshrInPlace(ShAmt);
1596 // High bits known zero.
1597 Known.Zero.setHighBits(ShAmt);
1598 }
1599 break;
1600 }
1601 case ISD::SRA: {
1602 SDValue Op0 = Op.getOperand(0);
1603 SDValue Op1 = Op.getOperand(1);
1604 EVT ShiftVT = Op1.getValueType();
1605
1606 // If we only want bits that already match the signbit then we don't need
1607 // to shift.
1608 unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1609 if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1610 NumHiDemandedBits)
1611 return TLO.CombineTo(Op, Op0);
1612
1613 // If this is an arithmetic shift right and only the low-bit is set, we can
1614 // always convert this into a logical shr, even if the shift amount is
1615 // variable. The low bit of the shift cannot be an input sign bit unless
1616 // the shift amount is >= the size of the datatype, which is undefined.
1617 if (DemandedBits.isOneValue())
1618 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1619
1620 if (const APInt *SA =
1621 TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1622 unsigned ShAmt = SA->getZExtValue();
1623 if (ShAmt == 0)
1624 return TLO.CombineTo(Op, Op0);
1625
1626 APInt InDemandedMask = (DemandedBits << ShAmt);
1627
1628 // If the shift is exact, then it does demand the low bits (and knows that
1629 // they are zero).
1630 if (Op->getFlags().hasExact())
1631 InDemandedMask.setLowBits(ShAmt);
1632
1633 // If any of the demanded bits are produced by the sign extension, we also
1634 // demand the input sign bit.
1635 if (DemandedBits.countLeadingZeros() < ShAmt)
1636 InDemandedMask.setSignBit();
1637
1638 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1639 Depth + 1))
1640 return true;
1641 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1642 Known.Zero.lshrInPlace(ShAmt);
1643 Known.One.lshrInPlace(ShAmt);
1644
1645 // If the input sign bit is known to be zero, or if none of the top bits
1646 // are demanded, turn this into an unsigned shift right.
1647 if (Known.Zero[BitWidth - ShAmt - 1] ||
1648 DemandedBits.countLeadingZeros() >= ShAmt) {
1649 SDNodeFlags Flags;
1650 Flags.setExact(Op->getFlags().hasExact());
1651 return TLO.CombineTo(
1652 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1653 }
1654
1655 int Log2 = DemandedBits.exactLogBase2();
1656 if (Log2 >= 0) {
1657 // The bit must come from the sign.
1658 SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1659 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1660 }
1661
1662 if (Known.One[BitWidth - ShAmt - 1])
1663 // New bits are known one.
1664 Known.One.setHighBits(ShAmt);
1665
1666 // Attempt to avoid multi-use ops if we don't need anything from them.
1667 if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1668 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1669 Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1670 if (DemandedOp0) {
1671 SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1672 return TLO.CombineTo(Op, NewOp);
1673 }
1674 }
1675 }
1676 break;
1677 }
1678 case ISD::FSHL:
1679 case ISD::FSHR: {
1680 SDValue Op0 = Op.getOperand(0);
1681 SDValue Op1 = Op.getOperand(1);
1682 SDValue Op2 = Op.getOperand(2);
1683 bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1684
1685 if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1686 unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1687
1688 // For fshl, 0-shift returns the 1st arg.
1689 // For fshr, 0-shift returns the 2nd arg.
1690 if (Amt == 0) {
1691 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1692 Known, TLO, Depth + 1))
1693 return true;
1694 break;
1695 }
1696
1697 // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1698 // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1699 APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1700 APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1701 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1702 Depth + 1))
1703 return true;
1704 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1705 Depth + 1))
1706 return true;
1707
1708 Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1709 Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1710 Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1711 Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1712 Known.One |= Known2.One;
1713 Known.Zero |= Known2.Zero;
1714 }
1715
1716 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1717 if (isPowerOf2_32(BitWidth)) {
1718 APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1719 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1720 Known2, TLO, Depth + 1))
1721 return true;
1722 }
1723 break;
1724 }
1725 case ISD::ROTL:
1726 case ISD::ROTR: {
1727 SDValue Op0 = Op.getOperand(0);
1728 SDValue Op1 = Op.getOperand(1);
1729
1730 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1731 if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1732 return TLO.CombineTo(Op, Op0);
1733
1734 // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1735 if (isPowerOf2_32(BitWidth)) {
1736 APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1737 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1738 Depth + 1))
1739 return true;
1740 }
1741 break;
1742 }
1743 case ISD::UMIN: {
1744 // Check if one arg is always less than (or equal) to the other arg.
1745 SDValue Op0 = Op.getOperand(0);
1746 SDValue Op1 = Op.getOperand(1);
1747 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1748 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1749 Known = KnownBits::umin(Known0, Known1);
1750 if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1751 return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1752 if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1753 return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1754 break;
1755 }
1756 case ISD::UMAX: {
1757 // Check if one arg is always greater than (or equal) to the other arg.
1758 SDValue Op0 = Op.getOperand(0);
1759 SDValue Op1 = Op.getOperand(1);
1760 KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1761 KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1762 Known = KnownBits::umax(Known0, Known1);
1763 if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1764 return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1765 if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1766 return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1767 break;
1768 }
1769 case ISD::BITREVERSE: {
1770 SDValue Src = Op.getOperand(0);
1771 APInt DemandedSrcBits = DemandedBits.reverseBits();
1772 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1773 Depth + 1))
1774 return true;
1775 Known.One = Known2.One.reverseBits();
1776 Known.Zero = Known2.Zero.reverseBits();
1777 break;
1778 }
1779 case ISD::BSWAP: {
1780 SDValue Src = Op.getOperand(0);
1781 APInt DemandedSrcBits = DemandedBits.byteSwap();
1782 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1783 Depth + 1))
1784 return true;
1785 Known.One = Known2.One.byteSwap();
1786 Known.Zero = Known2.Zero.byteSwap();
1787 break;
1788 }
1789 case ISD::CTPOP: {
1790 // If only 1 bit is demanded, replace with PARITY as long as we're before
1791 // op legalization.
1792 // FIXME: Limit to scalars for now.
1793 if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())
1794 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1795 Op.getOperand(0)));
1796
1797 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1798 break;
1799 }
1800 case ISD::SIGN_EXTEND_INREG: {
1801 SDValue Op0 = Op.getOperand(0);
1802 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1803 unsigned ExVTBits = ExVT.getScalarSizeInBits();
1804
1805 // If we only care about the highest bit, don't bother shifting right.
1806 if (DemandedBits.isSignMask()) {
1807 unsigned NumSignBits =
1808 TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1809 bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1810 // However if the input is already sign extended we expect the sign
1811 // extension to be dropped altogether later and do not simplify.
1812 if (!AlreadySignExtended) {
1813 // Compute the correct shift amount type, which must be getShiftAmountTy
1814 // for scalar types after legalization.
1815 EVT ShiftAmtTy = VT;
1816 if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1817 ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1818
1819 SDValue ShiftAmt =
1820 TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1821 return TLO.CombineTo(Op,
1822 TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1823 }
1824 }
1825
1826 // If none of the extended bits are demanded, eliminate the sextinreg.
1827 if (DemandedBits.getActiveBits() <= ExVTBits)
1828 return TLO.CombineTo(Op, Op0);
1829
1830 APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1831
1832 // Since the sign extended bits are demanded, we know that the sign
1833 // bit is demanded.
1834 InputDemandedBits.setBit(ExVTBits - 1);
1835
1836 if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1837 return true;
1838 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1839
1840 // If the sign bit of the input is known set or clear, then we know the
1841 // top bits of the result.
1842
1843 // If the input sign bit is known zero, convert this into a zero extension.
1844 if (Known.Zero[ExVTBits - 1])
1845 return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1846
1847 APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1848 if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1849 Known.One.setBitsFrom(ExVTBits);
1850 Known.Zero &= Mask;
1851 } else { // Input sign bit unknown
1852 Known.Zero &= Mask;
1853 Known.One &= Mask;
1854 }
1855 break;
1856 }
1857 case ISD::BUILD_PAIR: {
1858 EVT HalfVT = Op.getOperand(0).getValueType();
1859 unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1860
1861 APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1862 APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1863
1864 KnownBits KnownLo, KnownHi;
1865
1866 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1867 return true;
1868
1869 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1870 return true;
1871
1872 Known.Zero = KnownLo.Zero.zext(BitWidth) |
1873 KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1874
1875 Known.One = KnownLo.One.zext(BitWidth) |
1876 KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1877 break;
1878 }
1879 case ISD::ZERO_EXTEND:
1880 case ISD::ZERO_EXTEND_VECTOR_INREG: {
1881 SDValue Src = Op.getOperand(0);
1882 EVT SrcVT = Src.getValueType();
1883 unsigned InBits = SrcVT.getScalarSizeInBits();
1884 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1885 bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1886
1887 // If none of the top bits are demanded, convert this into an any_extend.
1888 if (DemandedBits.getActiveBits() <= InBits) {
1889 // If we only need the non-extended bits of the bottom element
1890 // then we can just bitcast to the result.
1891 if (IsVecInReg && DemandedElts == 1 &&
1892 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1893 TLO.DAG.getDataLayout().isLittleEndian())
1894 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1895
1896 unsigned Opc =
1897 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1898 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1899 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1900 }
1901
1902 APInt InDemandedBits = DemandedBits.trunc(InBits);
1903 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1904 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1905 Depth + 1))
1906 return true;
1907 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1908 assert(Known.getBitWidth() == InBits && "Src width has changed?");
1909 Known = Known.zext(BitWidth);
1910
1911 // Attempt to avoid multi-use ops if we don't need anything from them.
1912 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1913 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1914 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1915 break;
1916 }
1917 case ISD::SIGN_EXTEND:
1918 case ISD::SIGN_EXTEND_VECTOR_INREG: {
1919 SDValue Src = Op.getOperand(0);
1920 EVT SrcVT = Src.getValueType();
1921 unsigned InBits = SrcVT.getScalarSizeInBits();
1922 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1923 bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1924
1925 // If none of the top bits are demanded, convert this into an any_extend.
1926 if (DemandedBits.getActiveBits() <= InBits) {
1927 // If we only need the non-extended bits of the bottom element
1928 // then we can just bitcast to the result.
1929 if (IsVecInReg && DemandedElts == 1 &&
1930 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1931 TLO.DAG.getDataLayout().isLittleEndian())
1932 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1933
1934 unsigned Opc =
1935 IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND;
1936 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1937 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1938 }
1939
1940 APInt InDemandedBits = DemandedBits.trunc(InBits);
1941 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1942
1943 // Since some of the sign extended bits are demanded, we know that the sign
1944 // bit is demanded.
1945 InDemandedBits.setBit(InBits - 1);
1946
1947 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1948 Depth + 1))
1949 return true;
1950 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1951 assert(Known.getBitWidth() == InBits && "Src width has changed?");
1952
1953 // If the sign bit is known one, the top bits match.
1954 Known = Known.sext(BitWidth);
1955
1956 // If the sign bit is known zero, convert this to a zero extend.
1957 if (Known.isNonNegative()) {
1958 unsigned Opc =
1959 IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND;
1960 if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1961 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1962 }
1963
1964 // Attempt to avoid multi-use ops if we don't need anything from them.
1965 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1966 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1967 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1968 break;
1969 }
1970 case ISD::ANY_EXTEND:
1971 case ISD::ANY_EXTEND_VECTOR_INREG: {
1972 SDValue Src = Op.getOperand(0);
1973 EVT SrcVT = Src.getValueType();
1974 unsigned InBits = SrcVT.getScalarSizeInBits();
1975 unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1976 bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1977
1978 // If we only need the bottom element then we can just bitcast.
1979 // TODO: Handle ANY_EXTEND?
1980 if (IsVecInReg && DemandedElts == 1 &&
1981 VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1982 TLO.DAG.getDataLayout().isLittleEndian())
1983 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1984
1985 APInt InDemandedBits = DemandedBits.trunc(InBits);
1986 APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1987 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1988 Depth + 1))
1989 return true;
1990 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1991 assert(Known.getBitWidth() == InBits && "Src width has changed?");
1992 Known = Known.anyext(BitWidth);
1993
1994 // Attempt to avoid multi-use ops if we don't need anything from them.
1995 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1996 Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1997 return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1998 break;
1999 }
2000 case ISD::TRUNCATE: {
2001 SDValue Src = Op.getOperand(0);
2002
2003 // Simplify the input, using demanded bit information, and compute the known
2004 // zero/one bits live out.
2005 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2006 APInt TruncMask = DemandedBits.zext(OperandBitWidth);
2007 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2008 Depth + 1))
2009 return true;
2010 Known = Known.trunc(BitWidth);
2011
2012 // Attempt to avoid multi-use ops if we don't need anything from them.
2013 if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2014 Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2015 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2016
2017 // If the input is only used by this truncate, see if we can shrink it based
2018 // on the known demanded bits.
2019 if (Src.getNode()->hasOneUse()) {
2020 switch (Src.getOpcode()) {
2021 default:
2022 break;
2023 case ISD::SRL:
2024 // Shrink SRL by a constant if none of the high bits shifted in are
2025 // demanded.
2026 if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2027 // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2028 // undesirable.
2029 break;
2030
2031 const APInt *ShAmtC =
2032 TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2033 if (!ShAmtC || ShAmtC->uge(BitWidth))
2034 break;
2035 uint64_t ShVal = ShAmtC->getZExtValue();
2036
2037 APInt HighBits =
2038 APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2039 HighBits.lshrInPlace(ShVal);
2040 HighBits = HighBits.trunc(BitWidth);
2041
2042 if (!(HighBits & DemandedBits)) {
2043 // None of the shifted in bits are needed. Add a truncate of the
2044 // shift input, then shift it.
2045 SDValue NewShAmt = TLO.DAG.getConstant(
2046 ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2047 SDValue NewTrunc =
2048 TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2049 return TLO.CombineTo(
2050 Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2051 }
2052 break;
2053 }
2054 }
2055
2056 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2057 break;
2058 }
2059 case ISD::AssertZext: {
2060 // AssertZext demands all of the high bits, plus any of the low bits
2061 // demanded by its users.
2062 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2063 APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits());
2064 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2065 TLO, Depth + 1))
2066 return true;
2067 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2068
2069 Known.Zero |= ~InMask;
2070 break;
2071 }
2072 case ISD::EXTRACT_VECTOR_ELT: {
2073 SDValue Src = Op.getOperand(0);
2074 SDValue Idx = Op.getOperand(1);
2075 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2076 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2077
2078 if (SrcEltCnt.isScalable())
2079 return false;
2080
2081 // Demand the bits from every vector element without a constant index.
2082 unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2083 APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
2084 if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2085 if (CIdx->getAPIntValue().ult(NumSrcElts))
2086 DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2087
2088 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2089 // anything about the extended bits.
2090 APInt DemandedSrcBits = DemandedBits;
2091 if (BitWidth > EltBitWidth)
2092 DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2093
2094 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2095 Depth + 1))
2096 return true;
2097
2098 // Attempt to avoid multi-use ops if we don't need anything from them.
2099 if (!DemandedSrcBits.isAllOnesValue() ||
2100 !DemandedSrcElts.isAllOnesValue()) {
2101 if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2102 Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2103 SDValue NewOp =
2104 TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2105 return TLO.CombineTo(Op, NewOp);
2106 }
2107 }
2108
2109 Known = Known2;
2110 if (BitWidth > EltBitWidth)
2111 Known = Known.anyext(BitWidth);
2112 break;
2113 }
2114 case ISD::BITCAST: {
2115 SDValue Src = Op.getOperand(0);
2116 EVT SrcVT = Src.getValueType();
2117 unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2118
2119 // If this is an FP->Int bitcast and if the sign bit is the only
2120 // thing demanded, turn this into a FGETSIGN.
2121 if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2122 DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2123 SrcVT.isFloatingPoint()) {
2124 bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2125 bool i32Legal = isOperationLegalOrCustom(ISD::FGETSIGN, MVT::i32);
2126 if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2127 SrcVT != MVT::f128) {
2128 // Cannot eliminate/lower SHL for f128 yet.
2129 EVT Ty = OpVTLegal ? VT : MVT::i32;
2130 // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2131 // place. We expect the SHL to be eliminated by other optimizations.
2132 SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2133 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2134 if (!OpVTLegal && OpVTSizeInBits > 32)
2135 Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2136 unsigned ShVal = Op.getValueSizeInBits() - 1;
2137 SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2138 return TLO.CombineTo(Op,
2139 TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2140 }
2141 }
2142
2143 // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2144 // Demand the elt/bit if any of the original elts/bits are demanded.
2145 // TODO - bigendian once we have test coverage.
2146 if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
2147 TLO.DAG.getDataLayout().isLittleEndian()) {
2148 unsigned Scale = BitWidth / NumSrcEltBits;
2149 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2150 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2151 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2152 for (unsigned i = 0; i != Scale; ++i) {
2153 unsigned Offset = i * NumSrcEltBits;
2154 APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
2155 if (!Sub.isNullValue()) {
2156 DemandedSrcBits |= Sub;
2157 for (unsigned j = 0; j != NumElts; ++j)
2158 if (DemandedElts[j])
2159 DemandedSrcElts.setBit((j * Scale) + i);
2160 }
2161 }
2162
2163 APInt KnownSrcUndef, KnownSrcZero;
2164 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2165 KnownSrcZero, TLO, Depth + 1))
2166 return true;
2167
2168 KnownBits KnownSrcBits;
2169 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2170 KnownSrcBits, TLO, Depth + 1))
2171 return true;
2172 } else if ((NumSrcEltBits % BitWidth) == 0 &&
2173 TLO.DAG.getDataLayout().isLittleEndian()) {
2174 unsigned Scale = NumSrcEltBits / BitWidth;
2175 unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2176 APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2177 APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2178 for (unsigned i = 0; i != NumElts; ++i)
2179 if (DemandedElts[i]) {
2180 unsigned Offset = (i % Scale) * BitWidth;
2181 DemandedSrcBits.insertBits(DemandedBits, Offset);
2182 DemandedSrcElts.setBit(i / Scale);
2183 }
2184
2185 if (SrcVT.isVector()) {
2186 APInt KnownSrcUndef, KnownSrcZero;
2187 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2188 KnownSrcZero, TLO, Depth + 1))
2189 return true;
2190 }
2191
2192 KnownBits KnownSrcBits;
2193 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2194 KnownSrcBits, TLO, Depth + 1))
2195 return true;
2196 }
2197
2198 // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2199 // recursive call where Known may be useful to the caller.
2200 if (Depth > 0) {
2201 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2202 return false;
2203 }
2204 break;
2205 }
2206 case ISD::ADD:
2207 case ISD::MUL:
2208 case ISD::SUB: {
2209 // Add, Sub, and Mul don't demand any bits in positions beyond that
2210 // of the highest bit demanded of them.
2211 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2212 SDNodeFlags Flags = Op.getNode()->getFlags();
2213 unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2214 APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2215 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2216 Depth + 1) ||
2217 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2218 Depth + 1) ||
2219 // See if the operation should be performed at a smaller bit width.
2220 ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2221 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2222 // Disable the nsw and nuw flags. We can no longer guarantee that we
2223 // won't wrap after simplification.
2224 Flags.setNoSignedWrap(false);
2225 Flags.setNoUnsignedWrap(false);
2226 SDValue NewOp =
2227 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2228 return TLO.CombineTo(Op, NewOp);
2229 }
2230 return true;
2231 }
2232
2233 // Attempt to avoid multi-use ops if we don't need anything from them.
2234 if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
2235 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2236 Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2237 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2238 Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2239 if (DemandedOp0 || DemandedOp1) {
2240 Flags.setNoSignedWrap(false);
2241 Flags.setNoUnsignedWrap(false);
2242 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2243 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2244 SDValue NewOp =
2245 TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2246 return TLO.CombineTo(Op, NewOp);
2247 }
2248 }
2249
2250 // If we have a constant operand, we may be able to turn it into -1 if we
2251 // do not demand the high bits. This can make the constant smaller to
2252 // encode, allow more general folding, or match specialized instruction
2253 // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2254 // is probably not useful (and could be detrimental).
2255 ConstantSDNode *C = isConstOrConstSplat(Op1);
2256 APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2257 if (C && !C->isAllOnesValue() && !C->isOne() &&
2258 (C->getAPIntValue() | HighMask).isAllOnesValue()) {
2259 SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2260 // Disable the nsw and nuw flags. We can no longer guarantee that we
2261 // won't wrap after simplification.
2262 Flags.setNoSignedWrap(false);
2263 Flags.setNoUnsignedWrap(false);
2264 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2265 return TLO.CombineTo(Op, NewOp);
2266 }
2267
2268 LLVM_FALLTHROUGH;
2269 }
2270 default:
2271 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2272 if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2273 Known, TLO, Depth))
2274 return true;
2275 break;
2276 }
2277
2278 // Just use computeKnownBits to compute output bits.
2279 Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2280 break;
2281 }
2282
2283 // If we know the value of all of the demanded bits, return this as a
2284 // constant.
2285 if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2286 // Avoid folding to a constant if any OpaqueConstant is involved.
2287 const SDNode *N = Op.getNode();
2288 for (SDNode *Op :
2289 llvm::make_range(SDNodeIterator::begin(N), SDNodeIterator::end(N))) {
2290 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2291 if (C->isOpaque())
2292 return false;
2293 }
2294 if (VT.isInteger())
2295 return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2296 if (VT.isFloatingPoint())
2297 return TLO.CombineTo(
2298 Op,
2299 TLO.DAG.getConstantFP(
2300 APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2301 }
2302
2303 return false;
2304 }
2305
SimplifyDemandedVectorElts(SDValue Op,const APInt & DemandedElts,APInt & KnownUndef,APInt & KnownZero,DAGCombinerInfo & DCI) const2306 bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
2307 const APInt &DemandedElts,
2308 APInt &KnownUndef,
2309 APInt &KnownZero,
2310 DAGCombinerInfo &DCI) const {
2311 SelectionDAG &DAG = DCI.DAG;
2312 TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2313 !DCI.isBeforeLegalizeOps());
2314
2315 bool Simplified =
2316 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2317 if (Simplified) {
2318 DCI.AddToWorklist(Op.getNode());
2319 DCI.CommitTargetLoweringOpt(TLO);
2320 }
2321
2322 return Simplified;
2323 }
2324
2325 /// Given a vector binary operation and known undefined elements for each input
2326 /// operand, compute whether each element of the output is undefined.
getKnownUndefForVectorBinop(SDValue BO,SelectionDAG & DAG,const APInt & UndefOp0,const APInt & UndefOp1)2327 static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
2328 const APInt &UndefOp0,
2329 const APInt &UndefOp1) {
2330 EVT VT = BO.getValueType();
2331 assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
2332 "Vector binop only");
2333
2334 EVT EltVT = VT.getVectorElementType();
2335 unsigned NumElts = VT.getVectorNumElements();
2336 assert(UndefOp0.getBitWidth() == NumElts &&
2337 UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
2338
2339 auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2340 const APInt &UndefVals) {
2341 if (UndefVals[Index])
2342 return DAG.getUNDEF(EltVT);
2343
2344 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2345 // Try hard to make sure that the getNode() call is not creating temporary
2346 // nodes. Ignore opaque integers because they do not constant fold.
2347 SDValue Elt = BV->getOperand(Index);
2348 auto *C = dyn_cast<ConstantSDNode>(Elt);
2349 if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2350 return Elt;
2351 }
2352
2353 return SDValue();
2354 };
2355
2356 APInt KnownUndef = APInt::getNullValue(NumElts);
2357 for (unsigned i = 0; i != NumElts; ++i) {
2358 // If both inputs for this element are either constant or undef and match
2359 // the element type, compute the constant/undef result for this element of
2360 // the vector.
2361 // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2362 // not handle FP constants. The code within getNode() should be refactored
2363 // to avoid the danger of creating a bogus temporary node here.
2364 SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2365 SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2366 if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2367 if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2368 KnownUndef.setBit(i);
2369 }
2370 return KnownUndef;
2371 }
2372
SimplifyDemandedVectorElts(SDValue Op,const APInt & OriginalDemandedElts,APInt & KnownUndef,APInt & KnownZero,TargetLoweringOpt & TLO,unsigned Depth,bool AssumeSingleUse) const2373 bool TargetLowering::SimplifyDemandedVectorElts(
2374 SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2375 APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2376 bool AssumeSingleUse) const {
2377 EVT VT = Op.getValueType();
2378 unsigned Opcode = Op.getOpcode();
2379 APInt DemandedElts = OriginalDemandedElts;
2380 unsigned NumElts = DemandedElts.getBitWidth();
2381 assert(VT.isVector() && "Expected vector op");
2382
2383 KnownUndef = KnownZero = APInt::getNullValue(NumElts);
2384
2385 // TODO: For now we assume we know nothing about scalable vectors.
2386 if (VT.isScalableVector())
2387 return false;
2388
2389 assert(VT.getVectorNumElements() == NumElts &&
2390 "Mask size mismatches value type element count!");
2391
2392 // Undef operand.
2393 if (Op.isUndef()) {
2394 KnownUndef.setAllBits();
2395 return false;
2396 }
2397
2398 // If Op has other users, assume that all elements are needed.
2399 if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2400 DemandedElts.setAllBits();
2401
2402 // Not demanding any elements from Op.
2403 if (DemandedElts == 0) {
2404 KnownUndef.setAllBits();
2405 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2406 }
2407
2408 // Limit search depth.
2409 if (Depth >= SelectionDAG::MaxRecursionDepth)
2410 return false;
2411
2412 SDLoc DL(Op);
2413 unsigned EltSizeInBits = VT.getScalarSizeInBits();
2414
2415 // Helper for demanding the specified elements and all the bits of both binary
2416 // operands.
2417 auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2418 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2419 TLO.DAG, Depth + 1);
2420 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2421 TLO.DAG, Depth + 1);
2422 if (NewOp0 || NewOp1) {
2423 SDValue NewOp = TLO.DAG.getNode(
2424 Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2425 return TLO.CombineTo(Op, NewOp);
2426 }
2427 return false;
2428 };
2429
2430 switch (Opcode) {
2431 case ISD::SCALAR_TO_VECTOR: {
2432 if (!DemandedElts[0]) {
2433 KnownUndef.setAllBits();
2434 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2435 }
2436 KnownUndef.setHighBits(NumElts - 1);
2437 break;
2438 }
2439 case ISD::BITCAST: {
2440 SDValue Src = Op.getOperand(0);
2441 EVT SrcVT = Src.getValueType();
2442
2443 // We only handle vectors here.
2444 // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2445 if (!SrcVT.isVector())
2446 break;
2447
2448 // Fast handling of 'identity' bitcasts.
2449 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2450 if (NumSrcElts == NumElts)
2451 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2452 KnownZero, TLO, Depth + 1);
2453
2454 APInt SrcZero, SrcUndef;
2455 APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
2456
2457 // Bitcast from 'large element' src vector to 'small element' vector, we
2458 // must demand a source element if any DemandedElt maps to it.
2459 if ((NumElts % NumSrcElts) == 0) {
2460 unsigned Scale = NumElts / NumSrcElts;
2461 for (unsigned i = 0; i != NumElts; ++i)
2462 if (DemandedElts[i])
2463 SrcDemandedElts.setBit(i / Scale);
2464
2465 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2466 TLO, Depth + 1))
2467 return true;
2468
2469 // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2470 // of the large element.
2471 // TODO - bigendian once we have test coverage.
2472 if (TLO.DAG.getDataLayout().isLittleEndian()) {
2473 unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2474 APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
2475 for (unsigned i = 0; i != NumElts; ++i)
2476 if (DemandedElts[i]) {
2477 unsigned Ofs = (i % Scale) * EltSizeInBits;
2478 SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2479 }
2480
2481 KnownBits Known;
2482 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2483 TLO, Depth + 1))
2484 return true;
2485 }
2486
2487 // If the src element is zero/undef then all the output elements will be -
2488 // only demanded elements are guaranteed to be correct.
2489 for (unsigned i = 0; i != NumSrcElts; ++i) {
2490 if (SrcDemandedElts[i]) {
2491 if (SrcZero[i])
2492 KnownZero.setBits(i * Scale, (i + 1) * Scale);
2493 if (SrcUndef[i])
2494 KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2495 }
2496 }
2497 }
2498
2499 // Bitcast from 'small element' src vector to 'large element' vector, we
2500 // demand all smaller source elements covered by the larger demanded element
2501 // of this vector.
2502 if ((NumSrcElts % NumElts) == 0) {
2503 unsigned Scale = NumSrcElts / NumElts;
2504 for (unsigned i = 0; i != NumElts; ++i)
2505 if (DemandedElts[i])
2506 SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
2507
2508 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2509 TLO, Depth + 1))
2510 return true;
2511
2512 // If all the src elements covering an output element are zero/undef, then
2513 // the output element will be as well, assuming it was demanded.
2514 for (unsigned i = 0; i != NumElts; ++i) {
2515 if (DemandedElts[i]) {
2516 if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
2517 KnownZero.setBit(i);
2518 if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
2519 KnownUndef.setBit(i);
2520 }
2521 }
2522 }
2523 break;
2524 }
2525 case ISD::BUILD_VECTOR: {
2526 // Check all elements and simplify any unused elements with UNDEF.
2527 if (!DemandedElts.isAllOnesValue()) {
2528 // Don't simplify BROADCASTS.
2529 if (llvm::any_of(Op->op_values(),
2530 [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2531 SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2532 bool Updated = false;
2533 for (unsigned i = 0; i != NumElts; ++i) {
2534 if (!DemandedElts[i] && !Ops[i].isUndef()) {
2535 Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2536 KnownUndef.setBit(i);
2537 Updated = true;
2538 }
2539 }
2540 if (Updated)
2541 return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2542 }
2543 }
2544 for (unsigned i = 0; i != NumElts; ++i) {
2545 SDValue SrcOp = Op.getOperand(i);
2546 if (SrcOp.isUndef()) {
2547 KnownUndef.setBit(i);
2548 } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2549 (isNullConstant(SrcOp) || isNullFPConstant(SrcOp))) {
2550 KnownZero.setBit(i);
2551 }
2552 }
2553 break;
2554 }
2555 case ISD::CONCAT_VECTORS: {
2556 EVT SubVT = Op.getOperand(0).getValueType();
2557 unsigned NumSubVecs = Op.getNumOperands();
2558 unsigned NumSubElts = SubVT.getVectorNumElements();
2559 for (unsigned i = 0; i != NumSubVecs; ++i) {
2560 SDValue SubOp = Op.getOperand(i);
2561 APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2562 APInt SubUndef, SubZero;
2563 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2564 Depth + 1))
2565 return true;
2566 KnownUndef.insertBits(SubUndef, i * NumSubElts);
2567 KnownZero.insertBits(SubZero, i * NumSubElts);
2568 }
2569 break;
2570 }
2571 case ISD::INSERT_SUBVECTOR: {
2572 // Demand any elements from the subvector and the remainder from the src its
2573 // inserted into.
2574 SDValue Src = Op.getOperand(0);
2575 SDValue Sub = Op.getOperand(1);
2576 uint64_t Idx = Op.getConstantOperandVal(2);
2577 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2578 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2579 APInt DemandedSrcElts = DemandedElts;
2580 DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
2581
2582 APInt SubUndef, SubZero;
2583 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2584 Depth + 1))
2585 return true;
2586
2587 // If none of the src operand elements are demanded, replace it with undef.
2588 if (!DemandedSrcElts && !Src.isUndef())
2589 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2590 TLO.DAG.getUNDEF(VT), Sub,
2591 Op.getOperand(2)));
2592
2593 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2594 TLO, Depth + 1))
2595 return true;
2596 KnownUndef.insertBits(SubUndef, Idx);
2597 KnownZero.insertBits(SubZero, Idx);
2598
2599 // Attempt to avoid multi-use ops if we don't need anything from them.
2600 if (!DemandedSrcElts.isAllOnesValue() ||
2601 !DemandedSubElts.isAllOnesValue()) {
2602 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2603 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2604 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2605 Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2606 if (NewSrc || NewSub) {
2607 NewSrc = NewSrc ? NewSrc : Src;
2608 NewSub = NewSub ? NewSub : Sub;
2609 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2610 NewSub, Op.getOperand(2));
2611 return TLO.CombineTo(Op, NewOp);
2612 }
2613 }
2614 break;
2615 }
2616 case ISD::EXTRACT_SUBVECTOR: {
2617 // Offset the demanded elts by the subvector index.
2618 SDValue Src = Op.getOperand(0);
2619 if (Src.getValueType().isScalableVector())
2620 break;
2621 uint64_t Idx = Op.getConstantOperandVal(1);
2622 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2623 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2624
2625 APInt SrcUndef, SrcZero;
2626 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2627 Depth + 1))
2628 return true;
2629 KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2630 KnownZero = SrcZero.extractBits(NumElts, Idx);
2631
2632 // Attempt to avoid multi-use ops if we don't need anything from them.
2633 if (!DemandedElts.isAllOnesValue()) {
2634 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2635 Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2636 if (NewSrc) {
2637 SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2638 Op.getOperand(1));
2639 return TLO.CombineTo(Op, NewOp);
2640 }
2641 }
2642 break;
2643 }
2644 case ISD::INSERT_VECTOR_ELT: {
2645 SDValue Vec = Op.getOperand(0);
2646 SDValue Scl = Op.getOperand(1);
2647 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2648
2649 // For a legal, constant insertion index, if we don't need this insertion
2650 // then strip it, else remove it from the demanded elts.
2651 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2652 unsigned Idx = CIdx->getZExtValue();
2653 if (!DemandedElts[Idx])
2654 return TLO.CombineTo(Op, Vec);
2655
2656 APInt DemandedVecElts(DemandedElts);
2657 DemandedVecElts.clearBit(Idx);
2658 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2659 KnownZero, TLO, Depth + 1))
2660 return true;
2661
2662 KnownUndef.setBitVal(Idx, Scl.isUndef());
2663
2664 KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2665 break;
2666 }
2667
2668 APInt VecUndef, VecZero;
2669 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2670 Depth + 1))
2671 return true;
2672 // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2673 break;
2674 }
2675 case ISD::VSELECT: {
2676 // Try to transform the select condition based on the current demanded
2677 // elements.
2678 // TODO: If a condition element is undef, we can choose from one arm of the
2679 // select (and if one arm is undef, then we can propagate that to the
2680 // result).
2681 // TODO - add support for constant vselect masks (see IR version of this).
2682 APInt UnusedUndef, UnusedZero;
2683 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2684 UnusedZero, TLO, Depth + 1))
2685 return true;
2686
2687 // See if we can simplify either vselect operand.
2688 APInt DemandedLHS(DemandedElts);
2689 APInt DemandedRHS(DemandedElts);
2690 APInt UndefLHS, ZeroLHS;
2691 APInt UndefRHS, ZeroRHS;
2692 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2693 ZeroLHS, TLO, Depth + 1))
2694 return true;
2695 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2696 ZeroRHS, TLO, Depth + 1))
2697 return true;
2698
2699 KnownUndef = UndefLHS & UndefRHS;
2700 KnownZero = ZeroLHS & ZeroRHS;
2701 break;
2702 }
2703 case ISD::VECTOR_SHUFFLE: {
2704 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2705
2706 // Collect demanded elements from shuffle operands..
2707 APInt DemandedLHS(NumElts, 0);
2708 APInt DemandedRHS(NumElts, 0);
2709 for (unsigned i = 0; i != NumElts; ++i) {
2710 int M = ShuffleMask[i];
2711 if (M < 0 || !DemandedElts[i])
2712 continue;
2713 assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
2714 if (M < (int)NumElts)
2715 DemandedLHS.setBit(M);
2716 else
2717 DemandedRHS.setBit(M - NumElts);
2718 }
2719
2720 // See if we can simplify either shuffle operand.
2721 APInt UndefLHS, ZeroLHS;
2722 APInt UndefRHS, ZeroRHS;
2723 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2724 ZeroLHS, TLO, Depth + 1))
2725 return true;
2726 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2727 ZeroRHS, TLO, Depth + 1))
2728 return true;
2729
2730 // Simplify mask using undef elements from LHS/RHS.
2731 bool Updated = false;
2732 bool IdentityLHS = true, IdentityRHS = true;
2733 SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2734 for (unsigned i = 0; i != NumElts; ++i) {
2735 int &M = NewMask[i];
2736 if (M < 0)
2737 continue;
2738 if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2739 (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2740 Updated = true;
2741 M = -1;
2742 }
2743 IdentityLHS &= (M < 0) || (M == (int)i);
2744 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2745 }
2746
2747 // Update legal shuffle masks based on demanded elements if it won't reduce
2748 // to Identity which can cause premature removal of the shuffle mask.
2749 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2750 SDValue LegalShuffle =
2751 buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2752 NewMask, TLO.DAG);
2753 if (LegalShuffle)
2754 return TLO.CombineTo(Op, LegalShuffle);
2755 }
2756
2757 // Propagate undef/zero elements from LHS/RHS.
2758 for (unsigned i = 0; i != NumElts; ++i) {
2759 int M = ShuffleMask[i];
2760 if (M < 0) {
2761 KnownUndef.setBit(i);
2762 } else if (M < (int)NumElts) {
2763 if (UndefLHS[M])
2764 KnownUndef.setBit(i);
2765 if (ZeroLHS[M])
2766 KnownZero.setBit(i);
2767 } else {
2768 if (UndefRHS[M - NumElts])
2769 KnownUndef.setBit(i);
2770 if (ZeroRHS[M - NumElts])
2771 KnownZero.setBit(i);
2772 }
2773 }
2774 break;
2775 }
2776 case ISD::ANY_EXTEND_VECTOR_INREG:
2777 case ISD::SIGN_EXTEND_VECTOR_INREG:
2778 case ISD::ZERO_EXTEND_VECTOR_INREG: {
2779 APInt SrcUndef, SrcZero;
2780 SDValue Src = Op.getOperand(0);
2781 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2782 APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2783 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2784 Depth + 1))
2785 return true;
2786 KnownZero = SrcZero.zextOrTrunc(NumElts);
2787 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2788
2789 if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2790 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2791 DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2792 // aext - if we just need the bottom element then we can bitcast.
2793 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2794 }
2795
2796 if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2797 // zext(undef) upper bits are guaranteed to be zero.
2798 if (DemandedElts.isSubsetOf(KnownUndef))
2799 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2800 KnownUndef.clearAllBits();
2801 }
2802 break;
2803 }
2804
2805 // TODO: There are more binop opcodes that could be handled here - MIN,
2806 // MAX, saturated math, etc.
2807 case ISD::OR:
2808 case ISD::XOR:
2809 case ISD::ADD:
2810 case ISD::SUB:
2811 case ISD::FADD:
2812 case ISD::FSUB:
2813 case ISD::FMUL:
2814 case ISD::FDIV:
2815 case ISD::FREM: {
2816 SDValue Op0 = Op.getOperand(0);
2817 SDValue Op1 = Op.getOperand(1);
2818
2819 APInt UndefRHS, ZeroRHS;
2820 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2821 Depth + 1))
2822 return true;
2823 APInt UndefLHS, ZeroLHS;
2824 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2825 Depth + 1))
2826 return true;
2827
2828 KnownZero = ZeroLHS & ZeroRHS;
2829 KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2830
2831 // Attempt to avoid multi-use ops if we don't need anything from them.
2832 // TODO - use KnownUndef to relax the demandedelts?
2833 if (!DemandedElts.isAllOnesValue())
2834 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2835 return true;
2836 break;
2837 }
2838 case ISD::SHL:
2839 case ISD::SRL:
2840 case ISD::SRA:
2841 case ISD::ROTL:
2842 case ISD::ROTR: {
2843 SDValue Op0 = Op.getOperand(0);
2844 SDValue Op1 = Op.getOperand(1);
2845
2846 APInt UndefRHS, ZeroRHS;
2847 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2848 Depth + 1))
2849 return true;
2850 APInt UndefLHS, ZeroLHS;
2851 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2852 Depth + 1))
2853 return true;
2854
2855 KnownZero = ZeroLHS;
2856 KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2857
2858 // Attempt to avoid multi-use ops if we don't need anything from them.
2859 // TODO - use KnownUndef to relax the demandedelts?
2860 if (!DemandedElts.isAllOnesValue())
2861 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2862 return true;
2863 break;
2864 }
2865 case ISD::MUL:
2866 case ISD::AND: {
2867 SDValue Op0 = Op.getOperand(0);
2868 SDValue Op1 = Op.getOperand(1);
2869
2870 APInt SrcUndef, SrcZero;
2871 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2872 Depth + 1))
2873 return true;
2874 if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2875 TLO, Depth + 1))
2876 return true;
2877
2878 // If either side has a zero element, then the result element is zero, even
2879 // if the other is an UNDEF.
2880 // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2881 // and then handle 'and' nodes with the rest of the binop opcodes.
2882 KnownZero |= SrcZero;
2883 KnownUndef &= SrcUndef;
2884 KnownUndef &= ~KnownZero;
2885
2886 // Attempt to avoid multi-use ops if we don't need anything from them.
2887 // TODO - use KnownUndef to relax the demandedelts?
2888 if (!DemandedElts.isAllOnesValue())
2889 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2890 return true;
2891 break;
2892 }
2893 case ISD::TRUNCATE:
2894 case ISD::SIGN_EXTEND:
2895 case ISD::ZERO_EXTEND:
2896 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2897 KnownZero, TLO, Depth + 1))
2898 return true;
2899
2900 if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2901 // zext(undef) upper bits are guaranteed to be zero.
2902 if (DemandedElts.isSubsetOf(KnownUndef))
2903 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2904 KnownUndef.clearAllBits();
2905 }
2906 break;
2907 default: {
2908 if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2909 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2910 KnownZero, TLO, Depth))
2911 return true;
2912 } else {
2913 KnownBits Known;
2914 APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
2915 if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2916 TLO, Depth, AssumeSingleUse))
2917 return true;
2918 }
2919 break;
2920 }
2921 }
2922 assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
2923
2924 // Constant fold all undef cases.
2925 // TODO: Handle zero cases as well.
2926 if (DemandedElts.isSubsetOf(KnownUndef))
2927 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2928
2929 return false;
2930 }
2931
2932 /// Determine which of the bits specified in Mask are known to be either zero or
2933 /// one and return them in the Known.
computeKnownBitsForTargetNode(const SDValue Op,KnownBits & Known,const APInt & DemandedElts,const SelectionDAG & DAG,unsigned Depth) const2934 void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
2935 KnownBits &Known,
2936 const APInt &DemandedElts,
2937 const SelectionDAG &DAG,
2938 unsigned Depth) const {
2939 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2940 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2941 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2942 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2943 "Should use MaskedValueIsZero if you don't know whether Op"
2944 " is a target node!");
2945 Known.resetAll();
2946 }
2947
computeKnownBitsForTargetInstr(GISelKnownBits & Analysis,Register R,KnownBits & Known,const APInt & DemandedElts,const MachineRegisterInfo & MRI,unsigned Depth) const2948 void TargetLowering::computeKnownBitsForTargetInstr(
2949 GISelKnownBits &Analysis, Register R, KnownBits &Known,
2950 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2951 unsigned Depth) const {
2952 Known.resetAll();
2953 }
2954
computeKnownBitsForFrameIndex(const int FrameIdx,KnownBits & Known,const MachineFunction & MF) const2955 void TargetLowering::computeKnownBitsForFrameIndex(
2956 const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
2957 // The low bits are known zero if the pointer is aligned.
2958 Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
2959 }
2960
computeKnownAlignForTargetInstr(GISelKnownBits & Analysis,Register R,const MachineRegisterInfo & MRI,unsigned Depth) const2961 Align TargetLowering::computeKnownAlignForTargetInstr(
2962 GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
2963 unsigned Depth) const {
2964 return Align(1);
2965 }
2966
2967 /// This method can be implemented by targets that want to expose additional
2968 /// information about sign bits to the DAG Combiner.
ComputeNumSignBitsForTargetNode(SDValue Op,const APInt &,const SelectionDAG &,unsigned Depth) const2969 unsigned TargetLowering::ComputeNumSignBitsForTargetNode(SDValue Op,
2970 const APInt &,
2971 const SelectionDAG &,
2972 unsigned Depth) const {
2973 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2974 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2975 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2976 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2977 "Should use ComputeNumSignBits if you don't know whether Op"
2978 " is a target node!");
2979 return 1;
2980 }
2981
computeNumSignBitsForTargetInstr(GISelKnownBits & Analysis,Register R,const APInt & DemandedElts,const MachineRegisterInfo & MRI,unsigned Depth) const2982 unsigned TargetLowering::computeNumSignBitsForTargetInstr(
2983 GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
2984 const MachineRegisterInfo &MRI, unsigned Depth) const {
2985 return 1;
2986 }
2987
SimplifyDemandedVectorEltsForTargetNode(SDValue Op,const APInt & DemandedElts,APInt & KnownUndef,APInt & KnownZero,TargetLoweringOpt & TLO,unsigned Depth) const2988 bool TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
2989 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
2990 TargetLoweringOpt &TLO, unsigned Depth) const {
2991 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2992 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2993 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2994 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2995 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
2996 " is a target node!");
2997 return false;
2998 }
2999
SimplifyDemandedBitsForTargetNode(SDValue Op,const APInt & DemandedBits,const APInt & DemandedElts,KnownBits & Known,TargetLoweringOpt & TLO,unsigned Depth) const3000 bool TargetLowering::SimplifyDemandedBitsForTargetNode(
3001 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3002 KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3003 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3004 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3005 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3006 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3007 "Should use SimplifyDemandedBits if you don't know whether Op"
3008 " is a target node!");
3009 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3010 return false;
3011 }
3012
SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op,const APInt & DemandedBits,const APInt & DemandedElts,SelectionDAG & DAG,unsigned Depth) const3013 SDValue TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
3014 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3015 SelectionDAG &DAG, unsigned Depth) const {
3016 assert(
3017 (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3018 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3019 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3020 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3021 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3022 " is a target node!");
3023 return SDValue();
3024 }
3025
3026 SDValue
buildLegalVectorShuffle(EVT VT,const SDLoc & DL,SDValue N0,SDValue N1,MutableArrayRef<int> Mask,SelectionDAG & DAG) const3027 TargetLowering::buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0,
3028 SDValue N1, MutableArrayRef<int> Mask,
3029 SelectionDAG &DAG) const {
3030 bool LegalMask = isShuffleMaskLegal(Mask, VT);
3031 if (!LegalMask) {
3032 std::swap(N0, N1);
3033 ShuffleVectorSDNode::commuteMask(Mask);
3034 LegalMask = isShuffleMaskLegal(Mask, VT);
3035 }
3036
3037 if (!LegalMask)
3038 return SDValue();
3039
3040 return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3041 }
3042
getTargetConstantFromLoad(LoadSDNode *) const3043 const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const {
3044 return nullptr;
3045 }
3046
isKnownNeverNaNForTargetNode(SDValue Op,const SelectionDAG & DAG,bool SNaN,unsigned Depth) const3047 bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
3048 const SelectionDAG &DAG,
3049 bool SNaN,
3050 unsigned Depth) const {
3051 assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3052 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3053 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3054 Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3055 "Should use isKnownNeverNaN if you don't know whether Op"
3056 " is a target node!");
3057 return false;
3058 }
3059
3060 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3061 // work with truncating build vectors and vectors with elements of less than
3062 // 8 bits.
isConstTrueVal(const SDNode * N) const3063 bool TargetLowering::isConstTrueVal(const SDNode *N) const {
3064 if (!N)
3065 return false;
3066
3067 APInt CVal;
3068 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3069 CVal = CN->getAPIntValue();
3070 } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3071 auto *CN = BV->getConstantSplatNode();
3072 if (!CN)
3073 return false;
3074
3075 // If this is a truncating build vector, truncate the splat value.
3076 // Otherwise, we may fail to match the expected values below.
3077 unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3078 CVal = CN->getAPIntValue();
3079 if (BVEltWidth < CVal.getBitWidth())
3080 CVal = CVal.trunc(BVEltWidth);
3081 } else {
3082 return false;
3083 }
3084
3085 switch (getBooleanContents(N->getValueType(0))) {
3086 case UndefinedBooleanContent:
3087 return CVal[0];
3088 case ZeroOrOneBooleanContent:
3089 return CVal.isOneValue();
3090 case ZeroOrNegativeOneBooleanContent:
3091 return CVal.isAllOnesValue();
3092 }
3093
3094 llvm_unreachable("Invalid boolean contents");
3095 }
3096
isConstFalseVal(const SDNode * N) const3097 bool TargetLowering::isConstFalseVal(const SDNode *N) const {
3098 if (!N)
3099 return false;
3100
3101 const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3102 if (!CN) {
3103 const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3104 if (!BV)
3105 return false;
3106
3107 // Only interested in constant splats, we don't care about undef
3108 // elements in identifying boolean constants and getConstantSplatNode
3109 // returns NULL if all ops are undef;
3110 CN = BV->getConstantSplatNode();
3111 if (!CN)
3112 return false;
3113 }
3114
3115 if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3116 return !CN->getAPIntValue()[0];
3117
3118 return CN->isNullValue();
3119 }
3120
isExtendedTrueVal(const ConstantSDNode * N,EVT VT,bool SExt) const3121 bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
3122 bool SExt) const {
3123 if (VT == MVT::i1)
3124 return N->isOne();
3125
3126 TargetLowering::BooleanContent Cnt = getBooleanContents(VT);
3127 switch (Cnt) {
3128 case TargetLowering::ZeroOrOneBooleanContent:
3129 // An extended value of 1 is always true, unless its original type is i1,
3130 // in which case it will be sign extended to -1.
3131 return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3132 case TargetLowering::UndefinedBooleanContent:
3133 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3134 return N->isAllOnesValue() && SExt;
3135 }
3136 llvm_unreachable("Unexpected enumeration.");
3137 }
3138
3139 /// This helper function of SimplifySetCC tries to optimize the comparison when
3140 /// either operand of the SetCC node is a bitwise-and instruction.
foldSetCCWithAnd(EVT VT,SDValue N0,SDValue N1,ISD::CondCode Cond,const SDLoc & DL,DAGCombinerInfo & DCI) const3141 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3142 ISD::CondCode Cond, const SDLoc &DL,
3143 DAGCombinerInfo &DCI) const {
3144 // Match these patterns in any of their permutations:
3145 // (X & Y) == Y
3146 // (X & Y) != Y
3147 if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3148 std::swap(N0, N1);
3149
3150 EVT OpVT = N0.getValueType();
3151 if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3152 (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3153 return SDValue();
3154
3155 SDValue X, Y;
3156 if (N0.getOperand(0) == N1) {
3157 X = N0.getOperand(1);
3158 Y = N0.getOperand(0);
3159 } else if (N0.getOperand(1) == N1) {
3160 X = N0.getOperand(0);
3161 Y = N0.getOperand(1);
3162 } else {
3163 return SDValue();
3164 }
3165
3166 SelectionDAG &DAG = DCI.DAG;
3167 SDValue Zero = DAG.getConstant(0, DL, OpVT);
3168 if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3169 // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3170 // Note that where Y is variable and is known to have at most one bit set
3171 // (for example, if it is Z & 1) we cannot do this; the expressions are not
3172 // equivalent when Y == 0.
3173 assert(OpVT.isInteger());
3174 Cond = ISD::getSetCCInverse(Cond, OpVT);
3175 if (DCI.isBeforeLegalizeOps() ||
3176 isCondCodeLegal(Cond, N0.getSimpleValueType()))
3177 return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3178 } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3179 // If the target supports an 'and-not' or 'and-complement' logic operation,
3180 // try to use that to make a comparison operation more efficient.
3181 // But don't do this transform if the mask is a single bit because there are
3182 // more efficient ways to deal with that case (for example, 'bt' on x86 or
3183 // 'rlwinm' on PPC).
3184
3185 // Bail out if the compare operand that we want to turn into a zero is
3186 // already a zero (otherwise, infinite loop).
3187 auto *YConst = dyn_cast<ConstantSDNode>(Y);
3188 if (YConst && YConst->isNullValue())
3189 return SDValue();
3190
3191 // Transform this into: ~X & Y == 0.
3192 SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3193 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3194 return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3195 }
3196
3197 return SDValue();
3198 }
3199
3200 /// There are multiple IR patterns that could be checking whether certain
3201 /// truncation of a signed number would be lossy or not. The pattern which is
3202 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
3203 /// We are looking for the following pattern: (KeptBits is a constant)
3204 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3205 /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3206 /// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3207 /// We will unfold it into the natural trunc+sext pattern:
3208 /// ((%x << C) a>> C) dstcond %x
3209 /// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
optimizeSetCCOfSignedTruncationCheck(EVT SCCVT,SDValue N0,SDValue N1,ISD::CondCode Cond,DAGCombinerInfo & DCI,const SDLoc & DL) const3210 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3211 EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3212 const SDLoc &DL) const {
3213 // We must be comparing with a constant.
3214 ConstantSDNode *C1;
3215 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3216 return SDValue();
3217
3218 // N0 should be: add %x, (1 << (KeptBits-1))
3219 if (N0->getOpcode() != ISD::ADD)
3220 return SDValue();
3221
3222 // And we must be 'add'ing a constant.
3223 ConstantSDNode *C01;
3224 if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3225 return SDValue();
3226
3227 SDValue X = N0->getOperand(0);
3228 EVT XVT = X.getValueType();
3229
3230 // Validate constants ...
3231
3232 APInt I1 = C1->getAPIntValue();
3233
3234 ISD::CondCode NewCond;
3235 if (Cond == ISD::CondCode::SETULT) {
3236 NewCond = ISD::CondCode::SETEQ;
3237 } else if (Cond == ISD::CondCode::SETULE) {
3238 NewCond = ISD::CondCode::SETEQ;
3239 // But need to 'canonicalize' the constant.
3240 I1 += 1;
3241 } else if (Cond == ISD::CondCode::SETUGT) {
3242 NewCond = ISD::CondCode::SETNE;
3243 // But need to 'canonicalize' the constant.
3244 I1 += 1;
3245 } else if (Cond == ISD::CondCode::SETUGE) {
3246 NewCond = ISD::CondCode::SETNE;
3247 } else
3248 return SDValue();
3249
3250 APInt I01 = C01->getAPIntValue();
3251
3252 auto checkConstants = [&I1, &I01]() -> bool {
3253 // Both of them must be power-of-two, and the constant from setcc is bigger.
3254 return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3255 };
3256
3257 if (checkConstants()) {
3258 // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3259 } else {
3260 // What if we invert constants? (and the target predicate)
3261 I1.negate();
3262 I01.negate();
3263 assert(XVT.isInteger());
3264 NewCond = getSetCCInverse(NewCond, XVT);
3265 if (!checkConstants())
3266 return SDValue();
3267 // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3268 }
3269
3270 // They are power-of-two, so which bit is set?
3271 const unsigned KeptBits = I1.logBase2();
3272 const unsigned KeptBitsMinusOne = I01.logBase2();
3273
3274 // Magic!
3275 if (KeptBits != (KeptBitsMinusOne + 1))
3276 return SDValue();
3277 assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
3278
3279 // We don't want to do this in every single case.
3280 SelectionDAG &DAG = DCI.DAG;
3281 if (!DAG.getTargetLoweringInfo().shouldTransformSignedTruncationCheck(
3282 XVT, KeptBits))
3283 return SDValue();
3284
3285 const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3286 assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable");
3287
3288 // Unfold into: ((%x << C) a>> C) cond %x
3289 // Where 'cond' will be either 'eq' or 'ne'.
3290 SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3291 SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3292 SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3293 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3294
3295 return T2;
3296 }
3297
3298 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
optimizeSetCCByHoistingAndByConstFromLogicalShift(EVT SCCVT,SDValue N0,SDValue N1C,ISD::CondCode Cond,DAGCombinerInfo & DCI,const SDLoc & DL) const3299 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3300 EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3301 DAGCombinerInfo &DCI, const SDLoc &DL) const {
3302 assert(isConstOrConstSplat(N1C) &&
3303 isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
3304 "Should be a comparison with 0.");
3305 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3306 "Valid only for [in]equality comparisons.");
3307
3308 unsigned NewShiftOpcode;
3309 SDValue X, C, Y;
3310
3311 SelectionDAG &DAG = DCI.DAG;
3312 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3313
3314 // Look for '(C l>>/<< Y)'.
3315 auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3316 // The shift should be one-use.
3317 if (!V.hasOneUse())
3318 return false;
3319 unsigned OldShiftOpcode = V.getOpcode();
3320 switch (OldShiftOpcode) {
3321 case ISD::SHL:
3322 NewShiftOpcode = ISD::SRL;
3323 break;
3324 case ISD::SRL:
3325 NewShiftOpcode = ISD::SHL;
3326 break;
3327 default:
3328 return false; // must be a logical shift.
3329 }
3330 // We should be shifting a constant.
3331 // FIXME: best to use isConstantOrConstantVector().
3332 C = V.getOperand(0);
3333 ConstantSDNode *CC =
3334 isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3335 if (!CC)
3336 return false;
3337 Y = V.getOperand(1);
3338
3339 ConstantSDNode *XC =
3340 isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3341 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3342 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3343 };
3344
3345 // LHS of comparison should be an one-use 'and'.
3346 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3347 return SDValue();
3348
3349 X = N0.getOperand(0);
3350 SDValue Mask = N0.getOperand(1);
3351
3352 // 'and' is commutative!
3353 if (!Match(Mask)) {
3354 std::swap(X, Mask);
3355 if (!Match(Mask))
3356 return SDValue();
3357 }
3358
3359 EVT VT = X.getValueType();
3360
3361 // Produce:
3362 // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3363 SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3364 SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3365 SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3366 return T2;
3367 }
3368
3369 /// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3370 /// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3371 /// handle the commuted versions of these patterns.
foldSetCCWithBinOp(EVT VT,SDValue N0,SDValue N1,ISD::CondCode Cond,const SDLoc & DL,DAGCombinerInfo & DCI) const3372 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3373 ISD::CondCode Cond, const SDLoc &DL,
3374 DAGCombinerInfo &DCI) const {
3375 unsigned BOpcode = N0.getOpcode();
3376 assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
3377 "Unexpected binop");
3378 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
3379
3380 // (X + Y) == X --> Y == 0
3381 // (X - Y) == X --> Y == 0
3382 // (X ^ Y) == X --> Y == 0
3383 SelectionDAG &DAG = DCI.DAG;
3384 EVT OpVT = N0.getValueType();
3385 SDValue X = N0.getOperand(0);
3386 SDValue Y = N0.getOperand(1);
3387 if (X == N1)
3388 return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3389
3390 if (Y != N1)
3391 return SDValue();
3392
3393 // (X + Y) == Y --> X == 0
3394 // (X ^ Y) == Y --> X == 0
3395 if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3396 return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3397
3398 // The shift would not be valid if the operands are boolean (i1).
3399 if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3400 return SDValue();
3401
3402 // (X - Y) == Y --> X == Y << 1
3403 EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3404 !DCI.isBeforeLegalize());
3405 SDValue One = DAG.getConstant(1, DL, ShiftVT);
3406 SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3407 if (!DCI.isCalledByLegalizer())
3408 DCI.AddToWorklist(YShl1.getNode());
3409 return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3410 }
3411
simplifySetCCWithCTPOP(const TargetLowering & TLI,EVT VT,SDValue N0,const APInt & C1,ISD::CondCode Cond,const SDLoc & dl,SelectionDAG & DAG)3412 static SDValue simplifySetCCWithCTPOP(const TargetLowering &TLI, EVT VT,
3413 SDValue N0, const APInt &C1,
3414 ISD::CondCode Cond, const SDLoc &dl,
3415 SelectionDAG &DAG) {
3416 // Look through truncs that don't change the value of a ctpop.
3417 // FIXME: Add vector support? Need to be careful with setcc result type below.
3418 SDValue CTPOP = N0;
3419 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3420 N0.getScalarValueSizeInBits() > Log2_32(N0.getOperand(0).getScalarValueSizeInBits()))
3421 CTPOP = N0.getOperand(0);
3422
3423 if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3424 return SDValue();
3425
3426 EVT CTVT = CTPOP.getValueType();
3427 SDValue CTOp = CTPOP.getOperand(0);
3428
3429 // If this is a vector CTPOP, keep the CTPOP if it is legal.
3430 // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3431 if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3432 return SDValue();
3433
3434 // (ctpop x) u< 2 -> (x & x-1) == 0
3435 // (ctpop x) u> 1 -> (x & x-1) != 0
3436 if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3437 unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3438 if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3439 return SDValue();
3440 if (C1 == 0 && (Cond == ISD::SETULT))
3441 return SDValue(); // This is handled elsewhere.
3442
3443 unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3444
3445 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3446 SDValue Result = CTOp;
3447 for (unsigned i = 0; i < Passes; i++) {
3448 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3449 Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3450 }
3451 ISD::CondCode CC = Cond == ISD::SETULT ? ISD::SETEQ : ISD::SETNE;
3452 return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3453 }
3454
3455 // If ctpop is not supported, expand a power-of-2 comparison based on it.
3456 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3457 // For scalars, keep CTPOP if it is legal or custom.
3458 if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3459 return SDValue();
3460 // This is based on X86's custom lowering for CTPOP which produces more
3461 // instructions than the expansion here.
3462
3463 // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3464 // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3465 SDValue Zero = DAG.getConstant(0, dl, CTVT);
3466 SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3467 assert(CTVT.isInteger());
3468 ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3469 SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3470 SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3471 SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3472 SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3473 unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3474 return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3475 }
3476
3477 return SDValue();
3478 }
3479
3480 /// Try to simplify a setcc built with the specified operands and cc. If it is
3481 /// unable to simplify it, return a null SDValue.
SimplifySetCC(EVT VT,SDValue N0,SDValue N1,ISD::CondCode Cond,bool foldBooleans,DAGCombinerInfo & DCI,const SDLoc & dl) const3482 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
3483 ISD::CondCode Cond, bool foldBooleans,
3484 DAGCombinerInfo &DCI,
3485 const SDLoc &dl) const {
3486 SelectionDAG &DAG = DCI.DAG;
3487 const DataLayout &Layout = DAG.getDataLayout();
3488 EVT OpVT = N0.getValueType();
3489
3490 // Constant fold or commute setcc.
3491 if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3492 return Fold;
3493
3494 // Ensure that the constant occurs on the RHS and fold constant comparisons.
3495 // TODO: Handle non-splat vector constants. All undef causes trouble.
3496 // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3497 // infinite loop here when we encounter one.
3498 ISD::CondCode SwappedCC = ISD::getSetCCSwappedOperands(Cond);
3499 if (isConstOrConstSplat(N0) &&
3500 (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3501 (DCI.isBeforeLegalizeOps() ||
3502 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3503 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3504
3505 // If we have a subtract with the same 2 non-constant operands as this setcc
3506 // -- but in reverse order -- then try to commute the operands of this setcc
3507 // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3508 // instruction on some targets.
3509 if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3510 (DCI.isBeforeLegalizeOps() ||
3511 isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3512 DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3513 !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3514 return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3515
3516 if (auto *N1C = isConstOrConstSplat(N1)) {
3517 const APInt &C1 = N1C->getAPIntValue();
3518
3519 // Optimize some CTPOP cases.
3520 if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3521 return V;
3522
3523 // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3524 // equality comparison, then we're just comparing whether X itself is
3525 // zero.
3526 if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
3527 N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3528 isPowerOf2_32(N0.getScalarValueSizeInBits())) {
3529 if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3530 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3531 ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3532 if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3533 // (srl (ctlz x), 5) == 0 -> X != 0
3534 // (srl (ctlz x), 5) != 1 -> X != 0
3535 Cond = ISD::SETNE;
3536 } else {
3537 // (srl (ctlz x), 5) != 0 -> X == 0
3538 // (srl (ctlz x), 5) == 1 -> X == 0
3539 Cond = ISD::SETEQ;
3540 }
3541 SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3542 return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3543 Cond);
3544 }
3545 }
3546 }
3547 }
3548
3549 // FIXME: Support vectors.
3550 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3551 const APInt &C1 = N1C->getAPIntValue();
3552
3553 // (zext x) == C --> x == (trunc C)
3554 // (sext x) == C --> x == (trunc C)
3555 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3556 DCI.isBeforeLegalize() && N0->hasOneUse()) {
3557 unsigned MinBits = N0.getValueSizeInBits();
3558 SDValue PreExt;
3559 bool Signed = false;
3560 if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3561 // ZExt
3562 MinBits = N0->getOperand(0).getValueSizeInBits();
3563 PreExt = N0->getOperand(0);
3564 } else if (N0->getOpcode() == ISD::AND) {
3565 // DAGCombine turns costly ZExts into ANDs
3566 if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3567 if ((C->getAPIntValue()+1).isPowerOf2()) {
3568 MinBits = C->getAPIntValue().countTrailingOnes();
3569 PreExt = N0->getOperand(0);
3570 }
3571 } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3572 // SExt
3573 MinBits = N0->getOperand(0).getValueSizeInBits();
3574 PreExt = N0->getOperand(0);
3575 Signed = true;
3576 } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3577 // ZEXTLOAD / SEXTLOAD
3578 if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3579 MinBits = LN0->getMemoryVT().getSizeInBits();
3580 PreExt = N0;
3581 } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3582 Signed = true;
3583 MinBits = LN0->getMemoryVT().getSizeInBits();
3584 PreExt = N0;
3585 }
3586 }
3587
3588 // Figure out how many bits we need to preserve this constant.
3589 unsigned ReqdBits = Signed ?
3590 C1.getBitWidth() - C1.getNumSignBits() + 1 :
3591 C1.getActiveBits();
3592
3593 // Make sure we're not losing bits from the constant.
3594 if (MinBits > 0 &&
3595 MinBits < C1.getBitWidth() &&
3596 MinBits >= ReqdBits) {
3597 EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3598 if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3599 // Will get folded away.
3600 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3601 if (MinBits == 1 && C1 == 1)
3602 // Invert the condition.
3603 return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3604 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3605 SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3606 return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3607 }
3608
3609 // If truncating the setcc operands is not desirable, we can still
3610 // simplify the expression in some cases:
3611 // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3612 // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3613 // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3614 // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3615 // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3616 // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3617 SDValue TopSetCC = N0->getOperand(0);
3618 unsigned N0Opc = N0->getOpcode();
3619 bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3620 if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3621 TopSetCC.getOpcode() == ISD::SETCC &&
3622 (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3623 (isConstFalseVal(N1C) ||
3624 isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3625
3626 bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
3627 (!N1C->isNullValue() && Cond == ISD::SETNE);
3628
3629 if (!Inverse)
3630 return TopSetCC;
3631
3632 ISD::CondCode InvCond = ISD::getSetCCInverse(
3633 cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3634 TopSetCC.getOperand(0).getValueType());
3635 return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3636 TopSetCC.getOperand(1),
3637 InvCond);
3638 }
3639 }
3640 }
3641
3642 // If the LHS is '(and load, const)', the RHS is 0, the test is for
3643 // equality or unsigned, and all 1 bits of the const are in the same
3644 // partial word, see if we can shorten the load.
3645 if (DCI.isBeforeLegalize() &&
3646 !ISD::isSignedIntSetCC(Cond) &&
3647 N0.getOpcode() == ISD::AND && C1 == 0 &&
3648 N0.getNode()->hasOneUse() &&
3649 isa<LoadSDNode>(N0.getOperand(0)) &&
3650 N0.getOperand(0).getNode()->hasOneUse() &&
3651 isa<ConstantSDNode>(N0.getOperand(1))) {
3652 LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3653 APInt bestMask;
3654 unsigned bestWidth = 0, bestOffset = 0;
3655 if (Lod->isSimple() && Lod->isUnindexed()) {
3656 unsigned origWidth = N0.getValueSizeInBits();
3657 unsigned maskWidth = origWidth;
3658 // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3659 // 8 bits, but have to be careful...
3660 if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3661 origWidth = Lod->getMemoryVT().getSizeInBits();
3662 const APInt &Mask = N0.getConstantOperandAPInt(1);
3663 for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3664 APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3665 for (unsigned offset=0; offset<origWidth/width; offset++) {
3666 if (Mask.isSubsetOf(newMask)) {
3667 if (Layout.isLittleEndian())
3668 bestOffset = (uint64_t)offset * (width/8);
3669 else
3670 bestOffset = (origWidth/width - offset - 1) * (width/8);
3671 bestMask = Mask.lshr(offset * (width/8) * 8);
3672 bestWidth = width;
3673 break;
3674 }
3675 newMask <<= width;
3676 }
3677 }
3678 }
3679 if (bestWidth) {
3680 EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3681 if (newVT.isRound() &&
3682 shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3683 SDValue Ptr = Lod->getBasePtr();
3684 if (bestOffset != 0)
3685 Ptr =
3686 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3687 SDValue NewLoad =
3688 DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3689 Lod->getPointerInfo().getWithOffset(bestOffset),
3690 Lod->getOriginalAlign());
3691 return DAG.getSetCC(dl, VT,
3692 DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3693 DAG.getConstant(bestMask.trunc(bestWidth),
3694 dl, newVT)),
3695 DAG.getConstant(0LL, dl, newVT), Cond);
3696 }
3697 }
3698 }
3699
3700 // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3701 if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3702 unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3703
3704 // If the comparison constant has bits in the upper part, the
3705 // zero-extended value could never match.
3706 if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3707 C1.getBitWidth() - InSize))) {
3708 switch (Cond) {
3709 case ISD::SETUGT:
3710 case ISD::SETUGE:
3711 case ISD::SETEQ:
3712 return DAG.getConstant(0, dl, VT);
3713 case ISD::SETULT:
3714 case ISD::SETULE:
3715 case ISD::SETNE:
3716 return DAG.getConstant(1, dl, VT);
3717 case ISD::SETGT:
3718 case ISD::SETGE:
3719 // True if the sign bit of C1 is set.
3720 return DAG.getConstant(C1.isNegative(), dl, VT);
3721 case ISD::SETLT:
3722 case ISD::SETLE:
3723 // True if the sign bit of C1 isn't set.
3724 return DAG.getConstant(C1.isNonNegative(), dl, VT);
3725 default:
3726 break;
3727 }
3728 }
3729
3730 // Otherwise, we can perform the comparison with the low bits.
3731 switch (Cond) {
3732 case ISD::SETEQ:
3733 case ISD::SETNE:
3734 case ISD::SETUGT:
3735 case ISD::SETUGE:
3736 case ISD::SETULT:
3737 case ISD::SETULE: {
3738 EVT newVT = N0.getOperand(0).getValueType();
3739 if (DCI.isBeforeLegalizeOps() ||
3740 (isOperationLegal(ISD::SETCC, newVT) &&
3741 isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3742 EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3743 SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3744
3745 SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3746 NewConst, Cond);
3747 return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3748 }
3749 break;
3750 }
3751 default:
3752 break; // todo, be more careful with signed comparisons
3753 }
3754 } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3755 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3756 !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3757 OpVT)) {
3758 EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3759 unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3760 EVT ExtDstTy = N0.getValueType();
3761 unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3762
3763 // If the constant doesn't fit into the number of bits for the source of
3764 // the sign extension, it is impossible for both sides to be equal.
3765 if (C1.getMinSignedBits() > ExtSrcTyBits)
3766 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3767
3768 assert(ExtDstTy == N0.getOperand(0).getValueType() &&
3769 ExtDstTy != ExtSrcTy && "Unexpected types!");
3770 APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3771 SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3772 DAG.getConstant(Imm, dl, ExtDstTy));
3773 if (!DCI.isCalledByLegalizer())
3774 DCI.AddToWorklist(ZextOp.getNode());
3775 // Otherwise, make this a use of a zext.
3776 return DAG.getSetCC(dl, VT, ZextOp,
3777 DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3778 } else if ((N1C->isNullValue() || N1C->isOne()) &&
3779 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3780 // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3781 if (N0.getOpcode() == ISD::SETCC &&
3782 isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3783 (N0.getValueType() == MVT::i1 ||
3784 getBooleanContents(N0.getOperand(0).getValueType()) ==
3785 ZeroOrOneBooleanContent)) {
3786 bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3787 if (TrueWhenTrue)
3788 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3789 // Invert the condition.
3790 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3791 CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3792 if (DCI.isBeforeLegalizeOps() ||
3793 isCondCodeLegal(CC, N0.getOperand(0).getSimpleValueType()))
3794 return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3795 }
3796
3797 if ((N0.getOpcode() == ISD::XOR ||
3798 (N0.getOpcode() == ISD::AND &&
3799 N0.getOperand(0).getOpcode() == ISD::XOR &&
3800 N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3801 isOneConstant(N0.getOperand(1))) {
3802 // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3803 // can only do this if the top bits are known zero.
3804 unsigned BitWidth = N0.getValueSizeInBits();
3805 if (DAG.MaskedValueIsZero(N0,
3806 APInt::getHighBitsSet(BitWidth,
3807 BitWidth-1))) {
3808 // Okay, get the un-inverted input value.
3809 SDValue Val;
3810 if (N0.getOpcode() == ISD::XOR) {
3811 Val = N0.getOperand(0);
3812 } else {
3813 assert(N0.getOpcode() == ISD::AND &&
3814 N0.getOperand(0).getOpcode() == ISD::XOR);
3815 // ((X^1)&1)^1 -> X & 1
3816 Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3817 N0.getOperand(0).getOperand(0),
3818 N0.getOperand(1));
3819 }
3820
3821 return DAG.getSetCC(dl, VT, Val, N1,
3822 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3823 }
3824 } else if (N1C->isOne()) {
3825 SDValue Op0 = N0;
3826 if (Op0.getOpcode() == ISD::TRUNCATE)
3827 Op0 = Op0.getOperand(0);
3828
3829 if ((Op0.getOpcode() == ISD::XOR) &&
3830 Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3831 Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3832 SDValue XorLHS = Op0.getOperand(0);
3833 SDValue XorRHS = Op0.getOperand(1);
3834 // Ensure that the input setccs return an i1 type or 0/1 value.
3835 if (Op0.getValueType() == MVT::i1 ||
3836 (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3837 ZeroOrOneBooleanContent &&
3838 getBooleanContents(XorRHS.getOperand(0).getValueType()) ==
3839 ZeroOrOneBooleanContent)) {
3840 // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3841 Cond = (Cond == ISD::SETEQ) ? ISD::SETNE : ISD::SETEQ;
3842 return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3843 }
3844 }
3845 if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3846 // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3847 if (Op0.getValueType().bitsGT(VT))
3848 Op0 = DAG.getNode(ISD::AND, dl, VT,
3849 DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3850 DAG.getConstant(1, dl, VT));
3851 else if (Op0.getValueType().bitsLT(VT))
3852 Op0 = DAG.getNode(ISD::AND, dl, VT,
3853 DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3854 DAG.getConstant(1, dl, VT));
3855
3856 return DAG.getSetCC(dl, VT, Op0,
3857 DAG.getConstant(0, dl, Op0.getValueType()),
3858 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3859 }
3860 if (Op0.getOpcode() == ISD::AssertZext &&
3861 cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3862 return DAG.getSetCC(dl, VT, Op0,
3863 DAG.getConstant(0, dl, Op0.getValueType()),
3864 Cond == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ);
3865 }
3866 }
3867
3868 // Given:
3869 // icmp eq/ne (urem %x, %y), 0
3870 // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3871 // icmp eq/ne %x, 0
3872 if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
3873 (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3874 KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3875 KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3876 if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3877 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3878 }
3879
3880 if (SDValue V =
3881 optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3882 return V;
3883 }
3884
3885 // These simplifications apply to splat vectors as well.
3886 // TODO: Handle more splat vector cases.
3887 if (auto *N1C = isConstOrConstSplat(N1)) {
3888 const APInt &C1 = N1C->getAPIntValue();
3889
3890 APInt MinVal, MaxVal;
3891 unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3892 if (ISD::isSignedIntSetCC(Cond)) {
3893 MinVal = APInt::getSignedMinValue(OperandBitSize);
3894 MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3895 } else {
3896 MinVal = APInt::getMinValue(OperandBitSize);
3897 MaxVal = APInt::getMaxValue(OperandBitSize);
3898 }
3899
3900 // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3901 if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3902 // X >= MIN --> true
3903 if (C1 == MinVal)
3904 return DAG.getBoolConstant(true, dl, VT, OpVT);
3905
3906 if (!VT.isVector()) { // TODO: Support this for vectors.
3907 // X >= C0 --> X > (C0 - 1)
3908 APInt C = C1 - 1;
3909 ISD::CondCode NewCC = (Cond == ISD::SETGE) ? ISD::SETGT : ISD::SETUGT;
3910 if ((DCI.isBeforeLegalizeOps() ||
3911 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3912 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3913 isLegalICmpImmediate(C.getSExtValue())))) {
3914 return DAG.getSetCC(dl, VT, N0,
3915 DAG.getConstant(C, dl, N1.getValueType()),
3916 NewCC);
3917 }
3918 }
3919 }
3920
3921 if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3922 // X <= MAX --> true
3923 if (C1 == MaxVal)
3924 return DAG.getBoolConstant(true, dl, VT, OpVT);
3925
3926 // X <= C0 --> X < (C0 + 1)
3927 if (!VT.isVector()) { // TODO: Support this for vectors.
3928 APInt C = C1 + 1;
3929 ISD::CondCode NewCC = (Cond == ISD::SETLE) ? ISD::SETLT : ISD::SETULT;
3930 if ((DCI.isBeforeLegalizeOps() ||
3931 isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3932 (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3933 isLegalICmpImmediate(C.getSExtValue())))) {
3934 return DAG.getSetCC(dl, VT, N0,
3935 DAG.getConstant(C, dl, N1.getValueType()),
3936 NewCC);
3937 }
3938 }
3939 }
3940
3941 if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3942 if (C1 == MinVal)
3943 return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3944
3945 // TODO: Support this for vectors after legalize ops.
3946 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3947 // Canonicalize setlt X, Max --> setne X, Max
3948 if (C1 == MaxVal)
3949 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3950
3951 // If we have setult X, 1, turn it into seteq X, 0
3952 if (C1 == MinVal+1)
3953 return DAG.getSetCC(dl, VT, N0,
3954 DAG.getConstant(MinVal, dl, N0.getValueType()),
3955 ISD::SETEQ);
3956 }
3957 }
3958
3959 if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3960 if (C1 == MaxVal)
3961 return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3962
3963 // TODO: Support this for vectors after legalize ops.
3964 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3965 // Canonicalize setgt X, Min --> setne X, Min
3966 if (C1 == MinVal)
3967 return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3968
3969 // If we have setugt X, Max-1, turn it into seteq X, Max
3970 if (C1 == MaxVal-1)
3971 return DAG.getSetCC(dl, VT, N0,
3972 DAG.getConstant(MaxVal, dl, N0.getValueType()),
3973 ISD::SETEQ);
3974 }
3975 }
3976
3977 if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
3978 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3979 if (C1.isNullValue())
3980 if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
3981 VT, N0, N1, Cond, DCI, dl))
3982 return CC;
3983
3984 // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
3985 // For example, when high 32-bits of i64 X are known clear:
3986 // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
3987 // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
3988 bool CmpZero = N1C->getAPIntValue().isNullValue();
3989 bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();
3990 if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
3991 // Match or(lo,shl(hi,bw/2)) pattern.
3992 auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
3993 unsigned EltBits = V.getScalarValueSizeInBits();
3994 if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
3995 return false;
3996 SDValue LHS = V.getOperand(0);
3997 SDValue RHS = V.getOperand(1);
3998 APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
3999 // Unshifted element must have zero upperbits.
4000 if (RHS.getOpcode() == ISD::SHL &&
4001 isa<ConstantSDNode>(RHS.getOperand(1)) &&
4002 RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4003 DAG.MaskedValueIsZero(LHS, HiBits)) {
4004 Lo = LHS;
4005 Hi = RHS.getOperand(0);
4006 return true;
4007 }
4008 if (LHS.getOpcode() == ISD::SHL &&
4009 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4010 LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4011 DAG.MaskedValueIsZero(RHS, HiBits)) {
4012 Lo = RHS;
4013 Hi = LHS.getOperand(0);
4014 return true;
4015 }
4016 return false;
4017 };
4018
4019 auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4020 unsigned EltBits = N0.getScalarValueSizeInBits();
4021 unsigned HalfBits = EltBits / 2;
4022 APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4023 SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4024 SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4025 SDValue NewN0 =
4026 DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4027 SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4028 return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4029 };
4030
4031 SDValue Lo, Hi;
4032 if (IsConcat(N0, Lo, Hi))
4033 return MergeConcat(Lo, Hi);
4034
4035 if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4036 SDValue Lo0, Lo1, Hi0, Hi1;
4037 if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4038 IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4039 return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4040 DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4041 }
4042 }
4043 }
4044 }
4045
4046 // If we have "setcc X, C0", check to see if we can shrink the immediate
4047 // by changing cc.
4048 // TODO: Support this for vectors after legalize ops.
4049 if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4050 // SETUGT X, SINTMAX -> SETLT X, 0
4051 // SETUGE X, SINTMIN -> SETLT X, 0
4052 if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4053 (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4054 return DAG.getSetCC(dl, VT, N0,
4055 DAG.getConstant(0, dl, N1.getValueType()),
4056 ISD::SETLT);
4057
4058 // SETULT X, SINTMIN -> SETGT X, -1
4059 // SETULE X, SINTMAX -> SETGT X, -1
4060 if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4061 (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4062 return DAG.getSetCC(dl, VT, N0,
4063 DAG.getAllOnesConstant(dl, N1.getValueType()),
4064 ISD::SETGT);
4065 }
4066 }
4067
4068 // Back to non-vector simplifications.
4069 // TODO: Can we do these for vector splats?
4070 if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4071 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4072 const APInt &C1 = N1C->getAPIntValue();
4073 EVT ShValTy = N0.getValueType();
4074
4075 // Fold bit comparisons when we can. This will result in an
4076 // incorrect value when boolean false is negative one, unless
4077 // the bitsize is 1 in which case the false value is the same
4078 // in practice regardless of the representation.
4079 if ((VT.getSizeInBits() == 1 ||
4080 getBooleanContents(N0.getValueType()) == ZeroOrOneBooleanContent) &&
4081 (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4082 (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4083 N0.getOpcode() == ISD::AND) {
4084 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4085 EVT ShiftTy =
4086 getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4087 if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4088 // Perform the xform if the AND RHS is a single bit.
4089 unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4090 if (AndRHS->getAPIntValue().isPowerOf2() &&
4091 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4092 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4093 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4094 DAG.getConstant(ShCt, dl, ShiftTy)));
4095 }
4096 } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4097 // (X & 8) == 8 --> (X & 8) >> 3
4098 // Perform the xform if C1 is a single bit.
4099 unsigned ShCt = C1.logBase2();
4100 if (C1.isPowerOf2() &&
4101 !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4102 return DAG.getNode(ISD::TRUNCATE, dl, VT,
4103 DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4104 DAG.getConstant(ShCt, dl, ShiftTy)));
4105 }
4106 }
4107 }
4108 }
4109
4110 if (C1.getMinSignedBits() <= 64 &&
4111 !isLegalICmpImmediate(C1.getSExtValue())) {
4112 EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4113 // (X & -256) == 256 -> (X >> 8) == 1
4114 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4115 N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4116 if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4117 const APInt &AndRHSC = AndRHS->getAPIntValue();
4118 if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
4119 unsigned ShiftBits = AndRHSC.countTrailingZeros();
4120 if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4121 SDValue Shift =
4122 DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4123 DAG.getConstant(ShiftBits, dl, ShiftTy));
4124 SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4125 return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4126 }
4127 }
4128 }
4129 } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4130 Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4131 bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4132 // X < 0x100000000 -> (X >> 32) < 1
4133 // X >= 0x100000000 -> (X >> 32) >= 1
4134 // X <= 0x0ffffffff -> (X >> 32) < 1
4135 // X > 0x0ffffffff -> (X >> 32) >= 1
4136 unsigned ShiftBits;
4137 APInt NewC = C1;
4138 ISD::CondCode NewCond = Cond;
4139 if (AdjOne) {
4140 ShiftBits = C1.countTrailingOnes();
4141 NewC = NewC + 1;
4142 NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4143 } else {
4144 ShiftBits = C1.countTrailingZeros();
4145 }
4146 NewC.lshrInPlace(ShiftBits);
4147 if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4148 isLegalICmpImmediate(NewC.getSExtValue()) &&
4149 !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4150 SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4151 DAG.getConstant(ShiftBits, dl, ShiftTy));
4152 SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4153 return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4154 }
4155 }
4156 }
4157 }
4158
4159 if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4160 auto *CFP = cast<ConstantFPSDNode>(N1);
4161 assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
4162
4163 // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4164 // constant if knowing that the operand is non-nan is enough. We prefer to
4165 // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4166 // materialize 0.0.
4167 if (Cond == ISD::SETO || Cond == ISD::SETUO)
4168 return DAG.getSetCC(dl, VT, N0, N0, Cond);
4169
4170 // setcc (fneg x), C -> setcc swap(pred) x, -C
4171 if (N0.getOpcode() == ISD::FNEG) {
4172 ISD::CondCode SwapCond = ISD::getSetCCSwappedOperands(Cond);
4173 if (DCI.isBeforeLegalizeOps() ||
4174 isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4175 SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4176 return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4177 }
4178 }
4179
4180 // If the condition is not legal, see if we can find an equivalent one
4181 // which is legal.
4182 if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4183 // If the comparison was an awkward floating-point == or != and one of
4184 // the comparison operands is infinity or negative infinity, convert the
4185 // condition to a less-awkward <= or >=.
4186 if (CFP->getValueAPF().isInfinity()) {
4187 bool IsNegInf = CFP->getValueAPF().isNegative();
4188 ISD::CondCode NewCond = ISD::SETCC_INVALID;
4189 switch (Cond) {
4190 case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4191 case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4192 case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4193 case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4194 default: break;
4195 }
4196 if (NewCond != ISD::SETCC_INVALID &&
4197 isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4198 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4199 }
4200 }
4201 }
4202
4203 if (N0 == N1) {
4204 // The sext(setcc()) => setcc() optimization relies on the appropriate
4205 // constant being emitted.
4206 assert(!N0.getValueType().isInteger() &&
4207 "Integer types should be handled by FoldSetCC");
4208
4209 bool EqTrue = ISD::isTrueWhenEqual(Cond);
4210 unsigned UOF = ISD::getUnorderedFlavor(Cond);
4211 if (UOF == 2) // FP operators that are undefined on NaNs.
4212 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4213 if (UOF == unsigned(EqTrue))
4214 return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4215 // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4216 // if it is not already.
4217 ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4218 if (NewCond != Cond &&
4219 (DCI.isBeforeLegalizeOps() ||
4220 isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4221 return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4222 }
4223
4224 if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4225 N0.getValueType().isInteger()) {
4226 if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4227 N0.getOpcode() == ISD::XOR) {
4228 // Simplify (X+Y) == (X+Z) --> Y == Z
4229 if (N0.getOpcode() == N1.getOpcode()) {
4230 if (N0.getOperand(0) == N1.getOperand(0))
4231 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4232 if (N0.getOperand(1) == N1.getOperand(1))
4233 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4234 if (isCommutativeBinOp(N0.getOpcode())) {
4235 // If X op Y == Y op X, try other combinations.
4236 if (N0.getOperand(0) == N1.getOperand(1))
4237 return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4238 Cond);
4239 if (N0.getOperand(1) == N1.getOperand(0))
4240 return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4241 Cond);
4242 }
4243 }
4244
4245 // If RHS is a legal immediate value for a compare instruction, we need
4246 // to be careful about increasing register pressure needlessly.
4247 bool LegalRHSImm = false;
4248
4249 if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4250 if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4251 // Turn (X+C1) == C2 --> X == C2-C1
4252 if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4253 return DAG.getSetCC(dl, VT, N0.getOperand(0),
4254 DAG.getConstant(RHSC->getAPIntValue()-
4255 LHSR->getAPIntValue(),
4256 dl, N0.getValueType()), Cond);
4257 }
4258
4259 // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4260 if (N0.getOpcode() == ISD::XOR)
4261 // If we know that all of the inverted bits are zero, don't bother
4262 // performing the inversion.
4263 if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4264 return
4265 DAG.getSetCC(dl, VT, N0.getOperand(0),
4266 DAG.getConstant(LHSR->getAPIntValue() ^
4267 RHSC->getAPIntValue(),
4268 dl, N0.getValueType()),
4269 Cond);
4270 }
4271
4272 // Turn (C1-X) == C2 --> X == C1-C2
4273 if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4274 if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4275 return
4276 DAG.getSetCC(dl, VT, N0.getOperand(1),
4277 DAG.getConstant(SUBC->getAPIntValue() -
4278 RHSC->getAPIntValue(),
4279 dl, N0.getValueType()),
4280 Cond);
4281 }
4282 }
4283
4284 // Could RHSC fold directly into a compare?
4285 if (RHSC->getValueType(0).getSizeInBits() <= 64)
4286 LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4287 }
4288
4289 // (X+Y) == X --> Y == 0 and similar folds.
4290 // Don't do this if X is an immediate that can fold into a cmp
4291 // instruction and X+Y has other uses. It could be an induction variable
4292 // chain, and the transform would increase register pressure.
4293 if (!LegalRHSImm || N0.hasOneUse())
4294 if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4295 return V;
4296 }
4297
4298 if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4299 N1.getOpcode() == ISD::XOR)
4300 if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4301 return V;
4302
4303 if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4304 return V;
4305 }
4306
4307 // Fold remainder of division by a constant.
4308 if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4309 N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4310 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
4311
4312 // When division is cheap or optimizing for minimum size,
4313 // fall through to DIVREM creation by skipping this fold.
4314 if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {
4315 if (N0.getOpcode() == ISD::UREM) {
4316 if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4317 return Folded;
4318 } else if (N0.getOpcode() == ISD::SREM) {
4319 if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4320 return Folded;
4321 }
4322 }
4323 }
4324
4325 // Fold away ALL boolean setcc's.
4326 if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4327 SDValue Temp;
4328 switch (Cond) {
4329 default: llvm_unreachable("Unknown integer setcc!");
4330 case ISD::SETEQ: // X == Y -> ~(X^Y)
4331 Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4332 N0 = DAG.getNOT(dl, Temp, OpVT);
4333 if (!DCI.isCalledByLegalizer())
4334 DCI.AddToWorklist(Temp.getNode());
4335 break;
4336 case ISD::SETNE: // X != Y --> (X^Y)
4337 N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4338 break;
4339 case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4340 case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4341 Temp = DAG.getNOT(dl, N0, OpVT);
4342 N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4343 if (!DCI.isCalledByLegalizer())
4344 DCI.AddToWorklist(Temp.getNode());
4345 break;
4346 case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4347 case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4348 Temp = DAG.getNOT(dl, N1, OpVT);
4349 N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4350 if (!DCI.isCalledByLegalizer())
4351 DCI.AddToWorklist(Temp.getNode());
4352 break;
4353 case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4354 case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4355 Temp = DAG.getNOT(dl, N0, OpVT);
4356 N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4357 if (!DCI.isCalledByLegalizer())
4358 DCI.AddToWorklist(Temp.getNode());
4359 break;
4360 case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4361 case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4362 Temp = DAG.getNOT(dl, N1, OpVT);
4363 N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4364 break;
4365 }
4366 if (VT.getScalarType() != MVT::i1) {
4367 if (!DCI.isCalledByLegalizer())
4368 DCI.AddToWorklist(N0.getNode());
4369 // FIXME: If running after legalize, we probably can't do this.
4370 ISD::NodeType ExtendCode = getExtendForContent(getBooleanContents(OpVT));
4371 N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4372 }
4373 return N0;
4374 }
4375
4376 // Could not fold it.
4377 return SDValue();
4378 }
4379
4380 /// Returns true (and the GlobalValue and the offset) if the node is a
4381 /// GlobalAddress + offset.
isGAPlusOffset(SDNode * WN,const GlobalValue * & GA,int64_t & Offset) const4382 bool TargetLowering::isGAPlusOffset(SDNode *WN, const GlobalValue *&GA,
4383 int64_t &Offset) const {
4384
4385 SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4386
4387 if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4388 GA = GASD->getGlobal();
4389 Offset += GASD->getOffset();
4390 return true;
4391 }
4392
4393 if (N->getOpcode() == ISD::ADD) {
4394 SDValue N1 = N->getOperand(0);
4395 SDValue N2 = N->getOperand(1);
4396 if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4397 if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4398 Offset += V->getSExtValue();
4399 return true;
4400 }
4401 } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4402 if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4403 Offset += V->getSExtValue();
4404 return true;
4405 }
4406 }
4407 }
4408
4409 return false;
4410 }
4411
PerformDAGCombine(SDNode * N,DAGCombinerInfo & DCI) const4412 SDValue TargetLowering::PerformDAGCombine(SDNode *N,
4413 DAGCombinerInfo &DCI) const {
4414 // Default implementation: no optimization.
4415 return SDValue();
4416 }
4417
4418 //===----------------------------------------------------------------------===//
4419 // Inline Assembler Implementation Methods
4420 //===----------------------------------------------------------------------===//
4421
4422 TargetLowering::ConstraintType
getConstraintType(StringRef Constraint) const4423 TargetLowering::getConstraintType(StringRef Constraint) const {
4424 unsigned S = Constraint.size();
4425
4426 if (S == 1) {
4427 switch (Constraint[0]) {
4428 default: break;
4429 case 'r':
4430 return C_RegisterClass;
4431 case 'm': // memory
4432 case 'o': // offsetable
4433 case 'V': // not offsetable
4434 return C_Memory;
4435 case 'n': // Simple Integer
4436 case 'E': // Floating Point Constant
4437 case 'F': // Floating Point Constant
4438 return C_Immediate;
4439 case 'i': // Simple Integer or Relocatable Constant
4440 case 's': // Relocatable Constant
4441 case 'p': // Address.
4442 case 'X': // Allow ANY value.
4443 case 'I': // Target registers.
4444 case 'J':
4445 case 'K':
4446 case 'L':
4447 case 'M':
4448 case 'N':
4449 case 'O':
4450 case 'P':
4451 case '<':
4452 case '>':
4453 return C_Other;
4454 }
4455 }
4456
4457 if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') {
4458 if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}"
4459 return C_Memory;
4460 return C_Register;
4461 }
4462 return C_Unknown;
4463 }
4464
4465 /// Try to replace an X constraint, which matches anything, with another that
4466 /// has more specific requirements based on the type of the corresponding
4467 /// operand.
LowerXConstraint(EVT ConstraintVT) const4468 const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
4469 if (ConstraintVT.isInteger())
4470 return "r";
4471 if (ConstraintVT.isFloatingPoint())
4472 return "f"; // works for many targets
4473 return nullptr;
4474 }
4475
LowerAsmOutputForConstraint(SDValue & Chain,SDValue & Flag,const SDLoc & DL,const AsmOperandInfo & OpInfo,SelectionDAG & DAG) const4476 SDValue TargetLowering::LowerAsmOutputForConstraint(
4477 SDValue &Chain, SDValue &Flag, const SDLoc &DL,
4478 const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const {
4479 return SDValue();
4480 }
4481
4482 /// Lower the specified operand into the Ops vector.
4483 /// If it is invalid, don't add anything to Ops.
LowerAsmOperandForConstraint(SDValue Op,std::string & Constraint,std::vector<SDValue> & Ops,SelectionDAG & DAG) const4484 void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
4485 std::string &Constraint,
4486 std::vector<SDValue> &Ops,
4487 SelectionDAG &DAG) const {
4488
4489 if (Constraint.length() > 1) return;
4490
4491 char ConstraintLetter = Constraint[0];
4492 switch (ConstraintLetter) {
4493 default: break;
4494 case 'X': // Allows any operand; labels (basic block) use this.
4495 if (Op.getOpcode() == ISD::BasicBlock ||
4496 Op.getOpcode() == ISD::TargetBlockAddress) {
4497 Ops.push_back(Op);
4498 return;
4499 }
4500 LLVM_FALLTHROUGH;
4501 case 'i': // Simple Integer or Relocatable Constant
4502 case 'n': // Simple Integer
4503 case 's': { // Relocatable Constant
4504
4505 GlobalAddressSDNode *GA;
4506 ConstantSDNode *C;
4507 BlockAddressSDNode *BA;
4508 uint64_t Offset = 0;
4509
4510 // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C),
4511 // etc., since getelementpointer is variadic. We can't use
4512 // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible
4513 // while in this case the GA may be furthest from the root node which is
4514 // likely an ISD::ADD.
4515 while (1) {
4516 if ((GA = dyn_cast<GlobalAddressSDNode>(Op)) && ConstraintLetter != 'n') {
4517 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
4518 GA->getValueType(0),
4519 Offset + GA->getOffset()));
4520 return;
4521 }
4522 if ((C = dyn_cast<ConstantSDNode>(Op)) && ConstraintLetter != 's') {
4523 // gcc prints these as sign extended. Sign extend value to 64 bits
4524 // now; without this it would get ZExt'd later in
4525 // ScheduleDAGSDNodes::EmitNode, which is very generic.
4526 bool IsBool = C->getConstantIntValue()->getBitWidth() == 1;
4527 BooleanContent BCont = getBooleanContents(MVT::i64);
4528 ISD::NodeType ExtOpc =
4529 IsBool ? getExtendForContent(BCont) : ISD::SIGN_EXTEND;
4530 int64_t ExtVal =
4531 ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() : C->getSExtValue();
4532 Ops.push_back(
4533 DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64));
4534 return;
4535 }
4536 if ((BA = dyn_cast<BlockAddressSDNode>(Op)) && ConstraintLetter != 'n') {
4537 Ops.push_back(DAG.getTargetBlockAddress(
4538 BA->getBlockAddress(), BA->getValueType(0),
4539 Offset + BA->getOffset(), BA->getTargetFlags()));
4540 return;
4541 }
4542 const unsigned OpCode = Op.getOpcode();
4543 if (OpCode == ISD::ADD || OpCode == ISD::SUB) {
4544 if ((C = dyn_cast<ConstantSDNode>(Op.getOperand(0))))
4545 Op = Op.getOperand(1);
4546 // Subtraction is not commutative.
4547 else if (OpCode == ISD::ADD &&
4548 (C = dyn_cast<ConstantSDNode>(Op.getOperand(1))))
4549 Op = Op.getOperand(0);
4550 else
4551 return;
4552 Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue();
4553 continue;
4554 }
4555 return;
4556 }
4557 break;
4558 }
4559 }
4560 }
4561
4562 std::pair<unsigned, const TargetRegisterClass *>
getRegForInlineAsmConstraint(const TargetRegisterInfo * RI,StringRef Constraint,MVT VT) const4563 TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI,
4564 StringRef Constraint,
4565 MVT VT) const {
4566 if (Constraint.empty() || Constraint[0] != '{')
4567 return std::make_pair(0u, static_cast<TargetRegisterClass *>(nullptr));
4568 assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?");
4569
4570 // Remove the braces from around the name.
4571 StringRef RegName(Constraint.data() + 1, Constraint.size() - 2);
4572
4573 std::pair<unsigned, const TargetRegisterClass *> R =
4574 std::make_pair(0u, static_cast<const TargetRegisterClass *>(nullptr));
4575
4576 // Figure out which register class contains this reg.
4577 for (const TargetRegisterClass *RC : RI->regclasses()) {
4578 // If none of the value types for this register class are valid, we
4579 // can't use it. For example, 64-bit reg classes on 32-bit targets.
4580 if (!isLegalRC(*RI, *RC))
4581 continue;
4582
4583 for (const MCPhysReg &PR : *RC) {
4584 if (RegName.equals_lower(RI->getRegAsmName(PR))) {
4585 std::pair<unsigned, const TargetRegisterClass *> S =
4586 std::make_pair(PR, RC);
4587
4588 // If this register class has the requested value type, return it,
4589 // otherwise keep searching and return the first class found
4590 // if no other is found which explicitly has the requested type.
4591 if (RI->isTypeLegalForClass(*RC, VT))
4592 return S;
4593 if (!R.second)
4594 R = S;
4595 }
4596 }
4597 }
4598
4599 return R;
4600 }
4601
4602 //===----------------------------------------------------------------------===//
4603 // Constraint Selection.
4604
4605 /// Return true of this is an input operand that is a matching constraint like
4606 /// "4".
isMatchingInputConstraint() const4607 bool TargetLowering::AsmOperandInfo::isMatchingInputConstraint() const {
4608 assert(!ConstraintCode.empty() && "No known constraint!");
4609 return isdigit(static_cast<unsigned char>(ConstraintCode[0]));
4610 }
4611
4612 /// If this is an input matching constraint, this method returns the output
4613 /// operand it matches.
getMatchedOperand() const4614 unsigned TargetLowering::AsmOperandInfo::getMatchedOperand() const {
4615 assert(!ConstraintCode.empty() && "No known constraint!");
4616 return atoi(ConstraintCode.c_str());
4617 }
4618
4619 /// Split up the constraint string from the inline assembly value into the
4620 /// specific constraints and their prefixes, and also tie in the associated
4621 /// operand values.
4622 /// If this returns an empty vector, and if the constraint string itself
4623 /// isn't empty, there was an error parsing.
4624 TargetLowering::AsmOperandInfoVector
ParseConstraints(const DataLayout & DL,const TargetRegisterInfo * TRI,const CallBase & Call) const4625 TargetLowering::ParseConstraints(const DataLayout &DL,
4626 const TargetRegisterInfo *TRI,
4627 const CallBase &Call) const {
4628 /// Information about all of the constraints.
4629 AsmOperandInfoVector ConstraintOperands;
4630 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand());
4631 unsigned maCount = 0; // Largest number of multiple alternative constraints.
4632
4633 // Do a prepass over the constraints, canonicalizing them, and building up the
4634 // ConstraintOperands list.
4635 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
4636 unsigned ResNo = 0; // ResNo - The result number of the next output.
4637
4638 for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) {
4639 ConstraintOperands.emplace_back(std::move(CI));
4640 AsmOperandInfo &OpInfo = ConstraintOperands.back();
4641
4642 // Update multiple alternative constraint count.
4643 if (OpInfo.multipleAlternatives.size() > maCount)
4644 maCount = OpInfo.multipleAlternatives.size();
4645
4646 OpInfo.ConstraintVT = MVT::Other;
4647
4648 // Compute the value type for each operand.
4649 switch (OpInfo.Type) {
4650 case InlineAsm::isOutput:
4651 // Indirect outputs just consume an argument.
4652 if (OpInfo.isIndirect) {
4653 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4654 break;
4655 }
4656
4657 // The return value of the call is this value. As such, there is no
4658 // corresponding argument.
4659 assert(!Call.getType()->isVoidTy() && "Bad inline asm!");
4660 if (StructType *STy = dyn_cast<StructType>(Call.getType())) {
4661 OpInfo.ConstraintVT =
4662 getSimpleValueType(DL, STy->getElementType(ResNo));
4663 } else {
4664 assert(ResNo == 0 && "Asm only has one result!");
4665 OpInfo.ConstraintVT = getSimpleValueType(DL, Call.getType());
4666 }
4667 ++ResNo;
4668 break;
4669 case InlineAsm::isInput:
4670 OpInfo.CallOperandVal = Call.getArgOperand(ArgNo++);
4671 break;
4672 case InlineAsm::isClobber:
4673 // Nothing to do.
4674 break;
4675 }
4676
4677 if (OpInfo.CallOperandVal) {
4678 llvm::Type *OpTy = OpInfo.CallOperandVal->getType();
4679 if (OpInfo.isIndirect) {
4680 llvm::PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
4681 if (!PtrTy)
4682 report_fatal_error("Indirect operand for inline asm not a pointer!");
4683 OpTy = PtrTy->getElementType();
4684 }
4685
4686 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
4687 if (StructType *STy = dyn_cast<StructType>(OpTy))
4688 if (STy->getNumElements() == 1)
4689 OpTy = STy->getElementType(0);
4690
4691 // If OpTy is not a single value, it may be a struct/union that we
4692 // can tile with integers.
4693 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
4694 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
4695 switch (BitSize) {
4696 default: break;
4697 case 1:
4698 case 8:
4699 case 16:
4700 case 32:
4701 case 64:
4702 case 128:
4703 OpInfo.ConstraintVT =
4704 MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true);
4705 break;
4706 }
4707 } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
4708 unsigned PtrSize = DL.getPointerSizeInBits(PT->getAddressSpace());
4709 OpInfo.ConstraintVT = MVT::getIntegerVT(PtrSize);
4710 } else {
4711 OpInfo.ConstraintVT = MVT::getVT(OpTy, true);
4712 }
4713 }
4714 }
4715
4716 // If we have multiple alternative constraints, select the best alternative.
4717 if (!ConstraintOperands.empty()) {
4718 if (maCount) {
4719 unsigned bestMAIndex = 0;
4720 int bestWeight = -1;
4721 // weight: -1 = invalid match, and 0 = so-so match to 5 = good match.
4722 int weight = -1;
4723 unsigned maIndex;
4724 // Compute the sums of the weights for each alternative, keeping track
4725 // of the best (highest weight) one so far.
4726 for (maIndex = 0; maIndex < maCount; ++maIndex) {
4727 int weightSum = 0;
4728 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4729 cIndex != eIndex; ++cIndex) {
4730 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4731 if (OpInfo.Type == InlineAsm::isClobber)
4732 continue;
4733
4734 // If this is an output operand with a matching input operand,
4735 // look up the matching input. If their types mismatch, e.g. one
4736 // is an integer, the other is floating point, or their sizes are
4737 // different, flag it as an maCantMatch.
4738 if (OpInfo.hasMatchingInput()) {
4739 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4740 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4741 if ((OpInfo.ConstraintVT.isInteger() !=
4742 Input.ConstraintVT.isInteger()) ||
4743 (OpInfo.ConstraintVT.getSizeInBits() !=
4744 Input.ConstraintVT.getSizeInBits())) {
4745 weightSum = -1; // Can't match.
4746 break;
4747 }
4748 }
4749 }
4750 weight = getMultipleConstraintMatchWeight(OpInfo, maIndex);
4751 if (weight == -1) {
4752 weightSum = -1;
4753 break;
4754 }
4755 weightSum += weight;
4756 }
4757 // Update best.
4758 if (weightSum > bestWeight) {
4759 bestWeight = weightSum;
4760 bestMAIndex = maIndex;
4761 }
4762 }
4763
4764 // Now select chosen alternative in each constraint.
4765 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4766 cIndex != eIndex; ++cIndex) {
4767 AsmOperandInfo &cInfo = ConstraintOperands[cIndex];
4768 if (cInfo.Type == InlineAsm::isClobber)
4769 continue;
4770 cInfo.selectAlternative(bestMAIndex);
4771 }
4772 }
4773 }
4774
4775 // Check and hook up tied operands, choose constraint code to use.
4776 for (unsigned cIndex = 0, eIndex = ConstraintOperands.size();
4777 cIndex != eIndex; ++cIndex) {
4778 AsmOperandInfo &OpInfo = ConstraintOperands[cIndex];
4779
4780 // If this is an output operand with a matching input operand, look up the
4781 // matching input. If their types mismatch, e.g. one is an integer, the
4782 // other is floating point, or their sizes are different, flag it as an
4783 // error.
4784 if (OpInfo.hasMatchingInput()) {
4785 AsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
4786
4787 if (OpInfo.ConstraintVT != Input.ConstraintVT) {
4788 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
4789 getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
4790 OpInfo.ConstraintVT);
4791 std::pair<unsigned, const TargetRegisterClass *> InputRC =
4792 getRegForInlineAsmConstraint(TRI, Input.ConstraintCode,
4793 Input.ConstraintVT);
4794 if ((OpInfo.ConstraintVT.isInteger() !=
4795 Input.ConstraintVT.isInteger()) ||
4796 (MatchRC.second != InputRC.second)) {
4797 report_fatal_error("Unsupported asm: input constraint"
4798 " with a matching output constraint of"
4799 " incompatible type!");
4800 }
4801 }
4802 }
4803 }
4804
4805 return ConstraintOperands;
4806 }
4807
4808 /// Return an integer indicating how general CT is.
getConstraintGenerality(TargetLowering::ConstraintType CT)4809 static unsigned getConstraintGenerality(TargetLowering::ConstraintType CT) {
4810 switch (CT) {
4811 case TargetLowering::C_Immediate:
4812 case TargetLowering::C_Other:
4813 case TargetLowering::C_Unknown:
4814 return 0;
4815 case TargetLowering::C_Register:
4816 return 1;
4817 case TargetLowering::C_RegisterClass:
4818 return 2;
4819 case TargetLowering::C_Memory:
4820 return 3;
4821 }
4822 llvm_unreachable("Invalid constraint type");
4823 }
4824
4825 /// Examine constraint type and operand type and determine a weight value.
4826 /// This object must already have been set up with the operand type
4827 /// and the current alternative constraint selected.
4828 TargetLowering::ConstraintWeight
getMultipleConstraintMatchWeight(AsmOperandInfo & info,int maIndex) const4829 TargetLowering::getMultipleConstraintMatchWeight(
4830 AsmOperandInfo &info, int maIndex) const {
4831 InlineAsm::ConstraintCodeVector *rCodes;
4832 if (maIndex >= (int)info.multipleAlternatives.size())
4833 rCodes = &info.Codes;
4834 else
4835 rCodes = &info.multipleAlternatives[maIndex].Codes;
4836 ConstraintWeight BestWeight = CW_Invalid;
4837
4838 // Loop over the options, keeping track of the most general one.
4839 for (unsigned i = 0, e = rCodes->size(); i != e; ++i) {
4840 ConstraintWeight weight =
4841 getSingleConstraintMatchWeight(info, (*rCodes)[i].c_str());
4842 if (weight > BestWeight)
4843 BestWeight = weight;
4844 }
4845
4846 return BestWeight;
4847 }
4848
4849 /// Examine constraint type and operand type and determine a weight value.
4850 /// This object must already have been set up with the operand type
4851 /// and the current alternative constraint selected.
4852 TargetLowering::ConstraintWeight
getSingleConstraintMatchWeight(AsmOperandInfo & info,const char * constraint) const4853 TargetLowering::getSingleConstraintMatchWeight(
4854 AsmOperandInfo &info, const char *constraint) const {
4855 ConstraintWeight weight = CW_Invalid;
4856 Value *CallOperandVal = info.CallOperandVal;
4857 // If we don't have a value, we can't do a match,
4858 // but allow it at the lowest weight.
4859 if (!CallOperandVal)
4860 return CW_Default;
4861 // Look at the constraint type.
4862 switch (*constraint) {
4863 case 'i': // immediate integer.
4864 case 'n': // immediate integer with a known value.
4865 if (isa<ConstantInt>(CallOperandVal))
4866 weight = CW_Constant;
4867 break;
4868 case 's': // non-explicit intregal immediate.
4869 if (isa<GlobalValue>(CallOperandVal))
4870 weight = CW_Constant;
4871 break;
4872 case 'E': // immediate float if host format.
4873 case 'F': // immediate float.
4874 if (isa<ConstantFP>(CallOperandVal))
4875 weight = CW_Constant;
4876 break;
4877 case '<': // memory operand with autodecrement.
4878 case '>': // memory operand with autoincrement.
4879 case 'm': // memory operand.
4880 case 'o': // offsettable memory operand
4881 case 'V': // non-offsettable memory operand
4882 weight = CW_Memory;
4883 break;
4884 case 'r': // general register.
4885 case 'g': // general register, memory operand or immediate integer.
4886 // note: Clang converts "g" to "imr".
4887 if (CallOperandVal->getType()->isIntegerTy())
4888 weight = CW_Register;
4889 break;
4890 case 'X': // any operand.
4891 default:
4892 weight = CW_Default;
4893 break;
4894 }
4895 return weight;
4896 }
4897
4898 /// If there are multiple different constraints that we could pick for this
4899 /// operand (e.g. "imr") try to pick the 'best' one.
4900 /// This is somewhat tricky: constraints fall into four classes:
4901 /// Other -> immediates and magic values
4902 /// Register -> one specific register
4903 /// RegisterClass -> a group of regs
4904 /// Memory -> memory
4905 /// Ideally, we would pick the most specific constraint possible: if we have
4906 /// something that fits into a register, we would pick it. The problem here
4907 /// is that if we have something that could either be in a register or in
4908 /// memory that use of the register could cause selection of *other*
4909 /// operands to fail: they might only succeed if we pick memory. Because of
4910 /// this the heuristic we use is:
4911 ///
4912 /// 1) If there is an 'other' constraint, and if the operand is valid for
4913 /// that constraint, use it. This makes us take advantage of 'i'
4914 /// constraints when available.
4915 /// 2) Otherwise, pick the most general constraint present. This prefers
4916 /// 'm' over 'r', for example.
4917 ///
ChooseConstraint(TargetLowering::AsmOperandInfo & OpInfo,const TargetLowering & TLI,SDValue Op,SelectionDAG * DAG)4918 static void ChooseConstraint(TargetLowering::AsmOperandInfo &OpInfo,
4919 const TargetLowering &TLI,
4920 SDValue Op, SelectionDAG *DAG) {
4921 assert(OpInfo.Codes.size() > 1 && "Doesn't have multiple constraint options");
4922 unsigned BestIdx = 0;
4923 TargetLowering::ConstraintType BestType = TargetLowering::C_Unknown;
4924 int BestGenerality = -1;
4925
4926 // Loop over the options, keeping track of the most general one.
4927 for (unsigned i = 0, e = OpInfo.Codes.size(); i != e; ++i) {
4928 TargetLowering::ConstraintType CType =
4929 TLI.getConstraintType(OpInfo.Codes[i]);
4930
4931 // Indirect 'other' or 'immediate' constraints are not allowed.
4932 if (OpInfo.isIndirect && !(CType == TargetLowering::C_Memory ||
4933 CType == TargetLowering::C_Register ||
4934 CType == TargetLowering::C_RegisterClass))
4935 continue;
4936
4937 // If this is an 'other' or 'immediate' constraint, see if the operand is
4938 // valid for it. For example, on X86 we might have an 'rI' constraint. If
4939 // the operand is an integer in the range [0..31] we want to use I (saving a
4940 // load of a register), otherwise we must use 'r'.
4941 if ((CType == TargetLowering::C_Other ||
4942 CType == TargetLowering::C_Immediate) && Op.getNode()) {
4943 assert(OpInfo.Codes[i].size() == 1 &&
4944 "Unhandled multi-letter 'other' constraint");
4945 std::vector<SDValue> ResultOps;
4946 TLI.LowerAsmOperandForConstraint(Op, OpInfo.Codes[i],
4947 ResultOps, *DAG);
4948 if (!ResultOps.empty()) {
4949 BestType = CType;
4950 BestIdx = i;
4951 break;
4952 }
4953 }
4954
4955 // Things with matching constraints can only be registers, per gcc
4956 // documentation. This mainly affects "g" constraints.
4957 if (CType == TargetLowering::C_Memory && OpInfo.hasMatchingInput())
4958 continue;
4959
4960 // This constraint letter is more general than the previous one, use it.
4961 int Generality = getConstraintGenerality(CType);
4962 if (Generality > BestGenerality) {
4963 BestType = CType;
4964 BestIdx = i;
4965 BestGenerality = Generality;
4966 }
4967 }
4968
4969 OpInfo.ConstraintCode = OpInfo.Codes[BestIdx];
4970 OpInfo.ConstraintType = BestType;
4971 }
4972
4973 /// Determines the constraint code and constraint type to use for the specific
4974 /// AsmOperandInfo, setting OpInfo.ConstraintCode and OpInfo.ConstraintType.
ComputeConstraintToUse(AsmOperandInfo & OpInfo,SDValue Op,SelectionDAG * DAG) const4975 void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
4976 SDValue Op,
4977 SelectionDAG *DAG) const {
4978 assert(!OpInfo.Codes.empty() && "Must have at least one constraint");
4979
4980 // Single-letter constraints ('r') are very common.
4981 if (OpInfo.Codes.size() == 1) {
4982 OpInfo.ConstraintCode = OpInfo.Codes[0];
4983 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
4984 } else {
4985 ChooseConstraint(OpInfo, *this, Op, DAG);
4986 }
4987
4988 // 'X' matches anything.
4989 if (OpInfo.ConstraintCode == "X" && OpInfo.CallOperandVal) {
4990 // Labels and constants are handled elsewhere ('X' is the only thing
4991 // that matches labels). For Functions, the type here is the type of
4992 // the result, which is not what we want to look at; leave them alone.
4993 Value *v = OpInfo.CallOperandVal;
4994 if (isa<BasicBlock>(v) || isa<ConstantInt>(v) || isa<Function>(v)) {
4995 OpInfo.CallOperandVal = v;
4996 return;
4997 }
4998
4999 if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
5000 return;
5001
5002 // Otherwise, try to resolve it to something we know about by looking at
5003 // the actual operand type.
5004 if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
5005 OpInfo.ConstraintCode = Repl;
5006 OpInfo.ConstraintType = getConstraintType(OpInfo.ConstraintCode);
5007 }
5008 }
5009 }
5010
5011 /// Given an exact SDIV by a constant, create a multiplication
5012 /// with the multiplicative inverse of the constant.
BuildExactSDIV(const TargetLowering & TLI,SDNode * N,const SDLoc & dl,SelectionDAG & DAG,SmallVectorImpl<SDNode * > & Created)5013 static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N,
5014 const SDLoc &dl, SelectionDAG &DAG,
5015 SmallVectorImpl<SDNode *> &Created) {
5016 SDValue Op0 = N->getOperand(0);
5017 SDValue Op1 = N->getOperand(1);
5018 EVT VT = N->getValueType(0);
5019 EVT SVT = VT.getScalarType();
5020 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
5021 EVT ShSVT = ShVT.getScalarType();
5022
5023 bool UseSRA = false;
5024 SmallVector<SDValue, 16> Shifts, Factors;
5025
5026 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5027 if (C->isNullValue())
5028 return false;
5029 APInt Divisor = C->getAPIntValue();
5030 unsigned Shift = Divisor.countTrailingZeros();
5031 if (Shift) {
5032 Divisor.ashrInPlace(Shift);
5033 UseSRA = true;
5034 }
5035 // Calculate the multiplicative inverse, using Newton's method.
5036 APInt t;
5037 APInt Factor = Divisor;
5038 while ((t = Divisor * Factor) != 1)
5039 Factor *= APInt(Divisor.getBitWidth(), 2) - t;
5040 Shifts.push_back(DAG.getConstant(Shift, dl, ShSVT));
5041 Factors.push_back(DAG.getConstant(Factor, dl, SVT));
5042 return true;
5043 };
5044
5045 // Collect all magic values from the build vector.
5046 if (!ISD::matchUnaryPredicate(Op1, BuildSDIVPattern))
5047 return SDValue();
5048
5049 SDValue Shift, Factor;
5050 if (Op1.getOpcode() == ISD::BUILD_VECTOR) {
5051 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5052 Factor = DAG.getBuildVector(VT, dl, Factors);
5053 } else if (Op1.getOpcode() == ISD::SPLAT_VECTOR) {
5054 assert(Shifts.size() == 1 && Factors.size() == 1 &&
5055 "Expected matchUnaryPredicate to return one element for scalable "
5056 "vectors");
5057 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5058 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5059 } else {
5060 assert(isa<ConstantSDNode>(Op1) && "Expected a constant");
5061 Shift = Shifts[0];
5062 Factor = Factors[0];
5063 }
5064
5065 SDValue Res = Op0;
5066
5067 // Shift the value upfront if it is even, so the LSB is one.
5068 if (UseSRA) {
5069 // TODO: For UDIV use SRL instead of SRA.
5070 SDNodeFlags Flags;
5071 Flags.setExact(true);
5072 Res = DAG.getNode(ISD::SRA, dl, VT, Res, Shift, Flags);
5073 Created.push_back(Res.getNode());
5074 }
5075
5076 return DAG.getNode(ISD::MUL, dl, VT, Res, Factor);
5077 }
5078
BuildSDIVPow2(SDNode * N,const APInt & Divisor,SelectionDAG & DAG,SmallVectorImpl<SDNode * > & Created) const5079 SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
5080 SelectionDAG &DAG,
5081 SmallVectorImpl<SDNode *> &Created) const {
5082 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
5083 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5084 if (TLI.isIntDivCheap(N->getValueType(0), Attr))
5085 return SDValue(N, 0); // Lower SDIV as SDIV
5086 return SDValue();
5087 }
5088
5089 /// Given an ISD::SDIV node expressing a divide by constant,
5090 /// return a DAG expression to select that will generate the same value by
5091 /// multiplying by a magic number.
5092 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
BuildSDIV(SDNode * N,SelectionDAG & DAG,bool IsAfterLegalization,SmallVectorImpl<SDNode * > & Created) const5093 SDValue TargetLowering::BuildSDIV(SDNode *N, SelectionDAG &DAG,
5094 bool IsAfterLegalization,
5095 SmallVectorImpl<SDNode *> &Created) const {
5096 SDLoc dl(N);
5097 EVT VT = N->getValueType(0);
5098 EVT SVT = VT.getScalarType();
5099 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5100 EVT ShSVT = ShVT.getScalarType();
5101 unsigned EltBits = VT.getScalarSizeInBits();
5102 EVT MulVT;
5103
5104 // Check to see if we can do this.
5105 // FIXME: We should be more aggressive here.
5106 if (!isTypeLegal(VT)) {
5107 // Limit this to simple scalars for now.
5108 if (VT.isVector() || !VT.isSimple())
5109 return SDValue();
5110
5111 // If this type will be promoted to a large enough type with a legal
5112 // multiply operation, we can go ahead and do this transform.
5113 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5114 return SDValue();
5115
5116 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5117 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5118 !isOperationLegal(ISD::MUL, MulVT))
5119 return SDValue();
5120 }
5121
5122 // If the sdiv has an 'exact' bit we can use a simpler lowering.
5123 if (N->getFlags().hasExact())
5124 return BuildExactSDIV(*this, N, dl, DAG, Created);
5125
5126 SmallVector<SDValue, 16> MagicFactors, Factors, Shifts, ShiftMasks;
5127
5128 auto BuildSDIVPattern = [&](ConstantSDNode *C) {
5129 if (C->isNullValue())
5130 return false;
5131
5132 const APInt &Divisor = C->getAPIntValue();
5133 APInt::ms magics = Divisor.magic();
5134 int NumeratorFactor = 0;
5135 int ShiftMask = -1;
5136
5137 if (Divisor.isOneValue() || Divisor.isAllOnesValue()) {
5138 // If d is +1/-1, we just multiply the numerator by +1/-1.
5139 NumeratorFactor = Divisor.getSExtValue();
5140 magics.m = 0;
5141 magics.s = 0;
5142 ShiftMask = 0;
5143 } else if (Divisor.isStrictlyPositive() && magics.m.isNegative()) {
5144 // If d > 0 and m < 0, add the numerator.
5145 NumeratorFactor = 1;
5146 } else if (Divisor.isNegative() && magics.m.isStrictlyPositive()) {
5147 // If d < 0 and m > 0, subtract the numerator.
5148 NumeratorFactor = -1;
5149 }
5150
5151 MagicFactors.push_back(DAG.getConstant(magics.m, dl, SVT));
5152 Factors.push_back(DAG.getConstant(NumeratorFactor, dl, SVT));
5153 Shifts.push_back(DAG.getConstant(magics.s, dl, ShSVT));
5154 ShiftMasks.push_back(DAG.getConstant(ShiftMask, dl, SVT));
5155 return true;
5156 };
5157
5158 SDValue N0 = N->getOperand(0);
5159 SDValue N1 = N->getOperand(1);
5160
5161 // Collect the shifts / magic values from each element.
5162 if (!ISD::matchUnaryPredicate(N1, BuildSDIVPattern))
5163 return SDValue();
5164
5165 SDValue MagicFactor, Factor, Shift, ShiftMask;
5166 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5167 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5168 Factor = DAG.getBuildVector(VT, dl, Factors);
5169 Shift = DAG.getBuildVector(ShVT, dl, Shifts);
5170 ShiftMask = DAG.getBuildVector(VT, dl, ShiftMasks);
5171 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5172 assert(MagicFactors.size() == 1 && Factors.size() == 1 &&
5173 Shifts.size() == 1 && ShiftMasks.size() == 1 &&
5174 "Expected matchUnaryPredicate to return one element for scalable "
5175 "vectors");
5176 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5177 Factor = DAG.getSplatVector(VT, dl, Factors[0]);
5178 Shift = DAG.getSplatVector(ShVT, dl, Shifts[0]);
5179 ShiftMask = DAG.getSplatVector(VT, dl, ShiftMasks[0]);
5180 } else {
5181 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
5182 MagicFactor = MagicFactors[0];
5183 Factor = Factors[0];
5184 Shift = Shifts[0];
5185 ShiftMask = ShiftMasks[0];
5186 }
5187
5188 // Multiply the numerator (operand 0) by the magic value.
5189 // FIXME: We should support doing a MUL in a wider type.
5190 auto GetMULHS = [&](SDValue X, SDValue Y) {
5191 // If the type isn't legal, use a wider mul of the the type calculated
5192 // earlier.
5193 if (!isTypeLegal(VT)) {
5194 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, X);
5195 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MulVT, Y);
5196 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5197 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5198 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5199 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5200 }
5201
5202 if (isOperationLegalOrCustom(ISD::MULHS, VT, IsAfterLegalization))
5203 return DAG.getNode(ISD::MULHS, dl, VT, X, Y);
5204 if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT, IsAfterLegalization)) {
5205 SDValue LoHi =
5206 DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5207 return SDValue(LoHi.getNode(), 1);
5208 }
5209 return SDValue();
5210 };
5211
5212 SDValue Q = GetMULHS(N0, MagicFactor);
5213 if (!Q)
5214 return SDValue();
5215
5216 Created.push_back(Q.getNode());
5217
5218 // (Optionally) Add/subtract the numerator using Factor.
5219 Factor = DAG.getNode(ISD::MUL, dl, VT, N0, Factor);
5220 Created.push_back(Factor.getNode());
5221 Q = DAG.getNode(ISD::ADD, dl, VT, Q, Factor);
5222 Created.push_back(Q.getNode());
5223
5224 // Shift right algebraic by shift value.
5225 Q = DAG.getNode(ISD::SRA, dl, VT, Q, Shift);
5226 Created.push_back(Q.getNode());
5227
5228 // Extract the sign bit, mask it and add it to the quotient.
5229 SDValue SignShift = DAG.getConstant(EltBits - 1, dl, ShVT);
5230 SDValue T = DAG.getNode(ISD::SRL, dl, VT, Q, SignShift);
5231 Created.push_back(T.getNode());
5232 T = DAG.getNode(ISD::AND, dl, VT, T, ShiftMask);
5233 Created.push_back(T.getNode());
5234 return DAG.getNode(ISD::ADD, dl, VT, Q, T);
5235 }
5236
5237 /// Given an ISD::UDIV node expressing a divide by constant,
5238 /// return a DAG expression to select that will generate the same value by
5239 /// multiplying by a magic number.
5240 /// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
BuildUDIV(SDNode * N,SelectionDAG & DAG,bool IsAfterLegalization,SmallVectorImpl<SDNode * > & Created) const5241 SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG,
5242 bool IsAfterLegalization,
5243 SmallVectorImpl<SDNode *> &Created) const {
5244 SDLoc dl(N);
5245 EVT VT = N->getValueType(0);
5246 EVT SVT = VT.getScalarType();
5247 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
5248 EVT ShSVT = ShVT.getScalarType();
5249 unsigned EltBits = VT.getScalarSizeInBits();
5250 EVT MulVT;
5251
5252 // Check to see if we can do this.
5253 // FIXME: We should be more aggressive here.
5254 if (!isTypeLegal(VT)) {
5255 // Limit this to simple scalars for now.
5256 if (VT.isVector() || !VT.isSimple())
5257 return SDValue();
5258
5259 // If this type will be promoted to a large enough type with a legal
5260 // multiply operation, we can go ahead and do this transform.
5261 if (getTypeAction(VT.getSimpleVT()) != TypePromoteInteger)
5262 return SDValue();
5263
5264 MulVT = getTypeToTransformTo(*DAG.getContext(), VT);
5265 if (MulVT.getSizeInBits() < (2 * EltBits) ||
5266 !isOperationLegal(ISD::MUL, MulVT))
5267 return SDValue();
5268 }
5269
5270 bool UseNPQ = false;
5271 SmallVector<SDValue, 16> PreShifts, PostShifts, MagicFactors, NPQFactors;
5272
5273 auto BuildUDIVPattern = [&](ConstantSDNode *C) {
5274 if (C->isNullValue())
5275 return false;
5276 // FIXME: We should use a narrower constant when the upper
5277 // bits are known to be zero.
5278 const APInt& Divisor = C->getAPIntValue();
5279 APInt::mu magics = Divisor.magicu();
5280 unsigned PreShift = 0, PostShift = 0;
5281
5282 // If the divisor is even, we can avoid using the expensive fixup by
5283 // shifting the divided value upfront.
5284 if (magics.a != 0 && !Divisor[0]) {
5285 PreShift = Divisor.countTrailingZeros();
5286 // Get magic number for the shifted divisor.
5287 magics = Divisor.lshr(PreShift).magicu(PreShift);
5288 assert(magics.a == 0 && "Should use cheap fixup now");
5289 }
5290
5291 APInt Magic = magics.m;
5292
5293 unsigned SelNPQ;
5294 if (magics.a == 0 || Divisor.isOneValue()) {
5295 assert(magics.s < Divisor.getBitWidth() &&
5296 "We shouldn't generate an undefined shift!");
5297 PostShift = magics.s;
5298 SelNPQ = false;
5299 } else {
5300 PostShift = magics.s - 1;
5301 SelNPQ = true;
5302 }
5303
5304 PreShifts.push_back(DAG.getConstant(PreShift, dl, ShSVT));
5305 MagicFactors.push_back(DAG.getConstant(Magic, dl, SVT));
5306 NPQFactors.push_back(
5307 DAG.getConstant(SelNPQ ? APInt::getOneBitSet(EltBits, EltBits - 1)
5308 : APInt::getNullValue(EltBits),
5309 dl, SVT));
5310 PostShifts.push_back(DAG.getConstant(PostShift, dl, ShSVT));
5311 UseNPQ |= SelNPQ;
5312 return true;
5313 };
5314
5315 SDValue N0 = N->getOperand(0);
5316 SDValue N1 = N->getOperand(1);
5317
5318 // Collect the shifts/magic values from each element.
5319 if (!ISD::matchUnaryPredicate(N1, BuildUDIVPattern))
5320 return SDValue();
5321
5322 SDValue PreShift, PostShift, MagicFactor, NPQFactor;
5323 if (N1.getOpcode() == ISD::BUILD_VECTOR) {
5324 PreShift = DAG.getBuildVector(ShVT, dl, PreShifts);
5325 MagicFactor = DAG.getBuildVector(VT, dl, MagicFactors);
5326 NPQFactor = DAG.getBuildVector(VT, dl, NPQFactors);
5327 PostShift = DAG.getBuildVector(ShVT, dl, PostShifts);
5328 } else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
5329 assert(PreShifts.size() == 1 && MagicFactors.size() == 1 &&
5330 NPQFactors.size() == 1 && PostShifts.size() == 1 &&
5331 "Expected matchUnaryPredicate to return one for scalable vectors");
5332 PreShift = DAG.getSplatVector(ShVT, dl, PreShifts[0]);
5333 MagicFactor = DAG.getSplatVector(VT, dl, MagicFactors[0]);
5334 NPQFactor = DAG.getSplatVector(VT, dl, NPQFactors[0]);
5335 PostShift = DAG.getSplatVector(ShVT, dl, PostShifts[0]);
5336 } else {
5337 assert(isa<ConstantSDNode>(N1) && "Expected a constant");
5338 PreShift = PreShifts[0];
5339 MagicFactor = MagicFactors[0];
5340 PostShift = PostShifts[0];
5341 }
5342
5343 SDValue Q = N0;
5344 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PreShift);
5345 Created.push_back(Q.getNode());
5346
5347 // FIXME: We should support doing a MUL in a wider type.
5348 auto GetMULHU = [&](SDValue X, SDValue Y) {
5349 // If the type isn't legal, use a wider mul of the the type calculated
5350 // earlier.
5351 if (!isTypeLegal(VT)) {
5352 X = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, X);
5353 Y = DAG.getNode(ISD::ZERO_EXTEND, dl, MulVT, Y);
5354 Y = DAG.getNode(ISD::MUL, dl, MulVT, X, Y);
5355 Y = DAG.getNode(ISD::SRL, dl, MulVT, Y,
5356 DAG.getShiftAmountConstant(EltBits, MulVT, dl));
5357 return DAG.getNode(ISD::TRUNCATE, dl, VT, Y);
5358 }
5359
5360 if (isOperationLegalOrCustom(ISD::MULHU, VT, IsAfterLegalization))
5361 return DAG.getNode(ISD::MULHU, dl, VT, X, Y);
5362 if (isOperationLegalOrCustom(ISD::UMUL_LOHI, VT, IsAfterLegalization)) {
5363 SDValue LoHi =
5364 DAG.getNode(ISD::UMUL_LOHI, dl, DAG.getVTList(VT, VT), X, Y);
5365 return SDValue(LoHi.getNode(), 1);
5366 }
5367 return SDValue(); // No mulhu or equivalent
5368 };
5369
5370 // Multiply the numerator (operand 0) by the magic value.
5371 Q = GetMULHU(Q, MagicFactor);
5372 if (!Q)
5373 return SDValue();
5374
5375 Created.push_back(Q.getNode());
5376
5377 if (UseNPQ) {
5378 SDValue NPQ = DAG.getNode(ISD::SUB, dl, VT, N0, Q);
5379 Created.push_back(NPQ.getNode());
5380
5381 // For vectors we might have a mix of non-NPQ/NPQ paths, so use
5382 // MULHU to act as a SRL-by-1 for NPQ, else multiply by zero.
5383 if (VT.isVector())
5384 NPQ = GetMULHU(NPQ, NPQFactor);
5385 else
5386 NPQ = DAG.getNode(ISD::SRL, dl, VT, NPQ, DAG.getConstant(1, dl, ShVT));
5387
5388 Created.push_back(NPQ.getNode());
5389
5390 Q = DAG.getNode(ISD::ADD, dl, VT, NPQ, Q);
5391 Created.push_back(Q.getNode());
5392 }
5393
5394 Q = DAG.getNode(ISD::SRL, dl, VT, Q, PostShift);
5395 Created.push_back(Q.getNode());
5396
5397 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5398
5399 SDValue One = DAG.getConstant(1, dl, VT);
5400 SDValue IsOne = DAG.getSetCC(dl, SetCCVT, N1, One, ISD::SETEQ);
5401 return DAG.getSelect(dl, VT, IsOne, N0, Q);
5402 }
5403
5404 /// If all values in Values that *don't* match the predicate are same 'splat'
5405 /// value, then replace all values with that splat value.
5406 /// Else, if AlternativeReplacement was provided, then replace all values that
5407 /// do match predicate with AlternativeReplacement value.
5408 static void
turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,std::function<bool (SDValue)> Predicate,SDValue AlternativeReplacement=SDValue ())5409 turnVectorIntoSplatVector(MutableArrayRef<SDValue> Values,
5410 std::function<bool(SDValue)> Predicate,
5411 SDValue AlternativeReplacement = SDValue()) {
5412 SDValue Replacement;
5413 // Is there a value for which the Predicate does *NOT* match? What is it?
5414 auto SplatValue = llvm::find_if_not(Values, Predicate);
5415 if (SplatValue != Values.end()) {
5416 // Does Values consist only of SplatValue's and values matching Predicate?
5417 if (llvm::all_of(Values, [Predicate, SplatValue](SDValue Value) {
5418 return Value == *SplatValue || Predicate(Value);
5419 })) // Then we shall replace values matching predicate with SplatValue.
5420 Replacement = *SplatValue;
5421 }
5422 if (!Replacement) {
5423 // Oops, we did not find the "baseline" splat value.
5424 if (!AlternativeReplacement)
5425 return; // Nothing to do.
5426 // Let's replace with provided value then.
5427 Replacement = AlternativeReplacement;
5428 }
5429 std::replace_if(Values.begin(), Values.end(), Predicate, Replacement);
5430 }
5431
5432 /// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE
5433 /// where the divisor is constant and the comparison target is zero,
5434 /// return a DAG expression that will generate the same comparison result
5435 /// using only multiplications, additions and shifts/rotations.
5436 /// Ref: "Hacker's Delight" 10-17.
buildUREMEqFold(EVT SETCCVT,SDValue REMNode,SDValue CompTargetNode,ISD::CondCode Cond,DAGCombinerInfo & DCI,const SDLoc & DL) const5437 SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode,
5438 SDValue CompTargetNode,
5439 ISD::CondCode Cond,
5440 DAGCombinerInfo &DCI,
5441 const SDLoc &DL) const {
5442 SmallVector<SDNode *, 5> Built;
5443 if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5444 DCI, DL, Built)) {
5445 for (SDNode *N : Built)
5446 DCI.AddToWorklist(N);
5447 return Folded;
5448 }
5449
5450 return SDValue();
5451 }
5452
5453 SDValue
prepareUREMEqFold(EVT SETCCVT,SDValue REMNode,SDValue CompTargetNode,ISD::CondCode Cond,DAGCombinerInfo & DCI,const SDLoc & DL,SmallVectorImpl<SDNode * > & Created) const5454 TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode,
5455 SDValue CompTargetNode, ISD::CondCode Cond,
5456 DAGCombinerInfo &DCI, const SDLoc &DL,
5457 SmallVectorImpl<SDNode *> &Created) const {
5458 // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q)
5459 // - D must be constant, with D = D0 * 2^K where D0 is odd
5460 // - P is the multiplicative inverse of D0 modulo 2^W
5461 // - Q = floor(((2^W) - 1) / D)
5462 // where W is the width of the common type of N and D.
5463 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5464 "Only applicable for (in)equality comparisons.");
5465
5466 SelectionDAG &DAG = DCI.DAG;
5467
5468 EVT VT = REMNode.getValueType();
5469 EVT SVT = VT.getScalarType();
5470 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5471 EVT ShSVT = ShVT.getScalarType();
5472
5473 // If MUL is unavailable, we cannot proceed in any case.
5474 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5475 return SDValue();
5476
5477 bool ComparingWithAllZeros = true;
5478 bool AllComparisonsWithNonZerosAreTautological = true;
5479 bool HadTautologicalLanes = false;
5480 bool AllLanesAreTautological = true;
5481 bool HadEvenDivisor = false;
5482 bool AllDivisorsArePowerOfTwo = true;
5483 bool HadTautologicalInvertedLanes = false;
5484 SmallVector<SDValue, 16> PAmts, KAmts, QAmts, IAmts;
5485
5486 auto BuildUREMPattern = [&](ConstantSDNode *CDiv, ConstantSDNode *CCmp) {
5487 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5488 if (CDiv->isNullValue())
5489 return false;
5490
5491 const APInt &D = CDiv->getAPIntValue();
5492 const APInt &Cmp = CCmp->getAPIntValue();
5493
5494 ComparingWithAllZeros &= Cmp.isNullValue();
5495
5496 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5497 // if C2 is not less than C1, the comparison is always false.
5498 // But we will only be able to produce the comparison that will give the
5499 // opposive tautological answer. So this lane would need to be fixed up.
5500 bool TautologicalInvertedLane = D.ule(Cmp);
5501 HadTautologicalInvertedLanes |= TautologicalInvertedLane;
5502
5503 // If all lanes are tautological (either all divisors are ones, or divisor
5504 // is not greater than the constant we are comparing with),
5505 // we will prefer to avoid the fold.
5506 bool TautologicalLane = D.isOneValue() || TautologicalInvertedLane;
5507 HadTautologicalLanes |= TautologicalLane;
5508 AllLanesAreTautological &= TautologicalLane;
5509
5510 // If we are comparing with non-zero, we need'll need to subtract said
5511 // comparison value from the LHS. But there is no point in doing that if
5512 // every lane where we are comparing with non-zero is tautological..
5513 if (!Cmp.isNullValue())
5514 AllComparisonsWithNonZerosAreTautological &= TautologicalLane;
5515
5516 // Decompose D into D0 * 2^K
5517 unsigned K = D.countTrailingZeros();
5518 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.");
5519 APInt D0 = D.lshr(K);
5520
5521 // D is even if it has trailing zeros.
5522 HadEvenDivisor |= (K != 0);
5523 // D is a power-of-two if D0 is one.
5524 // If all divisors are power-of-two, we will prefer to avoid the fold.
5525 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5526
5527 // P = inv(D0, 2^W)
5528 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5529 unsigned W = D.getBitWidth();
5530 APInt P = D0.zext(W + 1)
5531 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5532 .trunc(W);
5533 assert(!P.isNullValue() && "No multiplicative inverse!"); // unreachable
5534 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.");
5535
5536 // Q = floor((2^W - 1) u/ D)
5537 // R = ((2^W - 1) u% D)
5538 APInt Q, R;
5539 APInt::udivrem(APInt::getAllOnesValue(W), D, Q, R);
5540
5541 // If we are comparing with zero, then that comparison constant is okay,
5542 // else it may need to be one less than that.
5543 if (Cmp.ugt(R))
5544 Q -= 1;
5545
5546 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
5547 "We are expecting that K is always less than all-ones for ShSVT");
5548
5549 // If the lane is tautological the result can be constant-folded.
5550 if (TautologicalLane) {
5551 // Set P and K amount to a bogus values so we can try to splat them.
5552 P = 0;
5553 K = -1;
5554 // And ensure that comparison constant is tautological,
5555 // it will always compare true/false.
5556 Q = -1;
5557 }
5558
5559 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5560 KAmts.push_back(
5561 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5562 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5563 return true;
5564 };
5565
5566 SDValue N = REMNode.getOperand(0);
5567 SDValue D = REMNode.getOperand(1);
5568
5569 // Collect the values from each element.
5570 if (!ISD::matchBinaryPredicate(D, CompTargetNode, BuildUREMPattern))
5571 return SDValue();
5572
5573 // If all lanes are tautological, the result can be constant-folded.
5574 if (AllLanesAreTautological)
5575 return SDValue();
5576
5577 // If this is a urem by a powers-of-two, avoid the fold since it can be
5578 // best implemented as a bit test.
5579 if (AllDivisorsArePowerOfTwo)
5580 return SDValue();
5581
5582 SDValue PVal, KVal, QVal;
5583 if (VT.isVector()) {
5584 if (HadTautologicalLanes) {
5585 // Try to turn PAmts into a splat, since we don't care about the values
5586 // that are currently '0'. If we can't, just keep '0'`s.
5587 turnVectorIntoSplatVector(PAmts, isNullConstant);
5588 // Try to turn KAmts into a splat, since we don't care about the values
5589 // that are currently '-1'. If we can't, change them to '0'`s.
5590 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5591 DAG.getConstant(0, DL, ShSVT));
5592 }
5593
5594 PVal = DAG.getBuildVector(VT, DL, PAmts);
5595 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5596 QVal = DAG.getBuildVector(VT, DL, QAmts);
5597 } else {
5598 PVal = PAmts[0];
5599 KVal = KAmts[0];
5600 QVal = QAmts[0];
5601 }
5602
5603 if (!ComparingWithAllZeros && !AllComparisonsWithNonZerosAreTautological) {
5604 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::SUB, VT))
5605 return SDValue(); // FIXME: Could/should use `ISD::ADD`?
5606 assert(CompTargetNode.getValueType() == N.getValueType() &&
5607 "Expecting that the types on LHS and RHS of comparisons match.");
5608 N = DAG.getNode(ISD::SUB, DL, VT, N, CompTargetNode);
5609 }
5610
5611 // (mul N, P)
5612 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5613 Created.push_back(Op0.getNode());
5614
5615 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5616 // divisors as a performance improvement, since rotating by 0 is a no-op.
5617 if (HadEvenDivisor) {
5618 // We need ROTR to do this.
5619 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
5620 return SDValue();
5621 SDNodeFlags Flags;
5622 Flags.setExact(true);
5623 // UREM: (rotr (mul N, P), K)
5624 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5625 Created.push_back(Op0.getNode());
5626 }
5627
5628 // UREM: (setule/setugt (rotr (mul N, P), K), Q)
5629 SDValue NewCC =
5630 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5631 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5632 if (!HadTautologicalInvertedLanes)
5633 return NewCC;
5634
5635 // If any lanes previously compared always-false, the NewCC will give
5636 // always-true result for them, so we need to fixup those lanes.
5637 // Or the other way around for inequality predicate.
5638 assert(VT.isVector() && "Can/should only get here for vectors.");
5639 Created.push_back(NewCC.getNode());
5640
5641 // x u% C1` is *always* less than C1. So given `x u% C1 == C2`,
5642 // if C2 is not less than C1, the comparison is always false.
5643 // But we have produced the comparison that will give the
5644 // opposive tautological answer. So these lanes would need to be fixed up.
5645 SDValue TautologicalInvertedChannels =
5646 DAG.getSetCC(DL, SETCCVT, D, CompTargetNode, ISD::SETULE);
5647 Created.push_back(TautologicalInvertedChannels.getNode());
5648
5649 // NOTE: we avoid letting illegal types through even if we're before legalize
5650 // ops – legalization has a hard time producing good code for this.
5651 if (isOperationLegalOrCustom(ISD::VSELECT, SETCCVT)) {
5652 // If we have a vector select, let's replace the comparison results in the
5653 // affected lanes with the correct tautological result.
5654 SDValue Replacement = DAG.getBoolConstant(Cond == ISD::SETEQ ? false : true,
5655 DL, SETCCVT, SETCCVT);
5656 return DAG.getNode(ISD::VSELECT, DL, SETCCVT, TautologicalInvertedChannels,
5657 Replacement, NewCC);
5658 }
5659
5660 // Else, we can just invert the comparison result in the appropriate lanes.
5661 //
5662 // NOTE: see the note above VSELECT above.
5663 if (isOperationLegalOrCustom(ISD::XOR, SETCCVT))
5664 return DAG.getNode(ISD::XOR, DL, SETCCVT, NewCC,
5665 TautologicalInvertedChannels);
5666
5667 return SDValue(); // Don't know how to lower.
5668 }
5669
5670 /// Given an ISD::SREM used only by an ISD::SETEQ or ISD::SETNE
5671 /// where the divisor is constant and the comparison target is zero,
5672 /// return a DAG expression that will generate the same comparison result
5673 /// using only multiplications, additions and shifts/rotations.
5674 /// Ref: "Hacker's Delight" 10-17.
buildSREMEqFold(EVT SETCCVT,SDValue REMNode,SDValue CompTargetNode,ISD::CondCode Cond,DAGCombinerInfo & DCI,const SDLoc & DL) const5675 SDValue TargetLowering::buildSREMEqFold(EVT SETCCVT, SDValue REMNode,
5676 SDValue CompTargetNode,
5677 ISD::CondCode Cond,
5678 DAGCombinerInfo &DCI,
5679 const SDLoc &DL) const {
5680 SmallVector<SDNode *, 7> Built;
5681 if (SDValue Folded = prepareSREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond,
5682 DCI, DL, Built)) {
5683 assert(Built.size() <= 7 && "Max size prediction failed.");
5684 for (SDNode *N : Built)
5685 DCI.AddToWorklist(N);
5686 return Folded;
5687 }
5688
5689 return SDValue();
5690 }
5691
5692 SDValue
prepareSREMEqFold(EVT SETCCVT,SDValue REMNode,SDValue CompTargetNode,ISD::CondCode Cond,DAGCombinerInfo & DCI,const SDLoc & DL,SmallVectorImpl<SDNode * > & Created) const5693 TargetLowering::prepareSREMEqFold(EVT SETCCVT, SDValue REMNode,
5694 SDValue CompTargetNode, ISD::CondCode Cond,
5695 DAGCombinerInfo &DCI, const SDLoc &DL,
5696 SmallVectorImpl<SDNode *> &Created) const {
5697 // Fold:
5698 // (seteq/ne (srem N, D), 0)
5699 // To:
5700 // (setule/ugt (rotr (add (mul N, P), A), K), Q)
5701 //
5702 // - D must be constant, with D = D0 * 2^K where D0 is odd
5703 // - P is the multiplicative inverse of D0 modulo 2^W
5704 // - A = bitwiseand(floor((2^(W - 1) - 1) / D0), (-(2^k)))
5705 // - Q = floor((2 * A) / (2^K))
5706 // where W is the width of the common type of N and D.
5707 assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
5708 "Only applicable for (in)equality comparisons.");
5709
5710 SelectionDAG &DAG = DCI.DAG;
5711
5712 EVT VT = REMNode.getValueType();
5713 EVT SVT = VT.getScalarType();
5714 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout(), !DCI.isBeforeLegalize());
5715 EVT ShSVT = ShVT.getScalarType();
5716
5717 // If we are after ops legalization, and MUL is unavailable, we can not
5718 // proceed.
5719 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::MUL, VT))
5720 return SDValue();
5721
5722 // TODO: Could support comparing with non-zero too.
5723 ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode);
5724 if (!CompTarget || !CompTarget->isNullValue())
5725 return SDValue();
5726
5727 bool HadIntMinDivisor = false;
5728 bool HadOneDivisor = false;
5729 bool AllDivisorsAreOnes = true;
5730 bool HadEvenDivisor = false;
5731 bool NeedToApplyOffset = false;
5732 bool AllDivisorsArePowerOfTwo = true;
5733 SmallVector<SDValue, 16> PAmts, AAmts, KAmts, QAmts;
5734
5735 auto BuildSREMPattern = [&](ConstantSDNode *C) {
5736 // Division by 0 is UB. Leave it to be constant-folded elsewhere.
5737 if (C->isNullValue())
5738 return false;
5739
5740 // FIXME: we don't fold `rem %X, -C` to `rem %X, C` in DAGCombine.
5741
5742 // WARNING: this fold is only valid for positive divisors!
5743 APInt D = C->getAPIntValue();
5744 if (D.isNegative())
5745 D.negate(); // `rem %X, -C` is equivalent to `rem %X, C`
5746
5747 HadIntMinDivisor |= D.isMinSignedValue();
5748
5749 // If all divisors are ones, we will prefer to avoid the fold.
5750 HadOneDivisor |= D.isOneValue();
5751 AllDivisorsAreOnes &= D.isOneValue();
5752
5753 // Decompose D into D0 * 2^K
5754 unsigned K = D.countTrailingZeros();
5755 assert((!D.isOneValue() || (K == 0)) && "For divisor '1' we won't rotate.");
5756 APInt D0 = D.lshr(K);
5757
5758 if (!D.isMinSignedValue()) {
5759 // D is even if it has trailing zeros; unless it's INT_MIN, in which case
5760 // we don't care about this lane in this fold, we'll special-handle it.
5761 HadEvenDivisor |= (K != 0);
5762 }
5763
5764 // D is a power-of-two if D0 is one. This includes INT_MIN.
5765 // If all divisors are power-of-two, we will prefer to avoid the fold.
5766 AllDivisorsArePowerOfTwo &= D0.isOneValue();
5767
5768 // P = inv(D0, 2^W)
5769 // 2^W requires W + 1 bits, so we have to extend and then truncate.
5770 unsigned W = D.getBitWidth();
5771 APInt P = D0.zext(W + 1)
5772 .multiplicativeInverse(APInt::getSignedMinValue(W + 1))
5773 .trunc(W);
5774 assert(!P.isNullValue() && "No multiplicative inverse!"); // unreachable
5775 assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check.");
5776
5777 // A = floor((2^(W - 1) - 1) / D0) & -2^K
5778 APInt A = APInt::getSignedMaxValue(W).udiv(D0);
5779 A.clearLowBits(K);
5780
5781 if (!D.isMinSignedValue()) {
5782 // If divisor INT_MIN, then we don't care about this lane in this fold,
5783 // we'll special-handle it.
5784 NeedToApplyOffset |= A != 0;
5785 }
5786
5787 // Q = floor((2 * A) / (2^K))
5788 APInt Q = (2 * A).udiv(APInt::getOneBitSet(W, K));
5789
5790 assert(APInt::getAllOnesValue(SVT.getSizeInBits()).ugt(A) &&
5791 "We are expecting that A is always less than all-ones for SVT");
5792 assert(APInt::getAllOnesValue(ShSVT.getSizeInBits()).ugt(K) &&
5793 "We are expecting that K is always less than all-ones for ShSVT");
5794
5795 // If the divisor is 1 the result can be constant-folded. Likewise, we
5796 // don't care about INT_MIN lanes, those can be set to undef if appropriate.
5797 if (D.isOneValue()) {
5798 // Set P, A and K to a bogus values so we can try to splat them.
5799 P = 0;
5800 A = -1;
5801 K = -1;
5802
5803 // x ?% 1 == 0 <--> true <--> x u<= -1
5804 Q = -1;
5805 }
5806
5807 PAmts.push_back(DAG.getConstant(P, DL, SVT));
5808 AAmts.push_back(DAG.getConstant(A, DL, SVT));
5809 KAmts.push_back(
5810 DAG.getConstant(APInt(ShSVT.getSizeInBits(), K), DL, ShSVT));
5811 QAmts.push_back(DAG.getConstant(Q, DL, SVT));
5812 return true;
5813 };
5814
5815 SDValue N = REMNode.getOperand(0);
5816 SDValue D = REMNode.getOperand(1);
5817
5818 // Collect the values from each element.
5819 if (!ISD::matchUnaryPredicate(D, BuildSREMPattern))
5820 return SDValue();
5821
5822 // If this is a srem by a one, avoid the fold since it can be constant-folded.
5823 if (AllDivisorsAreOnes)
5824 return SDValue();
5825
5826 // If this is a srem by a powers-of-two (including INT_MIN), avoid the fold
5827 // since it can be best implemented as a bit test.
5828 if (AllDivisorsArePowerOfTwo)
5829 return SDValue();
5830
5831 SDValue PVal, AVal, KVal, QVal;
5832 if (D.getOpcode() == ISD::BUILD_VECTOR) {
5833 if (HadOneDivisor) {
5834 // Try to turn PAmts into a splat, since we don't care about the values
5835 // that are currently '0'. If we can't, just keep '0'`s.
5836 turnVectorIntoSplatVector(PAmts, isNullConstant);
5837 // Try to turn AAmts into a splat, since we don't care about the
5838 // values that are currently '-1'. If we can't, change them to '0'`s.
5839 turnVectorIntoSplatVector(AAmts, isAllOnesConstant,
5840 DAG.getConstant(0, DL, SVT));
5841 // Try to turn KAmts into a splat, since we don't care about the values
5842 // that are currently '-1'. If we can't, change them to '0'`s.
5843 turnVectorIntoSplatVector(KAmts, isAllOnesConstant,
5844 DAG.getConstant(0, DL, ShSVT));
5845 }
5846
5847 PVal = DAG.getBuildVector(VT, DL, PAmts);
5848 AVal = DAG.getBuildVector(VT, DL, AAmts);
5849 KVal = DAG.getBuildVector(ShVT, DL, KAmts);
5850 QVal = DAG.getBuildVector(VT, DL, QAmts);
5851 } else if (D.getOpcode() == ISD::SPLAT_VECTOR) {
5852 assert(PAmts.size() == 1 && AAmts.size() == 1 && KAmts.size() == 1 &&
5853 QAmts.size() == 1 &&
5854 "Expected matchUnaryPredicate to return one element for scalable "
5855 "vectors");
5856 PVal = DAG.getSplatVector(VT, DL, PAmts[0]);
5857 AVal = DAG.getSplatVector(VT, DL, AAmts[0]);
5858 KVal = DAG.getSplatVector(ShVT, DL, KAmts[0]);
5859 QVal = DAG.getSplatVector(VT, DL, QAmts[0]);
5860 } else {
5861 assert(isa<ConstantSDNode>(D) && "Expected a constant");
5862 PVal = PAmts[0];
5863 AVal = AAmts[0];
5864 KVal = KAmts[0];
5865 QVal = QAmts[0];
5866 }
5867
5868 // (mul N, P)
5869 SDValue Op0 = DAG.getNode(ISD::MUL, DL, VT, N, PVal);
5870 Created.push_back(Op0.getNode());
5871
5872 if (NeedToApplyOffset) {
5873 // We need ADD to do this.
5874 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ADD, VT))
5875 return SDValue();
5876
5877 // (add (mul N, P), A)
5878 Op0 = DAG.getNode(ISD::ADD, DL, VT, Op0, AVal);
5879 Created.push_back(Op0.getNode());
5880 }
5881
5882 // Rotate right only if any divisor was even. We avoid rotates for all-odd
5883 // divisors as a performance improvement, since rotating by 0 is a no-op.
5884 if (HadEvenDivisor) {
5885 // We need ROTR to do this.
5886 if (!DCI.isBeforeLegalizeOps() && !isOperationLegalOrCustom(ISD::ROTR, VT))
5887 return SDValue();
5888 SDNodeFlags Flags;
5889 Flags.setExact(true);
5890 // SREM: (rotr (add (mul N, P), A), K)
5891 Op0 = DAG.getNode(ISD::ROTR, DL, VT, Op0, KVal, Flags);
5892 Created.push_back(Op0.getNode());
5893 }
5894
5895 // SREM: (setule/setugt (rotr (add (mul N, P), A), K), Q)
5896 SDValue Fold =
5897 DAG.getSetCC(DL, SETCCVT, Op0, QVal,
5898 ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT));
5899
5900 // If we didn't have lanes with INT_MIN divisor, then we're done.
5901 if (!HadIntMinDivisor)
5902 return Fold;
5903
5904 // That fold is only valid for positive divisors. Which effectively means,
5905 // it is invalid for INT_MIN divisors. So if we have such a lane,
5906 // we must fix-up results for said lanes.
5907 assert(VT.isVector() && "Can/should only get here for vectors.");
5908
5909 // NOTE: we avoid letting illegal types through even if we're before legalize
5910 // ops – legalization has a hard time producing good code for the code that
5911 // follows.
5912 if (!isOperationLegalOrCustom(ISD::SETEQ, VT) ||
5913 !isOperationLegalOrCustom(ISD::AND, VT) ||
5914 !isOperationLegalOrCustom(Cond, VT) ||
5915 !isOperationLegalOrCustom(ISD::VSELECT, VT))
5916 return SDValue();
5917
5918 Created.push_back(Fold.getNode());
5919
5920 SDValue IntMin = DAG.getConstant(
5921 APInt::getSignedMinValue(SVT.getScalarSizeInBits()), DL, VT);
5922 SDValue IntMax = DAG.getConstant(
5923 APInt::getSignedMaxValue(SVT.getScalarSizeInBits()), DL, VT);
5924 SDValue Zero =
5925 DAG.getConstant(APInt::getNullValue(SVT.getScalarSizeInBits()), DL, VT);
5926
5927 // Which lanes had INT_MIN divisors? Divisor is constant, so const-folded.
5928 SDValue DivisorIsIntMin = DAG.getSetCC(DL, SETCCVT, D, IntMin, ISD::SETEQ);
5929 Created.push_back(DivisorIsIntMin.getNode());
5930
5931 // (N s% INT_MIN) ==/!= 0 <--> (N & INT_MAX) ==/!= 0
5932 SDValue Masked = DAG.getNode(ISD::AND, DL, VT, N, IntMax);
5933 Created.push_back(Masked.getNode());
5934 SDValue MaskedIsZero = DAG.getSetCC(DL, SETCCVT, Masked, Zero, Cond);
5935 Created.push_back(MaskedIsZero.getNode());
5936
5937 // To produce final result we need to blend 2 vectors: 'SetCC' and
5938 // 'MaskedIsZero'. If the divisor for channel was *NOT* INT_MIN, we pick
5939 // from 'Fold', else pick from 'MaskedIsZero'. Since 'DivisorIsIntMin' is
5940 // constant-folded, select can get lowered to a shuffle with constant mask.
5941 SDValue Blended =
5942 DAG.getNode(ISD::VSELECT, DL, VT, DivisorIsIntMin, MaskedIsZero, Fold);
5943
5944 return Blended;
5945 }
5946
5947 bool TargetLowering::
verifyReturnAddressArgumentIsConstant(SDValue Op,SelectionDAG & DAG) const5948 verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
5949 if (!isa<ConstantSDNode>(Op.getOperand(0))) {
5950 DAG.getContext()->emitError("argument to '__builtin_return_address' must "
5951 "be a constant integer");
5952 return true;
5953 }
5954
5955 return false;
5956 }
5957
getSqrtInputTest(SDValue Op,SelectionDAG & DAG,const DenormalMode & Mode) const5958 SDValue TargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
5959 const DenormalMode &Mode) const {
5960 SDLoc DL(Op);
5961 EVT VT = Op.getValueType();
5962 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
5963 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
5964 // Testing it with denormal inputs to avoid wrong estimate.
5965 if (Mode.Input == DenormalMode::IEEE) {
5966 // This is specifically a check for the handling of denormal inputs,
5967 // not the result.
5968
5969 // Test = fabs(X) < SmallestNormal
5970 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
5971 APFloat SmallestNorm = APFloat::getSmallestNormalized(FltSem);
5972 SDValue NormC = DAG.getConstantFP(SmallestNorm, DL, VT);
5973 SDValue Fabs = DAG.getNode(ISD::FABS, DL, VT, Op);
5974 return DAG.getSetCC(DL, CCVT, Fabs, NormC, ISD::SETLT);
5975 }
5976 // Test = X == 0.0
5977 return DAG.getSetCC(DL, CCVT, Op, FPZero, ISD::SETEQ);
5978 }
5979
getNegatedExpression(SDValue Op,SelectionDAG & DAG,bool LegalOps,bool OptForSize,NegatibleCost & Cost,unsigned Depth) const5980 SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
5981 bool LegalOps, bool OptForSize,
5982 NegatibleCost &Cost,
5983 unsigned Depth) const {
5984 // fneg is removable even if it has multiple uses.
5985 if (Op.getOpcode() == ISD::FNEG) {
5986 Cost = NegatibleCost::Cheaper;
5987 return Op.getOperand(0);
5988 }
5989
5990 // Don't recurse exponentially.
5991 if (Depth > SelectionDAG::MaxRecursionDepth)
5992 return SDValue();
5993
5994 // Pre-increment recursion depth for use in recursive calls.
5995 ++Depth;
5996 const SDNodeFlags Flags = Op->getFlags();
5997 const TargetOptions &Options = DAG.getTarget().Options;
5998 EVT VT = Op.getValueType();
5999 unsigned Opcode = Op.getOpcode();
6000
6001 // Don't allow anything with multiple uses unless we know it is free.
6002 if (!Op.hasOneUse() && Opcode != ISD::ConstantFP) {
6003 bool IsFreeExtend = Opcode == ISD::FP_EXTEND &&
6004 isFPExtFree(VT, Op.getOperand(0).getValueType());
6005 if (!IsFreeExtend)
6006 return SDValue();
6007 }
6008
6009 auto RemoveDeadNode = [&](SDValue N) {
6010 if (N && N.getNode()->use_empty())
6011 DAG.RemoveDeadNode(N.getNode());
6012 };
6013
6014 SDLoc DL(Op);
6015
6016 // Because getNegatedExpression can delete nodes we need a handle to keep
6017 // temporary nodes alive in case the recursion manages to create an identical
6018 // node.
6019 std::list<HandleSDNode> Handles;
6020
6021 switch (Opcode) {
6022 case ISD::ConstantFP: {
6023 // Don't invert constant FP values after legalization unless the target says
6024 // the negated constant is legal.
6025 bool IsOpLegal =
6026 isOperationLegal(ISD::ConstantFP, VT) ||
6027 isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
6028 OptForSize);
6029
6030 if (LegalOps && !IsOpLegal)
6031 break;
6032
6033 APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
6034 V.changeSign();
6035 SDValue CFP = DAG.getConstantFP(V, DL, VT);
6036
6037 // If we already have the use of the negated floating constant, it is free
6038 // to negate it even it has multiple uses.
6039 if (!Op.hasOneUse() && CFP.use_empty())
6040 break;
6041 Cost = NegatibleCost::Neutral;
6042 return CFP;
6043 }
6044 case ISD::BUILD_VECTOR: {
6045 // Only permit BUILD_VECTOR of constants.
6046 if (llvm::any_of(Op->op_values(), [&](SDValue N) {
6047 return !N.isUndef() && !isa<ConstantFPSDNode>(N);
6048 }))
6049 break;
6050
6051 bool IsOpLegal =
6052 (isOperationLegal(ISD::ConstantFP, VT) &&
6053 isOperationLegal(ISD::BUILD_VECTOR, VT)) ||
6054 llvm::all_of(Op->op_values(), [&](SDValue N) {
6055 return N.isUndef() ||
6056 isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
6057 OptForSize);
6058 });
6059
6060 if (LegalOps && !IsOpLegal)
6061 break;
6062
6063 SmallVector<SDValue, 4> Ops;
6064 for (SDValue C : Op->op_values()) {
6065 if (C.isUndef()) {
6066 Ops.push_back(C);
6067 continue;
6068 }
6069 APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
6070 V.changeSign();
6071 Ops.push_back(DAG.getConstantFP(V, DL, C.getValueType()));
6072 }
6073 Cost = NegatibleCost::Neutral;
6074 return DAG.getBuildVector(VT, DL, Ops);
6075 }
6076 case ISD::FADD: {
6077 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6078 break;
6079
6080 // After operation legalization, it might not be legal to create new FSUBs.
6081 if (LegalOps && !isOperationLegalOrCustom(ISD::FSUB, VT))
6082 break;
6083 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6084
6085 // fold (fneg (fadd X, Y)) -> (fsub (fneg X), Y)
6086 NegatibleCost CostX = NegatibleCost::Expensive;
6087 SDValue NegX =
6088 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6089 // Prevent this node from being deleted by the next call.
6090 if (NegX)
6091 Handles.emplace_back(NegX);
6092
6093 // fold (fneg (fadd X, Y)) -> (fsub (fneg Y), X)
6094 NegatibleCost CostY = NegatibleCost::Expensive;
6095 SDValue NegY =
6096 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6097
6098 // We're done with the handles.
6099 Handles.clear();
6100
6101 // Negate the X if its cost is less or equal than Y.
6102 if (NegX && (CostX <= CostY)) {
6103 Cost = CostX;
6104 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegX, Y, Flags);
6105 if (NegY != N)
6106 RemoveDeadNode(NegY);
6107 return N;
6108 }
6109
6110 // Negate the Y if it is not expensive.
6111 if (NegY) {
6112 Cost = CostY;
6113 SDValue N = DAG.getNode(ISD::FSUB, DL, VT, NegY, X, Flags);
6114 if (NegX != N)
6115 RemoveDeadNode(NegX);
6116 return N;
6117 }
6118 break;
6119 }
6120 case ISD::FSUB: {
6121 // We can't turn -(A-B) into B-A when we honor signed zeros.
6122 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6123 break;
6124
6125 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6126 // fold (fneg (fsub 0, Y)) -> Y
6127 if (ConstantFPSDNode *C = isConstOrConstSplatFP(X, /*AllowUndefs*/ true))
6128 if (C->isZero()) {
6129 Cost = NegatibleCost::Cheaper;
6130 return Y;
6131 }
6132
6133 // fold (fneg (fsub X, Y)) -> (fsub Y, X)
6134 Cost = NegatibleCost::Neutral;
6135 return DAG.getNode(ISD::FSUB, DL, VT, Y, X, Flags);
6136 }
6137 case ISD::FMUL:
6138 case ISD::FDIV: {
6139 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
6140
6141 // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
6142 NegatibleCost CostX = NegatibleCost::Expensive;
6143 SDValue NegX =
6144 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6145 // Prevent this node from being deleted by the next call.
6146 if (NegX)
6147 Handles.emplace_back(NegX);
6148
6149 // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
6150 NegatibleCost CostY = NegatibleCost::Expensive;
6151 SDValue NegY =
6152 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6153
6154 // We're done with the handles.
6155 Handles.clear();
6156
6157 // Negate the X if its cost is less or equal than Y.
6158 if (NegX && (CostX <= CostY)) {
6159 Cost = CostX;
6160 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, Flags);
6161 if (NegY != N)
6162 RemoveDeadNode(NegY);
6163 return N;
6164 }
6165
6166 // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
6167 if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
6168 if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
6169 break;
6170
6171 // Negate the Y if it is not expensive.
6172 if (NegY) {
6173 Cost = CostY;
6174 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, Flags);
6175 if (NegX != N)
6176 RemoveDeadNode(NegX);
6177 return N;
6178 }
6179 break;
6180 }
6181 case ISD::FMA:
6182 case ISD::FMAD: {
6183 if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
6184 break;
6185
6186 SDValue X = Op.getOperand(0), Y = Op.getOperand(1), Z = Op.getOperand(2);
6187 NegatibleCost CostZ = NegatibleCost::Expensive;
6188 SDValue NegZ =
6189 getNegatedExpression(Z, DAG, LegalOps, OptForSize, CostZ, Depth);
6190 // Give up if fail to negate the Z.
6191 if (!NegZ)
6192 break;
6193
6194 // Prevent this node from being deleted by the next two calls.
6195 Handles.emplace_back(NegZ);
6196
6197 // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
6198 NegatibleCost CostX = NegatibleCost::Expensive;
6199 SDValue NegX =
6200 getNegatedExpression(X, DAG, LegalOps, OptForSize, CostX, Depth);
6201 // Prevent this node from being deleted by the next call.
6202 if (NegX)
6203 Handles.emplace_back(NegX);
6204
6205 // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
6206 NegatibleCost CostY = NegatibleCost::Expensive;
6207 SDValue NegY =
6208 getNegatedExpression(Y, DAG, LegalOps, OptForSize, CostY, Depth);
6209
6210 // We're done with the handles.
6211 Handles.clear();
6212
6213 // Negate the X if its cost is less or equal than Y.
6214 if (NegX && (CostX <= CostY)) {
6215 Cost = std::min(CostX, CostZ);
6216 SDValue N = DAG.getNode(Opcode, DL, VT, NegX, Y, NegZ, Flags);
6217 if (NegY != N)
6218 RemoveDeadNode(NegY);
6219 return N;
6220 }
6221
6222 // Negate the Y if it is not expensive.
6223 if (NegY) {
6224 Cost = std::min(CostY, CostZ);
6225 SDValue N = DAG.getNode(Opcode, DL, VT, X, NegY, NegZ, Flags);
6226 if (NegX != N)
6227 RemoveDeadNode(NegX);
6228 return N;
6229 }
6230 break;
6231 }
6232
6233 case ISD::FP_EXTEND:
6234 case ISD::FSIN:
6235 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
6236 OptForSize, Cost, Depth))
6237 return DAG.getNode(Opcode, DL, VT, NegV);
6238 break;
6239 case ISD::FP_ROUND:
6240 if (SDValue NegV = getNegatedExpression(Op.getOperand(0), DAG, LegalOps,
6241 OptForSize, Cost, Depth))
6242 return DAG.getNode(ISD::FP_ROUND, DL, VT, NegV, Op.getOperand(1));
6243 break;
6244 }
6245
6246 return SDValue();
6247 }
6248
6249 //===----------------------------------------------------------------------===//
6250 // Legalization Utilities
6251 //===----------------------------------------------------------------------===//
6252
expandMUL_LOHI(unsigned Opcode,EVT VT,const SDLoc & dl,SDValue LHS,SDValue RHS,SmallVectorImpl<SDValue> & Result,EVT HiLoVT,SelectionDAG & DAG,MulExpansionKind Kind,SDValue LL,SDValue LH,SDValue RL,SDValue RH) const6253 bool TargetLowering::expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl,
6254 SDValue LHS, SDValue RHS,
6255 SmallVectorImpl<SDValue> &Result,
6256 EVT HiLoVT, SelectionDAG &DAG,
6257 MulExpansionKind Kind, SDValue LL,
6258 SDValue LH, SDValue RL, SDValue RH) const {
6259 assert(Opcode == ISD::MUL || Opcode == ISD::UMUL_LOHI ||
6260 Opcode == ISD::SMUL_LOHI);
6261
6262 bool HasMULHS = (Kind == MulExpansionKind::Always) ||
6263 isOperationLegalOrCustom(ISD::MULHS, HiLoVT);
6264 bool HasMULHU = (Kind == MulExpansionKind::Always) ||
6265 isOperationLegalOrCustom(ISD::MULHU, HiLoVT);
6266 bool HasSMUL_LOHI = (Kind == MulExpansionKind::Always) ||
6267 isOperationLegalOrCustom(ISD::SMUL_LOHI, HiLoVT);
6268 bool HasUMUL_LOHI = (Kind == MulExpansionKind::Always) ||
6269 isOperationLegalOrCustom(ISD::UMUL_LOHI, HiLoVT);
6270
6271 if (!HasMULHU && !HasMULHS && !HasUMUL_LOHI && !HasSMUL_LOHI)
6272 return false;
6273
6274 unsigned OuterBitSize = VT.getScalarSizeInBits();
6275 unsigned InnerBitSize = HiLoVT.getScalarSizeInBits();
6276
6277 // LL, LH, RL, and RH must be either all NULL or all set to a value.
6278 assert((LL.getNode() && LH.getNode() && RL.getNode() && RH.getNode()) ||
6279 (!LL.getNode() && !LH.getNode() && !RL.getNode() && !RH.getNode()));
6280
6281 SDVTList VTs = DAG.getVTList(HiLoVT, HiLoVT);
6282 auto MakeMUL_LOHI = [&](SDValue L, SDValue R, SDValue &Lo, SDValue &Hi,
6283 bool Signed) -> bool {
6284 if ((Signed && HasSMUL_LOHI) || (!Signed && HasUMUL_LOHI)) {
6285 Lo = DAG.getNode(Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI, dl, VTs, L, R);
6286 Hi = SDValue(Lo.getNode(), 1);
6287 return true;
6288 }
6289 if ((Signed && HasMULHS) || (!Signed && HasMULHU)) {
6290 Lo = DAG.getNode(ISD::MUL, dl, HiLoVT, L, R);
6291 Hi = DAG.getNode(Signed ? ISD::MULHS : ISD::MULHU, dl, HiLoVT, L, R);
6292 return true;
6293 }
6294 return false;
6295 };
6296
6297 SDValue Lo, Hi;
6298
6299 if (!LL.getNode() && !RL.getNode() &&
6300 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
6301 LL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LHS);
6302 RL = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RHS);
6303 }
6304
6305 if (!LL.getNode())
6306 return false;
6307
6308 APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
6309 if (DAG.MaskedValueIsZero(LHS, HighMask) &&
6310 DAG.MaskedValueIsZero(RHS, HighMask)) {
6311 // The inputs are both zero-extended.
6312 if (MakeMUL_LOHI(LL, RL, Lo, Hi, false)) {
6313 Result.push_back(Lo);
6314 Result.push_back(Hi);
6315 if (Opcode != ISD::MUL) {
6316 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
6317 Result.push_back(Zero);
6318 Result.push_back(Zero);
6319 }
6320 return true;
6321 }
6322 }
6323
6324 if (!VT.isVector() && Opcode == ISD::MUL &&
6325 DAG.ComputeNumSignBits(LHS) > InnerBitSize &&
6326 DAG.ComputeNumSignBits(RHS) > InnerBitSize) {
6327 // The input values are both sign-extended.
6328 // TODO non-MUL case?
6329 if (MakeMUL_LOHI(LL, RL, Lo, Hi, true)) {
6330 Result.push_back(Lo);
6331 Result.push_back(Hi);
6332 return true;
6333 }
6334 }
6335
6336 unsigned ShiftAmount = OuterBitSize - InnerBitSize;
6337 EVT ShiftAmountTy = getShiftAmountTy(VT, DAG.getDataLayout());
6338 if (APInt::getMaxValue(ShiftAmountTy.getSizeInBits()).ult(ShiftAmount)) {
6339 // FIXME getShiftAmountTy does not always return a sensible result when VT
6340 // is an illegal type, and so the type may be too small to fit the shift
6341 // amount. Override it with i32. The shift will have to be legalized.
6342 ShiftAmountTy = MVT::i32;
6343 }
6344 SDValue Shift = DAG.getConstant(ShiftAmount, dl, ShiftAmountTy);
6345
6346 if (!LH.getNode() && !RH.getNode() &&
6347 isOperationLegalOrCustom(ISD::SRL, VT) &&
6348 isOperationLegalOrCustom(ISD::TRUNCATE, HiLoVT)) {
6349 LH = DAG.getNode(ISD::SRL, dl, VT, LHS, Shift);
6350 LH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, LH);
6351 RH = DAG.getNode(ISD::SRL, dl, VT, RHS, Shift);
6352 RH = DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, RH);
6353 }
6354
6355 if (!LH.getNode())
6356 return false;
6357
6358 if (!MakeMUL_LOHI(LL, RL, Lo, Hi, false))
6359 return false;
6360
6361 Result.push_back(Lo);
6362
6363 if (Opcode == ISD::MUL) {
6364 RH = DAG.getNode(ISD::MUL, dl, HiLoVT, LL, RH);
6365 LH = DAG.getNode(ISD::MUL, dl, HiLoVT, LH, RL);
6366 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, RH);
6367 Hi = DAG.getNode(ISD::ADD, dl, HiLoVT, Hi, LH);
6368 Result.push_back(Hi);
6369 return true;
6370 }
6371
6372 // Compute the full width result.
6373 auto Merge = [&](SDValue Lo, SDValue Hi) -> SDValue {
6374 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
6375 Hi = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6376 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
6377 return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
6378 };
6379
6380 SDValue Next = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Hi);
6381 if (!MakeMUL_LOHI(LL, RH, Lo, Hi, false))
6382 return false;
6383
6384 // This is effectively the add part of a multiply-add of half-sized operands,
6385 // so it cannot overflow.
6386 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
6387
6388 if (!MakeMUL_LOHI(LH, RL, Lo, Hi, false))
6389 return false;
6390
6391 SDValue Zero = DAG.getConstant(0, dl, HiLoVT);
6392 EVT BoolType = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
6393
6394 bool UseGlue = (isOperationLegalOrCustom(ISD::ADDC, VT) &&
6395 isOperationLegalOrCustom(ISD::ADDE, VT));
6396 if (UseGlue)
6397 Next = DAG.getNode(ISD::ADDC, dl, DAG.getVTList(VT, MVT::Glue), Next,
6398 Merge(Lo, Hi));
6399 else
6400 Next = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(VT, BoolType), Next,
6401 Merge(Lo, Hi), DAG.getConstant(0, dl, BoolType));
6402
6403 SDValue Carry = Next.getValue(1);
6404 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6405 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
6406
6407 if (!MakeMUL_LOHI(LH, RH, Lo, Hi, Opcode == ISD::SMUL_LOHI))
6408 return false;
6409
6410 if (UseGlue)
6411 Hi = DAG.getNode(ISD::ADDE, dl, DAG.getVTList(HiLoVT, MVT::Glue), Hi, Zero,
6412 Carry);
6413 else
6414 Hi = DAG.getNode(ISD::ADDCARRY, dl, DAG.getVTList(HiLoVT, BoolType), Hi,
6415 Zero, Carry);
6416
6417 Next = DAG.getNode(ISD::ADD, dl, VT, Next, Merge(Lo, Hi));
6418
6419 if (Opcode == ISD::SMUL_LOHI) {
6420 SDValue NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6421 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RL));
6422 Next = DAG.getSelectCC(dl, LH, Zero, NextSub, Next, ISD::SETLT);
6423
6424 NextSub = DAG.getNode(ISD::SUB, dl, VT, Next,
6425 DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LL));
6426 Next = DAG.getSelectCC(dl, RH, Zero, NextSub, Next, ISD::SETLT);
6427 }
6428
6429 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6430 Next = DAG.getNode(ISD::SRL, dl, VT, Next, Shift);
6431 Result.push_back(DAG.getNode(ISD::TRUNCATE, dl, HiLoVT, Next));
6432 return true;
6433 }
6434
expandMUL(SDNode * N,SDValue & Lo,SDValue & Hi,EVT HiLoVT,SelectionDAG & DAG,MulExpansionKind Kind,SDValue LL,SDValue LH,SDValue RL,SDValue RH) const6435 bool TargetLowering::expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT,
6436 SelectionDAG &DAG, MulExpansionKind Kind,
6437 SDValue LL, SDValue LH, SDValue RL,
6438 SDValue RH) const {
6439 SmallVector<SDValue, 2> Result;
6440 bool Ok = expandMUL_LOHI(N->getOpcode(), N->getValueType(0), SDLoc(N),
6441 N->getOperand(0), N->getOperand(1), Result, HiLoVT,
6442 DAG, Kind, LL, LH, RL, RH);
6443 if (Ok) {
6444 assert(Result.size() == 2);
6445 Lo = Result[0];
6446 Hi = Result[1];
6447 }
6448 return Ok;
6449 }
6450
6451 // Check that (every element of) Z is undef or not an exact multiple of BW.
isNonZeroModBitWidthOrUndef(SDValue Z,unsigned BW)6452 static bool isNonZeroModBitWidthOrUndef(SDValue Z, unsigned BW) {
6453 return ISD::matchUnaryPredicate(
6454 Z,
6455 [=](ConstantSDNode *C) { return !C || C->getAPIntValue().urem(BW) != 0; },
6456 true);
6457 }
6458
expandFunnelShift(SDNode * Node,SDValue & Result,SelectionDAG & DAG) const6459 bool TargetLowering::expandFunnelShift(SDNode *Node, SDValue &Result,
6460 SelectionDAG &DAG) const {
6461 EVT VT = Node->getValueType(0);
6462
6463 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SHL, VT) ||
6464 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6465 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6466 !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
6467 return false;
6468
6469 SDValue X = Node->getOperand(0);
6470 SDValue Y = Node->getOperand(1);
6471 SDValue Z = Node->getOperand(2);
6472
6473 unsigned BW = VT.getScalarSizeInBits();
6474 bool IsFSHL = Node->getOpcode() == ISD::FSHL;
6475 SDLoc DL(SDValue(Node, 0));
6476
6477 EVT ShVT = Z.getValueType();
6478
6479 // If a funnel shift in the other direction is more supported, use it.
6480 unsigned RevOpcode = IsFSHL ? ISD::FSHR : ISD::FSHL;
6481 if (!isOperationLegalOrCustom(Node->getOpcode(), VT) &&
6482 isOperationLegalOrCustom(RevOpcode, VT) && isPowerOf2_32(BW)) {
6483 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
6484 // fshl X, Y, Z -> fshr X, Y, -Z
6485 // fshr X, Y, Z -> fshl X, Y, -Z
6486 SDValue Zero = DAG.getConstant(0, DL, ShVT);
6487 Z = DAG.getNode(ISD::SUB, DL, VT, Zero, Z);
6488 } else {
6489 // fshl X, Y, Z -> fshr (srl X, 1), (fshr X, Y, 1), ~Z
6490 // fshr X, Y, Z -> fshl (fshl X, Y, 1), (shl Y, 1), ~Z
6491 SDValue One = DAG.getConstant(1, DL, ShVT);
6492 if (IsFSHL) {
6493 Y = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
6494 X = DAG.getNode(ISD::SRL, DL, VT, X, One);
6495 } else {
6496 X = DAG.getNode(RevOpcode, DL, VT, X, Y, One);
6497 Y = DAG.getNode(ISD::SHL, DL, VT, Y, One);
6498 }
6499 Z = DAG.getNOT(DL, Z, ShVT);
6500 }
6501 Result = DAG.getNode(RevOpcode, DL, VT, X, Y, Z);
6502 return true;
6503 }
6504
6505 SDValue ShX, ShY;
6506 SDValue ShAmt, InvShAmt;
6507 if (isNonZeroModBitWidthOrUndef(Z, BW)) {
6508 // fshl: X << C | Y >> (BW - C)
6509 // fshr: X << (BW - C) | Y >> C
6510 // where C = Z % BW is not zero
6511 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
6512 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
6513 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthC, ShAmt);
6514 ShX = DAG.getNode(ISD::SHL, DL, VT, X, IsFSHL ? ShAmt : InvShAmt);
6515 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, IsFSHL ? InvShAmt : ShAmt);
6516 } else {
6517 // fshl: X << (Z % BW) | Y >> 1 >> (BW - 1 - (Z % BW))
6518 // fshr: X << 1 << (BW - 1 - (Z % BW)) | Y >> (Z % BW)
6519 SDValue Mask = DAG.getConstant(BW - 1, DL, ShVT);
6520 if (isPowerOf2_32(BW)) {
6521 // Z % BW -> Z & (BW - 1)
6522 ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Z, Mask);
6523 // (BW - 1) - (Z % BW) -> ~Z & (BW - 1)
6524 InvShAmt = DAG.getNode(ISD::AND, DL, ShVT, DAG.getNOT(DL, Z, ShVT), Mask);
6525 } else {
6526 SDValue BitWidthC = DAG.getConstant(BW, DL, ShVT);
6527 ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Z, BitWidthC);
6528 InvShAmt = DAG.getNode(ISD::SUB, DL, ShVT, Mask, ShAmt);
6529 }
6530
6531 SDValue One = DAG.getConstant(1, DL, ShVT);
6532 if (IsFSHL) {
6533 ShX = DAG.getNode(ISD::SHL, DL, VT, X, ShAmt);
6534 SDValue ShY1 = DAG.getNode(ISD::SRL, DL, VT, Y, One);
6535 ShY = DAG.getNode(ISD::SRL, DL, VT, ShY1, InvShAmt);
6536 } else {
6537 SDValue ShX1 = DAG.getNode(ISD::SHL, DL, VT, X, One);
6538 ShX = DAG.getNode(ISD::SHL, DL, VT, ShX1, InvShAmt);
6539 ShY = DAG.getNode(ISD::SRL, DL, VT, Y, ShAmt);
6540 }
6541 }
6542 Result = DAG.getNode(ISD::OR, DL, VT, ShX, ShY);
6543 return true;
6544 }
6545
6546 // TODO: Merge with expandFunnelShift.
expandROT(SDNode * Node,bool AllowVectorOps,SDValue & Result,SelectionDAG & DAG) const6547 bool TargetLowering::expandROT(SDNode *Node, bool AllowVectorOps,
6548 SDValue &Result, SelectionDAG &DAG) const {
6549 EVT VT = Node->getValueType(0);
6550 unsigned EltSizeInBits = VT.getScalarSizeInBits();
6551 bool IsLeft = Node->getOpcode() == ISD::ROTL;
6552 SDValue Op0 = Node->getOperand(0);
6553 SDValue Op1 = Node->getOperand(1);
6554 SDLoc DL(SDValue(Node, 0));
6555
6556 EVT ShVT = Op1.getValueType();
6557 SDValue Zero = DAG.getConstant(0, DL, ShVT);
6558
6559 // If a rotate in the other direction is supported, use it.
6560 unsigned RevRot = IsLeft ? ISD::ROTR : ISD::ROTL;
6561 if (isOperationLegalOrCustom(RevRot, VT) && isPowerOf2_32(EltSizeInBits)) {
6562 SDValue Sub = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6563 Result = DAG.getNode(RevRot, DL, VT, Op0, Sub);
6564 return true;
6565 }
6566
6567 if (!AllowVectorOps && VT.isVector() &&
6568 (!isOperationLegalOrCustom(ISD::SHL, VT) ||
6569 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6570 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6571 !isOperationLegalOrCustomOrPromote(ISD::OR, VT) ||
6572 !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
6573 return false;
6574
6575 unsigned ShOpc = IsLeft ? ISD::SHL : ISD::SRL;
6576 unsigned HsOpc = IsLeft ? ISD::SRL : ISD::SHL;
6577 SDValue BitWidthMinusOneC = DAG.getConstant(EltSizeInBits - 1, DL, ShVT);
6578 SDValue ShVal;
6579 SDValue HsVal;
6580 if (isPowerOf2_32(EltSizeInBits)) {
6581 // (rotl x, c) -> x << (c & (w - 1)) | x >> (-c & (w - 1))
6582 // (rotr x, c) -> x >> (c & (w - 1)) | x << (-c & (w - 1))
6583 SDValue NegOp1 = DAG.getNode(ISD::SUB, DL, ShVT, Zero, Op1);
6584 SDValue ShAmt = DAG.getNode(ISD::AND, DL, ShVT, Op1, BitWidthMinusOneC);
6585 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
6586 SDValue HsAmt = DAG.getNode(ISD::AND, DL, ShVT, NegOp1, BitWidthMinusOneC);
6587 HsVal = DAG.getNode(HsOpc, DL, VT, Op0, HsAmt);
6588 } else {
6589 // (rotl x, c) -> x << (c % w) | x >> 1 >> (w - 1 - (c % w))
6590 // (rotr x, c) -> x >> (c % w) | x << 1 << (w - 1 - (c % w))
6591 SDValue BitWidthC = DAG.getConstant(EltSizeInBits, DL, ShVT);
6592 SDValue ShAmt = DAG.getNode(ISD::UREM, DL, ShVT, Op1, BitWidthC);
6593 ShVal = DAG.getNode(ShOpc, DL, VT, Op0, ShAmt);
6594 SDValue HsAmt = DAG.getNode(ISD::SUB, DL, ShVT, BitWidthMinusOneC, ShAmt);
6595 SDValue One = DAG.getConstant(1, DL, ShVT);
6596 HsVal =
6597 DAG.getNode(HsOpc, DL, VT, DAG.getNode(HsOpc, DL, VT, Op0, One), HsAmt);
6598 }
6599 Result = DAG.getNode(ISD::OR, DL, VT, ShVal, HsVal);
6600 return true;
6601 }
6602
expandShiftParts(SDNode * Node,SDValue & Lo,SDValue & Hi,SelectionDAG & DAG) const6603 void TargetLowering::expandShiftParts(SDNode *Node, SDValue &Lo, SDValue &Hi,
6604 SelectionDAG &DAG) const {
6605 assert(Node->getNumOperands() == 3 && "Not a double-shift!");
6606 EVT VT = Node->getValueType(0);
6607 unsigned VTBits = VT.getScalarSizeInBits();
6608 assert(isPowerOf2_32(VTBits) && "Power-of-two integer type expected");
6609
6610 bool IsSHL = Node->getOpcode() == ISD::SHL_PARTS;
6611 bool IsSRA = Node->getOpcode() == ISD::SRA_PARTS;
6612 SDValue ShOpLo = Node->getOperand(0);
6613 SDValue ShOpHi = Node->getOperand(1);
6614 SDValue ShAmt = Node->getOperand(2);
6615 EVT ShAmtVT = ShAmt.getValueType();
6616 EVT ShAmtCCVT =
6617 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ShAmtVT);
6618 SDLoc dl(Node);
6619
6620 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
6621 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's usually optimized
6622 // away during isel.
6623 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
6624 DAG.getConstant(VTBits - 1, dl, ShAmtVT));
6625 SDValue Tmp1 = IsSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
6626 DAG.getConstant(VTBits - 1, dl, ShAmtVT))
6627 : DAG.getConstant(0, dl, VT);
6628
6629 SDValue Tmp2, Tmp3;
6630 if (IsSHL) {
6631 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);
6632 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
6633 } else {
6634 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);
6635 Tmp3 = DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
6636 }
6637
6638 // If the shift amount is larger or equal than the width of a part we don't
6639 // use the result from the FSHL/FSHR. Insert a test and select the appropriate
6640 // values for large shift amounts.
6641 SDValue AndNode = DAG.getNode(ISD::AND, dl, ShAmtVT, ShAmt,
6642 DAG.getConstant(VTBits, dl, ShAmtVT));
6643 SDValue Cond = DAG.getSetCC(dl, ShAmtCCVT, AndNode,
6644 DAG.getConstant(0, dl, ShAmtVT), ISD::SETNE);
6645
6646 if (IsSHL) {
6647 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
6648 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
6649 } else {
6650 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
6651 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
6652 }
6653 }
6654
expandFP_TO_SINT(SDNode * Node,SDValue & Result,SelectionDAG & DAG) const6655 bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
6656 SelectionDAG &DAG) const {
6657 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
6658 SDValue Src = Node->getOperand(OpNo);
6659 EVT SrcVT = Src.getValueType();
6660 EVT DstVT = Node->getValueType(0);
6661 SDLoc dl(SDValue(Node, 0));
6662
6663 // FIXME: Only f32 to i64 conversions are supported.
6664 if (SrcVT != MVT::f32 || DstVT != MVT::i64)
6665 return false;
6666
6667 if (Node->isStrictFPOpcode())
6668 // When a NaN is converted to an integer a trap is allowed. We can't
6669 // use this expansion here because it would eliminate that trap. Other
6670 // traps are also allowed and cannot be eliminated. See
6671 // IEEE 754-2008 sec 5.8.
6672 return false;
6673
6674 // Expand f32 -> i64 conversion
6675 // This algorithm comes from compiler-rt's implementation of fixsfdi:
6676 // https://github.com/llvm/llvm-project/blob/main/compiler-rt/lib/builtins/fixsfdi.c
6677 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
6678 EVT IntVT = SrcVT.changeTypeToInteger();
6679 EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
6680
6681 SDValue ExponentMask = DAG.getConstant(0x7F800000, dl, IntVT);
6682 SDValue ExponentLoBit = DAG.getConstant(23, dl, IntVT);
6683 SDValue Bias = DAG.getConstant(127, dl, IntVT);
6684 SDValue SignMask = DAG.getConstant(APInt::getSignMask(SrcEltBits), dl, IntVT);
6685 SDValue SignLowBit = DAG.getConstant(SrcEltBits - 1, dl, IntVT);
6686 SDValue MantissaMask = DAG.getConstant(0x007FFFFF, dl, IntVT);
6687
6688 SDValue Bits = DAG.getNode(ISD::BITCAST, dl, IntVT, Src);
6689
6690 SDValue ExponentBits = DAG.getNode(
6691 ISD::SRL, dl, IntVT, DAG.getNode(ISD::AND, dl, IntVT, Bits, ExponentMask),
6692 DAG.getZExtOrTrunc(ExponentLoBit, dl, IntShVT));
6693 SDValue Exponent = DAG.getNode(ISD::SUB, dl, IntVT, ExponentBits, Bias);
6694
6695 SDValue Sign = DAG.getNode(ISD::SRA, dl, IntVT,
6696 DAG.getNode(ISD::AND, dl, IntVT, Bits, SignMask),
6697 DAG.getZExtOrTrunc(SignLowBit, dl, IntShVT));
6698 Sign = DAG.getSExtOrTrunc(Sign, dl, DstVT);
6699
6700 SDValue R = DAG.getNode(ISD::OR, dl, IntVT,
6701 DAG.getNode(ISD::AND, dl, IntVT, Bits, MantissaMask),
6702 DAG.getConstant(0x00800000, dl, IntVT));
6703
6704 R = DAG.getZExtOrTrunc(R, dl, DstVT);
6705
6706 R = DAG.getSelectCC(
6707 dl, Exponent, ExponentLoBit,
6708 DAG.getNode(ISD::SHL, dl, DstVT, R,
6709 DAG.getZExtOrTrunc(
6710 DAG.getNode(ISD::SUB, dl, IntVT, Exponent, ExponentLoBit),
6711 dl, IntShVT)),
6712 DAG.getNode(ISD::SRL, dl, DstVT, R,
6713 DAG.getZExtOrTrunc(
6714 DAG.getNode(ISD::SUB, dl, IntVT, ExponentLoBit, Exponent),
6715 dl, IntShVT)),
6716 ISD::SETGT);
6717
6718 SDValue Ret = DAG.getNode(ISD::SUB, dl, DstVT,
6719 DAG.getNode(ISD::XOR, dl, DstVT, R, Sign), Sign);
6720
6721 Result = DAG.getSelectCC(dl, Exponent, DAG.getConstant(0, dl, IntVT),
6722 DAG.getConstant(0, dl, DstVT), Ret, ISD::SETLT);
6723 return true;
6724 }
6725
expandFP_TO_UINT(SDNode * Node,SDValue & Result,SDValue & Chain,SelectionDAG & DAG) const6726 bool TargetLowering::expandFP_TO_UINT(SDNode *Node, SDValue &Result,
6727 SDValue &Chain,
6728 SelectionDAG &DAG) const {
6729 SDLoc dl(SDValue(Node, 0));
6730 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
6731 SDValue Src = Node->getOperand(OpNo);
6732
6733 EVT SrcVT = Src.getValueType();
6734 EVT DstVT = Node->getValueType(0);
6735 EVT SetCCVT =
6736 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
6737 EVT DstSetCCVT =
6738 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), DstVT);
6739
6740 // Only expand vector types if we have the appropriate vector bit operations.
6741 unsigned SIntOpcode = Node->isStrictFPOpcode() ? ISD::STRICT_FP_TO_SINT :
6742 ISD::FP_TO_SINT;
6743 if (DstVT.isVector() && (!isOperationLegalOrCustom(SIntOpcode, DstVT) ||
6744 !isOperationLegalOrCustomOrPromote(ISD::XOR, SrcVT)))
6745 return false;
6746
6747 // If the maximum float value is smaller then the signed integer range,
6748 // the destination signmask can't be represented by the float, so we can
6749 // just use FP_TO_SINT directly.
6750 const fltSemantics &APFSem = DAG.EVTToAPFloatSemantics(SrcVT);
6751 APFloat APF(APFSem, APInt::getNullValue(SrcVT.getScalarSizeInBits()));
6752 APInt SignMask = APInt::getSignMask(DstVT.getScalarSizeInBits());
6753 if (APFloat::opOverflow &
6754 APF.convertFromAPInt(SignMask, false, APFloat::rmNearestTiesToEven)) {
6755 if (Node->isStrictFPOpcode()) {
6756 Result = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
6757 { Node->getOperand(0), Src });
6758 Chain = Result.getValue(1);
6759 } else
6760 Result = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6761 return true;
6762 }
6763
6764 // Don't expand it if there isn't cheap fsub instruction.
6765 if (!isOperationLegalOrCustom(
6766 Node->isStrictFPOpcode() ? ISD::STRICT_FSUB : ISD::FSUB, SrcVT))
6767 return false;
6768
6769 SDValue Cst = DAG.getConstantFP(APF, dl, SrcVT);
6770 SDValue Sel;
6771
6772 if (Node->isStrictFPOpcode()) {
6773 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT,
6774 Node->getOperand(0), /*IsSignaling*/ true);
6775 Chain = Sel.getValue(1);
6776 } else {
6777 Sel = DAG.getSetCC(dl, SetCCVT, Src, Cst, ISD::SETLT);
6778 }
6779
6780 bool Strict = Node->isStrictFPOpcode() ||
6781 shouldUseStrictFP_TO_INT(SrcVT, DstVT, /*IsSigned*/ false);
6782
6783 if (Strict) {
6784 // Expand based on maximum range of FP_TO_SINT, if the value exceeds the
6785 // signmask then offset (the result of which should be fully representable).
6786 // Sel = Src < 0x8000000000000000
6787 // FltOfs = select Sel, 0, 0x8000000000000000
6788 // IntOfs = select Sel, 0, 0x8000000000000000
6789 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
6790
6791 // TODO: Should any fast-math-flags be set for the FSUB?
6792 SDValue FltOfs = DAG.getSelect(dl, SrcVT, Sel,
6793 DAG.getConstantFP(0.0, dl, SrcVT), Cst);
6794 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
6795 SDValue IntOfs = DAG.getSelect(dl, DstVT, Sel,
6796 DAG.getConstant(0, dl, DstVT),
6797 DAG.getConstant(SignMask, dl, DstVT));
6798 SDValue SInt;
6799 if (Node->isStrictFPOpcode()) {
6800 SDValue Val = DAG.getNode(ISD::STRICT_FSUB, dl, { SrcVT, MVT::Other },
6801 { Chain, Src, FltOfs });
6802 SInt = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { DstVT, MVT::Other },
6803 { Val.getValue(1), Val });
6804 Chain = SInt.getValue(1);
6805 } else {
6806 SDValue Val = DAG.getNode(ISD::FSUB, dl, SrcVT, Src, FltOfs);
6807 SInt = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Val);
6808 }
6809 Result = DAG.getNode(ISD::XOR, dl, DstVT, SInt, IntOfs);
6810 } else {
6811 // Expand based on maximum range of FP_TO_SINT:
6812 // True = fp_to_sint(Src)
6813 // False = 0x8000000000000000 + fp_to_sint(Src - 0x8000000000000000)
6814 // Result = select (Src < 0x8000000000000000), True, False
6815
6816 SDValue True = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, Src);
6817 // TODO: Should any fast-math-flags be set for the FSUB?
6818 SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT,
6819 DAG.getNode(ISD::FSUB, dl, SrcVT, Src, Cst));
6820 False = DAG.getNode(ISD::XOR, dl, DstVT, False,
6821 DAG.getConstant(SignMask, dl, DstVT));
6822 Sel = DAG.getBoolExtOrTrunc(Sel, dl, DstSetCCVT, DstVT);
6823 Result = DAG.getSelect(dl, DstVT, Sel, True, False);
6824 }
6825 return true;
6826 }
6827
expandUINT_TO_FP(SDNode * Node,SDValue & Result,SDValue & Chain,SelectionDAG & DAG) const6828 bool TargetLowering::expandUINT_TO_FP(SDNode *Node, SDValue &Result,
6829 SDValue &Chain,
6830 SelectionDAG &DAG) const {
6831 // This transform is not correct for converting 0 when rounding mode is set
6832 // to round toward negative infinity which will produce -0.0. So disable under
6833 // strictfp.
6834 if (Node->isStrictFPOpcode())
6835 return false;
6836
6837 SDValue Src = Node->getOperand(0);
6838 EVT SrcVT = Src.getValueType();
6839 EVT DstVT = Node->getValueType(0);
6840
6841 if (SrcVT.getScalarType() != MVT::i64 || DstVT.getScalarType() != MVT::f64)
6842 return false;
6843
6844 // Only expand vector types if we have the appropriate vector bit operations.
6845 if (SrcVT.isVector() && (!isOperationLegalOrCustom(ISD::SRL, SrcVT) ||
6846 !isOperationLegalOrCustom(ISD::FADD, DstVT) ||
6847 !isOperationLegalOrCustom(ISD::FSUB, DstVT) ||
6848 !isOperationLegalOrCustomOrPromote(ISD::OR, SrcVT) ||
6849 !isOperationLegalOrCustomOrPromote(ISD::AND, SrcVT)))
6850 return false;
6851
6852 SDLoc dl(SDValue(Node, 0));
6853 EVT ShiftVT = getShiftAmountTy(SrcVT, DAG.getDataLayout());
6854
6855 // Implementation of unsigned i64 to f64 following the algorithm in
6856 // __floatundidf in compiler_rt. This implementation performs rounding
6857 // correctly in all rounding modes with the exception of converting 0
6858 // when rounding toward negative infinity. In that case the fsub will produce
6859 // -0.0. This will be added to +0.0 and produce -0.0 which is incorrect.
6860 SDValue TwoP52 = DAG.getConstant(UINT64_C(0x4330000000000000), dl, SrcVT);
6861 SDValue TwoP84PlusTwoP52 = DAG.getConstantFP(
6862 BitsToDouble(UINT64_C(0x4530000000100000)), dl, DstVT);
6863 SDValue TwoP84 = DAG.getConstant(UINT64_C(0x4530000000000000), dl, SrcVT);
6864 SDValue LoMask = DAG.getConstant(UINT64_C(0x00000000FFFFFFFF), dl, SrcVT);
6865 SDValue HiShift = DAG.getConstant(32, dl, ShiftVT);
6866
6867 SDValue Lo = DAG.getNode(ISD::AND, dl, SrcVT, Src, LoMask);
6868 SDValue Hi = DAG.getNode(ISD::SRL, dl, SrcVT, Src, HiShift);
6869 SDValue LoOr = DAG.getNode(ISD::OR, dl, SrcVT, Lo, TwoP52);
6870 SDValue HiOr = DAG.getNode(ISD::OR, dl, SrcVT, Hi, TwoP84);
6871 SDValue LoFlt = DAG.getBitcast(DstVT, LoOr);
6872 SDValue HiFlt = DAG.getBitcast(DstVT, HiOr);
6873 SDValue HiSub =
6874 DAG.getNode(ISD::FSUB, dl, DstVT, HiFlt, TwoP84PlusTwoP52);
6875 Result = DAG.getNode(ISD::FADD, dl, DstVT, LoFlt, HiSub);
6876 return true;
6877 }
6878
expandFMINNUM_FMAXNUM(SDNode * Node,SelectionDAG & DAG) const6879 SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node,
6880 SelectionDAG &DAG) const {
6881 SDLoc dl(Node);
6882 unsigned NewOp = Node->getOpcode() == ISD::FMINNUM ?
6883 ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
6884 EVT VT = Node->getValueType(0);
6885
6886 if (VT.isScalableVector())
6887 report_fatal_error(
6888 "Expanding fminnum/fmaxnum for scalable vectors is undefined.");
6889
6890 if (isOperationLegalOrCustom(NewOp, VT)) {
6891 SDValue Quiet0 = Node->getOperand(0);
6892 SDValue Quiet1 = Node->getOperand(1);
6893
6894 if (!Node->getFlags().hasNoNaNs()) {
6895 // Insert canonicalizes if it's possible we need to quiet to get correct
6896 // sNaN behavior.
6897 if (!DAG.isKnownNeverSNaN(Quiet0)) {
6898 Quiet0 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet0,
6899 Node->getFlags());
6900 }
6901 if (!DAG.isKnownNeverSNaN(Quiet1)) {
6902 Quiet1 = DAG.getNode(ISD::FCANONICALIZE, dl, VT, Quiet1,
6903 Node->getFlags());
6904 }
6905 }
6906
6907 return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags());
6908 }
6909
6910 // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that
6911 // instead if there are no NaNs.
6912 if (Node->getFlags().hasNoNaNs()) {
6913 unsigned IEEE2018Op =
6914 Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM;
6915 if (isOperationLegalOrCustom(IEEE2018Op, VT)) {
6916 return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0),
6917 Node->getOperand(1), Node->getFlags());
6918 }
6919 }
6920
6921 // If none of the above worked, but there are no NaNs, then expand to
6922 // a compare/select sequence. This is required for correctness since
6923 // InstCombine might have canonicalized a fcmp+select sequence to a
6924 // FMINNUM/FMAXNUM node. If we were to fall through to the default
6925 // expansion to libcall, we might introduce a link-time dependency
6926 // on libm into a file that originally did not have one.
6927 if (Node->getFlags().hasNoNaNs()) {
6928 ISD::CondCode Pred =
6929 Node->getOpcode() == ISD::FMINNUM ? ISD::SETLT : ISD::SETGT;
6930 SDValue Op1 = Node->getOperand(0);
6931 SDValue Op2 = Node->getOperand(1);
6932 SDValue SelCC = DAG.getSelectCC(dl, Op1, Op2, Op1, Op2, Pred);
6933 // Copy FMF flags, but always set the no-signed-zeros flag
6934 // as this is implied by the FMINNUM/FMAXNUM semantics.
6935 SDNodeFlags Flags = Node->getFlags();
6936 Flags.setNoSignedZeros(true);
6937 SelCC->setFlags(Flags);
6938 return SelCC;
6939 }
6940
6941 return SDValue();
6942 }
6943
expandCTPOP(SDNode * Node,SDValue & Result,SelectionDAG & DAG) const6944 bool TargetLowering::expandCTPOP(SDNode *Node, SDValue &Result,
6945 SelectionDAG &DAG) const {
6946 SDLoc dl(Node);
6947 EVT VT = Node->getValueType(0);
6948 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
6949 SDValue Op = Node->getOperand(0);
6950 unsigned Len = VT.getScalarSizeInBits();
6951 assert(VT.isInteger() && "CTPOP not implemented for this type.");
6952
6953 // TODO: Add support for irregular type lengths.
6954 if (!(Len <= 128 && Len % 8 == 0))
6955 return false;
6956
6957 // Only expand vector types if we have the appropriate vector bit operations.
6958 if (VT.isVector() && (!isOperationLegalOrCustom(ISD::ADD, VT) ||
6959 !isOperationLegalOrCustom(ISD::SUB, VT) ||
6960 !isOperationLegalOrCustom(ISD::SRL, VT) ||
6961 (Len != 8 && !isOperationLegalOrCustom(ISD::MUL, VT)) ||
6962 !isOperationLegalOrCustomOrPromote(ISD::AND, VT)))
6963 return false;
6964
6965 // This is the "best" algorithm from
6966 // http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
6967 SDValue Mask55 =
6968 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x55)), dl, VT);
6969 SDValue Mask33 =
6970 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x33)), dl, VT);
6971 SDValue Mask0F =
6972 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x0F)), dl, VT);
6973 SDValue Mask01 =
6974 DAG.getConstant(APInt::getSplat(Len, APInt(8, 0x01)), dl, VT);
6975
6976 // v = v - ((v >> 1) & 0x55555555...)
6977 Op = DAG.getNode(ISD::SUB, dl, VT, Op,
6978 DAG.getNode(ISD::AND, dl, VT,
6979 DAG.getNode(ISD::SRL, dl, VT, Op,
6980 DAG.getConstant(1, dl, ShVT)),
6981 Mask55));
6982 // v = (v & 0x33333333...) + ((v >> 2) & 0x33333333...)
6983 Op = DAG.getNode(ISD::ADD, dl, VT, DAG.getNode(ISD::AND, dl, VT, Op, Mask33),
6984 DAG.getNode(ISD::AND, dl, VT,
6985 DAG.getNode(ISD::SRL, dl, VT, Op,
6986 DAG.getConstant(2, dl, ShVT)),
6987 Mask33));
6988 // v = (v + (v >> 4)) & 0x0F0F0F0F...
6989 Op = DAG.getNode(ISD::AND, dl, VT,
6990 DAG.getNode(ISD::ADD, dl, VT, Op,
6991 DAG.getNode(ISD::SRL, dl, VT, Op,
6992 DAG.getConstant(4, dl, ShVT))),
6993 Mask0F);
6994 // v = (v * 0x01010101...) >> (Len - 8)
6995 if (Len > 8)
6996 Op =
6997 DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::MUL, dl, VT, Op, Mask01),
6998 DAG.getConstant(Len - 8, dl, ShVT));
6999
7000 Result = Op;
7001 return true;
7002 }
7003
expandCTLZ(SDNode * Node,SDValue & Result,SelectionDAG & DAG) const7004 bool TargetLowering::expandCTLZ(SDNode *Node, SDValue &Result,
7005 SelectionDAG &DAG) const {
7006 SDLoc dl(Node);
7007 EVT VT = Node->getValueType(0);
7008 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7009 SDValue Op = Node->getOperand(0);
7010 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
7011
7012 // If the non-ZERO_UNDEF version is supported we can use that instead.
7013 if (Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF &&
7014 isOperationLegalOrCustom(ISD::CTLZ, VT)) {
7015 Result = DAG.getNode(ISD::CTLZ, dl, VT, Op);
7016 return true;
7017 }
7018
7019 // If the ZERO_UNDEF version is supported use that and handle the zero case.
7020 if (isOperationLegalOrCustom(ISD::CTLZ_ZERO_UNDEF, VT)) {
7021 EVT SetCCVT =
7022 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7023 SDValue CTLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, VT, Op);
7024 SDValue Zero = DAG.getConstant(0, dl, VT);
7025 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
7026 Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
7027 DAG.getConstant(NumBitsPerElt, dl, VT), CTLZ);
7028 return true;
7029 }
7030
7031 // Only expand vector types if we have the appropriate vector bit operations.
7032 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
7033 !isOperationLegalOrCustom(ISD::CTPOP, VT) ||
7034 !isOperationLegalOrCustom(ISD::SRL, VT) ||
7035 !isOperationLegalOrCustomOrPromote(ISD::OR, VT)))
7036 return false;
7037
7038 // for now, we do this:
7039 // x = x | (x >> 1);
7040 // x = x | (x >> 2);
7041 // ...
7042 // x = x | (x >>16);
7043 // x = x | (x >>32); // for 64-bit input
7044 // return popcount(~x);
7045 //
7046 // Ref: "Hacker's Delight" by Henry Warren
7047 for (unsigned i = 0; (1U << i) <= (NumBitsPerElt / 2); ++i) {
7048 SDValue Tmp = DAG.getConstant(1ULL << i, dl, ShVT);
7049 Op = DAG.getNode(ISD::OR, dl, VT, Op,
7050 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp));
7051 }
7052 Op = DAG.getNOT(dl, Op, VT);
7053 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
7054 return true;
7055 }
7056
expandCTTZ(SDNode * Node,SDValue & Result,SelectionDAG & DAG) const7057 bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,
7058 SelectionDAG &DAG) const {
7059 SDLoc dl(Node);
7060 EVT VT = Node->getValueType(0);
7061 SDValue Op = Node->getOperand(0);
7062 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
7063
7064 // If the non-ZERO_UNDEF version is supported we can use that instead.
7065 if (Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF &&
7066 isOperationLegalOrCustom(ISD::CTTZ, VT)) {
7067 Result = DAG.getNode(ISD::CTTZ, dl, VT, Op);
7068 return true;
7069 }
7070
7071 // If the ZERO_UNDEF version is supported use that and handle the zero case.
7072 if (isOperationLegalOrCustom(ISD::CTTZ_ZERO_UNDEF, VT)) {
7073 EVT SetCCVT =
7074 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
7075 SDValue CTTZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, VT, Op);
7076 SDValue Zero = DAG.getConstant(0, dl, VT);
7077 SDValue SrcIsZero = DAG.getSetCC(dl, SetCCVT, Op, Zero, ISD::SETEQ);
7078 Result = DAG.getNode(ISD::SELECT, dl, VT, SrcIsZero,
7079 DAG.getConstant(NumBitsPerElt, dl, VT), CTTZ);
7080 return true;
7081 }
7082
7083 // Only expand vector types if we have the appropriate vector bit operations.
7084 if (VT.isVector() && (!isPowerOf2_32(NumBitsPerElt) ||
7085 (!isOperationLegalOrCustom(ISD::CTPOP, VT) &&
7086 !isOperationLegalOrCustom(ISD::CTLZ, VT)) ||
7087 !isOperationLegalOrCustom(ISD::SUB, VT) ||
7088 !isOperationLegalOrCustomOrPromote(ISD::AND, VT) ||
7089 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
7090 return false;
7091
7092 // for now, we use: { return popcount(~x & (x - 1)); }
7093 // unless the target has ctlz but not ctpop, in which case we use:
7094 // { return 32 - nlz(~x & (x-1)); }
7095 // Ref: "Hacker's Delight" by Henry Warren
7096 SDValue Tmp = DAG.getNode(
7097 ISD::AND, dl, VT, DAG.getNOT(dl, Op, VT),
7098 DAG.getNode(ISD::SUB, dl, VT, Op, DAG.getConstant(1, dl, VT)));
7099
7100 // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
7101 if (isOperationLegal(ISD::CTLZ, VT) && !isOperationLegal(ISD::CTPOP, VT)) {
7102 Result =
7103 DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(NumBitsPerElt, dl, VT),
7104 DAG.getNode(ISD::CTLZ, dl, VT, Tmp));
7105 return true;
7106 }
7107
7108 Result = DAG.getNode(ISD::CTPOP, dl, VT, Tmp);
7109 return true;
7110 }
7111
expandABS(SDNode * N,SDValue & Result,SelectionDAG & DAG,bool IsNegative) const7112 bool TargetLowering::expandABS(SDNode *N, SDValue &Result,
7113 SelectionDAG &DAG, bool IsNegative) const {
7114 SDLoc dl(N);
7115 EVT VT = N->getValueType(0);
7116 EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
7117 SDValue Op = N->getOperand(0);
7118
7119 // abs(x) -> smax(x,sub(0,x))
7120 if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
7121 isOperationLegal(ISD::SMAX, VT)) {
7122 SDValue Zero = DAG.getConstant(0, dl, VT);
7123 Result = DAG.getNode(ISD::SMAX, dl, VT, Op,
7124 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7125 return true;
7126 }
7127
7128 // abs(x) -> umin(x,sub(0,x))
7129 if (!IsNegative && isOperationLegal(ISD::SUB, VT) &&
7130 isOperationLegal(ISD::UMIN, VT)) {
7131 SDValue Zero = DAG.getConstant(0, dl, VT);
7132 Result = DAG.getNode(ISD::UMIN, dl, VT, Op,
7133 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7134 return true;
7135 }
7136
7137 // 0 - abs(x) -> smin(x, sub(0,x))
7138 if (IsNegative && isOperationLegal(ISD::SUB, VT) &&
7139 isOperationLegal(ISD::SMIN, VT)) {
7140 SDValue Zero = DAG.getConstant(0, dl, VT);
7141 Result = DAG.getNode(ISD::SMIN, dl, VT, Op,
7142 DAG.getNode(ISD::SUB, dl, VT, Zero, Op));
7143 return true;
7144 }
7145
7146 // Only expand vector types if we have the appropriate vector operations.
7147 if (VT.isVector() &&
7148 (!isOperationLegalOrCustom(ISD::SRA, VT) ||
7149 (!IsNegative && !isOperationLegalOrCustom(ISD::ADD, VT)) ||
7150 (IsNegative && !isOperationLegalOrCustom(ISD::SUB, VT)) ||
7151 !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
7152 return false;
7153
7154 SDValue Shift =
7155 DAG.getNode(ISD::SRA, dl, VT, Op,
7156 DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT));
7157 if (!IsNegative) {
7158 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
7159 Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift);
7160 } else {
7161 // 0 - abs(x) -> Y = sra (X, size(X)-1); sub (Y, xor (X, Y))
7162 SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, Op, Shift);
7163 Result = DAG.getNode(ISD::SUB, dl, VT, Shift, Xor);
7164 }
7165 return true;
7166 }
7167
expandBSWAP(SDNode * N,SelectionDAG & DAG) const7168 SDValue TargetLowering::expandBSWAP(SDNode *N, SelectionDAG &DAG) const {
7169 SDLoc dl(N);
7170 EVT VT = N->getValueType(0);
7171 SDValue Op = N->getOperand(0);
7172
7173 if (!VT.isSimple())
7174 return SDValue();
7175
7176 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
7177 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
7178 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
7179 default:
7180 return SDValue();
7181 case MVT::i16:
7182 // Use a rotate by 8. This can be further expanded if necessary.
7183 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7184 case MVT::i32:
7185 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7186 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7187 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7188 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7189 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
7190 DAG.getConstant(0xFF0000, dl, VT));
7191 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
7192 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
7193 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
7194 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
7195 case MVT::i64:
7196 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
7197 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
7198 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7199 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7200 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
7201 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
7202 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
7203 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
7204 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
7205 DAG.getConstant(255ULL<<48, dl, VT));
7206 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
7207 DAG.getConstant(255ULL<<40, dl, VT));
7208 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
7209 DAG.getConstant(255ULL<<32, dl, VT));
7210 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
7211 DAG.getConstant(255ULL<<24, dl, VT));
7212 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
7213 DAG.getConstant(255ULL<<16, dl, VT));
7214 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
7215 DAG.getConstant(255ULL<<8 , dl, VT));
7216 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
7217 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
7218 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
7219 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
7220 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
7221 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
7222 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
7223 }
7224 }
7225
expandBITREVERSE(SDNode * N,SelectionDAG & DAG) const7226 SDValue TargetLowering::expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const {
7227 SDLoc dl(N);
7228 EVT VT = N->getValueType(0);
7229 SDValue Op = N->getOperand(0);
7230 EVT SHVT = getShiftAmountTy(VT, DAG.getDataLayout());
7231 unsigned Sz = VT.getScalarSizeInBits();
7232
7233 SDValue Tmp, Tmp2, Tmp3;
7234
7235 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
7236 // and finally the i1 pairs.
7237 // TODO: We can easily support i4/i2 legal types if any target ever does.
7238 if (Sz >= 8 && isPowerOf2_32(Sz)) {
7239 // Create the masks - repeating the pattern every byte.
7240 APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0));
7241 APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC));
7242 APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA));
7243 APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F));
7244 APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33));
7245 APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55));
7246
7247 // BSWAP if the type is wider than a single byte.
7248 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
7249
7250 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
7251 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
7252 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
7253 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT));
7254 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
7255 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7256
7257 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
7258 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
7259 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
7260 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT));
7261 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
7262 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7263
7264 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
7265 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
7266 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
7267 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT));
7268 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
7269 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
7270 return Tmp;
7271 }
7272
7273 Tmp = DAG.getConstant(0, dl, VT);
7274 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
7275 if (I < J)
7276 Tmp2 =
7277 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
7278 else
7279 Tmp2 =
7280 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
7281
7282 APInt Shift(Sz, 1);
7283 Shift <<= J;
7284 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
7285 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
7286 }
7287
7288 return Tmp;
7289 }
7290
7291 std::pair<SDValue, SDValue>
scalarizeVectorLoad(LoadSDNode * LD,SelectionDAG & DAG) const7292 TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
7293 SelectionDAG &DAG) const {
7294 SDLoc SL(LD);
7295 SDValue Chain = LD->getChain();
7296 SDValue BasePTR = LD->getBasePtr();
7297 EVT SrcVT = LD->getMemoryVT();
7298 EVT DstVT = LD->getValueType(0);
7299 ISD::LoadExtType ExtType = LD->getExtensionType();
7300
7301 if (SrcVT.isScalableVector())
7302 report_fatal_error("Cannot scalarize scalable vector loads");
7303
7304 unsigned NumElem = SrcVT.getVectorNumElements();
7305
7306 EVT SrcEltVT = SrcVT.getScalarType();
7307 EVT DstEltVT = DstVT.getScalarType();
7308
7309 // A vector must always be stored in memory as-is, i.e. without any padding
7310 // between the elements, since various code depend on it, e.g. in the
7311 // handling of a bitcast of a vector type to int, which may be done with a
7312 // vector store followed by an integer load. A vector that does not have
7313 // elements that are byte-sized must therefore be stored as an integer
7314 // built out of the extracted vector elements.
7315 if (!SrcEltVT.isByteSized()) {
7316 unsigned NumLoadBits = SrcVT.getStoreSizeInBits();
7317 EVT LoadVT = EVT::getIntegerVT(*DAG.getContext(), NumLoadBits);
7318
7319 unsigned NumSrcBits = SrcVT.getSizeInBits();
7320 EVT SrcIntVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcBits);
7321
7322 unsigned SrcEltBits = SrcEltVT.getSizeInBits();
7323 SDValue SrcEltBitMask = DAG.getConstant(
7324 APInt::getLowBitsSet(NumLoadBits, SrcEltBits), SL, LoadVT);
7325
7326 // Load the whole vector and avoid masking off the top bits as it makes
7327 // the codegen worse.
7328 SDValue Load =
7329 DAG.getExtLoad(ISD::EXTLOAD, SL, LoadVT, Chain, BasePTR,
7330 LD->getPointerInfo(), SrcIntVT, LD->getOriginalAlign(),
7331 LD->getMemOperand()->getFlags(), LD->getAAInfo());
7332
7333 SmallVector<SDValue, 8> Vals;
7334 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7335 unsigned ShiftIntoIdx =
7336 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
7337 SDValue ShiftAmount =
7338 DAG.getShiftAmountConstant(ShiftIntoIdx * SrcEltVT.getSizeInBits(),
7339 LoadVT, SL, /*LegalTypes=*/false);
7340 SDValue ShiftedElt = DAG.getNode(ISD::SRL, SL, LoadVT, Load, ShiftAmount);
7341 SDValue Elt =
7342 DAG.getNode(ISD::AND, SL, LoadVT, ShiftedElt, SrcEltBitMask);
7343 SDValue Scalar = DAG.getNode(ISD::TRUNCATE, SL, SrcEltVT, Elt);
7344
7345 if (ExtType != ISD::NON_EXTLOAD) {
7346 unsigned ExtendOp = ISD::getExtForLoadExtType(false, ExtType);
7347 Scalar = DAG.getNode(ExtendOp, SL, DstEltVT, Scalar);
7348 }
7349
7350 Vals.push_back(Scalar);
7351 }
7352
7353 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
7354 return std::make_pair(Value, Load.getValue(1));
7355 }
7356
7357 unsigned Stride = SrcEltVT.getSizeInBits() / 8;
7358 assert(SrcEltVT.isByteSized());
7359
7360 SmallVector<SDValue, 8> Vals;
7361 SmallVector<SDValue, 8> LoadChains;
7362
7363 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7364 SDValue ScalarLoad =
7365 DAG.getExtLoad(ExtType, SL, DstEltVT, Chain, BasePTR,
7366 LD->getPointerInfo().getWithOffset(Idx * Stride),
7367 SrcEltVT, LD->getOriginalAlign(),
7368 LD->getMemOperand()->getFlags(), LD->getAAInfo());
7369
7370 BasePTR = DAG.getObjectPtrOffset(SL, BasePTR, TypeSize::Fixed(Stride));
7371
7372 Vals.push_back(ScalarLoad.getValue(0));
7373 LoadChains.push_back(ScalarLoad.getValue(1));
7374 }
7375
7376 SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains);
7377 SDValue Value = DAG.getBuildVector(DstVT, SL, Vals);
7378
7379 return std::make_pair(Value, NewChain);
7380 }
7381
scalarizeVectorStore(StoreSDNode * ST,SelectionDAG & DAG) const7382 SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST,
7383 SelectionDAG &DAG) const {
7384 SDLoc SL(ST);
7385
7386 SDValue Chain = ST->getChain();
7387 SDValue BasePtr = ST->getBasePtr();
7388 SDValue Value = ST->getValue();
7389 EVT StVT = ST->getMemoryVT();
7390
7391 if (StVT.isScalableVector())
7392 report_fatal_error("Cannot scalarize scalable vector stores");
7393
7394 // The type of the data we want to save
7395 EVT RegVT = Value.getValueType();
7396 EVT RegSclVT = RegVT.getScalarType();
7397
7398 // The type of data as saved in memory.
7399 EVT MemSclVT = StVT.getScalarType();
7400
7401 unsigned NumElem = StVT.getVectorNumElements();
7402
7403 // A vector must always be stored in memory as-is, i.e. without any padding
7404 // between the elements, since various code depend on it, e.g. in the
7405 // handling of a bitcast of a vector type to int, which may be done with a
7406 // vector store followed by an integer load. A vector that does not have
7407 // elements that are byte-sized must therefore be stored as an integer
7408 // built out of the extracted vector elements.
7409 if (!MemSclVT.isByteSized()) {
7410 unsigned NumBits = StVT.getSizeInBits();
7411 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
7412
7413 SDValue CurrVal = DAG.getConstant(0, SL, IntVT);
7414
7415 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7416 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
7417 DAG.getVectorIdxConstant(Idx, SL));
7418 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SL, MemSclVT, Elt);
7419 SDValue ExtElt = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Trunc);
7420 unsigned ShiftIntoIdx =
7421 (DAG.getDataLayout().isBigEndian() ? (NumElem - 1) - Idx : Idx);
7422 SDValue ShiftAmount =
7423 DAG.getConstant(ShiftIntoIdx * MemSclVT.getSizeInBits(), SL, IntVT);
7424 SDValue ShiftedElt =
7425 DAG.getNode(ISD::SHL, SL, IntVT, ExtElt, ShiftAmount);
7426 CurrVal = DAG.getNode(ISD::OR, SL, IntVT, CurrVal, ShiftedElt);
7427 }
7428
7429 return DAG.getStore(Chain, SL, CurrVal, BasePtr, ST->getPointerInfo(),
7430 ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
7431 ST->getAAInfo());
7432 }
7433
7434 // Store Stride in bytes
7435 unsigned Stride = MemSclVT.getSizeInBits() / 8;
7436 assert(Stride && "Zero stride!");
7437 // Extract each of the elements from the original vector and save them into
7438 // memory individually.
7439 SmallVector<SDValue, 8> Stores;
7440 for (unsigned Idx = 0; Idx < NumElem; ++Idx) {
7441 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, RegSclVT, Value,
7442 DAG.getVectorIdxConstant(Idx, SL));
7443
7444 SDValue Ptr =
7445 DAG.getObjectPtrOffset(SL, BasePtr, TypeSize::Fixed(Idx * Stride));
7446
7447 // This scalar TruncStore may be illegal, but we legalize it later.
7448 SDValue Store = DAG.getTruncStore(
7449 Chain, SL, Elt, Ptr, ST->getPointerInfo().getWithOffset(Idx * Stride),
7450 MemSclVT, ST->getOriginalAlign(), ST->getMemOperand()->getFlags(),
7451 ST->getAAInfo());
7452
7453 Stores.push_back(Store);
7454 }
7455
7456 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Stores);
7457 }
7458
7459 std::pair<SDValue, SDValue>
expandUnalignedLoad(LoadSDNode * LD,SelectionDAG & DAG) const7460 TargetLowering::expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const {
7461 assert(LD->getAddressingMode() == ISD::UNINDEXED &&
7462 "unaligned indexed loads not implemented!");
7463 SDValue Chain = LD->getChain();
7464 SDValue Ptr = LD->getBasePtr();
7465 EVT VT = LD->getValueType(0);
7466 EVT LoadedVT = LD->getMemoryVT();
7467 SDLoc dl(LD);
7468 auto &MF = DAG.getMachineFunction();
7469
7470 if (VT.isFloatingPoint() || VT.isVector()) {
7471 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), LoadedVT.getSizeInBits());
7472 if (isTypeLegal(intVT) && isTypeLegal(LoadedVT)) {
7473 if (!isOperationLegalOrCustom(ISD::LOAD, intVT) &&
7474 LoadedVT.isVector()) {
7475 // Scalarize the load and let the individual components be handled.
7476 return scalarizeVectorLoad(LD, DAG);
7477 }
7478
7479 // Expand to a (misaligned) integer load of the same size,
7480 // then bitconvert to floating point or vector.
7481 SDValue newLoad = DAG.getLoad(intVT, dl, Chain, Ptr,
7482 LD->getMemOperand());
7483 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad);
7484 if (LoadedVT != VT)
7485 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND :
7486 ISD::ANY_EXTEND, dl, VT, Result);
7487
7488 return std::make_pair(Result, newLoad.getValue(1));
7489 }
7490
7491 // Copy the value to a (aligned) stack slot using (unaligned) integer
7492 // loads and stores, then do a (aligned) load from the stack slot.
7493 MVT RegVT = getRegisterType(*DAG.getContext(), intVT);
7494 unsigned LoadedBytes = LoadedVT.getStoreSize();
7495 unsigned RegBytes = RegVT.getSizeInBits() / 8;
7496 unsigned NumRegs = (LoadedBytes + RegBytes - 1) / RegBytes;
7497
7498 // Make sure the stack slot is also aligned for the register type.
7499 SDValue StackBase = DAG.CreateStackTemporary(LoadedVT, RegVT);
7500 auto FrameIndex = cast<FrameIndexSDNode>(StackBase.getNode())->getIndex();
7501 SmallVector<SDValue, 8> Stores;
7502 SDValue StackPtr = StackBase;
7503 unsigned Offset = 0;
7504
7505 EVT PtrVT = Ptr.getValueType();
7506 EVT StackPtrVT = StackPtr.getValueType();
7507
7508 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
7509 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
7510
7511 // Do all but one copies using the full register width.
7512 for (unsigned i = 1; i < NumRegs; i++) {
7513 // Load one integer register's worth from the original location.
7514 SDValue Load = DAG.getLoad(
7515 RegVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(Offset),
7516 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
7517 LD->getAAInfo());
7518 // Follow the load with a store to the stack slot. Remember the store.
7519 Stores.push_back(DAG.getStore(
7520 Load.getValue(1), dl, Load, StackPtr,
7521 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset)));
7522 // Increment the pointers.
7523 Offset += RegBytes;
7524
7525 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
7526 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
7527 }
7528
7529 // The last copy may be partial. Do an extending load.
7530 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(),
7531 8 * (LoadedBytes - Offset));
7532 SDValue Load =
7533 DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr,
7534 LD->getPointerInfo().getWithOffset(Offset), MemVT,
7535 LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
7536 LD->getAAInfo());
7537 // Follow the load with a store to the stack slot. Remember the store.
7538 // On big-endian machines this requires a truncating store to ensure
7539 // that the bits end up in the right place.
7540 Stores.push_back(DAG.getTruncStore(
7541 Load.getValue(1), dl, Load, StackPtr,
7542 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT));
7543
7544 // The order of the stores doesn't matter - say it with a TokenFactor.
7545 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7546
7547 // Finally, perform the original load only redirected to the stack slot.
7548 Load = DAG.getExtLoad(LD->getExtensionType(), dl, VT, TF, StackBase,
7549 MachinePointerInfo::getFixedStack(MF, FrameIndex, 0),
7550 LoadedVT);
7551
7552 // Callers expect a MERGE_VALUES node.
7553 return std::make_pair(Load, TF);
7554 }
7555
7556 assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
7557 "Unaligned load of unsupported type.");
7558
7559 // Compute the new VT that is half the size of the old one. This is an
7560 // integer MVT.
7561 unsigned NumBits = LoadedVT.getSizeInBits();
7562 EVT NewLoadedVT;
7563 NewLoadedVT = EVT::getIntegerVT(*DAG.getContext(), NumBits/2);
7564 NumBits >>= 1;
7565
7566 Align Alignment = LD->getOriginalAlign();
7567 unsigned IncrementSize = NumBits / 8;
7568 ISD::LoadExtType HiExtType = LD->getExtensionType();
7569
7570 // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
7571 if (HiExtType == ISD::NON_EXTLOAD)
7572 HiExtType = ISD::ZEXTLOAD;
7573
7574 // Load the value in two parts
7575 SDValue Lo, Hi;
7576 if (DAG.getDataLayout().isLittleEndian()) {
7577 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(),
7578 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7579 LD->getAAInfo());
7580
7581 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7582 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr,
7583 LD->getPointerInfo().getWithOffset(IncrementSize),
7584 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7585 LD->getAAInfo());
7586 } else {
7587 Hi = DAG.getExtLoad(HiExtType, dl, VT, Chain, Ptr, LD->getPointerInfo(),
7588 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7589 LD->getAAInfo());
7590
7591 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7592 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr,
7593 LD->getPointerInfo().getWithOffset(IncrementSize),
7594 NewLoadedVT, Alignment, LD->getMemOperand()->getFlags(),
7595 LD->getAAInfo());
7596 }
7597
7598 // aggregate the two parts
7599 SDValue ShiftAmount =
7600 DAG.getConstant(NumBits, dl, getShiftAmountTy(Hi.getValueType(),
7601 DAG.getDataLayout()));
7602 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount);
7603 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo);
7604
7605 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
7606 Hi.getValue(1));
7607
7608 return std::make_pair(Result, TF);
7609 }
7610
expandUnalignedStore(StoreSDNode * ST,SelectionDAG & DAG) const7611 SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST,
7612 SelectionDAG &DAG) const {
7613 assert(ST->getAddressingMode() == ISD::UNINDEXED &&
7614 "unaligned indexed stores not implemented!");
7615 SDValue Chain = ST->getChain();
7616 SDValue Ptr = ST->getBasePtr();
7617 SDValue Val = ST->getValue();
7618 EVT VT = Val.getValueType();
7619 Align Alignment = ST->getOriginalAlign();
7620 auto &MF = DAG.getMachineFunction();
7621 EVT StoreMemVT = ST->getMemoryVT();
7622
7623 SDLoc dl(ST);
7624 if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) {
7625 EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
7626 if (isTypeLegal(intVT)) {
7627 if (!isOperationLegalOrCustom(ISD::STORE, intVT) &&
7628 StoreMemVT.isVector()) {
7629 // Scalarize the store and let the individual components be handled.
7630 SDValue Result = scalarizeVectorStore(ST, DAG);
7631 return Result;
7632 }
7633 // Expand to a bitconvert of the value to the integer type of the
7634 // same size, then a (misaligned) int store.
7635 // FIXME: Does not handle truncating floating point stores!
7636 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val);
7637 Result = DAG.getStore(Chain, dl, Result, Ptr, ST->getPointerInfo(),
7638 Alignment, ST->getMemOperand()->getFlags());
7639 return Result;
7640 }
7641 // Do a (aligned) store to a stack slot, then copy from the stack slot
7642 // to the final destination using (unaligned) integer loads and stores.
7643 MVT RegVT = getRegisterType(
7644 *DAG.getContext(),
7645 EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits()));
7646 EVT PtrVT = Ptr.getValueType();
7647 unsigned StoredBytes = StoreMemVT.getStoreSize();
7648 unsigned RegBytes = RegVT.getSizeInBits() / 8;
7649 unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes;
7650
7651 // Make sure the stack slot is also aligned for the register type.
7652 SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT);
7653 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
7654
7655 // Perform the original store, only redirected to the stack slot.
7656 SDValue Store = DAG.getTruncStore(
7657 Chain, dl, Val, StackPtr,
7658 MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT);
7659
7660 EVT StackPtrVT = StackPtr.getValueType();
7661
7662 SDValue PtrIncrement = DAG.getConstant(RegBytes, dl, PtrVT);
7663 SDValue StackPtrIncrement = DAG.getConstant(RegBytes, dl, StackPtrVT);
7664 SmallVector<SDValue, 8> Stores;
7665 unsigned Offset = 0;
7666
7667 // Do all but one copies using the full register width.
7668 for (unsigned i = 1; i < NumRegs; i++) {
7669 // Load one integer register's worth from the stack slot.
7670 SDValue Load = DAG.getLoad(
7671 RegVT, dl, Store, StackPtr,
7672 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset));
7673 // Store it to the final location. Remember the store.
7674 Stores.push_back(DAG.getStore(Load.getValue(1), dl, Load, Ptr,
7675 ST->getPointerInfo().getWithOffset(Offset),
7676 ST->getOriginalAlign(),
7677 ST->getMemOperand()->getFlags()));
7678 // Increment the pointers.
7679 Offset += RegBytes;
7680 StackPtr = DAG.getObjectPtrOffset(dl, StackPtr, StackPtrIncrement);
7681 Ptr = DAG.getObjectPtrOffset(dl, Ptr, PtrIncrement);
7682 }
7683
7684 // The last store may be partial. Do a truncating store. On big-endian
7685 // machines this requires an extending load from the stack slot to ensure
7686 // that the bits are in the right place.
7687 EVT LoadMemVT =
7688 EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset));
7689
7690 // Load from the stack slot.
7691 SDValue Load = DAG.getExtLoad(
7692 ISD::EXTLOAD, dl, RegVT, Store, StackPtr,
7693 MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT);
7694
7695 Stores.push_back(
7696 DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr,
7697 ST->getPointerInfo().getWithOffset(Offset), LoadMemVT,
7698 ST->getOriginalAlign(),
7699 ST->getMemOperand()->getFlags(), ST->getAAInfo()));
7700 // The order of the stores doesn't matter - say it with a TokenFactor.
7701 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
7702 return Result;
7703 }
7704
7705 assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() &&
7706 "Unaligned store of unknown type.");
7707 // Get the half-size VT
7708 EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext());
7709 unsigned NumBits = NewStoredVT.getFixedSizeInBits();
7710 unsigned IncrementSize = NumBits / 8;
7711
7712 // Divide the stored value in two parts.
7713 SDValue ShiftAmount = DAG.getConstant(
7714 NumBits, dl, getShiftAmountTy(Val.getValueType(), DAG.getDataLayout()));
7715 SDValue Lo = Val;
7716 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount);
7717
7718 // Store the two parts
7719 SDValue Store1, Store2;
7720 Store1 = DAG.getTruncStore(Chain, dl,
7721 DAG.getDataLayout().isLittleEndian() ? Lo : Hi,
7722 Ptr, ST->getPointerInfo(), NewStoredVT, Alignment,
7723 ST->getMemOperand()->getFlags());
7724
7725 Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
7726 Store2 = DAG.getTruncStore(
7727 Chain, dl, DAG.getDataLayout().isLittleEndian() ? Hi : Lo, Ptr,
7728 ST->getPointerInfo().getWithOffset(IncrementSize), NewStoredVT, Alignment,
7729 ST->getMemOperand()->getFlags(), ST->getAAInfo());
7730
7731 SDValue Result =
7732 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
7733 return Result;
7734 }
7735
7736 SDValue
IncrementMemoryAddress(SDValue Addr,SDValue Mask,const SDLoc & DL,EVT DataVT,SelectionDAG & DAG,bool IsCompressedMemory) const7737 TargetLowering::IncrementMemoryAddress(SDValue Addr, SDValue Mask,
7738 const SDLoc &DL, EVT DataVT,
7739 SelectionDAG &DAG,
7740 bool IsCompressedMemory) const {
7741 SDValue Increment;
7742 EVT AddrVT = Addr.getValueType();
7743 EVT MaskVT = Mask.getValueType();
7744 assert(DataVT.getVectorElementCount() == MaskVT.getVectorElementCount() &&
7745 "Incompatible types of Data and Mask");
7746 if (IsCompressedMemory) {
7747 if (DataVT.isScalableVector())
7748 report_fatal_error(
7749 "Cannot currently handle compressed memory with scalable vectors");
7750 // Incrementing the pointer according to number of '1's in the mask.
7751 EVT MaskIntVT = EVT::getIntegerVT(*DAG.getContext(), MaskVT.getSizeInBits());
7752 SDValue MaskInIntReg = DAG.getBitcast(MaskIntVT, Mask);
7753 if (MaskIntVT.getSizeInBits() < 32) {
7754 MaskInIntReg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, MaskInIntReg);
7755 MaskIntVT = MVT::i32;
7756 }
7757
7758 // Count '1's with POPCNT.
7759 Increment = DAG.getNode(ISD::CTPOP, DL, MaskIntVT, MaskInIntReg);
7760 Increment = DAG.getZExtOrTrunc(Increment, DL, AddrVT);
7761 // Scale is an element size in bytes.
7762 SDValue Scale = DAG.getConstant(DataVT.getScalarSizeInBits() / 8, DL,
7763 AddrVT);
7764 Increment = DAG.getNode(ISD::MUL, DL, AddrVT, Increment, Scale);
7765 } else if (DataVT.isScalableVector()) {
7766 Increment = DAG.getVScale(DL, AddrVT,
7767 APInt(AddrVT.getFixedSizeInBits(),
7768 DataVT.getStoreSize().getKnownMinSize()));
7769 } else
7770 Increment = DAG.getConstant(DataVT.getStoreSize(), DL, AddrVT);
7771
7772 return DAG.getNode(ISD::ADD, DL, AddrVT, Addr, Increment);
7773 }
7774
clampDynamicVectorIndex(SelectionDAG & DAG,SDValue Idx,EVT VecVT,const SDLoc & dl)7775 static SDValue clampDynamicVectorIndex(SelectionDAG &DAG,
7776 SDValue Idx,
7777 EVT VecVT,
7778 const SDLoc &dl) {
7779 if (!VecVT.isScalableVector() && isa<ConstantSDNode>(Idx))
7780 return Idx;
7781
7782 EVT IdxVT = Idx.getValueType();
7783 unsigned NElts = VecVT.getVectorMinNumElements();
7784 if (VecVT.isScalableVector()) {
7785 // If this is a constant index and we know the value is less than the
7786 // minimum number of elements then it's safe to return Idx.
7787 if (auto *IdxCst = dyn_cast<ConstantSDNode>(Idx))
7788 if (IdxCst->getZExtValue() < NElts)
7789 return Idx;
7790 SDValue VS =
7791 DAG.getVScale(dl, IdxVT, APInt(IdxVT.getFixedSizeInBits(), NElts));
7792 SDValue Sub =
7793 DAG.getNode(ISD::SUB, dl, IdxVT, VS, DAG.getConstant(1, dl, IdxVT));
7794 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx, Sub);
7795 }
7796 if (isPowerOf2_32(NElts)) {
7797 APInt Imm = APInt::getLowBitsSet(IdxVT.getSizeInBits(), Log2_32(NElts));
7798 return DAG.getNode(ISD::AND, dl, IdxVT, Idx,
7799 DAG.getConstant(Imm, dl, IdxVT));
7800 }
7801 return DAG.getNode(ISD::UMIN, dl, IdxVT, Idx,
7802 DAG.getConstant(NElts - 1, dl, IdxVT));
7803 }
7804
getVectorElementPointer(SelectionDAG & DAG,SDValue VecPtr,EVT VecVT,SDValue Index) const7805 SDValue TargetLowering::getVectorElementPointer(SelectionDAG &DAG,
7806 SDValue VecPtr, EVT VecVT,
7807 SDValue Index) const {
7808 SDLoc dl(Index);
7809 // Make sure the index type is big enough to compute in.
7810 Index = DAG.getZExtOrTrunc(Index, dl, VecPtr.getValueType());
7811
7812 EVT EltVT = VecVT.getVectorElementType();
7813
7814 // Calculate the element offset and add it to the pointer.
7815 unsigned EltSize = EltVT.getFixedSizeInBits() / 8; // FIXME: should be ABI size.
7816 assert(EltSize * 8 == EltVT.getFixedSizeInBits() &&
7817 "Converting bits to bytes lost precision");
7818
7819 Index = clampDynamicVectorIndex(DAG, Index, VecVT, dl);
7820
7821 EVT IdxVT = Index.getValueType();
7822
7823 Index = DAG.getNode(ISD::MUL, dl, IdxVT, Index,
7824 DAG.getConstant(EltSize, dl, IdxVT));
7825 return DAG.getMemBasePlusOffset(VecPtr, Index, dl);
7826 }
7827
7828 //===----------------------------------------------------------------------===//
7829 // Implementation of Emulated TLS Model
7830 //===----------------------------------------------------------------------===//
7831
LowerToTLSEmulatedModel(const GlobalAddressSDNode * GA,SelectionDAG & DAG) const7832 SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA,
7833 SelectionDAG &DAG) const {
7834 // Access to address of TLS varialbe xyz is lowered to a function call:
7835 // __emutls_get_address( address of global variable named "__emutls_v.xyz" )
7836 EVT PtrVT = getPointerTy(DAG.getDataLayout());
7837 PointerType *VoidPtrType = Type::getInt8PtrTy(*DAG.getContext());
7838 SDLoc dl(GA);
7839
7840 ArgListTy Args;
7841 ArgListEntry Entry;
7842 std::string NameString = ("__emutls_v." + GA->getGlobal()->getName()).str();
7843 Module *VariableModule = const_cast<Module*>(GA->getGlobal()->getParent());
7844 StringRef EmuTlsVarName(NameString);
7845 GlobalVariable *EmuTlsVar = VariableModule->getNamedGlobal(EmuTlsVarName);
7846 assert(EmuTlsVar && "Cannot find EmuTlsVar ");
7847 Entry.Node = DAG.getGlobalAddress(EmuTlsVar, dl, PtrVT);
7848 Entry.Ty = VoidPtrType;
7849 Args.push_back(Entry);
7850
7851 SDValue EmuTlsGetAddr = DAG.getExternalSymbol("__emutls_get_address", PtrVT);
7852
7853 TargetLowering::CallLoweringInfo CLI(DAG);
7854 CLI.setDebugLoc(dl).setChain(DAG.getEntryNode());
7855 CLI.setLibCallee(CallingConv::C, VoidPtrType, EmuTlsGetAddr, std::move(Args));
7856 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
7857
7858 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
7859 // At last for X86 targets, maybe good for other targets too?
7860 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7861 MFI.setAdjustsStack(true); // Is this only for X86 target?
7862 MFI.setHasCalls(true);
7863
7864 assert((GA->getOffset() == 0) &&
7865 "Emulated TLS must have zero offset in GlobalAddressSDNode");
7866 return CallResult.first;
7867 }
7868
lowerCmpEqZeroToCtlzSrl(SDValue Op,SelectionDAG & DAG) const7869 SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
7870 SelectionDAG &DAG) const {
7871 assert((Op->getOpcode() == ISD::SETCC) && "Input has to be a SETCC node.");
7872 if (!isCtlzFast())
7873 return SDValue();
7874 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
7875 SDLoc dl(Op);
7876 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
7877 if (C->isNullValue() && CC == ISD::SETEQ) {
7878 EVT VT = Op.getOperand(0).getValueType();
7879 SDValue Zext = Op.getOperand(0);
7880 if (VT.bitsLT(MVT::i32)) {
7881 VT = MVT::i32;
7882 Zext = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Op.getOperand(0));
7883 }
7884 unsigned Log2b = Log2_32(VT.getSizeInBits());
7885 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Zext);
7886 SDValue Scc = DAG.getNode(ISD::SRL, dl, VT, Clz,
7887 DAG.getConstant(Log2b, dl, MVT::i32));
7888 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Scc);
7889 }
7890 }
7891 return SDValue();
7892 }
7893
7894 // Convert redundant addressing modes (e.g. scaling is redundant
7895 // when accessing bytes).
7896 ISD::MemIndexType
getCanonicalIndexType(ISD::MemIndexType IndexType,EVT MemVT,SDValue Offsets) const7897 TargetLowering::getCanonicalIndexType(ISD::MemIndexType IndexType, EVT MemVT,
7898 SDValue Offsets) const {
7899 bool IsScaledIndex =
7900 (IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::UNSIGNED_SCALED);
7901 bool IsSignedIndex =
7902 (IndexType == ISD::SIGNED_SCALED) || (IndexType == ISD::SIGNED_UNSCALED);
7903
7904 // Scaling is unimportant for bytes, canonicalize to unscaled.
7905 if (IsScaledIndex && MemVT.getScalarType() == MVT::i8) {
7906 IsScaledIndex = false;
7907 IndexType = IsSignedIndex ? ISD::SIGNED_UNSCALED : ISD::UNSIGNED_UNSCALED;
7908 }
7909
7910 return IndexType;
7911 }
7912
expandIntMINMAX(SDNode * Node,SelectionDAG & DAG) const7913 SDValue TargetLowering::expandIntMINMAX(SDNode *Node, SelectionDAG &DAG) const {
7914 SDValue Op0 = Node->getOperand(0);
7915 SDValue Op1 = Node->getOperand(1);
7916 EVT VT = Op0.getValueType();
7917 unsigned Opcode = Node->getOpcode();
7918 SDLoc DL(Node);
7919
7920 // umin(x,y) -> sub(x,usubsat(x,y))
7921 if (Opcode == ISD::UMIN && isOperationLegal(ISD::SUB, VT) &&
7922 isOperationLegal(ISD::USUBSAT, VT)) {
7923 return DAG.getNode(ISD::SUB, DL, VT, Op0,
7924 DAG.getNode(ISD::USUBSAT, DL, VT, Op0, Op1));
7925 }
7926
7927 // umax(x,y) -> add(x,usubsat(y,x))
7928 if (Opcode == ISD::UMAX && isOperationLegal(ISD::ADD, VT) &&
7929 isOperationLegal(ISD::USUBSAT, VT)) {
7930 return DAG.getNode(ISD::ADD, DL, VT, Op0,
7931 DAG.getNode(ISD::USUBSAT, DL, VT, Op1, Op0));
7932 }
7933
7934 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
7935 ISD::CondCode CC;
7936 switch (Opcode) {
7937 default: llvm_unreachable("How did we get here?");
7938 case ISD::SMAX: CC = ISD::SETGT; break;
7939 case ISD::SMIN: CC = ISD::SETLT; break;
7940 case ISD::UMAX: CC = ISD::SETUGT; break;
7941 case ISD::UMIN: CC = ISD::SETULT; break;
7942 }
7943
7944 // FIXME: Should really try to split the vector in case it's legal on a
7945 // subvector.
7946 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
7947 return DAG.UnrollVectorOp(Node);
7948
7949 SDValue Cond = DAG.getSetCC(DL, VT, Op0, Op1, CC);
7950 return DAG.getSelect(DL, VT, Cond, Op0, Op1);
7951 }
7952
expandAddSubSat(SDNode * Node,SelectionDAG & DAG) const7953 SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
7954 unsigned Opcode = Node->getOpcode();
7955 SDValue LHS = Node->getOperand(0);
7956 SDValue RHS = Node->getOperand(1);
7957 EVT VT = LHS.getValueType();
7958 SDLoc dl(Node);
7959
7960 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
7961 assert(VT.isInteger() && "Expected operands to be integers");
7962
7963 // usub.sat(a, b) -> umax(a, b) - b
7964 if (Opcode == ISD::USUBSAT && isOperationLegal(ISD::UMAX, VT)) {
7965 SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
7966 return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
7967 }
7968
7969 // uadd.sat(a, b) -> umin(a, ~b) + b
7970 if (Opcode == ISD::UADDSAT && isOperationLegal(ISD::UMIN, VT)) {
7971 SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
7972 SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
7973 return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
7974 }
7975
7976 unsigned OverflowOp;
7977 switch (Opcode) {
7978 case ISD::SADDSAT:
7979 OverflowOp = ISD::SADDO;
7980 break;
7981 case ISD::UADDSAT:
7982 OverflowOp = ISD::UADDO;
7983 break;
7984 case ISD::SSUBSAT:
7985 OverflowOp = ISD::SSUBO;
7986 break;
7987 case ISD::USUBSAT:
7988 OverflowOp = ISD::USUBO;
7989 break;
7990 default:
7991 llvm_unreachable("Expected method to receive signed or unsigned saturation "
7992 "addition or subtraction node.");
7993 }
7994
7995 // FIXME: Should really try to split the vector in case it's legal on a
7996 // subvector.
7997 if (VT.isVector() && !isOperationLegalOrCustom(ISD::VSELECT, VT))
7998 return DAG.UnrollVectorOp(Node);
7999
8000 unsigned BitWidth = LHS.getScalarValueSizeInBits();
8001 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8002 SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8003 SDValue SumDiff = Result.getValue(0);
8004 SDValue Overflow = Result.getValue(1);
8005 SDValue Zero = DAG.getConstant(0, dl, VT);
8006 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
8007
8008 if (Opcode == ISD::UADDSAT) {
8009 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
8010 // (LHS + RHS) | OverflowMask
8011 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
8012 return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask);
8013 }
8014 // Overflow ? 0xffff.... : (LHS + RHS)
8015 return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff);
8016 }
8017
8018 if (Opcode == ISD::USUBSAT) {
8019 if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) {
8020 // (LHS - RHS) & ~OverflowMask
8021 SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT);
8022 SDValue Not = DAG.getNOT(dl, OverflowMask, VT);
8023 return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not);
8024 }
8025 // Overflow ? 0 : (LHS - RHS)
8026 return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff);
8027 }
8028
8029 // SatMax -> Overflow && SumDiff < 0
8030 // SatMin -> Overflow && SumDiff >= 0
8031 APInt MinVal = APInt::getSignedMinValue(BitWidth);
8032 APInt MaxVal = APInt::getSignedMaxValue(BitWidth);
8033 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
8034 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
8035 SDValue SumNeg = DAG.getSetCC(dl, BoolVT, SumDiff, Zero, ISD::SETLT);
8036 Result = DAG.getSelect(dl, VT, SumNeg, SatMax, SatMin);
8037 return DAG.getSelect(dl, VT, Overflow, Result, SumDiff);
8038 }
8039
expandShlSat(SDNode * Node,SelectionDAG & DAG) const8040 SDValue TargetLowering::expandShlSat(SDNode *Node, SelectionDAG &DAG) const {
8041 unsigned Opcode = Node->getOpcode();
8042 bool IsSigned = Opcode == ISD::SSHLSAT;
8043 SDValue LHS = Node->getOperand(0);
8044 SDValue RHS = Node->getOperand(1);
8045 EVT VT = LHS.getValueType();
8046 SDLoc dl(Node);
8047
8048 assert((Node->getOpcode() == ISD::SSHLSAT ||
8049 Node->getOpcode() == ISD::USHLSAT) &&
8050 "Expected a SHLSAT opcode");
8051 assert(VT == RHS.getValueType() && "Expected operands to be the same type");
8052 assert(VT.isInteger() && "Expected operands to be integers");
8053
8054 // If LHS != (LHS << RHS) >> RHS, we have overflow and must saturate.
8055
8056 unsigned BW = VT.getScalarSizeInBits();
8057 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, LHS, RHS);
8058 SDValue Orig =
8059 DAG.getNode(IsSigned ? ISD::SRA : ISD::SRL, dl, VT, Result, RHS);
8060
8061 SDValue SatVal;
8062 if (IsSigned) {
8063 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(BW), dl, VT);
8064 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(BW), dl, VT);
8065 SatVal = DAG.getSelectCC(dl, LHS, DAG.getConstant(0, dl, VT),
8066 SatMin, SatMax, ISD::SETLT);
8067 } else {
8068 SatVal = DAG.getConstant(APInt::getMaxValue(BW), dl, VT);
8069 }
8070 Result = DAG.getSelectCC(dl, LHS, Orig, SatVal, Result, ISD::SETNE);
8071
8072 return Result;
8073 }
8074
8075 SDValue
expandFixedPointMul(SDNode * Node,SelectionDAG & DAG) const8076 TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
8077 assert((Node->getOpcode() == ISD::SMULFIX ||
8078 Node->getOpcode() == ISD::UMULFIX ||
8079 Node->getOpcode() == ISD::SMULFIXSAT ||
8080 Node->getOpcode() == ISD::UMULFIXSAT) &&
8081 "Expected a fixed point multiplication opcode");
8082
8083 SDLoc dl(Node);
8084 SDValue LHS = Node->getOperand(0);
8085 SDValue RHS = Node->getOperand(1);
8086 EVT VT = LHS.getValueType();
8087 unsigned Scale = Node->getConstantOperandVal(2);
8088 bool Saturating = (Node->getOpcode() == ISD::SMULFIXSAT ||
8089 Node->getOpcode() == ISD::UMULFIXSAT);
8090 bool Signed = (Node->getOpcode() == ISD::SMULFIX ||
8091 Node->getOpcode() == ISD::SMULFIXSAT);
8092 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8093 unsigned VTSize = VT.getScalarSizeInBits();
8094
8095 if (!Scale) {
8096 // [us]mul.fix(a, b, 0) -> mul(a, b)
8097 if (!Saturating) {
8098 if (isOperationLegalOrCustom(ISD::MUL, VT))
8099 return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8100 } else if (Signed && isOperationLegalOrCustom(ISD::SMULO, VT)) {
8101 SDValue Result =
8102 DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8103 SDValue Product = Result.getValue(0);
8104 SDValue Overflow = Result.getValue(1);
8105 SDValue Zero = DAG.getConstant(0, dl, VT);
8106
8107 APInt MinVal = APInt::getSignedMinValue(VTSize);
8108 APInt MaxVal = APInt::getSignedMaxValue(VTSize);
8109 SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
8110 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
8111 SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
8112 Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
8113 return DAG.getSelect(dl, VT, Overflow, Result, Product);
8114 } else if (!Signed && isOperationLegalOrCustom(ISD::UMULO, VT)) {
8115 SDValue Result =
8116 DAG.getNode(ISD::UMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
8117 SDValue Product = Result.getValue(0);
8118 SDValue Overflow = Result.getValue(1);
8119
8120 APInt MaxVal = APInt::getMaxValue(VTSize);
8121 SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
8122 return DAG.getSelect(dl, VT, Overflow, SatMax, Product);
8123 }
8124 }
8125
8126 assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
8127 "Expected scale to be less than the number of bits if signed or at "
8128 "most the number of bits if unsigned.");
8129 assert(LHS.getValueType() == RHS.getValueType() &&
8130 "Expected both operands to be the same type");
8131
8132 // Get the upper and lower bits of the result.
8133 SDValue Lo, Hi;
8134 unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
8135 unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
8136 if (isOperationLegalOrCustom(LoHiOp, VT)) {
8137 SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
8138 Lo = Result.getValue(0);
8139 Hi = Result.getValue(1);
8140 } else if (isOperationLegalOrCustom(HiOp, VT)) {
8141 Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8142 Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
8143 } else if (VT.isVector()) {
8144 return SDValue();
8145 } else {
8146 report_fatal_error("Unable to expand fixed point multiplication.");
8147 }
8148
8149 if (Scale == VTSize)
8150 // Result is just the top half since we'd be shifting by the width of the
8151 // operand. Overflow impossible so this works for both UMULFIX and
8152 // UMULFIXSAT.
8153 return Hi;
8154
8155 // The result will need to be shifted right by the scale since both operands
8156 // are scaled. The result is given to us in 2 halves, so we only want part of
8157 // both in the result.
8158 EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
8159 SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
8160 DAG.getConstant(Scale, dl, ShiftTy));
8161 if (!Saturating)
8162 return Result;
8163
8164 if (!Signed) {
8165 // Unsigned overflow happened if the upper (VTSize - Scale) bits (of the
8166 // widened multiplication) aren't all zeroes.
8167
8168 // Saturate to max if ((Hi >> Scale) != 0),
8169 // which is the same as if (Hi > ((1 << Scale) - 1))
8170 APInt MaxVal = APInt::getMaxValue(VTSize);
8171 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
8172 dl, VT);
8173 Result = DAG.getSelectCC(dl, Hi, LowMask,
8174 DAG.getConstant(MaxVal, dl, VT), Result,
8175 ISD::SETUGT);
8176
8177 return Result;
8178 }
8179
8180 // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
8181 // widened multiplication) aren't all ones or all zeroes.
8182
8183 SDValue SatMin = DAG.getConstant(APInt::getSignedMinValue(VTSize), dl, VT);
8184 SDValue SatMax = DAG.getConstant(APInt::getSignedMaxValue(VTSize), dl, VT);
8185
8186 if (Scale == 0) {
8187 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, Lo,
8188 DAG.getConstant(VTSize - 1, dl, ShiftTy));
8189 SDValue Overflow = DAG.getSetCC(dl, BoolVT, Hi, Sign, ISD::SETNE);
8190 // Saturated to SatMin if wide product is negative, and SatMax if wide
8191 // product is positive ...
8192 SDValue Zero = DAG.getConstant(0, dl, VT);
8193 SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
8194 ISD::SETLT);
8195 // ... but only if we overflowed.
8196 return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
8197 }
8198
8199 // We handled Scale==0 above so all the bits to examine is in Hi.
8200
8201 // Saturate to max if ((Hi >> (Scale - 1)) > 0),
8202 // which is the same as if (Hi > (1 << (Scale - 1)) - 1)
8203 SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
8204 dl, VT);
8205 Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);
8206 // Saturate to min if (Hi >> (Scale - 1)) < -1),
8207 // which is the same as if (HI < (-1 << (Scale - 1))
8208 SDValue HighMask =
8209 DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),
8210 dl, VT);
8211 Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);
8212 return Result;
8213 }
8214
8215 SDValue
expandFixedPointDiv(unsigned Opcode,const SDLoc & dl,SDValue LHS,SDValue RHS,unsigned Scale,SelectionDAG & DAG) const8216 TargetLowering::expandFixedPointDiv(unsigned Opcode, const SDLoc &dl,
8217 SDValue LHS, SDValue RHS,
8218 unsigned Scale, SelectionDAG &DAG) const {
8219 assert((Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT ||
8220 Opcode == ISD::UDIVFIX || Opcode == ISD::UDIVFIXSAT) &&
8221 "Expected a fixed point division opcode");
8222
8223 EVT VT = LHS.getValueType();
8224 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT;
8225 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT;
8226 EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8227
8228 // If there is enough room in the type to upscale the LHS or downscale the
8229 // RHS before the division, we can perform it in this type without having to
8230 // resize. For signed operations, the LHS headroom is the number of
8231 // redundant sign bits, and for unsigned ones it is the number of zeroes.
8232 // The headroom for the RHS is the number of trailing zeroes.
8233 unsigned LHSLead = Signed ? DAG.ComputeNumSignBits(LHS) - 1
8234 : DAG.computeKnownBits(LHS).countMinLeadingZeros();
8235 unsigned RHSTrail = DAG.computeKnownBits(RHS).countMinTrailingZeros();
8236
8237 // For signed saturating operations, we need to be able to detect true integer
8238 // division overflow; that is, when you have MIN / -EPS. However, this
8239 // is undefined behavior and if we emit divisions that could take such
8240 // values it may cause undesired behavior (arithmetic exceptions on x86, for
8241 // example).
8242 // Avoid this by requiring an extra bit so that we never get this case.
8243 // FIXME: This is a bit unfortunate as it means that for an 8-bit 7-scale
8244 // signed saturating division, we need to emit a whopping 32-bit division.
8245 if (LHSLead + RHSTrail < Scale + (unsigned)(Saturating && Signed))
8246 return SDValue();
8247
8248 unsigned LHSShift = std::min(LHSLead, Scale);
8249 unsigned RHSShift = Scale - LHSShift;
8250
8251 // At this point, we know that if we shift the LHS up by LHSShift and the
8252 // RHS down by RHSShift, we can emit a regular division with a final scaling
8253 // factor of Scale.
8254
8255 EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
8256 if (LHSShift)
8257 LHS = DAG.getNode(ISD::SHL, dl, VT, LHS,
8258 DAG.getConstant(LHSShift, dl, ShiftTy));
8259 if (RHSShift)
8260 RHS = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, VT, RHS,
8261 DAG.getConstant(RHSShift, dl, ShiftTy));
8262
8263 SDValue Quot;
8264 if (Signed) {
8265 // For signed operations, if the resulting quotient is negative and the
8266 // remainder is nonzero, subtract 1 from the quotient to round towards
8267 // negative infinity.
8268 SDValue Rem;
8269 // FIXME: Ideally we would always produce an SDIVREM here, but if the
8270 // type isn't legal, SDIVREM cannot be expanded. There is no reason why
8271 // we couldn't just form a libcall, but the type legalizer doesn't do it.
8272 if (isTypeLegal(VT) &&
8273 isOperationLegalOrCustom(ISD::SDIVREM, VT)) {
8274 Quot = DAG.getNode(ISD::SDIVREM, dl,
8275 DAG.getVTList(VT, VT),
8276 LHS, RHS);
8277 Rem = Quot.getValue(1);
8278 Quot = Quot.getValue(0);
8279 } else {
8280 Quot = DAG.getNode(ISD::SDIV, dl, VT,
8281 LHS, RHS);
8282 Rem = DAG.getNode(ISD::SREM, dl, VT,
8283 LHS, RHS);
8284 }
8285 SDValue Zero = DAG.getConstant(0, dl, VT);
8286 SDValue RemNonZero = DAG.getSetCC(dl, BoolVT, Rem, Zero, ISD::SETNE);
8287 SDValue LHSNeg = DAG.getSetCC(dl, BoolVT, LHS, Zero, ISD::SETLT);
8288 SDValue RHSNeg = DAG.getSetCC(dl, BoolVT, RHS, Zero, ISD::SETLT);
8289 SDValue QuotNeg = DAG.getNode(ISD::XOR, dl, BoolVT, LHSNeg, RHSNeg);
8290 SDValue Sub1 = DAG.getNode(ISD::SUB, dl, VT, Quot,
8291 DAG.getConstant(1, dl, VT));
8292 Quot = DAG.getSelect(dl, VT,
8293 DAG.getNode(ISD::AND, dl, BoolVT, RemNonZero, QuotNeg),
8294 Sub1, Quot);
8295 } else
8296 Quot = DAG.getNode(ISD::UDIV, dl, VT,
8297 LHS, RHS);
8298
8299 return Quot;
8300 }
8301
expandUADDSUBO(SDNode * Node,SDValue & Result,SDValue & Overflow,SelectionDAG & DAG) const8302 void TargetLowering::expandUADDSUBO(
8303 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
8304 SDLoc dl(Node);
8305 SDValue LHS = Node->getOperand(0);
8306 SDValue RHS = Node->getOperand(1);
8307 bool IsAdd = Node->getOpcode() == ISD::UADDO;
8308
8309 // If ADD/SUBCARRY is legal, use that instead.
8310 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
8311 if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
8312 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
8313 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
8314 { LHS, RHS, CarryIn });
8315 Result = SDValue(NodeCarry.getNode(), 0);
8316 Overflow = SDValue(NodeCarry.getNode(), 1);
8317 return;
8318 }
8319
8320 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
8321 LHS.getValueType(), LHS, RHS);
8322
8323 EVT ResultType = Node->getValueType(1);
8324 EVT SetCCType = getSetCCResultType(
8325 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
8326 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
8327 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC);
8328 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
8329 }
8330
expandSADDSUBO(SDNode * Node,SDValue & Result,SDValue & Overflow,SelectionDAG & DAG) const8331 void TargetLowering::expandSADDSUBO(
8332 SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const {
8333 SDLoc dl(Node);
8334 SDValue LHS = Node->getOperand(0);
8335 SDValue RHS = Node->getOperand(1);
8336 bool IsAdd = Node->getOpcode() == ISD::SADDO;
8337
8338 Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
8339 LHS.getValueType(), LHS, RHS);
8340
8341 EVT ResultType = Node->getValueType(1);
8342 EVT OType = getSetCCResultType(
8343 DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0));
8344
8345 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
8346 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
8347 if (isOperationLegalOrCustom(OpcSat, LHS.getValueType())) {
8348 SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
8349 SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE);
8350 Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType);
8351 return;
8352 }
8353
8354 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
8355
8356 // For an addition, the result should be less than one of the operands (LHS)
8357 // if and only if the other operand (RHS) is negative, otherwise there will
8358 // be overflow.
8359 // For a subtraction, the result should be less than one of the operands
8360 // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
8361 // otherwise there will be overflow.
8362 SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
8363 SDValue ConditionRHS =
8364 DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);
8365
8366 Overflow = DAG.getBoolExtOrTrunc(
8367 DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,
8368 ResultType, ResultType);
8369 }
8370
expandMULO(SDNode * Node,SDValue & Result,SDValue & Overflow,SelectionDAG & DAG) const8371 bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
8372 SDValue &Overflow, SelectionDAG &DAG) const {
8373 SDLoc dl(Node);
8374 EVT VT = Node->getValueType(0);
8375 EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8376 SDValue LHS = Node->getOperand(0);
8377 SDValue RHS = Node->getOperand(1);
8378 bool isSigned = Node->getOpcode() == ISD::SMULO;
8379
8380 // For power-of-two multiplications we can use a simpler shift expansion.
8381 if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) {
8382 const APInt &C = RHSC->getAPIntValue();
8383 // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X }
8384 if (C.isPowerOf2()) {
8385 // smulo(x, signed_min) is same as umulo(x, signed_min).
8386 bool UseArithShift = isSigned && !C.isMinSignedValue();
8387 EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout());
8388 SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy);
8389 Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt);
8390 Overflow = DAG.getSetCC(dl, SetCCVT,
8391 DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL,
8392 dl, VT, Result, ShiftAmt),
8393 LHS, ISD::SETNE);
8394 return true;
8395 }
8396 }
8397
8398 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2);
8399 if (VT.isVector())
8400 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
8401 VT.getVectorNumElements());
8402
8403 SDValue BottomHalf;
8404 SDValue TopHalf;
8405 static const unsigned Ops[2][3] =
8406 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND },
8407 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }};
8408 if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) {
8409 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
8410 TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS);
8411 } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) {
8412 BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS,
8413 RHS);
8414 TopHalf = BottomHalf.getValue(1);
8415 } else if (isTypeLegal(WideVT)) {
8416 LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS);
8417 RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS);
8418 SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS);
8419 BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
8420 SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl,
8421 getShiftAmountTy(WideVT, DAG.getDataLayout()));
8422 TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT,
8423 DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt));
8424 } else {
8425 if (VT.isVector())
8426 return false;
8427
8428 // We can fall back to a libcall with an illegal type for the MUL if we
8429 // have a libcall big enough.
8430 // Also, we can fall back to a division in some cases, but that's a big
8431 // performance hit in the general case.
8432 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
8433 if (WideVT == MVT::i16)
8434 LC = RTLIB::MUL_I16;
8435 else if (WideVT == MVT::i32)
8436 LC = RTLIB::MUL_I32;
8437 else if (WideVT == MVT::i64)
8438 LC = RTLIB::MUL_I64;
8439 else if (WideVT == MVT::i128)
8440 LC = RTLIB::MUL_I128;
8441 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!");
8442
8443 SDValue HiLHS;
8444 SDValue HiRHS;
8445 if (isSigned) {
8446 // The high part is obtained by SRA'ing all but one of the bits of low
8447 // part.
8448 unsigned LoSize = VT.getFixedSizeInBits();
8449 HiLHS =
8450 DAG.getNode(ISD::SRA, dl, VT, LHS,
8451 DAG.getConstant(LoSize - 1, dl,
8452 getPointerTy(DAG.getDataLayout())));
8453 HiRHS =
8454 DAG.getNode(ISD::SRA, dl, VT, RHS,
8455 DAG.getConstant(LoSize - 1, dl,
8456 getPointerTy(DAG.getDataLayout())));
8457 } else {
8458 HiLHS = DAG.getConstant(0, dl, VT);
8459 HiRHS = DAG.getConstant(0, dl, VT);
8460 }
8461
8462 // Here we're passing the 2 arguments explicitly as 4 arguments that are
8463 // pre-lowered to the correct types. This all depends upon WideVT not
8464 // being a legal type for the architecture and thus has to be split to
8465 // two arguments.
8466 SDValue Ret;
8467 TargetLowering::MakeLibCallOptions CallOptions;
8468 CallOptions.setSExt(isSigned);
8469 CallOptions.setIsPostTypeLegalization(true);
8470 if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) {
8471 // Halves of WideVT are packed into registers in different order
8472 // depending on platform endianness. This is usually handled by
8473 // the C calling convention, but we can't defer to it in
8474 // the legalizer.
8475 SDValue Args[] = { LHS, HiLHS, RHS, HiRHS };
8476 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
8477 } else {
8478 SDValue Args[] = { HiLHS, LHS, HiRHS, RHS };
8479 Ret = makeLibCall(DAG, LC, WideVT, Args, CallOptions, dl).first;
8480 }
8481 assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
8482 "Ret value is a collection of constituent nodes holding result.");
8483 if (DAG.getDataLayout().isLittleEndian()) {
8484 // Same as above.
8485 BottomHalf = Ret.getOperand(0);
8486 TopHalf = Ret.getOperand(1);
8487 } else {
8488 BottomHalf = Ret.getOperand(1);
8489 TopHalf = Ret.getOperand(0);
8490 }
8491 }
8492
8493 Result = BottomHalf;
8494 if (isSigned) {
8495 SDValue ShiftAmt = DAG.getConstant(
8496 VT.getScalarSizeInBits() - 1, dl,
8497 getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout()));
8498 SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt);
8499 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE);
8500 } else {
8501 Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf,
8502 DAG.getConstant(0, dl, VT), ISD::SETNE);
8503 }
8504
8505 // Truncate the result if SetCC returns a larger type than needed.
8506 EVT RType = Node->getValueType(1);
8507 if (RType.bitsLT(Overflow.getValueType()))
8508 Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow);
8509
8510 assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() &&
8511 "Unexpected result type for S/UMULO legalization");
8512 return true;
8513 }
8514
expandVecReduce(SDNode * Node,SelectionDAG & DAG) const8515 SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
8516 SDLoc dl(Node);
8517 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
8518 SDValue Op = Node->getOperand(0);
8519 EVT VT = Op.getValueType();
8520
8521 if (VT.isScalableVector())
8522 report_fatal_error(
8523 "Expanding reductions for scalable vectors is undefined.");
8524
8525 // Try to use a shuffle reduction for power of two vectors.
8526 if (VT.isPow2VectorType()) {
8527 while (VT.getVectorNumElements() > 1) {
8528 EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
8529 if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))
8530 break;
8531
8532 SDValue Lo, Hi;
8533 std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
8534 Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi);
8535 VT = HalfVT;
8536 }
8537 }
8538
8539 EVT EltVT = VT.getVectorElementType();
8540 unsigned NumElts = VT.getVectorNumElements();
8541
8542 SmallVector<SDValue, 8> Ops;
8543 DAG.ExtractVectorElements(Op, Ops, 0, NumElts);
8544
8545 SDValue Res = Ops[0];
8546 for (unsigned i = 1; i < NumElts; i++)
8547 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags());
8548
8549 // Result type may be wider than element type.
8550 if (EltVT != Node->getValueType(0))
8551 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res);
8552 return Res;
8553 }
8554
expandVecReduceSeq(SDNode * Node,SelectionDAG & DAG) const8555 SDValue TargetLowering::expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const {
8556 SDLoc dl(Node);
8557 SDValue AccOp = Node->getOperand(0);
8558 SDValue VecOp = Node->getOperand(1);
8559 SDNodeFlags Flags = Node->getFlags();
8560
8561 EVT VT = VecOp.getValueType();
8562 EVT EltVT = VT.getVectorElementType();
8563
8564 if (VT.isScalableVector())
8565 report_fatal_error(
8566 "Expanding reductions for scalable vectors is undefined.");
8567
8568 unsigned NumElts = VT.getVectorNumElements();
8569
8570 SmallVector<SDValue, 8> Ops;
8571 DAG.ExtractVectorElements(VecOp, Ops, 0, NumElts);
8572
8573 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Node->getOpcode());
8574
8575 SDValue Res = AccOp;
8576 for (unsigned i = 0; i < NumElts; i++)
8577 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Flags);
8578
8579 return Res;
8580 }
8581
expandREM(SDNode * Node,SDValue & Result,SelectionDAG & DAG) const8582 bool TargetLowering::expandREM(SDNode *Node, SDValue &Result,
8583 SelectionDAG &DAG) const {
8584 EVT VT = Node->getValueType(0);
8585 SDLoc dl(Node);
8586 bool isSigned = Node->getOpcode() == ISD::SREM;
8587 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
8588 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
8589 SDValue Dividend = Node->getOperand(0);
8590 SDValue Divisor = Node->getOperand(1);
8591 if (isOperationLegalOrCustom(DivRemOpc, VT)) {
8592 SDVTList VTs = DAG.getVTList(VT, VT);
8593 Result = DAG.getNode(DivRemOpc, dl, VTs, Dividend, Divisor).getValue(1);
8594 return true;
8595 }
8596 if (isOperationLegalOrCustom(DivOpc, VT)) {
8597 // X % Y -> X-X/Y*Y
8598 SDValue Divide = DAG.getNode(DivOpc, dl, VT, Dividend, Divisor);
8599 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Divide, Divisor);
8600 Result = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
8601 return true;
8602 }
8603 return false;
8604 }
8605
expandFP_TO_INT_SAT(SDNode * Node,SelectionDAG & DAG) const8606 SDValue TargetLowering::expandFP_TO_INT_SAT(SDNode *Node,
8607 SelectionDAG &DAG) const {
8608 bool IsSigned = Node->getOpcode() == ISD::FP_TO_SINT_SAT;
8609 SDLoc dl(SDValue(Node, 0));
8610 SDValue Src = Node->getOperand(0);
8611
8612 // DstVT is the result type, while SatVT is the size to which we saturate
8613 EVT SrcVT = Src.getValueType();
8614 EVT DstVT = Node->getValueType(0);
8615
8616 EVT SatVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
8617 unsigned SatWidth = SatVT.getScalarSizeInBits();
8618 unsigned DstWidth = DstVT.getScalarSizeInBits();
8619 assert(SatWidth <= DstWidth &&
8620 "Expected saturation width smaller than result width");
8621
8622 // Determine minimum and maximum integer values and their corresponding
8623 // floating-point values.
8624 APInt MinInt, MaxInt;
8625 if (IsSigned) {
8626 MinInt = APInt::getSignedMinValue(SatWidth).sextOrSelf(DstWidth);
8627 MaxInt = APInt::getSignedMaxValue(SatWidth).sextOrSelf(DstWidth);
8628 } else {
8629 MinInt = APInt::getMinValue(SatWidth).zextOrSelf(DstWidth);
8630 MaxInt = APInt::getMaxValue(SatWidth).zextOrSelf(DstWidth);
8631 }
8632
8633 // We cannot risk emitting FP_TO_XINT nodes with a source VT of f16, as
8634 // libcall emission cannot handle this. Large result types will fail.
8635 if (SrcVT == MVT::f16) {
8636 Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Src);
8637 SrcVT = Src.getValueType();
8638 }
8639
8640 APFloat MinFloat(DAG.EVTToAPFloatSemantics(SrcVT));
8641 APFloat MaxFloat(DAG.EVTToAPFloatSemantics(SrcVT));
8642
8643 APFloat::opStatus MinStatus =
8644 MinFloat.convertFromAPInt(MinInt, IsSigned, APFloat::rmTowardZero);
8645 APFloat::opStatus MaxStatus =
8646 MaxFloat.convertFromAPInt(MaxInt, IsSigned, APFloat::rmTowardZero);
8647 bool AreExactFloatBounds = !(MinStatus & APFloat::opStatus::opInexact) &&
8648 !(MaxStatus & APFloat::opStatus::opInexact);
8649
8650 SDValue MinFloatNode = DAG.getConstantFP(MinFloat, dl, SrcVT);
8651 SDValue MaxFloatNode = DAG.getConstantFP(MaxFloat, dl, SrcVT);
8652
8653 // If the integer bounds are exactly representable as floats and min/max are
8654 // legal, emit a min+max+fptoi sequence. Otherwise we have to use a sequence
8655 // of comparisons and selects.
8656 bool MinMaxLegal = isOperationLegal(ISD::FMINNUM, SrcVT) &&
8657 isOperationLegal(ISD::FMAXNUM, SrcVT);
8658 if (AreExactFloatBounds && MinMaxLegal) {
8659 SDValue Clamped = Src;
8660
8661 // Clamp Src by MinFloat from below. If Src is NaN the result is MinFloat.
8662 Clamped = DAG.getNode(ISD::FMAXNUM, dl, SrcVT, Clamped, MinFloatNode);
8663 // Clamp by MaxFloat from above. NaN cannot occur.
8664 Clamped = DAG.getNode(ISD::FMINNUM, dl, SrcVT, Clamped, MaxFloatNode);
8665 // Convert clamped value to integer.
8666 SDValue FpToInt = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT,
8667 dl, DstVT, Clamped);
8668
8669 // In the unsigned case we're done, because we mapped NaN to MinFloat,
8670 // which will cast to zero.
8671 if (!IsSigned)
8672 return FpToInt;
8673
8674 // Otherwise, select 0 if Src is NaN.
8675 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
8676 return DAG.getSelectCC(dl, Src, Src, ZeroInt, FpToInt,
8677 ISD::CondCode::SETUO);
8678 }
8679
8680 SDValue MinIntNode = DAG.getConstant(MinInt, dl, DstVT);
8681 SDValue MaxIntNode = DAG.getConstant(MaxInt, dl, DstVT);
8682
8683 // Result of direct conversion. The assumption here is that the operation is
8684 // non-trapping and it's fine to apply it to an out-of-range value if we
8685 // select it away later.
8686 SDValue FpToInt =
8687 DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, DstVT, Src);
8688
8689 SDValue Select = FpToInt;
8690
8691 // If Src ULT MinFloat, select MinInt. In particular, this also selects
8692 // MinInt if Src is NaN.
8693 Select = DAG.getSelectCC(dl, Src, MinFloatNode, MinIntNode, Select,
8694 ISD::CondCode::SETULT);
8695 // If Src OGT MaxFloat, select MaxInt.
8696 Select = DAG.getSelectCC(dl, Src, MaxFloatNode, MaxIntNode, Select,
8697 ISD::CondCode::SETOGT);
8698
8699 // In the unsigned case we are done, because we mapped NaN to MinInt, which
8700 // is already zero.
8701 if (!IsSigned)
8702 return Select;
8703
8704 // Otherwise, select 0 if Src is NaN.
8705 SDValue ZeroInt = DAG.getConstant(0, dl, DstVT);
8706 return DAG.getSelectCC(dl, Src, Src, ZeroInt, Select, ISD::CondCode::SETUO);
8707 }
8708
expandVectorSplice(SDNode * Node,SelectionDAG & DAG) const8709 SDValue TargetLowering::expandVectorSplice(SDNode *Node,
8710 SelectionDAG &DAG) const {
8711 assert(Node->getOpcode() == ISD::VECTOR_SPLICE && "Unexpected opcode!");
8712 assert(Node->getValueType(0).isScalableVector() &&
8713 "Fixed length vector types expected to use SHUFFLE_VECTOR!");
8714
8715 EVT VT = Node->getValueType(0);
8716 SDValue V1 = Node->getOperand(0);
8717 SDValue V2 = Node->getOperand(1);
8718 int64_t Imm = cast<ConstantSDNode>(Node->getOperand(2))->getSExtValue();
8719 SDLoc DL(Node);
8720
8721 // Expand through memory thusly:
8722 // Alloca CONCAT_VECTORS_TYPES(V1, V2) Ptr
8723 // Store V1, Ptr
8724 // Store V2, Ptr + sizeof(V1)
8725 // If (Imm < 0)
8726 // TrailingElts = -Imm
8727 // Ptr = Ptr + sizeof(V1) - (TrailingElts * sizeof(VT.Elt))
8728 // else
8729 // Ptr = Ptr + (Imm * sizeof(VT.Elt))
8730 // Res = Load Ptr
8731
8732 Align Alignment = DAG.getReducedAlign(VT, /*UseABI=*/false);
8733
8734 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
8735 VT.getVectorElementCount() * 2);
8736 SDValue StackPtr = DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
8737 EVT PtrVT = StackPtr.getValueType();
8738 auto &MF = DAG.getMachineFunction();
8739 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
8740 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
8741
8742 // Store the lo part of CONCAT_VECTORS(V1, V2)
8743 SDValue StoreV1 = DAG.getStore(DAG.getEntryNode(), DL, V1, StackPtr, PtrInfo);
8744 // Store the hi part of CONCAT_VECTORS(V1, V2)
8745 SDValue OffsetToV2 = DAG.getVScale(
8746 DL, PtrVT,
8747 APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));
8748 SDValue StackPtr2 = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, OffsetToV2);
8749 SDValue StoreV2 = DAG.getStore(StoreV1, DL, V2, StackPtr2, PtrInfo);
8750
8751 if (Imm >= 0) {
8752 // Load back the required element. getVectorElementPointer takes care of
8753 // clamping the index if it's out-of-bounds.
8754 StackPtr = getVectorElementPointer(DAG, StackPtr, VT, Node->getOperand(2));
8755 // Load the spliced result
8756 return DAG.getLoad(VT, DL, StoreV2, StackPtr,
8757 MachinePointerInfo::getUnknownStack(MF));
8758 }
8759
8760 uint64_t TrailingElts = -Imm;
8761
8762 // NOTE: TrailingElts must be clamped so as not to read outside of V1:V2.
8763 TypeSize EltByteSize = VT.getVectorElementType().getStoreSize();
8764 SDValue TrailingBytes =
8765 DAG.getConstant(TrailingElts * EltByteSize, DL, PtrVT);
8766
8767 if (TrailingElts > VT.getVectorMinNumElements()) {
8768 SDValue VLBytes = DAG.getVScale(
8769 DL, PtrVT,
8770 APInt(PtrVT.getFixedSizeInBits(), VT.getStoreSize().getKnownMinSize()));
8771 TrailingBytes = DAG.getNode(ISD::UMIN, DL, PtrVT, TrailingBytes, VLBytes);
8772 }
8773
8774 // Calculate the start address of the spliced result.
8775 StackPtr2 = DAG.getNode(ISD::SUB, DL, PtrVT, StackPtr2, TrailingBytes);
8776
8777 // Load the spliced result
8778 return DAG.getLoad(VT, DL, StoreV2, StackPtr2,
8779 MachinePointerInfo::getUnknownStack(MF));
8780 }
8781
LegalizeSetCCCondCode(SelectionDAG & DAG,EVT VT,SDValue & LHS,SDValue & RHS,SDValue & CC,bool & NeedInvert,const SDLoc & dl,SDValue & Chain,bool IsSignaling) const8782 bool TargetLowering::LegalizeSetCCCondCode(SelectionDAG &DAG, EVT VT,
8783 SDValue &LHS, SDValue &RHS,
8784 SDValue &CC, bool &NeedInvert,
8785 const SDLoc &dl, SDValue &Chain,
8786 bool IsSignaling) const {
8787 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8788 MVT OpVT = LHS.getSimpleValueType();
8789 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
8790 NeedInvert = false;
8791 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
8792 default:
8793 llvm_unreachable("Unknown condition code action!");
8794 case TargetLowering::Legal:
8795 // Nothing to do.
8796 break;
8797 case TargetLowering::Expand: {
8798 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
8799 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8800 std::swap(LHS, RHS);
8801 CC = DAG.getCondCode(InvCC);
8802 return true;
8803 }
8804 // Swapping operands didn't work. Try inverting the condition.
8805 bool NeedSwap = false;
8806 InvCC = getSetCCInverse(CCCode, OpVT);
8807 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8808 // If inverting the condition is not enough, try swapping operands
8809 // on top of it.
8810 InvCC = ISD::getSetCCSwappedOperands(InvCC);
8811 NeedSwap = true;
8812 }
8813 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
8814 CC = DAG.getCondCode(InvCC);
8815 NeedInvert = true;
8816 if (NeedSwap)
8817 std::swap(LHS, RHS);
8818 return true;
8819 }
8820
8821 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
8822 unsigned Opc = 0;
8823 switch (CCCode) {
8824 default:
8825 llvm_unreachable("Don't know how to expand this condition!");
8826 case ISD::SETUO:
8827 if (TLI.isCondCodeLegal(ISD::SETUNE, OpVT)) {
8828 CC1 = ISD::SETUNE;
8829 CC2 = ISD::SETUNE;
8830 Opc = ISD::OR;
8831 break;
8832 }
8833 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
8834 "If SETUE is expanded, SETOEQ or SETUNE must be legal!");
8835 NeedInvert = true;
8836 LLVM_FALLTHROUGH;
8837 case ISD::SETO:
8838 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) &&
8839 "If SETO is expanded, SETOEQ must be legal!");
8840 CC1 = ISD::SETOEQ;
8841 CC2 = ISD::SETOEQ;
8842 Opc = ISD::AND;
8843 break;
8844 case ISD::SETONE:
8845 case ISD::SETUEQ:
8846 // If the SETUO or SETO CC isn't legal, we might be able to use
8847 // SETOGT || SETOLT, inverting the result for SETUEQ. We only need one
8848 // of SETOGT/SETOLT to be legal, the other can be emulated by swapping
8849 // the operands.
8850 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
8851 if (!TLI.isCondCodeLegal(CC2, OpVT) &&
8852 (TLI.isCondCodeLegal(ISD::SETOGT, OpVT) ||
8853 TLI.isCondCodeLegal(ISD::SETOLT, OpVT))) {
8854 CC1 = ISD::SETOGT;
8855 CC2 = ISD::SETOLT;
8856 Opc = ISD::OR;
8857 NeedInvert = ((unsigned)CCCode & 0x8U);
8858 break;
8859 }
8860 LLVM_FALLTHROUGH;
8861 case ISD::SETOEQ:
8862 case ISD::SETOGT:
8863 case ISD::SETOGE:
8864 case ISD::SETOLT:
8865 case ISD::SETOLE:
8866 case ISD::SETUNE:
8867 case ISD::SETUGT:
8868 case ISD::SETUGE:
8869 case ISD::SETULT:
8870 case ISD::SETULE:
8871 // If we are floating point, assign and break, otherwise fall through.
8872 if (!OpVT.isInteger()) {
8873 // We can use the 4th bit to tell if we are the unordered
8874 // or ordered version of the opcode.
8875 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
8876 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
8877 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
8878 break;
8879 }
8880 // Fallthrough if we are unsigned integer.
8881 LLVM_FALLTHROUGH;
8882 case ISD::SETLE:
8883 case ISD::SETGT:
8884 case ISD::SETGE:
8885 case ISD::SETLT:
8886 case ISD::SETNE:
8887 case ISD::SETEQ:
8888 // If all combinations of inverting the condition and swapping operands
8889 // didn't work then we have no means to expand the condition.
8890 llvm_unreachable("Don't know how to expand this condition!");
8891 }
8892
8893 SDValue SetCC1, SetCC2;
8894 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
8895 // If we aren't the ordered or unorder operation,
8896 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
8897 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1, Chain, IsSignaling);
8898 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2, Chain, IsSignaling);
8899 } else {
8900 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
8901 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1, Chain, IsSignaling);
8902 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2, Chain, IsSignaling);
8903 }
8904 if (Chain)
8905 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, SetCC1.getValue(1),
8906 SetCC2.getValue(1));
8907 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
8908 RHS = SDValue();
8909 CC = SDValue();
8910 return true;
8911 }
8912 }
8913 return false;
8914 }
8915