1 //===-- NVPTXISelLowering.h - NVPTX DAG Lowering Interface ------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that NVPTX uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_LIB_TARGET_NVPTX_NVPTXISELLOWERING_H
15 #define LLVM_LIB_TARGET_NVPTX_NVPTXISELLOWERING_H
16 
17 #include "NVPTX.h"
18 #include "llvm/CodeGen/SelectionDAG.h"
19 #include "llvm/CodeGen/TargetLowering.h"
20 
21 namespace llvm {
22 namespace NVPTXISD {
23 enum NodeType : unsigned {
24   // Start the numbering from where ISD NodeType finishes.
25   FIRST_NUMBER = ISD::BUILTIN_OP_END,
26   Wrapper,
27   CALL,
28   RET_GLUE,
29   LOAD_PARAM,
30   DeclareParam,
31   DeclareScalarParam,
32   DeclareRetParam,
33   DeclareRet,
34   DeclareScalarRet,
35   PrintCall,
36   PrintConvergentCall,
37   PrintCallUni,
38   PrintConvergentCallUni,
39   CallArgBegin,
40   CallArg,
41   LastCallArg,
42   CallArgEnd,
43   CallVoid,
44   CallVal,
45   CallSymbol,
46   Prototype,
47   MoveParam,
48   PseudoUseParam,
49   RETURN,
50   CallSeqBegin,
51   CallSeqEnd,
52   CallPrototype,
53   ProxyReg,
54   FUN_SHFL_CLAMP,
55   FUN_SHFR_CLAMP,
56   MUL_WIDE_SIGNED,
57   MUL_WIDE_UNSIGNED,
58   IMAD,
59   SETP_F16X2,
60   SETP_BF16X2,
61   BFE,
62   BFI,
63   PRMT,
64   Dummy,
65 
66   LoadV2 = ISD::FIRST_TARGET_MEMORY_OPCODE,
67   LoadV4,
68   LDGV2, // LDG.v2
69   LDGV4, // LDG.v4
70   LDUV2, // LDU.v2
71   LDUV4, // LDU.v4
72   StoreV2,
73   StoreV4,
74   LoadParam,
75   LoadParamV2,
76   LoadParamV4,
77   StoreParam,
78   StoreParamV2,
79   StoreParamV4,
80   StoreParamS32, // to sext and store a <32bit value, not used currently
81   StoreParamU32, // to zext and store a <32bit value, not used currently
82   StoreRetval,
83   StoreRetvalV2,
84   StoreRetvalV4,
85 
86   // Texture intrinsics
87   Tex1DFloatS32,
88   Tex1DFloatFloat,
89   Tex1DFloatFloatLevel,
90   Tex1DFloatFloatGrad,
91   Tex1DS32S32,
92   Tex1DS32Float,
93   Tex1DS32FloatLevel,
94   Tex1DS32FloatGrad,
95   Tex1DU32S32,
96   Tex1DU32Float,
97   Tex1DU32FloatLevel,
98   Tex1DU32FloatGrad,
99   Tex1DArrayFloatS32,
100   Tex1DArrayFloatFloat,
101   Tex1DArrayFloatFloatLevel,
102   Tex1DArrayFloatFloatGrad,
103   Tex1DArrayS32S32,
104   Tex1DArrayS32Float,
105   Tex1DArrayS32FloatLevel,
106   Tex1DArrayS32FloatGrad,
107   Tex1DArrayU32S32,
108   Tex1DArrayU32Float,
109   Tex1DArrayU32FloatLevel,
110   Tex1DArrayU32FloatGrad,
111   Tex2DFloatS32,
112   Tex2DFloatFloat,
113   Tex2DFloatFloatLevel,
114   Tex2DFloatFloatGrad,
115   Tex2DS32S32,
116   Tex2DS32Float,
117   Tex2DS32FloatLevel,
118   Tex2DS32FloatGrad,
119   Tex2DU32S32,
120   Tex2DU32Float,
121   Tex2DU32FloatLevel,
122   Tex2DU32FloatGrad,
123   Tex2DArrayFloatS32,
124   Tex2DArrayFloatFloat,
125   Tex2DArrayFloatFloatLevel,
126   Tex2DArrayFloatFloatGrad,
127   Tex2DArrayS32S32,
128   Tex2DArrayS32Float,
129   Tex2DArrayS32FloatLevel,
130   Tex2DArrayS32FloatGrad,
131   Tex2DArrayU32S32,
132   Tex2DArrayU32Float,
133   Tex2DArrayU32FloatLevel,
134   Tex2DArrayU32FloatGrad,
135   Tex3DFloatS32,
136   Tex3DFloatFloat,
137   Tex3DFloatFloatLevel,
138   Tex3DFloatFloatGrad,
139   Tex3DS32S32,
140   Tex3DS32Float,
141   Tex3DS32FloatLevel,
142   Tex3DS32FloatGrad,
143   Tex3DU32S32,
144   Tex3DU32Float,
145   Tex3DU32FloatLevel,
146   Tex3DU32FloatGrad,
147   TexCubeFloatFloat,
148   TexCubeFloatFloatLevel,
149   TexCubeS32Float,
150   TexCubeS32FloatLevel,
151   TexCubeU32Float,
152   TexCubeU32FloatLevel,
153   TexCubeArrayFloatFloat,
154   TexCubeArrayFloatFloatLevel,
155   TexCubeArrayS32Float,
156   TexCubeArrayS32FloatLevel,
157   TexCubeArrayU32Float,
158   TexCubeArrayU32FloatLevel,
159   Tld4R2DFloatFloat,
160   Tld4G2DFloatFloat,
161   Tld4B2DFloatFloat,
162   Tld4A2DFloatFloat,
163   Tld4R2DS64Float,
164   Tld4G2DS64Float,
165   Tld4B2DS64Float,
166   Tld4A2DS64Float,
167   Tld4R2DU64Float,
168   Tld4G2DU64Float,
169   Tld4B2DU64Float,
170   Tld4A2DU64Float,
171   TexUnified1DFloatS32,
172   TexUnified1DFloatFloat,
173   TexUnified1DFloatFloatLevel,
174   TexUnified1DFloatFloatGrad,
175   TexUnified1DS32S32,
176   TexUnified1DS32Float,
177   TexUnified1DS32FloatLevel,
178   TexUnified1DS32FloatGrad,
179   TexUnified1DU32S32,
180   TexUnified1DU32Float,
181   TexUnified1DU32FloatLevel,
182   TexUnified1DU32FloatGrad,
183   TexUnified1DArrayFloatS32,
184   TexUnified1DArrayFloatFloat,
185   TexUnified1DArrayFloatFloatLevel,
186   TexUnified1DArrayFloatFloatGrad,
187   TexUnified1DArrayS32S32,
188   TexUnified1DArrayS32Float,
189   TexUnified1DArrayS32FloatLevel,
190   TexUnified1DArrayS32FloatGrad,
191   TexUnified1DArrayU32S32,
192   TexUnified1DArrayU32Float,
193   TexUnified1DArrayU32FloatLevel,
194   TexUnified1DArrayU32FloatGrad,
195   TexUnified2DFloatS32,
196   TexUnified2DFloatFloat,
197   TexUnified2DFloatFloatLevel,
198   TexUnified2DFloatFloatGrad,
199   TexUnified2DS32S32,
200   TexUnified2DS32Float,
201   TexUnified2DS32FloatLevel,
202   TexUnified2DS32FloatGrad,
203   TexUnified2DU32S32,
204   TexUnified2DU32Float,
205   TexUnified2DU32FloatLevel,
206   TexUnified2DU32FloatGrad,
207   TexUnified2DArrayFloatS32,
208   TexUnified2DArrayFloatFloat,
209   TexUnified2DArrayFloatFloatLevel,
210   TexUnified2DArrayFloatFloatGrad,
211   TexUnified2DArrayS32S32,
212   TexUnified2DArrayS32Float,
213   TexUnified2DArrayS32FloatLevel,
214   TexUnified2DArrayS32FloatGrad,
215   TexUnified2DArrayU32S32,
216   TexUnified2DArrayU32Float,
217   TexUnified2DArrayU32FloatLevel,
218   TexUnified2DArrayU32FloatGrad,
219   TexUnified3DFloatS32,
220   TexUnified3DFloatFloat,
221   TexUnified3DFloatFloatLevel,
222   TexUnified3DFloatFloatGrad,
223   TexUnified3DS32S32,
224   TexUnified3DS32Float,
225   TexUnified3DS32FloatLevel,
226   TexUnified3DS32FloatGrad,
227   TexUnified3DU32S32,
228   TexUnified3DU32Float,
229   TexUnified3DU32FloatLevel,
230   TexUnified3DU32FloatGrad,
231   TexUnifiedCubeFloatFloat,
232   TexUnifiedCubeFloatFloatLevel,
233   TexUnifiedCubeS32Float,
234   TexUnifiedCubeS32FloatLevel,
235   TexUnifiedCubeU32Float,
236   TexUnifiedCubeU32FloatLevel,
237   TexUnifiedCubeArrayFloatFloat,
238   TexUnifiedCubeArrayFloatFloatLevel,
239   TexUnifiedCubeArrayS32Float,
240   TexUnifiedCubeArrayS32FloatLevel,
241   TexUnifiedCubeArrayU32Float,
242   TexUnifiedCubeArrayU32FloatLevel,
243   TexUnifiedCubeFloatFloatGrad,
244   TexUnifiedCubeS32FloatGrad,
245   TexUnifiedCubeU32FloatGrad,
246   TexUnifiedCubeArrayFloatFloatGrad,
247   TexUnifiedCubeArrayS32FloatGrad,
248   TexUnifiedCubeArrayU32FloatGrad,
249   Tld4UnifiedR2DFloatFloat,
250   Tld4UnifiedG2DFloatFloat,
251   Tld4UnifiedB2DFloatFloat,
252   Tld4UnifiedA2DFloatFloat,
253   Tld4UnifiedR2DS64Float,
254   Tld4UnifiedG2DS64Float,
255   Tld4UnifiedB2DS64Float,
256   Tld4UnifiedA2DS64Float,
257   Tld4UnifiedR2DU64Float,
258   Tld4UnifiedG2DU64Float,
259   Tld4UnifiedB2DU64Float,
260   Tld4UnifiedA2DU64Float,
261 
262   // Surface intrinsics
263   Suld1DI8Clamp,
264   Suld1DI16Clamp,
265   Suld1DI32Clamp,
266   Suld1DI64Clamp,
267   Suld1DV2I8Clamp,
268   Suld1DV2I16Clamp,
269   Suld1DV2I32Clamp,
270   Suld1DV2I64Clamp,
271   Suld1DV4I8Clamp,
272   Suld1DV4I16Clamp,
273   Suld1DV4I32Clamp,
274 
275   Suld1DArrayI8Clamp,
276   Suld1DArrayI16Clamp,
277   Suld1DArrayI32Clamp,
278   Suld1DArrayI64Clamp,
279   Suld1DArrayV2I8Clamp,
280   Suld1DArrayV2I16Clamp,
281   Suld1DArrayV2I32Clamp,
282   Suld1DArrayV2I64Clamp,
283   Suld1DArrayV4I8Clamp,
284   Suld1DArrayV4I16Clamp,
285   Suld1DArrayV4I32Clamp,
286 
287   Suld2DI8Clamp,
288   Suld2DI16Clamp,
289   Suld2DI32Clamp,
290   Suld2DI64Clamp,
291   Suld2DV2I8Clamp,
292   Suld2DV2I16Clamp,
293   Suld2DV2I32Clamp,
294   Suld2DV2I64Clamp,
295   Suld2DV4I8Clamp,
296   Suld2DV4I16Clamp,
297   Suld2DV4I32Clamp,
298 
299   Suld2DArrayI8Clamp,
300   Suld2DArrayI16Clamp,
301   Suld2DArrayI32Clamp,
302   Suld2DArrayI64Clamp,
303   Suld2DArrayV2I8Clamp,
304   Suld2DArrayV2I16Clamp,
305   Suld2DArrayV2I32Clamp,
306   Suld2DArrayV2I64Clamp,
307   Suld2DArrayV4I8Clamp,
308   Suld2DArrayV4I16Clamp,
309   Suld2DArrayV4I32Clamp,
310 
311   Suld3DI8Clamp,
312   Suld3DI16Clamp,
313   Suld3DI32Clamp,
314   Suld3DI64Clamp,
315   Suld3DV2I8Clamp,
316   Suld3DV2I16Clamp,
317   Suld3DV2I32Clamp,
318   Suld3DV2I64Clamp,
319   Suld3DV4I8Clamp,
320   Suld3DV4I16Clamp,
321   Suld3DV4I32Clamp,
322 
323   Suld1DI8Trap,
324   Suld1DI16Trap,
325   Suld1DI32Trap,
326   Suld1DI64Trap,
327   Suld1DV2I8Trap,
328   Suld1DV2I16Trap,
329   Suld1DV2I32Trap,
330   Suld1DV2I64Trap,
331   Suld1DV4I8Trap,
332   Suld1DV4I16Trap,
333   Suld1DV4I32Trap,
334 
335   Suld1DArrayI8Trap,
336   Suld1DArrayI16Trap,
337   Suld1DArrayI32Trap,
338   Suld1DArrayI64Trap,
339   Suld1DArrayV2I8Trap,
340   Suld1DArrayV2I16Trap,
341   Suld1DArrayV2I32Trap,
342   Suld1DArrayV2I64Trap,
343   Suld1DArrayV4I8Trap,
344   Suld1DArrayV4I16Trap,
345   Suld1DArrayV4I32Trap,
346 
347   Suld2DI8Trap,
348   Suld2DI16Trap,
349   Suld2DI32Trap,
350   Suld2DI64Trap,
351   Suld2DV2I8Trap,
352   Suld2DV2I16Trap,
353   Suld2DV2I32Trap,
354   Suld2DV2I64Trap,
355   Suld2DV4I8Trap,
356   Suld2DV4I16Trap,
357   Suld2DV4I32Trap,
358 
359   Suld2DArrayI8Trap,
360   Suld2DArrayI16Trap,
361   Suld2DArrayI32Trap,
362   Suld2DArrayI64Trap,
363   Suld2DArrayV2I8Trap,
364   Suld2DArrayV2I16Trap,
365   Suld2DArrayV2I32Trap,
366   Suld2DArrayV2I64Trap,
367   Suld2DArrayV4I8Trap,
368   Suld2DArrayV4I16Trap,
369   Suld2DArrayV4I32Trap,
370 
371   Suld3DI8Trap,
372   Suld3DI16Trap,
373   Suld3DI32Trap,
374   Suld3DI64Trap,
375   Suld3DV2I8Trap,
376   Suld3DV2I16Trap,
377   Suld3DV2I32Trap,
378   Suld3DV2I64Trap,
379   Suld3DV4I8Trap,
380   Suld3DV4I16Trap,
381   Suld3DV4I32Trap,
382 
383   Suld1DI8Zero,
384   Suld1DI16Zero,
385   Suld1DI32Zero,
386   Suld1DI64Zero,
387   Suld1DV2I8Zero,
388   Suld1DV2I16Zero,
389   Suld1DV2I32Zero,
390   Suld1DV2I64Zero,
391   Suld1DV4I8Zero,
392   Suld1DV4I16Zero,
393   Suld1DV4I32Zero,
394 
395   Suld1DArrayI8Zero,
396   Suld1DArrayI16Zero,
397   Suld1DArrayI32Zero,
398   Suld1DArrayI64Zero,
399   Suld1DArrayV2I8Zero,
400   Suld1DArrayV2I16Zero,
401   Suld1DArrayV2I32Zero,
402   Suld1DArrayV2I64Zero,
403   Suld1DArrayV4I8Zero,
404   Suld1DArrayV4I16Zero,
405   Suld1DArrayV4I32Zero,
406 
407   Suld2DI8Zero,
408   Suld2DI16Zero,
409   Suld2DI32Zero,
410   Suld2DI64Zero,
411   Suld2DV2I8Zero,
412   Suld2DV2I16Zero,
413   Suld2DV2I32Zero,
414   Suld2DV2I64Zero,
415   Suld2DV4I8Zero,
416   Suld2DV4I16Zero,
417   Suld2DV4I32Zero,
418 
419   Suld2DArrayI8Zero,
420   Suld2DArrayI16Zero,
421   Suld2DArrayI32Zero,
422   Suld2DArrayI64Zero,
423   Suld2DArrayV2I8Zero,
424   Suld2DArrayV2I16Zero,
425   Suld2DArrayV2I32Zero,
426   Suld2DArrayV2I64Zero,
427   Suld2DArrayV4I8Zero,
428   Suld2DArrayV4I16Zero,
429   Suld2DArrayV4I32Zero,
430 
431   Suld3DI8Zero,
432   Suld3DI16Zero,
433   Suld3DI32Zero,
434   Suld3DI64Zero,
435   Suld3DV2I8Zero,
436   Suld3DV2I16Zero,
437   Suld3DV2I32Zero,
438   Suld3DV2I64Zero,
439   Suld3DV4I8Zero,
440   Suld3DV4I16Zero,
441   Suld3DV4I32Zero
442 };
443 }
444 
445 class NVPTXSubtarget;
446 
447 //===--------------------------------------------------------------------===//
448 // TargetLowering Implementation
449 //===--------------------------------------------------------------------===//
450 class NVPTXTargetLowering : public TargetLowering {
451 public:
452   explicit NVPTXTargetLowering(const NVPTXTargetMachine &TM,
453                                const NVPTXSubtarget &STI);
454   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
455 
456   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
457 
458   const char *getTargetNodeName(unsigned Opcode) const override;
459 
460   bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
461                           MachineFunction &MF,
462                           unsigned Intrinsic) const override;
463 
464   /// getFunctionParamOptimizedAlign - since function arguments are passed via
465   /// .param space, we may want to increase their alignment in a way that
466   /// ensures that we can effectively vectorize their loads & stores. We can
467   /// increase alignment only if the function has internal or has private
468   /// linkage as for other linkage types callers may already rely on default
469   /// alignment. To allow using 128-bit vectorized loads/stores, this function
470   /// ensures that alignment is 16 or greater.
471   Align getFunctionParamOptimizedAlign(const Function *F, Type *ArgTy,
472                                        const DataLayout &DL) const;
473 
474   /// Helper for computing alignment of a device function byval parameter.
475   Align getFunctionByValParamAlign(const Function *F, Type *ArgTy,
476                                    Align InitialAlign,
477                                    const DataLayout &DL) const;
478 
479   // Helper for getting a function parameter name. Name is composed from
480   // its index and the function name. Negative index corresponds to special
481   // parameter (unsized array) used for passing variable arguments.
482   std::string getParamName(const Function *F, int Idx) const;
483 
484   /// isLegalAddressingMode - Return true if the addressing mode represented
485   /// by AM is legal for this target, for a load/store of the specified type
486   /// Used to guide target specific optimizations, like loop strength
487   /// reduction (LoopStrengthReduce.cpp) and memory optimization for
488   /// address mode (CodeGenPrepare.cpp)
489   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
490                              unsigned AS,
491                              Instruction *I = nullptr) const override;
492 
isTruncateFree(Type * SrcTy,Type * DstTy)493   bool isTruncateFree(Type *SrcTy, Type *DstTy) const override {
494     // Truncating 64-bit to 32-bit is free in SASS.
495     if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
496       return false;
497     return SrcTy->getPrimitiveSizeInBits() == 64 &&
498            DstTy->getPrimitiveSizeInBits() == 32;
499   }
500 
getSetCCResultType(const DataLayout & DL,LLVMContext & Ctx,EVT VT)501   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Ctx,
502                          EVT VT) const override {
503     if (VT.isVector())
504       return EVT::getVectorVT(Ctx, MVT::i1, VT.getVectorNumElements());
505     return MVT::i1;
506   }
507 
508   ConstraintType getConstraintType(StringRef Constraint) const override;
509   std::pair<unsigned, const TargetRegisterClass *>
510   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
511                                StringRef Constraint, MVT VT) const override;
512 
513   SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
514                                bool isVarArg,
515                                const SmallVectorImpl<ISD::InputArg> &Ins,
516                                const SDLoc &dl, SelectionDAG &DAG,
517                                SmallVectorImpl<SDValue> &InVals) const override;
518 
519   SDValue LowerCall(CallLoweringInfo &CLI,
520                     SmallVectorImpl<SDValue> &InVals) const override;
521 
522   SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
523 
524   std::string
525   getPrototype(const DataLayout &DL, Type *, const ArgListTy &,
526                const SmallVectorImpl<ISD::OutputArg> &, MaybeAlign retAlignment,
527                std::optional<std::pair<unsigned, const APInt &>> VAInfo,
528                const CallBase &CB, unsigned UniqueCallSite) const;
529 
530   SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
531                       const SmallVectorImpl<ISD::OutputArg> &Outs,
532                       const SmallVectorImpl<SDValue> &OutVals, const SDLoc &dl,
533                       SelectionDAG &DAG) const override;
534 
535   void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
536                                     std::vector<SDValue> &Ops,
537                                     SelectionDAG &DAG) const override;
538 
539   const NVPTXTargetMachine *nvTM;
540 
541   // PTX always uses 32-bit shift amounts
getScalarShiftAmountTy(const DataLayout &,EVT)542   MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
543     return MVT::i32;
544   }
545 
546   TargetLoweringBase::LegalizeTypeAction
547   getPreferredVectorAction(MVT VT) const override;
548 
549   // Get the degree of precision we want from 32-bit floating point division
550   // operations.
551   //
552   //  0 - Use ptx div.approx
553   //  1 - Use ptx.div.full (approximate, but less so than div.approx)
554   //  2 - Use IEEE-compliant div instructions, if available.
555   int getDivF32Level() const;
556 
557   // Get whether we should use a precise or approximate 32-bit floating point
558   // sqrt instruction.
559   bool usePrecSqrtF32() const;
560 
561   // Get whether we should use instructions that flush floating-point denormals
562   // to sign-preserving zero.
563   bool useF32FTZ(const MachineFunction &MF) const;
564 
565   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
566                           int &ExtraSteps, bool &UseOneConst,
567                           bool Reciprocal) const override;
568 
combineRepeatedFPDivisors()569   unsigned combineRepeatedFPDivisors() const override { return 2; }
570 
571   bool allowFMA(MachineFunction &MF, CodeGenOptLevel OptLevel) const;
572   bool allowUnsafeFPMath(MachineFunction &MF) const;
573 
isFMAFasterThanFMulAndFAdd(const MachineFunction & MF,EVT)574   bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
575                                   EVT) const override {
576     return true;
577   }
578 
enableAggressiveFMAFusion(EVT VT)579   bool enableAggressiveFMAFusion(EVT VT) const override { return true; }
580 
581   // The default is to transform llvm.ctlz(x, false) (where false indicates that
582   // x == 0 is not undefined behavior) into a branch that checks whether x is 0
583   // and avoids calling ctlz in that case.  We have a dedicated ctlz
584   // instruction, so we say that ctlz is cheap to speculate.
isCheapToSpeculateCtlz(Type * Ty)585   bool isCheapToSpeculateCtlz(Type *Ty) const override { return true; }
586 
shouldCastAtomicLoadInIR(LoadInst * LI)587   AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override {
588     return AtomicExpansionKind::None;
589   }
590 
shouldCastAtomicStoreInIR(StoreInst * SI)591   AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override {
592     return AtomicExpansionKind::None;
593   }
594 
595   AtomicExpansionKind
596   shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
597 
aggressivelyPreferBuildVectorSources(EVT VecVT)598   bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override {
599     // There's rarely any point of packing something into a vector type if we
600     // already have the source data.
601     return true;
602   }
603 
604 private:
605   const NVPTXSubtarget &STI; // cache the subtarget here
606   SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;
607 
608   SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
609   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
610   SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
611   SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
612   SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
613 
614   SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const;
615   SDValue LowerFROUND32(SDValue Op, SelectionDAG &DAG) const;
616   SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const;
617 
618   SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
619   SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
620 
621   SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
622   SDValue LowerLOADi1(SDValue Op, SelectionDAG &DAG) const;
623 
624   SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
625   SDValue LowerSTOREi1(SDValue Op, SelectionDAG &DAG) const;
626   SDValue LowerSTOREVector(SDValue Op, SelectionDAG &DAG) const;
627 
628   SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
629   SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
630 
631   SDValue LowerSelect(SDValue Op, SelectionDAG &DAG) const;
632 
633   SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
634   SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
635 
636   void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
637                           SelectionDAG &DAG) const override;
638   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
639 
640   Align getArgumentAlignment(const CallBase *CB, Type *Ty, unsigned Idx,
641                              const DataLayout &DL) const;
642 };
643 
644 } // namespace llvm
645 
646 #endif
647