1*13fbcb42Sjoerg //===------ CGOpenMPRuntimeGPU.h - Interface to OpenMP GPU Runtimes ------===// 2*13fbcb42Sjoerg // 3*13fbcb42Sjoerg // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4*13fbcb42Sjoerg // See https://llvm.org/LICENSE.txt for license information. 5*13fbcb42Sjoerg // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6*13fbcb42Sjoerg // 7*13fbcb42Sjoerg //===----------------------------------------------------------------------===// 8*13fbcb42Sjoerg // 9*13fbcb42Sjoerg // This provides a generalized class for OpenMP runtime code generation 10*13fbcb42Sjoerg // specialized by GPU targets NVPTX and AMDGCN. 11*13fbcb42Sjoerg // 12*13fbcb42Sjoerg //===----------------------------------------------------------------------===// 13*13fbcb42Sjoerg 14*13fbcb42Sjoerg #ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H 15*13fbcb42Sjoerg #define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H 16*13fbcb42Sjoerg 17*13fbcb42Sjoerg #include "CGOpenMPRuntime.h" 18*13fbcb42Sjoerg #include "CodeGenFunction.h" 19*13fbcb42Sjoerg #include "clang/AST/StmtOpenMP.h" 20*13fbcb42Sjoerg #include "llvm/Frontend/OpenMP/OMPGridValues.h" 21*13fbcb42Sjoerg 22*13fbcb42Sjoerg namespace clang { 23*13fbcb42Sjoerg namespace CodeGen { 24*13fbcb42Sjoerg 25*13fbcb42Sjoerg class CGOpenMPRuntimeGPU : public CGOpenMPRuntime { 26*13fbcb42Sjoerg public: 27*13fbcb42Sjoerg /// Defines the execution mode. 28*13fbcb42Sjoerg enum ExecutionMode { 29*13fbcb42Sjoerg /// SPMD execution mode (all threads are worker threads). 30*13fbcb42Sjoerg EM_SPMD, 31*13fbcb42Sjoerg /// Non-SPMD execution mode (1 master thread, others are workers). 32*13fbcb42Sjoerg EM_NonSPMD, 33*13fbcb42Sjoerg /// Unknown execution mode (orphaned directive). 34*13fbcb42Sjoerg EM_Unknown, 35*13fbcb42Sjoerg }; 36*13fbcb42Sjoerg private: 37*13fbcb42Sjoerg /// Parallel outlined function work for workers to execute. 38*13fbcb42Sjoerg llvm::SmallVector<llvm::Function *, 16> Work; 39*13fbcb42Sjoerg 40*13fbcb42Sjoerg struct EntryFunctionState { 41*13fbcb42Sjoerg llvm::BasicBlock *ExitBB = nullptr; 42*13fbcb42Sjoerg }; 43*13fbcb42Sjoerg 44*13fbcb42Sjoerg class WorkerFunctionState { 45*13fbcb42Sjoerg public: 46*13fbcb42Sjoerg llvm::Function *WorkerFn; 47*13fbcb42Sjoerg const CGFunctionInfo &CGFI; 48*13fbcb42Sjoerg SourceLocation Loc; 49*13fbcb42Sjoerg 50*13fbcb42Sjoerg WorkerFunctionState(CodeGenModule &CGM, SourceLocation Loc); 51*13fbcb42Sjoerg 52*13fbcb42Sjoerg private: 53*13fbcb42Sjoerg void createWorkerFunction(CodeGenModule &CGM); 54*13fbcb42Sjoerg }; 55*13fbcb42Sjoerg 56*13fbcb42Sjoerg ExecutionMode getExecutionMode() const; 57*13fbcb42Sjoerg requiresFullRuntime()58*13fbcb42Sjoerg bool requiresFullRuntime() const { return RequiresFullRuntime; } 59*13fbcb42Sjoerg 60*13fbcb42Sjoerg /// Get barrier to synchronize all threads in a block. 61*13fbcb42Sjoerg void syncCTAThreads(CodeGenFunction &CGF); 62*13fbcb42Sjoerg 63*13fbcb42Sjoerg /// Emit the worker function for the current target region. 64*13fbcb42Sjoerg void emitWorkerFunction(WorkerFunctionState &WST); 65*13fbcb42Sjoerg 66*13fbcb42Sjoerg /// Helper for worker function. Emit body of worker loop. 67*13fbcb42Sjoerg void emitWorkerLoop(CodeGenFunction &CGF, WorkerFunctionState &WST); 68*13fbcb42Sjoerg 69*13fbcb42Sjoerg /// Helper for non-SPMD target entry function. Guide the master and 70*13fbcb42Sjoerg /// worker threads to their respective locations. 71*13fbcb42Sjoerg void emitNonSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST, 72*13fbcb42Sjoerg WorkerFunctionState &WST); 73*13fbcb42Sjoerg 74*13fbcb42Sjoerg /// Signal termination of OMP execution for non-SPMD target entry 75*13fbcb42Sjoerg /// function. 76*13fbcb42Sjoerg void emitNonSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST); 77*13fbcb42Sjoerg 78*13fbcb42Sjoerg /// Helper for generic variables globalization prolog. 79*13fbcb42Sjoerg void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc, 80*13fbcb42Sjoerg bool WithSPMDCheck = false); 81*13fbcb42Sjoerg 82*13fbcb42Sjoerg /// Helper for generic variables globalization epilog. 83*13fbcb42Sjoerg void emitGenericVarsEpilog(CodeGenFunction &CGF, bool WithSPMDCheck = false); 84*13fbcb42Sjoerg 85*13fbcb42Sjoerg /// Helper for SPMD mode target directive's entry function. 86*13fbcb42Sjoerg void emitSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST, 87*13fbcb42Sjoerg const OMPExecutableDirective &D); 88*13fbcb42Sjoerg 89*13fbcb42Sjoerg /// Signal termination of SPMD mode execution. 90*13fbcb42Sjoerg void emitSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST); 91*13fbcb42Sjoerg 92*13fbcb42Sjoerg // 93*13fbcb42Sjoerg // Base class overrides. 94*13fbcb42Sjoerg // 95*13fbcb42Sjoerg 96*13fbcb42Sjoerg /// Creates offloading entry for the provided entry ID \a ID, 97*13fbcb42Sjoerg /// address \a Addr, size \a Size, and flags \a Flags. 98*13fbcb42Sjoerg void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, 99*13fbcb42Sjoerg uint64_t Size, int32_t Flags, 100*13fbcb42Sjoerg llvm::GlobalValue::LinkageTypes Linkage) override; 101*13fbcb42Sjoerg 102*13fbcb42Sjoerg /// Emit outlined function specialized for the Fork-Join 103*13fbcb42Sjoerg /// programming model for applicable target directives on the NVPTX device. 104*13fbcb42Sjoerg /// \param D Directive to emit. 105*13fbcb42Sjoerg /// \param ParentName Name of the function that encloses the target region. 106*13fbcb42Sjoerg /// \param OutlinedFn Outlined function value to be defined by this call. 107*13fbcb42Sjoerg /// \param OutlinedFnID Outlined function ID value to be defined by this call. 108*13fbcb42Sjoerg /// \param IsOffloadEntry True if the outlined function is an offload entry. 109*13fbcb42Sjoerg /// An outlined function may not be an entry if, e.g. the if clause always 110*13fbcb42Sjoerg /// evaluates to false. 111*13fbcb42Sjoerg void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName, 112*13fbcb42Sjoerg llvm::Function *&OutlinedFn, 113*13fbcb42Sjoerg llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, 114*13fbcb42Sjoerg const RegionCodeGenTy &CodeGen); 115*13fbcb42Sjoerg 116*13fbcb42Sjoerg /// Emit outlined function specialized for the Single Program 117*13fbcb42Sjoerg /// Multiple Data programming model for applicable target directives on the 118*13fbcb42Sjoerg /// NVPTX device. 119*13fbcb42Sjoerg /// \param D Directive to emit. 120*13fbcb42Sjoerg /// \param ParentName Name of the function that encloses the target region. 121*13fbcb42Sjoerg /// \param OutlinedFn Outlined function value to be defined by this call. 122*13fbcb42Sjoerg /// \param OutlinedFnID Outlined function ID value to be defined by this call. 123*13fbcb42Sjoerg /// \param IsOffloadEntry True if the outlined function is an offload entry. 124*13fbcb42Sjoerg /// \param CodeGen Object containing the target statements. 125*13fbcb42Sjoerg /// An outlined function may not be an entry if, e.g. the if clause always 126*13fbcb42Sjoerg /// evaluates to false. 127*13fbcb42Sjoerg void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName, 128*13fbcb42Sjoerg llvm::Function *&OutlinedFn, 129*13fbcb42Sjoerg llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, 130*13fbcb42Sjoerg const RegionCodeGenTy &CodeGen); 131*13fbcb42Sjoerg 132*13fbcb42Sjoerg /// Emit outlined function for 'target' directive on the NVPTX 133*13fbcb42Sjoerg /// device. 134*13fbcb42Sjoerg /// \param D Directive to emit. 135*13fbcb42Sjoerg /// \param ParentName Name of the function that encloses the target region. 136*13fbcb42Sjoerg /// \param OutlinedFn Outlined function value to be defined by this call. 137*13fbcb42Sjoerg /// \param OutlinedFnID Outlined function ID value to be defined by this call. 138*13fbcb42Sjoerg /// \param IsOffloadEntry True if the outlined function is an offload entry. 139*13fbcb42Sjoerg /// An outlined function may not be an entry if, e.g. the if clause always 140*13fbcb42Sjoerg /// evaluates to false. 141*13fbcb42Sjoerg void emitTargetOutlinedFunction(const OMPExecutableDirective &D, 142*13fbcb42Sjoerg StringRef ParentName, 143*13fbcb42Sjoerg llvm::Function *&OutlinedFn, 144*13fbcb42Sjoerg llvm::Constant *&OutlinedFnID, 145*13fbcb42Sjoerg bool IsOffloadEntry, 146*13fbcb42Sjoerg const RegionCodeGenTy &CodeGen) override; 147*13fbcb42Sjoerg 148*13fbcb42Sjoerg /// Emits code for parallel or serial call of the \a OutlinedFn with 149*13fbcb42Sjoerg /// variables captured in a record which address is stored in \a 150*13fbcb42Sjoerg /// CapturedStruct. 151*13fbcb42Sjoerg /// This call is for the Non-SPMD Execution Mode. 152*13fbcb42Sjoerg /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 153*13fbcb42Sjoerg /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 154*13fbcb42Sjoerg /// \param CapturedVars A pointer to the record with the references to 155*13fbcb42Sjoerg /// variables used in \a OutlinedFn function. 156*13fbcb42Sjoerg /// \param IfCond Condition in the associated 'if' clause, if it was 157*13fbcb42Sjoerg /// specified, nullptr otherwise. 158*13fbcb42Sjoerg void emitNonSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 159*13fbcb42Sjoerg llvm::Value *OutlinedFn, 160*13fbcb42Sjoerg ArrayRef<llvm::Value *> CapturedVars, 161*13fbcb42Sjoerg const Expr *IfCond); 162*13fbcb42Sjoerg 163*13fbcb42Sjoerg /// Emits code for parallel or serial call of the \a OutlinedFn with 164*13fbcb42Sjoerg /// variables captured in a record which address is stored in \a 165*13fbcb42Sjoerg /// CapturedStruct. 166*13fbcb42Sjoerg /// This call is for a parallel directive within an SPMD target directive. 167*13fbcb42Sjoerg /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 168*13fbcb42Sjoerg /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 169*13fbcb42Sjoerg /// \param CapturedVars A pointer to the record with the references to 170*13fbcb42Sjoerg /// variables used in \a OutlinedFn function. 171*13fbcb42Sjoerg /// \param IfCond Condition in the associated 'if' clause, if it was 172*13fbcb42Sjoerg /// specified, nullptr otherwise. 173*13fbcb42Sjoerg /// 174*13fbcb42Sjoerg void emitSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 175*13fbcb42Sjoerg llvm::Function *OutlinedFn, 176*13fbcb42Sjoerg ArrayRef<llvm::Value *> CapturedVars, 177*13fbcb42Sjoerg const Expr *IfCond); 178*13fbcb42Sjoerg 179*13fbcb42Sjoerg protected: 180*13fbcb42Sjoerg /// Get the function name of an outlined region. 181*13fbcb42Sjoerg // The name can be customized depending on the target. 182*13fbcb42Sjoerg // getOutlinedHelperName()183*13fbcb42Sjoerg StringRef getOutlinedHelperName() const override { 184*13fbcb42Sjoerg return "__omp_outlined__"; 185*13fbcb42Sjoerg } 186*13fbcb42Sjoerg 187*13fbcb42Sjoerg /// Check if the default location must be constant. 188*13fbcb42Sjoerg /// Constant for NVPTX for better optimization. isDefaultLocationConstant()189*13fbcb42Sjoerg bool isDefaultLocationConstant() const override { return true; } 190*13fbcb42Sjoerg 191*13fbcb42Sjoerg /// Returns additional flags that can be stored in reserved_2 field of the 192*13fbcb42Sjoerg /// default location. 193*13fbcb42Sjoerg /// For NVPTX target contains data about SPMD/Non-SPMD execution mode + 194*13fbcb42Sjoerg /// Full/Lightweight runtime mode. Used for better optimization. 195*13fbcb42Sjoerg unsigned getDefaultLocationReserved2Flags() const override; 196*13fbcb42Sjoerg 197*13fbcb42Sjoerg public: 198*13fbcb42Sjoerg explicit CGOpenMPRuntimeGPU(CodeGenModule &CGM); 199*13fbcb42Sjoerg void clear() override; 200*13fbcb42Sjoerg 201*13fbcb42Sjoerg /// Declare generalized virtual functions which need to be defined 202*13fbcb42Sjoerg /// by all specializations of OpenMPGPURuntime Targets like AMDGCN 203*13fbcb42Sjoerg /// and NVPTX. 204*13fbcb42Sjoerg 205*13fbcb42Sjoerg /// Get the GPU warp size. 206*13fbcb42Sjoerg virtual llvm::Value *getGPUWarpSize(CodeGenFunction &CGF) = 0; 207*13fbcb42Sjoerg 208*13fbcb42Sjoerg /// Get the id of the current thread on the GPU. 209*13fbcb42Sjoerg virtual llvm::Value *getGPUThreadID(CodeGenFunction &CGF) = 0; 210*13fbcb42Sjoerg 211*13fbcb42Sjoerg /// Get the maximum number of threads in a block of the GPU. 212*13fbcb42Sjoerg virtual llvm::Value *getGPUNumThreads(CodeGenFunction &CGF) = 0; 213*13fbcb42Sjoerg 214*13fbcb42Sjoerg /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 215*13fbcb42Sjoerg /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. 216*13fbcb42Sjoerg virtual void emitProcBindClause(CodeGenFunction &CGF, 217*13fbcb42Sjoerg llvm::omp::ProcBindKind ProcBind, 218*13fbcb42Sjoerg SourceLocation Loc) override; 219*13fbcb42Sjoerg 220*13fbcb42Sjoerg /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 221*13fbcb42Sjoerg /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' 222*13fbcb42Sjoerg /// clause. 223*13fbcb42Sjoerg /// \param NumThreads An integer value of threads. 224*13fbcb42Sjoerg virtual void emitNumThreadsClause(CodeGenFunction &CGF, 225*13fbcb42Sjoerg llvm::Value *NumThreads, 226*13fbcb42Sjoerg SourceLocation Loc) override; 227*13fbcb42Sjoerg 228*13fbcb42Sjoerg /// This function ought to emit, in the general case, a call to 229*13fbcb42Sjoerg // the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed 230*13fbcb42Sjoerg // as these numbers are obtained through the PTX grid and block configuration. 231*13fbcb42Sjoerg /// \param NumTeams An integer expression of teams. 232*13fbcb42Sjoerg /// \param ThreadLimit An integer expression of threads. 233*13fbcb42Sjoerg void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, 234*13fbcb42Sjoerg const Expr *ThreadLimit, SourceLocation Loc) override; 235*13fbcb42Sjoerg 236*13fbcb42Sjoerg /// Emits inlined function for the specified OpenMP parallel 237*13fbcb42Sjoerg // directive. 238*13fbcb42Sjoerg /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 239*13fbcb42Sjoerg /// kmp_int32 BoundID, struct context_vars*). 240*13fbcb42Sjoerg /// \param D OpenMP directive. 241*13fbcb42Sjoerg /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 242*13fbcb42Sjoerg /// \param InnermostKind Kind of innermost directive (for simple directives it 243*13fbcb42Sjoerg /// is a directive itself, for combined - its innermost directive). 244*13fbcb42Sjoerg /// \param CodeGen Code generation sequence for the \a D directive. 245*13fbcb42Sjoerg llvm::Function * 246*13fbcb42Sjoerg emitParallelOutlinedFunction(const OMPExecutableDirective &D, 247*13fbcb42Sjoerg const VarDecl *ThreadIDVar, 248*13fbcb42Sjoerg OpenMPDirectiveKind InnermostKind, 249*13fbcb42Sjoerg const RegionCodeGenTy &CodeGen) override; 250*13fbcb42Sjoerg 251*13fbcb42Sjoerg /// Emits inlined function for the specified OpenMP teams 252*13fbcb42Sjoerg // directive. 253*13fbcb42Sjoerg /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 254*13fbcb42Sjoerg /// kmp_int32 BoundID, struct context_vars*). 255*13fbcb42Sjoerg /// \param D OpenMP directive. 256*13fbcb42Sjoerg /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 257*13fbcb42Sjoerg /// \param InnermostKind Kind of innermost directive (for simple directives it 258*13fbcb42Sjoerg /// is a directive itself, for combined - its innermost directive). 259*13fbcb42Sjoerg /// \param CodeGen Code generation sequence for the \a D directive. 260*13fbcb42Sjoerg llvm::Function * 261*13fbcb42Sjoerg emitTeamsOutlinedFunction(const OMPExecutableDirective &D, 262*13fbcb42Sjoerg const VarDecl *ThreadIDVar, 263*13fbcb42Sjoerg OpenMPDirectiveKind InnermostKind, 264*13fbcb42Sjoerg const RegionCodeGenTy &CodeGen) override; 265*13fbcb42Sjoerg 266*13fbcb42Sjoerg /// Emits code for teams call of the \a OutlinedFn with 267*13fbcb42Sjoerg /// variables captured in a record which address is stored in \a 268*13fbcb42Sjoerg /// CapturedStruct. 269*13fbcb42Sjoerg /// \param OutlinedFn Outlined function to be run by team masters. Type of 270*13fbcb42Sjoerg /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 271*13fbcb42Sjoerg /// \param CapturedVars A pointer to the record with the references to 272*13fbcb42Sjoerg /// variables used in \a OutlinedFn function. 273*13fbcb42Sjoerg /// 274*13fbcb42Sjoerg void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, 275*13fbcb42Sjoerg SourceLocation Loc, llvm::Function *OutlinedFn, 276*13fbcb42Sjoerg ArrayRef<llvm::Value *> CapturedVars) override; 277*13fbcb42Sjoerg 278*13fbcb42Sjoerg /// Emits code for parallel or serial call of the \a OutlinedFn with 279*13fbcb42Sjoerg /// variables captured in a record which address is stored in \a 280*13fbcb42Sjoerg /// CapturedStruct. 281*13fbcb42Sjoerg /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 282*13fbcb42Sjoerg /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 283*13fbcb42Sjoerg /// \param CapturedVars A pointer to the record with the references to 284*13fbcb42Sjoerg /// variables used in \a OutlinedFn function. 285*13fbcb42Sjoerg /// \param IfCond Condition in the associated 'if' clause, if it was 286*13fbcb42Sjoerg /// specified, nullptr otherwise. 287*13fbcb42Sjoerg void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 288*13fbcb42Sjoerg llvm::Function *OutlinedFn, 289*13fbcb42Sjoerg ArrayRef<llvm::Value *> CapturedVars, 290*13fbcb42Sjoerg const Expr *IfCond) override; 291*13fbcb42Sjoerg 292*13fbcb42Sjoerg /// Emit an implicit/explicit barrier for OpenMP threads. 293*13fbcb42Sjoerg /// \param Kind Directive for which this implicit barrier call must be 294*13fbcb42Sjoerg /// generated. Must be OMPD_barrier for explicit barrier generation. 295*13fbcb42Sjoerg /// \param EmitChecks true if need to emit checks for cancellation barriers. 296*13fbcb42Sjoerg /// \param ForceSimpleCall true simple barrier call must be emitted, false if 297*13fbcb42Sjoerg /// runtime class decides which one to emit (simple or with cancellation 298*13fbcb42Sjoerg /// checks). 299*13fbcb42Sjoerg /// 300*13fbcb42Sjoerg void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, 301*13fbcb42Sjoerg OpenMPDirectiveKind Kind, bool EmitChecks = true, 302*13fbcb42Sjoerg bool ForceSimpleCall = false) override; 303*13fbcb42Sjoerg 304*13fbcb42Sjoerg /// Emits a critical region. 305*13fbcb42Sjoerg /// \param CriticalName Name of the critical region. 306*13fbcb42Sjoerg /// \param CriticalOpGen Generator for the statement associated with the given 307*13fbcb42Sjoerg /// critical region. 308*13fbcb42Sjoerg /// \param Hint Value of the 'hint' clause (optional). 309*13fbcb42Sjoerg void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, 310*13fbcb42Sjoerg const RegionCodeGenTy &CriticalOpGen, 311*13fbcb42Sjoerg SourceLocation Loc, 312*13fbcb42Sjoerg const Expr *Hint = nullptr) override; 313*13fbcb42Sjoerg 314*13fbcb42Sjoerg /// Emit a code for reduction clause. 315*13fbcb42Sjoerg /// 316*13fbcb42Sjoerg /// \param Privates List of private copies for original reduction arguments. 317*13fbcb42Sjoerg /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. 318*13fbcb42Sjoerg /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. 319*13fbcb42Sjoerg /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' 320*13fbcb42Sjoerg /// or 'operator binop(LHS, RHS)'. 321*13fbcb42Sjoerg /// \param Options List of options for reduction codegen: 322*13fbcb42Sjoerg /// WithNowait true if parent directive has also nowait clause, false 323*13fbcb42Sjoerg /// otherwise. 324*13fbcb42Sjoerg /// SimpleReduction Emit reduction operation only. Used for omp simd 325*13fbcb42Sjoerg /// directive on the host. 326*13fbcb42Sjoerg /// ReductionKind The kind of reduction to perform. 327*13fbcb42Sjoerg virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, 328*13fbcb42Sjoerg ArrayRef<const Expr *> Privates, 329*13fbcb42Sjoerg ArrayRef<const Expr *> LHSExprs, 330*13fbcb42Sjoerg ArrayRef<const Expr *> RHSExprs, 331*13fbcb42Sjoerg ArrayRef<const Expr *> ReductionOps, 332*13fbcb42Sjoerg ReductionOptionsTy Options) override; 333*13fbcb42Sjoerg 334*13fbcb42Sjoerg /// Returns specified OpenMP runtime function for the current OpenMP 335*13fbcb42Sjoerg /// implementation. Specialized for the NVPTX device. 336*13fbcb42Sjoerg /// \param Function OpenMP runtime function. 337*13fbcb42Sjoerg /// \return Specified function. 338*13fbcb42Sjoerg llvm::FunctionCallee createNVPTXRuntimeFunction(unsigned Function); 339*13fbcb42Sjoerg 340*13fbcb42Sjoerg /// Translates the native parameter of outlined function if this is required 341*13fbcb42Sjoerg /// for target. 342*13fbcb42Sjoerg /// \param FD Field decl from captured record for the parameter. 343*13fbcb42Sjoerg /// \param NativeParam Parameter itself. 344*13fbcb42Sjoerg const VarDecl *translateParameter(const FieldDecl *FD, 345*13fbcb42Sjoerg const VarDecl *NativeParam) const override; 346*13fbcb42Sjoerg 347*13fbcb42Sjoerg /// Gets the address of the native argument basing on the address of the 348*13fbcb42Sjoerg /// target-specific parameter. 349*13fbcb42Sjoerg /// \param NativeParam Parameter itself. 350*13fbcb42Sjoerg /// \param TargetParam Corresponding target-specific parameter. 351*13fbcb42Sjoerg Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, 352*13fbcb42Sjoerg const VarDecl *TargetParam) const override; 353*13fbcb42Sjoerg 354*13fbcb42Sjoerg /// Emits call of the outlined function with the provided arguments, 355*13fbcb42Sjoerg /// translating these arguments to correct target-specific arguments. 356*13fbcb42Sjoerg void emitOutlinedFunctionCall( 357*13fbcb42Sjoerg CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, 358*13fbcb42Sjoerg ArrayRef<llvm::Value *> Args = llvm::None) const override; 359*13fbcb42Sjoerg 360*13fbcb42Sjoerg /// Emits OpenMP-specific function prolog. 361*13fbcb42Sjoerg /// Required for device constructs. 362*13fbcb42Sjoerg void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override; 363*13fbcb42Sjoerg 364*13fbcb42Sjoerg /// Gets the OpenMP-specific address of the local variable. 365*13fbcb42Sjoerg Address getAddressOfLocalVariable(CodeGenFunction &CGF, 366*13fbcb42Sjoerg const VarDecl *VD) override; 367*13fbcb42Sjoerg 368*13fbcb42Sjoerg /// Target codegen is specialized based on two data-sharing modes: CUDA, in 369*13fbcb42Sjoerg /// which the local variables are actually global threadlocal, and Generic, in 370*13fbcb42Sjoerg /// which the local variables are placed in global memory if they may escape 371*13fbcb42Sjoerg /// their declaration context. 372*13fbcb42Sjoerg enum DataSharingMode { 373*13fbcb42Sjoerg /// CUDA data sharing mode. 374*13fbcb42Sjoerg CUDA, 375*13fbcb42Sjoerg /// Generic data-sharing mode. 376*13fbcb42Sjoerg Generic, 377*13fbcb42Sjoerg }; 378*13fbcb42Sjoerg 379*13fbcb42Sjoerg /// Cleans up references to the objects in finished function. 380*13fbcb42Sjoerg /// 381*13fbcb42Sjoerg void functionFinished(CodeGenFunction &CGF) override; 382*13fbcb42Sjoerg 383*13fbcb42Sjoerg /// Choose a default value for the dist_schedule clause. 384*13fbcb42Sjoerg void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF, 385*13fbcb42Sjoerg const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind, 386*13fbcb42Sjoerg llvm::Value *&Chunk) const override; 387*13fbcb42Sjoerg 388*13fbcb42Sjoerg /// Choose a default value for the schedule clause. 389*13fbcb42Sjoerg void getDefaultScheduleAndChunk(CodeGenFunction &CGF, 390*13fbcb42Sjoerg const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, 391*13fbcb42Sjoerg const Expr *&ChunkExpr) const override; 392*13fbcb42Sjoerg 393*13fbcb42Sjoerg /// Adjust some parameters for the target-based directives, like addresses of 394*13fbcb42Sjoerg /// the variables captured by reference in lambdas. 395*13fbcb42Sjoerg void adjustTargetSpecificDataForLambdas( 396*13fbcb42Sjoerg CodeGenFunction &CGF, const OMPExecutableDirective &D) const override; 397*13fbcb42Sjoerg 398*13fbcb42Sjoerg /// Perform check on requires decl to ensure that target architecture 399*13fbcb42Sjoerg /// supports unified addressing 400*13fbcb42Sjoerg void processRequiresDirective(const OMPRequiresDecl *D) override; 401*13fbcb42Sjoerg 402*13fbcb42Sjoerg /// Returns default address space for the constant firstprivates, __constant__ 403*13fbcb42Sjoerg /// address space by default. 404*13fbcb42Sjoerg unsigned getDefaultFirstprivateAddressSpace() const override; 405*13fbcb42Sjoerg 406*13fbcb42Sjoerg /// Checks if the variable has associated OMPAllocateDeclAttr attribute with 407*13fbcb42Sjoerg /// the predefined allocator and translates it into the corresponding address 408*13fbcb42Sjoerg /// space. 409*13fbcb42Sjoerg bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) override; 410*13fbcb42Sjoerg 411*13fbcb42Sjoerg private: 412*13fbcb42Sjoerg /// Track the execution mode when codegening directives within a target 413*13fbcb42Sjoerg /// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the 414*13fbcb42Sjoerg /// target region and used by containing directives such as 'parallel' 415*13fbcb42Sjoerg /// to emit optimized code. 416*13fbcb42Sjoerg ExecutionMode CurrentExecutionMode = EM_Unknown; 417*13fbcb42Sjoerg 418*13fbcb42Sjoerg /// Check if the full runtime is required (default - yes). 419*13fbcb42Sjoerg bool RequiresFullRuntime = true; 420*13fbcb42Sjoerg 421*13fbcb42Sjoerg /// true if we're emitting the code for the target region and next parallel 422*13fbcb42Sjoerg /// region is L0 for sure. 423*13fbcb42Sjoerg bool IsInTargetMasterThreadRegion = false; 424*13fbcb42Sjoerg /// true if currently emitting code for target/teams/distribute region, false 425*13fbcb42Sjoerg /// - otherwise. 426*13fbcb42Sjoerg bool IsInTTDRegion = false; 427*13fbcb42Sjoerg /// true if we're definitely in the parallel region. 428*13fbcb42Sjoerg bool IsInParallelRegion = false; 429*13fbcb42Sjoerg 430*13fbcb42Sjoerg /// Map between an outlined function and its wrapper. 431*13fbcb42Sjoerg llvm::DenseMap<llvm::Function *, llvm::Function *> WrapperFunctionsMap; 432*13fbcb42Sjoerg 433*13fbcb42Sjoerg /// Emit function which wraps the outline parallel region 434*13fbcb42Sjoerg /// and controls the parameters which are passed to this function. 435*13fbcb42Sjoerg /// The wrapper ensures that the outlined function is called 436*13fbcb42Sjoerg /// with the correct arguments when data is shared. 437*13fbcb42Sjoerg llvm::Function *createParallelDataSharingWrapper( 438*13fbcb42Sjoerg llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D); 439*13fbcb42Sjoerg 440*13fbcb42Sjoerg /// The data for the single globalized variable. 441*13fbcb42Sjoerg struct MappedVarData { 442*13fbcb42Sjoerg /// Corresponding field in the global record. 443*13fbcb42Sjoerg const FieldDecl *FD = nullptr; 444*13fbcb42Sjoerg /// Corresponding address. 445*13fbcb42Sjoerg Address PrivateAddr = Address::invalid(); 446*13fbcb42Sjoerg /// true, if only one element is required (for latprivates in SPMD mode), 447*13fbcb42Sjoerg /// false, if need to create based on the warp-size. 448*13fbcb42Sjoerg bool IsOnePerTeam = false; 449*13fbcb42Sjoerg MappedVarData() = delete; 450*13fbcb42Sjoerg MappedVarData(const FieldDecl *FD, bool IsOnePerTeam = false) FDMappedVarData451*13fbcb42Sjoerg : FD(FD), IsOnePerTeam(IsOnePerTeam) {} 452*13fbcb42Sjoerg }; 453*13fbcb42Sjoerg /// The map of local variables to their addresses in the global memory. 454*13fbcb42Sjoerg using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>; 455*13fbcb42Sjoerg /// Set of the parameters passed by value escaping OpenMP context. 456*13fbcb42Sjoerg using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>; 457*13fbcb42Sjoerg struct FunctionData { 458*13fbcb42Sjoerg DeclToAddrMapTy LocalVarData; 459*13fbcb42Sjoerg llvm::Optional<DeclToAddrMapTy> SecondaryLocalVarData = llvm::None; 460*13fbcb42Sjoerg EscapedParamsTy EscapedParameters; 461*13fbcb42Sjoerg llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls; 462*13fbcb42Sjoerg llvm::SmallVector<llvm::Value *, 4> EscapedVariableLengthDeclsAddrs; 463*13fbcb42Sjoerg const RecordDecl *GlobalRecord = nullptr; 464*13fbcb42Sjoerg llvm::Optional<const RecordDecl *> SecondaryGlobalRecord = llvm::None; 465*13fbcb42Sjoerg llvm::Value *GlobalRecordAddr = nullptr; 466*13fbcb42Sjoerg llvm::Value *IsInSPMDModeFlag = nullptr; 467*13fbcb42Sjoerg std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams; 468*13fbcb42Sjoerg }; 469*13fbcb42Sjoerg /// Maps the function to the list of the globalized variables with their 470*13fbcb42Sjoerg /// addresses. 471*13fbcb42Sjoerg llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls; 472*13fbcb42Sjoerg /// List of records for the globalized variables in target/teams/distribute 473*13fbcb42Sjoerg /// contexts. Inner records are going to be joined into the single record, 474*13fbcb42Sjoerg /// while those resulting records are going to be joined into the single 475*13fbcb42Sjoerg /// union. This resulting union (one per CU) is the entry point for the static 476*13fbcb42Sjoerg /// memory management runtime functions. 477*13fbcb42Sjoerg struct GlobalPtrSizeRecsTy { 478*13fbcb42Sjoerg llvm::GlobalVariable *UseSharedMemory = nullptr; 479*13fbcb42Sjoerg llvm::GlobalVariable *RecSize = nullptr; 480*13fbcb42Sjoerg llvm::GlobalVariable *Buffer = nullptr; 481*13fbcb42Sjoerg SourceLocation Loc; 482*13fbcb42Sjoerg llvm::SmallVector<const RecordDecl *, 2> Records; 483*13fbcb42Sjoerg unsigned RegionCounter = 0; 484*13fbcb42Sjoerg }; 485*13fbcb42Sjoerg llvm::SmallVector<GlobalPtrSizeRecsTy, 8> GlobalizedRecords; 486*13fbcb42Sjoerg llvm::GlobalVariable *KernelTeamsReductionPtr = nullptr; 487*13fbcb42Sjoerg /// List of the records with the list of fields for the reductions across the 488*13fbcb42Sjoerg /// teams. Used to build the intermediate buffer for the fast teams 489*13fbcb42Sjoerg /// reductions. 490*13fbcb42Sjoerg /// All the records are gathered into a union `union.type` is created. 491*13fbcb42Sjoerg llvm::SmallVector<const RecordDecl *, 4> TeamsReductions; 492*13fbcb42Sjoerg /// Shared pointer for the global memory in the global memory buffer used for 493*13fbcb42Sjoerg /// the given kernel. 494*13fbcb42Sjoerg llvm::GlobalVariable *KernelStaticGlobalized = nullptr; 495*13fbcb42Sjoerg /// Pair of the Non-SPMD team and all reductions variables in this team 496*13fbcb42Sjoerg /// region. 497*13fbcb42Sjoerg std::pair<const Decl *, llvm::SmallVector<const ValueDecl *, 4>> 498*13fbcb42Sjoerg TeamAndReductions; 499*13fbcb42Sjoerg }; 500*13fbcb42Sjoerg 501*13fbcb42Sjoerg } // CodeGen namespace. 502*13fbcb42Sjoerg } // clang namespace. 503*13fbcb42Sjoerg 504*13fbcb42Sjoerg #endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMEGPU_H 505