1 //===- CoroInternal.h - Internal Coroutine interfaces ---------*- C++ -*---===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // Common definitions/declarations used internally by coroutine lowering passes. 9 //===----------------------------------------------------------------------===// 10 11 #ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H 12 #define LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H 13 14 #include "CoroInstr.h" 15 #include "llvm/IR/IRBuilder.h" 16 17 namespace llvm { 18 19 class CallGraph; 20 21 namespace coro { 22 23 bool declaresAnyIntrinsic(const Module &M); 24 bool declaresIntrinsics(const Module &M, 25 const std::initializer_list<StringRef>); 26 void replaceCoroFree(CoroIdInst *CoroId, bool Elide); 27 28 /// Attempts to rewrite the location operand of debug intrinsics in terms of 29 /// the coroutine frame pointer, folding pointer offsets into the DIExpression 30 /// of the intrinsic. 31 /// If the frame pointer is an Argument, store it into an alloca if 32 /// OptimizeFrame is false. 33 void salvageDebugInfo( 34 SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap, 35 DbgVariableIntrinsic &DVI, bool OptimizeFrame, bool IsEntryPoint); 36 void salvageDebugInfo( 37 SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap, DPValue &DPV, 38 bool OptimizeFrame, bool UseEntryValue); 39 40 // Keeps data and helper functions for lowering coroutine intrinsics. 41 struct LowererBase { 42 Module &TheModule; 43 LLVMContext &Context; 44 PointerType *const Int8Ptr; 45 FunctionType *const ResumeFnType; 46 ConstantPointerNull *const NullPtr; 47 48 LowererBase(Module &M); 49 Value *makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt); 50 }; 51 52 enum class ABI { 53 /// The "resume-switch" lowering, where there are separate resume and 54 /// destroy functions that are shared between all suspend points. The 55 /// coroutine frame implicitly stores the resume and destroy functions, 56 /// the current index, and any promise value. 57 Switch, 58 59 /// The "returned-continuation" lowering, where each suspend point creates a 60 /// single continuation function that is used for both resuming and 61 /// destroying. Does not support promises. 62 Retcon, 63 64 /// The "unique returned-continuation" lowering, where each suspend point 65 /// creates a single continuation function that is used for both resuming 66 /// and destroying. Does not support promises. The function is known to 67 /// suspend at most once during its execution, and the return value of 68 /// the continuation is void. 69 RetconOnce, 70 71 /// The "async continuation" lowering, where each suspend point creates a 72 /// single continuation function. The continuation function is available as an 73 /// intrinsic. 74 Async, 75 }; 76 77 // Holds structural Coroutine Intrinsics for a particular function and other 78 // values used during CoroSplit pass. 79 struct LLVM_LIBRARY_VISIBILITY Shape { 80 CoroBeginInst *CoroBegin; 81 SmallVector<AnyCoroEndInst *, 4> CoroEnds; 82 SmallVector<CoroSizeInst *, 2> CoroSizes; 83 SmallVector<CoroAlignInst *, 2> CoroAligns; 84 SmallVector<AnyCoroSuspendInst *, 4> CoroSuspends; 85 SmallVector<CallInst*, 2> SwiftErrorOps; 86 87 // Field indexes for special fields in the switch lowering. 88 struct SwitchFieldIndex { 89 enum { 90 Resume, 91 Destroy 92 93 // The promise field is always at a fixed offset from the start of 94 // frame given its type, but the index isn't a constant for all 95 // possible frames. 96 97 // The switch-index field isn't at a fixed offset or index, either; 98 // we just work it in where it fits best. 99 }; 100 }; 101 102 coro::ABI ABI; 103 104 StructType *FrameTy; 105 Align FrameAlign; 106 uint64_t FrameSize; 107 Value *FramePtr; 108 BasicBlock *AllocaSpillBlock; 109 110 /// This would only be true if optimization are enabled. 111 bool OptimizeFrame; 112 113 struct SwitchLoweringStorage { 114 SwitchInst *ResumeSwitch; 115 AllocaInst *PromiseAlloca; 116 BasicBlock *ResumeEntryBlock; 117 unsigned IndexField; 118 unsigned IndexAlign; 119 unsigned IndexOffset; 120 bool HasFinalSuspend; 121 bool HasUnwindCoroEnd; 122 }; 123 124 struct RetconLoweringStorage { 125 Function *ResumePrototype; 126 Function *Alloc; 127 Function *Dealloc; 128 BasicBlock *ReturnBlock; 129 bool IsFrameInlineInStorage; 130 }; 131 132 struct AsyncLoweringStorage { 133 Value *Context; 134 CallingConv::ID AsyncCC; 135 unsigned ContextArgNo; 136 uint64_t ContextHeaderSize; 137 uint64_t ContextAlignment; 138 uint64_t FrameOffset; // Start of the frame. 139 uint64_t ContextSize; // Includes frame size. 140 GlobalVariable *AsyncFuncPointer; 141 getContextAlignmentShape::AsyncLoweringStorage142 Align getContextAlignment() const { return Align(ContextAlignment); } 143 }; 144 145 union { 146 SwitchLoweringStorage SwitchLowering; 147 RetconLoweringStorage RetconLowering; 148 AsyncLoweringStorage AsyncLowering; 149 }; 150 getSwitchCoroIdShape151 CoroIdInst *getSwitchCoroId() const { 152 assert(ABI == coro::ABI::Switch); 153 return cast<CoroIdInst>(CoroBegin->getId()); 154 } 155 getRetconCoroIdShape156 AnyCoroIdRetconInst *getRetconCoroId() const { 157 assert(ABI == coro::ABI::Retcon || 158 ABI == coro::ABI::RetconOnce); 159 return cast<AnyCoroIdRetconInst>(CoroBegin->getId()); 160 } 161 getAsyncCoroIdShape162 CoroIdAsyncInst *getAsyncCoroId() const { 163 assert(ABI == coro::ABI::Async); 164 return cast<CoroIdAsyncInst>(CoroBegin->getId()); 165 } 166 getSwitchIndexFieldShape167 unsigned getSwitchIndexField() const { 168 assert(ABI == coro::ABI::Switch); 169 assert(FrameTy && "frame type not assigned"); 170 return SwitchLowering.IndexField; 171 } getIndexTypeShape172 IntegerType *getIndexType() const { 173 assert(ABI == coro::ABI::Switch); 174 assert(FrameTy && "frame type not assigned"); 175 return cast<IntegerType>(FrameTy->getElementType(getSwitchIndexField())); 176 } getIndexShape177 ConstantInt *getIndex(uint64_t Value) const { 178 return ConstantInt::get(getIndexType(), Value); 179 } 180 getSwitchResumePointerTypeShape181 PointerType *getSwitchResumePointerType() const { 182 assert(ABI == coro::ABI::Switch); 183 assert(FrameTy && "frame type not assigned"); 184 return cast<PointerType>(FrameTy->getElementType(SwitchFieldIndex::Resume)); 185 } 186 getResumeFunctionTypeShape187 FunctionType *getResumeFunctionType() const { 188 switch (ABI) { 189 case coro::ABI::Switch: 190 return FunctionType::get(Type::getVoidTy(FrameTy->getContext()), 191 PointerType::getUnqual(FrameTy->getContext()), 192 /*IsVarArg=*/false); 193 case coro::ABI::Retcon: 194 case coro::ABI::RetconOnce: 195 return RetconLowering.ResumePrototype->getFunctionType(); 196 case coro::ABI::Async: 197 // Not used. The function type depends on the active suspend. 198 return nullptr; 199 } 200 201 llvm_unreachable("Unknown coro::ABI enum"); 202 } 203 getRetconResultTypesShape204 ArrayRef<Type*> getRetconResultTypes() const { 205 assert(ABI == coro::ABI::Retcon || 206 ABI == coro::ABI::RetconOnce); 207 auto FTy = CoroBegin->getFunction()->getFunctionType(); 208 209 // The safety of all this is checked by checkWFRetconPrototype. 210 if (auto STy = dyn_cast<StructType>(FTy->getReturnType())) { 211 return STy->elements().slice(1); 212 } else { 213 return ArrayRef<Type*>(); 214 } 215 } 216 getRetconResumeTypesShape217 ArrayRef<Type*> getRetconResumeTypes() const { 218 assert(ABI == coro::ABI::Retcon || 219 ABI == coro::ABI::RetconOnce); 220 221 // The safety of all this is checked by checkWFRetconPrototype. 222 auto FTy = RetconLowering.ResumePrototype->getFunctionType(); 223 return FTy->params().slice(1); 224 } 225 getResumeFunctionCCShape226 CallingConv::ID getResumeFunctionCC() const { 227 switch (ABI) { 228 case coro::ABI::Switch: 229 return CallingConv::Fast; 230 231 case coro::ABI::Retcon: 232 case coro::ABI::RetconOnce: 233 return RetconLowering.ResumePrototype->getCallingConv(); 234 case coro::ABI::Async: 235 return AsyncLowering.AsyncCC; 236 } 237 llvm_unreachable("Unknown coro::ABI enum"); 238 } 239 getPromiseAllocaShape240 AllocaInst *getPromiseAlloca() const { 241 if (ABI == coro::ABI::Switch) 242 return SwitchLowering.PromiseAlloca; 243 return nullptr; 244 } 245 getInsertPtAfterFramePtrShape246 BasicBlock::iterator getInsertPtAfterFramePtr() const { 247 if (auto *I = dyn_cast<Instruction>(FramePtr)) { 248 BasicBlock::iterator It = std::next(I->getIterator()); 249 It.setHeadBit(true); // Copy pre-RemoveDIs behaviour. 250 return It; 251 } 252 return cast<Argument>(FramePtr)->getParent()->getEntryBlock().begin(); 253 } 254 255 /// Allocate memory according to the rules of the active lowering. 256 /// 257 /// \param CG - if non-null, will be updated for the new call 258 Value *emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const; 259 260 /// Deallocate memory according to the rules of the active lowering. 261 /// 262 /// \param CG - if non-null, will be updated for the new call 263 void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const; 264 265 Shape() = default; 266 explicit Shape(Function &F, bool OptimizeFrame = false) OptimizeFrameShape267 : OptimizeFrame(OptimizeFrame) { 268 buildFrom(F); 269 } 270 void buildFrom(Function &F); 271 }; 272 273 bool defaultMaterializable(Instruction &V); 274 void buildCoroutineFrame( 275 Function &F, Shape &Shape, 276 const std::function<bool(Instruction &)> &MaterializableCallback); 277 CallInst *createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, 278 ArrayRef<Value *> Arguments, IRBuilder<> &); 279 } // End namespace coro. 280 } // End namespace llvm 281 282 #endif 283