1 //===--- CGDeclCXX.cpp - Emit LLVM Code for C++ declarations --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code dealing with code generation of C++ declarations
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCXXABI.h"
14 #include "CGObjCRuntime.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/Attr.h"
19 #include "clang/Basic/LangOptions.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/IR/Intrinsics.h"
22 #include "llvm/IR/MDBuilder.h"
23 #include "llvm/Support/Path.h"
24 #include "llvm/Transforms/Utils/ModuleUtils.h"
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
EmitDeclInit(CodeGenFunction & CGF,const VarDecl & D,ConstantAddress DeclPtr)29 static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
30                          ConstantAddress DeclPtr) {
31   assert(
32       (D.hasGlobalStorage() ||
33        (D.hasLocalStorage() && CGF.getContext().getLangOpts().OpenCLCPlusPlus)) &&
34       "VarDecl must have global or local (in the case of OpenCL) storage!");
35   assert(!D.getType()->isReferenceType() &&
36          "Should not call EmitDeclInit on a reference!");
37 
38   QualType type = D.getType();
39   LValue lv = CGF.MakeAddrLValue(DeclPtr, type);
40 
41   const Expr *Init = D.getInit();
42   switch (CGF.getEvaluationKind(type)) {
43   case TEK_Scalar: {
44     CodeGenModule &CGM = CGF.CGM;
45     if (lv.isObjCStrong())
46       CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF, CGF.EmitScalarExpr(Init),
47                                                 DeclPtr, D.getTLSKind());
48     else if (lv.isObjCWeak())
49       CGM.getObjCRuntime().EmitObjCWeakAssign(CGF, CGF.EmitScalarExpr(Init),
50                                               DeclPtr);
51     else
52       CGF.EmitScalarInit(Init, &D, lv, false);
53     return;
54   }
55   case TEK_Complex:
56     CGF.EmitComplexExprIntoLValue(Init, lv, /*isInit*/ true);
57     return;
58   case TEK_Aggregate:
59     CGF.EmitAggExpr(Init,
60                     AggValueSlot::forLValue(lv, CGF, AggValueSlot::IsDestructed,
61                                             AggValueSlot::DoesNotNeedGCBarriers,
62                                             AggValueSlot::IsNotAliased,
63                                             AggValueSlot::DoesNotOverlap));
64     return;
65   }
66   llvm_unreachable("bad evaluation kind");
67 }
68 
69 /// Emit code to cause the destruction of the given variable with
70 /// static storage duration.
EmitDeclDestroy(CodeGenFunction & CGF,const VarDecl & D,ConstantAddress Addr)71 static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
72                             ConstantAddress Addr) {
73   // Honor __attribute__((no_destroy)) and bail instead of attempting
74   // to emit a reference to a possibly nonexistent destructor, which
75   // in turn can cause a crash. This will result in a global constructor
76   // that isn't balanced out by a destructor call as intended by the
77   // attribute. This also checks for -fno-c++-static-destructors and
78   // bails even if the attribute is not present.
79   QualType::DestructionKind DtorKind = D.needsDestruction(CGF.getContext());
80 
81   // FIXME:  __attribute__((cleanup)) ?
82 
83   switch (DtorKind) {
84   case QualType::DK_none:
85     return;
86 
87   case QualType::DK_cxx_destructor:
88     break;
89 
90   case QualType::DK_objc_strong_lifetime:
91   case QualType::DK_objc_weak_lifetime:
92   case QualType::DK_nontrivial_c_struct:
93     // We don't care about releasing objects during process teardown.
94     assert(!D.getTLSKind() && "should have rejected this");
95     return;
96   }
97 
98   llvm::FunctionCallee Func;
99   llvm::Constant *Argument;
100 
101   CodeGenModule &CGM = CGF.CGM;
102   QualType Type = D.getType();
103 
104   // Special-case non-array C++ destructors, if they have the right signature.
105   // Under some ABIs, destructors return this instead of void, and cannot be
106   // passed directly to __cxa_atexit if the target does not allow this
107   // mismatch.
108   const CXXRecordDecl *Record = Type->getAsCXXRecordDecl();
109   bool CanRegisterDestructor =
110       Record && (!CGM.getCXXABI().HasThisReturn(
111                      GlobalDecl(Record->getDestructor(), Dtor_Complete)) ||
112                  CGM.getCXXABI().canCallMismatchedFunctionType());
113   // If __cxa_atexit is disabled via a flag, a different helper function is
114   // generated elsewhere which uses atexit instead, and it takes the destructor
115   // directly.
116   bool UsingExternalHelper = !CGM.getCodeGenOpts().CXAAtExit;
117   if (Record && (CanRegisterDestructor || UsingExternalHelper)) {
118     assert(!Record->hasTrivialDestructor());
119     CXXDestructorDecl *Dtor = Record->getDestructor();
120 
121     Func = CGM.getAddrAndTypeOfCXXStructor(GlobalDecl(Dtor, Dtor_Complete));
122     if (CGF.getContext().getLangOpts().OpenCL) {
123       auto DestAS =
124           CGM.getTargetCodeGenInfo().getAddrSpaceOfCxaAtexitPtrParam();
125       auto DestTy = CGF.getTypes().ConvertType(Type)->getPointerTo(
126           CGM.getContext().getTargetAddressSpace(DestAS));
127       auto SrcAS = D.getType().getQualifiers().getAddressSpace();
128       if (DestAS == SrcAS)
129         Argument = llvm::ConstantExpr::getBitCast(Addr.getPointer(), DestTy);
130       else
131         // FIXME: On addr space mismatch we are passing NULL. The generation
132         // of the global destructor function should be adjusted accordingly.
133         Argument = llvm::ConstantPointerNull::get(DestTy);
134     } else {
135       Argument = llvm::ConstantExpr::getBitCast(
136           Addr.getPointer(), CGF.getTypes().ConvertType(Type)->getPointerTo());
137     }
138   // Otherwise, the standard logic requires a helper function.
139   } else {
140     Func = CodeGenFunction(CGM)
141            .generateDestroyHelper(Addr, Type, CGF.getDestroyer(DtorKind),
142                                   CGF.needsEHCleanup(DtorKind), &D);
143     Argument = llvm::Constant::getNullValue(CGF.Int8PtrTy);
144   }
145 
146   CGM.getCXXABI().registerGlobalDtor(CGF, D, Func, Argument);
147 }
148 
149 /// Emit code to cause the variable at the given address to be considered as
150 /// constant from this point onwards.
EmitDeclInvariant(CodeGenFunction & CGF,const VarDecl & D,llvm::Constant * Addr)151 static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D,
152                               llvm::Constant *Addr) {
153   return CGF.EmitInvariantStart(
154       Addr, CGF.getContext().getTypeSizeInChars(D.getType()));
155 }
156 
EmitInvariantStart(llvm::Constant * Addr,CharUnits Size)157 void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) {
158   // Do not emit the intrinsic if we're not optimizing.
159   if (!CGM.getCodeGenOpts().OptimizationLevel)
160     return;
161 
162   // Grab the llvm.invariant.start intrinsic.
163   llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
164   // Overloaded address space type.
165   llvm::Type *ObjectPtr[1] = {Int8PtrTy};
166   llvm::Function *InvariantStart = CGM.getIntrinsic(InvStartID, ObjectPtr);
167 
168   // Emit a call with the size in bytes of the object.
169   uint64_t Width = Size.getQuantity();
170   llvm::Value *Args[2] = { llvm::ConstantInt::getSigned(Int64Ty, Width),
171                            llvm::ConstantExpr::getBitCast(Addr, Int8PtrTy)};
172   Builder.CreateCall(InvariantStart, Args);
173 }
174 
EmitCXXGlobalVarDeclInit(const VarDecl & D,llvm::Constant * DeclPtr,bool PerformInit)175 void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
176                                                llvm::Constant *DeclPtr,
177                                                bool PerformInit) {
178 
179   const Expr *Init = D.getInit();
180   QualType T = D.getType();
181 
182   // The address space of a static local variable (DeclPtr) may be different
183   // from the address space of the "this" argument of the constructor. In that
184   // case, we need an addrspacecast before calling the constructor.
185   //
186   // struct StructWithCtor {
187   //   __device__ StructWithCtor() {...}
188   // };
189   // __device__ void foo() {
190   //   __shared__ StructWithCtor s;
191   //   ...
192   // }
193   //
194   // For example, in the above CUDA code, the static local variable s has a
195   // "shared" address space qualifier, but the constructor of StructWithCtor
196   // expects "this" in the "generic" address space.
197   unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T);
198   unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace();
199   if (ActualAddrSpace != ExpectedAddrSpace) {
200     llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T);
201     llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace);
202     DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy);
203   }
204 
205   ConstantAddress DeclAddr(DeclPtr, getContext().getDeclAlign(&D));
206 
207   if (!T->isReferenceType()) {
208     if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
209         D.hasAttr<OMPThreadPrivateDeclAttr>()) {
210       (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
211           &D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
212           PerformInit, this);
213     }
214     if (PerformInit)
215       EmitDeclInit(*this, D, DeclAddr);
216     if (CGM.isTypeConstant(D.getType(), true))
217       EmitDeclInvariant(*this, D, DeclPtr);
218     else
219       EmitDeclDestroy(*this, D, DeclAddr);
220     return;
221   }
222 
223   assert(PerformInit && "cannot have constant initializer which needs "
224          "destruction for reference");
225   RValue RV = EmitReferenceBindingToExpr(Init);
226   EmitStoreOfScalar(RV.getScalarVal(), DeclAddr, false, T);
227 }
228 
229 /// Create a stub function, suitable for being passed to atexit,
230 /// which passes the given address to the given destructor function.
createAtExitStub(const VarDecl & VD,llvm::FunctionCallee dtor,llvm::Constant * addr)231 llvm::Function *CodeGenFunction::createAtExitStub(const VarDecl &VD,
232                                                   llvm::FunctionCallee dtor,
233                                                   llvm::Constant *addr) {
234   // Get the destructor function type, void(*)(void).
235   llvm::FunctionType *ty = llvm::FunctionType::get(CGM.VoidTy, false);
236   SmallString<256> FnName;
237   {
238     llvm::raw_svector_ostream Out(FnName);
239     CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(&VD, Out);
240   }
241 
242   const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
243   llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
244       ty, FnName.str(), FI, VD.getLocation());
245 
246   CodeGenFunction CGF(CGM);
247 
248   CGF.StartFunction(GlobalDecl(&VD, DynamicInitKind::AtExit),
249                     CGM.getContext().VoidTy, fn, FI, FunctionArgList());
250 
251   llvm::CallInst *call = CGF.Builder.CreateCall(dtor, addr);
252 
253   // Make sure the call and the callee agree on calling convention.
254   if (auto *dtorFn = dyn_cast<llvm::Function>(
255           dtor.getCallee()->stripPointerCastsAndAliases()))
256     call->setCallingConv(dtorFn->getCallingConv());
257 
258   CGF.FinishFunction();
259 
260   return fn;
261 }
262 
263 /// Register a global destructor using the C atexit runtime function.
registerGlobalDtorWithAtExit(const VarDecl & VD,llvm::FunctionCallee dtor,llvm::Constant * addr)264 void CodeGenFunction::registerGlobalDtorWithAtExit(const VarDecl &VD,
265                                                    llvm::FunctionCallee dtor,
266                                                    llvm::Constant *addr) {
267   // Create a function which calls the destructor.
268   llvm::Constant *dtorStub = createAtExitStub(VD, dtor, addr);
269   registerGlobalDtorWithAtExit(dtorStub);
270 }
271 
registerGlobalDtorWithAtExit(llvm::Constant * dtorStub)272 void CodeGenFunction::registerGlobalDtorWithAtExit(llvm::Constant *dtorStub) {
273   // extern "C" int atexit(void (*f)(void));
274   assert(cast<llvm::Function>(dtorStub)->getFunctionType() ==
275              llvm::FunctionType::get(CGM.VoidTy, false) &&
276          "Argument to atexit has a wrong type.");
277 
278   llvm::FunctionType *atexitTy =
279       llvm::FunctionType::get(IntTy, dtorStub->getType(), false);
280 
281   llvm::FunctionCallee atexit =
282       CGM.CreateRuntimeFunction(atexitTy, "atexit", llvm::AttributeList(),
283                                 /*Local=*/true);
284   if (llvm::Function *atexitFn = dyn_cast<llvm::Function>(atexit.getCallee()))
285     atexitFn->setDoesNotThrow();
286 
287   EmitNounwindRuntimeCall(atexit, dtorStub);
288 }
289 
290 llvm::Value *
unregisterGlobalDtorWithUnAtExit(llvm::Function * dtorStub)291 CodeGenFunction::unregisterGlobalDtorWithUnAtExit(llvm::Function *dtorStub) {
292   // The unatexit subroutine unregisters __dtor functions that were previously
293   // registered by the atexit subroutine. If the referenced function is found,
294   // it is removed from the list of functions that are called at normal program
295   // termination and the unatexit returns a value of 0, otherwise a non-zero
296   // value is returned.
297   //
298   // extern "C" int unatexit(void (*f)(void));
299   assert(dtorStub->getFunctionType() ==
300              llvm::FunctionType::get(CGM.VoidTy, false) &&
301          "Argument to unatexit has a wrong type.");
302 
303   llvm::FunctionType *unatexitTy =
304       llvm::FunctionType::get(IntTy, {dtorStub->getType()}, /*isVarArg=*/false);
305 
306   llvm::FunctionCallee unatexit =
307       CGM.CreateRuntimeFunction(unatexitTy, "unatexit", llvm::AttributeList());
308 
309   cast<llvm::Function>(unatexit.getCallee())->setDoesNotThrow();
310 
311   return EmitNounwindRuntimeCall(unatexit, dtorStub);
312 }
313 
EmitCXXGuardedInit(const VarDecl & D,llvm::GlobalVariable * DeclPtr,bool PerformInit)314 void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
315                                          llvm::GlobalVariable *DeclPtr,
316                                          bool PerformInit) {
317   // If we've been asked to forbid guard variables, emit an error now.
318   // This diagnostic is hard-coded for Darwin's use case;  we can find
319   // better phrasing if someone else needs it.
320   if (CGM.getCodeGenOpts().ForbidGuardVariables)
321     CGM.Error(D.getLocation(),
322               "this initialization requires a guard variable, which "
323               "the kernel does not support");
324 
325   CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit);
326 }
327 
EmitCXXGuardedInitBranch(llvm::Value * NeedsInit,llvm::BasicBlock * InitBlock,llvm::BasicBlock * NoInitBlock,GuardKind Kind,const VarDecl * D)328 void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
329                                                llvm::BasicBlock *InitBlock,
330                                                llvm::BasicBlock *NoInitBlock,
331                                                GuardKind Kind,
332                                                const VarDecl *D) {
333   assert((Kind == GuardKind::TlsGuard || D) && "no guarded variable");
334 
335   // A guess at how many times we will enter the initialization of a
336   // variable, depending on the kind of variable.
337   static const uint64_t InitsPerTLSVar = 1024;
338   static const uint64_t InitsPerLocalVar = 1024 * 1024;
339 
340   llvm::MDNode *Weights;
341   if (Kind == GuardKind::VariableGuard && !D->isLocalVarDecl()) {
342     // For non-local variables, don't apply any weighting for now. Due to our
343     // use of COMDATs, we expect there to be at most one initialization of the
344     // variable per DSO, but we have no way to know how many DSOs will try to
345     // initialize the variable.
346     Weights = nullptr;
347   } else {
348     uint64_t NumInits;
349     // FIXME: For the TLS case, collect and use profiling information to
350     // determine a more accurate brach weight.
351     if (Kind == GuardKind::TlsGuard || D->getTLSKind())
352       NumInits = InitsPerTLSVar;
353     else
354       NumInits = InitsPerLocalVar;
355 
356     // The probability of us entering the initializer is
357     //   1 / (total number of times we attempt to initialize the variable).
358     llvm::MDBuilder MDHelper(CGM.getLLVMContext());
359     Weights = MDHelper.createBranchWeights(1, NumInits - 1);
360   }
361 
362   Builder.CreateCondBr(NeedsInit, InitBlock, NoInitBlock, Weights);
363 }
364 
CreateGlobalInitOrCleanUpFunction(llvm::FunctionType * FTy,const Twine & Name,const CGFunctionInfo & FI,SourceLocation Loc,bool TLS,bool IsExternalLinkage)365 llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction(
366     llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI,
367     SourceLocation Loc, bool TLS, bool IsExternalLinkage) {
368   llvm::Function *Fn = llvm::Function::Create(
369       FTy,
370       IsExternalLinkage ? llvm::GlobalValue::ExternalLinkage
371                         : llvm::GlobalValue::InternalLinkage,
372       Name, &getModule());
373 
374   if (!getLangOpts().AppleKext && !TLS) {
375     // Set the section if needed.
376     if (const char *Section = getTarget().getStaticInitSectionSpecifier())
377       Fn->setSection(Section);
378   }
379 
380   if (Fn->hasInternalLinkage())
381     SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
382 
383   Fn->setCallingConv(getRuntimeCC());
384 
385   if (!getLangOpts().Exceptions)
386     Fn->setDoesNotThrow();
387 
388   if (getLangOpts().Sanitize.has(SanitizerKind::Address) &&
389       !isInSanitizerBlacklist(SanitizerKind::Address, Fn, Loc))
390     Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
391 
392   if (getLangOpts().Sanitize.has(SanitizerKind::KernelAddress) &&
393       !isInSanitizerBlacklist(SanitizerKind::KernelAddress, Fn, Loc))
394     Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
395 
396   if (getLangOpts().Sanitize.has(SanitizerKind::HWAddress) &&
397       !isInSanitizerBlacklist(SanitizerKind::HWAddress, Fn, Loc))
398     Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
399 
400   if (getLangOpts().Sanitize.has(SanitizerKind::KernelHWAddress) &&
401       !isInSanitizerBlacklist(SanitizerKind::KernelHWAddress, Fn, Loc))
402     Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
403 
404   if (getLangOpts().Sanitize.has(SanitizerKind::MemTag) &&
405       !isInSanitizerBlacklist(SanitizerKind::MemTag, Fn, Loc))
406     Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
407 
408   if (getLangOpts().Sanitize.has(SanitizerKind::Thread) &&
409       !isInSanitizerBlacklist(SanitizerKind::Thread, Fn, Loc))
410     Fn->addFnAttr(llvm::Attribute::SanitizeThread);
411 
412   if (getLangOpts().Sanitize.has(SanitizerKind::Memory) &&
413       !isInSanitizerBlacklist(SanitizerKind::Memory, Fn, Loc))
414     Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
415 
416   if (getLangOpts().Sanitize.has(SanitizerKind::KernelMemory) &&
417       !isInSanitizerBlacklist(SanitizerKind::KernelMemory, Fn, Loc))
418     Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
419 
420   if (getLangOpts().Sanitize.has(SanitizerKind::SafeStack) &&
421       !isInSanitizerBlacklist(SanitizerKind::SafeStack, Fn, Loc))
422     Fn->addFnAttr(llvm::Attribute::SafeStack);
423 
424   if (getLangOpts().Sanitize.has(SanitizerKind::ShadowCallStack) &&
425       !isInSanitizerBlacklist(SanitizerKind::ShadowCallStack, Fn, Loc))
426     Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
427 
428   auto RASignKind = getLangOpts().getSignReturnAddressScope();
429   if (RASignKind != LangOptions::SignReturnAddressScopeKind::None) {
430     Fn->addFnAttr("sign-return-address",
431                   RASignKind == LangOptions::SignReturnAddressScopeKind::All
432                       ? "all"
433                       : "non-leaf");
434     auto RASignKey = getLangOpts().getSignReturnAddressKey();
435     Fn->addFnAttr("sign-return-address-key",
436                   RASignKey == LangOptions::SignReturnAddressKeyKind::AKey
437                       ? "a_key"
438                       : "b_key");
439   }
440 
441   if (getLangOpts().BranchTargetEnforcement)
442     Fn->addFnAttr("branch-target-enforcement");
443 
444   return Fn;
445 }
446 
447 /// Create a global pointer to a function that will initialize a global
448 /// variable.  The user has requested that this pointer be emitted in a specific
449 /// section.
EmitPointerToInitFunc(const VarDecl * D,llvm::GlobalVariable * GV,llvm::Function * InitFunc,InitSegAttr * ISA)450 void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D,
451                                           llvm::GlobalVariable *GV,
452                                           llvm::Function *InitFunc,
453                                           InitSegAttr *ISA) {
454   llvm::GlobalVariable *PtrArray = new llvm::GlobalVariable(
455       TheModule, InitFunc->getType(), /*isConstant=*/true,
456       llvm::GlobalValue::PrivateLinkage, InitFunc, "__cxx_init_fn_ptr");
457   PtrArray->setSection(ISA->getSection());
458   addUsedGlobal(PtrArray);
459 
460   // If the GV is already in a comdat group, then we have to join it.
461   if (llvm::Comdat *C = GV->getComdat())
462     PtrArray->setComdat(C);
463 }
464 
465 void
EmitCXXGlobalVarDeclInitFunc(const VarDecl * D,llvm::GlobalVariable * Addr,bool PerformInit)466 CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
467                                             llvm::GlobalVariable *Addr,
468                                             bool PerformInit) {
469 
470   // According to E.2.3.1 in CUDA-7.5 Programming guide: __device__,
471   // __constant__ and __shared__ variables defined in namespace scope,
472   // that are of class type, cannot have a non-empty constructor. All
473   // the checks have been done in Sema by now. Whatever initializers
474   // are allowed are empty and we just need to ignore them here.
475   if (getLangOpts().CUDAIsDevice && !getLangOpts().GPUAllowDeviceInit &&
476       (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
477        D->hasAttr<CUDASharedAttr>()))
478     return;
479 
480   if (getLangOpts().OpenMP &&
481       getOpenMPRuntime().emitDeclareTargetVarDefinition(D, Addr, PerformInit))
482     return;
483 
484   // Check if we've already initialized this decl.
485   auto I = DelayedCXXInitPosition.find(D);
486   if (I != DelayedCXXInitPosition.end() && I->second == ~0U)
487     return;
488 
489   llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
490   SmallString<256> FnName;
491   {
492     llvm::raw_svector_ostream Out(FnName);
493     getCXXABI().getMangleContext().mangleDynamicInitializer(D, Out);
494   }
495 
496   // Create a variable initialization function.
497   llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
498       FTy, FnName.str(), getTypes().arrangeNullaryFunction(), D->getLocation());
499 
500   auto *ISA = D->getAttr<InitSegAttr>();
501   CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr,
502                                                           PerformInit);
503 
504   llvm::GlobalVariable *COMDATKey =
505       supportsCOMDAT() && D->isExternallyVisible() ? Addr : nullptr;
506 
507   if (D->getTLSKind()) {
508     // FIXME: Should we support init_priority for thread_local?
509     // FIXME: We only need to register one __cxa_thread_atexit function for the
510     // entire TU.
511     CXXThreadLocalInits.push_back(Fn);
512     CXXThreadLocalInitVars.push_back(D);
513   } else if (PerformInit && ISA) {
514     EmitPointerToInitFunc(D, Addr, Fn, ISA);
515   } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
516     OrderGlobalInits Key(IPA->getPriority(), PrioritizedCXXGlobalInits.size());
517     PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn));
518   } else if (isTemplateInstantiation(D->getTemplateSpecializationKind()) ||
519              getContext().GetGVALinkageForVariable(D) == GVA_DiscardableODR) {
520     // C++ [basic.start.init]p2:
521     //   Definitions of explicitly specialized class template static data
522     //   members have ordered initialization. Other class template static data
523     //   members (i.e., implicitly or explicitly instantiated specializations)
524     //   have unordered initialization.
525     //
526     // As a consequence, we can put them into their own llvm.global_ctors entry.
527     //
528     // If the global is externally visible, put the initializer into a COMDAT
529     // group with the global being initialized.  On most platforms, this is a
530     // minor startup time optimization.  In the MS C++ ABI, there are no guard
531     // variables, so this COMDAT key is required for correctness.
532     AddGlobalCtor(Fn, 65535, COMDATKey);
533     if (getTarget().getCXXABI().isMicrosoft() && COMDATKey) {
534       // In The MS C++, MS add template static data member in the linker
535       // drective.
536       addUsedGlobal(COMDATKey);
537     }
538   } else if (D->hasAttr<SelectAnyAttr>()) {
539     // SelectAny globals will be comdat-folded. Put the initializer into a
540     // COMDAT group associated with the global, so the initializers get folded
541     // too.
542     AddGlobalCtor(Fn, 65535, COMDATKey);
543   } else {
544     I = DelayedCXXInitPosition.find(D); // Re-do lookup in case of re-hash.
545     if (I == DelayedCXXInitPosition.end()) {
546       CXXGlobalInits.push_back(Fn);
547     } else if (I->second != ~0U) {
548       assert(I->second < CXXGlobalInits.size() &&
549              CXXGlobalInits[I->second] == nullptr);
550       CXXGlobalInits[I->second] = Fn;
551     }
552   }
553 
554   // Remember that we already emitted the initializer for this global.
555   DelayedCXXInitPosition[D] = ~0U;
556 }
557 
EmitCXXThreadLocalInitFunc()558 void CodeGenModule::EmitCXXThreadLocalInitFunc() {
559   getCXXABI().EmitThreadLocalInitFuncs(
560       *this, CXXThreadLocals, CXXThreadLocalInits, CXXThreadLocalInitVars);
561 
562   CXXThreadLocalInits.clear();
563   CXXThreadLocalInitVars.clear();
564   CXXThreadLocals.clear();
565 }
566 
getTransformedFileName(llvm::Module & M)567 static SmallString<128> getTransformedFileName(llvm::Module &M) {
568   SmallString<128> FileName = llvm::sys::path::filename(M.getName());
569 
570   if (FileName.empty())
571     FileName = "<null>";
572 
573   for (size_t i = 0; i < FileName.size(); ++i) {
574     // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens
575     // to be the set of C preprocessing numbers.
576     if (!isPreprocessingNumberBody(FileName[i]))
577       FileName[i] = '_';
578   }
579 
580   return FileName;
581 }
582 
583 void
EmitCXXGlobalInitFunc()584 CodeGenModule::EmitCXXGlobalInitFunc() {
585   while (!CXXGlobalInits.empty() && !CXXGlobalInits.back())
586     CXXGlobalInits.pop_back();
587 
588   if (CXXGlobalInits.empty() && PrioritizedCXXGlobalInits.empty())
589     return;
590 
591   const bool UseSinitAndSterm = getCXXABI().useSinitAndSterm();
592   if (UseSinitAndSterm) {
593     GlobalUniqueModuleId = getUniqueModuleId(&getModule());
594 
595     // FIXME: We need to figure out what to hash on or encode into the unique ID
596     // we need.
597     if (GlobalUniqueModuleId.compare("") == 0)
598       llvm::report_fatal_error(
599           "cannot produce a unique identifier for this module"
600           " based on strong external symbols");
601     GlobalUniqueModuleId = GlobalUniqueModuleId.substr(1);
602   }
603 
604   llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
605   const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
606 
607   // Create our global prioritized initialization function.
608   if (!PrioritizedCXXGlobalInits.empty()) {
609     assert(!UseSinitAndSterm && "Prioritized sinit and sterm functions are not"
610                                 " supported yet.");
611 
612     SmallVector<llvm::Function *, 8> LocalCXXGlobalInits;
613     llvm::array_pod_sort(PrioritizedCXXGlobalInits.begin(),
614                          PrioritizedCXXGlobalInits.end());
615     // Iterate over "chunks" of ctors with same priority and emit each chunk
616     // into separate function. Note - everything is sorted first by priority,
617     // second - by lex order, so we emit ctor functions in proper order.
618     for (SmallVectorImpl<GlobalInitData >::iterator
619            I = PrioritizedCXXGlobalInits.begin(),
620            E = PrioritizedCXXGlobalInits.end(); I != E; ) {
621       SmallVectorImpl<GlobalInitData >::iterator
622         PrioE = std::upper_bound(I + 1, E, *I, GlobalInitPriorityCmp());
623 
624       LocalCXXGlobalInits.clear();
625       unsigned Priority = I->first.priority;
626       // Compute the function suffix from priority. Prepend with zeroes to make
627       // sure the function names are also ordered as priorities.
628       std::string PrioritySuffix = llvm::utostr(Priority);
629       // Priority is always <= 65535 (enforced by sema).
630       PrioritySuffix = std::string(6-PrioritySuffix.size(), '0')+PrioritySuffix;
631       llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
632           FTy, "_GLOBAL__I_" + PrioritySuffix, FI);
633 
634       for (; I < PrioE; ++I)
635         LocalCXXGlobalInits.push_back(I->second);
636 
637       CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, LocalCXXGlobalInits);
638       AddGlobalCtor(Fn, Priority);
639     }
640     PrioritizedCXXGlobalInits.clear();
641   }
642 
643   if (UseSinitAndSterm && CXXGlobalInits.empty())
644     return;
645 
646   // Create our global initialization function.
647   SmallString<128> FuncName;
648   bool IsExternalLinkage = false;
649   if (UseSinitAndSterm) {
650     llvm::Twine("__sinit80000000_clang_", GlobalUniqueModuleId)
651         .toVector(FuncName);
652     IsExternalLinkage = true;
653   } else {
654     // Include the filename in the symbol name. Including "sub_" matches gcc
655     // and makes sure these symbols appear lexicographically behind the symbols
656     // with priority emitted above.
657     llvm::Twine("_GLOBAL__sub_I_", getTransformedFileName(getModule()))
658         .toVector(FuncName);
659   }
660 
661   llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
662       FTy, FuncName, FI, SourceLocation(), false /* TLS */,
663       IsExternalLinkage);
664 
665   CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXGlobalInits);
666   AddGlobalCtor(Fn);
667 
668   // In OpenCL global init functions must be converted to kernels in order to
669   // be able to launch them from the host.
670   // FIXME: Some more work might be needed to handle destructors correctly.
671   // Current initialization function makes use of function pointers callbacks.
672   // We can't support function pointers especially between host and device.
673   // However it seems global destruction has little meaning without any
674   // dynamic resource allocation on the device and program scope variables are
675   // destroyed by the runtime when program is released.
676   if (getLangOpts().OpenCL) {
677     GenOpenCLArgMetadata(Fn);
678     Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
679   }
680 
681   if (getLangOpts().HIP) {
682     Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
683     Fn->addFnAttr("device-init");
684   }
685 
686   CXXGlobalInits.clear();
687 }
688 
EmitCXXGlobalCleanUpFunc()689 void CodeGenModule::EmitCXXGlobalCleanUpFunc() {
690   if (CXXGlobalDtorsOrStermFinalizers.empty())
691     return;
692 
693   llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
694   const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
695 
696   // Create our global cleanup function.
697   llvm::Function *Fn = nullptr;
698   if (getCXXABI().useSinitAndSterm()) {
699     if (GlobalUniqueModuleId.empty()) {
700       GlobalUniqueModuleId = getUniqueModuleId(&getModule());
701       // FIXME: We need to figure out what to hash on or encode into the unique
702       // ID we need.
703       if (GlobalUniqueModuleId.compare("") == 0)
704         llvm::report_fatal_error(
705             "cannot produce a unique identifier for this module"
706             " based on strong external symbols");
707       GlobalUniqueModuleId = GlobalUniqueModuleId.substr(1);
708     }
709 
710     Fn = CreateGlobalInitOrCleanUpFunction(
711         FTy, llvm::Twine("__sterm80000000_clang_", GlobalUniqueModuleId), FI,
712         SourceLocation(), false /* TLS */, true /* IsExternalLinkage */);
713   } else {
714     Fn = CreateGlobalInitOrCleanUpFunction(FTy, "_GLOBAL__D_a", FI);
715   }
716 
717   CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
718       Fn, CXXGlobalDtorsOrStermFinalizers);
719   AddGlobalDtor(Fn);
720   CXXGlobalDtorsOrStermFinalizers.clear();
721 }
722 
723 /// Emit the code necessary to initialize the given global variable.
GenerateCXXGlobalVarDeclInitFunc(llvm::Function * Fn,const VarDecl * D,llvm::GlobalVariable * Addr,bool PerformInit)724 void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
725                                                        const VarDecl *D,
726                                                  llvm::GlobalVariable *Addr,
727                                                        bool PerformInit) {
728   // Check if we need to emit debug info for variable initializer.
729   if (D->hasAttr<NoDebugAttr>())
730     DebugInfo = nullptr; // disable debug info indefinitely for this function
731 
732   CurEHLocation = D->getBeginLoc();
733 
734   StartFunction(GlobalDecl(D, DynamicInitKind::Initializer),
735                 getContext().VoidTy, Fn, getTypes().arrangeNullaryFunction(),
736                 FunctionArgList(), D->getLocation(),
737                 D->getInit()->getExprLoc());
738 
739   // Use guarded initialization if the global variable is weak. This
740   // occurs for, e.g., instantiated static data members and
741   // definitions explicitly marked weak.
742   //
743   // Also use guarded initialization for a variable with dynamic TLS and
744   // unordered initialization. (If the initialization is ordered, the ABI
745   // layer will guard the whole-TU initialization for us.)
746   if (Addr->hasWeakLinkage() || Addr->hasLinkOnceLinkage() ||
747       (D->getTLSKind() == VarDecl::TLS_Dynamic &&
748        isTemplateInstantiation(D->getTemplateSpecializationKind()))) {
749     EmitCXXGuardedInit(*D, Addr, PerformInit);
750   } else {
751     EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit);
752   }
753 
754   FinishFunction();
755 }
756 
757 void
GenerateCXXGlobalInitFunc(llvm::Function * Fn,ArrayRef<llvm::Function * > Decls,ConstantAddress Guard)758 CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
759                                            ArrayRef<llvm::Function *> Decls,
760                                            ConstantAddress Guard) {
761   {
762     auto NL = ApplyDebugLocation::CreateEmpty(*this);
763     StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
764                   getTypes().arrangeNullaryFunction(), FunctionArgList());
765     // Emit an artificial location for this function.
766     auto AL = ApplyDebugLocation::CreateArtificial(*this);
767 
768     llvm::BasicBlock *ExitBlock = nullptr;
769     if (Guard.isValid()) {
770       // If we have a guard variable, check whether we've already performed
771       // these initializations. This happens for TLS initialization functions.
772       llvm::Value *GuardVal = Builder.CreateLoad(Guard);
773       llvm::Value *Uninit = Builder.CreateIsNull(GuardVal,
774                                                  "guard.uninitialized");
775       llvm::BasicBlock *InitBlock = createBasicBlock("init");
776       ExitBlock = createBasicBlock("exit");
777       EmitCXXGuardedInitBranch(Uninit, InitBlock, ExitBlock,
778                                GuardKind::TlsGuard, nullptr);
779       EmitBlock(InitBlock);
780       // Mark as initialized before initializing anything else. If the
781       // initializers use previously-initialized thread_local vars, that's
782       // probably supposed to be OK, but the standard doesn't say.
783       Builder.CreateStore(llvm::ConstantInt::get(GuardVal->getType(),1), Guard);
784 
785       // The guard variable can't ever change again.
786       EmitInvariantStart(
787           Guard.getPointer(),
788           CharUnits::fromQuantity(
789               CGM.getDataLayout().getTypeAllocSize(GuardVal->getType())));
790     }
791 
792     RunCleanupsScope Scope(*this);
793 
794     // When building in Objective-C++ ARC mode, create an autorelease pool
795     // around the global initializers.
796     if (getLangOpts().ObjCAutoRefCount && getLangOpts().CPlusPlus) {
797       llvm::Value *token = EmitObjCAutoreleasePoolPush();
798       EmitObjCAutoreleasePoolCleanup(token);
799     }
800 
801     for (unsigned i = 0, e = Decls.size(); i != e; ++i)
802       if (Decls[i])
803         EmitRuntimeCall(Decls[i]);
804 
805     Scope.ForceCleanup();
806 
807     if (ExitBlock) {
808       Builder.CreateBr(ExitBlock);
809       EmitBlock(ExitBlock);
810     }
811   }
812 
813   FinishFunction();
814 }
815 
GenerateCXXGlobalCleanUpFunc(llvm::Function * Fn,const std::vector<std::tuple<llvm::FunctionType *,llvm::WeakTrackingVH,llvm::Constant * >> & DtorsOrStermFinalizers)816 void CodeGenFunction::GenerateCXXGlobalCleanUpFunc(
817     llvm::Function *Fn,
818     const std::vector<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
819                                  llvm::Constant *>> &DtorsOrStermFinalizers) {
820   {
821     auto NL = ApplyDebugLocation::CreateEmpty(*this);
822     StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
823                   getTypes().arrangeNullaryFunction(), FunctionArgList());
824     // Emit an artificial location for this function.
825     auto AL = ApplyDebugLocation::CreateArtificial(*this);
826 
827     // Emit the cleanups, in reverse order from construction.
828     for (unsigned i = 0, e = DtorsOrStermFinalizers.size(); i != e; ++i) {
829       llvm::FunctionType *CalleeTy;
830       llvm::Value *Callee;
831       llvm::Constant *Arg;
832       std::tie(CalleeTy, Callee, Arg) = DtorsOrStermFinalizers[e - i - 1];
833 
834       llvm::CallInst *CI = nullptr;
835       if (Arg == nullptr) {
836         assert(
837             CGM.getCXXABI().useSinitAndSterm() &&
838             "Arg could not be nullptr unless using sinit and sterm functions.");
839         CI = Builder.CreateCall(CalleeTy, Callee);
840       } else
841         CI = Builder.CreateCall(CalleeTy, Callee, Arg);
842 
843       // Make sure the call and the callee agree on calling convention.
844       if (llvm::Function *F = dyn_cast<llvm::Function>(Callee))
845         CI->setCallingConv(F->getCallingConv());
846     }
847   }
848 
849   FinishFunction();
850 }
851 
852 /// generateDestroyHelper - Generates a helper function which, when
853 /// invoked, destroys the given object.  The address of the object
854 /// should be in global memory.
generateDestroyHelper(Address addr,QualType type,Destroyer * destroyer,bool useEHCleanupForArray,const VarDecl * VD)855 llvm::Function *CodeGenFunction::generateDestroyHelper(
856     Address addr, QualType type, Destroyer *destroyer,
857     bool useEHCleanupForArray, const VarDecl *VD) {
858   FunctionArgList args;
859   ImplicitParamDecl Dst(getContext(), getContext().VoidPtrTy,
860                         ImplicitParamDecl::Other);
861   args.push_back(&Dst);
862 
863   const CGFunctionInfo &FI =
864     CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, args);
865   llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
866   llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
867       FTy, "__cxx_global_array_dtor", FI, VD->getLocation());
868 
869   CurEHLocation = VD->getBeginLoc();
870 
871   StartFunction(VD, getContext().VoidTy, fn, FI, args);
872 
873   emitDestroy(addr, type, destroyer, useEHCleanupForArray);
874 
875   FinishFunction();
876 
877   return fn;
878 }
879