1 //===--- CGDeclCXX.cpp - Emit LLVM Code for C++ declarations --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code dealing with code generation of C++ declarations
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "CGCXXABI.h"
14 #include "CGObjCRuntime.h"
15 #include "CGOpenMPRuntime.h"
16 #include "CodeGenFunction.h"
17 #include "TargetInfo.h"
18 #include "clang/AST/Attr.h"
19 #include "clang/Basic/LangOptions.h"
20 #include "llvm/ADT/StringExtras.h"
21 #include "llvm/IR/Intrinsics.h"
22 #include "llvm/IR/MDBuilder.h"
23 #include "llvm/Support/Path.h"
24
25 using namespace clang;
26 using namespace CodeGen;
27
EmitDeclInit(CodeGenFunction & CGF,const VarDecl & D,ConstantAddress DeclPtr)28 static void EmitDeclInit(CodeGenFunction &CGF, const VarDecl &D,
29 ConstantAddress DeclPtr) {
30 assert(
31 (D.hasGlobalStorage() ||
32 (D.hasLocalStorage() && CGF.getContext().getLangOpts().OpenCLCPlusPlus)) &&
33 "VarDecl must have global or local (in the case of OpenCL) storage!");
34 assert(!D.getType()->isReferenceType() &&
35 "Should not call EmitDeclInit on a reference!");
36
37 QualType type = D.getType();
38 LValue lv = CGF.MakeAddrLValue(DeclPtr, type);
39
40 const Expr *Init = D.getInit();
41 switch (CGF.getEvaluationKind(type)) {
42 case TEK_Scalar: {
43 CodeGenModule &CGM = CGF.CGM;
44 if (lv.isObjCStrong())
45 CGM.getObjCRuntime().EmitObjCGlobalAssign(CGF, CGF.EmitScalarExpr(Init),
46 DeclPtr, D.getTLSKind());
47 else if (lv.isObjCWeak())
48 CGM.getObjCRuntime().EmitObjCWeakAssign(CGF, CGF.EmitScalarExpr(Init),
49 DeclPtr);
50 else
51 CGF.EmitScalarInit(Init, &D, lv, false);
52 return;
53 }
54 case TEK_Complex:
55 CGF.EmitComplexExprIntoLValue(Init, lv, /*isInit*/ true);
56 return;
57 case TEK_Aggregate:
58 CGF.EmitAggExpr(Init,
59 AggValueSlot::forLValue(lv, CGF, AggValueSlot::IsDestructed,
60 AggValueSlot::DoesNotNeedGCBarriers,
61 AggValueSlot::IsNotAliased,
62 AggValueSlot::DoesNotOverlap));
63 return;
64 }
65 llvm_unreachable("bad evaluation kind");
66 }
67
68 /// Emit code to cause the destruction of the given variable with
69 /// static storage duration.
EmitDeclDestroy(CodeGenFunction & CGF,const VarDecl & D,ConstantAddress Addr)70 static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
71 ConstantAddress Addr) {
72 // Honor __attribute__((no_destroy)) and bail instead of attempting
73 // to emit a reference to a possibly nonexistent destructor, which
74 // in turn can cause a crash. This will result in a global constructor
75 // that isn't balanced out by a destructor call as intended by the
76 // attribute. This also checks for -fno-c++-static-destructors and
77 // bails even if the attribute is not present.
78 QualType::DestructionKind DtorKind = D.needsDestruction(CGF.getContext());
79
80 // FIXME: __attribute__((cleanup)) ?
81
82 switch (DtorKind) {
83 case QualType::DK_none:
84 return;
85
86 case QualType::DK_cxx_destructor:
87 break;
88
89 case QualType::DK_objc_strong_lifetime:
90 case QualType::DK_objc_weak_lifetime:
91 case QualType::DK_nontrivial_c_struct:
92 // We don't care about releasing objects during process teardown.
93 assert(!D.getTLSKind() && "should have rejected this");
94 return;
95 }
96
97 llvm::FunctionCallee Func;
98 llvm::Constant *Argument;
99
100 CodeGenModule &CGM = CGF.CGM;
101 QualType Type = D.getType();
102
103 // Special-case non-array C++ destructors, if they have the right signature.
104 // Under some ABIs, destructors return this instead of void, and cannot be
105 // passed directly to __cxa_atexit if the target does not allow this
106 // mismatch.
107 const CXXRecordDecl *Record = Type->getAsCXXRecordDecl();
108 bool CanRegisterDestructor =
109 Record && (!CGM.getCXXABI().HasThisReturn(
110 GlobalDecl(Record->getDestructor(), Dtor_Complete)) ||
111 CGM.getCXXABI().canCallMismatchedFunctionType());
112 // If __cxa_atexit is disabled via a flag, a different helper function is
113 // generated elsewhere which uses atexit instead, and it takes the destructor
114 // directly.
115 bool UsingExternalHelper = !CGM.getCodeGenOpts().CXAAtExit;
116 if (Record && (CanRegisterDestructor || UsingExternalHelper)) {
117 assert(!Record->hasTrivialDestructor());
118 CXXDestructorDecl *Dtor = Record->getDestructor();
119
120 Func = CGM.getAddrAndTypeOfCXXStructor(GlobalDecl(Dtor, Dtor_Complete));
121 if (CGF.getContext().getLangOpts().OpenCL) {
122 auto DestAS =
123 CGM.getTargetCodeGenInfo().getAddrSpaceOfCxaAtexitPtrParam();
124 auto DestTy = CGF.getTypes().ConvertType(Type)->getPointerTo(
125 CGM.getContext().getTargetAddressSpace(DestAS));
126 auto SrcAS = D.getType().getQualifiers().getAddressSpace();
127 if (DestAS == SrcAS)
128 Argument = llvm::ConstantExpr::getBitCast(Addr.getPointer(), DestTy);
129 else
130 // FIXME: On addr space mismatch we are passing NULL. The generation
131 // of the global destructor function should be adjusted accordingly.
132 Argument = llvm::ConstantPointerNull::get(DestTy);
133 } else {
134 Argument = llvm::ConstantExpr::getBitCast(
135 Addr.getPointer(), CGF.getTypes().ConvertType(Type)->getPointerTo());
136 }
137 // Otherwise, the standard logic requires a helper function.
138 } else {
139 Func = CodeGenFunction(CGM)
140 .generateDestroyHelper(Addr, Type, CGF.getDestroyer(DtorKind),
141 CGF.needsEHCleanup(DtorKind), &D);
142 Argument = llvm::Constant::getNullValue(CGF.Int8PtrTy);
143 }
144
145 CGM.getCXXABI().registerGlobalDtor(CGF, D, Func, Argument);
146 }
147
148 /// Emit code to cause the variable at the given address to be considered as
149 /// constant from this point onwards.
EmitDeclInvariant(CodeGenFunction & CGF,const VarDecl & D,llvm::Constant * Addr)150 static void EmitDeclInvariant(CodeGenFunction &CGF, const VarDecl &D,
151 llvm::Constant *Addr) {
152 return CGF.EmitInvariantStart(
153 Addr, CGF.getContext().getTypeSizeInChars(D.getType()));
154 }
155
EmitInvariantStart(llvm::Constant * Addr,CharUnits Size)156 void CodeGenFunction::EmitInvariantStart(llvm::Constant *Addr, CharUnits Size) {
157 // Do not emit the intrinsic if we're not optimizing.
158 if (!CGM.getCodeGenOpts().OptimizationLevel)
159 return;
160
161 // Grab the llvm.invariant.start intrinsic.
162 llvm::Intrinsic::ID InvStartID = llvm::Intrinsic::invariant_start;
163 // Overloaded address space type.
164 llvm::Type *ObjectPtr[1] = {Int8PtrTy};
165 llvm::Function *InvariantStart = CGM.getIntrinsic(InvStartID, ObjectPtr);
166
167 // Emit a call with the size in bytes of the object.
168 uint64_t Width = Size.getQuantity();
169 llvm::Value *Args[2] = { llvm::ConstantInt::getSigned(Int64Ty, Width),
170 llvm::ConstantExpr::getBitCast(Addr, Int8PtrTy)};
171 Builder.CreateCall(InvariantStart, Args);
172 }
173
EmitCXXGlobalVarDeclInit(const VarDecl & D,llvm::Constant * DeclPtr,bool PerformInit)174 void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
175 llvm::Constant *DeclPtr,
176 bool PerformInit) {
177
178 const Expr *Init = D.getInit();
179 QualType T = D.getType();
180
181 // The address space of a static local variable (DeclPtr) may be different
182 // from the address space of the "this" argument of the constructor. In that
183 // case, we need an addrspacecast before calling the constructor.
184 //
185 // struct StructWithCtor {
186 // __device__ StructWithCtor() {...}
187 // };
188 // __device__ void foo() {
189 // __shared__ StructWithCtor s;
190 // ...
191 // }
192 //
193 // For example, in the above CUDA code, the static local variable s has a
194 // "shared" address space qualifier, but the constructor of StructWithCtor
195 // expects "this" in the "generic" address space.
196 unsigned ExpectedAddrSpace = getContext().getTargetAddressSpace(T);
197 unsigned ActualAddrSpace = DeclPtr->getType()->getPointerAddressSpace();
198 if (ActualAddrSpace != ExpectedAddrSpace) {
199 llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(T);
200 llvm::PointerType *PTy = llvm::PointerType::get(LTy, ExpectedAddrSpace);
201 DeclPtr = llvm::ConstantExpr::getAddrSpaceCast(DeclPtr, PTy);
202 }
203
204 ConstantAddress DeclAddr(DeclPtr, getContext().getDeclAlign(&D));
205
206 if (!T->isReferenceType()) {
207 if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
208 D.hasAttr<OMPThreadPrivateDeclAttr>()) {
209 (void)CGM.getOpenMPRuntime().emitThreadPrivateVarDefinition(
210 &D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
211 PerformInit, this);
212 }
213 if (PerformInit)
214 EmitDeclInit(*this, D, DeclAddr);
215 if (CGM.isTypeConstant(D.getType(), true))
216 EmitDeclInvariant(*this, D, DeclPtr);
217 else
218 EmitDeclDestroy(*this, D, DeclAddr);
219 return;
220 }
221
222 assert(PerformInit && "cannot have constant initializer which needs "
223 "destruction for reference");
224 RValue RV = EmitReferenceBindingToExpr(Init);
225 EmitStoreOfScalar(RV.getScalarVal(), DeclAddr, false, T);
226 }
227
228 /// Create a stub function, suitable for being passed to atexit,
229 /// which passes the given address to the given destructor function.
createAtExitStub(const VarDecl & VD,llvm::FunctionCallee dtor,llvm::Constant * addr)230 llvm::Function *CodeGenFunction::createAtExitStub(const VarDecl &VD,
231 llvm::FunctionCallee dtor,
232 llvm::Constant *addr) {
233 // Get the destructor function type, void(*)(void).
234 llvm::FunctionType *ty = llvm::FunctionType::get(CGM.VoidTy, false);
235 SmallString<256> FnName;
236 {
237 llvm::raw_svector_ostream Out(FnName);
238 CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(&VD, Out);
239 }
240
241 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction();
242 llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
243 ty, FnName.str(), FI, VD.getLocation());
244
245 CodeGenFunction CGF(CGM);
246
247 CGF.StartFunction(GlobalDecl(&VD, DynamicInitKind::AtExit),
248 CGM.getContext().VoidTy, fn, FI, FunctionArgList(),
249 VD.getLocation(), VD.getInit()->getExprLoc());
250 // Emit an artificial location for this function.
251 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
252
253 llvm::CallInst *call = CGF.Builder.CreateCall(dtor, addr);
254
255 // Make sure the call and the callee agree on calling convention.
256 if (auto *dtorFn = dyn_cast<llvm::Function>(
257 dtor.getCallee()->stripPointerCastsAndAliases()))
258 call->setCallingConv(dtorFn->getCallingConv());
259
260 CGF.FinishFunction();
261
262 return fn;
263 }
264
265 /// Create a stub function, suitable for being passed to __pt_atexit_np,
266 /// which passes the given address to the given destructor function.
createTLSAtExitStub(const VarDecl & D,llvm::FunctionCallee Dtor,llvm::Constant * Addr,llvm::FunctionCallee & AtExit)267 llvm::Function *CodeGenFunction::createTLSAtExitStub(
268 const VarDecl &D, llvm::FunctionCallee Dtor, llvm::Constant *Addr,
269 llvm::FunctionCallee &AtExit) {
270 SmallString<256> FnName;
271 {
272 llvm::raw_svector_ostream Out(FnName);
273 CGM.getCXXABI().getMangleContext().mangleDynamicAtExitDestructor(&D, Out);
274 }
275
276 const CGFunctionInfo &FI = CGM.getTypes().arrangeLLVMFunctionInfo(
277 getContext().IntTy, /*instanceMethod=*/false, /*chainCall=*/false,
278 {getContext().IntTy}, FunctionType::ExtInfo(), {}, RequiredArgs::All);
279
280 // Get the stub function type, int(*)(int,...).
281 llvm::FunctionType *StubTy =
282 llvm::FunctionType::get(CGM.IntTy, {CGM.IntTy}, true);
283
284 llvm::Function *DtorStub = CGM.CreateGlobalInitOrCleanUpFunction(
285 StubTy, FnName.str(), FI, D.getLocation());
286
287 CodeGenFunction CGF(CGM);
288
289 FunctionArgList Args;
290 ImplicitParamDecl IPD(CGM.getContext(), CGM.getContext().IntTy,
291 ImplicitParamDecl::Other);
292 Args.push_back(&IPD);
293 QualType ResTy = CGM.getContext().IntTy;
294
295 CGF.StartFunction(GlobalDecl(&D, DynamicInitKind::AtExit), ResTy, DtorStub,
296 FI, Args, D.getLocation(), D.getInit()->getExprLoc());
297
298 // Emit an artificial location for this function.
299 auto AL = ApplyDebugLocation::CreateArtificial(CGF);
300
301 llvm::CallInst *call = CGF.Builder.CreateCall(Dtor, Addr);
302
303 // Make sure the call and the callee agree on calling convention.
304 if (auto *DtorFn = dyn_cast<llvm::Function>(
305 Dtor.getCallee()->stripPointerCastsAndAliases()))
306 call->setCallingConv(DtorFn->getCallingConv());
307
308 // Return 0 from function
309 CGF.Builder.CreateStore(llvm::Constant::getNullValue(CGM.IntTy),
310 CGF.ReturnValue);
311
312 CGF.FinishFunction();
313
314 return DtorStub;
315 }
316
317 /// Register a global destructor using the C atexit runtime function.
registerGlobalDtorWithAtExit(const VarDecl & VD,llvm::FunctionCallee dtor,llvm::Constant * addr)318 void CodeGenFunction::registerGlobalDtorWithAtExit(const VarDecl &VD,
319 llvm::FunctionCallee dtor,
320 llvm::Constant *addr) {
321 // Create a function which calls the destructor.
322 llvm::Constant *dtorStub = createAtExitStub(VD, dtor, addr);
323 registerGlobalDtorWithAtExit(dtorStub);
324 }
325
registerGlobalDtorWithAtExit(llvm::Constant * dtorStub)326 void CodeGenFunction::registerGlobalDtorWithAtExit(llvm::Constant *dtorStub) {
327 // extern "C" int atexit(void (*f)(void));
328 assert(dtorStub->getType() ==
329 llvm::PointerType::get(
330 llvm::FunctionType::get(CGM.VoidTy, false),
331 dtorStub->getType()->getPointerAddressSpace()) &&
332 "Argument to atexit has a wrong type.");
333
334 llvm::FunctionType *atexitTy =
335 llvm::FunctionType::get(IntTy, dtorStub->getType(), false);
336
337 llvm::FunctionCallee atexit =
338 CGM.CreateRuntimeFunction(atexitTy, "atexit", llvm::AttributeList(),
339 /*Local=*/true);
340 if (llvm::Function *atexitFn = dyn_cast<llvm::Function>(atexit.getCallee()))
341 atexitFn->setDoesNotThrow();
342
343 EmitNounwindRuntimeCall(atexit, dtorStub);
344 }
345
346 llvm::Value *
unregisterGlobalDtorWithUnAtExit(llvm::Constant * dtorStub)347 CodeGenFunction::unregisterGlobalDtorWithUnAtExit(llvm::Constant *dtorStub) {
348 // The unatexit subroutine unregisters __dtor functions that were previously
349 // registered by the atexit subroutine. If the referenced function is found,
350 // it is removed from the list of functions that are called at normal program
351 // termination and the unatexit returns a value of 0, otherwise a non-zero
352 // value is returned.
353 //
354 // extern "C" int unatexit(void (*f)(void));
355 assert(dtorStub->getType() ==
356 llvm::PointerType::get(
357 llvm::FunctionType::get(CGM.VoidTy, false),
358 dtorStub->getType()->getPointerAddressSpace()) &&
359 "Argument to unatexit has a wrong type.");
360
361 llvm::FunctionType *unatexitTy =
362 llvm::FunctionType::get(IntTy, {dtorStub->getType()}, /*isVarArg=*/false);
363
364 llvm::FunctionCallee unatexit =
365 CGM.CreateRuntimeFunction(unatexitTy, "unatexit", llvm::AttributeList());
366
367 cast<llvm::Function>(unatexit.getCallee())->setDoesNotThrow();
368
369 return EmitNounwindRuntimeCall(unatexit, dtorStub);
370 }
371
EmitCXXGuardedInit(const VarDecl & D,llvm::GlobalVariable * DeclPtr,bool PerformInit)372 void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
373 llvm::GlobalVariable *DeclPtr,
374 bool PerformInit) {
375 // If we've been asked to forbid guard variables, emit an error now.
376 // This diagnostic is hard-coded for Darwin's use case; we can find
377 // better phrasing if someone else needs it.
378 if (CGM.getCodeGenOpts().ForbidGuardVariables)
379 CGM.Error(D.getLocation(),
380 "this initialization requires a guard variable, which "
381 "the kernel does not support");
382
383 CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit);
384 }
385
EmitCXXGuardedInitBranch(llvm::Value * NeedsInit,llvm::BasicBlock * InitBlock,llvm::BasicBlock * NoInitBlock,GuardKind Kind,const VarDecl * D)386 void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
387 llvm::BasicBlock *InitBlock,
388 llvm::BasicBlock *NoInitBlock,
389 GuardKind Kind,
390 const VarDecl *D) {
391 assert((Kind == GuardKind::TlsGuard || D) && "no guarded variable");
392
393 // A guess at how many times we will enter the initialization of a
394 // variable, depending on the kind of variable.
395 static const uint64_t InitsPerTLSVar = 1024;
396 static const uint64_t InitsPerLocalVar = 1024 * 1024;
397
398 llvm::MDNode *Weights;
399 if (Kind == GuardKind::VariableGuard && !D->isLocalVarDecl()) {
400 // For non-local variables, don't apply any weighting for now. Due to our
401 // use of COMDATs, we expect there to be at most one initialization of the
402 // variable per DSO, but we have no way to know how many DSOs will try to
403 // initialize the variable.
404 Weights = nullptr;
405 } else {
406 uint64_t NumInits;
407 // FIXME: For the TLS case, collect and use profiling information to
408 // determine a more accurate brach weight.
409 if (Kind == GuardKind::TlsGuard || D->getTLSKind())
410 NumInits = InitsPerTLSVar;
411 else
412 NumInits = InitsPerLocalVar;
413
414 // The probability of us entering the initializer is
415 // 1 / (total number of times we attempt to initialize the variable).
416 llvm::MDBuilder MDHelper(CGM.getLLVMContext());
417 Weights = MDHelper.createBranchWeights(1, NumInits - 1);
418 }
419
420 Builder.CreateCondBr(NeedsInit, InitBlock, NoInitBlock, Weights);
421 }
422
CreateGlobalInitOrCleanUpFunction(llvm::FunctionType * FTy,const Twine & Name,const CGFunctionInfo & FI,SourceLocation Loc,bool TLS)423 llvm::Function *CodeGenModule::CreateGlobalInitOrCleanUpFunction(
424 llvm::FunctionType *FTy, const Twine &Name, const CGFunctionInfo &FI,
425 SourceLocation Loc, bool TLS) {
426 llvm::Function *Fn = llvm::Function::Create(
427 FTy, llvm::GlobalValue::InternalLinkage, Name, &getModule());
428
429 if (!getLangOpts().AppleKext && !TLS) {
430 // Set the section if needed.
431 if (const char *Section = getTarget().getStaticInitSectionSpecifier())
432 Fn->setSection(Section);
433 }
434
435 SetInternalFunctionAttributes(GlobalDecl(), Fn, FI);
436
437 Fn->setCallingConv(getRuntimeCC());
438
439 if (!getLangOpts().Exceptions)
440 Fn->setDoesNotThrow();
441
442 if (getLangOpts().Sanitize.has(SanitizerKind::Address) &&
443 !isInNoSanitizeList(SanitizerKind::Address, Fn, Loc))
444 Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
445
446 if (getLangOpts().Sanitize.has(SanitizerKind::KernelAddress) &&
447 !isInNoSanitizeList(SanitizerKind::KernelAddress, Fn, Loc))
448 Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
449
450 if (getLangOpts().Sanitize.has(SanitizerKind::HWAddress) &&
451 !isInNoSanitizeList(SanitizerKind::HWAddress, Fn, Loc))
452 Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
453
454 if (getLangOpts().Sanitize.has(SanitizerKind::KernelHWAddress) &&
455 !isInNoSanitizeList(SanitizerKind::KernelHWAddress, Fn, Loc))
456 Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
457
458 if (getLangOpts().Sanitize.has(SanitizerKind::MemTag) &&
459 !isInNoSanitizeList(SanitizerKind::MemTag, Fn, Loc))
460 Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
461
462 if (getLangOpts().Sanitize.has(SanitizerKind::Thread) &&
463 !isInNoSanitizeList(SanitizerKind::Thread, Fn, Loc))
464 Fn->addFnAttr(llvm::Attribute::SanitizeThread);
465
466 if (getLangOpts().Sanitize.has(SanitizerKind::Memory) &&
467 !isInNoSanitizeList(SanitizerKind::Memory, Fn, Loc))
468 Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
469
470 if (getLangOpts().Sanitize.has(SanitizerKind::KernelMemory) &&
471 !isInNoSanitizeList(SanitizerKind::KernelMemory, Fn, Loc))
472 Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
473
474 if (getLangOpts().Sanitize.has(SanitizerKind::SafeStack) &&
475 !isInNoSanitizeList(SanitizerKind::SafeStack, Fn, Loc))
476 Fn->addFnAttr(llvm::Attribute::SafeStack);
477
478 if (getLangOpts().Sanitize.has(SanitizerKind::ShadowCallStack) &&
479 !isInNoSanitizeList(SanitizerKind::ShadowCallStack, Fn, Loc))
480 Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
481
482 return Fn;
483 }
484
485 /// Create a global pointer to a function that will initialize a global
486 /// variable. The user has requested that this pointer be emitted in a specific
487 /// section.
EmitPointerToInitFunc(const VarDecl * D,llvm::GlobalVariable * GV,llvm::Function * InitFunc,InitSegAttr * ISA)488 void CodeGenModule::EmitPointerToInitFunc(const VarDecl *D,
489 llvm::GlobalVariable *GV,
490 llvm::Function *InitFunc,
491 InitSegAttr *ISA) {
492 llvm::GlobalVariable *PtrArray = new llvm::GlobalVariable(
493 TheModule, InitFunc->getType(), /*isConstant=*/true,
494 llvm::GlobalValue::PrivateLinkage, InitFunc, "__cxx_init_fn_ptr");
495 PtrArray->setSection(ISA->getSection());
496 addUsedGlobal(PtrArray);
497
498 // If the GV is already in a comdat group, then we have to join it.
499 if (llvm::Comdat *C = GV->getComdat())
500 PtrArray->setComdat(C);
501 }
502
503 void
EmitCXXGlobalVarDeclInitFunc(const VarDecl * D,llvm::GlobalVariable * Addr,bool PerformInit)504 CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
505 llvm::GlobalVariable *Addr,
506 bool PerformInit) {
507
508 // According to E.2.3.1 in CUDA-7.5 Programming guide: __device__,
509 // __constant__ and __shared__ variables defined in namespace scope,
510 // that are of class type, cannot have a non-empty constructor. All
511 // the checks have been done in Sema by now. Whatever initializers
512 // are allowed are empty and we just need to ignore them here.
513 if (getLangOpts().CUDAIsDevice && !getLangOpts().GPUAllowDeviceInit &&
514 (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
515 D->hasAttr<CUDASharedAttr>()))
516 return;
517
518 if (getLangOpts().OpenMP &&
519 getOpenMPRuntime().emitDeclareTargetVarDefinition(D, Addr, PerformInit))
520 return;
521
522 // Check if we've already initialized this decl.
523 auto I = DelayedCXXInitPosition.find(D);
524 if (I != DelayedCXXInitPosition.end() && I->second == ~0U)
525 return;
526
527 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
528 SmallString<256> FnName;
529 {
530 llvm::raw_svector_ostream Out(FnName);
531 getCXXABI().getMangleContext().mangleDynamicInitializer(D, Out);
532 }
533
534 // Create a variable initialization function.
535 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
536 FTy, FnName.str(), getTypes().arrangeNullaryFunction(), D->getLocation());
537
538 auto *ISA = D->getAttr<InitSegAttr>();
539 CodeGenFunction(*this).GenerateCXXGlobalVarDeclInitFunc(Fn, D, Addr,
540 PerformInit);
541
542 llvm::GlobalVariable *COMDATKey =
543 supportsCOMDAT() && D->isExternallyVisible() ? Addr : nullptr;
544
545 if (D->getTLSKind()) {
546 // FIXME: Should we support init_priority for thread_local?
547 // FIXME: We only need to register one __cxa_thread_atexit function for the
548 // entire TU.
549 CXXThreadLocalInits.push_back(Fn);
550 CXXThreadLocalInitVars.push_back(D);
551 } else if (PerformInit && ISA) {
552 EmitPointerToInitFunc(D, Addr, Fn, ISA);
553 } else if (auto *IPA = D->getAttr<InitPriorityAttr>()) {
554 OrderGlobalInitsOrStermFinalizers Key(IPA->getPriority(),
555 PrioritizedCXXGlobalInits.size());
556 PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn));
557 } else if (isTemplateInstantiation(D->getTemplateSpecializationKind()) ||
558 getContext().GetGVALinkageForVariable(D) == GVA_DiscardableODR ||
559 D->hasAttr<SelectAnyAttr>()) {
560 // C++ [basic.start.init]p2:
561 // Definitions of explicitly specialized class template static data
562 // members have ordered initialization. Other class template static data
563 // members (i.e., implicitly or explicitly instantiated specializations)
564 // have unordered initialization.
565 //
566 // As a consequence, we can put them into their own llvm.global_ctors entry.
567 //
568 // If the global is externally visible, put the initializer into a COMDAT
569 // group with the global being initialized. On most platforms, this is a
570 // minor startup time optimization. In the MS C++ ABI, there are no guard
571 // variables, so this COMDAT key is required for correctness.
572 //
573 // SelectAny globals will be comdat-folded. Put the initializer into a
574 // COMDAT group associated with the global, so the initializers get folded
575 // too.
576
577 AddGlobalCtor(Fn, 65535, COMDATKey);
578 if (COMDATKey && (getTriple().isOSBinFormatELF() ||
579 getTarget().getCXXABI().isMicrosoft())) {
580 // When COMDAT is used on ELF or in the MS C++ ABI, the key must be in
581 // llvm.used to prevent linker GC.
582 addUsedGlobal(COMDATKey);
583 }
584
585 // If we used a COMDAT key for the global ctor, the init function can be
586 // discarded if the global ctor entry is discarded.
587 // FIXME: Do we need to restrict this to ELF and Wasm?
588 llvm::Comdat *C = Addr->getComdat();
589 if (COMDATKey && C &&
590 (getTarget().getTriple().isOSBinFormatELF() ||
591 getTarget().getTriple().isOSBinFormatWasm())) {
592 Fn->setComdat(C);
593 }
594 } else {
595 I = DelayedCXXInitPosition.find(D); // Re-do lookup in case of re-hash.
596 if (I == DelayedCXXInitPosition.end()) {
597 CXXGlobalInits.push_back(Fn);
598 } else if (I->second != ~0U) {
599 assert(I->second < CXXGlobalInits.size() &&
600 CXXGlobalInits[I->second] == nullptr);
601 CXXGlobalInits[I->second] = Fn;
602 }
603 }
604
605 // Remember that we already emitted the initializer for this global.
606 DelayedCXXInitPosition[D] = ~0U;
607 }
608
EmitCXXThreadLocalInitFunc()609 void CodeGenModule::EmitCXXThreadLocalInitFunc() {
610 getCXXABI().EmitThreadLocalInitFuncs(
611 *this, CXXThreadLocals, CXXThreadLocalInits, CXXThreadLocalInitVars);
612
613 CXXThreadLocalInits.clear();
614 CXXThreadLocalInitVars.clear();
615 CXXThreadLocals.clear();
616 }
617
getTransformedFileName(llvm::Module & M)618 static SmallString<128> getTransformedFileName(llvm::Module &M) {
619 SmallString<128> FileName = llvm::sys::path::filename(M.getName());
620
621 if (FileName.empty())
622 FileName = "<null>";
623
624 for (size_t i = 0; i < FileName.size(); ++i) {
625 // Replace everything that's not [a-zA-Z0-9._] with a _. This set happens
626 // to be the set of C preprocessing numbers.
627 if (!isPreprocessingNumberBody(FileName[i]))
628 FileName[i] = '_';
629 }
630
631 return FileName;
632 }
633
getPrioritySuffix(unsigned int Priority)634 static std::string getPrioritySuffix(unsigned int Priority) {
635 assert(Priority <= 65535 && "Priority should always be <= 65535.");
636
637 // Compute the function suffix from priority. Prepend with zeroes to make
638 // sure the function names are also ordered as priorities.
639 std::string PrioritySuffix = llvm::utostr(Priority);
640 PrioritySuffix = std::string(6 - PrioritySuffix.size(), '0') + PrioritySuffix;
641
642 return PrioritySuffix;
643 }
644
645 void
EmitCXXGlobalInitFunc()646 CodeGenModule::EmitCXXGlobalInitFunc() {
647 while (!CXXGlobalInits.empty() && !CXXGlobalInits.back())
648 CXXGlobalInits.pop_back();
649
650 if (CXXGlobalInits.empty() && PrioritizedCXXGlobalInits.empty())
651 return;
652
653 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
654 const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
655
656 // Create our global prioritized initialization function.
657 if (!PrioritizedCXXGlobalInits.empty()) {
658 SmallVector<llvm::Function *, 8> LocalCXXGlobalInits;
659 llvm::array_pod_sort(PrioritizedCXXGlobalInits.begin(),
660 PrioritizedCXXGlobalInits.end());
661 // Iterate over "chunks" of ctors with same priority and emit each chunk
662 // into separate function. Note - everything is sorted first by priority,
663 // second - by lex order, so we emit ctor functions in proper order.
664 for (SmallVectorImpl<GlobalInitData >::iterator
665 I = PrioritizedCXXGlobalInits.begin(),
666 E = PrioritizedCXXGlobalInits.end(); I != E; ) {
667 SmallVectorImpl<GlobalInitData >::iterator
668 PrioE = std::upper_bound(I + 1, E, *I, GlobalInitPriorityCmp());
669
670 LocalCXXGlobalInits.clear();
671
672 unsigned int Priority = I->first.priority;
673 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
674 FTy, "_GLOBAL__I_" + getPrioritySuffix(Priority), FI);
675
676 for (; I < PrioE; ++I)
677 LocalCXXGlobalInits.push_back(I->second);
678
679 CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, LocalCXXGlobalInits);
680 AddGlobalCtor(Fn, Priority);
681 }
682 PrioritizedCXXGlobalInits.clear();
683 }
684
685 if (getCXXABI().useSinitAndSterm() && CXXGlobalInits.empty())
686 return;
687
688 // Include the filename in the symbol name. Including "sub_" matches gcc
689 // and makes sure these symbols appear lexicographically behind the symbols
690 // with priority emitted above.
691 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
692 FTy, llvm::Twine("_GLOBAL__sub_I_", getTransformedFileName(getModule())),
693 FI);
694
695 CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn, CXXGlobalInits);
696 AddGlobalCtor(Fn);
697
698 // In OpenCL global init functions must be converted to kernels in order to
699 // be able to launch them from the host.
700 // FIXME: Some more work might be needed to handle destructors correctly.
701 // Current initialization function makes use of function pointers callbacks.
702 // We can't support function pointers especially between host and device.
703 // However it seems global destruction has little meaning without any
704 // dynamic resource allocation on the device and program scope variables are
705 // destroyed by the runtime when program is released.
706 if (getLangOpts().OpenCL) {
707 GenOpenCLArgMetadata(Fn);
708 Fn->setCallingConv(llvm::CallingConv::SPIR_KERNEL);
709 }
710
711 assert(!getLangOpts().CUDA || !getLangOpts().CUDAIsDevice ||
712 getLangOpts().GPUAllowDeviceInit);
713 if (getLangOpts().HIP && getLangOpts().CUDAIsDevice) {
714 Fn->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL);
715 Fn->addFnAttr("device-init");
716 }
717
718 CXXGlobalInits.clear();
719 }
720
EmitCXXGlobalCleanUpFunc()721 void CodeGenModule::EmitCXXGlobalCleanUpFunc() {
722 if (CXXGlobalDtorsOrStermFinalizers.empty() &&
723 PrioritizedCXXStermFinalizers.empty())
724 return;
725
726 llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
727 const CGFunctionInfo &FI = getTypes().arrangeNullaryFunction();
728
729 // Create our global prioritized cleanup function.
730 if (!PrioritizedCXXStermFinalizers.empty()) {
731 SmallVector<CXXGlobalDtorsOrStermFinalizer_t, 8> LocalCXXStermFinalizers;
732 llvm::array_pod_sort(PrioritizedCXXStermFinalizers.begin(),
733 PrioritizedCXXStermFinalizers.end());
734 // Iterate over "chunks" of dtors with same priority and emit each chunk
735 // into separate function. Note - everything is sorted first by priority,
736 // second - by lex order, so we emit dtor functions in proper order.
737 for (SmallVectorImpl<StermFinalizerData>::iterator
738 I = PrioritizedCXXStermFinalizers.begin(),
739 E = PrioritizedCXXStermFinalizers.end();
740 I != E;) {
741 SmallVectorImpl<StermFinalizerData>::iterator PrioE =
742 std::upper_bound(I + 1, E, *I, StermFinalizerPriorityCmp());
743
744 LocalCXXStermFinalizers.clear();
745
746 unsigned int Priority = I->first.priority;
747 llvm::Function *Fn = CreateGlobalInitOrCleanUpFunction(
748 FTy, "_GLOBAL__a_" + getPrioritySuffix(Priority), FI);
749
750 for (; I < PrioE; ++I) {
751 llvm::FunctionCallee DtorFn = I->second;
752 LocalCXXStermFinalizers.emplace_back(DtorFn.getFunctionType(),
753 DtorFn.getCallee(), nullptr);
754 }
755
756 CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
757 Fn, LocalCXXStermFinalizers);
758 AddGlobalDtor(Fn, Priority);
759 }
760 PrioritizedCXXStermFinalizers.clear();
761 }
762
763 if (CXXGlobalDtorsOrStermFinalizers.empty())
764 return;
765
766 // Create our global cleanup function.
767 llvm::Function *Fn =
768 CreateGlobalInitOrCleanUpFunction(FTy, "_GLOBAL__D_a", FI);
769
770 CodeGenFunction(*this).GenerateCXXGlobalCleanUpFunc(
771 Fn, CXXGlobalDtorsOrStermFinalizers);
772 AddGlobalDtor(Fn);
773 CXXGlobalDtorsOrStermFinalizers.clear();
774 }
775
776 /// Emit the code necessary to initialize the given global variable.
GenerateCXXGlobalVarDeclInitFunc(llvm::Function * Fn,const VarDecl * D,llvm::GlobalVariable * Addr,bool PerformInit)777 void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
778 const VarDecl *D,
779 llvm::GlobalVariable *Addr,
780 bool PerformInit) {
781 // Check if we need to emit debug info for variable initializer.
782 if (D->hasAttr<NoDebugAttr>())
783 DebugInfo = nullptr; // disable debug info indefinitely for this function
784
785 CurEHLocation = D->getBeginLoc();
786
787 StartFunction(GlobalDecl(D, DynamicInitKind::Initializer),
788 getContext().VoidTy, Fn, getTypes().arrangeNullaryFunction(),
789 FunctionArgList());
790 // Emit an artificial location for this function.
791 auto AL = ApplyDebugLocation::CreateArtificial(*this);
792
793 // Use guarded initialization if the global variable is weak. This
794 // occurs for, e.g., instantiated static data members and
795 // definitions explicitly marked weak.
796 //
797 // Also use guarded initialization for a variable with dynamic TLS and
798 // unordered initialization. (If the initialization is ordered, the ABI
799 // layer will guard the whole-TU initialization for us.)
800 if (Addr->hasWeakLinkage() || Addr->hasLinkOnceLinkage() ||
801 (D->getTLSKind() == VarDecl::TLS_Dynamic &&
802 isTemplateInstantiation(D->getTemplateSpecializationKind()))) {
803 EmitCXXGuardedInit(*D, Addr, PerformInit);
804 } else {
805 EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit);
806 }
807
808 FinishFunction();
809 }
810
811 void
GenerateCXXGlobalInitFunc(llvm::Function * Fn,ArrayRef<llvm::Function * > Decls,ConstantAddress Guard)812 CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
813 ArrayRef<llvm::Function *> Decls,
814 ConstantAddress Guard) {
815 {
816 auto NL = ApplyDebugLocation::CreateEmpty(*this);
817 StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
818 getTypes().arrangeNullaryFunction(), FunctionArgList());
819 // Emit an artificial location for this function.
820 auto AL = ApplyDebugLocation::CreateArtificial(*this);
821
822 llvm::BasicBlock *ExitBlock = nullptr;
823 if (Guard.isValid()) {
824 // If we have a guard variable, check whether we've already performed
825 // these initializations. This happens for TLS initialization functions.
826 llvm::Value *GuardVal = Builder.CreateLoad(Guard);
827 llvm::Value *Uninit = Builder.CreateIsNull(GuardVal,
828 "guard.uninitialized");
829 llvm::BasicBlock *InitBlock = createBasicBlock("init");
830 ExitBlock = createBasicBlock("exit");
831 EmitCXXGuardedInitBranch(Uninit, InitBlock, ExitBlock,
832 GuardKind::TlsGuard, nullptr);
833 EmitBlock(InitBlock);
834 // Mark as initialized before initializing anything else. If the
835 // initializers use previously-initialized thread_local vars, that's
836 // probably supposed to be OK, but the standard doesn't say.
837 Builder.CreateStore(llvm::ConstantInt::get(GuardVal->getType(),1), Guard);
838
839 // The guard variable can't ever change again.
840 EmitInvariantStart(
841 Guard.getPointer(),
842 CharUnits::fromQuantity(
843 CGM.getDataLayout().getTypeAllocSize(GuardVal->getType())));
844 }
845
846 RunCleanupsScope Scope(*this);
847
848 // When building in Objective-C++ ARC mode, create an autorelease pool
849 // around the global initializers.
850 if (getLangOpts().ObjCAutoRefCount && getLangOpts().CPlusPlus) {
851 llvm::Value *token = EmitObjCAutoreleasePoolPush();
852 EmitObjCAutoreleasePoolCleanup(token);
853 }
854
855 for (unsigned i = 0, e = Decls.size(); i != e; ++i)
856 if (Decls[i])
857 EmitRuntimeCall(Decls[i]);
858
859 Scope.ForceCleanup();
860
861 if (ExitBlock) {
862 Builder.CreateBr(ExitBlock);
863 EmitBlock(ExitBlock);
864 }
865 }
866
867 FinishFunction();
868 }
869
GenerateCXXGlobalCleanUpFunc(llvm::Function * Fn,ArrayRef<std::tuple<llvm::FunctionType *,llvm::WeakTrackingVH,llvm::Constant * >> DtorsOrStermFinalizers)870 void CodeGenFunction::GenerateCXXGlobalCleanUpFunc(
871 llvm::Function *Fn,
872 ArrayRef<std::tuple<llvm::FunctionType *, llvm::WeakTrackingVH,
873 llvm::Constant *>>
874 DtorsOrStermFinalizers) {
875 {
876 auto NL = ApplyDebugLocation::CreateEmpty(*this);
877 StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
878 getTypes().arrangeNullaryFunction(), FunctionArgList());
879 // Emit an artificial location for this function.
880 auto AL = ApplyDebugLocation::CreateArtificial(*this);
881
882 // Emit the cleanups, in reverse order from construction.
883 for (unsigned i = 0, e = DtorsOrStermFinalizers.size(); i != e; ++i) {
884 llvm::FunctionType *CalleeTy;
885 llvm::Value *Callee;
886 llvm::Constant *Arg;
887 std::tie(CalleeTy, Callee, Arg) = DtorsOrStermFinalizers[e - i - 1];
888
889 llvm::CallInst *CI = nullptr;
890 if (Arg == nullptr) {
891 assert(
892 CGM.getCXXABI().useSinitAndSterm() &&
893 "Arg could not be nullptr unless using sinit and sterm functions.");
894 CI = Builder.CreateCall(CalleeTy, Callee);
895 } else
896 CI = Builder.CreateCall(CalleeTy, Callee, Arg);
897
898 // Make sure the call and the callee agree on calling convention.
899 if (llvm::Function *F = dyn_cast<llvm::Function>(Callee))
900 CI->setCallingConv(F->getCallingConv());
901 }
902 }
903
904 FinishFunction();
905 }
906
907 /// generateDestroyHelper - Generates a helper function which, when
908 /// invoked, destroys the given object. The address of the object
909 /// should be in global memory.
generateDestroyHelper(Address addr,QualType type,Destroyer * destroyer,bool useEHCleanupForArray,const VarDecl * VD)910 llvm::Function *CodeGenFunction::generateDestroyHelper(
911 Address addr, QualType type, Destroyer *destroyer,
912 bool useEHCleanupForArray, const VarDecl *VD) {
913 FunctionArgList args;
914 ImplicitParamDecl Dst(getContext(), getContext().VoidPtrTy,
915 ImplicitParamDecl::Other);
916 args.push_back(&Dst);
917
918 const CGFunctionInfo &FI =
919 CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, args);
920 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI);
921 llvm::Function *fn = CGM.CreateGlobalInitOrCleanUpFunction(
922 FTy, "__cxx_global_array_dtor", FI, VD->getLocation());
923
924 CurEHLocation = VD->getBeginLoc();
925
926 StartFunction(GlobalDecl(VD, DynamicInitKind::GlobalArrayDestructor),
927 getContext().VoidTy, fn, FI, args);
928 // Emit an artificial location for this function.
929 auto AL = ApplyDebugLocation::CreateArtificial(*this);
930
931 emitDestroy(addr, type, destroyer, useEHCleanupForArray);
932
933 FinishFunction();
934
935 return fn;
936 }
937