1 //===--- CodeGenFunction.cpp - Emit LLVM Code from ASTs for a Function ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This coordinates the per-function state used while generating code.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "CodeGenFunction.h"
14 #include "CGBlocks.h"
15 #include "CGCUDARuntime.h"
16 #include "CGCXXABI.h"
17 #include "CGCleanup.h"
18 #include "CGDebugInfo.h"
19 #include "CGOpenMPRuntime.h"
20 #include "CodeGenModule.h"
21 #include "CodeGenPGO.h"
22 #include "TargetInfo.h"
23 #include "clang/AST/ASTContext.h"
24 #include "clang/AST/ASTLambda.h"
25 #include "clang/AST/Attr.h"
26 #include "clang/AST/Decl.h"
27 #include "clang/AST/DeclCXX.h"
28 #include "clang/AST/Expr.h"
29 #include "clang/AST/StmtCXX.h"
30 #include "clang/AST/StmtObjC.h"
31 #include "clang/Basic/Builtins.h"
32 #include "clang/Basic/CodeGenOptions.h"
33 #include "clang/Basic/TargetInfo.h"
34 #include "clang/CodeGen/CGFunctionInfo.h"
35 #include "clang/Frontend/FrontendDiagnostic.h"
36 #include "llvm/ADT/ArrayRef.h"
37 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/IR/Dominators.h"
40 #include "llvm/IR/FPEnv.h"
41 #include "llvm/IR/IntrinsicInst.h"
42 #include "llvm/IR/Intrinsics.h"
43 #include "llvm/IR/MDBuilder.h"
44 #include "llvm/IR/Operator.h"
45 #include "llvm/Support/CRC.h"
46 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
47 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
48 using namespace clang;
49 using namespace CodeGen;
50
51 /// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
52 /// markers.
shouldEmitLifetimeMarkers(const CodeGenOptions & CGOpts,const LangOptions & LangOpts)53 static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
54 const LangOptions &LangOpts) {
55 if (CGOpts.DisableLifetimeMarkers)
56 return false;
57
58 // Sanitizers may use markers.
59 if (CGOpts.SanitizeAddressUseAfterScope ||
60 LangOpts.Sanitize.has(SanitizerKind::HWAddress) ||
61 LangOpts.Sanitize.has(SanitizerKind::Memory))
62 return true;
63
64 // For now, only in optimized builds.
65 return CGOpts.OptimizationLevel != 0;
66 }
67
CodeGenFunction(CodeGenModule & cgm,bool suppressNewContext)68 CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
69 : CodeGenTypeCache(cgm), CGM(cgm), Target(cgm.getTarget()),
70 Builder(cgm, cgm.getModule().getContext(), llvm::ConstantFolder(),
71 CGBuilderInserterTy(this)),
72 SanOpts(CGM.getLangOpts().Sanitize), CurFPFeatures(CGM.getLangOpts()),
73 DebugInfo(CGM.getModuleDebugInfo()), PGO(cgm),
74 ShouldEmitLifetimeMarkers(
75 shouldEmitLifetimeMarkers(CGM.getCodeGenOpts(), CGM.getLangOpts())) {
76 if (!suppressNewContext)
77 CGM.getCXXABI().getMangleContext().startNewFunction();
78 EHStack.setCGF(this);
79
80 SetFastMathFlags(CurFPFeatures);
81 SetFPModel();
82 }
83
~CodeGenFunction()84 CodeGenFunction::~CodeGenFunction() {
85 assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
86
87 if (getLangOpts().OpenMP && CurFn)
88 CGM.getOpenMPRuntime().functionFinished(*this);
89
90 // If we have an OpenMPIRBuilder we want to finalize functions (incl.
91 // outlining etc) at some point. Doing it once the function codegen is done
92 // seems to be a reasonable spot. We do it here, as opposed to the deletion
93 // time of the CodeGenModule, because we have to ensure the IR has not yet
94 // been "emitted" to the outside, thus, modifications are still sensible.
95 if (CGM.getLangOpts().OpenMPIRBuilder && CurFn)
96 CGM.getOpenMPRuntime().getOMPBuilder().finalize(CurFn);
97 }
98
99 // Map the LangOption for exception behavior into
100 // the corresponding enum in the IR.
101 llvm::fp::ExceptionBehavior
ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind)102 clang::ToConstrainedExceptMD(LangOptions::FPExceptionModeKind Kind) {
103
104 switch (Kind) {
105 case LangOptions::FPE_Ignore: return llvm::fp::ebIgnore;
106 case LangOptions::FPE_MayTrap: return llvm::fp::ebMayTrap;
107 case LangOptions::FPE_Strict: return llvm::fp::ebStrict;
108 }
109 llvm_unreachable("Unsupported FP Exception Behavior");
110 }
111
SetFPModel()112 void CodeGenFunction::SetFPModel() {
113 llvm::RoundingMode RM = getLangOpts().getFPRoundingMode();
114 auto fpExceptionBehavior = ToConstrainedExceptMD(
115 getLangOpts().getFPExceptionMode());
116
117 Builder.setDefaultConstrainedRounding(RM);
118 Builder.setDefaultConstrainedExcept(fpExceptionBehavior);
119 Builder.setIsFPConstrained(fpExceptionBehavior != llvm::fp::ebIgnore ||
120 RM != llvm::RoundingMode::NearestTiesToEven);
121 }
122
SetFastMathFlags(FPOptions FPFeatures)123 void CodeGenFunction::SetFastMathFlags(FPOptions FPFeatures) {
124 llvm::FastMathFlags FMF;
125 FMF.setAllowReassoc(FPFeatures.getAllowFPReassociate());
126 FMF.setNoNaNs(FPFeatures.getNoHonorNaNs());
127 FMF.setNoInfs(FPFeatures.getNoHonorInfs());
128 FMF.setNoSignedZeros(FPFeatures.getNoSignedZero());
129 FMF.setAllowReciprocal(FPFeatures.getAllowReciprocal());
130 FMF.setApproxFunc(FPFeatures.getAllowApproxFunc());
131 FMF.setAllowContract(FPFeatures.allowFPContractAcrossStatement());
132 Builder.setFastMathFlags(FMF);
133 }
134
CGFPOptionsRAII(CodeGenFunction & CGF,const Expr * E)135 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
136 const Expr *E)
137 : CGF(CGF) {
138 ConstructorHelper(E->getFPFeaturesInEffect(CGF.getLangOpts()));
139 }
140
CGFPOptionsRAII(CodeGenFunction & CGF,FPOptions FPFeatures)141 CodeGenFunction::CGFPOptionsRAII::CGFPOptionsRAII(CodeGenFunction &CGF,
142 FPOptions FPFeatures)
143 : CGF(CGF) {
144 ConstructorHelper(FPFeatures);
145 }
146
ConstructorHelper(FPOptions FPFeatures)147 void CodeGenFunction::CGFPOptionsRAII::ConstructorHelper(FPOptions FPFeatures) {
148 OldFPFeatures = CGF.CurFPFeatures;
149 CGF.CurFPFeatures = FPFeatures;
150
151 OldExcept = CGF.Builder.getDefaultConstrainedExcept();
152 OldRounding = CGF.Builder.getDefaultConstrainedRounding();
153
154 if (OldFPFeatures == FPFeatures)
155 return;
156
157 FMFGuard.emplace(CGF.Builder);
158
159 llvm::RoundingMode NewRoundingBehavior =
160 static_cast<llvm::RoundingMode>(FPFeatures.getRoundingMode());
161 CGF.Builder.setDefaultConstrainedRounding(NewRoundingBehavior);
162 auto NewExceptionBehavior =
163 ToConstrainedExceptMD(static_cast<LangOptions::FPExceptionModeKind>(
164 FPFeatures.getFPExceptionMode()));
165 CGF.Builder.setDefaultConstrainedExcept(NewExceptionBehavior);
166
167 CGF.SetFastMathFlags(FPFeatures);
168
169 assert((CGF.CurFuncDecl == nullptr || CGF.Builder.getIsFPConstrained() ||
170 isa<CXXConstructorDecl>(CGF.CurFuncDecl) ||
171 isa<CXXDestructorDecl>(CGF.CurFuncDecl) ||
172 (NewExceptionBehavior == llvm::fp::ebIgnore &&
173 NewRoundingBehavior == llvm::RoundingMode::NearestTiesToEven)) &&
174 "FPConstrained should be enabled on entire function");
175
176 auto mergeFnAttrValue = [&](StringRef Name, bool Value) {
177 auto OldValue =
178 CGF.CurFn->getFnAttribute(Name).getValueAsBool();
179 auto NewValue = OldValue & Value;
180 if (OldValue != NewValue)
181 CGF.CurFn->addFnAttr(Name, llvm::toStringRef(NewValue));
182 };
183 mergeFnAttrValue("no-infs-fp-math", FPFeatures.getNoHonorInfs());
184 mergeFnAttrValue("no-nans-fp-math", FPFeatures.getNoHonorNaNs());
185 mergeFnAttrValue("no-signed-zeros-fp-math", FPFeatures.getNoSignedZero());
186 mergeFnAttrValue("unsafe-fp-math", FPFeatures.getAllowFPReassociate() &&
187 FPFeatures.getAllowReciprocal() &&
188 FPFeatures.getAllowApproxFunc() &&
189 FPFeatures.getNoSignedZero());
190 }
191
~CGFPOptionsRAII()192 CodeGenFunction::CGFPOptionsRAII::~CGFPOptionsRAII() {
193 CGF.CurFPFeatures = OldFPFeatures;
194 CGF.Builder.setDefaultConstrainedExcept(OldExcept);
195 CGF.Builder.setDefaultConstrainedRounding(OldRounding);
196 }
197
MakeNaturalAlignAddrLValue(llvm::Value * V,QualType T)198 LValue CodeGenFunction::MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T) {
199 LValueBaseInfo BaseInfo;
200 TBAAAccessInfo TBAAInfo;
201 CharUnits Alignment = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo);
202 return LValue::MakeAddr(Address(V, Alignment), T, getContext(), BaseInfo,
203 TBAAInfo);
204 }
205
206 /// Given a value of type T* that may not be to a complete object,
207 /// construct an l-value with the natural pointee alignment of T.
208 LValue
MakeNaturalAlignPointeeAddrLValue(llvm::Value * V,QualType T)209 CodeGenFunction::MakeNaturalAlignPointeeAddrLValue(llvm::Value *V, QualType T) {
210 LValueBaseInfo BaseInfo;
211 TBAAAccessInfo TBAAInfo;
212 CharUnits Align = CGM.getNaturalTypeAlignment(T, &BaseInfo, &TBAAInfo,
213 /* forPointeeType= */ true);
214 return MakeAddrLValue(Address(V, Align), T, BaseInfo, TBAAInfo);
215 }
216
217
ConvertTypeForMem(QualType T)218 llvm::Type *CodeGenFunction::ConvertTypeForMem(QualType T) {
219 return CGM.getTypes().ConvertTypeForMem(T);
220 }
221
ConvertType(QualType T)222 llvm::Type *CodeGenFunction::ConvertType(QualType T) {
223 return CGM.getTypes().ConvertType(T);
224 }
225
getEvaluationKind(QualType type)226 TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
227 type = type.getCanonicalType();
228 while (true) {
229 switch (type->getTypeClass()) {
230 #define TYPE(name, parent)
231 #define ABSTRACT_TYPE(name, parent)
232 #define NON_CANONICAL_TYPE(name, parent) case Type::name:
233 #define DEPENDENT_TYPE(name, parent) case Type::name:
234 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
235 #include "clang/AST/TypeNodes.inc"
236 llvm_unreachable("non-canonical or dependent type in IR-generation");
237
238 case Type::Auto:
239 case Type::DeducedTemplateSpecialization:
240 llvm_unreachable("undeduced type in IR-generation");
241
242 // Various scalar types.
243 case Type::Builtin:
244 case Type::Pointer:
245 case Type::BlockPointer:
246 case Type::LValueReference:
247 case Type::RValueReference:
248 case Type::MemberPointer:
249 case Type::Vector:
250 case Type::ExtVector:
251 case Type::ConstantMatrix:
252 case Type::FunctionProto:
253 case Type::FunctionNoProto:
254 case Type::Enum:
255 case Type::ObjCObjectPointer:
256 case Type::Pipe:
257 case Type::ExtInt:
258 return TEK_Scalar;
259
260 // Complexes.
261 case Type::Complex:
262 return TEK_Complex;
263
264 // Arrays, records, and Objective-C objects.
265 case Type::ConstantArray:
266 case Type::IncompleteArray:
267 case Type::VariableArray:
268 case Type::Record:
269 case Type::ObjCObject:
270 case Type::ObjCInterface:
271 return TEK_Aggregate;
272
273 // We operate on atomic values according to their underlying type.
274 case Type::Atomic:
275 type = cast<AtomicType>(type)->getValueType();
276 continue;
277 }
278 llvm_unreachable("unknown type kind!");
279 }
280 }
281
EmitReturnBlock()282 llvm::DebugLoc CodeGenFunction::EmitReturnBlock() {
283 // For cleanliness, we try to avoid emitting the return block for
284 // simple cases.
285 llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
286
287 if (CurBB) {
288 assert(!CurBB->getTerminator() && "Unexpected terminated block.");
289
290 // We have a valid insert point, reuse it if it is empty or there are no
291 // explicit jumps to the return block.
292 if (CurBB->empty() || ReturnBlock.getBlock()->use_empty()) {
293 ReturnBlock.getBlock()->replaceAllUsesWith(CurBB);
294 delete ReturnBlock.getBlock();
295 ReturnBlock = JumpDest();
296 } else
297 EmitBlock(ReturnBlock.getBlock());
298 return llvm::DebugLoc();
299 }
300
301 // Otherwise, if the return block is the target of a single direct
302 // branch then we can just put the code in that block instead. This
303 // cleans up functions which started with a unified return block.
304 if (ReturnBlock.getBlock()->hasOneUse()) {
305 llvm::BranchInst *BI =
306 dyn_cast<llvm::BranchInst>(*ReturnBlock.getBlock()->user_begin());
307 if (BI && BI->isUnconditional() &&
308 BI->getSuccessor(0) == ReturnBlock.getBlock()) {
309 // Record/return the DebugLoc of the simple 'return' expression to be used
310 // later by the actual 'ret' instruction.
311 llvm::DebugLoc Loc = BI->getDebugLoc();
312 Builder.SetInsertPoint(BI->getParent());
313 BI->eraseFromParent();
314 delete ReturnBlock.getBlock();
315 ReturnBlock = JumpDest();
316 return Loc;
317 }
318 }
319
320 // FIXME: We are at an unreachable point, there is no reason to emit the block
321 // unless it has uses. However, we still need a place to put the debug
322 // region.end for now.
323
324 EmitBlock(ReturnBlock.getBlock());
325 return llvm::DebugLoc();
326 }
327
EmitIfUsed(CodeGenFunction & CGF,llvm::BasicBlock * BB)328 static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
329 if (!BB) return;
330 if (!BB->use_empty())
331 return CGF.CurFn->getBasicBlockList().push_back(BB);
332 delete BB;
333 }
334
FinishFunction(SourceLocation EndLoc)335 void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
336 assert(BreakContinueStack.empty() &&
337 "mismatched push/pop in break/continue stack!");
338
339 bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
340 && NumSimpleReturnExprs == NumReturnExprs
341 && ReturnBlock.getBlock()->use_empty();
342 // Usually the return expression is evaluated before the cleanup
343 // code. If the function contains only a simple return statement,
344 // such as a constant, the location before the cleanup code becomes
345 // the last useful breakpoint in the function, because the simple
346 // return expression will be evaluated after the cleanup code. To be
347 // safe, set the debug location for cleanup code to the location of
348 // the return statement. Otherwise the cleanup code should be at the
349 // end of the function's lexical scope.
350 //
351 // If there are multiple branches to the return block, the branch
352 // instructions will get the location of the return statements and
353 // all will be fine.
354 if (CGDebugInfo *DI = getDebugInfo()) {
355 if (OnlySimpleReturnStmts)
356 DI->EmitLocation(Builder, LastStopPoint);
357 else
358 DI->EmitLocation(Builder, EndLoc);
359 }
360
361 // Pop any cleanups that might have been associated with the
362 // parameters. Do this in whatever block we're currently in; it's
363 // important to do this before we enter the return block or return
364 // edges will be *really* confused.
365 bool HasCleanups = EHStack.stable_begin() != PrologueCleanupDepth;
366 bool HasOnlyLifetimeMarkers =
367 HasCleanups && EHStack.containsOnlyLifetimeMarkers(PrologueCleanupDepth);
368 bool EmitRetDbgLoc = !HasCleanups || HasOnlyLifetimeMarkers;
369 if (HasCleanups) {
370 // Make sure the line table doesn't jump back into the body for
371 // the ret after it's been at EndLoc.
372 Optional<ApplyDebugLocation> AL;
373 if (CGDebugInfo *DI = getDebugInfo()) {
374 if (OnlySimpleReturnStmts)
375 DI->EmitLocation(Builder, EndLoc);
376 else
377 // We may not have a valid end location. Try to apply it anyway, and
378 // fall back to an artificial location if needed.
379 AL = ApplyDebugLocation::CreateDefaultArtificial(*this, EndLoc);
380 }
381
382 PopCleanupBlocks(PrologueCleanupDepth);
383 }
384
385 // Emit function epilog (to return).
386 llvm::DebugLoc Loc = EmitReturnBlock();
387
388 if (ShouldInstrumentFunction()) {
389 if (CGM.getCodeGenOpts().InstrumentFunctions)
390 CurFn->addFnAttr("instrument-function-exit", "__cyg_profile_func_exit");
391 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
392 CurFn->addFnAttr("instrument-function-exit-inlined",
393 "__cyg_profile_func_exit");
394 }
395
396 // Emit debug descriptor for function end.
397 if (CGDebugInfo *DI = getDebugInfo())
398 DI->EmitFunctionEnd(Builder, CurFn);
399
400 // Reset the debug location to that of the simple 'return' expression, if any
401 // rather than that of the end of the function's scope '}'.
402 ApplyDebugLocation AL(*this, Loc);
403 EmitFunctionEpilog(*CurFnInfo, EmitRetDbgLoc, EndLoc);
404 EmitEndEHSpec(CurCodeDecl);
405
406 assert(EHStack.empty() &&
407 "did not remove all scopes from cleanup stack!");
408
409 // If someone did an indirect goto, emit the indirect goto block at the end of
410 // the function.
411 if (IndirectBranch) {
412 EmitBlock(IndirectBranch->getParent());
413 Builder.ClearInsertionPoint();
414 }
415
416 // If some of our locals escaped, insert a call to llvm.localescape in the
417 // entry block.
418 if (!EscapedLocals.empty()) {
419 // Invert the map from local to index into a simple vector. There should be
420 // no holes.
421 SmallVector<llvm::Value *, 4> EscapeArgs;
422 EscapeArgs.resize(EscapedLocals.size());
423 for (auto &Pair : EscapedLocals)
424 EscapeArgs[Pair.second] = Pair.first;
425 llvm::Function *FrameEscapeFn = llvm::Intrinsic::getDeclaration(
426 &CGM.getModule(), llvm::Intrinsic::localescape);
427 CGBuilderTy(*this, AllocaInsertPt).CreateCall(FrameEscapeFn, EscapeArgs);
428 }
429
430 // Remove the AllocaInsertPt instruction, which is just a convenience for us.
431 llvm::Instruction *Ptr = AllocaInsertPt;
432 AllocaInsertPt = nullptr;
433 Ptr->eraseFromParent();
434
435 // If someone took the address of a label but never did an indirect goto, we
436 // made a zero entry PHI node, which is illegal, zap it now.
437 if (IndirectBranch) {
438 llvm::PHINode *PN = cast<llvm::PHINode>(IndirectBranch->getAddress());
439 if (PN->getNumIncomingValues() == 0) {
440 PN->replaceAllUsesWith(llvm::UndefValue::get(PN->getType()));
441 PN->eraseFromParent();
442 }
443 }
444
445 EmitIfUsed(*this, EHResumeBlock);
446 EmitIfUsed(*this, TerminateLandingPad);
447 EmitIfUsed(*this, TerminateHandler);
448 EmitIfUsed(*this, UnreachableBlock);
449
450 for (const auto &FuncletAndParent : TerminateFunclets)
451 EmitIfUsed(*this, FuncletAndParent.second);
452
453 if (CGM.getCodeGenOpts().EmitDeclMetadata)
454 EmitDeclMetadata();
455
456 for (const auto &R : DeferredReplacements) {
457 if (llvm::Value *Old = R.first) {
458 Old->replaceAllUsesWith(R.second);
459 cast<llvm::Instruction>(Old)->eraseFromParent();
460 }
461 }
462 DeferredReplacements.clear();
463
464 // Eliminate CleanupDestSlot alloca by replacing it with SSA values and
465 // PHIs if the current function is a coroutine. We don't do it for all
466 // functions as it may result in slight increase in numbers of instructions
467 // if compiled with no optimizations. We do it for coroutine as the lifetime
468 // of CleanupDestSlot alloca make correct coroutine frame building very
469 // difficult.
470 if (NormalCleanupDest.isValid() && isCoroutine()) {
471 llvm::DominatorTree DT(*CurFn);
472 llvm::PromoteMemToReg(
473 cast<llvm::AllocaInst>(NormalCleanupDest.getPointer()), DT);
474 NormalCleanupDest = Address::invalid();
475 }
476
477 // Scan function arguments for vector width.
478 for (llvm::Argument &A : CurFn->args())
479 if (auto *VT = dyn_cast<llvm::VectorType>(A.getType()))
480 LargestVectorWidth =
481 std::max((uint64_t)LargestVectorWidth,
482 VT->getPrimitiveSizeInBits().getKnownMinSize());
483
484 // Update vector width based on return type.
485 if (auto *VT = dyn_cast<llvm::VectorType>(CurFn->getReturnType()))
486 LargestVectorWidth =
487 std::max((uint64_t)LargestVectorWidth,
488 VT->getPrimitiveSizeInBits().getKnownMinSize());
489
490 // Add the required-vector-width attribute. This contains the max width from:
491 // 1. min-vector-width attribute used in the source program.
492 // 2. Any builtins used that have a vector width specified.
493 // 3. Values passed in and out of inline assembly.
494 // 4. Width of vector arguments and return types for this function.
495 // 5. Width of vector aguments and return types for functions called by this
496 // function.
497 CurFn->addFnAttr("min-legal-vector-width", llvm::utostr(LargestVectorWidth));
498
499 // Add vscale attribute if appropriate.
500 if (getLangOpts().ArmSveVectorBits) {
501 unsigned VScale = getLangOpts().ArmSveVectorBits / 128;
502 CurFn->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(getLLVMContext(),
503 VScale, VScale));
504 }
505
506 // If we generated an unreachable return block, delete it now.
507 if (ReturnBlock.isValid() && ReturnBlock.getBlock()->use_empty()) {
508 Builder.ClearInsertionPoint();
509 ReturnBlock.getBlock()->eraseFromParent();
510 }
511 if (ReturnValue.isValid()) {
512 auto *RetAlloca = dyn_cast<llvm::AllocaInst>(ReturnValue.getPointer());
513 if (RetAlloca && RetAlloca->use_empty()) {
514 RetAlloca->eraseFromParent();
515 ReturnValue = Address::invalid();
516 }
517 }
518 }
519
520 /// ShouldInstrumentFunction - Return true if the current function should be
521 /// instrumented with __cyg_profile_func_* calls
ShouldInstrumentFunction()522 bool CodeGenFunction::ShouldInstrumentFunction() {
523 if (!CGM.getCodeGenOpts().InstrumentFunctions &&
524 !CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining &&
525 !CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
526 return false;
527 if (!CurFuncDecl || CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>())
528 return false;
529 return true;
530 }
531
532 /// ShouldXRayInstrument - Return true if the current function should be
533 /// instrumented with XRay nop sleds.
ShouldXRayInstrumentFunction() const534 bool CodeGenFunction::ShouldXRayInstrumentFunction() const {
535 return CGM.getCodeGenOpts().XRayInstrumentFunctions;
536 }
537
538 /// AlwaysEmitXRayCustomEvents - Return true if we should emit IR for calls to
539 /// the __xray_customevent(...) builtin calls, when doing XRay instrumentation.
AlwaysEmitXRayCustomEvents() const540 bool CodeGenFunction::AlwaysEmitXRayCustomEvents() const {
541 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
542 (CGM.getCodeGenOpts().XRayAlwaysEmitCustomEvents ||
543 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
544 XRayInstrKind::Custom);
545 }
546
AlwaysEmitXRayTypedEvents() const547 bool CodeGenFunction::AlwaysEmitXRayTypedEvents() const {
548 return CGM.getCodeGenOpts().XRayInstrumentFunctions &&
549 (CGM.getCodeGenOpts().XRayAlwaysEmitTypedEvents ||
550 CGM.getCodeGenOpts().XRayInstrumentationBundle.Mask ==
551 XRayInstrKind::Typed);
552 }
553
554 llvm::Constant *
EncodeAddrForUseInPrologue(llvm::Function * F,llvm::Constant * Addr)555 CodeGenFunction::EncodeAddrForUseInPrologue(llvm::Function *F,
556 llvm::Constant *Addr) {
557 // Addresses stored in prologue data can't require run-time fixups and must
558 // be PC-relative. Run-time fixups are undesirable because they necessitate
559 // writable text segments, which are unsafe. And absolute addresses are
560 // undesirable because they break PIE mode.
561
562 // Add a layer of indirection through a private global. Taking its address
563 // won't result in a run-time fixup, even if Addr has linkonce_odr linkage.
564 auto *GV = new llvm::GlobalVariable(CGM.getModule(), Addr->getType(),
565 /*isConstant=*/true,
566 llvm::GlobalValue::PrivateLinkage, Addr);
567
568 // Create a PC-relative address.
569 auto *GOTAsInt = llvm::ConstantExpr::getPtrToInt(GV, IntPtrTy);
570 auto *FuncAsInt = llvm::ConstantExpr::getPtrToInt(F, IntPtrTy);
571 auto *PCRelAsInt = llvm::ConstantExpr::getSub(GOTAsInt, FuncAsInt);
572 return (IntPtrTy == Int32Ty)
573 ? PCRelAsInt
574 : llvm::ConstantExpr::getTrunc(PCRelAsInt, Int32Ty);
575 }
576
577 llvm::Value *
DecodeAddrUsedInPrologue(llvm::Value * F,llvm::Value * EncodedAddr)578 CodeGenFunction::DecodeAddrUsedInPrologue(llvm::Value *F,
579 llvm::Value *EncodedAddr) {
580 // Reconstruct the address of the global.
581 auto *PCRelAsInt = Builder.CreateSExt(EncodedAddr, IntPtrTy);
582 auto *FuncAsInt = Builder.CreatePtrToInt(F, IntPtrTy, "func_addr.int");
583 auto *GOTAsInt = Builder.CreateAdd(PCRelAsInt, FuncAsInt, "global_addr.int");
584 auto *GOTAddr = Builder.CreateIntToPtr(GOTAsInt, Int8PtrPtrTy, "global_addr");
585
586 // Load the original pointer through the global.
587 return Builder.CreateLoad(Address(GOTAddr, getPointerAlign()),
588 "decoded_addr");
589 }
590
EmitOpenCLKernelMetadata(const FunctionDecl * FD,llvm::Function * Fn)591 void CodeGenFunction::EmitOpenCLKernelMetadata(const FunctionDecl *FD,
592 llvm::Function *Fn)
593 {
594 if (!FD->hasAttr<OpenCLKernelAttr>())
595 return;
596
597 llvm::LLVMContext &Context = getLLVMContext();
598
599 CGM.GenOpenCLArgMetadata(Fn, FD, this);
600
601 if (const VecTypeHintAttr *A = FD->getAttr<VecTypeHintAttr>()) {
602 QualType HintQTy = A->getTypeHint();
603 const ExtVectorType *HintEltQTy = HintQTy->getAs<ExtVectorType>();
604 bool IsSignedInteger =
605 HintQTy->isSignedIntegerType() ||
606 (HintEltQTy && HintEltQTy->getElementType()->isSignedIntegerType());
607 llvm::Metadata *AttrMDArgs[] = {
608 llvm::ConstantAsMetadata::get(llvm::UndefValue::get(
609 CGM.getTypes().ConvertType(A->getTypeHint()))),
610 llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
611 llvm::IntegerType::get(Context, 32),
612 llvm::APInt(32, (uint64_t)(IsSignedInteger ? 1 : 0))))};
613 Fn->setMetadata("vec_type_hint", llvm::MDNode::get(Context, AttrMDArgs));
614 }
615
616 if (const WorkGroupSizeHintAttr *A = FD->getAttr<WorkGroupSizeHintAttr>()) {
617 llvm::Metadata *AttrMDArgs[] = {
618 llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
619 llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
620 llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
621 Fn->setMetadata("work_group_size_hint", llvm::MDNode::get(Context, AttrMDArgs));
622 }
623
624 if (const ReqdWorkGroupSizeAttr *A = FD->getAttr<ReqdWorkGroupSizeAttr>()) {
625 llvm::Metadata *AttrMDArgs[] = {
626 llvm::ConstantAsMetadata::get(Builder.getInt32(A->getXDim())),
627 llvm::ConstantAsMetadata::get(Builder.getInt32(A->getYDim())),
628 llvm::ConstantAsMetadata::get(Builder.getInt32(A->getZDim()))};
629 Fn->setMetadata("reqd_work_group_size", llvm::MDNode::get(Context, AttrMDArgs));
630 }
631
632 if (const OpenCLIntelReqdSubGroupSizeAttr *A =
633 FD->getAttr<OpenCLIntelReqdSubGroupSizeAttr>()) {
634 llvm::Metadata *AttrMDArgs[] = {
635 llvm::ConstantAsMetadata::get(Builder.getInt32(A->getSubGroupSize()))};
636 Fn->setMetadata("intel_reqd_sub_group_size",
637 llvm::MDNode::get(Context, AttrMDArgs));
638 }
639 }
640
641 /// Determine whether the function F ends with a return stmt.
endsWithReturn(const Decl * F)642 static bool endsWithReturn(const Decl* F) {
643 const Stmt *Body = nullptr;
644 if (auto *FD = dyn_cast_or_null<FunctionDecl>(F))
645 Body = FD->getBody();
646 else if (auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(F))
647 Body = OMD->getBody();
648
649 if (auto *CS = dyn_cast_or_null<CompoundStmt>(Body)) {
650 auto LastStmt = CS->body_rbegin();
651 if (LastStmt != CS->body_rend())
652 return isa<ReturnStmt>(*LastStmt);
653 }
654 return false;
655 }
656
markAsIgnoreThreadCheckingAtRuntime(llvm::Function * Fn)657 void CodeGenFunction::markAsIgnoreThreadCheckingAtRuntime(llvm::Function *Fn) {
658 if (SanOpts.has(SanitizerKind::Thread)) {
659 Fn->addFnAttr("sanitize_thread_no_checking_at_run_time");
660 Fn->removeFnAttr(llvm::Attribute::SanitizeThread);
661 }
662 }
663
664 /// Check if the return value of this function requires sanitization.
requiresReturnValueCheck() const665 bool CodeGenFunction::requiresReturnValueCheck() const {
666 return requiresReturnValueNullabilityCheck() ||
667 (SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) && CurCodeDecl &&
668 CurCodeDecl->getAttr<ReturnsNonNullAttr>());
669 }
670
matchesStlAllocatorFn(const Decl * D,const ASTContext & Ctx)671 static bool matchesStlAllocatorFn(const Decl *D, const ASTContext &Ctx) {
672 auto *MD = dyn_cast_or_null<CXXMethodDecl>(D);
673 if (!MD || !MD->getDeclName().getAsIdentifierInfo() ||
674 !MD->getDeclName().getAsIdentifierInfo()->isStr("allocate") ||
675 (MD->getNumParams() != 1 && MD->getNumParams() != 2))
676 return false;
677
678 if (MD->parameters()[0]->getType().getCanonicalType() != Ctx.getSizeType())
679 return false;
680
681 if (MD->getNumParams() == 2) {
682 auto *PT = MD->parameters()[1]->getType()->getAs<PointerType>();
683 if (!PT || !PT->isVoidPointerType() ||
684 !PT->getPointeeType().isConstQualified())
685 return false;
686 }
687
688 return true;
689 }
690
691 /// Return the UBSan prologue signature for \p FD if one is available.
getPrologueSignature(CodeGenModule & CGM,const FunctionDecl * FD)692 static llvm::Constant *getPrologueSignature(CodeGenModule &CGM,
693 const FunctionDecl *FD) {
694 if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
695 if (!MD->isStatic())
696 return nullptr;
697 return CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM);
698 }
699
StartFunction(GlobalDecl GD,QualType RetTy,llvm::Function * Fn,const CGFunctionInfo & FnInfo,const FunctionArgList & Args,SourceLocation Loc,SourceLocation StartLoc)700 void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
701 llvm::Function *Fn,
702 const CGFunctionInfo &FnInfo,
703 const FunctionArgList &Args,
704 SourceLocation Loc,
705 SourceLocation StartLoc) {
706 assert(!CurFn &&
707 "Do not use a CodeGenFunction object for more than one function");
708
709 const Decl *D = GD.getDecl();
710
711 DidCallStackSave = false;
712 CurCodeDecl = D;
713 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(D))
714 if (FD->usesSEHTry())
715 CurSEHParent = FD;
716 CurFuncDecl = (D ? D->getNonClosureContext() : nullptr);
717 FnRetTy = RetTy;
718 CurFn = Fn;
719 CurFnInfo = &FnInfo;
720 assert(CurFn->isDeclaration() && "Function already has body?");
721
722 // If this function is ignored for any of the enabled sanitizers,
723 // disable the sanitizer for the function.
724 do {
725 #define SANITIZER(NAME, ID) \
726 if (SanOpts.empty()) \
727 break; \
728 if (SanOpts.has(SanitizerKind::ID)) \
729 if (CGM.isInNoSanitizeList(SanitizerKind::ID, Fn, Loc)) \
730 SanOpts.set(SanitizerKind::ID, false);
731
732 #include "clang/Basic/Sanitizers.def"
733 #undef SANITIZER
734 } while (0);
735
736 if (D) {
737 // Apply the no_sanitize* attributes to SanOpts.
738 for (auto Attr : D->specific_attrs<NoSanitizeAttr>()) {
739 SanitizerMask mask = Attr->getMask();
740 SanOpts.Mask &= ~mask;
741 if (mask & SanitizerKind::Address)
742 SanOpts.set(SanitizerKind::KernelAddress, false);
743 if (mask & SanitizerKind::KernelAddress)
744 SanOpts.set(SanitizerKind::Address, false);
745 if (mask & SanitizerKind::HWAddress)
746 SanOpts.set(SanitizerKind::KernelHWAddress, false);
747 if (mask & SanitizerKind::KernelHWAddress)
748 SanOpts.set(SanitizerKind::HWAddress, false);
749 }
750 }
751
752 // Apply sanitizer attributes to the function.
753 if (SanOpts.hasOneOf(SanitizerKind::Address | SanitizerKind::KernelAddress))
754 Fn->addFnAttr(llvm::Attribute::SanitizeAddress);
755 if (SanOpts.hasOneOf(SanitizerKind::HWAddress | SanitizerKind::KernelHWAddress))
756 Fn->addFnAttr(llvm::Attribute::SanitizeHWAddress);
757 if (SanOpts.has(SanitizerKind::MemTag))
758 Fn->addFnAttr(llvm::Attribute::SanitizeMemTag);
759 if (SanOpts.has(SanitizerKind::Thread))
760 Fn->addFnAttr(llvm::Attribute::SanitizeThread);
761 if (SanOpts.hasOneOf(SanitizerKind::Memory | SanitizerKind::KernelMemory))
762 Fn->addFnAttr(llvm::Attribute::SanitizeMemory);
763 if (SanOpts.has(SanitizerKind::SafeStack))
764 Fn->addFnAttr(llvm::Attribute::SafeStack);
765 if (SanOpts.has(SanitizerKind::ShadowCallStack))
766 Fn->addFnAttr(llvm::Attribute::ShadowCallStack);
767
768 // Apply fuzzing attribute to the function.
769 if (SanOpts.hasOneOf(SanitizerKind::Fuzzer | SanitizerKind::FuzzerNoLink))
770 Fn->addFnAttr(llvm::Attribute::OptForFuzzing);
771
772 // Ignore TSan memory acesses from within ObjC/ObjC++ dealloc, initialize,
773 // .cxx_destruct, __destroy_helper_block_ and all of their calees at run time.
774 if (SanOpts.has(SanitizerKind::Thread)) {
775 if (const auto *OMD = dyn_cast_or_null<ObjCMethodDecl>(D)) {
776 IdentifierInfo *II = OMD->getSelector().getIdentifierInfoForSlot(0);
777 if (OMD->getMethodFamily() == OMF_dealloc ||
778 OMD->getMethodFamily() == OMF_initialize ||
779 (OMD->getSelector().isUnarySelector() && II->isStr(".cxx_destruct"))) {
780 markAsIgnoreThreadCheckingAtRuntime(Fn);
781 }
782 }
783 }
784
785 // Ignore unrelated casts in STL allocate() since the allocator must cast
786 // from void* to T* before object initialization completes. Don't match on the
787 // namespace because not all allocators are in std::
788 if (D && SanOpts.has(SanitizerKind::CFIUnrelatedCast)) {
789 if (matchesStlAllocatorFn(D, getContext()))
790 SanOpts.Mask &= ~SanitizerKind::CFIUnrelatedCast;
791 }
792
793 // Ignore null checks in coroutine functions since the coroutines passes
794 // are not aware of how to move the extra UBSan instructions across the split
795 // coroutine boundaries.
796 if (D && SanOpts.has(SanitizerKind::Null))
797 if (const auto *FD = dyn_cast<FunctionDecl>(D))
798 if (FD->getBody() &&
799 FD->getBody()->getStmtClass() == Stmt::CoroutineBodyStmtClass)
800 SanOpts.Mask &= ~SanitizerKind::Null;
801
802 // Apply xray attributes to the function (as a string, for now)
803 bool AlwaysXRayAttr = false;
804 if (const auto *XRayAttr = D ? D->getAttr<XRayInstrumentAttr>() : nullptr) {
805 if (CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
806 XRayInstrKind::FunctionEntry) ||
807 CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
808 XRayInstrKind::FunctionExit)) {
809 if (XRayAttr->alwaysXRayInstrument() && ShouldXRayInstrumentFunction()) {
810 Fn->addFnAttr("function-instrument", "xray-always");
811 AlwaysXRayAttr = true;
812 }
813 if (XRayAttr->neverXRayInstrument())
814 Fn->addFnAttr("function-instrument", "xray-never");
815 if (const auto *LogArgs = D->getAttr<XRayLogArgsAttr>())
816 if (ShouldXRayInstrumentFunction())
817 Fn->addFnAttr("xray-log-args",
818 llvm::utostr(LogArgs->getArgumentCount()));
819 }
820 } else {
821 if (ShouldXRayInstrumentFunction() && !CGM.imbueXRayAttrs(Fn, Loc))
822 Fn->addFnAttr(
823 "xray-instruction-threshold",
824 llvm::itostr(CGM.getCodeGenOpts().XRayInstructionThreshold));
825 }
826
827 if (ShouldXRayInstrumentFunction()) {
828 if (CGM.getCodeGenOpts().XRayIgnoreLoops)
829 Fn->addFnAttr("xray-ignore-loops");
830
831 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
832 XRayInstrKind::FunctionExit))
833 Fn->addFnAttr("xray-skip-exit");
834
835 if (!CGM.getCodeGenOpts().XRayInstrumentationBundle.has(
836 XRayInstrKind::FunctionEntry))
837 Fn->addFnAttr("xray-skip-entry");
838
839 auto FuncGroups = CGM.getCodeGenOpts().XRayTotalFunctionGroups;
840 if (FuncGroups > 1) {
841 auto FuncName = llvm::makeArrayRef<uint8_t>(
842 CurFn->getName().bytes_begin(), CurFn->getName().bytes_end());
843 auto Group = crc32(FuncName) % FuncGroups;
844 if (Group != CGM.getCodeGenOpts().XRaySelectedFunctionGroup &&
845 !AlwaysXRayAttr)
846 Fn->addFnAttr("function-instrument", "xray-never");
847 }
848 }
849
850 if (CGM.getCodeGenOpts().getProfileInstr() != CodeGenOptions::ProfileNone)
851 if (CGM.isProfileInstrExcluded(Fn, Loc))
852 Fn->addFnAttr(llvm::Attribute::NoProfile);
853
854 unsigned Count, Offset;
855 if (const auto *Attr =
856 D ? D->getAttr<PatchableFunctionEntryAttr>() : nullptr) {
857 Count = Attr->getCount();
858 Offset = Attr->getOffset();
859 } else {
860 Count = CGM.getCodeGenOpts().PatchableFunctionEntryCount;
861 Offset = CGM.getCodeGenOpts().PatchableFunctionEntryOffset;
862 }
863 if (Count && Offset <= Count) {
864 Fn->addFnAttr("patchable-function-entry", std::to_string(Count - Offset));
865 if (Offset)
866 Fn->addFnAttr("patchable-function-prefix", std::to_string(Offset));
867 }
868
869 // Add no-jump-tables value.
870 if (CGM.getCodeGenOpts().NoUseJumpTables)
871 Fn->addFnAttr("no-jump-tables", "true");
872
873 // Add no-inline-line-tables value.
874 if (CGM.getCodeGenOpts().NoInlineLineTables)
875 Fn->addFnAttr("no-inline-line-tables");
876
877 // Add profile-sample-accurate value.
878 if (CGM.getCodeGenOpts().ProfileSampleAccurate)
879 Fn->addFnAttr("profile-sample-accurate");
880
881 if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
882 Fn->addFnAttr("use-sample-profile");
883
884 if (D && D->hasAttr<CFICanonicalJumpTableAttr>())
885 Fn->addFnAttr("cfi-canonical-jump-table");
886
887 if (getLangOpts().OpenCL) {
888 // Add metadata for a kernel function.
889 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
890 EmitOpenCLKernelMetadata(FD, Fn);
891 }
892
893 // If we are checking function types, emit a function type signature as
894 // prologue data.
895 if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function)) {
896 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
897 if (llvm::Constant *PrologueSig = getPrologueSignature(CGM, FD)) {
898 // Remove any (C++17) exception specifications, to allow calling e.g. a
899 // noexcept function through a non-noexcept pointer.
900 auto ProtoTy =
901 getContext().getFunctionTypeWithExceptionSpec(FD->getType(),
902 EST_None);
903 llvm::Constant *FTRTTIConst =
904 CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true);
905 llvm::Constant *FTRTTIConstEncoded =
906 EncodeAddrForUseInPrologue(Fn, FTRTTIConst);
907 llvm::Constant *PrologueStructElems[] = {PrologueSig,
908 FTRTTIConstEncoded};
909 llvm::Constant *PrologueStructConst =
910 llvm::ConstantStruct::getAnon(PrologueStructElems, /*Packed=*/true);
911 Fn->setPrologueData(PrologueStructConst);
912 }
913 }
914 }
915
916 // If we're checking nullability, we need to know whether we can check the
917 // return value. Initialize the flag to 'true' and refine it in EmitParmDecl.
918 if (SanOpts.has(SanitizerKind::NullabilityReturn)) {
919 auto Nullability = FnRetTy->getNullability(getContext());
920 if (Nullability && *Nullability == NullabilityKind::NonNull) {
921 if (!(SanOpts.has(SanitizerKind::ReturnsNonnullAttribute) &&
922 CurCodeDecl && CurCodeDecl->getAttr<ReturnsNonNullAttr>()))
923 RetValNullabilityPrecondition =
924 llvm::ConstantInt::getTrue(getLLVMContext());
925 }
926 }
927
928 // If we're in C++ mode and the function name is "main", it is guaranteed
929 // to be norecurse by the standard (3.6.1.3 "The function main shall not be
930 // used within a program").
931 //
932 // OpenCL C 2.0 v2.2-11 s6.9.i:
933 // Recursion is not supported.
934 //
935 // SYCL v1.2.1 s3.10:
936 // kernels cannot include RTTI information, exception classes,
937 // recursive code, virtual functions or make use of C++ libraries that
938 // are not compiled for the device.
939 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
940 if ((getLangOpts().CPlusPlus && FD->isMain()) || getLangOpts().OpenCL ||
941 getLangOpts().SYCLIsDevice ||
942 (getLangOpts().CUDA && FD->hasAttr<CUDAGlobalAttr>()))
943 Fn->addFnAttr(llvm::Attribute::NoRecurse);
944 }
945
946 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D)) {
947 Builder.setIsFPConstrained(FD->hasAttr<StrictFPAttr>());
948 if (FD->hasAttr<StrictFPAttr>())
949 Fn->addFnAttr(llvm::Attribute::StrictFP);
950 }
951
952 // If a custom alignment is used, force realigning to this alignment on
953 // any main function which certainly will need it.
954 if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
955 if ((FD->isMain() || FD->isMSVCRTEntryPoint()) &&
956 CGM.getCodeGenOpts().StackAlignment)
957 Fn->addFnAttr("stackrealign");
958
959 llvm::BasicBlock *EntryBB = createBasicBlock("entry", CurFn);
960
961 // Create a marker to make it easy to insert allocas into the entryblock
962 // later. Don't create this with the builder, because we don't want it
963 // folded.
964 llvm::Value *Undef = llvm::UndefValue::get(Int32Ty);
965 AllocaInsertPt = new llvm::BitCastInst(Undef, Int32Ty, "allocapt", EntryBB);
966
967 ReturnBlock = getJumpDestInCurrentScope("return");
968
969 Builder.SetInsertPoint(EntryBB);
970
971 // If we're checking the return value, allocate space for a pointer to a
972 // precise source location of the checked return statement.
973 if (requiresReturnValueCheck()) {
974 ReturnLocation = CreateDefaultAlignTempAlloca(Int8PtrTy, "return.sloc.ptr");
975 InitTempAlloca(ReturnLocation, llvm::ConstantPointerNull::get(Int8PtrTy));
976 }
977
978 // Emit subprogram debug descriptor.
979 if (CGDebugInfo *DI = getDebugInfo()) {
980 // Reconstruct the type from the argument list so that implicit parameters,
981 // such as 'this' and 'vtt', show up in the debug info. Preserve the calling
982 // convention.
983 CallingConv CC = CallingConv::CC_C;
984 if (auto *FD = dyn_cast_or_null<FunctionDecl>(D))
985 if (const auto *SrcFnTy = FD->getType()->getAs<FunctionType>())
986 CC = SrcFnTy->getCallConv();
987 SmallVector<QualType, 16> ArgTypes;
988 for (const VarDecl *VD : Args)
989 ArgTypes.push_back(VD->getType());
990 QualType FnType = getContext().getFunctionType(
991 RetTy, ArgTypes, FunctionProtoType::ExtProtoInfo(CC));
992 DI->emitFunctionStart(GD, Loc, StartLoc, FnType, CurFn, CurFuncIsThunk);
993 }
994
995 if (ShouldInstrumentFunction()) {
996 if (CGM.getCodeGenOpts().InstrumentFunctions)
997 CurFn->addFnAttr("instrument-function-entry", "__cyg_profile_func_enter");
998 if (CGM.getCodeGenOpts().InstrumentFunctionsAfterInlining)
999 CurFn->addFnAttr("instrument-function-entry-inlined",
1000 "__cyg_profile_func_enter");
1001 if (CGM.getCodeGenOpts().InstrumentFunctionEntryBare)
1002 CurFn->addFnAttr("instrument-function-entry-inlined",
1003 "__cyg_profile_func_enter_bare");
1004 }
1005
1006 // Since emitting the mcount call here impacts optimizations such as function
1007 // inlining, we just add an attribute to insert a mcount call in backend.
1008 // The attribute "counting-function" is set to mcount function name which is
1009 // architecture dependent.
1010 if (CGM.getCodeGenOpts().InstrumentForProfiling) {
1011 // Calls to fentry/mcount should not be generated if function has
1012 // the no_instrument_function attribute.
1013 if (!CurFuncDecl || !CurFuncDecl->hasAttr<NoInstrumentFunctionAttr>()) {
1014 if (CGM.getCodeGenOpts().CallFEntry)
1015 Fn->addFnAttr("fentry-call", "true");
1016 else {
1017 Fn->addFnAttr("instrument-function-entry-inlined",
1018 getTarget().getMCountName());
1019 }
1020 if (CGM.getCodeGenOpts().MNopMCount) {
1021 if (!CGM.getCodeGenOpts().CallFEntry)
1022 CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1023 << "-mnop-mcount" << "-mfentry";
1024 Fn->addFnAttr("mnop-mcount");
1025 }
1026
1027 if (CGM.getCodeGenOpts().RecordMCount) {
1028 if (!CGM.getCodeGenOpts().CallFEntry)
1029 CGM.getDiags().Report(diag::err_opt_not_valid_without_opt)
1030 << "-mrecord-mcount" << "-mfentry";
1031 Fn->addFnAttr("mrecord-mcount");
1032 }
1033 }
1034 }
1035
1036 if (CGM.getCodeGenOpts().PackedStack) {
1037 if (getContext().getTargetInfo().getTriple().getArch() !=
1038 llvm::Triple::systemz)
1039 CGM.getDiags().Report(diag::err_opt_not_valid_on_target)
1040 << "-mpacked-stack";
1041 Fn->addFnAttr("packed-stack");
1042 }
1043
1044 if (RetTy->isVoidType()) {
1045 // Void type; nothing to return.
1046 ReturnValue = Address::invalid();
1047
1048 // Count the implicit return.
1049 if (!endsWithReturn(D))
1050 ++NumReturnExprs;
1051 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::Indirect) {
1052 // Indirect return; emit returned value directly into sret slot.
1053 // This reduces code size, and affects correctness in C++.
1054 auto AI = CurFn->arg_begin();
1055 if (CurFnInfo->getReturnInfo().isSRetAfterThis())
1056 ++AI;
1057 ReturnValue = Address(&*AI, CurFnInfo->getReturnInfo().getIndirectAlign());
1058 if (!CurFnInfo->getReturnInfo().getIndirectByVal()) {
1059 ReturnValuePointer =
1060 CreateDefaultAlignTempAlloca(Int8PtrTy, "result.ptr");
1061 Builder.CreateStore(Builder.CreatePointerBitCastOrAddrSpaceCast(
1062 ReturnValue.getPointer(), Int8PtrTy),
1063 ReturnValuePointer);
1064 }
1065 } else if (CurFnInfo->getReturnInfo().getKind() == ABIArgInfo::InAlloca &&
1066 !hasScalarEvaluationKind(CurFnInfo->getReturnType())) {
1067 // Load the sret pointer from the argument struct and return into that.
1068 unsigned Idx = CurFnInfo->getReturnInfo().getInAllocaFieldIndex();
1069 llvm::Function::arg_iterator EI = CurFn->arg_end();
1070 --EI;
1071 llvm::Value *Addr = Builder.CreateStructGEP(nullptr, &*EI, Idx);
1072 llvm::Type *Ty =
1073 cast<llvm::GetElementPtrInst>(Addr)->getResultElementType();
1074 ReturnValuePointer = Address(Addr, getPointerAlign());
1075 Addr = Builder.CreateAlignedLoad(Ty, Addr, getPointerAlign(), "agg.result");
1076 ReturnValue = Address(Addr, CGM.getNaturalTypeAlignment(RetTy));
1077 } else {
1078 ReturnValue = CreateIRTemp(RetTy, "retval");
1079
1080 // Tell the epilog emitter to autorelease the result. We do this
1081 // now so that various specialized functions can suppress it
1082 // during their IR-generation.
1083 if (getLangOpts().ObjCAutoRefCount &&
1084 !CurFnInfo->isReturnsRetained() &&
1085 RetTy->isObjCRetainableType())
1086 AutoreleaseResult = true;
1087 }
1088
1089 EmitStartEHSpec(CurCodeDecl);
1090
1091 PrologueCleanupDepth = EHStack.stable_begin();
1092
1093 // Emit OpenMP specific initialization of the device functions.
1094 if (getLangOpts().OpenMP && CurCodeDecl)
1095 CGM.getOpenMPRuntime().emitFunctionProlog(*this, CurCodeDecl);
1096
1097 EmitFunctionProlog(*CurFnInfo, CurFn, Args);
1098
1099 if (D && isa<CXXMethodDecl>(D) && cast<CXXMethodDecl>(D)->isInstance()) {
1100 CGM.getCXXABI().EmitInstanceFunctionProlog(*this);
1101 const CXXMethodDecl *MD = cast<CXXMethodDecl>(D);
1102 if (MD->getParent()->isLambda() &&
1103 MD->getOverloadedOperator() == OO_Call) {
1104 // We're in a lambda; figure out the captures.
1105 MD->getParent()->getCaptureFields(LambdaCaptureFields,
1106 LambdaThisCaptureField);
1107 if (LambdaThisCaptureField) {
1108 // If the lambda captures the object referred to by '*this' - either by
1109 // value or by reference, make sure CXXThisValue points to the correct
1110 // object.
1111
1112 // Get the lvalue for the field (which is a copy of the enclosing object
1113 // or contains the address of the enclosing object).
1114 LValue ThisFieldLValue = EmitLValueForLambdaField(LambdaThisCaptureField);
1115 if (!LambdaThisCaptureField->getType()->isPointerType()) {
1116 // If the enclosing object was captured by value, just use its address.
1117 CXXThisValue = ThisFieldLValue.getAddress(*this).getPointer();
1118 } else {
1119 // Load the lvalue pointed to by the field, since '*this' was captured
1120 // by reference.
1121 CXXThisValue =
1122 EmitLoadOfLValue(ThisFieldLValue, SourceLocation()).getScalarVal();
1123 }
1124 }
1125 for (auto *FD : MD->getParent()->fields()) {
1126 if (FD->hasCapturedVLAType()) {
1127 auto *ExprArg = EmitLoadOfLValue(EmitLValueForLambdaField(FD),
1128 SourceLocation()).getScalarVal();
1129 auto VAT = FD->getCapturedVLAType();
1130 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
1131 }
1132 }
1133 } else {
1134 // Not in a lambda; just use 'this' from the method.
1135 // FIXME: Should we generate a new load for each use of 'this'? The
1136 // fast register allocator would be happier...
1137 CXXThisValue = CXXABIThisValue;
1138 }
1139
1140 // Check the 'this' pointer once per function, if it's available.
1141 if (CXXABIThisValue) {
1142 SanitizerSet SkippedChecks;
1143 SkippedChecks.set(SanitizerKind::ObjectSize, true);
1144 QualType ThisTy = MD->getThisType();
1145
1146 // If this is the call operator of a lambda with no capture-default, it
1147 // may have a static invoker function, which may call this operator with
1148 // a null 'this' pointer.
1149 if (isLambdaCallOperator(MD) &&
1150 MD->getParent()->getLambdaCaptureDefault() == LCD_None)
1151 SkippedChecks.set(SanitizerKind::Null, true);
1152
1153 EmitTypeCheck(
1154 isa<CXXConstructorDecl>(MD) ? TCK_ConstructorCall : TCK_MemberCall,
1155 Loc, CXXABIThisValue, ThisTy, CXXABIThisAlignment, SkippedChecks);
1156 }
1157 }
1158
1159 // If any of the arguments have a variably modified type, make sure to
1160 // emit the type size.
1161 for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
1162 i != e; ++i) {
1163 const VarDecl *VD = *i;
1164
1165 // Dig out the type as written from ParmVarDecls; it's unclear whether
1166 // the standard (C99 6.9.1p10) requires this, but we're following the
1167 // precedent set by gcc.
1168 QualType Ty;
1169 if (const ParmVarDecl *PVD = dyn_cast<ParmVarDecl>(VD))
1170 Ty = PVD->getOriginalType();
1171 else
1172 Ty = VD->getType();
1173
1174 if (Ty->isVariablyModifiedType())
1175 EmitVariablyModifiedType(Ty);
1176 }
1177 // Emit a location at the end of the prologue.
1178 if (CGDebugInfo *DI = getDebugInfo())
1179 DI->EmitLocation(Builder, StartLoc);
1180
1181 // TODO: Do we need to handle this in two places like we do with
1182 // target-features/target-cpu?
1183 if (CurFuncDecl)
1184 if (const auto *VecWidth = CurFuncDecl->getAttr<MinVectorWidthAttr>())
1185 LargestVectorWidth = VecWidth->getVectorWidth();
1186 }
1187
EmitFunctionBody(const Stmt * Body)1188 void CodeGenFunction::EmitFunctionBody(const Stmt *Body) {
1189 incrementProfileCounter(Body);
1190 if (const CompoundStmt *S = dyn_cast<CompoundStmt>(Body))
1191 EmitCompoundStmtWithoutScope(*S);
1192 else
1193 EmitStmt(Body);
1194
1195 // This is checked after emitting the function body so we know if there
1196 // are any permitted infinite loops.
1197 if (checkIfFunctionMustProgress())
1198 CurFn->addFnAttr(llvm::Attribute::MustProgress);
1199 }
1200
1201 /// When instrumenting to collect profile data, the counts for some blocks
1202 /// such as switch cases need to not include the fall-through counts, so
1203 /// emit a branch around the instrumentation code. When not instrumenting,
1204 /// this just calls EmitBlock().
EmitBlockWithFallThrough(llvm::BasicBlock * BB,const Stmt * S)1205 void CodeGenFunction::EmitBlockWithFallThrough(llvm::BasicBlock *BB,
1206 const Stmt *S) {
1207 llvm::BasicBlock *SkipCountBB = nullptr;
1208 if (HaveInsertPoint() && CGM.getCodeGenOpts().hasProfileClangInstr()) {
1209 // When instrumenting for profiling, the fallthrough to certain
1210 // statements needs to skip over the instrumentation code so that we
1211 // get an accurate count.
1212 SkipCountBB = createBasicBlock("skipcount");
1213 EmitBranch(SkipCountBB);
1214 }
1215 EmitBlock(BB);
1216 uint64_t CurrentCount = getCurrentProfileCount();
1217 incrementProfileCounter(S);
1218 setCurrentProfileCount(getCurrentProfileCount() + CurrentCount);
1219 if (SkipCountBB)
1220 EmitBlock(SkipCountBB);
1221 }
1222
1223 /// Tries to mark the given function nounwind based on the
1224 /// non-existence of any throwing calls within it. We believe this is
1225 /// lightweight enough to do at -O0.
TryMarkNoThrow(llvm::Function * F)1226 static void TryMarkNoThrow(llvm::Function *F) {
1227 // LLVM treats 'nounwind' on a function as part of the type, so we
1228 // can't do this on functions that can be overwritten.
1229 if (F->isInterposable()) return;
1230
1231 for (llvm::BasicBlock &BB : *F)
1232 for (llvm::Instruction &I : BB)
1233 if (I.mayThrow())
1234 return;
1235
1236 F->setDoesNotThrow();
1237 }
1238
BuildFunctionArgList(GlobalDecl GD,FunctionArgList & Args)1239 QualType CodeGenFunction::BuildFunctionArgList(GlobalDecl GD,
1240 FunctionArgList &Args) {
1241 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1242 QualType ResTy = FD->getReturnType();
1243
1244 const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD);
1245 if (MD && MD->isInstance()) {
1246 if (CGM.getCXXABI().HasThisReturn(GD))
1247 ResTy = MD->getThisType();
1248 else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
1249 ResTy = CGM.getContext().VoidPtrTy;
1250 CGM.getCXXABI().buildThisParam(*this, Args);
1251 }
1252
1253 // The base version of an inheriting constructor whose constructed base is a
1254 // virtual base is not passed any arguments (because it doesn't actually call
1255 // the inherited constructor).
1256 bool PassedParams = true;
1257 if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
1258 if (auto Inherited = CD->getInheritedConstructor())
1259 PassedParams =
1260 getTypes().inheritingCtorHasParams(Inherited, GD.getCtorType());
1261
1262 if (PassedParams) {
1263 for (auto *Param : FD->parameters()) {
1264 Args.push_back(Param);
1265 if (!Param->hasAttr<PassObjectSizeAttr>())
1266 continue;
1267
1268 auto *Implicit = ImplicitParamDecl::Create(
1269 getContext(), Param->getDeclContext(), Param->getLocation(),
1270 /*Id=*/nullptr, getContext().getSizeType(), ImplicitParamDecl::Other);
1271 SizeArguments[Param] = Implicit;
1272 Args.push_back(Implicit);
1273 }
1274 }
1275
1276 if (MD && (isa<CXXConstructorDecl>(MD) || isa<CXXDestructorDecl>(MD)))
1277 CGM.getCXXABI().addImplicitStructorParams(*this, ResTy, Args);
1278
1279 return ResTy;
1280 }
1281
GenerateCode(GlobalDecl GD,llvm::Function * Fn,const CGFunctionInfo & FnInfo)1282 void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
1283 const CGFunctionInfo &FnInfo) {
1284 const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
1285 CurGD = GD;
1286
1287 FunctionArgList Args;
1288 QualType ResTy = BuildFunctionArgList(GD, Args);
1289
1290 // Check if we should generate debug info for this function.
1291 if (FD->hasAttr<NoDebugAttr>())
1292 DebugInfo = nullptr; // disable debug info indefinitely for this function
1293
1294 // The function might not have a body if we're generating thunks for a
1295 // function declaration.
1296 SourceRange BodyRange;
1297 if (Stmt *Body = FD->getBody())
1298 BodyRange = Body->getSourceRange();
1299 else
1300 BodyRange = FD->getLocation();
1301 CurEHLocation = BodyRange.getEnd();
1302
1303 // Use the location of the start of the function to determine where
1304 // the function definition is located. By default use the location
1305 // of the declaration as the location for the subprogram. A function
1306 // may lack a declaration in the source code if it is created by code
1307 // gen. (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
1308 SourceLocation Loc = FD->getLocation();
1309
1310 // If this is a function specialization then use the pattern body
1311 // as the location for the function.
1312 if (const FunctionDecl *SpecDecl = FD->getTemplateInstantiationPattern())
1313 if (SpecDecl->hasBody(SpecDecl))
1314 Loc = SpecDecl->getLocation();
1315
1316 Stmt *Body = FD->getBody();
1317
1318 if (Body) {
1319 // Coroutines always emit lifetime markers.
1320 if (isa<CoroutineBodyStmt>(Body))
1321 ShouldEmitLifetimeMarkers = true;
1322
1323 // Initialize helper which will detect jumps which can cause invalid
1324 // lifetime markers.
1325 if (ShouldEmitLifetimeMarkers)
1326 Bypasses.Init(Body);
1327 }
1328
1329 // Emit the standard function prologue.
1330 StartFunction(GD, ResTy, Fn, FnInfo, Args, Loc, BodyRange.getBegin());
1331
1332 // Save parameters for coroutine function.
1333 if (Body && isa_and_nonnull<CoroutineBodyStmt>(Body))
1334 for (const auto *ParamDecl : FD->parameters())
1335 FnArgs.push_back(ParamDecl);
1336
1337 // Generate the body of the function.
1338 PGO.assignRegionCounters(GD, CurFn);
1339 if (isa<CXXDestructorDecl>(FD))
1340 EmitDestructorBody(Args);
1341 else if (isa<CXXConstructorDecl>(FD))
1342 EmitConstructorBody(Args);
1343 else if (getLangOpts().CUDA &&
1344 !getLangOpts().CUDAIsDevice &&
1345 FD->hasAttr<CUDAGlobalAttr>())
1346 CGM.getCUDARuntime().emitDeviceStub(*this, Args);
1347 else if (isa<CXXMethodDecl>(FD) &&
1348 cast<CXXMethodDecl>(FD)->isLambdaStaticInvoker()) {
1349 // The lambda static invoker function is special, because it forwards or
1350 // clones the body of the function call operator (but is actually static).
1351 EmitLambdaStaticInvokeBody(cast<CXXMethodDecl>(FD));
1352 } else if (FD->isDefaulted() && isa<CXXMethodDecl>(FD) &&
1353 (cast<CXXMethodDecl>(FD)->isCopyAssignmentOperator() ||
1354 cast<CXXMethodDecl>(FD)->isMoveAssignmentOperator())) {
1355 // Implicit copy-assignment gets the same special treatment as implicit
1356 // copy-constructors.
1357 emitImplicitAssignmentOperatorBody(Args);
1358 } else if (Body) {
1359 EmitFunctionBody(Body);
1360 } else
1361 llvm_unreachable("no definition for emitted function");
1362
1363 // C++11 [stmt.return]p2:
1364 // Flowing off the end of a function [...] results in undefined behavior in
1365 // a value-returning function.
1366 // C11 6.9.1p12:
1367 // If the '}' that terminates a function is reached, and the value of the
1368 // function call is used by the caller, the behavior is undefined.
1369 if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() && !SawAsmBlock &&
1370 !FD->getReturnType()->isVoidType() && Builder.GetInsertBlock()) {
1371 bool ShouldEmitUnreachable =
1372 CGM.getCodeGenOpts().StrictReturn ||
1373 !CGM.MayDropFunctionReturn(FD->getASTContext(), FD->getReturnType());
1374 if (SanOpts.has(SanitizerKind::Return)) {
1375 SanitizerScope SanScope(this);
1376 llvm::Value *IsFalse = Builder.getFalse();
1377 EmitCheck(std::make_pair(IsFalse, SanitizerKind::Return),
1378 SanitizerHandler::MissingReturn,
1379 EmitCheckSourceLocation(FD->getLocation()), None);
1380 } else if (ShouldEmitUnreachable) {
1381 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
1382 EmitTrapCall(llvm::Intrinsic::trap);
1383 }
1384 if (SanOpts.has(SanitizerKind::Return) || ShouldEmitUnreachable) {
1385 Builder.CreateUnreachable();
1386 Builder.ClearInsertionPoint();
1387 }
1388 }
1389
1390 // Emit the standard function epilogue.
1391 FinishFunction(BodyRange.getEnd());
1392
1393 // If we haven't marked the function nothrow through other means, do
1394 // a quick pass now to see if we can.
1395 if (!CurFn->doesNotThrow())
1396 TryMarkNoThrow(CurFn);
1397 }
1398
1399 /// ContainsLabel - Return true if the statement contains a label in it. If
1400 /// this statement is not executed normally, it not containing a label means
1401 /// that we can just remove the code.
ContainsLabel(const Stmt * S,bool IgnoreCaseStmts)1402 bool CodeGenFunction::ContainsLabel(const Stmt *S, bool IgnoreCaseStmts) {
1403 // Null statement, not a label!
1404 if (!S) return false;
1405
1406 // If this is a label, we have to emit the code, consider something like:
1407 // if (0) { ... foo: bar(); } goto foo;
1408 //
1409 // TODO: If anyone cared, we could track __label__'s, since we know that you
1410 // can't jump to one from outside their declared region.
1411 if (isa<LabelStmt>(S))
1412 return true;
1413
1414 // If this is a case/default statement, and we haven't seen a switch, we have
1415 // to emit the code.
1416 if (isa<SwitchCase>(S) && !IgnoreCaseStmts)
1417 return true;
1418
1419 // If this is a switch statement, we want to ignore cases below it.
1420 if (isa<SwitchStmt>(S))
1421 IgnoreCaseStmts = true;
1422
1423 // Scan subexpressions for verboten labels.
1424 for (const Stmt *SubStmt : S->children())
1425 if (ContainsLabel(SubStmt, IgnoreCaseStmts))
1426 return true;
1427
1428 return false;
1429 }
1430
1431 /// containsBreak - Return true if the statement contains a break out of it.
1432 /// If the statement (recursively) contains a switch or loop with a break
1433 /// inside of it, this is fine.
containsBreak(const Stmt * S)1434 bool CodeGenFunction::containsBreak(const Stmt *S) {
1435 // Null statement, not a label!
1436 if (!S) return false;
1437
1438 // If this is a switch or loop that defines its own break scope, then we can
1439 // include it and anything inside of it.
1440 if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) || isa<DoStmt>(S) ||
1441 isa<ForStmt>(S))
1442 return false;
1443
1444 if (isa<BreakStmt>(S))
1445 return true;
1446
1447 // Scan subexpressions for verboten breaks.
1448 for (const Stmt *SubStmt : S->children())
1449 if (containsBreak(SubStmt))
1450 return true;
1451
1452 return false;
1453 }
1454
mightAddDeclToScope(const Stmt * S)1455 bool CodeGenFunction::mightAddDeclToScope(const Stmt *S) {
1456 if (!S) return false;
1457
1458 // Some statement kinds add a scope and thus never add a decl to the current
1459 // scope. Note, this list is longer than the list of statements that might
1460 // have an unscoped decl nested within them, but this way is conservatively
1461 // correct even if more statement kinds are added.
1462 if (isa<IfStmt>(S) || isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
1463 isa<DoStmt>(S) || isa<ForStmt>(S) || isa<CompoundStmt>(S) ||
1464 isa<CXXForRangeStmt>(S) || isa<CXXTryStmt>(S) ||
1465 isa<ObjCForCollectionStmt>(S) || isa<ObjCAtTryStmt>(S))
1466 return false;
1467
1468 if (isa<DeclStmt>(S))
1469 return true;
1470
1471 for (const Stmt *SubStmt : S->children())
1472 if (mightAddDeclToScope(SubStmt))
1473 return true;
1474
1475 return false;
1476 }
1477
1478 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1479 /// to a constant, or if it does but contains a label, return false. If it
1480 /// constant folds return true and set the boolean result in Result.
ConstantFoldsToSimpleInteger(const Expr * Cond,bool & ResultBool,bool AllowLabels)1481 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1482 bool &ResultBool,
1483 bool AllowLabels) {
1484 llvm::APSInt ResultInt;
1485 if (!ConstantFoldsToSimpleInteger(Cond, ResultInt, AllowLabels))
1486 return false;
1487
1488 ResultBool = ResultInt.getBoolValue();
1489 return true;
1490 }
1491
1492 /// ConstantFoldsToSimpleInteger - If the specified expression does not fold
1493 /// to a constant, or if it does but contains a label, return false. If it
1494 /// constant folds return true and set the folded value.
ConstantFoldsToSimpleInteger(const Expr * Cond,llvm::APSInt & ResultInt,bool AllowLabels)1495 bool CodeGenFunction::ConstantFoldsToSimpleInteger(const Expr *Cond,
1496 llvm::APSInt &ResultInt,
1497 bool AllowLabels) {
1498 // FIXME: Rename and handle conversion of other evaluatable things
1499 // to bool.
1500 Expr::EvalResult Result;
1501 if (!Cond->EvaluateAsInt(Result, getContext()))
1502 return false; // Not foldable, not integer or not fully evaluatable.
1503
1504 llvm::APSInt Int = Result.Val.getInt();
1505 if (!AllowLabels && CodeGenFunction::ContainsLabel(Cond))
1506 return false; // Contains a label.
1507
1508 ResultInt = Int;
1509 return true;
1510 }
1511
1512 /// Determine whether the given condition is an instrumentable condition
1513 /// (i.e. no "&&" or "||").
isInstrumentedCondition(const Expr * C)1514 bool CodeGenFunction::isInstrumentedCondition(const Expr *C) {
1515 // Bypass simplistic logical-NOT operator before determining whether the
1516 // condition contains any other logical operator.
1517 if (const UnaryOperator *UnOp = dyn_cast<UnaryOperator>(C->IgnoreParens()))
1518 if (UnOp->getOpcode() == UO_LNot)
1519 C = UnOp->getSubExpr();
1520
1521 const BinaryOperator *BOp = dyn_cast<BinaryOperator>(C->IgnoreParens());
1522 return (!BOp || !BOp->isLogicalOp());
1523 }
1524
1525 /// EmitBranchToCounterBlock - Emit a conditional branch to a new block that
1526 /// increments a profile counter based on the semantics of the given logical
1527 /// operator opcode. This is used to instrument branch condition coverage for
1528 /// logical operators.
EmitBranchToCounterBlock(const Expr * Cond,BinaryOperator::Opcode LOp,llvm::BasicBlock * TrueBlock,llvm::BasicBlock * FalseBlock,uint64_t TrueCount,Stmt::Likelihood LH,const Expr * CntrIdx)1529 void CodeGenFunction::EmitBranchToCounterBlock(
1530 const Expr *Cond, BinaryOperator::Opcode LOp, llvm::BasicBlock *TrueBlock,
1531 llvm::BasicBlock *FalseBlock, uint64_t TrueCount /* = 0 */,
1532 Stmt::Likelihood LH /* =None */, const Expr *CntrIdx /* = nullptr */) {
1533 // If not instrumenting, just emit a branch.
1534 bool InstrumentRegions = CGM.getCodeGenOpts().hasProfileClangInstr();
1535 if (!InstrumentRegions || !isInstrumentedCondition(Cond))
1536 return EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount, LH);
1537
1538 llvm::BasicBlock *ThenBlock = NULL;
1539 llvm::BasicBlock *ElseBlock = NULL;
1540 llvm::BasicBlock *NextBlock = NULL;
1541
1542 // Create the block we'll use to increment the appropriate counter.
1543 llvm::BasicBlock *CounterIncrBlock = createBasicBlock("lop.rhscnt");
1544
1545 // Set block pointers according to Logical-AND (BO_LAnd) semantics. This
1546 // means we need to evaluate the condition and increment the counter on TRUE:
1547 //
1548 // if (Cond)
1549 // goto CounterIncrBlock;
1550 // else
1551 // goto FalseBlock;
1552 //
1553 // CounterIncrBlock:
1554 // Counter++;
1555 // goto TrueBlock;
1556
1557 if (LOp == BO_LAnd) {
1558 ThenBlock = CounterIncrBlock;
1559 ElseBlock = FalseBlock;
1560 NextBlock = TrueBlock;
1561 }
1562
1563 // Set block pointers according to Logical-OR (BO_LOr) semantics. This means
1564 // we need to evaluate the condition and increment the counter on FALSE:
1565 //
1566 // if (Cond)
1567 // goto TrueBlock;
1568 // else
1569 // goto CounterIncrBlock;
1570 //
1571 // CounterIncrBlock:
1572 // Counter++;
1573 // goto FalseBlock;
1574
1575 else if (LOp == BO_LOr) {
1576 ThenBlock = TrueBlock;
1577 ElseBlock = CounterIncrBlock;
1578 NextBlock = FalseBlock;
1579 } else {
1580 llvm_unreachable("Expected Opcode must be that of a Logical Operator");
1581 }
1582
1583 // Emit Branch based on condition.
1584 EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, TrueCount, LH);
1585
1586 // Emit the block containing the counter increment(s).
1587 EmitBlock(CounterIncrBlock);
1588
1589 // Increment corresponding counter; if index not provided, use Cond as index.
1590 incrementProfileCounter(CntrIdx ? CntrIdx : Cond);
1591
1592 // Go to the next block.
1593 EmitBranch(NextBlock);
1594 }
1595
1596 /// EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g. for an if
1597 /// statement) to the specified blocks. Based on the condition, this might try
1598 /// to simplify the codegen of the conditional based on the branch.
1599 /// \param LH The value of the likelihood attribute on the True branch.
EmitBranchOnBoolExpr(const Expr * Cond,llvm::BasicBlock * TrueBlock,llvm::BasicBlock * FalseBlock,uint64_t TrueCount,Stmt::Likelihood LH)1600 void CodeGenFunction::EmitBranchOnBoolExpr(const Expr *Cond,
1601 llvm::BasicBlock *TrueBlock,
1602 llvm::BasicBlock *FalseBlock,
1603 uint64_t TrueCount,
1604 Stmt::Likelihood LH) {
1605 Cond = Cond->IgnoreParens();
1606
1607 if (const BinaryOperator *CondBOp = dyn_cast<BinaryOperator>(Cond)) {
1608
1609 // Handle X && Y in a condition.
1610 if (CondBOp->getOpcode() == BO_LAnd) {
1611 // If we have "1 && X", simplify the code. "0 && X" would have constant
1612 // folded if the case was simple enough.
1613 bool ConstantBool = false;
1614 if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1615 ConstantBool) {
1616 // br(1 && X) -> br(X).
1617 incrementProfileCounter(CondBOp);
1618 return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1619 FalseBlock, TrueCount, LH);
1620 }
1621
1622 // If we have "X && 1", simplify the code to use an uncond branch.
1623 // "X && 0" would have been constant folded to 0.
1624 if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1625 ConstantBool) {
1626 // br(X && 1) -> br(X).
1627 return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LAnd, TrueBlock,
1628 FalseBlock, TrueCount, LH, CondBOp);
1629 }
1630
1631 // Emit the LHS as a conditional. If the LHS conditional is false, we
1632 // want to jump to the FalseBlock.
1633 llvm::BasicBlock *LHSTrue = createBasicBlock("land.lhs.true");
1634 // The counter tells us how often we evaluate RHS, and all of TrueCount
1635 // can be propagated to that branch.
1636 uint64_t RHSCount = getProfileCount(CondBOp->getRHS());
1637
1638 ConditionalEvaluation eval(*this);
1639 {
1640 ApplyDebugLocation DL(*this, Cond);
1641 // Propagate the likelihood attribute like __builtin_expect
1642 // __builtin_expect(X && Y, 1) -> X and Y are likely
1643 // __builtin_expect(X && Y, 0) -> only Y is unlikely
1644 EmitBranchOnBoolExpr(CondBOp->getLHS(), LHSTrue, FalseBlock, RHSCount,
1645 LH == Stmt::LH_Unlikely ? Stmt::LH_None : LH);
1646 EmitBlock(LHSTrue);
1647 }
1648
1649 incrementProfileCounter(CondBOp);
1650 setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1651
1652 // Any temporaries created here are conditional.
1653 eval.begin(*this);
1654 EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LAnd, TrueBlock,
1655 FalseBlock, TrueCount, LH);
1656 eval.end(*this);
1657
1658 return;
1659 }
1660
1661 if (CondBOp->getOpcode() == BO_LOr) {
1662 // If we have "0 || X", simplify the code. "1 || X" would have constant
1663 // folded if the case was simple enough.
1664 bool ConstantBool = false;
1665 if (ConstantFoldsToSimpleInteger(CondBOp->getLHS(), ConstantBool) &&
1666 !ConstantBool) {
1667 // br(0 || X) -> br(X).
1668 incrementProfileCounter(CondBOp);
1669 return EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock,
1670 FalseBlock, TrueCount, LH);
1671 }
1672
1673 // If we have "X || 0", simplify the code to use an uncond branch.
1674 // "X || 1" would have been constant folded to 1.
1675 if (ConstantFoldsToSimpleInteger(CondBOp->getRHS(), ConstantBool) &&
1676 !ConstantBool) {
1677 // br(X || 0) -> br(X).
1678 return EmitBranchToCounterBlock(CondBOp->getLHS(), BO_LOr, TrueBlock,
1679 FalseBlock, TrueCount, LH, CondBOp);
1680 }
1681
1682 // Emit the LHS as a conditional. If the LHS conditional is true, we
1683 // want to jump to the TrueBlock.
1684 llvm::BasicBlock *LHSFalse = createBasicBlock("lor.lhs.false");
1685 // We have the count for entry to the RHS and for the whole expression
1686 // being true, so we can divy up True count between the short circuit and
1687 // the RHS.
1688 uint64_t LHSCount =
1689 getCurrentProfileCount() - getProfileCount(CondBOp->getRHS());
1690 uint64_t RHSCount = TrueCount - LHSCount;
1691
1692 ConditionalEvaluation eval(*this);
1693 {
1694 // Propagate the likelihood attribute like __builtin_expect
1695 // __builtin_expect(X || Y, 1) -> only Y is likely
1696 // __builtin_expect(X || Y, 0) -> both X and Y are unlikely
1697 ApplyDebugLocation DL(*this, Cond);
1698 EmitBranchOnBoolExpr(CondBOp->getLHS(), TrueBlock, LHSFalse, LHSCount,
1699 LH == Stmt::LH_Likely ? Stmt::LH_None : LH);
1700 EmitBlock(LHSFalse);
1701 }
1702
1703 incrementProfileCounter(CondBOp);
1704 setCurrentProfileCount(getProfileCount(CondBOp->getRHS()));
1705
1706 // Any temporaries created here are conditional.
1707 eval.begin(*this);
1708 EmitBranchToCounterBlock(CondBOp->getRHS(), BO_LOr, TrueBlock, FalseBlock,
1709 RHSCount, LH);
1710
1711 eval.end(*this);
1712
1713 return;
1714 }
1715 }
1716
1717 if (const UnaryOperator *CondUOp = dyn_cast<UnaryOperator>(Cond)) {
1718 // br(!x, t, f) -> br(x, f, t)
1719 if (CondUOp->getOpcode() == UO_LNot) {
1720 // Negate the count.
1721 uint64_t FalseCount = getCurrentProfileCount() - TrueCount;
1722 // The values of the enum are chosen to make this negation possible.
1723 LH = static_cast<Stmt::Likelihood>(-LH);
1724 // Negate the condition and swap the destination blocks.
1725 return EmitBranchOnBoolExpr(CondUOp->getSubExpr(), FalseBlock, TrueBlock,
1726 FalseCount, LH);
1727 }
1728 }
1729
1730 if (const ConditionalOperator *CondOp = dyn_cast<ConditionalOperator>(Cond)) {
1731 // br(c ? x : y, t, f) -> br(c, br(x, t, f), br(y, t, f))
1732 llvm::BasicBlock *LHSBlock = createBasicBlock("cond.true");
1733 llvm::BasicBlock *RHSBlock = createBasicBlock("cond.false");
1734
1735 // The ConditionalOperator itself has no likelihood information for its
1736 // true and false branches. This matches the behavior of __builtin_expect.
1737 ConditionalEvaluation cond(*this);
1738 EmitBranchOnBoolExpr(CondOp->getCond(), LHSBlock, RHSBlock,
1739 getProfileCount(CondOp), Stmt::LH_None);
1740
1741 // When computing PGO branch weights, we only know the overall count for
1742 // the true block. This code is essentially doing tail duplication of the
1743 // naive code-gen, introducing new edges for which counts are not
1744 // available. Divide the counts proportionally between the LHS and RHS of
1745 // the conditional operator.
1746 uint64_t LHSScaledTrueCount = 0;
1747 if (TrueCount) {
1748 double LHSRatio =
1749 getProfileCount(CondOp) / (double)getCurrentProfileCount();
1750 LHSScaledTrueCount = TrueCount * LHSRatio;
1751 }
1752
1753 cond.begin(*this);
1754 EmitBlock(LHSBlock);
1755 incrementProfileCounter(CondOp);
1756 {
1757 ApplyDebugLocation DL(*this, Cond);
1758 EmitBranchOnBoolExpr(CondOp->getLHS(), TrueBlock, FalseBlock,
1759 LHSScaledTrueCount, LH);
1760 }
1761 cond.end(*this);
1762
1763 cond.begin(*this);
1764 EmitBlock(RHSBlock);
1765 EmitBranchOnBoolExpr(CondOp->getRHS(), TrueBlock, FalseBlock,
1766 TrueCount - LHSScaledTrueCount, LH);
1767 cond.end(*this);
1768
1769 return;
1770 }
1771
1772 if (const CXXThrowExpr *Throw = dyn_cast<CXXThrowExpr>(Cond)) {
1773 // Conditional operator handling can give us a throw expression as a
1774 // condition for a case like:
1775 // br(c ? throw x : y, t, f) -> br(c, br(throw x, t, f), br(y, t, f)
1776 // Fold this to:
1777 // br(c, throw x, br(y, t, f))
1778 EmitCXXThrowExpr(Throw, /*KeepInsertionPoint*/false);
1779 return;
1780 }
1781
1782 // Emit the code with the fully general case.
1783 llvm::Value *CondV;
1784 {
1785 ApplyDebugLocation DL(*this, Cond);
1786 CondV = EvaluateExprAsBool(Cond);
1787 }
1788
1789 llvm::MDNode *Weights = nullptr;
1790 llvm::MDNode *Unpredictable = nullptr;
1791
1792 // If the branch has a condition wrapped by __builtin_unpredictable,
1793 // create metadata that specifies that the branch is unpredictable.
1794 // Don't bother if not optimizing because that metadata would not be used.
1795 auto *Call = dyn_cast<CallExpr>(Cond->IgnoreImpCasts());
1796 if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
1797 auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
1798 if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
1799 llvm::MDBuilder MDHelper(getLLVMContext());
1800 Unpredictable = MDHelper.createUnpredictable();
1801 }
1802 }
1803
1804 // If there is a Likelihood knowledge for the cond, lower it.
1805 // Note that if not optimizing this won't emit anything.
1806 llvm::Value *NewCondV = emitCondLikelihoodViaExpectIntrinsic(CondV, LH);
1807 if (CondV != NewCondV)
1808 CondV = NewCondV;
1809 else {
1810 // Otherwise, lower profile counts. Note that we do this even at -O0.
1811 uint64_t CurrentCount = std::max(getCurrentProfileCount(), TrueCount);
1812 Weights = createProfileWeights(TrueCount, CurrentCount - TrueCount);
1813 }
1814
1815 Builder.CreateCondBr(CondV, TrueBlock, FalseBlock, Weights, Unpredictable);
1816 }
1817
1818 /// ErrorUnsupported - Print out an error that codegen doesn't support the
1819 /// specified stmt yet.
ErrorUnsupported(const Stmt * S,const char * Type)1820 void CodeGenFunction::ErrorUnsupported(const Stmt *S, const char *Type) {
1821 CGM.ErrorUnsupported(S, Type);
1822 }
1823
1824 /// emitNonZeroVLAInit - Emit the "zero" initialization of a
1825 /// variable-length array whose elements have a non-zero bit-pattern.
1826 ///
1827 /// \param baseType the inner-most element type of the array
1828 /// \param src - a char* pointing to the bit-pattern for a single
1829 /// base element of the array
1830 /// \param sizeInChars - the total size of the VLA, in chars
emitNonZeroVLAInit(CodeGenFunction & CGF,QualType baseType,Address dest,Address src,llvm::Value * sizeInChars)1831 static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
1832 Address dest, Address src,
1833 llvm::Value *sizeInChars) {
1834 CGBuilderTy &Builder = CGF.Builder;
1835
1836 CharUnits baseSize = CGF.getContext().getTypeSizeInChars(baseType);
1837 llvm::Value *baseSizeInChars
1838 = llvm::ConstantInt::get(CGF.IntPtrTy, baseSize.getQuantity());
1839
1840 Address begin =
1841 Builder.CreateElementBitCast(dest, CGF.Int8Ty, "vla.begin");
1842 llvm::Value *end = Builder.CreateInBoundsGEP(
1843 begin.getElementType(), begin.getPointer(), sizeInChars, "vla.end");
1844
1845 llvm::BasicBlock *originBB = CGF.Builder.GetInsertBlock();
1846 llvm::BasicBlock *loopBB = CGF.createBasicBlock("vla-init.loop");
1847 llvm::BasicBlock *contBB = CGF.createBasicBlock("vla-init.cont");
1848
1849 // Make a loop over the VLA. C99 guarantees that the VLA element
1850 // count must be nonzero.
1851 CGF.EmitBlock(loopBB);
1852
1853 llvm::PHINode *cur = Builder.CreatePHI(begin.getType(), 2, "vla.cur");
1854 cur->addIncoming(begin.getPointer(), originBB);
1855
1856 CharUnits curAlign =
1857 dest.getAlignment().alignmentOfArrayElement(baseSize);
1858
1859 // memcpy the individual element bit-pattern.
1860 Builder.CreateMemCpy(Address(cur, curAlign), src, baseSizeInChars,
1861 /*volatile*/ false);
1862
1863 // Go to the next element.
1864 llvm::Value *next =
1865 Builder.CreateInBoundsGEP(CGF.Int8Ty, cur, baseSizeInChars, "vla.next");
1866
1867 // Leave if that's the end of the VLA.
1868 llvm::Value *done = Builder.CreateICmpEQ(next, end, "vla-init.isdone");
1869 Builder.CreateCondBr(done, contBB, loopBB);
1870 cur->addIncoming(next, loopBB);
1871
1872 CGF.EmitBlock(contBB);
1873 }
1874
1875 void
EmitNullInitialization(Address DestPtr,QualType Ty)1876 CodeGenFunction::EmitNullInitialization(Address DestPtr, QualType Ty) {
1877 // Ignore empty classes in C++.
1878 if (getLangOpts().CPlusPlus) {
1879 if (const RecordType *RT = Ty->getAs<RecordType>()) {
1880 if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
1881 return;
1882 }
1883 }
1884
1885 // Cast the dest ptr to the appropriate i8 pointer type.
1886 if (DestPtr.getElementType() != Int8Ty)
1887 DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty);
1888
1889 // Get size and alignment info for this aggregate.
1890 CharUnits size = getContext().getTypeSizeInChars(Ty);
1891
1892 llvm::Value *SizeVal;
1893 const VariableArrayType *vla;
1894
1895 // Don't bother emitting a zero-byte memset.
1896 if (size.isZero()) {
1897 // But note that getTypeInfo returns 0 for a VLA.
1898 if (const VariableArrayType *vlaType =
1899 dyn_cast_or_null<VariableArrayType>(
1900 getContext().getAsArrayType(Ty))) {
1901 auto VlaSize = getVLASize(vlaType);
1902 SizeVal = VlaSize.NumElts;
1903 CharUnits eltSize = getContext().getTypeSizeInChars(VlaSize.Type);
1904 if (!eltSize.isOne())
1905 SizeVal = Builder.CreateNUWMul(SizeVal, CGM.getSize(eltSize));
1906 vla = vlaType;
1907 } else {
1908 return;
1909 }
1910 } else {
1911 SizeVal = CGM.getSize(size);
1912 vla = nullptr;
1913 }
1914
1915 // If the type contains a pointer to data member we can't memset it to zero.
1916 // Instead, create a null constant and copy it to the destination.
1917 // TODO: there are other patterns besides zero that we can usefully memset,
1918 // like -1, which happens to be the pattern used by member-pointers.
1919 if (!CGM.getTypes().isZeroInitializable(Ty)) {
1920 // For a VLA, emit a single element, then splat that over the VLA.
1921 if (vla) Ty = getContext().getBaseElementType(vla);
1922
1923 llvm::Constant *NullConstant = CGM.EmitNullConstant(Ty);
1924
1925 llvm::GlobalVariable *NullVariable =
1926 new llvm::GlobalVariable(CGM.getModule(), NullConstant->getType(),
1927 /*isConstant=*/true,
1928 llvm::GlobalVariable::PrivateLinkage,
1929 NullConstant, Twine());
1930 CharUnits NullAlign = DestPtr.getAlignment();
1931 NullVariable->setAlignment(NullAlign.getAsAlign());
1932 Address SrcPtr(Builder.CreateBitCast(NullVariable, Builder.getInt8PtrTy()),
1933 NullAlign);
1934
1935 if (vla) return emitNonZeroVLAInit(*this, Ty, DestPtr, SrcPtr, SizeVal);
1936
1937 // Get and call the appropriate llvm.memcpy overload.
1938 Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, false);
1939 return;
1940 }
1941
1942 // Otherwise, just memset the whole thing to zero. This is legal
1943 // because in LLVM, all default initializers (other than the ones we just
1944 // handled above) are guaranteed to have a bit pattern of all zeros.
1945 Builder.CreateMemSet(DestPtr, Builder.getInt8(0), SizeVal, false);
1946 }
1947
GetAddrOfLabel(const LabelDecl * L)1948 llvm::BlockAddress *CodeGenFunction::GetAddrOfLabel(const LabelDecl *L) {
1949 // Make sure that there is a block for the indirect goto.
1950 if (!IndirectBranch)
1951 GetIndirectGotoBlock();
1952
1953 llvm::BasicBlock *BB = getJumpDestForLabel(L).getBlock();
1954
1955 // Make sure the indirect branch includes all of the address-taken blocks.
1956 IndirectBranch->addDestination(BB);
1957 return llvm::BlockAddress::get(CurFn, BB);
1958 }
1959
GetIndirectGotoBlock()1960 llvm::BasicBlock *CodeGenFunction::GetIndirectGotoBlock() {
1961 // If we already made the indirect branch for indirect goto, return its block.
1962 if (IndirectBranch) return IndirectBranch->getParent();
1963
1964 CGBuilderTy TmpBuilder(*this, createBasicBlock("indirectgoto"));
1965
1966 // Create the PHI node that indirect gotos will add entries to.
1967 llvm::Value *DestVal = TmpBuilder.CreatePHI(Int8PtrTy, 0,
1968 "indirect.goto.dest");
1969
1970 // Create the indirect branch instruction.
1971 IndirectBranch = TmpBuilder.CreateIndirectBr(DestVal);
1972 return IndirectBranch->getParent();
1973 }
1974
1975 /// Computes the length of an array in elements, as well as the base
1976 /// element type and a properly-typed first element pointer.
emitArrayLength(const ArrayType * origArrayType,QualType & baseType,Address & addr)1977 llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
1978 QualType &baseType,
1979 Address &addr) {
1980 const ArrayType *arrayType = origArrayType;
1981
1982 // If it's a VLA, we have to load the stored size. Note that
1983 // this is the size of the VLA in bytes, not its size in elements.
1984 llvm::Value *numVLAElements = nullptr;
1985 if (isa<VariableArrayType>(arrayType)) {
1986 numVLAElements = getVLASize(cast<VariableArrayType>(arrayType)).NumElts;
1987
1988 // Walk into all VLAs. This doesn't require changes to addr,
1989 // which has type T* where T is the first non-VLA element type.
1990 do {
1991 QualType elementType = arrayType->getElementType();
1992 arrayType = getContext().getAsArrayType(elementType);
1993
1994 // If we only have VLA components, 'addr' requires no adjustment.
1995 if (!arrayType) {
1996 baseType = elementType;
1997 return numVLAElements;
1998 }
1999 } while (isa<VariableArrayType>(arrayType));
2000
2001 // We get out here only if we find a constant array type
2002 // inside the VLA.
2003 }
2004
2005 // We have some number of constant-length arrays, so addr should
2006 // have LLVM type [M x [N x [...]]]*. Build a GEP that walks
2007 // down to the first element of addr.
2008 SmallVector<llvm::Value*, 8> gepIndices;
2009
2010 // GEP down to the array type.
2011 llvm::ConstantInt *zero = Builder.getInt32(0);
2012 gepIndices.push_back(zero);
2013
2014 uint64_t countFromCLAs = 1;
2015 QualType eltType;
2016
2017 llvm::ArrayType *llvmArrayType =
2018 dyn_cast<llvm::ArrayType>(addr.getElementType());
2019 while (llvmArrayType) {
2020 assert(isa<ConstantArrayType>(arrayType));
2021 assert(cast<ConstantArrayType>(arrayType)->getSize().getZExtValue()
2022 == llvmArrayType->getNumElements());
2023
2024 gepIndices.push_back(zero);
2025 countFromCLAs *= llvmArrayType->getNumElements();
2026 eltType = arrayType->getElementType();
2027
2028 llvmArrayType =
2029 dyn_cast<llvm::ArrayType>(llvmArrayType->getElementType());
2030 arrayType = getContext().getAsArrayType(arrayType->getElementType());
2031 assert((!llvmArrayType || arrayType) &&
2032 "LLVM and Clang types are out-of-synch");
2033 }
2034
2035 if (arrayType) {
2036 // From this point onwards, the Clang array type has been emitted
2037 // as some other type (probably a packed struct). Compute the array
2038 // size, and just emit the 'begin' expression as a bitcast.
2039 while (arrayType) {
2040 countFromCLAs *=
2041 cast<ConstantArrayType>(arrayType)->getSize().getZExtValue();
2042 eltType = arrayType->getElementType();
2043 arrayType = getContext().getAsArrayType(eltType);
2044 }
2045
2046 llvm::Type *baseType = ConvertType(eltType);
2047 addr = Builder.CreateElementBitCast(addr, baseType, "array.begin");
2048 } else {
2049 // Create the actual GEP.
2050 addr = Address(Builder.CreateInBoundsGEP(
2051 addr.getElementType(), addr.getPointer(), gepIndices, "array.begin"),
2052 addr.getAlignment());
2053 }
2054
2055 baseType = eltType;
2056
2057 llvm::Value *numElements
2058 = llvm::ConstantInt::get(SizeTy, countFromCLAs);
2059
2060 // If we had any VLA dimensions, factor them in.
2061 if (numVLAElements)
2062 numElements = Builder.CreateNUWMul(numVLAElements, numElements);
2063
2064 return numElements;
2065 }
2066
getVLASize(QualType type)2067 CodeGenFunction::VlaSizePair CodeGenFunction::getVLASize(QualType type) {
2068 const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2069 assert(vla && "type was not a variable array type!");
2070 return getVLASize(vla);
2071 }
2072
2073 CodeGenFunction::VlaSizePair
getVLASize(const VariableArrayType * type)2074 CodeGenFunction::getVLASize(const VariableArrayType *type) {
2075 // The number of elements so far; always size_t.
2076 llvm::Value *numElements = nullptr;
2077
2078 QualType elementType;
2079 do {
2080 elementType = type->getElementType();
2081 llvm::Value *vlaSize = VLASizeMap[type->getSizeExpr()];
2082 assert(vlaSize && "no size for VLA!");
2083 assert(vlaSize->getType() == SizeTy);
2084
2085 if (!numElements) {
2086 numElements = vlaSize;
2087 } else {
2088 // It's undefined behavior if this wraps around, so mark it that way.
2089 // FIXME: Teach -fsanitize=undefined to trap this.
2090 numElements = Builder.CreateNUWMul(numElements, vlaSize);
2091 }
2092 } while ((type = getContext().getAsVariableArrayType(elementType)));
2093
2094 return { numElements, elementType };
2095 }
2096
2097 CodeGenFunction::VlaSizePair
getVLAElements1D(QualType type)2098 CodeGenFunction::getVLAElements1D(QualType type) {
2099 const VariableArrayType *vla = getContext().getAsVariableArrayType(type);
2100 assert(vla && "type was not a variable array type!");
2101 return getVLAElements1D(vla);
2102 }
2103
2104 CodeGenFunction::VlaSizePair
getVLAElements1D(const VariableArrayType * Vla)2105 CodeGenFunction::getVLAElements1D(const VariableArrayType *Vla) {
2106 llvm::Value *VlaSize = VLASizeMap[Vla->getSizeExpr()];
2107 assert(VlaSize && "no size for VLA!");
2108 assert(VlaSize->getType() == SizeTy);
2109 return { VlaSize, Vla->getElementType() };
2110 }
2111
EmitVariablyModifiedType(QualType type)2112 void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
2113 assert(type->isVariablyModifiedType() &&
2114 "Must pass variably modified type to EmitVLASizes!");
2115
2116 EnsureInsertPoint();
2117
2118 // We're going to walk down into the type and look for VLA
2119 // expressions.
2120 do {
2121 assert(type->isVariablyModifiedType());
2122
2123 const Type *ty = type.getTypePtr();
2124 switch (ty->getTypeClass()) {
2125
2126 #define TYPE(Class, Base)
2127 #define ABSTRACT_TYPE(Class, Base)
2128 #define NON_CANONICAL_TYPE(Class, Base)
2129 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
2130 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
2131 #include "clang/AST/TypeNodes.inc"
2132 llvm_unreachable("unexpected dependent type!");
2133
2134 // These types are never variably-modified.
2135 case Type::Builtin:
2136 case Type::Complex:
2137 case Type::Vector:
2138 case Type::ExtVector:
2139 case Type::ConstantMatrix:
2140 case Type::Record:
2141 case Type::Enum:
2142 case Type::Elaborated:
2143 case Type::TemplateSpecialization:
2144 case Type::ObjCTypeParam:
2145 case Type::ObjCObject:
2146 case Type::ObjCInterface:
2147 case Type::ObjCObjectPointer:
2148 case Type::ExtInt:
2149 llvm_unreachable("type class is never variably-modified!");
2150
2151 case Type::Adjusted:
2152 type = cast<AdjustedType>(ty)->getAdjustedType();
2153 break;
2154
2155 case Type::Decayed:
2156 type = cast<DecayedType>(ty)->getPointeeType();
2157 break;
2158
2159 case Type::Pointer:
2160 type = cast<PointerType>(ty)->getPointeeType();
2161 break;
2162
2163 case Type::BlockPointer:
2164 type = cast<BlockPointerType>(ty)->getPointeeType();
2165 break;
2166
2167 case Type::LValueReference:
2168 case Type::RValueReference:
2169 type = cast<ReferenceType>(ty)->getPointeeType();
2170 break;
2171
2172 case Type::MemberPointer:
2173 type = cast<MemberPointerType>(ty)->getPointeeType();
2174 break;
2175
2176 case Type::ConstantArray:
2177 case Type::IncompleteArray:
2178 // Losing element qualification here is fine.
2179 type = cast<ArrayType>(ty)->getElementType();
2180 break;
2181
2182 case Type::VariableArray: {
2183 // Losing element qualification here is fine.
2184 const VariableArrayType *vat = cast<VariableArrayType>(ty);
2185
2186 // Unknown size indication requires no size computation.
2187 // Otherwise, evaluate and record it.
2188 if (const Expr *size = vat->getSizeExpr()) {
2189 // It's possible that we might have emitted this already,
2190 // e.g. with a typedef and a pointer to it.
2191 llvm::Value *&entry = VLASizeMap[size];
2192 if (!entry) {
2193 llvm::Value *Size = EmitScalarExpr(size);
2194
2195 // C11 6.7.6.2p5:
2196 // If the size is an expression that is not an integer constant
2197 // expression [...] each time it is evaluated it shall have a value
2198 // greater than zero.
2199 if (SanOpts.has(SanitizerKind::VLABound) &&
2200 size->getType()->isSignedIntegerType()) {
2201 SanitizerScope SanScope(this);
2202 llvm::Value *Zero = llvm::Constant::getNullValue(Size->getType());
2203 llvm::Constant *StaticArgs[] = {
2204 EmitCheckSourceLocation(size->getBeginLoc()),
2205 EmitCheckTypeDescriptor(size->getType())};
2206 EmitCheck(std::make_pair(Builder.CreateICmpSGT(Size, Zero),
2207 SanitizerKind::VLABound),
2208 SanitizerHandler::VLABoundNotPositive, StaticArgs, Size);
2209 }
2210
2211 // Always zexting here would be wrong if it weren't
2212 // undefined behavior to have a negative bound.
2213 entry = Builder.CreateIntCast(Size, SizeTy, /*signed*/ false);
2214 }
2215 }
2216 type = vat->getElementType();
2217 break;
2218 }
2219
2220 case Type::FunctionProto:
2221 case Type::FunctionNoProto:
2222 type = cast<FunctionType>(ty)->getReturnType();
2223 break;
2224
2225 case Type::Paren:
2226 case Type::TypeOf:
2227 case Type::UnaryTransform:
2228 case Type::Attributed:
2229 case Type::SubstTemplateTypeParm:
2230 case Type::MacroQualified:
2231 // Keep walking after single level desugaring.
2232 type = type.getSingleStepDesugaredType(getContext());
2233 break;
2234
2235 case Type::Typedef:
2236 case Type::Decltype:
2237 case Type::Auto:
2238 case Type::DeducedTemplateSpecialization:
2239 // Stop walking: nothing to do.
2240 return;
2241
2242 case Type::TypeOfExpr:
2243 // Stop walking: emit typeof expression.
2244 EmitIgnoredExpr(cast<TypeOfExprType>(ty)->getUnderlyingExpr());
2245 return;
2246
2247 case Type::Atomic:
2248 type = cast<AtomicType>(ty)->getValueType();
2249 break;
2250
2251 case Type::Pipe:
2252 type = cast<PipeType>(ty)->getElementType();
2253 break;
2254 }
2255 } while (type->isVariablyModifiedType());
2256 }
2257
EmitVAListRef(const Expr * E)2258 Address CodeGenFunction::EmitVAListRef(const Expr* E) {
2259 if (getContext().getBuiltinVaListType()->isArrayType())
2260 return EmitPointerWithAlignment(E);
2261 return EmitLValue(E).getAddress(*this);
2262 }
2263
EmitMSVAListRef(const Expr * E)2264 Address CodeGenFunction::EmitMSVAListRef(const Expr *E) {
2265 return EmitLValue(E).getAddress(*this);
2266 }
2267
EmitDeclRefExprDbgValue(const DeclRefExpr * E,const APValue & Init)2268 void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
2269 const APValue &Init) {
2270 assert(Init.hasValue() && "Invalid DeclRefExpr initializer!");
2271 if (CGDebugInfo *Dbg = getDebugInfo())
2272 if (CGM.getCodeGenOpts().hasReducedDebugInfo())
2273 Dbg->EmitGlobalVariable(E->getDecl(), Init);
2274 }
2275
2276 CodeGenFunction::PeepholeProtection
protectFromPeepholes(RValue rvalue)2277 CodeGenFunction::protectFromPeepholes(RValue rvalue) {
2278 // At the moment, the only aggressive peephole we do in IR gen
2279 // is trunc(zext) folding, but if we add more, we can easily
2280 // extend this protection.
2281
2282 if (!rvalue.isScalar()) return PeepholeProtection();
2283 llvm::Value *value = rvalue.getScalarVal();
2284 if (!isa<llvm::ZExtInst>(value)) return PeepholeProtection();
2285
2286 // Just make an extra bitcast.
2287 assert(HaveInsertPoint());
2288 llvm::Instruction *inst = new llvm::BitCastInst(value, value->getType(), "",
2289 Builder.GetInsertBlock());
2290
2291 PeepholeProtection protection;
2292 protection.Inst = inst;
2293 return protection;
2294 }
2295
unprotectFromPeepholes(PeepholeProtection protection)2296 void CodeGenFunction::unprotectFromPeepholes(PeepholeProtection protection) {
2297 if (!protection.Inst) return;
2298
2299 // In theory, we could try to duplicate the peepholes now, but whatever.
2300 protection.Inst->eraseFromParent();
2301 }
2302
emitAlignmentAssumption(llvm::Value * PtrValue,QualType Ty,SourceLocation Loc,SourceLocation AssumptionLoc,llvm::Value * Alignment,llvm::Value * OffsetValue)2303 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2304 QualType Ty, SourceLocation Loc,
2305 SourceLocation AssumptionLoc,
2306 llvm::Value *Alignment,
2307 llvm::Value *OffsetValue) {
2308 if (Alignment->getType() != IntPtrTy)
2309 Alignment =
2310 Builder.CreateIntCast(Alignment, IntPtrTy, false, "casted.align");
2311 if (OffsetValue && OffsetValue->getType() != IntPtrTy)
2312 OffsetValue =
2313 Builder.CreateIntCast(OffsetValue, IntPtrTy, true, "casted.offset");
2314 llvm::Value *TheCheck = nullptr;
2315 if (SanOpts.has(SanitizerKind::Alignment)) {
2316 llvm::Value *PtrIntValue =
2317 Builder.CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");
2318
2319 if (OffsetValue) {
2320 bool IsOffsetZero = false;
2321 if (const auto *CI = dyn_cast<llvm::ConstantInt>(OffsetValue))
2322 IsOffsetZero = CI->isZero();
2323
2324 if (!IsOffsetZero)
2325 PtrIntValue = Builder.CreateSub(PtrIntValue, OffsetValue, "offsetptr");
2326 }
2327
2328 llvm::Value *Zero = llvm::ConstantInt::get(IntPtrTy, 0);
2329 llvm::Value *Mask =
2330 Builder.CreateSub(Alignment, llvm::ConstantInt::get(IntPtrTy, 1));
2331 llvm::Value *MaskedPtr = Builder.CreateAnd(PtrIntValue, Mask, "maskedptr");
2332 TheCheck = Builder.CreateICmpEQ(MaskedPtr, Zero, "maskcond");
2333 }
2334 llvm::Instruction *Assumption = Builder.CreateAlignmentAssumption(
2335 CGM.getDataLayout(), PtrValue, Alignment, OffsetValue);
2336
2337 if (!SanOpts.has(SanitizerKind::Alignment))
2338 return;
2339 emitAlignmentAssumptionCheck(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2340 OffsetValue, TheCheck, Assumption);
2341 }
2342
emitAlignmentAssumption(llvm::Value * PtrValue,const Expr * E,SourceLocation AssumptionLoc,llvm::Value * Alignment,llvm::Value * OffsetValue)2343 void CodeGenFunction::emitAlignmentAssumption(llvm::Value *PtrValue,
2344 const Expr *E,
2345 SourceLocation AssumptionLoc,
2346 llvm::Value *Alignment,
2347 llvm::Value *OffsetValue) {
2348 if (auto *CE = dyn_cast<CastExpr>(E))
2349 E = CE->getSubExprAsWritten();
2350 QualType Ty = E->getType();
2351 SourceLocation Loc = E->getExprLoc();
2352
2353 emitAlignmentAssumption(PtrValue, Ty, Loc, AssumptionLoc, Alignment,
2354 OffsetValue);
2355 }
2356
EmitAnnotationCall(llvm::Function * AnnotationFn,llvm::Value * AnnotatedVal,StringRef AnnotationStr,SourceLocation Location,const AnnotateAttr * Attr)2357 llvm::Value *CodeGenFunction::EmitAnnotationCall(llvm::Function *AnnotationFn,
2358 llvm::Value *AnnotatedVal,
2359 StringRef AnnotationStr,
2360 SourceLocation Location,
2361 const AnnotateAttr *Attr) {
2362 SmallVector<llvm::Value *, 5> Args = {
2363 AnnotatedVal,
2364 Builder.CreateBitCast(CGM.EmitAnnotationString(AnnotationStr), Int8PtrTy),
2365 Builder.CreateBitCast(CGM.EmitAnnotationUnit(Location), Int8PtrTy),
2366 CGM.EmitAnnotationLineNo(Location),
2367 };
2368 if (Attr)
2369 Args.push_back(CGM.EmitAnnotationArgs(Attr));
2370 return Builder.CreateCall(AnnotationFn, Args);
2371 }
2372
EmitVarAnnotations(const VarDecl * D,llvm::Value * V)2373 void CodeGenFunction::EmitVarAnnotations(const VarDecl *D, llvm::Value *V) {
2374 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2375 // FIXME We create a new bitcast for every annotation because that's what
2376 // llvm-gcc was doing.
2377 for (const auto *I : D->specific_attrs<AnnotateAttr>())
2378 EmitAnnotationCall(CGM.getIntrinsic(llvm::Intrinsic::var_annotation),
2379 Builder.CreateBitCast(V, CGM.Int8PtrTy, V->getName()),
2380 I->getAnnotation(), D->getLocation(), I);
2381 }
2382
EmitFieldAnnotations(const FieldDecl * D,Address Addr)2383 Address CodeGenFunction::EmitFieldAnnotations(const FieldDecl *D,
2384 Address Addr) {
2385 assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
2386 llvm::Value *V = Addr.getPointer();
2387 llvm::Type *VTy = V->getType();
2388 llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::ptr_annotation,
2389 CGM.Int8PtrTy);
2390
2391 for (const auto *I : D->specific_attrs<AnnotateAttr>()) {
2392 // FIXME Always emit the cast inst so we can differentiate between
2393 // annotation on the first field of a struct and annotation on the struct
2394 // itself.
2395 if (VTy != CGM.Int8PtrTy)
2396 V = Builder.CreateBitCast(V, CGM.Int8PtrTy);
2397 V = EmitAnnotationCall(F, V, I->getAnnotation(), D->getLocation(), I);
2398 V = Builder.CreateBitCast(V, VTy);
2399 }
2400
2401 return Address(V, Addr.getAlignment());
2402 }
2403
~CGCapturedStmtInfo()2404 CodeGenFunction::CGCapturedStmtInfo::~CGCapturedStmtInfo() { }
2405
SanitizerScope(CodeGenFunction * CGF)2406 CodeGenFunction::SanitizerScope::SanitizerScope(CodeGenFunction *CGF)
2407 : CGF(CGF) {
2408 assert(!CGF->IsSanitizerScope);
2409 CGF->IsSanitizerScope = true;
2410 }
2411
~SanitizerScope()2412 CodeGenFunction::SanitizerScope::~SanitizerScope() {
2413 CGF->IsSanitizerScope = false;
2414 }
2415
InsertHelper(llvm::Instruction * I,const llvm::Twine & Name,llvm::BasicBlock * BB,llvm::BasicBlock::iterator InsertPt) const2416 void CodeGenFunction::InsertHelper(llvm::Instruction *I,
2417 const llvm::Twine &Name,
2418 llvm::BasicBlock *BB,
2419 llvm::BasicBlock::iterator InsertPt) const {
2420 LoopStack.InsertHelper(I);
2421 if (IsSanitizerScope)
2422 CGM.getSanitizerMetadata()->disableSanitizerForInstruction(I);
2423 }
2424
InsertHelper(llvm::Instruction * I,const llvm::Twine & Name,llvm::BasicBlock * BB,llvm::BasicBlock::iterator InsertPt) const2425 void CGBuilderInserter::InsertHelper(
2426 llvm::Instruction *I, const llvm::Twine &Name, llvm::BasicBlock *BB,
2427 llvm::BasicBlock::iterator InsertPt) const {
2428 llvm::IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt);
2429 if (CGF)
2430 CGF->InsertHelper(I, Name, BB, InsertPt);
2431 }
2432
2433 // Emits an error if we don't have a valid set of target features for the
2434 // called function.
checkTargetFeatures(const CallExpr * E,const FunctionDecl * TargetDecl)2435 void CodeGenFunction::checkTargetFeatures(const CallExpr *E,
2436 const FunctionDecl *TargetDecl) {
2437 return checkTargetFeatures(E->getBeginLoc(), TargetDecl);
2438 }
2439
2440 // Emits an error if we don't have a valid set of target features for the
2441 // called function.
checkTargetFeatures(SourceLocation Loc,const FunctionDecl * TargetDecl)2442 void CodeGenFunction::checkTargetFeatures(SourceLocation Loc,
2443 const FunctionDecl *TargetDecl) {
2444 // Early exit if this is an indirect call.
2445 if (!TargetDecl)
2446 return;
2447
2448 // Get the current enclosing function if it exists. If it doesn't
2449 // we can't check the target features anyhow.
2450 const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurCodeDecl);
2451 if (!FD)
2452 return;
2453
2454 // Grab the required features for the call. For a builtin this is listed in
2455 // the td file with the default cpu, for an always_inline function this is any
2456 // listed cpu and any listed features.
2457 unsigned BuiltinID = TargetDecl->getBuiltinID();
2458 std::string MissingFeature;
2459 llvm::StringMap<bool> CallerFeatureMap;
2460 CGM.getContext().getFunctionFeatureMap(CallerFeatureMap, FD);
2461 if (BuiltinID) {
2462 StringRef FeatureList(
2463 CGM.getContext().BuiltinInfo.getRequiredFeatures(BuiltinID));
2464 // Return if the builtin doesn't have any required features.
2465 if (FeatureList.empty())
2466 return;
2467 assert(FeatureList.find(' ') == StringRef::npos &&
2468 "Space in feature list");
2469 TargetFeatures TF(CallerFeatureMap);
2470 if (!TF.hasRequiredFeatures(FeatureList))
2471 CGM.getDiags().Report(Loc, diag::err_builtin_needs_feature)
2472 << TargetDecl->getDeclName() << FeatureList;
2473 } else if (!TargetDecl->isMultiVersion() &&
2474 TargetDecl->hasAttr<TargetAttr>()) {
2475 // Get the required features for the callee.
2476
2477 const TargetAttr *TD = TargetDecl->getAttr<TargetAttr>();
2478 ParsedTargetAttr ParsedAttr =
2479 CGM.getContext().filterFunctionTargetAttrs(TD);
2480
2481 SmallVector<StringRef, 1> ReqFeatures;
2482 llvm::StringMap<bool> CalleeFeatureMap;
2483 CGM.getContext().getFunctionFeatureMap(CalleeFeatureMap, TargetDecl);
2484
2485 for (const auto &F : ParsedAttr.Features) {
2486 if (F[0] == '+' && CalleeFeatureMap.lookup(F.substr(1)))
2487 ReqFeatures.push_back(StringRef(F).substr(1));
2488 }
2489
2490 for (const auto &F : CalleeFeatureMap) {
2491 // Only positive features are "required".
2492 if (F.getValue())
2493 ReqFeatures.push_back(F.getKey());
2494 }
2495 if (!llvm::all_of(ReqFeatures, [&](StringRef Feature) {
2496 if (!CallerFeatureMap.lookup(Feature)) {
2497 MissingFeature = Feature.str();
2498 return false;
2499 }
2500 return true;
2501 }))
2502 CGM.getDiags().Report(Loc, diag::err_function_needs_feature)
2503 << FD->getDeclName() << TargetDecl->getDeclName() << MissingFeature;
2504 }
2505 }
2506
EmitSanitizerStatReport(llvm::SanitizerStatKind SSK)2507 void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
2508 if (!CGM.getCodeGenOpts().SanitizeStats)
2509 return;
2510
2511 llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
2512 IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
2513 CGM.getSanStats().create(IRB, SSK);
2514 }
2515
2516 llvm::Value *
FormResolverCondition(const MultiVersionResolverOption & RO)2517 CodeGenFunction::FormResolverCondition(const MultiVersionResolverOption &RO) {
2518 llvm::Value *Condition = nullptr;
2519
2520 if (!RO.Conditions.Architecture.empty())
2521 Condition = EmitX86CpuIs(RO.Conditions.Architecture);
2522
2523 if (!RO.Conditions.Features.empty()) {
2524 llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
2525 Condition =
2526 Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
2527 }
2528 return Condition;
2529 }
2530
CreateMultiVersionResolverReturn(CodeGenModule & CGM,llvm::Function * Resolver,CGBuilderTy & Builder,llvm::Function * FuncToReturn,bool SupportsIFunc)2531 static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
2532 llvm::Function *Resolver,
2533 CGBuilderTy &Builder,
2534 llvm::Function *FuncToReturn,
2535 bool SupportsIFunc) {
2536 if (SupportsIFunc) {
2537 Builder.CreateRet(FuncToReturn);
2538 return;
2539 }
2540
2541 llvm::SmallVector<llvm::Value *, 10> Args;
2542 llvm::for_each(Resolver->args(),
2543 [&](llvm::Argument &Arg) { Args.push_back(&Arg); });
2544
2545 llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
2546 Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
2547
2548 if (Resolver->getReturnType()->isVoidTy())
2549 Builder.CreateRetVoid();
2550 else
2551 Builder.CreateRet(Result);
2552 }
2553
EmitMultiVersionResolver(llvm::Function * Resolver,ArrayRef<MultiVersionResolverOption> Options)2554 void CodeGenFunction::EmitMultiVersionResolver(
2555 llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
2556 assert(getContext().getTargetInfo().getTriple().isX86() &&
2557 "Only implemented for x86 targets");
2558
2559 bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
2560
2561 // Main function's basic block.
2562 llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
2563 Builder.SetInsertPoint(CurBlock);
2564 EmitX86CpuInit();
2565
2566 for (const MultiVersionResolverOption &RO : Options) {
2567 Builder.SetInsertPoint(CurBlock);
2568 llvm::Value *Condition = FormResolverCondition(RO);
2569
2570 // The 'default' or 'generic' case.
2571 if (!Condition) {
2572 assert(&RO == Options.end() - 1 &&
2573 "Default or Generic case must be last");
2574 CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
2575 SupportsIFunc);
2576 return;
2577 }
2578
2579 llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
2580 CGBuilderTy RetBuilder(*this, RetBlock);
2581 CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
2582 SupportsIFunc);
2583 CurBlock = createBasicBlock("resolver_else", Resolver);
2584 Builder.CreateCondBr(Condition, RetBlock, CurBlock);
2585 }
2586
2587 // If no generic/default, emit an unreachable.
2588 Builder.SetInsertPoint(CurBlock);
2589 llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
2590 TrapCall->setDoesNotReturn();
2591 TrapCall->setDoesNotThrow();
2592 Builder.CreateUnreachable();
2593 Builder.ClearInsertionPoint();
2594 }
2595
2596 // Loc - where the diagnostic will point, where in the source code this
2597 // alignment has failed.
2598 // SecondaryLoc - if present (will be present if sufficiently different from
2599 // Loc), the diagnostic will additionally point a "Note:" to this location.
2600 // It should be the location where the __attribute__((assume_aligned))
2601 // was written e.g.
emitAlignmentAssumptionCheck(llvm::Value * Ptr,QualType Ty,SourceLocation Loc,SourceLocation SecondaryLoc,llvm::Value * Alignment,llvm::Value * OffsetValue,llvm::Value * TheCheck,llvm::Instruction * Assumption)2602 void CodeGenFunction::emitAlignmentAssumptionCheck(
2603 llvm::Value *Ptr, QualType Ty, SourceLocation Loc,
2604 SourceLocation SecondaryLoc, llvm::Value *Alignment,
2605 llvm::Value *OffsetValue, llvm::Value *TheCheck,
2606 llvm::Instruction *Assumption) {
2607 assert(Assumption && isa<llvm::CallInst>(Assumption) &&
2608 cast<llvm::CallInst>(Assumption)->getCalledOperand() ==
2609 llvm::Intrinsic::getDeclaration(
2610 Builder.GetInsertBlock()->getParent()->getParent(),
2611 llvm::Intrinsic::assume) &&
2612 "Assumption should be a call to llvm.assume().");
2613 assert(&(Builder.GetInsertBlock()->back()) == Assumption &&
2614 "Assumption should be the last instruction of the basic block, "
2615 "since the basic block is still being generated.");
2616
2617 if (!SanOpts.has(SanitizerKind::Alignment))
2618 return;
2619
2620 // Don't check pointers to volatile data. The behavior here is implementation-
2621 // defined.
2622 if (Ty->getPointeeType().isVolatileQualified())
2623 return;
2624
2625 // We need to temorairly remove the assumption so we can insert the
2626 // sanitizer check before it, else the check will be dropped by optimizations.
2627 Assumption->removeFromParent();
2628
2629 {
2630 SanitizerScope SanScope(this);
2631
2632 if (!OffsetValue)
2633 OffsetValue = Builder.getInt1(0); // no offset.
2634
2635 llvm::Constant *StaticData[] = {EmitCheckSourceLocation(Loc),
2636 EmitCheckSourceLocation(SecondaryLoc),
2637 EmitCheckTypeDescriptor(Ty)};
2638 llvm::Value *DynamicData[] = {EmitCheckValue(Ptr),
2639 EmitCheckValue(Alignment),
2640 EmitCheckValue(OffsetValue)};
2641 EmitCheck({std::make_pair(TheCheck, SanitizerKind::Alignment)},
2642 SanitizerHandler::AlignmentAssumption, StaticData, DynamicData);
2643 }
2644
2645 // We are now in the (new, empty) "cont" basic block.
2646 // Reintroduce the assumption.
2647 Builder.Insert(Assumption);
2648 // FIXME: Assumption still has it's original basic block as it's Parent.
2649 }
2650
SourceLocToDebugLoc(SourceLocation Location)2651 llvm::DebugLoc CodeGenFunction::SourceLocToDebugLoc(SourceLocation Location) {
2652 if (CGDebugInfo *DI = getDebugInfo())
2653 return DI->SourceLocToDebugLoc(Location);
2654
2655 return llvm::DebugLoc();
2656 }
2657
2658 llvm::Value *
emitCondLikelihoodViaExpectIntrinsic(llvm::Value * Cond,Stmt::Likelihood LH)2659 CodeGenFunction::emitCondLikelihoodViaExpectIntrinsic(llvm::Value *Cond,
2660 Stmt::Likelihood LH) {
2661 switch (LH) {
2662 case Stmt::LH_None:
2663 return Cond;
2664 case Stmt::LH_Likely:
2665 case Stmt::LH_Unlikely:
2666 // Don't generate llvm.expect on -O0 as the backend won't use it for
2667 // anything.
2668 if (CGM.getCodeGenOpts().OptimizationLevel == 0)
2669 return Cond;
2670 llvm::Type *CondTy = Cond->getType();
2671 assert(CondTy->isIntegerTy(1) && "expecting condition to be a boolean");
2672 llvm::Function *FnExpect =
2673 CGM.getIntrinsic(llvm::Intrinsic::expect, CondTy);
2674 llvm::Value *ExpectedValueOfCond =
2675 llvm::ConstantInt::getBool(CondTy, LH == Stmt::LH_Likely);
2676 return Builder.CreateCall(FnExpect, {Cond, ExpectedValueOfCond},
2677 Cond->getName() + ".expval");
2678 }
2679 llvm_unreachable("Unknown Likelihood");
2680 }
2681