xref: /openbsd/gnu/llvm/clang/lib/CodeGen/CGExpr.cpp (revision 12c85518)
1 //===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Expr nodes as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGCUDARuntime.h"
14 #include "CGCXXABI.h"
15 #include "CGCall.h"
16 #include "CGCleanup.h"
17 #include "CGDebugInfo.h"
18 #include "CGObjCRuntime.h"
19 #include "CGOpenMPRuntime.h"
20 #include "CGRecordLayout.h"
21 #include "CodeGenFunction.h"
22 #include "CodeGenModule.h"
23 #include "ConstantEmitter.h"
24 #include "TargetInfo.h"
25 #include "clang/AST/ASTContext.h"
26 #include "clang/AST/Attr.h"
27 #include "clang/AST/DeclObjC.h"
28 #include "clang/AST/NSAPI.h"
29 #include "clang/Basic/Builtins.h"
30 #include "clang/Basic/CodeGenOptions.h"
31 #include "clang/Basic/SourceManager.h"
32 #include "llvm/ADT/Hashing.h"
33 #include "llvm/ADT/StringExtras.h"
34 #include "llvm/IR/DataLayout.h"
35 #include "llvm/IR/Intrinsics.h"
36 #include "llvm/IR/LLVMContext.h"
37 #include "llvm/IR/MDBuilder.h"
38 #include "llvm/IR/MatrixBuilder.h"
39 #include "llvm/Support/ConvertUTF.h"
40 #include "llvm/Support/MathExtras.h"
41 #include "llvm/Support/Path.h"
42 #include "llvm/Support/SaveAndRestore.h"
43 #include "llvm/Transforms/Utils/SanitizerStats.h"
44 
45 #include <optional>
46 #include <string>
47 
48 using namespace clang;
49 using namespace CodeGen;
50 
51 //===--------------------------------------------------------------------===//
52 //                        Miscellaneous Helper Methods
53 //===--------------------------------------------------------------------===//
54 
EmitCastToVoidPtr(llvm::Value * value)55 llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
56   unsigned addressSpace =
57       cast<llvm::PointerType>(value->getType())->getAddressSpace();
58 
59   llvm::PointerType *destType = Int8PtrTy;
60   if (addressSpace)
61     destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
62 
63   if (value->getType() == destType) return value;
64   return Builder.CreateBitCast(value, destType);
65 }
66 
67 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
68 /// block.
CreateTempAllocaWithoutCast(llvm::Type * Ty,CharUnits Align,const Twine & Name,llvm::Value * ArraySize)69 Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
70                                                      CharUnits Align,
71                                                      const Twine &Name,
72                                                      llvm::Value *ArraySize) {
73   auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
74   Alloca->setAlignment(Align.getAsAlign());
75   return Address(Alloca, Ty, Align);
76 }
77 
78 /// CreateTempAlloca - This creates a alloca and inserts it into the entry
79 /// block. The alloca is casted to default address space if necessary.
CreateTempAlloca(llvm::Type * Ty,CharUnits Align,const Twine & Name,llvm::Value * ArraySize,Address * AllocaAddr)80 Address CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
81                                           const Twine &Name,
82                                           llvm::Value *ArraySize,
83                                           Address *AllocaAddr) {
84   auto Alloca = CreateTempAllocaWithoutCast(Ty, Align, Name, ArraySize);
85   if (AllocaAddr)
86     *AllocaAddr = Alloca;
87   llvm::Value *V = Alloca.getPointer();
88   // Alloca always returns a pointer in alloca address space, which may
89   // be different from the type defined by the language. For example,
90   // in C++ the auto variables are in the default address space. Therefore
91   // cast alloca to the default address space when necessary.
92   if (getASTAllocaAddressSpace() != LangAS::Default) {
93     auto DestAddrSpace = getContext().getTargetAddressSpace(LangAS::Default);
94     llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
95     // When ArraySize is nullptr, alloca is inserted at AllocaInsertPt,
96     // otherwise alloca is inserted at the current insertion point of the
97     // builder.
98     if (!ArraySize)
99       Builder.SetInsertPoint(getPostAllocaInsertPoint());
100     V = getTargetHooks().performAddrSpaceCast(
101         *this, V, getASTAllocaAddressSpace(), LangAS::Default,
102         Ty->getPointerTo(DestAddrSpace), /*non-null*/ true);
103   }
104 
105   return Address(V, Ty, Align);
106 }
107 
108 /// CreateTempAlloca - This creates an alloca and inserts it into the entry
109 /// block if \p ArraySize is nullptr, otherwise inserts it at the current
110 /// insertion point of the builder.
CreateTempAlloca(llvm::Type * Ty,const Twine & Name,llvm::Value * ArraySize)111 llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
112                                                     const Twine &Name,
113                                                     llvm::Value *ArraySize) {
114   if (ArraySize)
115     return Builder.CreateAlloca(Ty, ArraySize, Name);
116   return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
117                               ArraySize, Name, AllocaInsertPt);
118 }
119 
120 /// CreateDefaultAlignTempAlloca - This creates an alloca with the
121 /// default alignment of the corresponding LLVM type, which is *not*
122 /// guaranteed to be related in any way to the expected alignment of
123 /// an AST type that might have been lowered to Ty.
CreateDefaultAlignTempAlloca(llvm::Type * Ty,const Twine & Name)124 Address CodeGenFunction::CreateDefaultAlignTempAlloca(llvm::Type *Ty,
125                                                       const Twine &Name) {
126   CharUnits Align =
127       CharUnits::fromQuantity(CGM.getDataLayout().getPrefTypeAlign(Ty));
128   return CreateTempAlloca(Ty, Align, Name);
129 }
130 
CreateIRTemp(QualType Ty,const Twine & Name)131 Address CodeGenFunction::CreateIRTemp(QualType Ty, const Twine &Name) {
132   CharUnits Align = getContext().getTypeAlignInChars(Ty);
133   return CreateTempAlloca(ConvertType(Ty), Align, Name);
134 }
135 
CreateMemTemp(QualType Ty,const Twine & Name,Address * Alloca)136 Address CodeGenFunction::CreateMemTemp(QualType Ty, const Twine &Name,
137                                        Address *Alloca) {
138   // FIXME: Should we prefer the preferred type alignment here?
139   return CreateMemTemp(Ty, getContext().getTypeAlignInChars(Ty), Name, Alloca);
140 }
141 
CreateMemTemp(QualType Ty,CharUnits Align,const Twine & Name,Address * Alloca)142 Address CodeGenFunction::CreateMemTemp(QualType Ty, CharUnits Align,
143                                        const Twine &Name, Address *Alloca) {
144   Address Result = CreateTempAlloca(ConvertTypeForMem(Ty), Align, Name,
145                                     /*ArraySize=*/nullptr, Alloca);
146 
147   if (Ty->isConstantMatrixType()) {
148     auto *ArrayTy = cast<llvm::ArrayType>(Result.getElementType());
149     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
150                                                 ArrayTy->getNumElements());
151 
152     Result = Address(
153         Builder.CreateBitCast(Result.getPointer(), VectorTy->getPointerTo()),
154         VectorTy, Result.getAlignment());
155   }
156   return Result;
157 }
158 
CreateMemTempWithoutCast(QualType Ty,CharUnits Align,const Twine & Name)159 Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty, CharUnits Align,
160                                                   const Twine &Name) {
161   return CreateTempAllocaWithoutCast(ConvertTypeForMem(Ty), Align, Name);
162 }
163 
CreateMemTempWithoutCast(QualType Ty,const Twine & Name)164 Address CodeGenFunction::CreateMemTempWithoutCast(QualType Ty,
165                                                   const Twine &Name) {
166   return CreateMemTempWithoutCast(Ty, getContext().getTypeAlignInChars(Ty),
167                                   Name);
168 }
169 
170 /// EvaluateExprAsBool - Perform the usual unary conversions on the specified
171 /// expression and compare the result against zero, returning an Int1Ty value.
EvaluateExprAsBool(const Expr * E)172 llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
173   PGO.setCurrentStmt(E);
174   if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
175     llvm::Value *MemPtr = EmitScalarExpr(E);
176     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
177   }
178 
179   QualType BoolTy = getContext().BoolTy;
180   SourceLocation Loc = E->getExprLoc();
181   CGFPOptionsRAII FPOptsRAII(*this, E);
182   if (!E->getType()->isAnyComplexType())
183     return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy, Loc);
184 
185   return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(), BoolTy,
186                                        Loc);
187 }
188 
189 /// EmitIgnoredExpr - Emit code to compute the specified expression,
190 /// ignoring the result.
EmitIgnoredExpr(const Expr * E)191 void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
192   if (E->isPRValue())
193     return (void)EmitAnyExpr(E, AggValueSlot::ignored(), true);
194 
195   // if this is a bitfield-resulting conditional operator, we can special case
196   // emit this. The normal 'EmitLValue' version of this is particularly
197   // difficult to codegen for, since creating a single "LValue" for two
198   // different sized arguments here is not particularly doable.
199   if (const auto *CondOp = dyn_cast<AbstractConditionalOperator>(
200           E->IgnoreParenNoopCasts(getContext()))) {
201     if (CondOp->getObjectKind() == OK_BitField)
202       return EmitIgnoredConditionalOperator(CondOp);
203   }
204 
205   // Just emit it as an l-value and drop the result.
206   EmitLValue(E);
207 }
208 
209 /// EmitAnyExpr - Emit code to compute the specified expression which
210 /// can have any type.  The result is returned as an RValue struct.
211 /// If this is an aggregate expression, AggSlot indicates where the
212 /// result should be returned.
EmitAnyExpr(const Expr * E,AggValueSlot aggSlot,bool ignoreResult)213 RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
214                                     AggValueSlot aggSlot,
215                                     bool ignoreResult) {
216   switch (getEvaluationKind(E->getType())) {
217   case TEK_Scalar:
218     return RValue::get(EmitScalarExpr(E, ignoreResult));
219   case TEK_Complex:
220     return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));
221   case TEK_Aggregate:
222     if (!ignoreResult && aggSlot.isIgnored())
223       aggSlot = CreateAggTemp(E->getType(), "agg-temp");
224     EmitAggExpr(E, aggSlot);
225     return aggSlot.asRValue();
226   }
227   llvm_unreachable("bad evaluation kind");
228 }
229 
230 /// EmitAnyExprToTemp - Similar to EmitAnyExpr(), however, the result will
231 /// always be accessible even if no aggregate location is provided.
EmitAnyExprToTemp(const Expr * E)232 RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
233   AggValueSlot AggSlot = AggValueSlot::ignored();
234 
235   if (hasAggregateEvaluationKind(E->getType()))
236     AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
237   return EmitAnyExpr(E, AggSlot);
238 }
239 
240 /// EmitAnyExprToMem - Evaluate an expression into a given memory
241 /// location.
EmitAnyExprToMem(const Expr * E,Address Location,Qualifiers Quals,bool IsInit)242 void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
243                                        Address Location,
244                                        Qualifiers Quals,
245                                        bool IsInit) {
246   // FIXME: This function should take an LValue as an argument.
247   switch (getEvaluationKind(E->getType())) {
248   case TEK_Complex:
249     EmitComplexExprIntoLValue(E, MakeAddrLValue(Location, E->getType()),
250                               /*isInit*/ false);
251     return;
252 
253   case TEK_Aggregate: {
254     EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
255                                          AggValueSlot::IsDestructed_t(IsInit),
256                                          AggValueSlot::DoesNotNeedGCBarriers,
257                                          AggValueSlot::IsAliased_t(!IsInit),
258                                          AggValueSlot::MayOverlap));
259     return;
260   }
261 
262   case TEK_Scalar: {
263     RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
264     LValue LV = MakeAddrLValue(Location, E->getType());
265     EmitStoreThroughLValue(RV, LV);
266     return;
267   }
268   }
269   llvm_unreachable("bad evaluation kind");
270 }
271 
272 static void
pushTemporaryCleanup(CodeGenFunction & CGF,const MaterializeTemporaryExpr * M,const Expr * E,Address ReferenceTemporary)273 pushTemporaryCleanup(CodeGenFunction &CGF, const MaterializeTemporaryExpr *M,
274                      const Expr *E, Address ReferenceTemporary) {
275   // Objective-C++ ARC:
276   //   If we are binding a reference to a temporary that has ownership, we
277   //   need to perform retain/release operations on the temporary.
278   //
279   // FIXME: This should be looking at E, not M.
280   if (auto Lifetime = M->getType().getObjCLifetime()) {
281     switch (Lifetime) {
282     case Qualifiers::OCL_None:
283     case Qualifiers::OCL_ExplicitNone:
284       // Carry on to normal cleanup handling.
285       break;
286 
287     case Qualifiers::OCL_Autoreleasing:
288       // Nothing to do; cleaned up by an autorelease pool.
289       return;
290 
291     case Qualifiers::OCL_Strong:
292     case Qualifiers::OCL_Weak:
293       switch (StorageDuration Duration = M->getStorageDuration()) {
294       case SD_Static:
295         // Note: we intentionally do not register a cleanup to release
296         // the object on program termination.
297         return;
298 
299       case SD_Thread:
300         // FIXME: We should probably register a cleanup in this case.
301         return;
302 
303       case SD_Automatic:
304       case SD_FullExpression:
305         CodeGenFunction::Destroyer *Destroy;
306         CleanupKind CleanupKind;
307         if (Lifetime == Qualifiers::OCL_Strong) {
308           const ValueDecl *VD = M->getExtendingDecl();
309           bool Precise =
310               VD && isa<VarDecl>(VD) && VD->hasAttr<ObjCPreciseLifetimeAttr>();
311           CleanupKind = CGF.getARCCleanupKind();
312           Destroy = Precise ? &CodeGenFunction::destroyARCStrongPrecise
313                             : &CodeGenFunction::destroyARCStrongImprecise;
314         } else {
315           // __weak objects always get EH cleanups; otherwise, exceptions
316           // could cause really nasty crashes instead of mere leaks.
317           CleanupKind = NormalAndEHCleanup;
318           Destroy = &CodeGenFunction::destroyARCWeak;
319         }
320         if (Duration == SD_FullExpression)
321           CGF.pushDestroy(CleanupKind, ReferenceTemporary,
322                           M->getType(), *Destroy,
323                           CleanupKind & EHCleanup);
324         else
325           CGF.pushLifetimeExtendedDestroy(CleanupKind, ReferenceTemporary,
326                                           M->getType(),
327                                           *Destroy, CleanupKind & EHCleanup);
328         return;
329 
330       case SD_Dynamic:
331         llvm_unreachable("temporary cannot have dynamic storage duration");
332       }
333       llvm_unreachable("unknown storage duration");
334     }
335   }
336 
337   CXXDestructorDecl *ReferenceTemporaryDtor = nullptr;
338   if (const RecordType *RT =
339           E->getType()->getBaseElementTypeUnsafe()->getAs<RecordType>()) {
340     // Get the destructor for the reference temporary.
341     auto *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
342     if (!ClassDecl->hasTrivialDestructor())
343       ReferenceTemporaryDtor = ClassDecl->getDestructor();
344   }
345 
346   if (!ReferenceTemporaryDtor)
347     return;
348 
349   // Call the destructor for the temporary.
350   switch (M->getStorageDuration()) {
351   case SD_Static:
352   case SD_Thread: {
353     llvm::FunctionCallee CleanupFn;
354     llvm::Constant *CleanupArg;
355     if (E->getType()->isArrayType()) {
356       CleanupFn = CodeGenFunction(CGF.CGM).generateDestroyHelper(
357           ReferenceTemporary, E->getType(),
358           CodeGenFunction::destroyCXXObject, CGF.getLangOpts().Exceptions,
359           dyn_cast_or_null<VarDecl>(M->getExtendingDecl()));
360       CleanupArg = llvm::Constant::getNullValue(CGF.Int8PtrTy);
361     } else {
362       CleanupFn = CGF.CGM.getAddrAndTypeOfCXXStructor(
363           GlobalDecl(ReferenceTemporaryDtor, Dtor_Complete));
364       CleanupArg = cast<llvm::Constant>(ReferenceTemporary.getPointer());
365     }
366     CGF.CGM.getCXXABI().registerGlobalDtor(
367         CGF, *cast<VarDecl>(M->getExtendingDecl()), CleanupFn, CleanupArg);
368     break;
369   }
370 
371   case SD_FullExpression:
372     CGF.pushDestroy(NormalAndEHCleanup, ReferenceTemporary, E->getType(),
373                     CodeGenFunction::destroyCXXObject,
374                     CGF.getLangOpts().Exceptions);
375     break;
376 
377   case SD_Automatic:
378     CGF.pushLifetimeExtendedDestroy(NormalAndEHCleanup,
379                                     ReferenceTemporary, E->getType(),
380                                     CodeGenFunction::destroyCXXObject,
381                                     CGF.getLangOpts().Exceptions);
382     break;
383 
384   case SD_Dynamic:
385     llvm_unreachable("temporary cannot have dynamic storage duration");
386   }
387 }
388 
createReferenceTemporary(CodeGenFunction & CGF,const MaterializeTemporaryExpr * M,const Expr * Inner,Address * Alloca=nullptr)389 static Address createReferenceTemporary(CodeGenFunction &CGF,
390                                         const MaterializeTemporaryExpr *M,
391                                         const Expr *Inner,
392                                         Address *Alloca = nullptr) {
393   auto &TCG = CGF.getTargetHooks();
394   switch (M->getStorageDuration()) {
395   case SD_FullExpression:
396   case SD_Automatic: {
397     // If we have a constant temporary array or record try to promote it into a
398     // constant global under the same rules a normal constant would've been
399     // promoted. This is easier on the optimizer and generally emits fewer
400     // instructions.
401     QualType Ty = Inner->getType();
402     if (CGF.CGM.getCodeGenOpts().MergeAllConstants &&
403         (Ty->isArrayType() || Ty->isRecordType()) &&
404         CGF.CGM.isTypeConstant(Ty, true))
405       if (auto Init = ConstantEmitter(CGF).tryEmitAbstract(Inner, Ty)) {
406         auto AS = CGF.CGM.GetGlobalConstantAddressSpace();
407         auto *GV = new llvm::GlobalVariable(
408             CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true,
409             llvm::GlobalValue::PrivateLinkage, Init, ".ref.tmp", nullptr,
410             llvm::GlobalValue::NotThreadLocal,
411             CGF.getContext().getTargetAddressSpace(AS));
412         CharUnits alignment = CGF.getContext().getTypeAlignInChars(Ty);
413         GV->setAlignment(alignment.getAsAlign());
414         llvm::Constant *C = GV;
415         if (AS != LangAS::Default)
416           C = TCG.performAddrSpaceCast(
417               CGF.CGM, GV, AS, LangAS::Default,
418               GV->getValueType()->getPointerTo(
419                   CGF.getContext().getTargetAddressSpace(LangAS::Default)));
420         // FIXME: Should we put the new global into a COMDAT?
421         return Address(C, GV->getValueType(), alignment);
422       }
423     return CGF.CreateMemTemp(Ty, "ref.tmp", Alloca);
424   }
425   case SD_Thread:
426   case SD_Static:
427     return CGF.CGM.GetAddrOfGlobalTemporary(M, Inner);
428 
429   case SD_Dynamic:
430     llvm_unreachable("temporary can't have dynamic storage duration");
431   }
432   llvm_unreachable("unknown storage duration");
433 }
434 
435 /// Helper method to check if the underlying ABI is AAPCS
isAAPCS(const TargetInfo & TargetInfo)436 static bool isAAPCS(const TargetInfo &TargetInfo) {
437   return TargetInfo.getABI().startswith("aapcs");
438 }
439 
440 LValue CodeGenFunction::
EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr * M)441 EmitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *M) {
442   const Expr *E = M->getSubExpr();
443 
444   assert((!M->getExtendingDecl() || !isa<VarDecl>(M->getExtendingDecl()) ||
445           !cast<VarDecl>(M->getExtendingDecl())->isARCPseudoStrong()) &&
446          "Reference should never be pseudo-strong!");
447 
448   // FIXME: ideally this would use EmitAnyExprToMem, however, we cannot do so
449   // as that will cause the lifetime adjustment to be lost for ARC
450   auto ownership = M->getType().getObjCLifetime();
451   if (ownership != Qualifiers::OCL_None &&
452       ownership != Qualifiers::OCL_ExplicitNone) {
453     Address Object = createReferenceTemporary(*this, M, E);
454     if (auto *Var = dyn_cast<llvm::GlobalVariable>(Object.getPointer())) {
455       llvm::Type *Ty = ConvertTypeForMem(E->getType());
456       Object = Address(llvm::ConstantExpr::getBitCast(
457                            Var, Ty->getPointerTo(Object.getAddressSpace())),
458                        Ty, Object.getAlignment());
459 
460       // createReferenceTemporary will promote the temporary to a global with a
461       // constant initializer if it can.  It can only do this to a value of
462       // ARC-manageable type if the value is global and therefore "immune" to
463       // ref-counting operations.  Therefore we have no need to emit either a
464       // dynamic initialization or a cleanup and we can just return the address
465       // of the temporary.
466       if (Var->hasInitializer())
467         return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
468 
469       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
470     }
471     LValue RefTempDst = MakeAddrLValue(Object, M->getType(),
472                                        AlignmentSource::Decl);
473 
474     switch (getEvaluationKind(E->getType())) {
475     default: llvm_unreachable("expected scalar or aggregate expression");
476     case TEK_Scalar:
477       EmitScalarInit(E, M->getExtendingDecl(), RefTempDst, false);
478       break;
479     case TEK_Aggregate: {
480       EmitAggExpr(E, AggValueSlot::forAddr(Object,
481                                            E->getType().getQualifiers(),
482                                            AggValueSlot::IsDestructed,
483                                            AggValueSlot::DoesNotNeedGCBarriers,
484                                            AggValueSlot::IsNotAliased,
485                                            AggValueSlot::DoesNotOverlap));
486       break;
487     }
488     }
489 
490     pushTemporaryCleanup(*this, M, E, Object);
491     return RefTempDst;
492   }
493 
494   SmallVector<const Expr *, 2> CommaLHSs;
495   SmallVector<SubobjectAdjustment, 2> Adjustments;
496   E = E->skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);
497 
498   for (const auto &Ignored : CommaLHSs)
499     EmitIgnoredExpr(Ignored);
500 
501   if (const auto *opaque = dyn_cast<OpaqueValueExpr>(E)) {
502     if (opaque->getType()->isRecordType()) {
503       assert(Adjustments.empty());
504       return EmitOpaqueValueLValue(opaque);
505     }
506   }
507 
508   // Create and initialize the reference temporary.
509   Address Alloca = Address::invalid();
510   Address Object = createReferenceTemporary(*this, M, E, &Alloca);
511   if (auto *Var = dyn_cast<llvm::GlobalVariable>(
512           Object.getPointer()->stripPointerCasts())) {
513     llvm::Type *TemporaryType = ConvertTypeForMem(E->getType());
514     Object = Address(llvm::ConstantExpr::getBitCast(
515                          cast<llvm::Constant>(Object.getPointer()),
516                          TemporaryType->getPointerTo()),
517                      TemporaryType,
518                      Object.getAlignment());
519     // If the temporary is a global and has a constant initializer or is a
520     // constant temporary that we promoted to a global, we may have already
521     // initialized it.
522     if (!Var->hasInitializer()) {
523       Var->setInitializer(CGM.EmitNullConstant(E->getType()));
524       EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
525     }
526   } else {
527     switch (M->getStorageDuration()) {
528     case SD_Automatic:
529       if (auto *Size = EmitLifetimeStart(
530               CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
531               Alloca.getPointer())) {
532         pushCleanupAfterFullExpr<CallLifetimeEnd>(NormalEHLifetimeMarker,
533                                                   Alloca, Size);
534       }
535       break;
536 
537     case SD_FullExpression: {
538       if (!ShouldEmitLifetimeMarkers)
539         break;
540 
541       // Avoid creating a conditional cleanup just to hold an llvm.lifetime.end
542       // marker. Instead, start the lifetime of a conditional temporary earlier
543       // so that it's unconditional. Don't do this with sanitizers which need
544       // more precise lifetime marks.
545       ConditionalEvaluation *OldConditional = nullptr;
546       CGBuilderTy::InsertPoint OldIP;
547       if (isInConditionalBranch() && !E->getType().isDestructedType() &&
548           !SanOpts.has(SanitizerKind::HWAddress) &&
549           !SanOpts.has(SanitizerKind::Memory) &&
550           !CGM.getCodeGenOpts().SanitizeAddressUseAfterScope) {
551         OldConditional = OutermostConditional;
552         OutermostConditional = nullptr;
553 
554         OldIP = Builder.saveIP();
555         llvm::BasicBlock *Block = OldConditional->getStartingBlock();
556         Builder.restoreIP(CGBuilderTy::InsertPoint(
557             Block, llvm::BasicBlock::iterator(Block->back())));
558       }
559 
560       if (auto *Size = EmitLifetimeStart(
561               CGM.getDataLayout().getTypeAllocSize(Alloca.getElementType()),
562               Alloca.getPointer())) {
563         pushFullExprCleanup<CallLifetimeEnd>(NormalEHLifetimeMarker, Alloca,
564                                              Size);
565       }
566 
567       if (OldConditional) {
568         OutermostConditional = OldConditional;
569         Builder.restoreIP(OldIP);
570       }
571       break;
572     }
573 
574     default:
575       break;
576     }
577     EmitAnyExprToMem(E, Object, Qualifiers(), /*IsInit*/true);
578   }
579   pushTemporaryCleanup(*this, M, E, Object);
580 
581   // Perform derived-to-base casts and/or field accesses, to get from the
582   // temporary object we created (and, potentially, for which we extended
583   // the lifetime) to the subobject we're binding the reference to.
584   for (SubobjectAdjustment &Adjustment : llvm::reverse(Adjustments)) {
585     switch (Adjustment.Kind) {
586     case SubobjectAdjustment::DerivedToBaseAdjustment:
587       Object =
588           GetAddressOfBaseClass(Object, Adjustment.DerivedToBase.DerivedClass,
589                                 Adjustment.DerivedToBase.BasePath->path_begin(),
590                                 Adjustment.DerivedToBase.BasePath->path_end(),
591                                 /*NullCheckValue=*/ false, E->getExprLoc());
592       break;
593 
594     case SubobjectAdjustment::FieldAdjustment: {
595       LValue LV = MakeAddrLValue(Object, E->getType(), AlignmentSource::Decl);
596       LV = EmitLValueForField(LV, Adjustment.Field);
597       assert(LV.isSimple() &&
598              "materialized temporary field is not a simple lvalue");
599       Object = LV.getAddress(*this);
600       break;
601     }
602 
603     case SubobjectAdjustment::MemberPointerAdjustment: {
604       llvm::Value *Ptr = EmitScalarExpr(Adjustment.Ptr.RHS);
605       Object = EmitCXXMemberDataPointerAddress(E, Object, Ptr,
606                                                Adjustment.Ptr.MPT);
607       break;
608     }
609     }
610   }
611 
612   return MakeAddrLValue(Object, M->getType(), AlignmentSource::Decl);
613 }
614 
615 RValue
EmitReferenceBindingToExpr(const Expr * E)616 CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E) {
617   // Emit the expression as an lvalue.
618   LValue LV = EmitLValue(E);
619   assert(LV.isSimple());
620   llvm::Value *Value = LV.getPointer(*this);
621 
622   if (sanitizePerformTypeCheck() && !E->getType()->isFunctionType()) {
623     // C++11 [dcl.ref]p5 (as amended by core issue 453):
624     //   If a glvalue to which a reference is directly bound designates neither
625     //   an existing object or function of an appropriate type nor a region of
626     //   storage of suitable size and alignment to contain an object of the
627     //   reference's type, the behavior is undefined.
628     QualType Ty = E->getType();
629     EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty);
630   }
631 
632   return RValue::get(Value);
633 }
634 
635 
636 /// getAccessedFieldNo - Given an encoded value and a result number, return the
637 /// input field number being accessed.
getAccessedFieldNo(unsigned Idx,const llvm::Constant * Elts)638 unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
639                                              const llvm::Constant *Elts) {
640   return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
641       ->getZExtValue();
642 }
643 
644 /// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
emitHash16Bytes(CGBuilderTy & Builder,llvm::Value * Low,llvm::Value * High)645 static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
646                                     llvm::Value *High) {
647   llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
648   llvm::Value *K47 = Builder.getInt64(47);
649   llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
650   llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
651   llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
652   llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
653   return Builder.CreateMul(B1, KMul);
654 }
655 
isNullPointerAllowed(TypeCheckKind TCK)656 bool CodeGenFunction::isNullPointerAllowed(TypeCheckKind TCK) {
657   return TCK == TCK_DowncastPointer || TCK == TCK_Upcast ||
658          TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation;
659 }
660 
isVptrCheckRequired(TypeCheckKind TCK,QualType Ty)661 bool CodeGenFunction::isVptrCheckRequired(TypeCheckKind TCK, QualType Ty) {
662   CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
663   return (RD && RD->hasDefinition() && RD->isDynamicClass()) &&
664          (TCK == TCK_MemberAccess || TCK == TCK_MemberCall ||
665           TCK == TCK_DowncastPointer || TCK == TCK_DowncastReference ||
666           TCK == TCK_UpcastToVirtualBase || TCK == TCK_DynamicOperation);
667 }
668 
sanitizePerformTypeCheck() const669 bool CodeGenFunction::sanitizePerformTypeCheck() const {
670   return SanOpts.has(SanitizerKind::Null) ||
671          SanOpts.has(SanitizerKind::Alignment) ||
672          SanOpts.has(SanitizerKind::ObjectSize) ||
673          SanOpts.has(SanitizerKind::Vptr);
674 }
675 
EmitTypeCheck(TypeCheckKind TCK,SourceLocation Loc,llvm::Value * Ptr,QualType Ty,CharUnits Alignment,SanitizerSet SkippedChecks,llvm::Value * ArraySize)676 void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
677                                     llvm::Value *Ptr, QualType Ty,
678                                     CharUnits Alignment,
679                                     SanitizerSet SkippedChecks,
680                                     llvm::Value *ArraySize) {
681   if (!sanitizePerformTypeCheck())
682     return;
683 
684   // Don't check pointers outside the default address space. The null check
685   // isn't correct, the object-size check isn't supported by LLVM, and we can't
686   // communicate the addresses to the runtime handler for the vptr check.
687   if (Ptr->getType()->getPointerAddressSpace())
688     return;
689 
690   // Don't check pointers to volatile data. The behavior here is implementation-
691   // defined.
692   if (Ty.isVolatileQualified())
693     return;
694 
695   SanitizerScope SanScope(this);
696 
697   SmallVector<std::pair<llvm::Value *, SanitizerMask>, 3> Checks;
698   llvm::BasicBlock *Done = nullptr;
699 
700   // Quickly determine whether we have a pointer to an alloca. It's possible
701   // to skip null checks, and some alignment checks, for these pointers. This
702   // can reduce compile-time significantly.
703   auto PtrToAlloca = dyn_cast<llvm::AllocaInst>(Ptr->stripPointerCasts());
704 
705   llvm::Value *True = llvm::ConstantInt::getTrue(getLLVMContext());
706   llvm::Value *IsNonNull = nullptr;
707   bool IsGuaranteedNonNull =
708       SkippedChecks.has(SanitizerKind::Null) || PtrToAlloca;
709   bool AllowNullPointers = isNullPointerAllowed(TCK);
710   if ((SanOpts.has(SanitizerKind::Null) || AllowNullPointers) &&
711       !IsGuaranteedNonNull) {
712     // The glvalue must not be an empty glvalue.
713     IsNonNull = Builder.CreateIsNotNull(Ptr);
714 
715     // The IR builder can constant-fold the null check if the pointer points to
716     // a constant.
717     IsGuaranteedNonNull = IsNonNull == True;
718 
719     // Skip the null check if the pointer is known to be non-null.
720     if (!IsGuaranteedNonNull) {
721       if (AllowNullPointers) {
722         // When performing pointer casts, it's OK if the value is null.
723         // Skip the remaining checks in that case.
724         Done = createBasicBlock("null");
725         llvm::BasicBlock *Rest = createBasicBlock("not.null");
726         Builder.CreateCondBr(IsNonNull, Rest, Done);
727         EmitBlock(Rest);
728       } else {
729         Checks.push_back(std::make_pair(IsNonNull, SanitizerKind::Null));
730       }
731     }
732   }
733 
734   if (SanOpts.has(SanitizerKind::ObjectSize) &&
735       !SkippedChecks.has(SanitizerKind::ObjectSize) &&
736       !Ty->isIncompleteType()) {
737     uint64_t TySize = CGM.getMinimumObjectSize(Ty).getQuantity();
738     llvm::Value *Size = llvm::ConstantInt::get(IntPtrTy, TySize);
739     if (ArraySize)
740       Size = Builder.CreateMul(Size, ArraySize);
741 
742     // Degenerate case: new X[0] does not need an objectsize check.
743     llvm::Constant *ConstantSize = dyn_cast<llvm::Constant>(Size);
744     if (!ConstantSize || !ConstantSize->isNullValue()) {
745       // The glvalue must refer to a large enough storage region.
746       // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
747       //        to check this.
748       // FIXME: Get object address space
749       llvm::Type *Tys[2] = { IntPtrTy, Int8PtrTy };
750       llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, Tys);
751       llvm::Value *Min = Builder.getFalse();
752       llvm::Value *NullIsUnknown = Builder.getFalse();
753       llvm::Value *Dynamic = Builder.getFalse();
754       llvm::Value *CastAddr = Builder.CreateBitCast(Ptr, Int8PtrTy);
755       llvm::Value *LargeEnough = Builder.CreateICmpUGE(
756           Builder.CreateCall(F, {CastAddr, Min, NullIsUnknown, Dynamic}), Size);
757       Checks.push_back(std::make_pair(LargeEnough, SanitizerKind::ObjectSize));
758     }
759   }
760 
761   llvm::MaybeAlign AlignVal;
762   llvm::Value *PtrAsInt = nullptr;
763 
764   if (SanOpts.has(SanitizerKind::Alignment) &&
765       !SkippedChecks.has(SanitizerKind::Alignment)) {
766     AlignVal = Alignment.getAsMaybeAlign();
767     if (!Ty->isIncompleteType() && !AlignVal)
768       AlignVal = CGM.getNaturalTypeAlignment(Ty, nullptr, nullptr,
769                                              /*ForPointeeType=*/true)
770                      .getAsMaybeAlign();
771 
772     // The glvalue must be suitably aligned.
773     if (AlignVal && *AlignVal > llvm::Align(1) &&
774         (!PtrToAlloca || PtrToAlloca->getAlign() < *AlignVal)) {
775       PtrAsInt = Builder.CreatePtrToInt(Ptr, IntPtrTy);
776       llvm::Value *Align = Builder.CreateAnd(
777           PtrAsInt, llvm::ConstantInt::get(IntPtrTy, AlignVal->value() - 1));
778       llvm::Value *Aligned =
779           Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
780       if (Aligned != True)
781         Checks.push_back(std::make_pair(Aligned, SanitizerKind::Alignment));
782     }
783   }
784 
785   if (Checks.size() > 0) {
786     llvm::Constant *StaticData[] = {
787         EmitCheckSourceLocation(Loc), EmitCheckTypeDescriptor(Ty),
788         llvm::ConstantInt::get(Int8Ty, AlignVal ? llvm::Log2(*AlignVal) : 1),
789         llvm::ConstantInt::get(Int8Ty, TCK)};
790     EmitCheck(Checks, SanitizerHandler::TypeMismatch, StaticData,
791               PtrAsInt ? PtrAsInt : Ptr);
792   }
793 
794   // If possible, check that the vptr indicates that there is a subobject of
795   // type Ty at offset zero within this object.
796   //
797   // C++11 [basic.life]p5,6:
798   //   [For storage which does not refer to an object within its lifetime]
799   //   The program has undefined behavior if:
800   //    -- the [pointer or glvalue] is used to access a non-static data member
801   //       or call a non-static member function
802   if (SanOpts.has(SanitizerKind::Vptr) &&
803       !SkippedChecks.has(SanitizerKind::Vptr) && isVptrCheckRequired(TCK, Ty)) {
804     // Ensure that the pointer is non-null before loading it. If there is no
805     // compile-time guarantee, reuse the run-time null check or emit a new one.
806     if (!IsGuaranteedNonNull) {
807       if (!IsNonNull)
808         IsNonNull = Builder.CreateIsNotNull(Ptr);
809       if (!Done)
810         Done = createBasicBlock("vptr.null");
811       llvm::BasicBlock *VptrNotNull = createBasicBlock("vptr.not.null");
812       Builder.CreateCondBr(IsNonNull, VptrNotNull, Done);
813       EmitBlock(VptrNotNull);
814     }
815 
816     // Compute a hash of the mangled name of the type.
817     //
818     // FIXME: This is not guaranteed to be deterministic! Move to a
819     //        fingerprinting mechanism once LLVM provides one. For the time
820     //        being the implementation happens to be deterministic.
821     SmallString<64> MangledName;
822     llvm::raw_svector_ostream Out(MangledName);
823     CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(),
824                                                      Out);
825 
826     // Contained in NoSanitizeList based on the mangled type.
827     if (!CGM.getContext().getNoSanitizeList().containsType(SanitizerKind::Vptr,
828                                                            Out.str())) {
829       llvm::hash_code TypeHash = hash_value(Out.str());
830 
831       // Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
832       llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
833       llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
834       Address VPtrAddr(Builder.CreateBitCast(Ptr, VPtrTy), IntPtrTy,
835                        getPointerAlign());
836       llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
837       llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
838 
839       llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
840       Hash = Builder.CreateTrunc(Hash, IntPtrTy);
841 
842       // Look the hash up in our cache.
843       const int CacheSize = 128;
844       llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);
845       llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,
846                                                      "__ubsan_vptr_type_cache");
847       llvm::Value *Slot = Builder.CreateAnd(Hash,
848                                             llvm::ConstantInt::get(IntPtrTy,
849                                                                    CacheSize-1));
850       llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
851       llvm::Value *CacheVal = Builder.CreateAlignedLoad(
852           IntPtrTy, Builder.CreateInBoundsGEP(HashTable, Cache, Indices),
853           getPointerAlign());
854 
855       // If the hash isn't in the cache, call a runtime handler to perform the
856       // hard work of checking whether the vptr is for an object of the right
857       // type. This will either fill in the cache and return, or produce a
858       // diagnostic.
859       llvm::Value *EqualHash = Builder.CreateICmpEQ(CacheVal, Hash);
860       llvm::Constant *StaticData[] = {
861         EmitCheckSourceLocation(Loc),
862         EmitCheckTypeDescriptor(Ty),
863         CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()),
864         llvm::ConstantInt::get(Int8Ty, TCK)
865       };
866       llvm::Value *DynamicData[] = { Ptr, Hash };
867       EmitCheck(std::make_pair(EqualHash, SanitizerKind::Vptr),
868                 SanitizerHandler::DynamicTypeCacheMiss, StaticData,
869                 DynamicData);
870     }
871   }
872 
873   if (Done) {
874     Builder.CreateBr(Done);
875     EmitBlock(Done);
876   }
877 }
878 
LoadPassedObjectSize(const Expr * E,QualType EltTy)879 llvm::Value *CodeGenFunction::LoadPassedObjectSize(const Expr *E,
880                                                    QualType EltTy) {
881   ASTContext &C = getContext();
882   uint64_t EltSize = C.getTypeSizeInChars(EltTy).getQuantity();
883   if (!EltSize)
884     return nullptr;
885 
886   auto *ArrayDeclRef = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts());
887   if (!ArrayDeclRef)
888     return nullptr;
889 
890   auto *ParamDecl = dyn_cast<ParmVarDecl>(ArrayDeclRef->getDecl());
891   if (!ParamDecl)
892     return nullptr;
893 
894   auto *POSAttr = ParamDecl->getAttr<PassObjectSizeAttr>();
895   if (!POSAttr)
896     return nullptr;
897 
898   // Don't load the size if it's a lower bound.
899   int POSType = POSAttr->getType();
900   if (POSType != 0 && POSType != 1)
901     return nullptr;
902 
903   // Find the implicit size parameter.
904   auto PassedSizeIt = SizeArguments.find(ParamDecl);
905   if (PassedSizeIt == SizeArguments.end())
906     return nullptr;
907 
908   const ImplicitParamDecl *PassedSizeDecl = PassedSizeIt->second;
909   assert(LocalDeclMap.count(PassedSizeDecl) && "Passed size not loadable");
910   Address AddrOfSize = LocalDeclMap.find(PassedSizeDecl)->second;
911   llvm::Value *SizeInBytes = EmitLoadOfScalar(AddrOfSize, /*Volatile=*/false,
912                                               C.getSizeType(), E->getExprLoc());
913   llvm::Value *SizeOfElement =
914       llvm::ConstantInt::get(SizeInBytes->getType(), EltSize);
915   return Builder.CreateUDiv(SizeInBytes, SizeOfElement);
916 }
917 
918 /// If Base is known to point to the start of an array, return the length of
919 /// that array. Return 0 if the length cannot be determined.
getArrayIndexingBound(CodeGenFunction & CGF,const Expr * Base,QualType & IndexedType,LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel)920 static llvm::Value *getArrayIndexingBound(CodeGenFunction &CGF,
921                                           const Expr *Base,
922                                           QualType &IndexedType,
923                                           LangOptions::StrictFlexArraysLevelKind
924                                           StrictFlexArraysLevel) {
925   // For the vector indexing extension, the bound is the number of elements.
926   if (const VectorType *VT = Base->getType()->getAs<VectorType>()) {
927     IndexedType = Base->getType();
928     return CGF.Builder.getInt32(VT->getNumElements());
929   }
930 
931   Base = Base->IgnoreParens();
932 
933   if (const auto *CE = dyn_cast<CastExpr>(Base)) {
934     if (CE->getCastKind() == CK_ArrayToPointerDecay &&
935         !CE->getSubExpr()->isFlexibleArrayMemberLike(CGF.getContext(),
936                                                      StrictFlexArraysLevel)) {
937       IndexedType = CE->getSubExpr()->getType();
938       const ArrayType *AT = IndexedType->castAsArrayTypeUnsafe();
939       if (const auto *CAT = dyn_cast<ConstantArrayType>(AT))
940         return CGF.Builder.getInt(CAT->getSize());
941       else if (const auto *VAT = dyn_cast<VariableArrayType>(AT))
942         return CGF.getVLASize(VAT).NumElts;
943       // Ignore pass_object_size here. It's not applicable on decayed pointers.
944     }
945   }
946 
947   QualType EltTy{Base->getType()->getPointeeOrArrayElementType(), 0};
948   if (llvm::Value *POS = CGF.LoadPassedObjectSize(Base, EltTy)) {
949     IndexedType = Base->getType();
950     return POS;
951   }
952 
953   return nullptr;
954 }
955 
EmitBoundsCheck(const Expr * E,const Expr * Base,llvm::Value * Index,QualType IndexType,bool Accessed)956 void CodeGenFunction::EmitBoundsCheck(const Expr *E, const Expr *Base,
957                                       llvm::Value *Index, QualType IndexType,
958                                       bool Accessed) {
959   assert(SanOpts.has(SanitizerKind::ArrayBounds) &&
960          "should not be called unless adding bounds checks");
961   SanitizerScope SanScope(this);
962 
963   const LangOptions::StrictFlexArraysLevelKind StrictFlexArraysLevel =
964     getLangOpts().getStrictFlexArraysLevel();
965 
966   QualType IndexedType;
967   llvm::Value *Bound =
968       getArrayIndexingBound(*this, Base, IndexedType, StrictFlexArraysLevel);
969   if (!Bound)
970     return;
971 
972   bool IndexSigned = IndexType->isSignedIntegerOrEnumerationType();
973   llvm::Value *IndexVal = Builder.CreateIntCast(Index, SizeTy, IndexSigned);
974   llvm::Value *BoundVal = Builder.CreateIntCast(Bound, SizeTy, false);
975 
976   llvm::Constant *StaticData[] = {
977     EmitCheckSourceLocation(E->getExprLoc()),
978     EmitCheckTypeDescriptor(IndexedType),
979     EmitCheckTypeDescriptor(IndexType)
980   };
981   llvm::Value *Check = Accessed ? Builder.CreateICmpULT(IndexVal, BoundVal)
982                                 : Builder.CreateICmpULE(IndexVal, BoundVal);
983   EmitCheck(std::make_pair(Check, SanitizerKind::ArrayBounds),
984             SanitizerHandler::OutOfBounds, StaticData, Index);
985 }
986 
987 
988 CodeGenFunction::ComplexPairTy CodeGenFunction::
EmitComplexPrePostIncDec(const UnaryOperator * E,LValue LV,bool isInc,bool isPre)989 EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
990                          bool isInc, bool isPre) {
991   ComplexPairTy InVal = EmitLoadOfComplex(LV, E->getExprLoc());
992 
993   llvm::Value *NextVal;
994   if (isa<llvm::IntegerType>(InVal.first->getType())) {
995     uint64_t AmountVal = isInc ? 1 : -1;
996     NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
997 
998     // Add the inc/dec to the real part.
999     NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1000   } else {
1001     QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
1002     llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
1003     if (!isInc)
1004       FVal.changeSign();
1005     NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
1006 
1007     // Add the inc/dec to the real part.
1008     NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
1009   }
1010 
1011   ComplexPairTy IncVal(NextVal, InVal.second);
1012 
1013   // Store the updated result through the lvalue.
1014   EmitStoreOfComplex(IncVal, LV, /*init*/ false);
1015   if (getLangOpts().OpenMP)
1016     CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
1017                                                               E->getSubExpr());
1018 
1019   // If this is a postinc, return the value read from memory, otherwise use the
1020   // updated value.
1021   return isPre ? IncVal : InVal;
1022 }
1023 
EmitExplicitCastExprType(const ExplicitCastExpr * E,CodeGenFunction * CGF)1024 void CodeGenModule::EmitExplicitCastExprType(const ExplicitCastExpr *E,
1025                                              CodeGenFunction *CGF) {
1026   // Bind VLAs in the cast type.
1027   if (CGF && E->getType()->isVariablyModifiedType())
1028     CGF->EmitVariablyModifiedType(E->getType());
1029 
1030   if (CGDebugInfo *DI = getModuleDebugInfo())
1031     DI->EmitExplicitCastType(E->getType());
1032 }
1033 
1034 //===----------------------------------------------------------------------===//
1035 //                         LValue Expression Emission
1036 //===----------------------------------------------------------------------===//
1037 
1038 /// EmitPointerWithAlignment - Given an expression of pointer type, try to
1039 /// derive a more accurate bound on the alignment of the pointer.
EmitPointerWithAlignment(const Expr * E,LValueBaseInfo * BaseInfo,TBAAAccessInfo * TBAAInfo)1040 Address CodeGenFunction::EmitPointerWithAlignment(const Expr *E,
1041                                                   LValueBaseInfo *BaseInfo,
1042                                                   TBAAAccessInfo *TBAAInfo) {
1043   // We allow this with ObjC object pointers because of fragile ABIs.
1044   assert(E->getType()->isPointerType() ||
1045          E->getType()->isObjCObjectPointerType());
1046   E = E->IgnoreParens();
1047 
1048   // Casts:
1049   if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
1050     if (const auto *ECE = dyn_cast<ExplicitCastExpr>(CE))
1051       CGM.EmitExplicitCastExprType(ECE, this);
1052 
1053     switch (CE->getCastKind()) {
1054     // Non-converting casts (but not C's implicit conversion from void*).
1055     case CK_BitCast:
1056     case CK_NoOp:
1057     case CK_AddressSpaceConversion:
1058       if (auto PtrTy = CE->getSubExpr()->getType()->getAs<PointerType>()) {
1059         if (PtrTy->getPointeeType()->isVoidType())
1060           break;
1061 
1062         LValueBaseInfo InnerBaseInfo;
1063         TBAAAccessInfo InnerTBAAInfo;
1064         Address Addr = EmitPointerWithAlignment(CE->getSubExpr(),
1065                                                 &InnerBaseInfo,
1066                                                 &InnerTBAAInfo);
1067         if (BaseInfo) *BaseInfo = InnerBaseInfo;
1068         if (TBAAInfo) *TBAAInfo = InnerTBAAInfo;
1069 
1070         if (isa<ExplicitCastExpr>(CE)) {
1071           LValueBaseInfo TargetTypeBaseInfo;
1072           TBAAAccessInfo TargetTypeTBAAInfo;
1073           CharUnits Align = CGM.getNaturalPointeeTypeAlignment(
1074               E->getType(), &TargetTypeBaseInfo, &TargetTypeTBAAInfo);
1075           if (TBAAInfo)
1076             *TBAAInfo = CGM.mergeTBAAInfoForCast(*TBAAInfo,
1077                                                  TargetTypeTBAAInfo);
1078           // If the source l-value is opaque, honor the alignment of the
1079           // casted-to type.
1080           if (InnerBaseInfo.getAlignmentSource() != AlignmentSource::Decl) {
1081             if (BaseInfo)
1082               BaseInfo->mergeForCast(TargetTypeBaseInfo);
1083             Addr = Address(Addr.getPointer(), Addr.getElementType(), Align);
1084           }
1085         }
1086 
1087         if (SanOpts.has(SanitizerKind::CFIUnrelatedCast) &&
1088             CE->getCastKind() == CK_BitCast) {
1089           if (auto PT = E->getType()->getAs<PointerType>())
1090             EmitVTablePtrCheckForCast(PT->getPointeeType(), Addr,
1091                                       /*MayBeNull=*/true,
1092                                       CodeGenFunction::CFITCK_UnrelatedCast,
1093                                       CE->getBeginLoc());
1094         }
1095 
1096         llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType());
1097         Addr = Builder.CreateElementBitCast(Addr, ElemTy);
1098         if (CE->getCastKind() == CK_AddressSpaceConversion)
1099           Addr = Builder.CreateAddrSpaceCast(Addr, ConvertType(E->getType()));
1100         return Addr;
1101       }
1102       break;
1103 
1104     // Array-to-pointer decay.
1105     case CK_ArrayToPointerDecay:
1106       return EmitArrayToPointerDecay(CE->getSubExpr(), BaseInfo, TBAAInfo);
1107 
1108     // Derived-to-base conversions.
1109     case CK_UncheckedDerivedToBase:
1110     case CK_DerivedToBase: {
1111       // TODO: Support accesses to members of base classes in TBAA. For now, we
1112       // conservatively pretend that the complete object is of the base class
1113       // type.
1114       if (TBAAInfo)
1115         *TBAAInfo = CGM.getTBAAAccessInfo(E->getType());
1116       Address Addr = EmitPointerWithAlignment(CE->getSubExpr(), BaseInfo);
1117       auto Derived = CE->getSubExpr()->getType()->getPointeeCXXRecordDecl();
1118       return GetAddressOfBaseClass(Addr, Derived,
1119                                    CE->path_begin(), CE->path_end(),
1120                                    ShouldNullCheckClassCastValue(CE),
1121                                    CE->getExprLoc());
1122     }
1123 
1124     // TODO: Is there any reason to treat base-to-derived conversions
1125     // specially?
1126     default:
1127       break;
1128     }
1129   }
1130 
1131   // Unary &.
1132   if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(E)) {
1133     if (UO->getOpcode() == UO_AddrOf) {
1134       LValue LV = EmitLValue(UO->getSubExpr());
1135       if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1136       if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1137       return LV.getAddress(*this);
1138     }
1139   }
1140 
1141   // std::addressof and variants.
1142   if (auto *Call = dyn_cast<CallExpr>(E)) {
1143     switch (Call->getBuiltinCallee()) {
1144     default:
1145       break;
1146     case Builtin::BIaddressof:
1147     case Builtin::BI__addressof:
1148     case Builtin::BI__builtin_addressof: {
1149       LValue LV = EmitLValue(Call->getArg(0));
1150       if (BaseInfo) *BaseInfo = LV.getBaseInfo();
1151       if (TBAAInfo) *TBAAInfo = LV.getTBAAInfo();
1152       return LV.getAddress(*this);
1153     }
1154     }
1155   }
1156 
1157   // TODO: conditional operators, comma.
1158 
1159   // Otherwise, use the alignment of the type.
1160   CharUnits Align =
1161       CGM.getNaturalPointeeTypeAlignment(E->getType(), BaseInfo, TBAAInfo);
1162   llvm::Type *ElemTy = ConvertTypeForMem(E->getType()->getPointeeType());
1163   return Address(EmitScalarExpr(E), ElemTy, Align);
1164 }
1165 
EmitNonNullRValueCheck(RValue RV,QualType T)1166 llvm::Value *CodeGenFunction::EmitNonNullRValueCheck(RValue RV, QualType T) {
1167   llvm::Value *V = RV.getScalarVal();
1168   if (auto MPT = T->getAs<MemberPointerType>())
1169     return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, V, MPT);
1170   return Builder.CreateICmpNE(V, llvm::Constant::getNullValue(V->getType()));
1171 }
1172 
GetUndefRValue(QualType Ty)1173 RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
1174   if (Ty->isVoidType())
1175     return RValue::get(nullptr);
1176 
1177   switch (getEvaluationKind(Ty)) {
1178   case TEK_Complex: {
1179     llvm::Type *EltTy =
1180       ConvertType(Ty->castAs<ComplexType>()->getElementType());
1181     llvm::Value *U = llvm::UndefValue::get(EltTy);
1182     return RValue::getComplex(std::make_pair(U, U));
1183   }
1184 
1185   // If this is a use of an undefined aggregate type, the aggregate must have an
1186   // identifiable address.  Just because the contents of the value are undefined
1187   // doesn't mean that the address can't be taken and compared.
1188   case TEK_Aggregate: {
1189     Address DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
1190     return RValue::getAggregate(DestPtr);
1191   }
1192 
1193   case TEK_Scalar:
1194     return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
1195   }
1196   llvm_unreachable("bad evaluation kind");
1197 }
1198 
EmitUnsupportedRValue(const Expr * E,const char * Name)1199 RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
1200                                               const char *Name) {
1201   ErrorUnsupported(E, Name);
1202   return GetUndefRValue(E->getType());
1203 }
1204 
EmitUnsupportedLValue(const Expr * E,const char * Name)1205 LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
1206                                               const char *Name) {
1207   ErrorUnsupported(E, Name);
1208   llvm::Type *ElTy = ConvertType(E->getType());
1209   llvm::Type *Ty = llvm::PointerType::getUnqual(ElTy);
1210   return MakeAddrLValue(
1211       Address(llvm::UndefValue::get(Ty), ElTy, CharUnits::One()), E->getType());
1212 }
1213 
IsWrappedCXXThis(const Expr * Obj)1214 bool CodeGenFunction::IsWrappedCXXThis(const Expr *Obj) {
1215   const Expr *Base = Obj;
1216   while (!isa<CXXThisExpr>(Base)) {
1217     // The result of a dynamic_cast can be null.
1218     if (isa<CXXDynamicCastExpr>(Base))
1219       return false;
1220 
1221     if (const auto *CE = dyn_cast<CastExpr>(Base)) {
1222       Base = CE->getSubExpr();
1223     } else if (const auto *PE = dyn_cast<ParenExpr>(Base)) {
1224       Base = PE->getSubExpr();
1225     } else if (const auto *UO = dyn_cast<UnaryOperator>(Base)) {
1226       if (UO->getOpcode() == UO_Extension)
1227         Base = UO->getSubExpr();
1228       else
1229         return false;
1230     } else {
1231       return false;
1232     }
1233   }
1234   return true;
1235 }
1236 
EmitCheckedLValue(const Expr * E,TypeCheckKind TCK)1237 LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
1238   LValue LV;
1239   if (SanOpts.has(SanitizerKind::ArrayBounds) && isa<ArraySubscriptExpr>(E))
1240     LV = EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E), /*Accessed*/true);
1241   else
1242     LV = EmitLValue(E);
1243   if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple()) {
1244     SanitizerSet SkippedChecks;
1245     if (const auto *ME = dyn_cast<MemberExpr>(E)) {
1246       bool IsBaseCXXThis = IsWrappedCXXThis(ME->getBase());
1247       if (IsBaseCXXThis)
1248         SkippedChecks.set(SanitizerKind::Alignment, true);
1249       if (IsBaseCXXThis || isa<DeclRefExpr>(ME->getBase()))
1250         SkippedChecks.set(SanitizerKind::Null, true);
1251     }
1252     EmitTypeCheck(TCK, E->getExprLoc(), LV.getPointer(*this), E->getType(),
1253                   LV.getAlignment(), SkippedChecks);
1254   }
1255   return LV;
1256 }
1257 
1258 /// EmitLValue - Emit code to compute a designator that specifies the location
1259 /// of the expression.
1260 ///
1261 /// This can return one of two things: a simple address or a bitfield reference.
1262 /// In either case, the LLVM Value* in the LValue structure is guaranteed to be
1263 /// an LLVM pointer type.
1264 ///
1265 /// If this returns a bitfield reference, nothing about the pointee type of the
1266 /// LLVM value is known: For example, it may not be a pointer to an integer.
1267 ///
1268 /// If this returns a normal address, and if the lvalue's C type is fixed size,
1269 /// this method guarantees that the returned pointer type will point to an LLVM
1270 /// type of the same size of the lvalue's type.  If the lvalue has a variable
1271 /// length type, this is not possible.
1272 ///
EmitLValue(const Expr * E)1273 LValue CodeGenFunction::EmitLValue(const Expr *E) {
1274   ApplyDebugLocation DL(*this, E);
1275   switch (E->getStmtClass()) {
1276   default: return EmitUnsupportedLValue(E, "l-value expression");
1277 
1278   case Expr::ObjCPropertyRefExprClass:
1279     llvm_unreachable("cannot emit a property reference directly");
1280 
1281   case Expr::ObjCSelectorExprClass:
1282     return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
1283   case Expr::ObjCIsaExprClass:
1284     return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
1285   case Expr::BinaryOperatorClass:
1286     return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
1287   case Expr::CompoundAssignOperatorClass: {
1288     QualType Ty = E->getType();
1289     if (const AtomicType *AT = Ty->getAs<AtomicType>())
1290       Ty = AT->getValueType();
1291     if (!Ty->isAnyComplexType())
1292       return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1293     return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
1294   }
1295   case Expr::CallExprClass:
1296   case Expr::CXXMemberCallExprClass:
1297   case Expr::CXXOperatorCallExprClass:
1298   case Expr::UserDefinedLiteralClass:
1299     return EmitCallExprLValue(cast<CallExpr>(E));
1300   case Expr::CXXRewrittenBinaryOperatorClass:
1301     return EmitLValue(cast<CXXRewrittenBinaryOperator>(E)->getSemanticForm());
1302   case Expr::VAArgExprClass:
1303     return EmitVAArgExprLValue(cast<VAArgExpr>(E));
1304   case Expr::DeclRefExprClass:
1305     return EmitDeclRefLValue(cast<DeclRefExpr>(E));
1306   case Expr::ConstantExprClass: {
1307     const ConstantExpr *CE = cast<ConstantExpr>(E);
1308     if (llvm::Value *Result = ConstantEmitter(*this).tryEmitConstantExpr(CE)) {
1309       QualType RetType = cast<CallExpr>(CE->getSubExpr()->IgnoreImplicit())
1310                              ->getCallReturnType(getContext())
1311                              ->getPointeeType();
1312       return MakeNaturalAlignAddrLValue(Result, RetType);
1313     }
1314     return EmitLValue(cast<ConstantExpr>(E)->getSubExpr());
1315   }
1316   case Expr::ParenExprClass:
1317     return EmitLValue(cast<ParenExpr>(E)->getSubExpr());
1318   case Expr::GenericSelectionExprClass:
1319     return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr());
1320   case Expr::PredefinedExprClass:
1321     return EmitPredefinedLValue(cast<PredefinedExpr>(E));
1322   case Expr::StringLiteralClass:
1323     return EmitStringLiteralLValue(cast<StringLiteral>(E));
1324   case Expr::ObjCEncodeExprClass:
1325     return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
1326   case Expr::PseudoObjectExprClass:
1327     return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
1328   case Expr::InitListExprClass:
1329     return EmitInitListLValue(cast<InitListExpr>(E));
1330   case Expr::CXXTemporaryObjectExprClass:
1331   case Expr::CXXConstructExprClass:
1332     return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
1333   case Expr::CXXBindTemporaryExprClass:
1334     return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
1335   case Expr::CXXUuidofExprClass:
1336     return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));
1337   case Expr::LambdaExprClass:
1338     return EmitAggExprToLValue(E);
1339 
1340   case Expr::ExprWithCleanupsClass: {
1341     const auto *cleanups = cast<ExprWithCleanups>(E);
1342     RunCleanupsScope Scope(*this);
1343     LValue LV = EmitLValue(cleanups->getSubExpr());
1344     if (LV.isSimple()) {
1345       // Defend against branches out of gnu statement expressions surrounded by
1346       // cleanups.
1347       Address Addr = LV.getAddress(*this);
1348       llvm::Value *V = Addr.getPointer();
1349       Scope.ForceCleanup({&V});
1350       return LValue::MakeAddr(Addr.withPointer(V), LV.getType(), getContext(),
1351                               LV.getBaseInfo(), LV.getTBAAInfo());
1352     }
1353     // FIXME: Is it possible to create an ExprWithCleanups that produces a
1354     // bitfield lvalue or some other non-simple lvalue?
1355     return LV;
1356   }
1357 
1358   case Expr::CXXDefaultArgExprClass: {
1359     auto *DAE = cast<CXXDefaultArgExpr>(E);
1360     CXXDefaultArgExprScope Scope(*this, DAE);
1361     return EmitLValue(DAE->getExpr());
1362   }
1363   case Expr::CXXDefaultInitExprClass: {
1364     auto *DIE = cast<CXXDefaultInitExpr>(E);
1365     CXXDefaultInitExprScope Scope(*this, DIE);
1366     return EmitLValue(DIE->getExpr());
1367   }
1368   case Expr::CXXTypeidExprClass:
1369     return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
1370 
1371   case Expr::ObjCMessageExprClass:
1372     return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
1373   case Expr::ObjCIvarRefExprClass:
1374     return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
1375   case Expr::StmtExprClass:
1376     return EmitStmtExprLValue(cast<StmtExpr>(E));
1377   case Expr::UnaryOperatorClass:
1378     return EmitUnaryOpLValue(cast<UnaryOperator>(E));
1379   case Expr::ArraySubscriptExprClass:
1380     return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
1381   case Expr::MatrixSubscriptExprClass:
1382     return EmitMatrixSubscriptExpr(cast<MatrixSubscriptExpr>(E));
1383   case Expr::OMPArraySectionExprClass:
1384     return EmitOMPArraySectionExpr(cast<OMPArraySectionExpr>(E));
1385   case Expr::ExtVectorElementExprClass:
1386     return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
1387   case Expr::CXXThisExprClass:
1388     return MakeAddrLValue(LoadCXXThisAddress(), E->getType());
1389   case Expr::MemberExprClass:
1390     return EmitMemberExpr(cast<MemberExpr>(E));
1391   case Expr::CompoundLiteralExprClass:
1392     return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
1393   case Expr::ConditionalOperatorClass:
1394     return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
1395   case Expr::BinaryConditionalOperatorClass:
1396     return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
1397   case Expr::ChooseExprClass:
1398     return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr());
1399   case Expr::OpaqueValueExprClass:
1400     return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
1401   case Expr::SubstNonTypeTemplateParmExprClass:
1402     return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement());
1403   case Expr::ImplicitCastExprClass:
1404   case Expr::CStyleCastExprClass:
1405   case Expr::CXXFunctionalCastExprClass:
1406   case Expr::CXXStaticCastExprClass:
1407   case Expr::CXXDynamicCastExprClass:
1408   case Expr::CXXReinterpretCastExprClass:
1409   case Expr::CXXConstCastExprClass:
1410   case Expr::CXXAddrspaceCastExprClass:
1411   case Expr::ObjCBridgedCastExprClass:
1412     return EmitCastLValue(cast<CastExpr>(E));
1413 
1414   case Expr::MaterializeTemporaryExprClass:
1415     return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
1416 
1417   case Expr::CoawaitExprClass:
1418     return EmitCoawaitLValue(cast<CoawaitExpr>(E));
1419   case Expr::CoyieldExprClass:
1420     return EmitCoyieldLValue(cast<CoyieldExpr>(E));
1421   }
1422 }
1423 
1424 /// Given an object of the given canonical type, can we safely copy a
1425 /// value out of it based on its initializer?
isConstantEmittableObjectType(QualType type)1426 static bool isConstantEmittableObjectType(QualType type) {
1427   assert(type.isCanonical());
1428   assert(!type->isReferenceType());
1429 
1430   // Must be const-qualified but non-volatile.
1431   Qualifiers qs = type.getLocalQualifiers();
1432   if (!qs.hasConst() || qs.hasVolatile()) return false;
1433 
1434   // Otherwise, all object types satisfy this except C++ classes with
1435   // mutable subobjects or non-trivial copy/destroy behavior.
1436   if (const auto *RT = dyn_cast<RecordType>(type))
1437     if (const auto *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
1438       if (RD->hasMutableFields() || !RD->isTrivial())
1439         return false;
1440 
1441   return true;
1442 }
1443 
1444 /// Can we constant-emit a load of a reference to a variable of the
1445 /// given type?  This is different from predicates like
1446 /// Decl::mightBeUsableInConstantExpressions because we do want it to apply
1447 /// in situations that don't necessarily satisfy the language's rules
1448 /// for this (e.g. C++'s ODR-use rules).  For example, we want to able
1449 /// to do this with const float variables even if those variables
1450 /// aren't marked 'constexpr'.
1451 enum ConstantEmissionKind {
1452   CEK_None,
1453   CEK_AsReferenceOnly,
1454   CEK_AsValueOrReference,
1455   CEK_AsValueOnly
1456 };
checkVarTypeForConstantEmission(QualType type)1457 static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
1458   type = type.getCanonicalType();
1459   if (const auto *ref = dyn_cast<ReferenceType>(type)) {
1460     if (isConstantEmittableObjectType(ref->getPointeeType()))
1461       return CEK_AsValueOrReference;
1462     return CEK_AsReferenceOnly;
1463   }
1464   if (isConstantEmittableObjectType(type))
1465     return CEK_AsValueOnly;
1466   return CEK_None;
1467 }
1468 
1469 /// Try to emit a reference to the given value without producing it as
1470 /// an l-value.  This is just an optimization, but it avoids us needing
1471 /// to emit global copies of variables if they're named without triggering
1472 /// a formal use in a context where we can't emit a direct reference to them,
1473 /// for instance if a block or lambda or a member of a local class uses a
1474 /// const int variable or constexpr variable from an enclosing function.
1475 CodeGenFunction::ConstantEmission
tryEmitAsConstant(DeclRefExpr * refExpr)1476 CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
1477   ValueDecl *value = refExpr->getDecl();
1478 
1479   // The value needs to be an enum constant or a constant variable.
1480   ConstantEmissionKind CEK;
1481   if (isa<ParmVarDecl>(value)) {
1482     CEK = CEK_None;
1483   } else if (auto *var = dyn_cast<VarDecl>(value)) {
1484     CEK = checkVarTypeForConstantEmission(var->getType());
1485   } else if (isa<EnumConstantDecl>(value)) {
1486     CEK = CEK_AsValueOnly;
1487   } else {
1488     CEK = CEK_None;
1489   }
1490   if (CEK == CEK_None) return ConstantEmission();
1491 
1492   Expr::EvalResult result;
1493   bool resultIsReference;
1494   QualType resultType;
1495 
1496   // It's best to evaluate all the way as an r-value if that's permitted.
1497   if (CEK != CEK_AsReferenceOnly &&
1498       refExpr->EvaluateAsRValue(result, getContext())) {
1499     resultIsReference = false;
1500     resultType = refExpr->getType();
1501 
1502   // Otherwise, try to evaluate as an l-value.
1503   } else if (CEK != CEK_AsValueOnly &&
1504              refExpr->EvaluateAsLValue(result, getContext())) {
1505     resultIsReference = true;
1506     resultType = value->getType();
1507 
1508   // Failure.
1509   } else {
1510     return ConstantEmission();
1511   }
1512 
1513   // In any case, if the initializer has side-effects, abandon ship.
1514   if (result.HasSideEffects)
1515     return ConstantEmission();
1516 
1517   // In CUDA/HIP device compilation, a lambda may capture a reference variable
1518   // referencing a global host variable by copy. In this case the lambda should
1519   // make a copy of the value of the global host variable. The DRE of the
1520   // captured reference variable cannot be emitted as load from the host
1521   // global variable as compile time constant, since the host variable is not
1522   // accessible on device. The DRE of the captured reference variable has to be
1523   // loaded from captures.
1524   if (CGM.getLangOpts().CUDAIsDevice && result.Val.isLValue() &&
1525       refExpr->refersToEnclosingVariableOrCapture()) {
1526     auto *MD = dyn_cast_or_null<CXXMethodDecl>(CurCodeDecl);
1527     if (MD && MD->getParent()->isLambda() &&
1528         MD->getOverloadedOperator() == OO_Call) {
1529       const APValue::LValueBase &base = result.Val.getLValueBase();
1530       if (const ValueDecl *D = base.dyn_cast<const ValueDecl *>()) {
1531         if (const VarDecl *VD = dyn_cast<const VarDecl>(D)) {
1532           if (!VD->hasAttr<CUDADeviceAttr>()) {
1533             return ConstantEmission();
1534           }
1535         }
1536       }
1537     }
1538   }
1539 
1540   // Emit as a constant.
1541   auto C = ConstantEmitter(*this).emitAbstract(refExpr->getLocation(),
1542                                                result.Val, resultType);
1543 
1544   // Make sure we emit a debug reference to the global variable.
1545   // This should probably fire even for
1546   if (isa<VarDecl>(value)) {
1547     if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
1548       EmitDeclRefExprDbgValue(refExpr, result.Val);
1549   } else {
1550     assert(isa<EnumConstantDecl>(value));
1551     EmitDeclRefExprDbgValue(refExpr, result.Val);
1552   }
1553 
1554   // If we emitted a reference constant, we need to dereference that.
1555   if (resultIsReference)
1556     return ConstantEmission::forReference(C);
1557 
1558   return ConstantEmission::forValue(C);
1559 }
1560 
tryToConvertMemberExprToDeclRefExpr(CodeGenFunction & CGF,const MemberExpr * ME)1561 static DeclRefExpr *tryToConvertMemberExprToDeclRefExpr(CodeGenFunction &CGF,
1562                                                         const MemberExpr *ME) {
1563   if (auto *VD = dyn_cast<VarDecl>(ME->getMemberDecl())) {
1564     // Try to emit static variable member expressions as DREs.
1565     return DeclRefExpr::Create(
1566         CGF.getContext(), NestedNameSpecifierLoc(), SourceLocation(), VD,
1567         /*RefersToEnclosingVariableOrCapture=*/false, ME->getExprLoc(),
1568         ME->getType(), ME->getValueKind(), nullptr, nullptr, ME->isNonOdrUse());
1569   }
1570   return nullptr;
1571 }
1572 
1573 CodeGenFunction::ConstantEmission
tryEmitAsConstant(const MemberExpr * ME)1574 CodeGenFunction::tryEmitAsConstant(const MemberExpr *ME) {
1575   if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, ME))
1576     return tryEmitAsConstant(DRE);
1577   return ConstantEmission();
1578 }
1579 
emitScalarConstant(const CodeGenFunction::ConstantEmission & Constant,Expr * E)1580 llvm::Value *CodeGenFunction::emitScalarConstant(
1581     const CodeGenFunction::ConstantEmission &Constant, Expr *E) {
1582   assert(Constant && "not a constant");
1583   if (Constant.isReference())
1584     return EmitLoadOfLValue(Constant.getReferenceLValue(*this, E),
1585                             E->getExprLoc())
1586         .getScalarVal();
1587   return Constant.getValue();
1588 }
1589 
EmitLoadOfScalar(LValue lvalue,SourceLocation Loc)1590 llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue,
1591                                                SourceLocation Loc) {
1592   return EmitLoadOfScalar(lvalue.getAddress(*this), lvalue.isVolatile(),
1593                           lvalue.getType(), Loc, lvalue.getBaseInfo(),
1594                           lvalue.getTBAAInfo(), lvalue.isNontemporal());
1595 }
1596 
hasBooleanRepresentation(QualType Ty)1597 static bool hasBooleanRepresentation(QualType Ty) {
1598   if (Ty->isBooleanType())
1599     return true;
1600 
1601   if (const EnumType *ET = Ty->getAs<EnumType>())
1602     return ET->getDecl()->getIntegerType()->isBooleanType();
1603 
1604   if (const AtomicType *AT = Ty->getAs<AtomicType>())
1605     return hasBooleanRepresentation(AT->getValueType());
1606 
1607   return false;
1608 }
1609 
getRangeForType(CodeGenFunction & CGF,QualType Ty,llvm::APInt & Min,llvm::APInt & End,bool StrictEnums,bool IsBool)1610 static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
1611                             llvm::APInt &Min, llvm::APInt &End,
1612                             bool StrictEnums, bool IsBool) {
1613   const EnumType *ET = Ty->getAs<EnumType>();
1614   bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
1615                                 ET && !ET->getDecl()->isFixed();
1616   if (!IsBool && !IsRegularCPlusPlusEnum)
1617     return false;
1618 
1619   if (IsBool) {
1620     Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);
1621     End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);
1622   } else {
1623     const EnumDecl *ED = ET->getDecl();
1624     ED->getValueRange(End, Min);
1625   }
1626   return true;
1627 }
1628 
getRangeForLoadFromType(QualType Ty)1629 llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
1630   llvm::APInt Min, End;
1631   if (!getRangeForType(*this, Ty, Min, End, CGM.getCodeGenOpts().StrictEnums,
1632                        hasBooleanRepresentation(Ty)))
1633     return nullptr;
1634 
1635   llvm::MDBuilder MDHelper(getLLVMContext());
1636   return MDHelper.createRange(Min, End);
1637 }
1638 
EmitScalarRangeCheck(llvm::Value * Value,QualType Ty,SourceLocation Loc)1639 bool CodeGenFunction::EmitScalarRangeCheck(llvm::Value *Value, QualType Ty,
1640                                            SourceLocation Loc) {
1641   bool HasBoolCheck = SanOpts.has(SanitizerKind::Bool);
1642   bool HasEnumCheck = SanOpts.has(SanitizerKind::Enum);
1643   if (!HasBoolCheck && !HasEnumCheck)
1644     return false;
1645 
1646   bool IsBool = hasBooleanRepresentation(Ty) ||
1647                 NSAPI(CGM.getContext()).isObjCBOOLType(Ty);
1648   bool NeedsBoolCheck = HasBoolCheck && IsBool;
1649   bool NeedsEnumCheck = HasEnumCheck && Ty->getAs<EnumType>();
1650   if (!NeedsBoolCheck && !NeedsEnumCheck)
1651     return false;
1652 
1653   // Single-bit booleans don't need to be checked. Special-case this to avoid
1654   // a bit width mismatch when handling bitfield values. This is handled by
1655   // EmitFromMemory for the non-bitfield case.
1656   if (IsBool &&
1657       cast<llvm::IntegerType>(Value->getType())->getBitWidth() == 1)
1658     return false;
1659 
1660   llvm::APInt Min, End;
1661   if (!getRangeForType(*this, Ty, Min, End, /*StrictEnums=*/true, IsBool))
1662     return true;
1663 
1664   auto &Ctx = getLLVMContext();
1665   SanitizerScope SanScope(this);
1666   llvm::Value *Check;
1667   --End;
1668   if (!Min) {
1669     Check = Builder.CreateICmpULE(Value, llvm::ConstantInt::get(Ctx, End));
1670   } else {
1671     llvm::Value *Upper =
1672         Builder.CreateICmpSLE(Value, llvm::ConstantInt::get(Ctx, End));
1673     llvm::Value *Lower =
1674         Builder.CreateICmpSGE(Value, llvm::ConstantInt::get(Ctx, Min));
1675     Check = Builder.CreateAnd(Upper, Lower);
1676   }
1677   llvm::Constant *StaticArgs[] = {EmitCheckSourceLocation(Loc),
1678                                   EmitCheckTypeDescriptor(Ty)};
1679   SanitizerMask Kind =
1680       NeedsEnumCheck ? SanitizerKind::Enum : SanitizerKind::Bool;
1681   EmitCheck(std::make_pair(Check, Kind), SanitizerHandler::LoadInvalidValue,
1682             StaticArgs, EmitCheckValue(Value));
1683   return true;
1684 }
1685 
EmitLoadOfScalar(Address Addr,bool Volatile,QualType Ty,SourceLocation Loc,LValueBaseInfo BaseInfo,TBAAAccessInfo TBAAInfo,bool isNontemporal)1686 llvm::Value *CodeGenFunction::EmitLoadOfScalar(Address Addr, bool Volatile,
1687                                                QualType Ty,
1688                                                SourceLocation Loc,
1689                                                LValueBaseInfo BaseInfo,
1690                                                TBAAAccessInfo TBAAInfo,
1691                                                bool isNontemporal) {
1692   if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1693     if (GV->isThreadLocal())
1694       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV));
1695 
1696   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1697     // Boolean vectors use `iN` as storage type.
1698     if (ClangVecTy->isExtVectorBoolType()) {
1699       llvm::Type *ValTy = ConvertType(Ty);
1700       unsigned ValNumElems =
1701           cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1702       // Load the `iP` storage object (P is the padded vector size).
1703       auto *RawIntV = Builder.CreateLoad(Addr, Volatile, "load_bits");
1704       const auto *RawIntTy = RawIntV->getType();
1705       assert(RawIntTy->isIntegerTy() && "compressed iN storage for bitvectors");
1706       // Bitcast iP --> <P x i1>.
1707       auto *PaddedVecTy = llvm::FixedVectorType::get(
1708           Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1709       llvm::Value *V = Builder.CreateBitCast(RawIntV, PaddedVecTy);
1710       // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1711       V = emitBoolVecConversion(V, ValNumElems, "extractvec");
1712 
1713       return EmitFromMemory(V, Ty);
1714     }
1715 
1716     // Handle vectors of size 3 like size 4 for better performance.
1717     const llvm::Type *EltTy = Addr.getElementType();
1718     const auto *VTy = cast<llvm::FixedVectorType>(EltTy);
1719 
1720     if (!CGM.getCodeGenOpts().PreserveVec3Type && VTy->getNumElements() == 3) {
1721 
1722       // Bitcast to vec4 type.
1723       llvm::VectorType *vec4Ty =
1724           llvm::FixedVectorType::get(VTy->getElementType(), 4);
1725       Address Cast = Builder.CreateElementBitCast(Addr, vec4Ty, "castToVec4");
1726       // Now load value.
1727       llvm::Value *V = Builder.CreateLoad(Cast, Volatile, "loadVec4");
1728 
1729       // Shuffle vector to get vec3.
1730       V = Builder.CreateShuffleVector(V, ArrayRef<int>{0, 1, 2}, "extractVec");
1731       return EmitFromMemory(V, Ty);
1732     }
1733   }
1734 
1735   // Atomic operations have to be done on integral types.
1736   LValue AtomicLValue =
1737       LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1738   if (Ty->isAtomicType() || LValueIsSuitableForInlineAtomic(AtomicLValue)) {
1739     return EmitAtomicLoad(AtomicLValue, Loc).getScalarVal();
1740   }
1741 
1742   llvm::LoadInst *Load = Builder.CreateLoad(Addr, Volatile);
1743   if (isNontemporal) {
1744     llvm::MDNode *Node = llvm::MDNode::get(
1745         Load->getContext(), llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1746     Load->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
1747   }
1748 
1749   CGM.DecorateInstructionWithTBAA(Load, TBAAInfo);
1750 
1751   if (EmitScalarRangeCheck(Load, Ty, Loc)) {
1752     // In order to prevent the optimizer from throwing away the check, don't
1753     // attach range metadata to the load.
1754   } else if (CGM.getCodeGenOpts().OptimizationLevel > 0)
1755     if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty)) {
1756       Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
1757       Load->setMetadata(llvm::LLVMContext::MD_noundef,
1758                         llvm::MDNode::get(getLLVMContext(), std::nullopt));
1759     }
1760 
1761   return EmitFromMemory(Load, Ty);
1762 }
1763 
EmitToMemory(llvm::Value * Value,QualType Ty)1764 llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
1765   // Bool has a different representation in memory than in registers.
1766   if (hasBooleanRepresentation(Ty)) {
1767     // This should really always be an i1, but sometimes it's already
1768     // an i8, and it's awkward to track those cases down.
1769     if (Value->getType()->isIntegerTy(1))
1770       return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool");
1771     assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1772            "wrong value rep of bool");
1773   }
1774 
1775   return Value;
1776 }
1777 
EmitFromMemory(llvm::Value * Value,QualType Ty)1778 llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
1779   // Bool has a different representation in memory than in registers.
1780   if (hasBooleanRepresentation(Ty)) {
1781     assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1782            "wrong value rep of bool");
1783     return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
1784   }
1785   if (Ty->isExtVectorBoolType()) {
1786     const auto *RawIntTy = Value->getType();
1787     // Bitcast iP --> <P x i1>.
1788     auto *PaddedVecTy = llvm::FixedVectorType::get(
1789         Builder.getInt1Ty(), RawIntTy->getPrimitiveSizeInBits());
1790     auto *V = Builder.CreateBitCast(Value, PaddedVecTy);
1791     // Shuffle <P x i1> --> <N x i1> (N is the actual bit size).
1792     llvm::Type *ValTy = ConvertType(Ty);
1793     unsigned ValNumElems = cast<llvm::FixedVectorType>(ValTy)->getNumElements();
1794     return emitBoolVecConversion(V, ValNumElems, "extractvec");
1795   }
1796 
1797   return Value;
1798 }
1799 
1800 // Convert the pointer of \p Addr to a pointer to a vector (the value type of
1801 // MatrixType), if it points to a array (the memory type of MatrixType).
MaybeConvertMatrixAddress(Address Addr,CodeGenFunction & CGF,bool IsVector=true)1802 static Address MaybeConvertMatrixAddress(Address Addr, CodeGenFunction &CGF,
1803                                          bool IsVector = true) {
1804   auto *ArrayTy = dyn_cast<llvm::ArrayType>(Addr.getElementType());
1805   if (ArrayTy && IsVector) {
1806     auto *VectorTy = llvm::FixedVectorType::get(ArrayTy->getElementType(),
1807                                                 ArrayTy->getNumElements());
1808 
1809     return Address(CGF.Builder.CreateElementBitCast(Addr, VectorTy));
1810   }
1811   auto *VectorTy = dyn_cast<llvm::VectorType>(Addr.getElementType());
1812   if (VectorTy && !IsVector) {
1813     auto *ArrayTy = llvm::ArrayType::get(
1814         VectorTy->getElementType(),
1815         cast<llvm::FixedVectorType>(VectorTy)->getNumElements());
1816 
1817     return Address(CGF.Builder.CreateElementBitCast(Addr, ArrayTy));
1818   }
1819 
1820   return Addr;
1821 }
1822 
1823 // Emit a store of a matrix LValue. This may require casting the original
1824 // pointer to memory address (ArrayType) to a pointer to the value type
1825 // (VectorType).
EmitStoreOfMatrixScalar(llvm::Value * value,LValue lvalue,bool isInit,CodeGenFunction & CGF)1826 static void EmitStoreOfMatrixScalar(llvm::Value *value, LValue lvalue,
1827                                     bool isInit, CodeGenFunction &CGF) {
1828   Address Addr = MaybeConvertMatrixAddress(lvalue.getAddress(CGF), CGF,
1829                                            value->getType()->isVectorTy());
1830   CGF.EmitStoreOfScalar(value, Addr, lvalue.isVolatile(), lvalue.getType(),
1831                         lvalue.getBaseInfo(), lvalue.getTBAAInfo(), isInit,
1832                         lvalue.isNontemporal());
1833 }
1834 
EmitStoreOfScalar(llvm::Value * Value,Address Addr,bool Volatile,QualType Ty,LValueBaseInfo BaseInfo,TBAAAccessInfo TBAAInfo,bool isInit,bool isNontemporal)1835 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, Address Addr,
1836                                         bool Volatile, QualType Ty,
1837                                         LValueBaseInfo BaseInfo,
1838                                         TBAAAccessInfo TBAAInfo,
1839                                         bool isInit, bool isNontemporal) {
1840   if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr.getPointer()))
1841     if (GV->isThreadLocal())
1842       Addr = Addr.withPointer(Builder.CreateThreadLocalAddress(GV));
1843 
1844   llvm::Type *SrcTy = Value->getType();
1845   if (const auto *ClangVecTy = Ty->getAs<VectorType>()) {
1846     auto *VecTy = dyn_cast<llvm::FixedVectorType>(SrcTy);
1847     if (VecTy && ClangVecTy->isExtVectorBoolType()) {
1848       auto *MemIntTy = cast<llvm::IntegerType>(Addr.getElementType());
1849       // Expand to the memory bit width.
1850       unsigned MemNumElems = MemIntTy->getPrimitiveSizeInBits();
1851       // <N x i1> --> <P x i1>.
1852       Value = emitBoolVecConversion(Value, MemNumElems, "insertvec");
1853       // <P x i1> --> iP.
1854       Value = Builder.CreateBitCast(Value, MemIntTy);
1855     } else if (!CGM.getCodeGenOpts().PreserveVec3Type) {
1856       // Handle vec3 special.
1857       if (VecTy && cast<llvm::FixedVectorType>(VecTy)->getNumElements() == 3) {
1858         // Our source is a vec3, do a shuffle vector to make it a vec4.
1859         Value = Builder.CreateShuffleVector(Value, ArrayRef<int>{0, 1, 2, -1},
1860                                             "extractVec");
1861         SrcTy = llvm::FixedVectorType::get(VecTy->getElementType(), 4);
1862       }
1863       if (Addr.getElementType() != SrcTy) {
1864         Addr = Builder.CreateElementBitCast(Addr, SrcTy, "storetmp");
1865       }
1866     }
1867   }
1868 
1869   Value = EmitToMemory(Value, Ty);
1870 
1871   LValue AtomicLValue =
1872       LValue::MakeAddr(Addr, Ty, getContext(), BaseInfo, TBAAInfo);
1873   if (Ty->isAtomicType() ||
1874       (!isInit && LValueIsSuitableForInlineAtomic(AtomicLValue))) {
1875     EmitAtomicStore(RValue::get(Value), AtomicLValue, isInit);
1876     return;
1877   }
1878 
1879   llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
1880   if (isNontemporal) {
1881     llvm::MDNode *Node =
1882         llvm::MDNode::get(Store->getContext(),
1883                           llvm::ConstantAsMetadata::get(Builder.getInt32(1)));
1884     Store->setMetadata(CGM.getModule().getMDKindID("nontemporal"), Node);
1885   }
1886 
1887   CGM.DecorateInstructionWithTBAA(Store, TBAAInfo);
1888 }
1889 
EmitStoreOfScalar(llvm::Value * value,LValue lvalue,bool isInit)1890 void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
1891                                         bool isInit) {
1892   if (lvalue.getType()->isConstantMatrixType()) {
1893     EmitStoreOfMatrixScalar(value, lvalue, isInit, *this);
1894     return;
1895   }
1896 
1897   EmitStoreOfScalar(value, lvalue.getAddress(*this), lvalue.isVolatile(),
1898                     lvalue.getType(), lvalue.getBaseInfo(),
1899                     lvalue.getTBAAInfo(), isInit, lvalue.isNontemporal());
1900 }
1901 
1902 // Emit a load of a LValue of matrix type. This may require casting the pointer
1903 // to memory address (ArrayType) to a pointer to the value type (VectorType).
EmitLoadOfMatrixLValue(LValue LV,SourceLocation Loc,CodeGenFunction & CGF)1904 static RValue EmitLoadOfMatrixLValue(LValue LV, SourceLocation Loc,
1905                                      CodeGenFunction &CGF) {
1906   assert(LV.getType()->isConstantMatrixType());
1907   Address Addr = MaybeConvertMatrixAddress(LV.getAddress(CGF), CGF);
1908   LV.setAddress(Addr);
1909   return RValue::get(CGF.EmitLoadOfScalar(LV, Loc));
1910 }
1911 
1912 /// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
1913 /// method emits the address of the lvalue, then loads the result as an rvalue,
1914 /// returning the rvalue.
EmitLoadOfLValue(LValue LV,SourceLocation Loc)1915 RValue CodeGenFunction::EmitLoadOfLValue(LValue LV, SourceLocation Loc) {
1916   if (LV.isObjCWeak()) {
1917     // load of a __weak object.
1918     Address AddrWeakObj = LV.getAddress(*this);
1919     return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
1920                                                              AddrWeakObj));
1921   }
1922   if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
1923     // In MRC mode, we do a load+autorelease.
1924     if (!getLangOpts().ObjCAutoRefCount) {
1925       return RValue::get(EmitARCLoadWeak(LV.getAddress(*this)));
1926     }
1927 
1928     // In ARC mode, we load retained and then consume the value.
1929     llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress(*this));
1930     Object = EmitObjCConsumeObject(LV.getType(), Object);
1931     return RValue::get(Object);
1932   }
1933 
1934   if (LV.isSimple()) {
1935     assert(!LV.getType()->isFunctionType());
1936 
1937     if (LV.getType()->isConstantMatrixType())
1938       return EmitLoadOfMatrixLValue(LV, Loc, *this);
1939 
1940     // Everything needs a load.
1941     return RValue::get(EmitLoadOfScalar(LV, Loc));
1942   }
1943 
1944   if (LV.isVectorElt()) {
1945     llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddress(),
1946                                               LV.isVolatileQualified());
1947     return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
1948                                                     "vecext"));
1949   }
1950 
1951   // If this is a reference to a subset of the elements of a vector, either
1952   // shuffle the input or extract/insert them as appropriate.
1953   if (LV.isExtVectorElt()) {
1954     return EmitLoadOfExtVectorElementLValue(LV);
1955   }
1956 
1957   // Global Register variables always invoke intrinsics
1958   if (LV.isGlobalReg())
1959     return EmitLoadOfGlobalRegLValue(LV);
1960 
1961   if (LV.isMatrixElt()) {
1962     llvm::Value *Idx = LV.getMatrixIdx();
1963     if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
1964       const auto *const MatTy = LV.getType()->castAs<ConstantMatrixType>();
1965       llvm::MatrixBuilder MB(Builder);
1966       MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
1967     }
1968     llvm::LoadInst *Load =
1969         Builder.CreateLoad(LV.getMatrixAddress(), LV.isVolatileQualified());
1970     return RValue::get(Builder.CreateExtractElement(Load, Idx, "matrixext"));
1971   }
1972 
1973   assert(LV.isBitField() && "Unknown LValue type!");
1974   return EmitLoadOfBitfieldLValue(LV, Loc);
1975 }
1976 
EmitLoadOfBitfieldLValue(LValue LV,SourceLocation Loc)1977 RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV,
1978                                                  SourceLocation Loc) {
1979   const CGBitFieldInfo &Info = LV.getBitFieldInfo();
1980 
1981   // Get the output type.
1982   llvm::Type *ResLTy = ConvertType(LV.getType());
1983 
1984   Address Ptr = LV.getBitFieldAddress();
1985   llvm::Value *Val =
1986       Builder.CreateLoad(Ptr, LV.isVolatileQualified(), "bf.load");
1987 
1988   bool UseVolatile = LV.isVolatileQualified() &&
1989                      Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
1990   const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
1991   const unsigned StorageSize =
1992       UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
1993   if (Info.IsSigned) {
1994     assert(static_cast<unsigned>(Offset + Info.Size) <= StorageSize);
1995     unsigned HighBits = StorageSize - Offset - Info.Size;
1996     if (HighBits)
1997       Val = Builder.CreateShl(Val, HighBits, "bf.shl");
1998     if (Offset + HighBits)
1999       Val = Builder.CreateAShr(Val, Offset + HighBits, "bf.ashr");
2000   } else {
2001     if (Offset)
2002       Val = Builder.CreateLShr(Val, Offset, "bf.lshr");
2003     if (static_cast<unsigned>(Offset) + Info.Size < StorageSize)
2004       Val = Builder.CreateAnd(
2005           Val, llvm::APInt::getLowBitsSet(StorageSize, Info.Size), "bf.clear");
2006   }
2007   Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast");
2008   EmitScalarRangeCheck(Val, LV.getType(), Loc);
2009   return RValue::get(Val);
2010 }
2011 
2012 // If this is a reference to a subset of the elements of a vector, create an
2013 // appropriate shufflevector.
EmitLoadOfExtVectorElementLValue(LValue LV)2014 RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
2015   llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddress(),
2016                                         LV.isVolatileQualified());
2017 
2018   const llvm::Constant *Elts = LV.getExtVectorElts();
2019 
2020   // If the result of the expression is a non-vector type, we must be extracting
2021   // a single element.  Just codegen as an extractelement.
2022   const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
2023   if (!ExprVT) {
2024     unsigned InIdx = getAccessedFieldNo(0, Elts);
2025     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2026     return RValue::get(Builder.CreateExtractElement(Vec, Elt));
2027   }
2028 
2029   // Always use shuffle vector to try to retain the original program structure
2030   unsigned NumResultElts = ExprVT->getNumElements();
2031 
2032   SmallVector<int, 4> Mask;
2033   for (unsigned i = 0; i != NumResultElts; ++i)
2034     Mask.push_back(getAccessedFieldNo(i, Elts));
2035 
2036   Vec = Builder.CreateShuffleVector(Vec, Mask);
2037   return RValue::get(Vec);
2038 }
2039 
2040 /// Generates lvalue for partial ext_vector access.
EmitExtVectorElementLValue(LValue LV)2041 Address CodeGenFunction::EmitExtVectorElementLValue(LValue LV) {
2042   Address VectorAddress = LV.getExtVectorAddress();
2043   QualType EQT = LV.getType()->castAs<VectorType>()->getElementType();
2044   llvm::Type *VectorElementTy = CGM.getTypes().ConvertType(EQT);
2045 
2046   Address CastToPointerElement =
2047     Builder.CreateElementBitCast(VectorAddress, VectorElementTy,
2048                                  "conv.ptr.element");
2049 
2050   const llvm::Constant *Elts = LV.getExtVectorElts();
2051   unsigned ix = getAccessedFieldNo(0, Elts);
2052 
2053   Address VectorBasePtrPlusIx =
2054     Builder.CreateConstInBoundsGEP(CastToPointerElement, ix,
2055                                    "vector.elt");
2056 
2057   return VectorBasePtrPlusIx;
2058 }
2059 
2060 /// Load of global gamed gegisters are always calls to intrinsics.
EmitLoadOfGlobalRegLValue(LValue LV)2061 RValue CodeGenFunction::EmitLoadOfGlobalRegLValue(LValue LV) {
2062   assert((LV.getType()->isIntegerType() || LV.getType()->isPointerType()) &&
2063          "Bad type for register variable");
2064   llvm::MDNode *RegName = cast<llvm::MDNode>(
2065       cast<llvm::MetadataAsValue>(LV.getGlobalReg())->getMetadata());
2066 
2067   // We accept integer and pointer types only
2068   llvm::Type *OrigTy = CGM.getTypes().ConvertType(LV.getType());
2069   llvm::Type *Ty = OrigTy;
2070   if (OrigTy->isPointerTy())
2071     Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2072   llvm::Type *Types[] = { Ty };
2073 
2074   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::read_register, Types);
2075   llvm::Value *Call = Builder.CreateCall(
2076       F, llvm::MetadataAsValue::get(Ty->getContext(), RegName));
2077   if (OrigTy->isPointerTy())
2078     Call = Builder.CreateIntToPtr(Call, OrigTy);
2079   return RValue::get(Call);
2080 }
2081 
2082 /// EmitStoreThroughLValue - Store the specified rvalue into the specified
2083 /// lvalue, where both are guaranteed to the have the same type, and that type
2084 /// is 'Ty'.
EmitStoreThroughLValue(RValue Src,LValue Dst,bool isInit)2085 void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst,
2086                                              bool isInit) {
2087   if (!Dst.isSimple()) {
2088     if (Dst.isVectorElt()) {
2089       // Read/modify/write the vector, inserting the new element.
2090       llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddress(),
2091                                             Dst.isVolatileQualified());
2092       auto *IRStoreTy = dyn_cast<llvm::IntegerType>(Vec->getType());
2093       if (IRStoreTy) {
2094         auto *IRVecTy = llvm::FixedVectorType::get(
2095             Builder.getInt1Ty(), IRStoreTy->getPrimitiveSizeInBits());
2096         Vec = Builder.CreateBitCast(Vec, IRVecTy);
2097         // iN --> <N x i1>.
2098       }
2099       Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
2100                                         Dst.getVectorIdx(), "vecins");
2101       if (IRStoreTy) {
2102         // <N x i1> --> <iN>.
2103         Vec = Builder.CreateBitCast(Vec, IRStoreTy);
2104       }
2105       Builder.CreateStore(Vec, Dst.getVectorAddress(),
2106                           Dst.isVolatileQualified());
2107       return;
2108     }
2109 
2110     // If this is an update of extended vector elements, insert them as
2111     // appropriate.
2112     if (Dst.isExtVectorElt())
2113       return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
2114 
2115     if (Dst.isGlobalReg())
2116       return EmitStoreThroughGlobalRegLValue(Src, Dst);
2117 
2118     if (Dst.isMatrixElt()) {
2119       llvm::Value *Idx = Dst.getMatrixIdx();
2120       if (CGM.getCodeGenOpts().OptimizationLevel > 0) {
2121         const auto *const MatTy = Dst.getType()->castAs<ConstantMatrixType>();
2122         llvm::MatrixBuilder MB(Builder);
2123         MB.CreateIndexAssumption(Idx, MatTy->getNumElementsFlattened());
2124       }
2125       llvm::Instruction *Load = Builder.CreateLoad(Dst.getMatrixAddress());
2126       llvm::Value *Vec =
2127           Builder.CreateInsertElement(Load, Src.getScalarVal(), Idx, "matins");
2128       Builder.CreateStore(Vec, Dst.getMatrixAddress(),
2129                           Dst.isVolatileQualified());
2130       return;
2131     }
2132 
2133     assert(Dst.isBitField() && "Unknown LValue type");
2134     return EmitStoreThroughBitfieldLValue(Src, Dst);
2135   }
2136 
2137   // There's special magic for assigning into an ARC-qualified l-value.
2138   if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
2139     switch (Lifetime) {
2140     case Qualifiers::OCL_None:
2141       llvm_unreachable("present but none");
2142 
2143     case Qualifiers::OCL_ExplicitNone:
2144       // nothing special
2145       break;
2146 
2147     case Qualifiers::OCL_Strong:
2148       if (isInit) {
2149         Src = RValue::get(EmitARCRetain(Dst.getType(), Src.getScalarVal()));
2150         break;
2151       }
2152       EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
2153       return;
2154 
2155     case Qualifiers::OCL_Weak:
2156       if (isInit)
2157         // Initialize and then skip the primitive store.
2158         EmitARCInitWeak(Dst.getAddress(*this), Src.getScalarVal());
2159       else
2160         EmitARCStoreWeak(Dst.getAddress(*this), Src.getScalarVal(),
2161                          /*ignore*/ true);
2162       return;
2163 
2164     case Qualifiers::OCL_Autoreleasing:
2165       Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),
2166                                                      Src.getScalarVal()));
2167       // fall into the normal path
2168       break;
2169     }
2170   }
2171 
2172   if (Dst.isObjCWeak() && !Dst.isNonGC()) {
2173     // load of a __weak object.
2174     Address LvalueDst = Dst.getAddress(*this);
2175     llvm::Value *src = Src.getScalarVal();
2176      CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
2177     return;
2178   }
2179 
2180   if (Dst.isObjCStrong() && !Dst.isNonGC()) {
2181     // load of a __strong object.
2182     Address LvalueDst = Dst.getAddress(*this);
2183     llvm::Value *src = Src.getScalarVal();
2184     if (Dst.isObjCIvar()) {
2185       assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
2186       llvm::Type *ResultType = IntPtrTy;
2187       Address dst = EmitPointerWithAlignment(Dst.getBaseIvarExp());
2188       llvm::Value *RHS = dst.getPointer();
2189       RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
2190       llvm::Value *LHS =
2191         Builder.CreatePtrToInt(LvalueDst.getPointer(), ResultType,
2192                                "sub.ptr.lhs.cast");
2193       llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
2194       CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
2195                                               BytesBetween);
2196     } else if (Dst.isGlobalObjCRef()) {
2197       CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
2198                                                 Dst.isThreadLocalRef());
2199     }
2200     else
2201       CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
2202     return;
2203   }
2204 
2205   assert(Src.isScalar() && "Can't emit an agg store with this method");
2206   EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
2207 }
2208 
EmitStoreThroughBitfieldLValue(RValue Src,LValue Dst,llvm::Value ** Result)2209 void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
2210                                                      llvm::Value **Result) {
2211   const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
2212   llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
2213   Address Ptr = Dst.getBitFieldAddress();
2214 
2215   // Get the source value, truncated to the width of the bit-field.
2216   llvm::Value *SrcVal = Src.getScalarVal();
2217 
2218   // Cast the source to the storage type and shift it into place.
2219   SrcVal = Builder.CreateIntCast(SrcVal, Ptr.getElementType(),
2220                                  /*isSigned=*/false);
2221   llvm::Value *MaskedVal = SrcVal;
2222 
2223   const bool UseVolatile =
2224       CGM.getCodeGenOpts().AAPCSBitfieldWidth && Dst.isVolatileQualified() &&
2225       Info.VolatileStorageSize != 0 && isAAPCS(CGM.getTarget());
2226   const unsigned StorageSize =
2227       UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
2228   const unsigned Offset = UseVolatile ? Info.VolatileOffset : Info.Offset;
2229   // See if there are other bits in the bitfield's storage we'll need to load
2230   // and mask together with source before storing.
2231   if (StorageSize != Info.Size) {
2232     assert(StorageSize > Info.Size && "Invalid bitfield size.");
2233     llvm::Value *Val =
2234         Builder.CreateLoad(Ptr, Dst.isVolatileQualified(), "bf.load");
2235 
2236     // Mask the source value as needed.
2237     if (!hasBooleanRepresentation(Dst.getType()))
2238       SrcVal = Builder.CreateAnd(
2239           SrcVal, llvm::APInt::getLowBitsSet(StorageSize, Info.Size),
2240           "bf.value");
2241     MaskedVal = SrcVal;
2242     if (Offset)
2243       SrcVal = Builder.CreateShl(SrcVal, Offset, "bf.shl");
2244 
2245     // Mask out the original value.
2246     Val = Builder.CreateAnd(
2247         Val, ~llvm::APInt::getBitsSet(StorageSize, Offset, Offset + Info.Size),
2248         "bf.clear");
2249 
2250     // Or together the unchanged values and the source value.
2251     SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set");
2252   } else {
2253     assert(Offset == 0);
2254     // According to the AACPS:
2255     // When a volatile bit-field is written, and its container does not overlap
2256     // with any non-bit-field member, its container must be read exactly once
2257     // and written exactly once using the access width appropriate to the type
2258     // of the container. The two accesses are not atomic.
2259     if (Dst.isVolatileQualified() && isAAPCS(CGM.getTarget()) &&
2260         CGM.getCodeGenOpts().ForceAAPCSBitfieldLoad)
2261       Builder.CreateLoad(Ptr, true, "bf.load");
2262   }
2263 
2264   // Write the new value back out.
2265   Builder.CreateStore(SrcVal, Ptr, Dst.isVolatileQualified());
2266 
2267   // Return the new value of the bit-field, if requested.
2268   if (Result) {
2269     llvm::Value *ResultVal = MaskedVal;
2270 
2271     // Sign extend the value if needed.
2272     if (Info.IsSigned) {
2273       assert(Info.Size <= StorageSize);
2274       unsigned HighBits = StorageSize - Info.Size;
2275       if (HighBits) {
2276         ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl");
2277         ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr");
2278       }
2279     }
2280 
2281     ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned,
2282                                       "bf.result.cast");
2283     *Result = EmitFromMemory(ResultVal, Dst.getType());
2284   }
2285 }
2286 
EmitStoreThroughExtVectorComponentLValue(RValue Src,LValue Dst)2287 void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
2288                                                                LValue Dst) {
2289   // This access turns into a read/modify/write of the vector.  Load the input
2290   // value now.
2291   llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddress(),
2292                                         Dst.isVolatileQualified());
2293   const llvm::Constant *Elts = Dst.getExtVectorElts();
2294 
2295   llvm::Value *SrcVal = Src.getScalarVal();
2296 
2297   if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
2298     unsigned NumSrcElts = VTy->getNumElements();
2299     unsigned NumDstElts =
2300         cast<llvm::FixedVectorType>(Vec->getType())->getNumElements();
2301     if (NumDstElts == NumSrcElts) {
2302       // Use shuffle vector is the src and destination are the same number of
2303       // elements and restore the vector mask since it is on the side it will be
2304       // stored.
2305       SmallVector<int, 4> Mask(NumDstElts);
2306       for (unsigned i = 0; i != NumSrcElts; ++i)
2307         Mask[getAccessedFieldNo(i, Elts)] = i;
2308 
2309       Vec = Builder.CreateShuffleVector(SrcVal, Mask);
2310     } else if (NumDstElts > NumSrcElts) {
2311       // Extended the source vector to the same length and then shuffle it
2312       // into the destination.
2313       // FIXME: since we're shuffling with undef, can we just use the indices
2314       //        into that?  This could be simpler.
2315       SmallVector<int, 4> ExtMask;
2316       for (unsigned i = 0; i != NumSrcElts; ++i)
2317         ExtMask.push_back(i);
2318       ExtMask.resize(NumDstElts, -1);
2319       llvm::Value *ExtSrcVal = Builder.CreateShuffleVector(SrcVal, ExtMask);
2320       // build identity
2321       SmallVector<int, 4> Mask;
2322       for (unsigned i = 0; i != NumDstElts; ++i)
2323         Mask.push_back(i);
2324 
2325       // When the vector size is odd and .odd or .hi is used, the last element
2326       // of the Elts constant array will be one past the size of the vector.
2327       // Ignore the last element here, if it is greater than the mask size.
2328       if (getAccessedFieldNo(NumSrcElts - 1, Elts) == Mask.size())
2329         NumSrcElts--;
2330 
2331       // modify when what gets shuffled in
2332       for (unsigned i = 0; i != NumSrcElts; ++i)
2333         Mask[getAccessedFieldNo(i, Elts)] = i + NumDstElts;
2334       Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, Mask);
2335     } else {
2336       // We should never shorten the vector
2337       llvm_unreachable("unexpected shorten vector length");
2338     }
2339   } else {
2340     // If the Src is a scalar (not a vector) it must be updating one element.
2341     unsigned InIdx = getAccessedFieldNo(0, Elts);
2342     llvm::Value *Elt = llvm::ConstantInt::get(SizeTy, InIdx);
2343     Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
2344   }
2345 
2346   Builder.CreateStore(Vec, Dst.getExtVectorAddress(),
2347                       Dst.isVolatileQualified());
2348 }
2349 
2350 /// Store of global named registers are always calls to intrinsics.
EmitStoreThroughGlobalRegLValue(RValue Src,LValue Dst)2351 void CodeGenFunction::EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst) {
2352   assert((Dst.getType()->isIntegerType() || Dst.getType()->isPointerType()) &&
2353          "Bad type for register variable");
2354   llvm::MDNode *RegName = cast<llvm::MDNode>(
2355       cast<llvm::MetadataAsValue>(Dst.getGlobalReg())->getMetadata());
2356   assert(RegName && "Register LValue is not metadata");
2357 
2358   // We accept integer and pointer types only
2359   llvm::Type *OrigTy = CGM.getTypes().ConvertType(Dst.getType());
2360   llvm::Type *Ty = OrigTy;
2361   if (OrigTy->isPointerTy())
2362     Ty = CGM.getTypes().getDataLayout().getIntPtrType(OrigTy);
2363   llvm::Type *Types[] = { Ty };
2364 
2365   llvm::Function *F = CGM.getIntrinsic(llvm::Intrinsic::write_register, Types);
2366   llvm::Value *Value = Src.getScalarVal();
2367   if (OrigTy->isPointerTy())
2368     Value = Builder.CreatePtrToInt(Value, Ty);
2369   Builder.CreateCall(
2370       F, {llvm::MetadataAsValue::get(Ty->getContext(), RegName), Value});
2371 }
2372 
2373 // setObjCGCLValueClass - sets class of the lvalue for the purpose of
2374 // generating write-barries API. It is currently a global, ivar,
2375 // or neither.
setObjCGCLValueClass(const ASTContext & Ctx,const Expr * E,LValue & LV,bool IsMemberAccess=false)2376 static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
2377                                  LValue &LV,
2378                                  bool IsMemberAccess=false) {
2379   if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
2380     return;
2381 
2382   if (isa<ObjCIvarRefExpr>(E)) {
2383     QualType ExpTy = E->getType();
2384     if (IsMemberAccess && ExpTy->isPointerType()) {
2385       // If ivar is a structure pointer, assigning to field of
2386       // this struct follows gcc's behavior and makes it a non-ivar
2387       // writer-barrier conservatively.
2388       ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2389       if (ExpTy->isRecordType()) {
2390         LV.setObjCIvar(false);
2391         return;
2392       }
2393     }
2394     LV.setObjCIvar(true);
2395     auto *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr *>(E));
2396     LV.setBaseIvarExp(Exp->getBase());
2397     LV.setObjCArray(E->getType()->isArrayType());
2398     return;
2399   }
2400 
2401   if (const auto *Exp = dyn_cast<DeclRefExpr>(E)) {
2402     if (const auto *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
2403       if (VD->hasGlobalStorage()) {
2404         LV.setGlobalObjCRef(true);
2405         LV.setThreadLocalRef(VD->getTLSKind() != VarDecl::TLS_None);
2406       }
2407     }
2408     LV.setObjCArray(E->getType()->isArrayType());
2409     return;
2410   }
2411 
2412   if (const auto *Exp = dyn_cast<UnaryOperator>(E)) {
2413     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2414     return;
2415   }
2416 
2417   if (const auto *Exp = dyn_cast<ParenExpr>(E)) {
2418     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2419     if (LV.isObjCIvar()) {
2420       // If cast is to a structure pointer, follow gcc's behavior and make it
2421       // a non-ivar write-barrier.
2422       QualType ExpTy = E->getType();
2423       if (ExpTy->isPointerType())
2424         ExpTy = ExpTy->castAs<PointerType>()->getPointeeType();
2425       if (ExpTy->isRecordType())
2426         LV.setObjCIvar(false);
2427     }
2428     return;
2429   }
2430 
2431   if (const auto *Exp = dyn_cast<GenericSelectionExpr>(E)) {
2432     setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
2433     return;
2434   }
2435 
2436   if (const auto *Exp = dyn_cast<ImplicitCastExpr>(E)) {
2437     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2438     return;
2439   }
2440 
2441   if (const auto *Exp = dyn_cast<CStyleCastExpr>(E)) {
2442     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2443     return;
2444   }
2445 
2446   if (const auto *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
2447     setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
2448     return;
2449   }
2450 
2451   if (const auto *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
2452     setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
2453     if (LV.isObjCIvar() && !LV.isObjCArray())
2454       // Using array syntax to assigning to what an ivar points to is not
2455       // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
2456       LV.setObjCIvar(false);
2457     else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
2458       // Using array syntax to assigning to what global points to is not
2459       // same as assigning to the global itself. {id *G;} G[i] = 0;
2460       LV.setGlobalObjCRef(false);
2461     return;
2462   }
2463 
2464   if (const auto *Exp = dyn_cast<MemberExpr>(E)) {
2465     setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
2466     // We don't know if member is an 'ivar', but this flag is looked at
2467     // only in the context of LV.isObjCIvar().
2468     LV.setObjCArray(E->getType()->isArrayType());
2469     return;
2470   }
2471 }
2472 
2473 static llvm::Value *
EmitBitCastOfLValueToProperType(CodeGenFunction & CGF,llvm::Value * V,llvm::Type * IRType,StringRef Name=StringRef ())2474 EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
2475                                 llvm::Value *V, llvm::Type *IRType,
2476                                 StringRef Name = StringRef()) {
2477   unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
2478   return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
2479 }
2480 
EmitThreadPrivateVarDeclLValue(CodeGenFunction & CGF,const VarDecl * VD,QualType T,Address Addr,llvm::Type * RealVarTy,SourceLocation Loc)2481 static LValue EmitThreadPrivateVarDeclLValue(
2482     CodeGenFunction &CGF, const VarDecl *VD, QualType T, Address Addr,
2483     llvm::Type *RealVarTy, SourceLocation Loc) {
2484   if (CGF.CGM.getLangOpts().OpenMPIRBuilder)
2485     Addr = CodeGenFunction::OMPBuilderCBHelpers::getAddrOfThreadPrivate(
2486         CGF, VD, Addr, Loc);
2487   else
2488     Addr =
2489         CGF.CGM.getOpenMPRuntime().getAddrOfThreadPrivate(CGF, VD, Addr, Loc);
2490 
2491   Addr = CGF.Builder.CreateElementBitCast(Addr, RealVarTy);
2492   return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2493 }
2494 
emitDeclTargetVarDeclLValue(CodeGenFunction & CGF,const VarDecl * VD,QualType T)2495 static Address emitDeclTargetVarDeclLValue(CodeGenFunction &CGF,
2496                                            const VarDecl *VD, QualType T) {
2497   std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
2498       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
2499   // Return an invalid address if variable is MT_To (or MT_Enter starting with
2500   // OpenMP 5.2) and unified memory is not enabled. For all other cases: MT_Link
2501   // and MT_To (or MT_Enter) with unified memory, return a valid address.
2502   if (!Res || ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2503                 *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2504                !CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()))
2505     return Address::invalid();
2506   assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
2507           ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
2508             *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
2509            CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) &&
2510          "Expected link clause OR to clause with unified memory enabled.");
2511   QualType PtrTy = CGF.getContext().getPointerType(VD->getType());
2512   Address Addr = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
2513   return CGF.EmitLoadOfPointer(Addr, PtrTy->castAs<PointerType>());
2514 }
2515 
2516 Address
EmitLoadOfReference(LValue RefLVal,LValueBaseInfo * PointeeBaseInfo,TBAAAccessInfo * PointeeTBAAInfo)2517 CodeGenFunction::EmitLoadOfReference(LValue RefLVal,
2518                                      LValueBaseInfo *PointeeBaseInfo,
2519                                      TBAAAccessInfo *PointeeTBAAInfo) {
2520   llvm::LoadInst *Load =
2521       Builder.CreateLoad(RefLVal.getAddress(*this), RefLVal.isVolatile());
2522   CGM.DecorateInstructionWithTBAA(Load, RefLVal.getTBAAInfo());
2523 
2524   QualType PointeeType = RefLVal.getType()->getPointeeType();
2525   CharUnits Align = CGM.getNaturalTypeAlignment(
2526       PointeeType, PointeeBaseInfo, PointeeTBAAInfo,
2527       /* forPointeeType= */ true);
2528   return Address(Load, ConvertTypeForMem(PointeeType), Align);
2529 }
2530 
EmitLoadOfReferenceLValue(LValue RefLVal)2531 LValue CodeGenFunction::EmitLoadOfReferenceLValue(LValue RefLVal) {
2532   LValueBaseInfo PointeeBaseInfo;
2533   TBAAAccessInfo PointeeTBAAInfo;
2534   Address PointeeAddr = EmitLoadOfReference(RefLVal, &PointeeBaseInfo,
2535                                             &PointeeTBAAInfo);
2536   return MakeAddrLValue(PointeeAddr, RefLVal.getType()->getPointeeType(),
2537                         PointeeBaseInfo, PointeeTBAAInfo);
2538 }
2539 
EmitLoadOfPointer(Address Ptr,const PointerType * PtrTy,LValueBaseInfo * BaseInfo,TBAAAccessInfo * TBAAInfo)2540 Address CodeGenFunction::EmitLoadOfPointer(Address Ptr,
2541                                            const PointerType *PtrTy,
2542                                            LValueBaseInfo *BaseInfo,
2543                                            TBAAAccessInfo *TBAAInfo) {
2544   llvm::Value *Addr = Builder.CreateLoad(Ptr);
2545   return Address(Addr, ConvertTypeForMem(PtrTy->getPointeeType()),
2546                  CGM.getNaturalTypeAlignment(PtrTy->getPointeeType(), BaseInfo,
2547                                              TBAAInfo,
2548                                              /*forPointeeType=*/true));
2549 }
2550 
EmitLoadOfPointerLValue(Address PtrAddr,const PointerType * PtrTy)2551 LValue CodeGenFunction::EmitLoadOfPointerLValue(Address PtrAddr,
2552                                                 const PointerType *PtrTy) {
2553   LValueBaseInfo BaseInfo;
2554   TBAAAccessInfo TBAAInfo;
2555   Address Addr = EmitLoadOfPointer(PtrAddr, PtrTy, &BaseInfo, &TBAAInfo);
2556   return MakeAddrLValue(Addr, PtrTy->getPointeeType(), BaseInfo, TBAAInfo);
2557 }
2558 
EmitGlobalVarDeclLValue(CodeGenFunction & CGF,const Expr * E,const VarDecl * VD)2559 static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
2560                                       const Expr *E, const VarDecl *VD) {
2561   QualType T = E->getType();
2562 
2563   // If it's thread_local, emit a call to its wrapper function instead.
2564   if (VD->getTLSKind() == VarDecl::TLS_Dynamic &&
2565       CGF.CGM.getCXXABI().usesThreadWrapperFunction(VD))
2566     return CGF.CGM.getCXXABI().EmitThreadLocalVarDeclLValue(CGF, VD, T);
2567   // Check if the variable is marked as declare target with link clause in
2568   // device codegen.
2569   if (CGF.getLangOpts().OpenMPIsDevice) {
2570     Address Addr = emitDeclTargetVarDeclLValue(CGF, VD, T);
2571     if (Addr.isValid())
2572       return CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2573   }
2574 
2575   llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
2576 
2577   if (VD->getTLSKind() != VarDecl::TLS_None)
2578     V = CGF.Builder.CreateThreadLocalAddress(V);
2579 
2580   llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
2581   V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
2582   CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
2583   Address Addr(V, RealVarTy, Alignment);
2584   // Emit reference to the private copy of the variable if it is an OpenMP
2585   // threadprivate variable.
2586   if (CGF.getLangOpts().OpenMP && !CGF.getLangOpts().OpenMPSimd &&
2587       VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2588     return EmitThreadPrivateVarDeclLValue(CGF, VD, T, Addr, RealVarTy,
2589                                           E->getExprLoc());
2590   }
2591   LValue LV = VD->getType()->isReferenceType() ?
2592       CGF.EmitLoadOfReferenceLValue(Addr, VD->getType(),
2593                                     AlignmentSource::Decl) :
2594       CGF.MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2595   setObjCGCLValueClass(CGF.getContext(), E, LV);
2596   return LV;
2597 }
2598 
EmitFunctionDeclPointer(CodeGenModule & CGM,GlobalDecl GD)2599 static llvm::Constant *EmitFunctionDeclPointer(CodeGenModule &CGM,
2600                                                GlobalDecl GD) {
2601   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2602   if (FD->hasAttr<WeakRefAttr>()) {
2603     ConstantAddress aliasee = CGM.GetWeakRefReference(FD);
2604     return aliasee.getPointer();
2605   }
2606 
2607   llvm::Constant *V = CGM.GetAddrOfFunction(GD);
2608   if (!FD->hasPrototype()) {
2609     if (const FunctionProtoType *Proto =
2610             FD->getType()->getAs<FunctionProtoType>()) {
2611       // Ugly case: for a K&R-style definition, the type of the definition
2612       // isn't the same as the type of a use.  Correct for this with a
2613       // bitcast.
2614       QualType NoProtoType =
2615           CGM.getContext().getFunctionNoProtoType(Proto->getReturnType());
2616       NoProtoType = CGM.getContext().getPointerType(NoProtoType);
2617       V = llvm::ConstantExpr::getBitCast(V,
2618                                       CGM.getTypes().ConvertType(NoProtoType));
2619     }
2620   }
2621   return V;
2622 }
2623 
EmitFunctionDeclLValue(CodeGenFunction & CGF,const Expr * E,GlobalDecl GD)2624 static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF, const Expr *E,
2625                                      GlobalDecl GD) {
2626   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
2627   llvm::Value *V = EmitFunctionDeclPointer(CGF.CGM, GD);
2628   CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
2629   return CGF.MakeAddrLValue(V, E->getType(), Alignment,
2630                             AlignmentSource::Decl);
2631 }
2632 
EmitCapturedFieldLValue(CodeGenFunction & CGF,const FieldDecl * FD,llvm::Value * ThisValue)2633 static LValue EmitCapturedFieldLValue(CodeGenFunction &CGF, const FieldDecl *FD,
2634                                       llvm::Value *ThisValue) {
2635   QualType TagType = CGF.getContext().getTagDeclType(FD->getParent());
2636   LValue LV = CGF.MakeNaturalAlignAddrLValue(ThisValue, TagType);
2637   return CGF.EmitLValueForField(LV, FD);
2638 }
2639 
2640 /// Named Registers are named metadata pointing to the register name
2641 /// which will be read from/written to as an argument to the intrinsic
2642 /// @llvm.read/write_register.
2643 /// So far, only the name is being passed down, but other options such as
2644 /// register type, allocation type or even optimization options could be
2645 /// passed down via the metadata node.
EmitGlobalNamedRegister(const VarDecl * VD,CodeGenModule & CGM)2646 static LValue EmitGlobalNamedRegister(const VarDecl *VD, CodeGenModule &CGM) {
2647   SmallString<64> Name("llvm.named.register.");
2648   AsmLabelAttr *Asm = VD->getAttr<AsmLabelAttr>();
2649   assert(Asm->getLabel().size() < 64-Name.size() &&
2650       "Register name too big");
2651   Name.append(Asm->getLabel());
2652   llvm::NamedMDNode *M =
2653     CGM.getModule().getOrInsertNamedMetadata(Name);
2654   if (M->getNumOperands() == 0) {
2655     llvm::MDString *Str = llvm::MDString::get(CGM.getLLVMContext(),
2656                                               Asm->getLabel());
2657     llvm::Metadata *Ops[] = {Str};
2658     M->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
2659   }
2660 
2661   CharUnits Alignment = CGM.getContext().getDeclAlign(VD);
2662 
2663   llvm::Value *Ptr =
2664     llvm::MetadataAsValue::get(CGM.getLLVMContext(), M->getOperand(0));
2665   return LValue::MakeGlobalReg(Ptr, Alignment, VD->getType());
2666 }
2667 
2668 /// Determine whether we can emit a reference to \p VD from the current
2669 /// context, despite not necessarily having seen an odr-use of the variable in
2670 /// this context.
canEmitSpuriousReferenceToVariable(CodeGenFunction & CGF,const DeclRefExpr * E,const VarDecl * VD,bool IsConstant)2671 static bool canEmitSpuriousReferenceToVariable(CodeGenFunction &CGF,
2672                                                const DeclRefExpr *E,
2673                                                const VarDecl *VD,
2674                                                bool IsConstant) {
2675   // For a variable declared in an enclosing scope, do not emit a spurious
2676   // reference even if we have a capture, as that will emit an unwarranted
2677   // reference to our capture state, and will likely generate worse code than
2678   // emitting a local copy.
2679   if (E->refersToEnclosingVariableOrCapture())
2680     return false;
2681 
2682   // For a local declaration declared in this function, we can always reference
2683   // it even if we don't have an odr-use.
2684   if (VD->hasLocalStorage()) {
2685     return VD->getDeclContext() ==
2686            dyn_cast_or_null<DeclContext>(CGF.CurCodeDecl);
2687   }
2688 
2689   // For a global declaration, we can emit a reference to it if we know
2690   // for sure that we are able to emit a definition of it.
2691   VD = VD->getDefinition(CGF.getContext());
2692   if (!VD)
2693     return false;
2694 
2695   // Don't emit a spurious reference if it might be to a variable that only
2696   // exists on a different device / target.
2697   // FIXME: This is unnecessarily broad. Check whether this would actually be a
2698   // cross-target reference.
2699   if (CGF.getLangOpts().OpenMP || CGF.getLangOpts().CUDA ||
2700       CGF.getLangOpts().OpenCL) {
2701     return false;
2702   }
2703 
2704   // We can emit a spurious reference only if the linkage implies that we'll
2705   // be emitting a non-interposable symbol that will be retained until link
2706   // time.
2707   switch (CGF.CGM.getLLVMLinkageVarDefinition(VD, IsConstant)) {
2708   case llvm::GlobalValue::ExternalLinkage:
2709   case llvm::GlobalValue::LinkOnceODRLinkage:
2710   case llvm::GlobalValue::WeakODRLinkage:
2711   case llvm::GlobalValue::InternalLinkage:
2712   case llvm::GlobalValue::PrivateLinkage:
2713     return true;
2714   default:
2715     return false;
2716   }
2717 }
2718 
EmitDeclRefLValue(const DeclRefExpr * E)2719 LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
2720   const NamedDecl *ND = E->getDecl();
2721   QualType T = E->getType();
2722 
2723   assert(E->isNonOdrUse() != NOUR_Unevaluated &&
2724          "should not emit an unevaluated operand");
2725 
2726   if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2727     // Global Named registers access via intrinsics only
2728     if (VD->getStorageClass() == SC_Register &&
2729         VD->hasAttr<AsmLabelAttr>() && !VD->isLocalVarDecl())
2730       return EmitGlobalNamedRegister(VD, CGM);
2731 
2732     // If this DeclRefExpr does not constitute an odr-use of the variable,
2733     // we're not permitted to emit a reference to it in general, and it might
2734     // not be captured if capture would be necessary for a use. Emit the
2735     // constant value directly instead.
2736     if (E->isNonOdrUse() == NOUR_Constant &&
2737         (VD->getType()->isReferenceType() ||
2738          !canEmitSpuriousReferenceToVariable(*this, E, VD, true))) {
2739       VD->getAnyInitializer(VD);
2740       llvm::Constant *Val = ConstantEmitter(*this).emitAbstract(
2741           E->getLocation(), *VD->evaluateValue(), VD->getType());
2742       assert(Val && "failed to emit constant expression");
2743 
2744       Address Addr = Address::invalid();
2745       if (!VD->getType()->isReferenceType()) {
2746         // Spill the constant value to a global.
2747         Addr = CGM.createUnnamedGlobalFrom(*VD, Val,
2748                                            getContext().getDeclAlign(VD));
2749         llvm::Type *VarTy = getTypes().ConvertTypeForMem(VD->getType());
2750         auto *PTy = llvm::PointerType::get(
2751             VarTy, getTypes().getTargetAddressSpace(VD->getType()));
2752         Addr = Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, PTy, VarTy);
2753       } else {
2754         // Should we be using the alignment of the constant pointer we emitted?
2755         CharUnits Alignment =
2756             CGM.getNaturalTypeAlignment(E->getType(),
2757                                         /* BaseInfo= */ nullptr,
2758                                         /* TBAAInfo= */ nullptr,
2759                                         /* forPointeeType= */ true);
2760         Addr = Address(Val, ConvertTypeForMem(E->getType()), Alignment);
2761       }
2762       return MakeAddrLValue(Addr, T, AlignmentSource::Decl);
2763     }
2764 
2765     // FIXME: Handle other kinds of non-odr-use DeclRefExprs.
2766 
2767     // Check for captured variables.
2768     if (E->refersToEnclosingVariableOrCapture()) {
2769       VD = VD->getCanonicalDecl();
2770       if (auto *FD = LambdaCaptureFields.lookup(VD))
2771         return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2772       if (CapturedStmtInfo) {
2773         auto I = LocalDeclMap.find(VD);
2774         if (I != LocalDeclMap.end()) {
2775           LValue CapLVal;
2776           if (VD->getType()->isReferenceType())
2777             CapLVal = EmitLoadOfReferenceLValue(I->second, VD->getType(),
2778                                                 AlignmentSource::Decl);
2779           else
2780             CapLVal = MakeAddrLValue(I->second, T);
2781           // Mark lvalue as nontemporal if the variable is marked as nontemporal
2782           // in simd context.
2783           if (getLangOpts().OpenMP &&
2784               CGM.getOpenMPRuntime().isNontemporalDecl(VD))
2785             CapLVal.setNontemporal(/*Value=*/true);
2786           return CapLVal;
2787         }
2788         LValue CapLVal =
2789             EmitCapturedFieldLValue(*this, CapturedStmtInfo->lookup(VD),
2790                                     CapturedStmtInfo->getContextValue());
2791         Address LValueAddress = CapLVal.getAddress(*this);
2792         CapLVal = MakeAddrLValue(
2793             Address(LValueAddress.getPointer(), LValueAddress.getElementType(),
2794                     getContext().getDeclAlign(VD)),
2795             CapLVal.getType(), LValueBaseInfo(AlignmentSource::Decl),
2796             CapLVal.getTBAAInfo());
2797         // Mark lvalue as nontemporal if the variable is marked as nontemporal
2798         // in simd context.
2799         if (getLangOpts().OpenMP &&
2800             CGM.getOpenMPRuntime().isNontemporalDecl(VD))
2801           CapLVal.setNontemporal(/*Value=*/true);
2802         return CapLVal;
2803       }
2804 
2805       assert(isa<BlockDecl>(CurCodeDecl));
2806       Address addr = GetAddrOfBlockDecl(VD);
2807       return MakeAddrLValue(addr, T, AlignmentSource::Decl);
2808     }
2809   }
2810 
2811   // FIXME: We should be able to assert this for FunctionDecls as well!
2812   // FIXME: We should be able to assert this for all DeclRefExprs, not just
2813   // those with a valid source location.
2814   assert((ND->isUsed(false) || !isa<VarDecl>(ND) || E->isNonOdrUse() ||
2815           !E->getLocation().isValid()) &&
2816          "Should not use decl without marking it used!");
2817 
2818   if (ND->hasAttr<WeakRefAttr>()) {
2819     const auto *VD = cast<ValueDecl>(ND);
2820     ConstantAddress Aliasee = CGM.GetWeakRefReference(VD);
2821     return MakeAddrLValue(Aliasee, T, AlignmentSource::Decl);
2822   }
2823 
2824   if (const auto *VD = dyn_cast<VarDecl>(ND)) {
2825     // Check if this is a global variable.
2826     if (VD->hasLinkage() || VD->isStaticDataMember())
2827       return EmitGlobalVarDeclLValue(*this, E, VD);
2828 
2829     Address addr = Address::invalid();
2830 
2831     // The variable should generally be present in the local decl map.
2832     auto iter = LocalDeclMap.find(VD);
2833     if (iter != LocalDeclMap.end()) {
2834       addr = iter->second;
2835 
2836     // Otherwise, it might be static local we haven't emitted yet for
2837     // some reason; most likely, because it's in an outer function.
2838     } else if (VD->isStaticLocal()) {
2839       llvm::Constant *var = CGM.getOrCreateStaticVarDecl(
2840           *VD, CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false));
2841       addr = Address(
2842           var, ConvertTypeForMem(VD->getType()), getContext().getDeclAlign(VD));
2843 
2844     // No other cases for now.
2845     } else {
2846       llvm_unreachable("DeclRefExpr for Decl not entered in LocalDeclMap?");
2847     }
2848 
2849     // Handle threadlocal function locals.
2850     if (VD->getTLSKind() != VarDecl::TLS_None)
2851       addr =
2852           addr.withPointer(Builder.CreateThreadLocalAddress(addr.getPointer()));
2853 
2854     // Check for OpenMP threadprivate variables.
2855     if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd &&
2856         VD->hasAttr<OMPThreadPrivateDeclAttr>()) {
2857       return EmitThreadPrivateVarDeclLValue(
2858           *this, VD, T, addr, getTypes().ConvertTypeForMem(VD->getType()),
2859           E->getExprLoc());
2860     }
2861 
2862     // Drill into block byref variables.
2863     bool isBlockByref = VD->isEscapingByref();
2864     if (isBlockByref) {
2865       addr = emitBlockByrefAddress(addr, VD);
2866     }
2867 
2868     // Drill into reference types.
2869     LValue LV = VD->getType()->isReferenceType() ?
2870         EmitLoadOfReferenceLValue(addr, VD->getType(), AlignmentSource::Decl) :
2871         MakeAddrLValue(addr, T, AlignmentSource::Decl);
2872 
2873     bool isLocalStorage = VD->hasLocalStorage();
2874 
2875     bool NonGCable = isLocalStorage &&
2876                      !VD->getType()->isReferenceType() &&
2877                      !isBlockByref;
2878     if (NonGCable) {
2879       LV.getQuals().removeObjCGCAttr();
2880       LV.setNonGC(true);
2881     }
2882 
2883     bool isImpreciseLifetime =
2884       (isLocalStorage && !VD->hasAttr<ObjCPreciseLifetimeAttr>());
2885     if (isImpreciseLifetime)
2886       LV.setARCPreciseLifetime(ARCImpreciseLifetime);
2887     setObjCGCLValueClass(getContext(), E, LV);
2888     return LV;
2889   }
2890 
2891   if (const auto *FD = dyn_cast<FunctionDecl>(ND)) {
2892     LValue LV = EmitFunctionDeclLValue(*this, E, FD);
2893 
2894     // Emit debuginfo for the function declaration if the target wants to.
2895     if (getContext().getTargetInfo().allowDebugInfoForExternalRef()) {
2896       if (CGDebugInfo *DI = CGM.getModuleDebugInfo()) {
2897         auto *Fn =
2898             cast<llvm::Function>(LV.getPointer(*this)->stripPointerCasts());
2899         if (!Fn->getSubprogram())
2900           DI->EmitFunctionDecl(FD, FD->getLocation(), T, Fn);
2901       }
2902     }
2903 
2904     return LV;
2905   }
2906 
2907   // FIXME: While we're emitting a binding from an enclosing scope, all other
2908   // DeclRefExprs we see should be implicitly treated as if they also refer to
2909   // an enclosing scope.
2910   if (const auto *BD = dyn_cast<BindingDecl>(ND)) {
2911     if (E->refersToEnclosingVariableOrCapture()) {
2912       auto *FD = LambdaCaptureFields.lookup(BD);
2913       return EmitCapturedFieldLValue(*this, FD, CXXABIThisValue);
2914     }
2915     return EmitLValue(BD->getBinding());
2916   }
2917 
2918   // We can form DeclRefExprs naming GUID declarations when reconstituting
2919   // non-type template parameters into expressions.
2920   if (const auto *GD = dyn_cast<MSGuidDecl>(ND))
2921     return MakeAddrLValue(CGM.GetAddrOfMSGuidDecl(GD), T,
2922                           AlignmentSource::Decl);
2923 
2924   if (const auto *TPO = dyn_cast<TemplateParamObjectDecl>(ND))
2925     return MakeAddrLValue(CGM.GetAddrOfTemplateParamObject(TPO), T,
2926                           AlignmentSource::Decl);
2927 
2928   llvm_unreachable("Unhandled DeclRefExpr");
2929 }
2930 
EmitUnaryOpLValue(const UnaryOperator * E)2931 LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
2932   // __extension__ doesn't affect lvalue-ness.
2933   if (E->getOpcode() == UO_Extension)
2934     return EmitLValue(E->getSubExpr());
2935 
2936   QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
2937   switch (E->getOpcode()) {
2938   default: llvm_unreachable("Unknown unary operator lvalue!");
2939   case UO_Deref: {
2940     QualType T = E->getSubExpr()->getType()->getPointeeType();
2941     assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
2942 
2943     LValueBaseInfo BaseInfo;
2944     TBAAAccessInfo TBAAInfo;
2945     Address Addr = EmitPointerWithAlignment(E->getSubExpr(), &BaseInfo,
2946                                             &TBAAInfo);
2947     LValue LV = MakeAddrLValue(Addr, T, BaseInfo, TBAAInfo);
2948     LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
2949 
2950     // We should not generate __weak write barrier on indirect reference
2951     // of a pointer to object; as in void foo (__weak id *param); *param = 0;
2952     // But, we continue to generate __strong write barrier on indirect write
2953     // into a pointer to object.
2954     if (getLangOpts().ObjC &&
2955         getLangOpts().getGC() != LangOptions::NonGC &&
2956         LV.isObjCWeak())
2957       LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
2958     return LV;
2959   }
2960   case UO_Real:
2961   case UO_Imag: {
2962     LValue LV = EmitLValue(E->getSubExpr());
2963     assert(LV.isSimple() && "real/imag on non-ordinary l-value");
2964 
2965     // __real is valid on scalars.  This is a faster way of testing that.
2966     // __imag can only produce an rvalue on scalars.
2967     if (E->getOpcode() == UO_Real &&
2968         !LV.getAddress(*this).getElementType()->isStructTy()) {
2969       assert(E->getSubExpr()->getType()->isArithmeticType());
2970       return LV;
2971     }
2972 
2973     QualType T = ExprTy->castAs<ComplexType>()->getElementType();
2974 
2975     Address Component =
2976         (E->getOpcode() == UO_Real
2977              ? emitAddrOfRealComponent(LV.getAddress(*this), LV.getType())
2978              : emitAddrOfImagComponent(LV.getAddress(*this), LV.getType()));
2979     LValue ElemLV = MakeAddrLValue(Component, T, LV.getBaseInfo(),
2980                                    CGM.getTBAAInfoForSubobject(LV, T));
2981     ElemLV.getQuals().addQualifiers(LV.getQuals());
2982     return ElemLV;
2983   }
2984   case UO_PreInc:
2985   case UO_PreDec: {
2986     LValue LV = EmitLValue(E->getSubExpr());
2987     bool isInc = E->getOpcode() == UO_PreInc;
2988 
2989     if (E->getType()->isAnyComplexType())
2990       EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
2991     else
2992       EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
2993     return LV;
2994   }
2995   }
2996 }
2997 
EmitStringLiteralLValue(const StringLiteral * E)2998 LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
2999   return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
3000                         E->getType(), AlignmentSource::Decl);
3001 }
3002 
EmitObjCEncodeExprLValue(const ObjCEncodeExpr * E)3003 LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
3004   return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
3005                         E->getType(), AlignmentSource::Decl);
3006 }
3007 
EmitPredefinedLValue(const PredefinedExpr * E)3008 LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
3009   auto SL = E->getFunctionName();
3010   assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
3011   StringRef FnName = CurFn->getName();
3012   if (FnName.startswith("\01"))
3013     FnName = FnName.substr(1);
3014   StringRef NameItems[] = {
3015       PredefinedExpr::getIdentKindName(E->getIdentKind()), FnName};
3016   std::string GVName = llvm::join(NameItems, NameItems + 2, ".");
3017   if (auto *BD = dyn_cast_or_null<BlockDecl>(CurCodeDecl)) {
3018     std::string Name = std::string(SL->getString());
3019     if (!Name.empty()) {
3020       unsigned Discriminator =
3021           CGM.getCXXABI().getMangleContext().getBlockId(BD, true);
3022       if (Discriminator)
3023         Name += "_" + Twine(Discriminator + 1).str();
3024       auto C = CGM.GetAddrOfConstantCString(Name, GVName.c_str());
3025       return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3026     } else {
3027       auto C =
3028           CGM.GetAddrOfConstantCString(std::string(FnName), GVName.c_str());
3029       return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3030     }
3031   }
3032   auto C = CGM.GetAddrOfConstantStringFromLiteral(SL, GVName);
3033   return MakeAddrLValue(C, E->getType(), AlignmentSource::Decl);
3034 }
3035 
3036 /// Emit a type description suitable for use by a runtime sanitizer library. The
3037 /// format of a type descriptor is
3038 ///
3039 /// \code
3040 ///   { i16 TypeKind, i16 TypeInfo }
3041 /// \endcode
3042 ///
3043 /// followed by an array of i8 containing the type name. TypeKind is 0 for an
3044 /// integer, 1 for a floating point value, and -1 for anything else.
EmitCheckTypeDescriptor(QualType T)3045 llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
3046   // Only emit each type's descriptor once.
3047   if (llvm::Constant *C = CGM.getTypeDescriptorFromMap(T))
3048     return C;
3049 
3050   uint16_t TypeKind = -1;
3051   uint16_t TypeInfo = 0;
3052 
3053   if (T->isIntegerType()) {
3054     TypeKind = 0;
3055     TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
3056                (T->isSignedIntegerType() ? 1 : 0);
3057   } else if (T->isFloatingType()) {
3058     TypeKind = 1;
3059     TypeInfo = getContext().getTypeSize(T);
3060   }
3061 
3062   // Format the type name as if for a diagnostic, including quotes and
3063   // optionally an 'aka'.
3064   SmallString<32> Buffer;
3065   CGM.getDiags().ConvertArgToString(
3066       DiagnosticsEngine::ak_qualtype, (intptr_t)T.getAsOpaquePtr(), StringRef(),
3067       StringRef(), std::nullopt, Buffer, std::nullopt);
3068 
3069   llvm::Constant *Components[] = {
3070     Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
3071     llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
3072   };
3073   llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
3074 
3075   auto *GV = new llvm::GlobalVariable(
3076       CGM.getModule(), Descriptor->getType(),
3077       /*isConstant=*/true, llvm::GlobalVariable::PrivateLinkage, Descriptor);
3078   GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3079   CGM.getSanitizerMetadata()->disableSanitizerForGlobal(GV);
3080 
3081   // Remember the descriptor for this type.
3082   CGM.setTypeDescriptorInMap(T, GV);
3083 
3084   return GV;
3085 }
3086 
EmitCheckValue(llvm::Value * V)3087 llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
3088   llvm::Type *TargetTy = IntPtrTy;
3089 
3090   if (V->getType() == TargetTy)
3091     return V;
3092 
3093   // Floating-point types which fit into intptr_t are bitcast to integers
3094   // and then passed directly (after zero-extension, if necessary).
3095   if (V->getType()->isFloatingPointTy()) {
3096     unsigned Bits = V->getType()->getPrimitiveSizeInBits().getFixedValue();
3097     if (Bits <= TargetTy->getIntegerBitWidth())
3098       V = Builder.CreateBitCast(V, llvm::Type::getIntNTy(getLLVMContext(),
3099                                                          Bits));
3100   }
3101 
3102   // Integers which fit in intptr_t are zero-extended and passed directly.
3103   if (V->getType()->isIntegerTy() &&
3104       V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
3105     return Builder.CreateZExt(V, TargetTy);
3106 
3107   // Pointers are passed directly, everything else is passed by address.
3108   if (!V->getType()->isPointerTy()) {
3109     Address Ptr = CreateDefaultAlignTempAlloca(V->getType());
3110     Builder.CreateStore(V, Ptr);
3111     V = Ptr.getPointer();
3112   }
3113   return Builder.CreatePtrToInt(V, TargetTy);
3114 }
3115 
3116 /// Emit a representation of a SourceLocation for passing to a handler
3117 /// in a sanitizer runtime library. The format for this data is:
3118 /// \code
3119 ///   struct SourceLocation {
3120 ///     const char *Filename;
3121 ///     int32_t Line, Column;
3122 ///   };
3123 /// \endcode
3124 /// For an invalid SourceLocation, the Filename pointer is null.
EmitCheckSourceLocation(SourceLocation Loc)3125 llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
3126   llvm::Constant *Filename;
3127   int Line, Column;
3128 
3129   PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
3130   if (PLoc.isValid()) {
3131     StringRef FilenameString = PLoc.getFilename();
3132 
3133     int PathComponentsToStrip =
3134         CGM.getCodeGenOpts().EmitCheckPathComponentsToStrip;
3135     if (PathComponentsToStrip < 0) {
3136       assert(PathComponentsToStrip != INT_MIN);
3137       int PathComponentsToKeep = -PathComponentsToStrip;
3138       auto I = llvm::sys::path::rbegin(FilenameString);
3139       auto E = llvm::sys::path::rend(FilenameString);
3140       while (I != E && --PathComponentsToKeep)
3141         ++I;
3142 
3143       FilenameString = FilenameString.substr(I - E);
3144     } else if (PathComponentsToStrip > 0) {
3145       auto I = llvm::sys::path::begin(FilenameString);
3146       auto E = llvm::sys::path::end(FilenameString);
3147       while (I != E && PathComponentsToStrip--)
3148         ++I;
3149 
3150       if (I != E)
3151         FilenameString =
3152             FilenameString.substr(I - llvm::sys::path::begin(FilenameString));
3153       else
3154         FilenameString = llvm::sys::path::filename(FilenameString);
3155     }
3156 
3157     auto FilenameGV =
3158         CGM.GetAddrOfConstantCString(std::string(FilenameString), ".src");
3159     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(
3160         cast<llvm::GlobalVariable>(
3161             FilenameGV.getPointer()->stripPointerCasts()));
3162     Filename = FilenameGV.getPointer();
3163     Line = PLoc.getLine();
3164     Column = PLoc.getColumn();
3165   } else {
3166     Filename = llvm::Constant::getNullValue(Int8PtrTy);
3167     Line = Column = 0;
3168   }
3169 
3170   llvm::Constant *Data[] = {Filename, Builder.getInt32(Line),
3171                             Builder.getInt32(Column)};
3172 
3173   return llvm::ConstantStruct::getAnon(Data);
3174 }
3175 
3176 namespace {
3177 /// Specify under what conditions this check can be recovered
3178 enum class CheckRecoverableKind {
3179   /// Always terminate program execution if this check fails.
3180   Unrecoverable,
3181   /// Check supports recovering, runtime has both fatal (noreturn) and
3182   /// non-fatal handlers for this check.
3183   Recoverable,
3184   /// Runtime conditionally aborts, always need to support recovery.
3185   AlwaysRecoverable
3186 };
3187 }
3188 
getRecoverableKind(SanitizerMask Kind)3189 static CheckRecoverableKind getRecoverableKind(SanitizerMask Kind) {
3190   assert(Kind.countPopulation() == 1);
3191   if (Kind == SanitizerKind::Function || Kind == SanitizerKind::Vptr)
3192     return CheckRecoverableKind::AlwaysRecoverable;
3193   else if (Kind == SanitizerKind::Return || Kind == SanitizerKind::Unreachable)
3194     return CheckRecoverableKind::Unrecoverable;
3195   else
3196     return CheckRecoverableKind::Recoverable;
3197 }
3198 
3199 namespace {
3200 struct SanitizerHandlerInfo {
3201   char const *const Name;
3202   unsigned Version;
3203 };
3204 }
3205 
3206 const SanitizerHandlerInfo SanitizerHandlers[] = {
3207 #define SANITIZER_CHECK(Enum, Name, Version) {#Name, Version},
3208     LIST_SANITIZER_CHECKS
3209 #undef SANITIZER_CHECK
3210 };
3211 
emitCheckHandlerCall(CodeGenFunction & CGF,llvm::FunctionType * FnType,ArrayRef<llvm::Value * > FnArgs,SanitizerHandler CheckHandler,CheckRecoverableKind RecoverKind,bool IsFatal,llvm::BasicBlock * ContBB)3212 static void emitCheckHandlerCall(CodeGenFunction &CGF,
3213                                  llvm::FunctionType *FnType,
3214                                  ArrayRef<llvm::Value *> FnArgs,
3215                                  SanitizerHandler CheckHandler,
3216                                  CheckRecoverableKind RecoverKind, bool IsFatal,
3217                                  llvm::BasicBlock *ContBB) {
3218   assert(IsFatal || RecoverKind != CheckRecoverableKind::Unrecoverable);
3219   std::optional<ApplyDebugLocation> DL;
3220   if (!CGF.Builder.getCurrentDebugLocation()) {
3221     // Ensure that the call has at least an artificial debug location.
3222     DL.emplace(CGF, SourceLocation());
3223   }
3224   bool NeedsAbortSuffix =
3225       IsFatal && RecoverKind != CheckRecoverableKind::Unrecoverable;
3226   bool MinimalRuntime = CGF.CGM.getCodeGenOpts().SanitizeMinimalRuntime;
3227   const SanitizerHandlerInfo &CheckInfo = SanitizerHandlers[CheckHandler];
3228   const StringRef CheckName = CheckInfo.Name;
3229   std::string FnName = "__ubsan_handle_" + CheckName.str();
3230   if (CheckInfo.Version && !MinimalRuntime)
3231     FnName += "_v" + llvm::utostr(CheckInfo.Version);
3232   if (MinimalRuntime)
3233     FnName += "_minimal";
3234   if (NeedsAbortSuffix)
3235     FnName += "_abort";
3236   bool MayReturn =
3237       !IsFatal || RecoverKind == CheckRecoverableKind::AlwaysRecoverable;
3238 
3239   llvm::AttrBuilder B(CGF.getLLVMContext());
3240   if (!MayReturn) {
3241     B.addAttribute(llvm::Attribute::NoReturn)
3242         .addAttribute(llvm::Attribute::NoUnwind);
3243   }
3244   B.addUWTableAttr(llvm::UWTableKind::Default);
3245 
3246   llvm::FunctionCallee Fn = CGF.CGM.CreateRuntimeFunction(
3247       FnType, FnName,
3248       llvm::AttributeList::get(CGF.getLLVMContext(),
3249                                llvm::AttributeList::FunctionIndex, B),
3250       /*Local=*/true);
3251   llvm::CallInst *HandlerCall = CGF.EmitNounwindRuntimeCall(Fn, FnArgs);
3252   if (!MayReturn) {
3253     HandlerCall->setDoesNotReturn();
3254     CGF.Builder.CreateUnreachable();
3255   } else {
3256     CGF.Builder.CreateBr(ContBB);
3257   }
3258 }
3259 
EmitCheck(ArrayRef<std::pair<llvm::Value *,SanitizerMask>> Checked,SanitizerHandler CheckHandler,ArrayRef<llvm::Constant * > StaticArgs,ArrayRef<llvm::Value * > DynamicArgs)3260 void CodeGenFunction::EmitCheck(
3261     ArrayRef<std::pair<llvm::Value *, SanitizerMask>> Checked,
3262     SanitizerHandler CheckHandler, ArrayRef<llvm::Constant *> StaticArgs,
3263     ArrayRef<llvm::Value *> DynamicArgs) {
3264   assert(IsSanitizerScope);
3265   assert(Checked.size() > 0);
3266   assert(CheckHandler >= 0 &&
3267          size_t(CheckHandler) < std::size(SanitizerHandlers));
3268   const StringRef CheckName = SanitizerHandlers[CheckHandler].Name;
3269 
3270   llvm::Value *FatalCond = nullptr;
3271   llvm::Value *RecoverableCond = nullptr;
3272   llvm::Value *TrapCond = nullptr;
3273   for (int i = 0, n = Checked.size(); i < n; ++i) {
3274     llvm::Value *Check = Checked[i].first;
3275     // -fsanitize-trap= overrides -fsanitize-recover=.
3276     llvm::Value *&Cond =
3277         CGM.getCodeGenOpts().SanitizeTrap.has(Checked[i].second)
3278             ? TrapCond
3279             : CGM.getCodeGenOpts().SanitizeRecover.has(Checked[i].second)
3280                   ? RecoverableCond
3281                   : FatalCond;
3282     Cond = Cond ? Builder.CreateAnd(Cond, Check) : Check;
3283   }
3284 
3285   if (TrapCond)
3286     EmitTrapCheck(TrapCond, CheckHandler);
3287   if (!FatalCond && !RecoverableCond)
3288     return;
3289 
3290   llvm::Value *JointCond;
3291   if (FatalCond && RecoverableCond)
3292     JointCond = Builder.CreateAnd(FatalCond, RecoverableCond);
3293   else
3294     JointCond = FatalCond ? FatalCond : RecoverableCond;
3295   assert(JointCond);
3296 
3297   CheckRecoverableKind RecoverKind = getRecoverableKind(Checked[0].second);
3298   assert(SanOpts.has(Checked[0].second));
3299 #ifndef NDEBUG
3300   for (int i = 1, n = Checked.size(); i < n; ++i) {
3301     assert(RecoverKind == getRecoverableKind(Checked[i].second) &&
3302            "All recoverable kinds in a single check must be same!");
3303     assert(SanOpts.has(Checked[i].second));
3304   }
3305 #endif
3306 
3307   llvm::BasicBlock *Cont = createBasicBlock("cont");
3308   llvm::BasicBlock *Handlers = createBasicBlock("handler." + CheckName);
3309   llvm::Instruction *Branch = Builder.CreateCondBr(JointCond, Cont, Handlers);
3310   // Give hint that we very much don't expect to execute the handler
3311   // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
3312   llvm::MDBuilder MDHelper(getLLVMContext());
3313   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3314   Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);
3315   EmitBlock(Handlers);
3316 
3317   // Handler functions take an i8* pointing to the (handler-specific) static
3318   // information block, followed by a sequence of intptr_t arguments
3319   // representing operand values.
3320   SmallVector<llvm::Value *, 4> Args;
3321   SmallVector<llvm::Type *, 4> ArgTypes;
3322   if (!CGM.getCodeGenOpts().SanitizeMinimalRuntime) {
3323     Args.reserve(DynamicArgs.size() + 1);
3324     ArgTypes.reserve(DynamicArgs.size() + 1);
3325 
3326     // Emit handler arguments and create handler function type.
3327     if (!StaticArgs.empty()) {
3328       llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3329       auto *InfoPtr = new llvm::GlobalVariable(
3330           CGM.getModule(), Info->getType(), false,
3331           llvm::GlobalVariable::PrivateLinkage, Info, "", nullptr,
3332           llvm::GlobalVariable::NotThreadLocal,
3333           CGM.getDataLayout().getDefaultGlobalsAddressSpace());
3334       InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3335       CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
3336       Args.push_back(EmitCastToVoidPtr(InfoPtr));
3337       ArgTypes.push_back(Args.back()->getType());
3338     }
3339 
3340     for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
3341       Args.push_back(EmitCheckValue(DynamicArgs[i]));
3342       ArgTypes.push_back(IntPtrTy);
3343     }
3344   }
3345 
3346   llvm::FunctionType *FnType =
3347     llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);
3348 
3349   if (!FatalCond || !RecoverableCond) {
3350     // Simple case: we need to generate a single handler call, either
3351     // fatal, or non-fatal.
3352     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind,
3353                          (FatalCond != nullptr), Cont);
3354   } else {
3355     // Emit two handler calls: first one for set of unrecoverable checks,
3356     // another one for recoverable.
3357     llvm::BasicBlock *NonFatalHandlerBB =
3358         createBasicBlock("non_fatal." + CheckName);
3359     llvm::BasicBlock *FatalHandlerBB = createBasicBlock("fatal." + CheckName);
3360     Builder.CreateCondBr(FatalCond, NonFatalHandlerBB, FatalHandlerBB);
3361     EmitBlock(FatalHandlerBB);
3362     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, true,
3363                          NonFatalHandlerBB);
3364     EmitBlock(NonFatalHandlerBB);
3365     emitCheckHandlerCall(*this, FnType, Args, CheckHandler, RecoverKind, false,
3366                          Cont);
3367   }
3368 
3369   EmitBlock(Cont);
3370 }
3371 
EmitCfiSlowPathCheck(SanitizerMask Kind,llvm::Value * Cond,llvm::ConstantInt * TypeId,llvm::Value * Ptr,ArrayRef<llvm::Constant * > StaticArgs)3372 void CodeGenFunction::EmitCfiSlowPathCheck(
3373     SanitizerMask Kind, llvm::Value *Cond, llvm::ConstantInt *TypeId,
3374     llvm::Value *Ptr, ArrayRef<llvm::Constant *> StaticArgs) {
3375   llvm::BasicBlock *Cont = createBasicBlock("cfi.cont");
3376 
3377   llvm::BasicBlock *CheckBB = createBasicBlock("cfi.slowpath");
3378   llvm::BranchInst *BI = Builder.CreateCondBr(Cond, Cont, CheckBB);
3379 
3380   llvm::MDBuilder MDHelper(getLLVMContext());
3381   llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
3382   BI->setMetadata(llvm::LLVMContext::MD_prof, Node);
3383 
3384   EmitBlock(CheckBB);
3385 
3386   bool WithDiag = !CGM.getCodeGenOpts().SanitizeTrap.has(Kind);
3387 
3388   llvm::CallInst *CheckCall;
3389   llvm::FunctionCallee SlowPathFn;
3390   if (WithDiag) {
3391     llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
3392     auto *InfoPtr =
3393         new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
3394                                  llvm::GlobalVariable::PrivateLinkage, Info);
3395     InfoPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3396     CGM.getSanitizerMetadata()->disableSanitizerForGlobal(InfoPtr);
3397 
3398     SlowPathFn = CGM.getModule().getOrInsertFunction(
3399         "__cfi_slowpath_diag",
3400         llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy},
3401                                 false));
3402     CheckCall = Builder.CreateCall(
3403         SlowPathFn, {TypeId, Ptr, Builder.CreateBitCast(InfoPtr, Int8PtrTy)});
3404   } else {
3405     SlowPathFn = CGM.getModule().getOrInsertFunction(
3406         "__cfi_slowpath",
3407         llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy}, false));
3408     CheckCall = Builder.CreateCall(SlowPathFn, {TypeId, Ptr});
3409   }
3410 
3411   CGM.setDSOLocal(
3412       cast<llvm::GlobalValue>(SlowPathFn.getCallee()->stripPointerCasts()));
3413   CheckCall->setDoesNotThrow();
3414 
3415   EmitBlock(Cont);
3416 }
3417 
3418 // Emit a stub for __cfi_check function so that the linker knows about this
3419 // symbol in LTO mode.
EmitCfiCheckStub()3420 void CodeGenFunction::EmitCfiCheckStub() {
3421   llvm::Module *M = &CGM.getModule();
3422   auto &Ctx = M->getContext();
3423   llvm::Function *F = llvm::Function::Create(
3424       llvm::FunctionType::get(VoidTy, {Int64Ty, Int8PtrTy, Int8PtrTy}, false),
3425       llvm::GlobalValue::WeakAnyLinkage, "__cfi_check", M);
3426   CGM.setDSOLocal(F);
3427   llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", F);
3428   // FIXME: consider emitting an intrinsic call like
3429   // call void @llvm.cfi_check(i64 %0, i8* %1, i8* %2)
3430   // which can be lowered in CrossDSOCFI pass to the actual contents of
3431   // __cfi_check. This would allow inlining of __cfi_check calls.
3432   llvm::CallInst::Create(
3433       llvm::Intrinsic::getDeclaration(M, llvm::Intrinsic::trap), "", BB);
3434   llvm::ReturnInst::Create(Ctx, nullptr, BB);
3435 }
3436 
3437 // This function is basically a switch over the CFI failure kind, which is
3438 // extracted from CFICheckFailData (1st function argument). Each case is either
3439 // llvm.trap or a call to one of the two runtime handlers, based on
3440 // -fsanitize-trap and -fsanitize-recover settings.  Default case (invalid
3441 // failure kind) traps, but this should really never happen.  CFICheckFailData
3442 // can be nullptr if the calling module has -fsanitize-trap behavior for this
3443 // check kind; in this case __cfi_check_fail traps as well.
EmitCfiCheckFail()3444 void CodeGenFunction::EmitCfiCheckFail() {
3445   SanitizerScope SanScope(this);
3446   FunctionArgList Args;
3447   ImplicitParamDecl ArgData(getContext(), getContext().VoidPtrTy,
3448                             ImplicitParamDecl::Other);
3449   ImplicitParamDecl ArgAddr(getContext(), getContext().VoidPtrTy,
3450                             ImplicitParamDecl::Other);
3451   Args.push_back(&ArgData);
3452   Args.push_back(&ArgAddr);
3453 
3454   const CGFunctionInfo &FI =
3455     CGM.getTypes().arrangeBuiltinFunctionDeclaration(getContext().VoidTy, Args);
3456 
3457   llvm::Function *F = llvm::Function::Create(
3458       llvm::FunctionType::get(VoidTy, {VoidPtrTy, VoidPtrTy}, false),
3459       llvm::GlobalValue::WeakODRLinkage, "__cfi_check_fail", &CGM.getModule());
3460 
3461   CGM.SetLLVMFunctionAttributes(GlobalDecl(), FI, F, /*IsThunk=*/false);
3462   CGM.SetLLVMFunctionAttributesForDefinition(nullptr, F);
3463   F->setVisibility(llvm::GlobalValue::HiddenVisibility);
3464 
3465   StartFunction(GlobalDecl(), CGM.getContext().VoidTy, F, FI, Args,
3466                 SourceLocation());
3467 
3468   // This function is not affected by NoSanitizeList. This function does
3469   // not have a source location, but "src:*" would still apply. Revert any
3470   // changes to SanOpts made in StartFunction.
3471   SanOpts = CGM.getLangOpts().Sanitize;
3472 
3473   llvm::Value *Data =
3474       EmitLoadOfScalar(GetAddrOfLocalVar(&ArgData), /*Volatile=*/false,
3475                        CGM.getContext().VoidPtrTy, ArgData.getLocation());
3476   llvm::Value *Addr =
3477       EmitLoadOfScalar(GetAddrOfLocalVar(&ArgAddr), /*Volatile=*/false,
3478                        CGM.getContext().VoidPtrTy, ArgAddr.getLocation());
3479 
3480   // Data == nullptr means the calling module has trap behaviour for this check.
3481   llvm::Value *DataIsNotNullPtr =
3482       Builder.CreateICmpNE(Data, llvm::ConstantPointerNull::get(Int8PtrTy));
3483   EmitTrapCheck(DataIsNotNullPtr, SanitizerHandler::CFICheckFail);
3484 
3485   llvm::StructType *SourceLocationTy =
3486       llvm::StructType::get(VoidPtrTy, Int32Ty, Int32Ty);
3487   llvm::StructType *CfiCheckFailDataTy =
3488       llvm::StructType::get(Int8Ty, SourceLocationTy, VoidPtrTy);
3489 
3490   llvm::Value *V = Builder.CreateConstGEP2_32(
3491       CfiCheckFailDataTy,
3492       Builder.CreatePointerCast(Data, CfiCheckFailDataTy->getPointerTo(0)), 0,
3493       0);
3494 
3495   Address CheckKindAddr(V, Int8Ty, getIntAlign());
3496   llvm::Value *CheckKind = Builder.CreateLoad(CheckKindAddr);
3497 
3498   llvm::Value *AllVtables = llvm::MetadataAsValue::get(
3499       CGM.getLLVMContext(),
3500       llvm::MDString::get(CGM.getLLVMContext(), "all-vtables"));
3501   llvm::Value *ValidVtable = Builder.CreateZExt(
3502       Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::type_test),
3503                          {Addr, AllVtables}),
3504       IntPtrTy);
3505 
3506   const std::pair<int, SanitizerMask> CheckKinds[] = {
3507       {CFITCK_VCall, SanitizerKind::CFIVCall},
3508       {CFITCK_NVCall, SanitizerKind::CFINVCall},
3509       {CFITCK_DerivedCast, SanitizerKind::CFIDerivedCast},
3510       {CFITCK_UnrelatedCast, SanitizerKind::CFIUnrelatedCast},
3511       {CFITCK_ICall, SanitizerKind::CFIICall}};
3512 
3513   SmallVector<std::pair<llvm::Value *, SanitizerMask>, 5> Checks;
3514   for (auto CheckKindMaskPair : CheckKinds) {
3515     int Kind = CheckKindMaskPair.first;
3516     SanitizerMask Mask = CheckKindMaskPair.second;
3517     llvm::Value *Cond =
3518         Builder.CreateICmpNE(CheckKind, llvm::ConstantInt::get(Int8Ty, Kind));
3519     if (CGM.getLangOpts().Sanitize.has(Mask))
3520       EmitCheck(std::make_pair(Cond, Mask), SanitizerHandler::CFICheckFail, {},
3521                 {Data, Addr, ValidVtable});
3522     else
3523       EmitTrapCheck(Cond, SanitizerHandler::CFICheckFail);
3524   }
3525 
3526   FinishFunction();
3527   // The only reference to this function will be created during LTO link.
3528   // Make sure it survives until then.
3529   CGM.addUsedGlobal(F);
3530 }
3531 
EmitUnreachable(SourceLocation Loc)3532 void CodeGenFunction::EmitUnreachable(SourceLocation Loc) {
3533   if (SanOpts.has(SanitizerKind::Unreachable)) {
3534     SanitizerScope SanScope(this);
3535     EmitCheck(std::make_pair(static_cast<llvm::Value *>(Builder.getFalse()),
3536                              SanitizerKind::Unreachable),
3537               SanitizerHandler::BuiltinUnreachable,
3538               EmitCheckSourceLocation(Loc), std::nullopt);
3539   }
3540   Builder.CreateUnreachable();
3541 }
3542 
EmitTrapCheck(llvm::Value * Checked,SanitizerHandler CheckHandlerID)3543 void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked,
3544                                     SanitizerHandler CheckHandlerID) {
3545   llvm::BasicBlock *Cont = createBasicBlock("cont");
3546 
3547   // If we're optimizing, collapse all calls to trap down to just one per
3548   // check-type per function to save on code size.
3549   if (TrapBBs.size() <= CheckHandlerID)
3550     TrapBBs.resize(CheckHandlerID + 1);
3551   llvm::BasicBlock *&TrapBB = TrapBBs[CheckHandlerID];
3552 
3553   if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB ||
3554       (CurCodeDecl && CurCodeDecl->hasAttr<OptimizeNoneAttr>())) {
3555     TrapBB = createBasicBlock("trap");
3556     Builder.CreateCondBr(Checked, Cont, TrapBB);
3557     EmitBlock(TrapBB);
3558 
3559     llvm::CallInst *TrapCall =
3560         Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::ubsantrap),
3561                            llvm::ConstantInt::get(CGM.Int8Ty, CheckHandlerID));
3562 
3563     if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3564       auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3565                                     CGM.getCodeGenOpts().TrapFuncName);
3566       TrapCall->addFnAttr(A);
3567     }
3568     TrapCall->setDoesNotReturn();
3569     TrapCall->setDoesNotThrow();
3570     Builder.CreateUnreachable();
3571   } else {
3572     auto Call = TrapBB->begin();
3573     assert(isa<llvm::CallInst>(Call) && "Expected call in trap BB");
3574 
3575     Call->applyMergedLocation(Call->getDebugLoc(),
3576                               Builder.getCurrentDebugLocation());
3577     Builder.CreateCondBr(Checked, Cont, TrapBB);
3578   }
3579 
3580   EmitBlock(Cont);
3581 }
3582 
EmitTrapCall(llvm::Intrinsic::ID IntrID)3583 llvm::CallInst *CodeGenFunction::EmitTrapCall(llvm::Intrinsic::ID IntrID) {
3584   llvm::CallInst *TrapCall =
3585       Builder.CreateCall(CGM.getIntrinsic(IntrID));
3586 
3587   if (!CGM.getCodeGenOpts().TrapFuncName.empty()) {
3588     auto A = llvm::Attribute::get(getLLVMContext(), "trap-func-name",
3589                                   CGM.getCodeGenOpts().TrapFuncName);
3590     TrapCall->addFnAttr(A);
3591   }
3592 
3593   return TrapCall;
3594 }
3595 
EmitArrayToPointerDecay(const Expr * E,LValueBaseInfo * BaseInfo,TBAAAccessInfo * TBAAInfo)3596 Address CodeGenFunction::EmitArrayToPointerDecay(const Expr *E,
3597                                                  LValueBaseInfo *BaseInfo,
3598                                                  TBAAAccessInfo *TBAAInfo) {
3599   assert(E->getType()->isArrayType() &&
3600          "Array to pointer decay must have array source type!");
3601 
3602   // Expressions of array type can't be bitfields or vector elements.
3603   LValue LV = EmitLValue(E);
3604   Address Addr = LV.getAddress(*this);
3605 
3606   // If the array type was an incomplete type, we need to make sure
3607   // the decay ends up being the right type.
3608   llvm::Type *NewTy = ConvertType(E->getType());
3609   Addr = Builder.CreateElementBitCast(Addr, NewTy);
3610 
3611   // Note that VLA pointers are always decayed, so we don't need to do
3612   // anything here.
3613   if (!E->getType()->isVariableArrayType()) {
3614     assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3615            "Expected pointer to array");
3616     Addr = Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3617   }
3618 
3619   // The result of this decay conversion points to an array element within the
3620   // base lvalue. However, since TBAA currently does not support representing
3621   // accesses to elements of member arrays, we conservatively represent accesses
3622   // to the pointee object as if it had no any base lvalue specified.
3623   // TODO: Support TBAA for member arrays.
3624   QualType EltType = E->getType()->castAsArrayTypeUnsafe()->getElementType();
3625   if (BaseInfo) *BaseInfo = LV.getBaseInfo();
3626   if (TBAAInfo) *TBAAInfo = CGM.getTBAAAccessInfo(EltType);
3627 
3628   return Builder.CreateElementBitCast(Addr, ConvertTypeForMem(EltType));
3629 }
3630 
3631 /// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
3632 /// array to pointer, return the array subexpression.
isSimpleArrayDecayOperand(const Expr * E)3633 static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
3634   // If this isn't just an array->pointer decay, bail out.
3635   const auto *CE = dyn_cast<CastExpr>(E);
3636   if (!CE || CE->getCastKind() != CK_ArrayToPointerDecay)
3637     return nullptr;
3638 
3639   // If this is a decay from variable width array, bail out.
3640   const Expr *SubExpr = CE->getSubExpr();
3641   if (SubExpr->getType()->isVariableArrayType())
3642     return nullptr;
3643 
3644   return SubExpr;
3645 }
3646 
emitArraySubscriptGEP(CodeGenFunction & CGF,llvm::Type * elemType,llvm::Value * ptr,ArrayRef<llvm::Value * > indices,bool inbounds,bool signedIndices,SourceLocation loc,const llvm::Twine & name="arrayidx")3647 static llvm::Value *emitArraySubscriptGEP(CodeGenFunction &CGF,
3648                                           llvm::Type *elemType,
3649                                           llvm::Value *ptr,
3650                                           ArrayRef<llvm::Value*> indices,
3651                                           bool inbounds,
3652                                           bool signedIndices,
3653                                           SourceLocation loc,
3654                                     const llvm::Twine &name = "arrayidx") {
3655   if (inbounds) {
3656     return CGF.EmitCheckedInBoundsGEP(elemType, ptr, indices, signedIndices,
3657                                       CodeGenFunction::NotSubtraction, loc,
3658                                       name);
3659   } else {
3660     return CGF.Builder.CreateGEP(elemType, ptr, indices, name);
3661   }
3662 }
3663 
getArrayElementAlign(CharUnits arrayAlign,llvm::Value * idx,CharUnits eltSize)3664 static CharUnits getArrayElementAlign(CharUnits arrayAlign,
3665                                       llvm::Value *idx,
3666                                       CharUnits eltSize) {
3667   // If we have a constant index, we can use the exact offset of the
3668   // element we're accessing.
3669   if (auto constantIdx = dyn_cast<llvm::ConstantInt>(idx)) {
3670     CharUnits offset = constantIdx->getZExtValue() * eltSize;
3671     return arrayAlign.alignmentAtOffset(offset);
3672 
3673   // Otherwise, use the worst-case alignment for any element.
3674   } else {
3675     return arrayAlign.alignmentOfArrayElement(eltSize);
3676   }
3677 }
3678 
getFixedSizeElementType(const ASTContext & ctx,const VariableArrayType * vla)3679 static QualType getFixedSizeElementType(const ASTContext &ctx,
3680                                         const VariableArrayType *vla) {
3681   QualType eltType;
3682   do {
3683     eltType = vla->getElementType();
3684   } while ((vla = ctx.getAsVariableArrayType(eltType)));
3685   return eltType;
3686 }
3687 
3688 /// Given an array base, check whether its member access belongs to a record
3689 /// with preserve_access_index attribute or not.
IsPreserveAIArrayBase(CodeGenFunction & CGF,const Expr * ArrayBase)3690 static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) {
3691   if (!ArrayBase || !CGF.getDebugInfo())
3692     return false;
3693 
3694   // Only support base as either a MemberExpr or DeclRefExpr.
3695   // DeclRefExpr to cover cases like:
3696   //    struct s { int a; int b[10]; };
3697   //    struct s *p;
3698   //    p[1].a
3699   // p[1] will generate a DeclRefExpr and p[1].a is a MemberExpr.
3700   // p->b[5] is a MemberExpr example.
3701   const Expr *E = ArrayBase->IgnoreImpCasts();
3702   if (const auto *ME = dyn_cast<MemberExpr>(E))
3703     return ME->getMemberDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3704 
3705   if (const auto *DRE = dyn_cast<DeclRefExpr>(E)) {
3706     const auto *VarDef = dyn_cast<VarDecl>(DRE->getDecl());
3707     if (!VarDef)
3708       return false;
3709 
3710     const auto *PtrT = VarDef->getType()->getAs<PointerType>();
3711     if (!PtrT)
3712       return false;
3713 
3714     const auto *PointeeT = PtrT->getPointeeType()
3715                              ->getUnqualifiedDesugaredType();
3716     if (const auto *RecT = dyn_cast<RecordType>(PointeeT))
3717       return RecT->getDecl()->hasAttr<BPFPreserveAccessIndexAttr>();
3718     return false;
3719   }
3720 
3721   return false;
3722 }
3723 
emitArraySubscriptGEP(CodeGenFunction & CGF,Address addr,ArrayRef<llvm::Value * > indices,QualType eltType,bool inbounds,bool signedIndices,SourceLocation loc,QualType * arrayType=nullptr,const Expr * Base=nullptr,const llvm::Twine & name="arrayidx")3724 static Address emitArraySubscriptGEP(CodeGenFunction &CGF, Address addr,
3725                                      ArrayRef<llvm::Value *> indices,
3726                                      QualType eltType, bool inbounds,
3727                                      bool signedIndices, SourceLocation loc,
3728                                      QualType *arrayType = nullptr,
3729                                      const Expr *Base = nullptr,
3730                                      const llvm::Twine &name = "arrayidx") {
3731   // All the indices except that last must be zero.
3732 #ifndef NDEBUG
3733   for (auto *idx : indices.drop_back())
3734     assert(isa<llvm::ConstantInt>(idx) &&
3735            cast<llvm::ConstantInt>(idx)->isZero());
3736 #endif
3737 
3738   // Determine the element size of the statically-sized base.  This is
3739   // the thing that the indices are expressed in terms of.
3740   if (auto vla = CGF.getContext().getAsVariableArrayType(eltType)) {
3741     eltType = getFixedSizeElementType(CGF.getContext(), vla);
3742   }
3743 
3744   // We can use that to compute the best alignment of the element.
3745   CharUnits eltSize = CGF.getContext().getTypeSizeInChars(eltType);
3746   CharUnits eltAlign =
3747     getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize);
3748 
3749   llvm::Value *eltPtr;
3750   auto LastIndex = dyn_cast<llvm::ConstantInt>(indices.back());
3751   if (!LastIndex ||
3752       (!CGF.IsInPreservedAIRegion && !IsPreserveAIArrayBase(CGF, Base))) {
3753     eltPtr = emitArraySubscriptGEP(
3754         CGF, addr.getElementType(), addr.getPointer(), indices, inbounds,
3755         signedIndices, loc, name);
3756   } else {
3757     // Remember the original array subscript for bpf target
3758     unsigned idx = LastIndex->getZExtValue();
3759     llvm::DIType *DbgInfo = nullptr;
3760     if (arrayType)
3761       DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(*arrayType, loc);
3762     eltPtr = CGF.Builder.CreatePreserveArrayAccessIndex(addr.getElementType(),
3763                                                         addr.getPointer(),
3764                                                         indices.size() - 1,
3765                                                         idx, DbgInfo);
3766   }
3767 
3768   return Address(eltPtr, CGF.ConvertTypeForMem(eltType), eltAlign);
3769 }
3770 
EmitArraySubscriptExpr(const ArraySubscriptExpr * E,bool Accessed)3771 LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E,
3772                                                bool Accessed) {
3773   // The index must always be an integer, which is not an aggregate.  Emit it
3774   // in lexical order (this complexity is, sadly, required by C++17).
3775   llvm::Value *IdxPre =
3776       (E->getLHS() == E->getIdx()) ? EmitScalarExpr(E->getIdx()) : nullptr;
3777   bool SignedIndices = false;
3778   auto EmitIdxAfterBase = [&, IdxPre](bool Promote) -> llvm::Value * {
3779     auto *Idx = IdxPre;
3780     if (E->getLHS() != E->getIdx()) {
3781       assert(E->getRHS() == E->getIdx() && "index was neither LHS nor RHS");
3782       Idx = EmitScalarExpr(E->getIdx());
3783     }
3784 
3785     QualType IdxTy = E->getIdx()->getType();
3786     bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
3787     SignedIndices |= IdxSigned;
3788 
3789     if (SanOpts.has(SanitizerKind::ArrayBounds))
3790       EmitBoundsCheck(E, E->getBase(), Idx, IdxTy, Accessed);
3791 
3792     // Extend or truncate the index type to 32 or 64-bits.
3793     if (Promote && Idx->getType() != IntPtrTy)
3794       Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
3795 
3796     return Idx;
3797   };
3798   IdxPre = nullptr;
3799 
3800   // If the base is a vector type, then we are forming a vector element lvalue
3801   // with this subscript.
3802   if (E->getBase()->getType()->isVectorType() &&
3803       !isa<ExtVectorElementExpr>(E->getBase())) {
3804     // Emit the vector as an lvalue to get its address.
3805     LValue LHS = EmitLValue(E->getBase());
3806     auto *Idx = EmitIdxAfterBase(/*Promote*/false);
3807     assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
3808     return LValue::MakeVectorElt(LHS.getAddress(*this), Idx,
3809                                  E->getBase()->getType(), LHS.getBaseInfo(),
3810                                  TBAAAccessInfo());
3811   }
3812 
3813   // All the other cases basically behave like simple offsetting.
3814 
3815   // Handle the extvector case we ignored above.
3816   if (isa<ExtVectorElementExpr>(E->getBase())) {
3817     LValue LV = EmitLValue(E->getBase());
3818     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3819     Address Addr = EmitExtVectorElementLValue(LV);
3820 
3821     QualType EltType = LV.getType()->castAs<VectorType>()->getElementType();
3822     Addr = emitArraySubscriptGEP(*this, Addr, Idx, EltType, /*inbounds*/ true,
3823                                  SignedIndices, E->getExprLoc());
3824     return MakeAddrLValue(Addr, EltType, LV.getBaseInfo(),
3825                           CGM.getTBAAInfoForSubobject(LV, EltType));
3826   }
3827 
3828   LValueBaseInfo EltBaseInfo;
3829   TBAAAccessInfo EltTBAAInfo;
3830   Address Addr = Address::invalid();
3831   if (const VariableArrayType *vla =
3832            getContext().getAsVariableArrayType(E->getType())) {
3833     // The base must be a pointer, which is not an aggregate.  Emit
3834     // it.  It needs to be emitted first in case it's what captures
3835     // the VLA bounds.
3836     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3837     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3838 
3839     // The element count here is the total number of non-VLA elements.
3840     llvm::Value *numElements = getVLASize(vla).NumElts;
3841 
3842     // Effectively, the multiply by the VLA size is part of the GEP.
3843     // GEP indexes are signed, and scaling an index isn't permitted to
3844     // signed-overflow, so we use the same semantics for our explicit
3845     // multiply.  We suppress this if overflow is not undefined behavior.
3846     if (getLangOpts().isSignedOverflowDefined()) {
3847       Idx = Builder.CreateMul(Idx, numElements);
3848     } else {
3849       Idx = Builder.CreateNSWMul(Idx, numElements);
3850     }
3851 
3852     Addr = emitArraySubscriptGEP(*this, Addr, Idx, vla->getElementType(),
3853                                  !getLangOpts().isSignedOverflowDefined(),
3854                                  SignedIndices, E->getExprLoc());
3855 
3856   } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
3857     // Indexing over an interface, as in "NSString *P; P[4];"
3858 
3859     // Emit the base pointer.
3860     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3861     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3862 
3863     CharUnits InterfaceSize = getContext().getTypeSizeInChars(OIT);
3864     llvm::Value *InterfaceSizeVal =
3865         llvm::ConstantInt::get(Idx->getType(), InterfaceSize.getQuantity());
3866 
3867     llvm::Value *ScaledIdx = Builder.CreateMul(Idx, InterfaceSizeVal);
3868 
3869     // We don't necessarily build correct LLVM struct types for ObjC
3870     // interfaces, so we can't rely on GEP to do this scaling
3871     // correctly, so we need to cast to i8*.  FIXME: is this actually
3872     // true?  A lot of other things in the fragile ABI would break...
3873     llvm::Type *OrigBaseElemTy = Addr.getElementType();
3874     Addr = Builder.CreateElementBitCast(Addr, Int8Ty);
3875 
3876     // Do the GEP.
3877     CharUnits EltAlign =
3878       getArrayElementAlign(Addr.getAlignment(), Idx, InterfaceSize);
3879     llvm::Value *EltPtr =
3880         emitArraySubscriptGEP(*this, Addr.getElementType(), Addr.getPointer(),
3881                               ScaledIdx, false, SignedIndices, E->getExprLoc());
3882     Addr = Address(EltPtr, Addr.getElementType(), EltAlign);
3883 
3884     // Cast back.
3885     Addr = Builder.CreateElementBitCast(Addr, OrigBaseElemTy);
3886   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
3887     // If this is A[i] where A is an array, the frontend will have decayed the
3888     // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
3889     // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
3890     // "gep x, i" here.  Emit one "gep A, 0, i".
3891     assert(Array->getType()->isArrayType() &&
3892            "Array to pointer decay must have array source type!");
3893     LValue ArrayLV;
3894     // For simple multidimensional array indexing, set the 'accessed' flag for
3895     // better bounds-checking of the base expression.
3896     if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
3897       ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
3898     else
3899       ArrayLV = EmitLValue(Array);
3900     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3901 
3902     // Propagate the alignment from the array itself to the result.
3903     QualType arrayType = Array->getType();
3904     Addr = emitArraySubscriptGEP(
3905         *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
3906         E->getType(), !getLangOpts().isSignedOverflowDefined(), SignedIndices,
3907         E->getExprLoc(), &arrayType, E->getBase());
3908     EltBaseInfo = ArrayLV.getBaseInfo();
3909     EltTBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, E->getType());
3910   } else {
3911     // The base must be a pointer; emit it with an estimate of its alignment.
3912     Addr = EmitPointerWithAlignment(E->getBase(), &EltBaseInfo, &EltTBAAInfo);
3913     auto *Idx = EmitIdxAfterBase(/*Promote*/true);
3914     QualType ptrType = E->getBase()->getType();
3915     Addr = emitArraySubscriptGEP(*this, Addr, Idx, E->getType(),
3916                                  !getLangOpts().isSignedOverflowDefined(),
3917                                  SignedIndices, E->getExprLoc(), &ptrType,
3918                                  E->getBase());
3919   }
3920 
3921   LValue LV = MakeAddrLValue(Addr, E->getType(), EltBaseInfo, EltTBAAInfo);
3922 
3923   if (getLangOpts().ObjC &&
3924       getLangOpts().getGC() != LangOptions::NonGC) {
3925     LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
3926     setObjCGCLValueClass(getContext(), E, LV);
3927   }
3928   return LV;
3929 }
3930 
EmitMatrixSubscriptExpr(const MatrixSubscriptExpr * E)3931 LValue CodeGenFunction::EmitMatrixSubscriptExpr(const MatrixSubscriptExpr *E) {
3932   assert(
3933       !E->isIncomplete() &&
3934       "incomplete matrix subscript expressions should be rejected during Sema");
3935   LValue Base = EmitLValue(E->getBase());
3936   llvm::Value *RowIdx = EmitScalarExpr(E->getRowIdx());
3937   llvm::Value *ColIdx = EmitScalarExpr(E->getColumnIdx());
3938   llvm::Value *NumRows = Builder.getIntN(
3939       RowIdx->getType()->getScalarSizeInBits(),
3940       E->getBase()->getType()->castAs<ConstantMatrixType>()->getNumRows());
3941   llvm::Value *FinalIdx =
3942       Builder.CreateAdd(Builder.CreateMul(ColIdx, NumRows), RowIdx);
3943   return LValue::MakeMatrixElt(
3944       MaybeConvertMatrixAddress(Base.getAddress(*this), *this), FinalIdx,
3945       E->getBase()->getType(), Base.getBaseInfo(), TBAAAccessInfo());
3946 }
3947 
emitOMPArraySectionBase(CodeGenFunction & CGF,const Expr * Base,LValueBaseInfo & BaseInfo,TBAAAccessInfo & TBAAInfo,QualType BaseTy,QualType ElTy,bool IsLowerBound)3948 static Address emitOMPArraySectionBase(CodeGenFunction &CGF, const Expr *Base,
3949                                        LValueBaseInfo &BaseInfo,
3950                                        TBAAAccessInfo &TBAAInfo,
3951                                        QualType BaseTy, QualType ElTy,
3952                                        bool IsLowerBound) {
3953   LValue BaseLVal;
3954   if (auto *ASE = dyn_cast<OMPArraySectionExpr>(Base->IgnoreParenImpCasts())) {
3955     BaseLVal = CGF.EmitOMPArraySectionExpr(ASE, IsLowerBound);
3956     if (BaseTy->isArrayType()) {
3957       Address Addr = BaseLVal.getAddress(CGF);
3958       BaseInfo = BaseLVal.getBaseInfo();
3959 
3960       // If the array type was an incomplete type, we need to make sure
3961       // the decay ends up being the right type.
3962       llvm::Type *NewTy = CGF.ConvertType(BaseTy);
3963       Addr = CGF.Builder.CreateElementBitCast(Addr, NewTy);
3964 
3965       // Note that VLA pointers are always decayed, so we don't need to do
3966       // anything here.
3967       if (!BaseTy->isVariableArrayType()) {
3968         assert(isa<llvm::ArrayType>(Addr.getElementType()) &&
3969                "Expected pointer to array");
3970         Addr = CGF.Builder.CreateConstArrayGEP(Addr, 0, "arraydecay");
3971       }
3972 
3973       return CGF.Builder.CreateElementBitCast(Addr,
3974                                               CGF.ConvertTypeForMem(ElTy));
3975     }
3976     LValueBaseInfo TypeBaseInfo;
3977     TBAAAccessInfo TypeTBAAInfo;
3978     CharUnits Align =
3979         CGF.CGM.getNaturalTypeAlignment(ElTy, &TypeBaseInfo, &TypeTBAAInfo);
3980     BaseInfo.mergeForCast(TypeBaseInfo);
3981     TBAAInfo = CGF.CGM.mergeTBAAInfoForCast(TBAAInfo, TypeTBAAInfo);
3982     return Address(CGF.Builder.CreateLoad(BaseLVal.getAddress(CGF)),
3983                    CGF.ConvertTypeForMem(ElTy), Align);
3984   }
3985   return CGF.EmitPointerWithAlignment(Base, &BaseInfo, &TBAAInfo);
3986 }
3987 
EmitOMPArraySectionExpr(const OMPArraySectionExpr * E,bool IsLowerBound)3988 LValue CodeGenFunction::EmitOMPArraySectionExpr(const OMPArraySectionExpr *E,
3989                                                 bool IsLowerBound) {
3990   QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType(E->getBase());
3991   QualType ResultExprTy;
3992   if (auto *AT = getContext().getAsArrayType(BaseTy))
3993     ResultExprTy = AT->getElementType();
3994   else
3995     ResultExprTy = BaseTy->getPointeeType();
3996   llvm::Value *Idx = nullptr;
3997   if (IsLowerBound || E->getColonLocFirst().isInvalid()) {
3998     // Requesting lower bound or upper bound, but without provided length and
3999     // without ':' symbol for the default length -> length = 1.
4000     // Idx = LowerBound ?: 0;
4001     if (auto *LowerBound = E->getLowerBound()) {
4002       Idx = Builder.CreateIntCast(
4003           EmitScalarExpr(LowerBound), IntPtrTy,
4004           LowerBound->getType()->hasSignedIntegerRepresentation());
4005     } else
4006       Idx = llvm::ConstantInt::getNullValue(IntPtrTy);
4007   } else {
4008     // Try to emit length or lower bound as constant. If this is possible, 1
4009     // is subtracted from constant length or lower bound. Otherwise, emit LLVM
4010     // IR (LB + Len) - 1.
4011     auto &C = CGM.getContext();
4012     auto *Length = E->getLength();
4013     llvm::APSInt ConstLength;
4014     if (Length) {
4015       // Idx = LowerBound + Length - 1;
4016       if (std::optional<llvm::APSInt> CL = Length->getIntegerConstantExpr(C)) {
4017         ConstLength = CL->zextOrTrunc(PointerWidthInBits);
4018         Length = nullptr;
4019       }
4020       auto *LowerBound = E->getLowerBound();
4021       llvm::APSInt ConstLowerBound(PointerWidthInBits, /*isUnsigned=*/false);
4022       if (LowerBound) {
4023         if (std::optional<llvm::APSInt> LB =
4024                 LowerBound->getIntegerConstantExpr(C)) {
4025           ConstLowerBound = LB->zextOrTrunc(PointerWidthInBits);
4026           LowerBound = nullptr;
4027         }
4028       }
4029       if (!Length)
4030         --ConstLength;
4031       else if (!LowerBound)
4032         --ConstLowerBound;
4033 
4034       if (Length || LowerBound) {
4035         auto *LowerBoundVal =
4036             LowerBound
4037                 ? Builder.CreateIntCast(
4038                       EmitScalarExpr(LowerBound), IntPtrTy,
4039                       LowerBound->getType()->hasSignedIntegerRepresentation())
4040                 : llvm::ConstantInt::get(IntPtrTy, ConstLowerBound);
4041         auto *LengthVal =
4042             Length
4043                 ? Builder.CreateIntCast(
4044                       EmitScalarExpr(Length), IntPtrTy,
4045                       Length->getType()->hasSignedIntegerRepresentation())
4046                 : llvm::ConstantInt::get(IntPtrTy, ConstLength);
4047         Idx = Builder.CreateAdd(LowerBoundVal, LengthVal, "lb_add_len",
4048                                 /*HasNUW=*/false,
4049                                 !getLangOpts().isSignedOverflowDefined());
4050         if (Length && LowerBound) {
4051           Idx = Builder.CreateSub(
4052               Idx, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "idx_sub_1",
4053               /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4054         }
4055       } else
4056         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength + ConstLowerBound);
4057     } else {
4058       // Idx = ArraySize - 1;
4059       QualType ArrayTy = BaseTy->isPointerType()
4060                              ? E->getBase()->IgnoreParenImpCasts()->getType()
4061                              : BaseTy;
4062       if (auto *VAT = C.getAsVariableArrayType(ArrayTy)) {
4063         Length = VAT->getSizeExpr();
4064         if (std::optional<llvm::APSInt> L = Length->getIntegerConstantExpr(C)) {
4065           ConstLength = *L;
4066           Length = nullptr;
4067         }
4068       } else {
4069         auto *CAT = C.getAsConstantArrayType(ArrayTy);
4070         ConstLength = CAT->getSize();
4071       }
4072       if (Length) {
4073         auto *LengthVal = Builder.CreateIntCast(
4074             EmitScalarExpr(Length), IntPtrTy,
4075             Length->getType()->hasSignedIntegerRepresentation());
4076         Idx = Builder.CreateSub(
4077             LengthVal, llvm::ConstantInt::get(IntPtrTy, /*V=*/1), "len_sub_1",
4078             /*HasNUW=*/false, !getLangOpts().isSignedOverflowDefined());
4079       } else {
4080         ConstLength = ConstLength.zextOrTrunc(PointerWidthInBits);
4081         --ConstLength;
4082         Idx = llvm::ConstantInt::get(IntPtrTy, ConstLength);
4083       }
4084     }
4085   }
4086   assert(Idx);
4087 
4088   Address EltPtr = Address::invalid();
4089   LValueBaseInfo BaseInfo;
4090   TBAAAccessInfo TBAAInfo;
4091   if (auto *VLA = getContext().getAsVariableArrayType(ResultExprTy)) {
4092     // The base must be a pointer, which is not an aggregate.  Emit
4093     // it.  It needs to be emitted first in case it's what captures
4094     // the VLA bounds.
4095     Address Base =
4096         emitOMPArraySectionBase(*this, E->getBase(), BaseInfo, TBAAInfo,
4097                                 BaseTy, VLA->getElementType(), IsLowerBound);
4098     // The element count here is the total number of non-VLA elements.
4099     llvm::Value *NumElements = getVLASize(VLA).NumElts;
4100 
4101     // Effectively, the multiply by the VLA size is part of the GEP.
4102     // GEP indexes are signed, and scaling an index isn't permitted to
4103     // signed-overflow, so we use the same semantics for our explicit
4104     // multiply.  We suppress this if overflow is not undefined behavior.
4105     if (getLangOpts().isSignedOverflowDefined())
4106       Idx = Builder.CreateMul(Idx, NumElements);
4107     else
4108       Idx = Builder.CreateNSWMul(Idx, NumElements);
4109     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, VLA->getElementType(),
4110                                    !getLangOpts().isSignedOverflowDefined(),
4111                                    /*signedIndices=*/false, E->getExprLoc());
4112   } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
4113     // If this is A[i] where A is an array, the frontend will have decayed the
4114     // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
4115     // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
4116     // "gep x, i" here.  Emit one "gep A, 0, i".
4117     assert(Array->getType()->isArrayType() &&
4118            "Array to pointer decay must have array source type!");
4119     LValue ArrayLV;
4120     // For simple multidimensional array indexing, set the 'accessed' flag for
4121     // better bounds-checking of the base expression.
4122     if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Array))
4123       ArrayLV = EmitArraySubscriptExpr(ASE, /*Accessed*/ true);
4124     else
4125       ArrayLV = EmitLValue(Array);
4126 
4127     // Propagate the alignment from the array itself to the result.
4128     EltPtr = emitArraySubscriptGEP(
4129         *this, ArrayLV.getAddress(*this), {CGM.getSize(CharUnits::Zero()), Idx},
4130         ResultExprTy, !getLangOpts().isSignedOverflowDefined(),
4131         /*signedIndices=*/false, E->getExprLoc());
4132     BaseInfo = ArrayLV.getBaseInfo();
4133     TBAAInfo = CGM.getTBAAInfoForSubobject(ArrayLV, ResultExprTy);
4134   } else {
4135     Address Base = emitOMPArraySectionBase(*this, E->getBase(), BaseInfo,
4136                                            TBAAInfo, BaseTy, ResultExprTy,
4137                                            IsLowerBound);
4138     EltPtr = emitArraySubscriptGEP(*this, Base, Idx, ResultExprTy,
4139                                    !getLangOpts().isSignedOverflowDefined(),
4140                                    /*signedIndices=*/false, E->getExprLoc());
4141   }
4142 
4143   return MakeAddrLValue(EltPtr, ResultExprTy, BaseInfo, TBAAInfo);
4144 }
4145 
4146 LValue CodeGenFunction::
EmitExtVectorElementExpr(const ExtVectorElementExpr * E)4147 EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
4148   // Emit the base vector as an l-value.
4149   LValue Base;
4150 
4151   // ExtVectorElementExpr's base can either be a vector or pointer to vector.
4152   if (E->isArrow()) {
4153     // If it is a pointer to a vector, emit the address and form an lvalue with
4154     // it.
4155     LValueBaseInfo BaseInfo;
4156     TBAAAccessInfo TBAAInfo;
4157     Address Ptr = EmitPointerWithAlignment(E->getBase(), &BaseInfo, &TBAAInfo);
4158     const auto *PT = E->getBase()->getType()->castAs<PointerType>();
4159     Base = MakeAddrLValue(Ptr, PT->getPointeeType(), BaseInfo, TBAAInfo);
4160     Base.getQuals().removeObjCGCAttr();
4161   } else if (E->getBase()->isGLValue()) {
4162     // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
4163     // emit the base as an lvalue.
4164     assert(E->getBase()->getType()->isVectorType());
4165     Base = EmitLValue(E->getBase());
4166   } else {
4167     // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
4168     assert(E->getBase()->getType()->isVectorType() &&
4169            "Result must be a vector");
4170     llvm::Value *Vec = EmitScalarExpr(E->getBase());
4171 
4172     // Store the vector to memory (because LValue wants an address).
4173     Address VecMem = CreateMemTemp(E->getBase()->getType());
4174     Builder.CreateStore(Vec, VecMem);
4175     Base = MakeAddrLValue(VecMem, E->getBase()->getType(),
4176                           AlignmentSource::Decl);
4177   }
4178 
4179   QualType type =
4180     E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
4181 
4182   // Encode the element access list into a vector of unsigned indices.
4183   SmallVector<uint32_t, 4> Indices;
4184   E->getEncodedElementAccess(Indices);
4185 
4186   if (Base.isSimple()) {
4187     llvm::Constant *CV =
4188         llvm::ConstantDataVector::get(getLLVMContext(), Indices);
4189     return LValue::MakeExtVectorElt(Base.getAddress(*this), CV, type,
4190                                     Base.getBaseInfo(), TBAAAccessInfo());
4191   }
4192   assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
4193 
4194   llvm::Constant *BaseElts = Base.getExtVectorElts();
4195   SmallVector<llvm::Constant *, 4> CElts;
4196 
4197   for (unsigned i = 0, e = Indices.size(); i != e; ++i)
4198     CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
4199   llvm::Constant *CV = llvm::ConstantVector::get(CElts);
4200   return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,
4201                                   Base.getBaseInfo(), TBAAAccessInfo());
4202 }
4203 
EmitMemberExpr(const MemberExpr * E)4204 LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
4205   if (DeclRefExpr *DRE = tryToConvertMemberExprToDeclRefExpr(*this, E)) {
4206     EmitIgnoredExpr(E->getBase());
4207     return EmitDeclRefLValue(DRE);
4208   }
4209 
4210   Expr *BaseExpr = E->getBase();
4211   // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
4212   LValue BaseLV;
4213   if (E->isArrow()) {
4214     LValueBaseInfo BaseInfo;
4215     TBAAAccessInfo TBAAInfo;
4216     Address Addr = EmitPointerWithAlignment(BaseExpr, &BaseInfo, &TBAAInfo);
4217     QualType PtrTy = BaseExpr->getType()->getPointeeType();
4218     SanitizerSet SkippedChecks;
4219     bool IsBaseCXXThis = IsWrappedCXXThis(BaseExpr);
4220     if (IsBaseCXXThis)
4221       SkippedChecks.set(SanitizerKind::Alignment, true);
4222     if (IsBaseCXXThis || isa<DeclRefExpr>(BaseExpr))
4223       SkippedChecks.set(SanitizerKind::Null, true);
4224     EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Addr.getPointer(), PtrTy,
4225                   /*Alignment=*/CharUnits::Zero(), SkippedChecks);
4226     BaseLV = MakeAddrLValue(Addr, PtrTy, BaseInfo, TBAAInfo);
4227   } else
4228     BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
4229 
4230   NamedDecl *ND = E->getMemberDecl();
4231   if (auto *Field = dyn_cast<FieldDecl>(ND)) {
4232     LValue LV = EmitLValueForField(BaseLV, Field);
4233     setObjCGCLValueClass(getContext(), E, LV);
4234     if (getLangOpts().OpenMP) {
4235       // If the member was explicitly marked as nontemporal, mark it as
4236       // nontemporal. If the base lvalue is marked as nontemporal, mark access
4237       // to children as nontemporal too.
4238       if ((IsWrappedCXXThis(BaseExpr) &&
4239            CGM.getOpenMPRuntime().isNontemporalDecl(Field)) ||
4240           BaseLV.isNontemporal())
4241         LV.setNontemporal(/*Value=*/true);
4242     }
4243     return LV;
4244   }
4245 
4246   if (const auto *FD = dyn_cast<FunctionDecl>(ND))
4247     return EmitFunctionDeclLValue(*this, E, FD);
4248 
4249   llvm_unreachable("Unhandled member declaration!");
4250 }
4251 
4252 /// Given that we are currently emitting a lambda, emit an l-value for
4253 /// one of its members.
EmitLValueForLambdaField(const FieldDecl * Field)4254 LValue CodeGenFunction::EmitLValueForLambdaField(const FieldDecl *Field) {
4255   if (CurCodeDecl) {
4256     assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent()->isLambda());
4257     assert(cast<CXXMethodDecl>(CurCodeDecl)->getParent() == Field->getParent());
4258   }
4259   QualType LambdaTagType =
4260     getContext().getTagDeclType(Field->getParent());
4261   LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue, LambdaTagType);
4262   return EmitLValueForField(LambdaLV, Field);
4263 }
4264 
4265 /// Get the field index in the debug info. The debug info structure/union
4266 /// will ignore the unnamed bitfields.
getDebugInfoFIndex(const RecordDecl * Rec,unsigned FieldIndex)4267 unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec,
4268                                              unsigned FieldIndex) {
4269   unsigned I = 0, Skipped = 0;
4270 
4271   for (auto *F : Rec->getDefinition()->fields()) {
4272     if (I == FieldIndex)
4273       break;
4274     if (F->isUnnamedBitfield())
4275       Skipped++;
4276     I++;
4277   }
4278 
4279   return FieldIndex - Skipped;
4280 }
4281 
4282 /// Get the address of a zero-sized field within a record. The resulting
4283 /// address doesn't necessarily have the right type.
emitAddrOfZeroSizeField(CodeGenFunction & CGF,Address Base,const FieldDecl * Field)4284 static Address emitAddrOfZeroSizeField(CodeGenFunction &CGF, Address Base,
4285                                        const FieldDecl *Field) {
4286   CharUnits Offset = CGF.getContext().toCharUnitsFromBits(
4287       CGF.getContext().getFieldOffset(Field));
4288   if (Offset.isZero())
4289     return Base;
4290   Base = CGF.Builder.CreateElementBitCast(Base, CGF.Int8Ty);
4291   return CGF.Builder.CreateConstInBoundsByteGEP(Base, Offset);
4292 }
4293 
4294 /// Drill down to the storage of a field without walking into
4295 /// reference types.
4296 ///
4297 /// The resulting address doesn't necessarily have the right type.
emitAddrOfFieldStorage(CodeGenFunction & CGF,Address base,const FieldDecl * field)4298 static Address emitAddrOfFieldStorage(CodeGenFunction &CGF, Address base,
4299                                       const FieldDecl *field) {
4300   if (field->isZeroSize(CGF.getContext()))
4301     return emitAddrOfZeroSizeField(CGF, base, field);
4302 
4303   const RecordDecl *rec = field->getParent();
4304 
4305   unsigned idx =
4306     CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4307 
4308   return CGF.Builder.CreateStructGEP(base, idx, field->getName());
4309 }
4310 
emitPreserveStructAccess(CodeGenFunction & CGF,LValue base,Address addr,const FieldDecl * field)4311 static Address emitPreserveStructAccess(CodeGenFunction &CGF, LValue base,
4312                                         Address addr, const FieldDecl *field) {
4313   const RecordDecl *rec = field->getParent();
4314   llvm::DIType *DbgInfo = CGF.getDebugInfo()->getOrCreateStandaloneType(
4315       base.getType(), rec->getLocation());
4316 
4317   unsigned idx =
4318       CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
4319 
4320   return CGF.Builder.CreatePreserveStructAccessIndex(
4321       addr, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo);
4322 }
4323 
hasAnyVptr(const QualType Type,const ASTContext & Context)4324 static bool hasAnyVptr(const QualType Type, const ASTContext &Context) {
4325   const auto *RD = Type.getTypePtr()->getAsCXXRecordDecl();
4326   if (!RD)
4327     return false;
4328 
4329   if (RD->isDynamicClass())
4330     return true;
4331 
4332   for (const auto &Base : RD->bases())
4333     if (hasAnyVptr(Base.getType(), Context))
4334       return true;
4335 
4336   for (const FieldDecl *Field : RD->fields())
4337     if (hasAnyVptr(Field->getType(), Context))
4338       return true;
4339 
4340   return false;
4341 }
4342 
EmitLValueForField(LValue base,const FieldDecl * field)4343 LValue CodeGenFunction::EmitLValueForField(LValue base,
4344                                            const FieldDecl *field) {
4345   LValueBaseInfo BaseInfo = base.getBaseInfo();
4346 
4347   if (field->isBitField()) {
4348     const CGRecordLayout &RL =
4349         CGM.getTypes().getCGRecordLayout(field->getParent());
4350     const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
4351     const bool UseVolatile = isAAPCS(CGM.getTarget()) &&
4352                              CGM.getCodeGenOpts().AAPCSBitfieldWidth &&
4353                              Info.VolatileStorageSize != 0 &&
4354                              field->getType()
4355                                  .withCVRQualifiers(base.getVRQualifiers())
4356                                  .isVolatileQualified();
4357     Address Addr = base.getAddress(*this);
4358     unsigned Idx = RL.getLLVMFieldNo(field);
4359     const RecordDecl *rec = field->getParent();
4360     if (!UseVolatile) {
4361       if (!IsInPreservedAIRegion &&
4362           (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4363         if (Idx != 0)
4364           // For structs, we GEP to the field that the record layout suggests.
4365           Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
4366       } else {
4367         llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType(
4368             getContext().getRecordType(rec), rec->getLocation());
4369         Addr = Builder.CreatePreserveStructAccessIndex(
4370             Addr, Idx, getDebugInfoFIndex(rec, field->getFieldIndex()),
4371             DbgInfo);
4372       }
4373     }
4374     const unsigned SS =
4375         UseVolatile ? Info.VolatileStorageSize : Info.StorageSize;
4376     // Get the access type.
4377     llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), SS);
4378     if (Addr.getElementType() != FieldIntTy)
4379       Addr = Builder.CreateElementBitCast(Addr, FieldIntTy);
4380     if (UseVolatile) {
4381       const unsigned VolatileOffset = Info.VolatileStorageOffset.getQuantity();
4382       if (VolatileOffset)
4383         Addr = Builder.CreateConstInBoundsGEP(Addr, VolatileOffset);
4384     }
4385 
4386     QualType fieldType =
4387         field->getType().withCVRQualifiers(base.getVRQualifiers());
4388     // TODO: Support TBAA for bit fields.
4389     LValueBaseInfo FieldBaseInfo(BaseInfo.getAlignmentSource());
4390     return LValue::MakeBitfield(Addr, Info, fieldType, FieldBaseInfo,
4391                                 TBAAAccessInfo());
4392   }
4393 
4394   // Fields of may-alias structures are may-alias themselves.
4395   // FIXME: this should get propagated down through anonymous structs
4396   // and unions.
4397   QualType FieldType = field->getType();
4398   const RecordDecl *rec = field->getParent();
4399   AlignmentSource BaseAlignSource = BaseInfo.getAlignmentSource();
4400   LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(BaseAlignSource));
4401   TBAAAccessInfo FieldTBAAInfo;
4402   if (base.getTBAAInfo().isMayAlias() ||
4403           rec->hasAttr<MayAliasAttr>() || FieldType->isVectorType()) {
4404     FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4405   } else if (rec->isUnion()) {
4406     // TODO: Support TBAA for unions.
4407     FieldTBAAInfo = TBAAAccessInfo::getMayAliasInfo();
4408   } else {
4409     // If no base type been assigned for the base access, then try to generate
4410     // one for this base lvalue.
4411     FieldTBAAInfo = base.getTBAAInfo();
4412     if (!FieldTBAAInfo.BaseType) {
4413         FieldTBAAInfo.BaseType = CGM.getTBAABaseTypeInfo(base.getType());
4414         assert(!FieldTBAAInfo.Offset &&
4415                "Nonzero offset for an access with no base type!");
4416     }
4417 
4418     // Adjust offset to be relative to the base type.
4419     const ASTRecordLayout &Layout =
4420         getContext().getASTRecordLayout(field->getParent());
4421     unsigned CharWidth = getContext().getCharWidth();
4422     if (FieldTBAAInfo.BaseType)
4423       FieldTBAAInfo.Offset +=
4424           Layout.getFieldOffset(field->getFieldIndex()) / CharWidth;
4425 
4426     // Update the final access type and size.
4427     FieldTBAAInfo.AccessType = CGM.getTBAATypeInfo(FieldType);
4428     FieldTBAAInfo.Size =
4429         getContext().getTypeSizeInChars(FieldType).getQuantity();
4430   }
4431 
4432   Address addr = base.getAddress(*this);
4433   if (auto *ClassDef = dyn_cast<CXXRecordDecl>(rec)) {
4434     if (CGM.getCodeGenOpts().StrictVTablePointers &&
4435         ClassDef->isDynamicClass()) {
4436       // Getting to any field of dynamic object requires stripping dynamic
4437       // information provided by invariant.group.  This is because accessing
4438       // fields may leak the real address of dynamic object, which could result
4439       // in miscompilation when leaked pointer would be compared.
4440       auto *stripped = Builder.CreateStripInvariantGroup(addr.getPointer());
4441       addr = Address(stripped, addr.getElementType(), addr.getAlignment());
4442     }
4443   }
4444 
4445   unsigned RecordCVR = base.getVRQualifiers();
4446   if (rec->isUnion()) {
4447     // For unions, there is no pointer adjustment.
4448     if (CGM.getCodeGenOpts().StrictVTablePointers &&
4449         hasAnyVptr(FieldType, getContext()))
4450       // Because unions can easily skip invariant.barriers, we need to add
4451       // a barrier every time CXXRecord field with vptr is referenced.
4452       addr = Builder.CreateLaunderInvariantGroup(addr);
4453 
4454     if (IsInPreservedAIRegion ||
4455         (getDebugInfo() && rec->hasAttr<BPFPreserveAccessIndexAttr>())) {
4456       // Remember the original union field index
4457       llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateStandaloneType(base.getType(),
4458           rec->getLocation());
4459       addr = Address(
4460           Builder.CreatePreserveUnionAccessIndex(
4461               addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo),
4462           addr.getElementType(), addr.getAlignment());
4463     }
4464 
4465     if (FieldType->isReferenceType())
4466       addr = Builder.CreateElementBitCast(
4467           addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName());
4468   } else {
4469     if (!IsInPreservedAIRegion &&
4470         (!getDebugInfo() || !rec->hasAttr<BPFPreserveAccessIndexAttr>()))
4471       // For structs, we GEP to the field that the record layout suggests.
4472       addr = emitAddrOfFieldStorage(*this, addr, field);
4473     else
4474       // Remember the original struct field index
4475       addr = emitPreserveStructAccess(*this, base, addr, field);
4476   }
4477 
4478   // If this is a reference field, load the reference right now.
4479   if (FieldType->isReferenceType()) {
4480     LValue RefLVal =
4481         MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4482     if (RecordCVR & Qualifiers::Volatile)
4483       RefLVal.getQuals().addVolatile();
4484     addr = EmitLoadOfReference(RefLVal, &FieldBaseInfo, &FieldTBAAInfo);
4485 
4486     // Qualifiers on the struct don't apply to the referencee.
4487     RecordCVR = 0;
4488     FieldType = FieldType->getPointeeType();
4489   }
4490 
4491   // Make sure that the address is pointing to the right type.  This is critical
4492   // for both unions and structs.  A union needs a bitcast, a struct element
4493   // will need a bitcast if the LLVM type laid out doesn't match the desired
4494   // type.
4495   addr = Builder.CreateElementBitCast(
4496       addr, CGM.getTypes().ConvertTypeForMem(FieldType), field->getName());
4497 
4498   if (field->hasAttr<AnnotateAttr>())
4499     addr = EmitFieldAnnotations(field, addr);
4500 
4501   LValue LV = MakeAddrLValue(addr, FieldType, FieldBaseInfo, FieldTBAAInfo);
4502   LV.getQuals().addCVRQualifiers(RecordCVR);
4503 
4504   // __weak attribute on a field is ignored.
4505   if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
4506     LV.getQuals().removeObjCGCAttr();
4507 
4508   return LV;
4509 }
4510 
4511 LValue
EmitLValueForFieldInitialization(LValue Base,const FieldDecl * Field)4512 CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
4513                                                   const FieldDecl *Field) {
4514   QualType FieldType = Field->getType();
4515 
4516   if (!FieldType->isReferenceType())
4517     return EmitLValueForField(Base, Field);
4518 
4519   Address V = emitAddrOfFieldStorage(*this, Base.getAddress(*this), Field);
4520 
4521   // Make sure that the address is pointing to the right type.
4522   llvm::Type *llvmType = ConvertTypeForMem(FieldType);
4523   V = Builder.CreateElementBitCast(V, llvmType, Field->getName());
4524 
4525   // TODO: Generate TBAA information that describes this access as a structure
4526   // member access and not just an access to an object of the field's type. This
4527   // should be similar to what we do in EmitLValueForField().
4528   LValueBaseInfo BaseInfo = Base.getBaseInfo();
4529   AlignmentSource FieldAlignSource = BaseInfo.getAlignmentSource();
4530   LValueBaseInfo FieldBaseInfo(getFieldAlignmentSource(FieldAlignSource));
4531   return MakeAddrLValue(V, FieldType, FieldBaseInfo,
4532                         CGM.getTBAAInfoForSubobject(Base, FieldType));
4533 }
4534 
EmitCompoundLiteralLValue(const CompoundLiteralExpr * E)4535 LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
4536   if (E->isFileScope()) {
4537     ConstantAddress GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
4538     return MakeAddrLValue(GlobalPtr, E->getType(), AlignmentSource::Decl);
4539   }
4540   if (E->getType()->isVariablyModifiedType())
4541     // make sure to emit the VLA size.
4542     EmitVariablyModifiedType(E->getType());
4543 
4544   Address DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
4545   const Expr *InitExpr = E->getInitializer();
4546   LValue Result = MakeAddrLValue(DeclPtr, E->getType(), AlignmentSource::Decl);
4547 
4548   EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
4549                    /*Init*/ true);
4550 
4551   // Block-scope compound literals are destroyed at the end of the enclosing
4552   // scope in C.
4553   if (!getLangOpts().CPlusPlus)
4554     if (QualType::DestructionKind DtorKind = E->getType().isDestructedType())
4555       pushLifetimeExtendedDestroy(getCleanupKind(DtorKind), DeclPtr,
4556                                   E->getType(), getDestroyer(DtorKind),
4557                                   DtorKind & EHCleanup);
4558 
4559   return Result;
4560 }
4561 
EmitInitListLValue(const InitListExpr * E)4562 LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
4563   if (!E->isGLValue())
4564     // Initializing an aggregate temporary in C++11: T{...}.
4565     return EmitAggExprToLValue(E);
4566 
4567   // An lvalue initializer list must be initializing a reference.
4568   assert(E->isTransparent() && "non-transparent glvalue init list");
4569   return EmitLValue(E->getInit(0));
4570 }
4571 
4572 /// Emit the operand of a glvalue conditional operator. This is either a glvalue
4573 /// or a (possibly-parenthesized) throw-expression. If this is a throw, no
4574 /// LValue is returned and the current block has been terminated.
EmitLValueOrThrowExpression(CodeGenFunction & CGF,const Expr * Operand)4575 static std::optional<LValue> EmitLValueOrThrowExpression(CodeGenFunction &CGF,
4576                                                          const Expr *Operand) {
4577   if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Operand->IgnoreParens())) {
4578     CGF.EmitCXXThrowExpr(ThrowExpr, /*KeepInsertionPoint*/false);
4579     return std::nullopt;
4580   }
4581 
4582   return CGF.EmitLValue(Operand);
4583 }
4584 
4585 namespace {
4586 // Handle the case where the condition is a constant evaluatable simple integer,
4587 // which means we don't have to separately handle the true/false blocks.
HandleConditionalOperatorLValueSimpleCase(CodeGenFunction & CGF,const AbstractConditionalOperator * E)4588 std::optional<LValue> HandleConditionalOperatorLValueSimpleCase(
4589     CodeGenFunction &CGF, const AbstractConditionalOperator *E) {
4590   const Expr *condExpr = E->getCond();
4591   bool CondExprBool;
4592   if (CGF.ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
4593     const Expr *Live = E->getTrueExpr(), *Dead = E->getFalseExpr();
4594     if (!CondExprBool)
4595       std::swap(Live, Dead);
4596 
4597     if (!CGF.ContainsLabel(Dead)) {
4598       // If the true case is live, we need to track its region.
4599       if (CondExprBool)
4600         CGF.incrementProfileCounter(E);
4601       // If a throw expression we emit it and return an undefined lvalue
4602       // because it can't be used.
4603       if (auto *ThrowExpr = dyn_cast<CXXThrowExpr>(Live->IgnoreParens())) {
4604         CGF.EmitCXXThrowExpr(ThrowExpr);
4605         llvm::Type *ElemTy = CGF.ConvertType(Dead->getType());
4606         llvm::Type *Ty = llvm::PointerType::getUnqual(ElemTy);
4607         return CGF.MakeAddrLValue(
4608             Address(llvm::UndefValue::get(Ty), ElemTy, CharUnits::One()),
4609             Dead->getType());
4610       }
4611       return CGF.EmitLValue(Live);
4612     }
4613   }
4614   return std::nullopt;
4615 }
4616 struct ConditionalInfo {
4617   llvm::BasicBlock *lhsBlock, *rhsBlock;
4618   std::optional<LValue> LHS, RHS;
4619 };
4620 
4621 // Create and generate the 3 blocks for a conditional operator.
4622 // Leaves the 'current block' in the continuation basic block.
4623 template<typename FuncTy>
EmitConditionalBlocks(CodeGenFunction & CGF,const AbstractConditionalOperator * E,const FuncTy & BranchGenFunc)4624 ConditionalInfo EmitConditionalBlocks(CodeGenFunction &CGF,
4625                                       const AbstractConditionalOperator *E,
4626                                       const FuncTy &BranchGenFunc) {
4627   ConditionalInfo Info{CGF.createBasicBlock("cond.true"),
4628                        CGF.createBasicBlock("cond.false"), std::nullopt,
4629                        std::nullopt};
4630   llvm::BasicBlock *endBlock = CGF.createBasicBlock("cond.end");
4631 
4632   CodeGenFunction::ConditionalEvaluation eval(CGF);
4633   CGF.EmitBranchOnBoolExpr(E->getCond(), Info.lhsBlock, Info.rhsBlock,
4634                            CGF.getProfileCount(E));
4635 
4636   // Any temporaries created here are conditional.
4637   CGF.EmitBlock(Info.lhsBlock);
4638   CGF.incrementProfileCounter(E);
4639   eval.begin(CGF);
4640   Info.LHS = BranchGenFunc(CGF, E->getTrueExpr());
4641   eval.end(CGF);
4642   Info.lhsBlock = CGF.Builder.GetInsertBlock();
4643 
4644   if (Info.LHS)
4645     CGF.Builder.CreateBr(endBlock);
4646 
4647   // Any temporaries created here are conditional.
4648   CGF.EmitBlock(Info.rhsBlock);
4649   eval.begin(CGF);
4650   Info.RHS = BranchGenFunc(CGF, E->getFalseExpr());
4651   eval.end(CGF);
4652   Info.rhsBlock = CGF.Builder.GetInsertBlock();
4653   CGF.EmitBlock(endBlock);
4654 
4655   return Info;
4656 }
4657 } // namespace
4658 
EmitIgnoredConditionalOperator(const AbstractConditionalOperator * E)4659 void CodeGenFunction::EmitIgnoredConditionalOperator(
4660     const AbstractConditionalOperator *E) {
4661   if (!E->isGLValue()) {
4662     // ?: here should be an aggregate.
4663     assert(hasAggregateEvaluationKind(E->getType()) &&
4664            "Unexpected conditional operator!");
4665     return (void)EmitAggExprToLValue(E);
4666   }
4667 
4668   OpaqueValueMapping binding(*this, E);
4669   if (HandleConditionalOperatorLValueSimpleCase(*this, E))
4670     return;
4671 
4672   EmitConditionalBlocks(*this, E, [](CodeGenFunction &CGF, const Expr *E) {
4673     CGF.EmitIgnoredExpr(E);
4674     return LValue{};
4675   });
4676 }
EmitConditionalOperatorLValue(const AbstractConditionalOperator * expr)4677 LValue CodeGenFunction::EmitConditionalOperatorLValue(
4678     const AbstractConditionalOperator *expr) {
4679   if (!expr->isGLValue()) {
4680     // ?: here should be an aggregate.
4681     assert(hasAggregateEvaluationKind(expr->getType()) &&
4682            "Unexpected conditional operator!");
4683     return EmitAggExprToLValue(expr);
4684   }
4685 
4686   OpaqueValueMapping binding(*this, expr);
4687   if (std::optional<LValue> Res =
4688           HandleConditionalOperatorLValueSimpleCase(*this, expr))
4689     return *Res;
4690 
4691   ConditionalInfo Info = EmitConditionalBlocks(
4692       *this, expr, [](CodeGenFunction &CGF, const Expr *E) {
4693         return EmitLValueOrThrowExpression(CGF, E);
4694       });
4695 
4696   if ((Info.LHS && !Info.LHS->isSimple()) ||
4697       (Info.RHS && !Info.RHS->isSimple()))
4698     return EmitUnsupportedLValue(expr, "conditional operator");
4699 
4700   if (Info.LHS && Info.RHS) {
4701     Address lhsAddr = Info.LHS->getAddress(*this);
4702     Address rhsAddr = Info.RHS->getAddress(*this);
4703     llvm::PHINode *phi = Builder.CreatePHI(lhsAddr.getType(), 2, "cond-lvalue");
4704     phi->addIncoming(lhsAddr.getPointer(), Info.lhsBlock);
4705     phi->addIncoming(rhsAddr.getPointer(), Info.rhsBlock);
4706     Address result(phi, lhsAddr.getElementType(),
4707                    std::min(lhsAddr.getAlignment(), rhsAddr.getAlignment()));
4708     AlignmentSource alignSource =
4709         std::max(Info.LHS->getBaseInfo().getAlignmentSource(),
4710                  Info.RHS->getBaseInfo().getAlignmentSource());
4711     TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForConditionalOperator(
4712         Info.LHS->getTBAAInfo(), Info.RHS->getTBAAInfo());
4713     return MakeAddrLValue(result, expr->getType(), LValueBaseInfo(alignSource),
4714                           TBAAInfo);
4715   } else {
4716     assert((Info.LHS || Info.RHS) &&
4717            "both operands of glvalue conditional are throw-expressions?");
4718     return Info.LHS ? *Info.LHS : *Info.RHS;
4719   }
4720 }
4721 
4722 /// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
4723 /// type. If the cast is to a reference, we can have the usual lvalue result,
4724 /// otherwise if a cast is needed by the code generator in an lvalue context,
4725 /// then it must mean that we need the address of an aggregate in order to
4726 /// access one of its members.  This can happen for all the reasons that casts
4727 /// are permitted with aggregate result, including noop aggregate casts, and
4728 /// cast from scalar to union.
EmitCastLValue(const CastExpr * E)4729 LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
4730   switch (E->getCastKind()) {
4731   case CK_ToVoid:
4732   case CK_BitCast:
4733   case CK_LValueToRValueBitCast:
4734   case CK_ArrayToPointerDecay:
4735   case CK_FunctionToPointerDecay:
4736   case CK_NullToMemberPointer:
4737   case CK_NullToPointer:
4738   case CK_IntegralToPointer:
4739   case CK_PointerToIntegral:
4740   case CK_PointerToBoolean:
4741   case CK_VectorSplat:
4742   case CK_IntegralCast:
4743   case CK_BooleanToSignedIntegral:
4744   case CK_IntegralToBoolean:
4745   case CK_IntegralToFloating:
4746   case CK_FloatingToIntegral:
4747   case CK_FloatingToBoolean:
4748   case CK_FloatingCast:
4749   case CK_FloatingRealToComplex:
4750   case CK_FloatingComplexToReal:
4751   case CK_FloatingComplexToBoolean:
4752   case CK_FloatingComplexCast:
4753   case CK_FloatingComplexToIntegralComplex:
4754   case CK_IntegralRealToComplex:
4755   case CK_IntegralComplexToReal:
4756   case CK_IntegralComplexToBoolean:
4757   case CK_IntegralComplexCast:
4758   case CK_IntegralComplexToFloatingComplex:
4759   case CK_DerivedToBaseMemberPointer:
4760   case CK_BaseToDerivedMemberPointer:
4761   case CK_MemberPointerToBoolean:
4762   case CK_ReinterpretMemberPointer:
4763   case CK_AnyPointerToBlockPointerCast:
4764   case CK_ARCProduceObject:
4765   case CK_ARCConsumeObject:
4766   case CK_ARCReclaimReturnedObject:
4767   case CK_ARCExtendBlockObject:
4768   case CK_CopyAndAutoreleaseBlockObject:
4769   case CK_IntToOCLSampler:
4770   case CK_FloatingToFixedPoint:
4771   case CK_FixedPointToFloating:
4772   case CK_FixedPointCast:
4773   case CK_FixedPointToBoolean:
4774   case CK_FixedPointToIntegral:
4775   case CK_IntegralToFixedPoint:
4776   case CK_MatrixCast:
4777     return EmitUnsupportedLValue(E, "unexpected cast lvalue");
4778 
4779   case CK_Dependent:
4780     llvm_unreachable("dependent cast kind in IR gen!");
4781 
4782   case CK_BuiltinFnToFnPtr:
4783     llvm_unreachable("builtin functions are handled elsewhere");
4784 
4785   // These are never l-values; just use the aggregate emission code.
4786   case CK_NonAtomicToAtomic:
4787   case CK_AtomicToNonAtomic:
4788     return EmitAggExprToLValue(E);
4789 
4790   case CK_Dynamic: {
4791     LValue LV = EmitLValue(E->getSubExpr());
4792     Address V = LV.getAddress(*this);
4793     const auto *DCE = cast<CXXDynamicCastExpr>(E);
4794     return MakeNaturalAlignAddrLValue(EmitDynamicCast(V, DCE), E->getType());
4795   }
4796 
4797   case CK_ConstructorConversion:
4798   case CK_UserDefinedConversion:
4799   case CK_CPointerToObjCPointerCast:
4800   case CK_BlockPointerToObjCPointerCast:
4801   case CK_LValueToRValue:
4802     return EmitLValue(E->getSubExpr());
4803 
4804   case CK_NoOp: {
4805     // CK_NoOp can model a qualification conversion, which can remove an array
4806     // bound and change the IR type.
4807     // FIXME: Once pointee types are removed from IR, remove this.
4808     LValue LV = EmitLValue(E->getSubExpr());
4809     if (LV.isSimple()) {
4810       Address V = LV.getAddress(*this);
4811       if (V.isValid()) {
4812         llvm::Type *T = ConvertTypeForMem(E->getType());
4813         if (V.getElementType() != T)
4814           LV.setAddress(Builder.CreateElementBitCast(V, T));
4815       }
4816     }
4817     return LV;
4818   }
4819 
4820   case CK_UncheckedDerivedToBase:
4821   case CK_DerivedToBase: {
4822     const auto *DerivedClassTy =
4823         E->getSubExpr()->getType()->castAs<RecordType>();
4824     auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
4825 
4826     LValue LV = EmitLValue(E->getSubExpr());
4827     Address This = LV.getAddress(*this);
4828 
4829     // Perform the derived-to-base conversion
4830     Address Base = GetAddressOfBaseClass(
4831         This, DerivedClassDecl, E->path_begin(), E->path_end(),
4832         /*NullCheckValue=*/false, E->getExprLoc());
4833 
4834     // TODO: Support accesses to members of base classes in TBAA. For now, we
4835     // conservatively pretend that the complete object is of the base class
4836     // type.
4837     return MakeAddrLValue(Base, E->getType(), LV.getBaseInfo(),
4838                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4839   }
4840   case CK_ToUnion:
4841     return EmitAggExprToLValue(E);
4842   case CK_BaseToDerived: {
4843     const auto *DerivedClassTy = E->getType()->castAs<RecordType>();
4844     auto *DerivedClassDecl = cast<CXXRecordDecl>(DerivedClassTy->getDecl());
4845 
4846     LValue LV = EmitLValue(E->getSubExpr());
4847 
4848     // Perform the base-to-derived conversion
4849     Address Derived = GetAddressOfDerivedClass(
4850         LV.getAddress(*this), DerivedClassDecl, E->path_begin(), E->path_end(),
4851         /*NullCheckValue=*/false);
4852 
4853     // C++11 [expr.static.cast]p2: Behavior is undefined if a downcast is
4854     // performed and the object is not of the derived type.
4855     if (sanitizePerformTypeCheck())
4856       EmitTypeCheck(TCK_DowncastReference, E->getExprLoc(),
4857                     Derived.getPointer(), E->getType());
4858 
4859     if (SanOpts.has(SanitizerKind::CFIDerivedCast))
4860       EmitVTablePtrCheckForCast(E->getType(), Derived,
4861                                 /*MayBeNull=*/false, CFITCK_DerivedCast,
4862                                 E->getBeginLoc());
4863 
4864     return MakeAddrLValue(Derived, E->getType(), LV.getBaseInfo(),
4865                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4866   }
4867   case CK_LValueBitCast: {
4868     // This must be a reinterpret_cast (or c-style equivalent).
4869     const auto *CE = cast<ExplicitCastExpr>(E);
4870 
4871     CGM.EmitExplicitCastExprType(CE, this);
4872     LValue LV = EmitLValue(E->getSubExpr());
4873     Address V = Builder.CreateElementBitCast(
4874         LV.getAddress(*this),
4875         ConvertTypeForMem(CE->getTypeAsWritten()->getPointeeType()));
4876 
4877     if (SanOpts.has(SanitizerKind::CFIUnrelatedCast))
4878       EmitVTablePtrCheckForCast(E->getType(), V,
4879                                 /*MayBeNull=*/false, CFITCK_UnrelatedCast,
4880                                 E->getBeginLoc());
4881 
4882     return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
4883                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4884   }
4885   case CK_AddressSpaceConversion: {
4886     LValue LV = EmitLValue(E->getSubExpr());
4887     QualType DestTy = getContext().getPointerType(E->getType());
4888     llvm::Value *V = getTargetHooks().performAddrSpaceCast(
4889         *this, LV.getPointer(*this),
4890         E->getSubExpr()->getType().getAddressSpace(),
4891         E->getType().getAddressSpace(), ConvertType(DestTy));
4892     return MakeAddrLValue(Address(V, ConvertTypeForMem(E->getType()),
4893                                   LV.getAddress(*this).getAlignment()),
4894                           E->getType(), LV.getBaseInfo(), LV.getTBAAInfo());
4895   }
4896   case CK_ObjCObjectLValueCast: {
4897     LValue LV = EmitLValue(E->getSubExpr());
4898     Address V = Builder.CreateElementBitCast(LV.getAddress(*this),
4899                                              ConvertType(E->getType()));
4900     return MakeAddrLValue(V, E->getType(), LV.getBaseInfo(),
4901                           CGM.getTBAAInfoForSubobject(LV, E->getType()));
4902   }
4903   case CK_ZeroToOCLOpaqueType:
4904     llvm_unreachable("NULL to OpenCL opaque type lvalue cast is not valid");
4905   }
4906 
4907   llvm_unreachable("Unhandled lvalue cast kind?");
4908 }
4909 
EmitOpaqueValueLValue(const OpaqueValueExpr * e)4910 LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
4911   assert(OpaqueValueMappingData::shouldBindAsLValue(e));
4912   return getOrCreateOpaqueLValueMapping(e);
4913 }
4914 
4915 LValue
getOrCreateOpaqueLValueMapping(const OpaqueValueExpr * e)4916 CodeGenFunction::getOrCreateOpaqueLValueMapping(const OpaqueValueExpr *e) {
4917   assert(OpaqueValueMapping::shouldBindAsLValue(e));
4918 
4919   llvm::DenseMap<const OpaqueValueExpr*,LValue>::iterator
4920       it = OpaqueLValues.find(e);
4921 
4922   if (it != OpaqueLValues.end())
4923     return it->second;
4924 
4925   assert(e->isUnique() && "LValue for a nonunique OVE hasn't been emitted");
4926   return EmitLValue(e->getSourceExpr());
4927 }
4928 
4929 RValue
getOrCreateOpaqueRValueMapping(const OpaqueValueExpr * e)4930 CodeGenFunction::getOrCreateOpaqueRValueMapping(const OpaqueValueExpr *e) {
4931   assert(!OpaqueValueMapping::shouldBindAsLValue(e));
4932 
4933   llvm::DenseMap<const OpaqueValueExpr*,RValue>::iterator
4934       it = OpaqueRValues.find(e);
4935 
4936   if (it != OpaqueRValues.end())
4937     return it->second;
4938 
4939   assert(e->isUnique() && "RValue for a nonunique OVE hasn't been emitted");
4940   return EmitAnyExpr(e->getSourceExpr());
4941 }
4942 
EmitRValueForField(LValue LV,const FieldDecl * FD,SourceLocation Loc)4943 RValue CodeGenFunction::EmitRValueForField(LValue LV,
4944                                            const FieldDecl *FD,
4945                                            SourceLocation Loc) {
4946   QualType FT = FD->getType();
4947   LValue FieldLV = EmitLValueForField(LV, FD);
4948   switch (getEvaluationKind(FT)) {
4949   case TEK_Complex:
4950     return RValue::getComplex(EmitLoadOfComplex(FieldLV, Loc));
4951   case TEK_Aggregate:
4952     return FieldLV.asAggregateRValue(*this);
4953   case TEK_Scalar:
4954     // This routine is used to load fields one-by-one to perform a copy, so
4955     // don't load reference fields.
4956     if (FD->getType()->isReferenceType())
4957       return RValue::get(FieldLV.getPointer(*this));
4958     // Call EmitLoadOfScalar except when the lvalue is a bitfield to emit a
4959     // primitive load.
4960     if (FieldLV.isBitField())
4961       return EmitLoadOfLValue(FieldLV, Loc);
4962     return RValue::get(EmitLoadOfScalar(FieldLV, Loc));
4963   }
4964   llvm_unreachable("bad evaluation kind");
4965 }
4966 
4967 //===--------------------------------------------------------------------===//
4968 //                             Expression Emission
4969 //===--------------------------------------------------------------------===//
4970 
EmitCallExpr(const CallExpr * E,ReturnValueSlot ReturnValue)4971 RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
4972                                      ReturnValueSlot ReturnValue) {
4973   // Builtins never have block type.
4974   if (E->getCallee()->getType()->isBlockPointerType())
4975     return EmitBlockCallExpr(E, ReturnValue);
4976 
4977   if (const auto *CE = dyn_cast<CXXMemberCallExpr>(E))
4978     return EmitCXXMemberCallExpr(CE, ReturnValue);
4979 
4980   if (const auto *CE = dyn_cast<CUDAKernelCallExpr>(E))
4981     return EmitCUDAKernelCallExpr(CE, ReturnValue);
4982 
4983   if (const auto *CE = dyn_cast<CXXOperatorCallExpr>(E))
4984     if (const CXXMethodDecl *MD =
4985           dyn_cast_or_null<CXXMethodDecl>(CE->getCalleeDecl()))
4986       return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
4987 
4988   CGCallee callee = EmitCallee(E->getCallee());
4989 
4990   if (callee.isBuiltin()) {
4991     return EmitBuiltinExpr(callee.getBuiltinDecl(), callee.getBuiltinID(),
4992                            E, ReturnValue);
4993   }
4994 
4995   if (callee.isPseudoDestructor()) {
4996     return EmitCXXPseudoDestructorExpr(callee.getPseudoDestructorExpr());
4997   }
4998 
4999   return EmitCall(E->getCallee()->getType(), callee, E, ReturnValue);
5000 }
5001 
5002 /// Emit a CallExpr without considering whether it might be a subclass.
EmitSimpleCallExpr(const CallExpr * E,ReturnValueSlot ReturnValue)5003 RValue CodeGenFunction::EmitSimpleCallExpr(const CallExpr *E,
5004                                            ReturnValueSlot ReturnValue) {
5005   CGCallee Callee = EmitCallee(E->getCallee());
5006   return EmitCall(E->getCallee()->getType(), Callee, E, ReturnValue);
5007 }
5008 
5009 // Detect the unusual situation where an inline version is shadowed by a
5010 // non-inline version. In that case we should pick the external one
5011 // everywhere. That's GCC behavior too.
OnlyHasInlineBuiltinDeclaration(const FunctionDecl * FD)5012 static bool OnlyHasInlineBuiltinDeclaration(const FunctionDecl *FD) {
5013   for (const FunctionDecl *PD = FD; PD; PD = PD->getPreviousDecl())
5014     if (!PD->isInlineBuiltinDeclaration())
5015       return false;
5016   return true;
5017 }
5018 
EmitDirectCallee(CodeGenFunction & CGF,GlobalDecl GD)5019 static CGCallee EmitDirectCallee(CodeGenFunction &CGF, GlobalDecl GD) {
5020   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
5021 
5022   if (auto builtinID = FD->getBuiltinID()) {
5023     std::string NoBuiltinFD = ("no-builtin-" + FD->getName()).str();
5024     std::string NoBuiltins = "no-builtins";
5025 
5026     StringRef Ident = CGF.CGM.getMangledName(GD);
5027     std::string FDInlineName = (Ident + ".inline").str();
5028 
5029     bool IsPredefinedLibFunction =
5030         CGF.getContext().BuiltinInfo.isPredefinedLibFunction(builtinID);
5031     bool HasAttributeNoBuiltin =
5032         CGF.CurFn->getAttributes().hasFnAttr(NoBuiltinFD) ||
5033         CGF.CurFn->getAttributes().hasFnAttr(NoBuiltins);
5034 
5035     // When directing calling an inline builtin, call it through it's mangled
5036     // name to make it clear it's not the actual builtin.
5037     if (CGF.CurFn->getName() != FDInlineName &&
5038         OnlyHasInlineBuiltinDeclaration(FD)) {
5039       llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
5040       llvm::Function *Fn = llvm::cast<llvm::Function>(CalleePtr);
5041       llvm::Module *M = Fn->getParent();
5042       llvm::Function *Clone = M->getFunction(FDInlineName);
5043       if (!Clone) {
5044         Clone = llvm::Function::Create(Fn->getFunctionType(),
5045                                        llvm::GlobalValue::InternalLinkage,
5046                                        Fn->getAddressSpace(), FDInlineName, M);
5047         Clone->addFnAttr(llvm::Attribute::AlwaysInline);
5048       }
5049       return CGCallee::forDirect(Clone, GD);
5050     }
5051 
5052     // Replaceable builtins provide their own implementation of a builtin. If we
5053     // are in an inline builtin implementation, avoid trivial infinite
5054     // recursion. Honor __attribute__((no_builtin("foo"))) or
5055     // __attribute__((no_builtin)) on the current function unless foo is
5056     // not a predefined library function which means we must generate the
5057     // builtin no matter what.
5058     else if (!IsPredefinedLibFunction || !HasAttributeNoBuiltin)
5059       return CGCallee::forBuiltin(builtinID, FD);
5060   }
5061 
5062   llvm::Constant *CalleePtr = EmitFunctionDeclPointer(CGF.CGM, GD);
5063   if (CGF.CGM.getLangOpts().CUDA && !CGF.CGM.getLangOpts().CUDAIsDevice &&
5064       FD->hasAttr<CUDAGlobalAttr>())
5065     CalleePtr = CGF.CGM.getCUDARuntime().getKernelStub(
5066         cast<llvm::GlobalValue>(CalleePtr->stripPointerCasts()));
5067 
5068   return CGCallee::forDirect(CalleePtr, GD);
5069 }
5070 
EmitCallee(const Expr * E)5071 CGCallee CodeGenFunction::EmitCallee(const Expr *E) {
5072   E = E->IgnoreParens();
5073 
5074   // Look through function-to-pointer decay.
5075   if (auto ICE = dyn_cast<ImplicitCastExpr>(E)) {
5076     if (ICE->getCastKind() == CK_FunctionToPointerDecay ||
5077         ICE->getCastKind() == CK_BuiltinFnToFnPtr) {
5078       return EmitCallee(ICE->getSubExpr());
5079     }
5080 
5081   // Resolve direct calls.
5082   } else if (auto DRE = dyn_cast<DeclRefExpr>(E)) {
5083     if (auto FD = dyn_cast<FunctionDecl>(DRE->getDecl())) {
5084       return EmitDirectCallee(*this, FD);
5085     }
5086   } else if (auto ME = dyn_cast<MemberExpr>(E)) {
5087     if (auto FD = dyn_cast<FunctionDecl>(ME->getMemberDecl())) {
5088       EmitIgnoredExpr(ME->getBase());
5089       return EmitDirectCallee(*this, FD);
5090     }
5091 
5092   // Look through template substitutions.
5093   } else if (auto NTTP = dyn_cast<SubstNonTypeTemplateParmExpr>(E)) {
5094     return EmitCallee(NTTP->getReplacement());
5095 
5096   // Treat pseudo-destructor calls differently.
5097   } else if (auto PDE = dyn_cast<CXXPseudoDestructorExpr>(E)) {
5098     return CGCallee::forPseudoDestructor(PDE);
5099   }
5100 
5101   // Otherwise, we have an indirect reference.
5102   llvm::Value *calleePtr;
5103   QualType functionType;
5104   if (auto ptrType = E->getType()->getAs<PointerType>()) {
5105     calleePtr = EmitScalarExpr(E);
5106     functionType = ptrType->getPointeeType();
5107   } else {
5108     functionType = E->getType();
5109     calleePtr = EmitLValue(E).getPointer(*this);
5110   }
5111   assert(functionType->isFunctionType());
5112 
5113   GlobalDecl GD;
5114   if (const auto *VD =
5115           dyn_cast_or_null<VarDecl>(E->getReferencedDeclOfCallee()))
5116     GD = GlobalDecl(VD);
5117 
5118   CGCalleeInfo calleeInfo(functionType->getAs<FunctionProtoType>(), GD);
5119   CGCallee callee(calleeInfo, calleePtr);
5120   return callee;
5121 }
5122 
EmitBinaryOperatorLValue(const BinaryOperator * E)5123 LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
5124   // Comma expressions just emit their LHS then their RHS as an l-value.
5125   if (E->getOpcode() == BO_Comma) {
5126     EmitIgnoredExpr(E->getLHS());
5127     EnsureInsertPoint();
5128     return EmitLValue(E->getRHS());
5129   }
5130 
5131   if (E->getOpcode() == BO_PtrMemD ||
5132       E->getOpcode() == BO_PtrMemI)
5133     return EmitPointerToDataMemberBinaryExpr(E);
5134 
5135   assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
5136 
5137   // Note that in all of these cases, __block variables need the RHS
5138   // evaluated first just in case the variable gets moved by the RHS.
5139 
5140   switch (getEvaluationKind(E->getType())) {
5141   case TEK_Scalar: {
5142     switch (E->getLHS()->getType().getObjCLifetime()) {
5143     case Qualifiers::OCL_Strong:
5144       return EmitARCStoreStrong(E, /*ignored*/ false).first;
5145 
5146     case Qualifiers::OCL_Autoreleasing:
5147       return EmitARCStoreAutoreleasing(E).first;
5148 
5149     // No reason to do any of these differently.
5150     case Qualifiers::OCL_None:
5151     case Qualifiers::OCL_ExplicitNone:
5152     case Qualifiers::OCL_Weak:
5153       break;
5154     }
5155 
5156     RValue RV = EmitAnyExpr(E->getRHS());
5157     LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);
5158     if (RV.isScalar())
5159       EmitNullabilityCheck(LV, RV.getScalarVal(), E->getExprLoc());
5160     EmitStoreThroughLValue(RV, LV);
5161     if (getLangOpts().OpenMP)
5162       CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
5163                                                                 E->getLHS());
5164     return LV;
5165   }
5166 
5167   case TEK_Complex:
5168     return EmitComplexAssignmentLValue(E);
5169 
5170   case TEK_Aggregate:
5171     return EmitAggExprToLValue(E);
5172   }
5173   llvm_unreachable("bad evaluation kind");
5174 }
5175 
EmitCallExprLValue(const CallExpr * E)5176 LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
5177   RValue RV = EmitCallExpr(E);
5178 
5179   if (!RV.isScalar())
5180     return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5181                           AlignmentSource::Decl);
5182 
5183   assert(E->getCallReturnType(getContext())->isReferenceType() &&
5184          "Can't have a scalar return unless the return type is a "
5185          "reference type!");
5186 
5187   return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
5188 }
5189 
EmitVAArgExprLValue(const VAArgExpr * E)5190 LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
5191   // FIXME: This shouldn't require another copy.
5192   return EmitAggExprToLValue(E);
5193 }
5194 
EmitCXXConstructLValue(const CXXConstructExpr * E)5195 LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
5196   assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
5197          && "binding l-value to type which needs a temporary");
5198   AggValueSlot Slot = CreateAggTemp(E->getType());
5199   EmitCXXConstructExpr(E, Slot);
5200   return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5201 }
5202 
5203 LValue
EmitCXXTypeidLValue(const CXXTypeidExpr * E)5204 CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
5205   return MakeNaturalAlignAddrLValue(EmitCXXTypeidExpr(E), E->getType());
5206 }
5207 
EmitCXXUuidofExpr(const CXXUuidofExpr * E)5208 Address CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
5209   return Builder.CreateElementBitCast(CGM.GetAddrOfMSGuidDecl(E->getGuidDecl()),
5210                                       ConvertType(E->getType()));
5211 }
5212 
EmitCXXUuidofLValue(const CXXUuidofExpr * E)5213 LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
5214   return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType(),
5215                         AlignmentSource::Decl);
5216 }
5217 
5218 LValue
EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr * E)5219 CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
5220   AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
5221   Slot.setExternallyDestructed();
5222   EmitAggExpr(E->getSubExpr(), Slot);
5223   EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddress());
5224   return MakeAddrLValue(Slot.getAddress(), E->getType(), AlignmentSource::Decl);
5225 }
5226 
EmitObjCMessageExprLValue(const ObjCMessageExpr * E)5227 LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
5228   RValue RV = EmitObjCMessageExpr(E);
5229 
5230   if (!RV.isScalar())
5231     return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5232                           AlignmentSource::Decl);
5233 
5234   assert(E->getMethodDecl()->getReturnType()->isReferenceType() &&
5235          "Can't have a scalar return unless the return type is a "
5236          "reference type!");
5237 
5238   return MakeNaturalAlignPointeeAddrLValue(RV.getScalarVal(), E->getType());
5239 }
5240 
EmitObjCSelectorLValue(const ObjCSelectorExpr * E)5241 LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
5242   Address V =
5243     CGM.getObjCRuntime().GetAddrOfSelector(*this, E->getSelector());
5244   return MakeAddrLValue(V, E->getType(), AlignmentSource::Decl);
5245 }
5246 
EmitIvarOffset(const ObjCInterfaceDecl * Interface,const ObjCIvarDecl * Ivar)5247 llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
5248                                              const ObjCIvarDecl *Ivar) {
5249   return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);
5250 }
5251 
5252 llvm::Value *
EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl * Interface,const ObjCIvarDecl * Ivar)5253 CodeGenFunction::EmitIvarOffsetAsPointerDiff(const ObjCInterfaceDecl *Interface,
5254                                              const ObjCIvarDecl *Ivar) {
5255   llvm::Value *OffsetValue = EmitIvarOffset(Interface, Ivar);
5256   QualType PointerDiffType = getContext().getPointerDiffType();
5257   return Builder.CreateZExtOrTrunc(OffsetValue,
5258                                    getTypes().ConvertType(PointerDiffType));
5259 }
5260 
EmitLValueForIvar(QualType ObjectTy,llvm::Value * BaseValue,const ObjCIvarDecl * Ivar,unsigned CVRQualifiers)5261 LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
5262                                           llvm::Value *BaseValue,
5263                                           const ObjCIvarDecl *Ivar,
5264                                           unsigned CVRQualifiers) {
5265   return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,
5266                                                    Ivar, CVRQualifiers);
5267 }
5268 
EmitObjCIvarRefLValue(const ObjCIvarRefExpr * E)5269 LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
5270   // FIXME: A lot of the code below could be shared with EmitMemberExpr.
5271   llvm::Value *BaseValue = nullptr;
5272   const Expr *BaseExpr = E->getBase();
5273   Qualifiers BaseQuals;
5274   QualType ObjectTy;
5275   if (E->isArrow()) {
5276     BaseValue = EmitScalarExpr(BaseExpr);
5277     ObjectTy = BaseExpr->getType()->getPointeeType();
5278     BaseQuals = ObjectTy.getQualifiers();
5279   } else {
5280     LValue BaseLV = EmitLValue(BaseExpr);
5281     BaseValue = BaseLV.getPointer(*this);
5282     ObjectTy = BaseExpr->getType();
5283     BaseQuals = ObjectTy.getQualifiers();
5284   }
5285 
5286   LValue LV =
5287     EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),
5288                       BaseQuals.getCVRQualifiers());
5289   setObjCGCLValueClass(getContext(), E, LV);
5290   return LV;
5291 }
5292 
EmitStmtExprLValue(const StmtExpr * E)5293 LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
5294   // Can only get l-value for message expression returning aggregate type
5295   RValue RV = EmitAnyExprToTemp(E);
5296   return MakeAddrLValue(RV.getAggregateAddress(), E->getType(),
5297                         AlignmentSource::Decl);
5298 }
5299 
EmitCall(QualType CalleeType,const CGCallee & OrigCallee,const CallExpr * E,ReturnValueSlot ReturnValue,llvm::Value * Chain)5300 RValue CodeGenFunction::EmitCall(QualType CalleeType, const CGCallee &OrigCallee,
5301                                  const CallExpr *E, ReturnValueSlot ReturnValue,
5302                                  llvm::Value *Chain) {
5303   // Get the actual function type. The callee type will always be a pointer to
5304   // function type or a block pointer type.
5305   assert(CalleeType->isFunctionPointerType() &&
5306          "Call must have function pointer type!");
5307 
5308   const Decl *TargetDecl =
5309       OrigCallee.getAbstractInfo().getCalleeDecl().getDecl();
5310 
5311   CalleeType = getContext().getCanonicalType(CalleeType);
5312 
5313   auto PointeeType = cast<PointerType>(CalleeType)->getPointeeType();
5314 
5315   CGCallee Callee = OrigCallee;
5316 
5317   if (getLangOpts().CPlusPlus && SanOpts.has(SanitizerKind::Function) &&
5318       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5319     if (llvm::Constant *PrefixSig =
5320             CGM.getTargetCodeGenInfo().getUBSanFunctionSignature(CGM)) {
5321       SanitizerScope SanScope(this);
5322       // Remove any (C++17) exception specifications, to allow calling e.g. a
5323       // noexcept function through a non-noexcept pointer.
5324       auto ProtoTy =
5325         getContext().getFunctionTypeWithExceptionSpec(PointeeType, EST_None);
5326       llvm::Constant *FTRTTIConst =
5327           CGM.GetAddrOfRTTIDescriptor(ProtoTy, /*ForEH=*/true);
5328       llvm::Type *PrefixSigType = PrefixSig->getType();
5329       llvm::StructType *PrefixStructTy = llvm::StructType::get(
5330           CGM.getLLVMContext(), {PrefixSigType, Int32Ty}, /*isPacked=*/true);
5331 
5332       llvm::Value *CalleePtr = Callee.getFunctionPointer();
5333 
5334       llvm::Value *CalleePrefixStruct = Builder.CreateBitCast(
5335           CalleePtr, llvm::PointerType::getUnqual(PrefixStructTy));
5336       llvm::Value *CalleeSigPtr =
5337           Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 0);
5338       llvm::Value *CalleeSig =
5339           Builder.CreateAlignedLoad(PrefixSigType, CalleeSigPtr, getIntAlign());
5340       llvm::Value *CalleeSigMatch = Builder.CreateICmpEQ(CalleeSig, PrefixSig);
5341 
5342       llvm::BasicBlock *Cont = createBasicBlock("cont");
5343       llvm::BasicBlock *TypeCheck = createBasicBlock("typecheck");
5344       Builder.CreateCondBr(CalleeSigMatch, TypeCheck, Cont);
5345 
5346       EmitBlock(TypeCheck);
5347       llvm::Value *CalleeRTTIPtr =
5348           Builder.CreateConstGEP2_32(PrefixStructTy, CalleePrefixStruct, 0, 1);
5349       llvm::Value *CalleeRTTIEncoded =
5350           Builder.CreateAlignedLoad(Int32Ty, CalleeRTTIPtr, getPointerAlign());
5351       llvm::Value *CalleeRTTI =
5352           DecodeAddrUsedInPrologue(CalleePtr, CalleeRTTIEncoded);
5353       llvm::Value *CalleeRTTIMatch =
5354           Builder.CreateICmpEQ(CalleeRTTI, FTRTTIConst);
5355       llvm::Constant *StaticData[] = {EmitCheckSourceLocation(E->getBeginLoc()),
5356                                       EmitCheckTypeDescriptor(CalleeType)};
5357       EmitCheck(std::make_pair(CalleeRTTIMatch, SanitizerKind::Function),
5358                 SanitizerHandler::FunctionTypeMismatch, StaticData,
5359                 {CalleePtr, CalleeRTTI, FTRTTIConst});
5360 
5361       Builder.CreateBr(Cont);
5362       EmitBlock(Cont);
5363     }
5364   }
5365 
5366   const auto *FnType = cast<FunctionType>(PointeeType);
5367 
5368   // If we are checking indirect calls and this call is indirect, check that the
5369   // function pointer is a member of the bit set for the function type.
5370   if (SanOpts.has(SanitizerKind::CFIICall) &&
5371       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5372     SanitizerScope SanScope(this);
5373     EmitSanitizerStatReport(llvm::SanStat_CFI_ICall);
5374 
5375     llvm::Metadata *MD;
5376     if (CGM.getCodeGenOpts().SanitizeCfiICallGeneralizePointers)
5377       MD = CGM.CreateMetadataIdentifierGeneralized(QualType(FnType, 0));
5378     else
5379       MD = CGM.CreateMetadataIdentifierForType(QualType(FnType, 0));
5380 
5381     llvm::Value *TypeId = llvm::MetadataAsValue::get(getLLVMContext(), MD);
5382 
5383     llvm::Value *CalleePtr = Callee.getFunctionPointer();
5384     llvm::Value *CastedCallee = Builder.CreateBitCast(CalleePtr, Int8PtrTy);
5385     llvm::Value *TypeTest = Builder.CreateCall(
5386         CGM.getIntrinsic(llvm::Intrinsic::type_test), {CastedCallee, TypeId});
5387 
5388     auto CrossDsoTypeId = CGM.CreateCrossDsoCfiTypeId(MD);
5389     llvm::Constant *StaticData[] = {
5390         llvm::ConstantInt::get(Int8Ty, CFITCK_ICall),
5391         EmitCheckSourceLocation(E->getBeginLoc()),
5392         EmitCheckTypeDescriptor(QualType(FnType, 0)),
5393     };
5394     if (CGM.getCodeGenOpts().SanitizeCfiCrossDso && CrossDsoTypeId) {
5395       EmitCfiSlowPathCheck(SanitizerKind::CFIICall, TypeTest, CrossDsoTypeId,
5396                            CastedCallee, StaticData);
5397     } else {
5398       EmitCheck(std::make_pair(TypeTest, SanitizerKind::CFIICall),
5399                 SanitizerHandler::CFICheckFail, StaticData,
5400                 {CastedCallee, llvm::UndefValue::get(IntPtrTy)});
5401     }
5402   }
5403 
5404   CallArgList Args;
5405   if (Chain)
5406     Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)),
5407              CGM.getContext().VoidPtrTy);
5408 
5409   // C++17 requires that we evaluate arguments to a call using assignment syntax
5410   // right-to-left, and that we evaluate arguments to certain other operators
5411   // left-to-right. Note that we allow this to override the order dictated by
5412   // the calling convention on the MS ABI, which means that parameter
5413   // destruction order is not necessarily reverse construction order.
5414   // FIXME: Revisit this based on C++ committee response to unimplementability.
5415   EvaluationOrder Order = EvaluationOrder::Default;
5416   if (auto *OCE = dyn_cast<CXXOperatorCallExpr>(E)) {
5417     if (OCE->isAssignmentOp())
5418       Order = EvaluationOrder::ForceRightToLeft;
5419     else {
5420       switch (OCE->getOperator()) {
5421       case OO_LessLess:
5422       case OO_GreaterGreater:
5423       case OO_AmpAmp:
5424       case OO_PipePipe:
5425       case OO_Comma:
5426       case OO_ArrowStar:
5427         Order = EvaluationOrder::ForceLeftToRight;
5428         break;
5429       default:
5430         break;
5431       }
5432     }
5433   }
5434 
5435   EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), E->arguments(),
5436                E->getDirectCallee(), /*ParamsToSkip*/ 0, Order);
5437 
5438   const CGFunctionInfo &FnInfo = CGM.getTypes().arrangeFreeFunctionCall(
5439       Args, FnType, /*ChainCall=*/Chain);
5440 
5441   // C99 6.5.2.2p6:
5442   //   If the expression that denotes the called function has a type
5443   //   that does not include a prototype, [the default argument
5444   //   promotions are performed]. If the number of arguments does not
5445   //   equal the number of parameters, the behavior is undefined. If
5446   //   the function is defined with a type that includes a prototype,
5447   //   and either the prototype ends with an ellipsis (, ...) or the
5448   //   types of the arguments after promotion are not compatible with
5449   //   the types of the parameters, the behavior is undefined. If the
5450   //   function is defined with a type that does not include a
5451   //   prototype, and the types of the arguments after promotion are
5452   //   not compatible with those of the parameters after promotion,
5453   //   the behavior is undefined [except in some trivial cases].
5454   // That is, in the general case, we should assume that a call
5455   // through an unprototyped function type works like a *non-variadic*
5456   // call.  The way we make this work is to cast to the exact type
5457   // of the promoted arguments.
5458   //
5459   // Chain calls use this same code path to add the invisible chain parameter
5460   // to the function type.
5461   if (isa<FunctionNoProtoType>(FnType) || Chain) {
5462     llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
5463     int AS = Callee.getFunctionPointer()->getType()->getPointerAddressSpace();
5464     CalleeTy = CalleeTy->getPointerTo(AS);
5465 
5466     llvm::Value *CalleePtr = Callee.getFunctionPointer();
5467     CalleePtr = Builder.CreateBitCast(CalleePtr, CalleeTy, "callee.knr.cast");
5468     Callee.setFunctionPointer(CalleePtr);
5469   }
5470 
5471   // HIP function pointer contains kernel handle when it is used in triple
5472   // chevron. The kernel stub needs to be loaded from kernel handle and used
5473   // as callee.
5474   if (CGM.getLangOpts().HIP && !CGM.getLangOpts().CUDAIsDevice &&
5475       isa<CUDAKernelCallExpr>(E) &&
5476       (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
5477     llvm::Value *Handle = Callee.getFunctionPointer();
5478     auto *Cast =
5479         Builder.CreateBitCast(Handle, Handle->getType()->getPointerTo());
5480     auto *Stub = Builder.CreateLoad(
5481         Address(Cast, Handle->getType(), CGM.getPointerAlign()));
5482     Callee.setFunctionPointer(Stub);
5483   }
5484   llvm::CallBase *CallOrInvoke = nullptr;
5485   RValue Call = EmitCall(FnInfo, Callee, ReturnValue, Args, &CallOrInvoke,
5486                          E == MustTailCall, E->getExprLoc());
5487 
5488   // Generate function declaration DISuprogram in order to be used
5489   // in debug info about call sites.
5490   if (CGDebugInfo *DI = getDebugInfo()) {
5491     if (auto *CalleeDecl = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
5492       FunctionArgList Args;
5493       QualType ResTy = BuildFunctionArgList(CalleeDecl, Args);
5494       DI->EmitFuncDeclForCallSite(CallOrInvoke,
5495                                   DI->getFunctionType(CalleeDecl, ResTy, Args),
5496                                   CalleeDecl);
5497     }
5498   }
5499 
5500   return Call;
5501 }
5502 
5503 LValue CodeGenFunction::
EmitPointerToDataMemberBinaryExpr(const BinaryOperator * E)5504 EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
5505   Address BaseAddr = Address::invalid();
5506   if (E->getOpcode() == BO_PtrMemI) {
5507     BaseAddr = EmitPointerWithAlignment(E->getLHS());
5508   } else {
5509     BaseAddr = EmitLValue(E->getLHS()).getAddress(*this);
5510   }
5511 
5512   llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
5513   const auto *MPT = E->getRHS()->getType()->castAs<MemberPointerType>();
5514 
5515   LValueBaseInfo BaseInfo;
5516   TBAAAccessInfo TBAAInfo;
5517   Address MemberAddr =
5518     EmitCXXMemberDataPointerAddress(E, BaseAddr, OffsetV, MPT, &BaseInfo,
5519                                     &TBAAInfo);
5520 
5521   return MakeAddrLValue(MemberAddr, MPT->getPointeeType(), BaseInfo, TBAAInfo);
5522 }
5523 
5524 /// Given the address of a temporary variable, produce an r-value of
5525 /// its type.
convertTempToRValue(Address addr,QualType type,SourceLocation loc)5526 RValue CodeGenFunction::convertTempToRValue(Address addr,
5527                                             QualType type,
5528                                             SourceLocation loc) {
5529   LValue lvalue = MakeAddrLValue(addr, type, AlignmentSource::Decl);
5530   switch (getEvaluationKind(type)) {
5531   case TEK_Complex:
5532     return RValue::getComplex(EmitLoadOfComplex(lvalue, loc));
5533   case TEK_Aggregate:
5534     return lvalue.asAggregateRValue(*this);
5535   case TEK_Scalar:
5536     return RValue::get(EmitLoadOfScalar(lvalue, loc));
5537   }
5538   llvm_unreachable("bad evaluation kind");
5539 }
5540 
SetFPAccuracy(llvm::Value * Val,float Accuracy)5541 void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
5542   assert(Val->getType()->isFPOrFPVectorTy());
5543   if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
5544     return;
5545 
5546   llvm::MDBuilder MDHelper(getLLVMContext());
5547   llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
5548 
5549   cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node);
5550 }
5551 
5552 namespace {
5553   struct LValueOrRValue {
5554     LValue LV;
5555     RValue RV;
5556   };
5557 }
5558 
emitPseudoObjectExpr(CodeGenFunction & CGF,const PseudoObjectExpr * E,bool forLValue,AggValueSlot slot)5559 static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
5560                                            const PseudoObjectExpr *E,
5561                                            bool forLValue,
5562                                            AggValueSlot slot) {
5563   SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
5564 
5565   // Find the result expression, if any.
5566   const Expr *resultExpr = E->getResultExpr();
5567   LValueOrRValue result;
5568 
5569   for (PseudoObjectExpr::const_semantics_iterator
5570          i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
5571     const Expr *semantic = *i;
5572 
5573     // If this semantic expression is an opaque value, bind it
5574     // to the result of its source expression.
5575     if (const auto *ov = dyn_cast<OpaqueValueExpr>(semantic)) {
5576       // Skip unique OVEs.
5577       if (ov->isUnique()) {
5578         assert(ov != resultExpr &&
5579                "A unique OVE cannot be used as the result expression");
5580         continue;
5581       }
5582 
5583       // If this is the result expression, we may need to evaluate
5584       // directly into the slot.
5585       typedef CodeGenFunction::OpaqueValueMappingData OVMA;
5586       OVMA opaqueData;
5587       if (ov == resultExpr && ov->isPRValue() && !forLValue &&
5588           CodeGenFunction::hasAggregateEvaluationKind(ov->getType())) {
5589         CGF.EmitAggExpr(ov->getSourceExpr(), slot);
5590         LValue LV = CGF.MakeAddrLValue(slot.getAddress(), ov->getType(),
5591                                        AlignmentSource::Decl);
5592         opaqueData = OVMA::bind(CGF, ov, LV);
5593         result.RV = slot.asRValue();
5594 
5595       // Otherwise, emit as normal.
5596       } else {
5597         opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr());
5598 
5599         // If this is the result, also evaluate the result now.
5600         if (ov == resultExpr) {
5601           if (forLValue)
5602             result.LV = CGF.EmitLValue(ov);
5603           else
5604             result.RV = CGF.EmitAnyExpr(ov, slot);
5605         }
5606       }
5607 
5608       opaques.push_back(opaqueData);
5609 
5610     // Otherwise, if the expression is the result, evaluate it
5611     // and remember the result.
5612     } else if (semantic == resultExpr) {
5613       if (forLValue)
5614         result.LV = CGF.EmitLValue(semantic);
5615       else
5616         result.RV = CGF.EmitAnyExpr(semantic, slot);
5617 
5618     // Otherwise, evaluate the expression in an ignored context.
5619     } else {
5620       CGF.EmitIgnoredExpr(semantic);
5621     }
5622   }
5623 
5624   // Unbind all the opaques now.
5625   for (unsigned i = 0, e = opaques.size(); i != e; ++i)
5626     opaques[i].unbind(CGF);
5627 
5628   return result;
5629 }
5630 
EmitPseudoObjectRValue(const PseudoObjectExpr * E,AggValueSlot slot)5631 RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
5632                                                AggValueSlot slot) {
5633   return emitPseudoObjectExpr(*this, E, false, slot).RV;
5634 }
5635 
EmitPseudoObjectLValue(const PseudoObjectExpr * E)5636 LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
5637   return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;
5638 }
5639