1 //===--- CGExprComplex.cpp - Emit LLVM Code for Complex Exprs -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This contains code to emit Expr nodes with complex types as LLVM code.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "CGOpenMPRuntime.h"
14 #include "CodeGenFunction.h"
15 #include "CodeGenModule.h"
16 #include "ConstantEmitter.h"
17 #include "clang/AST/StmtVisitor.h"
18 #include "llvm/ADT/STLExtras.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Instructions.h"
21 #include "llvm/IR/MDBuilder.h"
22 #include "llvm/IR/Metadata.h"
23 #include <algorithm>
24 using namespace clang;
25 using namespace CodeGen;
26 
27 //===----------------------------------------------------------------------===//
28 //                        Complex Expression Emitter
29 //===----------------------------------------------------------------------===//
30 
31 typedef CodeGenFunction::ComplexPairTy ComplexPairTy;
32 
33 /// Return the complex type that we are meant to emit.
34 static const ComplexType *getComplexType(QualType type) {
35   type = type.getCanonicalType();
36   if (const ComplexType *comp = dyn_cast<ComplexType>(type)) {
37     return comp;
38   } else {
39     return cast<ComplexType>(cast<AtomicType>(type)->getValueType());
40   }
41 }
42 
43 namespace  {
44 class ComplexExprEmitter
45   : public StmtVisitor<ComplexExprEmitter, ComplexPairTy> {
46   CodeGenFunction &CGF;
47   CGBuilderTy &Builder;
48   bool IgnoreReal;
49   bool IgnoreImag;
50 public:
51   ComplexExprEmitter(CodeGenFunction &cgf, bool ir=false, bool ii=false)
52     : CGF(cgf), Builder(CGF.Builder), IgnoreReal(ir), IgnoreImag(ii) {
53   }
54 
55 
56   //===--------------------------------------------------------------------===//
57   //                               Utilities
58   //===--------------------------------------------------------------------===//
59 
60   bool TestAndClearIgnoreReal() {
61     bool I = IgnoreReal;
62     IgnoreReal = false;
63     return I;
64   }
65   bool TestAndClearIgnoreImag() {
66     bool I = IgnoreImag;
67     IgnoreImag = false;
68     return I;
69   }
70 
71   /// EmitLoadOfLValue - Given an expression with complex type that represents a
72   /// value l-value, this method emits the address of the l-value, then loads
73   /// and returns the result.
74   ComplexPairTy EmitLoadOfLValue(const Expr *E) {
75     return EmitLoadOfLValue(CGF.EmitLValue(E), E->getExprLoc());
76   }
77 
78   ComplexPairTy EmitLoadOfLValue(LValue LV, SourceLocation Loc);
79 
80   /// EmitStoreOfComplex - Store the specified real/imag parts into the
81   /// specified value pointer.
82   void EmitStoreOfComplex(ComplexPairTy Val, LValue LV, bool isInit);
83 
84   /// Emit a cast from complex value Val to DestType.
85   ComplexPairTy EmitComplexToComplexCast(ComplexPairTy Val, QualType SrcType,
86                                          QualType DestType, SourceLocation Loc);
87   /// Emit a cast from scalar value Val to DestType.
88   ComplexPairTy EmitScalarToComplexCast(llvm::Value *Val, QualType SrcType,
89                                         QualType DestType, SourceLocation Loc);
90 
91   //===--------------------------------------------------------------------===//
92   //                            Visitor Methods
93   //===--------------------------------------------------------------------===//
94 
95   ComplexPairTy Visit(Expr *E) {
96     ApplyDebugLocation DL(CGF, E);
97     return StmtVisitor<ComplexExprEmitter, ComplexPairTy>::Visit(E);
98   }
99 
100   ComplexPairTy VisitStmt(Stmt *S) {
101     S->dump(llvm::errs(), CGF.getContext());
102     llvm_unreachable("Stmt can't have complex result type!");
103   }
104   ComplexPairTy VisitExpr(Expr *S);
105   ComplexPairTy VisitConstantExpr(ConstantExpr *E) {
106     if (llvm::Constant *Result = ConstantEmitter(CGF).tryEmitConstantExpr(E))
107       return ComplexPairTy(Result->getAggregateElement(0U),
108                            Result->getAggregateElement(1U));
109     return Visit(E->getSubExpr());
110   }
111   ComplexPairTy VisitParenExpr(ParenExpr *PE) { return Visit(PE->getSubExpr());}
112   ComplexPairTy VisitGenericSelectionExpr(GenericSelectionExpr *GE) {
113     return Visit(GE->getResultExpr());
114   }
115   ComplexPairTy VisitImaginaryLiteral(const ImaginaryLiteral *IL);
116   ComplexPairTy
117   VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *PE) {
118     return Visit(PE->getReplacement());
119   }
120   ComplexPairTy VisitCoawaitExpr(CoawaitExpr *S) {
121     return CGF.EmitCoawaitExpr(*S).getComplexVal();
122   }
123   ComplexPairTy VisitCoyieldExpr(CoyieldExpr *S) {
124     return CGF.EmitCoyieldExpr(*S).getComplexVal();
125   }
126   ComplexPairTy VisitUnaryCoawait(const UnaryOperator *E) {
127     return Visit(E->getSubExpr());
128   }
129 
130   ComplexPairTy emitConstant(const CodeGenFunction::ConstantEmission &Constant,
131                              Expr *E) {
132     assert(Constant && "not a constant");
133     if (Constant.isReference())
134       return EmitLoadOfLValue(Constant.getReferenceLValue(CGF, E),
135                               E->getExprLoc());
136 
137     llvm::Constant *pair = Constant.getValue();
138     return ComplexPairTy(pair->getAggregateElement(0U),
139                          pair->getAggregateElement(1U));
140   }
141 
142   // l-values.
143   ComplexPairTy VisitDeclRefExpr(DeclRefExpr *E) {
144     if (CodeGenFunction::ConstantEmission Constant = CGF.tryEmitAsConstant(E))
145       return emitConstant(Constant, E);
146     return EmitLoadOfLValue(E);
147   }
148   ComplexPairTy VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) {
149     return EmitLoadOfLValue(E);
150   }
151   ComplexPairTy VisitObjCMessageExpr(ObjCMessageExpr *E) {
152     return CGF.EmitObjCMessageExpr(E).getComplexVal();
153   }
154   ComplexPairTy VisitArraySubscriptExpr(Expr *E) { return EmitLoadOfLValue(E); }
155   ComplexPairTy VisitMemberExpr(MemberExpr *ME) {
156     if (CodeGenFunction::ConstantEmission Constant =
157             CGF.tryEmitAsConstant(ME)) {
158       CGF.EmitIgnoredExpr(ME->getBase());
159       return emitConstant(Constant, ME);
160     }
161     return EmitLoadOfLValue(ME);
162   }
163   ComplexPairTy VisitOpaqueValueExpr(OpaqueValueExpr *E) {
164     if (E->isGLValue())
165       return EmitLoadOfLValue(CGF.getOrCreateOpaqueLValueMapping(E),
166                               E->getExprLoc());
167     return CGF.getOrCreateOpaqueRValueMapping(E).getComplexVal();
168   }
169 
170   ComplexPairTy VisitPseudoObjectExpr(PseudoObjectExpr *E) {
171     return CGF.EmitPseudoObjectRValue(E).getComplexVal();
172   }
173 
174   // FIXME: CompoundLiteralExpr
175 
176   ComplexPairTy EmitCast(CastKind CK, Expr *Op, QualType DestTy);
177   ComplexPairTy VisitImplicitCastExpr(ImplicitCastExpr *E) {
178     // Unlike for scalars, we don't have to worry about function->ptr demotion
179     // here.
180     return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
181   }
182   ComplexPairTy VisitCastExpr(CastExpr *E) {
183     if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E))
184       CGF.CGM.EmitExplicitCastExprType(ECE, &CGF);
185     return EmitCast(E->getCastKind(), E->getSubExpr(), E->getType());
186   }
187   ComplexPairTy VisitCallExpr(const CallExpr *E);
188   ComplexPairTy VisitStmtExpr(const StmtExpr *E);
189 
190   // Operators.
191   ComplexPairTy VisitPrePostIncDec(const UnaryOperator *E,
192                                    bool isInc, bool isPre) {
193     LValue LV = CGF.EmitLValue(E->getSubExpr());
194     return CGF.EmitComplexPrePostIncDec(E, LV, isInc, isPre);
195   }
196   ComplexPairTy VisitUnaryPostDec(const UnaryOperator *E) {
197     return VisitPrePostIncDec(E, false, false);
198   }
199   ComplexPairTy VisitUnaryPostInc(const UnaryOperator *E) {
200     return VisitPrePostIncDec(E, true, false);
201   }
202   ComplexPairTy VisitUnaryPreDec(const UnaryOperator *E) {
203     return VisitPrePostIncDec(E, false, true);
204   }
205   ComplexPairTy VisitUnaryPreInc(const UnaryOperator *E) {
206     return VisitPrePostIncDec(E, true, true);
207   }
208   ComplexPairTy VisitUnaryDeref(const Expr *E) { return EmitLoadOfLValue(E); }
209 
210   ComplexPairTy VisitUnaryPlus(const UnaryOperator *E,
211                                QualType PromotionType = QualType());
212   ComplexPairTy VisitPlus(const UnaryOperator *E, QualType PromotionType);
213   ComplexPairTy VisitUnaryMinus(const UnaryOperator *E,
214                                 QualType PromotionType = QualType());
215   ComplexPairTy VisitMinus(const UnaryOperator *E, QualType PromotionType);
216   ComplexPairTy VisitUnaryNot      (const UnaryOperator *E);
217   // LNot,Real,Imag never return complex.
218   ComplexPairTy VisitUnaryExtension(const UnaryOperator *E) {
219     return Visit(E->getSubExpr());
220   }
221   ComplexPairTy VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) {
222     CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE);
223     return Visit(DAE->getExpr());
224   }
225   ComplexPairTy VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) {
226     CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE);
227     return Visit(DIE->getExpr());
228   }
229   ComplexPairTy VisitExprWithCleanups(ExprWithCleanups *E) {
230     CodeGenFunction::RunCleanupsScope Scope(CGF);
231     ComplexPairTy Vals = Visit(E->getSubExpr());
232     // Defend against dominance problems caused by jumps out of expression
233     // evaluation through the shared cleanup block.
234     Scope.ForceCleanup({&Vals.first, &Vals.second});
235     return Vals;
236   }
237   ComplexPairTy VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) {
238     assert(E->getType()->isAnyComplexType() && "Expected complex type!");
239     QualType Elem = E->getType()->castAs<ComplexType>()->getElementType();
240     llvm::Constant *Null = llvm::Constant::getNullValue(CGF.ConvertType(Elem));
241     return ComplexPairTy(Null, Null);
242   }
243   ComplexPairTy VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) {
244     assert(E->getType()->isAnyComplexType() && "Expected complex type!");
245     QualType Elem = E->getType()->castAs<ComplexType>()->getElementType();
246     llvm::Constant *Null =
247                        llvm::Constant::getNullValue(CGF.ConvertType(Elem));
248     return ComplexPairTy(Null, Null);
249   }
250 
251   struct BinOpInfo {
252     ComplexPairTy LHS;
253     ComplexPairTy RHS;
254     QualType Ty;  // Computation Type.
255     FPOptions FPFeatures;
256   };
257 
258   BinOpInfo EmitBinOps(const BinaryOperator *E,
259                        QualType PromotionTy = QualType());
260   ComplexPairTy EmitPromoted(const Expr *E, QualType PromotionTy);
261   ComplexPairTy EmitPromotedComplexOperand(const Expr *E, QualType PromotionTy);
262   LValue EmitCompoundAssignLValue(const CompoundAssignOperator *E,
263                                   ComplexPairTy (ComplexExprEmitter::*Func)
264                                   (const BinOpInfo &),
265                                   RValue &Val);
266   ComplexPairTy EmitCompoundAssign(const CompoundAssignOperator *E,
267                                    ComplexPairTy (ComplexExprEmitter::*Func)
268                                    (const BinOpInfo &));
269 
270   ComplexPairTy EmitBinAdd(const BinOpInfo &Op);
271   ComplexPairTy EmitBinSub(const BinOpInfo &Op);
272   ComplexPairTy EmitBinMul(const BinOpInfo &Op);
273   ComplexPairTy EmitBinDiv(const BinOpInfo &Op);
274 
275   ComplexPairTy EmitComplexBinOpLibCall(StringRef LibCallName,
276                                         const BinOpInfo &Op);
277 
278   QualType getPromotionType(QualType Ty) {
279     if (auto *CT = Ty->getAs<ComplexType>()) {
280       QualType ElementType = CT->getElementType();
281       if (ElementType.UseExcessPrecision(CGF.getContext()))
282         return CGF.getContext().getComplexType(CGF.getContext().FloatTy);
283     }
284     if (Ty.UseExcessPrecision(CGF.getContext()))
285       return CGF.getContext().FloatTy;
286     return QualType();
287   }
288 
289 #define HANDLEBINOP(OP)                                                        \
290   ComplexPairTy VisitBin##OP(const BinaryOperator *E) {                        \
291     QualType promotionTy = getPromotionType(E->getType());                     \
292     ComplexPairTy result = EmitBin##OP(EmitBinOps(E, promotionTy));            \
293     if (!promotionTy.isNull())                                                 \
294       result =                                                                 \
295           CGF.EmitUnPromotedValue(result, E->getType());                       \
296     return result;                                                             \
297   }
298 
299   HANDLEBINOP(Mul)
300   HANDLEBINOP(Div)
301   HANDLEBINOP(Add)
302   HANDLEBINOP(Sub)
303 #undef HANDLEBINOP
304 
305   ComplexPairTy VisitCXXRewrittenBinaryOperator(CXXRewrittenBinaryOperator *E) {
306     return Visit(E->getSemanticForm());
307   }
308 
309   // Compound assignments.
310   ComplexPairTy VisitBinAddAssign(const CompoundAssignOperator *E) {
311     return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinAdd);
312   }
313   ComplexPairTy VisitBinSubAssign(const CompoundAssignOperator *E) {
314     return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinSub);
315   }
316   ComplexPairTy VisitBinMulAssign(const CompoundAssignOperator *E) {
317     return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinMul);
318   }
319   ComplexPairTy VisitBinDivAssign(const CompoundAssignOperator *E) {
320     return EmitCompoundAssign(E, &ComplexExprEmitter::EmitBinDiv);
321   }
322 
323   // GCC rejects rem/and/or/xor for integer complex.
324   // Logical and/or always return int, never complex.
325 
326   // No comparisons produce a complex result.
327 
328   LValue EmitBinAssignLValue(const BinaryOperator *E,
329                              ComplexPairTy &Val);
330   ComplexPairTy VisitBinAssign     (const BinaryOperator *E);
331   ComplexPairTy VisitBinComma      (const BinaryOperator *E);
332 
333 
334   ComplexPairTy
335   VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO);
336   ComplexPairTy VisitChooseExpr(ChooseExpr *CE);
337 
338   ComplexPairTy VisitInitListExpr(InitListExpr *E);
339 
340   ComplexPairTy VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
341     return EmitLoadOfLValue(E);
342   }
343 
344   ComplexPairTy VisitVAArgExpr(VAArgExpr *E);
345 
346   ComplexPairTy VisitAtomicExpr(AtomicExpr *E) {
347     return CGF.EmitAtomicExpr(E).getComplexVal();
348   }
349 };
350 }  // end anonymous namespace.
351 
352 //===----------------------------------------------------------------------===//
353 //                                Utilities
354 //===----------------------------------------------------------------------===//
355 
356 Address CodeGenFunction::emitAddrOfRealComponent(Address addr,
357                                                  QualType complexType) {
358   return Builder.CreateStructGEP(addr, 0, addr.getName() + ".realp");
359 }
360 
361 Address CodeGenFunction::emitAddrOfImagComponent(Address addr,
362                                                  QualType complexType) {
363   return Builder.CreateStructGEP(addr, 1, addr.getName() + ".imagp");
364 }
365 
366 /// EmitLoadOfLValue - Given an RValue reference for a complex, emit code to
367 /// load the real and imaginary pieces, returning them as Real/Imag.
368 ComplexPairTy ComplexExprEmitter::EmitLoadOfLValue(LValue lvalue,
369                                                    SourceLocation loc) {
370   assert(lvalue.isSimple() && "non-simple complex l-value?");
371   if (lvalue.getType()->isAtomicType())
372     return CGF.EmitAtomicLoad(lvalue, loc).getComplexVal();
373 
374   Address SrcPtr = lvalue.getAddress(CGF);
375   bool isVolatile = lvalue.isVolatileQualified();
376 
377   llvm::Value *Real = nullptr, *Imag = nullptr;
378 
379   if (!IgnoreReal || isVolatile) {
380     Address RealP = CGF.emitAddrOfRealComponent(SrcPtr, lvalue.getType());
381     Real = Builder.CreateLoad(RealP, isVolatile, SrcPtr.getName() + ".real");
382   }
383 
384   if (!IgnoreImag || isVolatile) {
385     Address ImagP = CGF.emitAddrOfImagComponent(SrcPtr, lvalue.getType());
386     Imag = Builder.CreateLoad(ImagP, isVolatile, SrcPtr.getName() + ".imag");
387   }
388 
389   return ComplexPairTy(Real, Imag);
390 }
391 
392 /// EmitStoreOfComplex - Store the specified real/imag parts into the
393 /// specified value pointer.
394 void ComplexExprEmitter::EmitStoreOfComplex(ComplexPairTy Val, LValue lvalue,
395                                             bool isInit) {
396   if (lvalue.getType()->isAtomicType() ||
397       (!isInit && CGF.LValueIsSuitableForInlineAtomic(lvalue)))
398     return CGF.EmitAtomicStore(RValue::getComplex(Val), lvalue, isInit);
399 
400   Address Ptr = lvalue.getAddress(CGF);
401   Address RealPtr = CGF.emitAddrOfRealComponent(Ptr, lvalue.getType());
402   Address ImagPtr = CGF.emitAddrOfImagComponent(Ptr, lvalue.getType());
403 
404   Builder.CreateStore(Val.first, RealPtr, lvalue.isVolatileQualified());
405   Builder.CreateStore(Val.second, ImagPtr, lvalue.isVolatileQualified());
406 }
407 
408 
409 
410 //===----------------------------------------------------------------------===//
411 //                            Visitor Methods
412 //===----------------------------------------------------------------------===//
413 
414 ComplexPairTy ComplexExprEmitter::VisitExpr(Expr *E) {
415   CGF.ErrorUnsupported(E, "complex expression");
416   llvm::Type *EltTy =
417     CGF.ConvertType(getComplexType(E->getType())->getElementType());
418   llvm::Value *U = llvm::UndefValue::get(EltTy);
419   return ComplexPairTy(U, U);
420 }
421 
422 ComplexPairTy ComplexExprEmitter::
423 VisitImaginaryLiteral(const ImaginaryLiteral *IL) {
424   llvm::Value *Imag = CGF.EmitScalarExpr(IL->getSubExpr());
425   return ComplexPairTy(llvm::Constant::getNullValue(Imag->getType()), Imag);
426 }
427 
428 
429 ComplexPairTy ComplexExprEmitter::VisitCallExpr(const CallExpr *E) {
430   if (E->getCallReturnType(CGF.getContext())->isReferenceType())
431     return EmitLoadOfLValue(E);
432 
433   return CGF.EmitCallExpr(E).getComplexVal();
434 }
435 
436 ComplexPairTy ComplexExprEmitter::VisitStmtExpr(const StmtExpr *E) {
437   CodeGenFunction::StmtExprEvaluation eval(CGF);
438   Address RetAlloca = CGF.EmitCompoundStmt(*E->getSubStmt(), true);
439   assert(RetAlloca.isValid() && "Expected complex return value");
440   return EmitLoadOfLValue(CGF.MakeAddrLValue(RetAlloca, E->getType()),
441                           E->getExprLoc());
442 }
443 
444 /// Emit a cast from complex value Val to DestType.
445 ComplexPairTy ComplexExprEmitter::EmitComplexToComplexCast(ComplexPairTy Val,
446                                                            QualType SrcType,
447                                                            QualType DestType,
448                                                            SourceLocation Loc) {
449   // Get the src/dest element type.
450   SrcType = SrcType->castAs<ComplexType>()->getElementType();
451   DestType = DestType->castAs<ComplexType>()->getElementType();
452 
453   // C99 6.3.1.6: When a value of complex type is converted to another
454   // complex type, both the real and imaginary parts follow the conversion
455   // rules for the corresponding real types.
456   if (Val.first)
457     Val.first = CGF.EmitScalarConversion(Val.first, SrcType, DestType, Loc);
458   if (Val.second)
459     Val.second = CGF.EmitScalarConversion(Val.second, SrcType, DestType, Loc);
460   return Val;
461 }
462 
463 ComplexPairTy ComplexExprEmitter::EmitScalarToComplexCast(llvm::Value *Val,
464                                                           QualType SrcType,
465                                                           QualType DestType,
466                                                           SourceLocation Loc) {
467   // Convert the input element to the element type of the complex.
468   DestType = DestType->castAs<ComplexType>()->getElementType();
469   Val = CGF.EmitScalarConversion(Val, SrcType, DestType, Loc);
470 
471   // Return (realval, 0).
472   return ComplexPairTy(Val, llvm::Constant::getNullValue(Val->getType()));
473 }
474 
475 ComplexPairTy ComplexExprEmitter::EmitCast(CastKind CK, Expr *Op,
476                                            QualType DestTy) {
477   switch (CK) {
478   case CK_Dependent: llvm_unreachable("dependent cast kind in IR gen!");
479 
480   // Atomic to non-atomic casts may be more than a no-op for some platforms and
481   // for some types.
482   case CK_AtomicToNonAtomic:
483   case CK_NonAtomicToAtomic:
484   case CK_NoOp:
485   case CK_LValueToRValue:
486   case CK_UserDefinedConversion:
487     return Visit(Op);
488 
489   case CK_LValueBitCast: {
490     LValue origLV = CGF.EmitLValue(Op);
491     Address V = origLV.getAddress(CGF).withElementType(CGF.ConvertType(DestTy));
492     return EmitLoadOfLValue(CGF.MakeAddrLValue(V, DestTy), Op->getExprLoc());
493   }
494 
495   case CK_LValueToRValueBitCast: {
496     LValue SourceLVal = CGF.EmitLValue(Op);
497     Address Addr = SourceLVal.getAddress(CGF).withElementType(
498         CGF.ConvertTypeForMem(DestTy));
499     LValue DestLV = CGF.MakeAddrLValue(Addr, DestTy);
500     DestLV.setTBAAInfo(TBAAAccessInfo::getMayAliasInfo());
501     return EmitLoadOfLValue(DestLV, Op->getExprLoc());
502   }
503 
504   case CK_BitCast:
505   case CK_BaseToDerived:
506   case CK_DerivedToBase:
507   case CK_UncheckedDerivedToBase:
508   case CK_Dynamic:
509   case CK_ToUnion:
510   case CK_ArrayToPointerDecay:
511   case CK_FunctionToPointerDecay:
512   case CK_NullToPointer:
513   case CK_NullToMemberPointer:
514   case CK_BaseToDerivedMemberPointer:
515   case CK_DerivedToBaseMemberPointer:
516   case CK_MemberPointerToBoolean:
517   case CK_ReinterpretMemberPointer:
518   case CK_ConstructorConversion:
519   case CK_IntegralToPointer:
520   case CK_PointerToIntegral:
521   case CK_PointerToBoolean:
522   case CK_ToVoid:
523   case CK_VectorSplat:
524   case CK_IntegralCast:
525   case CK_BooleanToSignedIntegral:
526   case CK_IntegralToBoolean:
527   case CK_IntegralToFloating:
528   case CK_FloatingToIntegral:
529   case CK_FloatingToBoolean:
530   case CK_FloatingCast:
531   case CK_CPointerToObjCPointerCast:
532   case CK_BlockPointerToObjCPointerCast:
533   case CK_AnyPointerToBlockPointerCast:
534   case CK_ObjCObjectLValueCast:
535   case CK_FloatingComplexToReal:
536   case CK_FloatingComplexToBoolean:
537   case CK_IntegralComplexToReal:
538   case CK_IntegralComplexToBoolean:
539   case CK_ARCProduceObject:
540   case CK_ARCConsumeObject:
541   case CK_ARCReclaimReturnedObject:
542   case CK_ARCExtendBlockObject:
543   case CK_CopyAndAutoreleaseBlockObject:
544   case CK_BuiltinFnToFnPtr:
545   case CK_ZeroToOCLOpaqueType:
546   case CK_AddressSpaceConversion:
547   case CK_IntToOCLSampler:
548   case CK_FloatingToFixedPoint:
549   case CK_FixedPointToFloating:
550   case CK_FixedPointCast:
551   case CK_FixedPointToBoolean:
552   case CK_FixedPointToIntegral:
553   case CK_IntegralToFixedPoint:
554   case CK_MatrixCast:
555     llvm_unreachable("invalid cast kind for complex value");
556 
557   case CK_FloatingRealToComplex:
558   case CK_IntegralRealToComplex: {
559     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op);
560     return EmitScalarToComplexCast(CGF.EmitScalarExpr(Op), Op->getType(),
561                                    DestTy, Op->getExprLoc());
562   }
563 
564   case CK_FloatingComplexCast:
565   case CK_FloatingComplexToIntegralComplex:
566   case CK_IntegralComplexCast:
567   case CK_IntegralComplexToFloatingComplex: {
568     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op);
569     return EmitComplexToComplexCast(Visit(Op), Op->getType(), DestTy,
570                                     Op->getExprLoc());
571   }
572   }
573 
574   llvm_unreachable("unknown cast resulting in complex value");
575 }
576 
577 ComplexPairTy ComplexExprEmitter::VisitUnaryPlus(const UnaryOperator *E,
578                                                  QualType PromotionType) {
579   QualType promotionTy = PromotionType.isNull()
580                              ? getPromotionType(E->getSubExpr()->getType())
581                              : PromotionType;
582   ComplexPairTy result = VisitPlus(E, promotionTy);
583   if (!promotionTy.isNull())
584     return CGF.EmitUnPromotedValue(result, E->getSubExpr()->getType());
585   return result;
586 }
587 
588 ComplexPairTy ComplexExprEmitter::VisitPlus(const UnaryOperator *E,
589                                             QualType PromotionType) {
590   TestAndClearIgnoreReal();
591   TestAndClearIgnoreImag();
592   if (!PromotionType.isNull())
593     return CGF.EmitPromotedComplexExpr(E->getSubExpr(), PromotionType);
594   return Visit(E->getSubExpr());
595 }
596 
597 ComplexPairTy ComplexExprEmitter::VisitUnaryMinus(const UnaryOperator *E,
598                                                   QualType PromotionType) {
599   QualType promotionTy = PromotionType.isNull()
600                              ? getPromotionType(E->getSubExpr()->getType())
601                              : PromotionType;
602   ComplexPairTy result = VisitMinus(E, promotionTy);
603   if (!promotionTy.isNull())
604     return CGF.EmitUnPromotedValue(result, E->getSubExpr()->getType());
605   return result;
606 }
607 ComplexPairTy ComplexExprEmitter::VisitMinus(const UnaryOperator *E,
608                                              QualType PromotionType) {
609   TestAndClearIgnoreReal();
610   TestAndClearIgnoreImag();
611   ComplexPairTy Op;
612   if (!PromotionType.isNull())
613     Op = CGF.EmitPromotedComplexExpr(E->getSubExpr(), PromotionType);
614   else
615     Op = Visit(E->getSubExpr());
616 
617   llvm::Value *ResR, *ResI;
618   if (Op.first->getType()->isFloatingPointTy()) {
619     ResR = Builder.CreateFNeg(Op.first,  "neg.r");
620     ResI = Builder.CreateFNeg(Op.second, "neg.i");
621   } else {
622     ResR = Builder.CreateNeg(Op.first,  "neg.r");
623     ResI = Builder.CreateNeg(Op.second, "neg.i");
624   }
625   return ComplexPairTy(ResR, ResI);
626 }
627 
628 ComplexPairTy ComplexExprEmitter::VisitUnaryNot(const UnaryOperator *E) {
629   TestAndClearIgnoreReal();
630   TestAndClearIgnoreImag();
631   // ~(a+ib) = a + i*-b
632   ComplexPairTy Op = Visit(E->getSubExpr());
633   llvm::Value *ResI;
634   if (Op.second->getType()->isFloatingPointTy())
635     ResI = Builder.CreateFNeg(Op.second, "conj.i");
636   else
637     ResI = Builder.CreateNeg(Op.second, "conj.i");
638 
639   return ComplexPairTy(Op.first, ResI);
640 }
641 
642 ComplexPairTy ComplexExprEmitter::EmitBinAdd(const BinOpInfo &Op) {
643   llvm::Value *ResR, *ResI;
644 
645   if (Op.LHS.first->getType()->isFloatingPointTy()) {
646     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
647     ResR = Builder.CreateFAdd(Op.LHS.first,  Op.RHS.first,  "add.r");
648     if (Op.LHS.second && Op.RHS.second)
649       ResI = Builder.CreateFAdd(Op.LHS.second, Op.RHS.second, "add.i");
650     else
651       ResI = Op.LHS.second ? Op.LHS.second : Op.RHS.second;
652     assert(ResI && "Only one operand may be real!");
653   } else {
654     ResR = Builder.CreateAdd(Op.LHS.first,  Op.RHS.first,  "add.r");
655     assert(Op.LHS.second && Op.RHS.second &&
656            "Both operands of integer complex operators must be complex!");
657     ResI = Builder.CreateAdd(Op.LHS.second, Op.RHS.second, "add.i");
658   }
659   return ComplexPairTy(ResR, ResI);
660 }
661 
662 ComplexPairTy ComplexExprEmitter::EmitBinSub(const BinOpInfo &Op) {
663   llvm::Value *ResR, *ResI;
664   if (Op.LHS.first->getType()->isFloatingPointTy()) {
665     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
666     ResR = Builder.CreateFSub(Op.LHS.first, Op.RHS.first, "sub.r");
667     if (Op.LHS.second && Op.RHS.second)
668       ResI = Builder.CreateFSub(Op.LHS.second, Op.RHS.second, "sub.i");
669     else
670       ResI = Op.LHS.second ? Op.LHS.second
671                            : Builder.CreateFNeg(Op.RHS.second, "sub.i");
672     assert(ResI && "Only one operand may be real!");
673   } else {
674     ResR = Builder.CreateSub(Op.LHS.first, Op.RHS.first, "sub.r");
675     assert(Op.LHS.second && Op.RHS.second &&
676            "Both operands of integer complex operators must be complex!");
677     ResI = Builder.CreateSub(Op.LHS.second, Op.RHS.second, "sub.i");
678   }
679   return ComplexPairTy(ResR, ResI);
680 }
681 
682 /// Emit a libcall for a binary operation on complex types.
683 ComplexPairTy ComplexExprEmitter::EmitComplexBinOpLibCall(StringRef LibCallName,
684                                                           const BinOpInfo &Op) {
685   CallArgList Args;
686   Args.add(RValue::get(Op.LHS.first),
687            Op.Ty->castAs<ComplexType>()->getElementType());
688   Args.add(RValue::get(Op.LHS.second),
689            Op.Ty->castAs<ComplexType>()->getElementType());
690   Args.add(RValue::get(Op.RHS.first),
691            Op.Ty->castAs<ComplexType>()->getElementType());
692   Args.add(RValue::get(Op.RHS.second),
693            Op.Ty->castAs<ComplexType>()->getElementType());
694 
695   // We *must* use the full CG function call building logic here because the
696   // complex type has special ABI handling. We also should not forget about
697   // special calling convention which may be used for compiler builtins.
698 
699   // We create a function qualified type to state that this call does not have
700   // any exceptions.
701   FunctionProtoType::ExtProtoInfo EPI;
702   EPI = EPI.withExceptionSpec(
703       FunctionProtoType::ExceptionSpecInfo(EST_BasicNoexcept));
704   SmallVector<QualType, 4> ArgsQTys(
705       4, Op.Ty->castAs<ComplexType>()->getElementType());
706   QualType FQTy = CGF.getContext().getFunctionType(Op.Ty, ArgsQTys, EPI);
707   const CGFunctionInfo &FuncInfo = CGF.CGM.getTypes().arrangeFreeFunctionCall(
708       Args, cast<FunctionType>(FQTy.getTypePtr()), false);
709 
710   llvm::FunctionType *FTy = CGF.CGM.getTypes().GetFunctionType(FuncInfo);
711   llvm::FunctionCallee Func = CGF.CGM.CreateRuntimeFunction(
712       FTy, LibCallName, llvm::AttributeList(), true);
713   CGCallee Callee = CGCallee::forDirect(Func, FQTy->getAs<FunctionProtoType>());
714 
715   llvm::CallBase *Call;
716   RValue Res = CGF.EmitCall(FuncInfo, Callee, ReturnValueSlot(), Args, &Call);
717   Call->setCallingConv(CGF.CGM.getRuntimeCC());
718   return Res.getComplexVal();
719 }
720 
721 /// Lookup the libcall name for a given floating point type complex
722 /// multiply.
723 static StringRef getComplexMultiplyLibCallName(llvm::Type *Ty) {
724   switch (Ty->getTypeID()) {
725   default:
726     llvm_unreachable("Unsupported floating point type!");
727   case llvm::Type::HalfTyID:
728     return "__mulhc3";
729   case llvm::Type::FloatTyID:
730     return "__mulsc3";
731   case llvm::Type::DoubleTyID:
732     return "__muldc3";
733   case llvm::Type::PPC_FP128TyID:
734     return "__multc3";
735   case llvm::Type::X86_FP80TyID:
736     return "__mulxc3";
737   case llvm::Type::FP128TyID:
738     return "__multc3";
739   }
740 }
741 
742 // See C11 Annex G.5.1 for the semantics of multiplicative operators on complex
743 // typed values.
744 ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) {
745   using llvm::Value;
746   Value *ResR, *ResI;
747   llvm::MDBuilder MDHelper(CGF.getLLVMContext());
748 
749   if (Op.LHS.first->getType()->isFloatingPointTy()) {
750     // The general formulation is:
751     // (a + ib) * (c + id) = (a * c - b * d) + i(a * d + b * c)
752     //
753     // But we can fold away components which would be zero due to a real
754     // operand according to C11 Annex G.5.1p2.
755     // FIXME: C11 also provides for imaginary types which would allow folding
756     // still more of this within the type system.
757 
758     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
759     if (Op.LHS.second && Op.RHS.second) {
760       // If both operands are complex, emit the core math directly, and then
761       // test for NaNs. If we find NaNs in the result, we delegate to a libcall
762       // to carefully re-compute the correct infinity representation if
763       // possible. The expectation is that the presence of NaNs here is
764       // *extremely* rare, and so the cost of the libcall is almost irrelevant.
765       // This is good, because the libcall re-computes the core multiplication
766       // exactly the same as we do here and re-tests for NaNs in order to be
767       // a generic complex*complex libcall.
768 
769       // First compute the four products.
770       Value *AC = Builder.CreateFMul(Op.LHS.first, Op.RHS.first, "mul_ac");
771       Value *BD = Builder.CreateFMul(Op.LHS.second, Op.RHS.second, "mul_bd");
772       Value *AD = Builder.CreateFMul(Op.LHS.first, Op.RHS.second, "mul_ad");
773       Value *BC = Builder.CreateFMul(Op.LHS.second, Op.RHS.first, "mul_bc");
774 
775       // The real part is the difference of the first two, the imaginary part is
776       // the sum of the second.
777       ResR = Builder.CreateFSub(AC, BD, "mul_r");
778       ResI = Builder.CreateFAdd(AD, BC, "mul_i");
779 
780       // Emit the test for the real part becoming NaN and create a branch to
781       // handle it. We test for NaN by comparing the number to itself.
782       Value *IsRNaN = Builder.CreateFCmpUNO(ResR, ResR, "isnan_cmp");
783       llvm::BasicBlock *ContBB = CGF.createBasicBlock("complex_mul_cont");
784       llvm::BasicBlock *INaNBB = CGF.createBasicBlock("complex_mul_imag_nan");
785       llvm::Instruction *Branch = Builder.CreateCondBr(IsRNaN, INaNBB, ContBB);
786       llvm::BasicBlock *OrigBB = Branch->getParent();
787 
788       // Give hint that we very much don't expect to see NaNs.
789       // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
790       llvm::MDNode *BrWeight = MDHelper.createBranchWeights(1, (1U << 20) - 1);
791       Branch->setMetadata(llvm::LLVMContext::MD_prof, BrWeight);
792 
793       // Now test the imaginary part and create its branch.
794       CGF.EmitBlock(INaNBB);
795       Value *IsINaN = Builder.CreateFCmpUNO(ResI, ResI, "isnan_cmp");
796       llvm::BasicBlock *LibCallBB = CGF.createBasicBlock("complex_mul_libcall");
797       Branch = Builder.CreateCondBr(IsINaN, LibCallBB, ContBB);
798       Branch->setMetadata(llvm::LLVMContext::MD_prof, BrWeight);
799 
800       // Now emit the libcall on this slowest of the slow paths.
801       CGF.EmitBlock(LibCallBB);
802       Value *LibCallR, *LibCallI;
803       std::tie(LibCallR, LibCallI) = EmitComplexBinOpLibCall(
804           getComplexMultiplyLibCallName(Op.LHS.first->getType()), Op);
805       Builder.CreateBr(ContBB);
806 
807       // Finally continue execution by phi-ing together the different
808       // computation paths.
809       CGF.EmitBlock(ContBB);
810       llvm::PHINode *RealPHI = Builder.CreatePHI(ResR->getType(), 3, "real_mul_phi");
811       RealPHI->addIncoming(ResR, OrigBB);
812       RealPHI->addIncoming(ResR, INaNBB);
813       RealPHI->addIncoming(LibCallR, LibCallBB);
814       llvm::PHINode *ImagPHI = Builder.CreatePHI(ResI->getType(), 3, "imag_mul_phi");
815       ImagPHI->addIncoming(ResI, OrigBB);
816       ImagPHI->addIncoming(ResI, INaNBB);
817       ImagPHI->addIncoming(LibCallI, LibCallBB);
818       return ComplexPairTy(RealPHI, ImagPHI);
819     }
820     assert((Op.LHS.second || Op.RHS.second) &&
821            "At least one operand must be complex!");
822 
823     // If either of the operands is a real rather than a complex, the
824     // imaginary component is ignored when computing the real component of the
825     // result.
826     ResR = Builder.CreateFMul(Op.LHS.first, Op.RHS.first, "mul.rl");
827 
828     ResI = Op.LHS.second
829                ? Builder.CreateFMul(Op.LHS.second, Op.RHS.first, "mul.il")
830                : Builder.CreateFMul(Op.LHS.first, Op.RHS.second, "mul.ir");
831   } else {
832     assert(Op.LHS.second && Op.RHS.second &&
833            "Both operands of integer complex operators must be complex!");
834     Value *ResRl = Builder.CreateMul(Op.LHS.first, Op.RHS.first, "mul.rl");
835     Value *ResRr = Builder.CreateMul(Op.LHS.second, Op.RHS.second, "mul.rr");
836     ResR = Builder.CreateSub(ResRl, ResRr, "mul.r");
837 
838     Value *ResIl = Builder.CreateMul(Op.LHS.second, Op.RHS.first, "mul.il");
839     Value *ResIr = Builder.CreateMul(Op.LHS.first, Op.RHS.second, "mul.ir");
840     ResI = Builder.CreateAdd(ResIl, ResIr, "mul.i");
841   }
842   return ComplexPairTy(ResR, ResI);
843 }
844 
845 // See C11 Annex G.5.1 for the semantics of multiplicative operators on complex
846 // typed values.
847 ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) {
848   llvm::Value *LHSr = Op.LHS.first, *LHSi = Op.LHS.second;
849   llvm::Value *RHSr = Op.RHS.first, *RHSi = Op.RHS.second;
850 
851   llvm::Value *DSTr, *DSTi;
852   if (LHSr->getType()->isFloatingPointTy()) {
853     // If we have a complex operand on the RHS and FastMath is not allowed, we
854     // delegate to a libcall to handle all of the complexities and minimize
855     // underflow/overflow cases. When FastMath is allowed we construct the
856     // divide inline using the same algorithm as for integer operands.
857     //
858     // FIXME: We would be able to avoid the libcall in many places if we
859     // supported imaginary types in addition to complex types.
860     CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures);
861     if (RHSi && !CGF.getLangOpts().FastMath) {
862       BinOpInfo LibCallOp = Op;
863       // If LHS was a real, supply a null imaginary part.
864       if (!LHSi)
865         LibCallOp.LHS.second = llvm::Constant::getNullValue(LHSr->getType());
866 
867       switch (LHSr->getType()->getTypeID()) {
868       default:
869         llvm_unreachable("Unsupported floating point type!");
870       case llvm::Type::HalfTyID:
871         return EmitComplexBinOpLibCall("__divhc3", LibCallOp);
872       case llvm::Type::FloatTyID:
873         return EmitComplexBinOpLibCall("__divsc3", LibCallOp);
874       case llvm::Type::DoubleTyID:
875         return EmitComplexBinOpLibCall("__divdc3", LibCallOp);
876       case llvm::Type::PPC_FP128TyID:
877         return EmitComplexBinOpLibCall("__divtc3", LibCallOp);
878       case llvm::Type::X86_FP80TyID:
879         return EmitComplexBinOpLibCall("__divxc3", LibCallOp);
880       case llvm::Type::FP128TyID:
881         return EmitComplexBinOpLibCall("__divtc3", LibCallOp);
882       }
883     } else if (RHSi) {
884       if (!LHSi)
885         LHSi = llvm::Constant::getNullValue(RHSi->getType());
886 
887       // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
888       llvm::Value *AC = Builder.CreateFMul(LHSr, RHSr); // a*c
889       llvm::Value *BD = Builder.CreateFMul(LHSi, RHSi); // b*d
890       llvm::Value *ACpBD = Builder.CreateFAdd(AC, BD); // ac+bd
891 
892       llvm::Value *CC = Builder.CreateFMul(RHSr, RHSr); // c*c
893       llvm::Value *DD = Builder.CreateFMul(RHSi, RHSi); // d*d
894       llvm::Value *CCpDD = Builder.CreateFAdd(CC, DD); // cc+dd
895 
896       llvm::Value *BC = Builder.CreateFMul(LHSi, RHSr); // b*c
897       llvm::Value *AD = Builder.CreateFMul(LHSr, RHSi); // a*d
898       llvm::Value *BCmAD = Builder.CreateFSub(BC, AD); // bc-ad
899 
900       DSTr = Builder.CreateFDiv(ACpBD, CCpDD);
901       DSTi = Builder.CreateFDiv(BCmAD, CCpDD);
902     } else {
903       assert(LHSi && "Can have at most one non-complex operand!");
904 
905       DSTr = Builder.CreateFDiv(LHSr, RHSr);
906       DSTi = Builder.CreateFDiv(LHSi, RHSr);
907     }
908   } else {
909     assert(Op.LHS.second && Op.RHS.second &&
910            "Both operands of integer complex operators must be complex!");
911     // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd))
912     llvm::Value *Tmp1 = Builder.CreateMul(LHSr, RHSr); // a*c
913     llvm::Value *Tmp2 = Builder.CreateMul(LHSi, RHSi); // b*d
914     llvm::Value *Tmp3 = Builder.CreateAdd(Tmp1, Tmp2); // ac+bd
915 
916     llvm::Value *Tmp4 = Builder.CreateMul(RHSr, RHSr); // c*c
917     llvm::Value *Tmp5 = Builder.CreateMul(RHSi, RHSi); // d*d
918     llvm::Value *Tmp6 = Builder.CreateAdd(Tmp4, Tmp5); // cc+dd
919 
920     llvm::Value *Tmp7 = Builder.CreateMul(LHSi, RHSr); // b*c
921     llvm::Value *Tmp8 = Builder.CreateMul(LHSr, RHSi); // a*d
922     llvm::Value *Tmp9 = Builder.CreateSub(Tmp7, Tmp8); // bc-ad
923 
924     if (Op.Ty->castAs<ComplexType>()->getElementType()->isUnsignedIntegerType()) {
925       DSTr = Builder.CreateUDiv(Tmp3, Tmp6);
926       DSTi = Builder.CreateUDiv(Tmp9, Tmp6);
927     } else {
928       DSTr = Builder.CreateSDiv(Tmp3, Tmp6);
929       DSTi = Builder.CreateSDiv(Tmp9, Tmp6);
930     }
931   }
932 
933   return ComplexPairTy(DSTr, DSTi);
934 }
935 
936 ComplexPairTy CodeGenFunction::EmitUnPromotedValue(ComplexPairTy result,
937                                                    QualType UnPromotionType) {
938   llvm::Type *ComplexElementTy =
939       ConvertType(UnPromotionType->castAs<ComplexType>()->getElementType());
940   if (result.first)
941     result.first =
942         Builder.CreateFPTrunc(result.first, ComplexElementTy, "unpromotion");
943   if (result.second)
944     result.second =
945         Builder.CreateFPTrunc(result.second, ComplexElementTy, "unpromotion");
946   return result;
947 }
948 
949 ComplexPairTy CodeGenFunction::EmitPromotedValue(ComplexPairTy result,
950                                                  QualType PromotionType) {
951   llvm::Type *ComplexElementTy =
952       ConvertType(PromotionType->castAs<ComplexType>()->getElementType());
953   if (result.first)
954     result.first = Builder.CreateFPExt(result.first, ComplexElementTy, "ext");
955   if (result.second)
956     result.second = Builder.CreateFPExt(result.second, ComplexElementTy, "ext");
957 
958   return result;
959 }
960 
961 ComplexPairTy ComplexExprEmitter::EmitPromoted(const Expr *E,
962                                                QualType PromotionType) {
963   E = E->IgnoreParens();
964   if (auto BO = dyn_cast<BinaryOperator>(E)) {
965     switch (BO->getOpcode()) {
966 #define HANDLE_BINOP(OP)                                                       \
967   case BO_##OP:                                                                \
968     return EmitBin##OP(EmitBinOps(BO, PromotionType));
969       HANDLE_BINOP(Add)
970       HANDLE_BINOP(Sub)
971       HANDLE_BINOP(Mul)
972       HANDLE_BINOP(Div)
973 #undef HANDLE_BINOP
974     default:
975       break;
976     }
977   } else if (auto UO = dyn_cast<UnaryOperator>(E)) {
978     switch (UO->getOpcode()) {
979     case UO_Minus:
980       return VisitMinus(UO, PromotionType);
981     case UO_Plus:
982       return VisitPlus(UO, PromotionType);
983     default:
984       break;
985     }
986   }
987   auto result = Visit(const_cast<Expr *>(E));
988   if (!PromotionType.isNull())
989     return CGF.EmitPromotedValue(result, PromotionType);
990   else
991     return result;
992 }
993 
994 ComplexPairTy CodeGenFunction::EmitPromotedComplexExpr(const Expr *E,
995                                                        QualType DstTy) {
996   return ComplexExprEmitter(*this).EmitPromoted(E, DstTy);
997 }
998 
999 ComplexPairTy
1000 ComplexExprEmitter::EmitPromotedComplexOperand(const Expr *E,
1001                                                QualType OverallPromotionType) {
1002   if (E->getType()->isAnyComplexType()) {
1003     if (!OverallPromotionType.isNull())
1004       return CGF.EmitPromotedComplexExpr(E, OverallPromotionType);
1005     else
1006       return Visit(const_cast<Expr *>(E));
1007   } else {
1008     if (!OverallPromotionType.isNull()) {
1009       QualType ComplexElementTy =
1010           OverallPromotionType->castAs<ComplexType>()->getElementType();
1011       return ComplexPairTy(CGF.EmitPromotedScalarExpr(E, ComplexElementTy),
1012                            nullptr);
1013     } else {
1014       return ComplexPairTy(CGF.EmitScalarExpr(E), nullptr);
1015     }
1016   }
1017 }
1018 
1019 ComplexExprEmitter::BinOpInfo
1020 ComplexExprEmitter::EmitBinOps(const BinaryOperator *E,
1021                                QualType PromotionType) {
1022   TestAndClearIgnoreReal();
1023   TestAndClearIgnoreImag();
1024   BinOpInfo Ops;
1025 
1026   Ops.LHS = EmitPromotedComplexOperand(E->getLHS(), PromotionType);
1027   Ops.RHS = EmitPromotedComplexOperand(E->getRHS(), PromotionType);
1028   if (!PromotionType.isNull())
1029     Ops.Ty = PromotionType;
1030   else
1031     Ops.Ty = E->getType();
1032   Ops.FPFeatures = E->getFPFeaturesInEffect(CGF.getLangOpts());
1033   return Ops;
1034 }
1035 
1036 
1037 LValue ComplexExprEmitter::
1038 EmitCompoundAssignLValue(const CompoundAssignOperator *E,
1039           ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&),
1040                          RValue &Val) {
1041   TestAndClearIgnoreReal();
1042   TestAndClearIgnoreImag();
1043   QualType LHSTy = E->getLHS()->getType();
1044   if (const AtomicType *AT = LHSTy->getAs<AtomicType>())
1045     LHSTy = AT->getValueType();
1046 
1047   BinOpInfo OpInfo;
1048   OpInfo.FPFeatures = E->getFPFeaturesInEffect(CGF.getLangOpts());
1049   CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, OpInfo.FPFeatures);
1050 
1051   // Load the RHS and LHS operands.
1052   // __block variables need to have the rhs evaluated first, plus this should
1053   // improve codegen a little.
1054   QualType PromotionTypeCR;
1055   PromotionTypeCR = getPromotionType(E->getComputationResultType());
1056   if (PromotionTypeCR.isNull())
1057     PromotionTypeCR = E->getComputationResultType();
1058   OpInfo.Ty = PromotionTypeCR;
1059   QualType ComplexElementTy =
1060       OpInfo.Ty->castAs<ComplexType>()->getElementType();
1061   QualType PromotionTypeRHS = getPromotionType(E->getRHS()->getType());
1062 
1063   // The RHS should have been converted to the computation type.
1064   if (E->getRHS()->getType()->isRealFloatingType()) {
1065     if (!PromotionTypeRHS.isNull())
1066       OpInfo.RHS = ComplexPairTy(
1067           CGF.EmitPromotedScalarExpr(E->getRHS(), PromotionTypeRHS), nullptr);
1068     else {
1069       assert(CGF.getContext().hasSameUnqualifiedType(ComplexElementTy,
1070                                                      E->getRHS()->getType()));
1071 
1072       OpInfo.RHS = ComplexPairTy(CGF.EmitScalarExpr(E->getRHS()), nullptr);
1073     }
1074   } else {
1075     if (!PromotionTypeRHS.isNull()) {
1076       OpInfo.RHS = ComplexPairTy(
1077           CGF.EmitPromotedComplexExpr(E->getRHS(), PromotionTypeRHS));
1078     } else {
1079       assert(CGF.getContext().hasSameUnqualifiedType(OpInfo.Ty,
1080                                                      E->getRHS()->getType()));
1081       OpInfo.RHS = Visit(E->getRHS());
1082     }
1083   }
1084 
1085   LValue LHS = CGF.EmitLValue(E->getLHS());
1086 
1087   // Load from the l-value and convert it.
1088   SourceLocation Loc = E->getExprLoc();
1089   QualType PromotionTypeLHS = getPromotionType(E->getComputationLHSType());
1090   if (LHSTy->isAnyComplexType()) {
1091     ComplexPairTy LHSVal = EmitLoadOfLValue(LHS, Loc);
1092     if (!PromotionTypeLHS.isNull())
1093       OpInfo.LHS =
1094           EmitComplexToComplexCast(LHSVal, LHSTy, PromotionTypeLHS, Loc);
1095     else
1096       OpInfo.LHS = EmitComplexToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc);
1097   } else {
1098     llvm::Value *LHSVal = CGF.EmitLoadOfScalar(LHS, Loc);
1099     // For floating point real operands we can directly pass the scalar form
1100     // to the binary operator emission and potentially get more efficient code.
1101     if (LHSTy->isRealFloatingType()) {
1102       QualType PromotedComplexElementTy;
1103       if (!PromotionTypeLHS.isNull()) {
1104         PromotedComplexElementTy =
1105             cast<ComplexType>(PromotionTypeLHS)->getElementType();
1106         if (!CGF.getContext().hasSameUnqualifiedType(PromotedComplexElementTy,
1107                                                      PromotionTypeLHS))
1108           LHSVal = CGF.EmitScalarConversion(LHSVal, LHSTy,
1109                                             PromotedComplexElementTy, Loc);
1110       } else {
1111         if (!CGF.getContext().hasSameUnqualifiedType(ComplexElementTy, LHSTy))
1112           LHSVal =
1113               CGF.EmitScalarConversion(LHSVal, LHSTy, ComplexElementTy, Loc);
1114       }
1115       OpInfo.LHS = ComplexPairTy(LHSVal, nullptr);
1116     } else {
1117       OpInfo.LHS = EmitScalarToComplexCast(LHSVal, LHSTy, OpInfo.Ty, Loc);
1118     }
1119   }
1120 
1121   // Expand the binary operator.
1122   ComplexPairTy Result = (this->*Func)(OpInfo);
1123 
1124   // Truncate the result and store it into the LHS lvalue.
1125   if (LHSTy->isAnyComplexType()) {
1126     ComplexPairTy ResVal =
1127         EmitComplexToComplexCast(Result, OpInfo.Ty, LHSTy, Loc);
1128     EmitStoreOfComplex(ResVal, LHS, /*isInit*/ false);
1129     Val = RValue::getComplex(ResVal);
1130   } else {
1131     llvm::Value *ResVal =
1132         CGF.EmitComplexToScalarConversion(Result, OpInfo.Ty, LHSTy, Loc);
1133     CGF.EmitStoreOfScalar(ResVal, LHS, /*isInit*/ false);
1134     Val = RValue::get(ResVal);
1135   }
1136 
1137   return LHS;
1138 }
1139 
1140 // Compound assignments.
1141 ComplexPairTy ComplexExprEmitter::
1142 EmitCompoundAssign(const CompoundAssignOperator *E,
1143                    ComplexPairTy (ComplexExprEmitter::*Func)(const BinOpInfo&)){
1144   RValue Val;
1145   LValue LV = EmitCompoundAssignLValue(E, Func, Val);
1146 
1147   // The result of an assignment in C is the assigned r-value.
1148   if (!CGF.getLangOpts().CPlusPlus)
1149     return Val.getComplexVal();
1150 
1151   // If the lvalue is non-volatile, return the computed value of the assignment.
1152   if (!LV.isVolatileQualified())
1153     return Val.getComplexVal();
1154 
1155   return EmitLoadOfLValue(LV, E->getExprLoc());
1156 }
1157 
1158 LValue ComplexExprEmitter::EmitBinAssignLValue(const BinaryOperator *E,
1159                                                ComplexPairTy &Val) {
1160   assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(),
1161                                                  E->getRHS()->getType()) &&
1162          "Invalid assignment");
1163   TestAndClearIgnoreReal();
1164   TestAndClearIgnoreImag();
1165 
1166   // Emit the RHS.  __block variables need the RHS evaluated first.
1167   Val = Visit(E->getRHS());
1168 
1169   // Compute the address to store into.
1170   LValue LHS = CGF.EmitLValue(E->getLHS());
1171 
1172   // Store the result value into the LHS lvalue.
1173   EmitStoreOfComplex(Val, LHS, /*isInit*/ false);
1174 
1175   return LHS;
1176 }
1177 
1178 ComplexPairTy ComplexExprEmitter::VisitBinAssign(const BinaryOperator *E) {
1179   ComplexPairTy Val;
1180   LValue LV = EmitBinAssignLValue(E, Val);
1181 
1182   // The result of an assignment in C is the assigned r-value.
1183   if (!CGF.getLangOpts().CPlusPlus)
1184     return Val;
1185 
1186   // If the lvalue is non-volatile, return the computed value of the assignment.
1187   if (!LV.isVolatileQualified())
1188     return Val;
1189 
1190   return EmitLoadOfLValue(LV, E->getExprLoc());
1191 }
1192 
1193 ComplexPairTy ComplexExprEmitter::VisitBinComma(const BinaryOperator *E) {
1194   CGF.EmitIgnoredExpr(E->getLHS());
1195   return Visit(E->getRHS());
1196 }
1197 
1198 ComplexPairTy ComplexExprEmitter::
1199 VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
1200   TestAndClearIgnoreReal();
1201   TestAndClearIgnoreImag();
1202   llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true");
1203   llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false");
1204   llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end");
1205 
1206   // Bind the common expression if necessary.
1207   CodeGenFunction::OpaqueValueMapping binding(CGF, E);
1208 
1209 
1210   CodeGenFunction::ConditionalEvaluation eval(CGF);
1211   CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock,
1212                            CGF.getProfileCount(E));
1213 
1214   eval.begin(CGF);
1215   CGF.EmitBlock(LHSBlock);
1216   CGF.incrementProfileCounter(E);
1217   ComplexPairTy LHS = Visit(E->getTrueExpr());
1218   LHSBlock = Builder.GetInsertBlock();
1219   CGF.EmitBranch(ContBlock);
1220   eval.end(CGF);
1221 
1222   eval.begin(CGF);
1223   CGF.EmitBlock(RHSBlock);
1224   ComplexPairTy RHS = Visit(E->getFalseExpr());
1225   RHSBlock = Builder.GetInsertBlock();
1226   CGF.EmitBlock(ContBlock);
1227   eval.end(CGF);
1228 
1229   // Create a PHI node for the real part.
1230   llvm::PHINode *RealPN = Builder.CreatePHI(LHS.first->getType(), 2, "cond.r");
1231   RealPN->addIncoming(LHS.first, LHSBlock);
1232   RealPN->addIncoming(RHS.first, RHSBlock);
1233 
1234   // Create a PHI node for the imaginary part.
1235   llvm::PHINode *ImagPN = Builder.CreatePHI(LHS.first->getType(), 2, "cond.i");
1236   ImagPN->addIncoming(LHS.second, LHSBlock);
1237   ImagPN->addIncoming(RHS.second, RHSBlock);
1238 
1239   return ComplexPairTy(RealPN, ImagPN);
1240 }
1241 
1242 ComplexPairTy ComplexExprEmitter::VisitChooseExpr(ChooseExpr *E) {
1243   return Visit(E->getChosenSubExpr());
1244 }
1245 
1246 ComplexPairTy ComplexExprEmitter::VisitInitListExpr(InitListExpr *E) {
1247     bool Ignore = TestAndClearIgnoreReal();
1248     (void)Ignore;
1249     assert (Ignore == false && "init list ignored");
1250     Ignore = TestAndClearIgnoreImag();
1251     (void)Ignore;
1252     assert (Ignore == false && "init list ignored");
1253 
1254   if (E->getNumInits() == 2) {
1255     llvm::Value *Real = CGF.EmitScalarExpr(E->getInit(0));
1256     llvm::Value *Imag = CGF.EmitScalarExpr(E->getInit(1));
1257     return ComplexPairTy(Real, Imag);
1258   } else if (E->getNumInits() == 1) {
1259     return Visit(E->getInit(0));
1260   }
1261 
1262   // Empty init list initializes to null
1263   assert(E->getNumInits() == 0 && "Unexpected number of inits");
1264   QualType Ty = E->getType()->castAs<ComplexType>()->getElementType();
1265   llvm::Type* LTy = CGF.ConvertType(Ty);
1266   llvm::Value* zeroConstant = llvm::Constant::getNullValue(LTy);
1267   return ComplexPairTy(zeroConstant, zeroConstant);
1268 }
1269 
1270 ComplexPairTy ComplexExprEmitter::VisitVAArgExpr(VAArgExpr *E) {
1271   Address ArgValue = Address::invalid();
1272   Address ArgPtr = CGF.EmitVAArg(E, ArgValue);
1273 
1274   if (!ArgPtr.isValid()) {
1275     CGF.ErrorUnsupported(E, "complex va_arg expression");
1276     llvm::Type *EltTy =
1277       CGF.ConvertType(E->getType()->castAs<ComplexType>()->getElementType());
1278     llvm::Value *U = llvm::UndefValue::get(EltTy);
1279     return ComplexPairTy(U, U);
1280   }
1281 
1282   return EmitLoadOfLValue(CGF.MakeAddrLValue(ArgPtr, E->getType()),
1283                           E->getExprLoc());
1284 }
1285 
1286 //===----------------------------------------------------------------------===//
1287 //                         Entry Point into this File
1288 //===----------------------------------------------------------------------===//
1289 
1290 /// EmitComplexExpr - Emit the computation of the specified expression of
1291 /// complex type, ignoring the result.
1292 ComplexPairTy CodeGenFunction::EmitComplexExpr(const Expr *E, bool IgnoreReal,
1293                                                bool IgnoreImag) {
1294   assert(E && getComplexType(E->getType()) &&
1295          "Invalid complex expression to emit");
1296 
1297   return ComplexExprEmitter(*this, IgnoreReal, IgnoreImag)
1298       .Visit(const_cast<Expr *>(E));
1299 }
1300 
1301 void CodeGenFunction::EmitComplexExprIntoLValue(const Expr *E, LValue dest,
1302                                                 bool isInit) {
1303   assert(E && getComplexType(E->getType()) &&
1304          "Invalid complex expression to emit");
1305   ComplexExprEmitter Emitter(*this);
1306   ComplexPairTy Val = Emitter.Visit(const_cast<Expr*>(E));
1307   Emitter.EmitStoreOfComplex(Val, dest, isInit);
1308 }
1309 
1310 /// EmitStoreOfComplex - Store a complex number into the specified l-value.
1311 void CodeGenFunction::EmitStoreOfComplex(ComplexPairTy V, LValue dest,
1312                                          bool isInit) {
1313   ComplexExprEmitter(*this).EmitStoreOfComplex(V, dest, isInit);
1314 }
1315 
1316 /// EmitLoadOfComplex - Load a complex number from the specified address.
1317 ComplexPairTy CodeGenFunction::EmitLoadOfComplex(LValue src,
1318                                                  SourceLocation loc) {
1319   return ComplexExprEmitter(*this).EmitLoadOfLValue(src, loc);
1320 }
1321 
1322 LValue CodeGenFunction::EmitComplexAssignmentLValue(const BinaryOperator *E) {
1323   assert(E->getOpcode() == BO_Assign);
1324   ComplexPairTy Val; // ignored
1325   LValue LVal = ComplexExprEmitter(*this).EmitBinAssignLValue(E, Val);
1326   if (getLangOpts().OpenMP)
1327     CGM.getOpenMPRuntime().checkAndEmitLastprivateConditional(*this,
1328                                                               E->getLHS());
1329   return LVal;
1330 }
1331 
1332 typedef ComplexPairTy (ComplexExprEmitter::*CompoundFunc)(
1333     const ComplexExprEmitter::BinOpInfo &);
1334 
1335 static CompoundFunc getComplexOp(BinaryOperatorKind Op) {
1336   switch (Op) {
1337   case BO_MulAssign: return &ComplexExprEmitter::EmitBinMul;
1338   case BO_DivAssign: return &ComplexExprEmitter::EmitBinDiv;
1339   case BO_SubAssign: return &ComplexExprEmitter::EmitBinSub;
1340   case BO_AddAssign: return &ComplexExprEmitter::EmitBinAdd;
1341   default:
1342     llvm_unreachable("unexpected complex compound assignment");
1343   }
1344 }
1345 
1346 LValue CodeGenFunction::
1347 EmitComplexCompoundAssignmentLValue(const CompoundAssignOperator *E) {
1348   CompoundFunc Op = getComplexOp(E->getOpcode());
1349   RValue Val;
1350   return ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);
1351 }
1352 
1353 LValue CodeGenFunction::
1354 EmitScalarCompoundAssignWithComplex(const CompoundAssignOperator *E,
1355                                     llvm::Value *&Result) {
1356   CompoundFunc Op = getComplexOp(E->getOpcode());
1357   RValue Val;
1358   LValue Ret = ComplexExprEmitter(*this).EmitCompoundAssignLValue(E, Op, Val);
1359   Result = Val.getScalarVal();
1360   return Ret;
1361 }
1362