1 /* 2 Copyright (c) 2011, Intel Corporation. All rights reserved. 3 Copyright (C) 2015 Gael Guennebaud <gael.guennebaud@inria.fr> 4 5 Redistribution and use in source and binary forms, with or without modification, 6 are permitted provided that the following conditions are met: 7 8 * Redistributions of source code must retain the above copyright notice, this 9 list of conditions and the following disclaimer. 10 * Redistributions in binary form must reproduce the above copyright notice, 11 this list of conditions and the following disclaimer in the documentation 12 and/or other materials provided with the distribution. 13 * Neither the name of Intel Corporation nor the names of its contributors may 14 be used to endorse or promote products derived from this software without 15 specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 21 ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 24 ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 26 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 28 ******************************************************************************** 29 * Content : Eigen bindings to Intel(R) MKL 30 * MKL VML support for coefficient-wise unary Eigen expressions like a=b.sin() 31 ******************************************************************************** 32 */ 33 34 #ifndef EIGEN_ASSIGN_VML_H 35 #define EIGEN_ASSIGN_VML_H 36 37 namespace Eigen { 38 39 namespace internal { 40 41 template<typename Dst, typename Src> 42 class vml_assign_traits 43 { 44 private: 45 enum { 46 DstHasDirectAccess = Dst::Flags & DirectAccessBit, 47 SrcHasDirectAccess = Src::Flags & DirectAccessBit, 48 StorageOrdersAgree = (int(Dst::IsRowMajor) == int(Src::IsRowMajor)), 49 InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime) 50 : int(Dst::Flags)&RowMajorBit ? int(Dst::ColsAtCompileTime) 51 : int(Dst::RowsAtCompileTime), 52 InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime) 53 : int(Dst::Flags)&RowMajorBit ? int(Dst::MaxColsAtCompileTime) 54 : int(Dst::MaxRowsAtCompileTime), 55 MaxSizeAtCompileTime = Dst::SizeAtCompileTime, 56 57 MightEnableVml = StorageOrdersAgree && DstHasDirectAccess && SrcHasDirectAccess && Src::InnerStrideAtCompileTime==1 && Dst::InnerStrideAtCompileTime==1, 58 MightLinearize = MightEnableVml && (int(Dst::Flags) & int(Src::Flags) & LinearAccessBit), 59 VmlSize = MightLinearize ? MaxSizeAtCompileTime : InnerMaxSize, 60 LargeEnough = VmlSize==Dynamic || VmlSize>=EIGEN_MKL_VML_THRESHOLD 61 }; 62 public: 63 enum { 64 EnableVml = MightEnableVml && LargeEnough, 65 Traversal = MightLinearize ? LinearTraversal : DefaultTraversal 66 }; 67 }; 68 69 #define EIGEN_PP_EXPAND(ARG) ARG 70 #if !defined (EIGEN_FAST_MATH) || (EIGEN_FAST_MATH != 1) 71 #define EIGEN_VMLMODE_EXPAND_LA , VML_HA 72 #else 73 #define EIGEN_VMLMODE_EXPAND_LA , VML_LA 74 #endif 75 76 #define EIGEN_VMLMODE_EXPAND__ 77 78 #define EIGEN_VMLMODE_PREFIX_LA vm 79 #define EIGEN_VMLMODE_PREFIX__ v 80 #define EIGEN_VMLMODE_PREFIX(VMLMODE) EIGEN_CAT(EIGEN_VMLMODE_PREFIX_,VMLMODE) 81 82 #define EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, VMLOP, EIGENTYPE, VMLTYPE, VMLMODE) \ 83 template< typename DstXprType, typename SrcXprNested> \ 84 struct Assignment<DstXprType, CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested>, assign_op<EIGENTYPE,EIGENTYPE>, \ 85 Dense2Dense, typename enable_if<vml_assign_traits<DstXprType,SrcXprNested>::EnableVml>::type> { \ 86 typedef CwiseUnaryOp<scalar_##EIGENOP##_op<EIGENTYPE>, SrcXprNested> SrcXprType; \ 87 static void run(DstXprType &dst, const SrcXprType &src, const assign_op<EIGENTYPE,EIGENTYPE> &/*func*/) { \ 88 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); \ 89 if(vml_assign_traits<DstXprType,SrcXprNested>::Traversal==LinearTraversal) { \ 90 VMLOP(dst.size(), (const VMLTYPE*)src.nestedExpression().data(), \ 91 (VMLTYPE*)dst.data() EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_##VMLMODE) ); \ 92 } else { \ 93 const Index outerSize = dst.outerSize(); \ 94 for(Index outer = 0; outer < outerSize; ++outer) { \ 95 const EIGENTYPE *src_ptr = src.IsRowMajor ? &(src.nestedExpression().coeffRef(outer,0)) : \ 96 &(src.nestedExpression().coeffRef(0, outer)); \ 97 EIGENTYPE *dst_ptr = dst.IsRowMajor ? &(dst.coeffRef(outer,0)) : &(dst.coeffRef(0, outer)); \ 98 VMLOP( dst.innerSize(), (const VMLTYPE*)src_ptr, \ 99 (VMLTYPE*)dst_ptr EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_##VMLMODE)); \ 100 } \ 101 } \ 102 } \ 103 }; \ 104 105 106 #define EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(EIGENOP, VMLOP, VMLMODE) \ 107 EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, EIGEN_CAT(EIGEN_VMLMODE_PREFIX(VMLMODE),s##VMLOP), float, float, VMLMODE) \ 108 EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, EIGEN_CAT(EIGEN_VMLMODE_PREFIX(VMLMODE),d##VMLOP), double, double, VMLMODE) 109 110 #define EIGEN_MKL_VML_DECLARE_UNARY_CALLS_CPLX(EIGENOP, VMLOP, VMLMODE) \ 111 EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, EIGEN_CAT(EIGEN_VMLMODE_PREFIX(VMLMODE),c##VMLOP), scomplex, MKL_Complex8, VMLMODE) \ 112 EIGEN_MKL_VML_DECLARE_UNARY_CALL(EIGENOP, EIGEN_CAT(EIGEN_VMLMODE_PREFIX(VMLMODE),z##VMLOP), dcomplex, MKL_Complex16, VMLMODE) 113 114 #define EIGEN_MKL_VML_DECLARE_UNARY_CALLS(EIGENOP, VMLOP, VMLMODE) \ 115 EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(EIGENOP, VMLOP, VMLMODE) \ 116 EIGEN_MKL_VML_DECLARE_UNARY_CALLS_CPLX(EIGENOP, VMLOP, VMLMODE) 117 118 119 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(sin, Sin, LA) 120 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(asin, Asin, LA) 121 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(sinh, Sinh, LA) 122 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(cos, Cos, LA) 123 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(acos, Acos, LA) 124 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(cosh, Cosh, LA) 125 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(tan, Tan, LA) 126 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(atan, Atan, LA) 127 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(tanh, Tanh, LA) 128 // EIGEN_MKL_VML_DECLARE_UNARY_CALLS(abs, Abs, _) 129 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(exp, Exp, LA) 130 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(log, Ln, LA) 131 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(log10, Log10, LA) 132 EIGEN_MKL_VML_DECLARE_UNARY_CALLS(sqrt, Sqrt, _) 133 134 EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(square, Sqr, _) 135 EIGEN_MKL_VML_DECLARE_UNARY_CALLS_CPLX(arg, Arg, _) 136 EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(round, Round, _) 137 EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(floor, Floor, _) 138 EIGEN_MKL_VML_DECLARE_UNARY_CALLS_REAL(ceil, Ceil, _) 139 140 #define EIGEN_MKL_VML_DECLARE_POW_CALL(EIGENOP, VMLOP, EIGENTYPE, VMLTYPE, VMLMODE) \ 141 template< typename DstXprType, typename SrcXprNested, typename Plain> \ 142 struct Assignment<DstXprType, CwiseBinaryOp<scalar_##EIGENOP##_op<EIGENTYPE,EIGENTYPE>, SrcXprNested, \ 143 const CwiseNullaryOp<internal::scalar_constant_op<EIGENTYPE>,Plain> >, assign_op<EIGENTYPE,EIGENTYPE>, \ 144 Dense2Dense, typename enable_if<vml_assign_traits<DstXprType,SrcXprNested>::EnableVml>::type> { \ 145 typedef CwiseBinaryOp<scalar_##EIGENOP##_op<EIGENTYPE,EIGENTYPE>, SrcXprNested, \ 146 const CwiseNullaryOp<internal::scalar_constant_op<EIGENTYPE>,Plain> > SrcXprType; \ 147 static void run(DstXprType &dst, const SrcXprType &src, const assign_op<EIGENTYPE,EIGENTYPE> &/*func*/) { \ 148 eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols()); \ 149 VMLTYPE exponent = reinterpret_cast<const VMLTYPE&>(src.rhs().functor().m_other); \ 150 if(vml_assign_traits<DstXprType,SrcXprNested>::Traversal==LinearTraversal) \ 151 { \ 152 VMLOP( dst.size(), (const VMLTYPE*)src.lhs().data(), exponent, \ 153 (VMLTYPE*)dst.data() EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_##VMLMODE) ); \ 154 } else { \ 155 const Index outerSize = dst.outerSize(); \ 156 for(Index outer = 0; outer < outerSize; ++outer) { \ 157 const EIGENTYPE *src_ptr = src.IsRowMajor ? &(src.lhs().coeffRef(outer,0)) : \ 158 &(src.lhs().coeffRef(0, outer)); \ 159 EIGENTYPE *dst_ptr = dst.IsRowMajor ? &(dst.coeffRef(outer,0)) : &(dst.coeffRef(0, outer)); \ 160 VMLOP( dst.innerSize(), (const VMLTYPE*)src_ptr, exponent, \ 161 (VMLTYPE*)dst_ptr EIGEN_PP_EXPAND(EIGEN_VMLMODE_EXPAND_##VMLMODE)); \ 162 } \ 163 } \ 164 } \ 165 }; 166 167 EIGEN_MKL_VML_DECLARE_POW_CALL(pow, vmsPowx, float, float, LA) 168 EIGEN_MKL_VML_DECLARE_POW_CALL(pow, vmdPowx, double, double, LA) 169 EIGEN_MKL_VML_DECLARE_POW_CALL(pow, vmcPowx, scomplex, MKL_Complex8, LA) 170 EIGEN_MKL_VML_DECLARE_POW_CALL(pow, vmzPowx, dcomplex, MKL_Complex16, LA) 171 172 } // end namespace internal 173 174 } // end namespace Eigen 175 176 #endif // EIGEN_ASSIGN_VML_H 177