1// This file is part of Eigen, a lightweight C++ template library 2// for linear algebra. 3// 4// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr> 5// Copyright (C) 2007-2011 Benoit Jacob <jacob.benoit.1@gmail.com> 6// 7// This Source Code Form is subject to the terms of the Mozilla 8// Public License v. 2.0. If a copy of the MPL was not distributed 9// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. 10 11#ifndef EIGEN_CORE_H 12#define EIGEN_CORE_H 13 14// first thing Eigen does: stop the compiler from committing suicide 15#include "src/Core/util/DisableStupidWarnings.h" 16 17#if defined(__CUDACC__) && !defined(EIGEN_NO_CUDA) 18 #define EIGEN_CUDACC __CUDACC__ 19#endif 20 21#if defined(__CUDA_ARCH__) && !defined(EIGEN_NO_CUDA) 22 #define EIGEN_CUDA_ARCH __CUDA_ARCH__ 23#endif 24 25#if defined(__CUDACC_VER_MAJOR__) && (__CUDACC_VER_MAJOR__ >= 9) 26#define EIGEN_CUDACC_VER ((__CUDACC_VER_MAJOR__ * 10000) + (__CUDACC_VER_MINOR__ * 100)) 27#elif defined(__CUDACC_VER__) 28#define EIGEN_CUDACC_VER __CUDACC_VER__ 29#else 30#define EIGEN_CUDACC_VER 0 31#endif 32 33// Handle NVCC/CUDA/SYCL 34#if defined(__CUDACC__) || defined(__SYCL_DEVICE_ONLY__) 35 // Do not try asserts on CUDA and SYCL! 36 #ifndef EIGEN_NO_DEBUG 37 #define EIGEN_NO_DEBUG 38 #endif 39 40 #ifdef EIGEN_INTERNAL_DEBUGGING 41 #undef EIGEN_INTERNAL_DEBUGGING 42 #endif 43 44 #ifdef EIGEN_EXCEPTIONS 45 #undef EIGEN_EXCEPTIONS 46 #endif 47 48 // All functions callable from CUDA code must be qualified with __device__ 49 #ifdef __CUDACC__ 50 // Do not try to vectorize on CUDA and SYCL! 51 #ifndef EIGEN_DONT_VECTORIZE 52 #define EIGEN_DONT_VECTORIZE 53 #endif 54 55 #define EIGEN_DEVICE_FUNC __host__ __device__ 56 // We need cuda_runtime.h to ensure that that EIGEN_USING_STD_MATH macro 57 // works properly on the device side 58 #include <cuda_runtime.h> 59 #else 60 #define EIGEN_DEVICE_FUNC 61 #endif 62 63#else 64 #define EIGEN_DEVICE_FUNC 65 66#endif 67 68// When compiling CUDA device code with NVCC, pull in math functions from the 69// global namespace. In host mode, and when device doee with clang, use the 70// std versions. 71#if defined(__CUDA_ARCH__) && defined(__NVCC__) 72 #define EIGEN_USING_STD_MATH(FUNC) using ::FUNC; 73#else 74 #define EIGEN_USING_STD_MATH(FUNC) using std::FUNC; 75#endif 76 77#if (defined(_CPPUNWIND) || defined(__EXCEPTIONS)) && !defined(__CUDA_ARCH__) && !defined(EIGEN_EXCEPTIONS) && !defined(EIGEN_USE_SYCL) 78 #define EIGEN_EXCEPTIONS 79#endif 80 81#ifdef EIGEN_EXCEPTIONS 82 #include <new> 83#endif 84 85// then include this file where all our macros are defined. It's really important to do it first because 86// it's where we do all the alignment settings (platform detection and honoring the user's will if he 87// defined e.g. EIGEN_DONT_ALIGN) so it needs to be done before we do anything with vectorization. 88#include "src/Core/util/Macros.h" 89 90// Disable the ipa-cp-clone optimization flag with MinGW 6.x or newer (enabled by default with -O3) 91// See http://eigen.tuxfamily.org/bz/show_bug.cgi?id=556 for details. 92#if EIGEN_COMP_MINGW && EIGEN_GNUC_AT_LEAST(4,6) 93 #pragma GCC optimize ("-fno-ipa-cp-clone") 94#endif 95 96#include <complex> 97 98// this include file manages BLAS and MKL related macros 99// and inclusion of their respective header files 100#include "src/Core/util/MKL_support.h" 101 102// if alignment is disabled, then disable vectorization. Note: EIGEN_MAX_ALIGN_BYTES is the proper check, it takes into 103// account both the user's will (EIGEN_MAX_ALIGN_BYTES,EIGEN_DONT_ALIGN) and our own platform checks 104#if EIGEN_MAX_ALIGN_BYTES==0 105 #ifndef EIGEN_DONT_VECTORIZE 106 #define EIGEN_DONT_VECTORIZE 107 #endif 108#endif 109 110#if EIGEN_COMP_MSVC 111 #include <malloc.h> // for _aligned_malloc -- need it regardless of whether vectorization is enabled 112 #if (EIGEN_COMP_MSVC >= 1500) // 2008 or later 113 // Remember that usage of defined() in a #define is undefined by the standard. 114 // a user reported that in 64-bit mode, MSVC doesn't care to define _M_IX86_FP. 115 #if (defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) || EIGEN_ARCH_x86_64 116 #define EIGEN_SSE2_ON_MSVC_2008_OR_LATER 117 #endif 118 #endif 119#else 120 // Remember that usage of defined() in a #define is undefined by the standard 121 #if (defined __SSE2__) && ( (!EIGEN_COMP_GNUC) || EIGEN_COMP_ICC || EIGEN_GNUC_AT_LEAST(4,2) ) 122 #define EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC 123 #endif 124#endif 125 126#ifndef EIGEN_DONT_VECTORIZE 127 128 #if defined (EIGEN_SSE2_ON_NON_MSVC_BUT_NOT_OLD_GCC) || defined(EIGEN_SSE2_ON_MSVC_2008_OR_LATER) 129 130 // Defines symbols for compile-time detection of which instructions are 131 // used. 132 // EIGEN_VECTORIZE_YY is defined if and only if the instruction set YY is used 133 #define EIGEN_VECTORIZE 134 #define EIGEN_VECTORIZE_SSE 135 #define EIGEN_VECTORIZE_SSE2 136 137 // Detect sse3/ssse3/sse4: 138 // gcc and icc defines __SSE3__, ... 139 // there is no way to know about this on msvc. You can define EIGEN_VECTORIZE_SSE* if you 140 // want to force the use of those instructions with msvc. 141 #ifdef __SSE3__ 142 #define EIGEN_VECTORIZE_SSE3 143 #endif 144 #ifdef __SSSE3__ 145 #define EIGEN_VECTORIZE_SSSE3 146 #endif 147 #ifdef __SSE4_1__ 148 #define EIGEN_VECTORIZE_SSE4_1 149 #endif 150 #ifdef __SSE4_2__ 151 #define EIGEN_VECTORIZE_SSE4_2 152 #endif 153 #ifdef __AVX__ 154 #define EIGEN_VECTORIZE_AVX 155 #define EIGEN_VECTORIZE_SSE3 156 #define EIGEN_VECTORIZE_SSSE3 157 #define EIGEN_VECTORIZE_SSE4_1 158 #define EIGEN_VECTORIZE_SSE4_2 159 #endif 160 #ifdef __AVX2__ 161 #define EIGEN_VECTORIZE_AVX2 162 #endif 163 #ifdef __FMA__ 164 #define EIGEN_VECTORIZE_FMA 165 #endif 166 #if defined(__AVX512F__) && defined(EIGEN_ENABLE_AVX512) 167 #define EIGEN_VECTORIZE_AVX512 168 #define EIGEN_VECTORIZE_AVX2 169 #define EIGEN_VECTORIZE_AVX 170 #define EIGEN_VECTORIZE_FMA 171 #ifdef __AVX512DQ__ 172 #define EIGEN_VECTORIZE_AVX512DQ 173 #endif 174 #ifdef __AVX512ER__ 175 #define EIGEN_VECTORIZE_AVX512ER 176 #endif 177 #endif 178 179 // include files 180 181 // This extern "C" works around a MINGW-w64 compilation issue 182 // https://sourceforge.net/tracker/index.php?func=detail&aid=3018394&group_id=202880&atid=983354 183 // In essence, intrin.h is included by windows.h and also declares intrinsics (just as emmintrin.h etc. below do). 184 // However, intrin.h uses an extern "C" declaration, and g++ thus complains of duplicate declarations 185 // with conflicting linkage. The linkage for intrinsics doesn't matter, but at that stage the compiler doesn't know; 186 // so, to avoid compile errors when windows.h is included after Eigen/Core, ensure intrinsics are extern "C" here too. 187 // notice that since these are C headers, the extern "C" is theoretically needed anyways. 188 extern "C" { 189 // In theory we should only include immintrin.h and not the other *mmintrin.h header files directly. 190 // Doing so triggers some issues with ICC. However old gcc versions seems to not have this file, thus: 191 #if EIGEN_COMP_ICC >= 1110 192 #include <immintrin.h> 193 #else 194 #include <mmintrin.h> 195 #include <emmintrin.h> 196 #include <xmmintrin.h> 197 #ifdef EIGEN_VECTORIZE_SSE3 198 #include <pmmintrin.h> 199 #endif 200 #ifdef EIGEN_VECTORIZE_SSSE3 201 #include <tmmintrin.h> 202 #endif 203 #ifdef EIGEN_VECTORIZE_SSE4_1 204 #include <smmintrin.h> 205 #endif 206 #ifdef EIGEN_VECTORIZE_SSE4_2 207 #include <nmmintrin.h> 208 #endif 209 #if defined(EIGEN_VECTORIZE_AVX) || defined(EIGEN_VECTORIZE_AVX512) 210 #include <immintrin.h> 211 #endif 212 #endif 213 } // end extern "C" 214 #elif defined __VSX__ 215 #define EIGEN_VECTORIZE 216 #define EIGEN_VECTORIZE_VSX 217 #include <altivec.h> 218 // We need to #undef all these ugly tokens defined in <altivec.h> 219 // => use __vector instead of vector 220 #undef bool 221 #undef vector 222 #undef pixel 223 #elif defined __ALTIVEC__ 224 #define EIGEN_VECTORIZE 225 #define EIGEN_VECTORIZE_ALTIVEC 226 #include <altivec.h> 227 // We need to #undef all these ugly tokens defined in <altivec.h> 228 // => use __vector instead of vector 229 #undef bool 230 #undef vector 231 #undef pixel 232 #elif (defined __ARM_NEON) || (defined __ARM_NEON__) 233 #define EIGEN_VECTORIZE 234 #define EIGEN_VECTORIZE_NEON 235 #include <arm_neon.h> 236 #elif (defined __s390x__ && defined __VEC__) 237 #define EIGEN_VECTORIZE 238 #define EIGEN_VECTORIZE_ZVECTOR 239 #include <vecintrin.h> 240 #endif 241#endif 242 243#if defined(__F16C__) && !defined(EIGEN_COMP_CLANG) 244 // We can use the optimized fp16 to float and float to fp16 conversion routines 245 #define EIGEN_HAS_FP16_C 246#endif 247 248#if defined __CUDACC__ 249 #define EIGEN_VECTORIZE_CUDA 250 #include <vector_types.h> 251 #if EIGEN_CUDACC_VER >= 70500 252 #define EIGEN_HAS_CUDA_FP16 253 #endif 254#endif 255 256#if defined EIGEN_HAS_CUDA_FP16 257 #include <host_defines.h> 258 #include <cuda_fp16.h> 259#endif 260 261#if (defined _OPENMP) && (!defined EIGEN_DONT_PARALLELIZE) 262 #define EIGEN_HAS_OPENMP 263#endif 264 265#ifdef EIGEN_HAS_OPENMP 266#include <omp.h> 267#endif 268 269// MSVC for windows mobile does not have the errno.h file 270#if !(EIGEN_COMP_MSVC && EIGEN_OS_WINCE) && !EIGEN_COMP_ARM 271#define EIGEN_HAS_ERRNO 272#endif 273 274#ifdef EIGEN_HAS_ERRNO 275#include <cerrno> 276#endif 277#include <cstddef> 278#include <cstdlib> 279#include <cmath> 280#include <cassert> 281#include <functional> 282#include <iosfwd> 283#include <cstring> 284#include <string> 285#include <limits> 286#include <climits> // for CHAR_BIT 287// for min/max: 288#include <algorithm> 289 290// for std::is_nothrow_move_assignable 291#ifdef EIGEN_INCLUDE_TYPE_TRAITS 292#include <type_traits> 293#endif 294 295// for outputting debug info 296#ifdef EIGEN_DEBUG_ASSIGN 297#include <iostream> 298#endif 299 300// required for __cpuid, needs to be included after cmath 301#if EIGEN_COMP_MSVC && EIGEN_ARCH_i386_OR_x86_64 && !EIGEN_OS_WINCE 302 #include <intrin.h> 303#endif 304 305/** \brief Namespace containing all symbols from the %Eigen library. */ 306namespace Eigen { 307 308inline static const char *SimdInstructionSetsInUse(void) { 309#if defined(EIGEN_VECTORIZE_AVX512) 310 return "AVX512, FMA, AVX2, AVX, SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; 311#elif defined(EIGEN_VECTORIZE_AVX) 312 return "AVX SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; 313#elif defined(EIGEN_VECTORIZE_SSE4_2) 314 return "SSE, SSE2, SSE3, SSSE3, SSE4.1, SSE4.2"; 315#elif defined(EIGEN_VECTORIZE_SSE4_1) 316 return "SSE, SSE2, SSE3, SSSE3, SSE4.1"; 317#elif defined(EIGEN_VECTORIZE_SSSE3) 318 return "SSE, SSE2, SSE3, SSSE3"; 319#elif defined(EIGEN_VECTORIZE_SSE3) 320 return "SSE, SSE2, SSE3"; 321#elif defined(EIGEN_VECTORIZE_SSE2) 322 return "SSE, SSE2"; 323#elif defined(EIGEN_VECTORIZE_ALTIVEC) 324 return "AltiVec"; 325#elif defined(EIGEN_VECTORIZE_VSX) 326 return "VSX"; 327#elif defined(EIGEN_VECTORIZE_NEON) 328 return "ARM NEON"; 329#elif defined(EIGEN_VECTORIZE_ZVECTOR) 330 return "S390X ZVECTOR"; 331#else 332 return "None"; 333#endif 334} 335 336} // end namespace Eigen 337 338#if defined EIGEN2_SUPPORT_STAGE40_FULL_EIGEN3_STRICTNESS || defined EIGEN2_SUPPORT_STAGE30_FULL_EIGEN3_API || defined EIGEN2_SUPPORT_STAGE20_RESOLVE_API_CONFLICTS || defined EIGEN2_SUPPORT_STAGE10_FULL_EIGEN2_API || defined EIGEN2_SUPPORT 339// This will generate an error message: 340#error Eigen2-support is only available up to version 3.2. Please go to "http://eigen.tuxfamily.org/index.php?title=Eigen2" for further information 341#endif 342 343namespace Eigen { 344 345// we use size_t frequently and we'll never remember to prepend it with std:: everytime just to 346// ensure QNX/QCC support 347using std::size_t; 348// gcc 4.6.0 wants std:: for ptrdiff_t 349using std::ptrdiff_t; 350 351} 352 353/** \defgroup Core_Module Core module 354 * This is the main module of Eigen providing dense matrix and vector support 355 * (both fixed and dynamic size) with all the features corresponding to a BLAS library 356 * and much more... 357 * 358 * \code 359 * #include <Eigen/Core> 360 * \endcode 361 */ 362 363#include "src/Core/util/Constants.h" 364#include "src/Core/util/Meta.h" 365#include "src/Core/util/ForwardDeclarations.h" 366#include "src/Core/util/StaticAssert.h" 367#include "src/Core/util/XprHelper.h" 368#include "src/Core/util/Memory.h" 369 370#include "src/Core/NumTraits.h" 371#include "src/Core/MathFunctions.h" 372#include "src/Core/GenericPacketMath.h" 373#include "src/Core/MathFunctionsImpl.h" 374#include "src/Core/arch/Default/ConjHelper.h" 375 376#if defined EIGEN_VECTORIZE_AVX512 377 #include "src/Core/arch/SSE/PacketMath.h" 378 #include "src/Core/arch/AVX/PacketMath.h" 379 #include "src/Core/arch/AVX512/PacketMath.h" 380 #include "src/Core/arch/AVX512/MathFunctions.h" 381#elif defined EIGEN_VECTORIZE_AVX 382 // Use AVX for floats and doubles, SSE for integers 383 #include "src/Core/arch/SSE/PacketMath.h" 384 #include "src/Core/arch/SSE/Complex.h" 385 #include "src/Core/arch/SSE/MathFunctions.h" 386 #include "src/Core/arch/AVX/PacketMath.h" 387 #include "src/Core/arch/AVX/MathFunctions.h" 388 #include "src/Core/arch/AVX/Complex.h" 389 #include "src/Core/arch/AVX/TypeCasting.h" 390 #include "src/Core/arch/SSE/TypeCasting.h" 391#elif defined EIGEN_VECTORIZE_SSE 392 #include "src/Core/arch/SSE/PacketMath.h" 393 #include "src/Core/arch/SSE/MathFunctions.h" 394 #include "src/Core/arch/SSE/Complex.h" 395 #include "src/Core/arch/SSE/TypeCasting.h" 396#elif defined(EIGEN_VECTORIZE_ALTIVEC) || defined(EIGEN_VECTORIZE_VSX) 397 #include "src/Core/arch/AltiVec/PacketMath.h" 398 #include "src/Core/arch/AltiVec/MathFunctions.h" 399 #include "src/Core/arch/AltiVec/Complex.h" 400#elif defined EIGEN_VECTORIZE_NEON 401 #include "src/Core/arch/NEON/PacketMath.h" 402 #include "src/Core/arch/NEON/MathFunctions.h" 403 #include "src/Core/arch/NEON/Complex.h" 404#elif defined EIGEN_VECTORIZE_ZVECTOR 405 #include "src/Core/arch/ZVector/PacketMath.h" 406 #include "src/Core/arch/ZVector/MathFunctions.h" 407 #include "src/Core/arch/ZVector/Complex.h" 408#endif 409 410// Half float support 411#include "src/Core/arch/CUDA/Half.h" 412#include "src/Core/arch/CUDA/PacketMathHalf.h" 413#include "src/Core/arch/CUDA/TypeCasting.h" 414 415#if defined EIGEN_VECTORIZE_CUDA 416 #include "src/Core/arch/CUDA/PacketMath.h" 417 #include "src/Core/arch/CUDA/MathFunctions.h" 418#endif 419 420#include "src/Core/arch/Default/Settings.h" 421 422#include "src/Core/functors/TernaryFunctors.h" 423#include "src/Core/functors/BinaryFunctors.h" 424#include "src/Core/functors/UnaryFunctors.h" 425#include "src/Core/functors/NullaryFunctors.h" 426#include "src/Core/functors/StlFunctors.h" 427#include "src/Core/functors/AssignmentFunctors.h" 428 429// Specialized functors to enable the processing of complex numbers 430// on CUDA devices 431#include "src/Core/arch/CUDA/Complex.h" 432 433#include "src/Core/IO.h" 434#include "src/Core/DenseCoeffsBase.h" 435#include "src/Core/DenseBase.h" 436#include "src/Core/MatrixBase.h" 437#include "src/Core/EigenBase.h" 438 439#include "src/Core/Product.h" 440#include "src/Core/CoreEvaluators.h" 441#include "src/Core/AssignEvaluator.h" 442 443#ifndef EIGEN_PARSED_BY_DOXYGEN // work around Doxygen bug triggered by Assign.h r814874 444 // at least confirmed with Doxygen 1.5.5 and 1.5.6 445 #include "src/Core/Assign.h" 446#endif 447 448#include "src/Core/ArrayBase.h" 449#include "src/Core/util/BlasUtil.h" 450#include "src/Core/DenseStorage.h" 451#include "src/Core/NestByValue.h" 452 453// #include "src/Core/ForceAlignedAccess.h" 454 455#include "src/Core/ReturnByValue.h" 456#include "src/Core/NoAlias.h" 457#include "src/Core/PlainObjectBase.h" 458#include "src/Core/Matrix.h" 459#include "src/Core/Array.h" 460#include "src/Core/CwiseTernaryOp.h" 461#include "src/Core/CwiseBinaryOp.h" 462#include "src/Core/CwiseUnaryOp.h" 463#include "src/Core/CwiseNullaryOp.h" 464#include "src/Core/CwiseUnaryView.h" 465#include "src/Core/SelfCwiseBinaryOp.h" 466#include "src/Core/Dot.h" 467#include "src/Core/StableNorm.h" 468#include "src/Core/Stride.h" 469#include "src/Core/MapBase.h" 470#include "src/Core/Map.h" 471#include "src/Core/Ref.h" 472#include "src/Core/Block.h" 473#include "src/Core/VectorBlock.h" 474#include "src/Core/Transpose.h" 475#include "src/Core/DiagonalMatrix.h" 476#include "src/Core/Diagonal.h" 477#include "src/Core/DiagonalProduct.h" 478#include "src/Core/Redux.h" 479#include "src/Core/Visitor.h" 480#include "src/Core/Fuzzy.h" 481#include "src/Core/Swap.h" 482#include "src/Core/CommaInitializer.h" 483#include "src/Core/GeneralProduct.h" 484#include "src/Core/Solve.h" 485#include "src/Core/Inverse.h" 486#include "src/Core/SolverBase.h" 487#include "src/Core/PermutationMatrix.h" 488#include "src/Core/Transpositions.h" 489#include "src/Core/TriangularMatrix.h" 490#include "src/Core/SelfAdjointView.h" 491#include "src/Core/products/GeneralBlockPanelKernel.h" 492#include "src/Core/products/Parallelizer.h" 493#include "src/Core/ProductEvaluators.h" 494#include "src/Core/products/GeneralMatrixVector.h" 495#include "src/Core/products/GeneralMatrixMatrix.h" 496#include "src/Core/SolveTriangular.h" 497#include "src/Core/products/GeneralMatrixMatrixTriangular.h" 498#include "src/Core/products/SelfadjointMatrixVector.h" 499#include "src/Core/products/SelfadjointMatrixMatrix.h" 500#include "src/Core/products/SelfadjointProduct.h" 501#include "src/Core/products/SelfadjointRank2Update.h" 502#include "src/Core/products/TriangularMatrixVector.h" 503#include "src/Core/products/TriangularMatrixMatrix.h" 504#include "src/Core/products/TriangularSolverMatrix.h" 505#include "src/Core/products/TriangularSolverVector.h" 506#include "src/Core/BandMatrix.h" 507#include "src/Core/CoreIterators.h" 508#include "src/Core/ConditionEstimator.h" 509 510#include "src/Core/BooleanRedux.h" 511#include "src/Core/Select.h" 512#include "src/Core/VectorwiseOp.h" 513#include "src/Core/Random.h" 514#include "src/Core/Replicate.h" 515#include "src/Core/Reverse.h" 516#include "src/Core/ArrayWrapper.h" 517 518#ifdef EIGEN_USE_BLAS 519#include "src/Core/products/GeneralMatrixMatrix_BLAS.h" 520#include "src/Core/products/GeneralMatrixVector_BLAS.h" 521#include "src/Core/products/GeneralMatrixMatrixTriangular_BLAS.h" 522#include "src/Core/products/SelfadjointMatrixMatrix_BLAS.h" 523#include "src/Core/products/SelfadjointMatrixVector_BLAS.h" 524#include "src/Core/products/TriangularMatrixMatrix_BLAS.h" 525#include "src/Core/products/TriangularMatrixVector_BLAS.h" 526#include "src/Core/products/TriangularSolverMatrix_BLAS.h" 527#endif // EIGEN_USE_BLAS 528 529#ifdef EIGEN_USE_MKL_VML 530#include "src/Core/Assign_MKL.h" 531#endif 532 533#include "src/Core/GlobalFunctions.h" 534 535#include "src/Core/util/ReenableStupidWarnings.h" 536 537#endif // EIGEN_CORE_H 538