1 // Copyright 2009-2021 Intel Corporation
2 // SPDX-License-Identifier: Apache-2.0
3 
4 #pragma once
5 
6 #include "../../common/sys/platform.h"
7 #include "../../common/sys/sysinfo.h"
8 
9 namespace embree
10 {
11 #define DEFINE_SYMBOL2(type,name)               \
12   typedef type (*name##Func)();                 \
13   name##Func name;
14 
15 #define DECLARE_SYMBOL2(type,name)                                       \
16   namespace sse2   { extern type name(); }                           \
17   namespace sse42  { extern type name(); }                           \
18   namespace avx    { extern type name(); }                           \
19   namespace avx2   { extern type name(); }                           \
20   namespace avx512 { extern type name(); }                           \
21   void name##_error2() { throw_RTCError(RTC_ERROR_UNKNOWN,"internal error in ISA selection for " TOSTRING(name)); } \
22   type name##_error() { return type(name##_error2); }                   \
23   type name##_zero() { return type(nullptr); }
24 
25 #define DECLARE_ISA_FUNCTION(type,symbol,args)                            \
26   namespace sse2   { extern type symbol(args); }                       \
27   namespace sse42  { extern type symbol(args); }                       \
28   namespace avx    { extern type symbol(args); }                       \
29   namespace avx2   { extern type symbol(args); }                       \
30   namespace avx512 { extern type symbol(args); }                     \
31   inline type symbol##_error(args) { throw_RTCError(RTC_ERROR_UNSUPPORTED_CPU,"function " TOSTRING(symbol) " not supported by your CPU"); } \
32   typedef type (*symbol##Ty)(args);                                       \
33 
34 #define DEFINE_ISA_FUNCTION(type,symbol,args)   \
35   typedef type (*symbol##Func)(args);           \
36   symbol##Func symbol;
37 
38 #define ZERO_SYMBOL(features,intersector)                      \
39   intersector = intersector##_zero;
40 
41 #define INIT_SYMBOL(features,intersector)                      \
42   intersector = decltype(intersector)(intersector##_error);
43 
44 #define SELECT_SYMBOL_DEFAULT(features,intersector) \
45   intersector = isa::intersector;
46 
47 #if defined(__SSE__)
48 #if !defined(EMBREE_TARGET_SIMD4)
49 #define EMBREE_TARGET_SIMD4
50 #endif
51 #endif
52 
53 #if defined(EMBREE_TARGET_SSE42)
54 #define SELECT_SYMBOL_SSE42(features,intersector) \
55   if ((features & SSE42) == SSE42) intersector = sse42::intersector;
56 #else
57 #define SELECT_SYMBOL_SSE42(features,intersector)
58 #endif
59 
60 #if defined(EMBREE_TARGET_AVX) || defined(__AVX__)
61 #if !defined(EMBREE_TARGET_SIMD8)
62 #define EMBREE_TARGET_SIMD8
63 #endif
64 #if defined(__AVX__) // if default ISA is >= AVX we treat AVX target as default target
65 #define SELECT_SYMBOL_AVX(features,intersector)                 \
66   if ((features & ISA) == ISA) intersector = isa::intersector;
67 #else
68 #define SELECT_SYMBOL_AVX(features,intersector)                 \
69   if ((features & AVX) == AVX) intersector = avx::intersector;
70 #endif
71 #else
72 #define SELECT_SYMBOL_AVX(features,intersector)
73 #endif
74 
75 #if defined(EMBREE_TARGET_AVX2)
76 #if !defined(EMBREE_TARGET_SIMD8)
77 #define EMBREE_TARGET_SIMD8
78 #endif
79 #define SELECT_SYMBOL_AVX2(features,intersector) \
80   if ((features & AVX2) == AVX2) intersector = avx2::intersector;
81 #else
82 #define SELECT_SYMBOL_AVX2(features,intersector)
83 #endif
84 
85 #if defined(EMBREE_TARGET_AVX512)
86 #if !defined(EMBREE_TARGET_SIMD16)
87 #define EMBREE_TARGET_SIMD16
88 #endif
89 #define SELECT_SYMBOL_AVX512(features,intersector) \
90   if ((features & AVX512) == AVX512) intersector = avx512::intersector;
91 #else
92 #define SELECT_SYMBOL_AVX512(features,intersector)
93 #endif
94 
95 #define SELECT_SYMBOL_DEFAULT_SSE42(features,intersector) \
96   SELECT_SYMBOL_DEFAULT(features,intersector);            \
97   SELECT_SYMBOL_SSE42(features,intersector);
98 
99 #define SELECT_SYMBOL_DEFAULT_SSE42_AVX(features,intersector) \
100   SELECT_SYMBOL_DEFAULT(features,intersector);                \
101   SELECT_SYMBOL_SSE42(features,intersector);                  \
102   SELECT_SYMBOL_AVX(features,intersector);
103 
104 #define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2(features,intersector) \
105   SELECT_SYMBOL_DEFAULT(features,intersector);                     \
106   SELECT_SYMBOL_SSE42(features,intersector);                       \
107   SELECT_SYMBOL_AVX(features,intersector);                         \
108   SELECT_SYMBOL_AVX2(features,intersector);
109 
110 #define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX512(features,intersector) \
111   SELECT_SYMBOL_DEFAULT(features,intersector);                          \
112   SELECT_SYMBOL_SSE42(features,intersector);                            \
113   SELECT_SYMBOL_AVX(features,intersector);                              \
114   SELECT_SYMBOL_AVX512(features,intersector);
115 
116 #define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \
117   SELECT_SYMBOL_DEFAULT(features,intersector);                                   \
118   SELECT_SYMBOL_AVX(features,intersector);                                       \
119   SELECT_SYMBOL_AVX2(features,intersector);                                      \
120   SELECT_SYMBOL_AVX512(features,intersector);
121 
122 #define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \
123   SELECT_SYMBOL_DEFAULT(features,intersector);                         \
124   SELECT_SYMBOL_AVX(features,intersector);                             \
125   SELECT_SYMBOL_AVX2(features,intersector);                            \
126   SELECT_SYMBOL_AVX512(features,intersector);
127 
128 #define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2_AVX512(features,intersector) \
129   SELECT_SYMBOL_DEFAULT(features,intersector);                                         \
130   SELECT_SYMBOL_SSE42(features,intersector);                                           \
131   SELECT_SYMBOL_AVX(features,intersector);                                             \
132   SELECT_SYMBOL_AVX2(features,intersector);                                            \
133   SELECT_SYMBOL_AVX512(features,intersector);
134 
135 #define SELECT_SYMBOL_DEFAULT_SSE42_AVX_AVX2_AVX512(features,intersector) \
136   SELECT_SYMBOL_DEFAULT(features,intersector);                               \
137   SELECT_SYMBOL_SSE42(features,intersector);                                 \
138   SELECT_SYMBOL_AVX(features,intersector);                                   \
139   SELECT_SYMBOL_AVX2(features,intersector);                                  \
140   SELECT_SYMBOL_AVX512(features,intersector);
141 
142 #define SELECT_SYMBOL_DEFAULT_AVX(features,intersector) \
143   SELECT_SYMBOL_DEFAULT(features,intersector);          \
144   SELECT_SYMBOL_AVX(features,intersector);
145 
146 #define SELECT_SYMBOL_DEFAULT_AVX_AVX2(features,intersector) \
147   SELECT_SYMBOL_DEFAULT(features,intersector);               \
148   SELECT_SYMBOL_AVX(features,intersector);                   \
149   SELECT_SYMBOL_AVX2(features,intersector);
150 
151 #define SELECT_SYMBOL_DEFAULT_AVX(features,intersector) \
152   SELECT_SYMBOL_DEFAULT(features,intersector);                    \
153   SELECT_SYMBOL_AVX(features,intersector);
154 
155 #define SELECT_SYMBOL_DEFAULT_AVX_AVX512(features,intersector) \
156   SELECT_SYMBOL_DEFAULT(features,intersector);                              \
157   SELECT_SYMBOL_AVX(features,intersector);                                  \
158   SELECT_SYMBOL_AVX512(features,intersector);
159 
160 #define SELECT_SYMBOL_DEFAULT_AVX_AVX512(features,intersector) \
161   SELECT_SYMBOL_DEFAULT(features,intersector);                    \
162   SELECT_SYMBOL_AVX(features,intersector);                        \
163   SELECT_SYMBOL_AVX512(features,intersector);
164 
165 #define SELECT_SYMBOL_INIT_AVX(features,intersector) \
166   INIT_SYMBOL(features,intersector);                 \
167   SELECT_SYMBOL_AVX(features,intersector);
168 
169 #define SELECT_SYMBOL_INIT_AVX_AVX2(features,intersector) \
170   INIT_SYMBOL(features,intersector);                      \
171   SELECT_SYMBOL_AVX(features,intersector);                \
172   SELECT_SYMBOL_AVX2(features,intersector);
173 
174 #define SELECT_SYMBOL_INIT_AVX_AVX2_AVX512(features,intersector) \
175   INIT_SYMBOL(features,intersector);                                \
176   SELECT_SYMBOL_AVX(features,intersector);                          \
177   SELECT_SYMBOL_AVX2(features,intersector);                         \
178   SELECT_SYMBOL_AVX512(features,intersector);
179 
180 #define SELECT_SYMBOL_INIT_SSE42_AVX_AVX2(features,intersector) \
181   INIT_SYMBOL(features,intersector);                            \
182   SELECT_SYMBOL_SSE42(features,intersector);                    \
183   SELECT_SYMBOL_AVX(features,intersector);                      \
184   SELECT_SYMBOL_AVX2(features,intersector);
185 
186 #define SELECT_SYMBOL_INIT_AVX(features,intersector) \
187   INIT_SYMBOL(features,intersector);                           \
188   SELECT_SYMBOL_AVX(features,intersector);
189 
190 #define SELECT_SYMBOL_INIT_AVX_AVX512(features,intersector) \
191   INIT_SYMBOL(features,intersector);                                     \
192   SELECT_SYMBOL_AVX(features,intersector);                               \
193   SELECT_SYMBOL_AVX512(features,intersector);
194 
195 #define SELECT_SYMBOL_INIT_AVX_AVX2(features,intersector) \
196   INIT_SYMBOL(features,intersector);                                \
197   SELECT_SYMBOL_AVX(features,intersector);                          \
198   SELECT_SYMBOL_AVX2(features,intersector);
199 
200 #define SELECT_SYMBOL_INIT_AVX_AVX2_AVX512(features,intersector) \
201   INIT_SYMBOL(features,intersector);                                          \
202   SELECT_SYMBOL_AVX(features,intersector);                                    \
203   SELECT_SYMBOL_AVX2(features,intersector);                                   \
204   SELECT_SYMBOL_AVX512(features,intersector);
205 
206 #define SELECT_SYMBOL_INIT_SSE42_AVX_AVX2_AVX512(features,intersector) \
207   INIT_SYMBOL(features,intersector);                                                \
208   SELECT_SYMBOL_SSE42(features,intersector);                                        \
209   SELECT_SYMBOL_AVX(features,intersector);                                          \
210   SELECT_SYMBOL_AVX2(features,intersector);                                         \
211   SELECT_SYMBOL_AVX512(features,intersector);
212 
213 #define SELECT_SYMBOL_ZERO_SSE42_AVX_AVX2_AVX512(features,intersector) \
214   ZERO_SYMBOL(features,intersector);                                    \
215   SELECT_SYMBOL_SSE42(features,intersector);                            \
216   SELECT_SYMBOL_AVX(features,intersector);                              \
217   SELECT_SYMBOL_AVX2(features,intersector);                             \
218   SELECT_SYMBOL_AVX512(features,intersector);
219 
220 #define SELECT_SYMBOL_DEFAULT_AVX_AVX2_AVX512(features,intersector) \
221   SELECT_SYMBOL_DEFAULT(features,intersector);                                   \
222   SELECT_SYMBOL_AVX(features,intersector);                                       \
223   SELECT_SYMBOL_AVX2(features,intersector);                                      \
224   SELECT_SYMBOL_AVX512(features,intersector);
225 
226 #define SELECT_SYMBOL_INIT_AVX512(features,intersector) \
227   INIT_SYMBOL(features,intersector);                                 \
228   SELECT_SYMBOL_AVX512(features,intersector);
229 
230 #define SELECT_SYMBOL_SSE42_AVX_AVX2(features,intersector) \
231   SELECT_SYMBOL_SSE42(features,intersector);               \
232   SELECT_SYMBOL_AVX(features,intersector);                 \
233   SELECT_SYMBOL_AVX2(features,intersector);
234 
235   struct VerifyMultiTargetLinking {
236     static __noinline int getISA(int depth = 5) {
237       if (depth == 0) return ISA;
238       else return getISA(depth-1);
239     }
240   };
241   namespace sse2   { int getISA(); };
242   namespace sse42  { int getISA(); };
243   namespace avx    { int getISA(); };
244   namespace avx2   { int getISA(); };
245   namespace avx512 { int getISA(); };
246 }
247