1 /*===---- __clang_cuda_runtime_wrapper.h - CUDA runtime support -------------===
2 *
3 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 * See https://llvm.org/LICENSE.txt for license information.
5 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 *
7 *===-----------------------------------------------------------------------===
8 */
9
10 /*
11 * WARNING: This header is intended to be directly -include'd by
12 * the compiler and is not supposed to be included by users.
13 *
14 * CUDA headers are implemented in a way that currently makes it
15 * impossible for user code to #include directly when compiling with
16 * Clang. They present different view of CUDA-supplied functions
17 * depending on where in NVCC's compilation pipeline the headers are
18 * included. Neither of these modes provides function definitions with
19 * correct attributes, so we use preprocessor to force the headers
20 * into a form that Clang can use.
21 *
22 * Similarly to NVCC which -include's cuda_runtime.h, Clang -include's
23 * this file during every CUDA compilation.
24 */
25
26 #ifndef __CLANG_CUDA_RUNTIME_WRAPPER_H__
27 #define __CLANG_CUDA_RUNTIME_WRAPPER_H__
28
29 #if defined(__CUDA__) && defined(__clang__)
30
31 // Include some forward declares that must come before cmath.
32 #include <__clang_cuda_math_forward_declares.h>
33
34 // Define __CUDACC__ early as libstdc++ standard headers with GNU extensions
35 // enabled depend on it to avoid using __float128, which is unsupported in
36 // CUDA.
37 #define __CUDACC__
38
39 // Include some standard headers to avoid CUDA headers including them
40 // while some required macros (like __THROW) are in a weird state.
41 #include <cmath>
42 #include <cstdlib>
43 #include <stdlib.h>
44 #include <string.h>
45 #undef __CUDACC__
46
47 // Preserve common macros that will be changed below by us or by CUDA
48 // headers.
49 #pragma push_macro("__THROW")
50 #pragma push_macro("__CUDA_ARCH__")
51
52 // WARNING: Preprocessor hacks below are based on specific details of
53 // CUDA-7.x headers and are not expected to work with any other
54 // version of CUDA headers.
55 #include "cuda.h"
56 #if !defined(CUDA_VERSION)
57 #error "cuda.h did not define CUDA_VERSION"
58 #elif CUDA_VERSION < 7000
59 #error "Unsupported CUDA version!"
60 #endif
61
62 #pragma push_macro("__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__")
63 #if CUDA_VERSION >= 10000
64 #define __CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__
65 #endif
66
67 // Make largest subset of device functions available during host
68 // compilation -- SM_35 for the time being.
69 #ifndef __CUDA_ARCH__
70 #define __CUDA_ARCH__ 350
71 #endif
72
73 #include "__clang_cuda_builtin_vars.h"
74
75 // No need for device_launch_parameters.h as __clang_cuda_builtin_vars.h above
76 // has taken care of builtin variables declared in the file.
77 #define __DEVICE_LAUNCH_PARAMETERS_H__
78
79 // {math,device}_functions.h only have declarations of the
80 // functions. We don't need them as we're going to pull in their
81 // definitions from .hpp files.
82 #define __DEVICE_FUNCTIONS_H__
83 #define __MATH_FUNCTIONS_H__
84 #define __COMMON_FUNCTIONS_H__
85 // device_functions_decls is replaced by __clang_cuda_device_functions.h
86 // included below.
87 #define __DEVICE_FUNCTIONS_DECLS_H__
88
89 #undef __CUDACC__
90 #if CUDA_VERSION < 9000
91 #define __CUDABE__
92 #else
93 #define __CUDACC__
94 #define __CUDA_LIBDEVICE__
95 #endif
96 // Disables definitions of device-side runtime support stubs in
97 // cuda_device_runtime_api.h
98 #include "host_defines.h"
99 #undef __CUDACC__
100 #include "driver_types.h"
101 #include "host_config.h"
102
103 // Temporarily replace "nv_weak" with weak, so __attribute__((nv_weak)) in
104 // cuda_device_runtime_api.h ends up being __attribute__((weak)) which is the
105 // functional equivalent of what we need.
106 #pragma push_macro("nv_weak")
107 #define nv_weak weak
108 #undef __CUDABE__
109 #undef __CUDA_LIBDEVICE__
110 #define __CUDACC__
111 #include "cuda_runtime.h"
112
113 #pragma pop_macro("nv_weak")
114 #undef __CUDACC__
115 #define __CUDABE__
116
117 // CUDA headers use __nvvm_memcpy and __nvvm_memset which Clang does
118 // not have at the moment. Emulate them with a builtin memcpy/memset.
119 #define __nvvm_memcpy(s, d, n, a) __builtin_memcpy(s, d, n)
120 #define __nvvm_memset(d, c, n, a) __builtin_memset(d, c, n)
121
122 #if CUDA_VERSION < 9000
123 #include "crt/device_runtime.h"
124 #endif
125 #include "crt/host_runtime.h"
126 // device_runtime.h defines __cxa_* macros that will conflict with
127 // cxxabi.h.
128 // FIXME: redefine these as __device__ functions.
129 #undef __cxa_vec_ctor
130 #undef __cxa_vec_cctor
131 #undef __cxa_vec_dtor
132 #undef __cxa_vec_new
133 #undef __cxa_vec_new2
134 #undef __cxa_vec_new3
135 #undef __cxa_vec_delete2
136 #undef __cxa_vec_delete
137 #undef __cxa_vec_delete3
138 #undef __cxa_pure_virtual
139
140 // math_functions.hpp expects this host function be defined on MacOS, but it
141 // ends up not being there because of the games we play here. Just define it
142 // ourselves; it's simple enough.
143 #ifdef __APPLE__
__signbitd(double x)144 inline __host__ double __signbitd(double x) {
145 return std::signbit(x);
146 }
147 #endif
148
149 // CUDA 9.1 no longer provides declarations for libdevice functions, so we need
150 // to provide our own.
151 #include <__clang_cuda_libdevice_declares.h>
152
153 // Wrappers for many device-side standard library functions, incl. math
154 // functions, became compiler builtins in CUDA-9 and have been removed from the
155 // CUDA headers. Clang now provides its own implementation of the wrappers.
156 #if CUDA_VERSION >= 9000
157 #include <__clang_cuda_device_functions.h>
158 #include <__clang_cuda_math.h>
159 #endif
160
161 // __THROW is redefined to be empty by device_functions_decls.h in CUDA. Clang's
162 // counterpart does not do it, so we need to make it empty here to keep
163 // following CUDA includes happy.
164 #undef __THROW
165 #define __THROW
166
167 // CUDA 8.0.41 relies on __USE_FAST_MATH__ and __CUDA_PREC_DIV's values.
168 // Previous versions used to check whether they are defined or not.
169 // CU_DEVICE_INVALID macro is only defined in 8.0.41, so we use it
170 // here to detect the switch.
171
172 #if defined(CU_DEVICE_INVALID)
173 #if !defined(__USE_FAST_MATH__)
174 #define __USE_FAST_MATH__ 0
175 #endif
176
177 #if !defined(__CUDA_PREC_DIV)
178 #define __CUDA_PREC_DIV 0
179 #endif
180 #endif
181
182 // Temporarily poison __host__ macro to ensure it's not used by any of
183 // the headers we're about to include.
184 #pragma push_macro("__host__")
185 #define __host__ UNEXPECTED_HOST_ATTRIBUTE
186
187 // device_functions.hpp and math_functions*.hpp use 'static
188 // __forceinline__' (with no __device__) for definitions of device
189 // functions. Temporarily redefine __forceinline__ to include
190 // __device__.
191 #pragma push_macro("__forceinline__")
192 #define __forceinline__ __device__ __inline__ __attribute__((always_inline))
193 #if CUDA_VERSION < 9000
194 #include "device_functions.hpp"
195 #endif
196
197 // math_function.hpp uses the __USE_FAST_MATH__ macro to determine whether we
198 // get the slow-but-accurate or fast-but-inaccurate versions of functions like
199 // sin and exp. This is controlled in clang by -fcuda-approx-transcendentals.
200 //
201 // device_functions.hpp uses __USE_FAST_MATH__ for a different purpose (fast vs.
202 // slow divides), so we need to scope our define carefully here.
203 #pragma push_macro("__USE_FAST_MATH__")
204 #if defined(__CLANG_CUDA_APPROX_TRANSCENDENTALS__)
205 #define __USE_FAST_MATH__ 1
206 #endif
207
208 #if CUDA_VERSION >= 9000
209 #include "crt/math_functions.hpp"
210 #else
211 #include "math_functions.hpp"
212 #endif
213
214 #pragma pop_macro("__USE_FAST_MATH__")
215
216 #if CUDA_VERSION < 9000
217 #include "math_functions_dbl_ptx3.hpp"
218 #endif
219 #pragma pop_macro("__forceinline__")
220
221 // Pull in host-only functions that are only available when neither
222 // __CUDACC__ nor __CUDABE__ are defined.
223 #undef __MATH_FUNCTIONS_HPP__
224 #undef __CUDABE__
225 #if CUDA_VERSION < 9000
226 #include "math_functions.hpp"
227 #endif
228 // Alas, additional overloads for these functions are hard to get to.
229 // Considering that we only need these overloads for a few functions,
230 // we can provide them here.
rsqrt(float __a)231 static inline float rsqrt(float __a) { return rsqrtf(__a); }
rcbrt(float __a)232 static inline float rcbrt(float __a) { return rcbrtf(__a); }
sinpi(float __a)233 static inline float sinpi(float __a) { return sinpif(__a); }
cospi(float __a)234 static inline float cospi(float __a) { return cospif(__a); }
sincospi(float __a,float * __b,float * __c)235 static inline void sincospi(float __a, float *__b, float *__c) {
236 return sincospif(__a, __b, __c);
237 }
erfcinv(float __a)238 static inline float erfcinv(float __a) { return erfcinvf(__a); }
normcdfinv(float __a)239 static inline float normcdfinv(float __a) { return normcdfinvf(__a); }
normcdf(float __a)240 static inline float normcdf(float __a) { return normcdff(__a); }
erfcx(float __a)241 static inline float erfcx(float __a) { return erfcxf(__a); }
242
243 #if CUDA_VERSION < 9000
244 // For some reason single-argument variant is not always declared by
245 // CUDA headers. Alas, device_functions.hpp included below needs it.
__brkpt(int __c)246 static inline __device__ void __brkpt(int __c) { __brkpt(); }
247 #endif
248
249 // Now include *.hpp with definitions of various GPU functions. Alas,
250 // a lot of thins get declared/defined with __host__ attribute which
251 // we don't want and we have to define it out. We also have to include
252 // {device,math}_functions.hpp again in order to extract the other
253 // branch of #if/else inside.
254 #define __host__
255 #undef __CUDABE__
256 #define __CUDACC__
257 #if CUDA_VERSION >= 9000
258 // Some atomic functions became compiler builtins in CUDA-9 , so we need their
259 // declarations.
260 #include "device_atomic_functions.h"
261 #endif
262 #undef __DEVICE_FUNCTIONS_HPP__
263 #include "device_atomic_functions.hpp"
264 #if CUDA_VERSION >= 9000
265 #include "crt/device_functions.hpp"
266 #include "crt/device_double_functions.hpp"
267 #else
268 #include "device_functions.hpp"
269 #define __CUDABE__
270 #include "device_double_functions.h"
271 #undef __CUDABE__
272 #endif
273 #include "sm_20_atomic_functions.hpp"
274 #include "sm_20_intrinsics.hpp"
275 #include "sm_32_atomic_functions.hpp"
276
277 // Don't include sm_30_intrinsics.h and sm_32_intrinsics.h. These define the
278 // __shfl and __ldg intrinsics using inline (volatile) asm, but we want to
279 // define them using builtins so that the optimizer can reason about and across
280 // these instructions. In particular, using intrinsics for ldg gets us the
281 // [addr+imm] addressing mode, which, although it doesn't actually exist in the
282 // hardware, seems to generate faster machine code because ptxas can more easily
283 // reason about our code.
284
285 #if CUDA_VERSION >= 8000
286 #pragma push_macro("__CUDA_ARCH__")
287 #undef __CUDA_ARCH__
288 #include "sm_60_atomic_functions.hpp"
289 #include "sm_61_intrinsics.hpp"
290 #pragma pop_macro("__CUDA_ARCH__")
291 #endif
292
293 #undef __MATH_FUNCTIONS_HPP__
294
295 // math_functions.hpp defines ::signbit as a __host__ __device__ function. This
296 // conflicts with libstdc++'s constexpr ::signbit, so we have to rename
297 // math_function.hpp's ::signbit. It's guarded by #undef signbit, but that's
298 // conditional on __GNUC__. :)
299 #pragma push_macro("signbit")
300 #pragma push_macro("__GNUC__")
301 #undef __GNUC__
302 #define signbit __ignored_cuda_signbit
303
304 // CUDA-9 omits device-side definitions of some math functions if it sees
305 // include guard from math.h wrapper from libstdc++. We have to undo the header
306 // guard temporarily to get the definitions we need.
307 #pragma push_macro("_GLIBCXX_MATH_H")
308 #pragma push_macro("_LIBCPP_VERSION")
309 #if CUDA_VERSION >= 9000
310 #undef _GLIBCXX_MATH_H
311 // We also need to undo another guard that checks for libc++ 3.8+
312 #ifdef _LIBCPP_VERSION
313 #define _LIBCPP_VERSION 3700
314 #endif
315 #endif
316
317 #if CUDA_VERSION >= 9000
318 #include "crt/math_functions.hpp"
319 #else
320 #include "math_functions.hpp"
321 #endif
322 #pragma pop_macro("_GLIBCXX_MATH_H")
323 #pragma pop_macro("_LIBCPP_VERSION")
324 #pragma pop_macro("__GNUC__")
325 #pragma pop_macro("signbit")
326
327 #pragma pop_macro("__host__")
328
329 #include "texture_indirect_functions.h"
330
331 // Restore state of __CUDA_ARCH__ and __THROW we had on entry.
332 #pragma pop_macro("__CUDA_ARCH__")
333 #pragma pop_macro("__THROW")
334
335 // Set up compiler macros expected to be seen during compilation.
336 #undef __CUDABE__
337 #define __CUDACC__
338
339 extern "C" {
340 // Device-side CUDA system calls.
341 // http://docs.nvidia.com/cuda/ptx-writers-guide-to-interoperability/index.html#system-calls
342 // We need these declarations and wrappers for device-side
343 // malloc/free/printf calls to work without relying on
344 // -fcuda-disable-target-call-checks option.
345 __device__ int vprintf(const char *, const char *);
346 __device__ void free(void *) __attribute((nothrow));
347 __device__ void *malloc(size_t) __attribute((nothrow)) __attribute__((malloc));
348
349 // __assertfail() used to have a `noreturn` attribute. Unfortunately that
350 // contributed to triggering the longstanding bug in ptxas when assert was used
351 // in sufficiently convoluted code. See
352 // https://bugs.llvm.org/show_bug.cgi?id=27738 for the details.
353 __device__ void __assertfail(const char *__message, const char *__file,
354 unsigned __line, const char *__function,
355 size_t __charSize);
356
357 // In order for standard assert() macro on linux to work we need to
358 // provide device-side __assert_fail()
__assert_fail(const char * __message,const char * __file,unsigned __line,const char * __function)359 __device__ static inline void __assert_fail(const char *__message,
360 const char *__file, unsigned __line,
361 const char *__function) {
362 __assertfail(__message, __file, __line, __function, sizeof(char));
363 }
364
365 // Clang will convert printf into vprintf, but we still need
366 // device-side declaration for it.
367 __device__ int printf(const char *, ...);
368 } // extern "C"
369
370 // We also need device-side std::malloc and std::free.
371 namespace std {
free(void * __ptr)372 __device__ static inline void free(void *__ptr) { ::free(__ptr); }
malloc(size_t __size)373 __device__ static inline void *malloc(size_t __size) {
374 return ::malloc(__size);
375 }
376 } // namespace std
377
378 // Out-of-line implementations from __clang_cuda_builtin_vars.h. These need to
379 // come after we've pulled in the definition of uint3 and dim3.
380
dim3()381 __device__ inline __cuda_builtin_threadIdx_t::operator dim3() const {
382 return dim3(x, y, z);
383 }
384
uint3()385 __device__ inline __cuda_builtin_threadIdx_t::operator uint3() const {
386 return {x, y, z};
387 }
388
dim3()389 __device__ inline __cuda_builtin_blockIdx_t::operator dim3() const {
390 return dim3(x, y, z);
391 }
392
uint3()393 __device__ inline __cuda_builtin_blockIdx_t::operator uint3() const {
394 return {x, y, z};
395 }
396
dim3()397 __device__ inline __cuda_builtin_blockDim_t::operator dim3() const {
398 return dim3(x, y, z);
399 }
400
uint3()401 __device__ inline __cuda_builtin_blockDim_t::operator uint3() const {
402 return {x, y, z};
403 }
404
dim3()405 __device__ inline __cuda_builtin_gridDim_t::operator dim3() const {
406 return dim3(x, y, z);
407 }
408
uint3()409 __device__ inline __cuda_builtin_gridDim_t::operator uint3() const {
410 return {x, y, z};
411 }
412
413 #include <__clang_cuda_cmath.h>
414 #include <__clang_cuda_intrinsics.h>
415 #include <__clang_cuda_complex_builtins.h>
416
417 // curand_mtgp32_kernel helpfully redeclares blockDim and threadIdx in host
418 // mode, giving them their "proper" types of dim3 and uint3. This is
419 // incompatible with the types we give in __clang_cuda_builtin_vars.h. As as
420 // hack, force-include the header (nvcc doesn't include it by default) but
421 // redefine dim3 and uint3 to our builtin types. (Thankfully dim3 and uint3 are
422 // only used here for the redeclarations of blockDim and threadIdx.)
423 #pragma push_macro("dim3")
424 #pragma push_macro("uint3")
425 #define dim3 __cuda_builtin_blockDim_t
426 #define uint3 __cuda_builtin_threadIdx_t
427 #include "curand_mtgp32_kernel.h"
428 #pragma pop_macro("dim3")
429 #pragma pop_macro("uint3")
430 #pragma pop_macro("__USE_FAST_MATH__")
431 #pragma pop_macro("__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__")
432
433 // CUDA runtime uses this undocumented function to access kernel launch
434 // configuration. The declaration is in crt/device_functions.h but that file
435 // includes a lot of other stuff we don't want. Instead, we'll provide our own
436 // declaration for it here.
437 #if CUDA_VERSION >= 9020
438 extern "C" unsigned __cudaPushCallConfiguration(dim3 gridDim, dim3 blockDim,
439 size_t sharedMem = 0,
440 void *stream = 0);
441 #endif
442
443 #endif // __CUDA__
444 #endif // __CLANG_CUDA_RUNTIME_WRAPPER_H__
445