1 //////////////////////////////////////////////////////////////////////////////////////
2 // This file is distributed under the University of Illinois/NCSA Open Source License.
3 // See LICENSE file in top directory for details.
4 //
5 // Copyright (c) 2016 Jeongnim Kim and QMCPACK developers.
6 //
7 // File developed by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
8 //                    Ying Wai Li, yingwaili@ornl.gov, Oak Ridge National Laboratory
9 //                    Ye Luo, yeluo@anl.gov, Argonne National Laboratory
10 //
11 // File created by: Jeongnim Kim, jeongnim.kim@gmail.com, University of Illinois at Urbana-Champaign
12 //////////////////////////////////////////////////////////////////////////////////////
13 
14 
15 #ifndef GPU_MISC_H
16 #define GPU_MISC_H
17 
18 #include <cstdlib>
19 #include <cstdio>
20 
21 #include <cuda_runtime_api.h>
22 #include <vector>
23 
24 #include <cublas_v2.h>
25 
26 namespace gpu
27 {
28 extern cudaStream_t kernelStream;
29 extern cudaStream_t memoryStream;
30 
31 extern cudaEvent_t syncEvent;
32 
33 extern cudaEvent_t gradientSyncDiracEvent;
34 extern cudaEvent_t gradientSyncOneBodyEvent;
35 extern cudaEvent_t gradientSyncTwoBodyEvent;
36 
37 extern cudaEvent_t ratioSyncDiracEvent;
38 extern cudaEvent_t ratioSyncOneBodyEvent;
39 extern cudaEvent_t ratioSyncTwoBodyEvent;
40 
41 extern cublasHandle_t cublasHandle;
42 
43 extern size_t MaxGPUSpineSizeMB;
44 extern int rank;
45 extern int relative_rank;
46 extern int device_group_size;
47 extern bool cudamps;
48 extern std::vector<int> device_group_numbers;
49 extern std::vector<int> device_rank_numbers;
50 
51 void initCUDAStreams();
52 void initCUDAEvents();
53 void initCublas();
54 
55 void finalizeCUDAStreams();
56 void finalizeCUDAEvents();
57 void finalizeCublas();
58 
59 void synchronize();
60 
61 void streamsSynchronize();
62 
63 } // namespace gpu
64 #endif
65