1 /* ----------------------------------------------------------------------
2 SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
3 http://sparta.sandia.gov
4 Steve Plimpton, sjplimp@sandia.gov, Michael Gallis, magalli@sandia.gov
5 Sandia National Laboratories
6
7 Copyright (2014) Sandia Corporation. Under the terms of Contract
8 DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
9 certain rights in this software. This software is distributed under
10 the GNU General Public License.
11
12 See the README file in the top-level SPARTA directory.
13 ------------------------------------------------------------------------- */
14
15 #include "string.h"
16 #include "compute_surf_kokkos.h"
17 #include "particle_kokkos.h"
18 #include "mixture.h"
19 #include "surf_kokkos.h"
20 #include "grid.h"
21 #include "update.h"
22 #include "modify.h"
23 #include "domain.h"
24 #include "memory_kokkos.h"
25 #include "error.h"
26 #include "sparta_masks.h"
27 #include "kokkos.h"
28
29 using namespace SPARTA_NS;
30
31 /* ---------------------------------------------------------------------- */
32
ComputeSurfKokkos(SPARTA * sparta,int narg,char ** arg)33 ComputeSurfKokkos::ComputeSurfKokkos(SPARTA *sparta, int narg, char **arg) :
34 ComputeSurf(sparta, narg, arg)
35 {
36 kokkos_flag = 1;
37 d_which = DAT::t_int_1d("surf:which",nvalue);
38 }
39
ComputeSurfKokkos(SPARTA * sparta)40 ComputeSurfKokkos::ComputeSurfKokkos(SPARTA *sparta) :
41 ComputeSurf(sparta)
42 {
43 hash = NULL;
44 which = NULL;
45 array_surf_tally = NULL;
46 tally2surf = NULL;
47 array_surf = NULL;
48 vector_surf = NULL;
49 normflux = NULL;
50 id = NULL;
51 style = NULL;
52 tlist = NULL;
53 }
54
55 /* ---------------------------------------------------------------------- */
56
~ComputeSurfKokkos()57 ComputeSurfKokkos::~ComputeSurfKokkos()
58 {
59 if (copy || copymode) return;
60
61 memoryKK->destroy_kokkos(k_tally2surf,tally2surf);
62 memoryKK->destroy_kokkos(k_array_surf_tally,array_surf_tally);
63 }
64
65 /* ---------------------------------------------------------------------- */
66
init()67 void ComputeSurfKokkos::init()
68 {
69 ComputeSurf::init();
70
71 auto h_which = Kokkos::create_mirror_view(d_which);
72 for (int n=0; n<nvalue; n++)
73 h_which(n) = which[n];
74 Kokkos::deep_copy(d_which,h_which);
75 }
76
77 /* ----------------------------------------------------------------------
78 set normflux for all surfs I store
79 all: just nlocal
80 distributed: nlocal + nghost
81 called by init before each run (in case dt or fnum has changed)
82 called whenever grid changes
83 ------------------------------------------------------------------------- */
84
init_normflux()85 void ComputeSurfKokkos::init_normflux()
86 {
87 ComputeSurf::init_normflux();
88
89 int nsurf = surf->nlocal + surf->nghost;
90
91 d_normflux = DAT::t_float_1d("surf:normflux",nsurf);
92 auto h_normflux = Kokkos::create_mirror_view(d_normflux);
93 for (int n=0; n<nsurf; n++)
94 h_normflux(n) = normflux[n];
95 Kokkos::deep_copy(d_normflux,h_normflux);
96
97 // Cannot realloc inside a Kokkos parallel region, so size tally2surf as nsurf
98 memoryKK->grow_kokkos(k_tally2surf,tally2surf,nsurf,"surf:tally2surf");
99 d_tally2surf = k_tally2surf.d_view;
100 d_surf2tally = DAT::t_int_1d("surf:surf2tally",nsurf);
101
102 memoryKK->grow_kokkos(k_array_surf_tally,array_surf_tally,nsurf,ntotal,"surf:array_surf_tally");
103 d_array_surf_tally = k_array_surf_tally.d_view;
104 }
105
106 /* ---------------------------------------------------------------------- */
107
clear()108 void ComputeSurfKokkos::clear()
109 {
110 // reset all set surf2tally values to -1
111 // called by Update at beginning of timesteps surf tallying is done
112
113 combined = 0;
114 Kokkos::deep_copy(d_array_surf_tally,0);
115
116 Kokkos::deep_copy(d_surf2tally,-1);
117 }
118
119 /* ---------------------------------------------------------------------- */
120
pre_surf_tally()121 void ComputeSurfKokkos::pre_surf_tally()
122 {
123 mvv2e = update->mvv2e;
124
125 ParticleKokkos* particle_kk = (ParticleKokkos*) particle;
126 particle_kk->sync(Device,SPECIES_MASK);
127 d_species = particle_kk->k_species.d_view;
128 d_s2g = particle_kk->k_species2group.d_view;
129
130 SurfKokkos* surf_kk = (SurfKokkos*) surf;
131 surf_kk->sync(Device,ALL_MASK);
132 d_lines = surf_kk->k_lines.d_view;
133 d_tris = surf_kk->k_tris.d_view;
134
135 need_dup = sparta->kokkos->need_dup<DeviceType>();
136 if (need_dup)
137 dup_array_surf_tally = Kokkos::Experimental::create_scatter_view<typename Kokkos::Experimental::ScatterSum, typename Kokkos::Experimental::ScatterDuplicated>(d_array_surf_tally);
138 else
139 ndup_array_surf_tally = Kokkos::Experimental::create_scatter_view<typename Kokkos::Experimental::ScatterSum, typename Kokkos::Experimental::ScatterNonDuplicated>(d_array_surf_tally);
140 }
141
142 /* ---------------------------------------------------------------------- */
143
post_surf_tally()144 void ComputeSurfKokkos::post_surf_tally()
145 {
146 if (need_dup) {
147 Kokkos::Experimental::contribute(d_array_surf_tally, dup_array_surf_tally);
148 dup_array_surf_tally = decltype(dup_array_surf_tally)(); // free duplicated memory
149 }
150
151 k_tally2surf.modify_device();
152 k_array_surf_tally.modify_device();
153 }
154
155 /* ----------------------------------------------------------------------
156 return ptr to norm vector used by column N
157 ------------------------------------------------------------------------- */
158
tallyinfo(surfint * & ptr)159 int ComputeSurfKokkos::tallyinfo(surfint *&ptr)
160 {
161 k_tally2surf.sync_host();
162 ptr = tally2surf;
163
164 k_array_surf_tally.sync_host();
165 auto h_surf2tally = Kokkos::create_mirror_view(d_surf2tally);
166 Kokkos::deep_copy(h_surf2tally,d_surf2tally);
167
168 // compress array_surf_tally
169
170 int nsurf = surf->nlocal + surf->nghost;
171 int istart = 0;
172 int iend = nsurf-1;
173
174 while (1) {
175 while (h_surf2tally[istart] != -1 && istart < nsurf-2) istart++;
176 while (h_surf2tally[iend] == -1 && iend > 0) iend--;
177 if (istart >= iend) {
178 ntally = istart;
179 break;
180 }
181 for (int k = 0; k < ntotal; k++) {
182 array_surf_tally[istart][k] = array_surf_tally[iend][k];
183 }
184 h_surf2tally[istart] = h_surf2tally[iend];
185 h_surf2tally[iend] = -1;
186 tally2surf[istart] = tally2surf[iend];
187 }
188
189 return ntally;
190 }
191
192 /* ---------------------------------------------------------------------- */
193
grow_tally()194 void ComputeSurfKokkos::grow_tally()
195 {
196 // Cannot realloc inside a Kokkos parallel region, so size tally2surf the
197 // same as surf2tally
198
199 int nsurf = surf->nlocal + surf->nghost;
200
201 memoryKK->grow_kokkos(k_tally2surf,tally2surf,nsurf,"surf:tally2surf");
202 d_tally2surf = k_tally2surf.d_view;
203
204 memoryKK->grow_kokkos(k_array_surf_tally,array_surf_tally,nsurf,ntotal,"surf:array_surf_tally");
205 d_array_surf_tally = k_array_surf_tally.d_view;
206 }
207
208