1 /* ----------------------------------------------------------------------
2    SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer
3    http://sparta.sandia.gov
4    Steve Plimpton, sjplimp@sandia.gov, Michael Gallis, magalli@sandia.gov
5    Sandia National Laboratories
6 
7    Copyright (2014) Sandia Corporation.  Under the terms of Contract
8    DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
9    certain rights in this software.  This software is distributed under
10    the GNU General Public License.
11 
12    See the README file in the top-level SPARTA directory.
13 ------------------------------------------------------------------------- */
14 
15 #include "ctype.h"
16 #include "surf_kokkos.h"
17 #include "style_surf_collide.h"
18 #include "style_surf_react.h"
19 #include "surf_collide.h"
20 #include "surf_react.h"
21 #include "domain.h"
22 #include "region.h"
23 #include "comm.h"
24 #include "geometry.h"
25 #include "input.h"
26 #include "math_extra.h"
27 #include "math_const.h"
28 #include "memory_kokkos.h"
29 #include "error.h"
30 #include "sparta_masks.h"
31 #include "kokkos.h"
32 
33 using namespace SPARTA_NS;
34 using namespace MathConst;
35 
36 enum{TALLYAUTO,TALLYREDUCE,TALLYLOCAL};         // same as Update
37 enum{REGION_ALL,REGION_ONE,REGION_CENTER};      // same as Grid
38 enum{TYPE,MOLECULE,ID};
39 enum{LT,LE,GT,GE,EQ,NEQ,BETWEEN};
40 
41 #define DELTA 4
42 #define EPSSQ 1.0e-12
43 #define BIG 1.0e20
44 #define MAXGROUP 32
45 
46 /* ---------------------------------------------------------------------- */
47 
SurfKokkos(SPARTA * sparta)48 SurfKokkos::SurfKokkos(SPARTA *sparta) : Surf(sparta)
49 {
50 
51 
52 }
53 
54 /* ---------------------------------------------------------------------- */
55 
~SurfKokkos()56 SurfKokkos::~SurfKokkos()
57 {
58   lines = NULL;
59   tris = NULL;
60 
61   mylines = NULL;
62   mytris = NULL;
63 }
64 
65 /* ---------------------------------------------------------------------- */
66 
wrap_kokkos()67 void SurfKokkos::wrap_kokkos()
68 {
69   if (domain->dimension == 2) {
70     if (lines != NULL && nmax > 0) {
71       if (lines != k_lines.h_view.data()) {
72         memoryKK->wrap_kokkos(k_lines,lines,nmax,"surf:lines");
73         k_lines.modify_host();
74         k_lines.sync_device();
75         memory->sfree(lines);
76         lines = k_lines.h_view.data();
77       }
78     }
79     if (mylines != NULL && nown > 0) {
80       if (mylines != k_mylines.h_view.data()) {
81         memoryKK->wrap_kokkos(k_mylines,mylines,nown,"surf:lines");
82         k_mylines.modify_host();
83         k_mylines.sync_device();
84         memory->sfree(mylines);
85         mylines = k_mylines.h_view.data();
86       }
87     }
88   } else {
89     if (tris != NULL && nmax > 0) {
90       if (tris != k_tris.h_view.data()) {
91         memoryKK->wrap_kokkos(k_tris,tris,nmax,"surf:tris");
92         k_tris.modify_host();
93         k_tris.sync_device();
94         memory->sfree(tris);
95         tris = k_tris.h_view.data();
96       }
97     }
98     if (mytris != NULL && nown > 0) {
99       if (mytris != k_mytris.h_view.data()) {
100         memoryKK->wrap_kokkos(k_mytris,mytris,nown,"surf:mytris");
101         k_mytris.modify_host();
102         k_mytris.sync_device();
103         memory->sfree(mytris);
104         mytris = k_mytris.h_view.data();
105       }
106     }
107   }
108 }
109 
110 /* ----------------------------------------------------------------------
111    grow surface data structures
112 ------------------------------------------------------------------------- */
grow(int old)113 void SurfKokkos::grow(int old)
114 {
115   if (sparta->kokkos->prewrap) {
116     Surf::grow(old);
117   } else {
118     SurfKokkos* surf_kk = (SurfKokkos*) surf;
119 
120     if (domain->dimension == 2) {
121       if (lines == NULL)
122           surf_kk->k_lines = tdual_line_1d("surf:lines",nmax);
123       else {
124         surf_kk->sync(Host,LINE_MASK);
125         surf_kk->modify(Host,LINE_MASK); // force resize on host
126         surf_kk->k_lines.resize(nmax);
127       }
128       lines = surf_kk->k_lines.h_view.data();
129     } else {
130       if (tris == NULL)
131           surf_kk->k_tris = tdual_tri_1d("surf:tris",nmax);
132       else {
133         surf_kk->sync(Host,TRI_MASK);
134         surf_kk->modify(Host,TRI_MASK); // force resize on host
135         surf_kk->k_tris.resize(nmax);
136       }
137       tris = surf_kk->k_tris.h_view.data();
138     }
139   }
140 }
141 
142 
143 /* ----------------------------------------------------------------------
144    grow surface data structures
145 ------------------------------------------------------------------------- */
grow_own(int old)146 void SurfKokkos::grow_own(int old)
147 {
148   if (sparta->kokkos->prewrap) {
149     Surf::grow_own(old);
150   } else {
151     SurfKokkos* surf_kk = (SurfKokkos*) surf;
152 
153     if (domain->dimension == 2) {
154       if (mylines == NULL)
155           surf_kk->k_mylines = tdual_line_1d("surf:mylines",nown);
156       else {
157         surf_kk->sync(Host,LINE_MASK);
158         surf_kk->modify(Host,LINE_MASK); // force resize on host
159         surf_kk->k_mylines.resize(nown);
160       }
161       mylines = surf_kk->k_mylines.h_view.data();
162     } else {
163       if (mytris == NULL)
164           surf_kk->k_mytris = tdual_tri_1d("surf:mytris",nown);
165       else {
166         surf_kk->sync(Host,TRI_MASK);
167         surf_kk->modify(Host,TRI_MASK); // force resize on host
168         surf_kk->k_mytris.resize(nown);
169       }
170       mytris = surf_kk->k_mytris.h_view.data();
171     }
172   }
173 }
174 
175 /* ---------------------------------------------------------------------- */
176 
sync(ExecutionSpace space,unsigned int mask)177 void SurfKokkos::sync(ExecutionSpace space, unsigned int mask)
178 {
179   if (sparta->kokkos->prewrap) {
180     if (space == Device)
181       error->one(FLERR,"Sync Device before wrap");
182     else
183       return;
184   }
185 
186   if (space == Device) {
187     if (sparta->kokkos->auto_sync)
188       modify(Host,mask);
189     if (mask & LINE_MASK) k_lines.sync_device();
190     if (mask & TRI_MASK) k_tris.sync_device();
191   } else {
192     if (mask & LINE_MASK) k_lines.sync_host();
193     if (mask & TRI_MASK) k_tris.sync_host();
194   }
195 }
196 
197 /* ---------------------------------------------------------------------- */
198 
modify(ExecutionSpace space,unsigned int mask)199 void SurfKokkos::modify(ExecutionSpace space, unsigned int mask)
200 {
201   if (sparta->kokkos->prewrap) {
202     if (space == Device)
203       error->one(FLERR,"Modify Device before wrap");
204     else
205       return;
206   }
207 
208   if (space == Device) {
209     if (mask & LINE_MASK) k_lines.modify_device();
210     if (mask & TRI_MASK) k_tris.modify_device();
211     if (sparta->kokkos->auto_sync)
212       sync(Host,mask);
213   } else {
214     if (mask & LINE_MASK) k_lines.modify_host();
215     if (mask & TRI_MASK) k_tris.modify_host();
216   }
217 }
218