1 /* ---------------------------------------------------------------------- 2 SPARTA - Stochastic PArallel Rarefied-gas Time-accurate Analyzer 3 http://sparta.sandia.gov 4 Steve Plimpton, sjplimp@sandia.gov, Michael Gallis, magalli@sandia.gov 5 Sandia National Laboratories 6 7 Copyright (2014) Sandia Corporation. Under the terms of Contract 8 DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains 9 certain rights in this software. This software is distributed under 10 the GNU General Public License. 11 12 See the README file in the top-level SPARTA directory. 13 ------------------------------------------------------------------------- */ 14 15 #ifndef SPARTA_GRID_KOKKOS_H 16 #define SPARTA_GRID_KOKKOS_H 17 18 #include "grid.h" 19 #include "kokkos_type.h" 20 #include <Kokkos_UnorderedMap.hpp> 21 22 namespace SPARTA_NS { 23 24 class GridKokkos : public Grid { 25 public: 26 27 // make into a view 28 //ChildCell *cells; // list of owned and ghost child cells 29 30 // methods 31 32 GridKokkos(class SPARTA *); 33 ~GridKokkos(); 34 void wrap_kokkos(); 35 void wrap_kokkos_graphs(); 36 void sync(ExecutionSpace, unsigned int); 37 void modify(ExecutionSpace, unsigned int); 38 39 // operations with grid cell IDs 40 void update_hash(); 41 42 /* ---------------------------------------------------------------------- 43 compute lo/hi extent of a specific child cell within a parent cell 44 plevel = level of parent 45 plo/phi = parent cell corner points 46 ichild ranges from 1 to Nx*Ny*Nz within parent cell 47 return clo/chi corner points, caller must allocate them 48 ------------------------------------------------------------------------- */ 49 50 KOKKOS_INLINE_FUNCTION id_child_lohi(int plevel,double * plo,double * phi,cellint ichild,double * clo,double * chi)51 void id_child_lohi(int plevel, double *plo, double *phi, 52 cellint ichild, double *clo, double *chi) const 53 { 54 int nx = k_plevels.d_view[plevel].nx; 55 int ny = k_plevels.d_view[plevel].ny; 56 int nz = k_plevels.d_view[plevel].nz; 57 58 ichild--; 59 int ix = ichild % nx; 60 int iy = (ichild/nx) % ny; 61 int iz = ichild / ((cellint) nx*ny); 62 63 clo[0] = plo[0] + ix*(phi[0]-plo[0])/nx; 64 clo[1] = plo[1] + iy*(phi[1]-plo[1])/ny; 65 clo[2] = plo[2] + iz*(phi[2]-plo[2])/nz; 66 67 chi[0] = plo[0] + (ix+1)*(phi[0]-plo[0])/nx; 68 chi[1] = plo[1] + (iy+1)*(phi[1]-plo[1])/ny; 69 chi[2] = plo[2] + (iz+1)*(phi[2]-plo[2])/nz; 70 71 if (ix == nx-1) chi[0] = phi[0]; 72 if (iy == ny-1) chi[1] = phi[1]; 73 if (iz == nz-1) chi[2] = phi[2]; 74 } 75 76 /* ---------------------------------------------------------------------- 77 find child cell within parentID which contains pt X 78 level = level of parent cell 79 oplo/ophi = original parent cell corner pts 80 pt X can be inside or on any boundary of parent cell 81 recurse from parent downward until find a child cell or reach maxlevel 82 if find child cell this proc stores (owned or ghost), return its local index 83 else return -1 for unknown 84 ------------------------------------------------------------------------- */ 85 86 KOKKOS_INLINE_FUNCTION id_find_child(cellint parentID,int plevel,double * oplo,double * ophi,double * x)87 int id_find_child(cellint parentID, int plevel, 88 double *oplo, double *ophi, double *x) const 89 { 90 typedef hash_type::size_type size_type; // uint32_t 91 typedef hash_type::key_type key_type; // cellint 92 typedef hash_type::value_type value_type; // int 93 94 int ix,iy,iz,nx,ny,nz; 95 double plo[3],phi[3],clo[3],chi[3]; 96 cellint childID,ichild; 97 98 cellint id = parentID; 99 int level = plevel; 100 double *lo = oplo; 101 double *hi = ophi; 102 103 while (level < maxlevel) { 104 nx = k_plevels.d_view[level].nx; 105 ny = k_plevels.d_view[level].ny; 106 nz = k_plevels.d_view[level].nz; 107 ix = static_cast<int> ((x[0]-lo[0]) * nx/(hi[0]-lo[0])); 108 iy = static_cast<int> ((x[1]-lo[1]) * ny/(hi[1]-lo[1])); 109 iz = static_cast<int> ((x[2]-lo[2]) * nz/(hi[2]-lo[2])); 110 if (ix == nx) ix--; 111 if (iy == ny) iy--; 112 if (iz == nz) iz--; 113 114 ichild = (cellint) iz*nx*ny + (cellint) iy*nx + ix + 1; 115 childID = (ichild << k_plevels.d_view[level].nbits) | id; 116 117 size_type h_index = hash_kk.find(static_cast<key_type>(childID)); 118 if (hash_kk.valid_at(h_index)) return static_cast<int>(hash_kk.value_at(h_index)); 119 120 id = childID; 121 id_child_lohi(level,lo,hi,ichild,clo,chi); 122 plo[0] = clo[0]; plo[1] = clo[1]; plo[2] = clo[2]; 123 phi[0] = chi[0]; phi[1] = chi[1]; phi[2] = chi[2]; 124 lo = plo; hi = phi; 125 level++; 126 } 127 128 return -1; 129 } 130 131 // extract/return neighbor flag for iface from per-cell nmask 132 // inlined for efficiency 133 134 KOKKOS_INLINE_FUNCTION neigh_decode(int nmask,int iface)135 int neigh_decode(int nmask, int iface) const { 136 return (nmask & neighmask[iface]) >> neighshift[iface]; 137 } 138 139 // overwrite neighbor flag for iface in per-cell nmask 140 // first line zeroes the iface bits via one's complement of mask 141 // inlined for efficiency 142 // return updated nmask 143 144 KOKKOS_INLINE_FUNCTION neigh_encode(int flag,int nmask,int iface)145 int neigh_encode(int flag, int nmask, int iface) const { 146 nmask &= ~neighmask[iface]; 147 nmask |= flag << neighshift[iface]; 148 return nmask; 149 } 150 151 tdual_cell_1d k_cells; 152 tdual_cinfo_1d k_cinfo; 153 tdual_sinfo_1d k_sinfo; 154 tdual_pcell_1d k_pcells; 155 tdual_plevel_1d k_plevels; 156 157 Kokkos::Crs<int, DeviceType, void, int> d_csurfs; 158 Kokkos::Crs<int, DeviceType, void, int> d_csplits; 159 Kokkos::Crs<int, DeviceType, void, int> d_csubs; 160 161 DAT::t_int_1d d_cellcount; 162 DAT::t_int_2d d_plist; 163 164 private: 165 void grow_cells(int, int); 166 void grow_sinfo(int); 167 void grow_pcells(); 168 169 // hash for all cell IDs (owned,ghost,parent). The _d postfix refers to the 170 // fact that this hash lives on "device" 171 172 typedef Kokkos::UnorderedMap<cellint,int> hash_type; 173 hash_type hash_kk; 174 175 }; 176 177 } 178 179 #endif 180 181 /* ERROR/WARNING messages: 182 183 */ 184