1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 3.0
6 // Copyright (2020) National Technology & Engineering
7 // Solutions of Sandia, LLC (NTESS).
8 //
9 // Under the terms of Contract DE-NA0003525 with NTESS,
10 // the U.S. Government retains certain rights in this software.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions are
14 // met:
15 //
16 // 1. Redistributions of source code must retain the above copyright
17 // notice, this list of conditions and the following disclaimer.
18 //
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 //
23 // 3. Neither the name of the Corporation nor the names of the
24 // contributors may be used to endorse or promote products derived from
25 // this software without specific prior written permission.
26 //
27 // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 //
39 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40 //
41 // ************************************************************************
42 //@HEADER
43 */
44
45 #ifndef KOKKOS_EXPERIMENTAL_VIEW_MAPPING_HPP
46 #define KOKKOS_EXPERIMENTAL_VIEW_MAPPING_HPP
47
48 #include <type_traits>
49 #include <initializer_list>
50
51 #include <Kokkos_Core_fwd.hpp>
52 #include <Kokkos_Pair.hpp>
53 #include <Kokkos_Layout.hpp>
54 #include <Kokkos_Extents.hpp>
55 #include <impl/Kokkos_Error.hpp>
56 #include <impl/Kokkos_Traits.hpp>
57 #include <impl/Kokkos_ViewTracker.hpp>
58 #include <impl/Kokkos_ViewCtor.hpp>
59 #include <impl/Kokkos_Atomic_View.hpp>
60 #include <impl/Kokkos_Tools.hpp>
61
62 //----------------------------------------------------------------------------
63 //----------------------------------------------------------------------------
64
65 namespace Kokkos {
66 namespace Impl {
67
68 template <unsigned I, size_t... Args>
69 struct variadic_size_t {
70 enum : size_t { value = KOKKOS_INVALID_INDEX };
71 };
72
73 template <size_t Val, size_t... Args>
74 struct variadic_size_t<0, Val, Args...> {
75 enum : size_t { value = Val };
76 };
77
78 template <unsigned I, size_t Val, size_t... Args>
79 struct variadic_size_t<I, Val, Args...> {
80 enum : size_t { value = variadic_size_t<I - 1, Args...>::value };
81 };
82
83 template <size_t... Args>
84 struct rank_dynamic;
85
86 template <>
87 struct rank_dynamic<> {
88 enum : unsigned { value = 0 };
89 };
90
91 template <size_t Val, size_t... Args>
92 struct rank_dynamic<Val, Args...> {
93 enum : unsigned { value = (Val == 0 ? 1 : 0) + rank_dynamic<Args...>::value };
94 };
95
96 #define KOKKOS_IMPL_VIEW_DIMENSION(R) \
97 template <size_t V, unsigned> \
98 struct ViewDimension##R { \
99 static constexpr size_t ArgN##R = (V != KOKKOS_INVALID_INDEX ? V : 1); \
100 static constexpr size_t N##R = (V != KOKKOS_INVALID_INDEX ? V : 1); \
101 KOKKOS_INLINE_FUNCTION explicit ViewDimension##R(size_t) {} \
102 ViewDimension##R() = default; \
103 ViewDimension##R(const ViewDimension##R&) = default; \
104 ViewDimension##R& operator=(const ViewDimension##R&) = default; \
105 }; \
106 template <size_t V, unsigned RD> \
107 constexpr size_t ViewDimension##R<V, RD>::ArgN##R; \
108 template <size_t V, unsigned RD> \
109 constexpr size_t ViewDimension##R<V, RD>::N##R; \
110 template <unsigned RD> \
111 struct ViewDimension##R<0u, RD> { \
112 static constexpr size_t ArgN##R = 0; \
113 typename std::conditional<(RD < 3), size_t, unsigned>::type N##R; \
114 ViewDimension##R() = default; \
115 ViewDimension##R(const ViewDimension##R&) = default; \
116 ViewDimension##R& operator=(const ViewDimension##R&) = default; \
117 KOKKOS_INLINE_FUNCTION explicit ViewDimension##R(size_t V) : N##R(V) {} \
118 }; \
119 template <unsigned RD> \
120 constexpr size_t ViewDimension##R<0u, RD>::ArgN##R;
121
122 KOKKOS_IMPL_VIEW_DIMENSION(0)
123 KOKKOS_IMPL_VIEW_DIMENSION(1)
124 KOKKOS_IMPL_VIEW_DIMENSION(2)
125 KOKKOS_IMPL_VIEW_DIMENSION(3)
126 KOKKOS_IMPL_VIEW_DIMENSION(4)
127 KOKKOS_IMPL_VIEW_DIMENSION(5)
128 KOKKOS_IMPL_VIEW_DIMENSION(6)
129 KOKKOS_IMPL_VIEW_DIMENSION(7)
130
131 #undef KOKKOS_IMPL_VIEW_DIMENSION
132
133 // MSVC does not do empty base class optimization by default.
134 // Per standard it is required for standard layout types
135 template <size_t... Vals>
136 struct KOKKOS_IMPL_ENFORCE_EMPTY_BASE_OPTIMIZATION ViewDimension
137 : public ViewDimension0<variadic_size_t<0u, Vals...>::value,
138 rank_dynamic<Vals...>::value>,
139 public ViewDimension1<variadic_size_t<1u, Vals...>::value,
140 rank_dynamic<Vals...>::value>,
141 public ViewDimension2<variadic_size_t<2u, Vals...>::value,
142 rank_dynamic<Vals...>::value>,
143 public ViewDimension3<variadic_size_t<3u, Vals...>::value,
144 rank_dynamic<Vals...>::value>,
145 public ViewDimension4<variadic_size_t<4u, Vals...>::value,
146 rank_dynamic<Vals...>::value>,
147 public ViewDimension5<variadic_size_t<5u, Vals...>::value,
148 rank_dynamic<Vals...>::value>,
149 public ViewDimension6<variadic_size_t<6u, Vals...>::value,
150 rank_dynamic<Vals...>::value>,
151 public ViewDimension7<variadic_size_t<7u, Vals...>::value,
152 rank_dynamic<Vals...>::value> {
153 using D0 = ViewDimension0<variadic_size_t<0U, Vals...>::value,
154 rank_dynamic<Vals...>::value>;
155 using D1 = ViewDimension1<variadic_size_t<1U, Vals...>::value,
156 rank_dynamic<Vals...>::value>;
157 using D2 = ViewDimension2<variadic_size_t<2U, Vals...>::value,
158 rank_dynamic<Vals...>::value>;
159 using D3 = ViewDimension3<variadic_size_t<3U, Vals...>::value,
160 rank_dynamic<Vals...>::value>;
161 using D4 = ViewDimension4<variadic_size_t<4U, Vals...>::value,
162 rank_dynamic<Vals...>::value>;
163 using D5 = ViewDimension5<variadic_size_t<5U, Vals...>::value,
164 rank_dynamic<Vals...>::value>;
165 using D6 = ViewDimension6<variadic_size_t<6U, Vals...>::value,
166 rank_dynamic<Vals...>::value>;
167 using D7 = ViewDimension7<variadic_size_t<7U, Vals...>::value,
168 rank_dynamic<Vals...>::value>;
169
170 using D0::ArgN0;
171 using D1::ArgN1;
172 using D2::ArgN2;
173 using D3::ArgN3;
174 using D4::ArgN4;
175 using D5::ArgN5;
176 using D6::ArgN6;
177 using D7::ArgN7;
178
179 using D0::N0;
180 using D1::N1;
181 using D2::N2;
182 using D3::N3;
183 using D4::N4;
184 using D5::N5;
185 using D6::N6;
186 using D7::N7;
187
188 enum : unsigned { rank = sizeof...(Vals) };
189 enum : unsigned { rank_dynamic = Impl::rank_dynamic<Vals...>::value };
190
191 ViewDimension() = default;
192 ViewDimension(const ViewDimension&) = default;
193 ViewDimension& operator=(const ViewDimension&) = default;
194
195 KOKKOS_INLINE_FUNCTION
ViewDimensionKokkos::Impl::ViewDimension196 constexpr ViewDimension(size_t n0, size_t n1, size_t n2, size_t n3, size_t n4,
197 size_t n5, size_t n6, size_t n7)
198 : D0(n0), D1(n1), D2(n2), D3(n3), D4(n4), D5(n5), D6(n6), D7(n7) {}
199
200 KOKKOS_INLINE_FUNCTION
extentKokkos::Impl::ViewDimension201 constexpr size_t extent(const unsigned r) const noexcept {
202 return r == 0
203 ? N0
204 : (r == 1
205 ? N1
206 : (r == 2
207 ? N2
208 : (r == 3
209 ? N3
210 : (r == 4
211 ? N4
212 : (r == 5
213 ? N5
214 : (r == 6
215 ? N6
216 : (r == 7 ? N7
217 : 0)))))));
218 }
219
static_extentKokkos::Impl::ViewDimension220 static KOKKOS_INLINE_FUNCTION constexpr size_t static_extent(
221 const unsigned r) noexcept {
222 return r == 0
223 ? ArgN0
224 : (r == 1
225 ? ArgN1
226 : (r == 2
227 ? ArgN2
228 : (r == 3
229 ? ArgN3
230 : (r == 4
231 ? ArgN4
232 : (r == 5
233 ? ArgN5
234 : (r == 6
235 ? ArgN6
236 : (r == 7 ? ArgN7
237 : 0)))))));
238 }
239
240 template <size_t N>
241 struct prepend {
242 using type = ViewDimension<N, Vals...>;
243 };
244
245 template <size_t N>
246 struct append {
247 using type = ViewDimension<Vals..., N>;
248 };
249 };
250
251 template <class A, class B>
252 struct ViewDimensionJoin;
253
254 template <size_t... A, size_t... B>
255 struct ViewDimensionJoin<ViewDimension<A...>, ViewDimension<B...>> {
256 using type = ViewDimension<A..., B...>;
257 };
258
259 //----------------------------------------------------------------------------
260
261 template <class DstDim, class SrcDim>
262 struct ViewDimensionAssignable;
263
264 template <size_t... DstArgs, size_t... SrcArgs>
265 struct ViewDimensionAssignable<ViewDimension<DstArgs...>,
266 ViewDimension<SrcArgs...>> {
267 using dst = ViewDimension<DstArgs...>;
268 using src = ViewDimension<SrcArgs...>;
269
270 enum {
271 value = unsigned(dst::rank) == unsigned(src::rank) &&
272 (
273 // Compile time check that potential static dimensions match
274 ((1 > dst::rank_dynamic && 1 > src::rank_dynamic)
275 ? (size_t(dst::ArgN0) == size_t(src::ArgN0))
276 : true) &&
277 ((2 > dst::rank_dynamic && 2 > src::rank_dynamic)
278 ? (size_t(dst::ArgN1) == size_t(src::ArgN1))
279 : true) &&
280 ((3 > dst::rank_dynamic && 3 > src::rank_dynamic)
281 ? (size_t(dst::ArgN2) == size_t(src::ArgN2))
282 : true) &&
283 ((4 > dst::rank_dynamic && 4 > src::rank_dynamic)
284 ? (size_t(dst::ArgN3) == size_t(src::ArgN3))
285 : true) &&
286 ((5 > dst::rank_dynamic && 5 > src::rank_dynamic)
287 ? (size_t(dst::ArgN4) == size_t(src::ArgN4))
288 : true) &&
289 ((6 > dst::rank_dynamic && 6 > src::rank_dynamic)
290 ? (size_t(dst::ArgN5) == size_t(src::ArgN5))
291 : true) &&
292 ((7 > dst::rank_dynamic && 7 > src::rank_dynamic)
293 ? (size_t(dst::ArgN6) == size_t(src::ArgN6))
294 : true) &&
295 ((8 > dst::rank_dynamic && 8 > src::rank_dynamic)
296 ? (size_t(dst::ArgN7) == size_t(src::ArgN7))
297 : true))
298 };
299 };
300
301 } // namespace Impl
302 } // namespace Kokkos
303
304 //----------------------------------------------------------------------------
305 //----------------------------------------------------------------------------
306
307 namespace Kokkos {
308 namespace Impl {
309
310 struct ALL_t {
311 KOKKOS_INLINE_FUNCTION
operator ()Kokkos::Impl::ALL_t312 constexpr const ALL_t& operator()() const { return *this; }
313
314 KOKKOS_INLINE_FUNCTION
operator ==Kokkos::Impl::ALL_t315 constexpr bool operator==(const ALL_t&) const { return true; }
316 };
317
318 } // namespace Impl
319 } // namespace Kokkos
320
321 namespace Kokkos {
322 namespace Impl {
323
324 template <class T>
325 struct is_integral_extent_type {
326 enum : bool { value = std::is_same<T, Kokkos::Impl::ALL_t>::value ? 1 : 0 };
327 };
328
329 template <class iType>
330 struct is_integral_extent_type<std::pair<iType, iType>> {
331 enum : bool { value = std::is_integral<iType>::value ? 1 : 0 };
332 };
333
334 template <class iType>
335 struct is_integral_extent_type<Kokkos::pair<iType, iType>> {
336 enum : bool { value = std::is_integral<iType>::value ? 1 : 0 };
337 };
338
339 // Assuming '2 == initializer_list<iType>::size()'
340 template <class iType>
341 struct is_integral_extent_type<std::initializer_list<iType>> {
342 enum : bool { value = std::is_integral<iType>::value ? 1 : 0 };
343 };
344
345 template <unsigned I, class... Args>
346 struct is_integral_extent {
347 // get_type is void when sizeof...(Args) <= I
348 using type = typename std::remove_cv<typename std::remove_reference<
349 typename Kokkos::Impl::get_type<I, Args...>::type>::type>::type;
350
351 enum : bool { value = is_integral_extent_type<type>::value };
352
353 static_assert(value || std::is_integral<type>::value ||
354 std::is_same<type, void>::value,
355 "subview argument must be either integral or integral extent");
356 };
357
358 // Rules for subview arguments and layouts matching
359
360 template <class LayoutDest, class LayoutSrc, int RankDest, int RankSrc,
361 int CurrentArg, class... SubViewArgs>
362 struct SubviewLegalArgsCompileTime;
363
364 // Rules which allow LayoutLeft to LayoutLeft assignment
365
366 template <int RankDest, int RankSrc, int CurrentArg, class Arg,
367 class... SubViewArgs>
368 struct SubviewLegalArgsCompileTime<Kokkos::LayoutLeft, Kokkos::LayoutLeft,
369 RankDest, RankSrc, CurrentArg, Arg,
370 SubViewArgs...> {
371 enum {
372 value = (((CurrentArg == RankDest - 1) &&
373 (Kokkos::Impl::is_integral_extent_type<Arg>::value)) ||
374 ((CurrentArg >= RankDest) && (std::is_integral<Arg>::value)) ||
375 ((CurrentArg < RankDest) &&
376 (std::is_same<Arg, Kokkos::Impl::ALL_t>::value)) ||
377 ((CurrentArg == 0) &&
378 (Kokkos::Impl::is_integral_extent_type<Arg>::value))) &&
379 (SubviewLegalArgsCompileTime<Kokkos::LayoutLeft, Kokkos::LayoutLeft,
380 RankDest, RankSrc, CurrentArg + 1,
381 SubViewArgs...>::value)
382 };
383 };
384
385 template <int RankDest, int RankSrc, int CurrentArg, class Arg>
386 struct SubviewLegalArgsCompileTime<Kokkos::LayoutLeft, Kokkos::LayoutLeft,
387 RankDest, RankSrc, CurrentArg, Arg> {
388 enum {
389 value = ((CurrentArg == RankDest - 1) || (std::is_integral<Arg>::value)) &&
390 (CurrentArg == RankSrc - 1)
391 };
392 };
393
394 // Rules which allow LayoutRight to LayoutRight assignment
395
396 template <int RankDest, int RankSrc, int CurrentArg, class Arg,
397 class... SubViewArgs>
398 struct SubviewLegalArgsCompileTime<Kokkos::LayoutRight, Kokkos::LayoutRight,
399 RankDest, RankSrc, CurrentArg, Arg,
400 SubViewArgs...> {
401 enum {
402 value = (((CurrentArg == RankSrc - RankDest) &&
403 (Kokkos::Impl::is_integral_extent_type<Arg>::value)) ||
404 ((CurrentArg < RankSrc - RankDest) &&
405 (std::is_integral<Arg>::value)) ||
406 ((CurrentArg >= RankSrc - RankDest) &&
407 (std::is_same<Arg, Kokkos::Impl::ALL_t>::value))) &&
408 (SubviewLegalArgsCompileTime<Kokkos::LayoutRight,
409 Kokkos::LayoutRight, RankDest, RankSrc,
410 CurrentArg + 1, SubViewArgs...>::value)
411 };
412 };
413
414 template <int RankDest, int RankSrc, int CurrentArg, class Arg>
415 struct SubviewLegalArgsCompileTime<Kokkos::LayoutRight, Kokkos::LayoutRight,
416 RankDest, RankSrc, CurrentArg, Arg> {
417 enum {
418 value = ((CurrentArg == RankSrc - 1) &&
419 (std::is_same<Arg, Kokkos::Impl::ALL_t>::value))
420 };
421 };
422
423 // Rules which allow assignment to LayoutStride
424
425 template <int RankDest, int RankSrc, int CurrentArg, class... SubViewArgs>
426 struct SubviewLegalArgsCompileTime<Kokkos::LayoutStride, Kokkos::LayoutLeft,
427 RankDest, RankSrc, CurrentArg,
428 SubViewArgs...> {
429 enum : bool { value = true };
430 };
431
432 template <int RankDest, int RankSrc, int CurrentArg, class... SubViewArgs>
433 struct SubviewLegalArgsCompileTime<Kokkos::LayoutStride, Kokkos::LayoutRight,
434 RankDest, RankSrc, CurrentArg,
435 SubViewArgs...> {
436 enum : bool { value = true };
437 };
438
439 template <int RankDest, int RankSrc, int CurrentArg, class... SubViewArgs>
440 struct SubviewLegalArgsCompileTime<Kokkos::LayoutStride, Kokkos::LayoutStride,
441 RankDest, RankSrc, CurrentArg,
442 SubViewArgs...> {
443 enum : bool { value = true };
444 };
445
446 template <unsigned DomainRank, unsigned RangeRank>
447 struct SubviewExtents {
448 private:
449 // Cannot declare zero-length arrays
450 // '+' is used to silence GCC 7.2.0 -Wduplicated-branches warning when
451 // RangeRank=1
452 enum { InternalRangeRank = RangeRank ? RangeRank : +1u };
453
454 size_t m_begin[DomainRank];
455 size_t m_length[InternalRangeRank];
456 unsigned m_index[InternalRangeRank];
457
458 template <size_t... DimArgs>
setKokkos::Impl::SubviewExtents459 KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned, unsigned,
460 const ViewDimension<DimArgs...>&) {
461 return true;
462 }
463
464 template <class T, size_t... DimArgs, class... Args>
setKokkos::Impl::SubviewExtents465 KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
466 unsigned range_rank,
467 const ViewDimension<DimArgs...>& dim,
468 const T& val, Args... args) {
469 const size_t v = static_cast<size_t>(val);
470
471 m_begin[domain_rank] = v;
472
473 return set(domain_rank + 1, range_rank, dim, args...)
474 #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
475 && (v < dim.extent(domain_rank))
476 #endif
477 ;
478 }
479
480 // ALL_t
481 template <size_t... DimArgs, class... Args>
setKokkos::Impl::SubviewExtents482 KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
483 unsigned range_rank,
484 const ViewDimension<DimArgs...>& dim,
485 const Kokkos::Impl::ALL_t,
486 Args... args) {
487 m_begin[domain_rank] = 0;
488 m_length[range_rank] = dim.extent(domain_rank);
489 m_index[range_rank] = domain_rank;
490
491 return set(domain_rank + 1, range_rank + 1, dim, args...);
492 }
493
494 // std::pair range
495 template <class T, size_t... DimArgs, class... Args>
setKokkos::Impl::SubviewExtents496 KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
497 unsigned range_rank,
498 const ViewDimension<DimArgs...>& dim,
499 const std::pair<T, T>& val,
500 Args... args) {
501 const size_t b = static_cast<size_t>(val.first);
502 const size_t e = static_cast<size_t>(val.second);
503
504 m_begin[domain_rank] = b;
505 m_length[range_rank] = e - b;
506 m_index[range_rank] = domain_rank;
507
508 return set(domain_rank + 1, range_rank + 1, dim, args...)
509 #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
510 && (e <= b + dim.extent(domain_rank))
511 #endif
512 ;
513 }
514
515 // Kokkos::pair range
516 template <class T, size_t... DimArgs, class... Args>
setKokkos::Impl::SubviewExtents517 KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
518 unsigned range_rank,
519 const ViewDimension<DimArgs...>& dim,
520 const Kokkos::pair<T, T>& val,
521 Args... args) {
522 const size_t b = static_cast<size_t>(val.first);
523 const size_t e = static_cast<size_t>(val.second);
524
525 m_begin[domain_rank] = b;
526 m_length[range_rank] = e - b;
527 m_index[range_rank] = domain_rank;
528
529 return set(domain_rank + 1, range_rank + 1, dim, args...)
530 #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
531 && (e <= b + dim.extent(domain_rank))
532 #endif
533 ;
534 }
535
536 // { begin , end } range
537 template <class T, size_t... DimArgs, class... Args>
setKokkos::Impl::SubviewExtents538 KOKKOS_FORCEINLINE_FUNCTION bool set(unsigned domain_rank,
539 unsigned range_rank,
540 const ViewDimension<DimArgs...>& dim,
541 const std::initializer_list<T>& val,
542 Args... args) {
543 const size_t b = static_cast<size_t>(val.begin()[0]);
544 const size_t e = static_cast<size_t>(val.begin()[1]);
545
546 m_begin[domain_rank] = b;
547 m_length[range_rank] = e - b;
548 m_index[range_rank] = domain_rank;
549
550 return set(domain_rank + 1, range_rank + 1, dim, args...)
551 #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
552 && (val.size() == 2) && (e <= b + dim.extent(domain_rank))
553 #endif
554 ;
555 }
556
557 //------------------------------
558
559 #if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK)
560
561 template <size_t... DimArgs>
errorKokkos::Impl::SubviewExtents562 void error(char*, int, unsigned, unsigned,
563 const ViewDimension<DimArgs...>&) const {}
564
565 template <class T, size_t... DimArgs, class... Args>
errorKokkos::Impl::SubviewExtents566 void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
567 const ViewDimension<DimArgs...>& dim, const T& val,
568 Args... args) const {
569 const int n = std::min(
570 buf_len,
571 snprintf(buf, buf_len, " %lu < %lu %c", static_cast<unsigned long>(val),
572 static_cast<unsigned long>(dim.extent(domain_rank)),
573 int(sizeof...(Args) ? ',' : ')')));
574
575 error(buf + n, buf_len - n, domain_rank + 1, range_rank, dim, args...);
576 }
577
578 // std::pair range
579 template <size_t... DimArgs, class... Args>
errorKokkos::Impl::SubviewExtents580 void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
581 const ViewDimension<DimArgs...>& dim, const Kokkos::Impl::ALL_t,
582 Args... args) const {
583 const int n = std::min(buf_len, snprintf(buf, buf_len, " Kokkos::ALL %c",
584 int(sizeof...(Args) ? ',' : ')')));
585
586 error(buf + n, buf_len - n, domain_rank + 1, range_rank + 1, dim, args...);
587 }
588
589 // std::pair range
590 template <class T, size_t... DimArgs, class... Args>
errorKokkos::Impl::SubviewExtents591 void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
592 const ViewDimension<DimArgs...>& dim, const std::pair<T, T>& val,
593 Args... args) const {
594 // d <= e - b
595 const int n = std::min(
596 buf_len, snprintf(buf, buf_len, " %lu <= %lu - %lu %c",
597 static_cast<unsigned long>(dim.extent(domain_rank)),
598 static_cast<unsigned long>(val.second),
599 static_cast<unsigned long>(val.first),
600 int(sizeof...(Args) ? ',' : ')')));
601
602 error(buf + n, buf_len - n, domain_rank + 1, range_rank + 1, dim, args...);
603 }
604
605 // Kokkos::pair range
606 template <class T, size_t... DimArgs, class... Args>
errorKokkos::Impl::SubviewExtents607 void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
608 const ViewDimension<DimArgs...>& dim,
609 const Kokkos::pair<T, T>& val, Args... args) const {
610 // d <= e - b
611 const int n = std::min(
612 buf_len, snprintf(buf, buf_len, " %lu <= %lu - %lu %c",
613 static_cast<unsigned long>(dim.extent(domain_rank)),
614 static_cast<unsigned long>(val.second),
615 static_cast<unsigned long>(val.first),
616 int(sizeof...(Args) ? ',' : ')')));
617
618 error(buf + n, buf_len - n, domain_rank + 1, range_rank + 1, dim, args...);
619 }
620
621 // { begin , end } range
622 template <class T, size_t... DimArgs, class... Args>
errorKokkos::Impl::SubviewExtents623 void error(char* buf, int buf_len, unsigned domain_rank, unsigned range_rank,
624 const ViewDimension<DimArgs...>& dim,
625 const std::initializer_list<T>& val, Args... args) const {
626 // d <= e - b
627 int n = 0;
628 if (val.size() == 2) {
629 n = std::min(buf_len,
630 snprintf(buf, buf_len, " %lu <= %lu - %lu %c",
631 static_cast<unsigned long>(dim.extent(domain_rank)),
632 static_cast<unsigned long>(val.begin()[0]),
633 static_cast<unsigned long>(val.begin()[1]),
634 int(sizeof...(Args) ? ',' : ')')));
635 } else {
636 n = std::min(buf_len, snprintf(buf, buf_len, " { ... }.size() == %u %c",
637 unsigned(val.size()),
638 int(sizeof...(Args) ? ',' : ')')));
639 }
640
641 error(buf + n, buf_len - n, domain_rank + 1, range_rank + 1, dim, args...);
642 }
643
644 #if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
645 template <size_t... DimArgs, class... Args>
errorKokkos::Impl::SubviewExtents646 KOKKOS_FORCEINLINE_FUNCTION void error(const ViewDimension<DimArgs...>& dim,
647 Args... args) const {
648 enum { LEN = 1024 };
649 char buffer[LEN];
650
651 const int n = snprintf(buffer, LEN, "Kokkos::subview bounds error (");
652 error(buffer + n, LEN - n, 0, 0, dim, args...);
653
654 Kokkos::Impl::throw_runtime_exception(std::string(buffer));
655 }
656 #else
657 template <size_t... DimArgs, class... Args>
errorKokkos::Impl::SubviewExtents658 KOKKOS_FORCEINLINE_FUNCTION void error(const ViewDimension<DimArgs...>&,
659 Args...) const {
660 Kokkos::abort("Kokkos::subview bounds error");
661 }
662 #endif
663
664 #else
665
666 template <size_t... DimArgs, class... Args>
errorKokkos::Impl::SubviewExtents667 KOKKOS_FORCEINLINE_FUNCTION void error(const ViewDimension<DimArgs...>&,
668 Args...) const {}
669
670 #endif
671
672 public:
673 template <size_t... DimArgs, class... Args>
SubviewExtentsKokkos::Impl::SubviewExtents674 KOKKOS_INLINE_FUNCTION SubviewExtents(const ViewDimension<DimArgs...>& dim,
675 Args... args) {
676 static_assert(DomainRank == sizeof...(DimArgs), "");
677 static_assert(DomainRank == sizeof...(Args), "");
678
679 // Verifies that all arguments, up to 8, are integral types,
680 // integral extents, or don't exist.
681 static_assert(
682 RangeRank == unsigned(is_integral_extent<0, Args...>::value) +
683 unsigned(is_integral_extent<1, Args...>::value) +
684 unsigned(is_integral_extent<2, Args...>::value) +
685 unsigned(is_integral_extent<3, Args...>::value) +
686 unsigned(is_integral_extent<4, Args...>::value) +
687 unsigned(is_integral_extent<5, Args...>::value) +
688 unsigned(is_integral_extent<6, Args...>::value) +
689 unsigned(is_integral_extent<7, Args...>::value),
690 "");
691
692 if (RangeRank == 0) {
693 m_length[0] = 0;
694 m_index[0] = ~0u;
695 }
696
697 if (!set(0, 0, dim, args...)) error(dim, args...);
698 }
699
700 template <typename iType>
domain_offsetKokkos::Impl::SubviewExtents701 KOKKOS_FORCEINLINE_FUNCTION constexpr size_t domain_offset(
702 const iType i) const {
703 return unsigned(i) < DomainRank ? m_begin[i] : 0;
704 }
705
706 template <typename iType>
range_extentKokkos::Impl::SubviewExtents707 KOKKOS_FORCEINLINE_FUNCTION constexpr size_t range_extent(
708 const iType i) const {
709 return unsigned(i) < InternalRangeRank ? m_length[i] : 0;
710 }
711
712 template <typename iType>
range_indexKokkos::Impl::SubviewExtents713 KOKKOS_FORCEINLINE_FUNCTION constexpr unsigned range_index(
714 const iType i) const {
715 return unsigned(i) < InternalRangeRank ? m_index[i] : ~0u;
716 }
717 };
718
719 } // namespace Impl
720 } // namespace Kokkos
721
722 //----------------------------------------------------------------------------
723 //----------------------------------------------------------------------------
724
725 namespace Kokkos {
726 namespace Impl {
727
728 /** \brief Given a value type and dimension generate the View data type */
729 template <class T, class Dim>
730 struct ViewDataType;
731
732 template <class T>
733 struct ViewDataType<T, ViewDimension<>> {
734 using type = T;
735 };
736
737 template <class T, size_t... Args>
738 struct ViewDataType<T, ViewDimension<0, Args...>> {
739 using type = typename ViewDataType<T*, ViewDimension<Args...>>::type;
740 };
741
742 template <class T, size_t N, size_t... Args>
743 struct ViewDataType<T, ViewDimension<N, Args...>> {
744 using type = typename ViewDataType<T, ViewDimension<Args...>>::type[N];
745 };
746
747 /**\brief Analysis of View data type.
748 *
749 * Data type conforms to one of the following patterns :
750 * {const} value_type [][#][#][#]
751 * {const} value_type ***[#][#][#]
752 * Where the sum of counts of '*' and '[#]' is at most ten.
753 *
754 * Provide alias for ViewDimension<...> and value_type.
755 */
756 template <class T>
757 struct ViewArrayAnalysis {
758 using value_type = T;
759 using const_value_type = typename std::add_const<T>::type;
760 using non_const_value_type = typename std::remove_const<T>::type;
761 using static_dimension = ViewDimension<>;
762 using dynamic_dimension = ViewDimension<>;
763 using dimension = ViewDimension<>;
764 };
765
766 template <class T, size_t N>
767 struct ViewArrayAnalysis<T[N]> {
768 private:
769 using nested = ViewArrayAnalysis<T>;
770
771 public:
772 using value_type = typename nested::value_type;
773 using const_value_type = typename nested::const_value_type;
774 using non_const_value_type = typename nested::non_const_value_type;
775
776 using static_dimension =
777 typename nested::static_dimension::template prepend<N>::type;
778
779 using dynamic_dimension = typename nested::dynamic_dimension;
780
781 using dimension =
782 typename ViewDimensionJoin<dynamic_dimension, static_dimension>::type;
783 };
784
785 template <class T>
786 struct ViewArrayAnalysis<T[]> {
787 private:
788 using nested = ViewArrayAnalysis<T>;
789 using nested_dimension = typename nested::dimension;
790
791 public:
792 using value_type = typename nested::value_type;
793 using const_value_type = typename nested::const_value_type;
794 using non_const_value_type = typename nested::non_const_value_type;
795
796 using dynamic_dimension =
797 typename nested::dynamic_dimension::template prepend<0>::type;
798
799 using static_dimension = typename nested::static_dimension;
800
801 using dimension =
802 typename ViewDimensionJoin<dynamic_dimension, static_dimension>::type;
803 };
804
805 template <class T>
806 struct ViewArrayAnalysis<T*> {
807 private:
808 using nested = ViewArrayAnalysis<T>;
809
810 public:
811 using value_type = typename nested::value_type;
812 using const_value_type = typename nested::const_value_type;
813 using non_const_value_type = typename nested::non_const_value_type;
814
815 using dynamic_dimension =
816 typename nested::dynamic_dimension::template prepend<0>::type;
817
818 using static_dimension = typename nested::static_dimension;
819
820 using dimension =
821 typename ViewDimensionJoin<dynamic_dimension, static_dimension>::type;
822 };
823
824 template <class DataType, class ArrayLayout, class ValueType>
825 struct ViewDataAnalysis {
826 private:
827 using array_analysis = ViewArrayAnalysis<DataType>;
828
829 // ValueType is opportunity for partial specialization.
830 // Must match array analysis when this default template is used.
831 static_assert(
832 std::is_same<ValueType,
833 typename array_analysis::non_const_value_type>::value,
834 "");
835
836 public:
837 using specialize = void; // No specialization
838
839 using dimension = typename array_analysis::dimension;
840 using value_type = typename array_analysis::value_type;
841 using const_value_type = typename array_analysis::const_value_type;
842 using non_const_value_type = typename array_analysis::non_const_value_type;
843
844 // Generate analogous multidimensional array specification type.
845 using type = typename ViewDataType<value_type, dimension>::type;
846 using const_type = typename ViewDataType<const_value_type, dimension>::type;
847 using non_const_type =
848 typename ViewDataType<non_const_value_type, dimension>::type;
849
850 // Generate "flattened" multidimensional array specification type.
851 using scalar_array_type = type;
852 using const_scalar_array_type = const_type;
853 using non_const_scalar_array_type = non_const_type;
854 };
855
856 } // namespace Impl
857 } // namespace Kokkos
858
859 //----------------------------------------------------------------------------
860 //----------------------------------------------------------------------------
861
862 namespace Kokkos {
863 namespace Impl {
864
865 template <class Dimension, class Layout, typename Enable = void>
866 struct ViewOffset {
867 using is_mapping_plugin = std::false_type;
868 };
869
870 //----------------------------------------------------------------------------
871 // LayoutLeft AND ( 1 >= rank OR 0 == rank_dynamic ) : no padding / striding
872 template <class Dimension>
873 struct ViewOffset<
874 Dimension, Kokkos::LayoutLeft,
875 typename std::enable_if<(1 >= Dimension::rank ||
876 0 == Dimension::rank_dynamic)>::type> {
877 using is_mapping_plugin = std::true_type;
878 using is_regular = std::true_type;
879
880 using size_type = size_t;
881 using dimension_type = Dimension;
882 using array_layout = Kokkos::LayoutLeft;
883
884 dimension_type m_dim;
885
886 //----------------------------------------
887
888 // rank 1
889 template <typename I0>
operator ()Kokkos::Impl::ViewOffset890 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
891 return i0;
892 }
893
894 // rank 2
895 template <typename I0, typename I1>
operator ()Kokkos::Impl::ViewOffset896 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
897 I1 const& i1) const {
898 return i0 + m_dim.N0 * i1;
899 }
900
901 // rank 3
902 template <typename I0, typename I1, typename I2>
operator ()Kokkos::Impl::ViewOffset903 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
904 I1 const& i1,
905 I2 const& i2) const {
906 return i0 + m_dim.N0 * (i1 + m_dim.N1 * i2);
907 }
908
909 // rank 4
910 template <typename I0, typename I1, typename I2, typename I3>
operator ()Kokkos::Impl::ViewOffset911 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
912 I1 const& i1,
913 I2 const& i2,
914 I3 const& i3) const {
915 return i0 + m_dim.N0 * (i1 + m_dim.N1 * (i2 + m_dim.N2 * i3));
916 }
917
918 // rank 5
919 template <typename I0, typename I1, typename I2, typename I3, typename I4>
operator ()Kokkos::Impl::ViewOffset920 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
921 I1 const& i1,
922 I2 const& i2,
923 I3 const& i3,
924 I4 const& i4) const {
925 return i0 +
926 m_dim.N0 * (i1 + m_dim.N1 * (i2 + m_dim.N2 * (i3 + m_dim.N3 * i4)));
927 }
928
929 // rank 6
930 template <typename I0, typename I1, typename I2, typename I3, typename I4,
931 typename I5>
operator ()Kokkos::Impl::ViewOffset932 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
933 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
934 I5 const& i5) const {
935 return i0 +
936 m_dim.N0 *
937 (i1 +
938 m_dim.N1 *
939 (i2 + m_dim.N2 * (i3 + m_dim.N3 * (i4 + m_dim.N4 * i5))));
940 }
941
942 // rank 7
943 template <typename I0, typename I1, typename I2, typename I3, typename I4,
944 typename I5, typename I6>
operator ()Kokkos::Impl::ViewOffset945 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
946 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
947 I5 const& i5, I6 const& i6) const {
948 return i0 +
949 m_dim.N0 *
950 (i1 + m_dim.N1 *
951 (i2 + m_dim.N2 *
952 (i3 + m_dim.N3 *
953 (i4 + m_dim.N4 *
954 (i5 + m_dim.N5 * i6)))));
955 }
956
957 // rank 8
958 template <typename I0, typename I1, typename I2, typename I3, typename I4,
959 typename I5, typename I6, typename I7>
operator ()Kokkos::Impl::ViewOffset960 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
961 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
962 I5 const& i5, I6 const& i6, I7 const& i7) const {
963 return i0 +
964 m_dim.N0 *
965 (i1 +
966 m_dim.N1 *
967 (i2 + m_dim.N2 *
968 (i3 + m_dim.N3 *
969 (i4 + m_dim.N4 *
970 (i5 + m_dim.N5 *
971 (i6 + m_dim.N6 *
972 i7))))));
973 }
974
975 //----------------------------------------
976
977 KOKKOS_INLINE_FUNCTION
layoutKokkos::Impl::ViewOffset978 constexpr array_layout layout() const {
979 return array_layout(m_dim.N0, m_dim.N1, m_dim.N2, m_dim.N3, m_dim.N4,
980 m_dim.N5, m_dim.N6, m_dim.N7);
981 }
982
dimension_0Kokkos::Impl::ViewOffset983 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
984 return m_dim.N0;
985 }
dimension_1Kokkos::Impl::ViewOffset986 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
987 return m_dim.N1;
988 }
dimension_2Kokkos::Impl::ViewOffset989 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
990 return m_dim.N2;
991 }
dimension_3Kokkos::Impl::ViewOffset992 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
993 return m_dim.N3;
994 }
dimension_4Kokkos::Impl::ViewOffset995 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
996 return m_dim.N4;
997 }
dimension_5Kokkos::Impl::ViewOffset998 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
999 return m_dim.N5;
1000 }
dimension_6Kokkos::Impl::ViewOffset1001 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
1002 return m_dim.N6;
1003 }
dimension_7Kokkos::Impl::ViewOffset1004 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
1005 return m_dim.N7;
1006 }
1007
1008 /* Cardinality of the domain index space */
1009 KOKKOS_INLINE_FUNCTION
sizeKokkos::Impl::ViewOffset1010 constexpr size_type size() const {
1011 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
1012 m_dim.N6 * m_dim.N7;
1013 }
1014
1015 /* Span of the range space */
1016 KOKKOS_INLINE_FUNCTION
spanKokkos::Impl::ViewOffset1017 constexpr size_type span() const {
1018 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
1019 m_dim.N6 * m_dim.N7;
1020 }
1021
span_is_contiguousKokkos::Impl::ViewOffset1022 KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
1023 return true;
1024 }
1025
1026 /* Strides of dimensions */
stride_0Kokkos::Impl::ViewOffset1027 KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const { return 1; }
stride_1Kokkos::Impl::ViewOffset1028 KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
1029 return m_dim.N0;
1030 }
stride_2Kokkos::Impl::ViewOffset1031 KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
1032 return m_dim.N0 * m_dim.N1;
1033 }
stride_3Kokkos::Impl::ViewOffset1034 KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
1035 return m_dim.N0 * m_dim.N1 * m_dim.N2;
1036 }
stride_4Kokkos::Impl::ViewOffset1037 KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
1038 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3;
1039 }
stride_5Kokkos::Impl::ViewOffset1040 KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
1041 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4;
1042 }
stride_6Kokkos::Impl::ViewOffset1043 KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
1044 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5;
1045 }
stride_7Kokkos::Impl::ViewOffset1046 KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const {
1047 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
1048 m_dim.N6;
1049 }
1050
1051 // Stride with [ rank ] value is the total length
1052 template <typename iType>
strideKokkos::Impl::ViewOffset1053 KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
1054 s[0] = 1;
1055 if (0 < dimension_type::rank) {
1056 s[1] = m_dim.N0;
1057 }
1058 if (1 < dimension_type::rank) {
1059 s[2] = s[1] * m_dim.N1;
1060 }
1061 if (2 < dimension_type::rank) {
1062 s[3] = s[2] * m_dim.N2;
1063 }
1064 if (3 < dimension_type::rank) {
1065 s[4] = s[3] * m_dim.N3;
1066 }
1067 if (4 < dimension_type::rank) {
1068 s[5] = s[4] * m_dim.N4;
1069 }
1070 if (5 < dimension_type::rank) {
1071 s[6] = s[5] * m_dim.N5;
1072 }
1073 if (6 < dimension_type::rank) {
1074 s[7] = s[6] * m_dim.N6;
1075 }
1076 if (7 < dimension_type::rank) {
1077 s[8] = s[7] * m_dim.N7;
1078 }
1079 }
1080
1081 //----------------------------------------
1082
1083 // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
1084 // correct and errors out during compilation. Same for the other places where
1085 // I changed this.
1086 #ifdef KOKKOS_IMPL_WINDOWS_CUDA
ViewOffsetKokkos::Impl::ViewOffset1087 KOKKOS_FUNCTION ViewOffset() : m_dim(dimension_type()) {}
ViewOffsetKokkos::Impl::ViewOffset1088 KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) { m_dim = src.m_dim; }
operator =Kokkos::Impl::ViewOffset1089 KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
1090 m_dim = src.m_dim;
1091 return *this;
1092 }
1093 #else
1094 ViewOffset() = default;
1095 ViewOffset(const ViewOffset&) = default;
1096 ViewOffset& operator=(const ViewOffset&) = default;
1097 #endif
1098
1099 template <unsigned TrivialScalarSize>
ViewOffsetKokkos::Impl::ViewOffset1100 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1101 std::integral_constant<unsigned, TrivialScalarSize> const&,
1102 Kokkos::LayoutLeft const& arg_layout)
1103 : m_dim(arg_layout.dimension[0], 0, 0, 0, 0, 0, 0, 0) {}
1104
1105 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1106 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1107 const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>& rhs)
1108 : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1109 rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7) {
1110 static_assert(int(DimRHS::rank) == int(dimension_type::rank),
1111 "ViewOffset assignment requires equal rank");
1112 // Also requires equal static dimensions ...
1113 }
1114
1115 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1116 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1117 const ViewOffset<DimRHS, Kokkos::LayoutRight, void>& rhs)
1118 : m_dim(rhs.m_dim.N0, 0, 0, 0, 0, 0, 0, 0) {
1119 static_assert((DimRHS::rank == 0 && dimension_type::rank == 0) ||
1120 (DimRHS::rank == 1 && dimension_type::rank == 1 &&
1121 dimension_type::rank_dynamic == 1),
1122 "ViewOffset LayoutLeft and LayoutRight are only compatible "
1123 "when rank <= 1");
1124 }
1125
1126 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1127 KOKKOS_INLINE_FUNCTION ViewOffset(
1128 const ViewOffset<DimRHS, Kokkos::LayoutStride, void>& rhs)
1129 : m_dim(rhs.m_dim.N0, 0, 0, 0, 0, 0, 0, 0) {
1130 if (rhs.m_stride.S0 != 1) {
1131 Kokkos::abort(
1132 "Kokkos::Impl::ViewOffset assignment of LayoutLeft from LayoutStride "
1133 " requires stride == 1");
1134 }
1135 }
1136
1137 //----------------------------------------
1138 // Subview construction
1139
1140 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1141 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1142 const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>&,
1143 const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
1144 : m_dim(sub.range_extent(0), 0, 0, 0, 0, 0, 0, 0) {
1145 static_assert((0 == dimension_type::rank_dynamic) ||
1146 (1 == dimension_type::rank &&
1147 1 == dimension_type::rank_dynamic && 1 <= DimRHS::rank),
1148 "ViewOffset subview construction requires compatible rank");
1149 }
1150 };
1151
1152 //----------------------------------------------------------------------------
1153 // LayoutLeft AND ( 1 < rank AND 0 < rank_dynamic ) : has padding / striding
1154 template <class Dimension>
1155 struct ViewOffset<
1156 Dimension, Kokkos::LayoutLeft,
1157 typename std::enable_if<(1 < Dimension::rank &&
1158 0 < Dimension::rank_dynamic)>::type> {
1159 using is_mapping_plugin = std::true_type;
1160 using is_regular = std::true_type;
1161
1162 using size_type = size_t;
1163 using dimension_type = Dimension;
1164 using array_layout = Kokkos::LayoutLeft;
1165
1166 dimension_type m_dim;
1167 size_type m_stride;
1168
1169 //----------------------------------------
1170
1171 // rank 1
1172 template <typename I0>
operator ()Kokkos::Impl::ViewOffset1173 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
1174 return i0;
1175 }
1176
1177 // rank 2
1178 template <typename I0, typename I1>
operator ()Kokkos::Impl::ViewOffset1179 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1180 I1 const& i1) const {
1181 return i0 + m_stride * i1;
1182 }
1183
1184 // rank 3
1185 template <typename I0, typename I1, typename I2>
operator ()Kokkos::Impl::ViewOffset1186 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1187 I1 const& i1,
1188 I2 const& i2) const {
1189 return i0 + m_stride * (i1 + m_dim.N1 * i2);
1190 }
1191
1192 // rank 4
1193 template <typename I0, typename I1, typename I2, typename I3>
operator ()Kokkos::Impl::ViewOffset1194 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1195 I1 const& i1,
1196 I2 const& i2,
1197 I3 const& i3) const {
1198 return i0 + m_stride * (i1 + m_dim.N1 * (i2 + m_dim.N2 * i3));
1199 }
1200
1201 // rank 5
1202 template <typename I0, typename I1, typename I2, typename I3, typename I4>
operator ()Kokkos::Impl::ViewOffset1203 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1204 I1 const& i1,
1205 I2 const& i2,
1206 I3 const& i3,
1207 I4 const& i4) const {
1208 return i0 +
1209 m_stride * (i1 + m_dim.N1 * (i2 + m_dim.N2 * (i3 + m_dim.N3 * i4)));
1210 }
1211
1212 // rank 6
1213 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1214 typename I5>
operator ()Kokkos::Impl::ViewOffset1215 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1216 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1217 I5 const& i5) const {
1218 return i0 +
1219 m_stride *
1220 (i1 +
1221 m_dim.N1 *
1222 (i2 + m_dim.N2 * (i3 + m_dim.N3 * (i4 + m_dim.N4 * i5))));
1223 }
1224
1225 // rank 7
1226 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1227 typename I5, typename I6>
operator ()Kokkos::Impl::ViewOffset1228 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1229 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1230 I5 const& i5, I6 const& i6) const {
1231 return i0 +
1232 m_stride *
1233 (i1 + m_dim.N1 *
1234 (i2 + m_dim.N2 *
1235 (i3 + m_dim.N3 *
1236 (i4 + m_dim.N4 *
1237 (i5 + m_dim.N5 * i6)))));
1238 }
1239
1240 // rank 8
1241 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1242 typename I5, typename I6, typename I7>
operator ()Kokkos::Impl::ViewOffset1243 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1244 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1245 I5 const& i5, I6 const& i6, I7 const& i7) const {
1246 return i0 +
1247 m_stride *
1248 (i1 +
1249 m_dim.N1 *
1250 (i2 + m_dim.N2 *
1251 (i3 + m_dim.N3 *
1252 (i4 + m_dim.N4 *
1253 (i5 + m_dim.N5 *
1254 (i6 + m_dim.N6 *
1255 i7))))));
1256 }
1257
1258 //----------------------------------------
1259
1260 KOKKOS_INLINE_FUNCTION
layoutKokkos::Impl::ViewOffset1261 constexpr array_layout layout() const {
1262 return array_layout(m_dim.N0, m_dim.N1, m_dim.N2, m_dim.N3, m_dim.N4,
1263 m_dim.N5, m_dim.N6, m_dim.N7);
1264 }
1265
dimension_0Kokkos::Impl::ViewOffset1266 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
1267 return m_dim.N0;
1268 }
dimension_1Kokkos::Impl::ViewOffset1269 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
1270 return m_dim.N1;
1271 }
dimension_2Kokkos::Impl::ViewOffset1272 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
1273 return m_dim.N2;
1274 }
dimension_3Kokkos::Impl::ViewOffset1275 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
1276 return m_dim.N3;
1277 }
dimension_4Kokkos::Impl::ViewOffset1278 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
1279 return m_dim.N4;
1280 }
dimension_5Kokkos::Impl::ViewOffset1281 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
1282 return m_dim.N5;
1283 }
dimension_6Kokkos::Impl::ViewOffset1284 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
1285 return m_dim.N6;
1286 }
dimension_7Kokkos::Impl::ViewOffset1287 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
1288 return m_dim.N7;
1289 }
1290
1291 /* Cardinality of the domain index space */
1292 KOKKOS_INLINE_FUNCTION
sizeKokkos::Impl::ViewOffset1293 constexpr size_type size() const {
1294 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
1295 m_dim.N6 * m_dim.N7;
1296 }
1297
1298 /* Span of the range space */
1299 KOKKOS_INLINE_FUNCTION
spanKokkos::Impl::ViewOffset1300 constexpr size_type span() const {
1301 return (m_dim.N0 > size_type(0) ? m_stride : size_type(0)) * m_dim.N1 *
1302 m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 * m_dim.N6 * m_dim.N7;
1303 }
1304
span_is_contiguousKokkos::Impl::ViewOffset1305 KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
1306 return m_stride == m_dim.N0;
1307 }
1308
1309 /* Strides of dimensions */
stride_0Kokkos::Impl::ViewOffset1310 KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const { return 1; }
stride_1Kokkos::Impl::ViewOffset1311 KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
1312 return m_stride;
1313 }
stride_2Kokkos::Impl::ViewOffset1314 KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
1315 return m_stride * m_dim.N1;
1316 }
stride_3Kokkos::Impl::ViewOffset1317 KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
1318 return m_stride * m_dim.N1 * m_dim.N2;
1319 }
stride_4Kokkos::Impl::ViewOffset1320 KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
1321 return m_stride * m_dim.N1 * m_dim.N2 * m_dim.N3;
1322 }
stride_5Kokkos::Impl::ViewOffset1323 KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
1324 return m_stride * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4;
1325 }
stride_6Kokkos::Impl::ViewOffset1326 KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
1327 return m_stride * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5;
1328 }
stride_7Kokkos::Impl::ViewOffset1329 KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const {
1330 return m_stride * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
1331 m_dim.N6;
1332 }
1333
1334 // Stride with [ rank ] value is the total length
1335 template <typename iType>
strideKokkos::Impl::ViewOffset1336 KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
1337 s[0] = 1;
1338 if (0 < dimension_type::rank) {
1339 s[1] = m_stride;
1340 }
1341 if (1 < dimension_type::rank) {
1342 s[2] = s[1] * m_dim.N1;
1343 }
1344 if (2 < dimension_type::rank) {
1345 s[3] = s[2] * m_dim.N2;
1346 }
1347 if (3 < dimension_type::rank) {
1348 s[4] = s[3] * m_dim.N3;
1349 }
1350 if (4 < dimension_type::rank) {
1351 s[5] = s[4] * m_dim.N4;
1352 }
1353 if (5 < dimension_type::rank) {
1354 s[6] = s[5] * m_dim.N5;
1355 }
1356 if (6 < dimension_type::rank) {
1357 s[7] = s[6] * m_dim.N6;
1358 }
1359 if (7 < dimension_type::rank) {
1360 s[8] = s[7] * m_dim.N7;
1361 }
1362 }
1363
1364 //----------------------------------------
1365
1366 private:
1367 template <unsigned TrivialScalarSize>
1368 struct Padding {
1369 enum {
1370 div = TrivialScalarSize == 0
1371 ? 0
1372 : Kokkos::Impl::MEMORY_ALIGNMENT /
1373 (TrivialScalarSize ? TrivialScalarSize : 1)
1374 };
1375 enum {
1376 mod = TrivialScalarSize == 0
1377 ? 0
1378 : Kokkos::Impl::MEMORY_ALIGNMENT %
1379 (TrivialScalarSize ? TrivialScalarSize : 1)
1380 };
1381
1382 // If memory alignment is a multiple of the trivial scalar size then attempt
1383 // to align.
1384 enum { align = 0 != TrivialScalarSize && 0 == mod ? div : 0 };
1385 enum {
1386 div_ok = (div != 0) ? div : 1
1387 }; // To valid modulo zero in constexpr
1388
1389 KOKKOS_INLINE_FUNCTION
strideKokkos::Impl::ViewOffset::Padding1390 static constexpr size_t stride(size_t const N) {
1391 return ((align != 0) &&
1392 ((Kokkos::Impl::MEMORY_ALIGNMENT_THRESHOLD * align) < N) &&
1393 ((N % div_ok) != 0))
1394 ? N + align - (N % div_ok)
1395 : N;
1396 }
1397 };
1398
1399 public:
1400 // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
1401 // correct and errors out during compilation. Same for the other places where
1402 // I changed this.
1403 #ifdef KOKKOS_IMPL_WINDOWS_CUDA
ViewOffsetKokkos::Impl::ViewOffset1404 KOKKOS_FUNCTION ViewOffset() : m_dim(dimension_type()), m_stride(0) {}
ViewOffsetKokkos::Impl::ViewOffset1405 KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) {
1406 m_dim = src.m_dim;
1407 m_stride = src.m_stride;
1408 }
operator =Kokkos::Impl::ViewOffset1409 KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
1410 m_dim = src.m_dim;
1411 m_stride = src.m_stride;
1412 return *this;
1413 }
1414 #else
1415
1416 ViewOffset() = default;
1417 ViewOffset(const ViewOffset&) = default;
1418 ViewOffset& operator=(const ViewOffset&) = default;
1419 #endif
1420
1421 /* Enable padding for trivial scalar types with non-zero trivial scalar size
1422 */
1423 template <unsigned TrivialScalarSize>
ViewOffsetKokkos::Impl::ViewOffset1424 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1425 std::integral_constant<unsigned, TrivialScalarSize> const&,
1426 Kokkos::LayoutLeft const& arg_layout)
1427 : m_dim(arg_layout.dimension[0], arg_layout.dimension[1],
1428 arg_layout.dimension[2], arg_layout.dimension[3],
1429 arg_layout.dimension[4], arg_layout.dimension[5],
1430 arg_layout.dimension[6], arg_layout.dimension[7]),
1431 m_stride(Padding<TrivialScalarSize>::stride(arg_layout.dimension[0])) {}
1432
1433 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1434 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1435 const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>& rhs)
1436 : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1437 rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
1438 m_stride(rhs.stride_1()) {
1439 static_assert(int(DimRHS::rank) == int(dimension_type::rank),
1440 "ViewOffset assignment requires equal rank");
1441 // Also requires equal static dimensions ...
1442 }
1443
1444 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1445 KOKKOS_INLINE_FUNCTION ViewOffset(
1446 const ViewOffset<DimRHS, Kokkos::LayoutStride, void>& rhs)
1447 : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1448 rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
1449 m_stride(rhs.stride_1()) {
1450 if (rhs.m_stride.S0 != 1) {
1451 Kokkos::abort(
1452 "Kokkos::Impl::ViewOffset assignment of LayoutLeft from LayoutStride "
1453 "requires stride == 1");
1454 }
1455 }
1456
1457 //----------------------------------------
1458 // Subview construction
1459 // This subview must be 2 == rank and 2 == rank_dynamic
1460 // due to only having stride #0.
1461 // The source dimension #0 must be non-zero for stride-one leading dimension.
1462 // At most subsequent dimension can be non-zero.
1463
1464 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1465 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1466 const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>& rhs,
1467 const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
1468 : m_dim(sub.range_extent(0), sub.range_extent(1), sub.range_extent(2),
1469 sub.range_extent(3), sub.range_extent(4), sub.range_extent(5),
1470 sub.range_extent(6), sub.range_extent(7)),
1471 m_stride(
1472 (1 == sub.range_index(1)
1473 ? rhs.stride_1()
1474 : (2 == sub.range_index(1)
1475 ? rhs.stride_2()
1476 : (3 == sub.range_index(1)
1477 ? rhs.stride_3()
1478 : (4 == sub.range_index(1)
1479 ? rhs.stride_4()
1480 : (5 == sub.range_index(1)
1481 ? rhs.stride_5()
1482 : (6 == sub.range_index(1)
1483 ? rhs.stride_6()
1484 : (7 == sub.range_index(1)
1485 ? rhs.stride_7()
1486 : 0)))))))) {
1487 // static_assert( ( 2 == dimension_type::rank ) &&
1488 // ( 2 == dimension_type::rank_dynamic ) &&
1489 // ( 2 <= DimRHS::rank )
1490 // , "ViewOffset subview construction requires compatible rank"
1491 // );
1492 }
1493 };
1494
1495 //----------------------------------------------------------------------------
1496 // LayoutRight AND ( 1 >= rank OR 0 == rank_dynamic ) : no padding / striding
1497 template <class Dimension>
1498 struct ViewOffset<
1499 Dimension, Kokkos::LayoutRight,
1500 typename std::enable_if<(1 >= Dimension::rank ||
1501 0 == Dimension::rank_dynamic)>::type> {
1502 using is_mapping_plugin = std::true_type;
1503 using is_regular = std::true_type;
1504
1505 using size_type = size_t;
1506 using dimension_type = Dimension;
1507 using array_layout = Kokkos::LayoutRight;
1508
1509 dimension_type m_dim;
1510
1511 //----------------------------------------
1512
1513 // rank 1
1514 template <typename I0>
operator ()Kokkos::Impl::ViewOffset1515 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
1516 return i0;
1517 }
1518
1519 // rank 2
1520 template <typename I0, typename I1>
operator ()Kokkos::Impl::ViewOffset1521 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1522 I1 const& i1) const {
1523 return i1 + m_dim.N1 * i0;
1524 }
1525
1526 // rank 3
1527 template <typename I0, typename I1, typename I2>
operator ()Kokkos::Impl::ViewOffset1528 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1529 I1 const& i1,
1530 I2 const& i2) const {
1531 return i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0));
1532 }
1533
1534 // rank 4
1535 template <typename I0, typename I1, typename I2, typename I3>
operator ()Kokkos::Impl::ViewOffset1536 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1537 I1 const& i1,
1538 I2 const& i2,
1539 I3 const& i3) const {
1540 return i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0)));
1541 }
1542
1543 // rank 5
1544 template <typename I0, typename I1, typename I2, typename I3, typename I4>
operator ()Kokkos::Impl::ViewOffset1545 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1546 I1 const& i1,
1547 I2 const& i2,
1548 I3 const& i3,
1549 I4 const& i4) const {
1550 return i4 + m_dim.N4 *
1551 (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0))));
1552 }
1553
1554 // rank 6
1555 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1556 typename I5>
operator ()Kokkos::Impl::ViewOffset1557 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1558 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1559 I5 const& i5) const {
1560 return i5 +
1561 m_dim.N5 *
1562 (i4 +
1563 m_dim.N4 *
1564 (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0)))));
1565 }
1566
1567 // rank 7
1568 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1569 typename I5, typename I6>
operator ()Kokkos::Impl::ViewOffset1570 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1571 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1572 I5 const& i5, I6 const& i6) const {
1573 return i6 +
1574 m_dim.N6 *
1575 (i5 +
1576 m_dim.N5 *
1577 (i4 +
1578 m_dim.N4 *
1579 (i3 + m_dim.N3 *
1580 (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0))))));
1581 }
1582
1583 // rank 8
1584 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1585 typename I5, typename I6, typename I7>
operator ()Kokkos::Impl::ViewOffset1586 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1587 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1588 I5 const& i5, I6 const& i6, I7 const& i7) const {
1589 return i7 +
1590 m_dim.N7 *
1591 (i6 +
1592 m_dim.N6 *
1593 (i5 +
1594 m_dim.N5 *
1595 (i4 +
1596 m_dim.N4 *
1597 (i3 +
1598 m_dim.N3 *
1599 (i2 + m_dim.N2 * (i1 + m_dim.N1 * (i0)))))));
1600 }
1601
1602 //----------------------------------------
1603
1604 KOKKOS_INLINE_FUNCTION
layoutKokkos::Impl::ViewOffset1605 constexpr array_layout layout() const {
1606 return array_layout(m_dim.N0, m_dim.N1, m_dim.N2, m_dim.N3, m_dim.N4,
1607 m_dim.N5, m_dim.N6, m_dim.N7);
1608 }
1609
dimension_0Kokkos::Impl::ViewOffset1610 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
1611 return m_dim.N0;
1612 }
dimension_1Kokkos::Impl::ViewOffset1613 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
1614 return m_dim.N1;
1615 }
dimension_2Kokkos::Impl::ViewOffset1616 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
1617 return m_dim.N2;
1618 }
dimension_3Kokkos::Impl::ViewOffset1619 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
1620 return m_dim.N3;
1621 }
dimension_4Kokkos::Impl::ViewOffset1622 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
1623 return m_dim.N4;
1624 }
dimension_5Kokkos::Impl::ViewOffset1625 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
1626 return m_dim.N5;
1627 }
dimension_6Kokkos::Impl::ViewOffset1628 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
1629 return m_dim.N6;
1630 }
dimension_7Kokkos::Impl::ViewOffset1631 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
1632 return m_dim.N7;
1633 }
1634
1635 /* Cardinality of the domain index space */
1636 KOKKOS_INLINE_FUNCTION
sizeKokkos::Impl::ViewOffset1637 constexpr size_type size() const {
1638 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
1639 m_dim.N6 * m_dim.N7;
1640 }
1641
1642 /* Span of the range space */
1643 KOKKOS_INLINE_FUNCTION
spanKokkos::Impl::ViewOffset1644 constexpr size_type span() const {
1645 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
1646 m_dim.N6 * m_dim.N7;
1647 }
1648
span_is_contiguousKokkos::Impl::ViewOffset1649 KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
1650 return true;
1651 }
1652
1653 /* Strides of dimensions */
stride_7Kokkos::Impl::ViewOffset1654 KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const { return 1; }
stride_6Kokkos::Impl::ViewOffset1655 KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
1656 return m_dim.N7;
1657 }
stride_5Kokkos::Impl::ViewOffset1658 KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
1659 return m_dim.N7 * m_dim.N6;
1660 }
stride_4Kokkos::Impl::ViewOffset1661 KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
1662 return m_dim.N7 * m_dim.N6 * m_dim.N5;
1663 }
stride_3Kokkos::Impl::ViewOffset1664 KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
1665 return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4;
1666 }
stride_2Kokkos::Impl::ViewOffset1667 KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
1668 return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3;
1669 }
stride_1Kokkos::Impl::ViewOffset1670 KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
1671 return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3 * m_dim.N2;
1672 }
stride_0Kokkos::Impl::ViewOffset1673 KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const {
1674 return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3 * m_dim.N2 *
1675 m_dim.N1;
1676 }
1677
1678 // Stride with [ rank ] value is the total length
1679 template <typename iType>
strideKokkos::Impl::ViewOffset1680 KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
1681 size_type n = 1;
1682 if (7 < dimension_type::rank) {
1683 s[7] = n;
1684 n *= m_dim.N7;
1685 }
1686 if (6 < dimension_type::rank) {
1687 s[6] = n;
1688 n *= m_dim.N6;
1689 }
1690 if (5 < dimension_type::rank) {
1691 s[5] = n;
1692 n *= m_dim.N5;
1693 }
1694 if (4 < dimension_type::rank) {
1695 s[4] = n;
1696 n *= m_dim.N4;
1697 }
1698 if (3 < dimension_type::rank) {
1699 s[3] = n;
1700 n *= m_dim.N3;
1701 }
1702 if (2 < dimension_type::rank) {
1703 s[2] = n;
1704 n *= m_dim.N2;
1705 }
1706 if (1 < dimension_type::rank) {
1707 s[1] = n;
1708 n *= m_dim.N1;
1709 }
1710 if (0 < dimension_type::rank) {
1711 s[0] = n;
1712 }
1713 s[dimension_type::rank] = n * m_dim.N0;
1714 }
1715
1716 //----------------------------------------
1717 // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
1718 // correct and errors out during compilation. Same for the other places where
1719 // I changed this.
1720
1721 #ifdef KOKKOS_IMPL_WINDOWS_CUDA
ViewOffsetKokkos::Impl::ViewOffset1722 KOKKOS_FUNCTION ViewOffset() : m_dim(dimension_type()) {}
ViewOffsetKokkos::Impl::ViewOffset1723 KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) { m_dim = src.m_dim; }
operator =Kokkos::Impl::ViewOffset1724 KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
1725 m_dim = src.m_dim;
1726 return *this;
1727 }
1728 #else
1729
1730 ViewOffset() = default;
1731 ViewOffset(const ViewOffset&) = default;
1732 ViewOffset& operator=(const ViewOffset&) = default;
1733 #endif
1734
1735 template <unsigned TrivialScalarSize>
ViewOffsetKokkos::Impl::ViewOffset1736 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1737 std::integral_constant<unsigned, TrivialScalarSize> const&,
1738 Kokkos::LayoutRight const& arg_layout)
1739 : m_dim(arg_layout.dimension[0], 0, 0, 0, 0, 0, 0, 0) {}
1740
1741 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1742 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1743 const ViewOffset<DimRHS, Kokkos::LayoutRight, void>& rhs)
1744 : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
1745 rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7) {
1746 static_assert(int(DimRHS::rank) == int(dimension_type::rank),
1747 "ViewOffset assignment requires equal rank");
1748 // Also requires equal static dimensions ...
1749 }
1750
1751 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1752 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1753 const ViewOffset<DimRHS, Kokkos::LayoutLeft, void>& rhs)
1754 : m_dim(rhs.m_dim.N0, 0, 0, 0, 0, 0, 0, 0) {
1755 static_assert((DimRHS::rank == 0 && dimension_type::rank == 0) ||
1756 (DimRHS::rank == 1 && dimension_type::rank == 1 &&
1757 dimension_type::rank_dynamic == 1),
1758 "ViewOffset LayoutRight and LayoutLeft are only compatible "
1759 "when rank <= 1");
1760 }
1761
1762 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1763 KOKKOS_INLINE_FUNCTION ViewOffset(
1764 const ViewOffset<DimRHS, Kokkos::LayoutStride, void>& rhs)
1765 : m_dim(rhs.m_dim.N0, 0, 0, 0, 0, 0, 0, 0) {}
1766
1767 //----------------------------------------
1768 // Subview construction
1769
1770 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset1771 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
1772 const ViewOffset<DimRHS, Kokkos::LayoutRight, void>&,
1773 const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
1774 : m_dim(sub.range_extent(0), 0, 0, 0, 0, 0, 0, 0) {
1775 static_assert((0 == dimension_type::rank_dynamic) ||
1776 (1 == dimension_type::rank &&
1777 1 == dimension_type::rank_dynamic && 1 <= DimRHS::rank),
1778 "ViewOffset subview construction requires compatible rank");
1779 }
1780 };
1781
1782 //----------------------------------------------------------------------------
1783 // LayoutRight AND ( 1 < rank AND 0 < rank_dynamic ) : has padding / striding
1784 template <class Dimension>
1785 struct ViewOffset<
1786 Dimension, Kokkos::LayoutRight,
1787 typename std::enable_if<(1 < Dimension::rank &&
1788 0 < Dimension::rank_dynamic)>::type> {
1789 using is_mapping_plugin = std::true_type;
1790 using is_regular = std::true_type;
1791
1792 using size_type = size_t;
1793 using dimension_type = Dimension;
1794 using array_layout = Kokkos::LayoutRight;
1795
1796 dimension_type m_dim;
1797 size_type m_stride;
1798
1799 //----------------------------------------
1800
1801 // rank 1
1802 template <typename I0>
operator ()Kokkos::Impl::ViewOffset1803 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
1804 return i0;
1805 }
1806
1807 // rank 2
1808 template <typename I0, typename I1>
operator ()Kokkos::Impl::ViewOffset1809 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1810 I1 const& i1) const {
1811 return i1 + i0 * m_stride;
1812 }
1813
1814 // rank 3
1815 template <typename I0, typename I1, typename I2>
operator ()Kokkos::Impl::ViewOffset1816 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1817 I1 const& i1,
1818 I2 const& i2) const {
1819 return i2 + m_dim.N2 * (i1) + i0 * m_stride;
1820 }
1821
1822 // rank 4
1823 template <typename I0, typename I1, typename I2, typename I3>
operator ()Kokkos::Impl::ViewOffset1824 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1825 I1 const& i1,
1826 I2 const& i2,
1827 I3 const& i3) const {
1828 return i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1)) + i0 * m_stride;
1829 }
1830
1831 // rank 5
1832 template <typename I0, typename I1, typename I2, typename I3, typename I4>
operator ()Kokkos::Impl::ViewOffset1833 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
1834 I1 const& i1,
1835 I2 const& i2,
1836 I3 const& i3,
1837 I4 const& i4) const {
1838 return i4 + m_dim.N4 * (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1))) +
1839 i0 * m_stride;
1840 }
1841
1842 // rank 6
1843 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1844 typename I5>
operator ()Kokkos::Impl::ViewOffset1845 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1846 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1847 I5 const& i5) const {
1848 return i5 +
1849 m_dim.N5 *
1850 (i4 + m_dim.N4 * (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1)))) +
1851 i0 * m_stride;
1852 }
1853
1854 // rank 7
1855 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1856 typename I5, typename I6>
operator ()Kokkos::Impl::ViewOffset1857 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1858 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1859 I5 const& i5, I6 const& i6) const {
1860 return i6 +
1861 m_dim.N6 *
1862 (i5 + m_dim.N5 *
1863 (i4 + m_dim.N4 *
1864 (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1))))) +
1865 i0 * m_stride;
1866 }
1867
1868 // rank 8
1869 template <typename I0, typename I1, typename I2, typename I3, typename I4,
1870 typename I5, typename I6, typename I7>
operator ()Kokkos::Impl::ViewOffset1871 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
1872 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
1873 I5 const& i5, I6 const& i6, I7 const& i7) const {
1874 return i7 +
1875 m_dim.N7 *
1876 (i6 +
1877 m_dim.N6 *
1878 (i5 +
1879 m_dim.N5 *
1880 (i4 + m_dim.N4 *
1881 (i3 + m_dim.N3 * (i2 + m_dim.N2 * (i1)))))) +
1882 i0 * m_stride;
1883 }
1884
1885 //----------------------------------------
1886
1887 KOKKOS_INLINE_FUNCTION
layoutKokkos::Impl::ViewOffset1888 constexpr array_layout layout() const {
1889 return array_layout(m_dim.N0, m_dim.N1, m_dim.N2, m_dim.N3, m_dim.N4,
1890 m_dim.N5, m_dim.N6, m_dim.N7);
1891 }
1892
dimension_0Kokkos::Impl::ViewOffset1893 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
1894 return m_dim.N0;
1895 }
dimension_1Kokkos::Impl::ViewOffset1896 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
1897 return m_dim.N1;
1898 }
dimension_2Kokkos::Impl::ViewOffset1899 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
1900 return m_dim.N2;
1901 }
dimension_3Kokkos::Impl::ViewOffset1902 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
1903 return m_dim.N3;
1904 }
dimension_4Kokkos::Impl::ViewOffset1905 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
1906 return m_dim.N4;
1907 }
dimension_5Kokkos::Impl::ViewOffset1908 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
1909 return m_dim.N5;
1910 }
dimension_6Kokkos::Impl::ViewOffset1911 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
1912 return m_dim.N6;
1913 }
dimension_7Kokkos::Impl::ViewOffset1914 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
1915 return m_dim.N7;
1916 }
1917
1918 /* Cardinality of the domain index space */
1919 KOKKOS_INLINE_FUNCTION
sizeKokkos::Impl::ViewOffset1920 constexpr size_type size() const {
1921 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
1922 m_dim.N6 * m_dim.N7;
1923 }
1924
1925 /* Span of the range space */
1926 KOKKOS_INLINE_FUNCTION
spanKokkos::Impl::ViewOffset1927 constexpr size_type span() const {
1928 return size() > 0 ? m_dim.N0 * m_stride : 0;
1929 }
1930
span_is_contiguousKokkos::Impl::ViewOffset1931 KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
1932 return m_stride == m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3 *
1933 m_dim.N2 * m_dim.N1;
1934 }
1935
1936 /* Strides of dimensions */
stride_7Kokkos::Impl::ViewOffset1937 KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const { return 1; }
stride_6Kokkos::Impl::ViewOffset1938 KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
1939 return m_dim.N7;
1940 }
stride_5Kokkos::Impl::ViewOffset1941 KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
1942 return m_dim.N7 * m_dim.N6;
1943 }
stride_4Kokkos::Impl::ViewOffset1944 KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
1945 return m_dim.N7 * m_dim.N6 * m_dim.N5;
1946 }
stride_3Kokkos::Impl::ViewOffset1947 KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
1948 return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4;
1949 }
stride_2Kokkos::Impl::ViewOffset1950 KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
1951 return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3;
1952 }
stride_1Kokkos::Impl::ViewOffset1953 KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
1954 return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3 * m_dim.N2;
1955 }
stride_0Kokkos::Impl::ViewOffset1956 KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const {
1957 return m_stride;
1958 }
1959
1960 // Stride with [ rank ] value is the total length
1961 template <typename iType>
strideKokkos::Impl::ViewOffset1962 KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
1963 size_type n = 1;
1964 if (7 < dimension_type::rank) {
1965 s[7] = n;
1966 n *= m_dim.N7;
1967 }
1968 if (6 < dimension_type::rank) {
1969 s[6] = n;
1970 n *= m_dim.N6;
1971 }
1972 if (5 < dimension_type::rank) {
1973 s[5] = n;
1974 n *= m_dim.N5;
1975 }
1976 if (4 < dimension_type::rank) {
1977 s[4] = n;
1978 n *= m_dim.N4;
1979 }
1980 if (3 < dimension_type::rank) {
1981 s[3] = n;
1982 n *= m_dim.N3;
1983 }
1984 if (2 < dimension_type::rank) {
1985 s[2] = n;
1986 n *= m_dim.N2;
1987 }
1988 if (1 < dimension_type::rank) {
1989 s[1] = n;
1990 }
1991 if (0 < dimension_type::rank) {
1992 s[0] = m_stride;
1993 }
1994 s[dimension_type::rank] = m_stride * m_dim.N0;
1995 }
1996
1997 //----------------------------------------
1998
1999 private:
2000 template <unsigned TrivialScalarSize>
2001 struct Padding {
2002 enum {
2003 div = TrivialScalarSize == 0
2004 ? 0
2005 : Kokkos::Impl::MEMORY_ALIGNMENT /
2006 (TrivialScalarSize ? TrivialScalarSize : 1)
2007 };
2008 enum {
2009 mod = TrivialScalarSize == 0
2010 ? 0
2011 : Kokkos::Impl::MEMORY_ALIGNMENT %
2012 (TrivialScalarSize ? TrivialScalarSize : 1)
2013 };
2014
2015 // If memory alignment is a multiple of the trivial scalar size then attempt
2016 // to align.
2017 enum { align = 0 != TrivialScalarSize && 0 == mod ? div : 0 };
2018 enum {
2019 div_ok = (div != 0) ? div : 1
2020 }; // To valid modulo zero in constexpr
2021
2022 KOKKOS_INLINE_FUNCTION
strideKokkos::Impl::ViewOffset::Padding2023 static constexpr size_t stride(size_t const N) {
2024 return ((align != 0) &&
2025 ((Kokkos::Impl::MEMORY_ALIGNMENT_THRESHOLD * align) < N) &&
2026 ((N % div_ok) != 0))
2027 ? N + align - (N % div_ok)
2028 : N;
2029 }
2030 };
2031
2032 public:
2033 // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
2034 // correct and errors out during compilation. Same for the other places where
2035 // I changed this.
2036
2037 #ifdef KOKKOS_IMPL_WINDOWS_CUDA
ViewOffsetKokkos::Impl::ViewOffset2038 KOKKOS_FUNCTION ViewOffset() : m_dim(dimension_type()), m_stride(0) {}
ViewOffsetKokkos::Impl::ViewOffset2039 KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) {
2040 m_dim = src.m_dim;
2041 m_stride = src.m_stride;
2042 }
operator =Kokkos::Impl::ViewOffset2043 KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
2044 m_dim = src.m_dim;
2045 m_stride = src.m_stride;
2046 return *this;
2047 }
2048 #else
2049
2050 ViewOffset() = default;
2051 ViewOffset(const ViewOffset&) = default;
2052 ViewOffset& operator=(const ViewOffset&) = default;
2053 #endif
2054
2055 /* Enable padding for trivial scalar types with non-zero trivial scalar size.
2056 */
2057 template <unsigned TrivialScalarSize>
ViewOffsetKokkos::Impl::ViewOffset2058 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
2059 std::integral_constant<unsigned, TrivialScalarSize> const&,
2060 Kokkos::LayoutRight const& arg_layout)
2061 : m_dim(arg_layout.dimension[0], arg_layout.dimension[1],
2062 arg_layout.dimension[2], arg_layout.dimension[3],
2063 arg_layout.dimension[4], arg_layout.dimension[5],
2064 arg_layout.dimension[6], arg_layout.dimension[7]),
2065 m_stride(
2066 Padding<TrivialScalarSize>::
2067 stride(/* 2 <= rank */
2068 m_dim.N1 *
2069 (dimension_type::rank == 2
2070 ? 1
2071 : m_dim.N2 *
2072 (dimension_type::rank == 3
2073 ? 1
2074 : m_dim.N3 *
2075 (dimension_type::rank == 4
2076 ? 1
2077 : m_dim.N4 *
2078 (dimension_type::rank ==
2079 5
2080 ? 1
2081 : m_dim.N5 *
2082 (dimension_type::
2083 rank ==
2084 6
2085 ? 1
2086 : m_dim.N6 *
2087 (dimension_type::
2088 rank ==
2089 7
2090 ? 1
2091 : m_dim
2092 .N7)))))))) {
2093 }
2094
2095 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset2096 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
2097 const ViewOffset<DimRHS, Kokkos::LayoutRight, void>& rhs)
2098 : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
2099 rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
2100 m_stride(rhs.stride_0()) {
2101 static_assert(int(DimRHS::rank) == int(dimension_type::rank),
2102 "ViewOffset assignment requires equal rank");
2103 // Also requires equal static dimensions ...
2104 }
2105
2106 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset2107 KOKKOS_INLINE_FUNCTION ViewOffset(
2108 const ViewOffset<DimRHS, Kokkos::LayoutStride, void>& rhs)
2109 : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
2110 rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
2111 m_stride(rhs.stride_0()) {
2112 if (((dimension_type::rank == 2)
2113 ? rhs.m_stride.S1
2114 : ((dimension_type::rank == 3)
2115 ? rhs.m_stride.S2
2116 : ((dimension_type::rank == 4)
2117 ? rhs.m_stride.S3
2118 : ((dimension_type::rank == 5)
2119 ? rhs.m_stride.S4
2120 : ((dimension_type::rank == 6)
2121 ? rhs.m_stride.S5
2122 : ((dimension_type::rank == 7)
2123 ? rhs.m_stride.S6
2124 : rhs.m_stride.S7)))))) != 1) {
2125 Kokkos::abort(
2126 "Kokkos::Impl::ViewOffset assignment of LayoutRight from "
2127 "LayoutStride requires right-most stride == 1");
2128 }
2129 }
2130
2131 //----------------------------------------
2132 // Subview construction
2133 // Last dimension must be non-zero
2134
2135 template <class DimRHS>
ViewOffsetKokkos::Impl::ViewOffset2136 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
2137 const ViewOffset<DimRHS, Kokkos::LayoutRight, void>& rhs,
2138 const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
2139 : m_dim(sub.range_extent(0), sub.range_extent(1), sub.range_extent(2),
2140 sub.range_extent(3), sub.range_extent(4), sub.range_extent(5),
2141 sub.range_extent(6), sub.range_extent(7)),
2142 m_stride(
2143 0 == sub.range_index(0)
2144 ? rhs.stride_0()
2145 : (1 == sub.range_index(0)
2146 ? rhs.stride_1()
2147 : (2 == sub.range_index(0)
2148 ? rhs.stride_2()
2149 : (3 == sub.range_index(0)
2150 ? rhs.stride_3()
2151 : (4 == sub.range_index(0)
2152 ? rhs.stride_4()
2153 : (5 == sub.range_index(0)
2154 ? rhs.stride_5()
2155 : (6 == sub.range_index(0)
2156 ? rhs.stride_6()
2157 : 0))))))) {
2158 /* // This subview must be 2 == rank and 2 == rank_dynamic
2159 // due to only having stride #0.
2160 // The source dimension #0 must be non-zero for stride-one leading
2161 dimension.
2162 // At most subsequent dimension can be non-zero.
2163
2164 static_assert( (( 2 == dimension_type::rank ) &&
2165 ( 2 <= DimRHS::rank )) ||
2166 ()
2167 , "ViewOffset subview construction requires compatible
2168 rank" );
2169 */
2170 }
2171 };
2172
2173 //----------------------------------------------------------------------------
2174 /* Strided array layout only makes sense for 0 < rank */
2175 /* rank = 0 included for DynRankView case */
2176
2177 template <unsigned Rank>
2178 struct ViewStride;
2179
2180 template <>
2181 struct ViewStride<0> {
2182 enum { S0 = 0, S1 = 0, S2 = 0, S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0 };
2183
2184 ViewStride() = default;
2185 ViewStride(const ViewStride&) = default;
2186 ViewStride& operator=(const ViewStride&) = default;
2187
2188 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2189 constexpr ViewStride(size_t, size_t, size_t, size_t, size_t, size_t, size_t,
2190 size_t) {}
2191 };
2192
2193 template <>
2194 struct ViewStride<1> {
2195 size_t S0;
2196 enum { S1 = 0, S2 = 0, S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0 };
2197
2198 ViewStride() = default;
2199 ViewStride(const ViewStride&) = default;
2200 ViewStride& operator=(const ViewStride&) = default;
2201
2202 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2203 constexpr ViewStride(size_t aS0, size_t, size_t, size_t, size_t, size_t,
2204 size_t, size_t)
2205 : S0(aS0) {}
2206 };
2207
2208 template <>
2209 struct ViewStride<2> {
2210 size_t S0, S1;
2211 enum { S2 = 0, S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0 };
2212
2213 ViewStride() = default;
2214 ViewStride(const ViewStride&) = default;
2215 ViewStride& operator=(const ViewStride&) = default;
2216
2217 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2218 constexpr ViewStride(size_t aS0, size_t aS1, size_t, size_t, size_t, size_t,
2219 size_t, size_t)
2220 : S0(aS0), S1(aS1) {}
2221 };
2222
2223 template <>
2224 struct ViewStride<3> {
2225 size_t S0, S1, S2;
2226 enum { S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0 };
2227
2228 ViewStride() = default;
2229 ViewStride(const ViewStride&) = default;
2230 ViewStride& operator=(const ViewStride&) = default;
2231
2232 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2233 constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t, size_t,
2234 size_t, size_t, size_t)
2235 : S0(aS0), S1(aS1), S2(aS2) {}
2236 };
2237
2238 template <>
2239 struct ViewStride<4> {
2240 size_t S0, S1, S2, S3;
2241 enum { S4 = 0, S5 = 0, S6 = 0, S7 = 0 };
2242
2243 ViewStride() = default;
2244 ViewStride(const ViewStride&) = default;
2245 ViewStride& operator=(const ViewStride&) = default;
2246
2247 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2248 constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3, size_t,
2249 size_t, size_t, size_t)
2250 : S0(aS0), S1(aS1), S2(aS2), S3(aS3) {}
2251 };
2252
2253 template <>
2254 struct ViewStride<5> {
2255 size_t S0, S1, S2, S3, S4;
2256 enum { S5 = 0, S6 = 0, S7 = 0 };
2257
2258 ViewStride() = default;
2259 ViewStride(const ViewStride&) = default;
2260 ViewStride& operator=(const ViewStride&) = default;
2261
2262 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2263 constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3,
2264 size_t aS4, size_t, size_t, size_t)
2265 : S0(aS0), S1(aS1), S2(aS2), S3(aS3), S4(aS4) {}
2266 };
2267
2268 template <>
2269 struct ViewStride<6> {
2270 size_t S0, S1, S2, S3, S4, S5;
2271 enum { S6 = 0, S7 = 0 };
2272
2273 ViewStride() = default;
2274 ViewStride(const ViewStride&) = default;
2275 ViewStride& operator=(const ViewStride&) = default;
2276
2277 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2278 constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3,
2279 size_t aS4, size_t aS5, size_t, size_t)
2280 : S0(aS0), S1(aS1), S2(aS2), S3(aS3), S4(aS4), S5(aS5) {}
2281 };
2282
2283 template <>
2284 struct ViewStride<7> {
2285 size_t S0, S1, S2, S3, S4, S5, S6;
2286 enum { S7 = 0 };
2287
2288 ViewStride() = default;
2289 ViewStride(const ViewStride&) = default;
2290 ViewStride& operator=(const ViewStride&) = default;
2291
2292 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2293 constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3,
2294 size_t aS4, size_t aS5, size_t aS6, size_t)
2295 : S0(aS0), S1(aS1), S2(aS2), S3(aS3), S4(aS4), S5(aS5), S6(aS6) {}
2296 };
2297
2298 template <>
2299 struct ViewStride<8> {
2300 size_t S0, S1, S2, S3, S4, S5, S6, S7;
2301
2302 ViewStride() = default;
2303 ViewStride(const ViewStride&) = default;
2304 ViewStride& operator=(const ViewStride&) = default;
2305
2306 KOKKOS_INLINE_FUNCTION
ViewStrideKokkos::Impl::ViewStride2307 constexpr ViewStride(size_t aS0, size_t aS1, size_t aS2, size_t aS3,
2308 size_t aS4, size_t aS5, size_t aS6, size_t aS7)
2309 : S0(aS0),
2310 S1(aS1),
2311 S2(aS2),
2312 S3(aS3),
2313 S4(aS4),
2314 S5(aS5),
2315 S6(aS6),
2316 S7(aS7) {}
2317 };
2318
2319 template <class Dimension>
2320 struct ViewOffset<Dimension, Kokkos::LayoutStride, void> {
2321 private:
2322 using stride_type = ViewStride<Dimension::rank>;
2323
2324 public:
2325 using is_mapping_plugin = std::true_type;
2326 using is_regular = std::true_type;
2327
2328 using size_type = size_t;
2329 using dimension_type = Dimension;
2330 using array_layout = Kokkos::LayoutStride;
2331
2332 dimension_type m_dim;
2333 stride_type m_stride;
2334
2335 //----------------------------------------
2336
2337 // rank 1
2338 template <typename I0>
operator ()Kokkos::Impl::ViewOffset2339 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0) const {
2340 return i0 * m_stride.S0;
2341 }
2342
2343 // rank 2
2344 template <typename I0, typename I1>
operator ()Kokkos::Impl::ViewOffset2345 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
2346 I1 const& i1) const {
2347 return i0 * m_stride.S0 + i1 * m_stride.S1;
2348 }
2349
2350 // rank 3
2351 template <typename I0, typename I1, typename I2>
operator ()Kokkos::Impl::ViewOffset2352 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
2353 I1 const& i1,
2354 I2 const& i2) const {
2355 return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2;
2356 }
2357
2358 // rank 4
2359 template <typename I0, typename I1, typename I2, typename I3>
operator ()Kokkos::Impl::ViewOffset2360 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
2361 I1 const& i1,
2362 I2 const& i2,
2363 I3 const& i3) const {
2364 return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2365 i3 * m_stride.S3;
2366 }
2367
2368 // rank 5
2369 template <typename I0, typename I1, typename I2, typename I3, typename I4>
operator ()Kokkos::Impl::ViewOffset2370 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(I0 const& i0,
2371 I1 const& i1,
2372 I2 const& i2,
2373 I3 const& i3,
2374 I4 const& i4) const {
2375 return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2376 i3 * m_stride.S3 + i4 * m_stride.S4;
2377 }
2378
2379 // rank 6
2380 template <typename I0, typename I1, typename I2, typename I3, typename I4,
2381 typename I5>
operator ()Kokkos::Impl::ViewOffset2382 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
2383 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
2384 I5 const& i5) const {
2385 return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2386 i3 * m_stride.S3 + i4 * m_stride.S4 + i5 * m_stride.S5;
2387 }
2388
2389 // rank 7
2390 template <typename I0, typename I1, typename I2, typename I3, typename I4,
2391 typename I5, typename I6>
operator ()Kokkos::Impl::ViewOffset2392 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
2393 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
2394 I5 const& i5, I6 const& i6) const {
2395 return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2396 i3 * m_stride.S3 + i4 * m_stride.S4 + i5 * m_stride.S5 +
2397 i6 * m_stride.S6;
2398 }
2399
2400 // rank 8
2401 template <typename I0, typename I1, typename I2, typename I3, typename I4,
2402 typename I5, typename I6, typename I7>
operator ()Kokkos::Impl::ViewOffset2403 KOKKOS_INLINE_FUNCTION constexpr size_type operator()(
2404 I0 const& i0, I1 const& i1, I2 const& i2, I3 const& i3, I4 const& i4,
2405 I5 const& i5, I6 const& i6, I7 const& i7) const {
2406 return i0 * m_stride.S0 + i1 * m_stride.S1 + i2 * m_stride.S2 +
2407 i3 * m_stride.S3 + i4 * m_stride.S4 + i5 * m_stride.S5 +
2408 i6 * m_stride.S6 + i7 * m_stride.S7;
2409 }
2410
2411 //----------------------------------------
2412
2413 KOKKOS_INLINE_FUNCTION
layoutKokkos::Impl::ViewOffset2414 constexpr array_layout layout() const {
2415 return array_layout(m_dim.N0, m_stride.S0, m_dim.N1, m_stride.S1, m_dim.N2,
2416 m_stride.S2, m_dim.N3, m_stride.S3, m_dim.N4,
2417 m_stride.S4, m_dim.N5, m_stride.S5, m_dim.N6,
2418 m_stride.S6, m_dim.N7, m_stride.S7);
2419 }
2420
dimension_0Kokkos::Impl::ViewOffset2421 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const {
2422 return m_dim.N0;
2423 }
dimension_1Kokkos::Impl::ViewOffset2424 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_1() const {
2425 return m_dim.N1;
2426 }
dimension_2Kokkos::Impl::ViewOffset2427 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_2() const {
2428 return m_dim.N2;
2429 }
dimension_3Kokkos::Impl::ViewOffset2430 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_3() const {
2431 return m_dim.N3;
2432 }
dimension_4Kokkos::Impl::ViewOffset2433 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_4() const {
2434 return m_dim.N4;
2435 }
dimension_5Kokkos::Impl::ViewOffset2436 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_5() const {
2437 return m_dim.N5;
2438 }
dimension_6Kokkos::Impl::ViewOffset2439 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_6() const {
2440 return m_dim.N6;
2441 }
dimension_7Kokkos::Impl::ViewOffset2442 KOKKOS_INLINE_FUNCTION constexpr size_type dimension_7() const {
2443 return m_dim.N7;
2444 }
2445
2446 /* Cardinality of the domain index space */
2447 KOKKOS_INLINE_FUNCTION
sizeKokkos::Impl::ViewOffset2448 constexpr size_type size() const {
2449 return m_dim.N0 * m_dim.N1 * m_dim.N2 * m_dim.N3 * m_dim.N4 * m_dim.N5 *
2450 m_dim.N6 * m_dim.N7;
2451 }
2452
2453 private:
2454 KOKKOS_INLINE_FUNCTION
MaxKokkos::Impl::ViewOffset2455 static constexpr size_type Max(size_type lhs, size_type rhs) {
2456 return lhs < rhs ? rhs : lhs;
2457 }
2458
2459 public:
2460 /* Span of the range space, largest stride * dimension */
2461 KOKKOS_INLINE_FUNCTION
spanKokkos::Impl::ViewOffset2462 constexpr size_type span() const {
2463 return size() == size_type(0)
2464 ? size_type(0)
2465 : Max(m_dim.N0 * m_stride.S0,
2466 Max(m_dim.N1 * m_stride.S1,
2467 Max(m_dim.N2 * m_stride.S2,
2468 Max(m_dim.N3 * m_stride.S3,
2469 Max(m_dim.N4 * m_stride.S4,
2470 Max(m_dim.N5 * m_stride.S5,
2471 Max(m_dim.N6 * m_stride.S6,
2472 m_dim.N7 * m_stride.S7)))))));
2473 }
2474
span_is_contiguousKokkos::Impl::ViewOffset2475 KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
2476 return span() == size();
2477 }
2478
2479 /* Strides of dimensions */
stride_0Kokkos::Impl::ViewOffset2480 KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const {
2481 return m_stride.S0;
2482 }
stride_1Kokkos::Impl::ViewOffset2483 KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const {
2484 return m_stride.S1;
2485 }
stride_2Kokkos::Impl::ViewOffset2486 KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const {
2487 return m_stride.S2;
2488 }
stride_3Kokkos::Impl::ViewOffset2489 KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const {
2490 return m_stride.S3;
2491 }
stride_4Kokkos::Impl::ViewOffset2492 KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const {
2493 return m_stride.S4;
2494 }
stride_5Kokkos::Impl::ViewOffset2495 KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const {
2496 return m_stride.S5;
2497 }
stride_6Kokkos::Impl::ViewOffset2498 KOKKOS_INLINE_FUNCTION constexpr size_type stride_6() const {
2499 return m_stride.S6;
2500 }
stride_7Kokkos::Impl::ViewOffset2501 KOKKOS_INLINE_FUNCTION constexpr size_type stride_7() const {
2502 return m_stride.S7;
2503 }
2504
2505 // Stride with [ rank ] value is the total length
2506 template <typename iType>
strideKokkos::Impl::ViewOffset2507 KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
2508 if (0 < dimension_type::rank) {
2509 s[0] = m_stride.S0;
2510 }
2511 if (1 < dimension_type::rank) {
2512 s[1] = m_stride.S1;
2513 }
2514 if (2 < dimension_type::rank) {
2515 s[2] = m_stride.S2;
2516 }
2517 if (3 < dimension_type::rank) {
2518 s[3] = m_stride.S3;
2519 }
2520 if (4 < dimension_type::rank) {
2521 s[4] = m_stride.S4;
2522 }
2523 if (5 < dimension_type::rank) {
2524 s[5] = m_stride.S5;
2525 }
2526 if (6 < dimension_type::rank) {
2527 s[6] = m_stride.S6;
2528 }
2529 if (7 < dimension_type::rank) {
2530 s[7] = m_stride.S7;
2531 }
2532 s[dimension_type::rank] = span();
2533 }
2534
2535 //----------------------------------------
2536 // MSVC (16.5.5) + CUDA (10.2) did not generate the defaulted functions
2537 // correct and errors out during compilation. Same for the other places where
2538 // I changed this.
2539
2540 #ifdef KOKKOS_IMPL_WINDOWS_CUDA
ViewOffsetKokkos::Impl::ViewOffset2541 KOKKOS_FUNCTION ViewOffset()
2542 : m_dim(dimension_type()), m_stride(stride_type()) {}
ViewOffsetKokkos::Impl::ViewOffset2543 KOKKOS_FUNCTION ViewOffset(const ViewOffset& src) {
2544 m_dim = src.m_dim;
2545 m_stride = src.m_stride;
2546 }
operator =Kokkos::Impl::ViewOffset2547 KOKKOS_FUNCTION ViewOffset& operator=(const ViewOffset& src) {
2548 m_dim = src.m_dim;
2549 m_stride = src.m_stride;
2550 return *this;
2551 }
2552 #else
2553
2554 ViewOffset() = default;
2555 ViewOffset(const ViewOffset&) = default;
2556 ViewOffset& operator=(const ViewOffset&) = default;
2557 #endif
2558
2559 KOKKOS_INLINE_FUNCTION
ViewOffsetKokkos::Impl::ViewOffset2560 constexpr ViewOffset(std::integral_constant<unsigned, 0> const&,
2561 Kokkos::LayoutStride const& rhs)
2562 : m_dim(rhs.dimension[0], rhs.dimension[1], rhs.dimension[2],
2563 rhs.dimension[3], rhs.dimension[4], rhs.dimension[5],
2564 rhs.dimension[6], rhs.dimension[7]),
2565 m_stride(rhs.stride[0], rhs.stride[1], rhs.stride[2], rhs.stride[3],
2566 rhs.stride[4], rhs.stride[5], rhs.stride[6], rhs.stride[7]) {}
2567
2568 template <class DimRHS, class LayoutRHS>
ViewOffsetKokkos::Impl::ViewOffset2569 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
2570 const ViewOffset<DimRHS, LayoutRHS, void>& rhs)
2571 : m_dim(rhs.m_dim.N0, rhs.m_dim.N1, rhs.m_dim.N2, rhs.m_dim.N3,
2572 rhs.m_dim.N4, rhs.m_dim.N5, rhs.m_dim.N6, rhs.m_dim.N7),
2573 m_stride(rhs.stride_0(), rhs.stride_1(), rhs.stride_2(), rhs.stride_3(),
2574 rhs.stride_4(), rhs.stride_5(), rhs.stride_6(),
2575 rhs.stride_7()) {
2576 static_assert(int(DimRHS::rank) == int(dimension_type::rank),
2577 "ViewOffset assignment requires equal rank");
2578 // Also requires equal static dimensions ...
2579 }
2580
2581 //----------------------------------------
2582 // Subview construction
2583
2584 private:
2585 template <class DimRHS, class LayoutRHS>
strideKokkos::Impl::ViewOffset2586 KOKKOS_INLINE_FUNCTION static constexpr size_t stride(
2587 unsigned r, const ViewOffset<DimRHS, LayoutRHS, void>& rhs) {
2588 return r > 7
2589 ? 0
2590 : (r == 0
2591 ? rhs.stride_0()
2592 : (r == 1
2593 ? rhs.stride_1()
2594 : (r == 2
2595 ? rhs.stride_2()
2596 : (r == 3
2597 ? rhs.stride_3()
2598 : (r == 4
2599 ? rhs.stride_4()
2600 : (r == 5
2601 ? rhs.stride_5()
2602 : (r == 6
2603 ? rhs.stride_6()
2604 : rhs.stride_7())))))));
2605 }
2606
2607 public:
2608 template <class DimRHS, class LayoutRHS>
ViewOffsetKokkos::Impl::ViewOffset2609 KOKKOS_INLINE_FUNCTION constexpr ViewOffset(
2610 const ViewOffset<DimRHS, LayoutRHS, void>& rhs,
2611 const SubviewExtents<DimRHS::rank, dimension_type::rank>& sub)
2612 // range_extent(r) returns 0 when dimension_type::rank <= r
2613 : m_dim(sub.range_extent(0), sub.range_extent(1), sub.range_extent(2),
2614 sub.range_extent(3), sub.range_extent(4), sub.range_extent(5),
2615 sub.range_extent(6), sub.range_extent(7))
2616 // range_index(r) returns ~0u when dimension_type::rank <= r
2617 ,
2618 m_stride(
2619 stride(sub.range_index(0), rhs), stride(sub.range_index(1), rhs),
2620 stride(sub.range_index(2), rhs), stride(sub.range_index(3), rhs),
2621 stride(sub.range_index(4), rhs), stride(sub.range_index(5), rhs),
2622 stride(sub.range_index(6), rhs), stride(sub.range_index(7), rhs)) {}
2623 };
2624
2625 } // namespace Impl
2626 } // namespace Kokkos
2627
2628 //----------------------------------------------------------------------------
2629 //----------------------------------------------------------------------------
2630
2631 namespace Kokkos {
2632 namespace Impl {
2633
2634 /** \brief ViewDataHandle provides the type of the 'data handle' which the view
2635 * uses to access data with the [] operator. It also provides
2636 * an allocate function and a function to extract a raw ptr from the
2637 * data handle. ViewDataHandle also defines an enum ReferenceAble which
2638 * specifies whether references/pointers to elements can be taken and a
2639 * 'return_type' which is what the view operators will give back.
2640 * Specialisation of this object allows three things depending
2641 * on ViewTraits and compiler options:
2642 * (i) Use special allocator (e.g. huge pages/small pages and pinned
2643 * memory) (ii) Use special data handle type (e.g. add Cuda Texture Object)
2644 * (iii) Use special access intrinsics (e.g. texture fetch and
2645 * non-caching loads)
2646 */
2647 template <class Traits, class Enable = void>
2648 struct ViewDataHandle {
2649 using value_type = typename Traits::value_type;
2650 using handle_type = typename Traits::value_type*;
2651 using return_type = typename Traits::value_type&;
2652 using track_type = Kokkos::Impl::SharedAllocationTracker;
2653
2654 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2655 static handle_type assign(value_type* arg_data_ptr,
2656 track_type const& /*arg_tracker*/) {
2657 return handle_type(arg_data_ptr);
2658 }
2659
2660 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2661 static handle_type assign(handle_type const arg_data_ptr, size_t offset) {
2662 return handle_type(arg_data_ptr + offset);
2663 }
2664 };
2665
2666 template <class Traits>
2667 struct ViewDataHandle<
2668 Traits, typename std::enable_if<(
2669 std::is_same<typename Traits::non_const_value_type,
2670 typename Traits::value_type>::value &&
2671 std::is_same<typename Traits::specialize, void>::value &&
2672 Traits::memory_traits::is_atomic)>::type> {
2673 using value_type = typename Traits::value_type;
2674 using handle_type = typename Kokkos::Impl::AtomicViewDataHandle<Traits>;
2675 using return_type = typename Kokkos::Impl::AtomicDataElement<Traits>;
2676 using track_type = Kokkos::Impl::SharedAllocationTracker;
2677
2678 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2679 static handle_type assign(value_type* arg_data_ptr,
2680 track_type const& /*arg_tracker*/) {
2681 return handle_type(arg_data_ptr);
2682 }
2683
2684 template <class SrcHandleType>
assignKokkos::Impl::ViewDataHandle2685 KOKKOS_INLINE_FUNCTION static handle_type assign(
2686 const SrcHandleType& arg_handle, size_t offset) {
2687 return handle_type(arg_handle + offset);
2688 }
2689 };
2690
2691 template <class Traits>
2692 struct ViewDataHandle<
2693 Traits, typename std::enable_if<(
2694 std::is_same<typename Traits::specialize, void>::value &&
2695 (!Traits::memory_traits::is_aligned) &&
2696 Traits::memory_traits::is_restrict
2697 #ifdef KOKKOS_ENABLE_CUDA
2698 && (!(std::is_same<typename Traits::memory_space,
2699 Kokkos::CudaSpace>::value ||
2700 std::is_same<typename Traits::memory_space,
2701 Kokkos::CudaUVMSpace>::value))
2702 #endif
2703 && (!Traits::memory_traits::is_atomic))>::type> {
2704 using value_type = typename Traits::value_type;
2705 using handle_type = typename Traits::value_type* KOKKOS_RESTRICT;
2706 using return_type = typename Traits::value_type& KOKKOS_RESTRICT;
2707 using track_type = Kokkos::Impl::SharedAllocationTracker;
2708
2709 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2710 static value_type* assign(value_type* arg_data_ptr,
2711 track_type const& /*arg_tracker*/) {
2712 return (value_type*)(arg_data_ptr);
2713 }
2714
2715 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2716 static value_type* assign(handle_type const arg_data_ptr, size_t offset) {
2717 return (value_type*)(arg_data_ptr + offset);
2718 }
2719 };
2720
2721 template <class Traits>
2722 struct ViewDataHandle<
2723 Traits, typename std::enable_if<(
2724 std::is_same<typename Traits::specialize, void>::value &&
2725 Traits::memory_traits::is_aligned &&
2726 (!Traits::memory_traits::is_restrict)
2727 #ifdef KOKKOS_ENABLE_CUDA
2728 && (!(std::is_same<typename Traits::memory_space,
2729 Kokkos::CudaSpace>::value ||
2730 std::is_same<typename Traits::memory_space,
2731 Kokkos::CudaUVMSpace>::value))
2732 #endif
2733 && (!Traits::memory_traits::is_atomic))>::type> {
2734 using value_type = typename Traits::value_type;
2735 // typedef work-around for intel compilers error #3186: expected typedef
2736 // declaration
2737 // NOLINTNEXTLINE(modernize-use-using)
2738 typedef value_type* KOKKOS_IMPL_ALIGN_PTR(KOKKOS_MEMORY_ALIGNMENT)
2739 handle_type;
2740 using return_type = typename Traits::value_type&;
2741 using track_type = Kokkos::Impl::SharedAllocationTracker;
2742
2743 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2744 static handle_type assign(value_type* arg_data_ptr,
2745 track_type const& /*arg_tracker*/) {
2746 if (reinterpret_cast<uintptr_t>(arg_data_ptr) % Impl::MEMORY_ALIGNMENT) {
2747 Kokkos::abort(
2748 "Assigning NonAligned View or Pointer to Kokkos::View with Aligned "
2749 "attribute");
2750 }
2751 return handle_type(arg_data_ptr);
2752 }
2753
2754 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2755 static handle_type assign(handle_type const arg_data_ptr, size_t offset) {
2756 if (reinterpret_cast<uintptr_t>(arg_data_ptr + offset) %
2757 Impl::MEMORY_ALIGNMENT) {
2758 Kokkos::abort(
2759 "Assigning NonAligned View or Pointer to Kokkos::View with Aligned "
2760 "attribute");
2761 }
2762 return handle_type(arg_data_ptr + offset);
2763 }
2764 };
2765
2766 template <class Traits>
2767 struct ViewDataHandle<
2768 Traits,
2769 typename std::enable_if<(
2770 std::is_same<typename Traits::specialize, void>::value &&
2771 Traits::memory_traits::is_aligned && Traits::memory_traits::is_restrict
2772 #ifdef KOKKOS_ENABLE_CUDA
2773 && (!(std::is_same<typename Traits::memory_space,
2774 Kokkos::CudaSpace>::value ||
2775 std::is_same<typename Traits::memory_space,
2776 Kokkos::CudaUVMSpace>::value))
2777 #endif
2778 && (!Traits::memory_traits::is_atomic))>::type> {
2779 using value_type = typename Traits::value_type;
2780 // typedef work-around for intel compilers error #3186: expected typedef
2781 // declaration
2782 // NOLINTNEXTLINE(modernize-use-using)
2783 typedef value_type* KOKKOS_IMPL_ALIGN_PTR(KOKKOS_MEMORY_ALIGNMENT)
2784 handle_type;
2785 using return_type = typename Traits::value_type& KOKKOS_RESTRICT;
2786 using track_type = Kokkos::Impl::SharedAllocationTracker;
2787
2788 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2789 static value_type* assign(value_type* arg_data_ptr,
2790 track_type const& /*arg_tracker*/) {
2791 if (reinterpret_cast<uintptr_t>(arg_data_ptr) % Impl::MEMORY_ALIGNMENT) {
2792 Kokkos::abort(
2793 "Assigning NonAligned View or Pointer to Kokkos::View with Aligned "
2794 "attribute");
2795 }
2796 return (value_type*)(arg_data_ptr);
2797 }
2798
2799 KOKKOS_INLINE_FUNCTION
assignKokkos::Impl::ViewDataHandle2800 static value_type* assign(handle_type const arg_data_ptr, size_t offset) {
2801 if (reinterpret_cast<uintptr_t>(arg_data_ptr + offset) %
2802 Impl::MEMORY_ALIGNMENT) {
2803 Kokkos::abort(
2804 "Assigning NonAligned View or Pointer to Kokkos::View with Aligned "
2805 "attribute");
2806 }
2807 return (value_type*)(arg_data_ptr + offset);
2808 }
2809 };
2810 } // namespace Impl
2811 } // namespace Kokkos
2812
2813 //----------------------------------------------------------------------------
2814 //----------------------------------------------------------------------------
2815
2816 namespace Kokkos {
2817 namespace Impl {
2818
2819 //----------------------------------------------------------------------------
2820
2821 /*
2822 * The construction, assignment to default, and destruction
2823 * are merged into a single functor.
2824 * Primarily to work around an unresolved CUDA back-end bug
2825 * that would lose the destruction cuda device function when
2826 * called from the shared memory tracking destruction.
2827 * Secondarily to have two fewer partial specializations.
2828 */
2829 template <class ExecSpace, class ValueType,
2830 bool IsScalar = std::is_scalar<ValueType>::value>
2831 struct ViewValueFunctor;
2832
2833 template <class ExecSpace, class ValueType>
2834 struct ViewValueFunctor<ExecSpace, ValueType, false /* is_scalar */> {
2835 using PolicyType = Kokkos::RangePolicy<ExecSpace, Kokkos::IndexType<int64_t>>;
2836 using Exec = typename ExecSpace::execution_space;
2837
2838 Exec space;
2839 ValueType* ptr;
2840 size_t n;
2841 bool destroy;
2842 std::string name;
2843
2844 KOKKOS_INLINE_FUNCTION
operator ()Kokkos::Impl::ViewValueFunctor2845 void operator()(const size_t i) const {
2846 if (destroy) {
2847 (ptr + i)->~ValueType();
2848 } // KOKKOS_IMPL_CUDA_CLANG_WORKAROUND this line causes ptax error
2849 // __cxa_begin_catch in nested_view unit-test
2850 else {
2851 new (ptr + i) ValueType();
2852 }
2853 }
2854
2855 ViewValueFunctor() = default;
2856 ViewValueFunctor(const ViewValueFunctor&) = default;
2857 ViewValueFunctor& operator=(const ViewValueFunctor&) = default;
2858
ViewValueFunctorKokkos::Impl::ViewValueFunctor2859 ViewValueFunctor(ExecSpace const& arg_space, ValueType* const arg_ptr,
2860 size_t const arg_n, std::string arg_name)
2861 : space(arg_space),
2862 ptr(arg_ptr),
2863 n(arg_n),
2864 destroy(false),
2865 name(std::move(arg_name)) {}
2866
executeKokkos::Impl::ViewValueFunctor2867 void execute(bool arg) {
2868 destroy = arg;
2869 PolicyType policy(0, n);
2870 std::string functor_name;
2871 if (!space.in_parallel()) {
2872 uint64_t kpID = 0;
2873 if (Kokkos::Profiling::profileLibraryLoaded()) {
2874 functor_name =
2875 (destroy ? "Kokkos::View::destruction [" + name + "]"
2876 : "Kokkos::View::initialization [" + name + "]");
2877 Kokkos::Tools::Impl::begin_parallel_for(policy, *this, functor_name,
2878 kpID);
2879 }
2880 #ifdef KOKKOS_ENABLE_CUDA
2881 if (std::is_same<ExecSpace, Kokkos::Cuda>::value) {
2882 Kokkos::Impl::cuda_prefetch_pointer(space, ptr, sizeof(ValueType) * n,
2883 true);
2884 }
2885 #endif
2886 const Kokkos::Impl::ParallelFor<ViewValueFunctor, PolicyType> closure(
2887 *this, policy);
2888 closure.execute();
2889 space.fence();
2890 if (Kokkos::Profiling::profileLibraryLoaded()) {
2891 Kokkos::Tools::Impl::end_parallel_for(policy, *this, functor_name,
2892 kpID);
2893 }
2894 } else {
2895 for (size_t i = 0; i < n; ++i) operator()(i);
2896 }
2897 }
2898
construct_shared_allocationKokkos::Impl::ViewValueFunctor2899 void construct_shared_allocation() { execute(false); }
2900
destroy_shared_allocationKokkos::Impl::ViewValueFunctor2901 void destroy_shared_allocation() { execute(true); }
2902 };
2903
2904 template <class ExecSpace, class ValueType>
2905 struct ViewValueFunctor<ExecSpace, ValueType, true /* is_scalar */> {
2906 using PolicyType = Kokkos::RangePolicy<ExecSpace, Kokkos::IndexType<int64_t>>;
2907
2908 ExecSpace space;
2909 ValueType* ptr;
2910 size_t n;
2911 std::string name;
2912
2913 KOKKOS_INLINE_FUNCTION
operator ()Kokkos::Impl::ViewValueFunctor2914 void operator()(const size_t i) const { ptr[i] = ValueType(); }
2915
2916 ViewValueFunctor() = default;
2917 ViewValueFunctor(const ViewValueFunctor&) = default;
2918 ViewValueFunctor& operator=(const ViewValueFunctor&) = default;
2919
ViewValueFunctorKokkos::Impl::ViewValueFunctor2920 ViewValueFunctor(ExecSpace const& arg_space, ValueType* const arg_ptr,
2921 size_t const arg_n, std::string arg_name)
2922 : space(arg_space), ptr(arg_ptr), n(arg_n), name(std::move(arg_name)) {}
2923
construct_shared_allocationKokkos::Impl::ViewValueFunctor2924 void construct_shared_allocation() {
2925 if (!space.in_parallel()) {
2926 uint64_t kpID = 0;
2927 if (Kokkos::Profiling::profileLibraryLoaded()) {
2928 Kokkos::Profiling::beginParallelFor(
2929 "Kokkos::View::initialization [" + name + "]", 0, &kpID);
2930 }
2931 #ifdef KOKKOS_ENABLE_CUDA
2932 if (std::is_same<ExecSpace, Kokkos::Cuda>::value) {
2933 Kokkos::Impl::cuda_prefetch_pointer(space, ptr, sizeof(ValueType) * n,
2934 true);
2935 }
2936 #endif
2937 const Kokkos::Impl::ParallelFor<ViewValueFunctor, PolicyType> closure(
2938 *this, PolicyType(0, n));
2939 closure.execute();
2940 space.fence();
2941 if (Kokkos::Profiling::profileLibraryLoaded()) {
2942 Kokkos::Profiling::endParallelFor(kpID);
2943 }
2944 } else {
2945 for (size_t i = 0; i < n; ++i) operator()(i);
2946 }
2947 }
2948
destroy_shared_allocationKokkos::Impl::ViewValueFunctor2949 void destroy_shared_allocation() {}
2950 };
2951
2952 //----------------------------------------------------------------------------
2953 /** \brief View mapping for non-specialized data type and standard layout */
2954 template <class Traits>
2955 class ViewMapping<
2956 Traits,
2957 typename std::enable_if<(
2958 std::is_same<typename Traits::specialize, void>::value &&
2959 ViewOffset<typename Traits::dimension, typename Traits::array_layout,
2960 void>::is_mapping_plugin::value)>::type> {
2961 public:
2962 using offset_type = ViewOffset<typename Traits::dimension,
2963 typename Traits::array_layout, void>;
2964
2965 using handle_type = typename ViewDataHandle<Traits>::handle_type;
2966
2967 handle_type m_impl_handle;
2968 offset_type m_impl_offset;
2969
2970 private:
2971 template <class, class...>
2972 friend class ViewMapping;
2973
2974 KOKKOS_INLINE_FUNCTION
ViewMapping(const handle_type & arg_handle,const offset_type & arg_offset)2975 ViewMapping(const handle_type& arg_handle, const offset_type& arg_offset)
2976 : m_impl_handle(arg_handle), m_impl_offset(arg_offset) {}
2977
2978 public:
2979 using printable_label_typedef = void;
2980 enum { is_managed = Traits::is_managed };
2981
2982 //----------------------------------------
2983 // Domain dimensions
2984
2985 enum { Rank = Traits::dimension::rank };
2986
2987 template <typename iType>
extent(const iType & r) const2988 KOKKOS_INLINE_FUNCTION constexpr size_t extent(const iType& r) const {
2989 return m_impl_offset.m_dim.extent(r);
2990 }
2991
static_extent(const unsigned r)2992 static KOKKOS_INLINE_FUNCTION constexpr size_t static_extent(
2993 const unsigned r) noexcept {
2994 using dim_type = typename offset_type::dimension_type;
2995 return dim_type::static_extent(r);
2996 }
2997
layout() const2998 KOKKOS_INLINE_FUNCTION constexpr typename Traits::array_layout layout()
2999 const {
3000 return m_impl_offset.layout();
3001 }
3002
dimension_0() const3003 KOKKOS_INLINE_FUNCTION constexpr size_t dimension_0() const {
3004 return m_impl_offset.dimension_0();
3005 }
dimension_1() const3006 KOKKOS_INLINE_FUNCTION constexpr size_t dimension_1() const {
3007 return m_impl_offset.dimension_1();
3008 }
dimension_2() const3009 KOKKOS_INLINE_FUNCTION constexpr size_t dimension_2() const {
3010 return m_impl_offset.dimension_2();
3011 }
dimension_3() const3012 KOKKOS_INLINE_FUNCTION constexpr size_t dimension_3() const {
3013 return m_impl_offset.dimension_3();
3014 }
dimension_4() const3015 KOKKOS_INLINE_FUNCTION constexpr size_t dimension_4() const {
3016 return m_impl_offset.dimension_4();
3017 }
dimension_5() const3018 KOKKOS_INLINE_FUNCTION constexpr size_t dimension_5() const {
3019 return m_impl_offset.dimension_5();
3020 }
dimension_6() const3021 KOKKOS_INLINE_FUNCTION constexpr size_t dimension_6() const {
3022 return m_impl_offset.dimension_6();
3023 }
dimension_7() const3024 KOKKOS_INLINE_FUNCTION constexpr size_t dimension_7() const {
3025 return m_impl_offset.dimension_7();
3026 }
3027
3028 // Is a regular layout with uniform striding for each index.
3029 using is_regular = typename offset_type::is_regular;
3030
stride_0() const3031 KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const {
3032 return m_impl_offset.stride_0();
3033 }
stride_1() const3034 KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const {
3035 return m_impl_offset.stride_1();
3036 }
stride_2() const3037 KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const {
3038 return m_impl_offset.stride_2();
3039 }
stride_3() const3040 KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const {
3041 return m_impl_offset.stride_3();
3042 }
stride_4() const3043 KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const {
3044 return m_impl_offset.stride_4();
3045 }
stride_5() const3046 KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const {
3047 return m_impl_offset.stride_5();
3048 }
stride_6() const3049 KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const {
3050 return m_impl_offset.stride_6();
3051 }
stride_7() const3052 KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const {
3053 return m_impl_offset.stride_7();
3054 }
3055
3056 template <typename iType>
stride(iType * const s) const3057 KOKKOS_INLINE_FUNCTION void stride(iType* const s) const {
3058 m_impl_offset.stride(s);
3059 }
3060
3061 //----------------------------------------
3062 // Range span
3063
3064 /** \brief Span of the mapped range */
span() const3065 KOKKOS_INLINE_FUNCTION constexpr size_t span() const {
3066 return m_impl_offset.span();
3067 }
3068
3069 /** \brief Is the mapped range span contiguous */
span_is_contiguous() const3070 KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const {
3071 return m_impl_offset.span_is_contiguous();
3072 }
3073
3074 using reference_type = typename ViewDataHandle<Traits>::return_type;
3075 using pointer_type = typename Traits::value_type*;
3076
3077 /** \brief Query raw pointer to memory */
data() const3078 KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const {
3079 return m_impl_handle;
3080 }
3081
3082 //----------------------------------------
3083 // The View class performs all rank and bounds checking before
3084 // calling these element reference methods.
3085
3086 KOKKOS_FORCEINLINE_FUNCTION
reference() const3087 reference_type reference() const { return m_impl_handle[0]; }
3088
3089 template <typename I0>
3090 KOKKOS_FORCEINLINE_FUNCTION
3091 typename std::enable_if<(std::is_integral<I0>::value &&
3092 // if layout is neither stride nor irregular,
3093 // then just use the handle directly
3094 !(std::is_same<typename Traits::array_layout,
3095 Kokkos::LayoutStride>::value ||
3096 !is_regular::value)),
3097 reference_type>::type
reference(const I0 & i0) const3098 reference(const I0& i0) const {
3099 return m_impl_handle[i0];
3100 }
3101
3102 template <typename I0>
3103 KOKKOS_FORCEINLINE_FUNCTION
3104 typename std::enable_if<(std::is_integral<I0>::value &&
3105 // if the layout is strided or irregular, then
3106 // we have to use the offset
3107 (std::is_same<typename Traits::array_layout,
3108 Kokkos::LayoutStride>::value ||
3109 !is_regular::value)),
3110 reference_type>::type
reference(const I0 & i0) const3111 reference(const I0& i0) const {
3112 return m_impl_handle[m_impl_offset(i0)];
3113 }
3114
3115 template <typename I0, typename I1>
reference(const I0 & i0,const I1 & i1) const3116 KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0& i0,
3117 const I1& i1) const {
3118 return m_impl_handle[m_impl_offset(i0, i1)];
3119 }
3120
3121 template <typename I0, typename I1, typename I2>
reference(const I0 & i0,const I1 & i1,const I2 & i2) const3122 KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0& i0,
3123 const I1& i1,
3124 const I2& i2) const {
3125 return m_impl_handle[m_impl_offset(i0, i1, i2)];
3126 }
3127
3128 template <typename I0, typename I1, typename I2, typename I3>
3129 KOKKOS_FORCEINLINE_FUNCTION reference_type
reference(const I0 & i0,const I1 & i1,const I2 & i2,const I3 & i3) const3130 reference(const I0& i0, const I1& i1, const I2& i2, const I3& i3) const {
3131 return m_impl_handle[m_impl_offset(i0, i1, i2, i3)];
3132 }
3133
3134 template <typename I0, typename I1, typename I2, typename I3, typename I4>
reference(const I0 & i0,const I1 & i1,const I2 & i2,const I3 & i3,const I4 & i4) const3135 KOKKOS_FORCEINLINE_FUNCTION reference_type reference(const I0& i0,
3136 const I1& i1,
3137 const I2& i2,
3138 const I3& i3,
3139 const I4& i4) const {
3140 return m_impl_handle[m_impl_offset(i0, i1, i2, i3, i4)];
3141 }
3142
3143 template <typename I0, typename I1, typename I2, typename I3, typename I4,
3144 typename I5>
3145 KOKKOS_FORCEINLINE_FUNCTION reference_type
reference(const I0 & i0,const I1 & i1,const I2 & i2,const I3 & i3,const I4 & i4,const I5 & i5) const3146 reference(const I0& i0, const I1& i1, const I2& i2, const I3& i3,
3147 const I4& i4, const I5& i5) const {
3148 return m_impl_handle[m_impl_offset(i0, i1, i2, i3, i4, i5)];
3149 }
3150
3151 template <typename I0, typename I1, typename I2, typename I3, typename I4,
3152 typename I5, typename I6>
3153 KOKKOS_FORCEINLINE_FUNCTION reference_type
reference(const I0 & i0,const I1 & i1,const I2 & i2,const I3 & i3,const I4 & i4,const I5 & i5,const I6 & i6) const3154 reference(const I0& i0, const I1& i1, const I2& i2, const I3& i3,
3155 const I4& i4, const I5& i5, const I6& i6) const {
3156 return m_impl_handle[m_impl_offset(i0, i1, i2, i3, i4, i5, i6)];
3157 }
3158
3159 template <typename I0, typename I1, typename I2, typename I3, typename I4,
3160 typename I5, typename I6, typename I7>
3161 KOKKOS_FORCEINLINE_FUNCTION reference_type
reference(const I0 & i0,const I1 & i1,const I2 & i2,const I3 & i3,const I4 & i4,const I5 & i5,const I6 & i6,const I7 & i7) const3162 reference(const I0& i0, const I1& i1, const I2& i2, const I3& i3,
3163 const I4& i4, const I5& i5, const I6& i6, const I7& i7) const {
3164 return m_impl_handle[m_impl_offset(i0, i1, i2, i3, i4, i5, i6, i7)];
3165 }
3166
3167 //----------------------------------------
3168
3169 private:
3170 enum { MemorySpanMask = 8 - 1 /* Force alignment on 8 byte boundary */ };
3171 enum { MemorySpanSize = sizeof(typename Traits::value_type) };
3172
3173 public:
3174 /** \brief Span, in bytes, of the referenced memory */
memory_span() const3175 KOKKOS_INLINE_FUNCTION constexpr size_t memory_span() const {
3176 return (m_impl_offset.span() * sizeof(typename Traits::value_type) +
3177 MemorySpanMask) &
3178 ~size_t(MemorySpanMask);
3179 }
3180
3181 //----------------------------------------
3182
3183 KOKKOS_DEFAULTED_FUNCTION ~ViewMapping() = default;
ViewMapping()3184 KOKKOS_INLINE_FUNCTION ViewMapping() : m_impl_handle(), m_impl_offset() {}
3185
3186 KOKKOS_DEFAULTED_FUNCTION ViewMapping(const ViewMapping&) = default;
3187 KOKKOS_DEFAULTED_FUNCTION ViewMapping& operator=(const ViewMapping&) =
3188 default;
3189
3190 KOKKOS_DEFAULTED_FUNCTION ViewMapping(ViewMapping&&) = default;
3191 KOKKOS_DEFAULTED_FUNCTION ViewMapping& operator=(ViewMapping&&) = default;
3192
3193 //----------------------------------------
3194
3195 /**\brief Span, in bytes, of the required memory */
3196 KOKKOS_INLINE_FUNCTION
memory_span(typename Traits::array_layout const & arg_layout)3197 static constexpr size_t memory_span(
3198 typename Traits::array_layout const& arg_layout) {
3199 using padding = std::integral_constant<unsigned int, 0>;
3200 return (offset_type(padding(), arg_layout).span() * MemorySpanSize +
3201 MemorySpanMask) &
3202 ~size_t(MemorySpanMask);
3203 }
3204
3205 /**\brief Wrap a span of memory */
3206 template <class... P>
ViewMapping(Kokkos::Impl::ViewCtorProp<P...> const & arg_prop,typename Traits::array_layout const & arg_layout)3207 KOKKOS_INLINE_FUNCTION ViewMapping(
3208 Kokkos::Impl::ViewCtorProp<P...> const& arg_prop,
3209 typename Traits::array_layout const& arg_layout)
3210 : m_impl_handle(
3211 ((Kokkos::Impl::ViewCtorProp<void, pointer_type> const&)arg_prop)
3212 .value),
3213 m_impl_offset(std::integral_constant<unsigned, 0>(), arg_layout) {}
3214
3215 /**\brief Assign data */
3216 KOKKOS_INLINE_FUNCTION
assign_data(pointer_type arg_ptr)3217 void assign_data(pointer_type arg_ptr) {
3218 m_impl_handle = handle_type(arg_ptr);
3219 }
3220
3221 //----------------------------------------
3222 /* Allocate and construct mapped array.
3223 * Allocate via shared allocation record and
3224 * return that record for allocation tracking.
3225 */
3226 template <class... P>
allocate_shared(Kokkos::Impl::ViewCtorProp<P...> const & arg_prop,typename Traits::array_layout const & arg_layout)3227 Kokkos::Impl::SharedAllocationRecord<>* allocate_shared(
3228 Kokkos::Impl::ViewCtorProp<P...> const& arg_prop,
3229 typename Traits::array_layout const& arg_layout) {
3230 using alloc_prop = Kokkos::Impl::ViewCtorProp<P...>;
3231
3232 using execution_space = typename alloc_prop::execution_space;
3233 using memory_space = typename Traits::memory_space;
3234 using value_type = typename Traits::value_type;
3235 using functor_type = ViewValueFunctor<execution_space, value_type>;
3236 using record_type =
3237 Kokkos::Impl::SharedAllocationRecord<memory_space, functor_type>;
3238
3239 // Query the mapping for byte-size of allocation.
3240 // If padding is allowed then pass in sizeof value type
3241 // for padding computation.
3242 using padding = std::integral_constant<
3243 unsigned int, alloc_prop::allow_padding ? sizeof(value_type) : 0>;
3244
3245 m_impl_offset = offset_type(padding(), arg_layout);
3246
3247 const size_t alloc_size =
3248 (m_impl_offset.span() * MemorySpanSize + MemorySpanMask) &
3249 ~size_t(MemorySpanMask);
3250 const std::string& alloc_name =
3251 static_cast<Kokkos::Impl::ViewCtorProp<void, std::string> const&>(
3252 arg_prop)
3253 .value;
3254 // Create shared memory tracking record with allocate memory from the memory
3255 // space
3256 record_type* const record = record_type::allocate(
3257 static_cast<Kokkos::Impl::ViewCtorProp<void, memory_space> const&>(
3258 arg_prop)
3259 .value,
3260 alloc_name, alloc_size);
3261
3262 m_impl_handle = handle_type(reinterpret_cast<pointer_type>(record->data()));
3263
3264 // Only initialize if the allocation is non-zero.
3265 // May be zero if one of the dimensions is zero.
3266 if (alloc_size && alloc_prop::initialize) {
3267 // Assume destruction is only required when construction is requested.
3268 // The ViewValueFunctor has both value construction and destruction
3269 // operators.
3270 record->m_destroy = functor_type(
3271 static_cast<Kokkos::Impl::ViewCtorProp<void, execution_space> const&>(
3272 arg_prop)
3273 .value,
3274 (value_type*)m_impl_handle, m_impl_offset.span(), alloc_name);
3275
3276 // Construct values
3277 record->m_destroy.construct_shared_allocation();
3278 }
3279
3280 return record;
3281 }
3282 };
3283
3284 //----------------------------------------------------------------------------
3285 //----------------------------------------------------------------------------
3286 /** \brief Assign compatible default mappings */
3287
3288 template <class DstTraits, class SrcTraits>
3289 class ViewMapping<
3290 DstTraits, SrcTraits,
3291 typename std::enable_if<(
3292 !(std::is_same<typename SrcTraits::array_layout, LayoutStride>::
3293 value) && // Added to have a new specialization for SrcType of
3294 // LayoutStride
3295 // default mappings
3296 std::is_same<typename DstTraits::specialize, void>::value &&
3297 std::is_same<typename SrcTraits::specialize, void>::value &&
3298 (
3299 // same layout
3300 std::is_same<typename DstTraits::array_layout,
3301 typename SrcTraits::array_layout>::value ||
3302 // known layout
3303 ((std::is_same<typename DstTraits::array_layout,
3304 Kokkos::LayoutLeft>::value ||
3305 std::is_same<typename DstTraits::array_layout,
3306 Kokkos::LayoutRight>::value ||
3307 std::is_same<typename DstTraits::array_layout,
3308 Kokkos::LayoutStride>::value) &&
3309 (std::is_same<typename SrcTraits::array_layout,
3310 Kokkos::LayoutLeft>::value ||
3311 std::is_same<typename SrcTraits::array_layout,
3312 Kokkos::LayoutRight>::value ||
3313 std::is_same<typename SrcTraits::array_layout,
3314 Kokkos::LayoutStride>::value))))>::type> {
3315 private:
3316 enum {
3317 is_assignable_space =
3318 #if 1
3319 Kokkos::Impl::MemorySpaceAccess<
3320 typename DstTraits::memory_space,
3321 typename SrcTraits::memory_space>::assignable
3322 };
3323 #else
3324 std::is_same<typename DstTraits::memory_space,
3325 typename SrcTraits::memory_space>::value
3326 };
3327 #endif
3328
3329 enum {
3330 is_assignable_value_type =
3331 std::is_same<typename DstTraits::value_type,
3332 typename SrcTraits::value_type>::value ||
3333 std::is_same<typename DstTraits::value_type,
3334 typename SrcTraits::const_value_type>::value
3335 };
3336
3337 enum {
3338 is_assignable_dimension =
3339 ViewDimensionAssignable<typename DstTraits::dimension,
3340 typename SrcTraits::dimension>::value
3341 };
3342
3343 enum {
3344 is_assignable_layout =
3345 std::is_same<typename DstTraits::array_layout,
3346 typename SrcTraits::array_layout>::value ||
3347 std::is_same<typename DstTraits::array_layout,
3348 Kokkos::LayoutStride>::value ||
3349 (DstTraits::dimension::rank == 0) ||
3350 (DstTraits::dimension::rank == 1 &&
3351 DstTraits::dimension::rank_dynamic == 1)
3352 };
3353
3354 public:
3355 enum {
3356 is_assignable_data_type =
3357 is_assignable_value_type && is_assignable_dimension
3358 };
3359 enum {
3360 is_assignable = is_assignable_space && is_assignable_value_type &&
3361 is_assignable_dimension && is_assignable_layout
3362 };
3363
3364 using TrackType = Kokkos::Impl::SharedAllocationTracker;
3365 using DstType = ViewMapping<DstTraits, void>;
3366 using SrcType = ViewMapping<SrcTraits, void>;
3367
3368 KOKKOS_INLINE_FUNCTION
assign(DstType & dst,const SrcType & src,const TrackType & src_track)3369 static void assign(DstType& dst, const SrcType& src,
3370 const TrackType& src_track) {
3371 static_assert(is_assignable_space,
3372 "View assignment must have compatible spaces");
3373
3374 static_assert(
3375 is_assignable_value_type,
3376 "View assignment must have same value type or const = non-const");
3377
3378 static_assert(is_assignable_dimension,
3379 "View assignment must have compatible dimensions");
3380
3381 static_assert(
3382 is_assignable_layout,
3383 "View assignment must have compatible layout or have rank <= 1");
3384
3385 using dst_offset_type = typename DstType::offset_type;
3386
3387 if (size_t(DstTraits::dimension::rank_dynamic) <
3388 size_t(SrcTraits::dimension::rank_dynamic)) {
3389 using dst_dim = typename DstTraits::dimension;
3390 bool assignable = ((1 > DstTraits::dimension::rank_dynamic &&
3391 1 <= SrcTraits::dimension::rank_dynamic)
3392 ? dst_dim::ArgN0 == src.dimension_0()
3393 : true) &&
3394 ((2 > DstTraits::dimension::rank_dynamic &&
3395 2 <= SrcTraits::dimension::rank_dynamic)
3396 ? dst_dim::ArgN1 == src.dimension_1()
3397 : true) &&
3398 ((3 > DstTraits::dimension::rank_dynamic &&
3399 3 <= SrcTraits::dimension::rank_dynamic)
3400 ? dst_dim::ArgN2 == src.dimension_2()
3401 : true) &&
3402 ((4 > DstTraits::dimension::rank_dynamic &&
3403 4 <= SrcTraits::dimension::rank_dynamic)
3404 ? dst_dim::ArgN3 == src.dimension_3()
3405 : true) &&
3406 ((5 > DstTraits::dimension::rank_dynamic &&
3407 5 <= SrcTraits::dimension::rank_dynamic)
3408 ? dst_dim::ArgN4 == src.dimension_4()
3409 : true) &&
3410 ((6 > DstTraits::dimension::rank_dynamic &&
3411 6 <= SrcTraits::dimension::rank_dynamic)
3412 ? dst_dim::ArgN5 == src.dimension_5()
3413 : true) &&
3414 ((7 > DstTraits::dimension::rank_dynamic &&
3415 7 <= SrcTraits::dimension::rank_dynamic)
3416 ? dst_dim::ArgN6 == src.dimension_6()
3417 : true) &&
3418 ((8 > DstTraits::dimension::rank_dynamic &&
3419 8 <= SrcTraits::dimension::rank_dynamic)
3420 ? dst_dim::ArgN7 == src.dimension_7()
3421 : true);
3422 if (!assignable)
3423 Kokkos::abort(
3424 "View Assignment: trying to assign runtime dimension to non "
3425 "matching compile time dimension.");
3426 }
3427 dst.m_impl_offset = dst_offset_type(src.m_impl_offset);
3428 dst.m_impl_handle = Kokkos::Impl::ViewDataHandle<DstTraits>::assign(
3429 src.m_impl_handle, src_track);
3430 }
3431 };
3432
3433 //----------------------------------------------------------------------------
3434 // Create new specialization for SrcType of LayoutStride. Runtime check for
3435 // compatible layout
3436 template <class DstTraits, class SrcTraits>
3437 class ViewMapping<
3438 DstTraits, SrcTraits,
3439 typename std::enable_if<(
3440 std::is_same<typename SrcTraits::array_layout,
3441 Kokkos::LayoutStride>::value &&
3442 std::is_same<typename DstTraits::specialize, void>::value &&
3443 std::is_same<typename SrcTraits::specialize, void>::value &&
3444 (
3445 // same layout
3446 std::is_same<typename DstTraits::array_layout,
3447 typename SrcTraits::array_layout>::value ||
3448 // known layout
3449 (std::is_same<typename DstTraits::array_layout,
3450 Kokkos::LayoutLeft>::value ||
3451 std::is_same<typename DstTraits::array_layout,
3452 Kokkos::LayoutRight>::value ||
3453 std::is_same<typename DstTraits::array_layout,
3454 Kokkos::LayoutStride>::value)))>::type> {
3455 private:
3456 enum {
3457 is_assignable_space = Kokkos::Impl::MemorySpaceAccess<
3458 typename DstTraits::memory_space,
3459 typename SrcTraits::memory_space>::assignable
3460 };
3461
3462 enum {
3463 is_assignable_value_type =
3464 std::is_same<typename DstTraits::value_type,
3465 typename SrcTraits::value_type>::value ||
3466 std::is_same<typename DstTraits::value_type,
3467 typename SrcTraits::const_value_type>::value
3468 };
3469
3470 enum {
3471 is_assignable_dimension =
3472 ViewDimensionAssignable<typename DstTraits::dimension,
3473 typename SrcTraits::dimension>::value
3474 };
3475
3476 public:
3477 enum {
3478 is_assignable_data_type =
3479 is_assignable_value_type && is_assignable_dimension
3480 };
3481 enum {
3482 is_assignable = is_assignable_space && is_assignable_value_type &&
3483 is_assignable_dimension
3484 };
3485
3486 using TrackType = Kokkos::Impl::SharedAllocationTracker;
3487 using DstType = ViewMapping<DstTraits, void>;
3488 using SrcType = ViewMapping<SrcTraits, void>;
3489
3490 KOKKOS_INLINE_FUNCTION
assignable_layout_check(DstType &,const SrcType & src)3491 static bool assignable_layout_check(DstType&,
3492 const SrcType& src) // Runtime check
3493 {
3494 size_t strides[9];
3495 bool assignable = true;
3496 src.stride(strides);
3497 size_t exp_stride = 1;
3498 if (std::is_same<typename DstTraits::array_layout,
3499 Kokkos::LayoutLeft>::value) {
3500 for (int i = 0; i < src.Rank; i++) {
3501 if (i > 0) exp_stride *= src.extent(i - 1);
3502 if (strides[i] != exp_stride) {
3503 assignable = false;
3504 break;
3505 }
3506 }
3507 } else if (std::is_same<typename DstTraits::array_layout,
3508 Kokkos::LayoutRight>::value) {
3509 for (int i = src.Rank - 1; i >= 0; i--) {
3510 if (i < src.Rank - 1) exp_stride *= src.extent(i + 1);
3511 if (strides[i] != exp_stride) {
3512 assignable = false;
3513 break;
3514 }
3515 }
3516 }
3517 return assignable;
3518 }
3519
3520 KOKKOS_INLINE_FUNCTION
assign(DstType & dst,const SrcType & src,const TrackType & src_track)3521 static void assign(DstType& dst, const SrcType& src,
3522 const TrackType& src_track) {
3523 static_assert(is_assignable_space,
3524 "View assignment must have compatible spaces");
3525
3526 static_assert(
3527 is_assignable_value_type,
3528 "View assignment must have same value type or const = non-const");
3529
3530 static_assert(is_assignable_dimension,
3531 "View assignment must have compatible dimensions");
3532
3533 bool assignable_layout = assignable_layout_check(dst, src); // Runtime
3534 // check
3535 if (!assignable_layout)
3536 Kokkos::abort("View assignment must have compatible layouts\n");
3537
3538 using dst_offset_type = typename DstType::offset_type;
3539
3540 if (size_t(DstTraits::dimension::rank_dynamic) <
3541 size_t(SrcTraits::dimension::rank_dynamic)) {
3542 using dst_dim = typename DstTraits::dimension;
3543 bool assignable = ((1 > DstTraits::dimension::rank_dynamic &&
3544 1 <= SrcTraits::dimension::rank_dynamic)
3545 ? dst_dim::ArgN0 == src.dimension_0()
3546 : true) &&
3547 ((2 > DstTraits::dimension::rank_dynamic &&
3548 2 <= SrcTraits::dimension::rank_dynamic)
3549 ? dst_dim::ArgN1 == src.dimension_1()
3550 : true) &&
3551 ((3 > DstTraits::dimension::rank_dynamic &&
3552 3 <= SrcTraits::dimension::rank_dynamic)
3553 ? dst_dim::ArgN2 == src.dimension_2()
3554 : true) &&
3555 ((4 > DstTraits::dimension::rank_dynamic &&
3556 4 <= SrcTraits::dimension::rank_dynamic)
3557 ? dst_dim::ArgN3 == src.dimension_3()
3558 : true) &&
3559 ((5 > DstTraits::dimension::rank_dynamic &&
3560 5 <= SrcTraits::dimension::rank_dynamic)
3561 ? dst_dim::ArgN4 == src.dimension_4()
3562 : true) &&
3563 ((6 > DstTraits::dimension::rank_dynamic &&
3564 6 <= SrcTraits::dimension::rank_dynamic)
3565 ? dst_dim::ArgN5 == src.dimension_5()
3566 : true) &&
3567 ((7 > DstTraits::dimension::rank_dynamic &&
3568 7 <= SrcTraits::dimension::rank_dynamic)
3569 ? dst_dim::ArgN6 == src.dimension_6()
3570 : true) &&
3571 ((8 > DstTraits::dimension::rank_dynamic &&
3572 8 <= SrcTraits::dimension::rank_dynamic)
3573 ? dst_dim::ArgN7 == src.dimension_7()
3574 : true);
3575 if (!assignable)
3576 Kokkos::abort(
3577 "View Assignment: trying to assign runtime dimension to non "
3578 "matching compile time dimension.");
3579 }
3580 dst.m_impl_offset = dst_offset_type(src.m_impl_offset);
3581 dst.m_impl_handle = Kokkos::Impl::ViewDataHandle<DstTraits>::assign(
3582 src.m_impl_handle, src_track);
3583 }
3584 };
3585
3586 //----------------------------------------------------------------------------
3587 //----------------------------------------------------------------------------
3588 // Subview mapping.
3589 // Deduce destination view type from source view traits and subview arguments
3590
3591 template <class, class ValueType, class Exts, class... Args>
3592 struct SubViewDataTypeImpl;
3593
3594 /* base case */
3595 template <class ValueType>
3596 struct SubViewDataTypeImpl<void, ValueType, Kokkos::Experimental::Extents<>> {
3597 using type = ValueType;
3598 };
3599
3600 /* for integral args, subview doesn't have that dimension */
3601 template <class ValueType, ptrdiff_t Ext, ptrdiff_t... Exts, class Integral,
3602 class... Args>
3603 struct SubViewDataTypeImpl<
3604 typename std::enable_if<
3605 std::is_integral<typename std::decay<Integral>::type>::value>::type,
3606 ValueType, Kokkos::Experimental::Extents<Ext, Exts...>, Integral, Args...>
3607 : SubViewDataTypeImpl<void, ValueType,
3608 Kokkos::Experimental::Extents<Exts...>, Args...> {};
3609
3610 /* for ALL slice, subview has the same dimension */
3611 template <class ValueType, ptrdiff_t Ext, ptrdiff_t... Exts, class... Args>
3612 struct SubViewDataTypeImpl<void, ValueType,
3613 Kokkos::Experimental::Extents<Ext, Exts...>, ALL_t,
3614 Args...>
3615 : SubViewDataTypeImpl<void, typename ApplyExtent<ValueType, Ext>::type,
3616 Kokkos::Experimental::Extents<Exts...>, Args...> {};
3617
3618 /* for pair-style slice, subview has dynamic dimension, since pair doesn't give
3619 * static sizes */
3620 /* Since we don't allow interleaving of dynamic and static extents, make all of
3621 * the dimensions to the left dynamic */
3622 template <class ValueType, ptrdiff_t Ext, ptrdiff_t... Exts, class PairLike,
3623 class... Args>
3624 struct SubViewDataTypeImpl<
3625 typename std::enable_if<is_pair_like<PairLike>::value>::type, ValueType,
3626 Kokkos::Experimental::Extents<Ext, Exts...>, PairLike, Args...>
3627 : SubViewDataTypeImpl<
3628 void, typename make_all_extents_into_pointers<ValueType>::type*,
3629 Kokkos::Experimental::Extents<Exts...>, Args...> {};
3630
3631 template <class ValueType, class Exts, class... Args>
3632 struct SubViewDataType : SubViewDataTypeImpl<void, ValueType, Exts, Args...> {};
3633
3634 //----------------------------------------------------------------------------
3635
3636 template <class SrcTraits, class... Args>
3637 class ViewMapping<
3638 typename std::enable_if<(
3639 std::is_same<typename SrcTraits::specialize, void>::value &&
3640 (std::is_same<typename SrcTraits::array_layout,
3641 Kokkos::LayoutLeft>::value ||
3642 std::is_same<typename SrcTraits::array_layout,
3643 Kokkos::LayoutRight>::value ||
3644 std::is_same<typename SrcTraits::array_layout,
3645 Kokkos::LayoutStride>::value))>::type,
3646 SrcTraits, Args...> {
3647 private:
3648 static_assert(SrcTraits::rank == sizeof...(Args),
3649 "Subview mapping requires one argument for each dimension of "
3650 "source View");
3651
3652 enum {
3653 RZ = false,
3654 R0 = bool(is_integral_extent<0, Args...>::value),
3655 R1 = bool(is_integral_extent<1, Args...>::value),
3656 R2 = bool(is_integral_extent<2, Args...>::value),
3657 R3 = bool(is_integral_extent<3, Args...>::value),
3658 R4 = bool(is_integral_extent<4, Args...>::value),
3659 R5 = bool(is_integral_extent<5, Args...>::value),
3660 R6 = bool(is_integral_extent<6, Args...>::value),
3661 R7 = bool(is_integral_extent<7, Args...>::value)
3662 };
3663
3664 enum {
3665 rank = unsigned(R0) + unsigned(R1) + unsigned(R2) + unsigned(R3) +
3666 unsigned(R4) + unsigned(R5) + unsigned(R6) + unsigned(R7)
3667 };
3668
3669 // Whether right-most rank is a range.
3670 enum {
3671 R0_rev =
3672 (0 == SrcTraits::rank
3673 ? RZ
3674 : (1 == SrcTraits::rank
3675 ? R0
3676 : (2 == SrcTraits::rank
3677 ? R1
3678 : (3 == SrcTraits::rank
3679 ? R2
3680 : (4 == SrcTraits::rank
3681 ? R3
3682 : (5 == SrcTraits::rank
3683 ? R4
3684 : (6 == SrcTraits::rank
3685 ? R5
3686 : (7 == SrcTraits::rank
3687 ? R6
3688 : R7))))))))
3689 };
3690
3691 // Subview's layout
3692 using array_layout = typename std::conditional<
3693 ( /* Same array layout IF */
3694 (rank == 0) /* output rank zero */
3695 || SubviewLegalArgsCompileTime<typename SrcTraits::array_layout,
3696 typename SrcTraits::array_layout, rank,
3697 SrcTraits::rank, 0, Args...>::value ||
3698 // OutputRank 1 or 2, InputLayout Left, Interval 0
3699 // because single stride one or second index has a stride.
3700 (rank <= 2 && R0 &&
3701 std::is_same<typename SrcTraits::array_layout,
3702 Kokkos::LayoutLeft>::value) // replace with input rank
3703 ||
3704 // OutputRank 1 or 2, InputLayout Right, Interval [InputRank-1]
3705 // because single stride one or second index has a stride.
3706 (rank <= 2 && R0_rev &&
3707 std::is_same<typename SrcTraits::array_layout,
3708 Kokkos::LayoutRight>::value) // replace input rank
3709 ),
3710 typename SrcTraits::array_layout, Kokkos::LayoutStride>::type;
3711
3712 using value_type = typename SrcTraits::value_type;
3713
3714 using data_type =
3715 typename SubViewDataType<value_type,
3716 typename Kokkos::Impl::ParseViewExtents<
3717 typename SrcTraits::data_type>::type,
3718 Args...>::type;
3719
3720 public:
3721 using traits_type = Kokkos::ViewTraits<data_type, array_layout,
3722 typename SrcTraits::device_type,
3723 typename SrcTraits::memory_traits>;
3724
3725 using type =
3726 Kokkos::View<data_type, array_layout, typename SrcTraits::device_type,
3727 typename SrcTraits::memory_traits>;
3728
3729 template <class MemoryTraits>
3730 struct apply {
3731 static_assert(Kokkos::Impl::is_memory_traits<MemoryTraits>::value, "");
3732
3733 using traits_type =
3734 Kokkos::ViewTraits<data_type, array_layout,
3735 typename SrcTraits::device_type, MemoryTraits>;
3736
3737 using type = Kokkos::View<data_type, array_layout,
3738 typename SrcTraits::device_type, MemoryTraits>;
3739 };
3740
3741 // The presumed type is 'ViewMapping< traits_type , void >'
3742 // However, a compatible ViewMapping is acceptable.
3743 template <class DstTraits>
assign(ViewMapping<DstTraits,void> & dst,ViewMapping<SrcTraits,void> const & src,Args...args)3744 KOKKOS_INLINE_FUNCTION static void assign(
3745 ViewMapping<DstTraits, void>& dst,
3746 ViewMapping<SrcTraits, void> const& src, Args... args) {
3747 static_assert(ViewMapping<DstTraits, traits_type, void>::is_assignable,
3748 "Subview destination type must be compatible with subview "
3749 "derived type");
3750
3751 using DstType = ViewMapping<DstTraits, void>;
3752
3753 using dst_offset_type = typename DstType::offset_type;
3754
3755 const SubviewExtents<SrcTraits::rank, rank> extents(src.m_impl_offset.m_dim,
3756 args...);
3757
3758 dst.m_impl_offset = dst_offset_type(src.m_impl_offset, extents);
3759
3760 dst.m_impl_handle = ViewDataHandle<DstTraits>::assign(
3761 src.m_impl_handle,
3762 src.m_impl_offset(extents.domain_offset(0), extents.domain_offset(1),
3763 extents.domain_offset(2), extents.domain_offset(3),
3764 extents.domain_offset(4), extents.domain_offset(5),
3765 extents.domain_offset(6), extents.domain_offset(7)));
3766 }
3767 };
3768
3769 //----------------------------------------------------------------------------
3770
3771 } // namespace Impl
3772 } // namespace Kokkos
3773
3774 //----------------------------------------------------------------------------
3775 //----------------------------------------------------------------------------
3776
3777 namespace Kokkos {
3778 namespace Impl {
3779
3780 template <unsigned, class MapType>
view_verify_operator_bounds(const MapType &)3781 KOKKOS_INLINE_FUNCTION bool view_verify_operator_bounds(const MapType&) {
3782 return true;
3783 }
3784
3785 template <unsigned R, class MapType, class iType, class... Args>
view_verify_operator_bounds(const MapType & map,const iType & i,Args...args)3786 KOKKOS_INLINE_FUNCTION bool view_verify_operator_bounds(const MapType& map,
3787 const iType& i,
3788 Args... args) {
3789 return (size_t(i) < map.extent(R)) &&
3790 view_verify_operator_bounds<R + 1>(map, args...);
3791 }
3792
3793 template <unsigned, class MapType>
view_error_operator_bounds(char *,int,const MapType &)3794 inline void view_error_operator_bounds(char*, int, const MapType&) {}
3795
3796 template <unsigned R, class MapType, class iType, class... Args>
view_error_operator_bounds(char * buf,int len,const MapType & map,const iType & i,Args...args)3797 inline void view_error_operator_bounds(char* buf, int len, const MapType& map,
3798 const iType& i, Args... args) {
3799 const int n = snprintf(
3800 buf, len, " %ld < %ld %c", static_cast<unsigned long>(i),
3801 static_cast<unsigned long>(map.extent(R)), (sizeof...(Args) ? ',' : ')'));
3802 view_error_operator_bounds<R + 1>(buf + n, len - n, map, args...);
3803 }
3804
3805 #if !defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
3806
3807 /* Check #3: is the View managed as determined by the MemoryTraits? */
3808 template <class MapType, bool is_managed = (MapType::is_managed != 0)>
3809 struct OperatorBoundsErrorOnDevice;
3810
3811 template <class MapType>
3812 struct OperatorBoundsErrorOnDevice<MapType, false> {
3813 KOKKOS_INLINE_FUNCTION
runKokkos::Impl::OperatorBoundsErrorOnDevice3814 static void run(MapType const&) { Kokkos::abort("View bounds error"); }
3815 };
3816
3817 template <class MapType>
3818 struct OperatorBoundsErrorOnDevice<MapType, true> {
3819 KOKKOS_INLINE_FUNCTION
runKokkos::Impl::OperatorBoundsErrorOnDevice3820 static void run(MapType const& map) {
3821 SharedAllocationHeader const* const header =
3822 SharedAllocationHeader::get_header((void*)(map.data()));
3823 char const* const label = header->label();
3824 enum { LEN = 128 };
3825 char msg[LEN];
3826 char const* const first_part = "View bounds error of view ";
3827 char* p = msg;
3828 char* const end = msg + LEN - 1;
3829 for (char const* p2 = first_part; (*p2 != '\0') && (p < end); ++p, ++p2) {
3830 *p = *p2;
3831 }
3832 for (char const* p2 = label; (*p2 != '\0') && (p < end); ++p, ++p2) {
3833 *p = *p2;
3834 }
3835 *p = '\0';
3836 Kokkos::abort(msg);
3837 }
3838 };
3839
3840 /* Check #2: does the ViewMapping have the printable_label_typedef defined?
3841 See above that only the non-specialized standard-layout ViewMapping has
3842 this defined by default.
3843 The existence of this alias indicates the existence of MapType::is_managed
3844 */
3845 template <class T, class Enable = void>
3846 struct has_printable_label_typedef : public std::false_type {};
3847
3848 template <class T>
3849 struct has_printable_label_typedef<T,
3850 void_t<typename T::printable_label_typedef>>
3851 : public std::true_type {};
3852
3853 template <class MapType>
operator_bounds_error_on_device(MapType const &,std::false_type)3854 KOKKOS_INLINE_FUNCTION void operator_bounds_error_on_device(MapType const&,
3855 std::false_type) {
3856 Kokkos::abort("View bounds error");
3857 }
3858
3859 template <class MapType>
operator_bounds_error_on_device(MapType const & map,std::true_type)3860 KOKKOS_INLINE_FUNCTION void operator_bounds_error_on_device(MapType const& map,
3861 std::true_type) {
3862 OperatorBoundsErrorOnDevice<MapType>::run(map);
3863 }
3864
3865 #endif // ! defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST )
3866
3867 template <class MemorySpace, class ViewType, class MapType, class... Args>
view_verify_operator_bounds(Kokkos::Impl::ViewTracker<ViewType> const & tracker,const MapType & map,Args...args)3868 KOKKOS_INLINE_FUNCTION void view_verify_operator_bounds(
3869 Kokkos::Impl::ViewTracker<ViewType> const& tracker, const MapType& map,
3870 Args... args) {
3871 if (!view_verify_operator_bounds<0>(map, args...)) {
3872 #if defined(KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST)
3873 enum { LEN = 1024 };
3874 char buffer[LEN];
3875 const std::string label =
3876 tracker.m_tracker.template get_label<MemorySpace>();
3877 int n =
3878 snprintf(buffer, LEN, "View bounds error of view %s (", label.c_str());
3879 view_error_operator_bounds<0>(buffer + n, LEN - n, map, args...);
3880 Kokkos::Impl::throw_runtime_exception(std::string(buffer));
3881 #else
3882 /* Check #1: is there a SharedAllocationRecord?
3883 (we won't use it, but if its not there then there isn't
3884 a corresponding SharedAllocationHeader containing a label).
3885 This check should cover the case of Views that don't
3886 have the Unmanaged trait but were initialized by pointer. */
3887 if (tracker.m_tracker.has_record()) {
3888 operator_bounds_error_on_device<MapType>(
3889 map, has_printable_label_typedef<MapType>());
3890 } else {
3891 Kokkos::abort("View bounds error");
3892 }
3893 #endif
3894 }
3895 }
3896
3897 } /* namespace Impl */
3898 } /* namespace Kokkos */
3899
3900 //----------------------------------------------------------------------------
3901 //----------------------------------------------------------------------------
3902
3903 #endif /* #ifndef KOKKOS_EXPERIMENTAL_VIEW_MAPPING_HPP */
3904