1 /*
2 //@HEADER
3 // ************************************************************************
4 //
5 // Kokkos v. 3.0
6 //       Copyright (2020) National Technology & Engineering
7 //               Solutions of Sandia, LLC (NTESS).
8 //
9 // Under the terms of Contract DE-NA0003525 with NTESS,
10 // the U.S. Government retains certain rights in this software.
11 //
12 // Redistribution and use in source and binary forms, with or without
13 // modification, are permitted provided that the following conditions are
14 // met:
15 //
16 // 1. Redistributions of source code must retain the above copyright
17 // notice, this list of conditions and the following disclaimer.
18 //
19 // 2. Redistributions in binary form must reproduce the above copyright
20 // notice, this list of conditions and the following disclaimer in the
21 // documentation and/or other materials provided with the distribution.
22 //
23 // 3. Neither the name of the Corporation nor the names of the
24 // contributors may be used to endorse or promote products derived from
25 // this software without specific prior written permission.
26 //
27 // THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY
28 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
29 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
30 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE
31 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
32 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
33 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
34 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 //
39 // Questions? Contact Christian R. Trott (crtrott@sandia.gov)
40 //
41 // ************************************************************************
42 //@HEADER
43 */
44 
45 // @Kokkos_Feature_Level_Required:12
46 // Unit test for hierarchical parallelism
47 // Create concurrent work hierarchically and verify if
48 // contributions of paticipating processing units corresponds to expected value
49 // Use a scratch pad memory for each team
50 #include <gtest/gtest.h>
51 #include <Kokkos_Core.hpp>
52 
53 namespace Test {
54 
55 template <class ExecSpace>
56 struct TeamScratch {
runTest::TeamScratch57   void run(const int pN, const int sX, const int sY) {
58     using policy_t = Kokkos::TeamPolicy<ExecSpace>;
59     using team_t   = typename Kokkos::TeamPolicy<ExecSpace>::member_type;
60     using data_t   = Kokkos::View<size_t **, ExecSpace>;
61     data_t v("Matrix", pN, sX);
62 
63     using scratch_t = Kokkos::View<size_t **, ExecSpace,
64                                    Kokkos::MemoryTraits<Kokkos::Unmanaged> >;
65     int scratchSize = scratch_t::shmem_size(sX, sY);
66 
67     const int scratch_level = 1;
68 
69     Kokkos::parallel_for(
70         "Team",
71         policy_t(pN, Kokkos::AUTO)
72             .set_scratch_size(scratch_level, Kokkos::PerTeam(scratchSize)),
73         KOKKOS_LAMBDA(const team_t &team) {
74           // Allocate and use scratch pad memory
75           scratch_t v_S(team.team_scratch(scratch_level), sX, sY);
76           int n = team.league_rank();
77 
78           Kokkos::parallel_for(
79               Kokkos::TeamThreadRange(team, sX), [&](const int m) {
80                 Kokkos::parallel_for(
81                     Kokkos::ThreadVectorRange(team, sY), [&](const int k) {
82                       v_S(m, k) = v_S.extent(0) * v_S.extent(1) * n +
83                                   v_S.extent(1) * m + k;
84                     });
85               });
86 
87           team.team_barrier();
88 
89           // Sum up contributions and reduce by one dimension
90           Kokkos::parallel_for(Kokkos::TeamThreadRange(team, sX),
91                                [&](const int m) {
92                                  for (int i = 0; i < sY; ++i)
93                                    v(n, m) += v_S(m, i);
94                                });
95         });
96 
97     Kokkos::fence();
98     auto v_H = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), v);
99 
100     size_t check   = 0;
101     const size_t s = pN * sX * sY;
102     for (int n = 0; n < pN; ++n)
103       for (int m = 0; m < sX; ++m) check += v_H(n, m);
104     ASSERT_EQ(check, s * (s - 1) / 2);
105   }
106 };
107 
TEST(TEST_CATEGORY,IncrTest_12b_TeamScratch)108 TEST(TEST_CATEGORY, IncrTest_12b_TeamScratch) {
109   TeamScratch<TEST_EXECSPACE> test;
110   // FIXME_OPENMPTARGET - team_size has to be a multiple of 32 for the tests to
111   // pass in the Release and RelWithDebInfo builds. Does not need the team_size
112   // to be a multiple of 32 for the Debug builds.
113 #ifdef KOKKOS_ENABLE_OPENMPTARGET
114   test.run(1, 32, 4);
115   test.run(4, 64, 10);
116   test.run(14, 128, 20);
117 #else
118   test.run(1, 4, 4);
119   test.run(4, 7, 10);
120   test.run(14, 277, 321);
121 #endif
122 }
123 
124 }  // namespace Test
125