1 /*
2  * Distributed under the OSI-approved Apache License, Version 2.0.  See
3  * accompanying file Copyright.txt for details.
4  *
5  * TestDataMan1D.cpp
6  *
7  *  Created on: Jul 12, 2018
8  *      Author: Jason Wang
9  */
10 
11 #include <numeric>
12 #include <thread>
13 
14 #include <adios2.h>
15 #include <adios2/common/ADIOSMacros.h>
16 #include <adios2/helper/adiosFunctions.h>
17 #include <gtest/gtest.h>
18 
19 using namespace adios2;
20 
21 size_t print_lines = 0;
22 size_t to_print_lines = 10;
23 
24 template <class T>
GenData(std::vector<std::complex<T>> & data,const size_t step)25 void GenData(std::vector<std::complex<T>> &data, const size_t step)
26 {
27     for (size_t i = 0; i < data.size(); ++i)
28     {
29         data[i] = {static_cast<T>(i + 10000 + step * 100),
30                    static_cast<T>(i + 10000)};
31     }
32 }
33 
34 template <class T>
GenData(std::vector<T> & data,const size_t step)35 void GenData(std::vector<T> &data, const size_t step)
36 {
37     for (size_t i = 0; i < data.size(); ++i)
38     {
39         data[i] = i + 10000 + step * 100;
40     }
41 }
42 
43 template <class T>
PrintData(const T * data,const size_t size,const size_t step)44 void PrintData(const T *data, const size_t size, const size_t step)
45 {
46     std::cout << "Step: " << step << " Size:" << size << "\n";
47     size_t printsize = 32;
48     if (size < printsize)
49     {
50         printsize = size;
51     }
52     for (size_t i = 0; i < printsize; ++i)
53     {
54         std::cout << data[i] << " ";
55     }
56     std::cout << "]" << std::endl;
57 }
58 
59 template <class T>
VerifyData(const std::complex<T> * data,const size_t size,size_t step)60 void VerifyData(const std::complex<T> *data, const size_t size, size_t step)
61 {
62     std::vector<std::complex<T>> tmpdata(size);
63     GenData(tmpdata, step);
64     for (size_t i = 0; i < size; ++i)
65     {
66         ASSERT_EQ(data[i], tmpdata[i]);
67     }
68 }
69 
70 template <class T>
VerifyData(const T * data,const size_t size,size_t step)71 void VerifyData(const T *data, const size_t size, size_t step)
72 {
73     std::vector<T> tmpdata(size);
74     GenData(tmpdata, step);
75     for (size_t i = 0; i < size; ++i)
76     {
77         ASSERT_EQ(data[i], tmpdata[i]);
78     }
79 }
80 
81 template <class T>
VerifyData(const std::vector<T> & data,const size_t step)82 void VerifyData(const std::vector<T> &data, const size_t step)
83 {
84     VerifyData(data.data(), data.size(), step);
85 }
86 
DataManWriter(const Dims & shape,const Dims & start,const Dims & count,const size_t steps,const adios2::Params & engineParams)87 void DataManWriter(const Dims &shape, const Dims &start, const Dims &count,
88                    const size_t steps, const adios2::Params &engineParams)
89 {
90     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
91                                       std::multiplies<size_t>());
92     adios2::ADIOS adios;
93     adios2::IO dataManIO = adios.DeclareIO("WAN");
94     dataManIO.SetEngine("DataMan");
95     dataManIO.SetParameters(engineParams);
96     std::vector<char> myChars(datasize);
97     std::vector<unsigned char> myUChars(datasize);
98     std::vector<short> myShorts(datasize);
99     std::vector<unsigned short> myUShorts(datasize);
100     std::vector<int> myInts(datasize);
101     std::vector<unsigned int> myUInts(datasize);
102     std::vector<float> myFloats(datasize);
103     std::vector<double> myDoubles(datasize);
104     std::vector<std::complex<float>> myComplexes(datasize);
105     std::vector<std::complex<double>> myDComplexes(datasize);
106     auto bpChars =
107         dataManIO.DefineVariable<char>("bpChars", shape, start, count);
108     auto bpUChars = dataManIO.DefineVariable<unsigned char>("bpUChars", shape,
109                                                             start, count);
110     auto bpShorts =
111         dataManIO.DefineVariable<short>("bpShorts", shape, start, count);
112     auto bpUShorts = dataManIO.DefineVariable<unsigned short>(
113         "bpUShorts", shape, start, count);
114     auto bpInts = dataManIO.DefineVariable<int>("bpInts", shape, start, count);
115     auto bpUInts =
116         dataManIO.DefineVariable<unsigned int>("bpUInts", shape, start, count);
117     auto bpFloats =
118         dataManIO.DefineVariable<float>("bpFloats", shape, start, count);
119     auto bpDoubles =
120         dataManIO.DefineVariable<double>("bpDoubles", shape, start, count);
121     auto bpComplexes = dataManIO.DefineVariable<std::complex<float>>(
122         "bpComplexes", shape, start, count);
123     auto bpDComplexes = dataManIO.DefineVariable<std::complex<double>>(
124         "bpDComplexes", shape, start, count);
125     auto bpUInt64s = dataManIO.DefineVariable<uint64_t>("bpUInt64s");
126     dataManIO.DefineAttribute<int>("AttInt", 110);
127     adios2::Engine dataManWriter =
128         dataManIO.Open("stream", adios2::Mode::Write);
129     for (uint64_t i = 0; i < steps; ++i)
130     {
131         dataManWriter.BeginStep();
132         GenData(myChars, i);
133         GenData(myUChars, i);
134         GenData(myShorts, i);
135         GenData(myUShorts, i);
136         GenData(myInts, i);
137         GenData(myUInts, i);
138         GenData(myFloats, i);
139         GenData(myDoubles, i);
140         GenData(myComplexes, i);
141         GenData(myDComplexes, i);
142         dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync);
143         dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync);
144         dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync);
145         dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
146         dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync);
147         dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync);
148         dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync);
149         dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
150         dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync);
151         dataManWriter.Put(bpDComplexes, myDComplexes.data(),
152                           adios2::Mode::Sync);
153         dataManWriter.Put(bpUInt64s, i);
154         dataManWriter.EndStep();
155     }
156     dataManWriter.Close();
157 }
158 
DataManReader(const Dims & shape,const Dims & start,const Dims & count,const size_t steps,const adios2::Params & engineParams)159 void DataManReader(const Dims &shape, const Dims &start, const Dims &count,
160                    const size_t steps, const adios2::Params &engineParams)
161 {
162     size_t datasize = std::accumulate(count.begin(), count.end(), 1,
163                                       std::multiplies<size_t>());
164     adios2::ADIOS adios;
165     adios2::IO dataManIO = adios.DeclareIO("WAN");
166     dataManIO.SetEngine("DataMan");
167     dataManIO.SetParameters(engineParams);
168     adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read);
169 
170     std::vector<char> myChars(datasize);
171     std::vector<unsigned char> myUChars(datasize);
172     std::vector<short> myShorts(datasize);
173     std::vector<unsigned short> myUShorts(datasize);
174     std::vector<int> myInts(datasize);
175     std::vector<unsigned int> myUInts(datasize);
176     std::vector<float> myFloats(datasize);
177     std::vector<double> myDoubles(datasize);
178     std::vector<std::complex<float>> myComplexes(datasize);
179     std::vector<std::complex<double>> myDComplexes(datasize);
180     bool received_steps = false;
181     size_t currentStep;
182     while (true)
183     {
184         adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5);
185         if (status == adios2::StepStatus::OK)
186         {
187             received_steps = true;
188             const auto &vars = dataManIO.AvailableVariables();
189             ASSERT_EQ(vars.size(), 11);
190             currentStep = dataManReader.CurrentStep();
191             adios2::Variable<char> bpChars =
192                 dataManIO.InquireVariable<char>("bpChars");
193             adios2::Variable<unsigned char> bpUChars =
194                 dataManIO.InquireVariable<unsigned char>("bpUChars");
195             adios2::Variable<short> bpShorts =
196                 dataManIO.InquireVariable<short>("bpShorts");
197             adios2::Variable<unsigned short> bpUShorts =
198                 dataManIO.InquireVariable<unsigned short>("bpUShorts");
199             adios2::Variable<int> bpInts =
200                 dataManIO.InquireVariable<int>("bpInts");
201             adios2::Variable<unsigned int> bpUInts =
202                 dataManIO.InquireVariable<unsigned int>("bpUInts");
203             adios2::Variable<float> bpFloats =
204                 dataManIO.InquireVariable<float>("bpFloats");
205             adios2::Variable<double> bpDoubles =
206                 dataManIO.InquireVariable<double>("bpDoubles");
207             adios2::Variable<std::complex<float>> bpComplexes =
208                 dataManIO.InquireVariable<std::complex<float>>("bpComplexes");
209             adios2::Variable<std::complex<double>> bpDComplexes =
210                 dataManIO.InquireVariable<std::complex<double>>("bpDComplexes");
211             adios2::Variable<uint64_t> bpUInt64s =
212                 dataManIO.InquireVariable<uint64_t>("bpUInt64s");
213             auto charsBlocksInfo = dataManReader.AllStepsBlocksInfo(bpChars);
214             bpChars.SetSelection({start, count});
215             bpUChars.SetSelection({start, count});
216             bpShorts.SetSelection({start, count});
217             bpUShorts.SetSelection({start, count});
218             bpInts.SetSelection({start, count});
219             bpUInts.SetSelection({start, count});
220             bpFloats.SetSelection({start, count});
221             bpDoubles.SetSelection({start, count});
222             bpComplexes.SetSelection({start, count});
223             bpDComplexes.SetSelection({start, count});
224             dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync);
225             dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync);
226             dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync);
227             dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync);
228             dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync);
229             dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync);
230             dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync);
231             dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync);
232             dataManReader.Get(bpComplexes, myComplexes.data(),
233                               adios2::Mode::Sync);
234             dataManReader.Get(bpDComplexes, myDComplexes.data(),
235                               adios2::Mode::Sync);
236             uint64_t stepValue;
237             dataManReader.Get(bpUInt64s, &stepValue, adios2::Mode::Sync);
238             ASSERT_EQ(currentStep, stepValue);
239             VerifyData(myChars, currentStep);
240             VerifyData(myUChars, currentStep);
241             VerifyData(myShorts, currentStep);
242             VerifyData(myUShorts, currentStep);
243             VerifyData(myInts, currentStep);
244             VerifyData(myUInts, currentStep);
245             VerifyData(myFloats, currentStep);
246             VerifyData(myDoubles, currentStep);
247             VerifyData(myComplexes, currentStep);
248             VerifyData(myDComplexes, currentStep);
249             dataManReader.EndStep();
250         }
251         else if (status == adios2::StepStatus::EndOfStream)
252         {
253             break;
254         }
255         else if (status == adios2::StepStatus::NotReady)
256         {
257             continue;
258         }
259     }
260     if (received_steps)
261     {
262         auto attInt = dataManIO.InquireAttribute<int>("AttInt");
263         ASSERT_EQ(110, attInt.Data()[0]);
264     }
265     dataManReader.Close();
266 }
267 
268 class DataManEngineTest : public ::testing::Test
269 {
270 public:
271     DataManEngineTest() = default;
272 };
273 
274 #ifdef ADIOS2_HAVE_ZEROMQ
275 TEST_F(DataManEngineTest, 1D)
276 {
277     // set parameters
278     Dims shape = {10};
279     Dims start = {0};
280     Dims count = {10};
281     size_t steps = 5000;
282     adios2::Params engineParams = {{"IPAddress", "127.0.0.1"},
283                                    {"Port", "12300"}};
284 
285     // run workflow
286     auto r =
287         std::thread(DataManReader, shape, start, count, steps, engineParams);
288     auto w =
289         std::thread(DataManWriter, shape, start, count, steps, engineParams);
290     w.join();
291     r.join();
292 }
293 #endif // ZEROMQ
294 
main(int argc,char ** argv)295 int main(int argc, char **argv)
296 {
297     int result;
298     ::testing::InitGoogleTest(&argc, argv);
299     result = RUN_ALL_TESTS();
300 
301     return result;
302 }
303