1 /*
2 * Distributed under the OSI-approved Apache License, Version 2.0. See
3 * accompanying file Copyright.txt for details.
4 */
5 #include <cstdint>
6 #include <cstring>
7
8 #include <fstream>
9 #include <iostream>
10 #include <numeric> //std::iota
11 #include <stdexcept>
12
13 #include <adios2.h>
14
15 #include <gtest/gtest.h>
16
17 //#include "../engine/SmallTestData.h"
18
19 // std::string engineName; // comes from command line
20
21 struct QueryTestData
22 {
23 std::vector<int> m_IntData;
24 std::vector<double> m_DoubleData;
25 };
26
WriteXmlQuery1D(const std::string & queryFile,const std::string & ioName,const std::string & varName)27 void WriteXmlQuery1D(const std::string &queryFile, const std::string &ioName,
28 const std::string &varName)
29 {
30 std::ofstream file(queryFile.c_str());
31 file << "<adios-query>" << std::endl;
32 file << " <io name=\"" << ioName << "\">" << std::endl;
33 file << " <var name=\"" << varName << "\">" << std::endl;
34 file << " <boundingbox start=\"5\" count=\"80\"/>" << std::endl;
35 file << " <op value=\"OR\">" << std::endl;
36 file << " <range compare=\"GT\" value=\"6.6\"/>" << std::endl;
37 file << " <range compare=\"LT\" value=\"-0.17\"/>" << std::endl;
38 file << " <op value=\"AND\">" << std::endl;
39 file << " <range compare=\"LT\" value=\"2.9\"/>" << std::endl;
40 file << " <range compare=\"GT\" value=\"2.8\"/>" << std::endl;
41 file << " </op>" << std::endl;
42 file << " </op>" << std::endl;
43 file << " </var>" << std::endl;
44 file << " </io>" << std::endl;
45 file << "</adios-query>" << std::endl;
46 file.close();
47 }
48
LoadTestData(QueryTestData & input,int step,int rank,int dataSize)49 void LoadTestData(QueryTestData &input, int step, int rank, int dataSize)
50 {
51 input.m_IntData.clear();
52 input.m_IntData.reserve(dataSize);
53 input.m_DoubleData.clear();
54 input.m_DoubleData.reserve(dataSize);
55
56 for (auto i = 0; i < dataSize; i++)
57 {
58 input.m_IntData.push_back(step * 1000 + rank * 100 + i);
59 input.m_DoubleData.push_back(step * 100 + rank * 10 + i * 0.01);
60 }
61 }
62
63 class BPQueryTest : public ::testing::Test
64 {
65 public:
66 BPQueryTest() = default;
67
68 void WriteFile(const std::string &fname, adios2::ADIOS &adios,
69 const std::string &engineName);
70 void QueryDoubleVar(const std::string &fname, adios2::ADIOS &adios,
71 const std::string &engineName);
72 void QueryIntVar(const std::string &fname, adios2::ADIOS &adios,
73 const std::string &engineName);
74
75 QueryTestData m_TestData;
76
77 // Number of rows
78 const size_t Nx = 100;
79 // Number of steps
80 const size_t NSteps = 3;
81
82 int mpiRank = 0, mpiSize = 1;
83 };
84
QueryIntVar(const std::string & fname,adios2::ADIOS & adios,const std::string & engineName)85 void BPQueryTest::QueryIntVar(const std::string &fname, adios2::ADIOS &adios,
86 const std::string &engineName)
87 {
88 std::string ioName = "IOQueryTestInt" + engineName;
89 adios2::IO io = adios.DeclareIO(ioName.c_str());
90
91 if (!engineName.empty())
92 {
93 io.SetEngine(engineName);
94 }
95
96 adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read);
97
98 EXPECT_EQ(bpReader.Steps(), NSteps);
99
100 std::string queryFile = "./" + ioName + "test.xml"; //"./test.xml";
101 std::cout << ioName << std::endl;
102 WriteXmlQuery1D(queryFile, ioName, "intV");
103 adios2::QueryWorker w = adios2::QueryWorker(queryFile, bpReader);
104
105 std::vector<size_t> rr;
106 if (engineName.compare("BP4") == 0)
107 rr = {9, 9, 9};
108 else
109 rr = {1, 1, 1};
110
111 while (bpReader.BeginStep() == adios2::StepStatus::OK)
112 {
113 std::vector<adios2::Box<adios2::Dims>> touched_blocks;
114 adios2::Box<adios2::Dims> empty;
115 w.GetResultCoverage(empty, touched_blocks);
116 ASSERT_EQ(touched_blocks.size(), rr[bpReader.CurrentStep()]);
117 bpReader.EndStep();
118 }
119 bpReader.Close();
120 }
121
QueryDoubleVar(const std::string & fname,adios2::ADIOS & adios,const std::string & engineName)122 void BPQueryTest::QueryDoubleVar(const std::string &fname, adios2::ADIOS &adios,
123 const std::string &engineName)
124 {
125 std::string ioName = "IOQueryTestDouble" + engineName;
126 adios2::IO io = adios.DeclareIO(ioName.c_str());
127
128 if (!engineName.empty())
129 {
130 io.SetEngine(engineName);
131 }
132
133 adios2::Engine bpReader = io.Open(fname, adios2::Mode::Read);
134
135 EXPECT_EQ(bpReader.Steps(), NSteps);
136
137 // std::string queryFile = "./.test.xml";
138 std::string queryFile = "./" + ioName + "test.xml";
139 WriteXmlQuery1D(queryFile, ioName, "doubleV");
140 adios2::QueryWorker w = adios2::QueryWorker(queryFile, bpReader);
141
142 std::vector<size_t> rr; //= {0,9,9};
143 if (engineName.compare("BP4") == 0)
144 rr = {0, 9, 9};
145 else
146 rr = {0, 1, 1};
147 while (bpReader.BeginStep() == adios2::StepStatus::OK)
148 {
149 std::vector<adios2::Box<adios2::Dims>> touched_blocks;
150 adios2::Box<adios2::Dims> empty;
151 w.GetResultCoverage(empty, touched_blocks);
152 ASSERT_EQ(touched_blocks.size(), rr[bpReader.CurrentStep()]);
153 bpReader.EndStep();
154 }
155 bpReader.Close();
156 }
157
WriteFile(const std::string & fname,adios2::ADIOS & adios,const std::string & engineName)158 void BPQueryTest::WriteFile(const std::string &fname, adios2::ADIOS &adios,
159 const std::string &engineName)
160 {
161
162 #if ADIOS2_USE_MPI
163 MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
164 MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
165 #endif
166
167 // Write test data using BP
168
169 {
170 adios2::IO io = adios.DeclareIO("TestQueryIOWriter");
171
172 // Declare 1D variables (NumOfProcesses * Nx)
173 // The local process' part (start, count) can be defined now or later
174 // before Write().
175 {
176 const adios2::Dims shape{static_cast<size_t>(Nx * mpiSize)};
177 const adios2::Dims start{static_cast<size_t>(Nx * mpiRank)};
178 const adios2::Dims count{Nx};
179
180 auto var_i32 =
181 io.DefineVariable<int32_t>("intV", shape, start, count);
182 auto var_r64 =
183 io.DefineVariable<double>("doubleV", shape, start, count);
184 }
185
186 if (!engineName.empty())
187 {
188 io.SetEngine(engineName);
189 }
190 else
191 {
192 // Create the BP Engine
193 io.SetEngine("BPFile");
194 }
195
196 io.SetParameter("AsyncThreads", "0");
197
198 if (engineName.compare("BP4") == 0)
199 {
200 io.SetParameters("statslevel=1");
201 io.SetParameters("statsblocksize=10");
202 }
203 io.AddTransport("file");
204
205 // QUESTION: It seems that BPFilterWriter cannot overwrite existing
206 // files
207 // Ex. if you tune Nx and NSteps, the test would fail. But if you clear
208 // the cache in
209 // ${adios2Build}/testing/adios2/engine/bp/ADIOS2BPWriteADIOS1Read1D800.bp.dir,
210 // then it works
211 adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write);
212
213 for (size_t step = 0; step < NSteps; ++step)
214 {
215 // Generate test data for each process uniquely
216 LoadTestData(m_TestData, static_cast<int>(step), mpiRank,
217 static_cast<int>(Nx));
218
219 auto var_i32 = io.InquireVariable<int32_t>("intV");
220 auto var_r64 = io.InquireVariable<double>("doubleV");
221
222 adios2::Box<adios2::Dims> sel({mpiRank * Nx}, {Nx});
223
224 var_i32.SetSelection(sel);
225 var_r64.SetSelection(sel);
226
227 bpWriter.BeginStep();
228 bpWriter.Put(var_i32, m_TestData.m_IntData.data());
229 bpWriter.Put(var_r64, m_TestData.m_DoubleData.data());
230 bpWriter.PerformPuts();
231
232 bpWriter.EndStep();
233 }
234
235 // Close the file
236 bpWriter.Close();
237 }
238 }
239 //******************************************************************************
240 // 1D test data
241 //******************************************************************************
242
TEST_F(BPQueryTest,BP3)243 TEST_F(BPQueryTest, BP3)
244 {
245 std::string engineName = "BP3";
246 // Each process would write a 1x8 array and all processes would
247 // form a mpiSize * Nx 1D array
248 const std::string fname(engineName + "Query1D.bp");
249
250 #if ADIOS2_USE_MPI
251 adios2::ADIOS adios(MPI_COMM_WORLD);
252 #else
253 adios2::ADIOS adios;
254 #endif
255
256 WriteFile(fname, adios, engineName);
257
258 if (mpiSize == 1)
259 {
260 QueryDoubleVar(fname, adios, engineName);
261 QueryIntVar(fname, adios, engineName);
262 }
263 }
264
265 //******************************************************************************
266 // 2D test data
267 //******************************************************************************
268
TEST_F(BPQueryTest,BP4)269 TEST_F(BPQueryTest, BP4)
270 {
271 std::string engineName = "BP4";
272 // Each process would write a 1x8 array and all processes would
273 // form a mpiSize * Nx 1D array
274 const std::string fname(engineName + "4Query1D.bp");
275
276 #if ADIOS2_USE_MPI
277 adios2::ADIOS adios(MPI_COMM_WORLD);
278 #else
279 adios2::ADIOS adios;
280 #endif
281
282 WriteFile(fname, adios, engineName);
283
284 if (mpiSize == 1)
285 {
286 QueryDoubleVar(fname, adios, engineName);
287 QueryIntVar(fname, adios, engineName);
288 }
289 }
290
291 //******************************************************************************
292 // main
293 //******************************************************************************
294
main(int argc,char ** argv)295 int main(int argc, char **argv)
296 {
297 #if ADIOS2_USE_MPI
298 // MPI_Init(nullptr, nullptr);
299 MPI_Init(&argc, &argv);
300 #endif
301
302 int result;
303 ::testing::InitGoogleTest(&argc, argv);
304
305 if (argc > 1)
306 {
307 // engineName = std::string(argv[1]);
308 }
309 result = RUN_ALL_TESTS();
310
311 #if ADIOS2_USE_MPI
312 MPI_Finalize();
313 #endif
314
315 return result;
316 }
317