1 /* 2 * This file is part of the GROMACS molecular simulation package. 3 * 4 * Copyright (c) 2011-2018, The GROMACS development team. 5 * Copyright (c) 2019,2020, by the GROMACS development team, led by 6 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl, 7 * and including many others, as listed in the AUTHORS file in the 8 * top-level source directory and at http://www.gromacs.org. 9 * 10 * GROMACS is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU Lesser General Public License 12 * as published by the Free Software Foundation; either version 2.1 13 * of the License, or (at your option) any later version. 14 * 15 * GROMACS is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 18 * Lesser General Public License for more details. 19 * 20 * You should have received a copy of the GNU Lesser General Public 21 * License along with GROMACS; if not, see 22 * http://www.gnu.org/licenses, or write to the Free Software Foundation, 23 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 24 * 25 * If you want to redistribute modifications to GROMACS, please 26 * consider that scientific software is very special. Version 27 * control is crucial - bugs must be traceable. We will be happy to 28 * consider code for inclusion in the official distribution, but 29 * derived work must not be called official GROMACS. Details are found 30 * in the README & COPYING files - if they are missing, get the 31 * official version at http://www.gromacs.org. 32 * 33 * To help us fund GROMACS development, we humbly ask that you cite 34 * the research papers on the package. Check out http://www.gromacs.org. 35 */ 36 /*! \libinternal \file 37 * \brief 38 * Functionality for writing tests that can produce their own reference data. 39 * 40 * See \ref page_refdata for more details. 41 * 42 * \author Teemu Murtola <teemu.murtola@gmail.com> 43 * \inlibraryapi 44 * \ingroup module_testutils 45 */ 46 #ifndef GMX_TESTUTILS_REFDATA_H 47 #define GMX_TESTUTILS_REFDATA_H 48 49 #include <iterator> 50 #include <memory> 51 #include <string> 52 53 #include "gromacs/utility/basedefinitions.h" 54 #include "gromacs/utility/classhelpers.h" 55 56 namespace gmx 57 { 58 59 class IOptionsContainer; 60 class KeyValueTreeObject; 61 class KeyValueTreeValue; 62 class Any; 63 64 template<typename ValueType> 65 class BasicVector; 66 67 namespace test 68 { 69 70 class FloatingPointTolerance; 71 72 /*! \libinternal \brief 73 * Mode of operation for reference data handling. 74 * 75 * There should be no need to use this type outside the test utility module. 76 * 77 * \ingroup module_testutils 78 */ 79 enum class ReferenceDataMode : int 80 { 81 /*! \brief 82 * Compare to existing reference data. 83 * 84 * If reference data does not exist, or if the test results differ from 85 * those in the reference data, the test fails. 86 */ 87 Compare, 88 /*! \brief 89 * Create missing reference data. 90 * 91 * If reference data does not exist for a test, that test behaves as if 92 * ReferenceDataMode::UpdateAll had been specified. Tests for which reference data 93 * exists, behave like with ReferenceDataMode::Compare. 94 */ 95 CreateMissing, 96 /*! \brief 97 * Update reference data that does not pass comparison. 98 * 99 * Tests utilizing reference data should always pass in this mode unless 100 * there is an I/O error. 101 */ 102 UpdateChanged, 103 /*! \brief 104 * Update reference data, overwriting old data. 105 * 106 * Tests utilizing reference data should always pass in this mode unless 107 * there is an I/O error. 108 */ 109 UpdateAll, 110 //! Marks the end of the enum 111 Count 112 }; 113 114 /*! \libinternal \brief 115 * Initializes reference data handling. 116 * 117 * Adds command-line options to \p options to set the reference data mode. 118 * By default, ReferenceDataMode::Compare is used, but ``--ref-data create`` or 119 * ``--ref-data update`` can be used to change it. 120 * 121 * This function is automatically called by initTestUtils(). 122 * 123 * \ingroup module_testutils 124 */ 125 void initReferenceData(IOptionsContainer* options); 126 127 class TestReferenceChecker; 128 129 namespace internal 130 { 131 class TestReferenceDataImpl; 132 } 133 134 /*! \libinternal \brief 135 * Handles creation of and comparison to test reference data. 136 * 137 * See \ref page_refdata for an overview of the functionality. 138 * 139 * This class provides functionality to use the same code to generate reference 140 * data and then on later runs compare the results of the code against that 141 * reference. The mode in which the class operates (writing reference data or 142 * comparing against existing data) is set using a command-line option that 143 * is automatically managed when using the testutils module to implement tests. 144 * Tests only need to create an instance of TestReferenceData, obtain a 145 * TestReferenceChecker using the rootChecker() method and use the various 146 * check*() methods in TestReferenceChecker to indicate values to check. If 147 * the test is running in reference data creation mode, it will produce an XML 148 * file with the values recorder. In comparison mode, it will read that same 149 * XML file and produce a Google Test non-fatal assertion for every discrepancy 150 * it detects with the reference data (including missing reference data file or 151 * individual item). Exceptions derived from TestException are thrown for I/O 152 * errors and syntax errors in the reference data. 153 * 154 * Simple example (using Google Test): 155 * \code 156 int functionToTest(int param); 157 158 namespace gmx 159 { 160 namespace test 161 { 162 TEST(MyTest, SimpleTest) 163 { 164 TestReferenceData data; 165 166 TestReferenceChecker checker(data.rootChecker()); 167 checker.checkInteger(functionToTest(3), "ValueWith3"); 168 checker.checkInteger(functionToTest(5), "ValueWith5"); 169 TestReferenceChecker compound( 170 checker.checkCompound("CustomCompound", "Item")); 171 compound.checkInteger(function2ToTest(3), "ValueWith3"); 172 compound.checkInteger(function2ToTest(5), "ValueWith5"); 173 checker.checkInteger(functionToTest(4), "ValueWith4"); 174 checker.checkVector(functionProducingRVec(), "Describe The RVec"); 175 } 176 } // namespace test 177 } // namespace gmx 178 * \endcode 179 * 180 * If rootChecker() is never called, no comparison is done (i.e., missing 181 * reference data file is not reported as an error, nor is empty reference data 182 * file created in write mode). 183 * 184 * For floating-point comparisons, the reference data should be generated in 185 * double precision (currently, no warning is provided even if this is not the 186 * case, but the double precision tests will then very likely fail). 187 * 188 * \inlibraryapi 189 * \ingroup module_testutils 190 */ 191 class TestReferenceData 192 { 193 public: 194 /*! \brief 195 * Initializes the reference data in the global mode. 196 */ 197 TestReferenceData(); 198 /*! \brief 199 * Initializes the reference data in a specific mode. 200 * 201 * This function is only useful for self-testing the reference data 202 * framework. As such, it also puts the framework in a state where it 203 * logs additional internal information for failures to help diagnosing 204 * problems in the framework, and stores the reference data in a 205 * temporary directory instead of the source tree. 206 * The default constructor should be used in tests utilizing this class. 207 */ 208 explicit TestReferenceData(ReferenceDataMode mode); 209 /*! \brief 210 * Frees reference data structures. 211 * 212 * The reference data is written out if necessary automatically when 213 * the test finishes. 214 */ 215 ~TestReferenceData(); 216 217 /*! \brief 218 * Returns a root-level checker object for comparisons. 219 * 220 * Each call returns an independent instance. 221 */ 222 TestReferenceChecker rootChecker(); 223 224 private: 225 std::shared_ptr<internal::TestReferenceDataImpl> impl_; 226 227 GMX_DISALLOW_COPY_AND_ASSIGN(TestReferenceData); 228 }; 229 230 /*! \libinternal \brief 231 * Handles comparison to test reference data. 232 * 233 * Every check*() method takes an id string as th 234 * e last parameter. This id is 235 * used to uniquely identify the value in the reference data, and it makes the 236 * output XML more human-friendly and more robust to errors. The id can be 237 * NULL; in this case, multiple elements with no id are created, and they will 238 * be matched in the same order as in which they are created. The 239 * checkCompound() method can be used to create a set of reference values 240 * grouped together. In this case, all check*() calls using the returned child 241 * TestReferenceChecker object will create the reference data within this 242 * group, and the ids only need to be unique within the compound. Compounds 243 * can be nested. 244 * 245 * For usage example, see TestReferenceData. 246 * 247 * Copies of this class behave have independent internal state. 248 * 249 * \inlibraryapi 250 * \ingroup module_testutils 251 */ 252 class TestReferenceChecker 253 { 254 public: 255 /*! \brief 256 * Creates a checker that cannot be used for checking. 257 * 258 * Attempting to call the check methods generates an assert. 259 * It is possible to check whether the checker is initialized by 260 * calling isValid(). 261 * This constructor exists to allow declaring checker variables that 262 * will receive their value later without resorting to dynamic 263 * allocation. 264 */ 265 TestReferenceChecker(); 266 //! Creates a deep copy of the other checker. 267 explicit TestReferenceChecker(const TestReferenceChecker& other); 268 //! Moves the checker. 269 TestReferenceChecker(TestReferenceChecker&& other) noexcept; 270 ~TestReferenceChecker(); 271 272 //! Prevents implicit copying during assignment. 273 TestReferenceChecker& operator=(const TestReferenceChecker&) = delete; 274 //! Assigns a test reference checker. 275 TestReferenceChecker& operator=(TestReferenceChecker&& other) noexcept; 276 277 //! Returns whether the checker is initialized. 278 bool isValid() const; 279 //! Allows testing whether the checker is initialized directly with if. 280 explicit operator bool() const { return isValid(); } 281 282 /*! \brief 283 * Sets the tolerance for floating-point comparisons. 284 * 285 * All following floating-point comparisons using this checker will use 286 * the new tolerance. Child checkers created with checkCompound() 287 * will inherit the tolerance from their parent checker at the time 288 * checkCompound() is called. 289 * 290 * Does not throw. 291 */ 292 void setDefaultTolerance(const FloatingPointTolerance& tolerance); 293 294 /*! \brief 295 * Checks that all reference values have been compared against. 296 * 297 * All values under the compound represented by this checker are 298 * checked, and a non-fatal Google Test assertion is produced if some 299 * values have not been used. 300 * 301 * If not called explicitly, the same check will be done for all 302 * reference data values when the test ends. 303 * 304 * This method also marks the values used, so that subsequent checks 305 * (including the check at the end of the test) will not produce 306 * another assertion about the same values. 307 */ 308 void checkUnusedEntries(); 309 310 /*! \brief Disables checking for unused entries 311 * 312 * \see checkUnusedEntries() 313 */ 314 void disableUnusedEntriesCheck(); 315 316 /*! \brief 317 * Checks whether a data item is present. 318 * 319 * \param[in] bPresent Whether to check for presence or absence. 320 * \param[in] id Unique identifier of the item to check. 321 * \returns true if bPresent was true and the data item was found. 322 * 323 * If \p bPresent is true, checks that a data item with \p id is 324 * present, otherwise checks that the data item is absent. 325 * If the check fails, a non-fatal Google Test assertion is generated. 326 * 327 * If reference data is being written, the check always succeeds and the 328 * return value is \p bPresent. 329 * 330 * The main use of this method is to assign meaning for missing 331 * reference data. Example use: 332 * \code 333 if (checker.checkPresent(bHaveVelocities, "Velocities")) 334 { 335 // <check the velocities> 336 } 337 * \endcode 338 */ 339 bool checkPresent(bool bPresent, const char* id); 340 341 /*! \brief 342 * Initializes comparison of a group of related data items. 343 * 344 * \param[in] type Informational type for the compound. 345 * \param[in] id Unique identifier for the compound among its 346 * siblings. 347 * \returns Checker to use for comparison within the compound. 348 * 349 * All checks performed with the returned checker only 350 * need to have unique ids within the compound, not globally. 351 * 352 * Compound structures can be nested. 353 */ 354 TestReferenceChecker checkCompound(const char* type, const char* id); 355 //! \copydoc checkCompound(const char *, const char *) 356 TestReferenceChecker checkCompound(const char* type, const std::string& id); 357 358 //! Check a single boolean value. 359 void checkBoolean(bool value, const char* id); 360 //! Check a single string value. 361 void checkString(const char* value, const char* id); 362 //! Check a single string value. 363 void checkString(const std::string& value, const char* id); 364 /*! \brief 365 * Check a multi-line string value. 366 * 367 * This method works as checkString(), but should be used for long 368 * strings that may contain, e.g., newlines. Typically used to check 369 * formatted output, and attempts to make the output XML such that it 370 * is easier to edit by hand to set the desired output formatting. 371 */ 372 void checkTextBlock(const std::string& value, const char* id); 373 //! Check a single unsigned char value. 374 void checkUChar(unsigned char value, const char* id); 375 //! Check a single integer value. 376 void checkInteger(int value, const char* id); 377 //! Check a single int32 value. 378 void checkInt32(int32_t value, const char* id); 379 //! Check a single uint32 value. 380 void checkUInt32(uint32_t value, const char* id); 381 //! Check a single int64 value. 382 void checkInt64(int64_t value, const char* id); 383 //! Check a single uint64 value. 384 void checkUInt64(uint64_t value, const char* id); 385 //! Check a single single-precision floating point value. 386 void checkFloat(float value, const char* id); 387 //! Check a single double-precision floating point value. 388 void checkDouble(double value, const char* id); 389 //! Check a single floating point value. 390 void checkReal(float value, const char* id); 391 //! Check a single floating point value. 392 void checkReal(double value, const char* id); 393 //! Check a vector of three integer values. 394 void checkVector(const int value[3], const char* id); 395 //! Check a vector of three single-precision floating point values. 396 void checkVector(const float value[3], const char* id); 397 //! Check a vector of three double-precision floating point values. 398 void checkVector(const double value[3], const char* id); 399 //! Check a BasicVector of ints, ie. IVec 400 void checkVector(const BasicVector<int>& value, const char* id); 401 //! Check a BasicVector of floats, ie. RVec 402 void checkVector(const BasicVector<float>& value, const char* id); 403 //! Check a BasicVector of doubles, ie. DVec 404 void checkVector(const BasicVector<double>& value, const char* id); 405 //! Check a single floating-point value from a string. 406 void checkRealFromString(const std::string& value, const char* id); 407 //! Checks a any value that contains a supported simple type. 408 void checkAny(const Any& value, const char* id); 409 //! Checks a key-value tree rooted at a object. 410 void checkKeyValueTreeObject(const KeyValueTreeObject& tree, const char* id); 411 //! Checks a generic key-value tree value. 412 void checkKeyValueTreeValue(const KeyValueTreeValue& value, const char* id); 413 414 /*! \name Methods to read values from reference data 415 * 416 * These methods assume that a value with the given `id` has already 417 * been created in the test with `check*()` methods, and that it has 418 * the correct type. 419 * 420 * Currently, these methods do not work correctly if the reference data 421 * file does not exist, so a test using them may fail with exceptions 422 * before the reference data has been generated. 423 * \{ 424 */ 425 //! Reads an unsigned char value. 426 unsigned char readUChar(const char* id); 427 //! Reads an integer value. 428 int readInteger(const char* id); 429 //! Reads a 32-bit integer value. 430 int32_t readInt32(const char* id); 431 //! Reads a 64-bit integer value. 432 int64_t readInt64(const char* id); 433 //! Reads a float value. 434 float readFloat(const char* id); 435 //! Reads a double value. 436 double readDouble(const char* id); 437 //! Reads a string value. 438 std::string readString(const char* id); 439 //! \} 440 441 /*! \name Overloaded versions of simple checker methods 442 * 443 * These methods provide overloads under a single name for all the 444 * methods checkBoolean(), checkString(), checkReal() and checkVector(). 445 * They are provided mainly to allow template implementations (such as 446 * checkSequence()). Typically callers should use the individually 447 * named versions for greater clarity. 448 * \{ 449 */ 450 //! Check a single boolean value. checkValue(bool value,const char * id)451 void checkValue(bool value, const char* id) { checkBoolean(value, id); } 452 //! Check a single string value. checkValue(const char * value,const char * id)453 void checkValue(const char* value, const char* id) { checkString(value, id); } 454 //! Check a single string value. checkValue(const std::string & value,const char * id)455 void checkValue(const std::string& value, const char* id) { checkString(value, id); } 456 //! Check a single signed integer value checkValue(int value,const char * id)457 void checkValue(int value, const char* id) { checkInteger(value, id); } 458 //! Check a single signed integer value of width 64 bits. checkValue(int64_t value,const char * id)459 void checkValue(int64_t value, const char* id) { checkInt64(value, id); } 460 //! Check a single unsigned integer value of width 64 bits. checkValue(uint64_t value,const char * id)461 void checkValue(uint64_t value, const char* id) { checkUInt64(value, id); } 462 //! Check a single single-precision floating point value. checkValue(float value,const char * id)463 void checkValue(float value, const char* id) { checkFloat(value, id); } 464 //! Check a single double-precision floating point value. checkValue(double value,const char * id)465 void checkValue(double value, const char* id) { checkDouble(value, id); } 466 //! Check a vector of three integer values. checkValue(const int value[3],const char * id)467 void checkValue(const int value[3], const char* id) { checkVector(value, id); } 468 //! Check a vector of three single-precision floating point values. checkValue(const float value[3],const char * id)469 void checkValue(const float value[3], const char* id) { checkVector(value, id); } 470 //! Check a vector of three double-precision floating point values. checkValue(const double value[3],const char * id)471 void checkValue(const double value[3], const char* id) { checkVector(value, id); } 472 //! Check a BasicVector of integer values, ie. IVec. checkValue(const BasicVector<int> & value,const char * id)473 void checkValue(const BasicVector<int>& value, const char* id) { checkVector(value, id); } 474 //! Check a BasicVector of float values, ie. RVec. checkValue(const BasicVector<float> & value,const char * id)475 void checkValue(const BasicVector<float>& value, const char* id) { checkVector(value, id); } 476 //! Check a BasicVector of double values, ie. DVec. checkValue(const BasicVector<double> & value,const char * id)477 void checkValue(const BasicVector<double>& value, const char* id) { checkVector(value, id); } 478 //! Check a generic key-value tree value. checkValue(const KeyValueTreeValue & value,const char * id)479 void checkValue(const KeyValueTreeValue& value, const char* id) 480 { 481 checkKeyValueTreeValue(value, id); 482 } 483 /*!\}*/ 484 485 /*! \brief 486 * Generic method to check a sequence of simple values. 487 * 488 * \tparam Iterator Input iterator that allows multiple (two) passes. 489 * Value type must be one of those accepted by checkValue(), or 490 * implicitly convertible to one. 491 * \param[in] begin Iterator to the start of the range to check. 492 * \param[in] end Iterator to the end of the range to check. 493 * \param[in] id Unique identifier for the sequence among its 494 * siblings. 495 */ 496 template<class Iterator> checkSequence(Iterator begin,Iterator end,const char * id)497 void checkSequence(Iterator begin, Iterator end, const char* id) 498 { 499 typename std::iterator_traits<Iterator>::difference_type length = std::distance(begin, end); 500 TestReferenceChecker compound(checkSequenceCompound(id, length)); 501 for (Iterator i = begin; i != end; ++i) 502 { 503 compound.checkValue(*i, nullptr); 504 } 505 } 506 /*! \brief 507 * Generic method to check a sequence of custom values. 508 * 509 * \tparam Iterator Input iterator that allows multiple (two) passes. 510 * \tparam ItemChecker Functor to check an individual value. Signature 511 * void(TestReferenceChecker *, const T &), where T is the value 512 * type of \p Iterator. 513 * \param[in] begin Iterator to the start of the range to check. 514 * \param[in] end Iterator to the end of the range to check. 515 * \param[in] id Unique identifier for the sequence among its 516 * siblings. 517 * \param[in] checkItem Functor to check an individual item. 518 * 519 * This method creates a compound checker \c compound within which all 520 * values of the sequence are checked. Calls \c checkItem(&compound, *i) 521 * with that compound for each iterator \c i in the range [begin, end). 522 * \p checkItem should use the various check methods in the passed 523 * checker to check each value. 524 * 525 * This method can be used to check a sequence made of compound types. 526 * Typically \p checkItem will create a compound within the passed 527 * checker to check different aspects of the value that was passed 528 * to it. Either NULL or a unique identifier string must be used for 529 * the id value of that compound. */ 530 template<class Iterator, class ItemChecker> checkSequence(Iterator begin,Iterator end,const char * id,ItemChecker checkItem)531 void checkSequence(Iterator begin, Iterator end, const char* id, ItemChecker checkItem) 532 { 533 typename std::iterator_traits<Iterator>::difference_type length = std::distance(begin, end); 534 TestReferenceChecker compound(checkSequenceCompound(id, length)); 535 for (Iterator i = begin; i != end; ++i) 536 { 537 checkItem(&compound, *i); 538 } 539 } 540 /*! \brief 541 * Check an array of values. 542 * 543 * \tparam T Type of values to check. Should be one of those accepted 544 * by checkValue(), or implicitly convertible to one. 545 * 546 * \param[in] length Number of values to check. 547 * \param[in] values Pointer to the first value to check. 548 * \param[in] id Unique identifier for the sequence among its 549 * siblings. 550 * 551 * This is a convenience method that delegates all work to 552 * checkSequence(). 553 */ 554 template<typename T> checkSequenceArray(size_t length,const T * values,const char * id)555 void checkSequenceArray(size_t length, const T* values, const char* id) 556 { 557 checkSequence(values, values + length, id); 558 } 559 /*! \brief 560 * Convenience method for checking that a sequence is empty. 561 * 562 * \param[in] id Unique identifier for the sequence among its 563 * siblings. 564 * 565 * This method provides a convenient solution for a case where there is 566 * implicitly a sequence to be checked, but there is no pointer 567 * available to the values since the sequence is empty. 568 * Since this method does not require the type of the values, it can be 569 * used in such cases easily. 570 */ 571 void checkEmptySequence(const char* id); 572 /*! \brief 573 * Initializes a compound for a sequence of items. 574 * 575 * \param[in] id Unique identifier for the sequence among its 576 * siblings. 577 * \param[in] length Number of items that will be in the sequence. 578 * \returns Checker to use for comparison within the sequence. 579 * 580 * This method can be used to check custom sequences where 581 * checkSequence() is not appropriate. 582 */ 583 TestReferenceChecker checkSequenceCompound(const char* id, size_t length); 584 585 private: 586 class Impl; 587 588 /*! \brief 589 * Constructs a checker with a specific internal state. 590 * 591 * Is private to only allow users of this class to create instances 592 * using TestReferenceData::rootChecker() or checkCompound() 593 * (or by copying). 594 */ 595 explicit TestReferenceChecker(Impl* impl); 596 597 PrivateImplPointer<Impl> impl_; 598 599 /*! \brief 600 * Needed to expose the constructor only to TestReferenceData. 601 */ 602 friend class TestReferenceData; 603 }; 604 605 } // namespace test 606 } // namespace gmx 607 608 #endif 609