1 /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying 2 file Copyright.txt or https://cmake.org/licensing for details. */ 3 #pragma once 4 5 #include "cmConfigure.h" // IWYU pragma: keep 6 7 #include <chrono> 8 #include <cstdint> 9 #include <iosfwd> 10 #include <map> 11 #include <set> 12 #include <string> 13 #include <utility> 14 #include <vector> 15 16 #include <stddef.h> 17 18 #include "cmsys/RegularExpression.hxx" 19 20 #include "cmCTest.h" 21 #include "cmCTestGenericHandler.h" 22 #include "cmCTestResourceSpec.h" 23 #include "cmDuration.h" 24 #include "cmListFileCache.h" 25 #include "cmValue.h" 26 27 class cmMakefile; 28 class cmXMLWriter; 29 30 /** \class cmCTestTestHandler 31 * \brief A class that handles ctest -S invocations 32 * 33 */ 34 class cmCTestTestHandler : public cmCTestGenericHandler 35 { 36 friend class cmCTestRunTest; 37 friend class cmCTestMultiProcessHandler; 38 39 public: 40 using Superclass = cmCTestGenericHandler; 41 42 /** 43 * The main entry point for this class 44 */ 45 int ProcessHandler() override; 46 47 /** 48 * When both -R and -I are used should the resulting test list be the 49 * intersection or the union of the lists. By default it is the 50 * intersection. 51 */ SetUseUnion(bool val)52 void SetUseUnion(bool val) { this->UseUnion = val; } 53 54 /** 55 * Set whether or not CTest should only execute the tests that failed 56 * on the previous run. By default this is false. 57 */ SetRerunFailed(bool val)58 void SetRerunFailed(bool val) { this->RerunFailed = val; } 59 60 /** 61 * This method is called when reading CTest custom file 62 */ 63 void PopulateCustomVectors(cmMakefile* mf) override; 64 65 //! Control the use of the regular expresisons, call these methods to turn 66 /// them on 67 void UseIncludeRegExp(); 68 void UseExcludeRegExp(); 69 void SetIncludeRegExp(const std::string&); 70 void SetExcludeRegExp(const std::string&); 71 SetMaxIndex(int n)72 void SetMaxIndex(int n) { this->MaxIndex = n; } GetMaxIndex()73 int GetMaxIndex() { return this->MaxIndex; } 74 SetTestOutputSizePassed(int n)75 void SetTestOutputSizePassed(int n) 76 { 77 this->CustomMaximumPassedTestOutputSize = n; 78 } SetTestOutputSizeFailed(int n)79 void SetTestOutputSizeFailed(int n) 80 { 81 this->CustomMaximumFailedTestOutputSize = n; 82 } 83 84 //! pass the -I argument down 85 void SetTestsToRunInformation(cmValue); 86 87 cmCTestTestHandler(); 88 89 /* 90 * Add the test to the list of tests to be executed 91 */ 92 bool AddTest(const std::vector<std::string>& args); 93 94 /* 95 * Set tests properties 96 */ 97 bool SetTestsProperties(const std::vector<std::string>& args); 98 99 /** 100 * Set directory properties 101 */ 102 bool SetDirectoryProperties(const std::vector<std::string>& args); 103 104 void Initialize() override; 105 106 struct cmCTestTestResourceRequirement 107 { 108 std::string ResourceType; 109 int SlotsNeeded; 110 int UnitsNeeded; 111 112 bool operator==(const cmCTestTestResourceRequirement& other) const; 113 bool operator!=(const cmCTestTestResourceRequirement& other) const; 114 }; 115 116 // NOTE: This struct is Saved/Restored 117 // in cmCTestTestHandler, if you add to this class 118 // then you must add the new members to that code or 119 // ctest -j N will break for that feature 120 struct cmCTestTestProperties 121 { 122 std::string Name; 123 std::string Directory; 124 std::vector<std::string> Args; 125 std::vector<std::string> RequiredFiles; 126 std::vector<std::string> Depends; 127 std::vector<std::string> AttachedFiles; 128 std::vector<std::string> AttachOnFail; 129 std::vector<std::pair<cmsys::RegularExpression, std::string>> 130 ErrorRegularExpressions; 131 std::vector<std::pair<cmsys::RegularExpression, std::string>> 132 RequiredRegularExpressions; 133 std::vector<std::pair<cmsys::RegularExpression, std::string>> 134 SkipRegularExpressions; 135 std::vector<std::pair<cmsys::RegularExpression, std::string>> 136 TimeoutRegularExpressions; 137 std::map<std::string, std::string> Measurements; 138 bool IsInBasedOnREOptions; 139 bool WillFail; 140 bool Disabled; 141 float Cost; 142 int PreviousRuns; 143 bool RunSerial; 144 cmDuration Timeout; 145 bool ExplicitTimeout; 146 cmDuration AlternateTimeout; 147 int Index; 148 // Requested number of process slots 149 int Processors; 150 bool WantAffinity; 151 std::vector<size_t> Affinity; 152 // return code of test which will mark test as "not run" 153 int SkipReturnCode; 154 std::vector<std::string> Environment; 155 std::vector<std::string> EnvironmentModification; 156 std::vector<std::string> Labels; 157 std::set<std::string> LockedResources; 158 std::set<std::string> FixturesSetup; 159 std::set<std::string> FixturesCleanup; 160 std::set<std::string> FixturesRequired; 161 std::set<std::string> RequireSuccessDepends; 162 std::vector<std::vector<cmCTestTestResourceRequirement>> ResourceGroups; 163 // Private test generator properties used to track backtraces 164 cmListFileBacktrace Backtrace; 165 }; 166 167 struct cmCTestTestResult 168 { 169 std::string Name; 170 std::string Path; 171 std::string Reason; 172 std::string FullCommandLine; 173 std::string Environment; 174 cmDuration ExecutionTime; 175 std::int64_t ReturnValue; 176 int Status; 177 std::string ExceptionStatus; 178 bool CompressOutput; 179 std::string CompletionStatus; 180 std::string CustomCompletionStatus; 181 std::string Output; 182 std::string TestMeasurementsOutput; 183 int TestCount; 184 cmCTestTestProperties* Properties; 185 }; 186 187 struct cmCTestTestResultLess 188 { operatorcmCTestTestResultLess189 bool operator()(const cmCTestTestResult& lhs, 190 const cmCTestTestResult& rhs) const 191 { 192 return lhs.TestCount < rhs.TestCount; 193 } 194 }; 195 196 // add configurations to a search path for an executable 197 static void AddConfigurations(cmCTest* ctest, 198 std::vector<std::string>& attempted, 199 std::vector<std::string>& attemptedConfigs, 200 std::string filepath, std::string& filename); 201 202 // full signature static method to find an executable 203 static std::string FindExecutable(cmCTest* ctest, 204 const std::string& testCommand, 205 std::string& resultingConfig, 206 std::vector<std::string>& extraPaths, 207 std::vector<std::string>& failed); 208 209 static bool ParseResourceGroupsProperty( 210 const std::string& val, 211 std::vector<std::vector<cmCTestTestResourceRequirement>>& resourceGroups); 212 213 using ListOfTests = std::vector<cmCTestTestProperties>; 214 215 // Support for writing test results in JUnit XML format. 216 void SetJUnitXMLFileName(const std::string& id); 217 218 protected: 219 using SetOfTests = 220 std::set<cmCTestTestHandler::cmCTestTestResult, cmCTestTestResultLess>; 221 222 // compute a final test list 223 virtual int PreProcessHandler(); 224 virtual int PostProcessHandler(); 225 virtual void GenerateTestCommand(std::vector<std::string>& args, int test); 226 int ExecuteCommands(std::vector<std::string>& vec); 227 228 bool ProcessOptions(); 229 void LogTestSummary(const std::vector<std::string>& passed, 230 const std::vector<std::string>& failed, 231 const cmDuration& durationInSecs); 232 void LogDisabledTests(const std::vector<cmCTestTestResult>& disabledTests); 233 void LogFailedTests(const std::vector<std::string>& failed, 234 const SetOfTests& resultsSet); 235 bool GenerateXML(); 236 237 void WriteTestResultHeader(cmXMLWriter& xml, 238 cmCTestTestResult const& result); 239 void WriteTestResultFooter(cmXMLWriter& xml, 240 cmCTestTestResult const& result); 241 // Write attached test files into the xml 242 void AttachFiles(cmXMLWriter& xml, cmCTestTestResult& result); 243 void AttachFile(cmXMLWriter& xml, std::string const& file, 244 std::string const& name); 245 246 //! Clean test output to specified length 247 void CleanTestOutput(std::string& output, size_t length); 248 249 cmDuration ElapsedTestingTime; 250 251 using TestResultsVector = std::vector<cmCTestTestResult>; 252 TestResultsVector TestResults; 253 254 std::vector<std::string> CustomTestsIgnore; 255 std::string StartTest; 256 std::string EndTest; 257 std::chrono::system_clock::time_point StartTestTime; 258 std::chrono::system_clock::time_point EndTestTime; 259 bool MemCheck; 260 int CustomMaximumPassedTestOutputSize; 261 int CustomMaximumFailedTestOutputSize; 262 int MaxIndex; 263 264 public: 265 enum 266 { // Program statuses 267 NOT_RUN = 0, 268 TIMEOUT, 269 SEGFAULT, 270 ILLEGAL, 271 INTERRUPT, 272 NUMERICAL, 273 OTHER_FAULT, 274 FAILED, 275 BAD_COMMAND, 276 COMPLETED 277 }; 278 279 private: 280 /** 281 * Write test results in CTest's Test.xml format 282 */ 283 virtual void GenerateCTestXML(cmXMLWriter& xml); 284 285 /** 286 * Write test results in JUnit XML format 287 */ 288 bool WriteJUnitXML(); 289 290 void PrintLabelOrSubprojectSummary(bool isSubProject); 291 292 /** 293 * Run the tests for a directory and any subdirectories 294 */ 295 bool ProcessDirectory(std::vector<std::string>& passed, 296 std::vector<std::string>& failed); 297 298 /** 299 * Get the list of tests in directory and subdirectories. 300 */ 301 bool GetListOfTests(); 302 // compute the lists of tests that will actually run 303 // based on union regex and -I stuff 304 bool ComputeTestList(); 305 306 // compute the lists of tests that will actually run 307 // based on LastTestFailed.log 308 void ComputeTestListForRerunFailed(); 309 310 // add required setup/cleanup tests not already in the 311 // list of tests to be run and update dependencies between 312 // tests to account for fixture setup/cleanup 313 void UpdateForFixtures(ListOfTests& tests) const; 314 315 void UpdateMaxTestNameWidth(); 316 317 bool GetValue(const char* tag, std::string& value, std::istream& fin); 318 bool GetValue(const char* tag, int& value, std::istream& fin); 319 bool GetValue(const char* tag, size_t& value, std::istream& fin); 320 bool GetValue(const char* tag, bool& value, std::istream& fin); 321 bool GetValue(const char* tag, double& value, std::istream& fin); 322 /** 323 * Find the executable for a test 324 */ 325 std::string FindTheExecutable(const std::string& exe); 326 327 std::string GetTestStatus(cmCTestTestResult const&); 328 void ExpandTestsToRunInformation(size_t numPossibleTests); 329 void ExpandTestsToRunInformationForRerunFailed(); 330 331 std::vector<std::string> CustomPreTest; 332 std::vector<std::string> CustomPostTest; 333 334 std::vector<int> TestsToRun; 335 336 bool UseIncludeRegExpFlag; 337 bool UseExcludeRegExpFlag; 338 bool UseExcludeRegExpFirst; 339 std::string IncludeRegExp; 340 std::string ExcludeRegExp; 341 std::string ExcludeFixtureRegExp; 342 std::string ExcludeFixtureSetupRegExp; 343 std::string ExcludeFixtureCleanupRegExp; 344 std::vector<cmsys::RegularExpression> IncludeLabelRegularExpressions; 345 std::vector<cmsys::RegularExpression> ExcludeLabelRegularExpressions; 346 cmsys::RegularExpression IncludeTestsRegularExpression; 347 cmsys::RegularExpression ExcludeTestsRegularExpression; 348 349 bool UseResourceSpec; 350 cmCTestResourceSpec ResourceSpec; 351 std::string ResourceSpecFile; 352 353 void RecordCustomTestMeasurements(cmXMLWriter& xml, std::string content); 354 void CheckLabelFilter(cmCTestTestProperties& it); 355 void CheckLabelFilterExclude(cmCTestTestProperties& it); 356 void CheckLabelFilterInclude(cmCTestTestProperties& it); 357 358 std::string TestsToRunString; 359 bool UseUnion; 360 ListOfTests TestList; 361 size_t TotalNumberOfTests; 362 cmsys::RegularExpression AllTestMeasurementsRegex; 363 cmsys::RegularExpression SingleTestMeasurementRegex; 364 cmsys::RegularExpression CustomCompletionStatusRegex; 365 cmsys::RegularExpression CustomLabelRegex; 366 367 std::ostream* LogFile; 368 369 cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never; 370 int RepeatCount = 1; 371 bool RerunFailed; 372 373 std::string JUnitXMLFileName; 374 }; 375