1 /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying 2 file Copyright.txt or https://cmake.org/licensing for details. */ 3 #pragma once 4 5 #include "cmConfigure.h" // IWYU pragma: keep 6 7 #include <chrono> 8 #include <ctime> 9 #include <map> 10 #include <memory> 11 #include <sstream> 12 #include <string> 13 #include <vector> 14 15 #include <cm/string_view> 16 17 #include "cmDuration.h" 18 #include "cmProcessOutput.h" 19 20 class cmCTestBuildHandler; 21 class cmCTestBuildAndTestHandler; 22 class cmCTestCoverageHandler; 23 class cmCTestScriptHandler; 24 class cmCTestTestHandler; 25 class cmCTestUpdateHandler; 26 class cmCTestConfigureHandler; 27 class cmCTestMemCheckHandler; 28 class cmCTestSubmitHandler; 29 class cmCTestUploadHandler; 30 class cmCTestStartCommand; 31 class cmGeneratedFileStream; 32 class cmMakefile; 33 class cmXMLWriter; 34 35 /** \class cmCTest 36 * \brief Represents a ctest invocation. 37 * 38 * This class represents a ctest invocation. It is the top level class when 39 * running ctest. 40 * 41 */ 42 class cmCTest 43 { 44 public: 45 using Encoding = cmProcessOutput::Encoding; 46 /** Enumerate parts of the testing and submission process. */ 47 enum Part 48 { 49 PartStart, 50 PartUpdate, 51 PartConfigure, 52 PartBuild, 53 PartTest, 54 PartCoverage, 55 PartMemCheck, 56 PartSubmit, 57 PartNotes, 58 PartExtraFiles, 59 PartUpload, 60 PartDone, 61 PartCount // Update names in constructor when adding a part 62 }; 63 64 /** Get a testing part id from its string name. Returns PartCount 65 if the string does not name a valid part. */ 66 Part GetPartFromName(const std::string& name); 67 68 /** Process Command line arguments */ 69 int Run(std::vector<std::string>&, std::string* output = nullptr); 70 71 /** 72 * Initialize and finalize testing 73 */ 74 bool InitializeFromCommand(cmCTestStartCommand* command); 75 void Finalize(); 76 77 /** 78 * Process the dashboard client steps. 79 * 80 * Steps are enabled using SetTest() 81 * 82 * The execution of the steps (or #Part) should look like this: 83 * 84 * /code 85 * ctest foo; 86 * foo.Initialize(); 87 * // Set some things on foo 88 * foo.ProcessSteps(); 89 * foo.Finalize(); 90 * /endcode 91 * 92 * \sa Initialize(), Finalize(), Part, PartInfo, SetTest() 93 */ 94 int ProcessSteps(); 95 96 /** 97 * A utility function that returns the nightly time 98 */ 99 struct tm* GetNightlyTime(std::string const& str, bool tomorrowtag); 100 101 /** 102 * Is the tomorrow tag set? 103 */ 104 bool GetTomorrowTag() const; 105 106 /** 107 * Try to run tests of the project 108 */ 109 int TestDirectory(bool memcheck); 110 111 /** what is the configuration type, e.g. Debug, Release etc. */ 112 std::string const& GetConfigType(); 113 cmDuration GetTimeOut() const; 114 void SetTimeOut(cmDuration t); 115 116 cmDuration GetGlobalTimeout() const; 117 118 /** how many test to run at the same time */ 119 int GetParallelLevel() const; 120 void SetParallelLevel(int); 121 122 unsigned long GetTestLoad() const; 123 void SetTestLoad(unsigned long); 124 125 /** 126 * Check if CTest file exists 127 */ 128 bool CTestFileExists(const std::string& filename); 129 bool AddIfExists(Part part, const std::string& file); 130 131 /** 132 * Set the cmake test 133 */ 134 bool SetTest(const std::string&, bool report = true); 135 136 /** 137 * Set the cmake test mode (experimental, nightly, continuous). 138 */ 139 void SetTestModel(int mode); 140 int GetTestModel() const; 141 142 std::string GetTestModelString(); 143 static int GetTestModelFromString(const std::string& str); 144 static std::string CleanString(const std::string& str, 145 std::string::size_type spos = 0); 146 std::string GetCTestConfiguration(const std::string& name); 147 void SetCTestConfiguration(const char* name, const std::string& value, 148 bool suppress = false); 149 void EmptyCTestConfiguration(); 150 151 std::string GetSubmitURL(); 152 153 /** 154 * constructor and destructor 155 */ 156 cmCTest(); 157 ~cmCTest(); 158 159 cmCTest(const cmCTest&) = delete; 160 cmCTest& operator=(const cmCTest&) = delete; 161 162 /** Set the notes files to be created. */ 163 void SetNotesFiles(const std::string& notes); 164 165 void PopulateCustomVector(cmMakefile* mf, const std::string& definition, 166 std::vector<std::string>& vec); 167 void PopulateCustomInteger(cmMakefile* mf, const std::string& def, int& val); 168 169 /** Get the current time as string */ 170 std::string CurrentTime(); 171 172 /** tar/gzip and then base 64 encode a file */ 173 std::string Base64GzipEncodeFile(std::string const& file); 174 /** base64 encode a file */ 175 std::string Base64EncodeFile(std::string const& file); 176 177 /** 178 * Return the time remaining that the script is allowed to run in 179 * seconds if the user has set the variable CTEST_TIME_LIMIT. If that has 180 * not been set it returns a very large duration. 181 */ 182 cmDuration GetRemainingTimeAllowed(); 183 184 static cmDuration MaxDuration(); 185 186 /** 187 * Open file in the output directory and set the stream 188 */ 189 bool OpenOutputFile(const std::string& path, const std::string& name, 190 cmGeneratedFileStream& stream, bool compress = false); 191 192 /** Should we only show what we would do? */ 193 bool GetShowOnly(); 194 195 bool GetOutputAsJson(); 196 197 int GetOutputAsJsonVersion(); 198 199 bool ShouldUseHTTP10() const; 200 201 bool ShouldPrintLabels() const; 202 203 bool ShouldCompressTestOutput(); 204 bool CompressString(std::string& str); 205 206 bool GetStopOnFailure() const; 207 void SetStopOnFailure(bool stop); 208 209 std::chrono::system_clock::time_point GetStopTime() const; 210 void SetStopTime(std::string const& time); 211 212 /** Used for parallel ctest job scheduling */ 213 std::string GetScheduleType() const; 214 void SetScheduleType(std::string const& type); 215 216 /** The max output width */ 217 int GetMaxTestNameWidth() const; 218 void SetMaxTestNameWidth(int w); 219 220 /** 221 * Run a single executable command and put the stdout and stderr 222 * in output. 223 * 224 * If verbose is false, no user-viewable output from the program 225 * being run will be generated. 226 * 227 * If timeout is specified, the command will be terminated after 228 * timeout expires. Timeout is specified in seconds. 229 * 230 * Argument retVal should be a pointer to the location where the 231 * exit code will be stored. If the retVal is not specified and 232 * the program exits with a code other than 0, then the this 233 * function will return false. 234 */ 235 bool RunCommand(std::vector<std::string> const& args, std::string* stdOut, 236 std::string* stdErr, int* retVal = nullptr, 237 const char* dir = nullptr, 238 cmDuration timeout = cmDuration::zero(), 239 Encoding encoding = cmProcessOutput::Auto); 240 241 /** 242 * Clean/make safe for xml the given value such that it may be used as 243 * one of the key fields by CDash when computing the buildid. 244 */ 245 static std::string SafeBuildIdField(const std::string& value); 246 247 /** Start CTest XML output file */ 248 void StartXML(cmXMLWriter& xml, bool append); 249 250 /** End CTest XML output file */ 251 void EndXML(cmXMLWriter& xml); 252 253 /** 254 * Run command specialized for make and configure. Returns process status 255 * and retVal is return value or exception. 256 */ 257 int RunMakeCommand(const std::string& command, std::string& output, 258 int* retVal, const char* dir, cmDuration timeout, 259 std::ostream& ofs, 260 Encoding encoding = cmProcessOutput::Auto); 261 262 /** Return the current tag */ 263 std::string GetCurrentTag(); 264 265 /** Get the path to the build tree */ 266 std::string GetBinaryDir(); 267 268 /** 269 * Get the short path to the file. 270 * 271 * This means if the file is in binary or 272 * source directory, it will become /.../relative/path/to/file 273 */ 274 std::string GetShortPathToFile(const std::string& fname); 275 276 enum 277 { 278 UNKNOWN = -1, 279 EXPERIMENTAL = 0, 280 NIGHTLY = 1, 281 CONTINUOUS = 2, 282 }; 283 284 /** provide some more detailed info on the return code for ctest */ 285 enum 286 { 287 UPDATE_ERRORS = 0x01, 288 CONFIGURE_ERRORS = 0x02, 289 BUILD_ERRORS = 0x04, 290 TEST_ERRORS = 0x08, 291 MEMORY_ERRORS = 0x10, 292 COVERAGE_ERRORS = 0x20, 293 SUBMIT_ERRORS = 0x40 294 }; 295 296 /** Are we producing XML */ 297 bool GetProduceXML(); 298 void SetProduceXML(bool v); 299 300 /** 301 * Run command specialized for tests. Returns process status and retVal is 302 * return value or exception. If environment is non-null, it is used to set 303 * environment variables prior to running the test. After running the test, 304 * environment variables are restored to their previous values. 305 */ 306 int RunTest(std::vector<const char*> args, std::string* output, int* retVal, 307 std::ostream* logfile, cmDuration testTimeOut, 308 std::vector<std::string>* environment, 309 Encoding encoding = cmProcessOutput::Auto); 310 311 /** 312 * Get the handler object 313 */ 314 cmCTestBuildHandler* GetBuildHandler(); 315 cmCTestBuildAndTestHandler* GetBuildAndTestHandler(); 316 cmCTestCoverageHandler* GetCoverageHandler(); 317 cmCTestScriptHandler* GetScriptHandler(); 318 cmCTestTestHandler* GetTestHandler(); 319 cmCTestUpdateHandler* GetUpdateHandler(); 320 cmCTestConfigureHandler* GetConfigureHandler(); 321 cmCTestMemCheckHandler* GetMemCheckHandler(); 322 cmCTestSubmitHandler* GetSubmitHandler(); 323 cmCTestUploadHandler* GetUploadHandler(); 324 325 /** 326 * Set the CTest variable from CMake variable 327 */ 328 bool SetCTestConfigurationFromCMakeVariable(cmMakefile* mf, 329 const char* dconfig, 330 const std::string& cmake_var, 331 bool suppress = false); 332 333 /** Decode a URL to the original string. */ 334 static std::string DecodeURL(const std::string&); 335 336 /** 337 * Should ctect configuration be updated. When using new style ctest 338 * script, this should be true. 339 */ 340 void SetSuppressUpdatingCTestConfiguration(bool val); 341 342 /** 343 * Add overwrite to ctest configuration. 344 * 345 * The format is key=value 346 */ 347 void AddCTestConfigurationOverwrite(const std::string& encstr); 348 349 /** Create XML file that contains all the notes specified */ 350 int GenerateNotesFile(std::vector<std::string> const& files); 351 352 /** Create XML file to indicate that build is complete */ 353 int GenerateDoneFile(); 354 355 /** Submit extra files to the server */ 356 bool SubmitExtraFiles(const std::string& files); 357 bool SubmitExtraFiles(std::vector<std::string> const& files); 358 359 /** Set the output log file name */ 360 void SetOutputLogFileName(const std::string& name); 361 362 /** Set the visual studio or Xcode config type */ 363 void SetConfigType(const std::string& ct); 364 365 /** Various log types */ 366 enum 367 { 368 DEBUG = 0, 369 OUTPUT, 370 HANDLER_OUTPUT, 371 HANDLER_PROGRESS_OUTPUT, 372 HANDLER_TEST_PROGRESS_OUTPUT, 373 HANDLER_VERBOSE_OUTPUT, 374 WARNING, 375 ERROR_MESSAGE, 376 OTHER 377 }; 378 379 /** Add log to the output */ 380 void Log(int logType, const char* file, int line, const char* msg, 381 bool suppress = false); 382 383 /** Color values */ 384 enum class Color 385 { 386 CLEAR_COLOR = 0, 387 RED = 31, 388 GREEN = 32, 389 YELLOW = 33, 390 BLUE = 34 391 }; 392 393 /** Get color code characters for a specific color */ 394 std::string GetColorCode(Color color) const; 395 396 /** The Build ID is assigned by CDash */ 397 void SetBuildID(const std::string& id); 398 std::string GetBuildID() const; 399 400 /** Add file to be submitted */ 401 void AddSubmitFile(Part part, const std::string& name); 402 std::vector<std::string> const& GetSubmitFiles(Part part) const; 403 void ClearSubmitFiles(Part part); 404 405 /** 406 * Read the custom configuration files and apply them to the current ctest 407 */ 408 int ReadCustomConfigurationFileTree(const std::string& dir, cmMakefile* mf); 409 410 std::vector<std::string>& GetInitialCommandLineArguments(); 411 412 /** Set the group to submit to */ 413 void SetSpecificGroup(const char* group); 414 const char* GetSpecificGroup(); 415 416 void SetFailover(bool failover); 417 bool GetFailover() const; 418 419 bool GetTestProgressOutput() const; 420 421 bool GetVerbose() const; 422 bool GetExtraVerbose() const; 423 424 /** Direct process output to given streams. */ 425 void SetStreams(std::ostream* out, std::ostream* err); 426 427 void AddSiteProperties(cmXMLWriter& xml); 428 429 bool GetLabelSummary() const; 430 bool GetSubprojectSummary() const; 431 432 std::string GetCostDataFile(); 433 434 bool GetOutputTestOutputOnTestFailure() const; 435 436 const std::map<std::string, std::string>& GetDefinitions() const; 437 438 /** Return the number of times a test should be run */ 439 int GetRepeatCount() const; 440 441 enum class Repeat 442 { 443 Never, 444 UntilFail, 445 UntilPass, 446 AfterTimeout, 447 }; 448 Repeat GetRepeatMode() const; 449 450 enum class NoTests 451 { 452 Legacy, 453 Error, 454 Ignore 455 }; 456 NoTests GetNoTestsMode() const; 457 458 void GenerateSubprojectsOutput(cmXMLWriter& xml); 459 std::vector<std::string> GetLabelsForSubprojects(); 460 461 void SetRunCurrentScript(bool value); 462 463 private: 464 void SetPersistentOptionIfNotEmpty(const std::string& value, 465 const std::string& optionName); 466 void AddPersistentMultiOptionIfNotEmpty(const std::string& value, 467 const std::string& optionName); 468 469 int GenerateNotesFile(const std::string& files); 470 471 void BlockTestErrorDiagnostics(); 472 473 /** 474 * Initialize a dashboard run in the given build tree. The "command" 475 * argument is non-NULL when running from a command-driven (ctest_start) 476 * dashboard script, and NULL when running from the CTest command 477 * line. Note that a declarative dashboard script does not actually 478 * call this method because it sets CTEST_COMMAND to drive a build 479 * through the ctest command line. 480 */ 481 int Initialize(const char* binary_dir, cmCTestStartCommand* command); 482 483 /** parse the option after -D and convert it into the appropriate steps */ 484 bool AddTestsForDashboardType(std::string& targ); 485 486 /** read as "emit an error message for an unknown -D value" */ 487 void ErrorMessageUnknownDashDValue(std::string& val); 488 489 /** add a variable definition from a command line -D value */ 490 bool AddVariableDefinition(const std::string& arg); 491 492 /** set command line arguments read from a test preset */ 493 bool SetArgsFromPreset(const std::string& presetName, bool listPresets); 494 495 /** parse and process most common command line arguments */ 496 bool HandleCommandLineArguments(size_t& i, std::vector<std::string>& args, 497 std::string& errormsg); 498 499 #if !defined(_WIN32) 500 /** returns true iff the console supports progress output */ 501 static bool ConsoleIsNotDumb(); 502 #endif 503 504 /** returns true iff the console supports progress output */ 505 static bool ProgressOutputSupportedByConsole(); 506 507 /** returns true iff the console supports colored output */ 508 static bool ColoredOutputSupportedByConsole(); 509 510 /** handle the -S -SP and -SR arguments */ 511 void HandleScriptArguments(size_t& i, std::vector<std::string>& args, 512 bool& SRArgumentSpecified); 513 514 /** Reread the configuration file */ 515 bool UpdateCTestConfiguration(); 516 517 /** Create note from files. */ 518 int GenerateCTestNotesOutput(cmXMLWriter& xml, 519 std::vector<std::string> const& files); 520 521 /** Check if the argument is the one specified */ 522 static bool CheckArgument(const std::string& arg, cm::string_view varg1, 523 const char* varg2 = nullptr); 524 525 /** Output errors from a test */ 526 void OutputTestErrors(std::vector<char> const& process_output); 527 528 /** Handle the --test-action command line argument */ 529 bool HandleTestActionArgument(const char* ctestExec, size_t& i, 530 const std::vector<std::string>& args); 531 532 /** Handle the --test-model command line argument */ 533 bool HandleTestModelArgument(const char* ctestExec, size_t& i, 534 const std::vector<std::string>& args); 535 536 int RunCMakeAndTest(std::string* output); 537 int ExecuteTests(); 538 539 /** return true iff change directory was successful */ 540 bool TryToChangeDirectory(std::string const& dir); 541 542 struct Private; 543 std::unique_ptr<Private> Impl; 544 }; 545 546 class cmCTestLogWrite 547 { 548 public: cmCTestLogWrite(const char * data,size_t length)549 cmCTestLogWrite(const char* data, size_t length) 550 : Data(data) 551 , Length(length) 552 { 553 } 554 555 const char* Data; 556 size_t Length; 557 }; 558 559 inline std::ostream& operator<<(std::ostream& os, const cmCTestLogWrite& c) 560 { 561 if (!c.Length) { 562 return os; 563 } 564 os.write(c.Data, c.Length); 565 os.flush(); 566 return os; 567 } 568 569 #define cmCTestLog(ctSelf, logType, msg) \ 570 do { \ 571 std::ostringstream cmCTestLog_msg; \ 572 cmCTestLog_msg << msg; \ 573 (ctSelf)->Log(cmCTest::logType, __FILE__, __LINE__, \ 574 cmCTestLog_msg.str().c_str()); \ 575 } while (false) 576 577 #define cmCTestOptionalLog(ctSelf, logType, msg, suppress) \ 578 do { \ 579 std::ostringstream cmCTestLog_msg; \ 580 cmCTestLog_msg << msg; \ 581 (ctSelf)->Log(cmCTest::logType, __FILE__, __LINE__, \ 582 cmCTestLog_msg.str().c_str(), suppress); \ 583 } while (false) 584