1 /* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
2    file Copyright.txt or https://cmake.org/licensing for details.  */
3 #pragma once
4 
5 #include "cmConfigure.h" // IWYU pragma: keep
6 
7 #include <chrono>
8 #include <ctime>
9 #include <map>
10 #include <memory>
11 #include <sstream>
12 #include <string>
13 #include <vector>
14 
15 #include <cm/string_view>
16 
17 #include "cmDuration.h"
18 #include "cmProcessOutput.h"
19 
20 class cmCTestBuildHandler;
21 class cmCTestBuildAndTestHandler;
22 class cmCTestCoverageHandler;
23 class cmCTestScriptHandler;
24 class cmCTestTestHandler;
25 class cmCTestUpdateHandler;
26 class cmCTestConfigureHandler;
27 class cmCTestMemCheckHandler;
28 class cmCTestSubmitHandler;
29 class cmCTestUploadHandler;
30 class cmCTestStartCommand;
31 class cmGeneratedFileStream;
32 class cmMakefile;
33 class cmXMLWriter;
34 
35 /** \class cmCTest
36  * \brief Represents a ctest invocation.
37  *
38  * This class represents a ctest invocation. It is the top level class when
39  * running ctest.
40  *
41  */
42 class cmCTest
43 {
44 public:
45   using Encoding = cmProcessOutput::Encoding;
46   /** Enumerate parts of the testing and submission process.  */
47   enum Part
48   {
49     PartStart,
50     PartUpdate,
51     PartConfigure,
52     PartBuild,
53     PartTest,
54     PartCoverage,
55     PartMemCheck,
56     PartSubmit,
57     PartNotes,
58     PartExtraFiles,
59     PartUpload,
60     PartDone,
61     PartCount // Update names in constructor when adding a part
62   };
63 
64   /** Get a testing part id from its string name.  Returns PartCount
65       if the string does not name a valid part.  */
66   Part GetPartFromName(const std::string& name);
67 
68   /** Process Command line arguments */
69   int Run(std::vector<std::string>&, std::string* output = nullptr);
70 
71   /**
72    * Initialize and finalize testing
73    */
74   bool InitializeFromCommand(cmCTestStartCommand* command);
75   void Finalize();
76 
77   /**
78    * Process the dashboard client steps.
79    *
80    * Steps are enabled using SetTest()
81    *
82    * The execution of the steps (or #Part) should look like this:
83    *
84    * /code
85    * ctest foo;
86    * foo.Initialize();
87    * // Set some things on foo
88    * foo.ProcessSteps();
89    * foo.Finalize();
90    * /endcode
91    *
92    * \sa Initialize(), Finalize(), Part, PartInfo, SetTest()
93    */
94   int ProcessSteps();
95 
96   /**
97    * A utility function that returns the nightly time
98    */
99   struct tm* GetNightlyTime(std::string const& str, bool tomorrowtag);
100 
101   /**
102    * Is the tomorrow tag set?
103    */
104   bool GetTomorrowTag() const;
105 
106   /**
107    * Try to run tests of the project
108    */
109   int TestDirectory(bool memcheck);
110 
111   /** what is the configuration type, e.g. Debug, Release etc. */
112   std::string const& GetConfigType();
113   cmDuration GetTimeOut() const;
114   void SetTimeOut(cmDuration t);
115 
116   cmDuration GetGlobalTimeout() const;
117 
118   /** how many test to run at the same time */
119   int GetParallelLevel() const;
120   void SetParallelLevel(int);
121 
122   unsigned long GetTestLoad() const;
123   void SetTestLoad(unsigned long);
124 
125   /**
126    * Check if CTest file exists
127    */
128   bool CTestFileExists(const std::string& filename);
129   bool AddIfExists(Part part, const std::string& file);
130 
131   /**
132    * Set the cmake test
133    */
134   bool SetTest(const std::string&, bool report = true);
135 
136   /**
137    * Set the cmake test mode (experimental, nightly, continuous).
138    */
139   void SetTestModel(int mode);
140   int GetTestModel() const;
141 
142   std::string GetTestModelString();
143   static int GetTestModelFromString(const std::string& str);
144   static std::string CleanString(const std::string& str,
145                                  std::string::size_type spos = 0);
146   std::string GetCTestConfiguration(const std::string& name);
147   void SetCTestConfiguration(const char* name, const std::string& value,
148                              bool suppress = false);
149   void EmptyCTestConfiguration();
150 
151   std::string GetSubmitURL();
152 
153   /**
154    * constructor and destructor
155    */
156   cmCTest();
157   ~cmCTest();
158 
159   cmCTest(const cmCTest&) = delete;
160   cmCTest& operator=(const cmCTest&) = delete;
161 
162   /** Set the notes files to be created. */
163   void SetNotesFiles(const std::string& notes);
164 
165   void PopulateCustomVector(cmMakefile* mf, const std::string& definition,
166                             std::vector<std::string>& vec);
167   void PopulateCustomInteger(cmMakefile* mf, const std::string& def, int& val);
168 
169   /** Get the current time as string */
170   std::string CurrentTime();
171 
172   /** tar/gzip and then base 64 encode a file */
173   std::string Base64GzipEncodeFile(std::string const& file);
174   /** base64 encode a file */
175   std::string Base64EncodeFile(std::string const& file);
176 
177   /**
178    * Return the time remaining that the script is allowed to run in
179    * seconds if the user has set the variable CTEST_TIME_LIMIT. If that has
180    * not been set it returns a very large duration.
181    */
182   cmDuration GetRemainingTimeAllowed();
183 
184   static cmDuration MaxDuration();
185 
186   /**
187    * Open file in the output directory and set the stream
188    */
189   bool OpenOutputFile(const std::string& path, const std::string& name,
190                       cmGeneratedFileStream& stream, bool compress = false);
191 
192   /** Should we only show what we would do? */
193   bool GetShowOnly();
194 
195   bool GetOutputAsJson();
196 
197   int GetOutputAsJsonVersion();
198 
199   bool ShouldUseHTTP10() const;
200 
201   bool ShouldPrintLabels() const;
202 
203   bool ShouldCompressTestOutput();
204   bool CompressString(std::string& str);
205 
206   bool GetStopOnFailure() const;
207   void SetStopOnFailure(bool stop);
208 
209   std::chrono::system_clock::time_point GetStopTime() const;
210   void SetStopTime(std::string const& time);
211 
212   /** Used for parallel ctest job scheduling */
213   std::string GetScheduleType() const;
214   void SetScheduleType(std::string const& type);
215 
216   /** The max output width */
217   int GetMaxTestNameWidth() const;
218   void SetMaxTestNameWidth(int w);
219 
220   /**
221    * Run a single executable command and put the stdout and stderr
222    * in output.
223    *
224    * If verbose is false, no user-viewable output from the program
225    * being run will be generated.
226    *
227    * If timeout is specified, the command will be terminated after
228    * timeout expires. Timeout is specified in seconds.
229    *
230    * Argument retVal should be a pointer to the location where the
231    * exit code will be stored. If the retVal is not specified and
232    * the program exits with a code other than 0, then the this
233    * function will return false.
234    */
235   bool RunCommand(std::vector<std::string> const& args, std::string* stdOut,
236                   std::string* stdErr, int* retVal = nullptr,
237                   const char* dir = nullptr,
238                   cmDuration timeout = cmDuration::zero(),
239                   Encoding encoding = cmProcessOutput::Auto);
240 
241   /**
242    * Clean/make safe for xml the given value such that it may be used as
243    * one of the key fields by CDash when computing the buildid.
244    */
245   static std::string SafeBuildIdField(const std::string& value);
246 
247   /** Start CTest XML output file */
248   void StartXML(cmXMLWriter& xml, bool append);
249 
250   /** End CTest XML output file */
251   void EndXML(cmXMLWriter& xml);
252 
253   /**
254    * Run command specialized for make and configure. Returns process status
255    * and retVal is return value or exception.
256    */
257   int RunMakeCommand(const std::string& command, std::string& output,
258                      int* retVal, const char* dir, cmDuration timeout,
259                      std::ostream& ofs,
260                      Encoding encoding = cmProcessOutput::Auto);
261 
262   /** Return the current tag */
263   std::string GetCurrentTag();
264 
265   /** Get the path to the build tree */
266   std::string GetBinaryDir();
267 
268   /**
269    * Get the short path to the file.
270    *
271    * This means if the file is in binary or
272    * source directory, it will become /.../relative/path/to/file
273    */
274   std::string GetShortPathToFile(const std::string& fname);
275 
276   enum
277   {
278     UNKNOWN = -1,
279     EXPERIMENTAL = 0,
280     NIGHTLY = 1,
281     CONTINUOUS = 2,
282   };
283 
284   /** provide some more detailed info on the return code for ctest */
285   enum
286   {
287     UPDATE_ERRORS = 0x01,
288     CONFIGURE_ERRORS = 0x02,
289     BUILD_ERRORS = 0x04,
290     TEST_ERRORS = 0x08,
291     MEMORY_ERRORS = 0x10,
292     COVERAGE_ERRORS = 0x20,
293     SUBMIT_ERRORS = 0x40
294   };
295 
296   /** Are we producing XML */
297   bool GetProduceXML();
298   void SetProduceXML(bool v);
299 
300   /**
301    * Run command specialized for tests. Returns process status and retVal is
302    * return value or exception. If environment is non-null, it is used to set
303    * environment variables prior to running the test. After running the test,
304    * environment variables are restored to their previous values.
305    */
306   int RunTest(std::vector<const char*> args, std::string* output, int* retVal,
307               std::ostream* logfile, cmDuration testTimeOut,
308               std::vector<std::string>* environment,
309               Encoding encoding = cmProcessOutput::Auto);
310 
311   /**
312    * Get the handler object
313    */
314   cmCTestBuildHandler* GetBuildHandler();
315   cmCTestBuildAndTestHandler* GetBuildAndTestHandler();
316   cmCTestCoverageHandler* GetCoverageHandler();
317   cmCTestScriptHandler* GetScriptHandler();
318   cmCTestTestHandler* GetTestHandler();
319   cmCTestUpdateHandler* GetUpdateHandler();
320   cmCTestConfigureHandler* GetConfigureHandler();
321   cmCTestMemCheckHandler* GetMemCheckHandler();
322   cmCTestSubmitHandler* GetSubmitHandler();
323   cmCTestUploadHandler* GetUploadHandler();
324 
325   /**
326    * Set the CTest variable from CMake variable
327    */
328   bool SetCTestConfigurationFromCMakeVariable(cmMakefile* mf,
329                                               const char* dconfig,
330                                               const std::string& cmake_var,
331                                               bool suppress = false);
332 
333   /** Decode a URL to the original string.  */
334   static std::string DecodeURL(const std::string&);
335 
336   /**
337    * Should ctect configuration be updated. When using new style ctest
338    * script, this should be true.
339    */
340   void SetSuppressUpdatingCTestConfiguration(bool val);
341 
342   /**
343    * Add overwrite to ctest configuration.
344    *
345    * The format is key=value
346    */
347   void AddCTestConfigurationOverwrite(const std::string& encstr);
348 
349   /** Create XML file that contains all the notes specified */
350   int GenerateNotesFile(std::vector<std::string> const& files);
351 
352   /** Create XML file to indicate that build is complete */
353   int GenerateDoneFile();
354 
355   /** Submit extra files to the server */
356   bool SubmitExtraFiles(const std::string& files);
357   bool SubmitExtraFiles(std::vector<std::string> const& files);
358 
359   /** Set the output log file name */
360   void SetOutputLogFileName(const std::string& name);
361 
362   /** Set the visual studio or Xcode config type */
363   void SetConfigType(const std::string& ct);
364 
365   /** Various log types */
366   enum
367   {
368     DEBUG = 0,
369     OUTPUT,
370     HANDLER_OUTPUT,
371     HANDLER_PROGRESS_OUTPUT,
372     HANDLER_TEST_PROGRESS_OUTPUT,
373     HANDLER_VERBOSE_OUTPUT,
374     WARNING,
375     ERROR_MESSAGE,
376     OTHER
377   };
378 
379   /** Add log to the output */
380   void Log(int logType, const char* file, int line, const char* msg,
381            bool suppress = false);
382 
383   /** Color values */
384   enum class Color
385   {
386     CLEAR_COLOR = 0,
387     RED = 31,
388     GREEN = 32,
389     YELLOW = 33,
390     BLUE = 34
391   };
392 
393   /** Get color code characters for a specific color */
394   std::string GetColorCode(Color color) const;
395 
396   /** The Build ID is assigned by CDash */
397   void SetBuildID(const std::string& id);
398   std::string GetBuildID() const;
399 
400   /** Add file to be submitted */
401   void AddSubmitFile(Part part, const std::string& name);
402   std::vector<std::string> const& GetSubmitFiles(Part part) const;
403   void ClearSubmitFiles(Part part);
404 
405   /**
406    * Read the custom configuration files and apply them to the current ctest
407    */
408   int ReadCustomConfigurationFileTree(const std::string& dir, cmMakefile* mf);
409 
410   std::vector<std::string>& GetInitialCommandLineArguments();
411 
412   /** Set the group to submit to */
413   void SetSpecificGroup(const char* group);
414   const char* GetSpecificGroup();
415 
416   void SetFailover(bool failover);
417   bool GetFailover() const;
418 
419   bool GetTestProgressOutput() const;
420 
421   bool GetVerbose() const;
422   bool GetExtraVerbose() const;
423 
424   /** Direct process output to given streams.  */
425   void SetStreams(std::ostream* out, std::ostream* err);
426 
427   void AddSiteProperties(cmXMLWriter& xml);
428 
429   bool GetLabelSummary() const;
430   bool GetSubprojectSummary() const;
431 
432   std::string GetCostDataFile();
433 
434   bool GetOutputTestOutputOnTestFailure() const;
435 
436   const std::map<std::string, std::string>& GetDefinitions() const;
437 
438   /** Return the number of times a test should be run */
439   int GetRepeatCount() const;
440 
441   enum class Repeat
442   {
443     Never,
444     UntilFail,
445     UntilPass,
446     AfterTimeout,
447   };
448   Repeat GetRepeatMode() const;
449 
450   enum class NoTests
451   {
452     Legacy,
453     Error,
454     Ignore
455   };
456   NoTests GetNoTestsMode() const;
457 
458   void GenerateSubprojectsOutput(cmXMLWriter& xml);
459   std::vector<std::string> GetLabelsForSubprojects();
460 
461   void SetRunCurrentScript(bool value);
462 
463 private:
464   void SetPersistentOptionIfNotEmpty(const std::string& value,
465                                      const std::string& optionName);
466   void AddPersistentMultiOptionIfNotEmpty(const std::string& value,
467                                           const std::string& optionName);
468 
469   int GenerateNotesFile(const std::string& files);
470 
471   void BlockTestErrorDiagnostics();
472 
473   /**
474    * Initialize a dashboard run in the given build tree.  The "command"
475    * argument is non-NULL when running from a command-driven (ctest_start)
476    * dashboard script, and NULL when running from the CTest command
477    * line.  Note that a declarative dashboard script does not actually
478    * call this method because it sets CTEST_COMMAND to drive a build
479    * through the ctest command line.
480    */
481   int Initialize(const char* binary_dir, cmCTestStartCommand* command);
482 
483   /** parse the option after -D and convert it into the appropriate steps */
484   bool AddTestsForDashboardType(std::string& targ);
485 
486   /** read as "emit an error message for an unknown -D value" */
487   void ErrorMessageUnknownDashDValue(std::string& val);
488 
489   /** add a variable definition from a command line -D value */
490   bool AddVariableDefinition(const std::string& arg);
491 
492   /** set command line arguments read from a test preset */
493   bool SetArgsFromPreset(const std::string& presetName, bool listPresets);
494 
495   /** parse and process most common command line arguments */
496   bool HandleCommandLineArguments(size_t& i, std::vector<std::string>& args,
497                                   std::string& errormsg);
498 
499 #if !defined(_WIN32)
500   /** returns true iff the console supports progress output */
501   static bool ConsoleIsNotDumb();
502 #endif
503 
504   /** returns true iff the console supports progress output */
505   static bool ProgressOutputSupportedByConsole();
506 
507   /** returns true iff the console supports colored output */
508   static bool ColoredOutputSupportedByConsole();
509 
510   /** handle the -S -SP and -SR arguments */
511   void HandleScriptArguments(size_t& i, std::vector<std::string>& args,
512                              bool& SRArgumentSpecified);
513 
514   /** Reread the configuration file */
515   bool UpdateCTestConfiguration();
516 
517   /** Create note from files. */
518   int GenerateCTestNotesOutput(cmXMLWriter& xml,
519                                std::vector<std::string> const& files);
520 
521   /** Check if the argument is the one specified */
522   static bool CheckArgument(const std::string& arg, cm::string_view varg1,
523                             const char* varg2 = nullptr);
524 
525   /** Output errors from a test */
526   void OutputTestErrors(std::vector<char> const& process_output);
527 
528   /** Handle the --test-action command line argument */
529   bool HandleTestActionArgument(const char* ctestExec, size_t& i,
530                                 const std::vector<std::string>& args);
531 
532   /** Handle the --test-model command line argument */
533   bool HandleTestModelArgument(const char* ctestExec, size_t& i,
534                                const std::vector<std::string>& args);
535 
536   int RunCMakeAndTest(std::string* output);
537   int ExecuteTests();
538 
539   /** return true iff change directory was successful */
540   bool TryToChangeDirectory(std::string const& dir);
541 
542   struct Private;
543   std::unique_ptr<Private> Impl;
544 };
545 
546 class cmCTestLogWrite
547 {
548 public:
cmCTestLogWrite(const char * data,size_t length)549   cmCTestLogWrite(const char* data, size_t length)
550     : Data(data)
551     , Length(length)
552   {
553   }
554 
555   const char* Data;
556   size_t Length;
557 };
558 
559 inline std::ostream& operator<<(std::ostream& os, const cmCTestLogWrite& c)
560 {
561   if (!c.Length) {
562     return os;
563   }
564   os.write(c.Data, c.Length);
565   os.flush();
566   return os;
567 }
568 
569 #define cmCTestLog(ctSelf, logType, msg)                                      \
570   do {                                                                        \
571     std::ostringstream cmCTestLog_msg;                                        \
572     cmCTestLog_msg << msg;                                                    \
573     (ctSelf)->Log(cmCTest::logType, __FILE__, __LINE__,                       \
574                   cmCTestLog_msg.str().c_str());                              \
575   } while (false)
576 
577 #define cmCTestOptionalLog(ctSelf, logType, msg, suppress)                    \
578   do {                                                                        \
579     std::ostringstream cmCTestLog_msg;                                        \
580     cmCTestLog_msg << msg;                                                    \
581     (ctSelf)->Log(cmCTest::logType, __FILE__, __LINE__,                       \
582                   cmCTestLog_msg.str().c_str(), suppress);                    \
583   } while (false)
584