1 /* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
2 file Copyright.txt or https://cmake.org/licensing for details. */
3 #include "cmCTestMultiProcessHandler.h"
4
5 #include <algorithm>
6 #include <cassert>
7 #include <chrono>
8 #include <cmath>
9 #include <cstddef> // IWYU pragma: keep
10 #include <cstdlib>
11 #include <cstring>
12 #include <iomanip>
13 #include <iostream>
14 #include <list>
15 #include <sstream>
16 #include <stack>
17 #include <unordered_map>
18 #include <utility>
19 #include <vector>
20
21 #include <cm/memory>
22 #include <cmext/algorithm>
23
24 #include <cm3p/json/value.h>
25 #include <cm3p/json/writer.h>
26 #include <cm3p/uv.h>
27
28 #include "cmsys/FStream.hxx"
29 #include "cmsys/SystemInformation.hxx"
30
31 #include "cmAffinity.h"
32 #include "cmCTest.h"
33 #include "cmCTestBinPacker.h"
34 #include "cmCTestRunTest.h"
35 #include "cmCTestTestHandler.h"
36 #include "cmDuration.h"
37 #include "cmListFileCache.h"
38 #include "cmRange.h"
39 #include "cmStringAlgorithms.h"
40 #include "cmSystemTools.h"
41 #include "cmUVSignalHackRAII.h" // IWYU pragma: keep
42 #include "cmWorkingDirectory.h"
43
44 namespace cmsys {
45 class RegularExpression;
46 }
47
48 class TestComparator
49 {
50 public:
TestComparator(cmCTestMultiProcessHandler * handler)51 TestComparator(cmCTestMultiProcessHandler* handler)
52 : Handler(handler)
53 {
54 }
55
56 // Sorts tests in descending order of cost
operator ()(int index1,int index2) const57 bool operator()(int index1, int index2) const
58 {
59 return this->Handler->Properties[index1]->Cost >
60 this->Handler->Properties[index2]->Cost;
61 }
62
63 private:
64 cmCTestMultiProcessHandler* Handler;
65 };
66
cmCTestMultiProcessHandler()67 cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
68 {
69 this->ParallelLevel = 1;
70 this->TestLoad = 0;
71 this->FakeLoadForTesting = 0;
72 this->Completed = 0;
73 this->RunningCount = 0;
74 this->ProcessorsAvailable = cmAffinity::GetProcessorsAvailable();
75 this->HaveAffinity = this->ProcessorsAvailable.size();
76 this->HasCycles = false;
77 this->SerialTestRunning = false;
78 }
79
80 cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() = default;
81
82 // Set the tests
SetTests(TestMap & tests,PropertiesMap & properties)83 void cmCTestMultiProcessHandler::SetTests(TestMap& tests,
84 PropertiesMap& properties)
85 {
86 this->Tests = tests;
87 this->Properties = properties;
88 this->Total = this->Tests.size();
89 // set test run map to false for all
90 for (auto const& t : this->Tests) {
91 this->TestRunningMap[t.first] = false;
92 this->TestFinishMap[t.first] = false;
93 }
94 if (!this->CTest->GetShowOnly()) {
95 this->ReadCostData();
96 this->HasCycles = !this->CheckCycles();
97 if (this->HasCycles) {
98 return;
99 }
100 this->CreateTestCostList();
101 }
102 }
103
104 // Set the max number of tests that can be run at the same time.
SetParallelLevel(size_t level)105 void cmCTestMultiProcessHandler::SetParallelLevel(size_t level)
106 {
107 this->ParallelLevel = level < 1 ? 1 : level;
108 }
109
SetTestLoad(unsigned long load)110 void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load)
111 {
112 this->TestLoad = load;
113
114 std::string fake_load_value;
115 if (cmSystemTools::GetEnv("__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING",
116 fake_load_value)) {
117 if (!cmStrToULong(fake_load_value, &this->FakeLoadForTesting)) {
118 cmSystemTools::Error("Failed to parse fake load value: " +
119 fake_load_value);
120 }
121 }
122 }
123
RunTests()124 void cmCTestMultiProcessHandler::RunTests()
125 {
126 this->CheckResume();
127 if (this->HasCycles) {
128 return;
129 }
130 #ifdef CMAKE_UV_SIGNAL_HACK
131 cmUVSignalHackRAII hackRAII;
132 #endif
133 this->TestHandler->SetMaxIndex(this->FindMaxIndex());
134
135 uv_loop_init(&this->Loop);
136 this->StartNextTests();
137 uv_run(&this->Loop, UV_RUN_DEFAULT);
138 uv_loop_close(&this->Loop);
139
140 if (!this->StopTimePassed && !this->CheckStopOnFailure()) {
141 assert(this->Completed == this->Total);
142 assert(this->Tests.empty());
143 }
144 assert(this->AllResourcesAvailable());
145
146 this->MarkFinished();
147 this->UpdateCostData();
148 }
149
StartTestProcess(int test)150 bool cmCTestMultiProcessHandler::StartTestProcess(int test)
151 {
152 if (this->HaveAffinity && this->Properties[test]->WantAffinity) {
153 size_t needProcessors = this->GetProcessorsUsed(test);
154 if (needProcessors > this->ProcessorsAvailable.size()) {
155 return false;
156 }
157 std::vector<size_t> affinity;
158 affinity.reserve(needProcessors);
159 for (size_t i = 0; i < needProcessors; ++i) {
160 auto p = this->ProcessorsAvailable.begin();
161 affinity.push_back(*p);
162 this->ProcessorsAvailable.erase(p);
163 }
164 this->Properties[test]->Affinity = std::move(affinity);
165 }
166
167 cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
168 "test " << test << "\n", this->Quiet);
169 this->TestRunningMap[test] = true; // mark the test as running
170 // now remove the test itself
171 this->EraseTest(test);
172 this->RunningCount += this->GetProcessorsUsed(test);
173
174 auto testRun = cm::make_unique<cmCTestRunTest>(*this);
175
176 if (this->RepeatMode != cmCTest::Repeat::Never) {
177 testRun->SetRepeatMode(this->RepeatMode);
178 testRun->SetNumberOfRuns(this->RepeatCount);
179 }
180 testRun->SetIndex(test);
181 testRun->SetTestProperties(this->Properties[test]);
182 if (this->TestHandler->UseResourceSpec) {
183 testRun->SetUseAllocatedResources(true);
184 testRun->SetAllocatedResources(this->AllocatedResources[test]);
185 }
186
187 // Find any failed dependencies for this test. We assume the more common
188 // scenario has no failed tests, so make it the outer loop.
189 for (std::string const& f : *this->Failed) {
190 if (cm::contains(this->Properties[test]->RequireSuccessDepends, f)) {
191 testRun->AddFailedDependency(f);
192 }
193 }
194
195 // Always lock the resources we'll be using, even if we fail to set the
196 // working directory because FinishTestProcess() will try to unlock them
197 this->LockResources(test);
198
199 if (!this->ResourceAllocationErrors[test].empty()) {
200 std::ostringstream e;
201 e << "Insufficient resources for test " << this->Properties[test]->Name
202 << ":\n\n";
203 for (auto const& it : this->ResourceAllocationErrors[test]) {
204 switch (it.second) {
205 case ResourceAllocationError::NoResourceType:
206 e << " Test requested resources of type '" << it.first
207 << "' which does not exist\n";
208 break;
209
210 case ResourceAllocationError::InsufficientResources:
211 e << " Test requested resources of type '" << it.first
212 << "' in the following amounts:\n";
213 for (auto const& group : this->Properties[test]->ResourceGroups) {
214 for (auto const& requirement : group) {
215 if (requirement.ResourceType == it.first) {
216 e << " " << requirement.SlotsNeeded
217 << (requirement.SlotsNeeded == 1 ? " slot\n" : " slots\n");
218 }
219 }
220 }
221 e << " but only the following units were available:\n";
222 for (auto const& res :
223 this->ResourceAllocator.GetResources().at(it.first)) {
224 e << " '" << res.first << "': " << res.second.Total
225 << (res.second.Total == 1 ? " slot\n" : " slots\n");
226 }
227 break;
228 }
229 e << "\n";
230 }
231 e << "Resource spec file:\n\n " << this->TestHandler->ResourceSpecFile;
232 cmCTestRunTest::StartFailure(std::move(testRun), e.str(),
233 "Insufficient resources");
234 return false;
235 }
236
237 cmWorkingDirectory workdir(this->Properties[test]->Directory);
238 if (workdir.Failed()) {
239 cmCTestRunTest::StartFailure(std::move(testRun),
240 "Failed to change working directory to " +
241 this->Properties[test]->Directory + " : " +
242 std::strerror(workdir.GetLastResult()),
243 "Failed to change working directory");
244 return false;
245 }
246
247 // Ownership of 'testRun' has moved to another structure.
248 // When the test finishes, FinishTestProcess will be called.
249 return cmCTestRunTest::StartTest(std::move(testRun), this->Completed,
250 this->Total);
251 }
252
AllocateResources(int index)253 bool cmCTestMultiProcessHandler::AllocateResources(int index)
254 {
255 if (!this->TestHandler->UseResourceSpec) {
256 return true;
257 }
258
259 std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
260 if (!this->TryAllocateResources(index, allocations)) {
261 return false;
262 }
263
264 auto& allocatedResources = this->AllocatedResources[index];
265 allocatedResources.resize(this->Properties[index]->ResourceGroups.size());
266 for (auto const& it : allocations) {
267 for (auto const& alloc : it.second) {
268 bool result = this->ResourceAllocator.AllocateResource(
269 it.first, alloc.Id, alloc.SlotsNeeded);
270 (void)result;
271 assert(result);
272 allocatedResources[alloc.ProcessIndex][it.first].push_back(
273 { alloc.Id, static_cast<unsigned int>(alloc.SlotsNeeded) });
274 }
275 }
276
277 return true;
278 }
279
TryAllocateResources(int index,std::map<std::string,std::vector<cmCTestBinPackerAllocation>> & allocations,std::map<std::string,ResourceAllocationError> * errors)280 bool cmCTestMultiProcessHandler::TryAllocateResources(
281 int index,
282 std::map<std::string, std::vector<cmCTestBinPackerAllocation>>& allocations,
283 std::map<std::string, ResourceAllocationError>* errors)
284 {
285 allocations.clear();
286
287 std::size_t processIndex = 0;
288 for (auto const& process : this->Properties[index]->ResourceGroups) {
289 for (auto const& requirement : process) {
290 for (int i = 0; i < requirement.UnitsNeeded; ++i) {
291 allocations[requirement.ResourceType].push_back(
292 { processIndex, requirement.SlotsNeeded, "" });
293 }
294 }
295 ++processIndex;
296 }
297
298 bool result = true;
299 auto const& availableResources = this->ResourceAllocator.GetResources();
300 for (auto& it : allocations) {
301 if (!availableResources.count(it.first)) {
302 if (errors) {
303 (*errors)[it.first] = ResourceAllocationError::NoResourceType;
304 result = false;
305 } else {
306 return false;
307 }
308 } else if (!cmAllocateCTestResourcesRoundRobin(
309 availableResources.at(it.first), it.second)) {
310 if (errors) {
311 (*errors)[it.first] = ResourceAllocationError::InsufficientResources;
312 result = false;
313 } else {
314 return false;
315 }
316 }
317 }
318
319 return result;
320 }
321
DeallocateResources(int index)322 void cmCTestMultiProcessHandler::DeallocateResources(int index)
323 {
324 if (!this->TestHandler->UseResourceSpec) {
325 return;
326 }
327
328 {
329 auto& allocatedResources = this->AllocatedResources[index];
330 for (auto const& processAlloc : allocatedResources) {
331 for (auto const& it : processAlloc) {
332 auto resourceType = it.first;
333 for (auto const& it2 : it.second) {
334 bool success = this->ResourceAllocator.DeallocateResource(
335 resourceType, it2.Id, it2.Slots);
336 (void)success;
337 assert(success);
338 }
339 }
340 }
341 }
342 this->AllocatedResources.erase(index);
343 }
344
AllResourcesAvailable()345 bool cmCTestMultiProcessHandler::AllResourcesAvailable()
346 {
347 for (auto const& it : this->ResourceAllocator.GetResources()) {
348 for (auto const& it2 : it.second) {
349 if (it2.second.Locked != 0) {
350 return false;
351 }
352 }
353 }
354
355 return true;
356 }
357
CheckResourcesAvailable()358 void cmCTestMultiProcessHandler::CheckResourcesAvailable()
359 {
360 if (this->TestHandler->UseResourceSpec) {
361 for (auto test : this->SortedTests) {
362 std::map<std::string, std::vector<cmCTestBinPackerAllocation>>
363 allocations;
364 this->TryAllocateResources(test, allocations,
365 &this->ResourceAllocationErrors[test]);
366 }
367 }
368 }
369
CheckStopOnFailure()370 bool cmCTestMultiProcessHandler::CheckStopOnFailure()
371 {
372 return this->CTest->GetStopOnFailure();
373 }
374
CheckStopTimePassed()375 bool cmCTestMultiProcessHandler::CheckStopTimePassed()
376 {
377 if (!this->StopTimePassed) {
378 std::chrono::system_clock::time_point stop_time =
379 this->CTest->GetStopTime();
380 if (stop_time != std::chrono::system_clock::time_point() &&
381 stop_time <= std::chrono::system_clock::now()) {
382 this->SetStopTimePassed();
383 }
384 }
385 return this->StopTimePassed;
386 }
387
SetStopTimePassed()388 void cmCTestMultiProcessHandler::SetStopTimePassed()
389 {
390 if (!this->StopTimePassed) {
391 cmCTestLog(this->CTest, ERROR_MESSAGE,
392 "The stop time has been passed. "
393 "Stopping all tests."
394 << std::endl);
395 this->StopTimePassed = true;
396 }
397 }
398
LockResources(int index)399 void cmCTestMultiProcessHandler::LockResources(int index)
400 {
401 this->LockedResources.insert(
402 this->Properties[index]->LockedResources.begin(),
403 this->Properties[index]->LockedResources.end());
404
405 if (this->Properties[index]->RunSerial) {
406 this->SerialTestRunning = true;
407 }
408 }
409
UnlockResources(int index)410 void cmCTestMultiProcessHandler::UnlockResources(int index)
411 {
412 for (std::string const& i : this->Properties[index]->LockedResources) {
413 this->LockedResources.erase(i);
414 }
415 if (this->Properties[index]->RunSerial) {
416 this->SerialTestRunning = false;
417 }
418 }
419
EraseTest(int test)420 void cmCTestMultiProcessHandler::EraseTest(int test)
421 {
422 this->Tests.erase(test);
423 this->SortedTests.erase(
424 std::find(this->SortedTests.begin(), this->SortedTests.end(), test));
425 }
426
GetProcessorsUsed(int test)427 inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test)
428 {
429 size_t processors = static_cast<int>(this->Properties[test]->Processors);
430 // If processors setting is set higher than the -j
431 // setting, we default to using all of the process slots.
432 if (processors > this->ParallelLevel) {
433 processors = this->ParallelLevel;
434 }
435 // Cap tests that want affinity to the maximum affinity available.
436 if (this->HaveAffinity && processors > this->HaveAffinity &&
437 this->Properties[test]->WantAffinity) {
438 processors = this->HaveAffinity;
439 }
440 return processors;
441 }
442
GetName(int test)443 std::string cmCTestMultiProcessHandler::GetName(int test)
444 {
445 return this->Properties[test]->Name;
446 }
447
StartTest(int test)448 bool cmCTestMultiProcessHandler::StartTest(int test)
449 {
450 // Check for locked resources
451 for (std::string const& i : this->Properties[test]->LockedResources) {
452 if (cm::contains(this->LockedResources, i)) {
453 return false;
454 }
455 }
456
457 // Allocate resources
458 if (this->ResourceAllocationErrors[test].empty() &&
459 !this->AllocateResources(test)) {
460 this->DeallocateResources(test);
461 return false;
462 }
463
464 // if there are no depends left then run this test
465 if (this->Tests[test].empty()) {
466 return this->StartTestProcess(test);
467 }
468 // This test was not able to start because it is waiting
469 // on depends to run
470 this->DeallocateResources(test);
471 return false;
472 }
473
StartNextTests()474 void cmCTestMultiProcessHandler::StartNextTests()
475 {
476 if (this->TestLoadRetryTimer.get() != nullptr) {
477 // This timer may be waiting to call StartNextTests again.
478 // Since we have been called it is no longer needed.
479 uv_timer_stop(this->TestLoadRetryTimer);
480 }
481
482 if (this->Tests.empty()) {
483 this->TestLoadRetryTimer.reset();
484 return;
485 }
486
487 if (this->CheckStopTimePassed()) {
488 return;
489 }
490
491 if (this->CheckStopOnFailure() && !this->Failed->empty()) {
492 return;
493 }
494
495 size_t numToStart = 0;
496
497 if (this->RunningCount < this->ParallelLevel) {
498 numToStart = this->ParallelLevel - this->RunningCount;
499 }
500
501 if (numToStart == 0) {
502 return;
503 }
504
505 // Don't start any new tests if one with the RUN_SERIAL property
506 // is already running.
507 if (this->SerialTestRunning) {
508 return;
509 }
510
511 bool allTestsFailedTestLoadCheck = false;
512 size_t minProcessorsRequired = this->ParallelLevel;
513 std::string testWithMinProcessors;
514
515 cmsys::SystemInformation info;
516
517 unsigned long systemLoad = 0;
518 size_t spareLoad = 0;
519 if (this->TestLoad > 0) {
520 // Activate possible wait.
521 allTestsFailedTestLoadCheck = true;
522
523 // Check for a fake load average value used in testing.
524 if (this->FakeLoadForTesting > 0) {
525 systemLoad = this->FakeLoadForTesting;
526 // Drop the fake load for the next iteration to a value low enough
527 // that the next iteration will start tests.
528 this->FakeLoadForTesting = 1;
529 }
530 // If it's not set, look up the true load average.
531 else {
532 systemLoad = static_cast<unsigned long>(ceil(info.GetLoadAverage()));
533 }
534 spareLoad =
535 (this->TestLoad > systemLoad ? this->TestLoad - systemLoad : 0);
536
537 // Don't start more tests than the spare load can support.
538 if (numToStart > spareLoad) {
539 numToStart = spareLoad;
540 }
541 }
542
543 TestList copy = this->SortedTests;
544 for (auto const& test : copy) {
545 // Take a nap if we're currently performing a RUN_SERIAL test.
546 if (this->SerialTestRunning) {
547 break;
548 }
549 // We can only start a RUN_SERIAL test if no other tests are also
550 // running.
551 if (this->Properties[test]->RunSerial && this->RunningCount > 0) {
552 continue;
553 }
554
555 size_t processors = this->GetProcessorsUsed(test);
556 bool testLoadOk = true;
557 if (this->TestLoad > 0) {
558 if (processors <= spareLoad) {
559 cmCTestLog(this->CTest, DEBUG,
560 "OK to run " << this->GetName(test) << ", it requires "
561 << processors << " procs & system load is: "
562 << systemLoad << std::endl);
563 allTestsFailedTestLoadCheck = false;
564 } else {
565 testLoadOk = false;
566 }
567 }
568
569 if (processors <= minProcessorsRequired) {
570 minProcessorsRequired = processors;
571 testWithMinProcessors = this->GetName(test);
572 }
573
574 if (testLoadOk && processors <= numToStart && this->StartTest(test)) {
575 numToStart -= processors;
576 } else if (numToStart == 0) {
577 break;
578 }
579 }
580
581 if (allTestsFailedTestLoadCheck) {
582 // Find out whether there are any non RUN_SERIAL tests left, so that the
583 // correct warning may be displayed.
584 bool onlyRunSerialTestsLeft = true;
585 for (auto const& test : copy) {
586 if (!this->Properties[test]->RunSerial) {
587 onlyRunSerialTestsLeft = false;
588 }
589 }
590 cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "***** WAITING, ");
591
592 if (this->SerialTestRunning) {
593 cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
594 "Waiting for RUN_SERIAL test to finish.");
595 } else if (onlyRunSerialTestsLeft) {
596 cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
597 "Only RUN_SERIAL tests remain, awaiting available slot.");
598 } else {
599 /* clang-format off */
600 cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
601 "System Load: " << systemLoad << ", "
602 "Max Allowed Load: " << this->TestLoad << ", "
603 "Smallest test " << testWithMinProcessors <<
604 " requires " << minProcessorsRequired);
605 /* clang-format on */
606 }
607 cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "*****" << std::endl);
608
609 // Wait between 1 and 5 seconds before trying again.
610 unsigned int milliseconds = (cmSystemTools::RandomSeed() % 5 + 1) * 1000;
611 if (this->FakeLoadForTesting) {
612 milliseconds = 10;
613 }
614 if (this->TestLoadRetryTimer.get() == nullptr) {
615 this->TestLoadRetryTimer.init(this->Loop, this);
616 }
617 this->TestLoadRetryTimer.start(
618 &cmCTestMultiProcessHandler::OnTestLoadRetryCB, milliseconds, 0);
619 }
620 }
621
OnTestLoadRetryCB(uv_timer_t * timer)622 void cmCTestMultiProcessHandler::OnTestLoadRetryCB(uv_timer_t* timer)
623 {
624 auto* self = static_cast<cmCTestMultiProcessHandler*>(timer->data);
625 self->StartNextTests();
626 }
627
FinishTestProcess(std::unique_ptr<cmCTestRunTest> runner,bool started)628 void cmCTestMultiProcessHandler::FinishTestProcess(
629 std::unique_ptr<cmCTestRunTest> runner, bool started)
630 {
631 this->Completed++;
632
633 int test = runner->GetIndex();
634 auto* properties = runner->GetTestProperties();
635
636 bool testResult = runner->EndTest(this->Completed, this->Total, started);
637 if (runner->TimedOutForStopTime()) {
638 this->SetStopTimePassed();
639 }
640 if (started) {
641 if (!this->StopTimePassed &&
642 cmCTestRunTest::StartAgain(std::move(runner), this->Completed)) {
643 this->Completed--; // remove the completed test because run again
644 return;
645 }
646 }
647
648 if (testResult) {
649 this->Passed->push_back(properties->Name);
650 } else if (!properties->Disabled) {
651 this->Failed->push_back(properties->Name);
652 }
653
654 for (auto& t : this->Tests) {
655 t.second.erase(test);
656 }
657
658 this->TestFinishMap[test] = true;
659 this->TestRunningMap[test] = false;
660 this->WriteCheckpoint(test);
661 this->DeallocateResources(test);
662 this->UnlockResources(test);
663 this->RunningCount -= this->GetProcessorsUsed(test);
664
665 for (auto p : properties->Affinity) {
666 this->ProcessorsAvailable.insert(p);
667 }
668 properties->Affinity.clear();
669
670 runner.reset();
671 if (started) {
672 this->StartNextTests();
673 }
674 }
675
UpdateCostData()676 void cmCTestMultiProcessHandler::UpdateCostData()
677 {
678 std::string fname = this->CTest->GetCostDataFile();
679 std::string tmpout = fname + ".tmp";
680 cmsys::ofstream fout;
681 fout.open(tmpout.c_str());
682
683 PropertiesMap temp = this->Properties;
684
685 if (cmSystemTools::FileExists(fname)) {
686 cmsys::ifstream fin;
687 fin.open(fname.c_str());
688
689 std::string line;
690 while (std::getline(fin, line)) {
691 if (line == "---") {
692 break;
693 }
694 std::vector<std::string> parts = cmSystemTools::SplitString(line, ' ');
695 // Format: <name> <previous_runs> <avg_cost>
696 if (parts.size() < 3) {
697 break;
698 }
699
700 std::string name = parts[0];
701 int prev = atoi(parts[1].c_str());
702 float cost = static_cast<float>(atof(parts[2].c_str()));
703
704 int index = this->SearchByName(name);
705 if (index == -1) {
706 // This test is not in memory. We just rewrite the entry
707 fout << name << " " << prev << " " << cost << "\n";
708 } else {
709 // Update with our new average cost
710 fout << name << " " << this->Properties[index]->PreviousRuns << " "
711 << this->Properties[index]->Cost << "\n";
712 temp.erase(index);
713 }
714 }
715 fin.close();
716 cmSystemTools::RemoveFile(fname);
717 }
718
719 // Add all tests not previously listed in the file
720 for (auto const& i : temp) {
721 fout << i.second->Name << " " << i.second->PreviousRuns << " "
722 << i.second->Cost << "\n";
723 }
724
725 // Write list of failed tests
726 fout << "---\n";
727 for (std::string const& f : *this->Failed) {
728 fout << f << "\n";
729 }
730 fout.close();
731 cmSystemTools::RenameFile(tmpout, fname);
732 }
733
ReadCostData()734 void cmCTestMultiProcessHandler::ReadCostData()
735 {
736 std::string fname = this->CTest->GetCostDataFile();
737
738 if (cmSystemTools::FileExists(fname, true)) {
739 cmsys::ifstream fin;
740 fin.open(fname.c_str());
741 std::string line;
742 while (std::getline(fin, line)) {
743 if (line == "---") {
744 break;
745 }
746
747 std::vector<std::string> parts = cmSystemTools::SplitString(line, ' ');
748
749 // Probably an older version of the file, will be fixed next run
750 if (parts.size() < 3) {
751 fin.close();
752 return;
753 }
754
755 std::string name = parts[0];
756 int prev = atoi(parts[1].c_str());
757 float cost = static_cast<float>(atof(parts[2].c_str()));
758
759 int index = this->SearchByName(name);
760 if (index == -1) {
761 continue;
762 }
763
764 this->Properties[index]->PreviousRuns = prev;
765 // When not running in parallel mode, don't use cost data
766 if (this->ParallelLevel > 1 && this->Properties[index] &&
767 this->Properties[index]->Cost == 0) {
768 this->Properties[index]->Cost = cost;
769 }
770 }
771 // Next part of the file is the failed tests
772 while (std::getline(fin, line)) {
773 if (!line.empty()) {
774 this->LastTestsFailed.push_back(line);
775 }
776 }
777 fin.close();
778 }
779 }
780
SearchByName(std::string const & name)781 int cmCTestMultiProcessHandler::SearchByName(std::string const& name)
782 {
783 int index = -1;
784
785 for (auto const& p : this->Properties) {
786 if (p.second->Name == name) {
787 index = p.first;
788 }
789 }
790 return index;
791 }
792
CreateTestCostList()793 void cmCTestMultiProcessHandler::CreateTestCostList()
794 {
795 if (this->ParallelLevel > 1) {
796 this->CreateParallelTestCostList();
797 } else {
798 this->CreateSerialTestCostList();
799 }
800 }
801
CreateParallelTestCostList()802 void cmCTestMultiProcessHandler::CreateParallelTestCostList()
803 {
804 TestSet alreadySortedTests;
805
806 std::list<TestSet> priorityStack;
807 priorityStack.emplace_back();
808 TestSet& topLevel = priorityStack.back();
809
810 // In parallel test runs add previously failed tests to the front
811 // of the cost list and queue other tests for further sorting
812 for (auto const& t : this->Tests) {
813 if (cm::contains(this->LastTestsFailed, this->Properties[t.first]->Name)) {
814 // If the test failed last time, it should be run first.
815 this->SortedTests.push_back(t.first);
816 alreadySortedTests.insert(t.first);
817 } else {
818 topLevel.insert(t.first);
819 }
820 }
821
822 // In parallel test runs repeatedly move dependencies of the tests on
823 // the current dependency level to the next level until no
824 // further dependencies exist.
825 while (!priorityStack.back().empty()) {
826 TestSet& previousSet = priorityStack.back();
827 priorityStack.emplace_back();
828 TestSet& currentSet = priorityStack.back();
829
830 for (auto const& i : previousSet) {
831 TestSet const& dependencies = this->Tests[i];
832 currentSet.insert(dependencies.begin(), dependencies.end());
833 }
834
835 for (auto const& i : currentSet) {
836 previousSet.erase(i);
837 }
838 }
839
840 // Remove the empty dependency level
841 priorityStack.pop_back();
842
843 // Reverse iterate over the different dependency levels (deepest first).
844 // Sort tests within each level by COST and append them to the cost list.
845 for (TestSet const& currentSet : cmReverseRange(priorityStack)) {
846 TestList sortedCopy;
847 cm::append(sortedCopy, currentSet);
848 std::stable_sort(sortedCopy.begin(), sortedCopy.end(),
849 TestComparator(this));
850
851 for (auto const& j : sortedCopy) {
852 if (!cm::contains(alreadySortedTests, j)) {
853 this->SortedTests.push_back(j);
854 alreadySortedTests.insert(j);
855 }
856 }
857 }
858 }
859
GetAllTestDependencies(int test,TestList & dependencies)860 void cmCTestMultiProcessHandler::GetAllTestDependencies(int test,
861 TestList& dependencies)
862 {
863 TestSet const& dependencySet = this->Tests[test];
864 for (int i : dependencySet) {
865 this->GetAllTestDependencies(i, dependencies);
866 dependencies.push_back(i);
867 }
868 }
869
CreateSerialTestCostList()870 void cmCTestMultiProcessHandler::CreateSerialTestCostList()
871 {
872 TestList presortedList;
873
874 for (auto const& i : this->Tests) {
875 presortedList.push_back(i.first);
876 }
877
878 std::stable_sort(presortedList.begin(), presortedList.end(),
879 TestComparator(this));
880
881 TestSet alreadySortedTests;
882
883 for (int test : presortedList) {
884 if (cm::contains(alreadySortedTests, test)) {
885 continue;
886 }
887
888 TestList dependencies;
889 this->GetAllTestDependencies(test, dependencies);
890
891 for (int testDependency : dependencies) {
892 if (!cm::contains(alreadySortedTests, testDependency)) {
893 alreadySortedTests.insert(testDependency);
894 this->SortedTests.push_back(testDependency);
895 }
896 }
897
898 alreadySortedTests.insert(test);
899 this->SortedTests.push_back(test);
900 }
901 }
902
WriteCheckpoint(int index)903 void cmCTestMultiProcessHandler::WriteCheckpoint(int index)
904 {
905 std::string fname =
906 this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
907 cmsys::ofstream fout;
908 fout.open(fname.c_str(), std::ios::app);
909 fout << index << "\n";
910 fout.close();
911 }
912
MarkFinished()913 void cmCTestMultiProcessHandler::MarkFinished()
914 {
915 std::string fname =
916 this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
917 cmSystemTools::RemoveFile(fname);
918 }
919
DumpToJsonArray(const std::set<std::string> & values)920 static Json::Value DumpToJsonArray(const std::set<std::string>& values)
921 {
922 Json::Value jsonArray = Json::arrayValue;
923 for (const auto& it : values) {
924 jsonArray.append(it);
925 }
926 return jsonArray;
927 }
928
DumpToJsonArray(const std::vector<std::string> & values)929 static Json::Value DumpToJsonArray(const std::vector<std::string>& values)
930 {
931 Json::Value jsonArray = Json::arrayValue;
932 for (const auto& it : values) {
933 jsonArray.append(it);
934 }
935 return jsonArray;
936 }
937
DumpRegExToJsonArray(const std::vector<std::pair<cmsys::RegularExpression,std::string>> & values)938 static Json::Value DumpRegExToJsonArray(
939 const std::vector<std::pair<cmsys::RegularExpression, std::string>>& values)
940 {
941 Json::Value jsonArray = Json::arrayValue;
942 for (const auto& it : values) {
943 jsonArray.append(it.second);
944 }
945 return jsonArray;
946 }
947
DumpMeasurementToJsonArray(const std::map<std::string,std::string> & values)948 static Json::Value DumpMeasurementToJsonArray(
949 const std::map<std::string, std::string>& values)
950 {
951 Json::Value jsonArray = Json::arrayValue;
952 for (const auto& it : values) {
953 Json::Value measurement = Json::objectValue;
954 measurement["measurement"] = it.first;
955 measurement["value"] = it.second;
956 jsonArray.append(measurement);
957 }
958 return jsonArray;
959 }
960
DumpTimeoutAfterMatch(cmCTestTestHandler::cmCTestTestProperties & testProperties)961 static Json::Value DumpTimeoutAfterMatch(
962 cmCTestTestHandler::cmCTestTestProperties& testProperties)
963 {
964 Json::Value timeoutAfterMatch = Json::objectValue;
965 timeoutAfterMatch["timeout"] = testProperties.AlternateTimeout.count();
966 timeoutAfterMatch["regex"] =
967 DumpRegExToJsonArray(testProperties.TimeoutRegularExpressions);
968 return timeoutAfterMatch;
969 }
970
DumpResourceGroupsToJsonArray(const std::vector<std::vector<cmCTestTestHandler::cmCTestTestResourceRequirement>> & resourceGroups)971 static Json::Value DumpResourceGroupsToJsonArray(
972 const std::vector<
973 std::vector<cmCTestTestHandler::cmCTestTestResourceRequirement>>&
974 resourceGroups)
975 {
976 Json::Value jsonResourceGroups = Json::arrayValue;
977 for (auto const& it : resourceGroups) {
978 Json::Value jsonResourceGroup = Json::objectValue;
979 Json::Value requirements = Json::arrayValue;
980 for (auto const& it2 : it) {
981 Json::Value res = Json::objectValue;
982 res[".type"] = it2.ResourceType;
983 // res[".units"] = it2.UnitsNeeded; // Intentionally commented out
984 res["slots"] = it2.SlotsNeeded;
985 requirements.append(res);
986 }
987 jsonResourceGroup["requirements"] = requirements;
988 jsonResourceGroups.append(jsonResourceGroup);
989 }
990 return jsonResourceGroups;
991 }
992
DumpCTestProperty(std::string const & name,Json::Value value)993 static Json::Value DumpCTestProperty(std::string const& name,
994 Json::Value value)
995 {
996 Json::Value property = Json::objectValue;
997 property["name"] = name;
998 property["value"] = std::move(value);
999 return property;
1000 }
1001
DumpCTestProperties(cmCTestTestHandler::cmCTestTestProperties & testProperties)1002 static Json::Value DumpCTestProperties(
1003 cmCTestTestHandler::cmCTestTestProperties& testProperties)
1004 {
1005 Json::Value properties = Json::arrayValue;
1006 if (!testProperties.AttachOnFail.empty()) {
1007 properties.append(DumpCTestProperty(
1008 "ATTACHED_FILES_ON_FAIL", DumpToJsonArray(testProperties.AttachOnFail)));
1009 }
1010 if (!testProperties.AttachedFiles.empty()) {
1011 properties.append(DumpCTestProperty(
1012 "ATTACHED_FILES", DumpToJsonArray(testProperties.AttachedFiles)));
1013 }
1014 if (testProperties.Cost != 0.0f) {
1015 properties.append(
1016 DumpCTestProperty("COST", static_cast<double>(testProperties.Cost)));
1017 }
1018 if (!testProperties.Depends.empty()) {
1019 properties.append(
1020 DumpCTestProperty("DEPENDS", DumpToJsonArray(testProperties.Depends)));
1021 }
1022 if (testProperties.Disabled) {
1023 properties.append(DumpCTestProperty("DISABLED", testProperties.Disabled));
1024 }
1025 if (!testProperties.Environment.empty()) {
1026 properties.append(DumpCTestProperty(
1027 "ENVIRONMENT", DumpToJsonArray(testProperties.Environment)));
1028 }
1029 if (!testProperties.EnvironmentModification.empty()) {
1030 properties.append(DumpCTestProperty(
1031 "ENVIRONMENT_MODIFICATION",
1032 DumpToJsonArray(testProperties.EnvironmentModification)));
1033 }
1034 if (!testProperties.ErrorRegularExpressions.empty()) {
1035 properties.append(DumpCTestProperty(
1036 "FAIL_REGULAR_EXPRESSION",
1037 DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
1038 }
1039 if (!testProperties.SkipRegularExpressions.empty()) {
1040 properties.append(DumpCTestProperty(
1041 "SKIP_REGULAR_EXPRESSION",
1042 DumpRegExToJsonArray(testProperties.SkipRegularExpressions)));
1043 }
1044 if (!testProperties.FixturesCleanup.empty()) {
1045 properties.append(DumpCTestProperty(
1046 "FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));
1047 }
1048 if (!testProperties.FixturesRequired.empty()) {
1049 properties.append(DumpCTestProperty(
1050 "FIXTURES_REQUIRED", DumpToJsonArray(testProperties.FixturesRequired)));
1051 }
1052 if (!testProperties.FixturesSetup.empty()) {
1053 properties.append(DumpCTestProperty(
1054 "FIXTURES_SETUP", DumpToJsonArray(testProperties.FixturesSetup)));
1055 }
1056 if (!testProperties.Labels.empty()) {
1057 properties.append(
1058 DumpCTestProperty("LABELS", DumpToJsonArray(testProperties.Labels)));
1059 }
1060 if (!testProperties.Measurements.empty()) {
1061 properties.append(DumpCTestProperty(
1062 "MEASUREMENT", DumpMeasurementToJsonArray(testProperties.Measurements)));
1063 }
1064 if (!testProperties.RequiredRegularExpressions.empty()) {
1065 properties.append(DumpCTestProperty(
1066 "PASS_REGULAR_EXPRESSION",
1067 DumpRegExToJsonArray(testProperties.RequiredRegularExpressions)));
1068 }
1069 if (!testProperties.ResourceGroups.empty()) {
1070 properties.append(DumpCTestProperty(
1071 "RESOURCE_GROUPS",
1072 DumpResourceGroupsToJsonArray(testProperties.ResourceGroups)));
1073 }
1074 if (testProperties.WantAffinity) {
1075 properties.append(
1076 DumpCTestProperty("PROCESSOR_AFFINITY", testProperties.WantAffinity));
1077 }
1078 if (testProperties.Processors != 1) {
1079 properties.append(
1080 DumpCTestProperty("PROCESSORS", testProperties.Processors));
1081 }
1082 if (!testProperties.RequiredFiles.empty()) {
1083 properties.append(DumpCTestProperty(
1084 "REQUIRED_FILES", DumpToJsonArray(testProperties.RequiredFiles)));
1085 }
1086 if (!testProperties.LockedResources.empty()) {
1087 properties.append(DumpCTestProperty(
1088 "RESOURCE_LOCK", DumpToJsonArray(testProperties.LockedResources)));
1089 }
1090 if (testProperties.RunSerial) {
1091 properties.append(
1092 DumpCTestProperty("RUN_SERIAL", testProperties.RunSerial));
1093 }
1094 if (testProperties.SkipReturnCode != -1) {
1095 properties.append(
1096 DumpCTestProperty("SKIP_RETURN_CODE", testProperties.SkipReturnCode));
1097 }
1098 if (testProperties.ExplicitTimeout) {
1099 properties.append(
1100 DumpCTestProperty("TIMEOUT", testProperties.Timeout.count()));
1101 }
1102 if (!testProperties.TimeoutRegularExpressions.empty()) {
1103 properties.append(DumpCTestProperty(
1104 "TIMEOUT_AFTER_MATCH", DumpTimeoutAfterMatch(testProperties)));
1105 }
1106 if (testProperties.WillFail) {
1107 properties.append(DumpCTestProperty("WILL_FAIL", testProperties.WillFail));
1108 }
1109 if (!testProperties.Directory.empty()) {
1110 properties.append(
1111 DumpCTestProperty("WORKING_DIRECTORY", testProperties.Directory));
1112 }
1113 return properties;
1114 }
1115
1116 class BacktraceData
1117 {
1118 std::unordered_map<std::string, Json::ArrayIndex> CommandMap;
1119 std::unordered_map<std::string, Json::ArrayIndex> FileMap;
1120 std::unordered_map<cmListFileContext const*, Json::ArrayIndex> NodeMap;
1121 Json::Value Commands = Json::arrayValue;
1122 Json::Value Files = Json::arrayValue;
1123 Json::Value Nodes = Json::arrayValue;
1124
AddCommand(std::string const & command)1125 Json::ArrayIndex AddCommand(std::string const& command)
1126 {
1127 auto i = this->CommandMap.find(command);
1128 if (i == this->CommandMap.end()) {
1129 i = this->CommandMap.emplace(command, this->Commands.size()).first;
1130 this->Commands.append(command);
1131 }
1132 return i->second;
1133 }
1134
AddFile(std::string const & file)1135 Json::ArrayIndex AddFile(std::string const& file)
1136 {
1137 auto i = this->FileMap.find(file);
1138 if (i == this->FileMap.end()) {
1139 i = this->FileMap.emplace(file, this->Files.size()).first;
1140 this->Files.append(file);
1141 }
1142 return i->second;
1143 }
1144
1145 public:
1146 bool Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index);
1147 Json::Value Dump();
1148 };
1149
Add(cmListFileBacktrace const & bt,Json::ArrayIndex & index)1150 bool BacktraceData::Add(cmListFileBacktrace const& bt, Json::ArrayIndex& index)
1151 {
1152 if (bt.Empty()) {
1153 return false;
1154 }
1155 cmListFileContext const* top = &bt.Top();
1156 auto found = this->NodeMap.find(top);
1157 if (found != this->NodeMap.end()) {
1158 index = found->second;
1159 return true;
1160 }
1161 Json::Value entry = Json::objectValue;
1162 entry["file"] = this->AddFile(top->FilePath);
1163 if (top->Line) {
1164 entry["line"] = static_cast<int>(top->Line);
1165 }
1166 if (!top->Name.empty()) {
1167 entry["command"] = this->AddCommand(top->Name);
1168 }
1169 Json::ArrayIndex parent;
1170 if (this->Add(bt.Pop(), parent)) {
1171 entry["parent"] = parent;
1172 }
1173 index = this->NodeMap[top] = this->Nodes.size();
1174 this->Nodes.append(std::move(entry)); // NOLINT(*)
1175 return true;
1176 }
1177
Dump()1178 Json::Value BacktraceData::Dump()
1179 {
1180 Json::Value backtraceGraph;
1181 this->CommandMap.clear();
1182 this->FileMap.clear();
1183 this->NodeMap.clear();
1184 backtraceGraph["commands"] = std::move(this->Commands);
1185 backtraceGraph["files"] = std::move(this->Files);
1186 backtraceGraph["nodes"] = std::move(this->Nodes);
1187 return backtraceGraph;
1188 }
1189
AddBacktrace(BacktraceData & backtraceGraph,Json::Value & object,cmListFileBacktrace const & bt)1190 static void AddBacktrace(BacktraceData& backtraceGraph, Json::Value& object,
1191 cmListFileBacktrace const& bt)
1192 {
1193 Json::ArrayIndex backtrace;
1194 if (backtraceGraph.Add(bt, backtrace)) {
1195 object["backtrace"] = backtrace;
1196 }
1197 }
1198
DumpCTestInfo(cmCTestRunTest & testRun,cmCTestTestHandler::cmCTestTestProperties & testProperties,BacktraceData & backtraceGraph)1199 static Json::Value DumpCTestInfo(
1200 cmCTestRunTest& testRun,
1201 cmCTestTestHandler::cmCTestTestProperties& testProperties,
1202 BacktraceData& backtraceGraph)
1203 {
1204 Json::Value testInfo = Json::objectValue;
1205 // test name should always be present
1206 testInfo["name"] = testProperties.Name;
1207 std::string const& config = testRun.GetCTest()->GetConfigType();
1208 if (!config.empty()) {
1209 testInfo["config"] = config;
1210 }
1211 std::string const& command = testRun.GetActualCommand();
1212 if (!command.empty()) {
1213 std::vector<std::string> commandAndArgs;
1214 commandAndArgs.push_back(command);
1215 const std::vector<std::string>& args = testRun.GetArguments();
1216 if (!args.empty()) {
1217 commandAndArgs.reserve(args.size() + 1);
1218 cm::append(commandAndArgs, args);
1219 }
1220 testInfo["command"] = DumpToJsonArray(commandAndArgs);
1221 }
1222 Json::Value properties = DumpCTestProperties(testProperties);
1223 if (!properties.empty()) {
1224 testInfo["properties"] = properties;
1225 }
1226 if (!testProperties.Backtrace.Empty()) {
1227 AddBacktrace(backtraceGraph, testInfo, testProperties.Backtrace);
1228 }
1229 return testInfo;
1230 }
1231
DumpVersion(int major,int minor)1232 static Json::Value DumpVersion(int major, int minor)
1233 {
1234 Json::Value version = Json::objectValue;
1235 version["major"] = major;
1236 version["minor"] = minor;
1237 return version;
1238 }
1239
PrintOutputAsJson()1240 void cmCTestMultiProcessHandler::PrintOutputAsJson()
1241 {
1242 this->TestHandler->SetMaxIndex(this->FindMaxIndex());
1243
1244 Json::Value result = Json::objectValue;
1245 result["kind"] = "ctestInfo";
1246 result["version"] = DumpVersion(1, 0);
1247
1248 BacktraceData backtraceGraph;
1249 Json::Value tests = Json::arrayValue;
1250 for (auto& it : this->Properties) {
1251 cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
1252
1253 // Don't worry if this fails, we are only showing the test list, not
1254 // running the tests
1255 cmWorkingDirectory workdir(p.Directory);
1256 cmCTestRunTest testRun(*this);
1257 testRun.SetIndex(p.Index);
1258 testRun.SetTestProperties(&p);
1259 testRun.ComputeArguments();
1260
1261 // Skip tests not available in this configuration.
1262 if (p.Args.size() >= 2 && p.Args[1] == "NOT_AVAILABLE") {
1263 continue;
1264 }
1265
1266 Json::Value testInfo = DumpCTestInfo(testRun, p, backtraceGraph);
1267 tests.append(testInfo);
1268 }
1269 result["backtraceGraph"] = backtraceGraph.Dump();
1270 result["tests"] = std::move(tests);
1271
1272 Json::StreamWriterBuilder builder;
1273 builder["indentation"] = " ";
1274 std::unique_ptr<Json::StreamWriter> jout(builder.newStreamWriter());
1275 jout->write(result, &std::cout);
1276 }
1277
1278 // For ShowOnly mode
PrintTestList()1279 void cmCTestMultiProcessHandler::PrintTestList()
1280 {
1281 if (this->CTest->GetOutputAsJson()) {
1282 this->PrintOutputAsJson();
1283 return;
1284 }
1285
1286 this->TestHandler->SetMaxIndex(this->FindMaxIndex());
1287 int count = 0;
1288
1289 for (auto& it : this->Properties) {
1290 count++;
1291 cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
1292
1293 // Don't worry if this fails, we are only showing the test list, not
1294 // running the tests
1295 cmWorkingDirectory workdir(p.Directory);
1296
1297 cmCTestRunTest testRun(*this);
1298 testRun.SetIndex(p.Index);
1299 testRun.SetTestProperties(&p);
1300 testRun.ComputeArguments(); // logs the command in verbose mode
1301
1302 if (!p.Labels.empty()) // print the labels
1303 {
1304 cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
1305 "Labels:", this->Quiet);
1306 }
1307 for (std::string const& label : p.Labels) {
1308 cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, " " << label,
1309 this->Quiet);
1310 }
1311 if (!p.Labels.empty()) // print the labels
1312 {
1313 cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl,
1314 this->Quiet);
1315 }
1316
1317 if (this->TestHandler->MemCheck) {
1318 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " Memory Check",
1319 this->Quiet);
1320 } else {
1321 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " Test", this->Quiet);
1322 }
1323 std::ostringstream indexStr;
1324 indexStr << " #" << p.Index << ":";
1325 cmCTestOptionalLog(
1326 this->CTest, HANDLER_OUTPUT,
1327 std::setw(3 + getNumWidth(this->TestHandler->GetMaxIndex()))
1328 << indexStr.str(),
1329 this->Quiet);
1330 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " " << p.Name,
1331 this->Quiet);
1332 if (p.Disabled) {
1333 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " (Disabled)",
1334 this->Quiet);
1335 }
1336 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, std::endl, this->Quiet);
1337 }
1338
1339 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
1340 std::endl
1341 << "Total Tests: " << this->Total << std::endl,
1342 this->Quiet);
1343 }
1344
PrintLabels()1345 void cmCTestMultiProcessHandler::PrintLabels()
1346 {
1347 std::set<std::string> allLabels;
1348 for (auto& it : this->Properties) {
1349 cmCTestTestHandler::cmCTestTestProperties& p = *it.second;
1350 allLabels.insert(p.Labels.begin(), p.Labels.end());
1351 }
1352
1353 if (!allLabels.empty()) {
1354 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "All Labels:" << std::endl,
1355 this->Quiet);
1356 } else {
1357 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
1358 "No Labels Exist" << std::endl, this->Quiet);
1359 }
1360 for (std::string const& label : allLabels) {
1361 cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, " " << label << std::endl,
1362 this->Quiet);
1363 }
1364 }
1365
CheckResume()1366 void cmCTestMultiProcessHandler::CheckResume()
1367 {
1368 std::string fname =
1369 this->CTest->GetBinaryDir() + "/Testing/Temporary/CTestCheckpoint.txt";
1370 if (this->CTest->GetFailover()) {
1371 if (cmSystemTools::FileExists(fname, true)) {
1372 *this->TestHandler->LogFile
1373 << "Resuming previously interrupted test set" << std::endl
1374 << "----------------------------------------------------------"
1375 << std::endl;
1376
1377 cmsys::ifstream fin;
1378 fin.open(fname.c_str());
1379 std::string line;
1380 while (std::getline(fin, line)) {
1381 int index = atoi(line.c_str());
1382 this->RemoveTest(index);
1383 }
1384 fin.close();
1385 }
1386 } else if (cmSystemTools::FileExists(fname, true)) {
1387 cmSystemTools::RemoveFile(fname);
1388 }
1389 }
1390
RemoveTest(int index)1391 void cmCTestMultiProcessHandler::RemoveTest(int index)
1392 {
1393 this->EraseTest(index);
1394 this->Properties.erase(index);
1395 this->TestRunningMap[index] = false;
1396 this->TestFinishMap[index] = true;
1397 this->Completed++;
1398 }
1399
FindMaxIndex()1400 int cmCTestMultiProcessHandler::FindMaxIndex()
1401 {
1402 int max = 0;
1403 for (auto const& i : this->Tests) {
1404 if (i.first > max) {
1405 max = i.first;
1406 }
1407 }
1408 return max;
1409 }
1410
1411 // Returns true if no cycles exist in the dependency graph
CheckCycles()1412 bool cmCTestMultiProcessHandler::CheckCycles()
1413 {
1414 cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
1415 "Checking test dependency graph..." << std::endl,
1416 this->Quiet);
1417 for (auto const& it : this->Tests) {
1418 // DFS from each element to itself
1419 int root = it.first;
1420 std::set<int> visited;
1421 std::stack<int> s;
1422 s.push(root);
1423 while (!s.empty()) {
1424 int test = s.top();
1425 s.pop();
1426 if (visited.insert(test).second) {
1427 for (auto const& d : this->Tests[test]) {
1428 if (d == root) {
1429 // cycle exists
1430 cmCTestLog(
1431 this->CTest, ERROR_MESSAGE,
1432 "Error: a cycle exists in the test dependency graph "
1433 "for the test \""
1434 << this->Properties[root]->Name
1435 << "\".\nPlease fix the cycle and run ctest again.\n");
1436 return false;
1437 }
1438 s.push(d);
1439 }
1440 }
1441 }
1442 }
1443 cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
1444 "Checking test dependency graph end" << std::endl,
1445 this->Quiet);
1446 return true;
1447 }
1448