1/**************************************************************************** 2** 3** Copyright (C) 2016 The Qt Company Ltd. 4** Contact: https://www.qt.io/licensing/ 5** 6** This file is part of the QtTest module of the Qt Toolkit. 7** 8** $QT_BEGIN_LICENSE:LGPL$ 9** Commercial License Usage 10** Licensees holding valid commercial Qt licenses may use this file in 11** accordance with the commercial license agreement provided with the 12** Software or, alternatively, in accordance with the terms contained in 13** a written agreement between you and The Qt Company. For licensing terms 14** and conditions see https://www.qt.io/terms-conditions. For further 15** information use the contact form at https://www.qt.io/contact-us. 16** 17** GNU Lesser General Public License Usage 18** Alternatively, this file may be used under the terms of the GNU Lesser 19** General Public License version 3 as published by the Free Software 20** Foundation and appearing in the file LICENSE.LGPL3 included in the 21** packaging of this file. Please review the following information to 22** ensure the GNU Lesser General Public License version 3 requirements 23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. 24** 25** GNU General Public License Usage 26** Alternatively, this file may be used under the terms of the GNU 27** General Public License version 2.0 or (at your option) the GNU General 28** Public license version 3 or any later version approved by the KDE Free 29** Qt Foundation. The licenses are as published by the Free Software 30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 31** included in the packaging of this file. Please review the following 32** information to ensure the GNU General Public License requirements will 33** be met: https://www.gnu.org/licenses/gpl-2.0.html and 34** https://www.gnu.org/licenses/gpl-3.0.html. 35** 36** $QT_END_LICENSE$ 37** 38****************************************************************************/ 39 40#include "qxctestlogger_p.h" 41 42#include <QtCore/qstring.h> 43 44#include <QtTest/private/qtestlog_p.h> 45#include <QtTest/private/qtestresult_p.h> 46 47#import <XCTest/XCTest.h> 48 49// --------------------------------------------------------- 50 51@interface XCTestProbe (Private) 52+ (BOOL)isTesting; 53+ (void)runTests:(id)unusedArgument; 54+ (NSString*)testScope; 55+ (BOOL)isInverseTestScope; 56@end 57 58@interface XCTestDriver : NSObject 59+ (XCTestDriver*)sharedTestDriver; 60@property (readonly, assign) NSObject *IDEConnection; 61@end 62 63@interface XCTest (Private) 64- (NSString *)nameForLegacyLogging; 65@end 66 67QT_WARNING_PUSH 68// Ignore XCTestProbe deprecation 69QT_WARNING_DISABLE_DEPRECATED 70 71// --------------------------------------------------------- 72 73@interface QtTestLibWrapper : XCTestCase 74@end 75 76@interface QtTestLibTests : XCTestSuite 77+ (XCTestSuiteRun*)testRun; 78@end 79 80@interface QtTestLibTest : XCTestCase 81@property (nonatomic, retain) NSString* testObjectName; 82@property (nonatomic, retain) NSString* testFunctionName; 83@end 84 85// --------------------------------------------------------- 86 87class ThreadBarriers 88{ 89public: 90 enum Barrier { 91 XCTestCanStartTesting, 92 XCTestHaveStarted, 93 QtTestsCanStartTesting, 94 QtTestsHaveCompleted, 95 XCTestsHaveCompleted, 96 BarrierCount 97 }; 98 99 static ThreadBarriers *get() 100 { 101 static ThreadBarriers instance; 102 return &instance; 103 } 104 105 static void initialize() { get(); } 106 107 void wait(Barrier barrier) { dispatch_semaphore_wait(barriers[barrier], DISPATCH_TIME_FOREVER); } 108 void signal(Barrier barrier) { dispatch_semaphore_signal(barriers[barrier]); } 109 110private: 111 #define FOREACH_BARRIER(cmd) for (int i = 0; i < BarrierCount; ++i) { cmd } 112 113 ThreadBarriers() { FOREACH_BARRIER(barriers[i] = dispatch_semaphore_create(0);) } 114 ~ThreadBarriers() { FOREACH_BARRIER(dispatch_release(barriers[i]);) } 115 116 dispatch_semaphore_t barriers[BarrierCount]; 117}; 118 119#define WAIT_FOR_BARRIER(b) ThreadBarriers::get()->wait(ThreadBarriers::b); 120#define SIGNAL_BARRIER(b) ThreadBarriers::get()->signal(ThreadBarriers::b); 121 122// --------------------------------------------------------- 123 124@implementation QtTestLibWrapper 125 126+ (void)load 127{ 128 NSAutoreleasePool *autoreleasepool = [[NSAutoreleasePool alloc] init]; 129 130 if (![XCTestProbe isTesting]) 131 return; 132 133 if (Q_UNLIKELY(!([NSDate timeIntervalSinceReferenceDate] > 0))) 134 qFatal("error: Device date '%s' is bad, likely set to update automatically. Please correct.", 135 [[NSDate date] description].UTF8String); 136 137 XCTestDriver *testDriver = nil; 138 if ([QtTestLibWrapper usingTestManager]) 139 testDriver = [XCTestDriver sharedTestDriver]; 140 141 // Spawn off task to run test infrastructure on separate thread so that we can 142 // let main() execute like normal on the main thread. The queue will never be 143 // destroyed, so there's no point in trying to keep a proper retain count. 144 dispatch_async(dispatch_queue_create("io.qt.QTestLib.xctest-wrapper", DISPATCH_QUEUE_SERIAL), ^{ 145 Q_ASSERT(![NSThread isMainThread]); 146 [XCTestProbe runTests:nil]; 147 Q_UNREACHABLE(); 148 }); 149 150 // Initialize barriers before registering exit handler so that the 151 // semaphores stay alive until after the exit handler completes. 152 ThreadBarriers::initialize(); 153 154 // We register an exit handler so that we can intercept when main() completes 155 // and let the XCTest thread finish up. For main() functions that never started 156 // testing using QtTestLib we also need to signal that xcTestsCanStart. 157 atexit_b(^{ 158 Q_ASSERT([NSThread isMainThread]); 159 160 // In case not started by startLogging 161 SIGNAL_BARRIER(XCTestCanStartTesting); 162 163 // [XCTestProbe runTests:] ends up calling [XCTestProbe exitTests:] after 164 // all test suites have completed, which calls exit(). We use that to signal 165 // to the main thread that it's free to continue its exit handler. 166 atexit_b(^{ 167 Q_ASSERT(![NSThread isMainThread]); 168 SIGNAL_BARRIER(XCTestsHaveCompleted); 169 170 // Block forever so that the main thread does all the cleanup 171 dispatch_semaphore_wait(dispatch_semaphore_create(0), DISPATCH_TIME_FOREVER); 172 }); 173 174 SIGNAL_BARRIER(QtTestsHaveCompleted); 175 176 // Ensure XCTest complets the top level tests suite 177 WAIT_FOR_BARRIER(XCTestsHaveCompleted); 178 }); 179 180 // Let test driver (Xcode) connection setup complete before continuing 181 if ([[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]) { 182 while (!testDriver.IDEConnection) 183 [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]]; 184 } 185 186 // Wait for our QtTestLib test suite to run before running main 187 WAIT_FOR_BARRIER(QtTestsCanStartTesting); 188 189 // Prevent XCTestProbe from re-launching runTests on application startup 190 [[NSNotificationCenter defaultCenter] removeObserver:[XCTestProbe class] 191 name:[NSString stringWithFormat:@"%@DidFinishLaunchingNotification", 192 #if defined(Q_OS_MACOS) 193 @"NSApplication" 194 #else 195 @"UIApplication" 196 #endif 197 ] 198 object:nil]; 199 200 [autoreleasepool release]; 201} 202 203+ (QTestLibTests *)defaultTestSuite 204{ 205 return [[QtTestLibTests alloc] initWithName:@"QtTestLib"]; 206} 207 208+ (BOOL)usingTestManager 209{ 210 return [[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]; 211} 212 213@end 214 215// --------------------------------------------------------- 216 217static XCTestSuiteRun *s_qtTestSuiteRun = 0; 218 219@implementation QtTestLibTests 220 221- (void)performTest:(XCTestSuiteRun *)testSuiteRun 222{ 223 Q_ASSERT(![NSThread isMainThread]); 224 225 Q_ASSERT(!s_qtTestSuiteRun); 226 s_qtTestSuiteRun = testSuiteRun; 227 228 SIGNAL_BARRIER(QtTestsCanStartTesting); 229 230 // Wait for main() to complete, or a QtTestLib test to start, so we 231 // know if we should start the QtTestLib test suite. 232 WAIT_FOR_BARRIER(XCTestCanStartTesting); 233 234 if (QXcodeTestLogger::isActive()) 235 [testSuiteRun start]; 236 237 SIGNAL_BARRIER(XCTestHaveStarted); 238 239 // All test reporting happens on main thread from now on. Wait until 240 // main() completes before allowing the XCTest thread to continue. 241 WAIT_FOR_BARRIER(QtTestsHaveCompleted); 242 243 if ([testSuiteRun startDate]) 244 [testSuiteRun stop]; 245} 246 247+ (XCTestSuiteRun*)testRun 248{ 249 return s_qtTestSuiteRun; 250} 251 252@end 253 254// --------------------------------------------------------- 255 256@implementation QtTestLibTest 257 258- (instancetype)initWithInvocation:(NSInvocation *)invocation 259{ 260 if (self = [super initWithInvocation:invocation]) { 261 // The test object name and function name are used by XCTest after QtTestLib has 262 // reset them, so we need to store them up front for each XCTestCase. 263 self.testObjectName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()]; 264 self.testFunctionName = [NSString stringWithUTF8String:QTestResult::currentTestFunction()]; 265 } 266 267 return self; 268} 269 270- (NSString *)testClassName 271{ 272 return self.testObjectName; 273} 274 275- (NSString *)testMethodName 276{ 277 return self.testFunctionName; 278} 279 280- (NSString *)nameForLegacyLogging 281{ 282 NSString *name = [NSString stringWithFormat:@"%@::%@", [self testClassName], [self testMethodName]]; 283 if (QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()) { 284 const char *currentDataTag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : ""; 285 const char *globalDataTag = QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : ""; 286 const char *filler = (currentDataTag[0] && globalDataTag[0]) ? ":" : ""; 287 name = [name stringByAppendingString:[NSString stringWithFormat:@"(%s%s%s)", 288 globalDataTag, filler, currentDataTag]]; 289 } 290 291 return name; 292} 293 294@end 295 296// --------------------------------------------------------- 297 298bool QXcodeTestLogger::canLogTestProgress() 299{ 300 return [XCTestProbe isTesting]; // FIXME: Exclude xctool 301} 302 303int QXcodeTestLogger::parseCommandLineArgument(const char *argument) 304{ 305 if (strncmp(argument, "-NS", 3) == 0 || strncmp(argument, "-Apple", 6) == 0) 306 return 2; // -NSTreatUnknownArgumentsAsOpen, -ApplePersistenceIgnoreState, etc, skip argument 307 else if (strcmp(argument, "--use-testmanagerd") == 0) 308 return 2; // Skip UID argument 309 else if (strncmp(argument, "-XCTest", 7) == 0) 310 return 2; // -XCTestInvertScope, -XCTest scope, etc, skip argument 311 else if (strcmp(argument + (strlen(argument) - 7), ".xctest") == 0) 312 return 1; // Skip test bundle 313 else 314 return 0; 315} 316 317// --------------------------------------------------------- 318 319QXcodeTestLogger *QXcodeTestLogger::s_currentTestLogger = 0; 320 321// --------------------------------------------------------- 322 323QXcodeTestLogger::QXcodeTestLogger() 324 : QAbstractTestLogger(0) 325 , m_testRuns([[NSMutableArray<XCTestRun *> arrayWithCapacity:2] retain]) 326 327{ 328 Q_ASSERT(!s_currentTestLogger); 329 s_currentTestLogger = this; 330} 331 332QXcodeTestLogger::~QXcodeTestLogger() 333{ 334 s_currentTestLogger = 0; 335 [m_testRuns release]; 336} 337 338void QXcodeTestLogger::startLogging() 339{ 340 SIGNAL_BARRIER(XCTestCanStartTesting); 341 342 static dispatch_once_t onceToken; 343 dispatch_once (&onceToken, ^{ 344 WAIT_FOR_BARRIER(XCTestHaveStarted); 345 }); 346 347 // Scope test object suite under top level QtTestLib test run 348 [m_testRuns addObject:[QtTestLibTests testRun]]; 349 350 NSString *suiteName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()]; 351 pushTestRunForTest([XCTestSuite testSuiteWithName:suiteName], true); 352} 353 354void QXcodeTestLogger::stopLogging() 355{ 356 popTestRun(); 357} 358 359static bool isTestFunctionInActiveScope(const char *function) 360{ 361 static NSString *testScope = [XCTestProbe testScope]; 362 363 enum TestScope { Unknown, All, None, Self, Selected }; 364 static TestScope activeScope = Unknown; 365 366 if (activeScope == Unknown) { 367 if ([testScope isEqualToString:@"All"]) 368 activeScope = All; 369 else if ([testScope isEqualToString:@"None"]) 370 activeScope = None; 371 else if ([testScope isEqualToString:@"Self"]) 372 activeScope = Self; 373 else 374 activeScope = Selected; 375 } 376 377 if (activeScope == All) 378 return true; 379 else if (activeScope == None) 380 return false; 381 else if (activeScope == Self) 382 return true; // Investigate 383 384 Q_ASSERT(activeScope == Selected); 385 386 static NSArray<NSString *> *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain]; 387 if ([forcedTests containsObject:[NSString stringWithUTF8String:function]]) 388 return true; 389 390 static NSArray<NSString *> *testsInScope = [[testScope componentsSeparatedByString:@","] retain]; 391 bool inScope = [testsInScope containsObject:[NSString stringWithFormat:@"%s/%s", 392 QTestResult::currentTestObjectName(), function]]; 393 394 if ([XCTestProbe isInverseTestScope]) 395 inScope = !inScope; 396 397 return inScope; 398} 399 400void QXcodeTestLogger::enterTestFunction(const char *function) 401{ 402 if (!isTestFunctionInActiveScope(function)) 403 QTestResult::setSkipCurrentTest(true); 404 405 XCTest *test = [QtTestLibTest testCaseWithInvocation:nil]; 406 pushTestRunForTest(test, !QTestResult::skipCurrentTest()); 407} 408 409void QXcodeTestLogger::leaveTestFunction() 410{ 411 popTestRun(); 412} 413 414void QXcodeTestLogger::addIncident(IncidentTypes type, const char *description, 415 const char *file, int line) 416{ 417 XCTestRun *testRun = [m_testRuns lastObject]; 418 419 // The 'expected' argument to recordFailureWithDescription refers to whether 420 // the failure was a regular failed assertion, or an unexpected exception, 421 // so in our case it's always 'YES', and we need to explicitly ignore XFail. 422 if (type == QAbstractTestLogger::XFail) { 423 QTestCharBuffer buf; 424 NSString *testCaseName = [[testRun test] nameForLegacyLogging]; 425 QTest::qt_asprintf(&buf, "Test Case '%s' failed expectedly (%s).\n", 426 [testCaseName UTF8String], description); 427 outputString(buf.constData()); 428 return; 429 } 430 431 if (type == QAbstractTestLogger::Pass) { 432 // We ignore non-data passes, as we're already reporting that as part of the 433 // normal test case start/stop cycle. 434 if (!(QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag())) 435 return; 436 437 QTestCharBuffer buf; 438 NSString *testCaseName = [[testRun test] nameForLegacyLogging]; 439 QTest::qt_asprintf(&buf, "Test Case '%s' passed.\n", [testCaseName UTF8String]); 440 outputString(buf.constData()); 441 return; 442 } 443 444 // FIXME: Handle blacklisted tests 445 446 if (!file || !description) 447 return; // Or report? 448 449 [testRun recordFailureWithDescription:[NSString stringWithUTF8String:description] 450 inFile:[NSString stringWithUTF8String:file] atLine:line expected:YES]; 451} 452 453void QXcodeTestLogger::addMessage(MessageTypes type, const QString &message, 454 const char *file, int line) 455{ 456 QTestCharBuffer buf; 457 458 if (QTestLog::verboseLevel() > 0 && (file && line)) { 459 QTest::qt_asprintf(&buf, "%s:%d: ", file, line); 460 outputString(buf.constData()); 461 } 462 463 if (type == QAbstractTestLogger::Skip) { 464 XCTestRun *testRun = [m_testRuns lastObject]; 465 NSString *testCaseName = [[testRun test] nameForLegacyLogging]; 466 QTest::qt_asprintf(&buf, "Test Case '%s' skipped (%s).\n", 467 [testCaseName UTF8String], message.toUtf8().constData()); 468 } else { 469 QTest::qt_asprintf(&buf, "%s\n", message.toUtf8().constData()); 470 } 471 472 outputString(buf.constData()); 473} 474 475void QXcodeTestLogger::addBenchmarkResult(const QBenchmarkResult &result) 476{ 477 Q_UNUSED(result); 478} 479 480void QXcodeTestLogger::pushTestRunForTest(XCTest *test, bool start) 481{ 482 XCTestRun *testRun = [[test testRunClass] testRunWithTest:test]; 483 [m_testRuns addObject:testRun]; 484 485 if (start) 486 [testRun start]; 487} 488 489XCTestRun *QXcodeTestLogger::popTestRun() 490{ 491 XCTestRun *testRun = [[m_testRuns lastObject] retain]; 492 [m_testRuns removeLastObject]; 493 494 if ([testRun startDate]) 495 [testRun stop]; 496 497 [[m_testRuns lastObject] addTestRun:testRun]; 498 [testRun release]; 499 500 return testRun; 501} 502 503bool QXcodeTestLogger::isActive() 504{ 505 return s_currentTestLogger; 506} 507 508QT_WARNING_POP 509