1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtTest module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39
40#include "qxctestlogger_p.h"
41
42#include <QtCore/qstring.h>
43
44#include <QtTest/private/qtestlog_p.h>
45#include <QtTest/private/qtestresult_p.h>
46
47#import <XCTest/XCTest.h>
48
49// ---------------------------------------------------------
50
51@interface XCTestProbe (Private)
52+ (BOOL)isTesting;
53+ (void)runTests:(id)unusedArgument;
54+ (NSString*)testScope;
55+ (BOOL)isInverseTestScope;
56@end
57
58@interface XCTestDriver : NSObject
59+ (XCTestDriver*)sharedTestDriver;
60@property (readonly, assign) NSObject *IDEConnection;
61@end
62
63@interface XCTest (Private)
64- (NSString *)nameForLegacyLogging;
65@end
66
67QT_WARNING_PUSH
68// Ignore XCTestProbe deprecation
69QT_WARNING_DISABLE_DEPRECATED
70
71// ---------------------------------------------------------
72
73@interface QtTestLibWrapper : XCTestCase
74@end
75
76@interface QtTestLibTests : XCTestSuite
77+ (XCTestSuiteRun*)testRun;
78@end
79
80@interface QtTestLibTest : XCTestCase
81@property (nonatomic, retain) NSString* testObjectName;
82@property (nonatomic, retain) NSString* testFunctionName;
83@end
84
85// ---------------------------------------------------------
86
87class ThreadBarriers
88{
89public:
90    enum Barrier {
91        XCTestCanStartTesting,
92        XCTestHaveStarted,
93        QtTestsCanStartTesting,
94        QtTestsHaveCompleted,
95        XCTestsHaveCompleted,
96        BarrierCount
97    };
98
99    static ThreadBarriers *get()
100    {
101        static ThreadBarriers instance;
102        return &instance;
103    }
104
105    static void initialize() { get(); }
106
107    void wait(Barrier barrier) { dispatch_semaphore_wait(barriers[barrier], DISPATCH_TIME_FOREVER); }
108    void signal(Barrier barrier) { dispatch_semaphore_signal(barriers[barrier]); }
109
110private:
111    #define FOREACH_BARRIER(cmd) for (int i = 0; i < BarrierCount; ++i) { cmd }
112
113    ThreadBarriers() { FOREACH_BARRIER(barriers[i] = dispatch_semaphore_create(0);) }
114    ~ThreadBarriers() { FOREACH_BARRIER(dispatch_release(barriers[i]);) }
115
116    dispatch_semaphore_t barriers[BarrierCount];
117};
118
119#define WAIT_FOR_BARRIER(b) ThreadBarriers::get()->wait(ThreadBarriers::b);
120#define SIGNAL_BARRIER(b) ThreadBarriers::get()->signal(ThreadBarriers::b);
121
122// ---------------------------------------------------------
123
124@implementation QtTestLibWrapper
125
126+ (void)load
127{
128    NSAutoreleasePool *autoreleasepool = [[NSAutoreleasePool alloc] init];
129
130    if (![XCTestProbe isTesting])
131        return;
132
133    if (Q_UNLIKELY(!([NSDate timeIntervalSinceReferenceDate] > 0)))
134        qFatal("error: Device date '%s' is bad, likely set to update automatically. Please correct.",
135            [[NSDate date] description].UTF8String);
136
137    XCTestDriver *testDriver = nil;
138    if ([QtTestLibWrapper usingTestManager])
139        testDriver = [XCTestDriver sharedTestDriver];
140
141    // Spawn off task to run test infrastructure on separate thread so that we can
142    // let main() execute like normal on the main thread. The queue will never be
143    // destroyed, so there's no point in trying to keep a proper retain count.
144    dispatch_async(dispatch_queue_create("io.qt.QTestLib.xctest-wrapper", DISPATCH_QUEUE_SERIAL), ^{
145        Q_ASSERT(![NSThread isMainThread]);
146        [XCTestProbe runTests:nil];
147        Q_UNREACHABLE();
148    });
149
150    // Initialize barriers before registering exit handler so that the
151    // semaphores stay alive until after the exit handler completes.
152    ThreadBarriers::initialize();
153
154    // We register an exit handler so that we can intercept when main() completes
155    // and let the XCTest thread finish up. For main() functions that never started
156    // testing using QtTestLib we also need to signal that xcTestsCanStart.
157    atexit_b(^{
158        Q_ASSERT([NSThread isMainThread]);
159
160        // In case not started by startLogging
161        SIGNAL_BARRIER(XCTestCanStartTesting);
162
163        // [XCTestProbe runTests:] ends up calling [XCTestProbe exitTests:] after
164        // all test suites have completed, which calls exit(). We use that to signal
165        // to the main thread that it's free to continue its exit handler.
166        atexit_b(^{
167            Q_ASSERT(![NSThread isMainThread]);
168            SIGNAL_BARRIER(XCTestsHaveCompleted);
169
170            // Block forever so that the main thread does all the cleanup
171            dispatch_semaphore_wait(dispatch_semaphore_create(0), DISPATCH_TIME_FOREVER);
172        });
173
174        SIGNAL_BARRIER(QtTestsHaveCompleted);
175
176        // Ensure XCTest complets the top level tests suite
177        WAIT_FOR_BARRIER(XCTestsHaveCompleted);
178    });
179
180    // Let test driver (Xcode) connection setup complete before continuing
181    if ([[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"]) {
182        while (!testDriver.IDEConnection)
183            [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate distantFuture]];
184    }
185
186    // Wait for our QtTestLib test suite to run before running main
187    WAIT_FOR_BARRIER(QtTestsCanStartTesting);
188
189    // Prevent XCTestProbe from re-launching runTests on application startup
190    [[NSNotificationCenter defaultCenter] removeObserver:[XCTestProbe class]
191        name:[NSString stringWithFormat:@"%@DidFinishLaunchingNotification",
192                #if defined(Q_OS_MACOS)
193                    @"NSApplication"
194                #else
195                    @"UIApplication"
196                #endif
197             ]
198        object:nil];
199
200    [autoreleasepool release];
201}
202
203+ (QTestLibTests *)defaultTestSuite
204{
205    return [[QtTestLibTests alloc] initWithName:@"QtTestLib"];
206}
207
208+ (BOOL)usingTestManager
209{
210    return [[[NSProcessInfo processInfo] arguments] containsObject:@"--use-testmanagerd"];
211}
212
213@end
214
215// ---------------------------------------------------------
216
217static XCTestSuiteRun *s_qtTestSuiteRun = 0;
218
219@implementation QtTestLibTests
220
221- (void)performTest:(XCTestSuiteRun *)testSuiteRun
222{
223    Q_ASSERT(![NSThread isMainThread]);
224
225    Q_ASSERT(!s_qtTestSuiteRun);
226    s_qtTestSuiteRun = testSuiteRun;
227
228    SIGNAL_BARRIER(QtTestsCanStartTesting);
229
230    // Wait for main() to complete, or a QtTestLib test to start, so we
231    // know if we should start the QtTestLib test suite.
232    WAIT_FOR_BARRIER(XCTestCanStartTesting);
233
234    if (QXcodeTestLogger::isActive())
235        [testSuiteRun start];
236
237    SIGNAL_BARRIER(XCTestHaveStarted);
238
239    // All test reporting happens on main thread from now on. Wait until
240    // main() completes before allowing the XCTest thread to continue.
241    WAIT_FOR_BARRIER(QtTestsHaveCompleted);
242
243    if ([testSuiteRun startDate])
244        [testSuiteRun stop];
245}
246
247+ (XCTestSuiteRun*)testRun
248{
249    return s_qtTestSuiteRun;
250}
251
252@end
253
254// ---------------------------------------------------------
255
256@implementation QtTestLibTest
257
258- (instancetype)initWithInvocation:(NSInvocation *)invocation
259{
260    if (self = [super initWithInvocation:invocation]) {
261        // The test object name and function name are used by XCTest after QtTestLib has
262        // reset them, so we need to store them up front for each XCTestCase.
263        self.testObjectName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()];
264        self.testFunctionName = [NSString stringWithUTF8String:QTestResult::currentTestFunction()];
265    }
266
267    return self;
268}
269
270- (NSString *)testClassName
271{
272    return self.testObjectName;
273}
274
275- (NSString *)testMethodName
276{
277    return self.testFunctionName;
278}
279
280- (NSString *)nameForLegacyLogging
281{
282    NSString *name = [NSString stringWithFormat:@"%@::%@", [self testClassName], [self testMethodName]];
283    if (QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()) {
284        const char *currentDataTag = QTestResult::currentDataTag() ? QTestResult::currentDataTag() : "";
285        const char *globalDataTag = QTestResult::currentGlobalDataTag() ? QTestResult::currentGlobalDataTag() : "";
286        const char *filler = (currentDataTag[0] && globalDataTag[0]) ? ":" : "";
287        name = [name stringByAppendingString:[NSString stringWithFormat:@"(%s%s%s)",
288            globalDataTag, filler, currentDataTag]];
289    }
290
291    return name;
292}
293
294@end
295
296// ---------------------------------------------------------
297
298bool QXcodeTestLogger::canLogTestProgress()
299{
300    return [XCTestProbe isTesting]; // FIXME: Exclude xctool
301}
302
303int QXcodeTestLogger::parseCommandLineArgument(const char *argument)
304{
305    if (strncmp(argument, "-NS", 3) == 0 || strncmp(argument, "-Apple", 6) == 0)
306        return 2; // -NSTreatUnknownArgumentsAsOpen, -ApplePersistenceIgnoreState, etc, skip argument
307    else if (strcmp(argument, "--use-testmanagerd") == 0)
308        return 2; // Skip UID argument
309    else if (strncmp(argument, "-XCTest", 7) == 0)
310        return 2; // -XCTestInvertScope, -XCTest scope, etc, skip argument
311    else if (strcmp(argument + (strlen(argument) - 7), ".xctest") == 0)
312        return 1; // Skip test bundle
313    else
314        return 0;
315}
316
317// ---------------------------------------------------------
318
319QXcodeTestLogger *QXcodeTestLogger::s_currentTestLogger = 0;
320
321// ---------------------------------------------------------
322
323QXcodeTestLogger::QXcodeTestLogger()
324    : QAbstractTestLogger(0)
325    , m_testRuns([[NSMutableArray<XCTestRun *> arrayWithCapacity:2] retain])
326
327{
328    Q_ASSERT(!s_currentTestLogger);
329    s_currentTestLogger = this;
330}
331
332QXcodeTestLogger::~QXcodeTestLogger()
333{
334    s_currentTestLogger = 0;
335    [m_testRuns release];
336}
337
338void QXcodeTestLogger::startLogging()
339{
340    SIGNAL_BARRIER(XCTestCanStartTesting);
341
342    static dispatch_once_t onceToken;
343    dispatch_once (&onceToken, ^{
344        WAIT_FOR_BARRIER(XCTestHaveStarted);
345    });
346
347    // Scope test object suite under top level QtTestLib test run
348    [m_testRuns addObject:[QtTestLibTests testRun]];
349
350    NSString *suiteName = [NSString stringWithUTF8String:QTestResult::currentTestObjectName()];
351    pushTestRunForTest([XCTestSuite testSuiteWithName:suiteName], true);
352}
353
354void QXcodeTestLogger::stopLogging()
355{
356    popTestRun();
357}
358
359static bool isTestFunctionInActiveScope(const char *function)
360{
361    static NSString *testScope = [XCTestProbe testScope];
362
363    enum TestScope { Unknown, All, None, Self, Selected };
364    static TestScope activeScope = Unknown;
365
366    if (activeScope == Unknown) {
367        if ([testScope isEqualToString:@"All"])
368            activeScope = All;
369        else if ([testScope isEqualToString:@"None"])
370            activeScope = None;
371        else if ([testScope isEqualToString:@"Self"])
372            activeScope = Self;
373        else
374            activeScope = Selected;
375    }
376
377    if (activeScope == All)
378        return true;
379    else if (activeScope == None)
380        return false;
381    else if (activeScope == Self)
382        return true; // Investigate
383
384    Q_ASSERT(activeScope == Selected);
385
386    static NSArray<NSString *> *forcedTests = [@[ @"initTestCase", @"initTestCase_data", @"cleanupTestCase" ] retain];
387    if ([forcedTests containsObject:[NSString stringWithUTF8String:function]])
388        return true;
389
390    static NSArray<NSString *> *testsInScope = [[testScope componentsSeparatedByString:@","] retain];
391    bool inScope = [testsInScope containsObject:[NSString stringWithFormat:@"%s/%s",
392                        QTestResult::currentTestObjectName(), function]];
393
394    if ([XCTestProbe isInverseTestScope])
395        inScope = !inScope;
396
397    return inScope;
398}
399
400void QXcodeTestLogger::enterTestFunction(const char *function)
401{
402    if (!isTestFunctionInActiveScope(function))
403        QTestResult::setSkipCurrentTest(true);
404
405    XCTest *test = [QtTestLibTest testCaseWithInvocation:nil];
406    pushTestRunForTest(test, !QTestResult::skipCurrentTest());
407}
408
409void QXcodeTestLogger::leaveTestFunction()
410{
411    popTestRun();
412}
413
414void QXcodeTestLogger::addIncident(IncidentTypes type, const char *description,
415                                   const char *file, int line)
416{
417    XCTestRun *testRun = [m_testRuns lastObject];
418
419    // The 'expected' argument to recordFailureWithDescription refers to whether
420    // the failure was a regular failed assertion, or an unexpected exception,
421    // so in our case it's always 'YES', and we need to explicitly ignore XFail.
422    if (type == QAbstractTestLogger::XFail) {
423        QTestCharBuffer buf;
424        NSString *testCaseName = [[testRun test] nameForLegacyLogging];
425        QTest::qt_asprintf(&buf, "Test Case '%s' failed expectedly (%s).\n",
426            [testCaseName UTF8String], description);
427        outputString(buf.constData());
428        return;
429    }
430
431    if (type == QAbstractTestLogger::Pass) {
432        // We ignore non-data passes, as we're already reporting that as part of the
433        // normal test case start/stop cycle.
434        if (!(QTestResult::currentDataTag() || QTestResult::currentGlobalDataTag()))
435            return;
436
437        QTestCharBuffer buf;
438        NSString *testCaseName = [[testRun test] nameForLegacyLogging];
439        QTest::qt_asprintf(&buf, "Test Case '%s' passed.\n", [testCaseName UTF8String]);
440        outputString(buf.constData());
441        return;
442    }
443
444    // FIXME: Handle blacklisted tests
445
446    if (!file || !description)
447        return; // Or report?
448
449    [testRun recordFailureWithDescription:[NSString stringWithUTF8String:description]
450        inFile:[NSString stringWithUTF8String:file] atLine:line expected:YES];
451}
452
453void QXcodeTestLogger::addMessage(MessageTypes type, const QString &message,
454                                  const char *file, int line)
455{
456    QTestCharBuffer buf;
457
458    if (QTestLog::verboseLevel() > 0 && (file && line)) {
459        QTest::qt_asprintf(&buf, "%s:%d: ", file, line);
460        outputString(buf.constData());
461    }
462
463    if (type == QAbstractTestLogger::Skip) {
464        XCTestRun *testRun = [m_testRuns lastObject];
465        NSString *testCaseName = [[testRun test] nameForLegacyLogging];
466        QTest::qt_asprintf(&buf, "Test Case '%s' skipped (%s).\n",
467            [testCaseName UTF8String], message.toUtf8().constData());
468    } else {
469        QTest::qt_asprintf(&buf, "%s\n", message.toUtf8().constData());
470    }
471
472    outputString(buf.constData());
473}
474
475void QXcodeTestLogger::addBenchmarkResult(const QBenchmarkResult &result)
476{
477    Q_UNUSED(result);
478}
479
480void QXcodeTestLogger::pushTestRunForTest(XCTest *test, bool start)
481{
482    XCTestRun *testRun = [[test testRunClass] testRunWithTest:test];
483    [m_testRuns addObject:testRun];
484
485    if (start)
486        [testRun start];
487}
488
489XCTestRun *QXcodeTestLogger::popTestRun()
490{
491    XCTestRun *testRun = [[m_testRuns lastObject] retain];
492    [m_testRuns removeLastObject];
493
494    if ([testRun startDate])
495        [testRun stop];
496
497    [[m_testRuns lastObject] addTestRun:testRun];
498    [testRun release];
499
500    return testRun;
501}
502
503bool QXcodeTestLogger::isActive()
504{
505    return s_currentTestLogger;
506}
507
508QT_WARNING_POP
509