1 /****************************************************************************
2 **
3 ** Copyright (C) 2016 The Qt Company Ltd.
4 ** Contact: https://www.qt.io/licensing/
5 **
6 ** This file is part of Qt Creator.
7 **
8 ** Commercial License Usage
9 ** Licensees holding valid commercial Qt licenses may use this file in
10 ** accordance with the commercial license agreement provided with the
11 ** Software or, alternatively, in accordance with the terms contained in
12 ** a written agreement between you and The Qt Company. For licensing terms
13 ** and conditions see https://www.qt.io/terms-conditions. For further
14 ** information use the contact form at https://www.qt.io/contact-us.
15 **
16 ** GNU General Public License Usage
17 ** Alternatively, this file may be used under the terms of the GNU
18 ** General Public License version 3 as published by the Free Software
19 ** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
20 ** included in the packaging of this file. Please review the following
21 ** information to ensure the GNU General Public License requirements will
22 ** be met: https://www.gnu.org/licenses/gpl-3.0.html.
23 **
24 ****************************************************************************/
25 
26 #include "testcodeparser.h"
27 
28 #include "autotestconstants.h"
29 #include "testframeworkmanager.h"
30 #include "testsettings.h"
31 #include "testtreemodel.h"
32 
33 #include <coreplugin/editormanager/editormanager.h>
34 #include <coreplugin/progressmanager/futureprogress.h>
35 #include <coreplugin/progressmanager/progressmanager.h>
36 #include <cpptools/cppmodelmanager.h>
37 #include <cpptools/cpptoolsconstants.h>
38 #include <projectexplorer/project.h>
39 #include <projectexplorer/session.h>
40 #include <qmljstools/qmljsmodelmanager.h>
41 
42 #include <utils/algorithm.h>
43 #include <utils/mapreduce.h>
44 #include <utils/qtcassert.h>
45 #include <utils/runextensions.h>
46 
47 #include <QFuture>
48 #include <QFutureInterface>
49 #include <QLoggingCategory>
50 
51 static Q_LOGGING_CATEGORY(LOG, "qtc.autotest.testcodeparser", QtWarningMsg)
52 
53 namespace Autotest {
54 namespace Internal {
55 
56 using namespace ProjectExplorer;
57 
TestCodeParser()58 TestCodeParser::TestCodeParser()
59     :  m_threadPool(new QThreadPool(this))
60 {
61     // connect to ProgressManager to postpone test parsing when CppModelManager is parsing
62     auto progressManager = qobject_cast<Core::ProgressManager *>(Core::ProgressManager::instance());
63     connect(progressManager, &Core::ProgressManager::taskStarted,
64             this, &TestCodeParser::onTaskStarted);
65     connect(progressManager, &Core::ProgressManager::allTasksFinished,
66             this, &TestCodeParser::onAllTasksFinished);
67     connect(&m_futureWatcher, &QFutureWatcher<TestParseResultPtr>::started,
68             this, &TestCodeParser::parsingStarted);
69     connect(&m_futureWatcher, &QFutureWatcher<TestParseResultPtr>::finished,
70             this, &TestCodeParser::onFinished);
71     connect(&m_futureWatcher, &QFutureWatcher<TestParseResultPtr>::resultReadyAt,
72             this, [this] (int index) {
73         emit testParseResultReady(m_futureWatcher.resultAt(index));
74     });
75     connect(this, &TestCodeParser::parsingFinished, this, &TestCodeParser::releaseParserInternals);
76     m_reparseTimer.setSingleShot(true);
77     connect(&m_reparseTimer, &QTimer::timeout, this, &TestCodeParser::parsePostponedFiles);
78     m_threadPool->setMaxThreadCount(std::max(QThread::idealThreadCount()/4, 1));
79 }
80 
setState(State state)81 void TestCodeParser::setState(State state)
82 {
83     if (m_parserState == Shutdown)
84         return;
85     qCDebug(LOG) << "setState(" << state << "), currentState:" << m_parserState;
86     // avoid triggering parse before code model parsing has finished, but mark as dirty
87     if (m_codeModelParsing) {
88         m_dirty = true;
89         qCDebug(LOG) << "Not setting new state - code model parsing is running, just marking dirty";
90         return;
91     }
92 
93     if ((state == Idle) && (m_parserState == PartialParse || m_parserState == FullParse)) {
94         qCDebug(LOG) << "Not setting state, parse is running";
95         return;
96     }
97     m_parserState = state;
98 
99     if (m_parserState == Idle && SessionManager::startupProject()) {
100         if (m_postponedUpdateType == UpdateType::FullUpdate || m_dirty) {
101             emitUpdateTestTree();
102         } else if (m_postponedUpdateType == UpdateType::PartialUpdate) {
103             m_postponedUpdateType = UpdateType::NoUpdate;
104             qCDebug(LOG) << "calling scanForTests with postponed files (setState)";
105             if (!m_reparseTimer.isActive())
106                 scanForTests(Utils::toList(m_postponedFiles));
107         }
108     }
109 }
110 
syncTestFrameworks(const QList<ITestParser * > & parsers)111 void TestCodeParser::syncTestFrameworks(const QList<ITestParser *> &parsers)
112 {
113     if (m_parserState != Idle) {
114         // there's a running parse
115         m_postponedUpdateType = UpdateType::NoUpdate;
116         m_postponedFiles.clear();
117         Core::ProgressManager::cancelTasks(Constants::TASK_PARSE);
118     }
119     qCDebug(LOG) << "Setting" << parsers << "as current parsers";
120     m_testCodeParsers = parsers;
121 }
122 
emitUpdateTestTree(ITestParser * parser)123 void TestCodeParser::emitUpdateTestTree(ITestParser *parser)
124 {
125     if (m_testCodeParsers.isEmpty())
126         return;
127     if (parser)
128         m_updateParsers.insert(parser);
129     else
130         m_updateParsers.clear();
131     if (m_singleShotScheduled) {
132         qCDebug(LOG) << "not scheduling another updateTestTree";
133         return;
134     }
135 
136     qCDebug(LOG) << "adding singleShot";
137     m_singleShotScheduled = true;
138     QTimer::singleShot(1000, this, [this]() { updateTestTree(m_updateParsers); });
139 }
140 
updateTestTree(const QSet<ITestParser * > & parsers)141 void TestCodeParser::updateTestTree(const QSet<ITestParser *> &parsers)
142 {
143     m_singleShotScheduled = false;
144     if (m_codeModelParsing) {
145         m_postponedUpdateType = UpdateType::FullUpdate;
146         m_postponedFiles.clear();
147         if (parsers.isEmpty()) {
148             m_updateParsers.clear();
149         } else {
150             for (ITestParser *parser : parsers)
151                 m_updateParsers.insert(parser);
152         }
153         return;
154     }
155 
156     if (!SessionManager::startupProject())
157         return;
158 
159     m_postponedUpdateType = UpdateType::NoUpdate;
160     qCDebug(LOG) << "calling scanForTests (updateTestTree)";
161     QList<ITestParser *> sortedParsers = Utils::toList(parsers);
162     Utils::sort(sortedParsers, [](const ITestParser *lhs, const ITestParser *rhs) {
163         return lhs->framework()->priority() < rhs->framework()->priority();
164     });
165     scanForTests(Utils::FilePaths(), sortedParsers);
166 }
167 
168 /****** threaded parsing stuff *******/
169 
onDocumentUpdated(const Utils::FilePath & fileName,bool isQmlFile)170 void TestCodeParser::onDocumentUpdated(const Utils::FilePath &fileName, bool isQmlFile)
171 {
172     if (m_codeModelParsing || m_postponedUpdateType == UpdateType::FullUpdate)
173         return;
174 
175     Project *project = SessionManager::startupProject();
176     if (!project)
177         return;
178     // Quick tests: qml files aren't necessarily listed inside project files
179     if (!isQmlFile && !project->isKnownFile(fileName))
180         return;
181 
182     scanForTests(Utils::FilePaths{fileName});
183 }
184 
onCppDocumentUpdated(const CPlusPlus::Document::Ptr & document)185 void TestCodeParser::onCppDocumentUpdated(const CPlusPlus::Document::Ptr &document)
186 {
187     onDocumentUpdated(Utils::FilePath::fromString(document->fileName()));
188 }
189 
onQmlDocumentUpdated(const QmlJS::Document::Ptr & document)190 void TestCodeParser::onQmlDocumentUpdated(const QmlJS::Document::Ptr &document)
191 {
192     const QString fileName = document->fileName();
193     if (!fileName.endsWith(".qbs"))
194         onDocumentUpdated(Utils::FilePath::fromString(fileName), true);
195 }
196 
onStartupProjectChanged(Project * project)197 void TestCodeParser::onStartupProjectChanged(Project *project)
198 {
199     if (m_parserState == FullParse || m_parserState == PartialParse) {
200         qCDebug(LOG) << "Canceling scanForTest (startup project changed)";
201         Core::ProgressManager::cancelTasks(Constants::TASK_PARSE);
202     }
203     emit aboutToPerformFullParse();
204     if (project)
205         emitUpdateTestTree();
206 }
207 
onProjectPartsUpdated(Project * project)208 void TestCodeParser::onProjectPartsUpdated(Project *project)
209 {
210     if (project != SessionManager::startupProject())
211         return;
212     if (m_codeModelParsing)
213         m_postponedUpdateType = UpdateType::FullUpdate;
214     else
215         emitUpdateTestTree();
216 }
217 
aboutToShutdown()218 void TestCodeParser::aboutToShutdown()
219 {
220     qCDebug(LOG) << "Disabling (immediately) - shutting down";
221     State oldState = m_parserState;
222     m_parserState = Shutdown;
223     if (oldState == PartialParse || oldState == FullParse) {
224         m_futureWatcher.cancel();
225         m_futureWatcher.waitForFinished();
226     }
227 }
228 
postponed(const Utils::FilePaths & fileList)229 bool TestCodeParser::postponed(const Utils::FilePaths &fileList)
230 {
231     switch (m_parserState) {
232     case Idle:
233         if (fileList.size() == 1) {
234             if (m_reparseTimerTimedOut)
235                 return false;
236             switch (m_postponedFiles.size()) {
237             case 0:
238                 m_postponedFiles.insert(fileList.first());
239                 m_reparseTimer.setInterval(1000);
240                 m_reparseTimer.start();
241                 return true;
242             case 1:
243                 if (m_postponedFiles.contains(fileList.first())) {
244                     m_reparseTimer.start();
245                     return true;
246                 }
247                 Q_FALLTHROUGH();
248             default:
249                 m_postponedFiles.insert(fileList.first());
250                 m_reparseTimer.stop();
251                 m_reparseTimer.setInterval(0);
252                 m_reparseTimerTimedOut = false;
253                 m_reparseTimer.start();
254                 return true;
255             }
256         }
257         return false;
258     case PartialParse:
259     case FullParse:
260         // parse is running, postponing a full parse
261         if (fileList.isEmpty()) {
262             m_postponedFiles.clear();
263             m_postponedUpdateType = UpdateType::FullUpdate;
264             qCDebug(LOG) << "Canceling scanForTest (full parse triggered while running a scan)";
265             Core::ProgressManager::cancelTasks(Constants::TASK_PARSE);
266         } else {
267             // partial parse triggered, but full parse is postponed already, ignoring this
268             if (m_postponedUpdateType == UpdateType::FullUpdate)
269                 return true;
270             // partial parse triggered, postpone or add current files to already postponed partial
271             for (const Utils::FilePath &file : fileList)
272                 m_postponedFiles.insert(file);
273             m_postponedUpdateType = UpdateType::PartialUpdate;
274         }
275         return true;
276     case Shutdown:
277         break;
278     }
279     QTC_ASSERT(false, return false); // should not happen at all
280 }
281 
parseFileForTests(const QList<ITestParser * > & parsers,QFutureInterface<TestParseResultPtr> & futureInterface,const Utils::FilePath & fileName)282 static void parseFileForTests(const QList<ITestParser *> &parsers,
283                               QFutureInterface<TestParseResultPtr> &futureInterface,
284                               const Utils::FilePath &fileName)
285 {
286     for (ITestParser *parser : parsers) {
287         if (futureInterface.isCanceled())
288             return;
289         if (parser->processDocument(futureInterface, fileName))
290             break;
291     }
292 }
293 
scanForTests(const Utils::FilePaths & fileList,const QList<ITestParser * > & parsers)294 void TestCodeParser::scanForTests(const Utils::FilePaths &fileList,
295                                   const QList<ITestParser *> &parsers)
296 {
297     if (m_parserState == Shutdown || m_testCodeParsers.isEmpty())
298         return;
299 
300     if (postponed(fileList))
301         return;
302 
303     m_reparseTimer.stop();
304     m_reparseTimerTimedOut = false;
305     m_postponedFiles.clear();
306     bool isFullParse = fileList.isEmpty();
307     Project *project = SessionManager::startupProject();
308     if (!project)
309         return;
310     Utils::FilePaths list;
311     if (isFullParse) {
312         list = project->files(Project::SourceFiles);
313         if (list.isEmpty()) {
314             // at least project file should be there, but might happen if parsing current project
315             // takes too long, especially when opening sessions holding multiple projects
316             qCDebug(LOG) << "File list empty (FullParse) - trying again in a sec";
317             emitUpdateTestTree();
318             return;
319         }
320         qCDebug(LOG) << "setting state to FullParse (scanForTests)";
321         m_parserState = FullParse;
322     } else {
323         list << fileList;
324         qCDebug(LOG) << "setting state to PartialParse (scanForTests)";
325         m_parserState = PartialParse;
326     }
327 
328     m_parsingHasFailed = false;
329     TestTreeModel::instance()->updateCheckStateCache();
330     if (isFullParse) {
331         // remove qml files as they will be found automatically by the referencing cpp file
332         list = Utils::filtered(list, [] (const Utils::FilePath &fn) {
333             return !fn.endsWith(".qml");
334         });
335         if (!parsers.isEmpty()) {
336             for (ITestParser *parser : parsers) {
337                 parser->framework()->rootNode()->markForRemovalRecursively(true);
338             }
339         } else {
340             emit requestRemoveAllFrameworkItems();
341         }
342     } else if (!parsers.isEmpty()) {
343         for (ITestParser *parser: parsers) {
344             for (const Utils::FilePath &filePath : qAsConst(list))
345                 parser->framework()->rootNode()->markForRemovalRecursively(filePath);
346         }
347     } else {
348         for (const Utils::FilePath &filePath : qAsConst(list))
349             emit requestRemoval(filePath);
350     }
351 
352     QTC_ASSERT(!(isFullParse && list.isEmpty()), onFinished(); return);
353 
354     // use only a single parser or all current active?
355     const QList<ITestParser *> codeParsers = parsers.isEmpty() ? m_testCodeParsers : parsers;
356     qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "StartParsing";
357     for (ITestParser *parser : codeParsers)
358         parser->init(list, isFullParse);
359 
360     QFuture<TestParseResultPtr> future = Utils::map(list,
361         [codeParsers](QFutureInterface<TestParseResultPtr> &fi, const Utils::FilePath &file) {
362             parseFileForTests(codeParsers, fi, file);
363         },
364         Utils::MapReduceOption::Unordered,
365         m_threadPool,
366         QThread::LowestPriority);
367     m_futureWatcher.setFuture(future);
368     if (list.size() > 5) {
369         Core::ProgressManager::addTask(future, tr("Scanning for Tests"),
370                                        Autotest::Constants::TASK_PARSE);
371     }
372 }
373 
onTaskStarted(Utils::Id type)374 void TestCodeParser::onTaskStarted(Utils::Id type)
375 {
376     if (type == CppTools::Constants::TASK_INDEX) {
377         m_codeModelParsing = true;
378         if (m_parserState == FullParse || m_parserState == PartialParse) {
379             m_postponedUpdateType = m_parserState == FullParse
380                     ? UpdateType::FullUpdate : UpdateType::PartialUpdate;
381             qCDebug(LOG) << "Canceling scan for test (CppModelParsing started)";
382             m_parsingHasFailed = true;
383             Core::ProgressManager::cancelTasks(Constants::TASK_PARSE);
384         }
385     }
386 }
387 
onAllTasksFinished(Utils::Id type)388 void TestCodeParser::onAllTasksFinished(Utils::Id type)
389 {
390     // if we cancel parsing ensure that progress animation is canceled as well
391     if (type == Constants::TASK_PARSE && m_parsingHasFailed)
392         emit parsingFailed();
393 
394     // only CPP parsing is relevant as we trigger Qml parsing internally anyway
395     if (type != CppTools::Constants::TASK_INDEX)
396         return;
397     m_codeModelParsing = false;
398 
399     // avoid illegal parser state if respective widgets became hidden while parsing
400     setState(Idle);
401 }
402 
onFinished()403 void TestCodeParser::onFinished()
404 {
405     if (m_futureWatcher.isCanceled())
406         m_parsingHasFailed = true;
407     switch (m_parserState) {
408     case PartialParse:
409         qCDebug(LOG) << "setting state to Idle (onFinished, PartialParse)";
410         m_parserState = Idle;
411         onPartialParsingFinished();
412         qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "PartParsingFin";
413         break;
414     case FullParse:
415         qCDebug(LOG) << "setting state to Idle (onFinished, FullParse)";
416         m_parserState = Idle;
417         m_dirty = m_parsingHasFailed;
418         if (m_postponedUpdateType != UpdateType::NoUpdate || m_parsingHasFailed) {
419             onPartialParsingFinished();
420         } else {
421             qCDebug(LOG) << "emitting parsingFinished"
422                          << "(onFinished, FullParse, nothing postponed, parsing succeeded)";
423             m_updateParsers.clear();
424             emit parsingFinished();
425             qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "ParsingFin";
426         }
427         m_dirty = false;
428         break;
429     case Shutdown:
430         qCDebug(LOG) << "Shutdown complete - not emitting parsingFinished (onFinished)";
431         break;
432     default:
433         qWarning("I should not be here... State: %d", m_parserState);
434         break;
435     }
436 }
437 
onPartialParsingFinished()438 void TestCodeParser::onPartialParsingFinished()
439 {
440     const UpdateType oldType = m_postponedUpdateType;
441     m_postponedUpdateType = UpdateType::NoUpdate;
442     switch (oldType) {
443     case UpdateType::FullUpdate:
444         qCDebug(LOG) << "calling updateTestTree (onPartialParsingFinished)";
445         updateTestTree(m_updateParsers);
446         break;
447     case UpdateType::PartialUpdate:
448         qCDebug(LOG) << "calling scanForTests with postponed files (onPartialParsingFinished)";
449         if (!m_reparseTimer.isActive())
450             scanForTests(Utils::toList(m_postponedFiles));
451         break;
452     case UpdateType::NoUpdate:
453         m_dirty |= m_codeModelParsing;
454         if (m_dirty) {
455             emit parsingFailed();
456             qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "ParsingFail";
457         } else if (!m_singleShotScheduled) {
458             qCDebug(LOG) << "emitting parsingFinished"
459                          << "(onPartialParsingFinished, nothing postponed, not dirty)";
460             m_updateParsers.clear();
461             emit parsingFinished();
462             qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "ParsingFin";
463         } else {
464             qCDebug(LOG) << "not emitting parsingFinished"
465                          << "(on PartialParsingFinished, singleshot scheduled)";
466         }
467         break;
468     }
469 }
470 
parsePostponedFiles()471 void TestCodeParser::parsePostponedFiles()
472 {
473     m_reparseTimerTimedOut = true;
474     scanForTests(Utils::toList(m_postponedFiles));
475 }
476 
releaseParserInternals()477 void TestCodeParser::releaseParserInternals()
478 {
479     for (ITestParser *parser : qAsConst(m_testCodeParsers))
480         parser->release();
481 }
482 
483 } // namespace Internal
484 } // namespace Autotest
485