1 /****************************************************************************
2 **
3 ** Copyright (C) 2016 The Qt Company Ltd.
4 ** Contact: https://www.qt.io/licensing/
5 **
6 ** This file is part of Qt Creator.
7 **
8 ** Commercial License Usage
9 ** Licensees holding valid commercial Qt licenses may use this file in
10 ** accordance with the commercial license agreement provided with the
11 ** Software or, alternatively, in accordance with the terms contained in
12 ** a written agreement between you and The Qt Company. For licensing terms
13 ** and conditions see https://www.qt.io/terms-conditions. For further
14 ** information use the contact form at https://www.qt.io/contact-us.
15 **
16 ** GNU General Public License Usage
17 ** Alternatively, this file may be used under the terms of the GNU
18 ** General Public License version 3 as published by the Free Software
19 ** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
20 ** included in the packaging of this file. Please review the following
21 ** information to ensure the GNU General Public License requirements will
22 ** be met: https://www.gnu.org/licenses/gpl-3.0.html.
23 **
24 ****************************************************************************/
25
26 #include "testcodeparser.h"
27
28 #include "autotestconstants.h"
29 #include "testframeworkmanager.h"
30 #include "testsettings.h"
31 #include "testtreemodel.h"
32
33 #include <coreplugin/editormanager/editormanager.h>
34 #include <coreplugin/progressmanager/futureprogress.h>
35 #include <coreplugin/progressmanager/progressmanager.h>
36 #include <cpptools/cppmodelmanager.h>
37 #include <cpptools/cpptoolsconstants.h>
38 #include <projectexplorer/project.h>
39 #include <projectexplorer/session.h>
40 #include <qmljstools/qmljsmodelmanager.h>
41
42 #include <utils/algorithm.h>
43 #include <utils/mapreduce.h>
44 #include <utils/qtcassert.h>
45 #include <utils/runextensions.h>
46
47 #include <QFuture>
48 #include <QFutureInterface>
49 #include <QLoggingCategory>
50
51 static Q_LOGGING_CATEGORY(LOG, "qtc.autotest.testcodeparser", QtWarningMsg)
52
53 namespace Autotest {
54 namespace Internal {
55
56 using namespace ProjectExplorer;
57
TestCodeParser()58 TestCodeParser::TestCodeParser()
59 : m_threadPool(new QThreadPool(this))
60 {
61 // connect to ProgressManager to postpone test parsing when CppModelManager is parsing
62 auto progressManager = qobject_cast<Core::ProgressManager *>(Core::ProgressManager::instance());
63 connect(progressManager, &Core::ProgressManager::taskStarted,
64 this, &TestCodeParser::onTaskStarted);
65 connect(progressManager, &Core::ProgressManager::allTasksFinished,
66 this, &TestCodeParser::onAllTasksFinished);
67 connect(&m_futureWatcher, &QFutureWatcher<TestParseResultPtr>::started,
68 this, &TestCodeParser::parsingStarted);
69 connect(&m_futureWatcher, &QFutureWatcher<TestParseResultPtr>::finished,
70 this, &TestCodeParser::onFinished);
71 connect(&m_futureWatcher, &QFutureWatcher<TestParseResultPtr>::resultReadyAt,
72 this, [this] (int index) {
73 emit testParseResultReady(m_futureWatcher.resultAt(index));
74 });
75 connect(this, &TestCodeParser::parsingFinished, this, &TestCodeParser::releaseParserInternals);
76 m_reparseTimer.setSingleShot(true);
77 connect(&m_reparseTimer, &QTimer::timeout, this, &TestCodeParser::parsePostponedFiles);
78 m_threadPool->setMaxThreadCount(std::max(QThread::idealThreadCount()/4, 1));
79 }
80
setState(State state)81 void TestCodeParser::setState(State state)
82 {
83 if (m_parserState == Shutdown)
84 return;
85 qCDebug(LOG) << "setState(" << state << "), currentState:" << m_parserState;
86 // avoid triggering parse before code model parsing has finished, but mark as dirty
87 if (m_codeModelParsing) {
88 m_dirty = true;
89 qCDebug(LOG) << "Not setting new state - code model parsing is running, just marking dirty";
90 return;
91 }
92
93 if ((state == Idle) && (m_parserState == PartialParse || m_parserState == FullParse)) {
94 qCDebug(LOG) << "Not setting state, parse is running";
95 return;
96 }
97 m_parserState = state;
98
99 if (m_parserState == Idle && SessionManager::startupProject()) {
100 if (m_postponedUpdateType == UpdateType::FullUpdate || m_dirty) {
101 emitUpdateTestTree();
102 } else if (m_postponedUpdateType == UpdateType::PartialUpdate) {
103 m_postponedUpdateType = UpdateType::NoUpdate;
104 qCDebug(LOG) << "calling scanForTests with postponed files (setState)";
105 if (!m_reparseTimer.isActive())
106 scanForTests(Utils::toList(m_postponedFiles));
107 }
108 }
109 }
110
syncTestFrameworks(const QList<ITestParser * > & parsers)111 void TestCodeParser::syncTestFrameworks(const QList<ITestParser *> &parsers)
112 {
113 if (m_parserState != Idle) {
114 // there's a running parse
115 m_postponedUpdateType = UpdateType::NoUpdate;
116 m_postponedFiles.clear();
117 Core::ProgressManager::cancelTasks(Constants::TASK_PARSE);
118 }
119 qCDebug(LOG) << "Setting" << parsers << "as current parsers";
120 m_testCodeParsers = parsers;
121 }
122
emitUpdateTestTree(ITestParser * parser)123 void TestCodeParser::emitUpdateTestTree(ITestParser *parser)
124 {
125 if (m_testCodeParsers.isEmpty())
126 return;
127 if (parser)
128 m_updateParsers.insert(parser);
129 else
130 m_updateParsers.clear();
131 if (m_singleShotScheduled) {
132 qCDebug(LOG) << "not scheduling another updateTestTree";
133 return;
134 }
135
136 qCDebug(LOG) << "adding singleShot";
137 m_singleShotScheduled = true;
138 QTimer::singleShot(1000, this, [this]() { updateTestTree(m_updateParsers); });
139 }
140
updateTestTree(const QSet<ITestParser * > & parsers)141 void TestCodeParser::updateTestTree(const QSet<ITestParser *> &parsers)
142 {
143 m_singleShotScheduled = false;
144 if (m_codeModelParsing) {
145 m_postponedUpdateType = UpdateType::FullUpdate;
146 m_postponedFiles.clear();
147 if (parsers.isEmpty()) {
148 m_updateParsers.clear();
149 } else {
150 for (ITestParser *parser : parsers)
151 m_updateParsers.insert(parser);
152 }
153 return;
154 }
155
156 if (!SessionManager::startupProject())
157 return;
158
159 m_postponedUpdateType = UpdateType::NoUpdate;
160 qCDebug(LOG) << "calling scanForTests (updateTestTree)";
161 QList<ITestParser *> sortedParsers = Utils::toList(parsers);
162 Utils::sort(sortedParsers, [](const ITestParser *lhs, const ITestParser *rhs) {
163 return lhs->framework()->priority() < rhs->framework()->priority();
164 });
165 scanForTests(Utils::FilePaths(), sortedParsers);
166 }
167
168 /****** threaded parsing stuff *******/
169
onDocumentUpdated(const Utils::FilePath & fileName,bool isQmlFile)170 void TestCodeParser::onDocumentUpdated(const Utils::FilePath &fileName, bool isQmlFile)
171 {
172 if (m_codeModelParsing || m_postponedUpdateType == UpdateType::FullUpdate)
173 return;
174
175 Project *project = SessionManager::startupProject();
176 if (!project)
177 return;
178 // Quick tests: qml files aren't necessarily listed inside project files
179 if (!isQmlFile && !project->isKnownFile(fileName))
180 return;
181
182 scanForTests(Utils::FilePaths{fileName});
183 }
184
onCppDocumentUpdated(const CPlusPlus::Document::Ptr & document)185 void TestCodeParser::onCppDocumentUpdated(const CPlusPlus::Document::Ptr &document)
186 {
187 onDocumentUpdated(Utils::FilePath::fromString(document->fileName()));
188 }
189
onQmlDocumentUpdated(const QmlJS::Document::Ptr & document)190 void TestCodeParser::onQmlDocumentUpdated(const QmlJS::Document::Ptr &document)
191 {
192 const QString fileName = document->fileName();
193 if (!fileName.endsWith(".qbs"))
194 onDocumentUpdated(Utils::FilePath::fromString(fileName), true);
195 }
196
onStartupProjectChanged(Project * project)197 void TestCodeParser::onStartupProjectChanged(Project *project)
198 {
199 if (m_parserState == FullParse || m_parserState == PartialParse) {
200 qCDebug(LOG) << "Canceling scanForTest (startup project changed)";
201 Core::ProgressManager::cancelTasks(Constants::TASK_PARSE);
202 }
203 emit aboutToPerformFullParse();
204 if (project)
205 emitUpdateTestTree();
206 }
207
onProjectPartsUpdated(Project * project)208 void TestCodeParser::onProjectPartsUpdated(Project *project)
209 {
210 if (project != SessionManager::startupProject())
211 return;
212 if (m_codeModelParsing)
213 m_postponedUpdateType = UpdateType::FullUpdate;
214 else
215 emitUpdateTestTree();
216 }
217
aboutToShutdown()218 void TestCodeParser::aboutToShutdown()
219 {
220 qCDebug(LOG) << "Disabling (immediately) - shutting down";
221 State oldState = m_parserState;
222 m_parserState = Shutdown;
223 if (oldState == PartialParse || oldState == FullParse) {
224 m_futureWatcher.cancel();
225 m_futureWatcher.waitForFinished();
226 }
227 }
228
postponed(const Utils::FilePaths & fileList)229 bool TestCodeParser::postponed(const Utils::FilePaths &fileList)
230 {
231 switch (m_parserState) {
232 case Idle:
233 if (fileList.size() == 1) {
234 if (m_reparseTimerTimedOut)
235 return false;
236 switch (m_postponedFiles.size()) {
237 case 0:
238 m_postponedFiles.insert(fileList.first());
239 m_reparseTimer.setInterval(1000);
240 m_reparseTimer.start();
241 return true;
242 case 1:
243 if (m_postponedFiles.contains(fileList.first())) {
244 m_reparseTimer.start();
245 return true;
246 }
247 Q_FALLTHROUGH();
248 default:
249 m_postponedFiles.insert(fileList.first());
250 m_reparseTimer.stop();
251 m_reparseTimer.setInterval(0);
252 m_reparseTimerTimedOut = false;
253 m_reparseTimer.start();
254 return true;
255 }
256 }
257 return false;
258 case PartialParse:
259 case FullParse:
260 // parse is running, postponing a full parse
261 if (fileList.isEmpty()) {
262 m_postponedFiles.clear();
263 m_postponedUpdateType = UpdateType::FullUpdate;
264 qCDebug(LOG) << "Canceling scanForTest (full parse triggered while running a scan)";
265 Core::ProgressManager::cancelTasks(Constants::TASK_PARSE);
266 } else {
267 // partial parse triggered, but full parse is postponed already, ignoring this
268 if (m_postponedUpdateType == UpdateType::FullUpdate)
269 return true;
270 // partial parse triggered, postpone or add current files to already postponed partial
271 for (const Utils::FilePath &file : fileList)
272 m_postponedFiles.insert(file);
273 m_postponedUpdateType = UpdateType::PartialUpdate;
274 }
275 return true;
276 case Shutdown:
277 break;
278 }
279 QTC_ASSERT(false, return false); // should not happen at all
280 }
281
parseFileForTests(const QList<ITestParser * > & parsers,QFutureInterface<TestParseResultPtr> & futureInterface,const Utils::FilePath & fileName)282 static void parseFileForTests(const QList<ITestParser *> &parsers,
283 QFutureInterface<TestParseResultPtr> &futureInterface,
284 const Utils::FilePath &fileName)
285 {
286 for (ITestParser *parser : parsers) {
287 if (futureInterface.isCanceled())
288 return;
289 if (parser->processDocument(futureInterface, fileName))
290 break;
291 }
292 }
293
scanForTests(const Utils::FilePaths & fileList,const QList<ITestParser * > & parsers)294 void TestCodeParser::scanForTests(const Utils::FilePaths &fileList,
295 const QList<ITestParser *> &parsers)
296 {
297 if (m_parserState == Shutdown || m_testCodeParsers.isEmpty())
298 return;
299
300 if (postponed(fileList))
301 return;
302
303 m_reparseTimer.stop();
304 m_reparseTimerTimedOut = false;
305 m_postponedFiles.clear();
306 bool isFullParse = fileList.isEmpty();
307 Project *project = SessionManager::startupProject();
308 if (!project)
309 return;
310 Utils::FilePaths list;
311 if (isFullParse) {
312 list = project->files(Project::SourceFiles);
313 if (list.isEmpty()) {
314 // at least project file should be there, but might happen if parsing current project
315 // takes too long, especially when opening sessions holding multiple projects
316 qCDebug(LOG) << "File list empty (FullParse) - trying again in a sec";
317 emitUpdateTestTree();
318 return;
319 }
320 qCDebug(LOG) << "setting state to FullParse (scanForTests)";
321 m_parserState = FullParse;
322 } else {
323 list << fileList;
324 qCDebug(LOG) << "setting state to PartialParse (scanForTests)";
325 m_parserState = PartialParse;
326 }
327
328 m_parsingHasFailed = false;
329 TestTreeModel::instance()->updateCheckStateCache();
330 if (isFullParse) {
331 // remove qml files as they will be found automatically by the referencing cpp file
332 list = Utils::filtered(list, [] (const Utils::FilePath &fn) {
333 return !fn.endsWith(".qml");
334 });
335 if (!parsers.isEmpty()) {
336 for (ITestParser *parser : parsers) {
337 parser->framework()->rootNode()->markForRemovalRecursively(true);
338 }
339 } else {
340 emit requestRemoveAllFrameworkItems();
341 }
342 } else if (!parsers.isEmpty()) {
343 for (ITestParser *parser: parsers) {
344 for (const Utils::FilePath &filePath : qAsConst(list))
345 parser->framework()->rootNode()->markForRemovalRecursively(filePath);
346 }
347 } else {
348 for (const Utils::FilePath &filePath : qAsConst(list))
349 emit requestRemoval(filePath);
350 }
351
352 QTC_ASSERT(!(isFullParse && list.isEmpty()), onFinished(); return);
353
354 // use only a single parser or all current active?
355 const QList<ITestParser *> codeParsers = parsers.isEmpty() ? m_testCodeParsers : parsers;
356 qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "StartParsing";
357 for (ITestParser *parser : codeParsers)
358 parser->init(list, isFullParse);
359
360 QFuture<TestParseResultPtr> future = Utils::map(list,
361 [codeParsers](QFutureInterface<TestParseResultPtr> &fi, const Utils::FilePath &file) {
362 parseFileForTests(codeParsers, fi, file);
363 },
364 Utils::MapReduceOption::Unordered,
365 m_threadPool,
366 QThread::LowestPriority);
367 m_futureWatcher.setFuture(future);
368 if (list.size() > 5) {
369 Core::ProgressManager::addTask(future, tr("Scanning for Tests"),
370 Autotest::Constants::TASK_PARSE);
371 }
372 }
373
onTaskStarted(Utils::Id type)374 void TestCodeParser::onTaskStarted(Utils::Id type)
375 {
376 if (type == CppTools::Constants::TASK_INDEX) {
377 m_codeModelParsing = true;
378 if (m_parserState == FullParse || m_parserState == PartialParse) {
379 m_postponedUpdateType = m_parserState == FullParse
380 ? UpdateType::FullUpdate : UpdateType::PartialUpdate;
381 qCDebug(LOG) << "Canceling scan for test (CppModelParsing started)";
382 m_parsingHasFailed = true;
383 Core::ProgressManager::cancelTasks(Constants::TASK_PARSE);
384 }
385 }
386 }
387
onAllTasksFinished(Utils::Id type)388 void TestCodeParser::onAllTasksFinished(Utils::Id type)
389 {
390 // if we cancel parsing ensure that progress animation is canceled as well
391 if (type == Constants::TASK_PARSE && m_parsingHasFailed)
392 emit parsingFailed();
393
394 // only CPP parsing is relevant as we trigger Qml parsing internally anyway
395 if (type != CppTools::Constants::TASK_INDEX)
396 return;
397 m_codeModelParsing = false;
398
399 // avoid illegal parser state if respective widgets became hidden while parsing
400 setState(Idle);
401 }
402
onFinished()403 void TestCodeParser::onFinished()
404 {
405 if (m_futureWatcher.isCanceled())
406 m_parsingHasFailed = true;
407 switch (m_parserState) {
408 case PartialParse:
409 qCDebug(LOG) << "setting state to Idle (onFinished, PartialParse)";
410 m_parserState = Idle;
411 onPartialParsingFinished();
412 qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "PartParsingFin";
413 break;
414 case FullParse:
415 qCDebug(LOG) << "setting state to Idle (onFinished, FullParse)";
416 m_parserState = Idle;
417 m_dirty = m_parsingHasFailed;
418 if (m_postponedUpdateType != UpdateType::NoUpdate || m_parsingHasFailed) {
419 onPartialParsingFinished();
420 } else {
421 qCDebug(LOG) << "emitting parsingFinished"
422 << "(onFinished, FullParse, nothing postponed, parsing succeeded)";
423 m_updateParsers.clear();
424 emit parsingFinished();
425 qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "ParsingFin";
426 }
427 m_dirty = false;
428 break;
429 case Shutdown:
430 qCDebug(LOG) << "Shutdown complete - not emitting parsingFinished (onFinished)";
431 break;
432 default:
433 qWarning("I should not be here... State: %d", m_parserState);
434 break;
435 }
436 }
437
onPartialParsingFinished()438 void TestCodeParser::onPartialParsingFinished()
439 {
440 const UpdateType oldType = m_postponedUpdateType;
441 m_postponedUpdateType = UpdateType::NoUpdate;
442 switch (oldType) {
443 case UpdateType::FullUpdate:
444 qCDebug(LOG) << "calling updateTestTree (onPartialParsingFinished)";
445 updateTestTree(m_updateParsers);
446 break;
447 case UpdateType::PartialUpdate:
448 qCDebug(LOG) << "calling scanForTests with postponed files (onPartialParsingFinished)";
449 if (!m_reparseTimer.isActive())
450 scanForTests(Utils::toList(m_postponedFiles));
451 break;
452 case UpdateType::NoUpdate:
453 m_dirty |= m_codeModelParsing;
454 if (m_dirty) {
455 emit parsingFailed();
456 qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "ParsingFail";
457 } else if (!m_singleShotScheduled) {
458 qCDebug(LOG) << "emitting parsingFinished"
459 << "(onPartialParsingFinished, nothing postponed, not dirty)";
460 m_updateParsers.clear();
461 emit parsingFinished();
462 qCDebug(LOG) << QDateTime::currentDateTime().toString("hh:mm:ss.zzz") << "ParsingFin";
463 } else {
464 qCDebug(LOG) << "not emitting parsingFinished"
465 << "(on PartialParsingFinished, singleshot scheduled)";
466 }
467 break;
468 }
469 }
470
parsePostponedFiles()471 void TestCodeParser::parsePostponedFiles()
472 {
473 m_reparseTimerTimedOut = true;
474 scanForTests(Utils::toList(m_postponedFiles));
475 }
476
releaseParserInternals()477 void TestCodeParser::releaseParserInternals()
478 {
479 for (ITestParser *parser : qAsConst(m_testCodeParsers))
480 parser->release();
481 }
482
483 } // namespace Internal
484 } // namespace Autotest
485