1 /****************************************************************************
2 **
3 ** Copyright (C) 2015 The Qt Company Ltd.
4 ** Contact: http://www.qt.io/licensing/
5 **
6 ** This file is part of the test suite of the Qt Toolkit.
7 **
8 ** $QT_BEGIN_LICENSE:LGPL$
9 ** Commercial License Usage
10 ** Licensees holding valid commercial Qt licenses may use this file in
11 ** accordance with the commercial license agreement provided with the
12 ** Software or, alternatively, in accordance with the terms contained in
13 ** a written agreement between you and The Qt Company. For licensing terms
14 ** and conditions see http://www.qt.io/terms-conditions. For further
15 ** information use the contact form at http://www.qt.io/contact-us.
16 **
17 ** GNU Lesser General Public License Usage
18 ** Alternatively, this file may be used under the terms of the GNU Lesser
19 ** General Public License version 2.1 or version 3 as published by the Free
20 ** Software Foundation and appearing in the file LICENSE.LGPLv21 and
21 ** LICENSE.LGPLv3 included in the packaging of this file. Please review the
22 ** following information to ensure the GNU Lesser General Public License
23 ** requirements will be met: https://www.gnu.org/licenses/lgpl.html and
24 ** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
25 **
26 ** As a special exception, The Qt Company gives you certain additional
27 ** rights. These rights are described in The Qt Company LGPL Exception
28 ** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
29 **
30 ** GNU General Public License Usage
31 ** Alternatively, this file may be used under the terms of the GNU
32 ** General Public License version 3.0 as published by the Free Software
33 ** Foundation and appearing in the file LICENSE.GPL included in the
34 ** packaging of this file. Please review the following information to
35 ** ensure the GNU General Public License version 3.0 requirements will be
36 ** met: http://www.gnu.org/copyleft/gpl.html.
37 **
38 ** $QT_END_LICENSE$
39 **
40 ****************************************************************************/
41
42
43 #include <atWrapper.h>
44 #include <datagenerator/datagenerator.h>
45
46 #include <QString>
47 #include <QHash>
48 #include <QFile>
49 #include <QFtp>
50 #include <QObject>
51 #include <QHostInfo>
52 #include <QWidget>
53 #include <QImage>
54 #include <QtTest/QSignalSpy>
55 #include <QLibraryInfo>
56
57 static const char *ArthurDir = "../../arthur";
58
59 #include <string.h>
60
atWrapper()61 atWrapper::atWrapper()
62 {
63
64 // initTests();
65
66 }
67
initTests(bool * haveBaseline)68 bool atWrapper::initTests(bool *haveBaseline)
69 {
70 qDebug() << "Running test on buildkey:" << QLibraryInfo::buildKey() << " qt version:" << qVersion();
71
72 qDebug( "Initializing tests..." );
73
74 if (!loadConfig( QHostInfo::localHostName().split( "." ).first() + ".ini" ))
75 return false;
76
77 //Reset the FTP environment where the results are stored
78 *haveBaseline = setupFTP();
79
80 // Retrieve the latest test result baseline from the FTP server.
81 downloadBaseline();
82 return true;
83 }
84
downloadBaseline()85 void atWrapper::downloadBaseline()
86 {
87
88 qDebug() << "Now downloading baseline...";
89
90 QFtp ftp;
91
92 QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpMgetAddToList(const QUrlInfo & ) ) );
93
94 //Making sure that the needed local directories exist.
95
96 QHashIterator<QString, QString> j(enginesToTest);
97
98 while ( j.hasNext() )
99 {
100 j.next();
101
102 QDir dir( output );
103
104 if ( !dir.cd( j.key() + ".baseline" ) )
105 dir.mkdir( j.key() + ".baseline" );
106
107 }
108
109 //FTP to the host specified in the config file, and retrieve the test result baseline.
110 ftp.connectToHost( ftpHost );
111 ftp.login( ftpUser, ftpPass );
112
113 ftp.cd( ftpBaseDir );
114
115 QHashIterator<QString, QString> i(enginesToTest);
116 while ( i.hasNext() )
117 {
118 i.next();
119 mgetDirList.clear();
120 mgetDirList << i.key() + ".baseline";
121 ftp.cd( i.key() + ".baseline" );
122 ftp.list();
123 ftp.cd( ".." );
124
125 while ( ftp.hasPendingCommands() )
126 QCoreApplication::instance()->processEvents();
127
128 ftpMgetDone( true );
129 }
130
131 ftp.close();
132 ftp.close();
133
134 while ( ftp.hasPendingCommands() )
135 QCoreApplication::instance()->processEvents();
136
137 }
138
ftpMgetAddToList(const QUrlInfo & urlInfo)139 void atWrapper::ftpMgetAddToList( const QUrlInfo &urlInfo )
140 {
141 //Simply adding to the list of files to download.
142 mgetDirList << urlInfo.name();
143
144 }
145
ftpMgetDone(bool error)146 void atWrapper::ftpMgetDone( bool error)
147 {
148 Q_UNUSED( error );
149
150 //Downloading the files listed in mgetDirList...
151 QFtp ftp;
152 ftp.connectToHost( ftpHost );
153 ftp.login( ftpUser, ftpPass );
154
155 QFile* file;
156
157 if ( mgetDirList.size() > 1 )
158 for ( int i = 1; i < mgetDirList.size(); ++i )
159 {
160 file = new QFile( QString( output ) + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ) );
161 if (file->open(QIODevice::WriteOnly)) {
162 ftp.get( ftpBaseDir + "/" + mgetDirList.at( 0 ) + "/" + mgetDirList.at( i ), file );
163 ftp.list(); //Only there to fill up a slot in the pendingCommands queue.
164 while ( ftp.hasPendingCommands() )
165 QCoreApplication::instance()->processEvents();
166 file->close();
167 } else {
168 qDebug() << "Couldn't open file for writing: " << file->fileName();
169 }
170 }
171
172
173 while ( ftp.hasPendingCommands() )
174 QCoreApplication::instance()->processEvents();
175 }
176
uploadFailed(QString dir,QString filename,QByteArray filedata)177 void atWrapper::uploadFailed( QString dir, QString filename, QByteArray filedata )
178 {
179 //Upload a failed test case image to the FTP server.
180 QFtp ftp;
181 ftp.connectToHost( ftpHost );
182 ftp.login( ftpUser, ftpPass );
183
184 ftp.cd( ftpBaseDir );
185 ftp.cd( dir );
186
187 ftp.put( filedata, filename, QFtp::Binary );
188
189 ftp.close();
190
191 while ( ftp.hasPendingCommands() )
192 QCoreApplication::instance()->processEvents();
193 }
194
195 // returns false if no baseline exists
setupFTP()196 bool atWrapper::setupFTP()
197 {
198 qDebug( "Setting up FTP environment" );
199
200 QString dir = "";
201 ftpMkDir( ftpBaseDir );
202
203 ftpBaseDir += "/" + QLibraryInfo::buildKey();
204
205 ftpMkDir( ftpBaseDir );
206
207 ftpBaseDir += "/" + QString( qVersion() );
208
209 ftpMkDir( ftpBaseDir );
210
211 QHashIterator<QString, QString> i(enginesToTest);
212 QHashIterator<QString, QString> j(enginesToTest);
213
214 bool haveBaseline = true;
215 //Creating the baseline directories for each engine
216 while ( i.hasNext() )
217 {
218 i.next();
219 //qDebug() << "Creating dir with key:" << i.key();
220 ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".failed" );
221 ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".diff" );
222 if (!ftpMkDir( ftpBaseDir + "/" + QString( i.key() ) + ".baseline" ))
223 haveBaseline = false;
224 }
225
226
227 QFtp ftp;
228 ftp.connectToHost( ftpHost );
229 ftp.login( ftpUser, ftpPass );
230
231 ftp.cd( ftpBaseDir );
232 //Deleting previous failed directory and all the files in it, then recreating it.
233 while ( j.hasNext() )
234 {
235 j.next();
236 rmDirList.clear();
237 rmDirList << ftpBaseDir + "/" + j.key() + ".failed" + "/";
238 ftpRmDir( j.key() + ".failed" );
239 ftp.rmdir( j.key() + ".failed" );
240 ftp.mkdir( j.key() + ".failed" );
241 ftp.list();
242
243 while ( ftp.hasPendingCommands() )
244 QCoreApplication::instance()->processEvents();
245
246 rmDirList.clear();
247 rmDirList << ftpBaseDir + "/" + j.key() + ".diff" + "/";
248 ftpRmDir( j.key() + ".diff" );
249 ftp.rmdir( j.key() + ".diff" );
250 ftp.mkdir( j.key() + ".diff" );
251 ftp.list();
252
253 while ( ftp.hasPendingCommands() )
254 QCoreApplication::instance()->processEvents();
255
256 }
257
258 ftp.close();
259
260 while ( ftp.hasPendingCommands() )
261 QCoreApplication::instance()->processEvents();
262
263 return haveBaseline;
264 }
265
ftpRmDir(QString dir)266 void atWrapper::ftpRmDir( QString dir )
267 {
268 //Hack to remove a populated directory. (caveat: containing only files and empty dirs, not recursive!)
269 qDebug() << "Now removing directory: " << dir;
270 QFtp ftp;
271 QObject::connect( &ftp, SIGNAL( listInfo( const QUrlInfo & ) ), this, SLOT( ftpRmDirAddToList(const QUrlInfo & ) ) );
272 QObject::connect( &ftp, SIGNAL( done( bool ) ), this, SLOT( ftpRmDirDone( bool ) ) );
273
274 ftp.connectToHost( ftpHost );
275 ftp.login( ftpUser, ftpPass );
276
277 ftp.list( ftpBaseDir + "/" + dir );
278 ftp.close();
279 ftp.close();
280
281 while ( ftp.hasPendingCommands() )
282 QCoreApplication::instance()->processEvents();
283 }
284
ftpRmDirDone(bool error)285 void atWrapper::ftpRmDirDone( bool error )
286 {
287 //Deleting each file in the directory listning, rmDirList.
288 Q_UNUSED( error );
289
290 QFtp ftp;
291 ftp.connectToHost( ftpHost );
292 ftp.login( ftpUser, ftpPass );
293
294 if ( rmDirList.size() > 1 )
295 for (int i = 1; i < rmDirList.size(); ++i)
296 ftp.remove( rmDirList.at(0) + rmDirList.at( i ) );
297
298 ftp.close();
299
300 while ( ftp.hasPendingCommands() )
301 QCoreApplication::instance()->processEvents();
302 }
303
304 // returns false if the directory already exists
ftpMkDir(QString dir)305 bool atWrapper::ftpMkDir( QString dir )
306 {
307 //Simply used to avoid QFTP from bailing out and loosing a queue of commands.
308 // IE: conveniance.
309 QFtp ftp;
310
311 QSignalSpy commandSpy(&ftp, SIGNAL(commandFinished(int, bool)));
312
313 ftp.connectToHost( ftpHost );
314 ftp.login( ftpUser, ftpPass );
315 const int command = ftp.mkdir( dir );
316 ftp.close();
317
318 while ( ftp.hasPendingCommands() )
319 QCoreApplication::instance()->processEvents();
320
321 for (int i = 0; i < commandSpy.count(); ++i)
322 if (commandSpy.at(i).at(0) == command)
323 return commandSpy.at(i).at(1).toBool();
324
325 return false;
326 }
327
328
ftpRmDirAddToList(const QUrlInfo & urlInfo)329 void atWrapper::ftpRmDirAddToList( const QUrlInfo &urlInfo )
330 {
331 //Just adding the file to the list for deletion
332 rmDirList << urlInfo.name();
333 }
334
335
executeTests()336 bool atWrapper::executeTests()
337 {
338 qDebug("Executing the tests...");
339
340 QHashIterator<QString, QString> i(enginesToTest);
341
342 DataGenerator generator;
343
344 //Running datagenerator against all the frameworks specified in the config file.
345 while ( i.hasNext() )
346 {
347
348 i.next();
349
350 qDebug( "Now testing: " + i.key().toLatin1() );
351
352 char* params[13];
353 //./bin/datagenerator -framework data/framework.ini -engine OpenGL -suite 1.1 -output outtest
354
355
356 QByteArray eng = i.key().toLatin1();
357 QByteArray fwk = framework.toLatin1();
358 QByteArray sut = suite.toLatin1();
359 QByteArray out = output.toLatin1();
360 QByteArray siz = size.toLatin1();
361 QByteArray fill = fillColor.toLatin1();
362
363 params[1] = "-framework";
364 params[2] = fwk.data();
365 params[3] = "-engine";
366 params[4] = eng.data();
367 params[5] = "-suite";
368 params[6] = sut.data();
369 params[7] = "-output";
370 params[8] = out.data();
371 params[9] = "-size";
372 params[10] = siz.data();
373 params[11] = "-fill";
374 params[12] = fill.data();
375
376 generator.run( 13, params );
377 }
378
379 return true;
380 }
381
createBaseline()382 void atWrapper::createBaseline()
383 {
384 qDebug( "Now uploading a baseline of only the latest test values" );
385
386 QHashIterator<QString, QString> i(enginesToTest);
387
388 QDir dir( output );
389 QFtp ftp;
390 ftp.connectToHost( ftpHost );
391 ftp.login( ftpUser, ftpPass );
392 ftp.cd( ftpBaseDir );
393 //Upload all the latest test results to the FTP server's baseline directory.
394 while ( i.hasNext() )
395 {
396
397 i.next();
398 dir.cd( i.key() );
399 ftp.cd( i.key() + ".baseline" );
400 dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);
401 dir.setNameFilters( QStringList() << "*.png" );
402 QFileInfoList list = dir.entryInfoList();
403 dir.cd( ".." );
404 for (int n = 0; n < list.size(); n++)
405 {
406 QFileInfo fileInfo = list.at( n );
407 QFile file( QString( output ) + "/" + i.key() + "/" + fileInfo.fileName() );
408 file.open( QIODevice::ReadOnly );
409 QByteArray fileData = file.readAll();
410 //qDebug() << "Sending up:" << fileInfo.fileName() << "with file size" << fileData.size();
411 file.close();
412 ftp.put( fileData, fileInfo.fileName(), QFtp::Binary );
413 }
414
415 ftp.cd( ".." );
416 }
417
418 ftp.close();
419
420 while ( ftp.hasPendingCommands() )
421 QCoreApplication::instance()->processEvents();
422 }
423
compare()424 bool atWrapper::compare()
425 {
426 qDebug( "Now comparing the results to the baseline" );
427
428 QHashIterator<QString, QString> i(enginesToTest);
429
430 while ( i.hasNext() )
431 {
432 i.next();
433
434 compareDirs( output , i.key() );
435
436 }
437
438 return true;
439 }
440
compareDirs(QString basedir,QString target)441 void atWrapper::compareDirs( QString basedir, QString target )
442 {
443
444 QDir dir( basedir );
445
446 /* The following should be redundant now.
447
448 if ( !dir.cd( target + ".failed" ) )
449 dir.mkdir( target + ".failed" );
450 else
451 dir.cdUp();
452
453 */
454
455 if ( !dir.cd( target + ".diff" ) )
456 dir.mkdir( target + ".diff" );
457 else
458 dir.cdUp();
459
460
461
462 //Perform comparisons between the two directories.
463
464 dir.setFilter(QDir::Files | QDir::Hidden | QDir::NoSymLinks);
465 dir.setNameFilters( QStringList() << "*.png" );
466 dir.cd( target + ".baseline" );
467 QFileInfoList list = dir.entryInfoList();
468
469 for (int i = 0; i < list.size(); ++i)
470 {
471 QFileInfo fileInfo = list.at(i);
472 diff ( basedir, target, fileInfo.fileName() );
473 }
474 }
475
diff(QString basedir,QString dir,QString target)476 bool atWrapper::diff( QString basedir, QString dir, QString target )
477 {
478 //Comparing the two specified files, and then uploading them to
479 //the ftp server if they differ
480
481 basedir += "/" + dir;
482 QString one = basedir + ".baseline/" + target;
483 QString two = basedir + "/" + target;
484
485 QFile file( one );
486
487 file.open( QIODevice::ReadOnly );
488 QByteArray contentsOfOne = file.readAll();
489 file.close();
490
491 file.setFileName( two );
492
493 file.open( QIODevice::ReadOnly );
494 QByteArray contentsOfTwo = file.readAll();
495 file.close();
496
497 if ( contentsOfTwo.size() == 0 )
498 {
499 qDebug() << "No test result found for baseline: " << one;
500 file.setFileName( one );
501 file.open( QIODevice::ReadOnly );
502 file.copy( basedir + ".failed/" + target + "_missing" );
503 uploadFailed( dir + ".failed", target + "_missing", contentsOfTwo );
504 return false;
505 }
506
507
508 if ( ( memcmp( contentsOfOne, contentsOfTwo, contentsOfOne.size() ) ) == 0 )
509 return true;
510 else
511 {
512 qDebug() << "Sorry, the result did not match: " << one;
513 file.setFileName( two );
514 file.open( QIODevice::ReadOnly );
515 file.copy( basedir + ".failed/" + target );
516 file.close();
517 uploadFailed( dir + ".failed", target, contentsOfTwo );
518 uploadDiff( basedir, dir, target );
519 return false;
520 }
521 }
522
uploadDiff(QString basedir,QString dir,QString filename)523 void atWrapper::uploadDiff( QString basedir, QString dir, QString filename )
524 {
525
526 qDebug() << basedir;
527 QImage im1( basedir + ".baseline/" + filename );
528 QImage im2( basedir + "/" + filename );
529
530 QImage im3(im1.size(), QImage::Format_ARGB32);
531
532 im1 = im1.convertToFormat(QImage::Format_ARGB32);
533 im2 = im2.convertToFormat(QImage::Format_ARGB32);
534
535 for ( int y=0; y<im1.height(); ++y )
536 {
537 uint *s = (uint *) im1.scanLine(y);
538 uint *d = (uint *) im2.scanLine(y);
539 uint *w = (uint *) im3.scanLine(y);
540
541 for ( int x=0; x<im1.width(); ++x )
542 {
543 if (*s != *d)
544 *w = 0xff000000;
545 else
546 *w = 0xffffffff;
547 w++;
548 s++;
549 d++;
550 }
551 }
552
553 im3.save( basedir + ".diff/" + filename ,"PNG");
554
555 QFile file( basedir + ".diff/" + filename );
556 file.open( QIODevice::ReadOnly );
557 QByteArray contents = file.readAll();
558 file.close();
559
560 uploadFailed( dir + ".diff", filename, contents );
561
562 }
563
loadConfig(QString path)564 bool atWrapper::loadConfig( QString path )
565 {
566 qDebug() << "Loading config file from ... " << path;
567 configPath = path;
568 //If there is no config file, dont proceed;
569 if ( !QFile::exists( path ) )
570 {
571 return false;
572 }
573
574
575 QSettings settings( path, QSettings::IniFormat, this );
576
577
578 //FIXME: Switch to QStringList or something, hash is not needed!
579 int numEngines = settings.beginReadArray("engines");
580
581 for ( int i = 0; i < numEngines; ++i )
582 {
583 settings.setArrayIndex(i);
584 enginesToTest.insert( settings.value( "engine" ).toString(), "Info here please :p" );
585 }
586
587 settings.endArray();
588
589 framework = QString(ArthurDir) + QDir::separator() + settings.value( "framework" ).toString();
590 suite = settings.value( "suite" ).toString();
591 output = settings.value( "output" ).toString();
592 size = settings.value( "size", "480,360" ).toString();
593 fillColor = settings.value( "fill", "white" ).toString();
594 ftpUser = settings.value( "ftpUser" ).toString();
595 ftpPass = settings.value( "ftpPass" ).toString();
596 ftpHost = settings.value( "ftpHost" ).toString();
597 ftpBaseDir = settings.value( "ftpBaseDir" ).toString();
598
599
600 QDir::current().mkdir( output );
601
602 output += "/" + QLibraryInfo::buildKey();
603
604 QDir::current().mkdir( output );
605
606 output += "/" + QString( qVersion() );
607
608 QDir::current().mkdir( output );
609
610
611 ftpBaseDir += "/" + QHostInfo::localHostName().split( "." ).first();
612
613
614 /*
615 framework = "data/framework.ini";
616 suite = "1.1";
617 output = "testresults";
618 ftpUser = "anonymous";
619 ftpPass = "anonymouspass";
620 ftpHost = "kramer.troll.no";
621 ftpBaseDir = "/arthurtest";
622 */
623 return true;
624 }
625
runAutoTests()626 bool atWrapper::runAutoTests()
627 {
628 //SVG needs this widget...
629 QWidget dummy;
630
631 bool haveBaseline = false;
632
633 if (!initTests(&haveBaseline))
634 return false;
635 executeTests();
636
637 if ( !haveBaseline )
638 {
639 qDebug( " First run! Creating baseline..." );
640 createBaseline();
641 }
642 else
643 {
644 qDebug( " Comparing results..." );
645 compare();
646 }
647 return true;
648 }
649