1 /***************************************************************************
2 * Copyright (C) 2003-2005 Max Howell <max.howell@methylblue.com> *
3 * (C) 2003-2010 Mark Kretschmann <kretschmann@kde.org> *
4 * (C) 2005-2007 Alexandre Oliveira <aleprj@gmail.com> *
5 * (C) 2008 Dan Meltzer <parallelgrapefruit@gmail.com> *
6 * (C) 2008-2009 Jeff Mitchell <mitchell@kde.org> *
7 * *
8 * This program is free software; you can redistribute it and/or modify *
9 * it under the terms of the GNU General Public License as published by *
10 * the Free Software Foundation; either version 2 of the License, or *
11 * (at your option) any later version. *
12 * *
13 * This program is distributed in the hope that it will be useful, *
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
16 * GNU General Public License for more details. *
17 * *
18 * You should have received a copy of the GNU General Public License *
19 * along with this program; if not, write to the *
20 * Free Software Foundation, Inc., *
21 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. *
22 ***************************************************************************/
23
24 #include "CollectionScanner.h"
25
26 #include "Version.h" // for AMAROK_VERSION
27 #include "collectionscanner/BatchFile.h"
28 #include "collectionscanner/Directory.h"
29 #include "collectionscanner/Track.h"
30
31 #include <QTimer>
32 #include <QThread>
33
34 #include <QString>
35 #include <QStringList>
36 #include <QDir>
37 #include <QFile>
38 #include <QDateTime>
39 #include <QXmlStreamReader>
40 #include <QXmlStreamWriter>
41 #include <QSharedMemory>
42 #include <QByteArray>
43 #include <QTextStream>
44 #include <QDataStream>
45 #include <QBuffer>
46 #include <QDebug>
47
48 #ifdef Q_OS_LINUX
49 // for ioprio
50 #include <unistd.h>
51 #include <sys/syscall.h>
52 enum {
53 IOPRIO_CLASS_NONE,
54 IOPRIO_CLASS_RT,
55 IOPRIO_CLASS_BE,
56 IOPRIO_CLASS_IDLE
57 };
58
59 enum {
60 IOPRIO_WHO_PROCESS = 1,
61 IOPRIO_WHO_PGRP,
62 IOPRIO_WHO_USER
63 };
64 #define IOPRIO_CLASS_SHIFT 13
65 #endif
66
67
68 int
main(int argc,char * argv[])69 main( int argc, char *argv[] )
70 {
71 CollectionScanner::Scanner scanner( argc, argv );
72 return scanner.exec();
73 }
74
Scanner(int & argc,char ** argv)75 CollectionScanner::Scanner::Scanner( int &argc, char **argv )
76 : QCoreApplication( argc, argv )
77 , m_charset( false )
78 , m_newerTime(0)
79 , m_incremental( false )
80 , m_recursively( false )
81 , m_restart( false )
82 , m_idlePriority( false )
83 {
84 setObjectName( QStringLiteral("amarokcollectionscanner") );
85
86 readArgs();
87
88 if( m_idlePriority )
89 {
90 bool ioPriorityWorked = false;
91 #if defined(Q_OS_LINUX) && defined(SYS_ioprio_set)
92 // try setting the idle priority class
93 ioPriorityWorked = ( syscall( SYS_ioprio_set, IOPRIO_WHO_PROCESS, 0,
94 IOPRIO_CLASS_IDLE << IOPRIO_CLASS_SHIFT ) >= 0 );
95 // try setting the lowest priority in the best-effort priority class (the default class)
96 if( !ioPriorityWorked )
97 ioPriorityWorked = ( syscall( SYS_ioprio_set, IOPRIO_WHO_PROCESS, 0,
98 7 | ( IOPRIO_CLASS_BE << IOPRIO_CLASS_SHIFT ) ) >= 0 );
99 #endif
100 if( !ioPriorityWorked && QThread::currentThread() )
101 QThread::currentThread()->setPriority( QThread::IdlePriority );
102 }
103 }
104
105
~Scanner()106 CollectionScanner::Scanner::~Scanner()
107 {
108 }
109
110 void
readBatchFile(const QString & path)111 CollectionScanner::Scanner::readBatchFile( const QString &path )
112 {
113 QFile batchFile( path );
114
115 if( !batchFile.exists() )
116 error( tr( "File \"%1\" not found." ).arg( path ) );
117
118 if( !batchFile.open( QIODevice::ReadOnly ) )
119 error( tr( "Could not open file \"%1\"." ).arg( path ) );
120
121 BatchFile batch( path );
122 foreach( const QString &str, batch.directories() )
123 {
124 m_folders.append( str );
125 }
126
127 foreach( const CollectionScanner::BatchFile::TimeDefinition &def, batch.timeDefinitions() )
128 {
129 m_mTimes.insert( def.first, def.second );
130 }
131 }
132
133 void
readNewerTime(const QString & path)134 CollectionScanner::Scanner::readNewerTime( const QString &path )
135 {
136 QFileInfo file( path );
137
138 if( !file.exists() )
139 error( tr( "File \"%1\" not found." ).arg( path ) );
140
141 m_newerTime = qMax<qint64>( m_newerTime, file.lastModified().toSecsSinceEpoch() );
142 }
143
144
145 void
doJob()146 CollectionScanner::Scanner::doJob() //SLOT
147 {
148 QFile xmlFile;
149 xmlFile.open( stdout, QIODevice::WriteOnly );
150 QXmlStreamWriter xmlWriter( &xmlFile );
151 xmlWriter.setAutoFormatting( true );
152
153 // get a list of folders to scan. We do it even if resuming because we don't want
154 // to save the (perhaps very big) list of directories into shared memory, bug 327812
155 QStringList entries;
156 {
157 QSet<QString> entriesSet;
158
159 foreach( QString dir, m_folders ) // krazy:exclude=foreach
160 {
161 if( dir.isEmpty() )
162 //apparently somewhere empty strings get into the mix
163 //which results in a full-system scan! Which we can't allow
164 continue;
165
166 // Make sure that all paths are absolute, not relative
167 if( QDir::isRelativePath( dir ) )
168 dir = QDir::cleanPath( QDir::currentPath() + QLatin1Char('/') + dir );
169
170 if( !dir.endsWith( QLatin1Char('/') ) )
171 dir += '/';
172
173 addDir( dir, &entriesSet ); // checks m_recursively
174 }
175
176 entries = entriesSet.toList();
177 qSort( entries ); // the sort is crucial because of restarts and lastDirectory handling
178 }
179
180 if( m_restart )
181 {
182 m_scanningState.readFull();
183 QString lastEntry = m_scanningState.lastDirectory();
184
185 int index = entries.indexOf( lastEntry );
186 if( index >= 0 )
187 // strip already processed entries, but *keep* the lastEntry
188 entries = entries.mid( index );
189 else
190 qWarning() << Q_FUNC_INFO << "restarting scan after a crash, but lastDirectory"
191 << lastEntry << "not found in folders to scan (size" << entries.size()
192 << "). Starting scanning from the beginning.";
193 }
194 else // first attempt
195 {
196 m_scanningState.writeFull(); // just trigger write to initialise memory
197
198 xmlWriter.writeStartDocument();
199 xmlWriter.writeStartElement(QStringLiteral("scanner"));
200 xmlWriter.writeAttribute(QStringLiteral("count"), QString::number( entries.count() ) );
201 if( m_incremental )
202 xmlWriter.writeAttribute(QStringLiteral("incremental"), QString());
203 // write some information into the file and close previous tag
204 xmlWriter.writeComment("Created by amarokcollectionscanner " AMAROK_VERSION " on "+QDateTime::currentDateTime().toString());
205 xmlFile.flush();
206 }
207
208 // --- now do the scanning
209 foreach( const QString &path, entries )
210 {
211 CollectionScanner::Directory dir( path, &m_scanningState,
212 m_incremental && !isModified( path ) );
213
214 xmlWriter.writeStartElement( QStringLiteral("directory") );
215 dir.toXml( &xmlWriter );
216 xmlWriter.writeEndElement();
217 xmlFile.flush();
218 }
219
220 // --- write the end element (must be done by hand as we might not have written the start element when restarting)
221 xmlFile.write("\n</scanner>\n");
222
223 quit();
224 }
225
226 void
addDir(const QString & dir,QSet<QString> * entries)227 CollectionScanner::Scanner::addDir( const QString& dir, QSet<QString>* entries )
228 {
229 // Linux specific, but this fits the 90% rule
230 if( dir.startsWith( QLatin1String("/dev") ) || dir.startsWith( QLatin1String("/sys") ) || dir.startsWith( QLatin1String("/proc") ) )
231 return;
232
233 if( entries->contains( dir ) )
234 return;
235
236 QDir d( dir );
237 if( !d.exists() )
238 {
239 QTextStream stream( stderr );
240 stream << "Directory \""<<dir<<"\" does not exist." << endl;
241 return;
242 }
243
244 entries->insert( dir );
245
246 if( !m_recursively )
247 return; // finished
248
249 d.setFilter( QDir::NoDotAndDotDot | QDir::Dirs );
250 const QFileInfoList fileInfos = d.entryInfoList();
251
252 for ( const QFileInfo &fi : fileInfos )
253 {
254 if( !fi.exists() )
255 continue;
256
257 const QFileInfo &f = fi.isSymLink() ? QFileInfo( fi.symLinkTarget() ) : fi;
258
259 if( !f.exists() )
260 continue;
261
262 if( f.isDir() )
263 {
264 addDir( QString( f.absoluteFilePath() + '/' ), entries );
265 }
266 }
267 }
268
269 bool
isModified(const QString & dir)270 CollectionScanner::Scanner::isModified( const QString& dir )
271 {
272 QFileInfo info( dir );
273 if( !info.exists() )
274 return false;
275
276 uint lastModified = info.lastModified().toSecsSinceEpoch();
277
278 if( m_mTimes.contains( dir ) )
279 return m_mTimes.value( dir ) != lastModified;
280 else
281 return m_newerTime < lastModified;
282 }
283
284 void
readArgs()285 CollectionScanner::Scanner::readArgs()
286 {
287 QStringList argslist = arguments();
288 if( argslist.size() < 2 )
289 displayHelp();
290
291 bool missingArg = false;
292
293 for( int argnum = 1; argnum < argslist.count(); argnum++ )
294 {
295 QString arg = argslist.at( argnum );
296
297 if( arg.startsWith( QLatin1String("--") ) )
298 {
299 QString myarg = QString( arg ).remove( 0, 2 );
300 if( myarg == QLatin1String("newer") )
301 {
302 if( argslist.count() > argnum + 1 )
303 readNewerTime( argslist.at( argnum + 1 ) );
304 else
305 missingArg = true;
306 argnum++;
307 }
308 else if( myarg == QLatin1String("batch") )
309 {
310 if( argslist.count() > argnum + 1 )
311 readBatchFile( argslist.at( argnum + 1 ) );
312 else
313 missingArg = true;
314 argnum++;
315 }
316 else if( myarg == QLatin1String("sharedmemory") )
317 {
318 if( argslist.count() > argnum + 1 )
319 m_scanningState.setKey( argslist.at( argnum + 1 ) );
320 else
321 missingArg = true;
322 argnum++;
323 }
324 else if( myarg == QLatin1String("version") )
325 displayVersion();
326 else if( myarg == QLatin1String("incremental") )
327 m_incremental = true;
328 else if( myarg == QLatin1String("recursive") )
329 m_recursively = true;
330 else if( myarg == QLatin1String("restart") )
331 m_restart = true;
332 else if( myarg == QLatin1String("idlepriority") )
333 m_idlePriority = true;
334 else if( myarg == QLatin1String("charset") )
335 m_charset = true;
336 else
337 displayHelp();
338
339 }
340 else if( arg.startsWith( '-' ) )
341 {
342 QString myarg = QString( arg ).remove( 0, 1 );
343 int pos = 0;
344 while( pos < myarg.length() )
345 {
346 if( myarg[pos] == 'r' )
347 m_recursively = true;
348 else if( myarg[pos] == 'v' )
349 displayVersion();
350 else if( myarg[pos] == 's' )
351 m_restart = true;
352 else if( myarg[pos] == 'c' )
353 m_charset = true;
354 else if( myarg[pos] == 'i' )
355 m_incremental = true;
356 else
357 displayHelp();
358
359 ++pos;
360 }
361 }
362 else
363 {
364 if( !arg.isEmpty() )
365 m_folders.append( arg );
366 }
367 }
368
369 if( missingArg )
370 displayHelp( tr( "Missing argument for option %1" ).arg( argslist.last() ) );
371
372
373 CollectionScanner::Track::setUseCharsetDetector( m_charset );
374
375 // Start the actual scanning job
376 QTimer::singleShot( 0, this, &Scanner::doJob );
377 }
378
379 void
error(const QString & str)380 CollectionScanner::Scanner::error( const QString &str )
381 {
382 QTextStream stream( stderr );
383 stream << str << endl;
384 stream.flush();
385
386 // Nothing else to do, so we exit directly
387 ::exit( 0 );
388 }
389
390 /** This function is called by Amarok to verify that Amarok an Scanner versions match */
391 void
displayVersion()392 CollectionScanner::Scanner::displayVersion()
393 {
394 QTextStream stream( stdout );
395 stream << AMAROK_VERSION << endl;
396 stream.flush();
397
398 // Nothing else to do, so we exit directly
399 ::exit( 0 );
400 }
401
402 void
displayHelp(const QString & error)403 CollectionScanner::Scanner::displayHelp( const QString &error )
404 {
405 QTextStream stream( error.isEmpty() ? stdout : stderr );
406 stream << error
407 << tr( "Amarok Collection Scanner\n"
408 "Scans directories and outputs a xml file with the results.\n"
409 "For more information see http://community.kde.org/Amarok/Development/BatchMode\n\n"
410 "Usage: amarokcollectionscanner [options] <Folder(s)>\n"
411 "User-modifiable Options:\n"
412 "<Folder(s)> : list of folders to scan\n"
413 "-h, --help : This help text\n"
414 "-v, --version : Print the version of this tool\n"
415 "-r, --recursive : Scan folders recursively\n"
416 "-i, --incremental : Incremental scan (modified folders only)\n"
417 "-s, --restart : After a crash, restart the scanner in its last position\n"
418 " --idlepriority : Run at idle priority\n"
419 " --sharedmemory <key> : A shared memory segment to be used for restarting a scan\n"
420 " --newer <path> : Only scan directories if modification time is newer than <path>\n"
421 " Only useful in incremental scan mode\n"
422 " --batch <path> : Add the directories from the batch xml file\n"
423 " batch file format should look like this:\n"
424 " <scanner>\n"
425 " <directory>\n"
426 " <path>/absolute/path/of/directory</path>\n"
427 " <mtime>1234</mtime> (this is optional)\n"
428 " </directory>\n"
429 " </scanner>\n"
430 " You can also use a previous scan result for that.\n"
431 )
432 << endl;
433 stream.flush();
434
435 ::exit(0);
436 }
437
438
439