1 /****************************************************************************************
2  * Copyright (c) 2009 Rick W. Chen <stuffcorpse@archlinux.us>                           *
3  *                                                                                      *
4  * This program is free software; you can redistribute it and/or modify it under        *
5  * the terms of the GNU General Public License as published by the Free Software        *
6  * Foundation; either version 2 of the License, or (at your option) any later           *
7  * version.                                                                             *
8  *                                                                                      *
9  * This program is distributed in the hope that it will be useful, but WITHOUT ANY      *
10  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A      *
11  * PARTICULAR PURPOSE. See the GNU General Public License for more details.             *
12  *                                                                                      *
13  * You should have received a copy of the GNU General Public License along with         *
14  * this program.  If not, see <http://www.gnu.org/licenses/>.                           *
15  ****************************************************************************************/
16 
17 #define DEBUG_PREFIX "CoverFetchUnit"
18 
19 #include "CoverFetchUnit.h"
20 
21 #include "core/support/Amarok.h"
22 #include "core/support/Debug.h"
23 
24 #include <QRegExp>
25 #include <QSet>
26 #include <QUrlQuery>
27 #include <QXmlStreamReader>
28 
29 #include <KLocalizedString>
30 
31 /*
32  * CoverFetchUnit
33  */
34 
CoverFetchUnit(const Meta::AlbumPtr & album,const CoverFetchPayload * payload,CoverFetch::Option opt)35 CoverFetchUnit::CoverFetchUnit( const Meta::AlbumPtr &album,
36                                 const CoverFetchPayload *payload,
37                                 CoverFetch::Option opt )
38     : m_album( album )
39     , m_options( opt )
40     , m_payload( payload )
41 {
42 }
43 
CoverFetchUnit(const CoverFetchPayload * payload,CoverFetch::Option opt)44 CoverFetchUnit::CoverFetchUnit( const CoverFetchPayload *payload, CoverFetch::Option opt )
45     : m_album( payload->album() )
46     , m_options( opt )
47     , m_payload( payload )
48 {
49 }
50 
CoverFetchUnit(const CoverFetchSearchPayload * payload)51 CoverFetchUnit::CoverFetchUnit( const CoverFetchSearchPayload *payload )
52     : m_album( payload->album() )
53     , m_options( CoverFetch::WildInteractive )
54     , m_payload( payload )
55 {
56 }
57 
~CoverFetchUnit()58 CoverFetchUnit::~CoverFetchUnit()
59 {
60     delete m_payload;
61 }
62 
63 Meta::AlbumPtr
album() const64 CoverFetchUnit::album() const
65 {
66     return m_album;
67 }
68 
69 const QStringList &
errors() const70 CoverFetchUnit::errors() const
71 {
72     return m_errors;
73 }
74 
75 CoverFetch::Option
options() const76 CoverFetchUnit::options() const
77 {
78     return m_options;
79 }
80 
81 const CoverFetchPayload *
payload() const82 CoverFetchUnit::payload() const
83 {
84     return m_payload;
85 }
86 
87 bool
isInteractive() const88 CoverFetchUnit::isInteractive() const
89 {
90     bool interactive( false );
91     switch( m_options )
92     {
93     case CoverFetch::Automatic:
94         interactive = false;
95         break;
96     case CoverFetch::Interactive:
97     case CoverFetch::WildInteractive:
98         interactive = true;
99         break;
100     }
101     return interactive;
102 }
103 
104 template< typename T >
105 void
addError(const T & error)106 CoverFetchUnit::addError( const T &error )
107 {
108     m_errors << error;
109 }
110 
operator ==(const CoverFetchUnit & other) const111 bool CoverFetchUnit::operator==( const CoverFetchUnit &other ) const
112 {
113     return (m_album == other.m_album) && (m_options == other.m_options) && (m_payload == other.m_payload);
114 }
115 
operator !=(const CoverFetchUnit & other) const116 bool CoverFetchUnit::operator!=( const CoverFetchUnit &other ) const
117 {
118     return !( *this == other );
119 }
120 
121 
122 /*
123  * CoverFetchPayload
124  */
125 
CoverFetchPayload(const Meta::AlbumPtr & album,CoverFetchPayload::Type type,CoverFetch::Source src)126 CoverFetchPayload::CoverFetchPayload( const Meta::AlbumPtr &album,
127                                       CoverFetchPayload::Type type,
128                                       CoverFetch::Source src )
129     : m_src( src )
130     , m_album( album )
131     , m_method( ( type == Search ) ? QString( "album.search" )
132                                    : album && album->hasAlbumArtist() ? QString( "album.getinfo" )
133                                                                       : QString( "album.search" ) )
134     , m_type( type )
135 {
136 }
137 
~CoverFetchPayload()138 CoverFetchPayload::~CoverFetchPayload()
139 {
140 }
141 
142 Meta::AlbumPtr
album() const143 CoverFetchPayload::album() const
144 {
145     return m_album;
146 }
147 
148 QString
sanitizeQuery(const QString & query)149 CoverFetchPayload::sanitizeQuery( const QString &query )
150 {
151     QString cooked( query );
152     cooked.remove( QChar('?') );
153     return cooked;
154 }
155 
156 CoverFetch::Source
source() const157 CoverFetchPayload::source() const
158 {
159     return m_src;
160 }
161 
162 CoverFetchPayload::Type
type() const163 CoverFetchPayload::type() const
164 {
165     return m_type;
166 }
167 
168 const CoverFetch::Urls &
urls() const169 CoverFetchPayload::urls() const
170 {
171     return m_urls;
172 }
173 
174 const QString
sourceString() const175 CoverFetchPayload::sourceString() const
176 {
177     QString source;
178     switch( m_src )
179     {
180     case CoverFetch::LastFm:
181         source = "Last.fm";
182         break;
183     case CoverFetch::Google:
184         source = "Google";
185         break;
186     case CoverFetch::Discogs:
187         source = "Discogs";
188         break;
189     default:
190         source = "Unknown";
191     }
192     return source;
193 }
194 
195 bool
isPrepared() const196 CoverFetchPayload::isPrepared() const
197 {
198     return !m_urls.isEmpty();
199 }
200 
201 /*
202  * CoverFetchInfoPayload
203  */
204 
CoverFetchInfoPayload(const Meta::AlbumPtr & album,const CoverFetch::Source src)205 CoverFetchInfoPayload::CoverFetchInfoPayload( const Meta::AlbumPtr &album, const CoverFetch::Source src )
206     : CoverFetchPayload( album, CoverFetchPayload::Info, src )
207 {
208     prepareUrls();
209 }
210 
CoverFetchInfoPayload(const CoverFetch::Source src,const QByteArray & data)211 CoverFetchInfoPayload::CoverFetchInfoPayload( const CoverFetch::Source src, const QByteArray &data )
212     : CoverFetchPayload( Meta::AlbumPtr( 0 ), CoverFetchPayload::Info, src )
213 {
214     switch( src )
215     {
216     default:
217         prepareUrls();
218         break;
219     case CoverFetch::Discogs:
220         prepareDiscogsUrls( data );
221         break;
222     }
223 }
224 
~CoverFetchInfoPayload()225 CoverFetchInfoPayload::~CoverFetchInfoPayload()
226 {
227 }
228 
229 void
prepareUrls()230 CoverFetchInfoPayload::prepareUrls()
231 {
232     QUrl url;
233     CoverFetch::Metadata metadata;
234 
235     switch( m_src )
236     {
237     default:
238     case CoverFetch::LastFm:
239         url.setScheme( "http" );
240         url.setHost( "ws.audioscrobbler.com" );
241         url.setPath( "/2.0/" );
242         QUrlQuery query;
243         query.addQueryItem( "api_key", Amarok::lastfmApiKey() );
244         query.addQueryItem( "album", sanitizeQuery( album()->name() ) );
245 
246         if( album()->hasAlbumArtist() )
247         {
248             query.addQueryItem( "artist", sanitizeQuery( album()->albumArtist()->name() ) );
249         }
250         query.addQueryItem( "method", method() );
251         url.setQuery( query );
252 
253         metadata[ "source" ] = "Last.fm";
254         metadata[ "method" ] = method();
255         break;
256     }
257 
258     if( url.isValid() )
259         m_urls.insert( url, metadata );
260 }
261 
262 void
prepareDiscogsUrls(const QByteArray & data)263 CoverFetchInfoPayload::prepareDiscogsUrls( const QByteArray &data )
264 {
265     QXmlStreamReader xml( QString::fromUtf8(data) );
266     while( !xml.atEnd() && !xml.hasError() )
267     {
268         xml.readNext();
269         if( xml.isStartElement() && xml.name() == "searchresults" )
270         {
271             while( !xml.atEnd() && !xml.hasError() )
272             {
273                 xml.readNext();
274                 const QStringRef &n = xml.name();
275                 if( xml.isEndElement() && n == "searchresults" )
276                     break;
277                 if( !xml.isStartElement() )
278                     continue;
279                 if( n == "result" )
280                 {
281                     while( !xml.atEnd() && !xml.hasError() )
282                     {
283                         xml.readNext();
284                         if( xml.isEndElement() && n == "result" )
285                             break;
286                         if( !xml.isStartElement() )
287                             continue;
288                         if( xml.name() == "uri" )
289                         {
290                             QUrl releaseUrl( xml.readElementText() );
291                             QString releaseStr = releaseUrl.adjusted(QUrl::StripTrailingSlash).toString();
292                             QString releaseId = releaseStr.split( QLatin1Char('/') ).last();
293 
294                             QUrl url;
295                             url.setScheme( "http" );
296                             url.setHost( "www.discogs.com" );
297                             url.setPath( "/release/" + releaseId );
298                             QUrlQuery query;
299                             query.addQueryItem( "api_key", Amarok::discogsApiKey() );
300                             query.addQueryItem( "f", "xml" );
301                             url.setQuery( query );
302 
303                             CoverFetch::Metadata metadata;
304                             metadata[ "source" ] = "Discogs";
305 
306                             if( url.isValid() )
307                                 m_urls.insert( url, metadata );
308                         }
309                         else
310                             xml.skipCurrentElement();
311                     }
312                 }
313                 else
314                     xml.skipCurrentElement();
315             }
316         }
317     }
318 }
319 
320 /*
321  * CoverFetchSearchPayload
322  */
323 
CoverFetchSearchPayload(const QString & query,const CoverFetch::Source src,unsigned int page,const Meta::AlbumPtr & album)324 CoverFetchSearchPayload::CoverFetchSearchPayload( const QString &query,
325                                                   const CoverFetch::Source src,
326                                                   unsigned int page,
327                                                   const Meta::AlbumPtr &album )
328     : CoverFetchPayload( album, CoverFetchPayload::Search, src )
329     , m_page( page )
330     , m_query( query )
331 {
332     prepareUrls();
333 }
334 
~CoverFetchSearchPayload()335 CoverFetchSearchPayload::~CoverFetchSearchPayload()
336 {
337 }
338 
339 QString
query() const340 CoverFetchSearchPayload::query() const
341 {
342     return m_query;
343 }
344 
345 void
prepareUrls()346 CoverFetchSearchPayload::prepareUrls()
347 {
348     QUrl url;
349     QUrlQuery query;
350     url.setScheme( "http" );
351     CoverFetch::Metadata metadata;
352 
353     switch( m_src )
354     {
355     default:
356     case CoverFetch::LastFm:
357         url.setHost( "ws.audioscrobbler.com" );
358         url.setPath( "/2.0/" );
359         query.addQueryItem( "api_key", Amarok::lastfmApiKey() );
360         query.addQueryItem( "limit", QString::number( 20 ) );
361         query.addQueryItem( "page", QString::number( m_page ) );
362         query.addQueryItem( "album", sanitizeQuery( m_query ) );
363         query.addQueryItem( "method", method() );
364         metadata[ "source" ] = "Last.fm";
365         metadata[ "method" ] = method();
366         break;
367 
368     case CoverFetch::Discogs:
369         debug() << "Setting up a Discogs fetch";
370         url.setHost( "www.discogs.com" );
371         url.setPath( "/search" );
372         query.addQueryItem( "api_key", Amarok::discogsApiKey() );
373         query.addQueryItem( "page", QString::number( m_page + 1 ) );
374         query.addQueryItem( "type", "all" );
375         query.addQueryItem( "q", sanitizeQuery( m_query ) );
376         query.addQueryItem( "f", "xml" );
377         debug() << "Discogs Url: " << url;
378         metadata[ "source" ] = "Discogs";
379         break;
380 
381     case CoverFetch::Google:
382         url.setHost( "images.google.com" );
383         url.setPath( "/images" );
384         query.addQueryItem( "q", sanitizeQuery( m_query ) );
385         query.addQueryItem( "gbv", QChar( '1' ) );
386         query.addQueryItem( "filter", QChar( '1' ) );
387         query.addQueryItem( "start", QString::number( 20 * m_page ) );
388         metadata[ "source" ] = "Google";
389         break;
390     }
391     url.setQuery( query );
392     debug() << "Fetching From URL: " << url;
393     if( url.isValid() )
394         m_urls.insert( url, metadata );
395 }
396 
397 /*
398  * CoverFetchArtPayload
399  */
400 
CoverFetchArtPayload(const Meta::AlbumPtr & album,const CoverFetch::ImageSize size,const CoverFetch::Source src,bool wild)401 CoverFetchArtPayload::CoverFetchArtPayload( const Meta::AlbumPtr &album,
402                                             const CoverFetch::ImageSize size,
403                                             const CoverFetch::Source src,
404                                             bool wild )
405     : CoverFetchPayload( album, CoverFetchPayload::Art, src )
406     , m_size( size )
407     , m_wild( wild )
408 {
409 }
410 
CoverFetchArtPayload(const CoverFetch::ImageSize size,const CoverFetch::Source src,bool wild)411 CoverFetchArtPayload::CoverFetchArtPayload( const CoverFetch::ImageSize size,
412                                             const CoverFetch::Source src,
413                                             bool wild )
414     : CoverFetchPayload( Meta::AlbumPtr( 0 ), CoverFetchPayload::Art, src )
415     , m_size( size )
416     , m_wild( wild )
417 {
418 }
419 
~CoverFetchArtPayload()420 CoverFetchArtPayload::~CoverFetchArtPayload()
421 {
422 }
423 
424 bool
isWild() const425 CoverFetchArtPayload::isWild() const
426 {
427     return m_wild;
428 }
429 
430 CoverFetch::ImageSize
imageSize() const431 CoverFetchArtPayload::imageSize() const
432 {
433     return m_size;
434 }
435 
436 void
setXml(const QByteArray & xml)437 CoverFetchArtPayload::setXml( const QByteArray &xml )
438 {
439     m_xml = QString::fromUtf8( xml );
440     prepareUrls();
441 }
442 
443 void
prepareUrls()444 CoverFetchArtPayload::prepareUrls()
445 {
446     if( m_src == CoverFetch::Google )
447     {
448         // google is special
449         prepareGoogleUrls();
450         return;
451     }
452 
453     QXmlStreamReader xml( m_xml );
454     xml.setNamespaceProcessing( false );
455     switch( m_src )
456     {
457     default:
458     case CoverFetch::LastFm:
459         prepareLastFmUrls( xml );
460         break;
461     case CoverFetch::Discogs:
462         prepareDiscogsUrls( xml );
463         break;
464     }
465 
466     if( xml.hasError() )
467     {
468         warning() << QString( "Error occurred when preparing %1 urls for %2: %3" )
469             .arg( sourceString(), (album() ? album()->name() : "'unknown'"), xml.errorString() );
470         debug() << urls();
471     }
472 }
473 
474 void
prepareDiscogsUrls(QXmlStreamReader & xml)475 CoverFetchArtPayload::prepareDiscogsUrls( QXmlStreamReader &xml )
476 {
477     while( !xml.atEnd() && !xml.hasError() )
478     {
479         xml.readNext();
480         if( !xml.isStartElement() || xml.name() != "release" )
481             continue;
482 
483         const QString releaseId = xml.attributes().value( "id" ).toString();
484         while( !xml.atEnd() && !xml.hasError() )
485         {
486             xml.readNext();
487             const QStringRef &n = xml.name();
488             if( xml.isEndElement() && n == "release" )
489                 break;
490             if( !xml.isStartElement() )
491                 continue;
492 
493             CoverFetch::Metadata metadata;
494             metadata[ "source" ] = "Discogs";
495             if( n == "title" )
496                 metadata[ "title" ] = xml.readElementText();
497             else if( n == "country" )
498                 metadata[ "country" ] = xml.readElementText();
499             else if( n == "released" )
500                 metadata[ "released" ] = xml.readElementText();
501             else if( n == "notes" )
502                 metadata[ "notes" ] = xml.readElementText();
503             else if( n == "images" )
504             {
505                 while( !xml.atEnd() && !xml.hasError() )
506                 {
507                     xml.readNext();
508                     if( xml.isEndElement() && xml.name() == "images" )
509                         break;
510                     if( !xml.isStartElement() )
511                         continue;
512                     if( xml.name() == "image" )
513                     {
514                         const QXmlStreamAttributes &attr = xml.attributes();
515                         const QUrl thburl( attr.value( "uri150" ).toString() );
516                         const QUrl uri( attr.value( "uri" ).toString() );
517                         const QUrl url = (m_size == CoverFetch::ThumbSize) ? thburl : uri;
518                         if( !url.isValid() )
519                             continue;
520 
521                         metadata[ "releaseid"    ] = releaseId;
522                         metadata[ "releaseurl"   ] = "http://discogs.com/release/" + releaseId;
523                         metadata[ "normalarturl" ] = uri.url();
524                         metadata[ "thumbarturl"  ] = thburl.url();
525                         metadata[ "width"        ] = attr.value( "width"  ).toString();
526                         metadata[ "height"       ] = attr.value( "height" ).toString();
527                         metadata[ "type"         ] = attr.value( "type"   ).toString();
528                         m_urls.insert( url, metadata );
529                     }
530                     else
531                         xml.skipCurrentElement();
532                 }
533             }
534             else
535                 xml.skipCurrentElement();
536         }
537     }
538 }
539 
540 void
prepareGoogleUrls()541 CoverFetchArtPayload::prepareGoogleUrls()
542 {
543     // code based on Audex CDDA Extractor
544     QRegExp rx( "<a\\shref=\"(\\/imgres\\?imgurl=[^\"]+)\">[\\s\\n]*<img[^>]+src=\"([^\"]+)\"" );
545     rx.setCaseSensitivity( Qt::CaseInsensitive );
546     rx.setMinimal( true );
547 
548     int pos = 0;
549     QString html = m_xml.replace( QLatin1String("&amp;"), QLatin1String("&") );
550 
551     while( ( (pos = rx.indexIn( html, pos ) ) != -1 ) )
552     {
553         QUrl url( "http://www.google.com" + rx.cap( 1 ) );
554         QUrlQuery query( url.query() );
555 
556         CoverFetch::Metadata metadata;
557         metadata[ "width" ] = query.queryItemValue( "w" );
558         metadata[ "height" ] = query.queryItemValue( "h" );
559         metadata[ "size" ] = query.queryItemValue( "sz" );
560         metadata[ "imgrefurl" ] = query.queryItemValue( "imgrefurl" );
561         metadata[ "normalarturl" ] = query.queryItemValue("imgurl");
562         metadata[ "source" ] = "Google";
563 
564         if( !rx.cap( 2 ).isEmpty() )
565             metadata[ "thumbarturl" ] = rx.cap( 2 );
566 
567         url.clear();
568         switch( m_size )
569         {
570         default:
571         case CoverFetch::ThumbSize:
572             url = QUrl( metadata.value( "thumbarturl" ) );
573             break;
574         case CoverFetch::NormalSize:
575             url = QUrl( metadata.value( "normalarturl" ) );
576             break;
577         }
578 
579         if( url.isValid() )
580             m_urls.insert( url, metadata );
581 
582         pos += rx.matchedLength();
583     }
584 }
585 
586 void
prepareLastFmUrls(QXmlStreamReader & xml)587 CoverFetchArtPayload::prepareLastFmUrls( QXmlStreamReader &xml )
588 {
589     QSet<QString> artistSet;
590     if( method() == "album.getinfo" )
591     {
592         artistSet << normalize( ( album() && album()->albumArtist() )
593                                 ? album()->albumArtist()->name()
594                                 : i18n( "Unknown Artist" ) );
595     }
596     else if( method() == "album.search" )
597     {
598         if( !m_wild && album() )
599         {
600             const Meta::TrackList tracks = album()->tracks();
601             QStringList artistNames( "Various Artists" );
602             foreach( const Meta::TrackPtr &track, tracks )
603                 artistNames << ( track->artist() ? track->artist()->name()
604                                                  : i18n( "Unknown Artist" ) );
605             artistSet = normalize( artistNames ).toSet();
606         }
607     }
608     else return;
609 
610     while( !xml.atEnd() && !xml.hasError() )
611     {
612         xml.readNext();
613         if( !xml.isStartElement() || xml.name() != "album" )
614             continue;
615 
616         QHash<QString, QString> coverUrlHash;
617         CoverFetch::Metadata metadata;
618         metadata[ "source" ] = "Last.fm";
619         while( !xml.atEnd() && !xml.hasError() )
620         {
621             xml.readNext();
622             const QStringRef &n = xml.name();
623             if( xml.isEndElement() && n == "album" )
624                 break;
625             if( !xml.isStartElement() )
626                 continue;
627 
628             if( n == "name" )
629             {
630                 metadata[ "name" ] = xml.readElementText();
631             }
632             else if( n == "artist" )
633             {
634                 const QString &artist = xml.readElementText();
635                 if( !artistSet.contains( artist ) )
636                     continue;
637                 metadata[ "artist" ] = artist;
638             }
639             else if( n == "url" )
640             {
641                 metadata[ "releaseurl" ] = xml.readElementText();
642             }
643             else if( n == "image" )
644             {
645                 QString sizeStr = xml.attributes().value("size").toString();
646                 coverUrlHash[ sizeStr ] = xml.readElementText();
647             }
648         }
649 
650         QStringList acceptableSizes;
651         acceptableSizes << "large" << "medium" << "small";
652         metadata[ "thumbarturl" ] = firstAvailableValue( acceptableSizes, coverUrlHash );
653 
654         acceptableSizes.clear();
655         acceptableSizes << "extralarge" << "large";
656         metadata[ "normalarturl" ] = firstAvailableValue( acceptableSizes, coverUrlHash );
657 
658         QUrl url( m_size == CoverFetch::ThumbSize ? metadata["thumbarturl"] : metadata["normalarturl"] );
659         if( url.isValid() )
660             m_urls.insert( url , metadata );
661     }
662 }
663 
664 QString
firstAvailableValue(const QStringList & keys,const QHash<QString,QString> & hash)665 CoverFetchArtPayload::firstAvailableValue( const QStringList &keys, const QHash<QString, QString> &hash )
666 {
667     for( int i = 0, size = keys.size(); i < size; ++i )
668     {
669         QString value( hash.value( keys.at(i) ) );
670         if( !value.isEmpty() )
671             return value;
672     }
673     return QString();
674 }
675 
676 QString
normalize(const QString & raw)677 CoverFetchArtPayload::normalize( const QString &raw )
678 {
679     const QRegExp spaceRegExp  = QRegExp( "\\s" );
680     return raw.toLower().remove( spaceRegExp ).normalized( QString::NormalizationForm_KC );
681 }
682 
683 QStringList
normalize(const QStringList & rawList)684 CoverFetchArtPayload::normalize( const QStringList &rawList )
685 {
686     QStringList cooked;
687     foreach( const QString &raw, rawList )
688     {
689         cooked << normalize( raw );
690     }
691     return cooked;
692 }
693 
694