1 /* This file is part of Clementine.
2 Copyright 2010-2013, David Sansome <me@davidsansome.com>
3 Copyright 2010, 2014, John Maguire <john.maguire@gmail.com>
4 Copyright 2011, Tyler Rhodes <tyler.s.rhodes@gmail.com>
5 Copyright 2011, Paweł Bara <keirangtp@gmail.com>
6 Copyright 2011, Andrea Decorte <adecorte@gmail.com>
7 Copyright 2014, Chocobozzz <florian.bigard@gmail.com>
8 Copyright 2014, Krzysztof Sobiecki <sobkas@gmail.com>
9
10 Clementine is free software: you can redistribute it and/or modify
11 it under the terms of the GNU General Public License as published by
12 the Free Software Foundation, either version 3 of the License, or
13 (at your option) any later version.
14
15 Clementine is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with Clementine. If not, see <http://www.gnu.org/licenses/>.
22 */
23
24 #include "jamendoservice.h"
25
26 #include <QDesktopServices>
27 #include <QMenu>
28 #include <QMessageBox>
29 #include <QNetworkReply>
30 #include <QSortFilterProxyModel>
31 #include <QtConcurrentRun>
32 #include <QXmlStreamReader>
33 #include "qtiocompressor.h"
34
35 #include "jamendodynamicplaylist.h"
36 #include "jamendoplaylistitem.h"
37 #include "internet/core/internetmodel.h"
38 #include "core/application.h"
39 #include "core/database.h"
40 #include "core/logging.h"
41 #include "core/mergedproxymodel.h"
42 #include "core/network.h"
43 #include "core/scopedtransaction.h"
44 #include "core/taskmanager.h"
45 #include "core/timeconstants.h"
46 #include "globalsearch/globalsearch.h"
47 #include "globalsearch/librarysearchprovider.h"
48 #include "library/librarybackend.h"
49 #include "library/libraryfilterwidget.h"
50 #include "library/librarymodel.h"
51 #include "smartplaylists/generator.h"
52 #include "smartplaylists/querygenerator.h"
53 #include "ui/iconloader.h"
54
55 const char* JamendoService::kServiceName = "Jamendo";
56 const char* JamendoService::kDirectoryUrl =
57 "https://imgjam.com/data/dbdump_artistalbumtrack.xml.gz";
58 const char* JamendoService::kMp3StreamUrl =
59 "http://api.jamendo.com/get2/stream/track/redirect/"
60 "?id=%1&streamencoding=mp31";
61 const char* JamendoService::kOggStreamUrl =
62 "http://api.jamendo.com/get2/stream/track/redirect/"
63 "?id=%1&streamencoding=ogg2";
64 const char* JamendoService::kAlbumCoverUrl =
65 "http://api.jamendo.com/get2/image/album/redirect/?id=%1&imagesize=300";
66 const char* JamendoService::kHomepage = "http://www.jamendo.com/";
67 const char* JamendoService::kAlbumInfoUrl = "http://www.jamendo.com/album/%1";
68 const char* JamendoService::kDownloadAlbumUrl =
69 "http://www.jamendo.com/download/album/%1";
70
71 const char* JamendoService::kSongsTable = "jamendo.songs";
72 const char* JamendoService::kFtsTable = "jamendo.songs_fts";
73 const char* JamendoService::kTrackIdsTable = "jamendo.track_ids";
74 const char* JamendoService::kTrackIdsColumn = "track_id";
75
76 const char* JamendoService::kSettingsGroup = "Jamendo";
77
78 const int JamendoService::kBatchSize = 10000;
79 const int JamendoService::kApproxDatabaseSize = 450000;
80
JamendoService(Application * app,InternetModel * parent)81 JamendoService::JamendoService(Application* app, InternetModel* parent)
82 : InternetService(kServiceName, app, parent, parent),
83 network_(new NetworkAccessManager(this)),
84 context_menu_(nullptr),
85 library_backend_(nullptr),
86 library_filter_(nullptr),
87 library_model_(nullptr),
88 library_sort_model_(new QSortFilterProxyModel(this)),
89 search_provider_(nullptr),
90 load_database_task_id_(0),
91 total_song_count_(0),
92 accepted_download_(false) {
93 library_backend_ = new LibraryBackend;
94 library_backend_->moveToThread(app_->database()->thread());
95 library_backend_->Init(app_->database(), kSongsTable, QString(), QString(),
96 kFtsTable);
97 connect(library_backend_, SIGNAL(TotalSongCountUpdated(int)),
98 SLOT(UpdateTotalSongCount(int)));
99
100 using smart_playlists::Generator;
101 using smart_playlists::GeneratorPtr;
102 using smart_playlists::QueryGenerator;
103 using smart_playlists::Search;
104 using smart_playlists::SearchTerm;
105
106 library_model_ = new LibraryModel(library_backend_, app_, this);
107 library_model_->set_show_various_artists(false);
108 library_model_->set_show_smart_playlists(false);
109 library_model_->set_default_smart_playlists(
110 LibraryModel::DefaultGenerators()
111 << (LibraryModel::GeneratorList()
112 << GeneratorPtr(new JamendoDynamicPlaylist(
113 tr("Jamendo Top Tracks of the Month"),
114 JamendoDynamicPlaylist::OrderBy_RatingMonth))
115 << GeneratorPtr(new JamendoDynamicPlaylist(
116 tr("Jamendo Top Tracks of the Week"),
117 JamendoDynamicPlaylist::OrderBy_RatingWeek))
118 << GeneratorPtr(new JamendoDynamicPlaylist(
119 tr("Jamendo Top Tracks"),
120 JamendoDynamicPlaylist::OrderBy_Rating))
121 << GeneratorPtr(new JamendoDynamicPlaylist(
122 tr("Jamendo Most Listened Tracks"),
123 JamendoDynamicPlaylist::OrderBy_Listened)))
124 << (LibraryModel::GeneratorList() << GeneratorPtr(new QueryGenerator(
125 tr("Dynamic random mix"),
126 Search(Search::Type_All, Search::TermList(), Search::Sort_Random,
127 SearchTerm::Field_Title),
128 true))));
129
130 library_sort_model_->setSourceModel(library_model_);
131 library_sort_model_->setSortRole(LibraryModel::Role_SortText);
132 library_sort_model_->setDynamicSortFilter(true);
133 library_sort_model_->setSortLocaleAware(true);
134 library_sort_model_->sort(0);
135
136 search_provider_ = new LibrarySearchProvider(
137 library_backend_, tr("Jamendo"), "jamendo",
138 IconLoader::Load("jamendo", IconLoader::Provider), false, app_, this);
139 app_->global_search()->AddProvider(search_provider_);
140 connect(app_->global_search(),
141 SIGNAL(ProviderToggled(const SearchProvider*, bool)),
142 SLOT(SearchProviderToggled(const SearchProvider*, bool)));
143 }
144
~JamendoService()145 JamendoService::~JamendoService() {}
146
CreateRootItem()147 QStandardItem* JamendoService::CreateRootItem() {
148 QStandardItem* item = new QStandardItem(
149 IconLoader::Load("jamendo", IconLoader::Provider), kServiceName);
150 item->setData(true, InternetModel::Role_CanLazyLoad);
151 return item;
152 }
153
LazyPopulate(QStandardItem * item)154 void JamendoService::LazyPopulate(QStandardItem* item) {
155 switch (item->data(InternetModel::Role_Type).toInt()) {
156 case InternetModel::Type_Service: {
157 if (total_song_count_ == 0 && !load_database_task_id_) {
158 DownloadDirectory();
159 }
160 model()->merged_model()->AddSubModel(item->index(), library_sort_model_);
161 break;
162 }
163 default:
164 break;
165 }
166 }
167
UpdateTotalSongCount(int count)168 void JamendoService::UpdateTotalSongCount(int count) {
169 total_song_count_ = count;
170 if (total_song_count_ > 0) {
171 library_model_->set_show_smart_playlists(true);
172 accepted_download_ = true; // the user has previously accepted
173 }
174 }
175
DownloadDirectory()176 void JamendoService::DownloadDirectory() {
177 // don't ask if we're refreshing the database
178 if (total_song_count_ == 0) {
179 if (QMessageBox::question(context_menu_, tr("Jamendo database"),
180 tr("This action will create a database which "
181 "could be as big as 150 MB.\n"
182 "Do you want to continue anyway?"),
183 QMessageBox::Ok | QMessageBox::Cancel) !=
184 QMessageBox::Ok)
185 return;
186 }
187 accepted_download_ = true;
188 QNetworkRequest req = QNetworkRequest(QUrl(kDirectoryUrl));
189 req.setAttribute(QNetworkRequest::CacheLoadControlAttribute,
190 QNetworkRequest::AlwaysNetwork);
191
192 QNetworkReply* reply = network_->get(req);
193 connect(reply, SIGNAL(finished()), SLOT(DownloadDirectoryFinished()));
194 connect(reply, SIGNAL(downloadProgress(qint64, qint64)),
195 SLOT(DownloadDirectoryProgress(qint64, qint64)));
196
197 if (!load_database_task_id_) {
198 load_database_task_id_ =
199 app_->task_manager()->StartTask(tr("Downloading Jamendo catalogue"));
200 }
201 }
202
DownloadDirectoryProgress(qint64 received,qint64 total)203 void JamendoService::DownloadDirectoryProgress(qint64 received, qint64 total) {
204 float progress = static_cast<float>(received) / total;
205 app_->task_manager()->SetTaskProgress(load_database_task_id_,
206 static_cast<int>(progress * 100), 100);
207 }
208
DownloadDirectoryFinished()209 void JamendoService::DownloadDirectoryFinished() {
210 QNetworkReply* reply = qobject_cast<QNetworkReply*>(sender());
211 Q_ASSERT(reply);
212
213 app_->task_manager()->SetTaskFinished(load_database_task_id_);
214 load_database_task_id_ = 0;
215
216 // TODO(John Maguire): Not leak reply.
217 QtIOCompressor* gzip = new QtIOCompressor(reply);
218 gzip->setStreamFormat(QtIOCompressor::GzipFormat);
219 if (!gzip->open(QIODevice::ReadOnly)) {
220 qLog(Warning) << "Jamendo library not in gzip format";
221 delete gzip;
222 return;
223 }
224
225 load_database_task_id_ =
226 app_->task_manager()->StartTask(tr("Parsing Jamendo catalogue"));
227
228 QFuture<void> future =
229 QtConcurrent::run(this, &JamendoService::ParseDirectory, gzip);
230 NewClosure(future, this, SLOT(ParseDirectoryFinished()));
231 }
232
ParseDirectory(QIODevice * device) const233 void JamendoService::ParseDirectory(QIODevice* device) const {
234 int total_count = 0;
235
236 // Bit of a hack: don't update the model while we're parsing the xml
237 disconnect(library_backend_, SIGNAL(SongsDiscovered(SongList)),
238 library_model_, SLOT(SongsDiscovered(SongList)));
239 disconnect(library_backend_, SIGNAL(TotalSongCountUpdated(int)), this,
240 SLOT(UpdateTotalSongCount(int)));
241
242 // Delete the database and recreate it. This is faster than dropping tables
243 // or removing rows.
244 library_backend_->db()->RecreateAttachedDb("jamendo");
245
246 TrackIdList track_ids;
247 SongList songs;
248 QXmlStreamReader reader(device);
249 while (!reader.atEnd()) {
250 reader.readNext();
251 if (reader.tokenType() == QXmlStreamReader::StartElement &&
252 reader.name() == "artist") {
253 songs << ReadArtist(&reader, &track_ids);
254 }
255
256 if (songs.count() >= kBatchSize) {
257 // Add the songs to the database in batches
258 library_backend_->AddOrUpdateSongs(songs);
259 InsertTrackIds(track_ids);
260
261 total_count += songs.count();
262 songs.clear();
263 track_ids.clear();
264
265 // Update progress info
266 app_->task_manager()->SetTaskProgress(load_database_task_id_, total_count,
267 kApproxDatabaseSize);
268 }
269 }
270
271 library_backend_->AddOrUpdateSongs(songs);
272 InsertTrackIds(track_ids);
273
274 connect(library_backend_, SIGNAL(SongsDiscovered(SongList)), library_model_,
275 SLOT(SongsDiscovered(SongList)));
276 connect(library_backend_, SIGNAL(TotalSongCountUpdated(int)),
277 SLOT(UpdateTotalSongCount(int)));
278
279 library_backend_->UpdateTotalSongCount();
280 }
281
InsertTrackIds(const TrackIdList & ids) const282 void JamendoService::InsertTrackIds(const TrackIdList& ids) const {
283 QMutexLocker l(library_backend_->db()->Mutex());
284 QSqlDatabase db(library_backend_->db()->Connect());
285
286 ScopedTransaction t(&db);
287
288 QSqlQuery insert(db);
289 insert.prepare(QString("INSERT INTO %1 (%2) VALUES (:id)")
290 .arg(kTrackIdsTable, kTrackIdsColumn));
291
292 for (int id : ids) {
293 insert.bindValue(":id", id);
294 if (!insert.exec()) {
295 qLog(Warning) << "Query failed" << insert.lastQuery();
296 }
297 }
298
299 t.Commit();
300 }
301
ReadArtist(QXmlStreamReader * reader,TrackIdList * track_ids) const302 SongList JamendoService::ReadArtist(QXmlStreamReader* reader,
303 TrackIdList* track_ids) const {
304 SongList ret;
305 QString current_artist;
306
307 while (!reader->atEnd()) {
308 reader->readNext();
309
310 if (reader->tokenType() == QXmlStreamReader::StartElement) {
311 QStringRef name = reader->name();
312 if (name == "name") {
313 current_artist = reader->readElementText().trimmed();
314 } else if (name == "album") {
315 ret << ReadAlbum(current_artist, reader, track_ids);
316 }
317 } else if (reader->isEndElement() && reader->name() == "artist") {
318 break;
319 }
320 }
321
322 return ret;
323 }
324
ReadAlbum(const QString & artist,QXmlStreamReader * reader,TrackIdList * track_ids) const325 SongList JamendoService::ReadAlbum(const QString& artist,
326 QXmlStreamReader* reader,
327 TrackIdList* track_ids) const {
328 SongList ret;
329 QString current_album;
330 QString cover;
331 int current_album_id = 0;
332
333 while (!reader->atEnd()) {
334 reader->readNext();
335
336 if (reader->tokenType() == QXmlStreamReader::StartElement) {
337 if (reader->name() == "name") {
338 current_album = reader->readElementText().trimmed();
339 } else if (reader->name() == "id") {
340 QString id = reader->readElementText();
341 cover = QString(kAlbumCoverUrl).arg(id);
342 current_album_id = id.toInt();
343 } else if (reader->name() == "track") {
344 ret << ReadTrack(artist, current_album, cover, current_album_id, reader,
345 track_ids);
346 }
347 } else if (reader->isEndElement() && reader->name() == "album") {
348 break;
349 }
350 }
351 return ret;
352 }
353
ReadTrack(const QString & artist,const QString & album,const QString & album_cover,int album_id,QXmlStreamReader * reader,TrackIdList * track_ids) const354 Song JamendoService::ReadTrack(const QString& artist, const QString& album,
355 const QString& album_cover, int album_id,
356 QXmlStreamReader* reader,
357 TrackIdList* track_ids) const {
358 Song song;
359 song.set_artist(artist);
360 song.set_album(album);
361 song.set_filetype(Song::Type_Stream);
362 song.set_directory_id(0);
363 song.set_mtime(0);
364 song.set_ctime(0);
365 song.set_filesize(0);
366
367 // Shoehorn the album ID into the comment field
368 song.set_comment(QString::number(album_id));
369
370 while (!reader->atEnd()) {
371 reader->readNext();
372 if (reader->isStartElement()) {
373 QStringRef name = reader->name();
374 if (name == "name") {
375 song.set_title(reader->readElementText().trimmed());
376 } else if (name == "duration") {
377 const int length = reader->readElementText().toFloat();
378 song.set_length_nanosec(length * kNsecPerSec);
379 } else if (name == "id3genre") {
380 int genre_id = reader->readElementText().toInt();
381 // In theory, genre 0 is "blues"; in practice it's invalid.
382 if (genre_id != 0) {
383 song.set_genre_id3(genre_id);
384 }
385 } else if (name == "id") {
386 QString id_text = reader->readElementText();
387 int id = id_text.toInt();
388 if (id == 0) continue;
389
390 QString mp3_url = QString(kMp3StreamUrl).arg(id_text);
391 song.set_url(QUrl(mp3_url));
392 song.set_art_automatic(album_cover);
393 song.set_valid(true);
394
395 // Rely on songs getting added in this exact order
396 track_ids->append(id);
397 }
398 } else if (reader->isEndElement() && reader->name() == "track") {
399 break;
400 }
401 }
402 return song;
403 }
404
ParseDirectoryFinished()405 void JamendoService::ParseDirectoryFinished() {
406 // show smart playlists
407 library_model_->set_show_smart_playlists(true);
408 library_model_->Reset();
409
410 app_->task_manager()->SetTaskFinished(load_database_task_id_);
411 load_database_task_id_ = 0;
412 }
413
EnsureMenuCreated()414 void JamendoService::EnsureMenuCreated() {
415 if (library_filter_) return;
416
417 context_menu_ = new QMenu;
418 context_menu_->addActions(GetPlaylistActions());
419 album_info_ = context_menu_->addAction(
420 IconLoader::Load("view-media-lyrics", IconLoader::Base),
421 tr("Album info on jamendo.com..."), this, SLOT(AlbumInfo()));
422 download_album_ = context_menu_->addAction(
423 IconLoader::Load("download", IconLoader::Base),
424 tr("Download this album..."), this, SLOT(DownloadAlbum()));
425 context_menu_->addSeparator();
426 context_menu_->addAction(IconLoader::Load("download", IconLoader::Base),
427 tr("Open %1 in browser").arg("jamendo.com"), this,
428 SLOT(Homepage()));
429 context_menu_->addAction(IconLoader::Load("view-refresh", IconLoader::Base),
430 tr("Refresh catalogue"), this,
431 SLOT(DownloadDirectory()));
432
433 if (accepted_download_) {
434 library_filter_ = new LibraryFilterWidget(0);
435 library_filter_->SetSettingsGroup(kSettingsGroup);
436 library_filter_->SetLibraryModel(library_model_);
437 library_filter_->SetFilterHint(tr("Search Jamendo"));
438 library_filter_->SetAgeFilterEnabled(false);
439
440 context_menu_->addSeparator();
441 context_menu_->addMenu(library_filter_->menu());
442 }
443 }
444
ShowContextMenu(const QPoint & global_pos)445 void JamendoService::ShowContextMenu(const QPoint& global_pos) {
446 EnsureMenuCreated();
447
448 const bool enabled = accepted_download_ &&
449 model()->current_index().model() == library_sort_model_;
450
451 // make menu items visible and enabled only when needed
452 GetAppendToPlaylistAction()->setVisible(accepted_download_);
453 GetAppendToPlaylistAction()->setEnabled(enabled);
454 GetReplacePlaylistAction()->setVisible(accepted_download_);
455 GetReplacePlaylistAction()->setEnabled(enabled);
456 GetOpenInNewPlaylistAction()->setEnabled(enabled);
457 GetOpenInNewPlaylistAction()->setVisible(accepted_download_);
458 album_info_->setEnabled(enabled);
459 album_info_->setVisible(accepted_download_);
460 download_album_->setEnabled(enabled);
461 download_album_->setVisible(accepted_download_);
462
463 context_menu_->popup(global_pos);
464 }
465
HeaderWidget() const466 QWidget* JamendoService::HeaderWidget() const {
467 const_cast<JamendoService*>(this)->EnsureMenuCreated();
468 return library_filter_;
469 }
470
AlbumInfo()471 void JamendoService::AlbumInfo() {
472 SongList songs(library_model_->GetChildSongs(
473 library_sort_model_->mapToSource(model()->current_index())));
474 if (songs.isEmpty()) return;
475
476 // We put the album ID into the comment field
477 int id = songs.first().comment().toInt();
478 if (!id) return;
479
480 QDesktopServices::openUrl(QUrl(QString(kAlbumInfoUrl).arg(id)));
481 }
482
DownloadAlbum()483 void JamendoService::DownloadAlbum() {
484 SongList songs(library_model_->GetChildSongs(
485 library_sort_model_->mapToSource(model()->current_index())));
486 if (songs.isEmpty()) return;
487
488 // We put the album ID into the comment field
489 int id = songs.first().comment().toInt();
490 if (!id) return;
491
492 QDesktopServices::openUrl(QUrl(QString(kDownloadAlbumUrl).arg(id)));
493 }
494
Homepage()495 void JamendoService::Homepage() { QDesktopServices::openUrl(QUrl(kHomepage)); }
496
SearchProviderToggled(const SearchProvider * provider,bool enabled)497 void JamendoService::SearchProviderToggled(const SearchProvider* provider,
498 bool enabled) {
499 // If the use enabled our provider and he hasn't downloaded the directory yet,
500 // prompt him to do so now.
501 if (provider == search_provider_ && enabled && total_song_count_ == 0) {
502 DownloadDirectory();
503 }
504 }
505