1 /* $BEGIN_LICENSE
2
3 This file is part of Minitube.
4 Copyright 2009, Flavio Tordini <flavio.tordini@gmail.com>
5
6 Minitube is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
10
11 Minitube is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with Minitube. If not, see <http://www.gnu.org/licenses/>.
18
19 $END_LICENSE */
20
21 #include "channelaggregator.h"
22 #include "database.h"
23 #include "searchparams.h"
24 #include "video.h"
25 #include "ytchannel.h"
26 #include "ytsearch.h"
27 #ifdef APP_MAC
28 #include "macutils.h"
29 #endif
30 #include "http.h"
31 #include "httputils.h"
32
33 #include "ivchannelsource.h"
34 #include "videoapi.h"
35 #include "ytjschannelsource.h"
36
ChannelAggregator(QObject * parent)37 ChannelAggregator::ChannelAggregator(QObject *parent)
38 : QObject(parent), unwatchedCount(-1), running(false), stopped(false), currentChannel(0) {
39 checkInterval = 3600;
40
41 timer = new QTimer(this);
42 timer->setInterval(60000 * 5);
43 connect(timer, SIGNAL(timeout()), SLOT(run()));
44 }
45
instance()46 ChannelAggregator *ChannelAggregator::instance() {
47 static ChannelAggregator *i = new ChannelAggregator();
48 return i;
49 }
50
start()51 void ChannelAggregator::start() {
52 stopped = false;
53 updateUnwatchedCount();
54 QTimer::singleShot(10000, this, SLOT(run()));
55 if (!timer->isActive()) timer->start();
56 }
57
stop()58 void ChannelAggregator::stop() {
59 timer->stop();
60 stopped = true;
61 }
62
getChannelToCheck()63 YTChannel *ChannelAggregator::getChannelToCheck() {
64 if (stopped) return 0;
65 QSqlDatabase db = Database::instance().getConnection();
66 QSqlQuery query(db);
67 query.prepare("select user_id from subscriptions where checked<? "
68 "order by checked limit 1");
69 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t() - checkInterval);
70 bool success = query.exec();
71 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
72 if (query.next()) return YTChannel::forId(query.value(0).toString());
73 return 0;
74 }
75
run()76 void ChannelAggregator::run() {
77 if (running) return;
78 if (!Database::exists()) return;
79 running = true;
80 newVideoCount = 0;
81 updatedChannels.clear();
82 updatedChannels.squeeze();
83
84 processNextChannel();
85 }
86
processNextChannel()87 void ChannelAggregator::processNextChannel() {
88 if (stopped) {
89 running = false;
90 return;
91 }
92 YTChannel *channel = getChannelToCheck();
93 if (channel) {
94 checkWebPage(channel);
95 } else
96 finish();
97 }
98
checkWebPage(YTChannel * channel)99 void ChannelAggregator::checkWebPage(YTChannel *channel) {
100 currentChannel = channel;
101
102 QString channelId = channel->getChannelId();
103 QString url;
104 if (channelId.startsWith("UC") && !channelId.contains(' ')) {
105 url = "https://www.youtube.com/channel/" + channelId + "/videos";
106 } else {
107 url = "https://www.youtube.com/user/" + channelId + "/videos";
108 }
109
110 QObject *reply = HttpUtils::yt().get(url);
111
112 connect(reply, SIGNAL(data(QByteArray)), SLOT(parseWebPage(QByteArray)));
113 connect(reply, SIGNAL(error(QString)), SLOT(errorWebPage(QString)));
114 }
115
parseWebPage(const QByteArray & bytes)116 void ChannelAggregator::parseWebPage(const QByteArray &bytes) {
117 bool hasNewVideos = true;
118 QRegExp re = QRegExp("[\\?&]v=([0-9A-Za-z_-]+)");
119 if (re.indexIn(bytes) != -1) {
120 QString videoId = re.cap(1);
121 QString latestVideoId = currentChannel->latestVideoId();
122 qDebug() << "Comparing" << videoId << latestVideoId;
123 hasNewVideos = videoId != latestVideoId;
124 } else {
125 qDebug() << "Cannot capture latest video id";
126 }
127 if (hasNewVideos) {
128 if (currentChannel) {
129 reallyProcessChannel(currentChannel);
130 currentChannel = 0;
131 }
132 } else {
133 currentChannel->updateChecked();
134 currentChannel = 0;
135 QTimer::singleShot(5000, this, &ChannelAggregator::processNextChannel);
136 }
137 }
138
errorWebPage(const QString & message)139 void ChannelAggregator::errorWebPage(const QString &message) {
140 Q_UNUSED(message);
141 reallyProcessChannel(currentChannel);
142 currentChannel = 0;
143 }
144
reallyProcessChannel(YTChannel * channel)145 void ChannelAggregator::reallyProcessChannel(YTChannel *channel) {
146 SearchParams *params = new SearchParams();
147 params->setChannelId(channel->getChannelId());
148 params->setSortBy(SearchParams::SortByNewest);
149 params->setTransient(true);
150 params->setPublishedAfter(channel->getChecked());
151
152 if (VideoAPI::impl() == VideoAPI::YT3) {
153 YTSearch *videoSource = new YTSearch(params);
154 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
155 SLOT(videosLoaded(QVector<Video *>)));
156 videoSource->loadVideos(50, 1);
157 } else if (VideoAPI::impl() == VideoAPI::IV) {
158 auto *videoSource = new IVChannelSource(params);
159 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
160 SLOT(videosLoaded(QVector<Video *>)));
161 videoSource->loadVideos(50, 1);
162 } else if (VideoAPI::impl() == VideoAPI::JS) {
163 auto *videoSource = new YTJSChannelSource(params);
164 connect(videoSource, SIGNAL(gotVideos(QVector<Video *>)),
165 SLOT(videosLoaded(QVector<Video *>)));
166 videoSource->loadVideos(50, 1);
167 }
168
169 channel->updateChecked();
170 }
171
finish()172 void ChannelAggregator::finish() {
173 currentChannel = 0;
174
175 #ifdef Q_OS_MAC
176 if (newVideoCount > 0 && unwatchedCount > 0 && mac::canNotify()) {
177 QString channelNames;
178 const int total = updatedChannels.size();
179 for (int i = 0; i < total; ++i) {
180 YTChannel *channel = updatedChannels.at(i);
181 channelNames += channel->getDisplayName();
182 if (i < total - 1) channelNames.append(", ");
183 }
184 channelNames = tr("By %1").arg(channelNames);
185 int actualNewVideoCount = qMin(newVideoCount, unwatchedCount);
186 mac::notify(tr("You have %n new video(s)", "", actualNewVideoCount), channelNames,
187 QString());
188 }
189 #endif
190
191 running = false;
192 }
193
videosLoaded(const QVector<Video * > & videos)194 void ChannelAggregator::videosLoaded(const QVector<Video *> &videos) {
195 sender()->deleteLater();
196
197 for (Video *video : videos) {
198 addVideo(video);
199 qApp->processEvents();
200 }
201
202 if (!videos.isEmpty()) {
203 YTChannel *channel = YTChannel::forId(videos.at(0)->getChannelId());
204 if (channel) {
205 channel->updateNotifyCount();
206 emit channelChanged(channel);
207 }
208 updateUnwatchedCount();
209 for (Video *video : videos)
210 video->deleteLater();
211 }
212
213 QTimer::singleShot(0, this, SLOT(processNextChannel()));
214 }
215
updateUnwatchedCount()216 void ChannelAggregator::updateUnwatchedCount() {
217 if (!Database::exists()) return;
218 QSqlDatabase db = Database::instance().getConnection();
219 QSqlQuery query(db);
220 query.prepare("select sum(notify_count) from subscriptions");
221 bool success = query.exec();
222 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
223 if (!query.next()) return;
224 int newUnwatchedCount = query.value(0).toInt();
225 if (newUnwatchedCount != unwatchedCount) {
226 unwatchedCount = newUnwatchedCount;
227 emit unwatchedCountChanged(unwatchedCount);
228 }
229 }
230
addVideo(Video * video)231 void ChannelAggregator::addVideo(Video *video) {
232 QSqlDatabase db = Database::instance().getConnection();
233
234 QSqlQuery query(db);
235 query.prepare("select count(*) from subscriptions_videos where video_id=?");
236 query.bindValue(0, video->getId());
237 bool success = query.exec();
238 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
239 if (!query.next()) return;
240 int count = query.value(0).toInt();
241 if (count > 0) return;
242
243 // qDebug() << "Inserting" << video->author() << video->title();
244
245 YTChannel *channel = YTChannel::forId(video->getChannelId());
246 if (!channel) {
247 qWarning() << "channelId not present in db" << video->getChannelId()
248 << video->getChannelTitle();
249 return;
250 }
251
252 if (!updatedChannels.contains(channel)) updatedChannels << channel;
253
254 uint now = QDateTime::currentDateTimeUtc().toTime_t();
255 uint published = video->getPublished().toTime_t();
256 if (published > now) {
257 qDebug() << "fixing publish time";
258 published = now;
259 }
260
261 query = QSqlQuery(db);
262 query.prepare("insert into subscriptions_videos "
263 "(video_id,channel_id,published,added,watched,"
264 "title,author,user_id,description,url,thumb_url,views,duration) "
265 "values (?,?,?,?,?,?,?,?,?,?,?,?,?)");
266 query.bindValue(0, video->getId());
267 query.bindValue(1, channel->getId());
268 query.bindValue(2, published);
269 query.bindValue(3, now);
270 query.bindValue(4, 0);
271 query.bindValue(5, video->getTitle());
272 query.bindValue(6, video->getChannelTitle());
273 query.bindValue(7, video->getChannelId());
274 query.bindValue(8, video->getDescription());
275 query.bindValue(9, video->getWebpage());
276
277 QJsonDocument thumbsDoc;
278 auto thumbsArray = thumbsDoc.array();
279 for (const auto &t : video->getThumbs()) {
280 thumbsArray.append(QJsonObject{
281 {"url", t.getUrl()},
282 {"width", t.getWidth()},
283 {"height", t.getHeight()},
284 });
285 }
286 thumbsDoc.setArray(thumbsArray);
287 query.bindValue(10, thumbsDoc.toJson(QJsonDocument::Compact));
288 query.bindValue(11, video->getViewCount());
289 query.bindValue(12, video->getDuration());
290 success = query.exec();
291 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
292
293 newVideoCount++;
294
295 query = QSqlQuery(db);
296 query.prepare("update subscriptions set updated=? where user_id=?");
297 query.bindValue(0, published);
298 query.bindValue(1, channel->getChannelId());
299 success = query.exec();
300 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
301 }
302
markAllAsWatched()303 void ChannelAggregator::markAllAsWatched() {
304 uint now = QDateTime::currentDateTimeUtc().toTime_t();
305
306 QSqlDatabase db = Database::instance().getConnection();
307 QSqlQuery query(db);
308 query.prepare("update subscriptions set watched=?, notify_count=0");
309 query.bindValue(0, now);
310 bool success = query.exec();
311 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
312 unwatchedCount = 0;
313
314 const auto &channels = YTChannel::getCachedChannels();
315 for (YTChannel *channel : channels) {
316 channel->setWatched(now);
317 channel->setNotifyCount(0);
318 }
319
320 emit unwatchedCountChanged(0);
321 }
322
videoWatched(Video * video)323 void ChannelAggregator::videoWatched(Video *video) {
324 if (!Database::exists()) return;
325 QSqlDatabase db = Database::instance().getConnection();
326 QSqlQuery query(db);
327 query.prepare("update subscriptions_videos set watched=? where video_id=?");
328 query.bindValue(0, QDateTime::currentDateTimeUtc().toTime_t());
329 query.bindValue(1, video->getId());
330 bool success = query.exec();
331 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
332 if (query.numRowsAffected() > 0) {
333 YTChannel *channel = YTChannel::forId(video->getChannelId());
334 if (channel) channel->updateNotifyCount();
335 }
336 }
337
cleanup()338 void ChannelAggregator::cleanup() {
339 const int maxVideos = 1000;
340 const int maxDeletions = 1000;
341 if (!Database::exists()) return;
342 QSqlDatabase db = Database::instance().getConnection();
343
344 QSqlQuery query(db);
345 bool success = query.exec("select count(*) from subscriptions_videos");
346 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
347 if (!query.next()) return;
348 int count = query.value(0).toInt();
349 if (count <= maxVideos) return;
350
351 query = QSqlQuery(db);
352 query.prepare("delete from subscriptions_videos where id in "
353 "(select id from subscriptions_videos "
354 "order by published desc limit ?,?)");
355 query.bindValue(0, maxVideos);
356 query.bindValue(1, maxDeletions);
357 success = query.exec();
358 if (!success) qWarning() << query.lastQuery() << query.lastError().text();
359 }
360