1 // For license of this file, see <project-root-folder>/LICENSE.md.
2
3 #include "services/greader/greadernetwork.h"
4
5 #include "3rd-party/boolinq/boolinq.h"
6 #include "database/databasequeries.h"
7 #include "exceptions/applicationexception.h"
8 #include "exceptions/feedfetchexception.h"
9 #include "exceptions/networkexception.h"
10 #include "miscellaneous/application.h"
11 #include "network-web/networkfactory.h"
12 #include "network-web/oauth2service.h"
13 #include "network-web/webfactory.h"
14 #include "services/abstract/category.h"
15 #include "services/abstract/label.h"
16 #include "services/abstract/labelsnode.h"
17 #include "services/greader/definitions.h"
18
19 #include <QJsonArray>
20 #include <QJsonDocument>
21 #include <QJsonObject>
22
GreaderNetwork(QObject * parent)23 GreaderNetwork::GreaderNetwork(QObject* parent)
24 : QObject(parent), m_root(nullptr), m_service(GreaderServiceRoot::Service::FreshRss), m_username(QString()),
25 m_password(QString()), m_baseUrl(QString()), m_batchSize(GREADER_DEFAULT_BATCH_SIZE), m_downloadOnlyUnreadMessages(false),
26 m_prefetchedMessages({}), m_prefetchedStatus(Feed::Status::Normal), m_performGlobalFetching(false),
27 m_intelligentSynchronization(true), m_newerThanFilter(QDate::currentDate().addYears(-1)),
28 m_oauth(new OAuth2Service(QSL(INO_OAUTH_AUTH_URL), QSL(INO_OAUTH_TOKEN_URL),
29 {}, {}, QSL(INO_OAUTH_SCOPE), this)) {
30 initializeOauth();
31 clearCredentials();
32 }
33
editLabels(const QString & state,bool assign,const QStringList & msg_custom_ids,const QNetworkProxy & proxy)34 QNetworkReply::NetworkError GreaderNetwork::editLabels(const QString& state,
35 bool assign,
36 const QStringList& msg_custom_ids,
37 const QNetworkProxy& proxy) {
38 QString full_url = generateFullUrl(Operations::EditTag);
39 int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
40
41 QNetworkReply::NetworkError network_err = QNetworkReply::NetworkError::UnknownNetworkError;
42
43 if (!ensureLogin(proxy, &network_err)) {
44 return network_err;
45 }
46
47 QStringList trimmed_ids; trimmed_ids.reserve(msg_custom_ids.size());
48
49 for (const QString& id : msg_custom_ids) {
50 trimmed_ids.append(QSL("i=") + id);
51 }
52
53 QStringList working_subset; working_subset.reserve(std::min(GREADER_API_EDIT_TAG_BATCH, trimmed_ids.size()));
54
55 // Now, we perform messages update in batches (max X messages per batch).
56 while (!trimmed_ids.isEmpty()) {
57 // We take X IDs.
58 for (int i = 0; i < GREADER_API_EDIT_TAG_BATCH && !trimmed_ids.isEmpty(); i++) {
59 working_subset.append(trimmed_ids.takeFirst());
60 }
61
62 QString args;
63
64 if (assign) {
65 args = QSL("a=") + state + "&";
66 }
67 else {
68 args = QSL("r=") + state + "&";
69 }
70
71 args += working_subset.join(QL1C('&'));
72
73 if (m_service == GreaderServiceRoot::Service::Reedah) {
74 args += QSL("&T=%1").arg(m_authToken);
75 }
76
77 // We send this batch.
78 QByteArray output;
79 auto result_edit = NetworkFactory::performNetworkOperation(full_url,
80 timeout,
81 args.toUtf8(),
82 output,
83 QNetworkAccessManager::Operation::PostOperation,
84 { authHeader(),
85 { QSL(HTTP_HEADERS_CONTENT_TYPE).toLocal8Bit(),
86 QSL("application/x-www-form-urlencoded").toLocal8Bit() } },
87 false,
88 {},
89 {},
90 proxy);
91
92 if (result_edit.first != QNetworkReply::NetworkError::NoError) {
93 return result_edit.first;
94 }
95
96 // Cleanup for next batch.
97 working_subset.clear();
98 }
99
100 return QNetworkReply::NetworkError::NoError;
101 }
102
userInfo(const QNetworkProxy & proxy)103 QVariantHash GreaderNetwork::userInfo(const QNetworkProxy& proxy) {
104 QString full_url = generateFullUrl(Operations::UserInfo);
105 int timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
106 QNetworkReply::NetworkError network_err = QNetworkReply::NetworkError::UnknownNetworkError;
107
108 if (!ensureLogin(proxy, &network_err)) {
109 throw NetworkException(network_err);
110 }
111
112 QByteArray output;
113 auto res = NetworkFactory::performNetworkOperation(full_url,
114 timeout,
115 {},
116 output,
117 QNetworkAccessManager::Operation::GetOperation,
118 { authHeader() },
119 false,
120 {},
121 {},
122 proxy);
123
124 if (res.first != QNetworkReply::NetworkError::NoError) {
125 throw NetworkException(res.first);
126 }
127
128 return QJsonDocument::fromJson(output).object().toVariantHash();
129 }
130
clearPrefetchedMessages()131 void GreaderNetwork::clearPrefetchedMessages() {
132 m_prefetchedMessages.clear();
133 m_prefetchedStatus = Feed::Status::Normal;
134 }
135
prepareFeedFetching(GreaderServiceRoot * root,const QList<Feed * > & feeds,const QHash<QString,QHash<ServiceRoot::BagOfMessages,QStringList>> & stated_messages,const QHash<QString,QStringList> & tagged_messages,const QNetworkProxy & proxy)136 void GreaderNetwork::prepareFeedFetching(GreaderServiceRoot* root,
137 const QList<Feed*>& feeds,
138 const QHash<QString, QHash<ServiceRoot::BagOfMessages, QStringList>>& stated_messages,
139 const QHash<QString, QStringList>& tagged_messages,
140 const QNetworkProxy& proxy) {
141 Q_UNUSED(tagged_messages)
142
143 m_prefetchedMessages.clear();
144 m_prefetchedStatus = Feed::Status::Normal;
145
146 try {
147
148 double perc_of_fetching = (feeds.size() * 1.0) / root->getSubTreeFeeds().size();
149
150 m_performGlobalFetching = perc_of_fetching > GREADER_GLOBAL_UPDATE_THRES;
151
152 qDebugNN << LOGSEC_GREADER
153 << "Percentage of feeds for fetching:"
154 << QUOTE_W_SPACE_DOT(perc_of_fetching);
155
156 auto remote_starred_ids_list = itemIds(QSL(GREADER_API_FULL_STATE_IMPORTANT), false, proxy, -1, m_newerThanFilter);
157
158 for (int i = 0; i < remote_starred_ids_list.size(); i++) {
159 remote_starred_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_starred_ids_list.at(i)));
160 }
161
162 QSet<QString> remote_starred_ids = FROM_LIST_TO_SET(QSet<QString>, remote_starred_ids_list);
163 QSet<QString> local_starred_ids;
164 QList<QHash<ServiceRoot::BagOfMessages, QStringList>> all_states = stated_messages.values();
165
166 for (auto& lst : all_states) {
167 auto s = lst.value(ServiceRoot::BagOfMessages::Starred);
168
169 local_starred_ids.unite(FROM_LIST_TO_SET(QSet<QString>, s));
170 }
171
172 auto starred_to_download((remote_starred_ids - local_starred_ids).unite(local_starred_ids - remote_starred_ids));
173 auto to_download = starred_to_download;
174
175 if (m_performGlobalFetching) {
176 qWarningNN << LOGSEC_GREADER << "Performing global contents fetching.";
177
178 QStringList remote_all_ids_list = m_downloadOnlyUnreadMessages
179 ? QStringList()
180 : itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), false, proxy, -1, m_newerThanFilter);
181 QStringList remote_unread_ids_list = itemIds(QSL(GREADER_API_FULL_STATE_READING_LIST), true, proxy, -1, m_newerThanFilter);
182
183 for (int i = 0; i < remote_all_ids_list.size(); i++) {
184 remote_all_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_all_ids_list.at(i)));
185 }
186
187 for (int i = 0; i < remote_unread_ids_list.size(); i++) {
188 remote_unread_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_unread_ids_list.at(i)));
189 }
190
191 QSet<QString> remote_all_ids = FROM_LIST_TO_SET(QSet<QString>, remote_all_ids_list);
192 QSet<QString> remote_unread_ids = FROM_LIST_TO_SET(QSet<QString>, remote_unread_ids_list);
193 QSet<QString> remote_read_ids = remote_all_ids - remote_unread_ids;
194 QSet<QString> local_unread_ids;
195 QSet<QString> local_read_ids;
196
197 for (auto& lst : all_states) {
198 auto u = lst.value(ServiceRoot::BagOfMessages::Unread);
199 auto r = lst.value(ServiceRoot::BagOfMessages::Read);
200
201 local_unread_ids.unite(FROM_LIST_TO_SET(QSet<QString>, u));
202 local_read_ids.unite(FROM_LIST_TO_SET(QSet<QString>, r));
203 }
204
205 if (!m_downloadOnlyUnreadMessages) {
206 to_download += remote_all_ids - local_read_ids - local_unread_ids;
207 }
208 else {
209 to_download += remote_unread_ids - local_read_ids - local_unread_ids;
210 }
211
212 auto moved_read = local_read_ids.intersect(remote_unread_ids);
213
214 to_download += moved_read;
215
216 if (!m_downloadOnlyUnreadMessages) {
217 auto moved_unread = local_unread_ids.intersect(remote_read_ids);
218
219 to_download += moved_unread;
220 }
221 }
222 else {
223 qWarningNN << LOGSEC_GREADER << "Performing feed-based contents fetching.";
224 }
225
226 Feed::Status error;
227 QList<QString> to_download_list(to_download.values());
228
229 if (!to_download_list.isEmpty()) {
230 if (m_service == GreaderServiceRoot::Service::Reedah) {
231 for (int i = 0; i < to_download_list.size(); i++) {
232 to_download_list.replace(i, convertLongStreamIdToShortStreamId(to_download_list.at(i)));
233 }
234 }
235
236 m_prefetchedMessages = itemContents(root, to_download_list, error, proxy);
237 }
238 }
239 catch (const FeedFetchException& fex) {
240 m_prefetchedStatus = fex.feedStatus();
241 }
242 }
243
getMessagesIntelligently(ServiceRoot * root,const QString & stream_id,const QHash<ServiceRoot::BagOfMessages,QStringList> & stated_messages,const QHash<QString,QStringList> & tagged_messages,Feed::Status & error,const QNetworkProxy & proxy)244 QList<Message> GreaderNetwork::getMessagesIntelligently(ServiceRoot* root,
245 const QString& stream_id,
246 const QHash<ServiceRoot::BagOfMessages, QStringList>& stated_messages,
247 const QHash<QString, QStringList>& tagged_messages,
248 Feed::Status& error,
249 const QNetworkProxy& proxy) {
250 Q_UNUSED(tagged_messages)
251
252 QList<Message> msgs;
253
254 if (m_prefetchedStatus != Feed::Status::Normal) {
255 error = m_prefetchedStatus;
256 return msgs;
257 }
258
259 if (!m_performGlobalFetching) {
260 // 1. Get unread IDs for a feed.
261 // 2. Get read IDs for a feed.
262 // 3. Download messages/contents for missing or changed IDs.
263 // 4. Add prefetched starred msgs.
264 QStringList remote_all_ids_list = m_downloadOnlyUnreadMessages
265 ? QStringList()
266 : itemIds(stream_id, false, proxy, -1, m_newerThanFilter);
267 QStringList remote_unread_ids_list = itemIds(stream_id, true, proxy, -1, m_newerThanFilter);
268
269 // Convert item IDs to long form.
270 for (int i = 0; i < remote_all_ids_list.size(); i++) {
271 remote_all_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_all_ids_list.at(i)));
272 }
273
274 for (int i = 0; i < remote_unread_ids_list.size(); i++) {
275 remote_unread_ids_list.replace(i, convertShortStreamIdToLongStreamId(remote_unread_ids_list.at(i)));
276 }
277
278 QSet<QString> remote_all_ids = FROM_LIST_TO_SET(QSet<QString>, remote_all_ids_list);
279
280 // 1.
281 auto local_unread_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Unread);
282 QSet<QString> remote_unread_ids = FROM_LIST_TO_SET(QSet<QString>, remote_unread_ids_list);
283 QSet<QString> local_unread_ids = FROM_LIST_TO_SET(QSet<QString>, local_unread_ids_list);
284
285 // 2.
286 auto local_read_ids_list = stated_messages.value(ServiceRoot::BagOfMessages::Read);
287 QSet<QString> remote_read_ids = remote_all_ids - remote_unread_ids;
288 QSet<QString> local_read_ids = FROM_LIST_TO_SET(QSet<QString>, local_read_ids_list);
289
290 // 3.
291 QSet<QString> to_download;
292
293 if (!m_downloadOnlyUnreadMessages) {
294 to_download += remote_all_ids - local_read_ids - local_unread_ids;
295 }
296 else {
297 to_download += remote_unread_ids - local_read_ids - local_unread_ids;
298 }
299
300 auto moved_read = local_read_ids.intersect(remote_unread_ids);
301
302 to_download += moved_read;
303
304 if (!m_downloadOnlyUnreadMessages) {
305 auto moved_unread = local_unread_ids.intersect(remote_read_ids);
306
307 to_download += moved_unread;
308 }
309
310 QList<QString> to_download_list(to_download.values());
311
312 if (!to_download_list.isEmpty()) {
313 if (m_service == GreaderServiceRoot::Service::Reedah) {
314 for (int i = 0; i < to_download_list.size(); i++) {
315 to_download_list.replace(i, convertLongStreamIdToShortStreamId(to_download_list.at(i)));
316 }
317 }
318
319 msgs = itemContents(root, to_download_list, error, proxy);
320 }
321 }
322
323 // Add prefetched messages.
324 for (int i = 0; i < m_prefetchedMessages.size(); i++) {
325 auto prefetched_msg = m_prefetchedMessages.at(i);
326
327 if (prefetched_msg.m_feedId == stream_id &&
328 !boolinq::from(msgs).any([&prefetched_msg](const Message& ms) {
329 return ms.m_customId == prefetched_msg.m_customId;
330 })) {
331 msgs.append(prefetched_msg);
332 m_prefetchedMessages.removeAt(i--);
333 }
334 }
335
336 return msgs;
337 }
338
markMessagesRead(RootItem::ReadStatus status,const QStringList & msg_custom_ids,const QNetworkProxy & proxy)339 QNetworkReply::NetworkError GreaderNetwork::markMessagesRead(RootItem::ReadStatus status,
340 const QStringList& msg_custom_ids,
341 const QNetworkProxy& proxy) {
342 return editLabels(QSL(GREADER_API_FULL_STATE_READ), status == RootItem::ReadStatus::Read, msg_custom_ids, proxy);
343 }
344
markMessagesStarred(RootItem::Importance importance,const QStringList & msg_custom_ids,const QNetworkProxy & proxy)345 QNetworkReply::NetworkError GreaderNetwork::markMessagesStarred(RootItem::Importance importance,
346 const QStringList& msg_custom_ids,
347 const QNetworkProxy& proxy) {
348 return editLabels(QSL(GREADER_API_FULL_STATE_IMPORTANT),
349 importance == RootItem::Importance::Important,
350 msg_custom_ids,
351 proxy);
352 }
353
itemIds(const QString & stream_id,bool unread_only,const QNetworkProxy & proxy,int max_count,QDate newer_than)354 QStringList GreaderNetwork::itemIds(const QString& stream_id, bool unread_only, const QNetworkProxy& proxy,
355 int max_count, QDate newer_than) {
356 QString continuation;
357
358 if (!ensureLogin(proxy)) {
359 throw FeedFetchException(Feed::Status::AuthError, tr("login failed"));
360 }
361
362 QStringList ids;
363
364 do {
365 QString full_url = generateFullUrl(Operations::ItemIds).arg(m_service == GreaderServiceRoot::Service::TheOldReader
366 ? stream_id
367 : QUrl::toPercentEncoding(stream_id),
368 QString::number(max_count <= 0
369 ? GREADET_API_ITEM_IDS_MAX
370 : max_count));
371 auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
372
373 if (unread_only) {
374 full_url += QSL("&xt=%1").arg(QSL(GREADER_API_FULL_STATE_READ));
375 }
376
377 if (!continuation.isEmpty()) {
378 full_url += QSL("&c=%1").arg(continuation);
379 }
380
381 if (newer_than.isValid()) {
382 full_url += QSL("&ot=%1").arg(
383 #if QT_VERSION < 0x050E00 // Qt < 5.14.0
384 QDateTime(newer_than)
385 #else
386 newer_than.startOfDay()
387 #endif
388 .toSecsSinceEpoch());
389 }
390
391 QByteArray output_stream;
392 auto result_stream = NetworkFactory::performNetworkOperation(full_url,
393 timeout,
394 {},
395 output_stream,
396 QNetworkAccessManager::Operation::GetOperation,
397 { authHeader() },
398 false,
399 {},
400 {},
401 proxy);
402
403 if (result_stream.first != QNetworkReply::NetworkError::NoError) {
404 qCriticalNN << LOGSEC_GREADER
405 << "Cannot download item IDs for "
406 << QUOTE_NO_SPACE(stream_id)
407 << ", network error:"
408 << QUOTE_W_SPACE_DOT(result_stream.first);
409 throw NetworkException(result_stream.first);
410 }
411 else {
412 ids.append(decodeItemIds(output_stream, continuation));
413 }
414 }
415 while (!continuation.isEmpty());
416
417 return ids;
418 }
419
itemContents(ServiceRoot * root,const QList<QString> & stream_ids,Feed::Status & error,const QNetworkProxy & proxy)420 QList<Message> GreaderNetwork::itemContents(ServiceRoot* root, const QList<QString>& stream_ids,
421 Feed::Status& error, const QNetworkProxy& proxy) {
422 QString continuation;
423
424 if (!ensureLogin(proxy)) {
425 error = Feed::Status::AuthError;
426 return {};
427 }
428
429 QList<Message> msgs;
430 QList<QString> my_stream_ids(stream_ids);
431
432 while (!my_stream_ids.isEmpty()) {
433 int batch = (m_service == GreaderServiceRoot::Service::TheOldReader ||
434 m_service == GreaderServiceRoot::Service::FreshRss)
435 ? TOR_ITEM_CONTENTS_BATCH
436 : (m_service == GreaderServiceRoot::Service::Inoreader
437 ? INO_ITEM_CONTENTS_BATCH
438 : GREADER_API_ITEM_CONTENTS_BATCH);
439 QList<QString> batch_ids = my_stream_ids.mid(0, batch);
440
441 my_stream_ids = my_stream_ids.mid(batch);
442
443 do {
444 QString full_url = generateFullUrl(Operations::ItemContents);
445 auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
446
447 if (!continuation.isEmpty()) {
448 full_url += QSL("&c=%1").arg(continuation);
449 }
450
451 std::list inp = boolinq::from(batch_ids).select([this](const QString& id) {
452 return QSL("i=%1").arg(m_service == GreaderServiceRoot::Service::TheOldReader
453 ? id
454 : QUrl::toPercentEncoding(id));
455 }).toStdList();
456 QByteArray input = FROM_STD_LIST(QStringList, inp).join(QSL("&")).toUtf8();
457 QByteArray output_stream;
458 auto result_stream = NetworkFactory::performNetworkOperation(full_url,
459 timeout,
460 input,
461 output_stream,
462 QNetworkAccessManager::Operation::PostOperation,
463 { authHeader() },
464 false,
465 {},
466 {},
467 proxy);
468
469 if (result_stream.first != QNetworkReply::NetworkError::NoError) {
470 qCriticalNN << LOGSEC_GREADER
471 << "Cannot download messages for "
472 << batch_ids
473 << ", network error:"
474 << QUOTE_W_SPACE_DOT(result_stream.first);
475 error = Feed::Status::NetworkError;
476 return {};
477 }
478 else {
479 msgs.append(decodeStreamContents(root, output_stream, QString(), continuation));
480 }
481 }
482 while (!continuation.isEmpty());
483 }
484
485 error = Feed::Status::Normal;
486 return msgs;
487 }
488
streamContents(ServiceRoot * root,const QString & stream_id,Feed::Status & error,const QNetworkProxy & proxy)489 QList<Message> GreaderNetwork::streamContents(ServiceRoot* root, const QString& stream_id,
490 Feed::Status& error, const QNetworkProxy& proxy) {
491 QString continuation;
492
493 if (!ensureLogin(proxy)) {
494 error = Feed::Status::AuthError;
495 return {};
496 }
497
498 QList<Message> msgs;
499 int target_msgs_size = batchSize() <= 0 ? 2000000: batchSize();
500
501 do {
502 QString full_url = generateFullUrl(Operations::StreamContents).arg(m_service == GreaderServiceRoot::Service::TheOldReader
503 ? stream_id
504 : QUrl::toPercentEncoding(stream_id),
505 QString::number(target_msgs_size));
506 auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
507
508 if (downloadOnlyUnreadMessages()) {
509 full_url += QSL("&xt=%1").arg(QSL(GREADER_API_FULL_STATE_READ));
510 }
511
512 if (!continuation.isEmpty()) {
513 full_url += QSL("&c=%1").arg(continuation);
514 }
515
516 if (m_newerThanFilter.isValid()) {
517 full_url += QSL("&ot=%1").arg(
518 #if QT_VERSION < 0x050E00 // Qt < 5.14.0
519 QDateTime(m_newerThanFilter)
520 #else
521 m_newerThanFilter.startOfDay()
522 #endif
523 .toSecsSinceEpoch());
524 }
525
526 QByteArray output_stream;
527 auto result_stream = NetworkFactory::performNetworkOperation(full_url,
528 timeout,
529 {},
530 output_stream,
531 QNetworkAccessManager::Operation::GetOperation,
532 { authHeader() },
533 false,
534 {},
535 {},
536 proxy);
537
538 if (result_stream.first != QNetworkReply::NetworkError::NoError) {
539 qCriticalNN << LOGSEC_GREADER
540 << "Cannot download messages for "
541 << QUOTE_NO_SPACE(stream_id)
542 << ", network error:"
543 << QUOTE_W_SPACE_DOT(result_stream.first);
544 error = Feed::Status::NetworkError;
545 return {};
546 }
547 else {
548 msgs.append(decodeStreamContents(root, output_stream, stream_id, continuation));
549 }
550 }
551 while (!continuation.isEmpty() && msgs.size() < target_msgs_size);
552
553 error = Feed::Status::Normal;
554 return msgs;
555 }
556
categoriesFeedsLabelsTree(bool obtain_icons,const QNetworkProxy & proxy)557 RootItem* GreaderNetwork::categoriesFeedsLabelsTree(bool obtain_icons, const QNetworkProxy& proxy) {
558 QString full_url = generateFullUrl(Operations::TagList);
559 auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
560
561 if (!ensureLogin(proxy)) {
562 return nullptr;
563 }
564
565 QByteArray output_labels;
566 auto result_labels = NetworkFactory::performNetworkOperation(full_url,
567 timeout,
568 {},
569 output_labels,
570 QNetworkAccessManager::Operation::GetOperation,
571 { authHeader() },
572 false,
573 {},
574 {},
575 proxy);
576
577 if (result_labels.first != QNetworkReply::NetworkError::NoError) {
578 return nullptr;
579 }
580
581 full_url = generateFullUrl(Operations::SubscriptionList);
582 QByteArray output_feeds;
583 auto result_feeds = NetworkFactory::performNetworkOperation(full_url,
584 timeout,
585 {},
586 output_feeds,
587 QNetworkAccessManager::Operation::GetOperation,
588 { authHeader() },
589 false,
590 {},
591 {},
592 proxy);
593
594 if (result_feeds.first != QNetworkReply::NetworkError::NoError) {
595 return nullptr;
596 }
597
598 return decodeTagsSubscriptions(output_labels, output_feeds, obtain_icons, proxy);
599 }
600
decodeTagsSubscriptions(const QString & categories,const QString & feeds,bool obtain_icons,const QNetworkProxy & proxy)601 RootItem* GreaderNetwork::decodeTagsSubscriptions(const QString& categories, const QString& feeds,
602 bool obtain_icons, const QNetworkProxy& proxy) {
603 auto* parent = new RootItem();
604 QMap<QString, RootItem*> cats;
605 QList<RootItem*> lbls;
606 QJsonArray json;
607
608 if (m_service == GreaderServiceRoot::Service::Bazqux ||
609 m_service == GreaderServiceRoot::Service::Reedah ||
610 m_service == GreaderServiceRoot::Service::Inoreader) {
611 // We need to process subscription list first and extract categories.
612 json = QJsonDocument::fromJson(feeds.toUtf8()).object()[QSL("subscriptions")].toArray();
613
614 for (const QJsonValue& feed : qAsConst(json)) {
615 auto subscription = feed.toObject();
616 auto json_cats = subscription[QSL("categories")].toArray();
617
618 for (const QJsonValue& cat : qAsConst(json_cats)) {
619 auto cat_obj = cat.toObject();
620 auto cat_id = cat_obj[QSL("id")].toString();
621
622 if (!cats.contains(cat_id)) {
623 auto* category = new Category();
624
625 category->setTitle(cat_id.mid(cat_id.lastIndexOf(QL1C('/')) + 1));
626 category->setCustomId(cat_id);
627
628 cats.insert(category->customId(), category);
629 parent->appendChild(category);
630 }
631 }
632 }
633 }
634
635 json = QJsonDocument::fromJson(categories.toUtf8()).object()[QSL("tags")].toArray();
636 cats.insert(QString(), parent);
637
638 for (const QJsonValue& obj : qAsConst(json)) {
639 auto label = obj.toObject();
640 QString label_id = label[QSL("id")].toString();
641
642 if ((label[QSL("type")].toString() == QSL("folder") ||
643 (m_service == GreaderServiceRoot::Service::TheOldReader &&
644 label_id.contains(QSL("/label/"))))) {
645
646 // We have category (not "state" or "tag" or "label").
647 auto* category = new Category();
648
649 category->setDescription(label[QSL("htmlUrl")].toString());
650 category->setTitle(label_id.mid(label_id.lastIndexOf(QL1C('/')) + 1));
651 category->setCustomId(label_id);
652
653 cats.insert(category->customId(), category);
654 parent->appendChild(category);
655 }
656 else if (label[QSL("type")] == QSL("tag")) {
657 QString plain_name = QRegularExpression(".+\\/([^\\/]+)").match(label_id).captured(1);
658 auto* new_lbl = new Label(plain_name, TextFactory::generateColorFromText(label_id));
659
660 new_lbl->setCustomId(label_id);
661 lbls.append(new_lbl);
662 }
663 else if ((m_service == GreaderServiceRoot::Service::Bazqux ||
664 m_service == GreaderServiceRoot::Service::Reedah ||
665 m_service == GreaderServiceRoot::Service::Inoreader) &&
666 label_id.contains(QSL("/label/"))) {
667 if (!cats.contains(label_id)) {
668 // This stream is not a category, it is label, bitches!
669 QString plain_name = QRegularExpression(QSL(".+\\/([^\\/]+)")).match(label_id).captured(1);
670 auto* new_lbl = new Label(plain_name, TextFactory::generateColorFromText(label_id));
671
672 new_lbl->setCustomId(label_id);
673 lbls.append(new_lbl);
674 }
675 }
676 }
677
678 json = QJsonDocument::fromJson(feeds.toUtf8()).object()[QSL("subscriptions")].toArray();
679
680 for (const QJsonValue& obj : qAsConst(json)) {
681 auto subscription = obj.toObject();
682 QString id = subscription[QSL("id")].toString();
683 QString title = subscription[QSL("title")].toString();
684 QString url = subscription[QSL("htmlUrl")].toString();
685 QString parent_label;
686 QJsonArray assigned_categories = subscription[QSL("categories")].toArray();
687
688 if (id.startsWith(TOR_SPONSORED_STREAM_ID)) {
689 continue;
690 }
691
692 for (const QJsonValue& cat : qAsConst(assigned_categories)) {
693 QString potential_id = cat.toObject()[QSL("id")].toString();
694
695 if (potential_id.contains(QSL("/label/"))) {
696 parent_label = potential_id;
697 break;
698 }
699 }
700
701 // We have label (not "state").
702 auto* feed = new Feed();
703
704 feed->setDescription(url);
705 feed->setSource(url);
706 feed->setTitle(title);
707 feed->setCustomId(id);
708
709 if (obtain_icons) {
710 QString icon_url = subscription[QSL("iconUrl")].toString();
711 QList<QPair<QString, bool>> icon_urls;
712
713 if (!icon_url.isEmpty()) {
714 if (icon_url.startsWith(QSL("//"))) {
715 icon_url = QUrl(baseUrl()).scheme() + QSL(":") + icon_url;
716 }
717 else if (service() == GreaderServiceRoot::Service::FreshRss) {
718 QUrl icon_url_obj(icon_url);
719 QUrl base_url(baseUrl());
720
721 if (icon_url_obj.host() == base_url.host()) {
722 icon_url_obj.setPort(base_url.port());
723 icon_url = icon_url_obj.toString();
724 }
725 }
726
727 icon_urls.append({ icon_url, true });
728 }
729
730 icon_urls.append({ url, false });
731
732 QIcon icon;
733
734 if (NetworkFactory::downloadIcon(icon_urls,
735 1000,
736 icon,
737 proxy) == QNetworkReply::NetworkError::NoError) {
738 feed->setIcon(icon);
739 }
740 }
741
742 if (cats.contains(parent_label)) {
743 cats[parent_label]->appendChild(feed);
744 }
745 }
746
747 auto* lblroot = new LabelsNode(parent);
748
749 lblroot->setChildItems(lbls);
750 parent->appendChild(lblroot);
751
752 return parent;
753 }
754
clientLogin(const QNetworkProxy & proxy)755 QNetworkReply::NetworkError GreaderNetwork::clientLogin(const QNetworkProxy& proxy) {
756 QString full_url = generateFullUrl(Operations::ClientLogin);
757 auto timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();
758 QByteArray output;
759 QByteArray args = QSL("Email=%1&Passwd=%2").arg(QString::fromLocal8Bit(QUrl::toPercentEncoding(username())),
760 QString::fromLocal8Bit(QUrl::toPercentEncoding(password()))).toLocal8Bit();
761 auto network_result = NetworkFactory::performNetworkOperation(full_url,
762 timeout,
763 args,
764 output,
765 QNetworkAccessManager::Operation::PostOperation,
766 { {
767 QSL(HTTP_HEADERS_CONTENT_TYPE).toLocal8Bit(),
768 QSL("application/x-www-form-urlencoded").toLocal8Bit()
769 } },
770 false,
771 {},
772 {},
773 proxy);
774
775 if (network_result.first == QNetworkReply::NetworkError::NoError) {
776 // Save credentials.
777 auto lines = QString::fromUtf8(output).replace(QSL("\r"), QString()).split('\n');
778
779 for (const QString& line : lines) {
780 int eq = line.indexOf('=');
781
782 if (eq > 0) {
783 QString id = line.mid(0, eq);
784
785 if (id == QSL("SID")) {
786 m_authSid = line.mid(eq + 1);
787 }
788 else if (id == QSL("Auth")) {
789 m_authAuth = line.mid(eq + 1);
790 }
791 }
792 }
793
794 QRegularExpression exp(QSL("^(NA|unused|none|null)$"));
795
796 if (exp.match(m_authSid).hasMatch()) {
797 m_authSid = QString();
798 }
799
800 if (exp.match(m_authAuth).hasMatch()) {
801 m_authAuth = QString();
802 }
803
804 if (m_authAuth.isEmpty()) {
805 clearCredentials();
806 return QNetworkReply::NetworkError::InternalServerError;
807 }
808
809 if (m_service == GreaderServiceRoot::Service::Reedah) {
810 // We need "T=" token for editing.
811 full_url = generateFullUrl(Operations::Token);
812
813 network_result = NetworkFactory::performNetworkOperation(full_url,
814 timeout,
815 args,
816 output,
817 QNetworkAccessManager::Operation::GetOperation,
818 { authHeader() },
819 false,
820 {},
821 {},
822 proxy);
823
824 if (network_result.first == QNetworkReply::NetworkError::NoError) {
825 m_authToken = output;
826 }
827 else {
828 clearCredentials();
829 }
830 }
831 }
832
833 return network_result.first;
834 }
835
service() const836 GreaderServiceRoot::Service GreaderNetwork::service() const {
837 return m_service;
838 }
839
setService(GreaderServiceRoot::Service service)840 void GreaderNetwork::setService(GreaderServiceRoot::Service service) {
841 m_service = service;
842 }
843
username() const844 QString GreaderNetwork::username() const {
845 return m_username;
846 }
847
setUsername(const QString & username)848 void GreaderNetwork::setUsername(const QString& username) {
849 m_username = username;
850 }
851
password() const852 QString GreaderNetwork::password() const {
853 return m_password;
854 }
855
setPassword(const QString & password)856 void GreaderNetwork::setPassword(const QString& password) {
857 m_password = password;
858 }
859
baseUrl() const860 QString GreaderNetwork::baseUrl() const {
861 return m_baseUrl;
862 }
863
setBaseUrl(const QString & base_url)864 void GreaderNetwork::setBaseUrl(const QString& base_url) {
865 m_baseUrl = base_url;
866 }
867
authHeader() const868 QPair<QByteArray, QByteArray> GreaderNetwork::authHeader() const {
869 if (m_service == GreaderServiceRoot::Service::Inoreader) {
870 return { QSL(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(),
871 m_oauth->bearer().toLocal8Bit() };
872 }
873 else {
874 return { QSL(HTTP_HEADERS_AUTHORIZATION).toLocal8Bit(),
875 QSL("GoogleLogin auth=%1").arg(m_authAuth).toLocal8Bit() };
876 }
877 }
878
ensureLogin(const QNetworkProxy & proxy,QNetworkReply::NetworkError * output)879 bool GreaderNetwork::ensureLogin(const QNetworkProxy& proxy, QNetworkReply::NetworkError* output) {
880 if (m_service == GreaderServiceRoot::Service::Inoreader) {
881 return !m_oauth->bearer().isEmpty();
882 }
883
884 if (m_authSid.isEmpty() && m_authAuth.isEmpty()) {
885 auto login = clientLogin(proxy);
886
887 if (output != nullptr) {
888 *output = login;
889 }
890
891 if (login != QNetworkReply::NetworkError::NoError) {
892 qCriticalNN << LOGSEC_GREADER
893 << "Login failed with error:"
894 << QUOTE_W_SPACE_DOT(NetworkFactory::networkErrorText(login));
895 return false;
896 }
897 else {
898 qDebugNN << LOGSEC_GREADER << "Login successful.";
899 }
900 }
901
902 return true;
903 }
904
convertLongStreamIdToShortStreamId(const QString & stream_id) const905 QString GreaderNetwork::convertLongStreamIdToShortStreamId(const QString& stream_id) const {
906 return QString::number(QString(stream_id).replace(QSL("tag:google.com,2005:reader/item/"),
907 QString()).toULongLong(nullptr, 16));
908 }
909
convertShortStreamIdToLongStreamId(const QString & stream_id) const910 QString GreaderNetwork::convertShortStreamIdToLongStreamId(const QString& stream_id) const {
911 if (stream_id.startsWith(QSL("tag:google.com,2005:reader/item/"))) {
912 return stream_id;
913 }
914
915 if (m_service == GreaderServiceRoot::Service::TheOldReader) {
916 return QSL("tag:google.com,2005:reader/item/%1").arg(stream_id);
917 }
918 else {
919 return QSL("tag:google.com,2005:reader/item/%1").arg(stream_id.toULongLong(),
920 16,
921 16,
922 QL1C('0'));
923 }
924 }
925
simplifyStreamId(const QString & stream_id) const926 QString GreaderNetwork::simplifyStreamId(const QString& stream_id) const {
927 return QString(stream_id).replace(QRegularExpression(QSL("\\/\\d+\\/")), QSL("/-/"));
928 }
929
decodeItemIds(const QString & stream_json_data,QString & continuation)930 QStringList GreaderNetwork::decodeItemIds(const QString& stream_json_data, QString& continuation) {
931 QStringList ids;
932 QJsonDocument json_doc = QJsonDocument::fromJson(stream_json_data.toUtf8());
933 QJsonArray json = json_doc.object()[QSL("itemRefs")].toArray();
934
935 continuation = json_doc.object()[QSL("continuation")].toString();
936 ids.reserve(json.count());
937
938 for (const QJsonValue& id : json) {
939 ids.append(id.toObject()[QSL("id")].toString());
940 }
941
942 return ids;
943 }
944
decodeStreamContents(ServiceRoot * root,const QString & stream_json_data,const QString & stream_id,QString & continuation)945 QList<Message> GreaderNetwork::decodeStreamContents(ServiceRoot* root,
946 const QString& stream_json_data,
947 const QString& stream_id,
948 QString& continuation) {
949 QList<Message> messages;
950 QJsonDocument json_doc = QJsonDocument::fromJson(stream_json_data.toUtf8());
951 QJsonArray json = json_doc.object()[QSL("items")].toArray();
952 auto active_labels = root->labelsNode() != nullptr ? root->labelsNode()->labels() : QList<Label*>();
953
954 continuation = json_doc.object()[QSL("continuation")].toString();
955 messages.reserve(json.count());
956
957 for (const QJsonValue& obj : json) {
958 auto message_obj = obj.toObject();
959 Message message;
960
961 message.m_title = qApp->web()->unescapeHtml(message_obj[QSL("title")].toString());
962 message.m_author = qApp->web()->unescapeHtml(message_obj[QSL("author")].toString());
963 message.m_created = QDateTime::fromSecsSinceEpoch(message_obj[QSL("published")].toInt(), Qt::UTC);
964 message.m_createdFromFeed = true;
965 message.m_customId = message_obj[QSL("id")].toString();
966
967 auto alternates = message_obj[QSL("alternate")].toArray();
968 auto enclosures = message_obj[QSL("enclosure")].toArray();
969 auto categories = message_obj[QSL("categories")].toArray();
970
971 for (const QJsonValue& alt : alternates) {
972 auto alt_obj = alt.toObject();
973 QString mime = alt_obj[QSL("type")].toString();
974 QString href = alt_obj[QSL("href")].toString();
975
976 if (mime.isEmpty() || mime == QL1S("text/html")) {
977 message.m_url = href;
978 }
979 else {
980 message.m_enclosures.append(Enclosure(href, mime));
981 }
982 }
983
984 for (const QJsonValue& enc : enclosures) {
985 auto enc_obj = enc.toObject();
986 QString mime = enc_obj[QSL("type")].toString();
987 QString href = enc_obj[QSL("href")].toString();
988
989 message.m_enclosures.append(Enclosure(href, mime));
990 }
991
992 for (const QJsonValue& cat : categories) {
993 QString category = cat.toString();
994
995 if (category.endsWith(QSL(GREADER_API_STATE_READ))) {
996 message.m_isRead = true;
997 }
998 else if (category.endsWith(QSL(GREADER_API_STATE_IMPORTANT))) {
999 message.m_isImportant = true;
1000 }
1001 else if (category.contains(QSL("label"))) {
1002 Label* label = boolinq::from(active_labels.begin(), active_labels.end()).firstOrDefault([category](Label* lbl) {
1003 return lbl->customId() == category;
1004 });
1005
1006 if (label != nullptr) {
1007 // We found live Label object for our assigned label.
1008 message.m_assignedLabels.append(label);
1009 }
1010 }
1011 }
1012
1013 message.m_contents = message_obj[QSL("summary")].toObject()[QSL("content")].toString();
1014 message.m_rawContents = QJsonDocument(message_obj).toJson(QJsonDocument::JsonFormat::Compact);
1015 message.m_feedId = stream_id.isEmpty()
1016 ? message_obj[QSL("origin")].toObject()[QSL("streamId")].toString()
1017 : stream_id;
1018
1019 if (message.m_title.isEmpty()) {
1020 message.m_title = message.m_url;
1021 }
1022
1023 messages.append(message);
1024 }
1025
1026 return messages;
1027 }
1028
batchSize() const1029 int GreaderNetwork::batchSize() const {
1030 return m_batchSize;
1031 }
1032
setBatchSize(int batch_size)1033 void GreaderNetwork::setBatchSize(int batch_size) {
1034 m_batchSize = batch_size;
1035 }
1036
clearCredentials()1037 void GreaderNetwork::clearCredentials() {
1038 m_authAuth = m_authSid = m_authToken = QString();
1039 }
1040
sanitizedBaseUrl() const1041 QString GreaderNetwork::sanitizedBaseUrl() const {
1042 QString base_url = m_service == GreaderServiceRoot::Service::Inoreader
1043 ? QSL(GREADER_URL_INOREADER)
1044 : m_baseUrl;
1045
1046 if (!base_url.endsWith('/')) {
1047 base_url = base_url + QL1C('/');
1048 }
1049
1050 switch (m_service) {
1051 case GreaderServiceRoot::Service::FreshRss:
1052 base_url += QSL(FRESHRSS_BASE_URL_PATH);
1053 break;
1054
1055 default:
1056 break;
1057 }
1058
1059 return base_url;
1060 }
1061
generateFullUrl(GreaderNetwork::Operations operation) const1062 QString GreaderNetwork::generateFullUrl(GreaderNetwork::Operations operation) const {
1063 switch (operation) {
1064 case Operations::ClientLogin:
1065 return sanitizedBaseUrl() + QSL(GREADER_API_CLIENT_LOGIN);
1066
1067 case Operations::Token:
1068 return sanitizedBaseUrl() + QSL(GREADER_API_TOKEN);
1069
1070 case Operations::TagList:
1071 return sanitizedBaseUrl() + QSL(GREADER_API_TAG_LIST);
1072
1073 case Operations::SubscriptionList:
1074 return sanitizedBaseUrl() + QSL(GREADER_API_SUBSCRIPTION_LIST);
1075
1076 case Operations::StreamContents:
1077 return sanitizedBaseUrl() + QSL(GREADER_API_STREAM_CONTENTS);
1078
1079 case Operations::UserInfo:
1080 return sanitizedBaseUrl() + QSL(GREADER_API_USER_INFO);
1081
1082 case Operations::EditTag:
1083 return sanitizedBaseUrl() + QSL(GREADER_API_EDIT_TAG);
1084
1085 case Operations::ItemIds:
1086 return sanitizedBaseUrl() + QSL(GREADER_API_ITEM_IDS);
1087
1088 case Operations::ItemContents:
1089 return sanitizedBaseUrl() + QSL(GREADER_API_ITEM_CONTENTS);
1090
1091 default:
1092 return sanitizedBaseUrl();
1093 }
1094 }
1095
onTokensError(const QString & error,const QString & error_description)1096 void GreaderNetwork::onTokensError(const QString& error, const QString& error_description) {
1097 Q_UNUSED(error)
1098
1099 qApp->showGuiMessage(Notification::Event::LoginFailure,
1100 tr("Inoreader: authentication error"),
1101 tr("Click this to login again. Error is: '%1'").arg(error_description),
1102 QSystemTrayIcon::MessageIcon::Critical,
1103 {}, {},
1104 tr("Login"),
1105 [this]() {
1106 m_oauth->setAccessToken(QString());
1107 m_oauth->setRefreshToken(QString());
1108 m_oauth->login();
1109 });
1110 }
1111
onAuthFailed()1112 void GreaderNetwork::onAuthFailed() {
1113 qApp->showGuiMessage(Notification::Event::LoginFailure,
1114 tr("Inoreader: authorization denied"),
1115 tr("Click this to login again."),
1116 QSystemTrayIcon::MessageIcon::Critical,
1117 {}, {},
1118 tr("Login"),
1119 [this]() {
1120 m_oauth->login();
1121 });
1122 }
1123
initializeOauth()1124 void GreaderNetwork::initializeOauth() {
1125 #if defined(INOREADER_OFFICIAL_SUPPORT)
1126 m_oauth->setClientSecretId(TextFactory::decrypt(QSL(INOREADER_CLIENT_ID), OAUTH_DECRYPTION_KEY));
1127 m_oauth->setClientSecretSecret(TextFactory::decrypt(QSL(INOREADER_CLIENT_SECRET), OAUTH_DECRYPTION_KEY));
1128 #endif
1129
1130 m_oauth->setRedirectUrl(QSL(OAUTH_REDIRECT_URI) +
1131 QL1C(':') +
1132 QString::number(INO_OAUTH_REDIRECT_URI_PORT),
1133 false);
1134
1135 connect(m_oauth, &OAuth2Service::tokensRetrieveError, this, &GreaderNetwork::onTokensError);
1136 connect(m_oauth, &OAuth2Service::authFailed, this, &GreaderNetwork::onAuthFailed);
1137 connect(m_oauth, &OAuth2Service::tokensRetrieved, this, [this](QString access_token, QString refresh_token, int expires_in) {
1138 Q_UNUSED(expires_in)
1139 Q_UNUSED(access_token)
1140
1141 if (m_root != nullptr && m_root->accountId() > 0 && !refresh_token.isEmpty()) {
1142 QSqlDatabase database = qApp->database()->driver()->connection(metaObject()->className());
1143
1144 DatabaseQueries::storeNewOauthTokens(database, refresh_token, m_root->accountId());
1145 }
1146 });
1147 }
1148
newerThanFilter() const1149 QDate GreaderNetwork::newerThanFilter() const {
1150 return m_newerThanFilter;
1151 }
1152
setNewerThanFilter(const QDate & newer_than)1153 void GreaderNetwork::setNewerThanFilter(const QDate& newer_than) {
1154 m_newerThanFilter = newer_than;
1155 }
1156
oauth() const1157 OAuth2Service* GreaderNetwork::oauth() const {
1158 return m_oauth;
1159 }
1160
setOauth(OAuth2Service * oauth)1161 void GreaderNetwork::setOauth(OAuth2Service* oauth) {
1162 m_oauth = oauth;
1163 }
1164
setRoot(GreaderServiceRoot * root)1165 void GreaderNetwork::setRoot(GreaderServiceRoot* root) {
1166 m_root = root;
1167 }
1168
intelligentSynchronization() const1169 bool GreaderNetwork::intelligentSynchronization() const {
1170 return m_intelligentSynchronization;
1171 }
1172
setIntelligentSynchronization(bool intelligent_synchronization)1173 void GreaderNetwork::setIntelligentSynchronization(bool intelligent_synchronization) {
1174 m_intelligentSynchronization = intelligent_synchronization;
1175 }
1176
downloadOnlyUnreadMessages() const1177 bool GreaderNetwork::downloadOnlyUnreadMessages() const {
1178 return m_downloadOnlyUnreadMessages;
1179 }
1180
setDownloadOnlyUnreadMessages(bool download_only_unread)1181 void GreaderNetwork::setDownloadOnlyUnreadMessages(bool download_only_unread) {
1182 m_downloadOnlyUnreadMessages = download_only_unread;
1183 }
1184