1 /*
2    SPDX-FileCopyrightText: 2016-2021 Laurent Montel <montel@kde.org>
3 
4    SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6 
7 #include "createphishingurldatabasejob.h"
8 #include "checkphishingurlutil.h"
9 #include "updatedatabaseinfo.h"
10 #include "webengineviewer_debug.h"
11 #include <PimCommon/NetworkManager>
12 
13 #include <QJsonDocument>
14 #include <QNetworkConfigurationManager>
15 #include <QUrlQuery>
16 
17 using namespace WebEngineViewer;
18 
19 WEBENGINEVIEWER_EXPORT bool webengineview_useCompactJson_CreatePhishingUrlDataBaseJob = true;
20 
21 class WebEngineViewer::CreatePhishingUrlDataBaseJobPrivate
22 {
23 public:
24     Q_REQUIRED_RESULT UpdateDataBaseInfo::CompressionType parseCompressionType(const QString &str);
25     Q_REQUIRED_RESULT RiceDeltaEncoding parseRiceDeltaEncoding(const QMap<QString, QVariant> &map);
26     Q_REQUIRED_RESULT QVector<Removal> parseRemovals(const QVariantList &lst);
27     Q_REQUIRED_RESULT QVector<Addition> parseAdditions(const QVariantList &lst);
28     QString mDataBaseState;
29     CreatePhishingUrlDataBaseJob::ContraintsCompressionType mContraintsCompressionType = CreatePhishingUrlDataBaseJob::RawCompression;
30     CreatePhishingUrlDataBaseJob::DataBaseDownloadType mDataBaseDownloadNeeded = CreatePhishingUrlDataBaseJob::FullDataBase;
31     QNetworkAccessManager *mNetworkAccessManager = nullptr;
32 };
33 
CreatePhishingUrlDataBaseJob(QObject * parent)34 CreatePhishingUrlDataBaseJob::CreatePhishingUrlDataBaseJob(QObject *parent)
35     : QObject(parent)
36     , d(new CreatePhishingUrlDataBaseJobPrivate)
37 {
38     d->mNetworkAccessManager = new QNetworkAccessManager(this);
39     d->mNetworkAccessManager->setRedirectPolicy(QNetworkRequest::NoLessSafeRedirectPolicy);
40     d->mNetworkAccessManager->setStrictTransportSecurityEnabled(true);
41     d->mNetworkAccessManager->enableStrictTransportSecurityStore(true);
42 
43     connect(d->mNetworkAccessManager, &QNetworkAccessManager::finished, this, &CreatePhishingUrlDataBaseJob::slotDownloadDataBaseFinished);
44     connect(d->mNetworkAccessManager, &QNetworkAccessManager::sslErrors, this, &CreatePhishingUrlDataBaseJob::slotSslErrors);
45 }
46 
47 CreatePhishingUrlDataBaseJob::~CreatePhishingUrlDataBaseJob() = default;
48 
slotSslErrors(QNetworkReply * reply,const QList<QSslError> & error)49 void CreatePhishingUrlDataBaseJob::slotSslErrors(QNetworkReply *reply, const QList<QSslError> &error)
50 {
51     qCDebug(WEBENGINEVIEWER_LOG) << " void CreatePhishingUrlDataBaseJob::slotSslErrors(QNetworkReply *reply, const QList<QSslError> &error)" << error.count();
52     reply->ignoreSslErrors(error);
53 }
54 
start()55 void CreatePhishingUrlDataBaseJob::start()
56 {
57     if (!PimCommon::NetworkManager::self()->networkConfigureManager()->isOnline()) {
58         Q_EMIT finished(UpdateDataBaseInfo(), BrokenNetwork);
59         deleteLater();
60     } else {
61         QUrlQuery query;
62         query.addQueryItem(QStringLiteral("key"), WebEngineViewer::CheckPhishingUrlUtil::apiKey());
63         QUrl safeUrl = QUrl(QStringLiteral("https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch"));
64         safeUrl.setQuery(query);
65         // qCDebug(WEBENGINEVIEWER_LOG) << " safeUrl" << safeUrl;
66         QNetworkRequest request(safeUrl);
67         request.setHeader(QNetworkRequest::ContentTypeHeader, QStringLiteral("application/json"));
68 
69         const QByteArray baPostData = jsonRequest();
70         Q_EMIT debugJson(baPostData);
71         qCDebug(WEBENGINEVIEWER_LOG) << " postData.toJson()" << baPostData;
72         // curl -H "Content-Type: application/json" -X POST -d
73         // '{"client":{"clientId":"KDE","clientVersion":"5.4.0"},"threatInfo":{"platformTypes":["WINDOWS"],"threatEntries":[{"url":"http://www.kde.org"}],"threatEntryTypes":["URL"],"threatTypes":["MALWARE"]}}'
74         // https://safebrowsing.googleapis.com/v4/threatMatches:find?key=AIzaSyBS62pXATjabbH2RM_jO2EzDg1mTMHlnyo
75         QNetworkReply *reply = d->mNetworkAccessManager->post(request, baPostData);
76         connect(reply, qOverload<QNetworkReply::NetworkError>(&QNetworkReply::errorOccurred), this, &CreatePhishingUrlDataBaseJob::slotError);
77     }
78 }
79 
setDataBaseState(const QString & value)80 void CreatePhishingUrlDataBaseJob::setDataBaseState(const QString &value)
81 {
82     d->mDataBaseState = value;
83 }
84 
slotError(QNetworkReply::NetworkError error)85 void CreatePhishingUrlDataBaseJob::slotError(QNetworkReply::NetworkError error)
86 {
87     auto reply = qobject_cast<QNetworkReply *>(sender());
88     qWarning() << " error " << error << " error string : " << reply->errorString();
89     reply->deleteLater();
90     deleteLater();
91 }
92 
jsonRequest() const93 QByteArray CreatePhishingUrlDataBaseJob::jsonRequest() const
94 {
95 #if 0
96     {
97         "client" : {
98             "clientId" :       "yourcompanyname",
99             "clientVersion" :  "1.5.2"
100         },
101         "listUpdateRequests" : [{
102                                     "threatType" :      "MALWARE",
103                                     "platformType" :    "WINDOWS",
104                                     "threatEntryType" : "URL",
105                                     "state" :           "Gg4IBBADIgYQgBAiAQEoAQ==",
106                                     "constraints" : {
107                                         "maxUpdateEntries" :      2048,
108                                         "maxDatabaseEntries" :    4096,
109                                         "region" :                "US",
110                                         "supportedCompressions" : ["RAW"]
111                                     }
112                                 }]
113     }
114 #endif
115     QVariantMap clientMap;
116     QVariantMap map;
117 
118     clientMap.insert(QStringLiteral("clientId"), QStringLiteral("KDE"));
119     clientMap.insert(QStringLiteral("clientVersion"), CheckPhishingUrlUtil::versionApps());
120     map.insert(QStringLiteral("client"), clientMap);
121 
122     QVariantList listUpdateRequests;
123 
124     QVariantMap threatMap;
125     threatMap.insert(QStringLiteral("platformType"), QStringLiteral("WINDOWS"));
126     threatMap.insert(QStringLiteral("threatType"), QStringLiteral("MALWARE"));
127     threatMap.insert(QStringLiteral("threatEntryType"), QStringLiteral("URL"));
128 
129     // Contrainsts
130     QVariantMap contraintsMap;
131     QVariantList contraintsCompressionList;
132     QString compressionStr;
133     switch (d->mContraintsCompressionType) {
134     case RiceCompression:
135         compressionStr = QStringLiteral("RICE");
136         break;
137     case RawCompression:
138         compressionStr = QStringLiteral("RAW");
139         break;
140     }
141     contraintsCompressionList.append(compressionStr);
142     contraintsMap.insert(QStringLiteral("supportedCompressions"), contraintsCompressionList);
143     threatMap.insert(QStringLiteral("constraints"), contraintsMap);
144 
145     // Define state when we want to define update database. Empty is full.
146     switch (d->mDataBaseDownloadNeeded) {
147     case FullDataBase:
148         qCDebug(WEBENGINEVIEWER_LOG) << " full update";
149         threatMap.insert(QStringLiteral("state"), QString());
150         break;
151     case UpdateDataBase:
152         qCDebug(WEBENGINEVIEWER_LOG) << " update database";
153         if (d->mDataBaseState.isEmpty()) {
154             qCWarning(WEBENGINEVIEWER_LOG) << "Partial Download asked but database set is empty";
155         }
156         threatMap.insert(QStringLiteral("state"), d->mDataBaseState);
157         break;
158     }
159 
160     listUpdateRequests.append(threatMap);
161 
162     map.insert(QStringLiteral("listUpdateRequests"), listUpdateRequests);
163 
164     const QJsonDocument postData = QJsonDocument::fromVariant(map);
165     const QByteArray baPostData = postData.toJson(webengineview_useCompactJson_CreatePhishingUrlDataBaseJob ? QJsonDocument::Compact : QJsonDocument::Indented);
166     return baPostData;
167 }
168 
setDataBaseDownloadNeeded(CreatePhishingUrlDataBaseJob::DataBaseDownloadType type)169 void CreatePhishingUrlDataBaseJob::setDataBaseDownloadNeeded(CreatePhishingUrlDataBaseJob::DataBaseDownloadType type)
170 {
171     d->mDataBaseDownloadNeeded = type;
172 }
173 
slotDownloadDataBaseFinished(QNetworkReply * reply)174 void CreatePhishingUrlDataBaseJob::slotDownloadDataBaseFinished(QNetworkReply *reply)
175 {
176     const QByteArray returnValue(reply->readAll());
177     Q_EMIT debugJsonResult(returnValue);
178     parseResult(returnValue);
179     reply->deleteLater();
180 }
181 
parseRiceDeltaEncoding(const QMap<QString,QVariant> & map)182 RiceDeltaEncoding CreatePhishingUrlDataBaseJobPrivate::parseRiceDeltaEncoding(const QMap<QString, QVariant> &map)
183 {
184     RiceDeltaEncoding riceDeltaEncodingTmp;
185     QMap<QString, QVariant>::const_iterator riceHashesIt = map.cbegin();
186     const QMap<QString, QVariant>::const_iterator riceHashesItEnd = map.cend();
187     for (; riceHashesIt != riceHashesItEnd; ++riceHashesIt) {
188         const QString key = riceHashesIt.key();
189         if (key == QLatin1String("firstValue")) {
190             riceDeltaEncodingTmp.firstValue = riceHashesIt.value().toByteArray();
191         } else if (key == QLatin1String("riceParameter")) {
192             riceDeltaEncodingTmp.riceParameter = riceHashesIt.value().toInt();
193         } else if (key == QLatin1String("numEntries")) {
194             riceDeltaEncodingTmp.numberEntries = riceHashesIt.value().toInt();
195         } else if (key == QLatin1String("encodedData")) {
196             riceDeltaEncodingTmp.encodingData = riceHashesIt.value().toByteArray();
197         } else {
198             qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRiceDeltaEncoding unknown riceDeltaEncoding key " << key;
199         }
200     }
201     return riceDeltaEncodingTmp;
202 }
203 
parseAdditions(const QVariantList & lst)204 QVector<Addition> CreatePhishingUrlDataBaseJobPrivate::parseAdditions(const QVariantList &lst)
205 {
206     QVector<Addition> additionList;
207     for (const QVariant &v : lst) {
208         if (v.canConvert<QVariantMap>()) {
209             QMapIterator<QString, QVariant> mapIt(v.toMap());
210             Addition tmp;
211             while (mapIt.hasNext()) {
212                 mapIt.next();
213                 const QString keyStr = mapIt.key();
214                 if (keyStr == QLatin1String("compressionType")) {
215                     tmp.compressionType = parseCompressionType(mapIt.value().toString());
216                 } else if (keyStr == QLatin1String("riceHashes")) {
217                     RiceDeltaEncoding riceDeltaEncodingTmp = parseRiceDeltaEncoding(mapIt.value().toMap());
218                     if (riceDeltaEncodingTmp.isValid()) {
219                         tmp.riceDeltaEncoding = riceDeltaEncodingTmp;
220                     }
221                 } else if (keyStr == QLatin1String("rawHashes")) {
222                     QMapIterator<QString, QVariant> rawHashesIt(mapIt.value().toMap());
223                     while (rawHashesIt.hasNext()) {
224                         rawHashesIt.next();
225                         const QString key = rawHashesIt.key();
226                         if (key == QLatin1String("rawHashes")) {
227                             tmp.hashString = QByteArray::fromBase64(rawHashesIt.value().toByteArray());
228                         } else if (key == QLatin1String("prefixSize")) {
229                             tmp.prefixSize = rawHashesIt.value().toInt();
230                         } else {
231                             qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions unknown rawHashes key " << key;
232                         }
233                     }
234                 } else {
235                     qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions unknown mapIt.key() " << keyStr;
236                 }
237             }
238             if (tmp.isValid()) {
239                 additionList.append(tmp);
240             }
241         } else {
242             qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions not parsing type " << v.typeName();
243         }
244     }
245     return additionList;
246 }
247 
parseCompressionType(const QString & str)248 UpdateDataBaseInfo::CompressionType CreatePhishingUrlDataBaseJobPrivate::parseCompressionType(const QString &str)
249 {
250     UpdateDataBaseInfo::CompressionType type(UpdateDataBaseInfo::UnknownCompression);
251     if (str == QLatin1String("COMPRESSION_TYPE_UNSPECIFIED")) {
252         type = UpdateDataBaseInfo::UnknownCompression;
253     } else if (str == QLatin1String("RICE")) {
254         type = UpdateDataBaseInfo::RiceCompression;
255     } else if (str == QLatin1String("RAW")) {
256         type = UpdateDataBaseInfo::RawCompression;
257     } else {
258         qCWarning(WEBENGINEVIEWER_LOG) << "CreatePhishingUrlDataBaseJob::parseCompressionType unknown compression type " << str;
259     }
260     return type;
261 }
262 
parseRemovals(const QVariantList & lst)263 QVector<Removal> CreatePhishingUrlDataBaseJobPrivate::parseRemovals(const QVariantList &lst)
264 {
265     QVector<Removal> removalList;
266     for (const QVariant &v : lst) {
267         if (v.canConvert<QVariantMap>()) {
268             Removal tmp;
269             QMapIterator<QString, QVariant> mapIt(v.toMap());
270             while (mapIt.hasNext()) {
271                 mapIt.next();
272                 const QString keyStr = mapIt.key();
273                 if (keyStr == QLatin1String("compressionType")) {
274                     tmp.compressionType = parseCompressionType(mapIt.value().toString());
275                 } else if (keyStr == QLatin1String("riceIndices")) {
276                     RiceDeltaEncoding riceDeltaEncodingTmp = parseRiceDeltaEncoding(mapIt.value().toMap());
277                     if (riceDeltaEncodingTmp.isValid()) {
278                         tmp.riceDeltaEncoding = riceDeltaEncodingTmp;
279                     }
280                 } else if (keyStr == QLatin1String("rawIndices")) {
281                     const QVariantMap map = mapIt.value().toMap();
282                     QMapIterator<QString, QVariant> rawIndicesIt(map);
283                     while (rawIndicesIt.hasNext()) {
284                         rawIndicesIt.next();
285                         if (rawIndicesIt.key() == QLatin1String("indices")) {
286                             const QVariantList rawList = rawIndicesIt.value().toList();
287                             QList<quint32> indexList;
288                             indexList.reserve(rawList.count());
289                             for (const QVariant &var : rawList) {
290                                 indexList.append(var.toUInt());
291                             }
292                             tmp.indexes = indexList;
293                         } else {
294                             qCDebug(WEBENGINEVIEWER_LOG) << "rawIndicesIt.key() unknown " << rawIndicesIt.key();
295                         }
296                     }
297                 } else {
298                     qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRemovals unknown mapIt.key() " << keyStr;
299                 }
300             }
301             if (tmp.isValid()) {
302                 removalList.append(tmp);
303             }
304         } else {
305             qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRemovals not parsing type " << v.typeName();
306         }
307     }
308     return removalList;
309 }
310 
parseResult(const QByteArray & value)311 void CreatePhishingUrlDataBaseJob::parseResult(const QByteArray &value)
312 {
313     UpdateDataBaseInfo databaseInfo;
314     QJsonDocument document = QJsonDocument::fromJson(value);
315     if (document.isNull()) {
316         Q_EMIT finished(databaseInfo, InvalidData);
317     } else {
318         const QVariantMap answer = document.toVariant().toMap();
319         if (answer.isEmpty()) {
320             Q_EMIT finished(databaseInfo, InvalidData);
321         } else {
322             QMapIterator<QString, QVariant> i(answer);
323             while (i.hasNext()) {
324                 i.next();
325                 if (i.key() == QLatin1String("listUpdateResponses")) {
326                     const QVariantList info = i.value().toList();
327                     if (info.count() == 1) {
328                         const QVariant infoVar = info.at(0);
329                         if (infoVar.canConvert<QVariantMap>()) {
330                             QMapIterator<QString, QVariant> mapIt(infoVar.toMap());
331                             while (mapIt.hasNext()) {
332                                 mapIt.next();
333                                 const QString mapKey = mapIt.key();
334                                 if (mapKey == QLatin1String("additions")) {
335                                     const QVariantList lst = mapIt.value().toList();
336                                     const QVector<Addition> addList = d->parseAdditions(lst);
337                                     if (!addList.isEmpty()) {
338                                         databaseInfo.additionList.append(addList);
339                                     }
340                                 } else if (mapKey == QLatin1String("removals")) {
341                                     const QVariantList lst = mapIt.value().toList();
342                                     const QVector<Removal> removeList = d->parseRemovals(lst);
343                                     if (!removeList.isEmpty()) {
344                                         databaseInfo.removalList.append(removeList);
345                                     }
346                                 } else if (mapKey == QLatin1String("checksum")) {
347                                     QMapIterator<QString, QVariant> mapCheckSum(mapIt.value().toMap());
348                                     while (mapCheckSum.hasNext()) {
349                                         mapCheckSum.next();
350                                         if (mapCheckSum.key() == QLatin1String("sha256")) {
351                                             databaseInfo.sha256 = mapCheckSum.value().toByteArray();
352                                         } else {
353                                             qCDebug(WEBENGINEVIEWER_LOG) << "Invalid checksum key" << mapCheckSum.key();
354                                         }
355                                     }
356                                 } else if (mapKey == QLatin1String("newClientState")) {
357                                     databaseInfo.newClientState = mapIt.value().toString();
358                                 } else if (mapKey == QLatin1String("platformType")) {
359                                     databaseInfo.platformType = mapIt.value().toString();
360                                 } else if (mapKey == QLatin1String("responseType")) {
361                                     const QString str = mapIt.value().toString();
362                                     if (str == QLatin1String("FULL_UPDATE")) {
363                                         databaseInfo.responseType = UpdateDataBaseInfo::FullUpdate;
364                                     } else if (str == QLatin1String("PARTIAL_UPDATE")) {
365                                         databaseInfo.responseType = UpdateDataBaseInfo::PartialUpdate;
366                                     } else {
367                                         qCDebug(WEBENGINEVIEWER_LOG) << " unknown responsetype " << str;
368                                         databaseInfo.responseType = UpdateDataBaseInfo::Unknown;
369                                     }
370                                 } else if (mapKey == QLatin1String("threatEntryType")) {
371                                     databaseInfo.threatEntryType = mapIt.value().toString();
372                                 } else if (mapKey == QLatin1String("threatType")) {
373                                     databaseInfo.threatType = mapIt.value().toString();
374                                 } else {
375                                     qCDebug(WEBENGINEVIEWER_LOG) << " unknown key " << mapKey;
376                                 }
377                             }
378                         }
379                     }
380                 } else if (i.key() == QLatin1String("minimumWaitDuration")) {
381                     databaseInfo.minimumWaitDuration = i.value().toString();
382                 } else {
383                     qCDebug(WEBENGINEVIEWER_LOG) << " map key unknown " << i.key();
384                 }
385             }
386             Q_EMIT finished(databaseInfo, ValidData);
387         }
388     }
389     deleteLater();
390 }
391 
setContraintsCompressionType(CreatePhishingUrlDataBaseJob::ContraintsCompressionType type)392 void CreatePhishingUrlDataBaseJob::setContraintsCompressionType(CreatePhishingUrlDataBaseJob::ContraintsCompressionType type)
393 {
394     d->mContraintsCompressionType = type;
395 }
396