1 /*
2 SPDX-FileCopyrightText: 2016-2021 Laurent Montel <montel@kde.org>
3
4 SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6
7 #include "createphishingurldatabasejob.h"
8 #include "checkphishingurlutil.h"
9 #include "updatedatabaseinfo.h"
10 #include "webengineviewer_debug.h"
11 #include <PimCommon/NetworkManager>
12
13 #include <QJsonDocument>
14 #include <QNetworkConfigurationManager>
15 #include <QUrlQuery>
16
17 using namespace WebEngineViewer;
18
19 WEBENGINEVIEWER_EXPORT bool webengineview_useCompactJson_CreatePhishingUrlDataBaseJob = true;
20
21 class WebEngineViewer::CreatePhishingUrlDataBaseJobPrivate
22 {
23 public:
24 Q_REQUIRED_RESULT UpdateDataBaseInfo::CompressionType parseCompressionType(const QString &str);
25 Q_REQUIRED_RESULT RiceDeltaEncoding parseRiceDeltaEncoding(const QMap<QString, QVariant> &map);
26 Q_REQUIRED_RESULT QVector<Removal> parseRemovals(const QVariantList &lst);
27 Q_REQUIRED_RESULT QVector<Addition> parseAdditions(const QVariantList &lst);
28 QString mDataBaseState;
29 CreatePhishingUrlDataBaseJob::ContraintsCompressionType mContraintsCompressionType = CreatePhishingUrlDataBaseJob::RawCompression;
30 CreatePhishingUrlDataBaseJob::DataBaseDownloadType mDataBaseDownloadNeeded = CreatePhishingUrlDataBaseJob::FullDataBase;
31 QNetworkAccessManager *mNetworkAccessManager = nullptr;
32 };
33
CreatePhishingUrlDataBaseJob(QObject * parent)34 CreatePhishingUrlDataBaseJob::CreatePhishingUrlDataBaseJob(QObject *parent)
35 : QObject(parent)
36 , d(new CreatePhishingUrlDataBaseJobPrivate)
37 {
38 d->mNetworkAccessManager = new QNetworkAccessManager(this);
39 d->mNetworkAccessManager->setRedirectPolicy(QNetworkRequest::NoLessSafeRedirectPolicy);
40 d->mNetworkAccessManager->setStrictTransportSecurityEnabled(true);
41 d->mNetworkAccessManager->enableStrictTransportSecurityStore(true);
42
43 connect(d->mNetworkAccessManager, &QNetworkAccessManager::finished, this, &CreatePhishingUrlDataBaseJob::slotDownloadDataBaseFinished);
44 connect(d->mNetworkAccessManager, &QNetworkAccessManager::sslErrors, this, &CreatePhishingUrlDataBaseJob::slotSslErrors);
45 }
46
47 CreatePhishingUrlDataBaseJob::~CreatePhishingUrlDataBaseJob() = default;
48
slotSslErrors(QNetworkReply * reply,const QList<QSslError> & error)49 void CreatePhishingUrlDataBaseJob::slotSslErrors(QNetworkReply *reply, const QList<QSslError> &error)
50 {
51 qCDebug(WEBENGINEVIEWER_LOG) << " void CreatePhishingUrlDataBaseJob::slotSslErrors(QNetworkReply *reply, const QList<QSslError> &error)" << error.count();
52 reply->ignoreSslErrors(error);
53 }
54
start()55 void CreatePhishingUrlDataBaseJob::start()
56 {
57 if (!PimCommon::NetworkManager::self()->networkConfigureManager()->isOnline()) {
58 Q_EMIT finished(UpdateDataBaseInfo(), BrokenNetwork);
59 deleteLater();
60 } else {
61 QUrlQuery query;
62 query.addQueryItem(QStringLiteral("key"), WebEngineViewer::CheckPhishingUrlUtil::apiKey());
63 QUrl safeUrl = QUrl(QStringLiteral("https://safebrowsing.googleapis.com/v4/threatListUpdates:fetch"));
64 safeUrl.setQuery(query);
65 // qCDebug(WEBENGINEVIEWER_LOG) << " safeUrl" << safeUrl;
66 QNetworkRequest request(safeUrl);
67 request.setHeader(QNetworkRequest::ContentTypeHeader, QStringLiteral("application/json"));
68
69 const QByteArray baPostData = jsonRequest();
70 Q_EMIT debugJson(baPostData);
71 qCDebug(WEBENGINEVIEWER_LOG) << " postData.toJson()" << baPostData;
72 // curl -H "Content-Type: application/json" -X POST -d
73 // '{"client":{"clientId":"KDE","clientVersion":"5.4.0"},"threatInfo":{"platformTypes":["WINDOWS"],"threatEntries":[{"url":"http://www.kde.org"}],"threatEntryTypes":["URL"],"threatTypes":["MALWARE"]}}'
74 // https://safebrowsing.googleapis.com/v4/threatMatches:find?key=AIzaSyBS62pXATjabbH2RM_jO2EzDg1mTMHlnyo
75 QNetworkReply *reply = d->mNetworkAccessManager->post(request, baPostData);
76 connect(reply, qOverload<QNetworkReply::NetworkError>(&QNetworkReply::errorOccurred), this, &CreatePhishingUrlDataBaseJob::slotError);
77 }
78 }
79
setDataBaseState(const QString & value)80 void CreatePhishingUrlDataBaseJob::setDataBaseState(const QString &value)
81 {
82 d->mDataBaseState = value;
83 }
84
slotError(QNetworkReply::NetworkError error)85 void CreatePhishingUrlDataBaseJob::slotError(QNetworkReply::NetworkError error)
86 {
87 auto reply = qobject_cast<QNetworkReply *>(sender());
88 qWarning() << " error " << error << " error string : " << reply->errorString();
89 reply->deleteLater();
90 deleteLater();
91 }
92
jsonRequest() const93 QByteArray CreatePhishingUrlDataBaseJob::jsonRequest() const
94 {
95 #if 0
96 {
97 "client" : {
98 "clientId" : "yourcompanyname",
99 "clientVersion" : "1.5.2"
100 },
101 "listUpdateRequests" : [{
102 "threatType" : "MALWARE",
103 "platformType" : "WINDOWS",
104 "threatEntryType" : "URL",
105 "state" : "Gg4IBBADIgYQgBAiAQEoAQ==",
106 "constraints" : {
107 "maxUpdateEntries" : 2048,
108 "maxDatabaseEntries" : 4096,
109 "region" : "US",
110 "supportedCompressions" : ["RAW"]
111 }
112 }]
113 }
114 #endif
115 QVariantMap clientMap;
116 QVariantMap map;
117
118 clientMap.insert(QStringLiteral("clientId"), QStringLiteral("KDE"));
119 clientMap.insert(QStringLiteral("clientVersion"), CheckPhishingUrlUtil::versionApps());
120 map.insert(QStringLiteral("client"), clientMap);
121
122 QVariantList listUpdateRequests;
123
124 QVariantMap threatMap;
125 threatMap.insert(QStringLiteral("platformType"), QStringLiteral("WINDOWS"));
126 threatMap.insert(QStringLiteral("threatType"), QStringLiteral("MALWARE"));
127 threatMap.insert(QStringLiteral("threatEntryType"), QStringLiteral("URL"));
128
129 // Contrainsts
130 QVariantMap contraintsMap;
131 QVariantList contraintsCompressionList;
132 QString compressionStr;
133 switch (d->mContraintsCompressionType) {
134 case RiceCompression:
135 compressionStr = QStringLiteral("RICE");
136 break;
137 case RawCompression:
138 compressionStr = QStringLiteral("RAW");
139 break;
140 }
141 contraintsCompressionList.append(compressionStr);
142 contraintsMap.insert(QStringLiteral("supportedCompressions"), contraintsCompressionList);
143 threatMap.insert(QStringLiteral("constraints"), contraintsMap);
144
145 // Define state when we want to define update database. Empty is full.
146 switch (d->mDataBaseDownloadNeeded) {
147 case FullDataBase:
148 qCDebug(WEBENGINEVIEWER_LOG) << " full update";
149 threatMap.insert(QStringLiteral("state"), QString());
150 break;
151 case UpdateDataBase:
152 qCDebug(WEBENGINEVIEWER_LOG) << " update database";
153 if (d->mDataBaseState.isEmpty()) {
154 qCWarning(WEBENGINEVIEWER_LOG) << "Partial Download asked but database set is empty";
155 }
156 threatMap.insert(QStringLiteral("state"), d->mDataBaseState);
157 break;
158 }
159
160 listUpdateRequests.append(threatMap);
161
162 map.insert(QStringLiteral("listUpdateRequests"), listUpdateRequests);
163
164 const QJsonDocument postData = QJsonDocument::fromVariant(map);
165 const QByteArray baPostData = postData.toJson(webengineview_useCompactJson_CreatePhishingUrlDataBaseJob ? QJsonDocument::Compact : QJsonDocument::Indented);
166 return baPostData;
167 }
168
setDataBaseDownloadNeeded(CreatePhishingUrlDataBaseJob::DataBaseDownloadType type)169 void CreatePhishingUrlDataBaseJob::setDataBaseDownloadNeeded(CreatePhishingUrlDataBaseJob::DataBaseDownloadType type)
170 {
171 d->mDataBaseDownloadNeeded = type;
172 }
173
slotDownloadDataBaseFinished(QNetworkReply * reply)174 void CreatePhishingUrlDataBaseJob::slotDownloadDataBaseFinished(QNetworkReply *reply)
175 {
176 const QByteArray returnValue(reply->readAll());
177 Q_EMIT debugJsonResult(returnValue);
178 parseResult(returnValue);
179 reply->deleteLater();
180 }
181
parseRiceDeltaEncoding(const QMap<QString,QVariant> & map)182 RiceDeltaEncoding CreatePhishingUrlDataBaseJobPrivate::parseRiceDeltaEncoding(const QMap<QString, QVariant> &map)
183 {
184 RiceDeltaEncoding riceDeltaEncodingTmp;
185 QMap<QString, QVariant>::const_iterator riceHashesIt = map.cbegin();
186 const QMap<QString, QVariant>::const_iterator riceHashesItEnd = map.cend();
187 for (; riceHashesIt != riceHashesItEnd; ++riceHashesIt) {
188 const QString key = riceHashesIt.key();
189 if (key == QLatin1String("firstValue")) {
190 riceDeltaEncodingTmp.firstValue = riceHashesIt.value().toByteArray();
191 } else if (key == QLatin1String("riceParameter")) {
192 riceDeltaEncodingTmp.riceParameter = riceHashesIt.value().toInt();
193 } else if (key == QLatin1String("numEntries")) {
194 riceDeltaEncodingTmp.numberEntries = riceHashesIt.value().toInt();
195 } else if (key == QLatin1String("encodedData")) {
196 riceDeltaEncodingTmp.encodingData = riceHashesIt.value().toByteArray();
197 } else {
198 qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRiceDeltaEncoding unknown riceDeltaEncoding key " << key;
199 }
200 }
201 return riceDeltaEncodingTmp;
202 }
203
parseAdditions(const QVariantList & lst)204 QVector<Addition> CreatePhishingUrlDataBaseJobPrivate::parseAdditions(const QVariantList &lst)
205 {
206 QVector<Addition> additionList;
207 for (const QVariant &v : lst) {
208 if (v.canConvert<QVariantMap>()) {
209 QMapIterator<QString, QVariant> mapIt(v.toMap());
210 Addition tmp;
211 while (mapIt.hasNext()) {
212 mapIt.next();
213 const QString keyStr = mapIt.key();
214 if (keyStr == QLatin1String("compressionType")) {
215 tmp.compressionType = parseCompressionType(mapIt.value().toString());
216 } else if (keyStr == QLatin1String("riceHashes")) {
217 RiceDeltaEncoding riceDeltaEncodingTmp = parseRiceDeltaEncoding(mapIt.value().toMap());
218 if (riceDeltaEncodingTmp.isValid()) {
219 tmp.riceDeltaEncoding = riceDeltaEncodingTmp;
220 }
221 } else if (keyStr == QLatin1String("rawHashes")) {
222 QMapIterator<QString, QVariant> rawHashesIt(mapIt.value().toMap());
223 while (rawHashesIt.hasNext()) {
224 rawHashesIt.next();
225 const QString key = rawHashesIt.key();
226 if (key == QLatin1String("rawHashes")) {
227 tmp.hashString = QByteArray::fromBase64(rawHashesIt.value().toByteArray());
228 } else if (key == QLatin1String("prefixSize")) {
229 tmp.prefixSize = rawHashesIt.value().toInt();
230 } else {
231 qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions unknown rawHashes key " << key;
232 }
233 }
234 } else {
235 qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions unknown mapIt.key() " << keyStr;
236 }
237 }
238 if (tmp.isValid()) {
239 additionList.append(tmp);
240 }
241 } else {
242 qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseAdditions not parsing type " << v.typeName();
243 }
244 }
245 return additionList;
246 }
247
parseCompressionType(const QString & str)248 UpdateDataBaseInfo::CompressionType CreatePhishingUrlDataBaseJobPrivate::parseCompressionType(const QString &str)
249 {
250 UpdateDataBaseInfo::CompressionType type(UpdateDataBaseInfo::UnknownCompression);
251 if (str == QLatin1String("COMPRESSION_TYPE_UNSPECIFIED")) {
252 type = UpdateDataBaseInfo::UnknownCompression;
253 } else if (str == QLatin1String("RICE")) {
254 type = UpdateDataBaseInfo::RiceCompression;
255 } else if (str == QLatin1String("RAW")) {
256 type = UpdateDataBaseInfo::RawCompression;
257 } else {
258 qCWarning(WEBENGINEVIEWER_LOG) << "CreatePhishingUrlDataBaseJob::parseCompressionType unknown compression type " << str;
259 }
260 return type;
261 }
262
parseRemovals(const QVariantList & lst)263 QVector<Removal> CreatePhishingUrlDataBaseJobPrivate::parseRemovals(const QVariantList &lst)
264 {
265 QVector<Removal> removalList;
266 for (const QVariant &v : lst) {
267 if (v.canConvert<QVariantMap>()) {
268 Removal tmp;
269 QMapIterator<QString, QVariant> mapIt(v.toMap());
270 while (mapIt.hasNext()) {
271 mapIt.next();
272 const QString keyStr = mapIt.key();
273 if (keyStr == QLatin1String("compressionType")) {
274 tmp.compressionType = parseCompressionType(mapIt.value().toString());
275 } else if (keyStr == QLatin1String("riceIndices")) {
276 RiceDeltaEncoding riceDeltaEncodingTmp = parseRiceDeltaEncoding(mapIt.value().toMap());
277 if (riceDeltaEncodingTmp.isValid()) {
278 tmp.riceDeltaEncoding = riceDeltaEncodingTmp;
279 }
280 } else if (keyStr == QLatin1String("rawIndices")) {
281 const QVariantMap map = mapIt.value().toMap();
282 QMapIterator<QString, QVariant> rawIndicesIt(map);
283 while (rawIndicesIt.hasNext()) {
284 rawIndicesIt.next();
285 if (rawIndicesIt.key() == QLatin1String("indices")) {
286 const QVariantList rawList = rawIndicesIt.value().toList();
287 QList<quint32> indexList;
288 indexList.reserve(rawList.count());
289 for (const QVariant &var : rawList) {
290 indexList.append(var.toUInt());
291 }
292 tmp.indexes = indexList;
293 } else {
294 qCDebug(WEBENGINEVIEWER_LOG) << "rawIndicesIt.key() unknown " << rawIndicesIt.key();
295 }
296 }
297 } else {
298 qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRemovals unknown mapIt.key() " << keyStr;
299 }
300 }
301 if (tmp.isValid()) {
302 removalList.append(tmp);
303 }
304 } else {
305 qCDebug(WEBENGINEVIEWER_LOG) << " CreatePhishingUrlDataBaseJob::parseRemovals not parsing type " << v.typeName();
306 }
307 }
308 return removalList;
309 }
310
parseResult(const QByteArray & value)311 void CreatePhishingUrlDataBaseJob::parseResult(const QByteArray &value)
312 {
313 UpdateDataBaseInfo databaseInfo;
314 QJsonDocument document = QJsonDocument::fromJson(value);
315 if (document.isNull()) {
316 Q_EMIT finished(databaseInfo, InvalidData);
317 } else {
318 const QVariantMap answer = document.toVariant().toMap();
319 if (answer.isEmpty()) {
320 Q_EMIT finished(databaseInfo, InvalidData);
321 } else {
322 QMapIterator<QString, QVariant> i(answer);
323 while (i.hasNext()) {
324 i.next();
325 if (i.key() == QLatin1String("listUpdateResponses")) {
326 const QVariantList info = i.value().toList();
327 if (info.count() == 1) {
328 const QVariant infoVar = info.at(0);
329 if (infoVar.canConvert<QVariantMap>()) {
330 QMapIterator<QString, QVariant> mapIt(infoVar.toMap());
331 while (mapIt.hasNext()) {
332 mapIt.next();
333 const QString mapKey = mapIt.key();
334 if (mapKey == QLatin1String("additions")) {
335 const QVariantList lst = mapIt.value().toList();
336 const QVector<Addition> addList = d->parseAdditions(lst);
337 if (!addList.isEmpty()) {
338 databaseInfo.additionList.append(addList);
339 }
340 } else if (mapKey == QLatin1String("removals")) {
341 const QVariantList lst = mapIt.value().toList();
342 const QVector<Removal> removeList = d->parseRemovals(lst);
343 if (!removeList.isEmpty()) {
344 databaseInfo.removalList.append(removeList);
345 }
346 } else if (mapKey == QLatin1String("checksum")) {
347 QMapIterator<QString, QVariant> mapCheckSum(mapIt.value().toMap());
348 while (mapCheckSum.hasNext()) {
349 mapCheckSum.next();
350 if (mapCheckSum.key() == QLatin1String("sha256")) {
351 databaseInfo.sha256 = mapCheckSum.value().toByteArray();
352 } else {
353 qCDebug(WEBENGINEVIEWER_LOG) << "Invalid checksum key" << mapCheckSum.key();
354 }
355 }
356 } else if (mapKey == QLatin1String("newClientState")) {
357 databaseInfo.newClientState = mapIt.value().toString();
358 } else if (mapKey == QLatin1String("platformType")) {
359 databaseInfo.platformType = mapIt.value().toString();
360 } else if (mapKey == QLatin1String("responseType")) {
361 const QString str = mapIt.value().toString();
362 if (str == QLatin1String("FULL_UPDATE")) {
363 databaseInfo.responseType = UpdateDataBaseInfo::FullUpdate;
364 } else if (str == QLatin1String("PARTIAL_UPDATE")) {
365 databaseInfo.responseType = UpdateDataBaseInfo::PartialUpdate;
366 } else {
367 qCDebug(WEBENGINEVIEWER_LOG) << " unknown responsetype " << str;
368 databaseInfo.responseType = UpdateDataBaseInfo::Unknown;
369 }
370 } else if (mapKey == QLatin1String("threatEntryType")) {
371 databaseInfo.threatEntryType = mapIt.value().toString();
372 } else if (mapKey == QLatin1String("threatType")) {
373 databaseInfo.threatType = mapIt.value().toString();
374 } else {
375 qCDebug(WEBENGINEVIEWER_LOG) << " unknown key " << mapKey;
376 }
377 }
378 }
379 }
380 } else if (i.key() == QLatin1String("minimumWaitDuration")) {
381 databaseInfo.minimumWaitDuration = i.value().toString();
382 } else {
383 qCDebug(WEBENGINEVIEWER_LOG) << " map key unknown " << i.key();
384 }
385 }
386 Q_EMIT finished(databaseInfo, ValidData);
387 }
388 }
389 deleteLater();
390 }
391
setContraintsCompressionType(CreatePhishingUrlDataBaseJob::ContraintsCompressionType type)392 void CreatePhishingUrlDataBaseJob::setContraintsCompressionType(CreatePhishingUrlDataBaseJob::ContraintsCompressionType type)
393 {
394 d->mContraintsCompressionType = type;
395 }
396