1 /*
2 * Bittorrent Client using Qt and libtorrent.
3 * Copyright (C) 2015 Vladimir Golovnev <glassez@yandex.ru>
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 *
19 * In addition, as a special exception, the copyright holders give permission to
20 * link this program with the OpenSSL project's "OpenSSL" library (or with
21 * modified versions of it that use the same license as the "OpenSSL" library),
22 * and distribute the linked executables. You must obey the GNU General Public
23 * License in all respects for all of the code used other than "OpenSSL". If you
24 * modify file(s), you may extend this exception to your version of the file(s),
25 * but you are not obligated to do so. If you do not wish to do so, delete this
26 * exception statement from your version.
27 */
28
29 #include <QDateTime>
30 #include <QDebug>
31 #include <QFile>
32 #include <QHostAddress>
33 #include <QVariant>
34
35 #include "geoipdatabase.h"
36
37 namespace
38 {
39 const qint32 MAX_FILE_SIZE = 67108864; // 64MB
40 const quint32 MAX_METADATA_SIZE = 131072; // 128KB
41 const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
42 const char DATA_SECTION_SEPARATOR[16] = {0};
43
44 enum class DataType
45 {
46 Unknown = 0,
47 Pointer = 1,
48 String = 2,
49 Double = 3,
50 Bytes = 4,
51 Integer16 = 5,
52 Integer32 = 6,
53 Map = 7,
54 SignedInteger32 = 8,
55 Integer64 = 9,
56 Integer128 = 10,
57 Array = 11,
58 DataCacheContainer = 12,
59 EndMarker = 13,
60 Boolean = 14,
61 Float = 15
62 };
63 }
64
65 struct DataFieldDescriptor
66 {
67 DataType fieldType {DataType::Unknown};
68 union
69 {
70 quint32 fieldSize = 0;
71 quint32 offset; // Pointer
72 };
73 };
74
GeoIPDatabase(const quint32 size)75 GeoIPDatabase::GeoIPDatabase(const quint32 size)
76 : m_ipVersion(0)
77 , m_recordSize(0)
78 , m_nodeCount(0)
79 , m_nodeSize(0)
80 , m_indexSize(0)
81 , m_recordBytes(0)
82 , m_size(size)
83 , m_data(new uchar[size])
84 {
85 }
86
load(const QString & filename,QString & error)87 GeoIPDatabase *GeoIPDatabase::load(const QString &filename, QString &error)
88 {
89 GeoIPDatabase *db = nullptr;
90 QFile file(filename);
91 if (file.size() > MAX_FILE_SIZE)
92 {
93 error = tr("Unsupported database file size.");
94 return nullptr;
95 }
96
97 if (!file.open(QFile::ReadOnly))
98 {
99 error = file.errorString();
100 return nullptr;
101 }
102
103 db = new GeoIPDatabase(file.size());
104
105 if (file.read(reinterpret_cast<char *>(db->m_data), db->m_size) != db->m_size)
106 {
107 error = file.errorString();
108 delete db;
109 return nullptr;
110 }
111
112
113 if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error))
114 {
115 delete db;
116 return nullptr;
117 }
118
119 return db;
120 }
121
load(const QByteArray & data,QString & error)122 GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error)
123 {
124 GeoIPDatabase *db = nullptr;
125 if (data.size() > MAX_FILE_SIZE)
126 {
127 error = tr("Unsupported database file size.");
128 return nullptr;
129 }
130
131 db = new GeoIPDatabase(data.size());
132
133 memcpy(reinterpret_cast<char *>(db->m_data), data.constData(), db->m_size);
134
135 if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error))
136 {
137 delete db;
138 return nullptr;
139 }
140
141 return db;
142 }
143
~GeoIPDatabase()144 GeoIPDatabase::~GeoIPDatabase()
145 {
146 delete [] m_data;
147 }
148
type() const149 QString GeoIPDatabase::type() const
150 {
151 return m_dbType;
152 }
153
ipVersion() const154 quint16 GeoIPDatabase::ipVersion() const
155 {
156 return m_ipVersion;
157 }
158
buildEpoch() const159 QDateTime GeoIPDatabase::buildEpoch() const
160 {
161 return m_buildEpoch;
162 }
163
lookup(const QHostAddress & hostAddr) const164 QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
165 {
166 Q_IPV6ADDR addr = hostAddr.toIPv6Address();
167
168 const uchar *ptr = m_data;
169
170 for (int i = 0; i < 16; ++i)
171 {
172 for (int j = 0; j < 8; ++j)
173 {
174 const bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1);
175 // Interpret the left/right record as number
176 if (right)
177 ptr += m_recordBytes;
178
179 quint32 id = 0;
180 auto *idPtr = reinterpret_cast<uchar *>(&id);
181 memcpy(&idPtr[4 - m_recordBytes], ptr, m_recordBytes);
182 fromBigEndian(idPtr, 4);
183
184 if (id == m_nodeCount)
185 {
186 return {};
187 }
188 if (id > m_nodeCount)
189 {
190 QString country = m_countries.value(id);
191 if (country.isEmpty())
192 {
193 const quint32 offset = id - m_nodeCount - sizeof(DATA_SECTION_SEPARATOR);
194 quint32 tmp = offset + m_indexSize + sizeof(DATA_SECTION_SEPARATOR);
195 const QVariant val = readDataField(tmp);
196 if (val.userType() == QMetaType::QVariantHash)
197 {
198 country = val.toHash()["country"].toHash()["iso_code"].toString();
199 m_countries[id] = country;
200 }
201 }
202 return country;
203 }
204
205 ptr = m_data + (id * m_nodeSize);
206 }
207 }
208
209 return {};
210 }
211
212 #define CHECK_METADATA_REQ(key, type) \
213 if (!metadata.contains(#key)) \
214 { \
215 error = errMsgNotFound.arg(#key); \
216 return false; \
217 } \
218 if (metadata.value(#key).userType() != QMetaType::type) \
219 { \
220 error = errMsgInvalid.arg(#key); \
221 return false; \
222 }
223
224 #define CHECK_METADATA_OPT(key, type) \
225 if (metadata.contains(#key)) \
226 { \
227 if (metadata.value(#key).userType() != QMetaType::type) \
228 { \
229 error = errMsgInvalid.arg(#key); \
230 return false; \
231 } \
232 }
233
parseMetadata(const QVariantHash & metadata,QString & error)234 bool GeoIPDatabase::parseMetadata(const QVariantHash &metadata, QString &error)
235 {
236 const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
237 const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
238
239 qDebug() << "Parsing MaxMindDB metadata...";
240
241 CHECK_METADATA_REQ(binary_format_major_version, UShort);
242 CHECK_METADATA_REQ(binary_format_minor_version, UShort);
243 const uint versionMajor = metadata.value("binary_format_major_version").toUInt();
244 const uint versionMinor = metadata.value("binary_format_minor_version").toUInt();
245 if (versionMajor != 2)
246 {
247 error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
248 return false;
249 }
250
251 CHECK_METADATA_REQ(ip_version, UShort);
252 m_ipVersion = metadata.value("ip_version").value<quint16>();
253 if (m_ipVersion != 6)
254 {
255 error = tr("Unsupported IP version: %1").arg(m_ipVersion);
256 return false;
257 }
258
259 CHECK_METADATA_REQ(record_size, UShort);
260 m_recordSize = metadata.value("record_size").value<quint16>();
261 if (m_recordSize != 24)
262 {
263 error = tr("Unsupported record size: %1").arg(m_recordSize);
264 return false;
265 }
266 m_nodeSize = m_recordSize / 4;
267 m_recordBytes = m_nodeSize / 2;
268
269 CHECK_METADATA_REQ(node_count, UInt);
270 m_nodeCount = metadata.value("node_count").value<quint32>();
271 m_indexSize = m_nodeCount * m_nodeSize;
272
273 CHECK_METADATA_REQ(database_type, QString);
274 m_dbType = metadata.value("database_type").toString();
275
276 CHECK_METADATA_REQ(build_epoch, ULongLong);
277 m_buildEpoch = QDateTime::fromSecsSinceEpoch(metadata.value("build_epoch").toULongLong());
278
279 CHECK_METADATA_OPT(languages, QVariantList);
280 CHECK_METADATA_OPT(description, QVariantHash);
281
282 return true;
283 }
284
loadDB(QString & error) const285 bool GeoIPDatabase::loadDB(QString &error) const
286 {
287 qDebug() << "Parsing IP geolocation database index tree...";
288
289 const int nodeSize = m_recordSize / 4; // in bytes
290 const int indexSize = m_nodeCount * nodeSize;
291 if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
292 || (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0))
293 {
294 error = tr("Database corrupted: no data section found.");
295 return false;
296 }
297
298 return true;
299 }
300
readMetadata() const301 QVariantHash GeoIPDatabase::readMetadata() const
302 {
303 const char *ptr = reinterpret_cast<const char *>(m_data);
304 quint32 size = m_size;
305 if (m_size > MAX_METADATA_SIZE)
306 {
307 ptr += m_size - MAX_METADATA_SIZE;
308 size = MAX_METADATA_SIZE;
309 }
310
311 const QByteArray data = QByteArray::fromRawData(ptr, size);
312 int index = data.lastIndexOf(METADATA_BEGIN_MARK);
313 if (index >= 0)
314 {
315 if (m_size > MAX_METADATA_SIZE)
316 index += (m_size - MAX_METADATA_SIZE); // from begin of all data
317 auto offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
318 const QVariant metadata = readDataField(offset);
319 if (metadata.userType() == QMetaType::QVariantHash)
320 return metadata.toHash();
321 }
322
323 return {};
324 }
325
readDataField(quint32 & offset) const326 QVariant GeoIPDatabase::readDataField(quint32 &offset) const
327 {
328 DataFieldDescriptor descr;
329 if (!readDataFieldDescriptor(offset, descr))
330 return {};
331
332 quint32 locOffset = offset;
333 bool usePointer = false;
334 if (descr.fieldType == DataType::Pointer)
335 {
336 usePointer = true;
337 // convert offset from data section to global
338 locOffset = descr.offset + (m_nodeCount * m_recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
339 if (!readDataFieldDescriptor(locOffset, descr))
340 return {};
341 }
342
343 QVariant fieldValue;
344 switch (descr.fieldType)
345 {
346 case DataType::Pointer:
347 qDebug() << "* Illegal Pointer using";
348 break;
349 case DataType::String:
350 fieldValue = QString::fromUtf8(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
351 locOffset += descr.fieldSize;
352 break;
353 case DataType::Double:
354 if (descr.fieldSize == 8)
355 fieldValue = readPlainValue<double>(locOffset, descr.fieldSize);
356 else
357 qDebug() << "* Invalid field size for type: Double";
358 break;
359 case DataType::Bytes:
360 fieldValue = QByteArray(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
361 locOffset += descr.fieldSize;
362 break;
363 case DataType::Integer16:
364 fieldValue = readPlainValue<quint16>(locOffset, descr.fieldSize);
365 break;
366 case DataType::Integer32:
367 fieldValue = readPlainValue<quint32>(locOffset, descr.fieldSize);
368 break;
369 case DataType::Map:
370 fieldValue = readMapValue(locOffset, descr.fieldSize);
371 break;
372 case DataType::SignedInteger32:
373 fieldValue = readPlainValue<qint32>(locOffset, descr.fieldSize);
374 break;
375 case DataType::Integer64:
376 fieldValue = readPlainValue<quint64>(locOffset, descr.fieldSize);
377 break;
378 case DataType::Integer128:
379 qDebug() << "* Unsupported data type: Integer128";
380 break;
381 case DataType::Array:
382 fieldValue = readArrayValue(locOffset, descr.fieldSize);
383 break;
384 case DataType::DataCacheContainer:
385 qDebug() << "* Unsupported data type: DataCacheContainer";
386 break;
387 case DataType::EndMarker:
388 qDebug() << "* Unsupported data type: EndMarker";
389 break;
390 case DataType::Boolean:
391 fieldValue = QVariant::fromValue(static_cast<bool>(descr.fieldSize));
392 break;
393 case DataType::Float:
394 if (descr.fieldSize == 4)
395 fieldValue = readPlainValue<float>(locOffset, descr.fieldSize);
396 else
397 qDebug() << "* Invalid field size for type: Float";
398 break;
399 default:
400 qDebug() << "* Unsupported data type: Unknown";
401 }
402
403 if (!usePointer)
404 offset = locOffset;
405 return fieldValue;
406 }
407
readDataFieldDescriptor(quint32 & offset,DataFieldDescriptor & out) const408 bool GeoIPDatabase::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const
409 {
410 const uchar *dataPtr = m_data + offset;
411 const int availSize = m_size - offset;
412 if (availSize < 1) return false;
413
414 out.fieldType = static_cast<DataType>((dataPtr[0] & 0xE0) >> 5);
415 if (out.fieldType == DataType::Pointer)
416 {
417 const int size = ((dataPtr[0] & 0x18) >> 3);
418 if (availSize < (size + 2)) return false;
419
420 if (size == 0)
421 out.offset = ((dataPtr[0] & 0x07) << 8) + dataPtr[1];
422 else if (size == 1)
423 out.offset = ((dataPtr[0] & 0x07) << 16) + (dataPtr[1] << 8) + dataPtr[2] + 2048;
424 else if (size == 2)
425 out.offset = ((dataPtr[0] & 0x07) << 24) + (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 526336;
426 else if (size == 3)
427 out.offset = (dataPtr[1] << 24) + (dataPtr[2] << 16) + (dataPtr[3] << 8) + dataPtr[4];
428
429 offset += size + 2;
430 return true;
431 }
432
433 out.fieldSize = dataPtr[0] & 0x1F;
434 if (out.fieldSize <= 28)
435 {
436 if (out.fieldType == DataType::Unknown)
437 {
438 out.fieldType = static_cast<DataType>(dataPtr[1] + 7);
439 if ((out.fieldType <= DataType::Map) || (out.fieldType > DataType::Float) || (availSize < 3))
440 return false;
441 offset += 2;
442 }
443 else
444 {
445 offset += 1;
446 }
447 }
448 else if (out.fieldSize == 29)
449 {
450 if (availSize < 2) return false;
451 out.fieldSize = dataPtr[1] + 29;
452 offset += 2;
453 }
454 else if (out.fieldSize == 30)
455 {
456 if (availSize < 3) return false;
457 out.fieldSize = (dataPtr[1] << 8) + dataPtr[2] + 285;
458 offset += 3;
459 }
460 else if (out.fieldSize == 31)
461 {
462 if (availSize < 4) return false;
463 out.fieldSize = (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 65821;
464 offset += 4;
465 }
466
467 return true;
468 }
469
fromBigEndian(uchar * buf,const quint32 len) const470 void GeoIPDatabase::fromBigEndian(uchar *buf, const quint32 len) const
471 {
472 #if (Q_BYTE_ORDER == Q_LITTLE_ENDIAN)
473 std::reverse(buf, buf + len);
474 #else
475 Q_UNUSED(buf);
476 Q_UNUSED(len);
477 #endif
478 }
479
readMapValue(quint32 & offset,const quint32 count) const480 QVariant GeoIPDatabase::readMapValue(quint32 &offset, const quint32 count) const
481 {
482 QVariantHash map;
483
484 for (quint32 i = 0; i < count; ++i)
485 {
486 QVariant field = readDataField(offset);
487 if (field.userType() != QMetaType::QString)
488 return {};
489
490 const QString key = field.toString();
491 field = readDataField(offset);
492 if (field.userType() == QVariant::Invalid)
493 return {};
494
495 map[key] = field;
496 }
497
498 return map;
499 }
500
readArrayValue(quint32 & offset,const quint32 count) const501 QVariant GeoIPDatabase::readArrayValue(quint32 &offset, const quint32 count) const
502 {
503 QVariantList array;
504
505 for (quint32 i = 0; i < count; ++i)
506 {
507 const QVariant field = readDataField(offset);
508 if (field.userType() == QVariant::Invalid)
509 return {};
510
511 array.append(field);
512 }
513
514 return array;
515 }
516