1 /*
2 * SPDX-FileCopyrightText: 2002-2005 David Faure <faure@kde.org>
3 *
4 * SPDX-License-Identifier: LGPL-2.0-or-later
5 */
6
7 #include "kfiltertest.h"
8
9 #include <QBuffer>
10 #include <QTest>
11 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
12 #include <QTextCodec>
13 #endif
14 #include <QRandomGenerator>
15 #include <QSaveFile>
16
17 #include "kcompressiondevice.h"
18 #include "kfilterbase.h"
19 #include <QDebug>
20 #include <QDir>
21 #include <QFile>
22 #include <QFileInfo>
23 #include <QTextStream>
24 #include <config-compression.h>
25 #include <zlib.h>
26
27 #ifdef Q_OS_UNIX
28 #include <limits.h>
29 #include <unistd.h>
30 #endif
31
QTEST_MAIN(KFilterTest)32 QTEST_MAIN(KFilterTest)
33
34 void KFilterTest::initTestCase()
35 {
36 qRegisterMetaType<KCompressionDevice::CompressionType>();
37 const QString currentdir = QDir::currentPath();
38 pathgz = currentdir + "/test.gz";
39 pathbz2 = currentdir + "/test.bz2";
40 pathxz = currentdir + "/test.xz";
41 pathnone = currentdir + "/test.txt";
42 pathzstd = currentdir + "/test.zst";
43
44 // warning, update the COMPAREs in test_block_write() if changing the test data...
45 testData = "hello world\n";
46 }
47
test_block_write(const QString & fileName,const QByteArray & data)48 void KFilterTest::test_block_write(const QString &fileName, const QByteArray &data)
49 {
50 KCompressionDevice dev(fileName);
51 bool ok = dev.open(QIODevice::WriteOnly);
52 QVERIFY(ok);
53
54 const int ret = dev.write(data);
55 QCOMPARE(ret, data.size());
56
57 dev.close();
58
59 QVERIFY(QFile::exists(fileName));
60 }
61
test_block_write()62 void KFilterTest::test_block_write()
63 {
64 qDebug() << " -- test_block_write gzip -- ";
65 test_block_write(pathgz, testData);
66 QCOMPARE(QFileInfo(pathgz).size(), 33LL); // size of test.gz
67
68 #if HAVE_BZIP2_SUPPORT
69 qDebug() << " -- test_block_write bzip2 -- ";
70 test_block_write(pathbz2, testData);
71 QCOMPARE(QFileInfo(pathbz2).size(), 52LL); // size of test.bz2
72 #endif
73
74 #if HAVE_XZ_SUPPORT
75 qDebug() << " -- test_block_write xz -- ";
76 test_block_write(pathxz, testData);
77 QCOMPARE(QFileInfo(pathxz).size(), 64LL); // size of test.lzma
78 #endif
79
80 qDebug() << " -- test_block_write none -- ";
81 test_block_write(pathnone, testData);
82 QCOMPARE(QFileInfo(pathnone).size(), 12LL); // size of test.txt
83
84 #if HAVE_ZSTD_SUPPORT
85 qDebug() << " -- test_block_write zstd -- ";
86 test_block_write(pathzstd, testData);
87 QCOMPARE(QFileInfo(pathzstd).size(), 24LL); // size of test.zst
88 #endif
89 }
90
test_biggerWrites()91 void KFilterTest::test_biggerWrites()
92 {
93 const QString currentdir = QDir::currentPath();
94 const QString outFile = currentdir + "/test_big.gz";
95 // Find the out-of-bounds from #157706/#188415
96 QByteArray data;
97 data.reserve(10000);
98 auto *generator = QRandomGenerator::global();
99 // Prepare test data
100 for (int i = 0; i < 8170; ++i) {
101 data.append((char)(generator->bounded(256)));
102 }
103 QCOMPARE(data.size(), 8170);
104 // 8170 random bytes compress to 8194 bytes due to the gzip header/footer.
105 // Now we can go one by one until we pass 8192.
106 // On 32 bit systems it crashed with data.size()=8173, before the "no room for footer yet" fix.
107 int compressedSize = 0;
108 while (compressedSize < 8200) {
109 test_block_write(outFile, data);
110 compressedSize = QFileInfo(outFile).size();
111 qDebug() << data.size() << "compressed into" << compressedSize;
112 // Test data is valid
113 test_readall(outFile, QString::fromLatin1("application/gzip"), data);
114
115 data.append((char)(generator->bounded(256)));
116 }
117 }
118
test_block_read(const QString & fileName)119 void KFilterTest::test_block_read(const QString &fileName)
120 {
121 KCompressionDevice dev(fileName);
122 bool ok = dev.open(QIODevice::ReadOnly);
123 QVERIFY(ok);
124
125 QByteArray array(1024, '\0');
126 QByteArray read;
127 int n;
128 while ((n = dev.read(array.data(), array.size()))) {
129 QVERIFY(n > 0);
130 read += QByteArray(array.constData(), n);
131 // qDebug() << "read returned " << n;
132 // qDebug() << "read='" << read << "'";
133
134 // pos() has no real meaning on sequential devices
135 // Ah, but kzip uses kfilterdev as a non-sequential device...
136
137 QCOMPARE((int)dev.pos(), (int)read.size());
138 // qDebug() << "dev.at = " << dev->at();
139 }
140 QCOMPARE(read, testData);
141
142 // Test seeking back
143 ok = dev.seek(0);
144 // test readAll
145 read = dev.readAll();
146 QCOMPARE(read.size(), testData.size());
147 QCOMPARE(read, testData);
148
149 dev.close();
150 }
151
test_block_read()152 void KFilterTest::test_block_read()
153 {
154 qDebug() << " -- test_block_read gzip -- ";
155 test_block_read(pathgz);
156 #if HAVE_BZIP2_SUPPORT
157 qDebug() << " -- test_block_read bzip2 -- ";
158 test_block_read(pathbz2);
159 #endif
160 #if HAVE_XZ_SUPPORT
161 qDebug() << " -- test_block_read lzma -- ";
162 test_block_read(pathxz);
163 #endif
164 qDebug() << " -- test_block_read none -- ";
165 test_block_read(pathnone);
166 #if HAVE_ZSTD_SUPPORT
167 qDebug() << " -- test_block_read zstd -- ";
168 test_block_read(pathzstd);
169 #endif
170 }
171
test_getch(const QString & fileName)172 void KFilterTest::test_getch(const QString &fileName)
173 {
174 KCompressionDevice dev(fileName);
175 bool ok = dev.open(QIODevice::ReadOnly);
176 QVERIFY(ok);
177 QByteArray read;
178 char ch;
179 while (dev.getChar(&ch)) {
180 // printf("%c",ch);
181 read += ch;
182 }
183 dev.close();
184 QCOMPARE(read, testData);
185 }
186
test_getch()187 void KFilterTest::test_getch()
188 {
189 qDebug() << " -- test_getch gzip -- ";
190 test_getch(pathgz);
191 #if HAVE_BZIP2_SUPPORT
192 qDebug() << " -- test_getch bzip2 -- ";
193 test_getch(pathbz2);
194 #endif
195 #if HAVE_XZ_SUPPORT
196 qDebug() << " -- test_getch lzma -- ";
197 test_getch(pathxz);
198 #endif
199 qDebug() << " -- test_getch none -- ";
200 test_getch(pathnone);
201 #if HAVE_ZSTD_SUPPORT
202 qDebug() << " -- test_getch zstd -- ";
203 test_getch(pathzstd);
204 #endif
205 }
206
test_textstream(const QString & fileName)207 void KFilterTest::test_textstream(const QString &fileName)
208 {
209 KCompressionDevice dev(fileName);
210 bool ok = dev.open(QIODevice::ReadOnly);
211 QVERIFY(ok);
212 QTextStream ts(&dev);
213 QString readStr = ts.readAll();
214 dev.close();
215
216 QByteArray read = readStr.toLatin1();
217 QCOMPARE(read, testData);
218 }
219
test_textstream()220 void KFilterTest::test_textstream()
221 {
222 qDebug() << " -- test_textstream gzip -- ";
223 test_textstream(pathgz);
224 #if HAVE_BZIP2_SUPPORT
225 qDebug() << " -- test_textstream bzip2 -- ";
226 test_textstream(pathbz2);
227 #endif
228 #if HAVE_XZ_SUPPORT
229 qDebug() << " -- test_textstream lzma -- ";
230 test_textstream(pathxz);
231 #endif
232 qDebug() << " -- test_textstream none -- ";
233 test_textstream(pathnone);
234 #if HAVE_ZSTD_SUPPORT
235 qDebug() << " -- test_textstream zstd -- ";
236 test_textstream(pathzstd);
237 #endif
238 }
239
test_readall(const QString & fileName,const QString & mimeType,const QByteArray & expectedData)240 void KFilterTest::test_readall(const QString &fileName, const QString &mimeType, const QByteArray &expectedData)
241 {
242 QFile file(fileName);
243 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(mimeType);
244 KCompressionDevice flt(&file, false, type);
245 bool ok = flt.open(QIODevice::ReadOnly);
246 QVERIFY(ok);
247 const QByteArray read = flt.readAll();
248 QCOMPARE(read.size(), expectedData.size());
249 QCOMPARE(read, expectedData);
250
251 // Now using QBuffer
252 file.seek(0);
253 QByteArray compressedData = file.readAll();
254 QVERIFY(!compressedData.isEmpty());
255 QBuffer buffer(&compressedData);
256 KCompressionDevice device(&buffer, false, type);
257 QVERIFY(device.open(QIODevice::ReadOnly));
258 QCOMPARE(device.readAll(), expectedData);
259 }
260
test_readall()261 void KFilterTest::test_readall()
262 {
263 qDebug() << " -- test_readall gzip -- ";
264 test_readall(pathgz, QString::fromLatin1("application/gzip"), testData);
265 #if HAVE_BZIP2_SUPPORT
266 qDebug() << " -- test_readall bzip2 -- ";
267 test_readall(pathbz2, QString::fromLatin1("application/x-bzip"), testData);
268 #endif
269 #if HAVE_XZ_SUPPORT
270 qDebug() << " -- test_readall lzma -- ";
271 test_readall(pathxz, QString::fromLatin1("application/x-xz"), testData);
272 #endif
273 qDebug() << " -- test_readall gzip-derived -- ";
274 test_readall(pathgz, QString::fromLatin1("image/svg+xml-compressed"), testData);
275
276 qDebug() << " -- test_readall none -- ";
277 test_readall(pathnone, QString::fromLatin1("text/plain"), testData);
278
279 #if HAVE_ZSTD_SUPPORT
280 qDebug() << " -- test_readall zstd -- ";
281 test_readall(pathzstd, QString::fromLatin1("application/zstd"), testData);
282 #endif
283 }
284
test_uncompressed()285 void KFilterTest::test_uncompressed()
286 {
287 // Can KCompressionDevice handle uncompressed data even when using gzip decompression?
288 qDebug() << " -- test_uncompressed -- ";
289 QBuffer buffer(&testData);
290 buffer.open(QIODevice::ReadOnly);
291 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(QString::fromLatin1("application/gzip"));
292 KCompressionDevice flt(&buffer, false, type);
293 bool ok = flt.open(QIODevice::ReadOnly);
294 QVERIFY(ok);
295 QByteArray read = flt.readAll();
296 QCOMPARE(read.size(), testData.size());
297 QCOMPARE(read, testData);
298 }
299
test_findFilterByMimeType_data()300 void KFilterTest::test_findFilterByMimeType_data()
301 {
302 QTest::addColumn<QString>("mimeType");
303 QTest::addColumn<KCompressionDevice::CompressionType>("type");
304
305 // direct mimetype name
306 QTest::newRow("application/gzip") << QString::fromLatin1("application/gzip") << KCompressionDevice::GZip;
307 #if HAVE_BZIP2_SUPPORT
308 QTest::newRow("application/x-bzip") << QString::fromLatin1("application/x-bzip") << KCompressionDevice::BZip2;
309 QTest::newRow("application/x-bzip2") << QString::fromLatin1("application/x-bzip2") << KCompressionDevice::BZip2;
310 #else
311 QTest::newRow("application/x-bzip") << QString::fromLatin1("application/x-bzip") << KCompressionDevice::None;
312 QTest::newRow("application/x-bzip2") << QString::fromLatin1("application/x-bzip2") << KCompressionDevice::None;
313 #endif
314 // indirect compressed mimetypes
315 QTest::newRow("application/x-gzdvi") << QString::fromLatin1("application/x-gzdvi") << KCompressionDevice::GZip;
316
317 // non-compressed mimetypes
318 QTest::newRow("text/plain") << QString::fromLatin1("text/plain") << KCompressionDevice::None;
319 QTest::newRow("application/x-tar") << QString::fromLatin1("application/x-tar") << KCompressionDevice::None;
320 }
321
test_findFilterByMimeType()322 void KFilterTest::test_findFilterByMimeType()
323 {
324 QFETCH(QString, mimeType);
325 QFETCH(KCompressionDevice::CompressionType, type);
326
327 KCompressionDevice::CompressionType compressionType = KCompressionDevice::compressionTypeForMimeType(mimeType);
328 QCOMPARE(compressionType, type);
329 }
330
getCompressedData(QByteArray & data,QByteArray & compressedData)331 static void getCompressedData(QByteArray &data, QByteArray &compressedData)
332 {
333 data = "Hello world, this is a test for deflate, from bug 114830 / 117683";
334 compressedData.resize(long(data.size() * 1.1f) + 12L); // requirements of zlib::compress2
335 unsigned long out_bufferlen = compressedData.size();
336 const int ret = compress2((Bytef *)compressedData.data(), &out_bufferlen, (const Bytef *)data.constData(), data.size(), 1);
337 QCOMPARE(ret, Z_OK);
338 compressedData.resize(out_bufferlen);
339 }
340
test_deflateWithZlibHeader()341 void KFilterTest::test_deflateWithZlibHeader()
342 {
343 QByteArray data;
344 QByteArray deflatedData;
345 getCompressedData(data, deflatedData);
346
347 #if 0 // Can't use KFilterDev for this, we need to call KGzipFilter::init(QIODevice::ReadOnly, KGzipFilter::ZlibHeader);
348 QBuffer buffer(&deflatedData);
349 QIODevice *flt = KFilterDev::device(&buffer, "application/gzip", false);
350 static_cast<KFilterDev *>(flt)->setSkipHeaders();
351 bool ok = flt->open(QIODevice::ReadOnly);
352 QVERIFY(ok);
353 const QByteArray read = flt->readAll();
354 #else
355 // Copied from HTTPFilter (which isn't linked into any kdelibs library)
356 KFilterBase *mFilterDevice = KCompressionDevice::filterForCompressionType(KCompressionDevice::GZip);
357 mFilterDevice->setFilterFlags(KFilterBase::ZlibHeaders);
358 mFilterDevice->init(QIODevice::ReadOnly);
359
360 mFilterDevice->setInBuffer(deflatedData.constData(), deflatedData.size());
361 char buf[8192];
362 mFilterDevice->setOutBuffer(buf, sizeof(buf));
363 KFilterBase::Result result = mFilterDevice->uncompress();
364 QCOMPARE(result, KFilterBase::End);
365 const int bytesOut = sizeof(buf) - mFilterDevice->outBufferAvailable();
366 QVERIFY(bytesOut);
367 QByteArray read(buf, bytesOut);
368 mFilterDevice->terminate();
369 delete mFilterDevice;
370 #endif
371 QCOMPARE(QString::fromLatin1(read.constData()), QString::fromLatin1(data.constData())); // more readable output than the line below
372 QCOMPARE(read, data);
373
374 // For the same test with HTTPFilter: see httpfiltertest.cpp
375 }
376
test_pushData()377 void KFilterTest::test_pushData() // ### UNFINISHED
378 {
379 // HTTPFilter says KFilterDev doesn't support the case where compressed data
380 // is arriving in chunks. Let's test that.
381 QFile file(pathgz);
382 QVERIFY(file.open(QIODevice::ReadOnly));
383 const QByteArray compressed = file.readAll();
384 const int firstChunkSize = compressed.size() / 2;
385 QByteArray firstData(compressed.constData(), firstChunkSize);
386 QBuffer inBuffer(&firstData);
387 QVERIFY(inBuffer.open(QIODevice::ReadWrite));
388 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(QString::fromLatin1("application/gzip"));
389 KCompressionDevice flt(&inBuffer, false, type);
390 QVERIFY(flt.open(QIODevice::ReadOnly));
391 QByteArray read = flt.readAll();
392 qDebug() << QString::fromLatin1(read.constData());
393
394 // And later...
395 inBuffer.write(QByteArray(compressed.data() + firstChunkSize, compressed.size() - firstChunkSize));
396 QCOMPARE(inBuffer.data().size(), compressed.size());
397 read += flt.readAll();
398 qDebug() << QString::fromLatin1(read.constData());
399 // ### indeed, doesn't work currently. So we use HTTPFilter instead, for now.
400 }
401
test_saveFile_data()402 void KFilterTest::test_saveFile_data()
403 {
404 QTest::addColumn<QString>("fileName");
405 QTest::addColumn<KCompressionDevice::CompressionType>("compressionType");
406
407 QTest::newRow("gz") << "test_saveFile.gz" << KCompressionDevice::GZip;
408 QTest::newRow("none") << "test_saveFile" << KCompressionDevice::None;
409 }
410
test_saveFile()411 void KFilterTest::test_saveFile()
412 {
413 QFETCH(QString, fileName);
414 QFETCH(KCompressionDevice::CompressionType, compressionType);
415
416 int numLines = 1000;
417 const QString lineTemplate = QStringLiteral("Hello world, this is the text for line %1");
418 const QString currentdir = QDir::currentPath();
419 const QString outFile = QDir::currentPath() + '/' + fileName;
420 {
421 QSaveFile file(outFile);
422 file.setDirectWriteFallback(true);
423 QVERIFY(file.open(QIODevice::WriteOnly));
424 KCompressionDevice device(&file, false, compressionType);
425 QVERIFY(device.open(QIODevice::WriteOnly));
426 QTextStream stream(&device);
427 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
428 stream.setCodec(QTextCodec::codecForName("UTF-8"));
429 #endif
430 for (int i = 0; i < numLines; ++i) {
431 stream << lineTemplate.arg(i);
432 stream << QString("\n");
433 }
434 stream.flush();
435 QCOMPARE(stream.status(), QTextStream::Ok);
436 // device.write("The data to be compressed");
437 device.close();
438 QVERIFY(file.commit());
439 }
440 QVERIFY(QFile::exists(outFile));
441 KCompressionDevice reader(outFile, compressionType);
442 QVERIFY(reader.open(QIODevice::ReadOnly));
443 QString expectedFullData;
444 for (int i = 0; i < numLines; ++i) {
445 QCOMPARE(QString::fromUtf8(reader.readLine()), QString(lineTemplate.arg(i) + '\n'));
446 expectedFullData += QString(lineTemplate.arg(i) + '\n');
447 }
448 KCompressionDevice otherReader(outFile);
449 QVERIFY(otherReader.open(QIODevice::ReadOnly));
450 QCOMPARE(QString::fromLatin1(otherReader.readAll()), expectedFullData);
451 QVERIFY(otherReader.atEnd());
452 }
453
test_twofilesgztogether()454 void KFilterTest::test_twofilesgztogether()
455 {
456 // Reported as 232843
457 // twofiles generated with
458 // echo foo > foo; echo bar > bar ; gzip -c foo > twofiles.gz; gzip -c bar >> twofiles.gz
459 // as documented in the gzip manpage
460 QString data = QFINDTESTDATA("data/twofiles.gz");
461 KCompressionDevice dev(data);
462 QVERIFY(dev.open(QIODevice::ReadOnly));
463 QByteArray extractedData = dev.readAll();
464 QByteArray expectedData{"foo\nbar\n"};
465 QCOMPARE(extractedData, expectedData);
466 }
467
test_threefilesgztogether()468 void KFilterTest::test_threefilesgztogether()
469 {
470 // Generated similarly to the one above
471 // This catches the case where there's more than two streams available in the same buffer fed to KGzipFilter
472 QString data = QFINDTESTDATA("data/threefiles.gz");
473 KCompressionDevice dev(data);
474 QVERIFY(dev.open(QIODevice::ReadOnly));
475 QByteArray extractedData = dev.readAll();
476 QByteArray expectedData{"foo\nbar\nbaz\n"};
477 QCOMPARE(extractedData, expectedData);
478 }
479