File indexing completed on 2024-04-21 03:52:29
0001 /* 0002 * SPDX-FileCopyrightText: 2002-2005 David Faure <faure@kde.org> 0003 * 0004 * SPDX-License-Identifier: LGPL-2.0-or-later 0005 */ 0006 0007 #include "kfiltertest.h" 0008 0009 #include <QBuffer> 0010 #include <QRandomGenerator> 0011 #include <QSaveFile> 0012 #include <QTest> 0013 0014 #include "kcompressiondevice.h" 0015 #include "kfilterbase.h" 0016 #include <QDebug> 0017 #include <QDir> 0018 #include <QFile> 0019 #include <QFileInfo> 0020 #include <QTextStream> 0021 #include <config-compression.h> 0022 #include <zlib.h> 0023 0024 #ifdef Q_OS_UNIX 0025 #include <limits.h> 0026 #include <unistd.h> 0027 #endif 0028 0029 QTEST_MAIN(KFilterTest) 0030 0031 void KFilterTest::initTestCase() 0032 { 0033 qRegisterMetaType<KCompressionDevice::CompressionType>(); 0034 const QString currentdir = QDir::currentPath(); 0035 pathgz = currentdir + "/test.gz"; 0036 pathbz2 = currentdir + "/test.bz2"; 0037 pathxz = currentdir + "/test.xz"; 0038 pathnone = currentdir + "/test.txt"; 0039 pathzstd = currentdir + "/test.zst"; 0040 0041 // warning, update the COMPAREs in test_block_write() if changing the test data... 0042 testData = "hello world\n"; 0043 } 0044 0045 void KFilterTest::test_block_write(const QString &fileName, const QByteArray &data, int nTimes) 0046 { 0047 KCompressionDevice dev(fileName); 0048 bool ok = dev.open(QIODevice::WriteOnly); 0049 QVERIFY(ok); 0050 0051 for (int i = 0; i < nTimes; ++i) { 0052 const int ret = dev.write(data); 0053 QCOMPARE(ret, data.size()); 0054 } 0055 0056 dev.close(); 0057 0058 QVERIFY(QFile::exists(fileName)); 0059 } 0060 0061 void KFilterTest::test_block_write() 0062 { 0063 // First we do it 50 times, to make sure the compressor 0064 // can "reuse" existing data, i.e. zstd used to have a bug 0065 // that made this use 513 bytes instead of 28 0066 qDebug() << " -- test_block_write gzip -- "; 0067 test_block_write(pathgz, testData, 50); 0068 QCOMPARE(QFileInfo(pathgz).size(), 40LL); // size of test.gz 0069 0070 #if HAVE_BZIP2_SUPPORT 0071 qDebug() << " -- test_block_write bzip2 -- "; 0072 test_block_write(pathbz2, testData, 50); 0073 QCOMPARE(QFileInfo(pathbz2).size(), 64LL); // size of test.bz2 0074 #endif 0075 0076 #if HAVE_XZ_SUPPORT 0077 qDebug() << " -- test_block_write xz -- "; 0078 test_block_write(pathxz, testData, 50); 0079 QCOMPARE(QFileInfo(pathxz).size(), 84LL); // size of test.lzma 0080 #endif 0081 0082 qDebug() << " -- test_block_write none -- "; 0083 test_block_write(pathnone, testData, 50); 0084 QCOMPARE(QFileInfo(pathnone).size(), 600LL); // size of test.txt 0085 0086 #if HAVE_ZSTD_SUPPORT 0087 qDebug() << " -- test_block_write zstd -- "; 0088 test_block_write(pathzstd, testData, 50); 0089 QCOMPARE(QFileInfo(pathzstd).size(), 28LL); // size of test.zst 0090 #endif 0091 0092 // Then do the write data just once because test_block_read 0093 // depends on this 0094 qDebug() << " -- test_block_write gzip -- "; 0095 test_block_write(pathgz, testData); 0096 QCOMPARE(QFileInfo(pathgz).size(), 33LL); // size of test.gz 0097 0098 #if HAVE_BZIP2_SUPPORT 0099 qDebug() << " -- test_block_write bzip2 -- "; 0100 test_block_write(pathbz2, testData); 0101 QCOMPARE(QFileInfo(pathbz2).size(), 52LL); // size of test.bz2 0102 #endif 0103 0104 #if HAVE_XZ_SUPPORT 0105 qDebug() << " -- test_block_write xz -- "; 0106 test_block_write(pathxz, testData); 0107 QCOMPARE(QFileInfo(pathxz).size(), 64LL); // size of test.lzma 0108 #endif 0109 0110 qDebug() << " -- test_block_write none -- "; 0111 test_block_write(pathnone, testData); 0112 QCOMPARE(QFileInfo(pathnone).size(), 12LL); // size of test.txt 0113 0114 #if HAVE_ZSTD_SUPPORT 0115 qDebug() << " -- test_block_write zstd -- "; 0116 test_block_write(pathzstd, testData); 0117 QCOMPARE(QFileInfo(pathzstd).size(), 21LL); // size of test.zst 0118 #endif 0119 } 0120 0121 void KFilterTest::test_biggerWrites() 0122 { 0123 const QString currentdir = QDir::currentPath(); 0124 const QString outFile = currentdir + "/test_big.gz"; 0125 // Find the out-of-bounds from #157706/#188415 0126 QByteArray data; 0127 data.reserve(10000); 0128 auto *generator = QRandomGenerator::global(); 0129 // Prepare test data 0130 for (int i = 0; i < 8170; ++i) { 0131 data.append((char)(generator->bounded(256))); 0132 } 0133 QCOMPARE(data.size(), 8170); 0134 // 8170 random bytes compress to 8194 bytes due to the gzip header/footer. 0135 // Now we can go one by one until we pass 8192. 0136 // On 32 bit systems it crashed with data.size()=8173, before the "no room for footer yet" fix. 0137 int compressedSize = 0; 0138 while (compressedSize < 8200) { 0139 test_block_write(outFile, data); 0140 compressedSize = QFileInfo(outFile).size(); 0141 qDebug() << data.size() << "compressed into" << compressedSize; 0142 // Test data is valid 0143 test_readall(outFile, QString::fromLatin1("application/gzip"), data); 0144 0145 data.append((char)(generator->bounded(256))); 0146 } 0147 } 0148 0149 void KFilterTest::test_block_read(const QString &fileName) 0150 { 0151 KCompressionDevice dev(fileName); 0152 bool ok = dev.open(QIODevice::ReadOnly); 0153 QVERIFY(ok); 0154 0155 QByteArray array(1024, '\0'); 0156 QByteArray read; 0157 int n; 0158 while ((n = dev.read(array.data(), array.size()))) { 0159 QVERIFY(n > 0); 0160 read += QByteArray(array.constData(), n); 0161 // qDebug() << "read returned " << n; 0162 // qDebug() << "read='" << read << "'"; 0163 0164 // pos() has no real meaning on sequential devices 0165 // Ah, but kzip uses kfilterdev as a non-sequential device... 0166 0167 QCOMPARE((int)dev.pos(), (int)read.size()); 0168 // qDebug() << "dev.at = " << dev->at(); 0169 } 0170 QCOMPARE(read, testData); 0171 0172 // Test seeking back 0173 ok = dev.seek(0); 0174 // test readAll 0175 read = dev.readAll(); 0176 QCOMPARE(read.size(), testData.size()); 0177 QCOMPARE(read, testData); 0178 0179 dev.close(); 0180 } 0181 0182 void KFilterTest::test_block_read() 0183 { 0184 qDebug() << " -- test_block_read gzip -- "; 0185 test_block_read(pathgz); 0186 #if HAVE_BZIP2_SUPPORT 0187 qDebug() << " -- test_block_read bzip2 -- "; 0188 test_block_read(pathbz2); 0189 #endif 0190 #if HAVE_XZ_SUPPORT 0191 qDebug() << " -- test_block_read lzma -- "; 0192 test_block_read(pathxz); 0193 #endif 0194 qDebug() << " -- test_block_read none -- "; 0195 test_block_read(pathnone); 0196 #if HAVE_ZSTD_SUPPORT 0197 qDebug() << " -- test_block_read zstd -- "; 0198 test_block_read(pathzstd); 0199 #endif 0200 } 0201 0202 void KFilterTest::test_getch(const QString &fileName) 0203 { 0204 KCompressionDevice dev(fileName); 0205 bool ok = dev.open(QIODevice::ReadOnly); 0206 QVERIFY(ok); 0207 QByteArray read; 0208 char ch; 0209 while (dev.getChar(&ch)) { 0210 // printf("%c",ch); 0211 read += ch; 0212 } 0213 dev.close(); 0214 QCOMPARE(read, testData); 0215 } 0216 0217 void KFilterTest::test_getch() 0218 { 0219 qDebug() << " -- test_getch gzip -- "; 0220 test_getch(pathgz); 0221 #if HAVE_BZIP2_SUPPORT 0222 qDebug() << " -- test_getch bzip2 -- "; 0223 test_getch(pathbz2); 0224 #endif 0225 #if HAVE_XZ_SUPPORT 0226 qDebug() << " -- test_getch lzma -- "; 0227 test_getch(pathxz); 0228 #endif 0229 qDebug() << " -- test_getch none -- "; 0230 test_getch(pathnone); 0231 #if HAVE_ZSTD_SUPPORT 0232 qDebug() << " -- test_getch zstd -- "; 0233 test_getch(pathzstd); 0234 #endif 0235 } 0236 0237 void KFilterTest::test_textstream(const QString &fileName) 0238 { 0239 KCompressionDevice dev(fileName); 0240 bool ok = dev.open(QIODevice::ReadOnly); 0241 QVERIFY(ok); 0242 QTextStream ts(&dev); 0243 QString readStr = ts.readAll(); 0244 dev.close(); 0245 0246 QByteArray read = readStr.toLatin1(); 0247 QCOMPARE(read, testData); 0248 } 0249 0250 void KFilterTest::test_textstream() 0251 { 0252 qDebug() << " -- test_textstream gzip -- "; 0253 test_textstream(pathgz); 0254 #if HAVE_BZIP2_SUPPORT 0255 qDebug() << " -- test_textstream bzip2 -- "; 0256 test_textstream(pathbz2); 0257 #endif 0258 #if HAVE_XZ_SUPPORT 0259 qDebug() << " -- test_textstream lzma -- "; 0260 test_textstream(pathxz); 0261 #endif 0262 qDebug() << " -- test_textstream none -- "; 0263 test_textstream(pathnone); 0264 #if HAVE_ZSTD_SUPPORT 0265 qDebug() << " -- test_textstream zstd -- "; 0266 test_textstream(pathzstd); 0267 #endif 0268 } 0269 0270 void KFilterTest::test_readall(const QString &fileName, const QString &mimeType, const QByteArray &expectedData) 0271 { 0272 QFile file(fileName); 0273 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(mimeType); 0274 KCompressionDevice flt(&file, false, type); 0275 bool ok = flt.open(QIODevice::ReadOnly); 0276 QVERIFY(ok); 0277 const QByteArray read = flt.readAll(); 0278 QCOMPARE(read.size(), expectedData.size()); 0279 QCOMPARE(read, expectedData); 0280 0281 // Now using QBuffer 0282 file.seek(0); 0283 QByteArray compressedData = file.readAll(); 0284 QVERIFY(!compressedData.isEmpty()); 0285 QBuffer buffer(&compressedData); 0286 KCompressionDevice device(&buffer, false, type); 0287 QVERIFY(device.open(QIODevice::ReadOnly)); 0288 QCOMPARE(device.readAll(), expectedData); 0289 } 0290 0291 void KFilterTest::test_readall() 0292 { 0293 qDebug() << " -- test_readall gzip -- "; 0294 test_readall(pathgz, QString::fromLatin1("application/gzip"), testData); 0295 #if HAVE_BZIP2_SUPPORT 0296 qDebug() << " -- test_readall bzip2 -- "; 0297 test_readall(pathbz2, QString::fromLatin1("application/x-bzip"), testData); 0298 #endif 0299 #if HAVE_XZ_SUPPORT 0300 qDebug() << " -- test_readall lzma -- "; 0301 test_readall(pathxz, QString::fromLatin1("application/x-xz"), testData); 0302 #endif 0303 qDebug() << " -- test_readall gzip-derived -- "; 0304 test_readall(pathgz, QString::fromLatin1("image/svg+xml-compressed"), testData); 0305 0306 qDebug() << " -- test_readall none -- "; 0307 test_readall(pathnone, QString::fromLatin1("text/plain"), testData); 0308 0309 #if HAVE_ZSTD_SUPPORT 0310 qDebug() << " -- test_readall zstd -- "; 0311 test_readall(pathzstd, QString::fromLatin1("application/zstd"), testData); 0312 #endif 0313 } 0314 0315 void KFilterTest::test_uncompressed() 0316 { 0317 // Can KCompressionDevice handle uncompressed data even when using gzip decompression? 0318 qDebug() << " -- test_uncompressed -- "; 0319 QBuffer buffer(&testData); 0320 buffer.open(QIODevice::ReadOnly); 0321 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(QString::fromLatin1("application/gzip")); 0322 KCompressionDevice flt(&buffer, false, type); 0323 bool ok = flt.open(QIODevice::ReadOnly); 0324 QVERIFY(ok); 0325 QByteArray read = flt.readAll(); 0326 QCOMPARE(read.size(), testData.size()); 0327 QCOMPARE(read, testData); 0328 } 0329 0330 void KFilterTest::test_findFilterByMimeType_data() 0331 { 0332 QTest::addColumn<QString>("mimeType"); 0333 QTest::addColumn<KCompressionDevice::CompressionType>("type"); 0334 0335 // direct mimetype name 0336 QTest::newRow("application/gzip") << QString::fromLatin1("application/gzip") << KCompressionDevice::GZip; 0337 #if HAVE_BZIP2_SUPPORT 0338 QTest::newRow("application/x-bzip") << QString::fromLatin1("application/x-bzip") << KCompressionDevice::BZip2; 0339 QTest::newRow("application/x-bzip2") << QString::fromLatin1("application/x-bzip2") << KCompressionDevice::BZip2; 0340 #else 0341 QTest::newRow("application/x-bzip") << QString::fromLatin1("application/x-bzip") << KCompressionDevice::None; 0342 QTest::newRow("application/x-bzip2") << QString::fromLatin1("application/x-bzip2") << KCompressionDevice::None; 0343 #endif 0344 // indirect compressed mimetypes 0345 QTest::newRow("application/x-gzdvi") << QString::fromLatin1("application/x-gzdvi") << KCompressionDevice::GZip; 0346 0347 // non-compressed mimetypes 0348 QTest::newRow("text/plain") << QString::fromLatin1("text/plain") << KCompressionDevice::None; 0349 QTest::newRow("application/x-tar") << QString::fromLatin1("application/x-tar") << KCompressionDevice::None; 0350 } 0351 0352 void KFilterTest::test_findFilterByMimeType() 0353 { 0354 QFETCH(QString, mimeType); 0355 QFETCH(KCompressionDevice::CompressionType, type); 0356 0357 KCompressionDevice::CompressionType compressionType = KCompressionDevice::compressionTypeForMimeType(mimeType); 0358 QCOMPARE(compressionType, type); 0359 } 0360 0361 static void getCompressedData(QByteArray &data, QByteArray &compressedData) 0362 { 0363 data = "Hello world, this is a test for deflate, from bug 114830 / 117683"; 0364 compressedData.resize(long(data.size() * 1.1f) + 12L); // requirements of zlib::compress2 0365 unsigned long out_bufferlen = compressedData.size(); 0366 const int ret = compress2((Bytef *)compressedData.data(), &out_bufferlen, (const Bytef *)data.constData(), data.size(), 1); 0367 QCOMPARE(ret, Z_OK); 0368 compressedData.resize(out_bufferlen); 0369 } 0370 0371 void KFilterTest::test_deflateWithZlibHeader() 0372 { 0373 QByteArray data; 0374 QByteArray deflatedData; 0375 getCompressedData(data, deflatedData); 0376 0377 #if 0 // Can't use KFilterDev for this, we need to call KGzipFilter::init(QIODevice::ReadOnly, KGzipFilter::ZlibHeader); 0378 QBuffer buffer(&deflatedData); 0379 QIODevice *flt = KFilterDev::device(&buffer, "application/gzip", false); 0380 static_cast<KFilterDev *>(flt)->setSkipHeaders(); 0381 bool ok = flt->open(QIODevice::ReadOnly); 0382 QVERIFY(ok); 0383 const QByteArray read = flt->readAll(); 0384 #else 0385 // Copied from HTTPFilter (which isn't linked into any kdelibs library) 0386 KFilterBase *mFilterDevice = KCompressionDevice::filterForCompressionType(KCompressionDevice::GZip); 0387 mFilterDevice->setFilterFlags(KFilterBase::ZlibHeaders); 0388 mFilterDevice->init(QIODevice::ReadOnly); 0389 0390 mFilterDevice->setInBuffer(deflatedData.constData(), deflatedData.size()); 0391 char buf[8192]; 0392 mFilterDevice->setOutBuffer(buf, sizeof(buf)); 0393 KFilterBase::Result result = mFilterDevice->uncompress(); 0394 QCOMPARE(result, KFilterBase::End); 0395 const int bytesOut = sizeof(buf) - mFilterDevice->outBufferAvailable(); 0396 QVERIFY(bytesOut); 0397 QByteArray read(buf, bytesOut); 0398 mFilterDevice->terminate(); 0399 delete mFilterDevice; 0400 #endif 0401 QCOMPARE(QString::fromLatin1(read.constData()), QString::fromLatin1(data.constData())); // more readable output than the line below 0402 QCOMPARE(read, data); 0403 0404 // For the same test with HTTPFilter: see httpfiltertest.cpp 0405 } 0406 0407 void KFilterTest::test_pushData() // ### UNFINISHED 0408 { 0409 // HTTPFilter says KFilterDev doesn't support the case where compressed data 0410 // is arriving in chunks. Let's test that. 0411 QFile file(pathgz); 0412 QVERIFY(file.open(QIODevice::ReadOnly)); 0413 const QByteArray compressed = file.readAll(); 0414 const int firstChunkSize = compressed.size() / 2; 0415 QByteArray firstData(compressed.constData(), firstChunkSize); 0416 QBuffer inBuffer(&firstData); 0417 QVERIFY(inBuffer.open(QIODevice::ReadWrite)); 0418 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(QString::fromLatin1("application/gzip")); 0419 KCompressionDevice flt(&inBuffer, false, type); 0420 QVERIFY(flt.open(QIODevice::ReadOnly)); 0421 QByteArray read = flt.readAll(); 0422 qDebug() << QString::fromLatin1(read.constData()); 0423 0424 // And later... 0425 inBuffer.write(QByteArray(compressed.data() + firstChunkSize, compressed.size() - firstChunkSize)); 0426 QCOMPARE(inBuffer.data().size(), compressed.size()); 0427 read += flt.readAll(); 0428 qDebug() << QString::fromLatin1(read.constData()); 0429 // ### indeed, doesn't work currently. So we use HTTPFilter instead, for now. 0430 } 0431 0432 void KFilterTest::test_saveFile_data() 0433 { 0434 QTest::addColumn<QString>("fileName"); 0435 QTest::addColumn<KCompressionDevice::CompressionType>("compressionType"); 0436 0437 QTest::newRow("gz") << "test_saveFile.gz" << KCompressionDevice::GZip; 0438 QTest::newRow("none") << "test_saveFile" << KCompressionDevice::None; 0439 } 0440 0441 void KFilterTest::test_saveFile() 0442 { 0443 QFETCH(QString, fileName); 0444 QFETCH(KCompressionDevice::CompressionType, compressionType); 0445 0446 int numLines = 1000; 0447 const QString lineTemplate = QStringLiteral("Hello world, this is the text for line %1"); 0448 const QString currentdir = QDir::currentPath(); 0449 const QString outFile = QDir::currentPath() + '/' + fileName; 0450 { 0451 QSaveFile file(outFile); 0452 file.setDirectWriteFallback(true); 0453 QVERIFY(file.open(QIODevice::WriteOnly)); 0454 KCompressionDevice device(&file, false, compressionType); 0455 QVERIFY(device.open(QIODevice::WriteOnly)); 0456 QTextStream stream(&device); 0457 for (int i = 0; i < numLines; ++i) { 0458 stream << lineTemplate.arg(i); 0459 stream << QString("\n"); 0460 } 0461 stream.flush(); 0462 QCOMPARE(stream.status(), QTextStream::Ok); 0463 // device.write("The data to be compressed"); 0464 device.close(); 0465 QVERIFY(file.commit()); 0466 } 0467 QVERIFY(QFile::exists(outFile)); 0468 KCompressionDevice reader(outFile, compressionType); 0469 QVERIFY(reader.open(QIODevice::ReadOnly)); 0470 QString expectedFullData; 0471 for (int i = 0; i < numLines; ++i) { 0472 QCOMPARE(QString::fromUtf8(reader.readLine()), QString(lineTemplate.arg(i) + '\n')); 0473 expectedFullData += QString(lineTemplate.arg(i) + '\n'); 0474 } 0475 KCompressionDevice otherReader(outFile); 0476 QVERIFY(otherReader.open(QIODevice::ReadOnly)); 0477 QCOMPARE(QString::fromLatin1(otherReader.readAll()), expectedFullData); 0478 QVERIFY(otherReader.atEnd()); 0479 } 0480 0481 void KFilterTest::test_twofilesgztogether() 0482 { 0483 // Reported as 232843 0484 // twofiles generated with 0485 // echo foo > foo; echo bar > bar ; gzip -c foo > twofiles.gz; gzip -c bar >> twofiles.gz 0486 // as documented in the gzip manpage 0487 QString data = QFINDTESTDATA("data/twofiles.gz"); 0488 KCompressionDevice dev(data); 0489 QVERIFY(dev.open(QIODevice::ReadOnly)); 0490 QByteArray extractedData = dev.readAll(); 0491 QByteArray expectedData{"foo\nbar\n"}; 0492 QCOMPARE(extractedData, expectedData); 0493 } 0494 0495 void KFilterTest::test_threefilesgztogether() 0496 { 0497 // Generated similarly to the one above 0498 // This catches the case where there's more than two streams available in the same buffer fed to KGzipFilter 0499 QString data = QFINDTESTDATA("data/threefiles.gz"); 0500 KCompressionDevice dev(data); 0501 QVERIFY(dev.open(QIODevice::ReadOnly)); 0502 QByteArray extractedData = dev.readAll(); 0503 QByteArray expectedData{"foo\nbar\nbaz\n"}; 0504 QCOMPARE(extractedData, expectedData); 0505 } 0506 0507 #include "moc_kfiltertest.cpp"