Warning, file /frameworks/karchive/autotests/kfiltertest.cpp was not indexed or was modified since last indexation (in which case cross-reference links may be missing, inaccurate or erroneous).
0001 /* 0002 * SPDX-FileCopyrightText: 2002-2005 David Faure <faure@kde.org> 0003 * 0004 * SPDX-License-Identifier: LGPL-2.0-or-later 0005 */ 0006 0007 #include "kfiltertest.h" 0008 0009 #include <QBuffer> 0010 #include <QTest> 0011 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) 0012 #include <QTextCodec> 0013 #endif 0014 #include <QRandomGenerator> 0015 #include <QSaveFile> 0016 0017 #include "kcompressiondevice.h" 0018 #include "kfilterbase.h" 0019 #include <QDebug> 0020 #include <QDir> 0021 #include <QFile> 0022 #include <QFileInfo> 0023 #include <QTextStream> 0024 #include <config-compression.h> 0025 #include <zlib.h> 0026 0027 #ifdef Q_OS_UNIX 0028 #include <limits.h> 0029 #include <unistd.h> 0030 #endif 0031 0032 QTEST_MAIN(KFilterTest) 0033 0034 void KFilterTest::initTestCase() 0035 { 0036 qRegisterMetaType<KCompressionDevice::CompressionType>(); 0037 const QString currentdir = QDir::currentPath(); 0038 pathgz = currentdir + "/test.gz"; 0039 pathbz2 = currentdir + "/test.bz2"; 0040 pathxz = currentdir + "/test.xz"; 0041 pathnone = currentdir + "/test.txt"; 0042 pathzstd = currentdir + "/test.zst"; 0043 0044 // warning, update the COMPAREs in test_block_write() if changing the test data... 0045 testData = "hello world\n"; 0046 } 0047 0048 void KFilterTest::test_block_write(const QString &fileName, const QByteArray &data, int nTimes) 0049 { 0050 KCompressionDevice dev(fileName); 0051 bool ok = dev.open(QIODevice::WriteOnly); 0052 QVERIFY(ok); 0053 0054 for (int i = 0; i < nTimes; ++i) { 0055 const int ret = dev.write(data); 0056 QCOMPARE(ret, data.size()); 0057 } 0058 0059 dev.close(); 0060 0061 QVERIFY(QFile::exists(fileName)); 0062 } 0063 0064 void KFilterTest::test_block_write() 0065 { 0066 // First we do it 50 times, to make sure the compressor 0067 // can "reuse" existing data, i.e. zstd used to have a bug 0068 // that made this use 513 bytes instead of 28 0069 qDebug() << " -- test_block_write gzip -- "; 0070 test_block_write(pathgz, testData, 50); 0071 QCOMPARE(QFileInfo(pathgz).size(), 40LL); // size of test.gz 0072 0073 #if HAVE_BZIP2_SUPPORT 0074 qDebug() << " -- test_block_write bzip2 -- "; 0075 test_block_write(pathbz2, testData, 50); 0076 QCOMPARE(QFileInfo(pathbz2).size(), 64LL); // size of test.bz2 0077 #endif 0078 0079 #if HAVE_XZ_SUPPORT 0080 qDebug() << " -- test_block_write xz -- "; 0081 test_block_write(pathxz, testData, 50); 0082 QCOMPARE(QFileInfo(pathxz).size(), 84LL); // size of test.lzma 0083 #endif 0084 0085 qDebug() << " -- test_block_write none -- "; 0086 test_block_write(pathnone, testData, 50); 0087 QCOMPARE(QFileInfo(pathnone).size(), 600LL); // size of test.txt 0088 0089 #if HAVE_ZSTD_SUPPORT 0090 qDebug() << " -- test_block_write zstd -- "; 0091 test_block_write(pathzstd, testData, 50); 0092 QCOMPARE(QFileInfo(pathzstd).size(), 28LL); // size of test.zst 0093 #endif 0094 0095 // Then do the write data just once because test_block_read 0096 // depends on this 0097 qDebug() << " -- test_block_write gzip -- "; 0098 test_block_write(pathgz, testData); 0099 QCOMPARE(QFileInfo(pathgz).size(), 33LL); // size of test.gz 0100 0101 #if HAVE_BZIP2_SUPPORT 0102 qDebug() << " -- test_block_write bzip2 -- "; 0103 test_block_write(pathbz2, testData); 0104 QCOMPARE(QFileInfo(pathbz2).size(), 52LL); // size of test.bz2 0105 #endif 0106 0107 #if HAVE_XZ_SUPPORT 0108 qDebug() << " -- test_block_write xz -- "; 0109 test_block_write(pathxz, testData); 0110 QCOMPARE(QFileInfo(pathxz).size(), 64LL); // size of test.lzma 0111 #endif 0112 0113 qDebug() << " -- test_block_write none -- "; 0114 test_block_write(pathnone, testData); 0115 QCOMPARE(QFileInfo(pathnone).size(), 12LL); // size of test.txt 0116 0117 #if HAVE_ZSTD_SUPPORT 0118 qDebug() << " -- test_block_write zstd -- "; 0119 test_block_write(pathzstd, testData); 0120 QCOMPARE(QFileInfo(pathzstd).size(), 21LL); // size of test.zst 0121 #endif 0122 } 0123 0124 void KFilterTest::test_biggerWrites() 0125 { 0126 const QString currentdir = QDir::currentPath(); 0127 const QString outFile = currentdir + "/test_big.gz"; 0128 // Find the out-of-bounds from #157706/#188415 0129 QByteArray data; 0130 data.reserve(10000); 0131 auto *generator = QRandomGenerator::global(); 0132 // Prepare test data 0133 for (int i = 0; i < 8170; ++i) { 0134 data.append((char)(generator->bounded(256))); 0135 } 0136 QCOMPARE(data.size(), 8170); 0137 // 8170 random bytes compress to 8194 bytes due to the gzip header/footer. 0138 // Now we can go one by one until we pass 8192. 0139 // On 32 bit systems it crashed with data.size()=8173, before the "no room for footer yet" fix. 0140 int compressedSize = 0; 0141 while (compressedSize < 8200) { 0142 test_block_write(outFile, data); 0143 compressedSize = QFileInfo(outFile).size(); 0144 qDebug() << data.size() << "compressed into" << compressedSize; 0145 // Test data is valid 0146 test_readall(outFile, QString::fromLatin1("application/gzip"), data); 0147 0148 data.append((char)(generator->bounded(256))); 0149 } 0150 } 0151 0152 void KFilterTest::test_block_read(const QString &fileName) 0153 { 0154 KCompressionDevice dev(fileName); 0155 bool ok = dev.open(QIODevice::ReadOnly); 0156 QVERIFY(ok); 0157 0158 QByteArray array(1024, '\0'); 0159 QByteArray read; 0160 int n; 0161 while ((n = dev.read(array.data(), array.size()))) { 0162 QVERIFY(n > 0); 0163 read += QByteArray(array.constData(), n); 0164 // qDebug() << "read returned " << n; 0165 // qDebug() << "read='" << read << "'"; 0166 0167 // pos() has no real meaning on sequential devices 0168 // Ah, but kzip uses kfilterdev as a non-sequential device... 0169 0170 QCOMPARE((int)dev.pos(), (int)read.size()); 0171 // qDebug() << "dev.at = " << dev->at(); 0172 } 0173 QCOMPARE(read, testData); 0174 0175 // Test seeking back 0176 ok = dev.seek(0); 0177 // test readAll 0178 read = dev.readAll(); 0179 QCOMPARE(read.size(), testData.size()); 0180 QCOMPARE(read, testData); 0181 0182 dev.close(); 0183 } 0184 0185 void KFilterTest::test_block_read() 0186 { 0187 qDebug() << " -- test_block_read gzip -- "; 0188 test_block_read(pathgz); 0189 #if HAVE_BZIP2_SUPPORT 0190 qDebug() << " -- test_block_read bzip2 -- "; 0191 test_block_read(pathbz2); 0192 #endif 0193 #if HAVE_XZ_SUPPORT 0194 qDebug() << " -- test_block_read lzma -- "; 0195 test_block_read(pathxz); 0196 #endif 0197 qDebug() << " -- test_block_read none -- "; 0198 test_block_read(pathnone); 0199 #if HAVE_ZSTD_SUPPORT 0200 qDebug() << " -- test_block_read zstd -- "; 0201 test_block_read(pathzstd); 0202 #endif 0203 } 0204 0205 void KFilterTest::test_getch(const QString &fileName) 0206 { 0207 KCompressionDevice dev(fileName); 0208 bool ok = dev.open(QIODevice::ReadOnly); 0209 QVERIFY(ok); 0210 QByteArray read; 0211 char ch; 0212 while (dev.getChar(&ch)) { 0213 // printf("%c",ch); 0214 read += ch; 0215 } 0216 dev.close(); 0217 QCOMPARE(read, testData); 0218 } 0219 0220 void KFilterTest::test_getch() 0221 { 0222 qDebug() << " -- test_getch gzip -- "; 0223 test_getch(pathgz); 0224 #if HAVE_BZIP2_SUPPORT 0225 qDebug() << " -- test_getch bzip2 -- "; 0226 test_getch(pathbz2); 0227 #endif 0228 #if HAVE_XZ_SUPPORT 0229 qDebug() << " -- test_getch lzma -- "; 0230 test_getch(pathxz); 0231 #endif 0232 qDebug() << " -- test_getch none -- "; 0233 test_getch(pathnone); 0234 #if HAVE_ZSTD_SUPPORT 0235 qDebug() << " -- test_getch zstd -- "; 0236 test_getch(pathzstd); 0237 #endif 0238 } 0239 0240 void KFilterTest::test_textstream(const QString &fileName) 0241 { 0242 KCompressionDevice dev(fileName); 0243 bool ok = dev.open(QIODevice::ReadOnly); 0244 QVERIFY(ok); 0245 QTextStream ts(&dev); 0246 QString readStr = ts.readAll(); 0247 dev.close(); 0248 0249 QByteArray read = readStr.toLatin1(); 0250 QCOMPARE(read, testData); 0251 } 0252 0253 void KFilterTest::test_textstream() 0254 { 0255 qDebug() << " -- test_textstream gzip -- "; 0256 test_textstream(pathgz); 0257 #if HAVE_BZIP2_SUPPORT 0258 qDebug() << " -- test_textstream bzip2 -- "; 0259 test_textstream(pathbz2); 0260 #endif 0261 #if HAVE_XZ_SUPPORT 0262 qDebug() << " -- test_textstream lzma -- "; 0263 test_textstream(pathxz); 0264 #endif 0265 qDebug() << " -- test_textstream none -- "; 0266 test_textstream(pathnone); 0267 #if HAVE_ZSTD_SUPPORT 0268 qDebug() << " -- test_textstream zstd -- "; 0269 test_textstream(pathzstd); 0270 #endif 0271 } 0272 0273 void KFilterTest::test_readall(const QString &fileName, const QString &mimeType, const QByteArray &expectedData) 0274 { 0275 QFile file(fileName); 0276 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(mimeType); 0277 KCompressionDevice flt(&file, false, type); 0278 bool ok = flt.open(QIODevice::ReadOnly); 0279 QVERIFY(ok); 0280 const QByteArray read = flt.readAll(); 0281 QCOMPARE(read.size(), expectedData.size()); 0282 QCOMPARE(read, expectedData); 0283 0284 // Now using QBuffer 0285 file.seek(0); 0286 QByteArray compressedData = file.readAll(); 0287 QVERIFY(!compressedData.isEmpty()); 0288 QBuffer buffer(&compressedData); 0289 KCompressionDevice device(&buffer, false, type); 0290 QVERIFY(device.open(QIODevice::ReadOnly)); 0291 QCOMPARE(device.readAll(), expectedData); 0292 } 0293 0294 void KFilterTest::test_readall() 0295 { 0296 qDebug() << " -- test_readall gzip -- "; 0297 test_readall(pathgz, QString::fromLatin1("application/gzip"), testData); 0298 #if HAVE_BZIP2_SUPPORT 0299 qDebug() << " -- test_readall bzip2 -- "; 0300 test_readall(pathbz2, QString::fromLatin1("application/x-bzip"), testData); 0301 #endif 0302 #if HAVE_XZ_SUPPORT 0303 qDebug() << " -- test_readall lzma -- "; 0304 test_readall(pathxz, QString::fromLatin1("application/x-xz"), testData); 0305 #endif 0306 qDebug() << " -- test_readall gzip-derived -- "; 0307 test_readall(pathgz, QString::fromLatin1("image/svg+xml-compressed"), testData); 0308 0309 qDebug() << " -- test_readall none -- "; 0310 test_readall(pathnone, QString::fromLatin1("text/plain"), testData); 0311 0312 #if HAVE_ZSTD_SUPPORT 0313 qDebug() << " -- test_readall zstd -- "; 0314 test_readall(pathzstd, QString::fromLatin1("application/zstd"), testData); 0315 #endif 0316 } 0317 0318 void KFilterTest::test_uncompressed() 0319 { 0320 // Can KCompressionDevice handle uncompressed data even when using gzip decompression? 0321 qDebug() << " -- test_uncompressed -- "; 0322 QBuffer buffer(&testData); 0323 buffer.open(QIODevice::ReadOnly); 0324 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(QString::fromLatin1("application/gzip")); 0325 KCompressionDevice flt(&buffer, false, type); 0326 bool ok = flt.open(QIODevice::ReadOnly); 0327 QVERIFY(ok); 0328 QByteArray read = flt.readAll(); 0329 QCOMPARE(read.size(), testData.size()); 0330 QCOMPARE(read, testData); 0331 } 0332 0333 void KFilterTest::test_findFilterByMimeType_data() 0334 { 0335 QTest::addColumn<QString>("mimeType"); 0336 QTest::addColumn<KCompressionDevice::CompressionType>("type"); 0337 0338 // direct mimetype name 0339 QTest::newRow("application/gzip") << QString::fromLatin1("application/gzip") << KCompressionDevice::GZip; 0340 #if HAVE_BZIP2_SUPPORT 0341 QTest::newRow("application/x-bzip") << QString::fromLatin1("application/x-bzip") << KCompressionDevice::BZip2; 0342 QTest::newRow("application/x-bzip2") << QString::fromLatin1("application/x-bzip2") << KCompressionDevice::BZip2; 0343 #else 0344 QTest::newRow("application/x-bzip") << QString::fromLatin1("application/x-bzip") << KCompressionDevice::None; 0345 QTest::newRow("application/x-bzip2") << QString::fromLatin1("application/x-bzip2") << KCompressionDevice::None; 0346 #endif 0347 // indirect compressed mimetypes 0348 QTest::newRow("application/x-gzdvi") << QString::fromLatin1("application/x-gzdvi") << KCompressionDevice::GZip; 0349 0350 // non-compressed mimetypes 0351 QTest::newRow("text/plain") << QString::fromLatin1("text/plain") << KCompressionDevice::None; 0352 QTest::newRow("application/x-tar") << QString::fromLatin1("application/x-tar") << KCompressionDevice::None; 0353 } 0354 0355 void KFilterTest::test_findFilterByMimeType() 0356 { 0357 QFETCH(QString, mimeType); 0358 QFETCH(KCompressionDevice::CompressionType, type); 0359 0360 KCompressionDevice::CompressionType compressionType = KCompressionDevice::compressionTypeForMimeType(mimeType); 0361 QCOMPARE(compressionType, type); 0362 } 0363 0364 static void getCompressedData(QByteArray &data, QByteArray &compressedData) 0365 { 0366 data = "Hello world, this is a test for deflate, from bug 114830 / 117683"; 0367 compressedData.resize(long(data.size() * 1.1f) + 12L); // requirements of zlib::compress2 0368 unsigned long out_bufferlen = compressedData.size(); 0369 const int ret = compress2((Bytef *)compressedData.data(), &out_bufferlen, (const Bytef *)data.constData(), data.size(), 1); 0370 QCOMPARE(ret, Z_OK); 0371 compressedData.resize(out_bufferlen); 0372 } 0373 0374 void KFilterTest::test_deflateWithZlibHeader() 0375 { 0376 QByteArray data; 0377 QByteArray deflatedData; 0378 getCompressedData(data, deflatedData); 0379 0380 #if 0 // Can't use KFilterDev for this, we need to call KGzipFilter::init(QIODevice::ReadOnly, KGzipFilter::ZlibHeader); 0381 QBuffer buffer(&deflatedData); 0382 QIODevice *flt = KFilterDev::device(&buffer, "application/gzip", false); 0383 static_cast<KFilterDev *>(flt)->setSkipHeaders(); 0384 bool ok = flt->open(QIODevice::ReadOnly); 0385 QVERIFY(ok); 0386 const QByteArray read = flt->readAll(); 0387 #else 0388 // Copied from HTTPFilter (which isn't linked into any kdelibs library) 0389 KFilterBase *mFilterDevice = KCompressionDevice::filterForCompressionType(KCompressionDevice::GZip); 0390 mFilterDevice->setFilterFlags(KFilterBase::ZlibHeaders); 0391 mFilterDevice->init(QIODevice::ReadOnly); 0392 0393 mFilterDevice->setInBuffer(deflatedData.constData(), deflatedData.size()); 0394 char buf[8192]; 0395 mFilterDevice->setOutBuffer(buf, sizeof(buf)); 0396 KFilterBase::Result result = mFilterDevice->uncompress(); 0397 QCOMPARE(result, KFilterBase::End); 0398 const int bytesOut = sizeof(buf) - mFilterDevice->outBufferAvailable(); 0399 QVERIFY(bytesOut); 0400 QByteArray read(buf, bytesOut); 0401 mFilterDevice->terminate(); 0402 delete mFilterDevice; 0403 #endif 0404 QCOMPARE(QString::fromLatin1(read.constData()), QString::fromLatin1(data.constData())); // more readable output than the line below 0405 QCOMPARE(read, data); 0406 0407 // For the same test with HTTPFilter: see httpfiltertest.cpp 0408 } 0409 0410 void KFilterTest::test_pushData() // ### UNFINISHED 0411 { 0412 // HTTPFilter says KFilterDev doesn't support the case where compressed data 0413 // is arriving in chunks. Let's test that. 0414 QFile file(pathgz); 0415 QVERIFY(file.open(QIODevice::ReadOnly)); 0416 const QByteArray compressed = file.readAll(); 0417 const int firstChunkSize = compressed.size() / 2; 0418 QByteArray firstData(compressed.constData(), firstChunkSize); 0419 QBuffer inBuffer(&firstData); 0420 QVERIFY(inBuffer.open(QIODevice::ReadWrite)); 0421 KCompressionDevice::CompressionType type = KCompressionDevice::compressionTypeForMimeType(QString::fromLatin1("application/gzip")); 0422 KCompressionDevice flt(&inBuffer, false, type); 0423 QVERIFY(flt.open(QIODevice::ReadOnly)); 0424 QByteArray read = flt.readAll(); 0425 qDebug() << QString::fromLatin1(read.constData()); 0426 0427 // And later... 0428 inBuffer.write(QByteArray(compressed.data() + firstChunkSize, compressed.size() - firstChunkSize)); 0429 QCOMPARE(inBuffer.data().size(), compressed.size()); 0430 read += flt.readAll(); 0431 qDebug() << QString::fromLatin1(read.constData()); 0432 // ### indeed, doesn't work currently. So we use HTTPFilter instead, for now. 0433 } 0434 0435 void KFilterTest::test_saveFile_data() 0436 { 0437 QTest::addColumn<QString>("fileName"); 0438 QTest::addColumn<KCompressionDevice::CompressionType>("compressionType"); 0439 0440 QTest::newRow("gz") << "test_saveFile.gz" << KCompressionDevice::GZip; 0441 QTest::newRow("none") << "test_saveFile" << KCompressionDevice::None; 0442 } 0443 0444 void KFilterTest::test_saveFile() 0445 { 0446 QFETCH(QString, fileName); 0447 QFETCH(KCompressionDevice::CompressionType, compressionType); 0448 0449 int numLines = 1000; 0450 const QString lineTemplate = QStringLiteral("Hello world, this is the text for line %1"); 0451 const QString currentdir = QDir::currentPath(); 0452 const QString outFile = QDir::currentPath() + '/' + fileName; 0453 { 0454 QSaveFile file(outFile); 0455 file.setDirectWriteFallback(true); 0456 QVERIFY(file.open(QIODevice::WriteOnly)); 0457 KCompressionDevice device(&file, false, compressionType); 0458 QVERIFY(device.open(QIODevice::WriteOnly)); 0459 QTextStream stream(&device); 0460 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0) 0461 stream.setCodec(QTextCodec::codecForName("UTF-8")); 0462 #endif 0463 for (int i = 0; i < numLines; ++i) { 0464 stream << lineTemplate.arg(i); 0465 stream << QString("\n"); 0466 } 0467 stream.flush(); 0468 QCOMPARE(stream.status(), QTextStream::Ok); 0469 // device.write("The data to be compressed"); 0470 device.close(); 0471 QVERIFY(file.commit()); 0472 } 0473 QVERIFY(QFile::exists(outFile)); 0474 KCompressionDevice reader(outFile, compressionType); 0475 QVERIFY(reader.open(QIODevice::ReadOnly)); 0476 QString expectedFullData; 0477 for (int i = 0; i < numLines; ++i) { 0478 QCOMPARE(QString::fromUtf8(reader.readLine()), QString(lineTemplate.arg(i) + '\n')); 0479 expectedFullData += QString(lineTemplate.arg(i) + '\n'); 0480 } 0481 KCompressionDevice otherReader(outFile); 0482 QVERIFY(otherReader.open(QIODevice::ReadOnly)); 0483 QCOMPARE(QString::fromLatin1(otherReader.readAll()), expectedFullData); 0484 QVERIFY(otherReader.atEnd()); 0485 } 0486 0487 void KFilterTest::test_twofilesgztogether() 0488 { 0489 // Reported as 232843 0490 // twofiles generated with 0491 // echo foo > foo; echo bar > bar ; gzip -c foo > twofiles.gz; gzip -c bar >> twofiles.gz 0492 // as documented in the gzip manpage 0493 QString data = QFINDTESTDATA("data/twofiles.gz"); 0494 KCompressionDevice dev(data); 0495 QVERIFY(dev.open(QIODevice::ReadOnly)); 0496 QByteArray extractedData = dev.readAll(); 0497 QByteArray expectedData{"foo\nbar\n"}; 0498 QCOMPARE(extractedData, expectedData); 0499 } 0500 0501 void KFilterTest::test_threefilesgztogether() 0502 { 0503 // Generated similarly to the one above 0504 // This catches the case where there's more than two streams available in the same buffer fed to KGzipFilter 0505 QString data = QFINDTESTDATA("data/threefiles.gz"); 0506 KCompressionDevice dev(data); 0507 QVERIFY(dev.open(QIODevice::ReadOnly)); 0508 QByteArray extractedData = dev.readAll(); 0509 QByteArray expectedData{"foo\nbar\nbaz\n"}; 0510 QCOMPARE(extractedData, expectedData); 0511 } 0512 0513 #include "moc_kfiltertest.cpp"