1/* This file is part of the KDE libraries
2 SPDX-FileCopyrightText: 2000 David Faure <faure@kde.org>
3 SPDX-FileCopyrightText: 2011 Mario Bensi <mbensi@ipsquad.net>
4
5 SPDX-License-Identifier: LGPL-2.0-or-later
6*/
7
8#include "kcompressiondevice.h"
9#include "kcompressiondevice_p.h"
10#include "kfilterbase.h"
11#include "loggingcategory.h"
12#include "kgzipfilter.h"
13#include "knonefilter.h"
14
15#include "config-compression.h"
16
17#if HAVE_BZIP2_SUPPORT
18#include "kbzip2filter.h"
19#endif
20#if HAVE_XZ_SUPPORT
21#include "klzfilter.h"
22#include "kxzfilter.h"
23#endif
24#if HAVE_ZSTD_SUPPORT
25#include "kzstdfilter.h"
26#endif
27
28#include <QDebug>
29#include <QFile>
30#include <QMimeDatabase>
31
32#include <assert.h>
33#include <stdio.h> // for EOF
34#include <stdlib.h>
35
36class KCompressionDevicePrivate
37{
38public:
39 KCompressionDevicePrivate(KCompressionDevice *qq)
40 : bNeedHeader(true)
41 , bSkipHeaders(false)
42 , bOpenedUnderlyingDevice(false)
43 , type(KCompressionDevice::None)
44 , errorCode(QFileDevice::NoError)
45 , deviceReadPos(0)
46 , q(qq)
47 {
48 }
49
50 void propagateErrorCode();
51
52 bool bNeedHeader;
53 bool bSkipHeaders;
54 bool bOpenedUnderlyingDevice;
55 QByteArray buffer; // Used as 'input buffer' when reading, as 'output buffer' when writing
56 QByteArray origFileName;
57 KFilterBase::Result result;
58 KFilterBase *filter;
59 KCompressionDevice::CompressionType type;
60 std::optional<qint64> size;
61 QFileDevice::FileError errorCode;
62 qint64 deviceReadPos;
63 KCompressionDevice *q;
64};
65
66void KCompressionDevicePrivate::propagateErrorCode()
67{
68 QIODevice *dev = filter->device();
69 if (QFileDevice *fileDev = qobject_cast<QFileDevice *>(object: dev)) {
70 if (fileDev->error() != QFileDevice::NoError) {
71 errorCode = fileDev->error();
72 q->setErrorString(dev->errorString());
73 }
74 }
75 // ... we have no generic way to propagate errors from other kinds of iodevices. Sucks, heh? :(
76}
77
78static KCompressionDevice::CompressionType findCompressionByFileName(const QString &fileName)
79{
80 if (fileName.endsWith(s: QLatin1String(".gz"), cs: Qt::CaseInsensitive)) {
81 return KCompressionDevice::GZip;
82 }
83#if HAVE_BZIP2_SUPPORT
84 if (fileName.endsWith(s: QLatin1String(".bz2"), cs: Qt::CaseInsensitive)) {
85 return KCompressionDevice::BZip2;
86 }
87#endif
88#if HAVE_XZ_SUPPORT
89 if (fileName.endsWith(s: QLatin1String(".lz"), cs: Qt::CaseInsensitive)) {
90 return KCompressionDevice::Lz;
91 }
92 if (fileName.endsWith(s: QLatin1String(".lzma"), cs: Qt::CaseInsensitive) || fileName.endsWith(s: QLatin1String(".xz"), cs: Qt::CaseInsensitive)) {
93 return KCompressionDevice::Xz;
94 }
95#endif
96#if HAVE_ZSTD_SUPPORT
97 if (fileName.endsWith(s: QLatin1String(".zst"), cs: Qt::CaseInsensitive)) {
98 return KCompressionDevice::Zstd;
99 }
100#endif
101 else {
102 // not a warning, since this is called often with other MIME types (see #88574)...
103 // maybe we can avoid that though?
104 // qCDebug(KArchiveLog) << "findCompressionByFileName : no compression found for " << fileName;
105 }
106
107 return KCompressionDevice::None;
108}
109
110KCompressionDevice::CompressionType KCompressionDevice::compressionTypeForMimeType(const QString &mimeType)
111{
112 if (mimeType == QLatin1String("application/gzip") //
113 || mimeType == QLatin1String("application/x-gzip") // legacy name, kept for compatibility
114 ) {
115 return KCompressionDevice::GZip;
116 }
117#if HAVE_BZIP2_SUPPORT
118 if (mimeType == QLatin1String("application/x-bzip") //
119 || mimeType == QLatin1String("application/x-bzip2") // old name, kept for compatibility
120 ) {
121 return KCompressionDevice::BZip2;
122 }
123#endif
124#if HAVE_XZ_SUPPORT
125 if (mimeType == QLatin1String("application/x-lzip")) {
126 return KCompressionDevice::Lz;
127 }
128 if (mimeType == QLatin1String("application/x-lzma") // legacy name, still used
129 || mimeType == QLatin1String("application/x-xz") // current naming
130 ) {
131 return KCompressionDevice::Xz;
132 }
133#endif
134#if HAVE_ZSTD_SUPPORT
135 if (mimeType == QLatin1String("application/zstd")) {
136 return KCompressionDevice::Zstd;
137 }
138#endif
139 QMimeDatabase db;
140 const QMimeType mime = db.mimeTypeForName(nameOrAlias: mimeType);
141 if (mime.isValid()) {
142 // use legacy MIME type for now, see comment in impl. of KTar(const QString &, const QString &_mimetype)
143 if (mime.inherits(QStringLiteral("application/x-gzip"))) {
144 return KCompressionDevice::GZip;
145 }
146#if HAVE_BZIP2_SUPPORT
147 if (mime.inherits(QStringLiteral("application/x-bzip"))) {
148 return KCompressionDevice::BZip2;
149 }
150#endif
151#if HAVE_XZ_SUPPORT
152 if (mime.inherits(QStringLiteral("application/x-lzip"))) {
153 return KCompressionDevice::Lz;
154 }
155
156 if (mime.inherits(QStringLiteral("application/x-lzma"))) {
157 return KCompressionDevice::Xz;
158 }
159
160 if (mime.inherits(QStringLiteral("application/x-xz"))) {
161 return KCompressionDevice::Xz;
162 }
163#endif
164 }
165
166 // not a warning, since this is called often with other MIME types (see #88574)...
167 // maybe we can avoid that though?
168 // qCDebug(KArchiveLog) << "no compression found for" << mimeType;
169 return KCompressionDevice::None;
170}
171
172KFilterBase *KCompressionDevice::filterForCompressionType(KCompressionDevice::CompressionType type)
173{
174 switch (type) {
175 case KCompressionDevice::GZip:
176 return new KGzipFilter;
177 case KCompressionDevice::BZip2:
178#if HAVE_BZIP2_SUPPORT
179 return new KBzip2Filter;
180#else
181 return nullptr;
182#endif
183 case KCompressionDevice::Lz:
184#if HAVE_XZ_SUPPORT
185 return new KLzFilter;
186#else
187 return nullptr;
188#endif
189 case KCompressionDevice::Xz:
190#if HAVE_XZ_SUPPORT
191 return new KXzFilter;
192#else
193 return nullptr;
194#endif
195 case KCompressionDevice::None:
196 return new KNoneFilter;
197 case KCompressionDevice::Zstd:
198#if HAVE_ZSTD_SUPPORT
199 return new KZstdFilter;
200#else
201 return nullptr;
202#endif
203 }
204 return nullptr;
205}
206
207KCompressionDevice::KCompressionDevice(QIODevice *inputDevice, bool autoDeleteInputDevice, CompressionType type)
208 : d(new KCompressionDevicePrivate(this))
209{
210 assert(inputDevice);
211 d->filter = filterForCompressionType(type);
212 if (d->filter) {
213 d->type = type;
214 d->filter->setDevice(dev: inputDevice, autodelete: autoDeleteInputDevice);
215 } else if (autoDeleteInputDevice) {
216 delete inputDevice;
217 }
218}
219
220KCompressionDevice::KCompressionDevice(const QString &fileName, CompressionType type)
221 : KCompressionDevice(new QFile(fileName), true, type)
222{
223}
224
225KCompressionDevice::KCompressionDevice(const QString &fileName)
226 : KCompressionDevice(fileName, findCompressionByFileName(fileName))
227{
228}
229
230KCompressionDevice::KCompressionDevice(std::unique_ptr<QIODevice> inputDevice, CompressionType type, std::optional<qint64> size)
231 : KCompressionDevice(inputDevice.release(), true, type)
232{
233 d->size = size;
234}
235
236KCompressionDevice::~KCompressionDevice()
237{
238 if (isOpen()) {
239 close();
240 }
241 delete d->filter;
242 delete d;
243}
244
245KCompressionDevice::CompressionType KCompressionDevice::compressionType() const
246{
247 return d->type;
248}
249
250bool KCompressionDevice::open(QIODevice::OpenMode mode)
251{
252 if (isOpen()) {
253 // qCWarning(KArchiveLog) << "KCompressionDevice::open: device is already open";
254 return true; // QFile returns false, but well, the device -is- open...
255 }
256 if (!d->filter) {
257 return false;
258 }
259 d->bOpenedUnderlyingDevice = false;
260 // qCDebug(KArchiveLog) << mode;
261 if (mode == QIODevice::ReadOnly) {
262 d->buffer.resize(size: 0);
263 } else {
264 d->buffer.resize(BUFFER_SIZE);
265 d->filter->setOutBuffer(data: d->buffer.data(), maxlen: d->buffer.size());
266 }
267 if (!d->filter->device()->isOpen()) {
268 if (!d->filter->device()->open(mode)) {
269 // qCWarning(KArchiveLog) << "KCompressionDevice::open: Couldn't open underlying device";
270 d->propagateErrorCode();
271 return false;
272 }
273 d->bOpenedUnderlyingDevice = true;
274 }
275 d->bNeedHeader = !d->bSkipHeaders;
276 d->filter->setFilterFlags(d->bSkipHeaders ? KFilterBase::NoHeaders : KFilterBase::WithHeaders);
277 if (!d->filter->init(mode: mode & ~QIODevice::Truncate)) {
278 return false;
279 }
280 d->result = KFilterBase::Ok;
281 setOpenMode(mode);
282 return true;
283}
284
285void KCompressionDevice::close()
286{
287 if (!isOpen()) {
288 return;
289 }
290 if (d->filter->mode() == QIODevice::WriteOnly && d->errorCode == QFileDevice::NoError) {
291 write(data: nullptr, len: 0); // finish writing
292 }
293 // qCDebug(KArchiveLog) << "Calling terminate().";
294
295 if (!d->filter->terminate()) {
296 // qCWarning(KArchiveLog) << "KCompressionDevice::close: terminate returned an error";
297 d->errorCode = QFileDevice::UnspecifiedError;
298 }
299 if (d->bOpenedUnderlyingDevice) {
300 QIODevice *dev = d->filter->device();
301 dev->close();
302 d->propagateErrorCode();
303 }
304 setOpenMode(QIODevice::NotOpen);
305}
306
307qint64 KCompressionDevice::size() const
308{
309 if (!d->size.has_value()) {
310 return QIODevice::size();
311 }
312
313 return d->size.value();
314}
315
316QFileDevice::FileError KCompressionDevice::error() const
317{
318 return d->errorCode;
319}
320
321bool KCompressionDevice::seek(qint64 pos)
322{
323 if (d->deviceReadPos == pos) {
324 return QIODevice::seek(pos);
325 }
326
327 // qCDebug(KArchiveLog) << "seek(" << pos << ") called, current pos=" << QIODevice::pos();
328
329 Q_ASSERT(d->filter->mode() == QIODevice::ReadOnly);
330
331 if (pos == 0) {
332 if (!QIODevice::seek(pos)) {
333 return false;
334 }
335
336 // We can forget about the cached data
337 d->bNeedHeader = !d->bSkipHeaders;
338 d->result = KFilterBase::Ok;
339 d->filter->setInBuffer(data: nullptr, size: 0);
340 d->filter->reset();
341 d->deviceReadPos = 0;
342 return d->filter->device()->reset();
343 }
344
345 qint64 bytesToRead;
346 if (d->deviceReadPos < pos) { // we can start from here
347 bytesToRead = pos - d->deviceReadPos;
348 // Since we're going to do a read() below
349 // we need to reset the internal QIODevice pos to the real position we are
350 // so that after read() we are indeed pointing to the pos seek
351 // asked us to be in
352 if (!QIODevice::seek(pos: d->deviceReadPos)) {
353 return false;
354 }
355 } else {
356 // we have to start from 0 ! Ugly and slow, but better than the previous
357 // solution (KTarGz was allocating everything into memory)
358 if (!seek(pos: 0)) { // recursive
359 return false;
360 }
361 bytesToRead = pos;
362 }
363
364 // qCDebug(KArchiveLog) << "reading " << bytesToRead << " dummy bytes";
365 QByteArray dummy(qMin(a: bytesToRead, b: qint64(SEEK_BUFFER_SIZE)), 0);
366 while (bytesToRead > 0) {
367 const qint64 bytesToReadThisTime = qMin(a: bytesToRead, b: qint64(dummy.size()));
368 const bool result = (read(data: dummy.data(), maxlen: bytesToReadThisTime) == bytesToReadThisTime);
369 if (!result) {
370 return false;
371 }
372 bytesToRead -= bytesToReadThisTime;
373 }
374 return true;
375}
376
377bool KCompressionDevice::atEnd() const
378{
379 return (d->type == KCompressionDevice::None || d->result == KFilterBase::End) //
380 && QIODevice::atEnd() // take QIODevice's internal buffer into account
381 && d->filter->device()->atEnd();
382}
383
384qint64 KCompressionDevice::readData(char *data, qint64 maxlen)
385{
386 Q_ASSERT(d->filter->mode() == QIODevice::ReadOnly);
387 // qCDebug(KArchiveLog) << "maxlen=" << maxlen;
388 KFilterBase *filter = d->filter;
389
390 uint dataReceived = 0;
391
392 // We came to the end of the stream
393 if (d->result == KFilterBase::End) {
394 return dataReceived;
395 }
396
397 // If we had an error, return -1.
398 if (d->result != KFilterBase::Ok) {
399 return -1;
400 }
401
402 qint64 availOut = maxlen;
403 filter->setOutBuffer(data, maxlen);
404
405 while (dataReceived < maxlen) {
406 if (filter->inBufferEmpty()) {
407 // Not sure about the best size to set there.
408 // For sure, it should be bigger than the header size (see comment in readHeader)
409 d->buffer.resize(BUFFER_SIZE);
410 // Request data from underlying device
411 int size = filter->device()->read(data: d->buffer.data(), maxlen: d->buffer.size());
412 // qCDebug(KArchiveLog) << "got" << size << "bytes from device";
413 if (size) {
414 filter->setInBuffer(data: d->buffer.data(), size);
415 } else {
416 // Not enough data available in underlying device for now
417 break;
418 }
419 }
420 if (d->bNeedHeader) {
421 (void)filter->readHeader();
422 d->bNeedHeader = false;
423 }
424
425 d->result = filter->uncompress();
426
427 if (d->result == KFilterBase::Error) {
428 // qCWarning(KArchiveLog) << "KCompressionDevice: Error when uncompressing data";
429 break;
430 }
431
432 // We got that much data since the last time we went here
433 uint outReceived = availOut - filter->outBufferAvailable();
434 // qCDebug(KArchiveLog) << "avail_out = " << filter->outBufferAvailable() << " result=" << d->result << " outReceived=" << outReceived;
435 if (availOut < uint(filter->outBufferAvailable())) {
436 // qCWarning(KArchiveLog) << " last availOut " << availOut << " smaller than new avail_out=" << filter->outBufferAvailable() << " !";
437 }
438
439 dataReceived += outReceived;
440 data += outReceived;
441 availOut = maxlen - dataReceived;
442 if (d->result == KFilterBase::End) {
443 // We're actually at the end, no more data to check
444 if (filter->device()->atEnd()) {
445 break;
446 }
447
448 // Still not done, re-init and try again
449 filter->init(mode: filter->mode());
450 }
451 filter->setOutBuffer(data, maxlen: availOut);
452 }
453
454 d->deviceReadPos += dataReceived;
455 return dataReceived;
456}
457
458qint64 KCompressionDevice::writeData(const char *data /*0 to finish*/, qint64 len)
459{
460 KFilterBase *filter = d->filter;
461 Q_ASSERT(filter->mode() == QIODevice::WriteOnly);
462 // If we had an error, return 0.
463 if (d->result != KFilterBase::Ok) {
464 return 0;
465 }
466
467 bool finish = (data == nullptr);
468 if (!finish) {
469 filter->setInBuffer(data, size: len);
470 if (d->bNeedHeader) {
471 (void)filter->writeHeader(filename: d->origFileName);
472 d->bNeedHeader = false;
473 }
474 }
475
476 uint dataWritten = 0;
477 uint availIn = len;
478 while (dataWritten < len || finish) {
479 d->result = filter->compress(finish);
480
481 if (d->result == KFilterBase::Error) {
482 // qCWarning(KArchiveLog) << "KCompressionDevice: Error when compressing data";
483 // What to do ?
484 break;
485 }
486
487 // Wrote everything ?
488 if (filter->inBufferEmpty() || (d->result == KFilterBase::End)) {
489 // We got that much data since the last time we went here
490 uint wrote = availIn - filter->inBufferAvailable();
491
492 // qCDebug(KArchiveLog) << " Wrote everything for now. avail_in=" << filter->inBufferAvailable() << "result=" << d->result << "wrote=" << wrote;
493
494 // Move on in the input buffer
495 data += wrote;
496 dataWritten += wrote;
497
498 availIn = len - dataWritten;
499 // qCDebug(KArchiveLog) << " availIn=" << availIn << "dataWritten=" << dataWritten << "pos=" << pos();
500 if (availIn > 0) {
501 filter->setInBuffer(data, size: availIn);
502 }
503 }
504
505 if (filter->outBufferFull() || (d->result == KFilterBase::End) || finish) {
506 // qCDebug(KArchiveLog) << " writing to underlying. avail_out=" << filter->outBufferAvailable();
507 int towrite = d->buffer.size() - filter->outBufferAvailable();
508 if (towrite > 0) {
509 // Write compressed data to underlying device
510 int size = filter->device()->write(data: d->buffer.data(), len: towrite);
511 if (size != towrite) {
512 // qCWarning(KArchiveLog) << "KCompressionDevice::write. Could only write " << size << " out of " << towrite << " bytes";
513 d->errorCode = QFileDevice::WriteError;
514 setErrorString(tr(s: "Could not write. Partition full?"));
515 return 0; // indicate an error
516 }
517 // qCDebug(KArchiveLog) << " wrote " << size << " bytes";
518 }
519 if (d->result == KFilterBase::End) {
520 Q_ASSERT(finish); // hopefully we don't get end before finishing
521 break;
522 }
523 d->buffer.resize(BUFFER_SIZE);
524 filter->setOutBuffer(data: d->buffer.data(), maxlen: d->buffer.size());
525 }
526 }
527
528 return dataWritten;
529}
530
531void KCompressionDevice::setOrigFileName(const QByteArray &fileName)
532{
533 d->origFileName = fileName;
534}
535
536void KCompressionDevice::setSkipHeaders()
537{
538 d->bSkipHeaders = true;
539}
540
541KFilterBase *KCompressionDevice::filterBase()
542{
543 return d->filter;
544}
545
546#include "moc_kcompressiondevice.cpp"
547

source code of karchive/src/kcompressiondevice.cpp