1 | /* This file is part of the KDE libraries |
2 | SPDX-FileCopyrightText: 2000 David Faure <faure@kde.org> |
3 | SPDX-FileCopyrightText: 2011 Mario Bensi <mbensi@ipsquad.net> |
4 | |
5 | SPDX-License-Identifier: LGPL-2.0-or-later |
6 | */ |
7 | |
8 | #include "kcompressiondevice.h" |
9 | #include "kcompressiondevice_p.h" |
10 | #include "kfilterbase.h" |
11 | #include "loggingcategory.h" |
12 | #include "kgzipfilter.h" |
13 | #include "knonefilter.h" |
14 | |
15 | #include "config-compression.h" |
16 | |
17 | #if HAVE_BZIP2_SUPPORT |
18 | #include "kbzip2filter.h" |
19 | #endif |
20 | #if HAVE_XZ_SUPPORT |
21 | #include "klzfilter.h" |
22 | #include "kxzfilter.h" |
23 | #endif |
24 | #if HAVE_ZSTD_SUPPORT |
25 | #include "kzstdfilter.h" |
26 | #endif |
27 | |
28 | #include <QDebug> |
29 | #include <QFile> |
30 | #include <QMimeDatabase> |
31 | |
32 | #include <assert.h> |
33 | #include <stdio.h> // for EOF |
34 | #include <stdlib.h> |
35 | |
36 | class KCompressionDevicePrivate |
37 | { |
38 | public: |
39 | KCompressionDevicePrivate(KCompressionDevice *qq) |
40 | : bNeedHeader(true) |
41 | , bSkipHeaders(false) |
42 | , bOpenedUnderlyingDevice(false) |
43 | , type(KCompressionDevice::None) |
44 | , errorCode(QFileDevice::NoError) |
45 | , deviceReadPos(0) |
46 | , q(qq) |
47 | { |
48 | } |
49 | |
50 | void propagateErrorCode(); |
51 | |
52 | bool ; |
53 | bool ; |
54 | bool bOpenedUnderlyingDevice; |
55 | QByteArray buffer; // Used as 'input buffer' when reading, as 'output buffer' when writing |
56 | QByteArray origFileName; |
57 | KFilterBase::Result result; |
58 | KFilterBase *filter; |
59 | KCompressionDevice::CompressionType type; |
60 | std::optional<qint64> size; |
61 | QFileDevice::FileError errorCode; |
62 | qint64 deviceReadPos; |
63 | KCompressionDevice *q; |
64 | }; |
65 | |
66 | void KCompressionDevicePrivate::propagateErrorCode() |
67 | { |
68 | QIODevice *dev = filter->device(); |
69 | if (QFileDevice *fileDev = qobject_cast<QFileDevice *>(object: dev)) { |
70 | if (fileDev->error() != QFileDevice::NoError) { |
71 | errorCode = fileDev->error(); |
72 | q->setErrorString(dev->errorString()); |
73 | } |
74 | } |
75 | // ... we have no generic way to propagate errors from other kinds of iodevices. Sucks, heh? :( |
76 | } |
77 | |
78 | static KCompressionDevice::CompressionType findCompressionByFileName(const QString &fileName) |
79 | { |
80 | if (fileName.endsWith(s: QLatin1String(".gz" ), cs: Qt::CaseInsensitive)) { |
81 | return KCompressionDevice::GZip; |
82 | } |
83 | #if HAVE_BZIP2_SUPPORT |
84 | if (fileName.endsWith(s: QLatin1String(".bz2" ), cs: Qt::CaseInsensitive)) { |
85 | return KCompressionDevice::BZip2; |
86 | } |
87 | #endif |
88 | #if HAVE_XZ_SUPPORT |
89 | if (fileName.endsWith(s: QLatin1String(".lz" ), cs: Qt::CaseInsensitive)) { |
90 | return KCompressionDevice::Lz; |
91 | } |
92 | if (fileName.endsWith(s: QLatin1String(".lzma" ), cs: Qt::CaseInsensitive) || fileName.endsWith(s: QLatin1String(".xz" ), cs: Qt::CaseInsensitive)) { |
93 | return KCompressionDevice::Xz; |
94 | } |
95 | #endif |
96 | #if HAVE_ZSTD_SUPPORT |
97 | if (fileName.endsWith(s: QLatin1String(".zst" ), cs: Qt::CaseInsensitive)) { |
98 | return KCompressionDevice::Zstd; |
99 | } |
100 | #endif |
101 | else { |
102 | // not a warning, since this is called often with other MIME types (see #88574)... |
103 | // maybe we can avoid that though? |
104 | // qCDebug(KArchiveLog) << "findCompressionByFileName : no compression found for " << fileName; |
105 | } |
106 | |
107 | return KCompressionDevice::None; |
108 | } |
109 | |
110 | KCompressionDevice::CompressionType KCompressionDevice::compressionTypeForMimeType(const QString &mimeType) |
111 | { |
112 | if (mimeType == QLatin1String("application/gzip" ) // |
113 | || mimeType == QLatin1String("application/x-gzip" ) // legacy name, kept for compatibility |
114 | ) { |
115 | return KCompressionDevice::GZip; |
116 | } |
117 | #if HAVE_BZIP2_SUPPORT |
118 | if (mimeType == QLatin1String("application/x-bzip" ) // |
119 | || mimeType == QLatin1String("application/x-bzip2" ) // old name, kept for compatibility |
120 | ) { |
121 | return KCompressionDevice::BZip2; |
122 | } |
123 | #endif |
124 | #if HAVE_XZ_SUPPORT |
125 | if (mimeType == QLatin1String("application/x-lzip" )) { |
126 | return KCompressionDevice::Lz; |
127 | } |
128 | if (mimeType == QLatin1String("application/x-lzma" ) // legacy name, still used |
129 | || mimeType == QLatin1String("application/x-xz" ) // current naming |
130 | ) { |
131 | return KCompressionDevice::Xz; |
132 | } |
133 | #endif |
134 | #if HAVE_ZSTD_SUPPORT |
135 | if (mimeType == QLatin1String("application/zstd" )) { |
136 | return KCompressionDevice::Zstd; |
137 | } |
138 | #endif |
139 | QMimeDatabase db; |
140 | const QMimeType mime = db.mimeTypeForName(nameOrAlias: mimeType); |
141 | if (mime.isValid()) { |
142 | // use legacy MIME type for now, see comment in impl. of KTar(const QString &, const QString &_mimetype) |
143 | if (mime.inherits(QStringLiteral("application/x-gzip" ))) { |
144 | return KCompressionDevice::GZip; |
145 | } |
146 | #if HAVE_BZIP2_SUPPORT |
147 | if (mime.inherits(QStringLiteral("application/x-bzip" ))) { |
148 | return KCompressionDevice::BZip2; |
149 | } |
150 | #endif |
151 | #if HAVE_XZ_SUPPORT |
152 | if (mime.inherits(QStringLiteral("application/x-lzip" ))) { |
153 | return KCompressionDevice::Lz; |
154 | } |
155 | |
156 | if (mime.inherits(QStringLiteral("application/x-lzma" ))) { |
157 | return KCompressionDevice::Xz; |
158 | } |
159 | |
160 | if (mime.inherits(QStringLiteral("application/x-xz" ))) { |
161 | return KCompressionDevice::Xz; |
162 | } |
163 | #endif |
164 | } |
165 | |
166 | // not a warning, since this is called often with other MIME types (see #88574)... |
167 | // maybe we can avoid that though? |
168 | // qCDebug(KArchiveLog) << "no compression found for" << mimeType; |
169 | return KCompressionDevice::None; |
170 | } |
171 | |
172 | KFilterBase *KCompressionDevice::filterForCompressionType(KCompressionDevice::CompressionType type) |
173 | { |
174 | switch (type) { |
175 | case KCompressionDevice::GZip: |
176 | return new KGzipFilter; |
177 | case KCompressionDevice::BZip2: |
178 | #if HAVE_BZIP2_SUPPORT |
179 | return new KBzip2Filter; |
180 | #else |
181 | return nullptr; |
182 | #endif |
183 | case KCompressionDevice::Lz: |
184 | #if HAVE_XZ_SUPPORT |
185 | return new KLzFilter; |
186 | #else |
187 | return nullptr; |
188 | #endif |
189 | case KCompressionDevice::Xz: |
190 | #if HAVE_XZ_SUPPORT |
191 | return new KXzFilter; |
192 | #else |
193 | return nullptr; |
194 | #endif |
195 | case KCompressionDevice::None: |
196 | return new KNoneFilter; |
197 | case KCompressionDevice::Zstd: |
198 | #if HAVE_ZSTD_SUPPORT |
199 | return new KZstdFilter; |
200 | #else |
201 | return nullptr; |
202 | #endif |
203 | } |
204 | return nullptr; |
205 | } |
206 | |
207 | KCompressionDevice::KCompressionDevice(QIODevice *inputDevice, bool autoDeleteInputDevice, CompressionType type) |
208 | : d(new KCompressionDevicePrivate(this)) |
209 | { |
210 | assert(inputDevice); |
211 | d->filter = filterForCompressionType(type); |
212 | if (d->filter) { |
213 | d->type = type; |
214 | d->filter->setDevice(dev: inputDevice, autodelete: autoDeleteInputDevice); |
215 | } else if (autoDeleteInputDevice) { |
216 | delete inputDevice; |
217 | } |
218 | } |
219 | |
220 | KCompressionDevice::KCompressionDevice(const QString &fileName, CompressionType type) |
221 | : KCompressionDevice(new QFile(fileName), true, type) |
222 | { |
223 | } |
224 | |
225 | KCompressionDevice::KCompressionDevice(const QString &fileName) |
226 | : KCompressionDevice(fileName, findCompressionByFileName(fileName)) |
227 | { |
228 | } |
229 | |
230 | KCompressionDevice::KCompressionDevice(std::unique_ptr<QIODevice> inputDevice, CompressionType type, std::optional<qint64> size) |
231 | : KCompressionDevice(inputDevice.release(), true, type) |
232 | { |
233 | d->size = size; |
234 | } |
235 | |
236 | KCompressionDevice::~KCompressionDevice() |
237 | { |
238 | if (isOpen()) { |
239 | close(); |
240 | } |
241 | delete d->filter; |
242 | delete d; |
243 | } |
244 | |
245 | KCompressionDevice::CompressionType KCompressionDevice::compressionType() const |
246 | { |
247 | return d->type; |
248 | } |
249 | |
250 | bool KCompressionDevice::open(QIODevice::OpenMode mode) |
251 | { |
252 | if (isOpen()) { |
253 | // qCWarning(KArchiveLog) << "KCompressionDevice::open: device is already open"; |
254 | return true; // QFile returns false, but well, the device -is- open... |
255 | } |
256 | if (!d->filter) { |
257 | return false; |
258 | } |
259 | d->bOpenedUnderlyingDevice = false; |
260 | // qCDebug(KArchiveLog) << mode; |
261 | if (mode == QIODevice::ReadOnly) { |
262 | d->buffer.resize(size: 0); |
263 | } else { |
264 | d->buffer.resize(BUFFER_SIZE); |
265 | d->filter->setOutBuffer(data: d->buffer.data(), maxlen: d->buffer.size()); |
266 | } |
267 | if (!d->filter->device()->isOpen()) { |
268 | if (!d->filter->device()->open(mode)) { |
269 | // qCWarning(KArchiveLog) << "KCompressionDevice::open: Couldn't open underlying device"; |
270 | d->propagateErrorCode(); |
271 | return false; |
272 | } |
273 | d->bOpenedUnderlyingDevice = true; |
274 | } |
275 | d->bNeedHeader = !d->bSkipHeaders; |
276 | d->filter->setFilterFlags(d->bSkipHeaders ? KFilterBase::NoHeaders : KFilterBase::WithHeaders); |
277 | if (!d->filter->init(mode: mode & ~QIODevice::Truncate)) { |
278 | return false; |
279 | } |
280 | d->result = KFilterBase::Ok; |
281 | setOpenMode(mode); |
282 | return true; |
283 | } |
284 | |
285 | void KCompressionDevice::close() |
286 | { |
287 | if (!isOpen()) { |
288 | return; |
289 | } |
290 | if (d->filter->mode() == QIODevice::WriteOnly && d->errorCode == QFileDevice::NoError) { |
291 | write(data: nullptr, len: 0); // finish writing |
292 | } |
293 | // qCDebug(KArchiveLog) << "Calling terminate()."; |
294 | |
295 | if (!d->filter->terminate()) { |
296 | // qCWarning(KArchiveLog) << "KCompressionDevice::close: terminate returned an error"; |
297 | d->errorCode = QFileDevice::UnspecifiedError; |
298 | } |
299 | if (d->bOpenedUnderlyingDevice) { |
300 | QIODevice *dev = d->filter->device(); |
301 | dev->close(); |
302 | d->propagateErrorCode(); |
303 | } |
304 | setOpenMode(QIODevice::NotOpen); |
305 | } |
306 | |
307 | qint64 KCompressionDevice::size() const |
308 | { |
309 | if (!d->size.has_value()) { |
310 | return QIODevice::size(); |
311 | } |
312 | |
313 | return d->size.value(); |
314 | } |
315 | |
316 | QFileDevice::FileError KCompressionDevice::error() const |
317 | { |
318 | return d->errorCode; |
319 | } |
320 | |
321 | bool KCompressionDevice::seek(qint64 pos) |
322 | { |
323 | if (d->deviceReadPos == pos) { |
324 | return QIODevice::seek(pos); |
325 | } |
326 | |
327 | // qCDebug(KArchiveLog) << "seek(" << pos << ") called, current pos=" << QIODevice::pos(); |
328 | |
329 | Q_ASSERT(d->filter->mode() == QIODevice::ReadOnly); |
330 | |
331 | if (pos == 0) { |
332 | if (!QIODevice::seek(pos)) { |
333 | return false; |
334 | } |
335 | |
336 | // We can forget about the cached data |
337 | d->bNeedHeader = !d->bSkipHeaders; |
338 | d->result = KFilterBase::Ok; |
339 | d->filter->setInBuffer(data: nullptr, size: 0); |
340 | d->filter->reset(); |
341 | d->deviceReadPos = 0; |
342 | return d->filter->device()->reset(); |
343 | } |
344 | |
345 | qint64 bytesToRead; |
346 | if (d->deviceReadPos < pos) { // we can start from here |
347 | bytesToRead = pos - d->deviceReadPos; |
348 | // Since we're going to do a read() below |
349 | // we need to reset the internal QIODevice pos to the real position we are |
350 | // so that after read() we are indeed pointing to the pos seek |
351 | // asked us to be in |
352 | if (!QIODevice::seek(pos: d->deviceReadPos)) { |
353 | return false; |
354 | } |
355 | } else { |
356 | // we have to start from 0 ! Ugly and slow, but better than the previous |
357 | // solution (KTarGz was allocating everything into memory) |
358 | if (!seek(pos: 0)) { // recursive |
359 | return false; |
360 | } |
361 | bytesToRead = pos; |
362 | } |
363 | |
364 | // qCDebug(KArchiveLog) << "reading " << bytesToRead << " dummy bytes"; |
365 | QByteArray dummy(qMin(a: bytesToRead, b: qint64(SEEK_BUFFER_SIZE)), 0); |
366 | while (bytesToRead > 0) { |
367 | const qint64 bytesToReadThisTime = qMin(a: bytesToRead, b: qint64(dummy.size())); |
368 | const bool result = (read(data: dummy.data(), maxlen: bytesToReadThisTime) == bytesToReadThisTime); |
369 | if (!result) { |
370 | return false; |
371 | } |
372 | bytesToRead -= bytesToReadThisTime; |
373 | } |
374 | return true; |
375 | } |
376 | |
377 | bool KCompressionDevice::atEnd() const |
378 | { |
379 | return (d->type == KCompressionDevice::None || d->result == KFilterBase::End) // |
380 | && QIODevice::atEnd() // take QIODevice's internal buffer into account |
381 | && d->filter->device()->atEnd(); |
382 | } |
383 | |
384 | qint64 KCompressionDevice::readData(char *data, qint64 maxlen) |
385 | { |
386 | Q_ASSERT(d->filter->mode() == QIODevice::ReadOnly); |
387 | // qCDebug(KArchiveLog) << "maxlen=" << maxlen; |
388 | KFilterBase *filter = d->filter; |
389 | |
390 | uint dataReceived = 0; |
391 | |
392 | // We came to the end of the stream |
393 | if (d->result == KFilterBase::End) { |
394 | return dataReceived; |
395 | } |
396 | |
397 | // If we had an error, return -1. |
398 | if (d->result != KFilterBase::Ok) { |
399 | return -1; |
400 | } |
401 | |
402 | qint64 availOut = maxlen; |
403 | filter->setOutBuffer(data, maxlen); |
404 | |
405 | while (dataReceived < maxlen) { |
406 | if (filter->inBufferEmpty()) { |
407 | // Not sure about the best size to set there. |
408 | // For sure, it should be bigger than the header size (see comment in readHeader) |
409 | d->buffer.resize(BUFFER_SIZE); |
410 | // Request data from underlying device |
411 | int size = filter->device()->read(data: d->buffer.data(), maxlen: d->buffer.size()); |
412 | // qCDebug(KArchiveLog) << "got" << size << "bytes from device"; |
413 | if (size) { |
414 | filter->setInBuffer(data: d->buffer.data(), size); |
415 | } else { |
416 | // Not enough data available in underlying device for now |
417 | break; |
418 | } |
419 | } |
420 | if (d->bNeedHeader) { |
421 | (void)filter->readHeader(); |
422 | d->bNeedHeader = false; |
423 | } |
424 | |
425 | d->result = filter->uncompress(); |
426 | |
427 | if (d->result == KFilterBase::Error) { |
428 | // qCWarning(KArchiveLog) << "KCompressionDevice: Error when uncompressing data"; |
429 | break; |
430 | } |
431 | |
432 | // We got that much data since the last time we went here |
433 | uint outReceived = availOut - filter->outBufferAvailable(); |
434 | // qCDebug(KArchiveLog) << "avail_out = " << filter->outBufferAvailable() << " result=" << d->result << " outReceived=" << outReceived; |
435 | if (availOut < uint(filter->outBufferAvailable())) { |
436 | // qCWarning(KArchiveLog) << " last availOut " << availOut << " smaller than new avail_out=" << filter->outBufferAvailable() << " !"; |
437 | } |
438 | |
439 | dataReceived += outReceived; |
440 | data += outReceived; |
441 | availOut = maxlen - dataReceived; |
442 | if (d->result == KFilterBase::End) { |
443 | // We're actually at the end, no more data to check |
444 | if (filter->device()->atEnd()) { |
445 | break; |
446 | } |
447 | |
448 | // Still not done, re-init and try again |
449 | filter->init(mode: filter->mode()); |
450 | } |
451 | filter->setOutBuffer(data, maxlen: availOut); |
452 | } |
453 | |
454 | d->deviceReadPos += dataReceived; |
455 | return dataReceived; |
456 | } |
457 | |
458 | qint64 KCompressionDevice::writeData(const char *data /*0 to finish*/, qint64 len) |
459 | { |
460 | KFilterBase *filter = d->filter; |
461 | Q_ASSERT(filter->mode() == QIODevice::WriteOnly); |
462 | // If we had an error, return 0. |
463 | if (d->result != KFilterBase::Ok) { |
464 | return 0; |
465 | } |
466 | |
467 | bool finish = (data == nullptr); |
468 | if (!finish) { |
469 | filter->setInBuffer(data, size: len); |
470 | if (d->bNeedHeader) { |
471 | (void)filter->writeHeader(filename: d->origFileName); |
472 | d->bNeedHeader = false; |
473 | } |
474 | } |
475 | |
476 | uint dataWritten = 0; |
477 | uint availIn = len; |
478 | while (dataWritten < len || finish) { |
479 | d->result = filter->compress(finish); |
480 | |
481 | if (d->result == KFilterBase::Error) { |
482 | // qCWarning(KArchiveLog) << "KCompressionDevice: Error when compressing data"; |
483 | // What to do ? |
484 | break; |
485 | } |
486 | |
487 | // Wrote everything ? |
488 | if (filter->inBufferEmpty() || (d->result == KFilterBase::End)) { |
489 | // We got that much data since the last time we went here |
490 | uint wrote = availIn - filter->inBufferAvailable(); |
491 | |
492 | // qCDebug(KArchiveLog) << " Wrote everything for now. avail_in=" << filter->inBufferAvailable() << "result=" << d->result << "wrote=" << wrote; |
493 | |
494 | // Move on in the input buffer |
495 | data += wrote; |
496 | dataWritten += wrote; |
497 | |
498 | availIn = len - dataWritten; |
499 | // qCDebug(KArchiveLog) << " availIn=" << availIn << "dataWritten=" << dataWritten << "pos=" << pos(); |
500 | if (availIn > 0) { |
501 | filter->setInBuffer(data, size: availIn); |
502 | } |
503 | } |
504 | |
505 | if (filter->outBufferFull() || (d->result == KFilterBase::End) || finish) { |
506 | // qCDebug(KArchiveLog) << " writing to underlying. avail_out=" << filter->outBufferAvailable(); |
507 | int towrite = d->buffer.size() - filter->outBufferAvailable(); |
508 | if (towrite > 0) { |
509 | // Write compressed data to underlying device |
510 | int size = filter->device()->write(data: d->buffer.data(), len: towrite); |
511 | if (size != towrite) { |
512 | // qCWarning(KArchiveLog) << "KCompressionDevice::write. Could only write " << size << " out of " << towrite << " bytes"; |
513 | d->errorCode = QFileDevice::WriteError; |
514 | setErrorString(tr(s: "Could not write. Partition full?" )); |
515 | return 0; // indicate an error |
516 | } |
517 | // qCDebug(KArchiveLog) << " wrote " << size << " bytes"; |
518 | } |
519 | if (d->result == KFilterBase::End) { |
520 | Q_ASSERT(finish); // hopefully we don't get end before finishing |
521 | break; |
522 | } |
523 | d->buffer.resize(BUFFER_SIZE); |
524 | filter->setOutBuffer(data: d->buffer.data(), maxlen: d->buffer.size()); |
525 | } |
526 | } |
527 | |
528 | return dataWritten; |
529 | } |
530 | |
531 | void KCompressionDevice::setOrigFileName(const QByteArray &fileName) |
532 | { |
533 | d->origFileName = fileName; |
534 | } |
535 | |
536 | void KCompressionDevice::() |
537 | { |
538 | d->bSkipHeaders = true; |
539 | } |
540 | |
541 | KFilterBase *KCompressionDevice::filterBase() |
542 | { |
543 | return d->filter; |
544 | } |
545 | |
546 | #include "moc_kcompressiondevice.cpp" |
547 | |