1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the test suite of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:GPL-EXCEPT$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU
19** General Public License version 3 as published by the Free Software
20** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
21** included in the packaging of this file. Please review the following
22** information to ensure the GNU General Public License requirements will
23** be met: https://www.gnu.org/licenses/gpl-3.0.html.
24**
25** $QT_END_LICENSE$
26**
27****************************************************************************/
28
29#include <QTest>
30
31#include <QFile>
32#include <QFileInfo>
33#include <QRandomGenerator>
34#include <qplatformdefs.h>
35
36#include <QDebug>
37
38#include <algorithm>
39#include <cstdlib>
40#include <cstdio>
41
42#ifdef Q_OS_WIN
43# include <qt_windows.h>
44# include <io.h>
45# ifndef FSCTL_SET_SPARSE
46// MinGW doesn't define this.
47# define FSCTL_SET_SPARSE (0x900C4)
48# endif
49#endif // Q_OS_WIN
50
51#include "emulationdetector.h"
52
53class tst_LargeFile
54 : public QObject
55{
56 Q_OBJECT
57
58public:
59 tst_LargeFile()
60 : blockSize(1 << 12)
61 , maxSizeBits()
62 , fd_(-1)
63 , stream_(0)
64 {
65 #if defined(QT_LARGEFILE_SUPPORT) && !defined(Q_OS_MAC) && !defined(Q_OS_WINRT)
66 maxSizeBits = 36; // 64 GiB
67 #elif defined(Q_OS_MAC)
68 // HFS+ does not support sparse files, so we limit file size for the test
69 // on Mac OS.
70 maxSizeBits = 24; // 16 MiB
71 #else
72 maxSizeBits = 24; // 16 MiB
73 #endif
74
75 // QEMU only supports < 4GB files
76 if (EmulationDetector::isRunningArmOnX86())
77 maxSizeBits = qMin(a: maxSizeBits, b: 28);
78 }
79
80private:
81 void sparseFileData();
82 QByteArray const &getDataBlock(int index, qint64 position);
83
84private slots:
85 // The LargeFile test case was designed to be run in order as a single unit
86
87 void initTestCase();
88 void cleanupTestCase();
89
90 void init();
91 void cleanup();
92
93 // Create and fill large file
94 void createSparseFile();
95 void fillFileSparsely();
96 void closeSparseFile();
97
98 // Verify file was created
99 void fileCreated();
100
101 // Positioning in large files
102 void filePositioning();
103 void fdPositioning();
104 void streamPositioning();
105
106 // Read data from file
107 void openFileForReading();
108 void readFile();
109
110 // Map/unmap large file
111 void mapFile();
112 void mapOffsetOverflow();
113
114 void closeFile() { largeFile.close(); }
115
116 // Test data
117 void fillFileSparsely_data() { sparseFileData(); }
118 void filePositioning_data() { sparseFileData(); }
119 void fdPositioning_data() { sparseFileData(); }
120 void streamPositioning_data() { sparseFileData(); }
121 void readFile_data() { sparseFileData(); }
122 void mapFile_data() { sparseFileData(); }
123
124private:
125 const int blockSize;
126 int maxSizeBits;
127
128 QFile largeFile;
129
130 QVector<QByteArray> generatedBlocks;
131
132 int fd_;
133 FILE *stream_;
134
135 QSharedPointer<QTemporaryDir> m_tempDir;
136 QString m_previousCurrent;
137};
138
139/*
140 Convenience function to hide reinterpret_cast when copying a POD directly
141 into a QByteArray.
142 */
143template <class T>
144static inline void appendRaw(QByteArray &array, T data)
145{
146 array.append(s: reinterpret_cast<char *>(&data), len: sizeof(T));
147}
148
149/*
150 Pad array with filler up to size. On return, array.size() returns size.
151 */
152static inline void topUpWith(QByteArray &array, QByteArray filler, int size)
153{
154 for (int i = (size - array.size()) / filler.size(); i > 0; --i)
155 array.append(a: filler);
156
157 if (array.size() < size) {
158 array.append(a: filler.left(len: size - array.size()));
159 }
160}
161
162/*
163 Generate a unique data block containing identifiable data. Unaligned,
164 overlapping and partial blocks should not compare equal.
165 */
166static inline QByteArray generateDataBlock(int blockSize, QString text, qint64 userBits = -1)
167{
168 QByteArray block;
169 block.reserve(asize: blockSize);
170
171 // Use of counter and randomBits means content of block will be dependent
172 // on the generation order. For (file-)systems that do not support sparse
173 // files, these can be removed so the test file can be reused and doesn't
174 // have to be generated for every run.
175
176 static qint64 counter = 0;
177
178 qint64 randomBits = QRandomGenerator::global()->generate64();
179
180 appendRaw(array&: block, data: randomBits);
181 appendRaw(array&: block, data: userBits);
182 appendRaw(array&: block, data: counter);
183 appendRaw(array&: block, data: (qint32)0xdeadbeef);
184 appendRaw(array&: block, data: blockSize);
185
186 QByteArray userContent = text.toUtf8();
187 appendRaw(array&: block, data: userContent.size());
188 block.append(a: userContent);
189 appendRaw(array&: block, data: (qint64)0);
190
191 // size, so far
192 appendRaw(array&: block, data: block.size());
193
194 QByteArray filler("0123456789");
195 block.append(a: filler.right(len: 10 - block.size() % 10));
196 topUpWith(array&: block, filler, size: blockSize - 3 * sizeof(qint64));
197
198 appendRaw(array&: block, data: counter);
199 appendRaw(array&: block, data: userBits);
200 appendRaw(array&: block, data: randomBits);
201
202 ++counter;
203 return block;
204}
205
206/*
207 Generates data blocks the first time they are requested. Keeps copies for reuse.
208 */
209QByteArray const &tst_LargeFile::getDataBlock(int index, qint64 position)
210{
211 if (index >= generatedBlocks.size())
212 generatedBlocks.resize(asize: index + 1);
213
214 if (generatedBlocks[index].isNull()) {
215 QString text = QString("Current %1-byte block (index = %2) "
216 "starts %3 bytes into the file '%4'.")
217 .arg(a: blockSize)
218 .arg(a: index)
219 .arg(a: position)
220 .arg(a: "qt_largefile.tmp");
221
222 generatedBlocks[index] = generateDataBlock(blockSize, text, userBits: (qint64)1 << index);
223 }
224
225 return generatedBlocks[index];
226}
227
228void tst_LargeFile::initTestCase()
229{
230 m_previousCurrent = QDir::currentPath();
231 m_tempDir = QSharedPointer<QTemporaryDir>::create();
232 QVERIFY2(!m_tempDir.isNull(), qPrintable("Could not create temporary directory."));
233 QVERIFY2(QDir::setCurrent(m_tempDir->path()), qPrintable("Could not switch current directory"));
234
235 QFile file("qt_largefile.tmp");
236 QVERIFY( !file.exists() || file.remove() );
237}
238
239void tst_LargeFile::cleanupTestCase()
240{
241 if (largeFile.isOpen())
242 largeFile.close();
243
244 QFile file("qt_largefile.tmp");
245 QVERIFY( !file.exists() || file.remove() );
246
247 QDir::setCurrent(m_previousCurrent);
248}
249
250void tst_LargeFile::init()
251{
252 fd_ = -1;
253 stream_ = 0;
254}
255
256void tst_LargeFile::cleanup()
257{
258 if (-1 != fd_)
259 QT_CLOSE(fd: fd_);
260 if (stream_)
261 ::fclose(stream: stream_);
262}
263
264void tst_LargeFile::sparseFileData()
265{
266 QTest::addColumn<int>(name: "index");
267 QTest::addColumn<qint64>(name: "position");
268 QTest::addColumn<QByteArray>(name: "block");
269
270 QTest::newRow(dataTag: QString("block[%1] @%2)")
271 .arg(a: 0).arg(a: 0)
272 .toLocal8Bit().constData())
273 << 0 << (qint64)0 << getDataBlock(index: 0, position: 0);
274
275 // While on Linux sparse files scale well, on Windows, testing at every
276 // power of 2 leads to very large files. i += 4 gives us a good coverage
277 // without taxing too much on resources.
278 for (int index = 12; index <= maxSizeBits; index += 4) {
279 qint64 position = (qint64)1 << index;
280 QByteArray block = getDataBlock(index, position);
281
282 QTest::newRow(
283 dataTag: QString("block[%1] @%2)")
284 .arg(a: index).arg(a: position)
285 .toLocal8Bit().constData())
286 << index << position << block;
287 }
288}
289
290void tst_LargeFile::createSparseFile()
291{
292#if defined(Q_OS_WIN32)
293 // On Windows platforms, we must explicitly set the file to be sparse,
294 // so disk space is not allocated for the full file when writing to it.
295 HANDLE handle = ::CreateFileA("qt_largefile.tmp",
296 GENERIC_WRITE, 0, 0, CREATE_ALWAYS, 0, 0);
297 QVERIFY( INVALID_HANDLE_VALUE != handle );
298
299 DWORD bytes;
300 if (!::DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, NULL, 0,
301 &bytes, NULL)) {
302 QWARN("Unable to set test file as sparse. "
303 "Limiting test file to 16MiB.");
304 maxSizeBits = 24;
305 }
306
307 int fd = ::_open_osfhandle((intptr_t)handle, 0);
308 QVERIFY( -1 != fd );
309 QVERIFY( largeFile.open(fd, QIODevice::WriteOnly | QIODevice::Unbuffered) );
310#else // !Q_OS_WIN32
311 largeFile.setFileName("qt_largefile.tmp");
312 QVERIFY( largeFile.open(QIODevice::WriteOnly | QIODevice::Unbuffered) );
313#endif
314}
315
316void tst_LargeFile::closeSparseFile()
317{
318#if defined(Q_OS_WIN32)
319 int fd = largeFile.handle();
320#endif
321
322 largeFile.close();
323
324#if defined(Q_OS_WIN32)
325 if (-1 != fd)
326 ::_close(fd);
327#endif
328}
329
330void tst_LargeFile::fillFileSparsely()
331{
332 QFETCH( qint64, position );
333 QFETCH( QByteArray, block );
334 QCOMPARE( block.size(), blockSize );
335
336 static int lastKnownGoodIndex = 0;
337 struct ScopeGuard {
338 ScopeGuard(tst_LargeFile* test)
339 : this_(test)
340 , failed(true)
341 {
342 QFETCH( int, index );
343 index_ = index;
344 }
345
346 ~ScopeGuard()
347 {
348 if (failed) {
349 this_->maxSizeBits = lastKnownGoodIndex;
350 QWARN( qPrintable(
351 QString("QFile::error %1: '%2'. Maximum size bits reset to %3.")
352 .arg(this_->largeFile.error())
353 .arg(this_->largeFile.errorString())
354 .arg(this_->maxSizeBits)) );
355 } else
356 lastKnownGoodIndex = qMax<int>(a: index_, b: lastKnownGoodIndex);
357 }
358
359 private:
360 tst_LargeFile * const this_;
361 int index_;
362
363 public:
364 bool failed;
365 };
366
367 ScopeGuard resetMaxSizeBitsOnFailure(this);
368
369 QVERIFY( largeFile.seek(position) );
370 QCOMPARE( largeFile.pos(), position );
371
372 QCOMPARE( largeFile.write(block), (qint64)blockSize );
373 QCOMPARE( largeFile.pos(), position + blockSize );
374 QVERIFY( largeFile.flush() );
375
376 resetMaxSizeBitsOnFailure.failed = false;
377}
378
379void tst_LargeFile::fileCreated()
380{
381 QFileInfo info("qt_largefile.tmp");
382
383 QVERIFY( info.exists() );
384 QVERIFY( info.isFile() );
385 QVERIFY( info.size() >= ((qint64)1 << maxSizeBits) + blockSize );
386}
387
388void tst_LargeFile::filePositioning()
389{
390 QFETCH( qint64, position );
391
392 QFile file("qt_largefile.tmp");
393 QVERIFY( file.open(QIODevice::ReadOnly) );
394
395 QVERIFY( file.seek(position) );
396 QCOMPARE( file.pos(), position );
397}
398
399void tst_LargeFile::fdPositioning()
400{
401 QFETCH( qint64, position );
402
403 fd_ = QT_OPEN(file: "qt_largefile.tmp",
404 QT_OPEN_RDONLY | QT_OPEN_LARGEFILE);
405 QVERIFY( -1 != fd_ );
406
407 QFile file;
408 QVERIFY( file.open(fd_, QIODevice::ReadOnly) );
409 QCOMPARE( file.pos(), (qint64)0 );
410 QVERIFY( file.seek(position) );
411 QCOMPARE( file.pos(), position );
412
413 file.close();
414
415 QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(0), SEEK_SET)), QT_OFF_T(0) );
416 QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(position), SEEK_SET)), QT_OFF_T(position) );
417
418 QVERIFY( file.open(fd_, QIODevice::ReadOnly) );
419 QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(0), SEEK_CUR)), QT_OFF_T(position) );
420 QCOMPARE( file.pos(), position );
421 QVERIFY( file.seek(0) );
422 QCOMPARE( file.pos(), (qint64)0 );
423
424 file.close();
425
426 QVERIFY( !QT_CLOSE(fd_) );
427 fd_ = -1;
428}
429
430void tst_LargeFile::streamPositioning()
431{
432 QFETCH( qint64, position );
433
434 stream_ = QT_FOPEN(filename: "qt_largefile.tmp", modes: "rb");
435 QVERIFY( 0 != stream_ );
436
437 QFile file;
438 QVERIFY( file.open(stream_, QIODevice::ReadOnly) );
439 QCOMPARE( file.pos(), (qint64)0 );
440 QVERIFY( file.seek(position) );
441 QCOMPARE( file.pos(), position );
442
443 file.close();
444
445 QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(0), SEEK_SET) );
446 QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(0) );
447 QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(position), SEEK_SET) );
448 QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(position) );
449
450 QVERIFY( file.open(stream_, QIODevice::ReadOnly) );
451 QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(position) );
452 QCOMPARE( file.pos(), position );
453 QVERIFY( file.seek(0) );
454 QCOMPARE( file.pos(), (qint64)0 );
455
456 file.close();
457
458 QVERIFY( !::fclose(stream_) );
459 stream_ = 0;
460}
461
462void tst_LargeFile::openFileForReading()
463{
464 largeFile.setFileName("qt_largefile.tmp");
465 QVERIFY( largeFile.open(QIODevice::ReadOnly) );
466}
467
468void tst_LargeFile::readFile()
469{
470 QFETCH( qint64, position );
471 QFETCH( QByteArray, block );
472 QCOMPARE( block.size(), blockSize );
473
474 QVERIFY( largeFile.size() >= position + blockSize );
475
476 QVERIFY( largeFile.seek(position) );
477 QCOMPARE( largeFile.pos(), position );
478
479 QCOMPARE( largeFile.read(blockSize), block );
480 QCOMPARE( largeFile.pos(), position + blockSize );
481}
482
483void tst_LargeFile::mapFile()
484{
485 QFETCH( qint64, position );
486 QFETCH( QByteArray, block );
487 QCOMPARE( block.size(), blockSize );
488
489 // Keep full block mapped to facilitate OS and/or internal reuse by Qt.
490 uchar *baseAddress = largeFile.map(offset: position, size: blockSize);
491 QVERIFY( baseAddress );
492 QVERIFY( std::equal(block.begin(), block.end(), reinterpret_cast<char*>(baseAddress)) );
493
494 for (int offset = 1; offset < blockSize; ++offset) {
495 uchar *address = largeFile.map(offset: position + offset, size: blockSize - offset);
496
497 QVERIFY( address );
498 if ( !std::equal(first1: block.begin() + offset, last1: block.end(), first2: reinterpret_cast<char*>(address)) ) {
499 qDebug() << "Expected:" << block.toHex();
500 qDebug() << "Actual :" << QByteArray(reinterpret_cast<char*>(address), blockSize).toHex();
501 QVERIFY(false);
502 }
503
504 QVERIFY( largeFile.unmap( address ) );
505 }
506
507 QVERIFY( largeFile.unmap( baseAddress ) );
508}
509
510//Mac: memory-mapping beyond EOF may succeed but it could generate bus error on access
511//FreeBSD: same
512//Linux: memory-mapping beyond EOF usually succeeds, but depends on the filesystem
513// 32-bit: limited to 44-bit offsets (when sizeof(off_t) == 8)
514//Windows: memory-mapping beyond EOF is not allowed
515void tst_LargeFile::mapOffsetOverflow()
516{
517 enum {
518#ifdef Q_OS_WIN
519 Succeeds = false,
520 MaxOffset = 63
521#else
522 Succeeds = true,
523# if (defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)) && Q_PROCESSOR_WORDSIZE == 4
524 MaxOffset = sizeof(QT_OFF_T) > 4 ? 43 : 30
525# else
526 MaxOffset = 8 * sizeof(QT_OFF_T) - 1
527# endif
528#endif
529 };
530
531 QByteArray zeroPage(blockSize, '\0');
532 for (int i = maxSizeBits + 1; i < 63; ++i) {
533 bool succeeds = Succeeds && (i <= MaxOffset);
534 uchar *address = 0;
535 qint64 offset = Q_INT64_C(1) << i;
536
537 if (succeeds)
538 QTest::ignoreMessage(type: QtWarningMsg, message: "QFSFileEngine::map: Mapping a file beyond its size is not portable");
539 address = largeFile.map(offset, size: blockSize);
540 QCOMPARE(!!address, succeeds);
541
542 if (succeeds)
543 QTest::ignoreMessage(type: QtWarningMsg, message: "QFSFileEngine::map: Mapping a file beyond its size is not portable");
544 address = largeFile.map(offset: offset + blockSize, size: blockSize);
545 QCOMPARE(!!address, succeeds);
546 }
547}
548
549QTEST_APPLESS_MAIN(tst_LargeFile)
550#include "tst_largefile.moc"
551
552

source code of qtbase/tests/auto/corelib/io/largefile/tst_largefile.cpp