1 | /**************************************************************************** |
2 | ** |
3 | ** Copyright (C) 2016 The Qt Company Ltd. |
4 | ** Contact: https://www.qt.io/licensing/ |
5 | ** |
6 | ** This file is part of the test suite of the Qt Toolkit. |
7 | ** |
8 | ** $QT_BEGIN_LICENSE:GPL-EXCEPT$ |
9 | ** Commercial License Usage |
10 | ** Licensees holding valid commercial Qt licenses may use this file in |
11 | ** accordance with the commercial license agreement provided with the |
12 | ** Software or, alternatively, in accordance with the terms contained in |
13 | ** a written agreement between you and The Qt Company. For licensing terms |
14 | ** and conditions see https://www.qt.io/terms-conditions. For further |
15 | ** information use the contact form at https://www.qt.io/contact-us. |
16 | ** |
17 | ** GNU General Public License Usage |
18 | ** Alternatively, this file may be used under the terms of the GNU |
19 | ** General Public License version 3 as published by the Free Software |
20 | ** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT |
21 | ** included in the packaging of this file. Please review the following |
22 | ** information to ensure the GNU General Public License requirements will |
23 | ** be met: https://www.gnu.org/licenses/gpl-3.0.html. |
24 | ** |
25 | ** $QT_END_LICENSE$ |
26 | ** |
27 | ****************************************************************************/ |
28 | |
29 | #include <QTest> |
30 | |
31 | #include <QFile> |
32 | #include <QFileInfo> |
33 | #include <QRandomGenerator> |
34 | #include <qplatformdefs.h> |
35 | |
36 | #include <QDebug> |
37 | |
38 | #include <algorithm> |
39 | #include <cstdlib> |
40 | #include <cstdio> |
41 | |
42 | #ifdef Q_OS_WIN |
43 | # include <qt_windows.h> |
44 | # include <io.h> |
45 | # ifndef FSCTL_SET_SPARSE |
46 | // MinGW doesn't define this. |
47 | # define FSCTL_SET_SPARSE (0x900C4) |
48 | # endif |
49 | #endif // Q_OS_WIN |
50 | |
51 | #include "emulationdetector.h" |
52 | |
53 | class tst_LargeFile |
54 | : public QObject |
55 | { |
56 | Q_OBJECT |
57 | |
58 | public: |
59 | tst_LargeFile() |
60 | : blockSize(1 << 12) |
61 | , maxSizeBits() |
62 | , fd_(-1) |
63 | , stream_(0) |
64 | { |
65 | #if defined(QT_LARGEFILE_SUPPORT) && !defined(Q_OS_MAC) && !defined(Q_OS_WINRT) |
66 | maxSizeBits = 36; // 64 GiB |
67 | #elif defined(Q_OS_MAC) |
68 | // HFS+ does not support sparse files, so we limit file size for the test |
69 | // on Mac OS. |
70 | maxSizeBits = 24; // 16 MiB |
71 | #else |
72 | maxSizeBits = 24; // 16 MiB |
73 | #endif |
74 | |
75 | // QEMU only supports < 4GB files |
76 | if (EmulationDetector::isRunningArmOnX86()) |
77 | maxSizeBits = qMin(a: maxSizeBits, b: 28); |
78 | } |
79 | |
80 | private: |
81 | void sparseFileData(); |
82 | QByteArray const &getDataBlock(int index, qint64 position); |
83 | |
84 | private slots: |
85 | // The LargeFile test case was designed to be run in order as a single unit |
86 | |
87 | void initTestCase(); |
88 | void cleanupTestCase(); |
89 | |
90 | void init(); |
91 | void cleanup(); |
92 | |
93 | // Create and fill large file |
94 | void createSparseFile(); |
95 | void fillFileSparsely(); |
96 | void closeSparseFile(); |
97 | |
98 | // Verify file was created |
99 | void fileCreated(); |
100 | |
101 | // Positioning in large files |
102 | void filePositioning(); |
103 | void fdPositioning(); |
104 | void streamPositioning(); |
105 | |
106 | // Read data from file |
107 | void openFileForReading(); |
108 | void readFile(); |
109 | |
110 | // Map/unmap large file |
111 | void mapFile(); |
112 | void mapOffsetOverflow(); |
113 | |
114 | void closeFile() { largeFile.close(); } |
115 | |
116 | // Test data |
117 | void fillFileSparsely_data() { sparseFileData(); } |
118 | void filePositioning_data() { sparseFileData(); } |
119 | void fdPositioning_data() { sparseFileData(); } |
120 | void streamPositioning_data() { sparseFileData(); } |
121 | void readFile_data() { sparseFileData(); } |
122 | void mapFile_data() { sparseFileData(); } |
123 | |
124 | private: |
125 | const int blockSize; |
126 | int maxSizeBits; |
127 | |
128 | QFile largeFile; |
129 | |
130 | QVector<QByteArray> generatedBlocks; |
131 | |
132 | int fd_; |
133 | FILE *stream_; |
134 | |
135 | QSharedPointer<QTemporaryDir> m_tempDir; |
136 | QString m_previousCurrent; |
137 | }; |
138 | |
139 | /* |
140 | Convenience function to hide reinterpret_cast when copying a POD directly |
141 | into a QByteArray. |
142 | */ |
143 | template <class T> |
144 | static inline void appendRaw(QByteArray &array, T data) |
145 | { |
146 | array.append(s: reinterpret_cast<char *>(&data), len: sizeof(T)); |
147 | } |
148 | |
149 | /* |
150 | Pad array with filler up to size. On return, array.size() returns size. |
151 | */ |
152 | static inline void topUpWith(QByteArray &array, QByteArray filler, int size) |
153 | { |
154 | for (int i = (size - array.size()) / filler.size(); i > 0; --i) |
155 | array.append(a: filler); |
156 | |
157 | if (array.size() < size) { |
158 | array.append(a: filler.left(len: size - array.size())); |
159 | } |
160 | } |
161 | |
162 | /* |
163 | Generate a unique data block containing identifiable data. Unaligned, |
164 | overlapping and partial blocks should not compare equal. |
165 | */ |
166 | static inline QByteArray generateDataBlock(int blockSize, QString text, qint64 userBits = -1) |
167 | { |
168 | QByteArray block; |
169 | block.reserve(asize: blockSize); |
170 | |
171 | // Use of counter and randomBits means content of block will be dependent |
172 | // on the generation order. For (file-)systems that do not support sparse |
173 | // files, these can be removed so the test file can be reused and doesn't |
174 | // have to be generated for every run. |
175 | |
176 | static qint64 counter = 0; |
177 | |
178 | qint64 randomBits = QRandomGenerator::global()->generate64(); |
179 | |
180 | appendRaw(array&: block, data: randomBits); |
181 | appendRaw(array&: block, data: userBits); |
182 | appendRaw(array&: block, data: counter); |
183 | appendRaw(array&: block, data: (qint32)0xdeadbeef); |
184 | appendRaw(array&: block, data: blockSize); |
185 | |
186 | QByteArray userContent = text.toUtf8(); |
187 | appendRaw(array&: block, data: userContent.size()); |
188 | block.append(a: userContent); |
189 | appendRaw(array&: block, data: (qint64)0); |
190 | |
191 | // size, so far |
192 | appendRaw(array&: block, data: block.size()); |
193 | |
194 | QByteArray filler("0123456789" ); |
195 | block.append(a: filler.right(len: 10 - block.size() % 10)); |
196 | topUpWith(array&: block, filler, size: blockSize - 3 * sizeof(qint64)); |
197 | |
198 | appendRaw(array&: block, data: counter); |
199 | appendRaw(array&: block, data: userBits); |
200 | appendRaw(array&: block, data: randomBits); |
201 | |
202 | ++counter; |
203 | return block; |
204 | } |
205 | |
206 | /* |
207 | Generates data blocks the first time they are requested. Keeps copies for reuse. |
208 | */ |
209 | QByteArray const &tst_LargeFile::getDataBlock(int index, qint64 position) |
210 | { |
211 | if (index >= generatedBlocks.size()) |
212 | generatedBlocks.resize(asize: index + 1); |
213 | |
214 | if (generatedBlocks[index].isNull()) { |
215 | QString text = QString("Current %1-byte block (index = %2) " |
216 | "starts %3 bytes into the file '%4'." ) |
217 | .arg(a: blockSize) |
218 | .arg(a: index) |
219 | .arg(a: position) |
220 | .arg(a: "qt_largefile.tmp" ); |
221 | |
222 | generatedBlocks[index] = generateDataBlock(blockSize, text, userBits: (qint64)1 << index); |
223 | } |
224 | |
225 | return generatedBlocks[index]; |
226 | } |
227 | |
228 | void tst_LargeFile::initTestCase() |
229 | { |
230 | m_previousCurrent = QDir::currentPath(); |
231 | m_tempDir = QSharedPointer<QTemporaryDir>::create(); |
232 | QVERIFY2(!m_tempDir.isNull(), qPrintable("Could not create temporary directory." )); |
233 | QVERIFY2(QDir::setCurrent(m_tempDir->path()), qPrintable("Could not switch current directory" )); |
234 | |
235 | QFile file("qt_largefile.tmp" ); |
236 | QVERIFY( !file.exists() || file.remove() ); |
237 | } |
238 | |
239 | void tst_LargeFile::cleanupTestCase() |
240 | { |
241 | if (largeFile.isOpen()) |
242 | largeFile.close(); |
243 | |
244 | QFile file("qt_largefile.tmp" ); |
245 | QVERIFY( !file.exists() || file.remove() ); |
246 | |
247 | QDir::setCurrent(m_previousCurrent); |
248 | } |
249 | |
250 | void tst_LargeFile::init() |
251 | { |
252 | fd_ = -1; |
253 | stream_ = 0; |
254 | } |
255 | |
256 | void tst_LargeFile::cleanup() |
257 | { |
258 | if (-1 != fd_) |
259 | QT_CLOSE(fd: fd_); |
260 | if (stream_) |
261 | ::fclose(stream: stream_); |
262 | } |
263 | |
264 | void tst_LargeFile::sparseFileData() |
265 | { |
266 | QTest::addColumn<int>(name: "index" ); |
267 | QTest::addColumn<qint64>(name: "position" ); |
268 | QTest::addColumn<QByteArray>(name: "block" ); |
269 | |
270 | QTest::newRow(dataTag: QString("block[%1] @%2)" ) |
271 | .arg(a: 0).arg(a: 0) |
272 | .toLocal8Bit().constData()) |
273 | << 0 << (qint64)0 << getDataBlock(index: 0, position: 0); |
274 | |
275 | // While on Linux sparse files scale well, on Windows, testing at every |
276 | // power of 2 leads to very large files. i += 4 gives us a good coverage |
277 | // without taxing too much on resources. |
278 | for (int index = 12; index <= maxSizeBits; index += 4) { |
279 | qint64 position = (qint64)1 << index; |
280 | QByteArray block = getDataBlock(index, position); |
281 | |
282 | QTest::newRow( |
283 | dataTag: QString("block[%1] @%2)" ) |
284 | .arg(a: index).arg(a: position) |
285 | .toLocal8Bit().constData()) |
286 | << index << position << block; |
287 | } |
288 | } |
289 | |
290 | void tst_LargeFile::createSparseFile() |
291 | { |
292 | #if defined(Q_OS_WIN32) |
293 | // On Windows platforms, we must explicitly set the file to be sparse, |
294 | // so disk space is not allocated for the full file when writing to it. |
295 | HANDLE handle = ::CreateFileA("qt_largefile.tmp" , |
296 | GENERIC_WRITE, 0, 0, CREATE_ALWAYS, 0, 0); |
297 | QVERIFY( INVALID_HANDLE_VALUE != handle ); |
298 | |
299 | DWORD bytes; |
300 | if (!::DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, |
301 | &bytes, NULL)) { |
302 | QWARN("Unable to set test file as sparse. " |
303 | "Limiting test file to 16MiB." ); |
304 | maxSizeBits = 24; |
305 | } |
306 | |
307 | int fd = ::_open_osfhandle((intptr_t)handle, 0); |
308 | QVERIFY( -1 != fd ); |
309 | QVERIFY( largeFile.open(fd, QIODevice::WriteOnly | QIODevice::Unbuffered) ); |
310 | #else // !Q_OS_WIN32 |
311 | largeFile.setFileName("qt_largefile.tmp" ); |
312 | QVERIFY( largeFile.open(QIODevice::WriteOnly | QIODevice::Unbuffered) ); |
313 | #endif |
314 | } |
315 | |
316 | void tst_LargeFile::closeSparseFile() |
317 | { |
318 | #if defined(Q_OS_WIN32) |
319 | int fd = largeFile.handle(); |
320 | #endif |
321 | |
322 | largeFile.close(); |
323 | |
324 | #if defined(Q_OS_WIN32) |
325 | if (-1 != fd) |
326 | ::_close(fd); |
327 | #endif |
328 | } |
329 | |
330 | void tst_LargeFile::fillFileSparsely() |
331 | { |
332 | QFETCH( qint64, position ); |
333 | QFETCH( QByteArray, block ); |
334 | QCOMPARE( block.size(), blockSize ); |
335 | |
336 | static int lastKnownGoodIndex = 0; |
337 | struct ScopeGuard { |
338 | ScopeGuard(tst_LargeFile* test) |
339 | : this_(test) |
340 | , failed(true) |
341 | { |
342 | QFETCH( int, index ); |
343 | index_ = index; |
344 | } |
345 | |
346 | ~ScopeGuard() |
347 | { |
348 | if (failed) { |
349 | this_->maxSizeBits = lastKnownGoodIndex; |
350 | QWARN( qPrintable( |
351 | QString("QFile::error %1: '%2'. Maximum size bits reset to %3." ) |
352 | .arg(this_->largeFile.error()) |
353 | .arg(this_->largeFile.errorString()) |
354 | .arg(this_->maxSizeBits)) ); |
355 | } else |
356 | lastKnownGoodIndex = qMax<int>(a: index_, b: lastKnownGoodIndex); |
357 | } |
358 | |
359 | private: |
360 | tst_LargeFile * const this_; |
361 | int index_; |
362 | |
363 | public: |
364 | bool failed; |
365 | }; |
366 | |
367 | ScopeGuard resetMaxSizeBitsOnFailure(this); |
368 | |
369 | QVERIFY( largeFile.seek(position) ); |
370 | QCOMPARE( largeFile.pos(), position ); |
371 | |
372 | QCOMPARE( largeFile.write(block), (qint64)blockSize ); |
373 | QCOMPARE( largeFile.pos(), position + blockSize ); |
374 | QVERIFY( largeFile.flush() ); |
375 | |
376 | resetMaxSizeBitsOnFailure.failed = false; |
377 | } |
378 | |
379 | void tst_LargeFile::fileCreated() |
380 | { |
381 | QFileInfo info("qt_largefile.tmp" ); |
382 | |
383 | QVERIFY( info.exists() ); |
384 | QVERIFY( info.isFile() ); |
385 | QVERIFY( info.size() >= ((qint64)1 << maxSizeBits) + blockSize ); |
386 | } |
387 | |
388 | void tst_LargeFile::filePositioning() |
389 | { |
390 | QFETCH( qint64, position ); |
391 | |
392 | QFile file("qt_largefile.tmp" ); |
393 | QVERIFY( file.open(QIODevice::ReadOnly) ); |
394 | |
395 | QVERIFY( file.seek(position) ); |
396 | QCOMPARE( file.pos(), position ); |
397 | } |
398 | |
399 | void tst_LargeFile::fdPositioning() |
400 | { |
401 | QFETCH( qint64, position ); |
402 | |
403 | fd_ = QT_OPEN(file: "qt_largefile.tmp" , |
404 | QT_OPEN_RDONLY | QT_OPEN_LARGEFILE); |
405 | QVERIFY( -1 != fd_ ); |
406 | |
407 | QFile file; |
408 | QVERIFY( file.open(fd_, QIODevice::ReadOnly) ); |
409 | QCOMPARE( file.pos(), (qint64)0 ); |
410 | QVERIFY( file.seek(position) ); |
411 | QCOMPARE( file.pos(), position ); |
412 | |
413 | file.close(); |
414 | |
415 | QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(0), SEEK_SET)), QT_OFF_T(0) ); |
416 | QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(position), SEEK_SET)), QT_OFF_T(position) ); |
417 | |
418 | QVERIFY( file.open(fd_, QIODevice::ReadOnly) ); |
419 | QCOMPARE( QT_OFF_T(QT_LSEEK(fd_, QT_OFF_T(0), SEEK_CUR)), QT_OFF_T(position) ); |
420 | QCOMPARE( file.pos(), position ); |
421 | QVERIFY( file.seek(0) ); |
422 | QCOMPARE( file.pos(), (qint64)0 ); |
423 | |
424 | file.close(); |
425 | |
426 | QVERIFY( !QT_CLOSE(fd_) ); |
427 | fd_ = -1; |
428 | } |
429 | |
430 | void tst_LargeFile::streamPositioning() |
431 | { |
432 | QFETCH( qint64, position ); |
433 | |
434 | stream_ = QT_FOPEN(filename: "qt_largefile.tmp" , modes: "rb" ); |
435 | QVERIFY( 0 != stream_ ); |
436 | |
437 | QFile file; |
438 | QVERIFY( file.open(stream_, QIODevice::ReadOnly) ); |
439 | QCOMPARE( file.pos(), (qint64)0 ); |
440 | QVERIFY( file.seek(position) ); |
441 | QCOMPARE( file.pos(), position ); |
442 | |
443 | file.close(); |
444 | |
445 | QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(0), SEEK_SET) ); |
446 | QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(0) ); |
447 | QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(position), SEEK_SET) ); |
448 | QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(position) ); |
449 | |
450 | QVERIFY( file.open(stream_, QIODevice::ReadOnly) ); |
451 | QCOMPARE( QT_OFF_T(QT_FTELL(stream_)), QT_OFF_T(position) ); |
452 | QCOMPARE( file.pos(), position ); |
453 | QVERIFY( file.seek(0) ); |
454 | QCOMPARE( file.pos(), (qint64)0 ); |
455 | |
456 | file.close(); |
457 | |
458 | QVERIFY( !::fclose(stream_) ); |
459 | stream_ = 0; |
460 | } |
461 | |
462 | void tst_LargeFile::openFileForReading() |
463 | { |
464 | largeFile.setFileName("qt_largefile.tmp" ); |
465 | QVERIFY( largeFile.open(QIODevice::ReadOnly) ); |
466 | } |
467 | |
468 | void tst_LargeFile::readFile() |
469 | { |
470 | QFETCH( qint64, position ); |
471 | QFETCH( QByteArray, block ); |
472 | QCOMPARE( block.size(), blockSize ); |
473 | |
474 | QVERIFY( largeFile.size() >= position + blockSize ); |
475 | |
476 | QVERIFY( largeFile.seek(position) ); |
477 | QCOMPARE( largeFile.pos(), position ); |
478 | |
479 | QCOMPARE( largeFile.read(blockSize), block ); |
480 | QCOMPARE( largeFile.pos(), position + blockSize ); |
481 | } |
482 | |
483 | void tst_LargeFile::mapFile() |
484 | { |
485 | QFETCH( qint64, position ); |
486 | QFETCH( QByteArray, block ); |
487 | QCOMPARE( block.size(), blockSize ); |
488 | |
489 | // Keep full block mapped to facilitate OS and/or internal reuse by Qt. |
490 | uchar *baseAddress = largeFile.map(offset: position, size: blockSize); |
491 | QVERIFY( baseAddress ); |
492 | QVERIFY( std::equal(block.begin(), block.end(), reinterpret_cast<char*>(baseAddress)) ); |
493 | |
494 | for (int offset = 1; offset < blockSize; ++offset) { |
495 | uchar *address = largeFile.map(offset: position + offset, size: blockSize - offset); |
496 | |
497 | QVERIFY( address ); |
498 | if ( !std::equal(first1: block.begin() + offset, last1: block.end(), first2: reinterpret_cast<char*>(address)) ) { |
499 | qDebug() << "Expected:" << block.toHex(); |
500 | qDebug() << "Actual :" << QByteArray(reinterpret_cast<char*>(address), blockSize).toHex(); |
501 | QVERIFY(false); |
502 | } |
503 | |
504 | QVERIFY( largeFile.unmap( address ) ); |
505 | } |
506 | |
507 | QVERIFY( largeFile.unmap( baseAddress ) ); |
508 | } |
509 | |
510 | //Mac: memory-mapping beyond EOF may succeed but it could generate bus error on access |
511 | //FreeBSD: same |
512 | //Linux: memory-mapping beyond EOF usually succeeds, but depends on the filesystem |
513 | // 32-bit: limited to 44-bit offsets (when sizeof(off_t) == 8) |
514 | //Windows: memory-mapping beyond EOF is not allowed |
515 | void tst_LargeFile::mapOffsetOverflow() |
516 | { |
517 | enum { |
518 | #ifdef Q_OS_WIN |
519 | Succeeds = false, |
520 | MaxOffset = 63 |
521 | #else |
522 | Succeeds = true, |
523 | # if (defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)) && Q_PROCESSOR_WORDSIZE == 4 |
524 | MaxOffset = sizeof(QT_OFF_T) > 4 ? 43 : 30 |
525 | # else |
526 | MaxOffset = 8 * sizeof(QT_OFF_T) - 1 |
527 | # endif |
528 | #endif |
529 | }; |
530 | |
531 | QByteArray zeroPage(blockSize, '\0'); |
532 | for (int i = maxSizeBits + 1; i < 63; ++i) { |
533 | bool succeeds = Succeeds && (i <= MaxOffset); |
534 | uchar *address = 0; |
535 | qint64 offset = Q_INT64_C(1) << i; |
536 | |
537 | if (succeeds) |
538 | QTest::ignoreMessage(type: QtWarningMsg, message: "QFSFileEngine::map: Mapping a file beyond its size is not portable" ); |
539 | address = largeFile.map(offset, size: blockSize); |
540 | QCOMPARE(!!address, succeeds); |
541 | |
542 | if (succeeds) |
543 | QTest::ignoreMessage(type: QtWarningMsg, message: "QFSFileEngine::map: Mapping a file beyond its size is not portable" ); |
544 | address = largeFile.map(offset: offset + blockSize, size: blockSize); |
545 | QCOMPARE(!!address, succeeds); |
546 | } |
547 | } |
548 | |
549 | QTEST_APPLESS_MAIN(tst_LargeFile) |
550 | #include "tst_largefile.moc" |
551 | |
552 | |