1// Copyright (C) 2025 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
3// Qt-Security score:critical reason:data-parser
4
5#include "qssglightmapio_p.h"
6
7#include <private/qssgrenderloadedtexture_p.h>
8#include <private/qssgassert_p.h>
9
10#include <QDataStream>
11#include <QDebug>
12#include <QtEndian>
13#include <QFile>
14
15#include <algorithm>
16#include <cstring>
17
18QT_BEGIN_NAMESPACE
19
20using IndexKey = std::tuple<QSSGLightmapIODataTag /* dataTag */, qint32 /* keySize */, QByteArray /* key */>;
21
22struct IndexEntry
23{
24 qint64 keyOffset; // 8 bytes
25 qint64 keySize; // 8 bytes
26 qint64 dataOffset; // 8 bytes
27 qint64 dataSize; // 8 bytes
28 QSSGLightmapIODataTag dataTag; // 4 bytes
29 quint32 padding;
30};
31
32struct QSSGLightmapIOPrivate
33{
34 QByteArray readKey(const IndexKey &indexKey) const;
35 bool writeHeader() const;
36 bool writeData(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &data);
37 bool writeFooter();
38 bool decodeHeaders();
39 QList<std::pair<QString, QSSGLightmapIODataTag>> getKeys() const;
40
41 QSharedPointer<QIODevice> stream;
42 QMap<IndexKey, IndexEntry> entries; // Maps from name -> entry
43 qint64 entryCount = -1;
44 qint64 indexOffset = -1;
45 qint64 fileVersion = -1;
46 qint64 fileSize = -1;
47};
48
49static_assert(offsetof(IndexEntry, keyOffset) == 0, "Unexpected alignment");
50static_assert(offsetof(IndexEntry, keySize) == 8, "Unexpected alignment");
51static_assert(offsetof(IndexEntry, dataOffset) == 16, "Unexpected alignment");
52static_assert(offsetof(IndexEntry, dataSize) == 24, "Unexpected alignment");
53static_assert(offsetof(IndexEntry, dataTag) == 32, "Unexpected alignment");
54static_assert(offsetof(IndexEntry, padding) == 36, "Unexpected alignment");
55static_assert(sizeof(IndexEntry) == 40, "Unexpected size");
56
57constexpr char fileSignature[] = "QTLTMP";
58
59static IndexKey keyToIndexKey(const QString &key, QSSGLightmapIODataTag tag)
60{
61 QByteArray keyBuffer = key.toUtf8();
62 return std::make_tuple(args&: tag, args: keyBuffer.size(), args&: keyBuffer);
63}
64
65static QByteArray mapToByteArray(const QVariantMap &map)
66{
67 QByteArray byteArray;
68 QDataStream out(&byteArray, QIODevice::WriteOnly);
69 out.setVersion(QDataStream::Qt_6_9);
70 out.setByteOrder(QDataStream::LittleEndian);
71 out << QVariant(map);
72 return byteArray;
73}
74
75static void convertEndian(QByteArray &buffer, int sizeOfDataType)
76{
77 if (QSysInfo::ByteOrder == QSysInfo::LittleEndian) {
78 return; // No need to flip
79 } else if (QSysInfo::ByteOrder == QSysInfo::BigEndian) {
80 // Need to flip
81 } else {
82 qWarning() << "QSSGLightmapLoader::convertEndian: Unknown endianness";
83 return;
84 }
85
86 Q_ASSERT(buffer.size() % sizeOfDataType == 0);
87 if (buffer.size() % sizeOfDataType != 0) {
88 qWarning() << "QSSGLightmapLoader::convertEndian: Unexpected buffer size";
89 return;
90 }
91
92 if (sizeOfDataType == 1)
93 return;
94
95 const int buffSize = buffer.size();
96 for (int offset = 0; offset < buffSize; offset += sizeOfDataType) {
97 for (int i = 0; i < sizeOfDataType / 2; ++i) {
98 std::swap(a&: buffer[offset + i], b&: buffer[offset + sizeOfDataType - 1 - i]);
99 }
100 }
101}
102
103static QVariantMap byteArrayToMap(QByteArray input)
104{
105 QVariant variant;
106 QDataStream in(input);
107 in.setVersion(QDataStream::Qt_6_9);
108 in.setByteOrder(QDataStream::LittleEndian);
109 in >> variant;
110 return variant.toMap();
111}
112
113bool QSSGLightmapIOPrivate::decodeHeaders()
114{
115 Q_ASSERT(stream);
116
117 if (!stream->isOpen())
118 stream->open(mode: QIODeviceBase::OpenModeFlag::ReadOnly);
119
120 if (!stream->isOpen()) {
121 qWarning() << "QSSGLightmapIO: Stream is not openable";
122 return false;
123 }
124
125 if (!stream->isReadable()) {
126 qWarning() << "QSSGLightmapIO: Stream is not readable";
127 return false;
128 }
129
130 // [Magic Bytes][Version]
131 constexpr int headerSize = 6 * sizeof(char) + sizeof(qint32);
132 // [Entry Count][Offset Index]
133 constexpr int footerSize = 2 * sizeof(qint64);
134
135 const qint64 fileSize = stream->size();
136 if (fileSize < headerSize + footerSize) {
137 qWarning() << "QSSGLightmapIO: File too small to contain header and footer";
138 return false;
139 }
140
141 stream->seek(pos: 0);
142 QByteArray headerData = stream->read(maxlen: headerSize);
143 if (headerData.size() != qsizetype(headerSize)) {
144 qWarning() << "Failed to read header";
145 return false;
146 }
147
148 // Verify file signature
149 if (QByteArrayView(headerData.constData(), 6) != QByteArray::fromRawData(data: fileSignature, size: 6)) {
150 qWarning() << "QSSGLightmapIO: Invalid file signature";
151 return false;
152 }
153
154 qint32 fileVersion = -1;
155
156 // Read file version (4 bytes after 6 magic bytes)
157 const char *versionPtr = headerData.constData() + 6;
158 fileVersion = qFromLittleEndian<qint32>(src: versionPtr);
159
160 if (fileVersion != 1) {
161 qWarning() << "QSSGLightmapIO: Invalid file version";
162 return false;
163 }
164
165 // Seek to the last bytes (footer)
166 if (!stream->seek(pos: fileSize - footerSize)) {
167 qWarning() << "Failed to seek to footer";
168 return false;
169 }
170
171 QByteArray footerData = stream->read(maxlen: footerSize);
172 if (footerData.size() != qsizetype(footerSize)) {
173 qWarning() << "Failed to read footer";
174 return false;
175 }
176
177 const char *footerPtr = footerData.constData();
178 const qint64 indexOffset = qFromLittleEndian<qint64>(src: footerPtr);
179 const qint64 entryCount = qFromLittleEndian<qint64>(src: footerPtr + sizeof(qint64));
180
181 this->entryCount = entryCount;
182 this->indexOffset = indexOffset;
183 this->fileVersion = fileVersion;
184 this->fileSize = fileSize;
185
186 return true;
187}
188
189bool QSSGLightmapIOPrivate::writeHeader() const
190{
191 Q_ASSERT(stream);
192 if (!stream->isOpen())
193 stream->open(mode: QIODeviceBase::OpenModeFlag::WriteOnly);
194
195 if (!stream->isOpen()) {
196 qWarning() << "QSSGLightmapIO: File is not openable";
197 return false;
198 }
199
200 if (!stream->isWritable()) {
201 qWarning() << "QSSGLightmapIO: File is not writable";
202 return false;
203 }
204
205 stream->seek(pos: 0);
206
207 // Write file signature
208 if (stream->write(data: fileSignature, len: sizeof(fileSignature) - 1) != sizeof(fileSignature) - 1) {
209 qWarning() << "QSSGLightmapIO: Failed to write file signature";
210 stream->close();
211 return false;
212 }
213
214 // Write file version
215 const qint32 version = qToLittleEndian<qint32>(source: 1);
216 if (stream->write(data: reinterpret_cast<const char *>(&version), len: sizeof(version)) != sizeof(version)) {
217 qWarning() << "QSSGLightmapIO: Failed to write file version";
218 stream->close();
219 return false;
220 }
221
222 return true;
223}
224
225bool QSSGLightmapIOPrivate::writeData(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &data)
226{
227 Q_ASSERT(stream->isOpen() && stream->isWritable());
228 IndexKey keyBytes = keyToIndexKey(key, tag);
229 Q_ASSERT(!entries.contains(keyBytes));
230
231 IndexEntry &entry = entries[keyBytes];
232 entry.dataOffset = stream->pos();
233 entry.dataSize = data.size();
234 entry.dataTag = tag;
235
236 if (stream->write(data) != data.size()) {
237 qWarning() << "QSSGLightmapIO: Failed to write entry data";
238 return false;
239 }
240
241 return true;
242}
243
244template<typename T>
245bool writeType(const QSharedPointer<QIODevice> &stream, T value)
246{
247 return stream->write(data: reinterpret_cast<const char *>(&value), len: sizeof(value)) == sizeof(value);
248}
249
250bool QSSGLightmapIOPrivate::writeFooter()
251{
252 Q_ASSERT(stream);
253 Q_ASSERT(stream->isOpen());
254 Q_ASSERT(stream->isWritable());
255
256 // Store the key strings
257 for (auto it = entries.begin(); it != entries.end(); ++it) {
258 auto [dataTag, keySize, key] = it.key();
259 IndexEntry &entry = it.value();
260 entry.keyOffset = stream->pos();
261 entry.keySize = keySize;
262 if (stream->write(data: key) != key.size()) {
263 qWarning() << "QSSGLightmapIO: Failed to write key";
264 return false;
265 }
266 }
267
268 // The file should be seeked to the end of the data segment so we can just
269 // add indices and footer now
270 const qint64 indexOffset = qToLittleEndian<qint64>(source: stream->pos());
271 const qint64 indexCount = qToLittleEndian<qint64>(source: entries.size());
272
273 for (const IndexEntry &entry : std::as_const(t&: entries)) {
274 const qint64 keyOffset = qToLittleEndian<qint64>(source: entry.keyOffset);
275 const qint64 keySize = qToLittleEndian<qint64>(source: entry.keySize);
276 const qint64 dataOffset = qToLittleEndian<qint64>(source: entry.dataOffset);
277 const qint64 dataSize = qToLittleEndian<qint64>(source: entry.dataSize);
278 const quint32 dataTag = qToLittleEndian<quint32>(source: std::underlying_type_t<QSSGLightmapIODataTag>(entry.dataTag));
279 const quint32 padding = qToLittleEndian<quint32>(source: entry.padding);
280 if (!writeType(stream, value: keyOffset) || !writeType(stream, value: keySize) || !writeType(stream, value: dataOffset)
281 || !writeType(stream, value: dataSize) || !writeType(stream, value: dataTag) || !writeType(stream, value: padding)) {
282 qWarning() << "QSSGLightmapIO: Failed to write entry";
283 return false;
284 }
285 }
286
287 // Write footer
288 if (!writeType(stream, value: indexOffset) || !writeType(stream, value: indexCount)) {
289 qWarning() << "QSSGLightmapIO: Failed to write footer";
290 return false;
291 }
292 stream->close();
293 return true;
294}
295
296QByteArray QSSGLightmapIOPrivate::readKey(const IndexKey &indexKey) const
297{
298 Q_ASSERT(stream);
299 Q_ASSERT(entryCount >= 0);
300 Q_ASSERT(indexOffset >= 0);
301 Q_ASSERT(fileVersion >= 0);
302 Q_ASSERT(fileSize >= 0);
303
304 // Perform binary search by reading one IndexEntry at a time
305 qint64 low = 0;
306 qint64 high = entryCount;
307 IndexEntry entry;
308 auto [dataTag, keySize, key] = indexKey;
309 bool found = false;
310 qint64 matchOffset = 0;
311 qint64 matchSize = 0;
312
313 while (low < high) {
314 qint64 mid = (low + high) / 2;
315 qint64 offset = indexOffset + mid * sizeof(IndexEntry);
316
317 if (!stream->seek(pos: offset)) {
318 qWarning() << "Failed to seek to index entry";
319 return {};
320 }
321
322 if (stream->read(data: reinterpret_cast<char *>(&entry), maxlen: sizeof(IndexEntry)) != sizeof(IndexEntry)) {
323 qWarning() << "Failed to read index entry";
324 return {};
325 }
326
327 // Sort by dataTag, keySize and name (matching the order of IndexKey)
328 int cmp = qint64(entry.dataTag) - qint64(dataTag);
329 if (cmp == 0) {
330 cmp = entry.keySize - keySize;
331 }
332 if (cmp == 0) {
333 if (!stream->seek(pos: entry.keyOffset)) {
334 qWarning() << "Failed to seek to key entry";
335 return {};
336 }
337 const QByteArray entryKey = stream->read(maxlen: entry.keySize);
338 if (entryKey.size() != entry.keySize) {
339 qWarning() << "Failed to read to key entry";
340 return {};
341 }
342 for (int i = 0, n = entry.keySize; i < n; ++i) {
343 cmp = int(entryKey[i]) - int(key[i]);
344 if (cmp != 0)
345 break;
346 }
347 }
348
349 if (cmp < 0) {
350 low = mid + 1;
351 } else if (cmp > 0) {
352 high = mid;
353 } else {
354 // Found
355 found = true;
356 matchOffset = qFromLittleEndian(source: entry.dataOffset);
357 matchSize = qFromLittleEndian(source: entry.dataSize);
358 break;
359 }
360 }
361
362 if (!found) {
363 qWarning() << "Key not found:" << key;
364 return {};
365 }
366
367 if (matchOffset + matchSize > fileSize) {
368 qWarning() << "Asset data out of bounds";
369 return {};
370 }
371
372 // Seek and read asset
373 if (!stream->seek(pos: matchOffset)) {
374 qWarning() << "Failed to seek to asset data";
375 return {};
376 }
377
378 QByteArray assetData = stream->read(maxlen: matchSize);
379 if (assetData.size() != static_cast<int>(matchSize)) {
380 qWarning() << "Failed to read full asset data";
381 return {};
382 }
383
384 return assetData;
385}
386
387QList<std::pair<QString, QSSGLightmapIODataTag>> QSSGLightmapIOPrivate::getKeys() const
388{
389 Q_ASSERT(stream);
390 Q_ASSERT(entryCount >= 0);
391 Q_ASSERT(indexOffset >= 0);
392 Q_ASSERT(fileVersion >= 0);
393 Q_ASSERT(fileSize >= 0);
394
395 QList<std::pair<QString, QSSGLightmapIODataTag>> keys;
396 keys.resize(size: entryCount);
397
398 IndexEntry entry;
399
400 for (int i = 0; i < entryCount; ++i) {
401 const qint64 offset = indexOffset + i * sizeof(IndexEntry);
402
403 if (!stream->seek(pos: offset)) {
404 qWarning() << "Failed to seek to index entry";
405 return {};
406 }
407
408 if (stream->read(data: reinterpret_cast<char *>(&entry), maxlen: sizeof(IndexEntry)) != sizeof(IndexEntry)) {
409 qWarning() << "Failed to read index entry";
410 return {};
411 }
412 if (!stream->seek(pos: entry.keyOffset)) {
413 qWarning() << "Failed to seek to key entry";
414 return {};
415 }
416 const QByteArray entryKey = stream->read(maxlen: entry.keySize);
417 if (entryKey.size() != entry.keySize) {
418 qWarning() << "Failed to read to key entry";
419 return {};
420 }
421
422 keys[i] = std::make_pair(x: QString::fromUtf8(ba: entryKey), y: static_cast<QSSGLightmapIODataTag>(entry.dataTag));
423 }
424
425 std::sort(first: keys.begin(), last: keys.end(), comp: [](const auto &a, const auto &b) { return a.first < b.first; });
426
427 return keys;
428}
429
430/////////////////////////////// Loader /////////////////////////////////
431
432QSSGLightmapLoader::QSSGLightmapLoader() : d(new QSSGLightmapIOPrivate) { };
433
434QSSGLightmapLoader::~QSSGLightmapLoader()
435{
436 if (d->stream)
437 d->stream->close();
438 delete d;
439}
440
441QSharedPointer<QSSGLightmapLoader> QSSGLightmapLoader::open(const QSharedPointer<QIODevice> &stream)
442{
443 if (!stream) {
444 qWarning() << "Failed to open lightmap: invalid stream";
445 return nullptr;
446 }
447
448 auto loader = QSharedPointer<QSSGLightmapLoader>(new QSSGLightmapLoader);
449 loader->d->stream = stream;
450
451 if (!loader->d->decodeHeaders()) {
452 if (loader->d->stream->isOpen())
453 loader->d->stream->close();
454 return {};
455 }
456
457 return loader;
458}
459
460QSharedPointer<QSSGLightmapLoader> QSSGLightmapLoader::open(const QString &path)
461{
462 if (QSharedPointer<QIODevice> source = QSSGInputUtil::getStreamForFile(inPath: path))
463 return open(stream: source);
464
465 return nullptr;
466}
467
468QByteArray QSSGLightmapLoader::readF32Image(const QString &key, QSSGLightmapIODataTag tag) const
469{
470 QByteArray buffer = d->readKey(indexKey: keyToIndexKey(key, tag));
471 convertEndian(buffer, sizeOfDataType: sizeof(float));
472 return buffer;
473}
474
475QByteArray QSSGLightmapLoader::readU32Image(const QString &key, QSSGLightmapIODataTag tag) const
476{
477 QByteArray buffer = d->readKey(indexKey: keyToIndexKey(key, tag));
478 convertEndian(buffer, sizeOfDataType: sizeof(quint32));
479 return buffer;
480}
481
482QByteArray QSSGLightmapLoader::readData(const QString &key, QSSGLightmapIODataTag tag) const
483{
484 return d->readKey(indexKey: keyToIndexKey(key, tag));
485}
486
487QVariantMap QSSGLightmapLoader::readMetadata(const QString &key) const
488{
489 QByteArray metadataBuffer = d->readKey(indexKey: keyToIndexKey(key, tag: QSSGLightmapIODataTag::Metadata));
490 QVariantMap metadata = byteArrayToMap(input: metadataBuffer);
491 return metadata;
492}
493
494QList<std::pair<QString, QSSGLightmapIODataTag>> QSSGLightmapLoader::getKeys() const
495{
496 return d->getKeys();
497}
498
499inline int calculateLine(int width, int bitdepth)
500{
501 return ((width * bitdepth) + 7) / 8;
502}
503inline int calculatePitch(int line)
504{
505 return (line + 3) & ~3;
506}
507
508QSSGLoadedTexture *QSSGLightmapLoader::createTexture(QSharedPointer<QIODevice> stream,
509 const QSSGRenderTextureFormat &format,
510 const QString &key)
511{
512 QSharedPointer<QSSGLightmapLoader> loader = QSSGLightmapLoader::open(stream);
513 if (!loader)
514 return nullptr;
515
516 QVariantMap metadata = loader->readMetadata(key);
517 if (metadata.isEmpty())
518 return nullptr;
519
520 bool ok = true;
521 const int w = metadata[QStringLiteral("width")].toInt(ok: &ok);
522 Q_ASSERT(ok);
523 const int h = metadata[QStringLiteral("height")].toInt(ok: &ok);
524 Q_ASSERT(ok);
525 QSize pixelSize = QSize(w, h);
526 QByteArray imageFP32 = loader->readF32Image(key, tag: QSSGLightmapIODataTag::Texture_Final);
527 if (imageFP32.isEmpty())
528 return nullptr;
529
530 // Setup Output container
531 const int bytesPerPixel = format.getSizeofFormat();
532 const int bitCount = bytesPerPixel * 8;
533 const int pitch = calculatePitch(line: calculateLine(width: pixelSize.width(), bitdepth: bitCount));
534 const size_t dataSize = pixelSize.height() * pitch;
535 QSSG_CHECK_X(dataSize <= std::numeric_limits<quint32>::max(), "Requested data size exceeds 4GB limit!");
536 QSSGLoadedTexture *imageData = new QSSGLoadedTexture;
537 imageData->dataSizeInBytes = quint32(dataSize);
538 imageData->data = ::malloc(size: imageData->dataSizeInBytes);
539 imageData->width = pixelSize.width();
540 imageData->height = pixelSize.height();
541 imageData->format = format;
542 imageData->components = format.getNumberOfComponent();
543 imageData->isSRGB = false;
544
545 std::array<float, 4> *source = reinterpret_cast<std::array<float, 4> *>(imageFP32.data());
546 quint8 *target = reinterpret_cast<quint8 *>(imageData->data);
547
548 int idx = 0;
549 float rgbaF32[4];
550
551 for (int y = imageData->height - 1; y >= 0; --y) {
552 for (int x = 0; x < imageData->width; x++) {
553 const int i = y * imageData->width + x;
554 const int lh = i * 4 * sizeof(float);
555 const int rh = imageFP32.size();
556 Q_ASSERT(lh < rh);
557 std::array<float, 4> v = source[i];
558 rgbaF32[0] = v[0];
559 rgbaF32[1] = v[1];
560 rgbaF32[2] = v[2];
561 rgbaF32[3] = v[3];
562
563 format.encodeToPixel(inPtr: rgbaF32, outPtr: target, byteOfs: idx * bytesPerPixel);
564 ++idx;
565 }
566 }
567
568 return imageData;
569}
570
571///////////////////////////// Writer ///////////////////////////////////
572
573QSSGLightmapWriter::QSSGLightmapWriter() : d(new QSSGLightmapIOPrivate) { }
574
575QSSGLightmapWriter::~QSSGLightmapWriter()
576{
577 if (d->stream->isOpen()) {
578 qWarning() << "Lightmap file open on destruction, closing";
579 close();
580 }
581 delete d;
582}
583
584QSharedPointer<QSSGLightmapWriter> QSSGLightmapWriter::open(const QString &path)
585{
586 return open(stream: QSharedPointer<QIODevice>(new QFile(path)));
587}
588
589QSharedPointer<QSSGLightmapWriter> QSSGLightmapWriter::open(const QSharedPointer<QIODevice> &stream)
590{
591 if (!stream) {
592 qWarning() << "Failed to open lightmap file";
593 return nullptr;
594 }
595
596 QSharedPointer<QSSGLightmapWriter> writer = QSharedPointer<QSSGLightmapWriter>(new QSSGLightmapWriter);
597 if (!stream->isOpen() && !stream->open(mode: QIODeviceBase::WriteOnly)) {
598 qWarning() << "Failed to open lightmap file";
599 return nullptr;
600 }
601
602 writer->d->stream = stream;
603 if (!writer->d->writeHeader())
604 return nullptr;
605
606 return writer;
607}
608
609bool QSSGLightmapWriter::writeF32Image(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &imageFP32)
610{
611 QByteArray buffer = QByteArray(imageFP32.constData(), imageFP32.size());
612 convertEndian(buffer, sizeOfDataType: sizeof(float));
613 return d->writeData(key, tag, data: buffer);
614}
615
616bool QSSGLightmapWriter::writeU32Image(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &imageU32)
617{
618 QByteArray buffer = QByteArray(imageU32.constData(), imageU32.size());
619 convertEndian(buffer, sizeOfDataType: sizeof(quint32));
620 return d->writeData(key, tag, data: buffer);
621}
622
623bool QSSGLightmapWriter::writeData(const QString &key, QSSGLightmapIODataTag tag, const QByteArray &buffer)
624{
625 return d->writeData(key, tag, data: buffer);
626}
627
628bool QSSGLightmapWriter::writeMetadata(const QString &key, const QVariantMap &metadata)
629{
630 return d->writeData(key, tag: QSSGLightmapIODataTag::Metadata, data: mapToByteArray(map: metadata));
631}
632
633bool QSSGLightmapWriter::close() const
634{
635 return d->writeFooter();
636}
637
638QT_END_NAMESPACE
639

source code of qtquick3d/src/runtimerender/qssglightmapio.cpp