1// Copyright (C) 2020 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only
3
4#include "qqsbcollection_p.h"
5#include <QtCore/QLockFile>
6#include <QtCore/QSaveFile>
7#include <QtCore/QCryptographicHash>
8#include <rhi/qrhi.h>
9
10QT_BEGIN_NAMESPACE
11
12QQsbCollection::~QQsbCollection()
13{
14}
15
16QDataStream &operator<<(QDataStream &stream, const QQsbCollection::Entry &entry)
17{
18 return (stream << entry.key << entry.value);
19}
20
21QDataStream &operator>>(QDataStream &stream, QQsbCollection::Entry &entry)
22{
23 QByteArray key;
24 qint64 value;
25 stream >> key >> value;
26 entry = QQsbCollection::Entry(key, value);
27 return stream;
28}
29
30size_t qHash(const QQsbCollection::Entry &entry, size_t)
31{
32 return entry.hashKey;
33}
34
35bool operator==(const QQsbCollection::Entry &l, const QQsbCollection::Entry &r)
36{
37 return (l.key == r.key);
38}
39
40QDataStream &operator<<(QDataStream &stream, const QQsbCollection::EntryDesc &entryDesc)
41{
42 return (stream << entryDesc.materialKey
43 << entryDesc.featureSet
44 << entryDesc.vertShader.serialized()
45 << entryDesc.fragShader.serialized());
46}
47
48QDataStream &operator>>(QDataStream &stream, QQsbCollection::EntryDesc &entryDesc)
49{
50 QByteArray desc;
51 QQsbCollection::FeatureSet fs;
52 QByteArray vertData;
53 QByteArray fragData;
54 stream >> desc >> fs >> vertData >> fragData;
55 entryDesc.materialKey = desc;
56 entryDesc.featureSet = fs;
57 entryDesc.vertShader = QShader::fromSerialized(data: vertData);
58 entryDesc.fragShader = QShader::fromSerialized(data: fragData);
59 return stream;
60}
61
62static constexpr quint64 MagicaDS = 0x3933333335346337;
63static constexpr qint64 HeaderSize = sizeof(qint64 /*startOffs*/) + sizeof(quint8 /*version*/) + sizeof(quint32 /*qtVersion*/) + sizeof(MagicaDS);
64static constexpr quint32 QtVersion = (QT_VERSION_MAJOR << 16) | (QT_VERSION_MINOR << 8) | (QT_VERSION_PATCH);
65
66bool QQsbCollection::readEndHeader(QDataStream &ds, qint64 *startPos, quint8 *version)
67{
68 quint64 fileId = 0;
69 quint32 qtver = 0;
70 ds >> *startPos >> *version >> qtver >> fileId;
71 if (fileId != MagicaDS) {
72 qWarning(msg: "Corrupt qsbc file");
73 return false;
74 }
75 if (*version != Version::Two) {
76 qWarning(msg: "qsbc file has an unsupported version");
77 return false;
78 }
79 if (qtver != QtVersion) {
80 qWarning(msg: "qsbc file is for a different Qt version");
81 return false;
82 }
83 return true;
84}
85
86bool QQsbCollection::readEndHeader(QIODevice *device, EntryMap *entries, quint8 *version)
87{
88 bool result = false;
89 const qint64 size = device->size();
90 if (device->seek(pos: size - HeaderSize)) {
91 QDataStream ds(device);
92 ds.setVersion(QDataStream::Qt_6_0);
93 qint64 startPos = 0;
94 if (readEndHeader(ds, startPos: &startPos, version)) {
95 if (startPos >= 0 && startPos < size && device->seek(pos: startPos)) {
96 ds >> *entries;
97 result = true;
98 }
99 }
100 }
101 return result;
102}
103
104void QQsbCollection::writeEndHeader(QDataStream &ds, qint64 startPos, quint8 version, quint64 magic)
105{
106 ds << startPos << version << QtVersion << magic;
107}
108
109void QQsbCollection::writeEndHeader(QIODevice *device, const EntryMap &entries)
110{
111 if (!device->atEnd()) {
112 device->seek(pos: device->size() - 1);
113 Q_ASSERT(device->atEnd());
114 }
115 QDataStream ds(device);
116 ds.setVersion(QDataStream::Qt_6_0);
117 const qint64 startPos = device->pos();
118 ds << entries;
119 writeEndHeader(ds, startPos, version: quint8(Version::Two), magic: MagicaDS);
120}
121
122QByteArray QQsbCollection::EntryDesc::generateSha(const QByteArray &materialKey, const FeatureSet &featureSet)
123{
124 QCryptographicHash h(QCryptographicHash::Algorithm::Sha1);
125 h.addData(data: materialKey);
126 for (auto it = featureSet.cbegin(), end = featureSet.cend(); it != end; ++it) {
127 if (it.value())
128 h.addData(data: it.key());
129 }
130 return h.result().toHex();
131}
132
133QByteArray QQsbCollection::EntryDesc::generateSha() const
134{
135 return generateSha(materialKey, featureSet);
136}
137
138QQsbCollection::EntryMap QQsbInMemoryCollection::availableEntries() const
139{
140 return EntryMap(entries.keyBegin(), entries.keyEnd());
141}
142
143QQsbCollection::Entry QQsbInMemoryCollection::addEntry(const QByteArray &key, const EntryDesc &entryDesc)
144{
145 Entry e(key);
146 if (!entries.contains(key: e)) {
147 entries.insert(key: e, value: entryDesc);
148 return e;
149 }
150 return {}; // can only add with a given key once
151}
152
153bool QQsbInMemoryCollection::extractEntry(Entry entry, EntryDesc &entryDesc)
154{
155 auto it = entries.constFind(key: entry);
156 if (it != entries.constEnd()) {
157 entryDesc = *it;
158 return true;
159 }
160 return false;
161}
162
163void QQsbInMemoryCollection::clear()
164{
165 entries.clear();
166}
167
168static inline QString lockFileName(const QString &name)
169{
170 return name + QLatin1String(".lck");
171}
172
173bool QQsbInMemoryCollection::load(const QString &filename)
174{
175 QLockFile lock(lockFileName(name: filename));
176 if (!lock.lock()) {
177 qWarning(msg: "Could not create shader cache lock file '%s'",
178 qPrintable(lock.fileName()));
179 return false;
180 }
181
182 QFile f(filename);
183 if (!f.open(flags: QIODevice::ReadOnly)) {
184 qWarning(msg: "Failed to open qsbc file %s", qPrintable(filename));
185 return false;
186 }
187
188 EntryMap entryMap;
189 quint8 version = 0;
190 if (!readEndHeader(device: &f, entries: &entryMap, version: &version)) {
191 qWarning(msg: "Ignoring qsbc file %s", qPrintable(filename));
192 return false;
193 }
194
195 f.seek(offset: 0);
196 const qint64 size = f.size();
197
198 clear();
199
200 for (const Entry &e : entryMap) {
201 const qint64 offset = e.value;
202 if (e.isValid() && offset >= 0 && size > offset && f.seek(offset)) {
203 QDataStream ds(&f);
204 ds.setVersion(QDataStream::Qt_6_0);
205 EntryDesc entryDesc;
206 ds >> entryDesc;
207 entries.insert(key: Entry(e.key), value: entryDesc);
208 }
209 }
210
211 return true;
212}
213
214bool QQsbInMemoryCollection::save(const QString &filename)
215{
216 QLockFile lock(lockFileName(name: filename));
217 if (!lock.lock()) {
218 qWarning(msg: "Could not create shader cache lock file '%s'",
219 qPrintable(lock.fileName()));
220 return false;
221 }
222
223#if QT_CONFIG(temporaryfile)
224 QSaveFile f(filename);
225#else
226 QFile f(filename);
227#endif
228 if (!f.open(flags: QIODevice::WriteOnly | QIODevice::Truncate)) {
229 qWarning(msg: "Failed to write qsbc file %s", qPrintable(filename));
230 return false;
231 }
232
233 QDataStream ds(&f);
234 ds.setVersion(QDataStream::Qt_6_0);
235
236 EntryMap entryMap;
237 for (auto it = entries.cbegin(), end = entries.cend(); it != end; ++it) {
238 const qint64 offset = f.pos();
239 ds << it.value();
240 entryMap.insert(value: Entry(it.key().key, offset));
241 }
242
243 writeEndHeader(device: &f, entries: entryMap);
244
245#if QT_CONFIG(temporaryfile)
246 return f.commit();
247#else
248 return true;
249#endif
250}
251
252QQsbIODeviceCollection::QQsbIODeviceCollection(const QString &filePath)
253 : file(filePath)
254 , device(file)
255{
256}
257
258QQsbIODeviceCollection::QQsbIODeviceCollection(QIODevice &dev)
259 : device(dev)
260 , devOwner(DeviceOwner::Extern)
261{
262
263}
264
265QQsbIODeviceCollection::~QQsbIODeviceCollection()
266{
267 if (!entries.isEmpty() || device.isOpen())
268 unmap();
269}
270
271bool QQsbIODeviceCollection::map(MapMode mode)
272{
273 if (device.isOpen()) {
274 // Make sure Truncate is set if we're writing.
275 if ((device.openMode() & QIODevice::WriteOnly) != 0) {
276 if ((device.openMode() & QIODevice::Truncate) == 0) {
277 qWarning(msg: "Open mode needs to have Truncate set for writing!");
278 return false;
279 }
280 if ((device.openMode() & QIODevice::Text) != 0) {
281 qWarning(msg: "Open mode can't have Text mode set!");
282 return false;
283 }
284 }
285 } else if (!device.open(mode: QIODevice::OpenMode(mode))) {
286 qWarning(msg: "Unable to open device!");
287 return false;
288 }
289
290 if (mode == Write)
291 return true;
292
293 Q_ASSERT(mode == Read);
294
295 const bool ret = readEndHeader(device: &device, entries: &entries, version: &version);
296
297 if (!ret)
298 unmap();
299
300 return ret;
301}
302
303void QQsbIODeviceCollection::unmap()
304{
305 if (device.isOpen() && ((device.openMode() & Write) == Write)) {
306 if (!entries.isEmpty()) {
307 writeEndHeader(device: &device, entries);
308 } else {
309 if (devOwner == DeviceOwner::Self)
310 file.remove();
311 }
312 }
313 device.close();
314 entries.clear();
315}
316
317QQsbCollection::EntryMap QQsbIODeviceCollection::availableEntries() const
318{
319 return entries;
320}
321
322QQsbCollection::Entry QQsbIODeviceCollection::addEntry(const QByteArray &key, const EntryDesc &entryDesc)
323{
324 if (entries.contains(value: Entry(key)) || !map(mode: MapMode::Write))
325 return {};
326
327 QDataStream ds(&device);
328 ds.setVersion(QDataStream::Qt_6_0);
329 const auto offset = device.pos();
330 ds << entryDesc;
331 Entry e(key, offset);
332 entries.insert(value: e);
333 return e;
334}
335
336bool QQsbIODeviceCollection::extractEntry(Entry entry, EntryDesc &entryDesc)
337{
338 if (device.isOpen() && device.isReadable()) {
339 const qint64 offset = entry.value;
340 if (entry.isValid() && offset >= 0) {
341 const qint64 size = device.size();
342 if (size > offset && device.seek(pos: offset)) {
343 QDataStream ds(&device);
344 ds.setVersion(QDataStream::Qt_6_0);
345 ds >> entryDesc;
346 return true;
347 }
348 } else {
349 qWarning(msg: "Entry not found id(%s), offset(%lld)", entry.key.constData(), entry.value);
350 }
351 } else {
352 qWarning(msg: "Unable to open file for reading");
353 }
354
355 return false;
356}
357
358static const char *borderText() { return "--------------------------------------------------------------------------------"; }
359
360void QQsbIODeviceCollection::dumpInfo()
361{
362 if (map(mode: QQsbIODeviceCollection::Read)) {
363 qDebug(msg: "Number of entries in collection: %zu\n", size_t(entries.size()));
364 int i = 0;
365 qDebug(msg: "Qsbc version: %u", version);
366 for (const auto &e : std::as_const(t&: entries)) {
367 qDebug(msg: "%s\n"
368 "Entry %d\n%s\n"
369 "Key: %s\n"
370 "Offset: %llu", borderText(), i++, borderText(), e.key.constData(), e.value);
371
372 QQsbCollection::EntryDesc ed;
373 if (extractEntry(entry: e, entryDesc&: ed)) {
374 qDebug() << ed.materialKey << Qt::endl
375 << ed.featureSet << Qt::endl
376 << ed.vertShader << Qt::endl
377 << ed.fragShader;
378 } else {
379 qWarning(msg: "Extracting Qsb entry failed!");
380 }
381 }
382 }
383 unmap();
384}
385
386void QQsbIODeviceCollection::dumpInfo(const QString &file)
387{
388 QQsbIODeviceCollection qsbc(file);
389 qsbc.dumpInfo();
390}
391
392void QQsbIODeviceCollection::dumpInfo(QIODevice &device)
393{
394 QQsbIODeviceCollection qsbc(device);
395 qsbc.dumpInfo();
396}
397
398QT_END_NAMESPACE
399

source code of qtquick3d/src/utils/qqsbcollection.cpp