1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qopenglprogrambinarycache_p.h"
5#include <QOpenGLContext>
6#include <QOpenGLExtraFunctions>
7#include <QSysInfo>
8#include <QStandardPaths>
9#include <QDir>
10#include <QSaveFile>
11#include <QCoreApplication>
12#include <QCryptographicHash>
13
14#ifdef Q_OS_UNIX
15#include <sys/mman.h>
16#include <private/qcore_unix_p.h>
17#endif
18
19QT_BEGIN_NAMESPACE
20
21using namespace Qt::StringLiterals;
22
23Q_LOGGING_CATEGORY(lcOpenGLProgramDiskCache, "qt.opengl.diskcache")
24
25#ifndef GL_CONTEXT_LOST
26#define GL_CONTEXT_LOST 0x0507
27#endif
28
29#ifndef GL_PROGRAM_BINARY_LENGTH
30#define GL_PROGRAM_BINARY_LENGTH 0x8741
31#endif
32
33#ifndef GL_NUM_PROGRAM_BINARY_FORMATS
34#define GL_NUM_PROGRAM_BINARY_FORMATS 0x87FE
35#endif
36
37const quint32 BINSHADER_MAGIC = 0x5174;
38const quint32 BINSHADER_VERSION = 0x3;
39const quint32 BINSHADER_QTVERSION = QT_VERSION;
40
41namespace {
42struct GLEnvInfo
43{
44 GLEnvInfo();
45
46 QByteArray glvendor;
47 QByteArray glrenderer;
48 QByteArray glversion;
49};
50}
51
52GLEnvInfo::GLEnvInfo()
53{
54 QOpenGLContext *ctx = QOpenGLContext::currentContext();
55 Q_ASSERT(ctx);
56 QOpenGLFunctions *f = ctx->functions();
57 const char *vendor = reinterpret_cast<const char *>(f->glGetString(GL_VENDOR));
58 const char *renderer = reinterpret_cast<const char *>(f->glGetString(GL_RENDERER));
59 const char *version = reinterpret_cast<const char *>(f->glGetString(GL_VERSION));
60 if (vendor)
61 glvendor = QByteArray(vendor);
62 if (renderer)
63 glrenderer = QByteArray(renderer);
64 if (version)
65 glversion = QByteArray(version);
66}
67
68QByteArray QOpenGLProgramBinaryCache::ProgramDesc::cacheKey() const
69{
70 QCryptographicHash keyBuilder(QCryptographicHash::Sha1);
71 for (const QOpenGLProgramBinaryCache::ShaderDesc &shader : shaders)
72 keyBuilder.addData(data: shader.source);
73
74 return keyBuilder.result().toHex();
75}
76
77static inline bool qt_ensureWritableDir(const QString &name)
78{
79 QDir::root().mkpath(dirPath: name);
80 return QFileInfo(name).isWritable();
81}
82
83QOpenGLProgramBinaryCache::QOpenGLProgramBinaryCache()
84 : m_cacheWritable(false)
85{
86 const QString subPath = "/qtshadercache-"_L1 + QSysInfo::buildAbi() + u'/';
87 const QString sharedCachePath = QStandardPaths::writableLocation(type: QStandardPaths::GenericCacheLocation);
88 m_globalCacheDir = sharedCachePath + subPath;
89 m_localCacheDir = QStandardPaths::writableLocation(type: QStandardPaths::CacheLocation) + subPath;
90
91 if (!sharedCachePath.isEmpty()) {
92 m_currentCacheDir = m_globalCacheDir;
93 m_cacheWritable = qt_ensureWritableDir(name: m_currentCacheDir);
94 }
95 if (!m_cacheWritable) {
96 m_currentCacheDir = m_localCacheDir;
97 m_cacheWritable = qt_ensureWritableDir(name: m_currentCacheDir);
98 }
99
100 qCDebug(lcOpenGLProgramDiskCache, "Cache location '%s' writable = %d", qPrintable(m_currentCacheDir), m_cacheWritable);
101}
102
103QString QOpenGLProgramBinaryCache::cacheFileName(const QByteArray &cacheKey) const
104{
105 return m_currentCacheDir + QString::fromUtf8(ba: cacheKey);
106}
107
108#define BASE_HEADER_SIZE (int(4 * sizeof(quint32)))
109#define FULL_HEADER_SIZE(stringsSize) (BASE_HEADER_SIZE + 12 + stringsSize + 8)
110#define PADDING_SIZE(fullHeaderSize) (((fullHeaderSize + 3) & ~3) - fullHeaderSize)
111
112static inline quint32 readUInt(const uchar **p)
113{
114 quint32 v;
115 memcpy(dest: &v, src: *p, n: sizeof(quint32));
116 *p += sizeof(quint32);
117 return v;
118}
119
120static inline QByteArray readStr(const uchar **p)
121{
122 quint32 len = readUInt(p);
123 QByteArray ba = QByteArray::fromRawData(data: reinterpret_cast<const char *>(*p), size: len);
124 *p += len;
125 return ba;
126}
127
128bool QOpenGLProgramBinaryCache::verifyHeader(const QByteArray &buf) const
129{
130 if (buf.size() < BASE_HEADER_SIZE) {
131 qCDebug(lcOpenGLProgramDiskCache, "Cached size too small");
132 return false;
133 }
134 const uchar *p = reinterpret_cast<const uchar *>(buf.constData());
135 if (readUInt(p: &p) != BINSHADER_MAGIC) {
136 qCDebug(lcOpenGLProgramDiskCache, "Magic does not match");
137 return false;
138 }
139 if (readUInt(p: &p) != BINSHADER_VERSION) {
140 qCDebug(lcOpenGLProgramDiskCache, "Version does not match");
141 return false;
142 }
143 if (readUInt(p: &p) != BINSHADER_QTVERSION) {
144 qCDebug(lcOpenGLProgramDiskCache, "Qt version does not match");
145 return false;
146 }
147 if (readUInt(p: &p) != sizeof(quintptr)) {
148 qCDebug(lcOpenGLProgramDiskCache, "Architecture does not match");
149 return false;
150 }
151 return true;
152}
153
154bool QOpenGLProgramBinaryCache::setProgramBinary(uint programId, uint blobFormat, const void *p, uint blobSize)
155{
156 QOpenGLContext *context = QOpenGLContext::currentContext();
157 QOpenGLExtraFunctions *funcs = context->extraFunctions();
158 while (true) {
159 GLenum error = funcs->glGetError();
160 if (error == GL_NO_ERROR || error == GL_CONTEXT_LOST)
161 break;
162 }
163#if QT_CONFIG(opengles2)
164 if (context->isOpenGLES() && context->format().majorVersion() < 3) {
165 initializeProgramBinaryOES(context);
166 programBinaryOES(programId, blobFormat, p, blobSize);
167 } else
168#endif
169 funcs->glProgramBinary(program: programId, binaryFormat: blobFormat, binary: p, length: blobSize);
170
171 GLenum err = funcs->glGetError();
172 if (err != GL_NO_ERROR) {
173 qCDebug(lcOpenGLProgramDiskCache, "Program binary failed to load for program %u, size %d, "
174 "format 0x%x, err = 0x%x",
175 programId, blobSize, blobFormat, err);
176 return false;
177 }
178 GLint linkStatus = 0;
179 funcs->glGetProgramiv(program: programId, GL_LINK_STATUS, params: &linkStatus);
180 if (linkStatus != GL_TRUE) {
181 qCDebug(lcOpenGLProgramDiskCache, "Program binary failed to load for program %u, size %d, "
182 "format 0x%x, linkStatus = 0x%x, err = 0x%x",
183 programId, blobSize, blobFormat, linkStatus, err);
184 return false;
185 }
186
187 qCDebug(lcOpenGLProgramDiskCache, "Program binary set for program %u, size %d, format 0x%x, err = 0x%x",
188 programId, blobSize, blobFormat, err);
189 return true;
190}
191
192#ifdef Q_OS_UNIX
193class FdWrapper
194{
195public:
196 FdWrapper(const QString &fn)
197 : ptr(MAP_FAILED)
198 {
199 fd = qt_safe_open(pathname: QFile::encodeName(fileName: fn).constData(), O_RDONLY);
200 }
201 ~FdWrapper()
202 {
203 if (ptr != MAP_FAILED)
204 munmap(addr: ptr, len: mapSize);
205 if (fd != -1)
206 qt_safe_close(fd);
207 }
208 bool map()
209 {
210 off_t offs = lseek(fd: fd, offset: 0, SEEK_END);
211 if (offs == (off_t) -1) {
212 qErrnoWarning(errno, msg: "lseek failed for program binary");
213 return false;
214 }
215 mapSize = static_cast<size_t>(offs);
216 ptr = mmap(addr: nullptr, len: mapSize, PROT_READ, MAP_SHARED, fd: fd, offset: 0);
217 return ptr != MAP_FAILED;
218 }
219
220 int fd;
221 void *ptr;
222 size_t mapSize;
223};
224#endif
225
226class DeferredFileRemove
227{
228public:
229 DeferredFileRemove(const QString &fn)
230 : fn(fn),
231 active(false)
232 {
233 }
234 ~DeferredFileRemove()
235 {
236 if (active)
237 QFile(fn).remove();
238 }
239 void setActive()
240 {
241 active = true;
242 }
243
244 QString fn;
245 bool active;
246};
247
248bool QOpenGLProgramBinaryCache::load(const QByteArray &cacheKey, uint programId)
249{
250 QMutexLocker lock(&m_mutex);
251 if (const MemCacheEntry *e = m_memCache.object(key: cacheKey))
252 return setProgramBinary(programId, blobFormat: e->format, p: e->blob.constData(), blobSize: e->blob.size());
253
254 QByteArray buf;
255 const QString fn = cacheFileName(cacheKey);
256 DeferredFileRemove undertaker(fn);
257#ifdef Q_OS_UNIX
258 FdWrapper fdw(fn);
259 if (fdw.fd == -1)
260 return false;
261 char header[BASE_HEADER_SIZE];
262 qint64 bytesRead = qt_safe_read(fd: fdw.fd, data: header, BASE_HEADER_SIZE);
263 if (bytesRead == BASE_HEADER_SIZE)
264 buf = QByteArray::fromRawData(data: header, BASE_HEADER_SIZE);
265#else
266 QFile f(fn);
267 if (!f.open(QIODevice::ReadOnly))
268 return false;
269 buf = f.read(BASE_HEADER_SIZE);
270#endif
271
272 if (!verifyHeader(buf)) {
273 undertaker.setActive();
274 return false;
275 }
276
277 const uchar *p;
278#ifdef Q_OS_UNIX
279 if (!fdw.map()) {
280 undertaker.setActive();
281 return false;
282 }
283 p = static_cast<const uchar *>(fdw.ptr) + BASE_HEADER_SIZE;
284#else
285 buf = f.readAll();
286 p = reinterpret_cast<const uchar *>(buf.constData());
287#endif
288
289 GLEnvInfo info;
290
291 QByteArray vendor = readStr(p: &p);
292 if (vendor != info.glvendor) {
293 // readStr returns non-null terminated strings just pointing to inside
294 // 'p' so must print these via the stream qCDebug and not constData().
295 qCDebug(lcOpenGLProgramDiskCache) << "GL_VENDOR does not match" << vendor << info.glvendor;
296 undertaker.setActive();
297 return false;
298 }
299 QByteArray renderer = readStr(p: &p);
300 if (renderer != info.glrenderer) {
301 qCDebug(lcOpenGLProgramDiskCache) << "GL_RENDERER does not match" << renderer << info.glrenderer;
302 undertaker.setActive();
303 return false;
304 }
305 QByteArray version = readStr(p: &p);
306 if (version != info.glversion) {
307 qCDebug(lcOpenGLProgramDiskCache) << "GL_VERSION does not match" << version << info.glversion;
308 undertaker.setActive();
309 return false;
310 }
311
312 quint32 blobFormat = readUInt(p: &p);
313 quint32 blobSize = readUInt(p: &p);
314
315 p += PADDING_SIZE(FULL_HEADER_SIZE(vendor.size() + renderer.size() + version.size()));
316
317 return setProgramBinary(programId, blobFormat, p, blobSize)
318 && m_memCache.insert(key: cacheKey, object: new MemCacheEntry(p, blobSize, blobFormat));
319}
320
321static inline void writeUInt(uchar **p, quint32 value)
322{
323 memcpy(dest: *p, src: &value, n: sizeof(quint32));
324 *p += sizeof(quint32);
325}
326
327static inline void writeStr(uchar **p, const QByteArray &str)
328{
329 writeUInt(p, value: str.size());
330 memcpy(dest: *p, src: str.constData(), n: str.size());
331 *p += str.size();
332}
333
334static inline bool writeFile(const QString &filename, const QByteArray &data)
335{
336#if QT_CONFIG(temporaryfile)
337 QSaveFile f(filename);
338 if (f.open(flags: QIODevice::WriteOnly | QIODevice::Truncate)) {
339 f.write(data);
340 if (f.commit())
341 return true;
342 }
343#else
344 QFile f(filename);
345 if (f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
346 if (f.write(data) == data.length())
347 return true;
348 }
349#endif
350 return false;
351}
352
353void QOpenGLProgramBinaryCache::save(const QByteArray &cacheKey, uint programId)
354{
355 if (!m_cacheWritable)
356 return;
357
358 GLEnvInfo info;
359
360 QOpenGLContext *context = QOpenGLContext::currentContext();
361 QOpenGLExtraFunctions *funcs = context->extraFunctions();
362 GLint blobSize = 0;
363 while (true) {
364 GLenum error = funcs->glGetError();
365 if (error == GL_NO_ERROR || error == GL_CONTEXT_LOST)
366 break;
367 }
368 funcs->glGetProgramiv(program: programId, GL_PROGRAM_BINARY_LENGTH, params: &blobSize);
369
370 const int headerSize = FULL_HEADER_SIZE(info.glvendor.size() + info.glrenderer.size() + info.glversion.size());
371
372 // Add padding to make the blob start 4-byte aligned in order to support
373 // OpenGL implementations on ARM that choke on non-aligned pointers passed
374 // to glProgramBinary.
375 const int paddingSize = PADDING_SIZE(headerSize);
376
377 const int totalSize = headerSize + paddingSize + blobSize;
378
379 qCDebug(lcOpenGLProgramDiskCache, "Program binary is %d bytes, err = 0x%x, total %d", blobSize, funcs->glGetError(), totalSize);
380 if (!blobSize)
381 return;
382
383 QByteArray blob(totalSize, Qt::Uninitialized);
384 uchar *p = reinterpret_cast<uchar *>(blob.data());
385
386 writeUInt(p: &p, value: BINSHADER_MAGIC);
387 writeUInt(p: &p, value: BINSHADER_VERSION);
388 writeUInt(p: &p, value: BINSHADER_QTVERSION);
389 writeUInt(p: &p, value: sizeof(quintptr));
390
391 writeStr(p: &p, str: info.glvendor);
392 writeStr(p: &p, str: info.glrenderer);
393 writeStr(p: &p, str: info.glversion);
394
395 quint32 blobFormat = 0;
396 uchar *blobFormatPtr = p;
397 writeUInt(p: &p, value: blobFormat);
398 writeUInt(p: &p, value: blobSize);
399
400 for (int i = 0; i < paddingSize; ++i)
401 *p++ = 0;
402
403 GLint outSize = 0;
404#if QT_CONFIG(opengles2)
405 if (context->isOpenGLES() && context->format().majorVersion() < 3) {
406 QMutexLocker lock(&m_mutex);
407 initializeProgramBinaryOES(context);
408 getProgramBinaryOES(programId, blobSize, &outSize, &blobFormat, p);
409 } else
410#endif
411 funcs->glGetProgramBinary(program: programId, bufSize: blobSize, length: &outSize, binaryFormat: &blobFormat, binary: p);
412 if (blobSize != outSize) {
413 qCDebug(lcOpenGLProgramDiskCache, "glGetProgramBinary returned size %d instead of %d", outSize, blobSize);
414 return;
415 }
416
417 writeUInt(p: &blobFormatPtr, value: blobFormat);
418
419 QString filename = cacheFileName(cacheKey);
420 bool ok = writeFile(filename, data: blob);
421 if (!ok && m_currentCacheDir == m_globalCacheDir) {
422 m_currentCacheDir = m_localCacheDir;
423 m_cacheWritable = qt_ensureWritableDir(name: m_currentCacheDir);
424 qCDebug(lcOpenGLProgramDiskCache, "Cache location changed to '%s' writable = %d",
425 qPrintable(m_currentCacheDir), m_cacheWritable);
426 if (m_cacheWritable) {
427 filename = cacheFileName(cacheKey);
428 ok = writeFile(filename, data: blob);
429 }
430 }
431 if (!ok)
432 qCDebug(lcOpenGLProgramDiskCache, "Failed to write %s to shader cache", qPrintable(filename));
433}
434
435#if QT_CONFIG(opengles2)
436void QOpenGLProgramBinaryCache::initializeProgramBinaryOES(QOpenGLContext *context)
437{
438 if (m_programBinaryOESInitialized)
439 return;
440 m_programBinaryOESInitialized = true;
441
442 Q_ASSERT(context);
443 getProgramBinaryOES = (void (QOPENGLF_APIENTRYP)(GLuint program, GLsizei bufSize, GLsizei *length, GLenum *binaryFormat, GLvoid *binary))context->getProcAddress("glGetProgramBinaryOES");
444 programBinaryOES = (void (QOPENGLF_APIENTRYP)(GLuint program, GLenum binaryFormat, const GLvoid *binary, GLint length))context->getProcAddress("glProgramBinaryOES");
445}
446#endif
447
448QOpenGLProgramBinarySupportCheck::QOpenGLProgramBinarySupportCheck(QOpenGLContext *context)
449 : QOpenGLSharedResource(context->shareGroup()),
450 m_supported(false)
451{
452 if (QCoreApplication::testAttribute(attribute: Qt::AA_DisableShaderDiskCache)) {
453 qCDebug(lcOpenGLProgramDiskCache, "Shader cache disabled via app attribute");
454 return;
455 }
456 if (qEnvironmentVariableIntValue(varName: "QT_DISABLE_SHADER_DISK_CACHE")) {
457 qCDebug(lcOpenGLProgramDiskCache, "Shader cache disabled via env var");
458 return;
459 }
460
461 QOpenGLContext *ctx = QOpenGLContext::currentContext();
462 if (ctx) {
463 if (ctx->isOpenGLES()) {
464 qCDebug(lcOpenGLProgramDiskCache, "OpenGL ES v%d context", ctx->format().majorVersion());
465 if (ctx->format().majorVersion() >= 3) {
466 m_supported = true;
467 } else {
468 const bool hasExt = ctx->hasExtension(extension: "GL_OES_get_program_binary");
469 qCDebug(lcOpenGLProgramDiskCache, "GL_OES_get_program_binary support = %d", hasExt);
470 if (hasExt)
471 m_supported = true;
472 }
473 } else {
474 const bool hasExt = ctx->hasExtension(extension: "GL_ARB_get_program_binary");
475 qCDebug(lcOpenGLProgramDiskCache, "GL_ARB_get_program_binary support = %d", hasExt);
476 if (hasExt)
477 m_supported = true;
478 }
479 if (m_supported) {
480 GLint fmtCount = 0;
481 ctx->functions()->glGetIntegerv(GL_NUM_PROGRAM_BINARY_FORMATS, params: &fmtCount);
482 qCDebug(lcOpenGLProgramDiskCache, "Supported binary format count = %d", fmtCount);
483 m_supported = fmtCount > 0;
484 }
485 }
486 qCDebug(lcOpenGLProgramDiskCache, "Shader cache supported = %d", m_supported);
487}
488
489QT_END_NAMESPACE
490

source code of qtbase/src/gui/opengl/qopenglprogrambinarycache.cpp