1// Copyright (C) 2020 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qbinaryjson_p.h"
5
6#include <QtCore/qjsonobject.h>
7#include <QtCore/qjsonarray.h>
8
9#include <private/qbinaryjsonarray_p.h>
10#include <private/qbinaryjsonobject_p.h>
11
12QT_BEGIN_NAMESPACE
13
14/*!
15 \namespace QBinaryJson
16 \inmodule QtCore5Compat
17 \brief Contains functions for converting QJsonDocument to and from JSON binary format.
18
19 This namespace provides utility functions to keep compatibility with
20 older code, which uses the JSON binary format for serializing JSON. Qt JSON
21 types can be converted to Qt CBOR types, which can in turn be serialized
22 into the CBOR binary format and vice versa.
23*/
24
25/*! \enum QBinaryJson::DataValidation
26
27 This enum is used to tell QJsonDocument whether to validate the binary data
28 when converting to a QJsonDocument using fromBinaryData() or fromRawData().
29
30 \value Validate Validate the data before using it. This is the default.
31 \value BypassValidation Bypasses data validation. Only use if you received the
32 data from a trusted place and know it's valid, as using of invalid data can crash
33 the application.
34*/
35
36namespace QBinaryJson {
37
38/*!
39 Creates a QJsonDocument that uses the first \a size bytes from
40 \a data. It assumes \a data contains a binary encoded JSON document.
41 The created document does not take ownership of \a data. The data is
42 copied into a different data structure, and the original data can be
43 deleted or modified afterwards.
44
45 \a data has to be aligned to a 4 byte boundary.
46
47 \a validation decides whether the data is checked for validity before being used.
48 By default the data is validated. If the \a data is not valid, the method returns
49 a null document.
50
51 Returns a QJsonDocument representing the data.
52
53 \note The binary JSON encoding is only retained for backwards
54 compatibility. It is undocumented and restrictive in the maximum size of JSON
55 documents that can be encoded. Qt JSON types can be converted to Qt CBOR types,
56 which can in turn be serialized into the CBOR binary format and vice versa. The
57 CBOR format is a well-defined and less restrictive binary representation for a
58 superset of JSON.
59
60 \note Before Qt 5.15, the caller had to guarantee that \a data would not be
61 deleted or modified as long as any QJsonDocument, QJsonObject or QJsonArray
62 still referenced the data. From Qt 5.15 on, this is not necessary anymore.
63
64 \sa toRawData(), fromBinaryData(), DataValidation, QCborValue
65*/
66QJsonDocument fromRawData(const char *data, int size, DataValidation validation)
67{
68 if (quintptr(data) & 3) {
69 qWarning(msg: "QJsonDocument::fromRawData: data has to have 4 byte alignment");
70 return QJsonDocument();
71 }
72
73 if (size < 0 || uint(size) < sizeof(QBinaryJsonPrivate::Header) + sizeof(QBinaryJsonPrivate::Base))
74 return QJsonDocument();
75
76 std::unique_ptr<QBinaryJsonPrivate::ConstData> binaryData
77 = std::make_unique<QBinaryJsonPrivate::ConstData>(args&: data, args&: size);
78
79 return (validation == BypassValidation || binaryData->isValid())
80 ? binaryData->toJsonDocument()
81 : QJsonDocument();
82}
83
84/*!
85 Returns the raw binary representation of \a document.
86 \a size will contain the size of the returned data.
87
88 This method is useful to e.g. stream the JSON document
89 in its binary form to a file.
90
91 \note The binary JSON encoding is only retained for backwards
92 compatibility. It is undocumented and restrictive in the maximum size of JSON
93 documents that can be encoded. Qt JSON types can be converted to Qt CBOR types,
94 which can in turn be serialized into the CBOR binary format and vice versa. The
95 CBOR format is a well-defined and less restrictive binary representation for a
96 superset of JSON.
97
98 \sa fromRawData(), fromBinaryData(), toBinaryData(), QCborValue
99*/
100const char *toRawData(const QJsonDocument &document, int *size)
101{
102 if (document.isNull()) {
103 *size = 0;
104 return nullptr;
105 }
106
107 char *rawData = nullptr;
108 uint rawDataSize = 0;
109 if (document.isObject()) {
110 QBinaryJsonObject o = QBinaryJsonObject::fromJsonObject(object: document.object());
111 rawData = o.takeRawData(size: &rawDataSize);
112 } else {
113 QBinaryJsonArray a = QBinaryJsonArray::fromJsonArray(array: document.array());
114 rawData = a.takeRawData(size: &rawDataSize);
115 }
116
117 // It would be quite miraculous if not, as we should have hit the 128MB limit then.
118 Q_ASSERT(rawDataSize <= uint(std::numeric_limits<int>::max()));
119
120 *size = static_cast<int>(rawDataSize);
121 return rawData;
122}
123
124/*!
125 Creates a QJsonDocument from \a data.
126
127 \a validation decides whether the data is checked for validity before being used.
128 By default the data is validated. If the \a data is not valid, the method returns
129 a null document.
130
131 \note The binary JSON encoding is only retained for backwards
132 compatibility. It is undocumented and restrictive in the maximum size of JSON
133 documents that can be encoded. Qt JSON types can be converted to Qt CBOR types,
134 which can in turn be serialized into the CBOR binary format and vice versa. The
135 CBOR format is a well-defined and less restrictive binary representation for a
136 superset of JSON.
137
138 \sa toBinaryData(), fromRawData(), DataValidation, QCborValue
139*/
140QJsonDocument fromBinaryData(const QByteArray &data, DataValidation validation)
141{
142 if (uint(data.size()) < sizeof(QBinaryJsonPrivate::Header) + sizeof(QBinaryJsonPrivate::Base))
143 return QJsonDocument();
144
145 QBinaryJsonPrivate::Header h;
146 memcpy(dest: &h, src: data.constData(), n: sizeof(QBinaryJsonPrivate::Header));
147 QBinaryJsonPrivate::Base root;
148 memcpy(dest: &root, src: data.constData() + sizeof(QBinaryJsonPrivate::Header),
149 n: sizeof(QBinaryJsonPrivate::Base));
150
151 const uint size = sizeof(QBinaryJsonPrivate::Header) + root.size;
152 if (h.tag != QJsonDocument::BinaryFormatTag || h.version != 1U || size > uint(data.size()))
153 return QJsonDocument();
154
155 std::unique_ptr<QBinaryJsonPrivate::ConstData> d
156 = std::make_unique<QBinaryJsonPrivate::ConstData>(args: data.constData(), args: size);
157
158 return (validation == BypassValidation || d->isValid())
159 ? d->toJsonDocument()
160 : QJsonDocument();
161}
162
163/*!
164 Returns a binary representation of \a document.
165
166 The binary representation is also the native format used internally in Qt,
167 and is very efficient and fast to convert to and from.
168
169 The binary format can be stored on disk and interchanged with other applications
170 or computers. fromBinaryData() can be used to convert it back into a
171 JSON document.
172
173 \note The binary JSON encoding is only retained for backwards
174 compatibility. It is undocumented and restrictive in the maximum size of JSON
175 documents that can be encoded. Qt JSON types can be converted to Qt CBOR types,
176 which can in turn be serialized into the CBOR binary format and vice versa. The
177 CBOR format is a well-defined and less restrictive binary representation for a
178 superset of JSON.
179
180 \sa fromBinaryData(), QCborValue
181*/
182QByteArray toBinaryData(const QJsonDocument &document)
183{
184 int size = 0;
185 const char *raw = toRawData(document, size: &size);
186 return QByteArray(raw, size);
187}
188
189} // namespace QBinaryJson
190
191namespace QBinaryJsonPrivate {
192
193static Q_CONSTEXPR Base emptyArray = {
194 .size: { qle_uint(sizeof(Base)) },
195 .isObjectAndLength: { 0 },
196 .tableOffset: { qle_uint(0) }
197};
198
199static Q_CONSTEXPR Base emptyObject = {
200 .size: { qle_uint(sizeof(Base)) },
201 .isObjectAndLength: { qToLittleEndian(source: 1U) },
202 .tableOffset: { qle_uint(0) }
203};
204
205void MutableData::compact()
206{
207 static_assert(sizeof(Value) == sizeof(offset));
208
209 Base *base = header->root();
210 int reserve = 0;
211 if (base->isObject()) {
212 auto *o = static_cast<Object *>(base);
213 for (uint i = 0; i < o->length(); ++i)
214 reserve += o->entryAt(i)->usedStorage(b: o);
215 } else {
216 auto *a = static_cast<Array *>(base);
217 for (uint i = 0; i < a->length(); ++i)
218 reserve += a->at(i)->usedStorage(b: a);
219 }
220
221 uint size = sizeof(Base) + reserve + base->length() * sizeof(offset);
222 uint alloc = sizeof(Header) + size;
223 auto *h = reinterpret_cast<Header *>(malloc(size: alloc));
224 Q_CHECK_PTR(h);
225 h->tag = QJsonDocument::BinaryFormatTag;
226 h->version = 1;
227 Base *b = new (h->root()) Base{};
228 b->size = size;
229 if (header->root()->isObject())
230 b->setIsObject();
231 else
232 b->setIsArray();
233 b->setLength(base->length());
234 b->tableOffset = reserve + sizeof(Array);
235
236 uint offset = sizeof(Base);
237 if (b->isObject()) {
238 const auto *o = static_cast<const Object *>(base);
239 auto *no = static_cast<Object *>(b);
240
241 for (uint i = 0; i < o->length(); ++i) {
242 no->table()[i] = offset;
243
244 const Entry *e = o->entryAt(i);
245 Entry *ne = no->entryAt(i);
246 uint s = e->size();
247 memcpy(dest: ne, src: e, n: s);
248 offset += s;
249 uint dataSize = e->value.usedStorage(b: o);
250 if (dataSize) {
251 memcpy(dest: reinterpret_cast<char *>(no) + offset, src: e->value.data(b: o), n: dataSize);
252 ne->value.setValue(offset);
253 offset += dataSize;
254 }
255 }
256 } else {
257 const auto *a = static_cast<const Array *>(base);
258 auto *na = static_cast<Array *>(b);
259
260 for (uint i = 0; i < a->length(); ++i) {
261 const Value *v = a->at(i);
262 Value *nv = na->at(i);
263 *nv = *v;
264 uint dataSize = v->usedStorage(b: a);
265 if (dataSize) {
266 memcpy(dest: reinterpret_cast<char *>(na) + offset, src: v->data(b: a), n: dataSize);
267 nv->setValue(offset);
268 offset += dataSize;
269 }
270 }
271 }
272 Q_ASSERT(offset == uint(b->tableOffset));
273
274 free(ptr: header);
275 header = h;
276 this->alloc = alloc;
277 compactionCounter = 0;
278}
279
280bool ConstData::isValid() const
281{
282 if (header->tag != QJsonDocument::BinaryFormatTag || header->version != 1U)
283 return false;
284
285 const Base *root = header->root();
286 const uint maxSize = alloc - sizeof(Header);
287 return root->isObject()
288 ? static_cast<const Object *>(root)->isValid(maxSize)
289 : static_cast<const Array *>(root)->isValid(maxSize);
290}
291
292QJsonDocument ConstData::toJsonDocument() const
293{
294 const Base *root = header->root();
295 return root->isObject()
296 ? QJsonDocument(static_cast<const Object *>(root)->toJsonObject())
297 : QJsonDocument(static_cast<const Array *>(root)->toJsonArray());
298}
299
300uint Base::reserveSpace(uint dataSize, uint posInTable, uint numItems, bool replace)
301{
302 Q_ASSERT(posInTable <= length());
303 if (size + dataSize >= Value::MaxSize) {
304 qWarning(msg: "QJson: Document too large to store in data structure %d %d %d",
305 uint(size), dataSize, Value::MaxSize);
306 return 0;
307 }
308
309 offset off = tableOffset;
310 // move table to new position
311 if (replace) {
312 memmove(dest: reinterpret_cast<char *>(table()) + dataSize, src: table(), n: length() * sizeof(offset));
313 } else {
314 memmove(dest: reinterpret_cast<char *>(table() + posInTable + numItems) + dataSize,
315 src: table() + posInTable, n: (length() - posInTable) * sizeof(offset));
316 memmove(dest: reinterpret_cast<char *>(table()) + dataSize, src: table(), n: posInTable * sizeof(offset));
317 }
318 tableOffset += dataSize;
319 for (uint i = 0; i < numItems; ++i)
320 table()[posInTable + i] = off;
321 size += dataSize;
322 if (!replace) {
323 setLength(length() + numItems);
324 size += numItems * sizeof(offset);
325 }
326 return off;
327}
328
329uint Object::indexOf(QStringView key, bool *exists) const
330{
331 uint min = 0;
332 uint n = length();
333 while (n > 0) {
334 uint half = n >> 1;
335 uint middle = min + half;
336 if (*entryAt(i: middle) >= key) {
337 n = half;
338 } else {
339 min = middle + 1;
340 n -= half + 1;
341 }
342 }
343 if (min < length() && *entryAt(i: min) == key) {
344 *exists = true;
345 return min;
346 }
347 *exists = false;
348 return min;
349}
350
351QJsonObject Object::toJsonObject() const
352{
353 QJsonObject object;
354 for (uint i = 0; i < length(); ++i) {
355 const Entry *e = entryAt(i);
356 object.insert(key: e->key(), value: e->value.toJsonValue(b: this));
357 }
358 return object;
359}
360
361bool Object::isValid(uint maxSize) const
362{
363 if (size > maxSize || tableOffset + length() * sizeof(offset) > size)
364 return false;
365
366 QString lastKey;
367 for (uint i = 0; i < length(); ++i) {
368 if (table()[i] + sizeof(Entry) >= tableOffset)
369 return false;
370 const Entry *e = entryAt(i);
371 if (!e->isValid(maxSize: tableOffset - table()[i]))
372 return false;
373 const QString key = e->key();
374 if (key < lastKey)
375 return false;
376 if (!e->value.isValid(b: this))
377 return false;
378 lastKey = key;
379 }
380 return true;
381}
382
383QJsonArray Array::toJsonArray() const
384{
385 QJsonArray array;
386 const offset *values = table();
387 for (uint i = 0; i < length(); ++i)
388 array.append(value: reinterpret_cast<const Value *>(values + i)->toJsonValue(b: this));
389 return array;
390}
391
392bool Array::isValid(uint maxSize) const
393{
394 if (size > maxSize || tableOffset + length() * sizeof(offset) > size)
395 return false;
396
397 const offset *values = table();
398 for (uint i = 0; i < length(); ++i) {
399 if (!reinterpret_cast<const Value *>(values + i)->isValid(b: this))
400 return false;
401 }
402 return true;
403}
404
405uint Value::usedStorage(const Base *b) const
406{
407 uint s = 0;
408 switch (type()) {
409 case QJsonValue::Double:
410 if (!isLatinOrIntValue())
411 s = sizeof(double);
412 break;
413 case QJsonValue::String: {
414 const char *d = data(b);
415 s = isLatinOrIntValue()
416 ? (sizeof(ushort)
417 + qFromLittleEndian(source: *reinterpret_cast<const ushort *>(d)))
418 : (sizeof(int)
419 + sizeof(ushort) * qFromLittleEndian(source: *reinterpret_cast<const int *>(d)));
420 break;
421 }
422 case QJsonValue::Array:
423 case QJsonValue::Object:
424 s = base(b)->size;
425 break;
426 case QJsonValue::Null:
427 case QJsonValue::Bool:
428 default:
429 break;
430 }
431 return alignedSize(size: s);
432}
433
434QJsonValue Value::toJsonValue(const Base *b) const
435{
436 switch (type()) {
437 case QJsonValue::Null:
438 return QJsonValue(QJsonValue::Null);
439 case QJsonValue::Bool:
440 return QJsonValue(toBoolean());
441 case QJsonValue::Double:
442 return QJsonValue(toDouble(b));
443 case QJsonValue::String:
444 return QJsonValue(toString(b));
445 case QJsonValue::Array:
446 return static_cast<const Array *>(base(b))->toJsonArray();
447 case QJsonValue::Object:
448 return static_cast<const Object *>(base(b))->toJsonObject();
449 case QJsonValue::Undefined:
450 return QJsonValue(QJsonValue::Undefined);
451 }
452 Q_UNREACHABLE_RETURN(QJsonValue(QJsonValue::Undefined));
453}
454
455inline bool isValidValueOffset(uint offset, uint tableOffset)
456{
457 return offset >= sizeof(Base)
458 && offset + sizeof(uint) <= tableOffset;
459}
460
461bool Value::isValid(const Base *b) const
462{
463 switch (type()) {
464 case QJsonValue::Null:
465 case QJsonValue::Bool:
466 return true;
467 case QJsonValue::Double:
468 return isLatinOrIntValue() || isValidValueOffset(offset: value(), tableOffset: b->tableOffset);
469 case QJsonValue::String:
470 if (!isValidValueOffset(offset: value(), tableOffset: b->tableOffset))
471 return false;
472 if (isLatinOrIntValue())
473 return asLatin1String(b).isValid(maxSize: b->tableOffset - value());
474 return asString(b).isValid(maxSize: b->tableOffset - value());
475 case QJsonValue::Array:
476 return isValidValueOffset(offset: value(), tableOffset: b->tableOffset)
477 && static_cast<const Array *>(base(b))->isValid(maxSize: b->tableOffset - value());
478 case QJsonValue::Object:
479 return isValidValueOffset(offset: value(), tableOffset: b->tableOffset)
480 && static_cast<const Object *>(base(b))->isValid(maxSize: b->tableOffset - value());
481 default:
482 return false;
483 }
484}
485
486uint Value::requiredStorage(const QBinaryJsonValue &v, bool *compressed)
487{
488 *compressed = false;
489 switch (v.type()) {
490 case QJsonValue::Double:
491 if (QBinaryJsonPrivate::compressedNumber(d: v.toDouble()) != INT_MAX) {
492 *compressed = true;
493 return 0;
494 }
495 return sizeof(double);
496 case QJsonValue::String: {
497 QString s = v.toString();
498 *compressed = QBinaryJsonPrivate::useCompressed(s);
499 return QBinaryJsonPrivate::qStringSize(string: s, compress: *compressed);
500 }
501 case QJsonValue::Array:
502 case QJsonValue::Object:
503 return v.base ? uint(v.base->size) : sizeof(QBinaryJsonPrivate::Base);
504 case QJsonValue::Undefined:
505 case QJsonValue::Null:
506 case QJsonValue::Bool:
507 break;
508 }
509 return 0;
510}
511
512uint Value::valueToStore(const QBinaryJsonValue &v, uint offset)
513{
514 switch (v.type()) {
515 case QJsonValue::Undefined:
516 case QJsonValue::Null:
517 break;
518 case QJsonValue::Bool:
519 return v.toBool();
520 case QJsonValue::Double: {
521 int c = QBinaryJsonPrivate::compressedNumber(d: v.toDouble());
522 if (c != INT_MAX)
523 return c;
524 }
525 Q_FALLTHROUGH();
526 case QJsonValue::String:
527 case QJsonValue::Array:
528 case QJsonValue::Object:
529 return offset;
530 }
531 return 0;
532}
533
534void Value::copyData(const QBinaryJsonValue &v, char *dest, bool compressed)
535{
536 switch (v.type()) {
537 case QJsonValue::Double:
538 if (!compressed)
539 qToLittleEndian(src: v.toDouble(), dest);
540 break;
541 case QJsonValue::String: {
542 const QString str = v.toString();
543 QBinaryJsonPrivate::copyString(dest, str, compress: compressed);
544 break;
545 }
546 case QJsonValue::Array:
547 case QJsonValue::Object: {
548 const QBinaryJsonPrivate::Base *b = v.base;
549 if (!b)
550 b = (v.type() == QJsonValue::Array ? &emptyArray : &emptyObject);
551 memcpy(dest: dest, src: b, n: b->size);
552 break;
553 }
554 default:
555 break;
556 }
557}
558
559} // namespace QBinaryJsonPrivate
560
561QT_END_NAMESPACE
562

source code of qt5compat/src/core5/serialization/qbinaryjson.cpp