1// Copyright (C) 2022 The Qt Company Ltd.
2// Copyright (C) 2019 Alexey Edelev <semlanik@gmail.com>, Viktor Kopp <vifactor@gmail.com>
3// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
4
5#ifndef QPROTOBUFSERIALIZER_P_H
6#define QPROTOBUFSERIALIZER_P_H
7
8//
9// W A R N I N G
10// -------------
11//
12// This file is not part of the Qt API. It exists purely as an
13// implementation detail. This header file may change from version to
14// version without notice, or even be removed.
15//
16// We mean it.
17//
18
19#include <QtCore/qstring.h>
20#include <QtCore/qbytearray.h>
21#include <QtCore/private/qbytearray_p.h>
22#include <QtCore/qhash.h>
23#include <QtCore/qendian.h>
24#include <QtCore/qvariant.h>
25
26#include <QtProtobuf/qprotobufselfcheckiterator.h>
27#include <QtProtobuf/qprotobufserializer.h>
28#include <QtProtobuf/qtprotobuftypes.h>
29#include <QtProtobuf/private/qtprotobuflogging_p.h>
30#include <QtProtobuf/qabstractprotobufserializer.h>
31
32#include <optional>
33#include <type_traits>
34
35QT_BEGIN_NAMESPACE
36
37using QtProtobufPrivate::QProtobufSelfcheckIterator;
38
39class QProtobufSerializerPrivate
40{
41 // The below type trait structures help to determine the required encoding method for protobuf
42 // types.
43 // See https://protobuf.dev/programming-guides/encoding for details.
44
45 // Tests if V is a Varint-compatible unsigned integer type.
46 // uint32 | uint64
47 template<typename V>
48 struct IsUnsignedInt
49 : std::integral_constant<bool, std::is_integral<V>::value && std::is_unsigned<V>::value>
50 {};
51
52 // Tests if V is a Varint-compatible signed integer type.
53 // int32 | int64
54 template<typename V>
55 struct IsSignedInt
56 : std::integral_constant<bool,
57 std::is_same<V, QtProtobuf::int32>::value
58 || std::is_same<V, QtProtobuf::int64>::value>
59 {};
60
61 // Tests if V is a Varint-compatible signed integer type, with ZigZag encoding support.
62 // sint32 | sint64
63 template<typename V>
64 struct IsZigZagInt
65 : std::integral_constant<bool, std::is_integral<V>::value && std::is_signed<V>::value>
66 {};
67
68 // Tests if V is a Varint-compatible integer type.
69 // int32 | int64 | uint32 | uint64 | sint32 | sint64
70 template<typename V>
71 struct IsInt : std::integral_constant<bool, std::is_integral<V>::value || IsSignedInt<V>::value>
72 {};
73
74 // Tests if V is a 32-bit integer without the need for encoding.
75 // sfixed32 | fixed32
76 template<typename V>
77 struct IsFixed32Int
78 : std::integral_constant<bool,
79 std::is_same<V, QtProtobuf::fixed32>::value
80 || std::is_same<V, QtProtobuf::sfixed32>::value>
81 {};
82
83 // Tests if V is a 64-bit integer without the need for encoding.
84 // sfixed64 | fixed64
85 template<typename V>
86 struct IsFixed64Int
87 : std::integral_constant<bool,
88 std::is_same<V, QtProtobuf::fixed64>::value
89 || std::is_same<V, QtProtobuf::sfixed64>::value>
90 {};
91
92 // Tests if V is an integer without the need for encoding.
93 // sfixed32 | fixed32 | sfixed64 | fixed64
94 template<typename V>
95 struct IsFixedInt
96 : std::integral_constant<bool, IsFixed32Int<V>::value || IsFixed64Int<V>::value>
97 {};
98
99 // Tests if V is a 32-bit type without the need for encoding.
100 // sfixed32 | fixed32 | float
101 template<typename V>
102 struct IsI32
103 : std::integral_constant<bool, IsFixed32Int<V>::value || std::is_same<V, float>::value>
104 {};
105
106 // Tests if V is a 64-bit type without the need for encoding.
107 // sfixed64 | fixed64 | double
108 template<typename V>
109 struct IsI64
110 : std::integral_constant<bool, IsFixed64Int<V>::value || std::is_same<V, double>::value>
111 {};
112
113 // Tests if V is a type without the need for encoding.
114 // sfixed32 | fixed32 | float | sfixed64 | fixed64 | double
115 template<typename V>
116 struct IsI32OrI64 : std::integral_constant<bool, IsI64<V>::value || IsI32<V>::value>
117 {};
118
119 // Tests if V is the length-delimited non-message, non-list type.
120 // QString | QByteArray
121 template<typename V>
122 struct IsLengthDelimited
123 : std::integral_constant<
124 bool, std::is_same<V, QString>::value || std::is_same<V, QByteArray>::value>
125 {
126 };
127
128public:
129 // Serializer is interface function for serialize method
130 using Serializer = QByteArray (*)(const QVariant &, const QByteArray &);
131 // Deserializer is interface function for deserialize method
132 using Deserializer = bool (*)(QProtobufSelfcheckIterator &, QVariant &);
133 // Function checks if value in QVariant is considered to be non-ignorable.
134 using IsPresentChecker = bool (*)(const QVariant &);
135
136 // SerializationHandlers contains set of objects that required for class
137 // serializaion/deserialization
138 struct ProtobufSerializationHandler
139 {
140 QMetaType metaType;
141 Serializer serializer; // serializer assigned to class
142 Deserializer deserializer; // deserializer assigned to class
143 IsPresentChecker isPresent; // checks if contains non-ignorable value
144 QtProtobuf::WireTypes wireType; // Serialization WireType
145 };
146
147 template<typename V, std::enable_if_t<IsI32OrI64<V>::value || IsInt<V>::value, int> = 0>
148 static bool isPresent(const QVariant &value)
149 {
150 return value.value<V>() != 0;
151 }
152
153 template<typename V, std::enable_if_t<!IsI32OrI64<V>::value && !IsInt<V>::value, int> = 0>
154 static bool isPresent(const QVariant &value)
155 {
156 return !value.value<V>().isEmpty();
157 }
158
159 explicit QProtobufSerializerPrivate(QProtobufSerializer *q);
160 ~QProtobufSerializerPrivate() = default;
161 // ###########################################################################
162 // Serializers
163 // ###########################################################################
164
165 template<typename V, typename std::enable_if_t<IsUnsignedInt<V>::value, int> = 0>
166 static QByteArray serializeVarintCommon(const V &value)
167 {
168 if (value == 0)
169 return QByteArray(1, char(0));
170
171 qProtoDebug() << "value" << value;
172 quint64 varint = value;
173 QByteArray result;
174
175 while (varint != 0) {
176 // Put 7 bits to result buffer and mark as "not last" (0b10000000)
177 result.append(c: (varint & 0b01111111) | 0b10000000);
178 // Divide values to chunks of 7 bits and move to next chunk
179 varint >>= 7;
180 }
181
182 result.data()[result.size() - 1] &= ~0b10000000;
183
184 return result;
185 }
186
187 //---------------Integral and floating point types serializers---------------
188 /*
189 Serialization of fixed-length primitive types
190
191 Natural layout of bits is used: value is encoded in a byte array same way as it is located in
192 memory
193
194 value: Value to serialize
195 returns a byte array with 'value' encoded
196 */
197 template<typename V, typename std::enable_if_t<IsI32OrI64<V>::value, int> = 0>
198 static QByteArray serializeBasic(const V &value)
199 {
200 qProtoDebug() << "value" << value;
201
202 // Reserve required number of bytes
203 QByteArray result(sizeof(V), Qt::Uninitialized);
204 qToUnaligned(qToLittleEndian(value), result.data());
205 return result;
206 }
207
208 /*
209 Serialization of signed integral types
210 Uses ZigZag encoding[0] first then apply serialization as for unsigned integral types
211
212 [0]: https://protobuf.dev/programming-guides/encoding
213
214 value: Value to serialize
215 Returns a byte array with 'value' encoded
216 */
217 template<typename V, typename std::enable_if_t<IsZigZagInt<V>::value, int> = 0>
218 static QByteArray serializeBasic(const V &value)
219 {
220 qProtoDebug() << "value" << value;
221 using UV = std::make_unsigned_t<V>;
222 UV uValue = 0;
223
224 // Use ZigZag convertion first and apply unsigned variant next
225 V zigZagValue = (value << 1) ^ (value >> (sizeof(UV) * 8 - 1));
226 uValue = static_cast<UV>(zigZagValue);
227 return serializeBasic(uValue);
228 }
229
230 template<typename V, typename std::enable_if_t<IsSignedInt<V>::value, int> = 0>
231 static QByteArray serializeBasic(const V &value)
232 {
233 qProtoDebug() << "value" << value;
234 // Non-ZigZag signed integers should always be (de)serialized as the
235 // QtProtobuf::uint64
236 return serializeBasic(value: static_cast<QtProtobuf::uint64>(value));
237 }
238
239 /*!
240 Serialization of unsigned integral types
241
242 Use Varint encoding[0]:
243 "Varints are a method of serializing integers using one or more bytes. Smaller numbers
244 [regardless its type] take a smaller number of bytes."
245
246 [0]: https://protobuf.dev/programming-guides/encoding
247
248 value: Value to serialize
249 Returns a byte array with 'value' encoded
250 */
251 template<typename V, typename std::enable_if_t<IsUnsignedInt<V>::value, int> = 0>
252 static QByteArray serializeBasic(const V &value)
253 {
254 qProtoDebug() << "value" << value;
255 return serializeVarintCommon(value);
256 }
257
258 //------------------QString and QByteArray types serializers-----------------
259 template<typename V, typename std::enable_if_t<IsLengthDelimited<V>::value, int> = 0>
260 static QByteArray serializeBasic(const V &value)
261 {
262 qProtoDebug("data.size: %" PRIdQSIZETYPE, value.size());
263 // Varint serialize field size and apply result as starting point
264 if constexpr (std::is_same<V, QString>::value)
265 return prependLengthDelimitedSize(data: value.toUtf8());
266 else
267 return prependLengthDelimitedSize(data: value);
268 }
269
270 //--------------------------List types serializers---------------------------
271 template<typename V, typename std::enable_if_t<!IsLengthDelimited<V>::value, int> = 0>
272 static QByteArray serializeListType(const QList<V> &listValue)
273 {
274 qProtoDebug("listValue.count %" PRIdQSIZETYPE, listValue.count());
275
276 if (listValue.isEmpty()) {
277 return {};
278 }
279
280 QByteArray serializedList;
281 for (auto &value : listValue) {
282 serializedList.append(serializeBasic<V>(value));
283 }
284 // If internal field type is not LengthDelimited, exact amount of fields to be specified
285 serializedList = prependLengthDelimitedSize(data: serializedList);
286 return serializedList;
287 }
288
289 template<typename V,
290 typename std::enable_if_t<
291 IsInt<V>::value || IsI32OrI64<V>::value || IsLengthDelimited<V>::value, int> =
292 0>
293 static QByteArray serializeNonPackedList(const QList<V> &listValue, const QByteArray &header)
294 {
295 qProtoDebug("listValue.count %" PRIdQSIZETYPE, listValue.count());
296 QByteArray serializedList;
297 for (auto &value : listValue) {
298 serializedList.append(a: header);
299 serializedList.append(serializeBasic<V>(value));
300 }
301 return serializedList;
302 }
303
304 // ###########################################################################
305 // Deserializers
306 // ###########################################################################
307 template<typename V, typename std::enable_if_t<IsUnsignedInt<V>::value, int> = 0>
308 Q_REQUIRED_RESULT static std::optional<V>
309 deserializeVarintCommon(QProtobufSelfcheckIterator &it)
310 {
311 qProtoDebug("currentByte: 0x%x", *it);
312
313 quint64 value = 0;
314 int k = 0;
315 while (true) {
316 if (it.bytesLeft() == 0)
317 return std::nullopt;
318 quint64 byte = quint64(static_cast<unsigned char>(*it));
319 value += (byte & 0b01111111) << k;
320 k += 7;
321 if (((*it) & 0b10000000) == 0)
322 break;
323 ++it;
324 }
325 ++it;
326 return { V(value) };
327 }
328
329 //-------------Integral and floating point types deserializers---------------
330 template<typename V, typename std::enable_if_t<IsI32OrI64<V>::value, int> = 0>
331 Q_REQUIRED_RESULT static bool deserializeBasic(QProtobufSelfcheckIterator &it,
332 QVariant &variantValue)
333 {
334 qsizetype size = sizeof(V);
335 if (it.bytesLeft() < size)
336 return false;
337 qProtoDebug("currentByte: 0x%x", *it);
338 variantValue = QVariant::fromValue(qFromLittleEndian(qFromUnaligned<V>(it.data())));
339 it += size;
340 return true;
341 }
342
343 template<typename V, typename std::enable_if_t<IsUnsignedInt<V>::value, int> = 0>
344 Q_REQUIRED_RESULT static bool deserializeBasic(QProtobufSelfcheckIterator &it,
345 QVariant &variantValue)
346 {
347 qProtoDebug("currentByte: 0x%x", *it);
348
349 auto opt = deserializeVarintCommon<V>(it);
350 if (!opt)
351 return false;
352 variantValue = QVariant::fromValue(opt.value());
353 return true;
354 }
355
356 template<typename V, typename std::enable_if_t<IsZigZagInt<V>::value, int> = 0>
357 Q_REQUIRED_RESULT static bool deserializeBasic(QProtobufSelfcheckIterator &it,
358 QVariant &variantValue)
359 {
360 qProtoDebug("currentByte: 0x%x", *it);
361 using UV = std::make_unsigned_t<V>;
362 auto opt = deserializeVarintCommon<UV>(it);
363 if (!opt)
364 return false;
365 UV unsignedValue = opt.value();
366 V value = (unsignedValue >> 1) ^ (-1 * (unsignedValue & 1));
367 variantValue = QVariant::fromValue<V>(value);
368 return true;
369 }
370
371 template<typename V, typename std::enable_if_t<IsSignedInt<V>::value, int> = 0>
372 Q_REQUIRED_RESULT static bool deserializeBasic(QProtobufSelfcheckIterator &it,
373 QVariant &variantValue)
374 {
375 qProtoDebug("currentByte: 0x%x", *it);
376 // Non-ZigZag signed integers should always be (de)serialized as the
377 // QtProtobuf::uint64
378 auto opt = deserializeVarintCommon<QtProtobuf::uint64>(it);
379 if (!opt)
380 return false;
381 QtProtobuf::uint64 unsignedValue = opt.value();
382 V value = static_cast<V>(unsignedValue);
383 variantValue = QVariant::fromValue(value);
384 return true;
385 }
386
387 //-----------------QString and QByteArray types deserializers----------------
388 template<typename V, typename std::enable_if_t<IsLengthDelimited<V>::value, int> = 0>
389 Q_REQUIRED_RESULT static bool deserializeBasic(QProtobufSelfcheckIterator &it,
390 QVariant &variantValue)
391 {
392 std::optional<QByteArray> result = deserializeLengthDelimited(it);
393 if (!result) {
394 variantValue = QVariant();
395 return false;
396 }
397 if constexpr (std::is_same<QString, V>::value)
398 variantValue = QVariant::fromValue(value: QString::fromUtf8(ba: *result));
399 else
400 variantValue = QVariant::fromValue(value: *result);
401 return true;
402 }
403
404 //-------------------------List types deserializers--------------------------
405 template<typename V>
406 Q_REQUIRED_RESULT static bool deserializeList(QProtobufSelfcheckIterator &it,
407 QVariant &previousValue)
408 {
409 qProtoDebug("currentByte: 0x%x", *it);
410
411 QList<V> out;
412 auto opt = deserializeVarintCommon<QtProtobuf::uint64>(it);
413 if (!opt)
414 return false;
415 quint64 count = opt.value();
416 if (count > quint64(std::numeric_limits<qsizetype>::max()))
417 return false;
418 QProtobufSelfcheckIterator lastVarint = it + count;
419 if (!lastVarint.isValid())
420 return false;
421 while (it != lastVarint) {
422 QVariant variantValue;
423 if (!deserializeBasic<V>(it, variantValue))
424 return false;
425 out.append(variantValue.value<V>());
426 }
427 previousValue.setValue(out);
428 return true;
429 }
430
431 template<typename V>
432 Q_REQUIRED_RESULT static bool deserializeNonPackedList(QProtobufSelfcheckIterator &it,
433 QVariant &previousValue)
434 {
435 qProtoDebug("currentByte: 0x%x", *it);
436 QVariant variantValue;
437 if (deserializeBasic<V>(it, variantValue)) {
438 auto out = previousValue.value<QList<V>>();
439 qProtoDebug() << out;
440 out.append(variantValue.value<V>());
441 previousValue.setValue(out);
442 return true;
443 }
444 return false;
445 }
446
447 // ###########################################################################
448 // Common functions
449 // ###########################################################################
450 static std::optional<QByteArray> deserializeLengthDelimited(QProtobufSelfcheckIterator &it)
451 {
452 if (it.bytesLeft() == 0)
453 return std::nullopt;
454 qProtoDebug("currentByte: 0x%x", *it);
455
456 auto opt = deserializeVarintCommon<QtProtobuf::uint64>(it);
457 if (!opt)
458 return std::nullopt;
459 quint64 length = opt.value();
460 if (!it.isValid() || quint64(it.bytesLeft()) < length || length > quint64(MaxByteArraySize))
461 return std::nullopt;
462 QByteArray result(it.data(), qsizetype(length));
463 it += length;
464 return { result };
465 }
466
467 Q_REQUIRED_RESULT
468 static bool decodeHeader(QProtobufSelfcheckIterator &it, int &fieldIndex,
469 QtProtobuf::WireTypes &wireType);
470 static QByteArray encodeHeader(int fieldIndex, QtProtobuf::WireTypes wireType);
471
472 /*!
473 Gets length of a byte-array and prepends to it its serialized length value
474 using the appropriate serialization algorithm
475
476 Returns 'data' with its length prepended
477 */
478 static QByteArray prependLengthDelimitedSize(const QByteArray &data)
479 {
480 return serializeVarintCommon<uint32_t>(value: data.size()) + data;
481 }
482
483 template<typename T, QByteArray (*s)(const T &)>
484 static QByteArray serializeWrapper(const QVariant &variantValue, const QByteArray &header)
485 {
486 return header + s(variantValue.value<T>());
487 }
488
489 template<typename T, QByteArray (*s)(const T &, const QByteArray &)>
490 static QByteArray serializeNonPackedWrapper(const QVariant &variantValue,
491 const QByteArray &header)
492 {
493 return s(variantValue.value<T>(), header);
494 }
495
496 // this set of 3 methods is used to skip bytes corresponding to an unexpected property
497 // in a serialized message met while the message being deserialized
498 static qsizetype skipSerializedFieldBytes(QProtobufSelfcheckIterator &it,
499 QtProtobuf::WireTypes type);
500 static void skipVarint(QProtobufSelfcheckIterator &it);
501 static void skipLengthDelimited(QProtobufSelfcheckIterator &it);
502
503 QByteArray serializeProperty(const QVariant &propertyValue,
504 const QtProtobufPrivate::QProtobufPropertyOrderingInfo &fieldInfo);
505 Q_REQUIRED_RESULT
506 bool deserializeProperty(QProtobufMessage *message,
507 const QtProtobufPrivate::QProtobufPropertyOrdering &ordering,
508 QProtobufSelfcheckIterator &it);
509
510 void setDeserializationError(QAbstractProtobufSerializer::DeserializationError error,
511 const QString &errorString);
512 void clearError();
513 void setUnexpectedEndOfStreamError();
514
515 Q_REQUIRED_RESULT
516 bool deserializeMapPair(QVariant &key, QVariant &value, QProtobufSelfcheckIterator &it);
517
518 QAbstractProtobufSerializer::DeserializationError deserializationError =
519 QAbstractProtobufSerializer::NoDeserializerError;
520 QString deserializationErrorString;
521
522private:
523 Q_DISABLE_COPY_MOVE(QProtobufSerializerPrivate)
524 QProtobufSerializer *q_ptr;
525};
526
527QT_END_NAMESPACE
528
529#endif // QPROTOBUFSERIALIZER_P_H
530

source code of qtgrpc/src/protobuf/qprotobufserializer_p.h