1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
3
4#include "translator.h"
5
6#include <QtCore/QByteArray>
7#include <QtCore/QDebug>
8#include <QtCore/QRegularExpression>
9#include <QtCore/QTextStream>
10
11#include <QtCore/QXmlStreamReader>
12
13#include <algorithm>
14
15using namespace Qt::StringLiterals;
16
17QT_BEGIN_NAMESPACE
18
19QDebug &operator<<(QDebug &d, const QXmlStreamAttribute &attr)
20{
21 return d << "[" << attr.name().toString() << "," << attr.value().toString() << "]";
22}
23
24
25class TSReader : public QXmlStreamReader
26{
27public:
28 TSReader(QIODevice &dev, ConversionData &cd)
29 : QXmlStreamReader(&dev), m_cd(cd)
30 {}
31
32 // the "real thing"
33 bool read(Translator &translator);
34
35private:
36 bool elementStarts(const QString &str) const
37 {
38 return isStartElement() && name() == str;
39 }
40
41 bool isWhiteSpace() const
42 {
43 return isCharacters() && text().toString().trimmed().isEmpty();
44 }
45
46 // needed to expand <byte ... />
47 QString readContents();
48 // needed to join <lengthvariant>s
49 QString readTransContents();
50
51 void handleError();
52
53 ConversionData &m_cd;
54};
55
56void TSReader::handleError()
57{
58 if (isComment())
59 return;
60 if (hasError() && error() == CustomError) // raised by readContents
61 return;
62
63 const QString loc = QString::fromLatin1(ba: "at %3:%1:%2")
64 .arg(a: lineNumber()).arg(a: columnNumber()).arg(a: m_cd.m_sourceFileName);
65
66 switch (tokenType()) {
67 case NoToken: // Cannot happen
68 default: // likewise
69 case Invalid:
70 raiseError(message: QString::fromLatin1(ba: "Parse error %1: %2").arg(args: loc, args: errorString()));
71 break;
72 case StartElement:
73 raiseError(message: QString::fromLatin1(ba: "Unexpected tag <%1> %2").arg(args: name().toString(), args: loc));
74 break;
75 case Characters:
76 {
77 QString tok = text().toString();
78 if (tok.size() > 30)
79 tok = tok.left(n: 30) + "[...]"_L1;
80 raiseError(message: QString::fromLatin1(ba: "Unexpected characters '%1' %2").arg(args&: tok, args: loc));
81 }
82 break;
83 case EntityReference:
84 raiseError(message: QString::fromLatin1(ba: "Unexpected entity '&%1;' %2").arg(args: name().toString(), args: loc));
85 break;
86 case ProcessingInstruction:
87 raiseError(message: QString::fromLatin1(ba: "Unexpected processing instruction %1").arg(a: loc));
88 break;
89 }
90}
91
92static QString byteValue(QString value)
93{
94 int base = 10;
95 if (value.startsWith(s: "x"_L1)) {
96 base = 16;
97 value.remove(i: 0, len: 1);
98 }
99 int n = value.toUInt(ok: 0, base);
100 return (n != 0) ? QString(QChar(n)) : QString();
101}
102
103QString TSReader::readContents()
104{
105 static const QString strbyte = u"byte"_s;
106 static const QString strvalue = u"value"_s;
107
108 QString result;
109 while (!atEnd()) {
110 readNext();
111 if (isEndElement()) {
112 break;
113 } else if (isCharacters()) {
114 result += text();
115 } else if (elementStarts(str: strbyte)) {
116 // <byte value="...">
117 result += byteValue(value: attributes().value(qualifiedName: strvalue).toString());
118 readNext();
119 if (!isEndElement()) {
120 handleError();
121 break;
122 }
123 } else {
124 handleError();
125 break;
126 }
127 }
128 //qDebug() << "TEXT: " << result;
129 return result;
130}
131
132QString TSReader::readTransContents()
133{
134 static const QString strlengthvariant = u"lengthvariant"_s;
135 static const QString strvariants = u"variants"_s;
136 static const QString stryes = u"yes"_s;
137
138 if (attributes().value(qualifiedName: strvariants) == stryes) {
139 QString result;
140 while (!atEnd()) {
141 readNext();
142 if (isEndElement()) {
143 break;
144 } else if (isWhiteSpace()) {
145 // ignore these, just whitespace
146 } else if (elementStarts(str: strlengthvariant)) {
147 if (!result.isEmpty())
148 result += QChar(Translator::BinaryVariantSeparator);
149 result += readContents();
150 } else {
151 handleError();
152 break;
153 }
154 }
155 return result;
156 } else {
157 return readContents();
158 }
159}
160
161bool TSReader::read(Translator &translator)
162{
163 static const QString strcatalog = u"catalog"_s;
164 static const QString strcomment = u"comment"_s;
165 static const QString strcontext = u"context"_s;
166 static const QString strdependencies = u"dependencies"_s;
167 static const QString strdependency = u"dependency"_s;
168 static const QString strextracomment = u"extracomment"_s;
169 static const QString strlabel = u"label"_s;
170 static const QString strfilename = u"filename"_s;
171 static const QString strid = u"id"_s;
172 static const QString strlanguage = u"language"_s;
173 static const QString strline = u"line"_s;
174 static const QString strlocation = u"location"_s;
175 static const QString strmessage = u"message"_s;
176 static const QString strname = u"name"_s;
177 static const QString strnumerus = u"numerus"_s;
178 static const QString strnumerusform = u"numerusform"_s;
179 static const QString strobsolete = u"obsolete"_s;
180 static const QString stroldcomment = u"oldcomment"_s;
181 static const QString stroldsource = u"oldsource"_s;
182 static const QString strsource = u"source"_s;
183 static const QString strsourcelanguage = u"sourcelanguage"_s;
184 static const QString strtranslation = u"translation"_s;
185 static const QString strtranslatorcomment = u"translatorcomment"_s;
186 static const QString strTS = u"TS"_s;
187 static const QString strtype = u"type"_s;
188 static const QString strunfinished = u"unfinished"_s;
189 static const QString struserdata = u"userdata"_s;
190 static const QString strvanished = u"vanished"_s;
191 //static const QString strversion = u"version"_s;
192 static const QString stryes = u"yes"_s;
193
194 static const QString strextrans("extra-"_L1);
195
196 while (!atEnd()) {
197 readNext();
198 if (isStartDocument()) {
199 // <!DOCTYPE TS>
200 //qDebug() << attributes();
201 } else if (isEndDocument()) {
202 // <!DOCTYPE TS>
203 //qDebug() << attributes();
204 } else if (isDTD()) {
205 // <!DOCTYPE TS>
206 //qDebug() << tokenString();
207 } else if (elementStarts(str: strTS)) {
208 // <TS>
209 //qDebug() << "TS " << attributes();
210 QHash<QString, int> currentLine;
211 QString currentFile;
212 bool maybeRelative = false, maybeAbsolute = false;
213
214 QXmlStreamAttributes atts = attributes();
215 //QString version = atts.value(strversion).toString();
216 translator.setLanguageCode(atts.value(qualifiedName: strlanguage).toString());
217 translator.setSourceLanguageCode(atts.value(qualifiedName: strsourcelanguage).toString());
218 while (!atEnd()) {
219 readNext();
220 if (isEndElement()) {
221 // </TS> found, finish local loop
222 break;
223 } else if (isWhiteSpace()) {
224 // ignore these, just whitespace
225 } else if (isStartElement()
226 && name().toString().startsWith(s: strextrans)) {
227 // <extra-...>
228 QString tag = name().toString();
229 translator.setExtra(ba: tag.mid(position: 6), var: readContents());
230 // </extra-...>
231 } else if (elementStarts(str: strdependencies)) {
232 /*
233 * <dependencies>
234 * <dependency catalog="qtsystems_no"/>
235 * <dependency catalog="qtbase_no"/>
236 * </dependencies>
237 **/
238 QStringList dependencies;
239 while (!atEnd()) {
240 readNext();
241 if (isEndElement()) {
242 // </dependencies> found, finish local loop
243 break;
244 } else if (elementStarts(str: strdependency)) {
245 // <dependency>
246 QXmlStreamAttributes atts = attributes();
247 dependencies.append(t: atts.value(qualifiedName: strcatalog).toString());
248 while (!atEnd()) {
249 readNext();
250 if (isEndElement()) {
251 // </dependency> found, finish local loop
252 break;
253 }
254 }
255 }
256 }
257 translator.setDependencies(dependencies);
258 } else if (elementStarts(str: strcontext)) {
259 // <context>
260 QString context;
261 while (!atEnd()) {
262 readNext();
263 if (isEndElement()) {
264 // </context> found, finish local loop
265 break;
266 } else if (isWhiteSpace()) {
267 // ignore these, just whitespace
268 } else if (elementStarts(str: strname)) {
269 // <name>
270 context = readElementText();
271 // </name>
272 } else if (elementStarts(str: strmessage)) {
273 // <message>
274 TranslatorMessage::References refs;
275 QString currentMsgFile = currentFile;
276
277 TranslatorMessage msg;
278 msg.setId(attributes().value(qualifiedName: strid).toString());
279 msg.setContext(context);
280 msg.setType(TranslatorMessage::Finished);
281 msg.setPlural(attributes().value(qualifiedName: strnumerus) == stryes);
282 msg.setTsLineNumber(lineNumber());
283 while (!atEnd()) {
284 readNext();
285 if (isEndElement()) {
286 // </message> found, finish local loop
287 msg.setReferences(refs);
288 translator.append(msg);
289 break;
290 } else if (isWhiteSpace()) {
291 // ignore these, just whitespace
292 } else if (elementStarts(str: strsource)) {
293 // <source>...</source>
294 msg.setSourceText(readContents());
295 } else if (elementStarts(str: stroldsource)) {
296 // <oldsource>...</oldsource>
297 msg.setOldSourceText(readContents());
298 } else if (elementStarts(str: stroldcomment)) {
299 // <oldcomment>...</oldcomment>
300 msg.setOldComment(readContents());
301 } else if (elementStarts(str: strextracomment)) {
302 // <extracomment>...</extracomment>
303 msg.setExtraComment(readContents());
304 } else if (elementStarts(str: strlabel)) {
305 // <label>...</label>
306 msg.setLabel(readContents());
307 } else if (elementStarts(str: strtranslatorcomment)) {
308 // <translatorcomment>...</translatorcomment>
309 msg.setTranslatorComment(readContents());
310 } else if (elementStarts(str: strlocation)) {
311 // <location/>
312 maybeAbsolute = true;
313 QXmlStreamAttributes atts = attributes();
314 QString fileName = atts.value(qualifiedName: strfilename).toString();
315 if (fileName.isEmpty()) {
316 fileName = currentMsgFile;
317 maybeRelative = true;
318 } else {
319 if (refs.isEmpty())
320 currentFile = fileName;
321 currentMsgFile = fileName;
322 }
323 const QString lin = atts.value(qualifiedName: strline).toString();
324 if (lin.isEmpty()) {
325 refs.append(t: TranslatorMessage::Reference(fileName, -1));
326 } else {
327 bool bOK;
328 int lineNo = lin.toInt(ok: &bOK);
329 if (bOK) {
330 if (lin.startsWith(c: u'+') || lin.startsWith(c: u'-')) {
331 lineNo = (currentLine[fileName] += lineNo);
332 maybeRelative = true;
333 }
334 refs.append(t: TranslatorMessage::Reference(fileName, lineNo));
335 }
336 }
337 readContents();
338 } else if (elementStarts(str: strcomment)) {
339 // <comment>...</comment>
340 msg.setComment(readContents());
341 } else if (elementStarts(str: struserdata)) {
342 // <userdata>...</userdata>
343 msg.setUserData(readContents());
344 } else if (elementStarts(str: strtranslation)) {
345 // <translation>
346 QXmlStreamAttributes atts = attributes();
347 QStringView type = atts.value(qualifiedName: strtype);
348 if (type == strunfinished)
349 msg.setType(TranslatorMessage::Unfinished);
350 else if (type == strvanished)
351 msg.setType(TranslatorMessage::Vanished);
352 else if (type == strobsolete)
353 msg.setType(TranslatorMessage::Obsolete);
354 if (msg.isPlural()) {
355 QStringList translations;
356 while (!atEnd()) {
357 readNext();
358 if (isEndElement()) {
359 break;
360 } else if (isWhiteSpace()) {
361 // ignore these, just whitespace
362 } else if (elementStarts(str: strnumerusform)) {
363 translations.append(t: readTransContents());
364 } else {
365 handleError();
366 break;
367 }
368 }
369 msg.setTranslations(translations);
370 } else {
371 msg.setTranslation(readTransContents());
372 }
373 // </translation>
374 } else if (isStartElement()
375 && name().toString().startsWith(s: strextrans)) {
376 // <extra-...>
377 QString tag = name().toString();
378 msg.setExtra(ba: tag.mid(position: 6), var: readContents());
379 // </extra-...>
380 } else {
381 handleError();
382 }
383 }
384 // </message>
385 } else {
386 handleError();
387 }
388 }
389 // </context>
390 } else {
391 handleError();
392 }
393 // if the file is empty adopt AbsoluteLocation (default location type for Translator)
394 if (translator.messageCount() == 0)
395 maybeAbsolute = true;
396 translator.setLocationsType(maybeRelative ? Translator::RelativeLocations :
397 maybeAbsolute ? Translator::AbsoluteLocations :
398 Translator::NoLocations);
399 } // </TS>
400 } else {
401 handleError();
402 }
403 }
404 if (hasError()) {
405 m_cd.appendError(error: errorString());
406 return false;
407 }
408 return true;
409}
410
411static QString tsNumericEntity(int ch)
412{
413 return QString(ch <= 0x20 ? QLatin1String("<byte value=\"x%1\"/>") : "&#x%1;"_L1)
414 .arg(a: ch, fieldWidth: 0, base: 16);
415}
416
417static QString tsProtect(const QString &str)
418{
419 QString result;
420 result.reserve(asize: str.size() * 12 / 10);
421 for (int i = 0; i != str.size(); ++i) {
422 const QChar ch = str[i];
423 uint c = ch.unicode();
424 switch (c) {
425 case '\"':
426 result += "&quot;"_L1;
427 break;
428 case '&':
429 result += "&amp;"_L1;
430 break;
431 case '>':
432 result += "&gt;"_L1;
433 break;
434 case '<':
435 result += "&lt;"_L1;
436 break;
437 case '\'':
438 result += "&apos;"_L1;
439 break;
440 default:
441 if ((c < 0x20 || (ch > QChar(0x7f) && ch.isSpace())) && c != '\n' && c != '\t')
442 result += tsNumericEntity(ch: c);
443 else // this also covers surrogates
444 result += QChar(c);
445 }
446 }
447 return result;
448}
449
450static void writeExtras(QTextStream &t, const char *indent,
451 const TranslatorMessage::ExtraData &extras, QRegularExpression drops)
452{
453 QStringList outs;
454 for (auto it = extras.cbegin(), end = extras.cend(); it != end; ++it) {
455 if (!drops.match(subject: it.key()).hasMatch()) {
456 outs << (QStringLiteral("<extra-") + it.key() + u'>' + tsProtect(str: it.value())
457 + QStringLiteral("</extra-") + it.key() + u'>');
458 }
459 }
460 outs.sort();
461 for (const QString &out : std::as_const(t&: outs))
462 t << indent << out << Qt::endl;
463}
464
465static void writeVariants(QTextStream &t, const char *indent, const QString &input)
466{
467 int offset;
468 if ((offset = input.indexOf(ch: Translator::BinaryVariantSeparator)) >= 0) {
469 t << " variants=\"yes\">";
470 int start = 0;
471 forever {
472 t << "\n " << indent << "<lengthvariant>"
473 << tsProtect(str: input.mid(position: start, n: offset - start))
474 << "</lengthvariant>";
475 if (offset == input.size())
476 break;
477 start = offset + 1;
478 offset = input.indexOf(ch: Translator::BinaryVariantSeparator, from: start);
479 if (offset < 0)
480 offset = input.size();
481 }
482 t << "\n" << indent;
483 } else {
484 t << ">" << tsProtect(str: input);
485 }
486}
487
488bool saveTS(const Translator &translator, QIODevice &dev, ConversionData &cd)
489{
490 bool result = true;
491 QTextStream t(&dev);
492
493 // The xml prolog allows processors to easily detect the correct encoding
494 t << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!DOCTYPE TS>\n";
495
496 t << "<TS version=\"2.1\"";
497
498 QString languageCode = translator.languageCode();
499 if (!languageCode.isEmpty() && languageCode != "C"_L1)
500 t << " language=\"" << languageCode << "\"";
501 languageCode = translator.sourceLanguageCode();
502 if (!languageCode.isEmpty() && languageCode != "C"_L1)
503 t << " sourcelanguage=\"" << languageCode << "\"";
504 t << ">\n";
505
506 const QStringList deps = translator.dependencies();
507 if (!deps.isEmpty()) {
508 t << "<dependencies>\n";
509 for (const QString &dep : deps)
510 t << " <dependency catalog=\"" << dep << "\"/>\n";
511 t << "</dependencies>\n";
512 }
513
514 QRegularExpression drops(QRegularExpression::anchoredPattern(expression: cd.dropTags().join(sep: u'|')));
515
516 writeExtras(t, indent: " ", extras: translator.extras(), drops);
517
518 QHash<QString, QList<TranslatorMessage> > messageOrder;
519 QList<QString> contextOrder;
520 for (const TranslatorMessage &msg : translator.messages()) {
521 // no need for such noise
522 if ((msg.type() == TranslatorMessage::Obsolete || msg.type() == TranslatorMessage::Vanished)
523 && msg.translation().isEmpty()) {
524 continue;
525 }
526
527 QList<TranslatorMessage> &context = messageOrder[msg.context()];
528 if (context.isEmpty())
529 contextOrder.append(t: msg.context());
530 context.append(t: msg);
531 }
532 if (cd.sortContexts())
533 std::sort(first: contextOrder.begin(), last: contextOrder.end());
534 if (cd.sortMessages()) {
535 auto messageComparator = [](const TranslatorMessage &m1, const TranslatorMessage &m2) {
536 return m1.sourceText() < m2.sourceText();
537 };
538 for (QList<TranslatorMessage> &contextMessages : messageOrder)
539 std::sort(first: contextMessages.begin(), last: contextMessages.end(), comp: messageComparator);
540 }
541
542 QHash<QString, int> currentLine;
543 QString currentFile;
544 for (const QString &context : std::as_const(t&: contextOrder)) {
545 t << "<context>\n"
546 " <name>"
547 << tsProtect(str: context)
548 << "</name>\n";
549 for (const TranslatorMessage &msg : std::as_const(t&: messageOrder[context])) {
550 //msg.dump();
551
552 t << " <message";
553 if (!msg.id().isEmpty())
554 t << " id=\"" << tsProtect(str: msg.id()) << "\"";
555 if (msg.isPlural())
556 t << " numerus=\"yes\"";
557 t << ">\n";
558 if (translator.locationsType() != Translator::NoLocations) {
559 QString cfile = currentFile;
560 bool first = true;
561 for (const TranslatorMessage::Reference &ref : msg.allReferences()) {
562 QString fn = cd.m_targetDir.relativeFilePath(fileName: ref.fileName())
563 .replace(before: u'\\', after: u'/');
564 int ln = ref.lineNumber();
565 QString ld;
566 if (translator.locationsType() == Translator::RelativeLocations) {
567 if (ln != -1) {
568 int dlt = ln - currentLine[fn];
569 if (dlt >= 0)
570 ld.append(c: u'+');
571 ld.append(s: QString::number(dlt));
572 currentLine[fn] = ln;
573 }
574
575 if (fn != cfile) {
576 if (first)
577 currentFile = fn;
578 cfile = fn;
579 } else {
580 fn.clear();
581 }
582 first = false;
583 } else {
584 if (ln != -1)
585 ld = QString::number(ln);
586 }
587
588 if (!ld.isEmpty()) {
589 t << " <location";
590 if (!fn.isEmpty())
591 t << " filename=\"" << fn << "\"";
592 t << " line=\"" << ld << "\"";
593 t << "/>\n";
594 }
595 }
596 }
597
598 t << " <source>"
599 << tsProtect(str: msg.sourceText())
600 << "</source>\n";
601
602 if (!msg.oldSourceText().isEmpty())
603 t << " <oldsource>" << tsProtect(str: msg.oldSourceText()) << "</oldsource>\n";
604
605 if (!msg.comment().isEmpty()) {
606 t << " <comment>"
607 << tsProtect(str: msg.comment())
608 << "</comment>\n";
609 }
610
611 if (!msg.oldComment().isEmpty())
612 t << " <oldcomment>" << tsProtect(str: msg.oldComment()) << "</oldcomment>\n";
613
614 if (!msg.extraComment().isEmpty())
615 t << " <extracomment>" << tsProtect(str: msg.extraComment())
616 << "</extracomment>\n";
617
618 if (!msg.label().isEmpty())
619 t << " <label>" << tsProtect(str: msg.label()) << "</label>\n";
620
621 if (!msg.translatorComment().isEmpty())
622 t << " <translatorcomment>" << tsProtect(str: msg.translatorComment())
623 << "</translatorcomment>\n";
624
625 t << " <translation";
626 if (msg.type() == TranslatorMessage::Unfinished)
627 t << " type=\"unfinished\"";
628 else if (msg.type() == TranslatorMessage::Vanished)
629 t << " type=\"vanished\"";
630 else if (msg.type() == TranslatorMessage::Obsolete)
631 t << " type=\"obsolete\"";
632 if (msg.isPlural()) {
633 t << ">";
634 const QStringList &translns = msg.translations();
635 for (int j = 0; j < translns.size(); ++j) {
636 t << "\n <numerusform";
637 writeVariants(t, indent: " ", input: translns[j]);
638 t << "</numerusform>";
639 }
640 t << "\n ";
641 } else {
642 writeVariants(t, indent: " ", input: msg.translation());
643 }
644 t << "</translation>\n";
645
646 writeExtras(t, indent: " ", extras: msg.extras(), drops);
647
648 if (!msg.userData().isEmpty())
649 t << " <userdata>" << msg.userData() << "</userdata>\n";
650 t << " </message>\n";
651 }
652 t << "</context>\n";
653 }
654
655 t << "</TS>\n";
656 return result;
657}
658
659bool loadTS(Translator &translator, QIODevice &dev, ConversionData &cd)
660{
661 TSReader reader(dev, cd);
662 return reader.read(translator);
663}
664
665int initTS()
666{
667 Translator::FileFormat format;
668
669 format.extension = "ts"_L1;
670 format.fileType = Translator::FileFormat::TranslationSource;
671 format.priority = 0;
672 format.untranslatedDescription = QT_TRANSLATE_NOOP("FMT", "Qt translation sources");
673 format.loader = &loadTS;
674 format.saver = &saveTS;
675 Translator::registerFileFormat(format);
676
677 return 1;
678}
679
680Q_CONSTRUCTOR_FUNCTION(initTS)
681
682QT_END_NAMESPACE
683

source code of qttools/src/linguist/shared/ts.cpp