1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the Qt Linguist of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:GPL-EXCEPT$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU
19** General Public License version 3 as published by the Free Software
20** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
21** included in the packaging of this file. Please review the following
22** information to ensure the GNU General Public License requirements will
23** be met: https://www.gnu.org/licenses/gpl-3.0.html.
24**
25** $QT_END_LICENSE$
26**
27****************************************************************************/
28
29#include "translator.h"
30
31#include <QtCore/QByteArray>
32#include <QtCore/QDebug>
33#include <QtCore/QRegExp>
34#include <QtCore/QTextCodec>
35#include <QtCore/QTextStream>
36
37#include <QtCore/QXmlStreamReader>
38
39#include <algorithm>
40
41#define STRINGIFY_INTERNAL(x) #x
42#define STRINGIFY(x) STRINGIFY_INTERNAL(x)
43#define STRING(s) static QString str##s(QLatin1String(STRINGIFY(s)))
44
45QT_BEGIN_NAMESPACE
46
47QDebug &operator<<(QDebug &d, const QXmlStreamAttribute &attr)
48{
49 return d << "[" << attr.name().toString() << "," << attr.value().toString() << "]";
50}
51
52
53class TSReader : public QXmlStreamReader
54{
55public:
56 TSReader(QIODevice &dev, ConversionData &cd)
57 : QXmlStreamReader(&dev), m_cd(cd)
58 {}
59
60 // the "real thing"
61 bool read(Translator &translator);
62
63private:
64 bool elementStarts(const QString &str) const
65 {
66 return isStartElement() && name() == str;
67 }
68
69 bool isWhiteSpace() const
70 {
71 return isCharacters() && text().toString().trimmed().isEmpty();
72 }
73
74 // needed to expand <byte ... />
75 QString readContents();
76 // needed to join <lengthvariant>s
77 QString readTransContents();
78
79 void handleError();
80
81 ConversionData &m_cd;
82};
83
84void TSReader::handleError()
85{
86 if (isComment())
87 return;
88 if (hasError() && error() == CustomError) // raised by readContents
89 return;
90
91 const QString loc = QString::fromLatin1(str: "at %3:%1:%2")
92 .arg(a: lineNumber()).arg(a: columnNumber()).arg(a: m_cd.m_sourceFileName);
93
94 switch (tokenType()) {
95 case NoToken: // Cannot happen
96 default: // likewise
97 case Invalid:
98 raiseError(message: QString::fromLatin1(str: "Parse error %1: %2").arg(args: loc, args: errorString()));
99 break;
100 case StartElement:
101 raiseError(message: QString::fromLatin1(str: "Unexpected tag <%1> %2").arg(args: name().toString(), args: loc));
102 break;
103 case Characters:
104 {
105 QString tok = text().toString();
106 if (tok.length() > 30)
107 tok = tok.left(n: 30) + QLatin1String("[...]");
108 raiseError(message: QString::fromLatin1(str: "Unexpected characters '%1' %2").arg(args&: tok, args: loc));
109 }
110 break;
111 case EntityReference:
112 raiseError(message: QString::fromLatin1(str: "Unexpected entity '&%1;' %2").arg(args: name().toString(), args: loc));
113 break;
114 case ProcessingInstruction:
115 raiseError(message: QString::fromLatin1(str: "Unexpected processing instruction %1").arg(a: loc));
116 break;
117 }
118}
119
120static QString byteValue(QString value)
121{
122 int base = 10;
123 if (value.startsWith(s: QLatin1String("x"))) {
124 base = 16;
125 value.remove(i: 0, len: 1);
126 }
127 int n = value.toUInt(ok: 0, base);
128 return (n != 0) ? QString(QChar(n)) : QString();
129}
130
131QString TSReader::readContents()
132{
133 STRING(byte);
134 STRING(value);
135
136 QString result;
137 while (!atEnd()) {
138 readNext();
139 if (isEndElement()) {
140 break;
141 } else if (isCharacters()) {
142 result += text();
143 } else if (elementStarts(str: strbyte)) {
144 // <byte value="...">
145 result += byteValue(value: attributes().value(qualifiedName: strvalue).toString());
146 readNext();
147 if (!isEndElement()) {
148 handleError();
149 break;
150 }
151 } else {
152 handleError();
153 break;
154 }
155 }
156 //qDebug() << "TEXT: " << result;
157 return result;
158}
159
160QString TSReader::readTransContents()
161{
162 STRING(lengthvariant);
163 STRING(variants);
164 STRING(yes);
165
166 if (attributes().value(qualifiedName: strvariants) == stryes) {
167 QString result;
168 while (!atEnd()) {
169 readNext();
170 if (isEndElement()) {
171 break;
172 } else if (isWhiteSpace()) {
173 // ignore these, just whitespace
174 } else if (elementStarts(str: strlengthvariant)) {
175 if (!result.isEmpty())
176 result += QChar(Translator::BinaryVariantSeparator);
177 result += readContents();
178 } else {
179 handleError();
180 break;
181 }
182 }
183 return result;
184 } else {
185 return readContents();
186 }
187}
188
189bool TSReader::read(Translator &translator)
190{
191 STRING(catalog);
192 STRING(comment);
193 STRING(context);
194 STRING(defaultcodec);
195 STRING(dependencies);
196 STRING(dependency);
197 STRING(extracomment);
198 STRING(filename);
199 STRING(id);
200 STRING(language);
201 STRING(line);
202 STRING(location);
203 STRING(message);
204 STRING(name);
205 STRING(numerus);
206 STRING(numerusform);
207 STRING(obsolete);
208 STRING(oldcomment);
209 STRING(oldsource);
210 STRING(source);
211 STRING(sourcelanguage);
212 STRING(translation);
213 STRING(translatorcomment);
214 STRING(TS);
215 STRING(type);
216 STRING(unfinished);
217 STRING(userdata);
218 STRING(vanished);
219 //STRING(version);
220 STRING(yes);
221
222 static const QString strextrans(QLatin1String("extra-"));
223
224 while (!atEnd()) {
225 readNext();
226 if (isStartDocument()) {
227 // <!DOCTYPE TS>
228 //qDebug() << attributes();
229 } else if (isEndDocument()) {
230 // <!DOCTYPE TS>
231 //qDebug() << attributes();
232 } else if (isDTD()) {
233 // <!DOCTYPE TS>
234 //qDebug() << tokenString();
235 } else if (elementStarts(str: strTS)) {
236 // <TS>
237 //qDebug() << "TS " << attributes();
238 QHash<QString, int> currentLine;
239 QString currentFile;
240 bool maybeRelative = false, maybeAbsolute = false;
241
242 QXmlStreamAttributes atts = attributes();
243 //QString version = atts.value(strversion).toString();
244 translator.setLanguageCode(atts.value(qualifiedName: strlanguage).toString());
245 translator.setSourceLanguageCode(atts.value(qualifiedName: strsourcelanguage).toString());
246 while (!atEnd()) {
247 readNext();
248 if (isEndElement()) {
249 // </TS> found, finish local loop
250 break;
251 } else if (isWhiteSpace()) {
252 // ignore these, just whitespace
253 } else if (elementStarts(str: strdefaultcodec)) {
254 // <defaultcodec>
255 readElementText();
256 m_cd.appendError(error: QString::fromLatin1(str: "Warning: ignoring <defaultcodec> element"));
257 // </defaultcodec>
258 } else if (isStartElement()
259 && name().toString().startsWith(s: strextrans)) {
260 // <extra-...>
261 QString tag = name().toString();
262 translator.setExtra(ba: tag.mid(position: 6), var: readContents());
263 // </extra-...>
264 } else if (elementStarts(str: strdependencies)) {
265 /*
266 * <dependencies>
267 * <dependency catalog="qtsystems_no"/>
268 * <dependency catalog="qtbase_no"/>
269 * </dependencies>
270 **/
271 QStringList dependencies;
272 while (!atEnd()) {
273 readNext();
274 if (isEndElement()) {
275 // </dependencies> found, finish local loop
276 break;
277 } else if (elementStarts(str: strdependency)) {
278 // <dependency>
279 QXmlStreamAttributes atts = attributes();
280 dependencies.append(t: atts.value(qualifiedName: strcatalog).toString());
281 while (!atEnd()) {
282 readNext();
283 if (isEndElement()) {
284 // </dependency> found, finish local loop
285 break;
286 }
287 }
288 }
289 }
290 translator.setDependencies(dependencies);
291 } else if (elementStarts(str: strcontext)) {
292 // <context>
293 QString context;
294 while (!atEnd()) {
295 readNext();
296 if (isEndElement()) {
297 // </context> found, finish local loop
298 break;
299 } else if (isWhiteSpace()) {
300 // ignore these, just whitespace
301 } else if (elementStarts(str: strname)) {
302 // <name>
303 context = readElementText();
304 // </name>
305 } else if (elementStarts(str: strmessage)) {
306 // <message>
307 TranslatorMessage::References refs;
308 QString currentMsgFile = currentFile;
309
310 TranslatorMessage msg;
311 msg.setId(attributes().value(qualifiedName: strid).toString());
312 msg.setContext(context);
313 msg.setType(TranslatorMessage::Finished);
314 msg.setPlural(attributes().value(qualifiedName: strnumerus) == stryes);
315 while (!atEnd()) {
316 readNext();
317 if (isEndElement()) {
318 // </message> found, finish local loop
319 msg.setReferences(refs);
320 translator.append(msg);
321 break;
322 } else if (isWhiteSpace()) {
323 // ignore these, just whitespace
324 } else if (elementStarts(str: strsource)) {
325 // <source>...</source>
326 msg.setSourceText(readContents());
327 } else if (elementStarts(str: stroldsource)) {
328 // <oldsource>...</oldsource>
329 msg.setOldSourceText(readContents());
330 } else if (elementStarts(str: stroldcomment)) {
331 // <oldcomment>...</oldcomment>
332 msg.setOldComment(readContents());
333 } else if (elementStarts(str: strextracomment)) {
334 // <extracomment>...</extracomment>
335 msg.setExtraComment(readContents());
336 } else if (elementStarts(str: strtranslatorcomment)) {
337 // <translatorcomment>...</translatorcomment>
338 msg.setTranslatorComment(readContents());
339 } else if (elementStarts(str: strlocation)) {
340 // <location/>
341 maybeAbsolute = true;
342 QXmlStreamAttributes atts = attributes();
343 QString fileName = atts.value(qualifiedName: strfilename).toString();
344 if (fileName.isEmpty()) {
345 fileName = currentMsgFile;
346 maybeRelative = true;
347 } else {
348 if (refs.isEmpty())
349 currentFile = fileName;
350 currentMsgFile = fileName;
351 }
352 const QString lin = atts.value(qualifiedName: strline).toString();
353 if (lin.isEmpty()) {
354 refs.append(t: TranslatorMessage::Reference(fileName, -1));
355 } else {
356 bool bOK;
357 int lineNo = lin.toInt(ok: &bOK);
358 if (bOK) {
359 if (lin.startsWith(c: QLatin1Char('+')) || lin.startsWith(c: QLatin1Char('-'))) {
360 lineNo = (currentLine[fileName] += lineNo);
361 maybeRelative = true;
362 }
363 refs.append(t: TranslatorMessage::Reference(fileName, lineNo));
364 }
365 }
366 readContents();
367 } else if (elementStarts(str: strcomment)) {
368 // <comment>...</comment>
369 msg.setComment(readContents());
370 } else if (elementStarts(str: struserdata)) {
371 // <userdata>...</userdata>
372 msg.setUserData(readContents());
373 } else if (elementStarts(str: strtranslation)) {
374 // <translation>
375 QXmlStreamAttributes atts = attributes();
376 QStringRef type = atts.value(qualifiedName: strtype);
377 if (type == strunfinished)
378 msg.setType(TranslatorMessage::Unfinished);
379 else if (type == strvanished)
380 msg.setType(TranslatorMessage::Vanished);
381 else if (type == strobsolete)
382 msg.setType(TranslatorMessage::Obsolete);
383 if (msg.isPlural()) {
384 QStringList translations;
385 while (!atEnd()) {
386 readNext();
387 if (isEndElement()) {
388 break;
389 } else if (isWhiteSpace()) {
390 // ignore these, just whitespace
391 } else if (elementStarts(str: strnumerusform)) {
392 translations.append(t: readTransContents());
393 } else {
394 handleError();
395 break;
396 }
397 }
398 msg.setTranslations(translations);
399 } else {
400 msg.setTranslation(readTransContents());
401 }
402 // </translation>
403 } else if (isStartElement()
404 && name().toString().startsWith(s: strextrans)) {
405 // <extra-...>
406 QString tag = name().toString();
407 msg.setExtra(ba: tag.mid(position: 6), var: readContents());
408 // </extra-...>
409 } else {
410 handleError();
411 }
412 }
413 // </message>
414 } else {
415 handleError();
416 }
417 }
418 // </context>
419 } else {
420 handleError();
421 }
422 translator.setLocationsType(maybeRelative ? Translator::RelativeLocations :
423 maybeAbsolute ? Translator::AbsoluteLocations :
424 Translator::NoLocations);
425 } // </TS>
426 } else {
427 handleError();
428 }
429 }
430 if (hasError()) {
431 m_cd.appendError(error: errorString());
432 return false;
433 }
434 return true;
435}
436
437static QString numericEntity(int ch)
438{
439 return QString(ch <= 0x20 ? QLatin1String("<byte value=\"x%1\"/>")
440 : QLatin1String("&#x%1;")) .arg(a: ch, fieldWidth: 0, base: 16);
441}
442
443static QString protect(const QString &str)
444{
445 QString result;
446 result.reserve(asize: str.length() * 12 / 10);
447 for (int i = 0; i != str.size(); ++i) {
448 const QChar ch = str[i];
449 uint c = ch.unicode();
450 switch (c) {
451 case '\"':
452 result += QLatin1String("&quot;");
453 break;
454 case '&':
455 result += QLatin1String("&amp;");
456 break;
457 case '>':
458 result += QLatin1String("&gt;");
459 break;
460 case '<':
461 result += QLatin1String("&lt;");
462 break;
463 case '\'':
464 result += QLatin1String("&apos;");
465 break;
466 default:
467 if ((c < 0x20 || (ch > QChar(0x7f) && ch.isSpace())) && c != '\n' && c != '\t')
468 result += numericEntity(ch: c);
469 else // this also covers surrogates
470 result += QChar(c);
471 }
472 }
473 return result;
474}
475
476static void writeExtras(QTextStream &t, const char *indent,
477 const TranslatorMessage::ExtraData &extras, QRegExp drops)
478{
479 QStringList outs;
480 for (Translator::ExtraData::ConstIterator it = extras.begin(); it != extras.end(); ++it) {
481 if (!drops.exactMatch(str: it.key())) {
482 outs << (QStringLiteral("<extra-") + it.key() + QLatin1Char('>')
483 + protect(str: it.value())
484 + QStringLiteral("</extra-") + it.key() + QLatin1Char('>'));
485 }
486 }
487 outs.sort();
488 foreach (const QString &out, outs)
489 t << indent << out << Qt::endl;
490}
491
492static void writeVariants(QTextStream &t, const char *indent, const QString &input)
493{
494 int offset;
495 if ((offset = input.indexOf(c: QChar(Translator::BinaryVariantSeparator))) >= 0) {
496 t << " variants=\"yes\">";
497 int start = 0;
498 forever {
499 t << "\n " << indent << "<lengthvariant>"
500 << protect(str: input.mid(position: start, n: offset - start))
501 << "</lengthvariant>";
502 if (offset == input.length())
503 break;
504 start = offset + 1;
505 offset = input.indexOf(c: QChar(Translator::BinaryVariantSeparator), from: start);
506 if (offset < 0)
507 offset = input.length();
508 }
509 t << "\n" << indent;
510 } else {
511 t << ">" << protect(str: input);
512 }
513}
514
515bool saveTS(const Translator &translator, QIODevice &dev, ConversionData &cd)
516{
517 bool result = true;
518 QTextStream t(&dev);
519 t.setCodec(QTextCodec::codecForName(name: "UTF-8"));
520 //qDebug() << translator.codecName();
521
522 // The xml prolog allows processors to easily detect the correct encoding
523 t << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<!DOCTYPE TS>\n";
524
525 t << "<TS version=\"2.1\"";
526
527 QString languageCode = translator.languageCode();
528 if (!languageCode.isEmpty() && languageCode != QLatin1String("C"))
529 t << " language=\"" << languageCode << "\"";
530 languageCode = translator.sourceLanguageCode();
531 if (!languageCode.isEmpty() && languageCode != QLatin1String("C"))
532 t << " sourcelanguage=\"" << languageCode << "\"";
533 t << ">\n";
534
535 QStringList deps = translator.dependencies();
536 if (!deps.isEmpty()) {
537 t << "<dependencies>\n";
538 foreach (const QString &dep, deps)
539 t << "<dependency catalog=\"" << dep << "\"/>\n";
540 t << "</dependencies>\n";
541 }
542
543 QRegExp drops(cd.dropTags().join(sep: QLatin1Char('|')));
544
545 writeExtras(t, indent: " ", extras: translator.extras(), drops);
546
547 QHash<QString, QList<TranslatorMessage> > messageOrder;
548 QList<QString> contextOrder;
549 foreach (const TranslatorMessage &msg, translator.messages()) {
550 // no need for such noise
551 if ((msg.type() == TranslatorMessage::Obsolete || msg.type() == TranslatorMessage::Vanished)
552 && msg.translation().isEmpty()) {
553 continue;
554 }
555
556 QList<TranslatorMessage> &context = messageOrder[msg.context()];
557 if (context.isEmpty())
558 contextOrder.append(t: msg.context());
559 context.append(t: msg);
560 }
561 if (cd.sortContexts())
562 std::sort(first: contextOrder.begin(), last: contextOrder.end());
563
564 QHash<QString, int> currentLine;
565 QString currentFile;
566 foreach (const QString &context, contextOrder) {
567 t << "<context>\n"
568 " <name>"
569 << protect(str: context)
570 << "</name>\n";
571 foreach (const TranslatorMessage &msg, messageOrder[context]) {
572 //msg.dump();
573
574 t << " <message";
575 if (!msg.id().isEmpty())
576 t << " id=\"" << msg.id() << "\"";
577 if (msg.isPlural())
578 t << " numerus=\"yes\"";
579 t << ">\n";
580 if (translator.locationsType() != Translator::NoLocations) {
581 QString cfile = currentFile;
582 bool first = true;
583 foreach (const TranslatorMessage::Reference &ref, msg.allReferences()) {
584 QString fn = cd.m_targetDir.relativeFilePath(fileName: ref.fileName())
585 .replace(before: QLatin1Char('\\'),after: QLatin1Char('/'));
586 int ln = ref.lineNumber();
587 QString ld;
588 if (translator.locationsType() == Translator::RelativeLocations) {
589 if (ln != -1) {
590 int dlt = ln - currentLine[fn];
591 if (dlt >= 0)
592 ld.append(c: QLatin1Char('+'));
593 ld.append(s: QString::number(dlt));
594 currentLine[fn] = ln;
595 }
596
597 if (fn != cfile) {
598 if (first)
599 currentFile = fn;
600 cfile = fn;
601 } else {
602 fn.clear();
603 }
604 first = false;
605 } else {
606 if (ln != -1)
607 ld = QString::number(ln);
608 }
609 t << " <location";
610 if (!fn.isEmpty())
611 t << " filename=\"" << fn << "\"";
612 if (!ld.isEmpty())
613 t << " line=\"" << ld << "\"";
614 t << "/>\n";
615 }
616 }
617
618 t << " <source>"
619 << protect(str: msg.sourceText())
620 << "</source>\n";
621
622 if (!msg.oldSourceText().isEmpty())
623 t << " <oldsource>" << protect(str: msg.oldSourceText()) << "</oldsource>\n";
624
625 if (!msg.comment().isEmpty()) {
626 t << " <comment>"
627 << protect(str: msg.comment())
628 << "</comment>\n";
629 }
630
631 if (!msg.oldComment().isEmpty())
632 t << " <oldcomment>" << protect(str: msg.oldComment()) << "</oldcomment>\n";
633
634 if (!msg.extraComment().isEmpty())
635 t << " <extracomment>" << protect(str: msg.extraComment())
636 << "</extracomment>\n";
637
638 if (!msg.translatorComment().isEmpty())
639 t << " <translatorcomment>" << protect(str: msg.translatorComment())
640 << "</translatorcomment>\n";
641
642 t << " <translation";
643 if (msg.type() == TranslatorMessage::Unfinished)
644 t << " type=\"unfinished\"";
645 else if (msg.type() == TranslatorMessage::Vanished)
646 t << " type=\"vanished\"";
647 else if (msg.type() == TranslatorMessage::Obsolete)
648 t << " type=\"obsolete\"";
649 if (msg.isPlural()) {
650 t << ">";
651 const QStringList &translns = msg.translations();
652 for (int j = 0; j < translns.count(); ++j) {
653 t << "\n <numerusform";
654 writeVariants(t, indent: " ", input: translns[j]);
655 t << "</numerusform>";
656 }
657 t << "\n ";
658 } else {
659 writeVariants(t, indent: " ", input: msg.translation());
660 }
661 t << "</translation>\n";
662
663 writeExtras(t, indent: " ", extras: msg.extras(), drops);
664
665 if (!msg.userData().isEmpty())
666 t << " <userdata>" << msg.userData() << "</userdata>\n";
667 t << " </message>\n";
668 }
669 t << "</context>\n";
670 }
671
672 t << "</TS>\n";
673 return result;
674}
675
676bool loadTS(Translator &translator, QIODevice &dev, ConversionData &cd)
677{
678 TSReader reader(dev, cd);
679 return reader.read(translator);
680}
681
682int initTS()
683{
684 Translator::FileFormat format;
685
686 format.extension = QLatin1String("ts");
687 format.fileType = Translator::FileFormat::TranslationSource;
688 format.priority = 0;
689 format.untranslatedDescription = QT_TRANSLATE_NOOP("FMT", "Qt translation sources");
690 format.loader = &loadTS;
691 format.saver = &saveTS;
692 Translator::registerFileFormat(format);
693
694 return 1;
695}
696
697Q_CONSTRUCTOR_FUNCTION(initTS)
698
699QT_END_NAMESPACE
700

source code of qttools/src/linguist/shared/ts.cpp