1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
3
4#include "lupdate.h"
5
6#include <translator.h>
7
8#include <QtCore/QDebug>
9#include <QtCore/QFile>
10#include <QtCore/QString>
11#include <QtCore/QTextStream>
12
13#include <private/qqmljsengine_p.h>
14#include <private/qqmljsparser_p.h>
15#include <private/qqmljslexer_p.h>
16#include <private/qqmljsastvisitor_p.h>
17#include <private/qqmljsast_p.h>
18
19#include <QCoreApplication>
20#include <QFile>
21#include <QFileInfo>
22#include <QtDebug>
23#include <QStringList>
24
25#include <iostream>
26#include <cstdlib>
27#include <cctype>
28
29QT_BEGIN_NAMESPACE
30
31using namespace QQmlJS;
32
33using namespace Qt::StringLiterals;
34
35static QString QmlMagicComment = u"TRANSLATOR"_s;
36
37class FindTrCalls: protected AST::Visitor
38{
39public:
40 FindTrCalls(Engine *engine, ConversionData &cd)
41 : engine(engine)
42 , m_cd(cd)
43 {
44 }
45
46 void operator()(Translator *translator, const QString &fileName, AST::Node *node)
47 {
48 m_todo = engine->comments();
49 m_translator = translator;
50 m_fileName = fileName;
51 m_component = QFileInfo(fileName).completeBaseName();
52 accept(node);
53
54 // process the trailing comments
55 processComments(offset: 0, /*flush*/ true);
56 }
57
58protected:
59 using AST::Visitor::visit;
60 using AST::Visitor::endVisit;
61
62 void accept(AST::Node *node)
63 { AST::Node::accept(node, visitor: this); }
64
65 void endVisit(AST::CallExpression *node) override
66 {
67 QString name;
68 AST::ExpressionNode *base = node->base;
69
70 while (base && base->kind == AST::Node::Kind_FieldMemberExpression) {
71 auto memberExpr = static_cast<AST::FieldMemberExpression *>(base);
72 name.prepend(v: memberExpr->name);
73 name.prepend(c: QLatin1Char('.'));
74 base = memberExpr->base;
75 }
76
77 if (AST::IdentifierExpression *idExpr = AST::cast<AST::IdentifierExpression *>(ast: base)) {
78 processComments(offset: idExpr->identifierToken.begin());
79
80 name = idExpr->name.toString() + name;
81 const int identLineNo = idExpr->identifierToken.startLine;
82 switch (trFunctionAliasManager.trFunctionByName(trFunctionName: name)) {
83 case TrFunctionAliasManager::Function_qsTr:
84 case TrFunctionAliasManager::Function_QT_TR_NOOP: {
85 if (!node->arguments) {
86 yyMsg(line: identLineNo)
87 << qPrintable(QStringLiteral("%1() requires at least one argument.\n")
88 .arg(name));
89 return;
90 }
91 if (AST::cast<AST::TemplateLiteral *>(ast: node->arguments->expression)) {
92 yyMsg(line: identLineNo)
93 << qPrintable(QStringLiteral("%1() cannot be used with template literals. "
94 "Ignoring\n").arg(name));
95 return;
96 }
97
98 QString source;
99 if (!createString(ast: node->arguments->expression, out: &source))
100 return;
101
102 QString comment;
103 bool plural = false;
104 if (AST::ArgumentList *commentNode = node->arguments->next) {
105 if (!createString(ast: commentNode->expression, out: &comment)) {
106 comment.clear(); // clear possible invalid comments
107 }
108 if (commentNode->next)
109 plural = true;
110 }
111
112 if (!sourcetext.isEmpty())
113 yyMsg(line: identLineNo) << qPrintable(QStringLiteral("//% cannot be used with %1(). Ignoring\n").arg(name));
114
115 TranslatorMessage msg(m_component, ParserTool::transcode(str: source),
116 comment, QString(), m_fileName,
117 node->firstSourceLocation().startLine, QStringList(),
118 TranslatorMessage::Unfinished, plural);
119 msg.setExtraComment(ParserTool::transcode(str: extracomment.simplified()));
120 msg.setId(msgid);
121 msg.setExtras(extra);
122 m_translator->extend(msg, cd&: m_cd);
123 consumeComment();
124 break; }
125 case TrFunctionAliasManager::Function_qsTranslate:
126 case TrFunctionAliasManager::Function_QT_TRANSLATE_NOOP: {
127 if (! (node->arguments && node->arguments->next)) {
128 yyMsg(line: identLineNo) << qPrintable(QStringLiteral("%1() requires at least two arguments.\n").arg(name));
129 return;
130 }
131
132 QString context;
133 if (!createString(ast: node->arguments->expression, out: &context))
134 return;
135
136 AST::ArgumentList *sourceNode = node->arguments->next; // we know that it is a valid pointer.
137
138 QString source;
139 if (!createString(ast: sourceNode->expression, out: &source))
140 return;
141
142 if (!sourcetext.isEmpty())
143 yyMsg(line: identLineNo) << qPrintable(QStringLiteral("//% cannot be used with %1(). Ignoring\n").arg(name));
144
145 QString comment;
146 bool plural = false;
147 if (AST::ArgumentList *commentNode = sourceNode->next) {
148 if (!createString(ast: commentNode->expression, out: &comment)) {
149 comment.clear(); // clear possible invalid comments
150 }
151
152 if (commentNode->next)
153 plural = true;
154 }
155
156 TranslatorMessage msg(context, ParserTool::transcode(str: source),
157 comment, QString(), m_fileName,
158 node->firstSourceLocation().startLine, QStringList(),
159 TranslatorMessage::Unfinished, plural);
160 msg.setExtraComment(ParserTool::transcode(str: extracomment.simplified()));
161 msg.setId(msgid);
162 msg.setExtras(extra);
163 m_translator->extend(msg, cd&: m_cd);
164 consumeComment();
165 break; }
166 case TrFunctionAliasManager::Function_qsTrId:
167 case TrFunctionAliasManager::Function_QT_TRID_NOOP: {
168 if (!node->arguments) {
169 yyMsg(line: identLineNo) << qPrintable(QStringLiteral("%1() requires at least one argument.\n").arg(name));
170 return;
171 }
172
173 QString id;
174 if (!createString(ast: node->arguments->expression, out: &id))
175 return;
176
177 if (!msgid.isEmpty()) {
178 yyMsg(line: identLineNo) << qPrintable(QStringLiteral("//= cannot be used with %1(). Ignoring\n").arg(name));
179 return;
180 }
181
182 bool plural = node->arguments->next;
183
184 TranslatorMessage msg(QString(), ParserTool::transcode(str: sourcetext),
185 QString(), QString(), m_fileName,
186 node->firstSourceLocation().startLine, QStringList(),
187 TranslatorMessage::Unfinished, plural);
188 msg.setExtraComment(ParserTool::transcode(str: extracomment.simplified()));
189 msg.setId(id);
190 msg.setExtras(extra);
191 m_translator->extend(msg, cd&: m_cd);
192 consumeComment();
193 break; }
194 }
195 }
196 }
197
198 void postVisit(AST::Node *node) override;
199
200private:
201 std::ostream &yyMsg(int line)
202 {
203 return std::cerr << qPrintable(m_fileName) << ':' << line << ": ";
204 }
205
206 void throwRecursionDepthError() final
207 {
208 std::cerr << qPrintable(m_fileName) << ": "
209 << "Maximum statement or expression depth exceeded";
210 }
211
212
213 void processComments(quint32 offset, bool flush = false);
214 void processComment(const SourceLocation &loc);
215 void consumeComment();
216
217 bool createString(AST::ExpressionNode *ast, QString *out)
218 {
219 if (AST::StringLiteral *literal = AST::cast<AST::StringLiteral *>(ast)) {
220 out->append(v: literal->value);
221 return true;
222 } else if (AST::BinaryExpression *binop = AST::cast<AST::BinaryExpression *>(ast)) {
223 if (binop->op == QSOperator::Add && createString(ast: binop->left, out)) {
224 if (createString(ast: binop->right, out))
225 return true;
226 }
227 }
228
229 return false;
230 }
231
232 Engine *engine;
233 Translator *m_translator;
234 ConversionData &m_cd;
235 QString m_fileName;
236 QString m_component;
237
238 // comments
239 QString extracomment;
240 QString msgid;
241 TranslatorMessage::ExtraData extra;
242 QString sourcetext;
243 QString trcontext;
244 QList<SourceLocation> m_todo;
245};
246
247QString createErrorString(const QString &filename, const QString &code, Parser &parser)
248{
249 // print out error
250 QStringList lines = code.split(sep: QLatin1Char('\n'));
251 lines.append(t: QLatin1String("\n")); // sentinel.
252 QString errorString;
253
254 const auto messages = parser.diagnosticMessages();
255 for (const DiagnosticMessage &m : messages) {
256
257 if (m.isWarning())
258 continue;
259
260 const int line = m.loc.startLine;
261 const int column = m.loc.startColumn;
262 QString error = filename + QLatin1Char(':')
263 + QString::number(line) + QLatin1Char(':') + QString::number(column)
264 + QLatin1String(": error: ") + m.message + QLatin1Char('\n');
265
266 const QString textLine = lines.at(line > 0 ? line - 1 : 0);
267 error += textLine + QLatin1Char('\n');
268 for (int i = 0, end = qMin(column > 0 ? column - 1 : 0, textLine.size()); i < end; ++i) {
269 const QChar ch = textLine.at(i);
270 if (ch.isSpace())
271 error += ch;
272 else
273 error += QLatin1Char(' ');
274 }
275 error += QLatin1String("^\n");
276 errorString += error;
277 }
278 return errorString;
279}
280
281void FindTrCalls::postVisit(AST::Node *node)
282{
283 if (node->statementCast() != 0 || node->uiObjectMemberCast()) {
284 processComments(offset: node->lastSourceLocation().end());
285
286 if (!sourcetext.isEmpty() || !extracomment.isEmpty() || !msgid.isEmpty() || !extra.isEmpty()) {
287 yyMsg(line: node->lastSourceLocation().startLine) << "Discarding unconsumed meta data\n";
288 consumeComment();
289 }
290 }
291}
292
293void FindTrCalls::processComments(quint32 offset, bool flush)
294{
295 for (; !m_todo.isEmpty(); m_todo.removeFirst()) {
296 SourceLocation loc = m_todo.first();
297 if (! flush && (loc.begin() >= offset))
298 break;
299
300 processComment(loc);
301 }
302}
303
304void FindTrCalls::consumeComment()
305{
306 // keep the current `trcontext'
307 extracomment.clear();
308 msgid.clear();
309 extra.clear();
310 sourcetext.clear();
311}
312
313void FindTrCalls::processComment(const SourceLocation &loc)
314{
315 if (!loc.length)
316 return;
317
318 const QStringView commentStr = engine->midRef(position: loc.begin(), size: loc.length);
319 const QChar *chars = commentStr.constData();
320 const int length = commentStr.size();
321
322 // Try to match the logic of the C++ parser.
323 if (*chars == QLatin1Char(':') && chars[1].isSpace()) {
324 if (!extracomment.isEmpty())
325 extracomment += QLatin1Char(' ');
326 extracomment += QString(chars+2, length-2);
327 } else if (*chars == QLatin1Char('=') && chars[1].isSpace()) {
328 msgid = QString(chars+2, length-2).simplified();
329 } else if (*chars == QLatin1Char('~') && chars[1].isSpace()) {
330 QString text = QString(chars+2, length-2).trimmed();
331 int k = text.indexOf(c: QLatin1Char(' '));
332 if (k > -1) {
333 QString commentvalue = text.mid(position: k + 1).trimmed();
334 if (commentvalue.startsWith(c: QLatin1Char('"')) && commentvalue.endsWith(c: QLatin1Char('"'))
335 && commentvalue.size() != 1) {
336 commentvalue = commentvalue.sliced(pos: 1, n: commentvalue.size() - 2);
337 }
338 extra.insert(key: text.left(n: k), value: commentvalue);
339 }
340 } else if (*chars == QLatin1Char('%') && chars[1].isSpace()) {
341 sourcetext.reserve(asize: sourcetext.size() + length-2);
342 ushort *ptr = (ushort *)sourcetext.data() + sourcetext.size();
343 int p = 2, c;
344 forever {
345 if (p >= length)
346 break;
347 c = chars[p++].unicode();
348 if (std::isspace(c))
349 continue;
350 if (c != '"') {
351 yyMsg(line: loc.startLine) << "Unexpected character in meta string\n";
352 break;
353 }
354 forever {
355 if (p >= length) {
356 whoops:
357 yyMsg(line: loc.startLine) << "Unterminated meta string\n";
358 break;
359 }
360 c = chars[p++].unicode();
361 if (c == '"')
362 break;
363 if (c == '\\') {
364 if (p >= length)
365 goto whoops;
366 c = chars[p++].unicode();
367 if (c == '\r' || c == '\n')
368 goto whoops;
369 *ptr++ = '\\';
370 }
371 *ptr++ = c;
372 }
373 }
374 sourcetext.resize(size: ptr - (ushort *)sourcetext.data());
375 } else {
376 int idx = 0;
377 ushort c;
378 while ((c = chars[idx].unicode()) == ' ' || c == '\t' || c == '\r' || c == '\n')
379 ++idx;
380 if (!memcmp(s1: chars + idx, s2: QmlMagicComment.unicode(), n: QmlMagicComment.size() * 2)) {
381 idx += QmlMagicComment.size();
382 QString comment = QString(chars + idx, length - idx).simplified();
383 int k = comment.indexOf(c: QLatin1Char(' '));
384 if (k == -1) {
385 trcontext = comment;
386 } else {
387 trcontext = comment.left(n: k);
388 comment.remove(i: 0, len: k + 1);
389 TranslatorMessage msg(
390 trcontext, QString(),
391 comment, QString(),
392 m_fileName, loc.startLine, QStringList(),
393 TranslatorMessage::Finished, /*plural=*/false);
394 msg.setExtraComment(extracomment.simplified());
395 extracomment.clear();
396 m_translator->append(msg);
397 m_translator->setExtras(extra);
398 extra.clear();
399 }
400
401 m_component = trcontext;
402 }
403 }
404}
405
406class HasDirectives: public Directives
407{
408public:
409 HasDirectives(Lexer *lexer)
410 : lexer(lexer)
411 , directives(0)
412 {
413 }
414
415 bool operator()() const { return directives != 0; }
416 int end() const { return lastOffset; }
417
418 void pragmaLibrary() override { consumeDirective(); }
419 void importFile(const QString &, const QString &, int, int) override { consumeDirective(); }
420 void importModule(const QString &, const QString &, const QString &, int, int) override { consumeDirective(); }
421
422private:
423 void consumeDirective()
424 {
425 ++directives;
426 lastOffset = lexer->tokenOffset() + lexer->tokenLength();
427 }
428
429private:
430 Lexer *lexer;
431 int directives;
432 int lastOffset;
433};
434
435static bool load(Translator &translator, const QString &filename, ConversionData &cd, bool qmlMode)
436{
437 cd.m_sourceFileName = filename;
438 QFile file(filename);
439 if (!file.open(flags: QIODevice::ReadOnly)) {
440 cd.appendError(QStringLiteral("Cannot open %1: %2").arg(args: filename, args: file.errorString()));
441 return false;
442 }
443
444 QString code;
445 if (!qmlMode) {
446 code = QTextStream(&file).readAll();
447 } else {
448 QTextStream ts(&file);
449 code = ts.readAll();
450 }
451
452 Engine driver;
453 Parser parser(&driver);
454
455 Lexer lexer(&driver);
456 lexer.setCode(code, /*line = */ lineno: 1, qmlMode);
457 driver.setLexer(&lexer);
458
459 if (qmlMode ? parser.parse() : parser.parseProgram()) {
460 FindTrCalls trCalls(&driver, cd);
461
462 //find all tr calls in the code
463 trCalls(&translator, filename, parser.rootNode());
464 } else {
465 QString error = createErrorString(filename, code, parser);
466 cd.appendError(error);
467 return false;
468 }
469 return true;
470}
471
472bool loadQml(Translator &translator, const QString &filename, ConversionData &cd)
473{
474 return load(translator, filename, cd, /*qmlMode=*/ true);
475}
476
477bool loadQScript(Translator &translator, const QString &filename, ConversionData &cd)
478{
479 return load(translator, filename, cd, /*qmlMode=*/ false);
480}
481
482QT_END_NAMESPACE
483

source code of qttools/src/linguist/lupdate/qdeclarative.cpp