1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qqmldomscanner_p.h"
5#include "qqmldomerrormessage_p.h"
6#include "qqmljsgrammar_p.h"
7
8#include <QtCore/QMetaEnum>
9
10#include <algorithm>
11
12QT_BEGIN_NAMESPACE
13
14using namespace QQmlJS::Dom;
15
16static void addLexToken(QList<Token> &tokens, int tokenKind, QQmlJS::Lexer &lexer,
17 bool &regexpMayFollow)
18{
19 switch (tokenKind) {
20 case QQmlJSGrammar::T_DIVIDE_:
21 case QQmlJSGrammar::T_DIVIDE_EQ:
22 if (lexer.state().currentChar.isSpace()) {
23 regexpMayFollow = false;
24 break;
25 }
26 if (regexpMayFollow) {
27 QQmlJS::Lexer::RegExpBodyPrefix prefix;
28 if (tokenKind == QQmlJSGrammar::T_DIVIDE_)
29 prefix = QQmlJS::Lexer::NoPrefix;
30 else
31 prefix = QQmlJS::Lexer::EqualPrefix;
32 if (lexer.scanRegExp(prefix)) {
33 regexpMayFollow = false;
34 break;
35 } else {
36 qCWarning(domLog) << "lexing error scannign regexp in" << lexer.code()
37 << lexer.errorCode() << lexer.errorMessage();
38 }
39 break;
40 } else if (tokenKind == QQmlJSGrammar::T_DIVIDE_) {
41 regexpMayFollow = true;
42 }
43 Q_FALLTHROUGH();
44
45 case QQmlJSGrammar::T_AND:
46 case QQmlJSGrammar::T_AND_AND:
47 case QQmlJSGrammar::T_AND_EQ:
48 case QQmlJSGrammar::T_ARROW:
49 case QQmlJSGrammar::T_EQ:
50 case QQmlJSGrammar::T_EQ_EQ:
51 case QQmlJSGrammar::T_EQ_EQ_EQ:
52 case QQmlJSGrammar::T_GE:
53 case QQmlJSGrammar::T_GT:
54 case QQmlJSGrammar::T_GT_GT:
55 case QQmlJSGrammar::T_GT_GT_EQ:
56 case QQmlJSGrammar::T_GT_GT_GT:
57 case QQmlJSGrammar::T_GT_GT_GT_EQ:
58 case QQmlJSGrammar::T_LE:
59 case QQmlJSGrammar::T_LT:
60 case QQmlJSGrammar::T_LT_LT:
61 case QQmlJSGrammar::T_LT_LT_EQ:
62 case QQmlJSGrammar::T_MINUS:
63 case QQmlJSGrammar::T_MINUS_EQ:
64 case QQmlJSGrammar::T_MINUS_MINUS:
65 case QQmlJSGrammar::T_NOT:
66 case QQmlJSGrammar::T_NOT_EQ:
67 case QQmlJSGrammar::T_NOT_EQ_EQ:
68 case QQmlJSGrammar::T_OR:
69 case QQmlJSGrammar::T_OR_EQ:
70 case QQmlJSGrammar::T_OR_OR:
71 case QQmlJSGrammar::T_PLUS:
72 case QQmlJSGrammar::T_PLUS_EQ:
73 case QQmlJSGrammar::T_PLUS_PLUS:
74 case QQmlJSGrammar::T_QUESTION:
75 case QQmlJSGrammar::T_QUESTION_DOT:
76 case QQmlJSGrammar::T_QUESTION_QUESTION:
77 case QQmlJSGrammar::T_REMAINDER:
78 case QQmlJSGrammar::T_REMAINDER_EQ:
79 case QQmlJSGrammar::T_STAR:
80 case QQmlJSGrammar::T_STAR_EQ:
81 case QQmlJSGrammar::T_STAR_STAR:
82 case QQmlJSGrammar::T_STAR_STAR_EQ:
83 case QQmlJSGrammar::T_TILDE:
84 case QQmlJSGrammar::T_XOR:
85 case QQmlJSGrammar::T_XOR_EQ:
86
87 case QQmlJSGrammar::T_AT:
88
89 case QQmlJSGrammar::T_AUTOMATIC_SEMICOLON:
90 case QQmlJSGrammar::T_COMPATIBILITY_SEMICOLON:
91 case QQmlJSGrammar::T_SEMICOLON:
92
93 case QQmlJSGrammar::T_COLON:
94 case QQmlJSGrammar::T_COMMA:
95 case QQmlJSGrammar::T_LBRACE:
96 case QQmlJSGrammar::T_LBRACKET:
97 case QQmlJSGrammar::T_LPAREN:
98
99 case QQmlJSGrammar::T_ELLIPSIS:
100 regexpMayFollow = true;
101 break;
102
103 case QQmlJSGrammar::T_FUNCTION:
104 // might contain a space at the end...
105 tokens.append(t: Token(lexer.tokenStartColumn() - 1,
106 lexer.tokenLength() - ((lexer.tokenText().endsWith(c: u' ')) ? 1 : 0),
107 tokenKind));
108 return;
109
110 case QQmlJSGrammar::T_DOT:
111 case QQmlJSGrammar::T_RBRACE:
112 case QQmlJSGrammar::T_RBRACKET:
113 case QQmlJSGrammar::T_RPAREN:
114 regexpMayFollow = false;
115 break;
116
117 // template used to expand to a string plus a delimiter for the ${ and }, now
118 // we use a + as delimiter
119 case QQmlJSGrammar::T_TEMPLATE_HEAD:
120 regexpMayFollow = true;
121 tokens.append(t: Token(lexer.tokenStartColumn() - 1, lexer.tokenLength() - 2, tokenKind));
122 tokens.append(Token(lexer.tokenStartColumn() + lexer.tokenLength() - 3, 2,
123 QQmlJSGrammar::T_PLUS));
124 return;
125 case QQmlJSGrammar::T_TEMPLATE_MIDDLE:
126 regexpMayFollow = true;
127 tokens.append(Token(lexer.tokenStartColumn() - 1, 1, QQmlJSGrammar::T_PLUS));
128 tokens.append(t: Token(lexer.tokenStartColumn(), lexer.tokenLength() - 3, tokenKind));
129 tokens.append(Token(lexer.tokenStartColumn() + lexer.tokenLength() - 3, 2,
130 QQmlJSGrammar::T_PLUS));
131 return;
132 case QQmlJSGrammar::T_TEMPLATE_TAIL:
133 regexpMayFollow = true;
134 tokens.append(Token(lexer.tokenStartColumn() - 1, 1, QQmlJSGrammar::T_PLUS));
135 tokens.append(t: Token(lexer.tokenStartColumn(), lexer.tokenLength() - 1, tokenKind));
136 return;
137 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_MIDDLE:
138 regexpMayFollow = true;
139 tokens.append(Token(lexer.tokenStartColumn() - 1, 1, QQmlJSGrammar::T_PLUS));
140 tokens.append(t: Token(lexer.tokenStartColumn(), lexer.tokenLength() - 1, tokenKind));
141 return;
142 case QQmlJSGrammar::T_MULTILINE_STRING_LITERAL:
143 case QQmlJSGrammar::T_NO_SUBSTITUTION_TEMPLATE:
144 case QQmlJSGrammar::T_STRING_LITERAL:
145 case QQmlJSGrammar::T_PARTIAL_SINGLE_QUOTE_STRING_LITERAL:
146 case QQmlJSGrammar::T_PARTIAL_DOUBLE_QUOTE_STRING_LITERAL:
147 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_HEAD:
148 regexpMayFollow = (tokenKind == QQmlJSGrammar::T_TEMPLATE_MIDDLE
149 || tokenKind == QQmlJSGrammar::T_TEMPLATE_HEAD);
150 break;
151
152 case QQmlJSGrammar::T_VERSION_NUMBER:
153 if (lexer.state().currentChar == u'.') {
154 int offset = lexer.tokenStartColumn() - 1;
155 int length = lexer.tokenLength();
156 tokenKind = lexer.lex();
157 Q_ASSERT(tokenKind == QQmlJSGrammar::T_DOT);
158 tokenKind = lexer.lex();
159 Q_ASSERT(tokenKind == QQmlJSGrammar::T_VERSION_NUMBER);
160 length += 1 + lexer.tokenLength();
161 tokens.append(Token(offset, length, QQmlJSGrammar::T_NUMERIC_LITERAL));
162 return;
163 }
164 break;
165
166 default:
167 break;
168 }
169 // avoid newline (on multiline comments/strings)
170 qsizetype len = lexer.code().size();
171 if (lexer.code().endsWith(c: u'\n'))
172 --len;
173 len -= lexer.tokenStartColumn() - 1;
174 if (len < 0)
175 len = 0;
176 if (lexer.tokenLength() < len)
177 len = lexer.tokenLength();
178 tokens.append(t: Token(lexer.tokenStartColumn() - 1, len, tokenKind));
179}
180
181bool Token::lexKindIsDelimiter(int kind)
182{
183 switch (kind) {
184 case QQmlJSGrammar::T_AND:
185 case QQmlJSGrammar::T_AND_AND:
186 case QQmlJSGrammar::T_AND_EQ:
187 case QQmlJSGrammar::T_ARROW:
188 case QQmlJSGrammar::T_EQ:
189 case QQmlJSGrammar::T_EQ_EQ:
190 case QQmlJSGrammar::T_EQ_EQ_EQ:
191 case QQmlJSGrammar::T_GE:
192 case QQmlJSGrammar::T_GT:
193 case QQmlJSGrammar::T_GT_GT:
194 case QQmlJSGrammar::T_GT_GT_EQ:
195 case QQmlJSGrammar::T_GT_GT_GT:
196 case QQmlJSGrammar::T_GT_GT_GT_EQ:
197 case QQmlJSGrammar::T_LE:
198 case QQmlJSGrammar::T_LT:
199 case QQmlJSGrammar::T_LT_LT:
200 case QQmlJSGrammar::T_LT_LT_EQ:
201 case QQmlJSGrammar::T_MINUS:
202 case QQmlJSGrammar::T_MINUS_EQ:
203 case QQmlJSGrammar::T_NOT:
204 case QQmlJSGrammar::T_NOT_EQ:
205 case QQmlJSGrammar::T_NOT_EQ_EQ:
206 case QQmlJSGrammar::T_OR:
207 case QQmlJSGrammar::T_OR_EQ:
208 case QQmlJSGrammar::T_OR_OR:
209 case QQmlJSGrammar::T_PLUS:
210 case QQmlJSGrammar::T_PLUS_EQ:
211 case QQmlJSGrammar::T_QUESTION:
212 case QQmlJSGrammar::T_QUESTION_DOT:
213 case QQmlJSGrammar::T_QUESTION_QUESTION:
214 case QQmlJSGrammar::T_REMAINDER:
215 case QQmlJSGrammar::T_REMAINDER_EQ:
216 case QQmlJSGrammar::T_STAR:
217 case QQmlJSGrammar::T_STAR_EQ:
218 case QQmlJSGrammar::T_STAR_STAR:
219 case QQmlJSGrammar::T_STAR_STAR_EQ:
220 case QQmlJSGrammar::T_TILDE:
221 case QQmlJSGrammar::T_XOR:
222 case QQmlJSGrammar::T_XOR_EQ:
223 case QQmlJSGrammar::T_AT:
224 case QQmlJSGrammar::T_COMMA:
225 case QQmlJSGrammar::T_COLON:
226 case QQmlJSGrammar::T_LPAREN:
227 case QQmlJSGrammar::T_DIVIDE_:
228 case QQmlJSGrammar::T_DIVIDE_EQ:
229 return true;
230 default:
231 break;
232 }
233 return false;
234}
235
236bool Token::lexKindIsQmlReserved(int kind)
237{
238 switch (kind) {
239 case QQmlJSGrammar::T_AS:
240 case QQmlJSGrammar::T_IMPORT:
241 case QQmlJSGrammar::T_SIGNAL:
242 case QQmlJSGrammar::T_PROPERTY:
243 case QQmlJSGrammar::T_READONLY:
244 case QQmlJSGrammar::T_COMPONENT:
245 case QQmlJSGrammar::T_REQUIRED:
246 case QQmlJSGrammar::T_ON:
247 case QQmlJSGrammar::T_ENUM:
248 return true;
249 default:
250 break;
251 }
252 return false;
253}
254
255bool Token::lexKindIsComment(int kind)
256{
257 switch (kind) {
258 case QQmlJSGrammar::T_COMMENT:
259 case QQmlJSGrammar::T_PARTIAL_COMMENT:
260 return true;
261 default:
262 break;
263 }
264 return false;
265}
266
267bool Token::lexKindIsJSKeyword(int kind)
268{
269 switch (kind) {
270 case QQmlJSGrammar::T_BREAK:
271 case QQmlJSGrammar::T_CASE:
272 case QQmlJSGrammar::T_CATCH:
273 case QQmlJSGrammar::T_CLASS:
274 case QQmlJSGrammar::T_CONST:
275 case QQmlJSGrammar::T_CONTINUE:
276 case QQmlJSGrammar::T_DEBUGGER:
277 case QQmlJSGrammar::T_DEFAULT:
278 case QQmlJSGrammar::T_DELETE:
279 case QQmlJSGrammar::T_DO:
280 case QQmlJSGrammar::T_ELSE:
281 case QQmlJSGrammar::T_ENUM:
282 case QQmlJSGrammar::T_EXPORT:
283 case QQmlJSGrammar::T_EXTENDS:
284 case QQmlJSGrammar::T_FALSE:
285 case QQmlJSGrammar::T_FINALLY:
286 case QQmlJSGrammar::T_FOR:
287 case QQmlJSGrammar::T_FROM:
288 case QQmlJSGrammar::T_GET:
289 case QQmlJSGrammar::T_IF:
290 case QQmlJSGrammar::T_IN:
291 case QQmlJSGrammar::T_INSTANCEOF:
292 case QQmlJSGrammar::T_LET:
293 case QQmlJSGrammar::T_NEW:
294 case QQmlJSGrammar::T_RETURN:
295 case QQmlJSGrammar::T_SUPER:
296 case QQmlJSGrammar::T_SWITCH:
297 case QQmlJSGrammar::T_THEN:
298 case QQmlJSGrammar::T_THIS:
299 case QQmlJSGrammar::T_THROW:
300 case QQmlJSGrammar::T_VOID:
301 case QQmlJSGrammar::T_WHILE:
302 case QQmlJSGrammar::T_WITH:
303 case QQmlJSGrammar::T_YIELD:
304 case QQmlJSGrammar::T_VAR:
305 case QQmlJSGrammar::T_FUNCTION:
306 return true;
307 default:
308 break;
309 }
310 return false;
311}
312
313bool Token::lexKindIsIdentifier(int kind)
314{
315 switch (kind) {
316 case QQmlJSGrammar::T_IDENTIFIER:
317 case QQmlJSGrammar::T_COMPONENT:
318 case QQmlJSGrammar::T_REQUIRED:
319 case QQmlJSGrammar::T_AS:
320 case QQmlJSGrammar::T_PRAGMA:
321 case QQmlJSGrammar::T_IMPORT:
322 case QQmlJSGrammar::T_RESERVED_WORD:
323 case QQmlJSGrammar::T_SET:
324 case QQmlJSGrammar::T_SIGNAL:
325 case QQmlJSGrammar::T_PROPERTY:
326 case QQmlJSGrammar::T_PUBLIC:
327 case QQmlJSGrammar::T_READONLY:
328 case QQmlJSGrammar::T_NULL:
329 case QQmlJSGrammar::T_OF:
330 case QQmlJSGrammar::T_ON:
331 case QQmlJSGrammar::T_STATIC:
332 case QQmlJSGrammar::T_TRUE:
333 case QQmlJSGrammar::T_TRY:
334 case QQmlJSGrammar::T_TYPEOF:
335 case QQmlJSGrammar::T_WITHOUTAS:
336 case QQmlJSGrammar::T_FROM:
337 return true;
338 default:
339 break;
340 }
341 return false;
342}
343
344bool Token::lexKindIsStringType(int kind)
345{
346 switch (kind) {
347 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_MIDDLE:
348 case QQmlJSGrammar::T_MULTILINE_STRING_LITERAL:
349 case QQmlJSGrammar::T_NO_SUBSTITUTION_TEMPLATE:
350 case QQmlJSGrammar::T_STRING_LITERAL:
351 case QQmlJSGrammar::T_PARTIAL_SINGLE_QUOTE_STRING_LITERAL:
352 case QQmlJSGrammar::T_PARTIAL_DOUBLE_QUOTE_STRING_LITERAL:
353 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_HEAD:
354 return true;
355 default:
356 break;
357 }
358 return false;
359}
360
361bool Token::lexKindIsInvalid(int kind)
362{
363 switch (kind) {
364 case QQmlJSGrammar::T_NONE:
365 case QQmlJSGrammar::T_EOL:
366 case QQmlJSGrammar::EOF_SYMBOL:
367 case QQmlJSGrammar::T_ERROR:
368 case QQmlJSGrammar::T_FEED_JS_EXPRESSION:
369 case QQmlJSGrammar::T_FEED_JS_MODULE:
370 case QQmlJSGrammar::T_FEED_JS_SCRIPT:
371 case QQmlJSGrammar::T_FEED_JS_STATEMENT:
372 case QQmlJSGrammar::T_FEED_UI_OBJECT_MEMBER:
373 case QQmlJSGrammar::T_FEED_UI_PROGRAM:
374 case QQmlJSGrammar::REDUCE_HERE:
375 case QQmlJSGrammar::T_FORCE_BLOCK:
376 case QQmlJSGrammar::T_FORCE_DECLARATION:
377 case QQmlJSGrammar::T_FOR_LOOKAHEAD_OK:
378 return true;
379 default:
380 break;
381 }
382 return false;
383}
384
385void Token::dump(const Sink &s, QStringView line) const
386{
387 s(u"{");
388 sinkInt(s, i: offset);
389 s(u", ");
390 sinkInt(s, i: length);
391 s(u", Token::");
392 s(QString::number(lexKind));
393 s(u"}");
394 QStringView value = line.mid(pos: offset, n: length);
395 if (!value.isEmpty()) {
396 s(u":");
397 sinkEscaped(sink: s, s: value);
398 }
399}
400
401QList<Token> Scanner::operator()(QStringView text, const Scanner::State &startState)
402{
403 _state = startState;
404 QList<Token> tokens;
405
406 {
407 QQmlJS::Lexer lexer(nullptr, QQmlJS::Lexer::LexMode::LineByLine);
408 lexer.setState(startState.state);
409 QString line = text.toString();
410 if (!(line.endsWith(s: u"\n") || line.endsWith(s: u"\r")))
411 line += u'\n';
412 lexer.setCode(code: line, lineno: -1, qmlMode: _qmlMode, codeContinuation: QQmlJS::Lexer::CodeContinuation::Continue);
413 while (true) {
414 int tokenKind = lexer.lex();
415 if (tokenKind == QQmlJSGrammar::T_EOL || tokenKind == QQmlJSGrammar::EOF_SYMBOL)
416 break;
417 addLexToken(tokens, tokenKind, lexer, regexpMayFollow&: _state.regexpMightFollow);
418 }
419 _state.state = lexer.state();
420 }
421 return tokens;
422}
423
424Scanner::State Scanner::state() const
425{
426 return _state;
427}
428
429bool Scanner::State::isMultiline() const
430{
431 switch (state.tokenKind) {
432 case QQmlJSGrammar::T_PARTIAL_COMMENT:
433 case QQmlJSGrammar::T_PARTIAL_DOUBLE_QUOTE_STRING_LITERAL:
434 case QQmlJSGrammar::T_PARTIAL_SINGLE_QUOTE_STRING_LITERAL:
435 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_HEAD:
436 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_MIDDLE:
437 return true;
438 default:
439 break;
440 }
441 return false;
442}
443
444bool Scanner::State::isMultilineComment() const
445{
446 switch (state.tokenKind) {
447 case QQmlJSGrammar::T_PARTIAL_COMMENT:
448 return true;
449 default:
450 break;
451 }
452 return false;
453}
454
455QT_END_NAMESPACE
456

source code of qtdeclarative/src/qmldom/qqmldomscanner.cpp