1// Copyright (C) 2022 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qqmldomscanner_p.h"
5#include "qqmldomerrormessage_p.h"
6
7#include <QtCore/QMetaEnum>
8
9#include <algorithm>
10
11QT_BEGIN_NAMESPACE
12
13using namespace QQmlJS::Dom;
14
15static void addLexToken(QList<Token> &tokens, int tokenKind, QQmlJS::Lexer &lexer,
16 bool &regexpMayFollow)
17{
18 switch (tokenKind) {
19 case QQmlJSGrammar::T_DIVIDE_:
20 case QQmlJSGrammar::T_DIVIDE_EQ:
21 if (regexpMayFollow) {
22 QQmlJS::Lexer::RegExpBodyPrefix prefix;
23 if (tokenKind == QQmlJSGrammar::T_DIVIDE_)
24 prefix = QQmlJS::Lexer::NoPrefix;
25 else
26 prefix = QQmlJS::Lexer::EqualPrefix;
27 if (lexer.scanRegExp(prefix)) {
28 regexpMayFollow = false;
29 break;
30 } else {
31 qCWarning(domLog) << "lexing error scannign regexp in" << lexer.code()
32 << lexer.errorCode() << lexer.errorMessage();
33 }
34 break;
35 } else if (tokenKind == QQmlJSGrammar::T_DIVIDE_) {
36 regexpMayFollow = true;
37 }
38 Q_FALLTHROUGH();
39
40 case QQmlJSGrammar::T_AND:
41 case QQmlJSGrammar::T_AND_AND:
42 case QQmlJSGrammar::T_AND_EQ:
43 case QQmlJSGrammar::T_ARROW:
44 case QQmlJSGrammar::T_EQ:
45 case QQmlJSGrammar::T_EQ_EQ:
46 case QQmlJSGrammar::T_EQ_EQ_EQ:
47 case QQmlJSGrammar::T_GE:
48 case QQmlJSGrammar::T_GT:
49 case QQmlJSGrammar::T_GT_GT:
50 case QQmlJSGrammar::T_GT_GT_EQ:
51 case QQmlJSGrammar::T_GT_GT_GT:
52 case QQmlJSGrammar::T_GT_GT_GT_EQ:
53 case QQmlJSGrammar::T_LE:
54 case QQmlJSGrammar::T_LT:
55 case QQmlJSGrammar::T_LT_LT:
56 case QQmlJSGrammar::T_LT_LT_EQ:
57 case QQmlJSGrammar::T_MINUS:
58 case QQmlJSGrammar::T_MINUS_EQ:
59 case QQmlJSGrammar::T_MINUS_MINUS:
60 case QQmlJSGrammar::T_NOT:
61 case QQmlJSGrammar::T_NOT_EQ:
62 case QQmlJSGrammar::T_NOT_EQ_EQ:
63 case QQmlJSGrammar::T_OR:
64 case QQmlJSGrammar::T_OR_EQ:
65 case QQmlJSGrammar::T_OR_OR:
66 case QQmlJSGrammar::T_PLUS:
67 case QQmlJSGrammar::T_PLUS_EQ:
68 case QQmlJSGrammar::T_PLUS_PLUS:
69 case QQmlJSGrammar::T_QUESTION:
70 case QQmlJSGrammar::T_QUESTION_DOT:
71 case QQmlJSGrammar::T_QUESTION_QUESTION:
72 case QQmlJSGrammar::T_REMAINDER:
73 case QQmlJSGrammar::T_REMAINDER_EQ:
74 case QQmlJSGrammar::T_STAR:
75 case QQmlJSGrammar::T_STAR_EQ:
76 case QQmlJSGrammar::T_STAR_STAR:
77 case QQmlJSGrammar::T_STAR_STAR_EQ:
78 case QQmlJSGrammar::T_TILDE:
79 case QQmlJSGrammar::T_XOR:
80 case QQmlJSGrammar::T_XOR_EQ:
81
82 case QQmlJSGrammar::T_AT:
83
84 case QQmlJSGrammar::T_AUTOMATIC_SEMICOLON:
85 case QQmlJSGrammar::T_COMPATIBILITY_SEMICOLON:
86 case QQmlJSGrammar::T_SEMICOLON:
87
88 case QQmlJSGrammar::T_COLON:
89 case QQmlJSGrammar::T_COMMA:
90 case QQmlJSGrammar::T_LBRACE:
91 case QQmlJSGrammar::T_LBRACKET:
92 case QQmlJSGrammar::T_LPAREN:
93
94 case QQmlJSGrammar::T_ELLIPSIS:
95 regexpMayFollow = true;
96 break;
97
98 case QQmlJSGrammar::T_FUNCTION:
99 // might contain a space at the end...
100 tokens.append(t: Token(lexer.tokenStartColumn() - 1,
101 lexer.tokenLength() - ((lexer.tokenText().endsWith(c: u' ')) ? 1 : 0),
102 tokenKind));
103 return;
104
105 case QQmlJSGrammar::T_DOT:
106 case QQmlJSGrammar::T_RBRACE:
107 case QQmlJSGrammar::T_RBRACKET:
108 case QQmlJSGrammar::T_RPAREN:
109 regexpMayFollow = false;
110 break;
111
112 // template used to expand to a string plus a delimiter for the ${ and }, now
113 // we use a + as delimiter
114 case QQmlJSGrammar::T_TEMPLATE_HEAD:
115 regexpMayFollow = true;
116 tokens.append(t: Token(lexer.tokenStartColumn() - 1, lexer.tokenLength() - 2, tokenKind));
117 tokens.append(Token(lexer.tokenStartColumn() + lexer.tokenLength() - 3, 2,
118 QQmlJSGrammar::T_PLUS));
119 return;
120 case QQmlJSGrammar::T_TEMPLATE_MIDDLE:
121 regexpMayFollow = true;
122 tokens.append(Token(lexer.tokenStartColumn() - 1, 1, QQmlJSGrammar::T_PLUS));
123 tokens.append(t: Token(lexer.tokenStartColumn(), lexer.tokenLength() - 3, tokenKind));
124 tokens.append(Token(lexer.tokenStartColumn() + lexer.tokenLength() - 3, 2,
125 QQmlJSGrammar::T_PLUS));
126 return;
127 case QQmlJSGrammar::T_TEMPLATE_TAIL:
128 regexpMayFollow = true;
129 tokens.append(Token(lexer.tokenStartColumn() - 1, 1, QQmlJSGrammar::T_PLUS));
130 tokens.append(t: Token(lexer.tokenStartColumn(), lexer.tokenLength() - 1, tokenKind));
131 return;
132 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_MIDDLE:
133 regexpMayFollow = true;
134 tokens.append(Token(lexer.tokenStartColumn() - 1, 1, QQmlJSGrammar::T_PLUS));
135 tokens.append(t: Token(lexer.tokenStartColumn(), lexer.tokenLength() - 1, tokenKind));
136 return;
137 case QQmlJSGrammar::T_MULTILINE_STRING_LITERAL:
138 case QQmlJSGrammar::T_NO_SUBSTITUTION_TEMPLATE:
139 case QQmlJSGrammar::T_STRING_LITERAL:
140 case QQmlJSGrammar::T_PARTIAL_SINGLE_QUOTE_STRING_LITERAL:
141 case QQmlJSGrammar::T_PARTIAL_DOUBLE_QUOTE_STRING_LITERAL:
142 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_HEAD:
143 regexpMayFollow = (tokenKind == QQmlJSGrammar::T_TEMPLATE_MIDDLE
144 || tokenKind == QQmlJSGrammar::T_TEMPLATE_HEAD);
145 break;
146
147 case QQmlJSGrammar::T_VERSION_NUMBER:
148 if (lexer.state().currentChar == u'.') {
149 int offset = lexer.tokenStartColumn() - 1;
150 int length = lexer.tokenLength();
151 tokenKind = lexer.lex();
152 Q_ASSERT(tokenKind == QQmlJSGrammar::T_DOT);
153 tokenKind = lexer.lex();
154 Q_ASSERT(tokenKind == QQmlJSGrammar::T_VERSION_NUMBER);
155 length += 1 + lexer.tokenLength();
156 tokens.append(Token(offset, length, QQmlJSGrammar::T_NUMERIC_LITERAL));
157 return;
158 }
159 break;
160
161 default:
162 break;
163 }
164 // avoid newline (on multiline comments/strings)
165 qsizetype len = lexer.code().size();
166 if (lexer.code().endsWith(c: u'\n'))
167 --len;
168 len -= lexer.tokenStartColumn() - 1;
169 if (len < 0)
170 len = 0;
171 if (lexer.tokenLength() < len)
172 len = lexer.tokenLength();
173 tokens.append(t: Token(lexer.tokenStartColumn() - 1, len, tokenKind));
174}
175
176bool Token::lexKindIsDelimiter(int kind)
177{
178 switch (kind) {
179 case QQmlJSGrammar::T_AND:
180 case QQmlJSGrammar::T_AND_AND:
181 case QQmlJSGrammar::T_AND_EQ:
182 case QQmlJSGrammar::T_ARROW:
183 case QQmlJSGrammar::T_EQ:
184 case QQmlJSGrammar::T_EQ_EQ:
185 case QQmlJSGrammar::T_EQ_EQ_EQ:
186 case QQmlJSGrammar::T_GE:
187 case QQmlJSGrammar::T_GT:
188 case QQmlJSGrammar::T_GT_GT:
189 case QQmlJSGrammar::T_GT_GT_EQ:
190 case QQmlJSGrammar::T_GT_GT_GT:
191 case QQmlJSGrammar::T_GT_GT_GT_EQ:
192 case QQmlJSGrammar::T_LE:
193 case QQmlJSGrammar::T_LT:
194 case QQmlJSGrammar::T_LT_LT:
195 case QQmlJSGrammar::T_LT_LT_EQ:
196 case QQmlJSGrammar::T_MINUS:
197 case QQmlJSGrammar::T_MINUS_EQ:
198 case QQmlJSGrammar::T_MINUS_MINUS:
199 case QQmlJSGrammar::T_NOT:
200 case QQmlJSGrammar::T_NOT_EQ:
201 case QQmlJSGrammar::T_NOT_EQ_EQ:
202 case QQmlJSGrammar::T_OR:
203 case QQmlJSGrammar::T_OR_EQ:
204 case QQmlJSGrammar::T_OR_OR:
205 case QQmlJSGrammar::T_PLUS:
206 case QQmlJSGrammar::T_PLUS_EQ:
207 case QQmlJSGrammar::T_PLUS_PLUS:
208 case QQmlJSGrammar::T_QUESTION:
209 case QQmlJSGrammar::T_QUESTION_DOT:
210 case QQmlJSGrammar::T_QUESTION_QUESTION:
211 case QQmlJSGrammar::T_REMAINDER:
212 case QQmlJSGrammar::T_REMAINDER_EQ:
213 case QQmlJSGrammar::T_STAR:
214 case QQmlJSGrammar::T_STAR_EQ:
215 case QQmlJSGrammar::T_STAR_STAR:
216 case QQmlJSGrammar::T_STAR_STAR_EQ:
217 case QQmlJSGrammar::T_TILDE:
218 case QQmlJSGrammar::T_XOR:
219 case QQmlJSGrammar::T_XOR_EQ:
220
221 case QQmlJSGrammar::T_AT:
222 return true;
223 default:
224 break;
225 }
226 return false;
227}
228
229bool Token::lexKindIsQmlReserved(int kind)
230{
231 switch (kind) {
232 case QQmlJSGrammar::T_AS:
233 case QQmlJSGrammar::T_IMPORT:
234 case QQmlJSGrammar::T_SIGNAL:
235 case QQmlJSGrammar::T_PROPERTY:
236 case QQmlJSGrammar::T_READONLY:
237 case QQmlJSGrammar::T_COMPONENT:
238 case QQmlJSGrammar::T_REQUIRED:
239 case QQmlJSGrammar::T_ON:
240 case QQmlJSGrammar::T_ENUM:
241 return true;
242 default:
243 break;
244 }
245 return false;
246}
247
248bool Token::lexKindIsComment(int kind)
249{
250 switch (kind) {
251 case QQmlJSGrammar::T_COMMENT:
252 case QQmlJSGrammar::T_PARTIAL_COMMENT:
253 return true;
254 default:
255 break;
256 }
257 return false;
258}
259
260bool Token::lexKindIsJSKeyword(int kind)
261{
262 switch (kind) {
263 case QQmlJSGrammar::T_BREAK:
264 case QQmlJSGrammar::T_CASE:
265 case QQmlJSGrammar::T_CATCH:
266 case QQmlJSGrammar::T_CLASS:
267 case QQmlJSGrammar::T_CONST:
268 case QQmlJSGrammar::T_CONTINUE:
269 case QQmlJSGrammar::T_DEBUGGER:
270 case QQmlJSGrammar::T_DEFAULT:
271 case QQmlJSGrammar::T_DELETE:
272 case QQmlJSGrammar::T_DO:
273 case QQmlJSGrammar::T_ELSE:
274 case QQmlJSGrammar::T_ENUM:
275 case QQmlJSGrammar::T_EXPORT:
276 case QQmlJSGrammar::T_EXTENDS:
277 case QQmlJSGrammar::T_FALSE:
278 case QQmlJSGrammar::T_FINALLY:
279 case QQmlJSGrammar::T_FOR:
280 case QQmlJSGrammar::T_FROM:
281 case QQmlJSGrammar::T_GET:
282 case QQmlJSGrammar::T_IF:
283 case QQmlJSGrammar::T_IN:
284 case QQmlJSGrammar::T_INSTANCEOF:
285 case QQmlJSGrammar::T_LET:
286 case QQmlJSGrammar::T_NEW:
287 case QQmlJSGrammar::T_RETURN:
288 case QQmlJSGrammar::T_SUPER:
289 case QQmlJSGrammar::T_SWITCH:
290 case QQmlJSGrammar::T_THEN:
291 case QQmlJSGrammar::T_THIS:
292 case QQmlJSGrammar::T_THROW:
293 case QQmlJSGrammar::T_VOID:
294 case QQmlJSGrammar::T_WHILE:
295 case QQmlJSGrammar::T_WITH:
296 case QQmlJSGrammar::T_YIELD:
297 case QQmlJSGrammar::T_VAR:
298 case QQmlJSGrammar::T_FUNCTION_STAR:
299 case QQmlJSGrammar::T_FUNCTION:
300 return true;
301 default:
302 break;
303 }
304 return false;
305}
306
307bool Token::lexKindIsIdentifier(int kind)
308{
309 switch (kind) {
310 case QQmlJSGrammar::T_IDENTIFIER:
311 case QQmlJSGrammar::T_COMPONENT:
312 case QQmlJSGrammar::T_REQUIRED:
313 case QQmlJSGrammar::T_AS:
314 case QQmlJSGrammar::T_PRAGMA:
315 case QQmlJSGrammar::T_IMPORT:
316 case QQmlJSGrammar::T_RESERVED_WORD:
317 case QQmlJSGrammar::T_SET:
318 case QQmlJSGrammar::T_SIGNAL:
319 case QQmlJSGrammar::T_PROPERTY:
320 case QQmlJSGrammar::T_PUBLIC:
321 case QQmlJSGrammar::T_READONLY:
322 case QQmlJSGrammar::T_NULL:
323 case QQmlJSGrammar::T_OF:
324 case QQmlJSGrammar::T_ON:
325 case QQmlJSGrammar::T_STATIC:
326 case QQmlJSGrammar::T_TRUE:
327 case QQmlJSGrammar::T_TRY:
328 case QQmlJSGrammar::T_TYPEOF:
329 case QQmlJSGrammar::T_WITHOUTAS:
330 return true;
331 default:
332 break;
333 }
334 return false;
335}
336
337bool Token::lexKindIsStringType(int kind)
338{
339 switch (kind) {
340 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_MIDDLE:
341 case QQmlJSGrammar::T_MULTILINE_STRING_LITERAL:
342 case QQmlJSGrammar::T_NO_SUBSTITUTION_TEMPLATE:
343 case QQmlJSGrammar::T_STRING_LITERAL:
344 case QQmlJSGrammar::T_PARTIAL_SINGLE_QUOTE_STRING_LITERAL:
345 case QQmlJSGrammar::T_PARTIAL_DOUBLE_QUOTE_STRING_LITERAL:
346 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_HEAD:
347 return true;
348 default:
349 break;
350 }
351 return false;
352}
353
354bool Token::lexKindIsInvalid(int kind)
355{
356 switch (kind) {
357 case QQmlJSGrammar::T_NONE:
358 case QQmlJSGrammar::T_EOL:
359 case QQmlJSGrammar::EOF_SYMBOL:
360 case QQmlJSGrammar::T_ERROR:
361 case QQmlJSGrammar::T_FEED_JS_EXPRESSION:
362 case QQmlJSGrammar::T_FEED_JS_MODULE:
363 case QQmlJSGrammar::T_FEED_JS_SCRIPT:
364 case QQmlJSGrammar::T_FEED_JS_STATEMENT:
365 case QQmlJSGrammar::T_FEED_UI_OBJECT_MEMBER:
366 case QQmlJSGrammar::T_FEED_UI_PROGRAM:
367 case QQmlJSGrammar::REDUCE_HERE:
368 case QQmlJSGrammar::T_FORCE_BLOCK:
369 case QQmlJSGrammar::T_FORCE_DECLARATION:
370 case QQmlJSGrammar::T_FOR_LOOKAHEAD_OK:
371 return true;
372 default:
373 break;
374 }
375 return false;
376}
377
378void Token::dump(Sink s, QStringView line) const
379{
380 s(u"{");
381 sinkInt(s, i: offset);
382 s(u", ");
383 sinkInt(s, i: length);
384 s(u", Token::");
385 s(QString::number(lexKind));
386 s(u"}");
387 QStringView value = line.mid(pos: offset, n: length);
388 if (!value.isEmpty()) {
389 s(u":");
390 sinkEscaped(sink: s, s: value);
391 }
392}
393
394QList<Token> Scanner::operator()(QStringView text, const Scanner::State &startState)
395{
396 _state = startState;
397 QList<Token> tokens;
398
399 {
400 QQmlJS::Lexer lexer(nullptr, QQmlJS::Lexer::LexMode::LineByLine);
401 lexer.setState(startState.state);
402 QString line = text.toString();
403 if (!(line.endsWith(s: u"\n") || line.endsWith(s: u"\r")))
404 line += u'\n';
405 lexer.setCode(code: line, lineno: -1, qmlMode: _qmlMode, codeContinuation: QQmlJS::Lexer::CodeContinuation::Continue);
406 while (true) {
407 int tokenKind = lexer.lex();
408 if (tokenKind == QQmlJSGrammar::T_EOL || tokenKind == QQmlJSGrammar::EOF_SYMBOL)
409 break;
410 addLexToken(tokens, tokenKind, lexer, regexpMayFollow&: _state.regexpMightFollow);
411 }
412 _state.state = lexer.state();
413 }
414 return tokens;
415}
416
417Scanner::State Scanner::state() const
418{
419 return _state;
420}
421
422bool Scanner::State::isMultiline() const
423{
424 switch (state.tokenKind) {
425 case QQmlJSGrammar::T_PARTIAL_COMMENT:
426 case QQmlJSGrammar::T_PARTIAL_DOUBLE_QUOTE_STRING_LITERAL:
427 case QQmlJSGrammar::T_PARTIAL_SINGLE_QUOTE_STRING_LITERAL:
428 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_HEAD:
429 case QQmlJSGrammar::T_PARTIAL_TEMPLATE_MIDDLE:
430 return true;
431 default:
432 break;
433 }
434 return false;
435}
436
437bool Scanner::State::isMultilineComment() const
438{
439 switch (state.tokenKind) {
440 case QQmlJSGrammar::T_PARTIAL_COMMENT:
441 return true;
442 default:
443 break;
444 }
445 return false;
446}
447
448QT_END_NAMESPACE
449

source code of qtdeclarative/src/qmldom/qqmldomscanner.cpp