1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#ifndef QQMLJSLEXER_P_H
5#define QQMLJSLEXER_P_H
6
7//
8// W A R N I N G
9// -------------
10//
11// This file is not part of the Qt API. It exists purely as an
12// implementation detail. This header file may change from version to
13// version without notice, or even be removed.
14//
15// We mean it.
16//
17
18#include <private/qqmljsglobal_p.h>
19#include <private/qqmljsgrammar_p.h>
20
21#include <QtCore/qstring.h>
22#include <QtCore/qstack.h>
23
24QT_BEGIN_NAMESPACE
25
26class QDebug;
27
28namespace QQmlJS {
29
30class Engine;
31struct DiagnosticMessage;
32class Directives;
33
34class QML_PARSER_EXPORT Lexer: public QQmlJSGrammar
35{
36public:
37 enum {
38 T_ABSTRACT = T_RESERVED_WORD,
39 T_BOOLEAN = T_RESERVED_WORD,
40 T_BYTE = T_RESERVED_WORD,
41 T_CHAR = T_RESERVED_WORD,
42 T_DOUBLE = T_RESERVED_WORD,
43 T_FINAL = T_RESERVED_WORD,
44 T_FLOAT = T_RESERVED_WORD,
45 T_GOTO = T_RESERVED_WORD,
46 T_IMPLEMENTS = T_RESERVED_WORD,
47 T_INT = T_RESERVED_WORD,
48 T_INTERFACE = T_RESERVED_WORD,
49 T_LONG = T_RESERVED_WORD,
50 T_NATIVE = T_RESERVED_WORD,
51 T_PACKAGE = T_RESERVED_WORD,
52 T_PRIVATE = T_RESERVED_WORD,
53 T_PROTECTED = T_RESERVED_WORD,
54 T_SHORT = T_RESERVED_WORD,
55 T_SYNCHRONIZED = T_RESERVED_WORD,
56 T_THROWS = T_RESERVED_WORD,
57 T_TRANSIENT = T_RESERVED_WORD,
58 T_VOLATILE = T_RESERVED_WORD
59 };
60
61 enum Error {
62 NoError,
63 IllegalCharacter,
64 IllegalNumber,
65 UnclosedStringLiteral,
66 IllegalEscapeSequence,
67 IllegalUnicodeEscapeSequence,
68 UnclosedComment,
69 IllegalExponentIndicator,
70 IllegalIdentifier,
71 IllegalHexadecimalEscapeSequence
72 };
73
74 enum RegExpBodyPrefix {
75 NoPrefix,
76 EqualPrefix
77 };
78
79 enum RegExpFlag {
80 RegExp_Global = 0x01,
81 RegExp_IgnoreCase = 0x02,
82 RegExp_Multiline = 0x04,
83 RegExp_Unicode = 0x08,
84 RegExp_Sticky = 0x10
85 };
86
87 enum ParseModeFlags {
88 QmlMode = 0x1,
89 YieldIsKeyword = 0x2,
90 StaticIsKeyword = 0x4
91 };
92
93 enum class ImportState {
94 SawImport,
95 NoQmlImport
96 };
97
98 enum class LexMode { WholeCode, LineByLine };
99
100 enum class CodeContinuation { Reset, Continue };
101
102public:
103 Lexer(Engine *engine, LexMode lexMode = LexMode::WholeCode);
104
105 int parseModeFlags() const {
106 int flags = 0;
107 if (qmlMode())
108 flags |= QmlMode|StaticIsKeyword;
109 if (yieldIsKeyWord())
110 flags |= YieldIsKeyword;
111 if (_staticIsKeyword)
112 flags |= StaticIsKeyword;
113 return flags;
114 }
115
116 bool qmlMode() const;
117 bool yieldIsKeyWord() const { return _state.generatorLevel != 0; }
118 void setStaticIsKeyword(bool b) { _staticIsKeyword = b; }
119
120 QString code() const;
121 void setCode(const QString &code, int lineno, bool qmlMode = true,
122 CodeContinuation codeContinuation = CodeContinuation::Reset);
123
124 int lex();
125
126 bool scanRegExp(RegExpBodyPrefix prefix = NoPrefix);
127 bool scanDirectives(Directives *directives, DiagnosticMessage *error);
128
129 int regExpFlags() const { return _state.patternFlags; }
130 QString regExpPattern() const { return _tokenText; }
131
132 int tokenKind() const { return _state.tokenKind; }
133 int tokenOffset() const { return _currentOffset + _tokenStartPtr - _code.unicode(); }
134 int tokenLength() const { return _tokenLength; }
135
136 int tokenStartLine() const { return _tokenLine; }
137 int tokenStartColumn() const { return _tokenColumn; }
138
139 inline QStringView tokenSpell() const { return _tokenSpell; }
140 inline QStringView rawString() const { return _rawString; }
141 double tokenValue() const { return _state.tokenValue; }
142 QString tokenText() const;
143
144 Error errorCode() const;
145 QString errorMessage() const;
146
147 bool prevTerminator() const;
148 bool followsClosingBrace() const;
149 bool canInsertAutomaticSemicolon(int token) const;
150
151 enum ParenthesesState {
152 IgnoreParentheses,
153 CountParentheses,
154 BalancedParentheses
155 };
156
157 enum class CommentState { NoComment, HadComment, InMultilineComment };
158
159 void enterGeneratorBody() { ++_state.generatorLevel; }
160 void leaveGeneratorBody() { --_state.generatorLevel; }
161
162 struct State
163 {
164 Error errorCode = NoError;
165
166 QChar currentChar = u'\n';
167 double tokenValue = 0;
168
169 // parentheses state
170 ParenthesesState parenthesesState = IgnoreParentheses;
171 int parenthesesCount = 0;
172
173 // template string stack
174 QStack<int> outerTemplateBraceCount;
175 int bracesCount = -1;
176
177 int stackToken = -1;
178
179 int patternFlags = 0;
180 int tokenKind = 0;
181 ImportState importState = ImportState::NoQmlImport;
182
183 bool validTokenText = false;
184 bool prohibitAutomaticSemicolon = false;
185 bool restrictedKeyword = false;
186 bool terminator = false;
187 bool followsClosingBrace = false;
188 bool delimited = true;
189 bool handlingDirectives = false;
190 CommentState comments = CommentState::NoComment;
191 int generatorLevel = 0;
192
193 friend bool operator==(State const &s1, State const &s2)
194 {
195 if (s1.errorCode != s2.errorCode)
196 return false;
197 if (s1.currentChar != s2.currentChar)
198 return false;
199 if (s1.tokenValue != s2.tokenValue)
200 return false;
201 if (s1.parenthesesState != s2.parenthesesState)
202 return false;
203 if (s1.parenthesesCount != s2.parenthesesCount)
204 return false;
205 if (s1.outerTemplateBraceCount != s2.outerTemplateBraceCount)
206 return false;
207 if (s1.bracesCount != s2.bracesCount)
208 return false;
209 if (s1.stackToken != s2.stackToken)
210 return false;
211 if (s1.patternFlags != s2.patternFlags)
212 return false;
213 if (s1.tokenKind != s2.tokenKind)
214 return false;
215 if (s1.importState != s2.importState)
216 return false;
217 if (s1.validTokenText != s2.validTokenText)
218 return false;
219 if (s1.prohibitAutomaticSemicolon != s2.prohibitAutomaticSemicolon)
220 return false;
221 if (s1.restrictedKeyword != s2.restrictedKeyword)
222 return false;
223 if (s1.terminator != s2.terminator)
224 return false;
225 if (s1.followsClosingBrace != s2.followsClosingBrace)
226 return false;
227 if (s1.delimited != s2.delimited)
228 return false;
229 if (s1.handlingDirectives != s2.handlingDirectives)
230 return false;
231 if (s1.generatorLevel != s2.generatorLevel)
232 return false;
233 return true;
234 }
235
236 friend bool operator!=(State const &s1, State const &s2) { return !(s1 == s2); }
237
238 friend QML_PARSER_EXPORT QDebug operator<<(QDebug dbg, State const &s);
239 };
240
241 const State &state() const;
242 void setState(const State &state);
243
244protected:
245 static int classify(const QChar *s, int n, int parseModeFlags);
246
247private:
248 inline void scanChar();
249 inline QChar peekChar();
250 int scanToken();
251 int scanNumber(QChar ch);
252 int scanVersionNumber(QChar ch);
253 enum ScanStringMode {
254 SingleQuote = '\'',
255 DoubleQuote = '"',
256 TemplateHead = '`',
257 TemplateContinuation = 0
258 };
259 int scanString(ScanStringMode mode);
260
261 bool isLineTerminator() const;
262 unsigned isLineTerminatorSequence() const;
263 static bool isIdentLetter(QChar c);
264 static bool isDecimalDigit(ushort c);
265 static bool isHexDigit(QChar c);
266 static bool isOctalDigit(ushort c);
267
268 void syncProhibitAutomaticSemicolon();
269 uint decodeUnicodeEscapeCharacter(bool *ok);
270 QChar decodeHexEscapeCharacter(bool *ok);
271
272 friend QML_PARSER_EXPORT QDebug operator<<(QDebug dbg, const Lexer &l);
273
274private:
275 Engine *_engine;
276
277 LexMode _lexMode = LexMode::WholeCode;
278 QString _code;
279 const QChar *_endPtr;
280 bool _qmlMode;
281 bool _staticIsKeyword = false;
282
283 bool _skipLinefeed = false;
284
285 int _currentLineNumber = 0;
286 int _currentColumnNumber = 0;
287 int _currentOffset = 0;
288
289 int _tokenLength = 0;
290 int _tokenLine = 0;
291 int _tokenColumn = 0;
292
293 QString _tokenText;
294 QString _errorMessage;
295 QStringView _tokenSpell;
296 QStringView _rawString;
297
298 const QChar *_codePtr = nullptr;
299 const QChar *_tokenStartPtr = nullptr;
300
301 State _state;
302};
303
304} // end of namespace QQmlJS
305
306QT_END_NAMESPACE
307
308#endif // LEXER_H
309

source code of qtdeclarative/src/qml/parser/qqmljslexer_p.h