1/****************************************************************************
2**
3** Copyright (C) 2016 The Qt Company Ltd.
4** Contact: https://www.qt.io/licensing/
5**
6** This file is part of the QtXmlPatterns module of the Qt Toolkit.
7**
8** $QT_BEGIN_LICENSE:LGPL$
9** Commercial License Usage
10** Licensees holding valid commercial Qt licenses may use this file in
11** accordance with the commercial license agreement provided with the
12** Software or, alternatively, in accordance with the terms contained in
13** a written agreement between you and The Qt Company. For licensing terms
14** and conditions see https://www.qt.io/terms-conditions. For further
15** information use the contact form at https://www.qt.io/contact-us.
16**
17** GNU Lesser General Public License Usage
18** Alternatively, this file may be used under the terms of the GNU Lesser
19** General Public License version 3 as published by the Free Software
20** Foundation and appearing in the file LICENSE.LGPL3 included in the
21** packaging of this file. Please review the following information to
22** ensure the GNU Lesser General Public License version 3 requirements
23** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
24**
25** GNU General Public License Usage
26** Alternatively, this file may be used under the terms of the GNU
27** General Public License version 2.0 or (at your option) the GNU General
28** Public license version 3 or any later version approved by the KDE Free
29** Qt Foundation. The licenses are as published by the Free Software
30** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
31** included in the packaging of this file. Please review the following
32** information to ensure the GNU General Public License requirements will
33** be met: https://www.gnu.org/licenses/gpl-2.0.html and
34** https://www.gnu.org/licenses/gpl-3.0.html.
35**
36** $QT_END_LICENSE$
37**
38****************************************************************************/
39
40//
41// W A R N I N G
42// -------------
43//
44// This file is not part of the Qt API. It exists purely as an
45// implementation detail. This header file may change from version to
46// version without notice, or even be removed.
47//
48// We mean it.
49
50#ifndef Patternist_Tokenizer_H
51#define Patternist_Tokenizer_H
52
53#include <QPair>
54#include <QSharedData>
55#include <QString>
56#include <QUrl>
57
58#include <private/qparsercontext_p.h>
59#include <private/qtokensource_p.h>
60#include <private/qtokenvalue_p.h>
61
62/**
63 * Macro for the data type of semantic values; int by default.
64 * See section Data Types of Semantic Values.
65 */
66#define XPATHSTYPE QPatternist::TokenValue
67
68#include <private/qquerytransformparser_p.h> /* This inclusion must be after TokenValue. */
69
70QT_BEGIN_NAMESPACE
71
72namespace QPatternist
73{
74 /**
75 * @short Base class for all tokenizers.
76 *
77 * The main entry point is nextToken(), which ones calls to retrieve the stream
78 * of tokens this Tokenizer delivers.
79 *
80 * @see <a href="http://www.w3.org/TR/xquery-xpath-parsing/">Building a
81 * Tokenizer for XPath or XQuery</a>
82 * @author Frans Englich <frans.englich@nokia.com>
83 */
84 class Tokenizer : public TokenSource
85 {
86 public:
87 inline Tokenizer(const QUrl &queryU) : m_queryURI(queryU)
88 {
89 Q_ASSERT(queryU.isValid());
90 }
91
92 typedef QExplicitlySharedDataPointer<Tokenizer> Ptr;
93
94 /**
95 * Switches the Tokenizer to only do scanning, and returns complete
96 * strings for attribute value templates as opposed to the tokens for
97 * the contained expressions.
98 *
99 * The current position in the stream is returned. It can be used to
100 * later resume regular tokenization.
101 */
102 virtual int commenceScanOnly() = 0;
103
104 /**
105 * Resumes regular parsing from @p position. The tokenizer must be in
106 * the scan-only state, which the commenceScanOnly() call transists to.
107 *
108 * The tokenizer will return the token POSITION_SET once after this
109 * function has been called.
110 */
111 virtual void resumeTokenizationFrom(const int position) = 0;
112
113 /**
114 * @returns the URI of the resource being tokenized.
115 */
116 inline const QUrl &queryURI() const
117 {
118 return m_queryURI;
119 }
120
121 virtual void setParserContext(const ParserContext::Ptr &parseInfo) = 0;
122
123 protected:
124 /**
125 * Returns a string representation of @p token.
126 *
127 * This function is used for debugging purposes. The implementation of
128 * this function is in querytransformparser.ypp.
129 */
130 static QString tokenToString(const Token &token);
131
132 private:
133 Q_DISABLE_COPY(Tokenizer)
134 const QUrl m_queryURI;
135 };
136
137}
138
139#undef Patternist_DEBUG_PARSER // disable it for now
140
141QT_END_NAMESPACE
142
143#endif
144

source code of qtxmlpatterns/src/xmlpatterns/parser/qtokenizer_p.h