1// Copyright (c) 2001-2011 Hartmut Kaiser
2//
3// Distributed under the Boost Software License, Version 1.0. (See accompanying
4// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
5
6#if !defined(BOOST_SPIRIT_LEX_TOKEN_FEB_10_2008_0751PM)
7#define BOOST_SPIRIT_LEX_TOKEN_FEB_10_2008_0751PM
8
9#if defined(_MSC_VER)
10#pragma once
11#endif
12
13#include <boost/config.hpp>
14#include <boost/detail/workaround.hpp>
15#include <boost/spirit/home/qi/detail/assign_to.hpp>
16#include <boost/spirit/home/support/attributes.hpp>
17#include <boost/spirit/home/support/argument.hpp>
18#include <boost/spirit/home/support/detail/lexer/generator.hpp>
19#include <boost/spirit/home/support/detail/lexer/rules.hpp>
20#include <boost/spirit/home/support/detail/lexer/consts.hpp>
21#include <boost/spirit/home/support/utree/utree_traits_fwd.hpp>
22#include <boost/spirit/home/lex/lexer/terminals.hpp>
23#include <boost/fusion/include/vector.hpp>
24#include <boost/fusion/include/at.hpp>
25#include <boost/fusion/include/value_at.hpp>
26#include <boost/variant.hpp>
27#include <boost/mpl/bool.hpp>
28#include <boost/mpl/vector.hpp>
29#include <boost/mpl/is_sequence.hpp>
30#include <boost/mpl/begin.hpp>
31#include <boost/mpl/insert.hpp>
32#include <boost/mpl/vector.hpp>
33#include <boost/mpl/if.hpp>
34#include <boost/mpl/or.hpp>
35#include <boost/range/iterator_range_core.hpp>
36#include <boost/type_traits/is_same.hpp>
37#include <boost/type_traits/make_unsigned.hpp>
38#include <boost/static_assert.hpp>
39
40#if defined(BOOST_SPIRIT_DEBUG)
41#include <iosfwd>
42#endif
43
44namespace boost { namespace spirit { namespace lex { namespace lexertl
45{
46 ///////////////////////////////////////////////////////////////////////////
47 //
48 // The token is the type of the objects returned by default by the
49 // iterator.
50 //
51 // template parameters:
52 // Iterator The type of the iterator used to access the
53 // underlying character stream.
54 // AttributeTypes A mpl sequence containing the types of all
55 // required different token values to be supported
56 // by this token type.
57 // HasState A mpl::bool_ indicating, whether this token type
58 // should support lexer states.
59 // Idtype The type to use for the token id (defaults to
60 // std::size_t).
61 //
62 // It is possible to use other token types with the spirit::lex
63 // framework as well. If you plan to use a different type as your token
64 // type, you'll need to expose the following things from your token type
65 // to make it compatible with spirit::lex:
66 //
67 // typedefs
68 // iterator_type The type of the iterator used to access the
69 // underlying character stream.
70 //
71 // id_type The type of the token id used.
72 //
73 // methods
74 // default constructor
75 // This should initialize the token as an end of
76 // input token.
77 // constructors The prototype of the other required
78 // constructors should be:
79 //
80 // token(int)
81 // This constructor should initialize the token as
82 // an invalid token (not carrying any specific
83 // values)
84 //
85 // where: the int is used as a tag only and its value is
86 // ignored
87 //
88 // and:
89 //
90 // token(Idtype id, std::size_t state,
91 // iterator_type first, iterator_type last);
92 //
93 // where: id: token id
94 // state: lexer state this token was matched in
95 // first, last: pair of iterators marking the matched
96 // range in the underlying input stream
97 //
98 // accessors
99 // id() return the token id of the matched input sequence
100 // id(newid) set the token id of the token instance
101 //
102 // state() return the lexer state this token was matched in
103 //
104 // value() return the token value
105 //
106 // Additionally, you will have to implement a couple of helper functions
107 // in the same namespace as the token type: a comparison operator==() to
108 // compare your token instances, a token_is_valid() function and different
109 // specializations of the Spirit customization point
110 // assign_to_attribute_from_value as shown below.
111 //
112 ///////////////////////////////////////////////////////////////////////////
113 template <typename Iterator = char const*
114 , typename AttributeTypes = mpl::vector0<>
115 , typename HasState = mpl::true_
116 , typename Idtype = std::size_t>
117 struct token;
118
119 ///////////////////////////////////////////////////////////////////////////
120 // This specialization of the token type doesn't contain any item data and
121 // doesn't support working with lexer states.
122 ///////////////////////////////////////////////////////////////////////////
123 template <typename Iterator, typename Idtype>
124 struct token<Iterator, lex::omit, mpl::false_, Idtype>
125 {
126 typedef Iterator iterator_type;
127 typedef mpl::false_ has_state;
128 typedef Idtype id_type;
129 typedef unused_type token_value_type;
130 typedef typename make_unsigned<id_type>::type uid_type;
131
132 // default constructed tokens correspond to EOI tokens
133 token() : id_(boost::lexer::npos) {}
134
135 // construct an invalid token
136 explicit token(int) : id_(0) {}
137
138 token(id_type id, std::size_t) : id_(id) {}
139
140 token(id_type id, std::size_t, token_value_type)
141 : id_(id) {}
142
143 token_value_type& value() { static token_value_type u; return u; }
144 token_value_type const& value() const { return unused; }
145
146#if defined(BOOST_SPIRIT_DEBUG)
147 token(id_type id, std::size_t, Iterator const& first
148 , Iterator const& last)
149 : matched_(first, last)
150 , id_(id) {}
151#else
152 token(id_type id, std::size_t, Iterator const&, Iterator const&)
153 : id_(id) {}
154#endif
155
156 // this default conversion operator is needed to allow the direct
157 // usage of tokens in conjunction with the primitive parsers defined
158 // in Qi
159 operator id_type() const { return id_type(uid_type(id_)); }
160
161 // Retrieve or set the token id of this token instance.
162 id_type id() const { return id_type(uid_type(id_)); }
163 void id(id_type newid) { id_ = uid_type(newid); }
164
165 std::size_t state() const { return 0; } // always '0' (INITIAL state)
166
167 bool is_valid() const
168 {
169 return 0 != id_ && boost::lexer::npos != id_;
170 }
171
172#if defined(BOOST_SPIRIT_DEBUG)
173#if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
174 // workaround for MSVC10 which has problems copying a default
175 // constructed iterator_range
176 token& operator= (token const& rhs)
177 {
178 if (this != &rhs)
179 {
180 id_ = rhs.id_;
181 if (is_valid())
182 matched_ = rhs.matched_;
183 }
184 return *this;
185 }
186#endif
187 std::pair<Iterator, Iterator> matched_;
188#endif
189
190 protected:
191 std::size_t id_; // token id, 0 if nothing has been matched
192 };
193
194#if defined(BOOST_SPIRIT_DEBUG)
195 template <typename Char, typename Traits, typename Iterator
196 , typename AttributeTypes, typename HasState, typename Idtype>
197 inline std::basic_ostream<Char, Traits>&
198 operator<< (std::basic_ostream<Char, Traits>& os
199 , token<Iterator, AttributeTypes, HasState, Idtype> const& t)
200 {
201 if (t.is_valid()) {
202 Iterator end = t.matched_.second;
203 for (Iterator it = t.matched_.first; it != end; ++it)
204 os << *it;
205 }
206 else {
207 os << "<invalid token>";
208 }
209 return os;
210 }
211#endif
212
213 ///////////////////////////////////////////////////////////////////////////
214 // This specialization of the token type doesn't contain any item data but
215 // supports working with lexer states.
216 ///////////////////////////////////////////////////////////////////////////
217 template <typename Iterator, typename Idtype>
218 struct token<Iterator, lex::omit, mpl::true_, Idtype>
219 : token<Iterator, lex::omit, mpl::false_, Idtype>
220 {
221 private:
222 typedef token<Iterator, lex::omit, mpl::false_, Idtype> base_type;
223
224 public:
225 typedef typename base_type::id_type id_type;
226 typedef Iterator iterator_type;
227 typedef mpl::true_ has_state;
228 typedef unused_type token_value_type;
229
230 // default constructed tokens correspond to EOI tokens
231 token() : state_(boost::lexer::npos) {}
232
233 // construct an invalid token
234 explicit token(int) : base_type(0), state_(boost::lexer::npos) {}
235
236 token(id_type id, std::size_t state)
237 : base_type(id, boost::lexer::npos), state_(state) {}
238
239 token(id_type id, std::size_t state, token_value_type)
240 : base_type(id, boost::lexer::npos, unused)
241 , state_(state) {}
242
243 token(id_type id, std::size_t state
244 , Iterator const& first, Iterator const& last)
245 : base_type(id, boost::lexer::npos, first, last)
246 , state_(state) {}
247
248 std::size_t state() const { return state_; }
249
250#if defined(BOOST_SPIRIT_DEBUG) && BOOST_WORKAROUND(BOOST_MSVC, == 1600)
251 // workaround for MSVC10 which has problems copying a default
252 // constructed iterator_range
253 token& operator= (token const& rhs)
254 {
255 if (this != &rhs)
256 {
257 this->base_type::operator=(static_cast<base_type const&>(rhs));
258 state_ = rhs.state_;
259 }
260 return *this;
261 }
262#endif
263
264 protected:
265 std::size_t state_; // lexer state this token was matched in
266 };
267
268 ///////////////////////////////////////////////////////////////////////////
269 // The generic version of the token type derives from the
270 // specialization above and adds a single data member holding the item
271 // data carried by the token instance.
272 ///////////////////////////////////////////////////////////////////////////
273 namespace detail
274 {
275 ///////////////////////////////////////////////////////////////////////
276 // Meta-function to calculate the type of the variant data item to be
277 // stored with each token instance.
278 //
279 // Note: The iterator pair needs to be the first type in the list of
280 // types supported by the generated variant type (this is being
281 // used to identify whether the stored data item in a particular
282 // token instance needs to be converted from the pair of
283 // iterators (see the first of the assign_to_attribute_from_value
284 // specializations below).
285 ///////////////////////////////////////////////////////////////////////
286 template <typename IteratorPair, typename AttributeTypes>
287 struct token_value_typesequence
288 {
289 typedef typename mpl::insert<
290 AttributeTypes
291 , typename mpl::begin<AttributeTypes>::type
292 , IteratorPair
293 >::type sequence_type;
294 typedef typename make_variant_over<sequence_type>::type type;
295 };
296
297 ///////////////////////////////////////////////////////////////////////
298 // The type of the data item stored with a token instance is defined
299 // by the template parameter 'AttributeTypes' and may be:
300 //
301 // lex::omit: no data item is stored with the token
302 // instance (this is handled by the
303 // specializations of the token class
304 // below)
305 // mpl::vector0<>: each token instance stores a pair of
306 // iterators pointing to the matched input
307 // sequence
308 // mpl::vector<...>: each token instance stores a variant being
309 // able to store the pair of iterators pointing
310 // to the matched input sequence, or any of the
311 // types a specified in the mpl::vector<>
312 //
313 // All this is done to ensure the token type is as small (in terms
314 // of its byte-size) as possible.
315 ///////////////////////////////////////////////////////////////////////
316 template <typename IteratorPair, typename AttributeTypes>
317 struct token_value_type
318 : mpl::eval_if<
319 mpl::or_<
320 is_same<AttributeTypes, mpl::vector0<> >
321 , is_same<AttributeTypes, mpl::vector<> > >
322 , mpl::identity<IteratorPair>
323 , token_value_typesequence<IteratorPair, AttributeTypes> >
324 {};
325 }
326
327 template <typename Iterator, typename AttributeTypes, typename HasState
328 , typename Idtype>
329 struct token : token<Iterator, lex::omit, HasState, Idtype>
330 {
331 private: // precondition assertions
332 BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value ||
333 is_same<AttributeTypes, lex::omit>::value));
334 typedef token<Iterator, lex::omit, HasState, Idtype> base_type;
335
336 protected:
337 // If no additional token value types are given, the token will
338 // hold the plain pair of iterators pointing to the matched range
339 // in the underlying input sequence. Otherwise the token value is
340 // stored as a variant and will again hold the pair of iterators but
341 // is able to hold any of the given data types as well. The conversion
342 // from the iterator pair to the required data type is done when it is
343 // accessed for the first time.
344 typedef iterator_range<Iterator> iterpair_type;
345
346 public:
347 typedef typename base_type::id_type id_type;
348 typedef typename detail::token_value_type<
349 iterpair_type, AttributeTypes
350 >::type token_value_type;
351
352 typedef Iterator iterator_type;
353
354 // default constructed tokens correspond to EOI tokens
355 token() : value_(iterpair_type(iterator_type(), iterator_type())) {}
356
357 // construct an invalid token
358 explicit token(int)
359 : base_type(0)
360 , value_(iterpair_type(iterator_type(), iterator_type())) {}
361
362 token(id_type id, std::size_t state, token_value_type const& value)
363 : base_type(id, state, value)
364 , value_(value) {}
365
366 token(id_type id, std::size_t state, Iterator const& first
367 , Iterator const& last)
368 : base_type(id, state, first, last)
369 , value_(iterpair_type(first, last)) {}
370
371 token_value_type& value() { return value_; }
372 token_value_type const& value() const { return value_; }
373
374#if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
375 // workaround for MSVC10 which has problems copying a default
376 // constructed iterator_range
377 token& operator= (token const& rhs)
378 {
379 if (this != &rhs)
380 {
381 this->base_type::operator=(static_cast<base_type const&>(rhs));
382 if (this->is_valid())
383 value_ = rhs.value_;
384 }
385 return *this;
386 }
387#endif
388
389 protected:
390 token_value_type value_; // token value, by default a pair of iterators
391 };
392
393 ///////////////////////////////////////////////////////////////////////////
394 // tokens are considered equal, if their id's match (these are unique)
395 template <typename Iterator, typename AttributeTypes, typename HasState
396 , typename Idtype>
397 inline bool
398 operator== (token<Iterator, AttributeTypes, HasState, Idtype> const& lhs,
399 token<Iterator, AttributeTypes, HasState, Idtype> const& rhs)
400 {
401 return lhs.id() == rhs.id();
402 }
403
404 ///////////////////////////////////////////////////////////////////////////
405 // This overload is needed by the multi_pass/functor_input_policy to
406 // validate a token instance. It has to be defined in the same namespace
407 // as the token class itself to allow ADL to find it.
408 ///////////////////////////////////////////////////////////////////////////
409 template <typename Iterator, typename AttributeTypes, typename HasState
410 , typename Idtype>
411 inline bool
412 token_is_valid(token<Iterator, AttributeTypes, HasState, Idtype> const& t)
413 {
414 return t.is_valid();
415 }
416}}}}
417
418namespace boost { namespace spirit { namespace traits
419{
420 ///////////////////////////////////////////////////////////////////////////
421 // We have to provide specializations for the customization point
422 // assign_to_attribute_from_value allowing to extract the needed value
423 // from the token.
424 ///////////////////////////////////////////////////////////////////////////
425
426 // This is called from the parse function of token_def if the token_def
427 // has been defined to carry a special attribute type
428 template <typename Attribute, typename Iterator, typename AttributeTypes
429 , typename HasState, typename Idtype>
430 struct assign_to_attribute_from_value<Attribute
431 , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
432 {
433 static void
434 call(lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> const& t
435 , Attribute& attr)
436 {
437 // The goal of this function is to avoid the conversion of the pair of
438 // iterators (to the matched character sequence) into the token value
439 // of the required type being done more than once. For this purpose it
440 // checks whether the stored value type is still the default one (pair
441 // of iterators) and if yes, replaces the pair of iterators with the
442 // converted value to be returned from subsequent calls.
443
444 if (0 == t.value().which()) {
445 // first access to the token value
446 typedef iterator_range<Iterator> iterpair_type;
447 iterpair_type const& ip = boost::get<iterpair_type>(t.value());
448
449 // Interestingly enough we use the assign_to() framework defined in
450 // Spirit.Qi allowing to convert the pair of iterators to almost any
451 // required type (assign_to(), if available, uses the standard Spirit
452 // parsers to do the conversion).
453 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
454
455 // If you get an error during the compilation of the following
456 // assignment expression, you probably forgot to list one or more
457 // types used as token value types (in your token_def<...>
458 // definitions) in your definition of the token class. I.e. any token
459 // value type used for a token_def<...> definition has to be listed
460 // during the declaration of the token type to use. For instance let's
461 // assume we have two token_def's:
462 //
463 // token_def<int> number; number = "...";
464 // token_def<std::string> identifier; identifier = "...";
465 //
466 // Then you'll have to use the following token type definition
467 // (assuming you are using the token class):
468 //
469 // typedef mpl::vector<int, std::string> token_values;
470 // typedef token<base_iter_type, token_values> token_type;
471 //
472 // where: base_iter_type is the iterator type used to expose the
473 // underlying input stream.
474 //
475 // This token_type has to be used as the second template parameter
476 // to the lexer class:
477 //
478 // typedef lexer<base_iter_type, token_type> lexer_type;
479 //
480 // again, assuming you're using the lexer<> template for your
481 // tokenization.
482
483 typedef lex::lexertl::token<
484 Iterator, AttributeTypes, HasState, Idtype> token_type;
485 spirit::traits::assign_to(
486 attr, const_cast<token_type&>(t).value()); // re-assign value
487 }
488 else {
489 // reuse the already assigned value
490 spirit::traits::assign_to(boost::get<Attribute>(t.value()), attr);
491 }
492 }
493 };
494
495 template <typename Attribute, typename Iterator, typename AttributeTypes
496 , typename HasState, typename Idtype>
497 struct assign_to_container_from_value<Attribute
498 , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
499 : assign_to_attribute_from_value<Attribute
500 , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
501 {};
502
503 template <typename Iterator, typename AttributeTypes
504 , typename HasState, typename Idtype>
505 struct assign_to_container_from_value<utree
506 , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
507 : assign_to_attribute_from_value<utree
508 , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
509 {};
510
511 template <typename Iterator>
512 struct assign_to_container_from_value<
513 iterator_range<Iterator>, iterator_range<Iterator> >
514 {
515 static void
516 call(iterator_range<Iterator> const& val, iterator_range<Iterator>& attr)
517 {
518 attr = val;
519 }
520 };
521
522 // These are called from the parse function of token_def if the token type
523 // has no special attribute type assigned
524 template <typename Attribute, typename Iterator, typename HasState
525 , typename Idtype>
526 struct assign_to_attribute_from_value<Attribute
527 , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> >
528 {
529 static void
530 call(lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> const& t
531 , Attribute& attr)
532 {
533 // The default type returned by the token_def parser component (if
534 // it has no token value type assigned) is the pair of iterators
535 // to the matched character sequence.
536 spirit::traits::assign_to(t.value().begin(), t.value().end(), attr);
537 }
538 };
539
540// template <typename Attribute, typename Iterator, typename HasState
541// , typename Idtype>
542// struct assign_to_container_from_value<Attribute
543// , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> >
544// : assign_to_attribute_from_value<Attribute
545// , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> >
546// {};
547
548 // same as above but using mpl::vector<> instead of mpl::vector0<>
549 template <typename Attribute, typename Iterator, typename HasState
550 , typename Idtype>
551 struct assign_to_attribute_from_value<Attribute
552 , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> >
553 {
554 static void
555 call(lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> const& t
556 , Attribute& attr)
557 {
558 // The default type returned by the token_def parser component (if
559 // it has no token value type assigned) is the pair of iterators
560 // to the matched character sequence.
561 spirit::traits::assign_to(t.value().begin(), t.value().end(), attr);
562 }
563 };
564
565// template <typename Attribute, typename Iterator, typename HasState
566// , typename Idtype>
567// struct assign_to_container_from_value<Attribute
568// , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> >
569// : assign_to_attribute_from_value<Attribute
570// , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> >
571// {};
572
573 // This is called from the parse function of token_def if the token type
574 // has been explicitly omitted (i.e. no attribute value is used), which
575 // essentially means that every attribute gets initialized using default
576 // constructed values.
577 template <typename Attribute, typename Iterator, typename HasState
578 , typename Idtype>
579 struct assign_to_attribute_from_value<Attribute
580 , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> >
581 {
582 static void
583 call(lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> const&
584 , Attribute&)
585 {
586 // do nothing
587 }
588 };
589
590 template <typename Attribute, typename Iterator, typename HasState
591 , typename Idtype>
592 struct assign_to_container_from_value<Attribute
593 , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> >
594 : assign_to_attribute_from_value<Attribute
595 , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> >
596 {};
597
598 // This is called from the parse function of lexer_def_
599 template <typename Iterator, typename AttributeTypes, typename HasState
600 , typename Idtype_, typename Idtype>
601 struct assign_to_attribute_from_value<
602 fusion::vector2<Idtype_, iterator_range<Iterator> >
603 , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
604 {
605 static void
606 call(lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> const& t
607 , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr)
608 {
609 // The type returned by the lexer_def_ parser components is a
610 // fusion::vector containing the token id of the matched token
611 // and the pair of iterators to the matched character sequence.
612 typedef iterator_range<Iterator> iterpair_type;
613 typedef fusion::vector2<Idtype_, iterator_range<Iterator> >
614 attribute_type;
615
616 iterpair_type const& ip = boost::get<iterpair_type>(t.value());
617 attr = attribute_type(t.id(), ip);
618 }
619 };
620
621 template <typename Iterator, typename AttributeTypes, typename HasState
622 , typename Idtype_, typename Idtype>
623 struct assign_to_container_from_value<
624 fusion::vector2<Idtype_, iterator_range<Iterator> >
625 , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
626 : assign_to_attribute_from_value<
627 fusion::vector2<Idtype_, iterator_range<Iterator> >
628 , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> >
629 {};
630
631 ///////////////////////////////////////////////////////////////////////////
632 // Overload debug output for a single token, this integrates lexer tokens
633 // with Qi's simple_trace debug facilities
634 template <typename Iterator, typename Attribute, typename HasState
635 , typename Idtype>
636 struct token_printer_debug<
637 lex::lexertl::token<Iterator, Attribute, HasState, Idtype> >
638 {
639 typedef lex::lexertl::token<Iterator, Attribute, HasState, Idtype> token_type;
640
641 template <typename Out>
642 static void print(Out& out, token_type const& val)
643 {
644 out << '[';
645 spirit::traits::print_token(out, val.value());
646 out << ']';
647 }
648 };
649}}}
650
651#endif
652

source code of boost/libs/spirit/include/boost/spirit/home/lex/lexer/lexertl/token.hpp