1// Copyright (c) 2001-2011 Hartmut Kaiser
2//
3// Distributed under the Boost Software License, Version 1.0. (See accompanying
4// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
5
6#if !defined(BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM)
7#define BOOST_SPIRIT_LEX_POSITION_TOKEN_MAY_13_2011_0846PM
8
9#if defined(_MSC_VER)
10#pragma once
11#endif
12
13#include <boost/config.hpp>
14#include <boost/detail/workaround.hpp>
15#include <boost/spirit/home/qi/detail/assign_to.hpp>
16#include <boost/spirit/home/support/attributes.hpp>
17#include <boost/spirit/home/support/argument.hpp>
18#include <boost/spirit/home/support/detail/lexer/generator.hpp>
19#include <boost/spirit/home/support/detail/lexer/rules.hpp>
20#include <boost/spirit/home/support/detail/lexer/consts.hpp>
21#include <boost/spirit/home/support/utree/utree_traits_fwd.hpp>
22#include <boost/spirit/home/lex/lexer/terminals.hpp>
23#include <boost/fusion/include/vector.hpp>
24#include <boost/fusion/include/at.hpp>
25#include <boost/fusion/include/value_at.hpp>
26#include <boost/variant.hpp>
27#include <boost/mpl/vector.hpp>
28#include <boost/mpl/bool.hpp>
29#include <boost/mpl/is_sequence.hpp>
30#include <boost/mpl/begin.hpp>
31#include <boost/mpl/insert.hpp>
32#include <boost/mpl/vector.hpp>
33#include <boost/mpl/if.hpp>
34#include <boost/mpl/or.hpp>
35#include <boost/type_traits/is_same.hpp>
36#include <boost/range/iterator_range_core.hpp>
37#include <boost/static_assert.hpp>
38
39#if defined(BOOST_SPIRIT_DEBUG)
40#include <iosfwd>
41#endif
42
43namespace boost { namespace spirit { namespace lex { namespace lexertl
44{
45 ///////////////////////////////////////////////////////////////////////////
46 //
47 // The position_token is the type of the objects returned by the
48 // iterator if it has been specified while instantiating the lexer object.
49 //
50 // template parameters:
51 // Iterator The type of the iterator used to access the
52 // underlying character stream.
53 // AttributeTypes A mpl sequence containing the types of all
54 // required different token values to be supported
55 // by this token type.
56 // HasState A mpl::bool_ indicating, whether this token type
57 // should support lexer states.
58 // Idtype The type to use for the token id (defaults to
59 // std::size_t).
60 //
61 // It is possible to use other token types with the spirit::lex
62 // framework as well. If you plan to use a different type as your token
63 // type, you'll need to expose the following things from your token type
64 // to make it compatible with spirit::lex:
65 //
66 // typedefs
67 // iterator_type The type of the iterator used to access the
68 // underlying character stream.
69 //
70 // id_type The type of the token id used.
71 //
72 // methods
73 // default constructor
74 // This should initialize the token as an end of
75 // input token.
76 // constructors The prototype of the other required
77 // constructors should be:
78 //
79 // token(int)
80 // This constructor should initialize the token as
81 // an invalid token (not carrying any specific
82 // values)
83 //
84 // where: the int is used as a tag only and its value is
85 // ignored
86 //
87 // and:
88 //
89 // token(Idtype id, std::size_t state,
90 // iterator_type first, iterator_type last);
91 //
92 // where: id: token id
93 // state: lexer state this token was matched in
94 // first, last: pair of iterators marking the matched
95 // range in the underlying input stream
96 //
97 // accessors
98 // id() return the token id of the matched input sequence
99 // id(newid) set the token id of the token instance
100 //
101 // state() return the lexer state this token was matched in
102 //
103 // value() return the token value
104 //
105 // Additionally, you will have to implement a couple of helper functions
106 // in the same namespace as the token type: a comparison operator==() to
107 // compare your token instances, a token_is_valid() function and different
108 // specializations of the Spirit customization point
109 // assign_to_attribute_from_value as shown below.
110 //
111 ///////////////////////////////////////////////////////////////////////////
112 template <typename Iterator = char const*
113 , typename AttributeTypes = mpl::vector0<>
114 , typename HasState = mpl::true_
115 , typename Idtype = std::size_t>
116 struct position_token;
117
118 ///////////////////////////////////////////////////////////////////////////
119 // This specialization of the token type doesn't contain any item data and
120 // doesn't support working with lexer states. Although, like all other
121 // variants of position_token, it carries a pair of iterators marking the
122 // begin and the end of the matched character sequence.
123 ///////////////////////////////////////////////////////////////////////////
124 template <typename Iterator, typename Idtype>
125 struct position_token<Iterator, lex::omit, mpl::false_, Idtype>
126 {
127 typedef Iterator iterator_type;
128 typedef iterator_range<iterator_type> iterpair_type;
129 typedef mpl::false_ has_state;
130 typedef Idtype id_type;
131 typedef unused_type token_value_type;
132
133 // default constructed tokens correspond to EOI tokens
134 position_token()
135 : id_(id_type(boost::lexer::npos)) {}
136
137 // construct an invalid token
138 explicit position_token(int)
139 : id_(id_type(0)) {}
140
141 position_token(id_type id, std::size_t)
142 : id_(id) {}
143
144 position_token(id_type id, std::size_t, token_value_type)
145 : id_(id) {}
146
147 position_token(id_type id, std::size_t, Iterator const& first
148 , Iterator const& last)
149 : id_(id), matched_(first, last) {}
150
151 // this default conversion operator is needed to allow the direct
152 // usage of tokens in conjunction with the primitive parsers defined
153 // in Qi
154 operator id_type() const { return id_; }
155
156 // Retrieve or set the token id of this token instance.
157 id_type id() const { return id_; }
158 void id(id_type newid) { id_ = newid; }
159
160 std::size_t state() const { return 0; } // always '0' (INITIAL state)
161
162 bool is_valid() const
163 {
164 return 0 != id_ && id_type(boost::lexer::npos) != id_;
165 }
166
167 // access the stored iterator range of the matched input sequence
168 iterator_type begin() const { return matched_.begin(); }
169 iterator_type end() const { return matched_.end(); }
170
171 iterpair_type& matched() { return matched_; }
172 iterpair_type const& matched() const { return matched_; }
173
174 token_value_type& value() { static token_value_type u; return u; }
175 token_value_type const& value() const { return unused; }
176
177#if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
178 // workaround for MSVC10 which has problems copying a default
179 // constructed iterator_range
180 position_token& operator= (position_token const& rhs)
181 {
182 if (this != &rhs)
183 {
184 id_ = rhs.id_;
185 if (is_valid())
186 matched_ = rhs.matched_;
187 }
188 return *this;
189 }
190#endif
191
192 protected:
193 id_type id_; // token id, 0 if nothing has been matched
194 iterpair_type matched_; // matched input sequence
195 };
196
197#if defined(BOOST_SPIRIT_DEBUG)
198 template <typename Char, typename Traits, typename Iterator
199 , typename AttributeTypes, typename HasState, typename Idtype>
200 inline std::basic_ostream<Char, Traits>&
201 operator<< (std::basic_ostream<Char, Traits>& os
202 , position_token<Iterator, AttributeTypes, HasState, Idtype> const& t)
203 {
204 if (t.is_valid()) {
205 Iterator end = t.end();
206 for (Iterator it = t.begin(); it != end; ++it)
207 os << *it;
208 }
209 else {
210 os << "<invalid token>";
211 }
212 return os;
213 }
214#endif
215
216 ///////////////////////////////////////////////////////////////////////////
217 // This specialization of the token type doesn't contain any item data but
218 // supports working with lexer states.
219 ///////////////////////////////////////////////////////////////////////////
220 template <typename Iterator, typename Idtype>
221 struct position_token<Iterator, lex::omit, mpl::true_, Idtype>
222 : position_token<Iterator, lex::omit, mpl::false_, Idtype>
223 {
224 private:
225 typedef position_token<Iterator, lex::omit, mpl::false_, Idtype>
226 base_type;
227
228 public:
229 typedef typename base_type::id_type id_type;
230 typedef Iterator iterator_type;
231 typedef mpl::true_ has_state;
232 typedef unused_type token_value_type;
233
234 // default constructed tokens correspond to EOI tokens
235 position_token() : state_(boost::lexer::npos) {}
236
237 // construct an invalid token
238 explicit position_token(int)
239 : base_type(0), state_(boost::lexer::npos) {}
240
241 position_token(id_type id, std::size_t state)
242 : base_type(id, boost::lexer::npos), state_(state) {}
243
244 position_token(id_type id, std::size_t state, token_value_type)
245 : base_type(id, boost::lexer::npos, unused)
246 , state_(state) {}
247
248 position_token(id_type id, std::size_t state
249 , Iterator const& first, Iterator const& last)
250 : base_type(id, boost::lexer::npos, first, last)
251 , state_(state) {}
252
253 std::size_t state() const { return state_; }
254
255#if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
256 // workaround for MSVC10 which has problems copying a default
257 // constructed iterator_range
258 position_token& operator= (position_token const& rhs)
259 {
260 if (this != &rhs)
261 {
262 this->base_type::operator=(static_cast<base_type const&>(rhs));
263 state_ = rhs.state_;
264 }
265 return *this;
266 }
267#endif
268
269 protected:
270 std::size_t state_; // lexer state this token was matched in
271 };
272
273 ///////////////////////////////////////////////////////////////////////////
274 // These specializations for an empty attribute list cause all token
275 // instances to expose as it attribute the iterator_range pointing to the
276 // matched input sequence.
277 ///////////////////////////////////////////////////////////////////////////
278 template <typename Iterator, typename HasState, typename Idtype>
279 struct position_token<Iterator, mpl::vector<>, HasState, Idtype>
280 : position_token<Iterator, lex::omit, HasState, Idtype>
281 {
282 private:
283 typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
284
285 public:
286 typedef typename base_type::id_type id_type;
287 typedef typename base_type::iterator_type iterator_type;
288 typedef typename base_type::iterpair_type iterpair_type;
289 typedef HasState has_state;
290 typedef iterpair_type token_value_type;
291
292 // default constructed tokens correspond to EOI tokens
293 position_token() {}
294
295 // construct an invalid token
296 explicit position_token(int)
297 : base_type(0) {}
298
299 position_token(id_type id, std::size_t state)
300 : base_type(id, state) {}
301
302 position_token(id_type id, std::size_t state, token_value_type)
303 : base_type(id, state, unused) {}
304
305 position_token(id_type id, std::size_t state
306 , Iterator const& first, Iterator const& last)
307 : base_type(id, state, first, last) {}
308
309 token_value_type& value() { return this->base_type::matched(); }
310 token_value_type const& value() const { return this->base_type::matched(); }
311 };
312
313 template <typename Iterator, typename HasState, typename Idtype>
314 struct position_token<Iterator, mpl::vector0<>, HasState, Idtype>
315 : position_token<Iterator, lex::omit, HasState, Idtype>
316 {
317 private:
318 typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
319
320 public:
321 typedef typename base_type::id_type id_type;
322 typedef typename base_type::iterator_type iterator_type;
323 typedef typename base_type::iterpair_type iterpair_type;
324 typedef HasState has_state;
325 typedef iterpair_type token_value_type;
326
327 // default constructed tokens correspond to EOI tokens
328 position_token() {}
329
330 // construct an invalid token
331 explicit position_token(int)
332 : base_type(0) {}
333
334 position_token(id_type id, std::size_t state)
335 : base_type(id, state) {}
336
337 position_token(id_type id, std::size_t state, token_value_type)
338 : base_type(id, state, unused) {}
339
340 position_token(id_type id, std::size_t state
341 , Iterator const& first, Iterator const& last)
342 : base_type(id, state, first, last) {}
343
344 token_value_type& value() { return this->base_type::matched(); }
345 token_value_type const& value() const { return this->base_type::matched(); }
346 };
347
348 ///////////////////////////////////////////////////////////////////////////
349 // These specializations for an attribute list of length one cause all token
350 // instances to expose the specified type as its attribute.
351 ///////////////////////////////////////////////////////////////////////////
352 template <typename Iterator, typename Attribute, typename HasState
353 , typename Idtype>
354 struct position_token<Iterator, mpl::vector<Attribute>, HasState, Idtype>
355 : position_token<Iterator, lex::omit, HasState, Idtype>
356 {
357 private:
358 typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
359
360 public:
361 typedef typename base_type::id_type id_type;
362 typedef typename base_type::iterator_type iterator_type;
363 typedef typename base_type::iterpair_type iterpair_type;
364 typedef HasState has_state;
365 typedef boost::optional<Attribute> token_value_type;
366
367 // default constructed tokens correspond to EOI tokens
368 position_token() {}
369
370 // construct an invalid token
371 explicit position_token(int)
372 : base_type(0) {}
373
374 position_token(id_type id, std::size_t state)
375 : base_type(id, state) {}
376
377 position_token(id_type id, std::size_t state, token_value_type const& v)
378 : base_type(id, state, unused), value_(v) {}
379
380 position_token(id_type id, std::size_t state
381 , Iterator const& first, Iterator const& last)
382 : base_type(id, state, first, last) {}
383
384 token_value_type& value() { return value_; }
385 token_value_type const& value() const { return value_; }
386
387 bool has_value() const { return !!value_; }
388
389#if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
390 // workaround for MSVC10 which has problems copying a default
391 // constructed iterator_range
392 position_token& operator= (position_token const& rhs)
393 {
394 if (this != &rhs)
395 {
396 this->base_type::operator=(static_cast<base_type const&>(rhs));
397 if (this->is_valid())
398 value_ = rhs.value_;
399 }
400 return *this;
401 }
402#endif
403
404 protected:
405 token_value_type value_; // token value
406 };
407
408 template <typename Iterator, typename Attribute, typename HasState
409 , typename Idtype>
410 struct position_token<Iterator, mpl::vector1<Attribute>, HasState, Idtype>
411 : position_token<Iterator, lex::omit, HasState, Idtype>
412 {
413 private:
414 typedef position_token<Iterator, lex::omit, HasState, Idtype> base_type;
415
416 public:
417 typedef typename base_type::id_type id_type;
418 typedef typename base_type::iterator_type iterator_type;
419 typedef typename base_type::iterpair_type iterpair_type;
420 typedef HasState has_state;
421 typedef boost::optional<Attribute> token_value_type;
422
423 // default constructed tokens correspond to EOI tokens
424 position_token() {}
425
426 // construct an invalid token
427 explicit position_token(int)
428 : base_type(0) {}
429
430 position_token(id_type id, std::size_t state)
431 : base_type(id, state) {}
432
433 position_token(id_type id, std::size_t state, token_value_type const& v)
434 : base_type(id, state, unused), value_(v) {}
435
436 position_token(id_type id, std::size_t state
437 , Iterator const& first, Iterator const& last)
438 : base_type(id, state, first, last) {}
439
440 token_value_type& value() { return value_; }
441 token_value_type const& value() const { return value_; }
442
443 bool has_value() const { return value_; }
444
445#if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
446 // workaround for MSVC10 which has problems copying a default
447 // constructed iterator_range
448 position_token& operator= (position_token const& rhs)
449 {
450 if (this != &rhs)
451 {
452 this->base_type::operator=(static_cast<base_type const&>(rhs));
453 if (this->is_valid())
454 value_ = rhs.value_;
455 }
456 return *this;
457 }
458#endif
459
460 protected:
461 token_value_type value_; // token value
462 };
463
464 ///////////////////////////////////////////////////////////////////////////
465 // The generic version of the position_token type derives from the
466 // specialization above and adds a single data member holding the item
467 // data carried by the token instance.
468 ///////////////////////////////////////////////////////////////////////////
469 namespace detail
470 {
471 ///////////////////////////////////////////////////////////////////////
472 // Meta-function to calculate the type of the variant data item to be
473 // stored with each token instance.
474 //
475 // Note: The iterator pair needs to be the first type in the list of
476 // types supported by the generated variant type (this is being
477 // used to identify whether the stored data item in a particular
478 // token instance needs to be converted from the pair of
479 // iterators (see the first of the assign_to_attribute_from_value
480 // specializations below).
481 ///////////////////////////////////////////////////////////////////////
482 template <typename IteratorPair, typename AttributeTypes>
483 struct position_token_value_typesequence
484 {
485 typedef typename mpl::insert<
486 AttributeTypes
487 , typename mpl::begin<AttributeTypes>::type
488 , IteratorPair
489 >::type sequence_type;
490 typedef typename make_variant_over<sequence_type>::type type;
491 };
492
493 ///////////////////////////////////////////////////////////////////////
494 // The type of the data item stored with a token instance is defined
495 // by the template parameter 'AttributeTypes' and may be:
496 //
497 // lex::omit: no data item is stored with the token
498 // instance (this is handled by the
499 // specializations of the token class
500 // below)
501 // mpl::vector0<>: each token instance stores a pair of
502 // iterators pointing to the matched input
503 // sequence
504 // mpl::vector<...>: each token instance stores a variant being
505 // able to store the pair of iterators pointing
506 // to the matched input sequence, or any of the
507 // types a specified in the mpl::vector<>
508 //
509 // All this is done to ensure the token type is as small (in terms
510 // of its byte-size) as possible.
511 ///////////////////////////////////////////////////////////////////////
512 template <typename IteratorPair, typename AttributeTypes>
513 struct position_token_value
514 : mpl::eval_if<
515 mpl::or_<
516 is_same<AttributeTypes, mpl::vector0<> >
517 , is_same<AttributeTypes, mpl::vector<> > >
518 , mpl::identity<IteratorPair>
519 , position_token_value_typesequence<IteratorPair, AttributeTypes> >
520 {};
521 }
522
523 template <typename Iterator, typename AttributeTypes, typename HasState
524 , typename Idtype>
525 struct position_token
526 : position_token<Iterator, lex::omit, HasState, Idtype>
527 {
528 private: // precondition assertions
529 BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value ||
530 is_same<AttributeTypes, lex::omit>::value));
531 typedef position_token<Iterator, lex::omit, HasState, Idtype>
532 base_type;
533
534 protected:
535 // If no additional token value types are given, the token will
536 // hold no token value at all as the base class already has the
537 // iterator pair of the matched range in the underlying input sequence.
538 // Otherwise the token value is stored as a variant and will
539 // initially hold an unused_type but is able to hold any of
540 // the given data types as well. The conversion from the iterator pair
541 // to the required data type is done when it is accessed for the first
542 // time.
543 typedef iterator_range<Iterator> iterpair_type;
544
545 public:
546 typedef typename base_type::id_type id_type;
547 typedef typename detail::position_token_value<
548 iterpair_type, AttributeTypes>::type token_value_type;
549
550 typedef Iterator iterator_type;
551
552 // default constructed tokens correspond to EOI tokens
553 position_token() {}
554
555 // construct an invalid token
556 explicit position_token(int)
557 : base_type(0) {}
558
559 position_token(id_type id, std::size_t state, token_value_type const& value)
560 : base_type(id, state, value), value_(value) {}
561
562 position_token(id_type id, std::size_t state, Iterator const& first
563 , Iterator const& last)
564 : base_type(id, state, first, last)
565 , value_(iterpair_type(first, last))
566 {}
567
568 token_value_type& value() { return value_; }
569 token_value_type const& value() const { return value_; }
570
571#if BOOST_WORKAROUND(BOOST_MSVC, == 1600)
572 // workaround for MSVC10 which has problems copying a default
573 // constructed iterator_range
574 position_token& operator= (position_token const& rhs)
575 {
576 if (this != &rhs)
577 {
578 this->base_type::operator=(static_cast<base_type const&>(rhs));
579 if (this->is_valid())
580 value_ = rhs.value_;
581 }
582 return *this;
583 }
584#endif
585
586 protected:
587 token_value_type value_; // token value, by default a pair of iterators
588 };
589
590 ///////////////////////////////////////////////////////////////////////////
591 // tokens are considered equal, if their id's match (these are unique)
592 template <typename Iterator, typename AttributeTypes, typename HasState
593 , typename Idtype>
594 inline bool
595 operator== (position_token<Iterator, AttributeTypes, HasState, Idtype> const& lhs,
596 position_token<Iterator, AttributeTypes, HasState, Idtype> const& rhs)
597 {
598 return lhs.id() == rhs.id();
599 }
600
601 ///////////////////////////////////////////////////////////////////////////
602 // This overload is needed by the multi_pass/functor_input_policy to
603 // validate a token instance. It has to be defined in the same namespace
604 // as the token class itself to allow ADL to find it.
605 ///////////////////////////////////////////////////////////////////////////
606 template <typename Iterator, typename AttributeTypes, typename HasState
607 , typename Idtype>
608 inline bool
609 token_is_valid(position_token<Iterator, AttributeTypes, HasState, Idtype> const& t)
610 {
611 return t.is_valid();
612 }
613}}}}
614
615namespace boost { namespace spirit { namespace traits
616{
617 ///////////////////////////////////////////////////////////////////////////
618 // We have to provide specializations for the customization point
619 // assign_to_attribute_from_value allowing to extract the needed value
620 // from the token.
621 ///////////////////////////////////////////////////////////////////////////
622
623 // This is called from the parse function of token_def if the token_def
624 // has been defined to carry a special attribute type
625 template <typename Attribute, typename Iterator, typename AttributeTypes
626 , typename HasState, typename Idtype>
627 struct assign_to_attribute_from_value<Attribute
628 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
629 {
630 static void
631 call(lex::lexertl::position_token<
632 Iterator, AttributeTypes, HasState, Idtype> const& t
633 , Attribute& attr)
634 {
635 // The goal of this function is to avoid the conversion of the pair of
636 // iterators (to the matched character sequence) into the token value
637 // of the required type being done more than once. For this purpose it
638 // checks whether the stored value type is still the default one (pair
639 // of iterators) and if yes, replaces the pair of iterators with the
640 // converted value to be returned from subsequent calls.
641
642 if (0 == t.value().which()) {
643 // first access to the token value
644 typedef iterator_range<Iterator> iterpair_type;
645 iterpair_type const& ip = t.matched();
646
647 // Interestingly enough we use the assign_to() framework defined in
648 // Spirit.Qi allowing to convert the pair of iterators to almost any
649 // required type (assign_to(), if available, uses the standard Spirit
650 // parsers to do the conversion).
651 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
652
653 // If you get an error during the compilation of the following
654 // assignment expression, you probably forgot to list one or more
655 // types used as token value types (in your token_def<...>
656 // definitions) in your definition of the token class. I.e. any token
657 // value type used for a token_def<...> definition has to be listed
658 // during the declaration of the token type to use. For instance let's
659 // assume we have two token_def's:
660 //
661 // token_def<int> number; number = "...";
662 // token_def<std::string> identifier; identifier = "...";
663 //
664 // Then you'll have to use the following token type definition
665 // (assuming you are using the token class):
666 //
667 // typedef mpl::vector<int, std::string> token_values;
668 // typedef token<base_iter_type, token_values> token_type;
669 //
670 // where: base_iter_type is the iterator type used to expose the
671 // underlying input stream.
672 //
673 // This token_type has to be used as the second template parameter
674 // to the lexer class:
675 //
676 // typedef lexer<base_iter_type, token_type> lexer_type;
677 //
678 // again, assuming you're using the lexer<> template for your
679 // tokenization.
680
681 typedef lex::lexertl::position_token<
682 Iterator, AttributeTypes, HasState, Idtype> token_type;
683 spirit::traits::assign_to(
684 attr, const_cast<token_type&>(t).value()); // re-assign value
685 }
686 else {
687 // reuse the already assigned value
688 spirit::traits::assign_to(get<Attribute>(t.value()), attr);
689 }
690 }
691 };
692
693 template <typename Attribute, typename Iterator, typename AttributeTypes
694 , typename HasState, typename Idtype>
695 struct assign_to_container_from_value<Attribute
696 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
697 : assign_to_attribute_from_value<Attribute
698 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
699 {};
700
701 ///////////////////////////////////////////////////////////////////////////
702 // These are called from the parse function of token_def if the token type
703 // has no special attribute type assigned
704 template <typename Attribute, typename Iterator, typename HasState
705 , typename Idtype>
706 struct assign_to_attribute_from_value<Attribute
707 , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
708 {
709 static void
710 call(lex::lexertl::position_token<
711 Iterator, mpl::vector0<>, HasState, Idtype> const& t
712 , Attribute& attr)
713 {
714 // The default type returned by the token_def parser component (if
715 // it has no token value type assigned) is the pair of iterators
716 // to the matched character sequence.
717 spirit::traits::assign_to(t.begin(), t.end(), attr);
718 }
719 };
720
721// template <typename Attribute, typename Iterator, typename HasState
722// , typename Idtype>
723// struct assign_to_container_from_value<Attribute
724// , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
725// : assign_to_attribute_from_value<Attribute
726// , lex::lexertl::position_token<Iterator, mpl::vector0<>, HasState, Idtype> >
727// {};
728
729 // same as above but using mpl::vector<> instead of mpl::vector0<>
730 template <typename Attribute, typename Iterator, typename HasState
731 , typename Idtype>
732 struct assign_to_attribute_from_value<Attribute
733 , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
734 {
735 static void
736 call(lex::lexertl::position_token<
737 Iterator, mpl::vector<>, HasState, Idtype> const& t
738 , Attribute& attr)
739 {
740 // The default type returned by the token_def parser component (if
741 // it has no token value type assigned) is the pair of iterators
742 // to the matched character sequence.
743 spirit::traits::assign_to(t.begin(), t.end(), attr);
744 }
745 };
746
747// template <typename Attribute, typename Iterator, typename HasState
748// , typename Idtype>
749// struct assign_to_container_from_value<Attribute
750// , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
751// : assign_to_attribute_from_value<Attribute
752// , lex::lexertl::position_token<Iterator, mpl::vector<>, HasState, Idtype> >
753// {};
754
755 ///////////////////////////////////////////////////////////////////////////
756 // These are called from the parse function of token_def if the token type
757 // has no special attribute type assigned
758 template <typename Attribute, typename Iterator, typename Attr
759 , typename HasState, typename Idtype>
760 struct assign_to_attribute_from_value<Attribute
761 , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
762 {
763 static void
764 call(lex::lexertl::position_token<
765 Iterator, mpl::vector1<Attr>, HasState, Idtype> const& t
766 , Attribute& attr)
767 {
768 // The goal of this function is to avoid the conversion of the pair of
769 // iterators (to the matched character sequence) into the token value
770 // of the required type being done more than once.
771
772 if (!t.has_value()) {
773 // first access to the token value
774 typedef iterator_range<Iterator> iterpair_type;
775 iterpair_type const& ip = t.matched();
776
777 // Interestingly enough we use the assign_to() framework defined in
778 // Spirit.Qi allowing to convert the pair of iterators to almost any
779 // required type (assign_to(), if available, uses the standard Spirit
780 // parsers to do the conversion).
781 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
782
783 // Re-assign the attribute to the stored value
784 typedef lex::lexertl::position_token<
785 Iterator, mpl::vector1<Attr>, HasState, Idtype> token_type;
786 spirit::traits::assign_to(
787 attr, const_cast<token_type&>(t).value());
788 }
789 else {
790 // reuse the already assigned value
791 spirit::traits::assign_to(t.value(), attr);
792 }
793 }
794 };
795
796// template <typename Attribute, typename Iterator, typename Attr
797// , typename HasState, typename Idtype>
798// struct assign_to_container_from_value<Attribute
799// , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
800// : assign_to_attribute_from_value<Attribute
801// , lex::lexertl::position_token<Iterator, mpl::vector1<Attr>, HasState, Idtype> >
802// {};
803
804 // same as above but using mpl::vector<Attr> instead of mpl::vector1<Attr>
805 template <typename Attribute, typename Iterator, typename Attr
806 , typename HasState, typename Idtype>
807 struct assign_to_attribute_from_value<Attribute
808 , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
809 {
810 static void
811 call(lex::lexertl::position_token<
812 Iterator, mpl::vector<Attr>, HasState, Idtype> const& t
813 , Attribute& attr)
814 {
815 // The goal of this function is to avoid the conversion of the pair of
816 // iterators (to the matched character sequence) into the token value
817 // of the required type being done more than once.
818
819 if (!t.has_value()) {
820 // first access to the token value
821 typedef iterator_range<Iterator> iterpair_type;
822 iterpair_type const& ip = t.matched();
823
824 // Interestingly enough we use the assign_to() framework defined in
825 // Spirit.Qi allowing to convert the pair of iterators to almost any
826 // required type (assign_to(), if available, uses the standard Spirit
827 // parsers to do the conversion).
828 spirit::traits::assign_to(ip.begin(), ip.end(), attr);
829
830 // Re-assign the attribute to the stored value
831 typedef lex::lexertl::position_token<
832 Iterator, mpl::vector<Attr>, HasState, Idtype> token_type;
833 spirit::traits::assign_to(
834 attr, const_cast<token_type&>(t).value());
835 }
836 else {
837 // reuse the already assigned value
838 spirit::traits::assign_to(t.value(), attr);
839 }
840 }
841 };
842
843// template <typename Attribute, typename Iterator, typename Attr
844// , typename HasState, typename Idtype>
845// struct assign_to_container_from_value<Attribute
846// , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
847// : assign_to_attribute_from_value<Attribute
848// , lex::lexertl::position_token<Iterator, mpl::vector<Attr>, HasState, Idtype> >
849// {};
850
851 // This is called from the parse function of token_def if the token type
852 // has been explicitly omitted (i.e. no attribute value is used), which
853 // essentially means that every attribute gets initialized using default
854 // constructed values.
855 template <typename Attribute, typename Iterator, typename HasState
856 , typename Idtype>
857 struct assign_to_attribute_from_value<Attribute
858 , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
859 {
860 static void
861 call(lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> const&
862 , Attribute&)
863 {
864 // do nothing
865 }
866 };
867
868 template <typename Attribute, typename Iterator, typename HasState
869 , typename Idtype>
870 struct assign_to_container_from_value<Attribute
871 , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
872 : assign_to_attribute_from_value<Attribute
873 , lex::lexertl::position_token<Iterator, lex::omit, HasState, Idtype> >
874 {};
875
876 // This is called from the parse function of lexer_def_
877 template <typename Iterator, typename AttributeTypes, typename HasState
878 , typename Idtype_, typename Idtype>
879 struct assign_to_attribute_from_value<
880 fusion::vector2<Idtype_, iterator_range<Iterator> >
881 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
882 {
883 static void
884 call(lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> const& t
885 , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr)
886 {
887 // The type returned by the lexer_def_ parser components is a
888 // fusion::vector containing the token id of the matched token
889 // and the pair of iterators to the matched character sequence.
890 typedef iterator_range<Iterator> iterpair_type;
891 typedef fusion::vector2<Idtype_, iterator_range<Iterator> >
892 attribute_type;
893
894 iterpair_type const& ip = t.matched();
895 attr = attribute_type(t.id(), ip);
896 }
897 };
898
899 template <typename Iterator, typename AttributeTypes, typename HasState
900 , typename Idtype_, typename Idtype>
901 struct assign_to_container_from_value<
902 fusion::vector2<Idtype_, iterator_range<Iterator> >
903 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
904 : assign_to_attribute_from_value<
905 fusion::vector2<Idtype_, iterator_range<Iterator> >
906 , lex::lexertl::position_token<Iterator, AttributeTypes, HasState, Idtype> >
907 {};
908
909 ///////////////////////////////////////////////////////////////////////////
910 // Overload debug output for a single token, this integrates lexer tokens
911 // with Qi's simple_trace debug facilities
912 template <typename Iterator, typename Attribute, typename HasState
913 , typename Idtype>
914 struct token_printer_debug<
915 lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> >
916 {
917 typedef lex::lexertl::position_token<Iterator, Attribute, HasState, Idtype> token_type;
918
919 template <typename Out>
920 static void print(Out& out, token_type const& val)
921 {
922 out << '[';
923 spirit::traits::print_token(out, val.value());
924 out << ']';
925 }
926 };
927}}}
928
929#endif
930

source code of boost/libs/spirit/include/boost/spirit/home/lex/lexer/lexertl/position_token.hpp