1 | // Copyright (c) 2001-2011 Hartmut Kaiser |
2 | // |
3 | // Distributed under the Boost Software License, Version 1.0. (See accompanying |
4 | // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) |
5 | |
6 | #if !defined(BOOST_SPIRIT_LEX_TOKEN_FEB_10_2008_0751PM) |
7 | #define BOOST_SPIRIT_LEX_TOKEN_FEB_10_2008_0751PM |
8 | |
9 | #if defined(_MSC_VER) |
10 | #pragma once |
11 | #endif |
12 | |
13 | #include <boost/config.hpp> |
14 | #include <boost/detail/workaround.hpp> |
15 | #include <boost/spirit/home/qi/detail/assign_to.hpp> |
16 | #include <boost/spirit/home/support/attributes.hpp> |
17 | #include <boost/spirit/home/support/argument.hpp> |
18 | #include <boost/spirit/home/support/detail/lexer/generator.hpp> |
19 | #include <boost/spirit/home/support/detail/lexer/rules.hpp> |
20 | #include <boost/spirit/home/support/detail/lexer/consts.hpp> |
21 | #include <boost/spirit/home/support/utree/utree_traits_fwd.hpp> |
22 | #include <boost/spirit/home/lex/lexer/terminals.hpp> |
23 | #include <boost/fusion/include/vector.hpp> |
24 | #include <boost/fusion/include/at.hpp> |
25 | #include <boost/fusion/include/value_at.hpp> |
26 | #include <boost/variant.hpp> |
27 | #include <boost/mpl/bool.hpp> |
28 | #include <boost/mpl/vector.hpp> |
29 | #include <boost/mpl/is_sequence.hpp> |
30 | #include <boost/mpl/begin.hpp> |
31 | #include <boost/mpl/insert.hpp> |
32 | #include <boost/mpl/vector.hpp> |
33 | #include <boost/mpl/if.hpp> |
34 | #include <boost/mpl/or.hpp> |
35 | #include <boost/range/iterator_range_core.hpp> |
36 | #include <boost/type_traits/is_same.hpp> |
37 | #include <boost/type_traits/make_unsigned.hpp> |
38 | #include <boost/static_assert.hpp> |
39 | |
40 | #if defined(BOOST_SPIRIT_DEBUG) |
41 | #include <iosfwd> |
42 | #endif |
43 | |
44 | namespace boost { namespace spirit { namespace lex { namespace lexertl |
45 | { |
46 | /////////////////////////////////////////////////////////////////////////// |
47 | // |
48 | // The token is the type of the objects returned by default by the |
49 | // iterator. |
50 | // |
51 | // template parameters: |
52 | // Iterator The type of the iterator used to access the |
53 | // underlying character stream. |
54 | // AttributeTypes A mpl sequence containing the types of all |
55 | // required different token values to be supported |
56 | // by this token type. |
57 | // HasState A mpl::bool_ indicating, whether this token type |
58 | // should support lexer states. |
59 | // Idtype The type to use for the token id (defaults to |
60 | // std::size_t). |
61 | // |
62 | // It is possible to use other token types with the spirit::lex |
63 | // framework as well. If you plan to use a different type as your token |
64 | // type, you'll need to expose the following things from your token type |
65 | // to make it compatible with spirit::lex: |
66 | // |
67 | // typedefs |
68 | // iterator_type The type of the iterator used to access the |
69 | // underlying character stream. |
70 | // |
71 | // id_type The type of the token id used. |
72 | // |
73 | // methods |
74 | // default constructor |
75 | // This should initialize the token as an end of |
76 | // input token. |
77 | // constructors The prototype of the other required |
78 | // constructors should be: |
79 | // |
80 | // token(int) |
81 | // This constructor should initialize the token as |
82 | // an invalid token (not carrying any specific |
83 | // values) |
84 | // |
85 | // where: the int is used as a tag only and its value is |
86 | // ignored |
87 | // |
88 | // and: |
89 | // |
90 | // token(Idtype id, std::size_t state, |
91 | // iterator_type first, iterator_type last); |
92 | // |
93 | // where: id: token id |
94 | // state: lexer state this token was matched in |
95 | // first, last: pair of iterators marking the matched |
96 | // range in the underlying input stream |
97 | // |
98 | // accessors |
99 | // id() return the token id of the matched input sequence |
100 | // id(newid) set the token id of the token instance |
101 | // |
102 | // state() return the lexer state this token was matched in |
103 | // |
104 | // value() return the token value |
105 | // |
106 | // Additionally, you will have to implement a couple of helper functions |
107 | // in the same namespace as the token type: a comparison operator==() to |
108 | // compare your token instances, a token_is_valid() function and different |
109 | // specializations of the Spirit customization point |
110 | // assign_to_attribute_from_value as shown below. |
111 | // |
112 | /////////////////////////////////////////////////////////////////////////// |
113 | template <typename Iterator = char const* |
114 | , typename AttributeTypes = mpl::vector0<> |
115 | , typename HasState = mpl::true_ |
116 | , typename Idtype = std::size_t> |
117 | struct token; |
118 | |
119 | /////////////////////////////////////////////////////////////////////////// |
120 | // This specialization of the token type doesn't contain any item data and |
121 | // doesn't support working with lexer states. |
122 | /////////////////////////////////////////////////////////////////////////// |
123 | template <typename Iterator, typename Idtype> |
124 | struct token<Iterator, lex::omit, mpl::false_, Idtype> |
125 | { |
126 | typedef Iterator iterator_type; |
127 | typedef mpl::false_ has_state; |
128 | typedef Idtype id_type; |
129 | typedef unused_type token_value_type; |
130 | typedef typename make_unsigned<id_type>::type uid_type; |
131 | |
132 | // default constructed tokens correspond to EOI tokens |
133 | token() : id_(boost::lexer::npos) {} |
134 | |
135 | // construct an invalid token |
136 | explicit token(int) : id_(0) {} |
137 | |
138 | token(id_type id, std::size_t) : id_(id) {} |
139 | |
140 | token(id_type id, std::size_t, token_value_type) |
141 | : id_(id) {} |
142 | |
143 | token_value_type& value() { static token_value_type u; return u; } |
144 | token_value_type const& value() const { return unused; } |
145 | |
146 | #if defined(BOOST_SPIRIT_DEBUG) |
147 | token(id_type id, std::size_t, Iterator const& first |
148 | , Iterator const& last) |
149 | : matched_(first, last) |
150 | , id_(id) {} |
151 | #else |
152 | token(id_type id, std::size_t, Iterator const&, Iterator const&) |
153 | : id_(id) {} |
154 | #endif |
155 | |
156 | // this default conversion operator is needed to allow the direct |
157 | // usage of tokens in conjunction with the primitive parsers defined |
158 | // in Qi |
159 | operator id_type() const { return id_type(uid_type(id_)); } |
160 | |
161 | // Retrieve or set the token id of this token instance. |
162 | id_type id() const { return id_type(uid_type(id_)); } |
163 | void id(id_type newid) { id_ = uid_type(newid); } |
164 | |
165 | std::size_t state() const { return 0; } // always '0' (INITIAL state) |
166 | |
167 | bool is_valid() const |
168 | { |
169 | return 0 != id_ && boost::lexer::npos != id_; |
170 | } |
171 | |
172 | #if defined(BOOST_SPIRIT_DEBUG) |
173 | #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) |
174 | // workaround for MSVC10 which has problems copying a default |
175 | // constructed iterator_range |
176 | token& operator= (token const& rhs) |
177 | { |
178 | if (this != &rhs) |
179 | { |
180 | id_ = rhs.id_; |
181 | if (is_valid()) |
182 | matched_ = rhs.matched_; |
183 | } |
184 | return *this; |
185 | } |
186 | #endif |
187 | std::pair<Iterator, Iterator> matched_; |
188 | #endif |
189 | |
190 | protected: |
191 | std::size_t id_; // token id, 0 if nothing has been matched |
192 | }; |
193 | |
194 | #if defined(BOOST_SPIRIT_DEBUG) |
195 | template <typename Char, typename Traits, typename Iterator |
196 | , typename AttributeTypes, typename HasState, typename Idtype> |
197 | inline std::basic_ostream<Char, Traits>& |
198 | operator<< (std::basic_ostream<Char, Traits>& os |
199 | , token<Iterator, AttributeTypes, HasState, Idtype> const& t) |
200 | { |
201 | if (t.is_valid()) { |
202 | Iterator end = t.matched_.second; |
203 | for (Iterator it = t.matched_.first; it != end; ++it) |
204 | os << *it; |
205 | } |
206 | else { |
207 | os << "<invalid token>" ; |
208 | } |
209 | return os; |
210 | } |
211 | #endif |
212 | |
213 | /////////////////////////////////////////////////////////////////////////// |
214 | // This specialization of the token type doesn't contain any item data but |
215 | // supports working with lexer states. |
216 | /////////////////////////////////////////////////////////////////////////// |
217 | template <typename Iterator, typename Idtype> |
218 | struct token<Iterator, lex::omit, mpl::true_, Idtype> |
219 | : token<Iterator, lex::omit, mpl::false_, Idtype> |
220 | { |
221 | private: |
222 | typedef token<Iterator, lex::omit, mpl::false_, Idtype> base_type; |
223 | |
224 | public: |
225 | typedef typename base_type::id_type id_type; |
226 | typedef Iterator iterator_type; |
227 | typedef mpl::true_ has_state; |
228 | typedef unused_type token_value_type; |
229 | |
230 | // default constructed tokens correspond to EOI tokens |
231 | token() : state_(boost::lexer::npos) {} |
232 | |
233 | // construct an invalid token |
234 | explicit token(int) : base_type(0), state_(boost::lexer::npos) {} |
235 | |
236 | token(id_type id, std::size_t state) |
237 | : base_type(id, boost::lexer::npos), state_(state) {} |
238 | |
239 | token(id_type id, std::size_t state, token_value_type) |
240 | : base_type(id, boost::lexer::npos, unused) |
241 | , state_(state) {} |
242 | |
243 | token(id_type id, std::size_t state |
244 | , Iterator const& first, Iterator const& last) |
245 | : base_type(id, boost::lexer::npos, first, last) |
246 | , state_(state) {} |
247 | |
248 | std::size_t state() const { return state_; } |
249 | |
250 | #if defined(BOOST_SPIRIT_DEBUG) && BOOST_WORKAROUND(BOOST_MSVC, == 1600) |
251 | // workaround for MSVC10 which has problems copying a default |
252 | // constructed iterator_range |
253 | token& operator= (token const& rhs) |
254 | { |
255 | if (this != &rhs) |
256 | { |
257 | this->base_type::operator=(static_cast<base_type const&>(rhs)); |
258 | state_ = rhs.state_; |
259 | } |
260 | return *this; |
261 | } |
262 | #endif |
263 | |
264 | protected: |
265 | std::size_t state_; // lexer state this token was matched in |
266 | }; |
267 | |
268 | /////////////////////////////////////////////////////////////////////////// |
269 | // The generic version of the token type derives from the |
270 | // specialization above and adds a single data member holding the item |
271 | // data carried by the token instance. |
272 | /////////////////////////////////////////////////////////////////////////// |
273 | namespace detail |
274 | { |
275 | /////////////////////////////////////////////////////////////////////// |
276 | // Meta-function to calculate the type of the variant data item to be |
277 | // stored with each token instance. |
278 | // |
279 | // Note: The iterator pair needs to be the first type in the list of |
280 | // types supported by the generated variant type (this is being |
281 | // used to identify whether the stored data item in a particular |
282 | // token instance needs to be converted from the pair of |
283 | // iterators (see the first of the assign_to_attribute_from_value |
284 | // specializations below). |
285 | /////////////////////////////////////////////////////////////////////// |
286 | template <typename IteratorPair, typename AttributeTypes> |
287 | struct token_value_typesequence |
288 | { |
289 | typedef typename mpl::insert< |
290 | AttributeTypes |
291 | , typename mpl::begin<AttributeTypes>::type |
292 | , IteratorPair |
293 | >::type sequence_type; |
294 | typedef typename make_variant_over<sequence_type>::type type; |
295 | }; |
296 | |
297 | /////////////////////////////////////////////////////////////////////// |
298 | // The type of the data item stored with a token instance is defined |
299 | // by the template parameter 'AttributeTypes' and may be: |
300 | // |
301 | // lex::omit: no data item is stored with the token |
302 | // instance (this is handled by the |
303 | // specializations of the token class |
304 | // below) |
305 | // mpl::vector0<>: each token instance stores a pair of |
306 | // iterators pointing to the matched input |
307 | // sequence |
308 | // mpl::vector<...>: each token instance stores a variant being |
309 | // able to store the pair of iterators pointing |
310 | // to the matched input sequence, or any of the |
311 | // types a specified in the mpl::vector<> |
312 | // |
313 | // All this is done to ensure the token type is as small (in terms |
314 | // of its byte-size) as possible. |
315 | /////////////////////////////////////////////////////////////////////// |
316 | template <typename IteratorPair, typename AttributeTypes> |
317 | struct token_value_type |
318 | : mpl::eval_if< |
319 | mpl::or_< |
320 | is_same<AttributeTypes, mpl::vector0<> > |
321 | , is_same<AttributeTypes, mpl::vector<> > > |
322 | , mpl::identity<IteratorPair> |
323 | , token_value_typesequence<IteratorPair, AttributeTypes> > |
324 | {}; |
325 | } |
326 | |
327 | template <typename Iterator, typename AttributeTypes, typename HasState |
328 | , typename Idtype> |
329 | struct token : token<Iterator, lex::omit, HasState, Idtype> |
330 | { |
331 | private: // precondition assertions |
332 | BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value || |
333 | is_same<AttributeTypes, lex::omit>::value)); |
334 | typedef token<Iterator, lex::omit, HasState, Idtype> base_type; |
335 | |
336 | protected: |
337 | // If no additional token value types are given, the token will |
338 | // hold the plain pair of iterators pointing to the matched range |
339 | // in the underlying input sequence. Otherwise the token value is |
340 | // stored as a variant and will again hold the pair of iterators but |
341 | // is able to hold any of the given data types as well. The conversion |
342 | // from the iterator pair to the required data type is done when it is |
343 | // accessed for the first time. |
344 | typedef iterator_range<Iterator> iterpair_type; |
345 | |
346 | public: |
347 | typedef typename base_type::id_type id_type; |
348 | typedef typename detail::token_value_type< |
349 | iterpair_type, AttributeTypes |
350 | >::type token_value_type; |
351 | |
352 | typedef Iterator iterator_type; |
353 | |
354 | // default constructed tokens correspond to EOI tokens |
355 | token() : value_(iterpair_type(iterator_type(), iterator_type())) {} |
356 | |
357 | // construct an invalid token |
358 | explicit token(int) |
359 | : base_type(0) |
360 | , value_(iterpair_type(iterator_type(), iterator_type())) {} |
361 | |
362 | token(id_type id, std::size_t state, token_value_type const& value) |
363 | : base_type(id, state, value) |
364 | , value_(value) {} |
365 | |
366 | token(id_type id, std::size_t state, Iterator const& first |
367 | , Iterator const& last) |
368 | : base_type(id, state, first, last) |
369 | , value_(iterpair_type(first, last)) {} |
370 | |
371 | token_value_type& value() { return value_; } |
372 | token_value_type const& value() const { return value_; } |
373 | |
374 | #if BOOST_WORKAROUND(BOOST_MSVC, == 1600) |
375 | // workaround for MSVC10 which has problems copying a default |
376 | // constructed iterator_range |
377 | token& operator= (token const& rhs) |
378 | { |
379 | if (this != &rhs) |
380 | { |
381 | this->base_type::operator=(static_cast<base_type const&>(rhs)); |
382 | if (this->is_valid()) |
383 | value_ = rhs.value_; |
384 | } |
385 | return *this; |
386 | } |
387 | #endif |
388 | |
389 | protected: |
390 | token_value_type value_; // token value, by default a pair of iterators |
391 | }; |
392 | |
393 | /////////////////////////////////////////////////////////////////////////// |
394 | // tokens are considered equal, if their id's match (these are unique) |
395 | template <typename Iterator, typename AttributeTypes, typename HasState |
396 | , typename Idtype> |
397 | inline bool |
398 | operator== (token<Iterator, AttributeTypes, HasState, Idtype> const& lhs, |
399 | token<Iterator, AttributeTypes, HasState, Idtype> const& rhs) |
400 | { |
401 | return lhs.id() == rhs.id(); |
402 | } |
403 | |
404 | /////////////////////////////////////////////////////////////////////////// |
405 | // This overload is needed by the multi_pass/functor_input_policy to |
406 | // validate a token instance. It has to be defined in the same namespace |
407 | // as the token class itself to allow ADL to find it. |
408 | /////////////////////////////////////////////////////////////////////////// |
409 | template <typename Iterator, typename AttributeTypes, typename HasState |
410 | , typename Idtype> |
411 | inline bool |
412 | token_is_valid(token<Iterator, AttributeTypes, HasState, Idtype> const& t) |
413 | { |
414 | return t.is_valid(); |
415 | } |
416 | }}}} |
417 | |
418 | namespace boost { namespace spirit { namespace traits |
419 | { |
420 | /////////////////////////////////////////////////////////////////////////// |
421 | // We have to provide specializations for the customization point |
422 | // assign_to_attribute_from_value allowing to extract the needed value |
423 | // from the token. |
424 | /////////////////////////////////////////////////////////////////////////// |
425 | |
426 | // This is called from the parse function of token_def if the token_def |
427 | // has been defined to carry a special attribute type |
428 | template <typename Attribute, typename Iterator, typename AttributeTypes |
429 | , typename HasState, typename Idtype> |
430 | struct assign_to_attribute_from_value<Attribute |
431 | , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> > |
432 | { |
433 | static void |
434 | call(lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> const& t |
435 | , Attribute& attr) |
436 | { |
437 | // The goal of this function is to avoid the conversion of the pair of |
438 | // iterators (to the matched character sequence) into the token value |
439 | // of the required type being done more than once. For this purpose it |
440 | // checks whether the stored value type is still the default one (pair |
441 | // of iterators) and if yes, replaces the pair of iterators with the |
442 | // converted value to be returned from subsequent calls. |
443 | |
444 | if (0 == t.value().which()) { |
445 | // first access to the token value |
446 | typedef iterator_range<Iterator> iterpair_type; |
447 | iterpair_type const& ip = boost::get<iterpair_type>(t.value()); |
448 | |
449 | // Interestingly enough we use the assign_to() framework defined in |
450 | // Spirit.Qi allowing to convert the pair of iterators to almost any |
451 | // required type (assign_to(), if available, uses the standard Spirit |
452 | // parsers to do the conversion). |
453 | spirit::traits::assign_to(ip.begin(), ip.end(), attr); |
454 | |
455 | // If you get an error during the compilation of the following |
456 | // assignment expression, you probably forgot to list one or more |
457 | // types used as token value types (in your token_def<...> |
458 | // definitions) in your definition of the token class. I.e. any token |
459 | // value type used for a token_def<...> definition has to be listed |
460 | // during the declaration of the token type to use. For instance let's |
461 | // assume we have two token_def's: |
462 | // |
463 | // token_def<int> number; number = "..."; |
464 | // token_def<std::string> identifier; identifier = "..."; |
465 | // |
466 | // Then you'll have to use the following token type definition |
467 | // (assuming you are using the token class): |
468 | // |
469 | // typedef mpl::vector<int, std::string> token_values; |
470 | // typedef token<base_iter_type, token_values> token_type; |
471 | // |
472 | // where: base_iter_type is the iterator type used to expose the |
473 | // underlying input stream. |
474 | // |
475 | // This token_type has to be used as the second template parameter |
476 | // to the lexer class: |
477 | // |
478 | // typedef lexer<base_iter_type, token_type> lexer_type; |
479 | // |
480 | // again, assuming you're using the lexer<> template for your |
481 | // tokenization. |
482 | |
483 | typedef lex::lexertl::token< |
484 | Iterator, AttributeTypes, HasState, Idtype> token_type; |
485 | spirit::traits::assign_to( |
486 | attr, const_cast<token_type&>(t).value()); // re-assign value |
487 | } |
488 | else { |
489 | // reuse the already assigned value |
490 | spirit::traits::assign_to(boost::get<Attribute>(t.value()), attr); |
491 | } |
492 | } |
493 | }; |
494 | |
495 | template <typename Attribute, typename Iterator, typename AttributeTypes |
496 | , typename HasState, typename Idtype> |
497 | struct assign_to_container_from_value<Attribute |
498 | , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> > |
499 | : assign_to_attribute_from_value<Attribute |
500 | , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> > |
501 | {}; |
502 | |
503 | template <typename Iterator, typename AttributeTypes |
504 | , typename HasState, typename Idtype> |
505 | struct assign_to_container_from_value<utree |
506 | , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> > |
507 | : assign_to_attribute_from_value<utree |
508 | , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> > |
509 | {}; |
510 | |
511 | template <typename Iterator> |
512 | struct assign_to_container_from_value< |
513 | iterator_range<Iterator>, iterator_range<Iterator> > |
514 | { |
515 | static void |
516 | call(iterator_range<Iterator> const& val, iterator_range<Iterator>& attr) |
517 | { |
518 | attr = val; |
519 | } |
520 | }; |
521 | |
522 | // These are called from the parse function of token_def if the token type |
523 | // has no special attribute type assigned |
524 | template <typename Attribute, typename Iterator, typename HasState |
525 | , typename Idtype> |
526 | struct assign_to_attribute_from_value<Attribute |
527 | , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> > |
528 | { |
529 | static void |
530 | call(lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> const& t |
531 | , Attribute& attr) |
532 | { |
533 | // The default type returned by the token_def parser component (if |
534 | // it has no token value type assigned) is the pair of iterators |
535 | // to the matched character sequence. |
536 | spirit::traits::assign_to(t.value().begin(), t.value().end(), attr); |
537 | } |
538 | }; |
539 | |
540 | // template <typename Attribute, typename Iterator, typename HasState |
541 | // , typename Idtype> |
542 | // struct assign_to_container_from_value<Attribute |
543 | // , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> > |
544 | // : assign_to_attribute_from_value<Attribute |
545 | // , lex::lexertl::token<Iterator, mpl::vector0<>, HasState, Idtype> > |
546 | // {}; |
547 | |
548 | // same as above but using mpl::vector<> instead of mpl::vector0<> |
549 | template <typename Attribute, typename Iterator, typename HasState |
550 | , typename Idtype> |
551 | struct assign_to_attribute_from_value<Attribute |
552 | , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> > |
553 | { |
554 | static void |
555 | call(lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> const& t |
556 | , Attribute& attr) |
557 | { |
558 | // The default type returned by the token_def parser component (if |
559 | // it has no token value type assigned) is the pair of iterators |
560 | // to the matched character sequence. |
561 | spirit::traits::assign_to(t.value().begin(), t.value().end(), attr); |
562 | } |
563 | }; |
564 | |
565 | // template <typename Attribute, typename Iterator, typename HasState |
566 | // , typename Idtype> |
567 | // struct assign_to_container_from_value<Attribute |
568 | // , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> > |
569 | // : assign_to_attribute_from_value<Attribute |
570 | // , lex::lexertl::token<Iterator, mpl::vector<>, HasState, Idtype> > |
571 | // {}; |
572 | |
573 | // This is called from the parse function of token_def if the token type |
574 | // has been explicitly omitted (i.e. no attribute value is used), which |
575 | // essentially means that every attribute gets initialized using default |
576 | // constructed values. |
577 | template <typename Attribute, typename Iterator, typename HasState |
578 | , typename Idtype> |
579 | struct assign_to_attribute_from_value<Attribute |
580 | , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> > |
581 | { |
582 | static void |
583 | call(lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> const& |
584 | , Attribute&) |
585 | { |
586 | // do nothing |
587 | } |
588 | }; |
589 | |
590 | template <typename Attribute, typename Iterator, typename HasState |
591 | , typename Idtype> |
592 | struct assign_to_container_from_value<Attribute |
593 | , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> > |
594 | : assign_to_attribute_from_value<Attribute |
595 | , lex::lexertl::token<Iterator, lex::omit, HasState, Idtype> > |
596 | {}; |
597 | |
598 | // This is called from the parse function of lexer_def_ |
599 | template <typename Iterator, typename AttributeTypes, typename HasState |
600 | , typename Idtype_, typename Idtype> |
601 | struct assign_to_attribute_from_value< |
602 | fusion::vector2<Idtype_, iterator_range<Iterator> > |
603 | , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> > |
604 | { |
605 | static void |
606 | call(lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> const& t |
607 | , fusion::vector2<Idtype_, iterator_range<Iterator> >& attr) |
608 | { |
609 | // The type returned by the lexer_def_ parser components is a |
610 | // fusion::vector containing the token id of the matched token |
611 | // and the pair of iterators to the matched character sequence. |
612 | typedef iterator_range<Iterator> iterpair_type; |
613 | typedef fusion::vector2<Idtype_, iterator_range<Iterator> > |
614 | attribute_type; |
615 | |
616 | iterpair_type const& ip = boost::get<iterpair_type>(t.value()); |
617 | attr = attribute_type(t.id(), ip); |
618 | } |
619 | }; |
620 | |
621 | template <typename Iterator, typename AttributeTypes, typename HasState |
622 | , typename Idtype_, typename Idtype> |
623 | struct assign_to_container_from_value< |
624 | fusion::vector2<Idtype_, iterator_range<Iterator> > |
625 | , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> > |
626 | : assign_to_attribute_from_value< |
627 | fusion::vector2<Idtype_, iterator_range<Iterator> > |
628 | , lex::lexertl::token<Iterator, AttributeTypes, HasState, Idtype> > |
629 | {}; |
630 | |
631 | /////////////////////////////////////////////////////////////////////////// |
632 | // Overload debug output for a single token, this integrates lexer tokens |
633 | // with Qi's simple_trace debug facilities |
634 | template <typename Iterator, typename Attribute, typename HasState |
635 | , typename Idtype> |
636 | struct token_printer_debug< |
637 | lex::lexertl::token<Iterator, Attribute, HasState, Idtype> > |
638 | { |
639 | typedef lex::lexertl::token<Iterator, Attribute, HasState, Idtype> token_type; |
640 | |
641 | template <typename Out> |
642 | static void print(Out& out, token_type const& val) |
643 | { |
644 | out << '['; |
645 | spirit::traits::print_token(out, val.value()); |
646 | out << ']'; |
647 | } |
648 | }; |
649 | }}} |
650 | |
651 | #endif |
652 | |