| 1 | // Copyright (c) 2001-2010 Hartmut Kaiser |
|---|
| 2 | // |
|---|
| 3 | // Distributed under the Boost Software License, Version 1.0. (See accompanying |
|---|
| 4 | // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) |
|---|
| 5 | |
|---|
| 6 | #if !defined(SPIRIT_LEXER_EXAMPLE_WORD_COUNT_LEXER_TOKENS_FEB_10_2008_0739PM) |
|---|
| 7 | #define SPIRIT_LEXER_EXAMPLE_WORD_COUNT_LEXER_TOKENS_FEB_10_2008_0739PM |
|---|
| 8 | |
|---|
| 9 | #include <boost/phoenix/operator.hpp> |
|---|
| 10 | #include <boost/phoenix/statement.hpp> |
|---|
| 11 | #include <boost/phoenix/core.hpp> |
|---|
| 12 | #include <boost/iterator/iterator_traits.hpp> |
|---|
| 13 | |
|---|
| 14 | /////////////////////////////////////////////////////////////////////////////// |
|---|
| 15 | // Token definition: We use the lexertl based lexer engine as the underlying |
|---|
| 16 | // lexer type. |
|---|
| 17 | // |
|---|
| 18 | // Note, the token definition type is derived from the 'lexertl_actor_lexer' |
|---|
| 19 | // template, which is a necessary to being able to use lexer semantic actions. |
|---|
| 20 | /////////////////////////////////////////////////////////////////////////////// |
|---|
| 21 | struct distance_func |
|---|
| 22 | { |
|---|
| 23 | template <typename Iterator1, typename Iterator2> |
|---|
| 24 | struct result : boost::iterator_difference<Iterator1> {}; |
|---|
| 25 | |
|---|
| 26 | template <typename Iterator1, typename Iterator2> |
|---|
| 27 | typename result<Iterator1, Iterator2>::type |
|---|
| 28 | operator()(Iterator1& begin, Iterator2& end) const |
|---|
| 29 | { |
|---|
| 30 | return std::distance(begin, end); |
|---|
| 31 | } |
|---|
| 32 | }; |
|---|
| 33 | boost::phoenix::function<distance_func> const distance = distance_func(); |
|---|
| 34 | |
|---|
| 35 | //[wcl_static_token_definition |
|---|
| 36 | template <typename Lexer> |
|---|
| 37 | struct word_count_lexer_tokens : boost::spirit::lex::lexer<Lexer> |
|---|
| 38 | { |
|---|
| 39 | word_count_lexer_tokens() |
|---|
| 40 | : c(0), w(0), l(0) |
|---|
| 41 | , word("[^ \t\n]+" ) // define tokens |
|---|
| 42 | , eol("\n" ) |
|---|
| 43 | , any("." ) |
|---|
| 44 | { |
|---|
| 45 | using boost::spirit::lex::_start; |
|---|
| 46 | using boost::spirit::lex::_end; |
|---|
| 47 | using boost::phoenix::ref; |
|---|
| 48 | |
|---|
| 49 | // associate tokens with the lexer |
|---|
| 50 | this->self |
|---|
| 51 | = word [++ref(w), ref(c) += distance(_start, _end)] |
|---|
| 52 | | eol [++ref(c), ++ref(l)] |
|---|
| 53 | | any [++ref(c)] |
|---|
| 54 | ; |
|---|
| 55 | } |
|---|
| 56 | |
|---|
| 57 | std::size_t c, w, l; |
|---|
| 58 | boost::spirit::lex::token_def<> word, eol, any; |
|---|
| 59 | }; |
|---|
| 60 | //] |
|---|
| 61 | |
|---|
| 62 | #endif |
|---|
| 63 | |
|---|
