1// Copyright (c) 2001-2010 Hartmut Kaiser
2//
3// Distributed under the Boost Software License, Version 1.0. (See accompanying
4// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
5
6// The purpose of this example is to demonstrate how custom, user defined types
7// can be easily integrated with the lexer as token value types. Moreover, the
8// custom token values are properly exposed to the parser as well, allowing to
9// retrieve the custom values using the built in parser attribute propagation
10// rules.
11
12#include <boost/spirit/include/lex_lexertl.hpp>
13#include <boost/spirit/include/qi.hpp>
14
15namespace lex = boost::spirit::lex;
16namespace qi = boost::spirit::qi;
17namespace mpl = boost::mpl;
18
19///////////////////////////////////////////////////////////////////////////////
20// This is just a simple custom rational data structure holding two ints to be
21// interpreted as a rational number
22struct rational
23{
24 rational(int n = 0, int d = 0)
25 : nominator_(n), denominator_(d)
26 {}
27
28 int nominator_;
29 int denominator_;
30};
31
32///////////////////////////////////////////////////////////////////////////////
33// A rational is represented as "{n,d}", where 'n' and 'd' are the nominator
34// and denominator of the number. We use Spirit.Qi to do the low level parsing
35// of the input sequence as matched by the lexer. Certainly, any other
36// conversion could be used instead.
37//
38// The lexer uses the template assign_to_attribute_from_iterators<> to convert
39// the matched input sequence (pair of iterators) to the token value type as
40// specified while defining the lex::token_def<>.
41//
42// Our specialization of assign_to_attribute_from_iterators<> for the rational
43// data type defined above has to be placed into the
44// namespace boost::spirit::traits, otherwise it won't be found by the library.
45namespace boost { namespace spirit { namespace traits
46{
47 template <typename Iterator>
48 struct assign_to_attribute_from_iterators<rational, Iterator>
49 {
50 static void
51 call(Iterator const& first, Iterator const& last, rational& attr)
52 {
53 int x, y;
54 Iterator b = first;
55 qi::parse(b, last,
56 '{' >> qi::int_ >> ',' >> qi::int_ >> '}', x, y);
57 attr = rational(x, y);
58 }
59 };
60}}}
61
62///////////////////////////////////////////////////////////////////////////////
63// a lexer recognizing a single token type: rational
64template <typename Lexer>
65struct lex_rational : lex::lexer<Lexer>
66{
67 lex_rational()
68 {
69 this->self.add_pattern("INT", "[1-9][0-9]*");
70
71 rt = "\\{{INT},{INT}\\}";
72 this->self.add(rt);
73 }
74 lex::token_def<rational> rt;
75};
76
77
78int main()
79{
80 // the token type needs to know the iterator type of the underlying
81 // input and the set of used token value types
82 typedef lex::lexertl::token<std::string::iterator,
83 mpl::vector<rational> > token_type;
84
85 // use actor_lexer<> here if your token definitions have semantic
86 // actions
87 typedef lex::lexertl::lexer<token_type> lexer_type;
88
89 // this is the iterator exposed by the lexer, we use this for parsing
90 typedef lexer_type::iterator_type iterator_type;
91
92 // create a lexer instance
93 std::string input("{3,4}");
94 std::string::iterator s = input.begin();
95
96 lex_rational<lexer_type> lex;
97 iterator_type b = lex.begin(first&: s, last: input.end());
98
99 // use the embedded token_def as a parser, it exposes its token value type
100 // as its parser attribute type
101 rational r;
102 if (!qi::parse(first&: b, last: lex.end(), expr: lex.rt, attr&: r))
103 {
104 std::cerr << "Parsing failed!" << std::endl;
105 return -1;
106 }
107
108 std::cout << "Parsing succeeded: {"
109 << r.nominator_ << ", " << r.denominator_ << "}" << std::endl;
110 return 0;
111}
112
113

source code of boost/libs/spirit/example/lex/custom_token_attribute.cpp