1// Copyright (c) 2001-2010 Hartmut Kaiser
2//
3// Distributed under the Boost Software License, Version 1.0. (See accompanying
4// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
5
6// The purpose of this example is to show, how it is possible to use a lexer
7// token definition for two purposes:
8//
9// . To generate C++ code implementing a static lexical analyzer allowing
10// to recognize all defined tokens
11// . To integrate the generated C++ lexer into the /Spirit/ framework.
12//
13
14// #define BOOST_SPIRIT_LEXERTL_DEBUG
15#define BOOST_VARIANT_MINIMIZE_SIZE
16
17#include <boost/spirit/include/qi.hpp>
18//[wc_static_include
19#include <boost/spirit/include/lex_static_lexertl.hpp>
20//]
21#include <boost/phoenix/operator.hpp>
22#include <boost/phoenix/statement.hpp>
23#include <boost/phoenix/stl/container.hpp>
24
25#include <iostream>
26#include <string>
27
28#include "../example.hpp"
29#include "word_count_tokens.hpp" // token definition
30#include "word_count_static.hpp" // generated tokenizer
31
32using namespace boost::spirit;
33using namespace boost::spirit::ascii;
34
35///////////////////////////////////////////////////////////////////////////////
36// Grammar definition
37///////////////////////////////////////////////////////////////////////////////
38//[wc_static_grammar
39// This is an ordinary grammar definition following the rules defined by
40// Spirit.Qi. There is nothing specific about it, except it gets the token
41// definition class instance passed to the constructor to allow accessing the
42// embedded token_def<> instances.
43template <typename Iterator>
44struct word_count_grammar : qi::grammar<Iterator>
45{
46 template <typename TokenDef>
47 word_count_grammar(TokenDef const& tok)
48 : word_count_grammar::base_type(start)
49 , c(0), w(0), l(0)
50 {
51 using boost::phoenix::ref;
52 using boost::phoenix::size;
53
54 // associate the defined tokens with the lexer, at the same time
55 // defining the actions to be executed
56 start = *( tok.word [ ++ref(w), ref(c) += size(_1) ]
57 | lit('\n') [ ++ref(l), ++ref(c) ]
58 | qi::token(IDANY) [ ++ref(c) ]
59 )
60 ;
61 }
62
63 std::size_t c, w, l; // counter for characters, words, and lines
64 qi::rule<Iterator> start;
65};
66//]
67
68///////////////////////////////////////////////////////////////////////////////
69//[wc_static_main
70int main(int argc, char* argv[])
71{
72 // Define the token type to be used: 'std::string' is available as the type
73 // of the token value.
74 typedef lex::lexertl::token<
75 char const*, boost::mpl::vector<std::string>
76 > token_type;
77
78 // Define the lexer type to be used as the base class for our token
79 // definition.
80 //
81 // This is the only place where the code is different from an equivalent
82 // dynamic lexical analyzer. We use the `lexertl::static_lexer<>` instead of
83 // the `lexertl::lexer<>` as the base class for our token definition type.
84 //
85 // As we specified the suffix "wc" while generating the static tables we
86 // need to pass the type lexertl::static_::lexer_wc as the second template
87 // parameter below (see word_count_generate.cpp).
88 typedef lex::lexertl::static_lexer<
89 token_type, lex::lexertl::static_::lexer_wc
90 > lexer_type;
91
92 // Define the iterator type exposed by the lexer.
93 typedef word_count_tokens<lexer_type>::iterator_type iterator_type;
94
95 // Now we use the types defined above to create the lexer and grammar
96 // object instances needed to invoke the parsing process.
97 word_count_tokens<lexer_type> word_count; // Our lexer
98 word_count_grammar<iterator_type> g (word_count); // Our parser
99
100 // Read in the file into memory.
101 std::string str (read_from_file(infile: 1 == argc ? "word_count.input" : argv[1]));
102 char const* first = str.c_str();
103 char const* last = &first[str.size()];
104
105 // Parsing is done based on the token stream, not the character stream.
106 bool r = lex::tokenize_and_parse(first, last, lex: word_count, xpr: g);
107
108 if (r) { // success
109 std::cout << "lines: " << g.l << ", words: " << g.w
110 << ", characters: " << g.c << "\n";
111 }
112 else {
113 std::string rest(first, last);
114 std::cerr << "Parsing failed\n" << "stopped at: \""
115 << rest << "\"\n";
116 }
117 return 0;
118}
119//]
120

source code of boost/libs/spirit/example/lex/static_lexer/word_count_static.cpp