1// Boost tokenizer.hpp -----------------------------------------------------//
2
3// (c) Copyright Jeremy Siek and John R. Bandela 2001.
4
5// Distributed under the Boost Software License, Version 1.0. (See
6// accompanying file LICENSE_1_0.txt or copy at
7// http://www.boost.org/LICENSE_1_0.txt)
8
9// See http://www.boost.org/libs/tokenizer for documenation
10
11// Revision History:
12// 03 Jul 2003 John Bandela
13// Converted to new iterator adapter
14// 02 Feb 2002 Jeremy Siek
15// Removed tabs and a little cleanup.
16
17#ifndef BOOST_TOKENIZER_JRB070303_HPP_
18#define BOOST_TOKENIZER_JRB070303_HPP_
19
20#include <boost/token_iterator.hpp>
21
22namespace boost {
23
24
25 //===========================================================================
26 // A container-view of a tokenized "sequence"
27 template <
28 typename TokenizerFunc = char_delimiters_separator<char>,
29 typename Iterator = std::string::const_iterator,
30 typename Type = std::string
31 >
32 class tokenizer {
33 private:
34 typedef token_iterator_generator<TokenizerFunc,Iterator,Type> TGen;
35
36 // It seems that MSVC does not like the unqualified use of iterator,
37 // Thus we use iter internally when it is used unqualified and
38 // the users of this class will always qualify iterator.
39 typedef typename TGen::type iter;
40
41 public:
42
43 typedef iter iterator;
44 typedef iter const_iterator;
45 typedef Type value_type;
46 typedef value_type& reference;
47 typedef const value_type& const_reference;
48 typedef value_type* pointer;
49 typedef const pointer const_pointer;
50 typedef void size_type;
51 typedef void difference_type;
52
53 tokenizer(Iterator first, Iterator last,
54 const TokenizerFunc& f = TokenizerFunc())
55 : first_(first), last_(last), f_(f) { }
56
57 template <typename Container>
58 tokenizer(const Container& c)
59 : first_(c.begin()), last_(c.end()), f_() { }
60
61 template <typename Container>
62 tokenizer(const Container& c,const TokenizerFunc& f)
63 : first_(c.begin()), last_(c.end()), f_(f) { }
64
65 void assign(Iterator first, Iterator last){
66 first_ = first;
67 last_ = last;
68 }
69
70 void assign(Iterator first, Iterator last, const TokenizerFunc& f){
71 assign(first,last);
72 f_ = f;
73 }
74
75 template <typename Container>
76 void assign(const Container& c){
77 assign(c.begin(),c.end());
78 }
79
80
81 template <typename Container>
82 void assign(const Container& c, const TokenizerFunc& f){
83 assign(c.begin(),c.end(),f);
84 }
85
86 iter begin() const { return iter(f_,first_,last_); }
87 iter end() const { return iter(f_,last_,last_); }
88
89 private:
90 Iterator first_;
91 Iterator last_;
92 TokenizerFunc f_;
93 };
94
95
96} // namespace boost
97
98#endif
99

source code of include/boost/tokenizer.hpp