1// Boost tokenizer examples -------------------------------------------------//
2
3// (c) Copyright John R. Bandela 2001.
4
5// Distributed under the Boost Software License, Version 1.0. (See
6// accompanying file LICENSE_1_0.txt or copy at
7// http://www.boost.org/LICENSE_1_0.txt)
8
9// See http://www.boost.org for updates, documentation, and revision history.
10
11#include <iostream>
12#include <iterator>
13#include <string>
14#include <algorithm>
15#include <boost/tokenizer.hpp>
16#include <boost/array.hpp>
17
18#include <boost/core/lightweight_test.hpp>
19
20int main()
21{
22 using namespace boost;
23
24 // Use tokenizer
25 {
26 const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|";
27 std::string answer[] = { "Hello", "world", "foo", "bar", "yow", "baz" };
28 typedef tokenizer<char_separator<char> > Tok;
29 char_separator<char> sep("-;|");
30 Tok t(test_string, sep);
31 BOOST_TEST(std::equal(t.begin(),t.end(),answer));
32 }
33 {
34 const std::string test_string = ";;Hello|world||-foo--bar;yow;baz|";
35 std::string answer[] = { "", "", "Hello", "|", "world", "|", "", "|", "",
36 "foo", "", "bar", "yow", "baz", "|", "" };
37 typedef tokenizer<char_separator<char> > Tok;
38 char_separator<char> sep("-;", "|", boost::keep_empty_tokens);
39 Tok t(test_string, sep);
40 BOOST_TEST(std::equal(t.begin(), t.end(), answer));
41 }
42 {
43 const std::string test_string = "This,,is, a.test..";
44 std::string answer[] = {"This","is","a","test"};
45 typedef tokenizer<> Tok;
46 Tok t(test_string);
47 BOOST_TEST(std::equal(t.begin(),t.end(),answer));
48 }
49
50 {
51 const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
52 std::string answer[] = {"Field 1","embedded,comma","quote \""," escape \\"};
53 typedef tokenizer<escaped_list_separator<char> > Tok;
54 Tok t(test_string);
55 BOOST_TEST(std::equal(t.begin(),t.end(),answer));
56 }
57
58 {
59 const std::string test_string = ",1,;2\\\";3\\;,4,5^\\,\'6,7\';";
60 std::string answer[] = {"","1","","2\"","3;","4","5\\","6,7",""};
61 typedef tokenizer<escaped_list_separator<char> > Tok;
62 escaped_list_separator<char> sep("\\^",",;","\"\'");
63 Tok t(test_string,sep);
64 BOOST_TEST(std::equal(t.begin(),t.end(),answer));
65 }
66
67 {
68 const std::string test_string = "12252001";
69 std::string answer[] = {"12","25","2001"};
70 typedef tokenizer<offset_separator > Tok;
71 boost::array<int,3> offsets = {.elems: {2,2,4}};
72 offset_separator func(offsets.begin(),offsets.end());
73 Tok t(test_string,func);
74 BOOST_TEST(std::equal(t.begin(),t.end(),answer));
75 }
76
77 // Use token_iterator_generator
78 {
79 const std::string test_string = "This,,is, a.test..";
80 std::string answer[] = {"This","is","a","test"};
81 typedef token_iterator_generator<char_delimiters_separator<char> >::type Iter;
82 Iter begin = make_token_iterator<std::string>(begin: test_string.begin(),
83 end: test_string.end(),fun: char_delimiters_separator<char>());
84 Iter end;
85 BOOST_TEST(std::equal(begin,end,answer));
86 }
87
88 {
89 const std::string test_string = "Field 1,\"embedded,comma\",quote \\\", escape \\\\";
90 std::string answer[] = {"Field 1","embedded,comma","quote \""," escape \\"};
91 typedef token_iterator_generator<escaped_list_separator<char> >::type Iter;
92 Iter begin = make_token_iterator<std::string>(begin: test_string.begin(),
93 end: test_string.end(),fun: escaped_list_separator<char>());
94 Iter begin_c(begin);
95 Iter end;
96 BOOST_TEST(std::equal(begin,end,answer));
97
98 while(begin_c != end)
99 {
100 BOOST_TEST(begin_c.at_end() == 0);
101 ++begin_c;
102 }
103 BOOST_TEST(begin_c.at_end());
104 }
105
106 {
107 const std::string test_string = "12252001";
108 std::string answer[] = {"12","25","2001"};
109 typedef token_iterator_generator<offset_separator>::type Iter;
110 boost::array<int,3> offsets = {.elems: {2,2,4}};
111 offset_separator func(offsets.begin(),offsets.end());
112 Iter begin = make_token_iterator<std::string>(begin: test_string.begin(),
113 end: test_string.end(),fun: func);
114 Iter end= make_token_iterator<std::string>(begin: test_string.end(),
115 end: test_string.end(),fun: func);
116 BOOST_TEST(std::equal(begin,end,answer));
117 }
118
119 // Test copying
120 {
121 const std::string test_string = "abcdef";
122 token_iterator_generator<offset_separator>::type beg, end, other;
123 boost::array<int,3> ar = {.elems: {1,2,3}};
124 offset_separator f(ar.begin(),ar.end());
125 beg = make_token_iterator<std::string>(begin: test_string.begin(),end: test_string.end(),fun: f);
126
127 ++beg;
128 other = beg;
129 ++other;
130
131 BOOST_TEST(*beg=="bc");
132 BOOST_TEST(*other=="def");
133
134 other = make_token_iterator<std::string>(begin: test_string.begin(),
135 end: test_string.end(),fun: f);
136
137 BOOST_TEST(*other=="a");
138 }
139
140 // Test non-default constructed char_delimiters_separator
141 {
142 const std::string test_string = "how,are you, doing";
143 std::string answer[] = {"how",",","are you",","," doing"};
144 tokenizer<> t(test_string,char_delimiters_separator<char>(true,",",""));
145 BOOST_TEST(std::equal(t.begin(),t.end(),answer));
146 }
147
148 return boost::report_errors();
149}
150

source code of boost/libs/tokenizer/test/examples.cpp