1 | // pest. The Elegant Parser |
2 | // Copyright (c) 2018 Dragoș Tiselice |
3 | // |
4 | // Licensed under the Apache License, Version 2.0 |
5 | // <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT |
6 | // license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your |
7 | // option. All files in the project carrying such notice may not be copied, |
8 | // modified, or distributed except according to those terms. |
9 | |
10 | use alloc::rc::Rc; |
11 | use alloc::vec::Vec; |
12 | use core::fmt; |
13 | |
14 | use super::line_index::LineIndex; |
15 | use super::pair::{self, Pair}; |
16 | use super::queueable_token::QueueableToken; |
17 | use super::tokens::{self, Tokens}; |
18 | use crate::RuleType; |
19 | |
20 | /// An iterator over [`Pair`]s. It is created by [`Pairs::flatten`]. |
21 | /// |
22 | /// [`Pair`]: struct.Pair.html |
23 | /// [`Pairs::flatten`]: struct.Pairs.html#method.flatten |
24 | pub struct FlatPairs<'i, R> { |
25 | /// # Safety |
26 | /// |
27 | /// All `QueueableToken`s' `input_pos` must be valid character boundary indices into `input`. |
28 | queue: Rc<Vec<QueueableToken<'i, R>>>, |
29 | input: &'i str, |
30 | start: usize, |
31 | end: usize, |
32 | line_index: Rc<LineIndex>, |
33 | } |
34 | |
35 | /// # Safety |
36 | /// |
37 | /// All `QueueableToken`s' `input_pos` must be valid character boundary indices into `input`. |
38 | pub unsafe fn new<'i, R: RuleType>( |
39 | queue: Rc<Vec<QueueableToken<'i, R>>>, |
40 | input: &'i str, |
41 | start: usize, |
42 | end: usize, |
43 | ) -> FlatPairs<'i, R> { |
44 | FlatPairs { |
45 | queue, |
46 | input, |
47 | line_index: Rc::new(LineIndex::new(text:input)), |
48 | start, |
49 | end, |
50 | } |
51 | } |
52 | |
53 | impl<'i, R: RuleType> FlatPairs<'i, R> { |
54 | /// Returns the `Tokens` for these pairs. |
55 | /// |
56 | /// # Examples |
57 | /// |
58 | /// ``` |
59 | /// # use std::rc::Rc; |
60 | /// # use pest; |
61 | /// # #[allow (non_camel_case_types)] |
62 | /// # #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] |
63 | /// enum Rule { |
64 | /// a |
65 | /// } |
66 | /// |
67 | /// let input = "" ; |
68 | /// let pairs = pest::state(input, |state| { |
69 | /// // generating Token pair with Rule::a ... |
70 | /// # state.rule(Rule::a, |s| Ok(s)) |
71 | /// }).unwrap(); |
72 | /// let tokens: Vec<_> = pairs.flatten().tokens().collect(); |
73 | /// |
74 | /// assert_eq!(tokens.len(), 2); |
75 | /// ``` |
76 | #[inline ] |
77 | pub fn tokens(self) -> Tokens<'i, R> { |
78 | tokens::new(self.queue, self.input, self.start, self.end) |
79 | } |
80 | |
81 | fn next_start(&mut self) { |
82 | self.start += 1; |
83 | |
84 | while self.start < self.end && !self.is_start(self.start) { |
85 | self.start += 1; |
86 | } |
87 | } |
88 | |
89 | fn next_start_from_end(&mut self) { |
90 | self.end -= 1; |
91 | |
92 | while self.end >= self.start && !self.is_start(self.end) { |
93 | self.end -= 1; |
94 | } |
95 | } |
96 | |
97 | fn is_start(&self, index: usize) -> bool { |
98 | match self.queue[index] { |
99 | QueueableToken::Start { .. } => true, |
100 | QueueableToken::End { .. } => false, |
101 | } |
102 | } |
103 | } |
104 | |
105 | impl<'i, R: RuleType> ExactSizeIterator for FlatPairs<'i, R> { |
106 | fn len(&self) -> usize { |
107 | // Tokens len is exactly twice as flatten pairs len |
108 | (self.end - self.start) >> 1 |
109 | } |
110 | } |
111 | |
112 | impl<'i, R: RuleType> Iterator for FlatPairs<'i, R> { |
113 | type Item = Pair<'i, R>; |
114 | |
115 | fn next(&mut self) -> Option<Self::Item> { |
116 | if self.start >= self.end { |
117 | return None; |
118 | } |
119 | |
120 | let pair = unsafe { |
121 | pair::new( |
122 | Rc::clone(&self.queue), |
123 | self.input, |
124 | Rc::clone(&self.line_index), |
125 | self.start, |
126 | ) |
127 | }; |
128 | self.next_start(); |
129 | |
130 | Some(pair) |
131 | } |
132 | |
133 | fn size_hint(&self) -> (usize, Option<usize>) { |
134 | let len = <Self as ExactSizeIterator>::len(self); |
135 | (len, Some(len)) |
136 | } |
137 | } |
138 | |
139 | impl<'i, R: RuleType> DoubleEndedIterator for FlatPairs<'i, R> { |
140 | fn next_back(&mut self) -> Option<Self::Item> { |
141 | if self.end <= self.start { |
142 | return None; |
143 | } |
144 | |
145 | self.next_start_from_end(); |
146 | |
147 | let pair: Pair<'_, R> = unsafe { |
148 | pair::new( |
149 | queue:Rc::clone(&self.queue), |
150 | self.input, |
151 | line_index:Rc::clone(&self.line_index), |
152 | self.end, |
153 | ) |
154 | }; |
155 | |
156 | Some(pair) |
157 | } |
158 | } |
159 | |
160 | impl<'i, R: RuleType> fmt::Debug for FlatPairs<'i, R> { |
161 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
162 | f&mut DebugStruct<'_, '_>.debug_struct("FlatPairs" ) |
163 | .field(name:"pairs" , &self.clone().collect::<Vec<_>>()) |
164 | .finish() |
165 | } |
166 | } |
167 | |
168 | impl<'i, R: Clone> Clone for FlatPairs<'i, R> { |
169 | fn clone(&self) -> FlatPairs<'i, R> { |
170 | FlatPairs { |
171 | queue: Rc::clone(&self.queue), |
172 | input: self.input, |
173 | line_index: Rc::clone(&self.line_index), |
174 | start: self.start, |
175 | end: self.end, |
176 | } |
177 | } |
178 | } |
179 | |
180 | #[cfg (test)] |
181 | mod tests { |
182 | use super::super::super::macros::tests::*; |
183 | use super::super::super::Parser; |
184 | use alloc::vec; |
185 | use alloc::vec::Vec; |
186 | |
187 | #[test ] |
188 | fn iter_for_flat_pairs() { |
189 | let pairs = AbcParser::parse(Rule::a, "abcde" ).unwrap(); |
190 | |
191 | assert_eq!( |
192 | pairs.flatten().map(|p| p.as_rule()).collect::<Vec<Rule>>(), |
193 | vec![Rule::a, Rule::b, Rule::c] |
194 | ); |
195 | } |
196 | |
197 | #[test ] |
198 | fn double_ended_iter_for_flat_pairs() { |
199 | let pairs = AbcParser::parse(Rule::a, "abcde" ).unwrap(); |
200 | assert_eq!( |
201 | pairs |
202 | .flatten() |
203 | .rev() |
204 | .map(|p| p.as_rule()) |
205 | .collect::<Vec<Rule>>(), |
206 | vec![Rule::c, Rule::b, Rule::a] |
207 | ); |
208 | } |
209 | |
210 | #[test ] |
211 | fn test_line_col() { |
212 | let mut pairs = AbcParser::parse(Rule::a, "abcNe \nabcde" ).unwrap().flatten(); |
213 | |
214 | let pair = pairs.next().unwrap(); |
215 | assert_eq!(pair.as_str(), "abc" ); |
216 | assert_eq!(pair.line_col(), (1, 1)); |
217 | assert_eq!(pair.line_col(), pair.as_span().start_pos().line_col()); |
218 | |
219 | let pair = pairs.next().unwrap(); |
220 | assert_eq!(pair.as_str(), "b" ); |
221 | assert_eq!(pair.line_col(), (1, 2)); |
222 | assert_eq!(pair.line_col(), pair.as_span().start_pos().line_col()); |
223 | |
224 | let pair = pairs.next().unwrap(); |
225 | assert_eq!(pair.as_str(), "e" ); |
226 | assert_eq!(pair.line_col(), (1, 5)); |
227 | assert_eq!(pair.line_col(), pair.as_span().start_pos().line_col()); |
228 | } |
229 | |
230 | #[test ] |
231 | fn exact_size_iter_for_pairs() { |
232 | let pairs = AbcParser::parse(Rule::a, "abc \nefgh" ).unwrap().flatten(); |
233 | assert_eq!(pairs.len(), pairs.count()); |
234 | |
235 | let pairs = AbcParser::parse(Rule::a, "我很漂亮efgh" ).unwrap().flatten(); |
236 | assert_eq!(pairs.len(), pairs.count()); |
237 | |
238 | let pairs = AbcParser::parse(Rule::a, "abc \nefgh" ).unwrap().flatten(); |
239 | let pairs = pairs.rev(); |
240 | assert_eq!(pairs.len(), pairs.count()); |
241 | |
242 | let mut pairs = AbcParser::parse(Rule::a, "abc \nefgh" ).unwrap().flatten(); |
243 | let pairs_len = pairs.len(); |
244 | let _ = pairs.next().unwrap(); |
245 | assert_eq!(pairs.count() + 1, pairs_len); |
246 | } |
247 | } |
248 | |