1// pest. The Elegant Parser
2// Copyright (c) 2018 Dragoș Tiselice
3//
4// Licensed under the Apache License, Version 2.0
5// <LICENSE-APACHE or http://www.apache.org/licenses/LICENSE-2.0> or the MIT
6// license <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
7// option. All files in the project carrying such notice may not be copied,
8// modified, or distributed except according to those terms.
9
10use alloc::rc::Rc;
11use alloc::vec::Vec;
12use core::fmt;
13use core::str;
14
15use super::queueable_token::QueueableToken;
16use crate::position;
17use crate::token::Token;
18use crate::RuleType;
19
20/// An iterator over [`Token`]s. It is created by [`Pair::tokens`] and [`Pairs::tokens`].
21///
22/// [`Token`]: ../enum.Token.html
23/// [`Pair::tokens`]: struct.Pair.html#method.tokens
24/// [`Pairs::tokens`]: struct.Pairs.html#method.tokens
25#[derive(Clone)]
26pub struct Tokens<'i, R> {
27 queue: Rc<Vec<QueueableToken<'i, R>>>,
28 input: &'i str,
29 start: usize,
30 end: usize,
31}
32
33pub fn new<'i, R: RuleType>(
34 queue: Rc<Vec<QueueableToken<'i, R>>>,
35 input: &'i str,
36 start: usize,
37 end: usize,
38) -> Tokens<'i, R> {
39 if cfg!(debug_assertions) {
40 for tok: &QueueableToken<'_, R> in queue.iter() {
41 match *tok {
42 QueueableToken::Start { input_pos: usize, .. } | QueueableToken::End { input_pos: usize, .. } => {
43 assert!(
44 input.get(input_pos..).is_some(),
45 "💥 INVALID `Tokens` CREATED 💥"
46 )
47 }
48 }
49 }
50 }
51
52 Tokens {
53 queue,
54 input,
55 start,
56 end,
57 }
58}
59
60impl<'i, R: RuleType> Tokens<'i, R> {
61 fn create_token(&self, index: usize) -> Token<'i, R> {
62 match self.queue[index] {
63 QueueableToken::Start {
64 end_token_index,
65 input_pos,
66 } => {
67 let rule = match self.queue[end_token_index] {
68 QueueableToken::End { rule, .. } => rule,
69 _ => unreachable!(),
70 };
71
72 Token::Start {
73 rule,
74 pos: position::Position::new_internal(self.input, input_pos),
75 }
76 }
77 QueueableToken::End {
78 rule, input_pos, ..
79 } => Token::End {
80 rule,
81 pos: position::Position::new_internal(self.input, input_pos),
82 },
83 }
84 }
85}
86
87impl<'i, R: RuleType> ExactSizeIterator for Tokens<'i, R> {
88 fn len(&self) -> usize {
89 self.end - self.start
90 }
91}
92
93impl<'i, R: RuleType> Iterator for Tokens<'i, R> {
94 type Item = Token<'i, R>;
95
96 fn next(&mut self) -> Option<Self::Item> {
97 if self.start >= self.end {
98 return None;
99 }
100
101 let token: Token<'_, R> = self.create_token(self.start);
102
103 self.start += 1;
104
105 Some(token)
106 }
107
108 fn size_hint(&self) -> (usize, Option<usize>) {
109 let len: usize = <Self as ExactSizeIterator>::len(self);
110 (len, Some(len))
111 }
112}
113
114impl<'i, R: RuleType> DoubleEndedIterator for Tokens<'i, R> {
115 fn next_back(&mut self) -> Option<Self::Item> {
116 if self.end <= self.start {
117 return None;
118 }
119
120 let token: Token<'_, R> = self.create_token(self.end - 1);
121
122 self.end -= 1;
123
124 Some(token)
125 }
126}
127
128impl<'i, R: RuleType> fmt::Debug for Tokens<'i, R> {
129 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
130 f.debug_list().entries(self.clone()).finish()
131 }
132}
133
134#[cfg(test)]
135mod tests {
136 use super::super::super::macros::tests::*;
137 use super::super::super::Parser;
138 use super::Token;
139 use alloc::vec::Vec;
140
141 #[test]
142 fn double_ended_iter_for_tokens() {
143 let pairs = AbcParser::parse(Rule::a, "abcde").unwrap();
144 let mut tokens = pairs.clone().tokens().collect::<Vec<Token<'_, Rule>>>();
145 tokens.reverse();
146 let reverse_tokens = pairs.tokens().rev().collect::<Vec<Token<'_, Rule>>>();
147 assert_eq!(tokens, reverse_tokens);
148 }
149
150 #[test]
151 fn exact_size_iter_for_tokens() {
152 let tokens = AbcParser::parse(Rule::a, "abcde").unwrap().tokens();
153 assert_eq!(tokens.len(), tokens.count());
154
155 let tokens = AbcParser::parse(Rule::a, "我很漂亮e").unwrap().tokens();
156 assert_eq!(tokens.len(), tokens.count());
157
158 let tokens = AbcParser::parse(Rule::a, "abcde").unwrap().tokens().rev();
159 assert_eq!(tokens.len(), tokens.count());
160
161 let mut tokens = AbcParser::parse(Rule::a, "abcde").unwrap().tokens();
162 let tokens_len = tokens.len();
163 let _ = tokens.next().unwrap();
164 assert_eq!(tokens.count() + 1, tokens_len);
165 }
166}
167