1use crate::error::Needed;
2use crate::lib::std::iter::Enumerate;
3use crate::lib::std::slice::Iter;
4use crate::stream::Checkpoint;
5use crate::stream::Compare;
6use crate::stream::CompareResult;
7use crate::stream::Location;
8use crate::stream::Offset;
9#[cfg(feature = "unstable-recover")]
10#[cfg(feature = "std")]
11use crate::stream::Recover;
12use crate::stream::SliceLen;
13use crate::stream::Stream;
14use crate::stream::StreamIsPartial;
15use crate::stream::UpdateSlice;
16
17/// Specialized input for parsing lexed tokens
18///
19/// Helpful impls
20/// - Any `PartialEq` type (e.g. a `TokenKind` or `&str`) can be used with
21/// [`literal`][crate::token::literal]
22/// - A `PartialEq` for `&str` allows for using `&str` as a parser for tokens
23///
24/// See also [Lexing and Parsing][crate::_topic::lexing].
25#[derive(Debug, Copy, Clone, PartialEq, Eq)]
26pub struct TokenSlice<'t, T> {
27 initial: &'t [T],
28 input: &'t [T],
29}
30
31impl<'t, T> TokenSlice<'t, T>
32where
33 T: crate::lib::std::fmt::Debug + Clone,
34{
35 /// Make a stream to parse tokens
36 #[inline]
37 pub fn new(input: &'t [T]) -> Self {
38 Self {
39 initial: input,
40 input,
41 }
42 }
43
44 /// Reset the stream to the start
45 ///
46 /// This is useful for formats that encode a graph with addresses relative to the start of the
47 /// input.
48 #[doc(alias = "fseek")]
49 #[inline]
50 pub fn reset_to_start(&mut self) {
51 let start = self.initial.checkpoint();
52 self.input.reset(&start);
53 }
54
55 /// Iterate over consumed tokens starting with the last emitted
56 ///
57 /// This is intended to help build up appropriate context when reporting errors.
58 #[inline]
59 pub fn previous_tokens(&self) -> impl Iterator<Item = &'t T> {
60 let offset = self.input.offset_from(&self.initial);
61 self.initial[0..offset].iter().rev()
62 }
63}
64
65/// Track locations by implementing [`Location`] on the Token.
66impl<T> TokenSlice<'_, T>
67where
68 T: Location,
69{
70 #[inline(always)]
71 fn previous_token_end(&self) -> Option<usize> {
72 let index: usize = self.input.offset_from(&self.initial);
73 indexOption
74 .checked_sub(1)
75 .map(|i: usize| self.initial[i].previous_token_end())
76 }
77
78 #[inline(always)]
79 fn current_token_start(&self) -> Option<usize> {
80 self.input.first().map(|t: &T| t.current_token_start())
81 }
82}
83
84impl<T> Default for TokenSlice<'_, T>
85where
86 T: crate::lib::std::fmt::Debug + Clone,
87{
88 fn default() -> Self {
89 Self::new(&[])
90 }
91}
92
93impl<T> crate::lib::std::ops::Deref for TokenSlice<'_, T> {
94 type Target = [T];
95
96 fn deref(&self) -> &Self::Target {
97 self.input
98 }
99}
100
101impl<T> SliceLen for TokenSlice<'_, T> {
102 #[inline(always)]
103 fn slice_len(&self) -> usize {
104 self.input.slice_len()
105 }
106}
107
108impl<'t, T> Stream for TokenSlice<'t, T>
109where
110 T: crate::lib::std::fmt::Debug + Clone,
111{
112 type Token = &'t T;
113 type Slice = &'t [T];
114
115 type IterOffsets = Enumerate<Iter<'t, T>>;
116
117 type Checkpoint = Checkpoint<&'t [T], Self>;
118
119 #[inline(always)]
120 fn iter_offsets(&self) -> Self::IterOffsets {
121 self.input.iter().enumerate()
122 }
123 #[inline(always)]
124 fn eof_offset(&self) -> usize {
125 self.input.eof_offset()
126 }
127
128 #[inline(always)]
129 fn next_token(&mut self) -> Option<Self::Token> {
130 let (token, next) = self.input.split_first()?;
131 self.input = next;
132 Some(token)
133 }
134
135 #[inline(always)]
136 fn peek_token(&self) -> Option<Self::Token> {
137 self.input.first()
138 }
139
140 #[inline(always)]
141 fn offset_for<P>(&self, predicate: P) -> Option<usize>
142 where
143 P: Fn(Self::Token) -> bool,
144 {
145 self.input.iter().position(predicate)
146 }
147 #[inline(always)]
148 fn offset_at(&self, tokens: usize) -> Result<usize, Needed> {
149 self.input.offset_at(tokens)
150 }
151 #[inline(always)]
152 fn next_slice(&mut self, offset: usize) -> Self::Slice {
153 self.input.next_slice(offset)
154 }
155 #[inline(always)]
156 unsafe fn next_slice_unchecked(&mut self, offset: usize) -> Self::Slice {
157 // SAFETY: Passing up invariants
158 unsafe { self.input.next_slice_unchecked(offset) }
159 }
160 #[inline(always)]
161 fn peek_slice(&self, offset: usize) -> Self::Slice {
162 self.input.peek_slice(offset)
163 }
164 #[inline(always)]
165 unsafe fn peek_slice_unchecked(&self, offset: usize) -> Self::Slice {
166 // SAFETY: Passing up invariants
167 unsafe { self.input.peek_slice_unchecked(offset) }
168 }
169
170 #[inline(always)]
171 fn checkpoint(&self) -> Self::Checkpoint {
172 Checkpoint::<_, Self>::new(self.input)
173 }
174 #[inline(always)]
175 fn reset(&mut self, checkpoint: &Self::Checkpoint) {
176 self.input = checkpoint.inner;
177 }
178
179 #[inline(always)]
180 fn raw(&self) -> &dyn crate::lib::std::fmt::Debug {
181 &self.input
182 }
183}
184
185impl<T> Location for TokenSlice<'_, T>
186where
187 T: Location,
188{
189 #[inline(always)]
190 fn previous_token_end(&self) -> usize {
191 self.previous_token_end()
192 .or_else(|| self.current_token_start())
193 .unwrap_or(default:0)
194 }
195 #[inline(always)]
196 fn current_token_start(&self) -> usize {
197 self.current_token_start()
198 .or_else(|| self.previous_token_end())
199 .unwrap_or(default:0)
200 }
201}
202
203#[cfg(feature = "unstable-recover")]
204#[cfg(feature = "std")]
205impl<T, E> Recover<E> for TokenSlice<'_, T>
206where
207 T: crate::lib::std::fmt::Debug + Clone,
208{
209 #[inline(always)]
210 fn record_err(
211 &mut self,
212 _token_start: &Self::Checkpoint,
213 _err_start: &Self::Checkpoint,
214 err: E,
215 ) -> Result<(), E> {
216 Err(err)
217 }
218
219 /// Report whether the [`Stream`] can save off errors for recovery
220 #[inline(always)]
221 fn is_recovery_supported() -> bool {
222 false
223 }
224}
225
226impl<'t, T> StreamIsPartial for TokenSlice<'t, T>
227where
228 T: crate::lib::std::fmt::Debug + Clone,
229{
230 type PartialState = <&'t [T] as StreamIsPartial>::PartialState;
231
232 #[inline]
233 fn complete(&mut self) -> Self::PartialState {
234 #![allow(clippy::semicolon_if_nothing_returned)]
235 self.input.complete()
236 }
237
238 #[inline]
239 fn restore_partial(&mut self, state: Self::PartialState) {
240 self.input.restore_partial(state);
241 }
242
243 #[inline(always)]
244 fn is_partial_supported() -> bool {
245 <&[T] as StreamIsPartial>::is_partial_supported()
246 }
247
248 #[inline(always)]
249 fn is_partial(&self) -> bool {
250 self.input.is_partial()
251 }
252}
253
254impl<T> Offset for TokenSlice<'_, T>
255where
256 T: crate::lib::std::fmt::Debug + Clone,
257{
258 #[inline(always)]
259 fn offset_from(&self, other: &Self) -> usize {
260 self.offset_from(&other.checkpoint())
261 }
262}
263
264impl<T> Offset<<TokenSlice<'_, T> as Stream>::Checkpoint> for TokenSlice<'_, T>
265where
266 T: crate::lib::std::fmt::Debug + Clone,
267{
268 #[inline(always)]
269 fn offset_from(&self, other: &<TokenSlice<'_, T> as Stream>::Checkpoint) -> usize {
270 self.checkpoint().offset_from(start:other)
271 }
272}
273
274impl<T, O> Compare<O> for TokenSlice<'_, T>
275where
276 T: PartialEq<O> + Eq,
277{
278 #[inline]
279 fn compare(&self, t: O) -> CompareResult {
280 if let Some(token: &T) = self.first() {
281 if *token == t {
282 CompareResult::Ok(1)
283 } else {
284 CompareResult::Error
285 }
286 } else {
287 CompareResult::Incomplete
288 }
289 }
290}
291
292impl<T> UpdateSlice for TokenSlice<'_, T>
293where
294 T: crate::lib::std::fmt::Debug + Clone,
295{
296 #[inline(always)]
297 fn update_slice(mut self, inner: Self::Slice) -> Self {
298 self.input = <&[T] as UpdateSlice>::update_slice(self.input, inner);
299 self
300 }
301}
302