1//! Parsing interface for parsing a token stream into a syntax tree node.
2//!
3//! Parsing in Syn is built on parser functions that take in a [`ParseStream`]
4//! and produce a [`Result<T>`] where `T` is some syntax tree node. Underlying
5//! these parser functions is a lower level mechanism built around the
6//! [`Cursor`] type. `Cursor` is a cheaply copyable cursor over a range of
7//! tokens in a token stream.
8//!
9//! [`Result<T>`]: Result
10//! [`Cursor`]: crate::buffer::Cursor
11//!
12//! # Example
13//!
14//! Here is a snippet of parsing code to get a feel for the style of the
15//! library. We define data structures for a subset of Rust syntax including
16//! enums (not shown) and structs, then provide implementations of the [`Parse`]
17//! trait to parse these syntax tree data structures from a token stream.
18//!
19//! Once `Parse` impls have been defined, they can be called conveniently from a
20//! procedural macro through [`parse_macro_input!`] as shown at the bottom of
21//! the snippet. If the caller provides syntactically invalid input to the
22//! procedural macro, they will receive a helpful compiler error message
23//! pointing out the exact token that triggered the failure to parse.
24//!
25//! [`parse_macro_input!`]: crate::parse_macro_input!
26//!
27//! ```
28//! # extern crate proc_macro;
29//! #
30//! use proc_macro::TokenStream;
31//! use syn::{braced, parse_macro_input, token, Field, Ident, Result, Token};
32//! use syn::parse::{Parse, ParseStream};
33//! use syn::punctuated::Punctuated;
34//!
35//! enum Item {
36//! Struct(ItemStruct),
37//! Enum(ItemEnum),
38//! }
39//!
40//! struct ItemStruct {
41//! struct_token: Token![struct],
42//! ident: Ident,
43//! brace_token: token::Brace,
44//! fields: Punctuated<Field, Token![,]>,
45//! }
46//! #
47//! # enum ItemEnum {}
48//!
49//! impl Parse for Item {
50//! fn parse(input: ParseStream) -> Result<Self> {
51//! let lookahead = input.lookahead1();
52//! if lookahead.peek(Token![struct]) {
53//! input.parse().map(Item::Struct)
54//! } else if lookahead.peek(Token![enum]) {
55//! input.parse().map(Item::Enum)
56//! } else {
57//! Err(lookahead.error())
58//! }
59//! }
60//! }
61//!
62//! impl Parse for ItemStruct {
63//! fn parse(input: ParseStream) -> Result<Self> {
64//! let content;
65//! Ok(ItemStruct {
66//! struct_token: input.parse()?,
67//! ident: input.parse()?,
68//! brace_token: braced!(content in input),
69//! fields: content.parse_terminated(Field::parse_named, Token![,])?,
70//! })
71//! }
72//! }
73//! #
74//! # impl Parse for ItemEnum {
75//! # fn parse(input: ParseStream) -> Result<Self> {
76//! # unimplemented!()
77//! # }
78//! # }
79//!
80//! # const IGNORE: &str = stringify! {
81//! #[proc_macro]
82//! # };
83//! pub fn my_macro(tokens: TokenStream) -> TokenStream {
84//! let input = parse_macro_input!(tokens as Item);
85//!
86//! /* ... */
87//! # TokenStream::new()
88//! }
89//! ```
90//!
91//! # The `syn::parse*` functions
92//!
93//! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
94//! as an entry point for parsing syntax tree nodes that can be parsed in an
95//! obvious default way. These functions can return any syntax tree node that
96//! implements the [`Parse`] trait, which includes most types in Syn.
97//!
98//! [`syn::parse`]: crate::parse()
99//! [`syn::parse2`]: crate::parse2()
100//! [`syn::parse_str`]: crate::parse_str()
101//!
102//! ```
103//! use syn::Type;
104//!
105//! # fn run_parser() -> syn::Result<()> {
106//! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
107//! # Ok(())
108//! # }
109//! #
110//! # run_parser().unwrap();
111//! ```
112//!
113//! The [`parse_quote!`] macro also uses this approach.
114//!
115//! [`parse_quote!`]: crate::parse_quote!
116//!
117//! # The `Parser` trait
118//!
119//! Some types can be parsed in several ways depending on context. For example
120//! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
121//! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
122//! may or may not allow trailing punctuation, and parsing it the wrong way
123//! would either reject valid input or accept invalid input.
124//!
125//! [`Attribute`]: crate::Attribute
126//! [`Punctuated`]: crate::punctuated
127//!
128//! The `Parse` trait is not implemented in these cases because there is no good
129//! behavior to consider the default.
130//!
131//! ```compile_fail
132//! # extern crate proc_macro;
133//! #
134//! # use syn::punctuated::Punctuated;
135//! # use syn::{PathSegment, Result, Token};
136//! #
137//! # fn f(tokens: proc_macro::TokenStream) -> Result<()> {
138//! #
139//! // Can't parse `Punctuated` without knowing whether trailing punctuation
140//! // should be allowed in this context.
141//! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
142//! #
143//! # Ok(())
144//! # }
145//! ```
146//!
147//! In these cases the types provide a choice of parser functions rather than a
148//! single `Parse` implementation, and those parser functions can be invoked
149//! through the [`Parser`] trait.
150//!
151//!
152//! ```
153//! # extern crate proc_macro;
154//! #
155//! use proc_macro::TokenStream;
156//! use syn::parse::Parser;
157//! use syn::punctuated::Punctuated;
158//! use syn::{Attribute, Expr, PathSegment, Result, Token};
159//!
160//! fn call_some_parser_methods(input: TokenStream) -> Result<()> {
161//! // Parse a nonempty sequence of path segments separated by `::` punctuation
162//! // with no trailing punctuation.
163//! let tokens = input.clone();
164//! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
165//! let _path = parser.parse(tokens)?;
166//!
167//! // Parse a possibly empty sequence of expressions terminated by commas with
168//! // an optional trailing punctuation.
169//! let tokens = input.clone();
170//! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
171//! let _args = parser.parse(tokens)?;
172//!
173//! // Parse zero or more outer attributes but not inner attributes.
174//! let tokens = input.clone();
175//! let parser = Attribute::parse_outer;
176//! let _attrs = parser.parse(tokens)?;
177//!
178//! Ok(())
179//! }
180//! ```
181
182#[path = "discouraged.rs"]
183pub mod discouraged;
184
185use crate::buffer::{Cursor, TokenBuffer};
186use crate::error;
187use crate::lookahead;
188#[cfg(feature = "proc-macro")]
189use crate::proc_macro;
190use crate::punctuated::Punctuated;
191use crate::token::Token;
192use proc_macro2::{self, Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree};
193use std::cell::Cell;
194use std::fmt::{self, Debug, Display};
195#[cfg(feature = "extra-traits")]
196use std::hash::{Hash, Hasher};
197use std::marker::PhantomData;
198use std::mem;
199use std::ops::Deref;
200use std::rc::Rc;
201use std::str::FromStr;
202
203pub use crate::error::{Error, Result};
204pub use crate::lookahead::{Lookahead1, Peek};
205
206/// Parsing interface implemented by all types that can be parsed in a default
207/// way from a token stream.
208///
209/// Refer to the [module documentation] for details about implementing and using
210/// the `Parse` trait.
211///
212/// [module documentation]: self
213pub trait Parse: Sized {
214 fn parse(input: ParseStream) -> Result<Self>;
215}
216
217/// Input to a Syn parser function.
218///
219/// See the methods of this type under the documentation of [`ParseBuffer`]. For
220/// an overview of parsing in Syn, refer to the [module documentation].
221///
222/// [module documentation]: self
223pub type ParseStream<'a> = &'a ParseBuffer<'a>;
224
225/// Cursor position within a buffered token stream.
226///
227/// This type is more commonly used through the type alias [`ParseStream`] which
228/// is an alias for `&ParseBuffer`.
229///
230/// `ParseStream` is the input type for all parser functions in Syn. They have
231/// the signature `fn(ParseStream) -> Result<T>`.
232///
233/// ## Calling a parser function
234///
235/// There is no public way to construct a `ParseBuffer`. Instead, if you are
236/// looking to invoke a parser function that requires `ParseStream` as input,
237/// you will need to go through one of the public parsing entry points.
238///
239/// - The [`parse_macro_input!`] macro if parsing input of a procedural macro;
240/// - One of [the `syn::parse*` functions][syn-parse]; or
241/// - A method of the [`Parser`] trait.
242///
243/// [`parse_macro_input!`]: crate::parse_macro_input!
244/// [syn-parse]: self#the-synparse-functions
245pub struct ParseBuffer<'a> {
246 scope: Span,
247 // Instead of Cell<Cursor<'a>> so that ParseBuffer<'a> is covariant in 'a.
248 // The rest of the code in this module needs to be careful that only a
249 // cursor derived from this `cell` is ever assigned to this `cell`.
250 //
251 // Cell<Cursor<'a>> cannot be covariant in 'a because then we could take a
252 // ParseBuffer<'a>, upcast to ParseBuffer<'short> for some lifetime shorter
253 // than 'a, and then assign a Cursor<'short> into the Cell.
254 //
255 // By extension, it would not be safe to expose an API that accepts a
256 // Cursor<'a> and trusts that it lives as long as the cursor currently in
257 // the cell.
258 cell: Cell<Cursor<'static>>,
259 marker: PhantomData<Cursor<'a>>,
260 unexpected: Cell<Option<Rc<Cell<Unexpected>>>>,
261}
262
263impl<'a> Drop for ParseBuffer<'a> {
264 fn drop(&mut self) {
265 if let Some(unexpected_span: Span) = span_of_unexpected_ignoring_nones(self.cursor()) {
266 let (inner: Rc>, old_span: Option) = inner_unexpected(self);
267 if old_span.is_none() {
268 inner.set(val:Unexpected::Some(unexpected_span));
269 }
270 }
271 }
272}
273
274impl<'a> Display for ParseBuffer<'a> {
275 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
276 Display::fmt(&self.cursor().token_stream(), f)
277 }
278}
279
280impl<'a> Debug for ParseBuffer<'a> {
281 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
282 Debug::fmt(&self.cursor().token_stream(), f)
283 }
284}
285
286/// Cursor state associated with speculative parsing.
287///
288/// This type is the input of the closure provided to [`ParseStream::step`].
289///
290/// [`ParseStream::step`]: ParseBuffer::step
291///
292/// # Example
293///
294/// ```
295/// use proc_macro2::TokenTree;
296/// use syn::Result;
297/// use syn::parse::ParseStream;
298///
299/// // This function advances the stream past the next occurrence of `@`. If
300/// // no `@` is present in the stream, the stream position is unchanged and
301/// // an error is returned.
302/// fn skip_past_next_at(input: ParseStream) -> Result<()> {
303/// input.step(|cursor| {
304/// let mut rest = *cursor;
305/// while let Some((tt, next)) = rest.token_tree() {
306/// match &tt {
307/// TokenTree::Punct(punct) if punct.as_char() == '@' => {
308/// return Ok(((), next));
309/// }
310/// _ => rest = next,
311/// }
312/// }
313/// Err(cursor.error("no `@` was found after this point"))
314/// })
315/// }
316/// #
317/// # fn remainder_after_skipping_past_next_at(
318/// # input: ParseStream,
319/// # ) -> Result<proc_macro2::TokenStream> {
320/// # skip_past_next_at(input)?;
321/// # input.parse()
322/// # }
323/// #
324/// # use syn::parse::Parser;
325/// # let remainder = remainder_after_skipping_past_next_at
326/// # .parse_str("a @ b c")
327/// # .unwrap();
328/// # assert_eq!(remainder.to_string(), "b c");
329/// ```
330pub struct StepCursor<'c, 'a> {
331 scope: Span,
332 // This field is covariant in 'c.
333 cursor: Cursor<'c>,
334 // This field is contravariant in 'c. Together these make StepCursor
335 // invariant in 'c. Also covariant in 'a. The user cannot cast 'c to a
336 // different lifetime but can upcast into a StepCursor with a shorter
337 // lifetime 'a.
338 //
339 // As long as we only ever construct a StepCursor for which 'c outlives 'a,
340 // this means if ever a StepCursor<'c, 'a> exists we are guaranteed that 'c
341 // outlives 'a.
342 marker: PhantomData<fn(Cursor<'c>) -> Cursor<'a>>,
343}
344
345impl<'c, 'a> Deref for StepCursor<'c, 'a> {
346 type Target = Cursor<'c>;
347
348 fn deref(&self) -> &Self::Target {
349 &self.cursor
350 }
351}
352
353impl<'c, 'a> Copy for StepCursor<'c, 'a> {}
354
355impl<'c, 'a> Clone for StepCursor<'c, 'a> {
356 fn clone(&self) -> Self {
357 *self
358 }
359}
360
361impl<'c, 'a> StepCursor<'c, 'a> {
362 /// Triggers an error at the current position of the parse stream.
363 ///
364 /// The `ParseStream::step` invocation will return this same error without
365 /// advancing the stream state.
366 pub fn error<T: Display>(self, message: T) -> Error {
367 error::new_at(self.scope, self.cursor, message)
368 }
369}
370
371pub(crate) fn advance_step_cursor<'c, 'a>(proof: StepCursor<'c, 'a>, to: Cursor<'c>) -> Cursor<'a> {
372 // Refer to the comments within the StepCursor definition. We use the
373 // fact that a StepCursor<'c, 'a> exists as proof that 'c outlives 'a.
374 // Cursor is covariant in its lifetime parameter so we can cast a
375 // Cursor<'c> to one with the shorter lifetime Cursor<'a>.
376 let _ = proof;
377 unsafe { mem::transmute::<Cursor<'c>, Cursor<'a>>(src:to) }
378}
379
380pub(crate) fn new_parse_buffer(
381 scope: Span,
382 cursor: Cursor,
383 unexpected: Rc<Cell<Unexpected>>,
384) -> ParseBuffer {
385 ParseBuffer {
386 scope,
387 // See comment on `cell` in the struct definition.
388 cell: Cell::new(unsafe { mem::transmute::<Cursor, Cursor<'static>>(src:cursor) }),
389 marker: PhantomData,
390 unexpected: Cell::new(Some(unexpected)),
391 }
392}
393
394pub(crate) enum Unexpected {
395 None,
396 Some(Span),
397 Chain(Rc<Cell<Unexpected>>),
398}
399
400impl Default for Unexpected {
401 fn default() -> Self {
402 Unexpected::None
403 }
404}
405
406impl Clone for Unexpected {
407 fn clone(&self) -> Self {
408 match self {
409 Unexpected::None => Unexpected::None,
410 Unexpected::Some(span: &Span) => Unexpected::Some(*span),
411 Unexpected::Chain(next: &Rc>) => Unexpected::Chain(next.clone()),
412 }
413 }
414}
415
416// We call this on Cell<Unexpected> and Cell<Option<T>> where temporarily
417// swapping in a None is cheap.
418fn cell_clone<T: Default + Clone>(cell: &Cell<T>) -> T {
419 let prev: T = cell.take();
420 let ret: T = prev.clone();
421 cell.set(val:prev);
422 ret
423}
424
425fn inner_unexpected(buffer: &ParseBuffer) -> (Rc<Cell<Unexpected>>, Option<Span>) {
426 let mut unexpected: Rc> = get_unexpected(buffer);
427 loop {
428 match cell_clone(&unexpected) {
429 Unexpected::None => return (unexpected, None),
430 Unexpected::Some(span: Span) => return (unexpected, Some(span)),
431 Unexpected::Chain(next: Rc>) => unexpected = next,
432 }
433 }
434}
435
436pub(crate) fn get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Unexpected>> {
437 cell_clone(&buffer.unexpected).unwrap()
438}
439
440fn span_of_unexpected_ignoring_nones(mut cursor: Cursor) -> Option<Span> {
441 if cursor.eof() {
442 return None;
443 }
444 while let Some((inner: Cursor<'_>, _span: DelimSpan, rest: Cursor<'_>)) = cursor.group(delim:Delimiter::None) {
445 if let Some(unexpected: Span) = span_of_unexpected_ignoring_nones(cursor:inner) {
446 return Some(unexpected);
447 }
448 cursor = rest;
449 }
450 if cursor.eof() {
451 None
452 } else {
453 Some(cursor.span())
454 }
455}
456
457impl<'a> ParseBuffer<'a> {
458 /// Parses a syntax tree node of type `T`, advancing the position of our
459 /// parse stream past it.
460 pub fn parse<T: Parse>(&self) -> Result<T> {
461 T::parse(self)
462 }
463
464 /// Calls the given parser function to parse a syntax tree node of type `T`
465 /// from this stream.
466 ///
467 /// # Example
468 ///
469 /// The parser below invokes [`Attribute::parse_outer`] to parse a vector of
470 /// zero or more outer attributes.
471 ///
472 /// [`Attribute::parse_outer`]: crate::Attribute::parse_outer
473 ///
474 /// ```
475 /// use syn::{Attribute, Ident, Result, Token};
476 /// use syn::parse::{Parse, ParseStream};
477 ///
478 /// // Parses a unit struct with attributes.
479 /// //
480 /// // #[path = "s.tmpl"]
481 /// // struct S;
482 /// struct UnitStruct {
483 /// attrs: Vec<Attribute>,
484 /// struct_token: Token![struct],
485 /// name: Ident,
486 /// semi_token: Token![;],
487 /// }
488 ///
489 /// impl Parse for UnitStruct {
490 /// fn parse(input: ParseStream) -> Result<Self> {
491 /// Ok(UnitStruct {
492 /// attrs: input.call(Attribute::parse_outer)?,
493 /// struct_token: input.parse()?,
494 /// name: input.parse()?,
495 /// semi_token: input.parse()?,
496 /// })
497 /// }
498 /// }
499 /// ```
500 pub fn call<T>(&self, function: fn(ParseStream) -> Result<T>) -> Result<T> {
501 function(self)
502 }
503
504 /// Looks at the next token in the parse stream to determine whether it
505 /// matches the requested type of token.
506 ///
507 /// Does not advance the position of the parse stream.
508 ///
509 /// # Syntax
510 ///
511 /// Note that this method does not use turbofish syntax. Pass the peek type
512 /// inside of parentheses.
513 ///
514 /// - `input.peek(Token![struct])`
515 /// - `input.peek(Token![==])`
516 /// - `input.peek(Ident)`&emsp;*(does not accept keywords)*
517 /// - `input.peek(Ident::peek_any)`
518 /// - `input.peek(Lifetime)`
519 /// - `input.peek(token::Brace)`
520 ///
521 /// # Example
522 ///
523 /// In this example we finish parsing the list of supertraits when the next
524 /// token in the input is either `where` or an opening curly brace.
525 ///
526 /// ```
527 /// use syn::{braced, token, Generics, Ident, Result, Token, TypeParamBound};
528 /// use syn::parse::{Parse, ParseStream};
529 /// use syn::punctuated::Punctuated;
530 ///
531 /// // Parses a trait definition containing no associated items.
532 /// //
533 /// // trait Marker<'de, T>: A + B<'de> where Box<T>: Clone {}
534 /// struct MarkerTrait {
535 /// trait_token: Token![trait],
536 /// ident: Ident,
537 /// generics: Generics,
538 /// colon_token: Option<Token![:]>,
539 /// supertraits: Punctuated<TypeParamBound, Token![+]>,
540 /// brace_token: token::Brace,
541 /// }
542 ///
543 /// impl Parse for MarkerTrait {
544 /// fn parse(input: ParseStream) -> Result<Self> {
545 /// let trait_token: Token![trait] = input.parse()?;
546 /// let ident: Ident = input.parse()?;
547 /// let mut generics: Generics = input.parse()?;
548 /// let colon_token: Option<Token![:]> = input.parse()?;
549 ///
550 /// let mut supertraits = Punctuated::new();
551 /// if colon_token.is_some() {
552 /// loop {
553 /// supertraits.push_value(input.parse()?);
554 /// if input.peek(Token![where]) || input.peek(token::Brace) {
555 /// break;
556 /// }
557 /// supertraits.push_punct(input.parse()?);
558 /// }
559 /// }
560 ///
561 /// generics.where_clause = input.parse()?;
562 /// let content;
563 /// let empty_brace_token = braced!(content in input);
564 ///
565 /// Ok(MarkerTrait {
566 /// trait_token,
567 /// ident,
568 /// generics,
569 /// colon_token,
570 /// supertraits,
571 /// brace_token: empty_brace_token,
572 /// })
573 /// }
574 /// }
575 /// ```
576 pub fn peek<T: Peek>(&self, token: T) -> bool {
577 let _ = token;
578 T::Token::peek(self.cursor())
579 }
580
581 /// Looks at the second-next token in the parse stream.
582 ///
583 /// This is commonly useful as a way to implement contextual keywords.
584 ///
585 /// # Example
586 ///
587 /// This example needs to use `peek2` because the symbol `union` is not a
588 /// keyword in Rust. We can't use just `peek` and decide to parse a union if
589 /// the very next token is `union`, because someone is free to write a `mod
590 /// union` and a macro invocation that looks like `union::some_macro! { ...
591 /// }`. In other words `union` is a contextual keyword.
592 ///
593 /// ```
594 /// use syn::{Ident, ItemUnion, Macro, Result, Token};
595 /// use syn::parse::{Parse, ParseStream};
596 ///
597 /// // Parses either a union or a macro invocation.
598 /// enum UnionOrMacro {
599 /// // union MaybeUninit<T> { uninit: (), value: T }
600 /// Union(ItemUnion),
601 /// // lazy_static! { ... }
602 /// Macro(Macro),
603 /// }
604 ///
605 /// impl Parse for UnionOrMacro {
606 /// fn parse(input: ParseStream) -> Result<Self> {
607 /// if input.peek(Token![union]) && input.peek2(Ident) {
608 /// input.parse().map(UnionOrMacro::Union)
609 /// } else {
610 /// input.parse().map(UnionOrMacro::Macro)
611 /// }
612 /// }
613 /// }
614 /// ```
615 pub fn peek2<T: Peek>(&self, token: T) -> bool {
616 fn peek2(buffer: &ParseBuffer, peek: fn(Cursor) -> bool) -> bool {
617 if let Some(group) = buffer.cursor().group(Delimiter::None) {
618 if group.0.skip().map_or(false, peek) {
619 return true;
620 }
621 }
622 buffer.cursor().skip().map_or(false, peek)
623 }
624
625 let _ = token;
626 peek2(self, T::Token::peek)
627 }
628
629 /// Looks at the third-next token in the parse stream.
630 pub fn peek3<T: Peek>(&self, token: T) -> bool {
631 fn peek3(buffer: &ParseBuffer, peek: fn(Cursor) -> bool) -> bool {
632 if let Some(group) = buffer.cursor().group(Delimiter::None) {
633 if group.0.skip().and_then(Cursor::skip).map_or(false, peek) {
634 return true;
635 }
636 }
637 buffer
638 .cursor()
639 .skip()
640 .and_then(Cursor::skip)
641 .map_or(false, peek)
642 }
643
644 let _ = token;
645 peek3(self, T::Token::peek)
646 }
647
648 /// Parses zero or more occurrences of `T` separated by punctuation of type
649 /// `P`, with optional trailing punctuation.
650 ///
651 /// Parsing continues until the end of this parse stream. The entire content
652 /// of this parse stream must consist of `T` and `P`.
653 ///
654 /// # Example
655 ///
656 /// ```
657 /// # use quote::quote;
658 /// #
659 /// use syn::{parenthesized, token, Ident, Result, Token, Type};
660 /// use syn::parse::{Parse, ParseStream};
661 /// use syn::punctuated::Punctuated;
662 ///
663 /// // Parse a simplified tuple struct syntax like:
664 /// //
665 /// // struct S(A, B);
666 /// struct TupleStruct {
667 /// struct_token: Token![struct],
668 /// ident: Ident,
669 /// paren_token: token::Paren,
670 /// fields: Punctuated<Type, Token![,]>,
671 /// semi_token: Token![;],
672 /// }
673 ///
674 /// impl Parse for TupleStruct {
675 /// fn parse(input: ParseStream) -> Result<Self> {
676 /// let content;
677 /// Ok(TupleStruct {
678 /// struct_token: input.parse()?,
679 /// ident: input.parse()?,
680 /// paren_token: parenthesized!(content in input),
681 /// fields: content.parse_terminated(Type::parse, Token![,])?,
682 /// semi_token: input.parse()?,
683 /// })
684 /// }
685 /// }
686 /// #
687 /// # let input = quote! {
688 /// # struct S(A, B);
689 /// # };
690 /// # syn::parse2::<TupleStruct>(input).unwrap();
691 /// ```
692 ///
693 /// # See also
694 ///
695 /// If your separator is anything more complicated than an invocation of the
696 /// `Token!` macro, this method won't be applicable and you can instead
697 /// directly use `Punctuated`'s parser functions: [`parse_terminated`],
698 /// [`parse_separated_nonempty`] etc.
699 ///
700 /// [`parse_terminated`]: Punctuated::parse_terminated
701 /// [`parse_separated_nonempty`]: Punctuated::parse_separated_nonempty
702 ///
703 /// ```
704 /// use syn::{custom_keyword, Expr, Result, Token};
705 /// use syn::parse::{Parse, ParseStream};
706 /// use syn::punctuated::Punctuated;
707 ///
708 /// mod kw {
709 /// syn::custom_keyword!(fin);
710 /// }
711 ///
712 /// struct Fin(kw::fin, Token![;]);
713 ///
714 /// impl Parse for Fin {
715 /// fn parse(input: ParseStream) -> Result<Self> {
716 /// Ok(Self(input.parse()?, input.parse()?))
717 /// }
718 /// }
719 ///
720 /// struct Thing {
721 /// steps: Punctuated<Expr, Fin>,
722 /// }
723 ///
724 /// impl Parse for Thing {
725 /// fn parse(input: ParseStream) -> Result<Self> {
726 /// # if true {
727 /// Ok(Thing {
728 /// steps: Punctuated::parse_terminated(input)?,
729 /// })
730 /// # } else {
731 /// // or equivalently, this means the same thing:
732 /// # Ok(Thing {
733 /// steps: input.call(Punctuated::parse_terminated)?,
734 /// # })
735 /// # }
736 /// }
737 /// }
738 /// ```
739 pub fn parse_terminated<T, P>(
740 &self,
741 parser: fn(ParseStream) -> Result<T>,
742 separator: P,
743 ) -> Result<Punctuated<T, P::Token>>
744 where
745 P: Peek,
746 P::Token: Parse,
747 {
748 let _ = separator;
749 Punctuated::parse_terminated_with(self, parser)
750 }
751
752 /// Returns whether there are tokens remaining in this stream.
753 ///
754 /// This method returns true at the end of the content of a set of
755 /// delimiters, as well as at the very end of the complete macro input.
756 ///
757 /// # Example
758 ///
759 /// ```
760 /// use syn::{braced, token, Ident, Item, Result, Token};
761 /// use syn::parse::{Parse, ParseStream};
762 ///
763 /// // Parses a Rust `mod m { ... }` containing zero or more items.
764 /// struct Mod {
765 /// mod_token: Token![mod],
766 /// name: Ident,
767 /// brace_token: token::Brace,
768 /// items: Vec<Item>,
769 /// }
770 ///
771 /// impl Parse for Mod {
772 /// fn parse(input: ParseStream) -> Result<Self> {
773 /// let content;
774 /// Ok(Mod {
775 /// mod_token: input.parse()?,
776 /// name: input.parse()?,
777 /// brace_token: braced!(content in input),
778 /// items: {
779 /// let mut items = Vec::new();
780 /// while !content.is_empty() {
781 /// items.push(content.parse()?);
782 /// }
783 /// items
784 /// },
785 /// })
786 /// }
787 /// }
788 /// ```
789 pub fn is_empty(&self) -> bool {
790 self.cursor().eof()
791 }
792
793 /// Constructs a helper for peeking at the next token in this stream and
794 /// building an error message if it is not one of a set of expected tokens.
795 ///
796 /// # Example
797 ///
798 /// ```
799 /// use syn::{ConstParam, Ident, Lifetime, LifetimeParam, Result, Token, TypeParam};
800 /// use syn::parse::{Parse, ParseStream};
801 ///
802 /// // A generic parameter, a single one of the comma-separated elements inside
803 /// // angle brackets in:
804 /// //
805 /// // fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... }
806 /// //
807 /// // On invalid input, lookahead gives us a reasonable error message.
808 /// //
809 /// // error: expected one of: identifier, lifetime, `const`
810 /// // |
811 /// // 5 | fn f<!Sized>() {}
812 /// // | ^
813 /// enum GenericParam {
814 /// Type(TypeParam),
815 /// Lifetime(LifetimeParam),
816 /// Const(ConstParam),
817 /// }
818 ///
819 /// impl Parse for GenericParam {
820 /// fn parse(input: ParseStream) -> Result<Self> {
821 /// let lookahead = input.lookahead1();
822 /// if lookahead.peek(Ident) {
823 /// input.parse().map(GenericParam::Type)
824 /// } else if lookahead.peek(Lifetime) {
825 /// input.parse().map(GenericParam::Lifetime)
826 /// } else if lookahead.peek(Token![const]) {
827 /// input.parse().map(GenericParam::Const)
828 /// } else {
829 /// Err(lookahead.error())
830 /// }
831 /// }
832 /// }
833 /// ```
834 pub fn lookahead1(&self) -> Lookahead1<'a> {
835 lookahead::new(self.scope, self.cursor())
836 }
837
838 /// Forks a parse stream so that parsing tokens out of either the original
839 /// or the fork does not advance the position of the other.
840 ///
841 /// # Performance
842 ///
843 /// Forking a parse stream is a cheap fixed amount of work and does not
844 /// involve copying token buffers. Where you might hit performance problems
845 /// is if your macro ends up parsing a large amount of content more than
846 /// once.
847 ///
848 /// ```
849 /// # use syn::{Expr, Result};
850 /// # use syn::parse::ParseStream;
851 /// #
852 /// # fn bad(input: ParseStream) -> Result<Expr> {
853 /// // Do not do this.
854 /// if input.fork().parse::<Expr>().is_ok() {
855 /// return input.parse::<Expr>();
856 /// }
857 /// # unimplemented!()
858 /// # }
859 /// ```
860 ///
861 /// As a rule, avoid parsing an unbounded amount of tokens out of a forked
862 /// parse stream. Only use a fork when the amount of work performed against
863 /// the fork is small and bounded.
864 ///
865 /// When complex speculative parsing against the forked stream is
866 /// unavoidable, use [`parse::discouraged::Speculative`] to advance the
867 /// original stream once the fork's parse is determined to have been
868 /// successful.
869 ///
870 /// For a lower level way to perform speculative parsing at the token level,
871 /// consider using [`ParseStream::step`] instead.
872 ///
873 /// [`parse::discouraged::Speculative`]: discouraged::Speculative
874 /// [`ParseStream::step`]: ParseBuffer::step
875 ///
876 /// # Example
877 ///
878 /// The parse implementation shown here parses possibly restricted `pub`
879 /// visibilities.
880 ///
881 /// - `pub`
882 /// - `pub(crate)`
883 /// - `pub(self)`
884 /// - `pub(super)`
885 /// - `pub(in some::path)`
886 ///
887 /// To handle the case of visibilities inside of tuple structs, the parser
888 /// needs to distinguish parentheses that specify visibility restrictions
889 /// from parentheses that form part of a tuple type.
890 ///
891 /// ```
892 /// # struct A;
893 /// # struct B;
894 /// # struct C;
895 /// #
896 /// struct S(pub(crate) A, pub (B, C));
897 /// ```
898 ///
899 /// In this example input the first tuple struct element of `S` has
900 /// `pub(crate)` visibility while the second tuple struct element has `pub`
901 /// visibility; the parentheses around `(B, C)` are part of the type rather
902 /// than part of a visibility restriction.
903 ///
904 /// The parser uses a forked parse stream to check the first token inside of
905 /// parentheses after the `pub` keyword. This is a small bounded amount of
906 /// work performed against the forked parse stream.
907 ///
908 /// ```
909 /// use syn::{parenthesized, token, Ident, Path, Result, Token};
910 /// use syn::ext::IdentExt;
911 /// use syn::parse::{Parse, ParseStream};
912 ///
913 /// struct PubVisibility {
914 /// pub_token: Token![pub],
915 /// restricted: Option<Restricted>,
916 /// }
917 ///
918 /// struct Restricted {
919 /// paren_token: token::Paren,
920 /// in_token: Option<Token![in]>,
921 /// path: Path,
922 /// }
923 ///
924 /// impl Parse for PubVisibility {
925 /// fn parse(input: ParseStream) -> Result<Self> {
926 /// let pub_token: Token![pub] = input.parse()?;
927 ///
928 /// if input.peek(token::Paren) {
929 /// let ahead = input.fork();
930 /// let mut content;
931 /// parenthesized!(content in ahead);
932 ///
933 /// if content.peek(Token![crate])
934 /// || content.peek(Token![self])
935 /// || content.peek(Token![super])
936 /// {
937 /// return Ok(PubVisibility {
938 /// pub_token,
939 /// restricted: Some(Restricted {
940 /// paren_token: parenthesized!(content in input),
941 /// in_token: None,
942 /// path: Path::from(content.call(Ident::parse_any)?),
943 /// }),
944 /// });
945 /// } else if content.peek(Token![in]) {
946 /// return Ok(PubVisibility {
947 /// pub_token,
948 /// restricted: Some(Restricted {
949 /// paren_token: parenthesized!(content in input),
950 /// in_token: Some(content.parse()?),
951 /// path: content.call(Path::parse_mod_style)?,
952 /// }),
953 /// });
954 /// }
955 /// }
956 ///
957 /// Ok(PubVisibility {
958 /// pub_token,
959 /// restricted: None,
960 /// })
961 /// }
962 /// }
963 /// ```
964 pub fn fork(&self) -> Self {
965 ParseBuffer {
966 scope: self.scope,
967 cell: self.cell.clone(),
968 marker: PhantomData,
969 // Not the parent's unexpected. Nothing cares whether the clone
970 // parses all the way unless we `advance_to`.
971 unexpected: Cell::new(Some(Rc::new(Cell::new(Unexpected::None)))),
972 }
973 }
974
975 /// Triggers an error at the current position of the parse stream.
976 ///
977 /// # Example
978 ///
979 /// ```
980 /// use syn::{Expr, Result, Token};
981 /// use syn::parse::{Parse, ParseStream};
982 ///
983 /// // Some kind of loop: `while` or `for` or `loop`.
984 /// struct Loop {
985 /// expr: Expr,
986 /// }
987 ///
988 /// impl Parse for Loop {
989 /// fn parse(input: ParseStream) -> Result<Self> {
990 /// if input.peek(Token![while])
991 /// || input.peek(Token![for])
992 /// || input.peek(Token![loop])
993 /// {
994 /// Ok(Loop {
995 /// expr: input.parse()?,
996 /// })
997 /// } else {
998 /// Err(input.error("expected some kind of loop"))
999 /// }
1000 /// }
1001 /// }
1002 /// ```
1003 pub fn error<T: Display>(&self, message: T) -> Error {
1004 error::new_at(self.scope, self.cursor(), message)
1005 }
1006
1007 /// Speculatively parses tokens from this parse stream, advancing the
1008 /// position of this stream only if parsing succeeds.
1009 ///
1010 /// This is a powerful low-level API used for defining the `Parse` impls of
1011 /// the basic built-in token types. It is not something that will be used
1012 /// widely outside of the Syn codebase.
1013 ///
1014 /// # Example
1015 ///
1016 /// ```
1017 /// use proc_macro2::TokenTree;
1018 /// use syn::Result;
1019 /// use syn::parse::ParseStream;
1020 ///
1021 /// // This function advances the stream past the next occurrence of `@`. If
1022 /// // no `@` is present in the stream, the stream position is unchanged and
1023 /// // an error is returned.
1024 /// fn skip_past_next_at(input: ParseStream) -> Result<()> {
1025 /// input.step(|cursor| {
1026 /// let mut rest = *cursor;
1027 /// while let Some((tt, next)) = rest.token_tree() {
1028 /// match &tt {
1029 /// TokenTree::Punct(punct) if punct.as_char() == '@' => {
1030 /// return Ok(((), next));
1031 /// }
1032 /// _ => rest = next,
1033 /// }
1034 /// }
1035 /// Err(cursor.error("no `@` was found after this point"))
1036 /// })
1037 /// }
1038 /// #
1039 /// # fn remainder_after_skipping_past_next_at(
1040 /// # input: ParseStream,
1041 /// # ) -> Result<proc_macro2::TokenStream> {
1042 /// # skip_past_next_at(input)?;
1043 /// # input.parse()
1044 /// # }
1045 /// #
1046 /// # use syn::parse::Parser;
1047 /// # let remainder = remainder_after_skipping_past_next_at
1048 /// # .parse_str("a @ b c")
1049 /// # .unwrap();
1050 /// # assert_eq!(remainder.to_string(), "b c");
1051 /// ```
1052 pub fn step<F, R>(&self, function: F) -> Result<R>
1053 where
1054 F: for<'c> FnOnce(StepCursor<'c, 'a>) -> Result<(R, Cursor<'c>)>,
1055 {
1056 // Since the user's function is required to work for any 'c, we know
1057 // that the Cursor<'c> they return is either derived from the input
1058 // StepCursor<'c, 'a> or from a Cursor<'static>.
1059 //
1060 // It would not be legal to write this function without the invariant
1061 // lifetime 'c in StepCursor<'c, 'a>. If this function were written only
1062 // in terms of 'a, the user could take our ParseBuffer<'a>, upcast it to
1063 // a ParseBuffer<'short> which some shorter lifetime than 'a, invoke
1064 // `step` on their ParseBuffer<'short> with a closure that returns
1065 // Cursor<'short>, and we would wrongly write that Cursor<'short> into
1066 // the Cell intended to hold Cursor<'a>.
1067 //
1068 // In some cases it may be necessary for R to contain a Cursor<'a>.
1069 // Within Syn we solve this using `advance_step_cursor` which uses the
1070 // existence of a StepCursor<'c, 'a> as proof that it is safe to cast
1071 // from Cursor<'c> to Cursor<'a>. If needed outside of Syn, it would be
1072 // safe to expose that API as a method on StepCursor.
1073 let (node, rest) = function(StepCursor {
1074 scope: self.scope,
1075 cursor: self.cell.get(),
1076 marker: PhantomData,
1077 })?;
1078 self.cell.set(rest);
1079 Ok(node)
1080 }
1081
1082 /// Returns the `Span` of the next token in the parse stream, or
1083 /// `Span::call_site()` if this parse stream has completely exhausted its
1084 /// input `TokenStream`.
1085 pub fn span(&self) -> Span {
1086 let cursor = self.cursor();
1087 if cursor.eof() {
1088 self.scope
1089 } else {
1090 crate::buffer::open_span_of_group(cursor)
1091 }
1092 }
1093
1094 /// Provides low-level access to the token representation underlying this
1095 /// parse stream.
1096 ///
1097 /// Cursors are immutable so no operations you perform against the cursor
1098 /// will affect the state of this parse stream.
1099 pub fn cursor(&self) -> Cursor<'a> {
1100 self.cell.get()
1101 }
1102
1103 fn check_unexpected(&self) -> Result<()> {
1104 match inner_unexpected(self).1 {
1105 Some(span) => Err(Error::new(span, "unexpected token")),
1106 None => Ok(()),
1107 }
1108 }
1109}
1110
1111#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
1112impl<T: Parse> Parse for Box<T> {
1113 fn parse(input: ParseStream) -> Result<Self> {
1114 input.parse().map(op:Box::new)
1115 }
1116}
1117
1118#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
1119impl<T: Parse + Token> Parse for Option<T> {
1120 fn parse(input: ParseStream) -> Result<Self> {
1121 if T::peek(input.cursor()) {
1122 Ok(Some(input.parse()?))
1123 } else {
1124 Ok(None)
1125 }
1126 }
1127}
1128
1129#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
1130impl Parse for TokenStream {
1131 fn parse(input: ParseStream) -> Result<Self> {
1132 input.step(|cursor: StepCursor<'_, '_>| Ok((cursor.token_stream(), Cursor::empty())))
1133 }
1134}
1135
1136#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
1137impl Parse for TokenTree {
1138 fn parse(input: ParseStream) -> Result<Self> {
1139 input.step(|cursor: StepCursor<'_, '_>| match cursor.token_tree() {
1140 Some((tt: TokenTree, rest: Cursor<'_>)) => Ok((tt, rest)),
1141 None => Err(cursor.error(message:"expected token tree")),
1142 })
1143 }
1144}
1145
1146#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
1147impl Parse for Group {
1148 fn parse(input: ParseStream) -> Result<Self> {
1149 input.step(|cursor: StepCursor<'_, '_>| {
1150 if let Some((group: Group, rest: Cursor<'_>)) = cursor.any_group_token() {
1151 if group.delimiter() != Delimiter::None {
1152 return Ok((group, rest));
1153 }
1154 }
1155 Err(cursor.error(message:"expected group token"))
1156 })
1157 }
1158}
1159
1160#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
1161impl Parse for Punct {
1162 fn parse(input: ParseStream) -> Result<Self> {
1163 input.step(|cursor: StepCursor<'_, '_>| match cursor.punct() {
1164 Some((punct: Punct, rest: Cursor<'_>)) => Ok((punct, rest)),
1165 None => Err(cursor.error(message:"expected punctuation token")),
1166 })
1167 }
1168}
1169
1170#[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))]
1171impl Parse for Literal {
1172 fn parse(input: ParseStream) -> Result<Self> {
1173 input.step(|cursor: StepCursor<'_, '_>| match cursor.literal() {
1174 Some((literal: Literal, rest: Cursor<'_>)) => Ok((literal, rest)),
1175 None => Err(cursor.error(message:"expected literal token")),
1176 })
1177 }
1178}
1179
1180/// Parser that can parse Rust tokens into a particular syntax tree node.
1181///
1182/// Refer to the [module documentation] for details about parsing in Syn.
1183///
1184/// [module documentation]: self
1185pub trait Parser: Sized {
1186 type Output;
1187
1188 /// Parse a proc-macro2 token stream into the chosen syntax tree node.
1189 ///
1190 /// This function will check that the input is fully parsed. If there are
1191 /// any unparsed tokens at the end of the stream, an error is returned.
1192 fn parse2(self, tokens: TokenStream) -> Result<Self::Output>;
1193
1194 /// Parse tokens of source code into the chosen syntax tree node.
1195 ///
1196 /// This function will check that the input is fully parsed. If there are
1197 /// any unparsed tokens at the end of the stream, an error is returned.
1198 #[cfg(feature = "proc-macro")]
1199 #[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))]
1200 fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> {
1201 self.parse2(proc_macro2::TokenStream::from(tokens))
1202 }
1203
1204 /// Parse a string of Rust code into the chosen syntax tree node.
1205 ///
1206 /// This function will check that the input is fully parsed. If there are
1207 /// any unparsed tokens at the end of the string, an error is returned.
1208 ///
1209 /// # Hygiene
1210 ///
1211 /// Every span in the resulting syntax tree will be set to resolve at the
1212 /// macro call site.
1213 fn parse_str(self, s: &str) -> Result<Self::Output> {
1214 self.parse2(proc_macro2::TokenStream::from_str(s)?)
1215 }
1216
1217 // Not public API.
1218 #[doc(hidden)]
1219 #[cfg(any(feature = "full", feature = "derive"))]
1220 fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> {
1221 let _ = scope;
1222 self.parse2(tokens)
1223 }
1224}
1225
1226fn tokens_to_parse_buffer(tokens: &TokenBuffer) -> ParseBuffer {
1227 let scope: Span = Span::call_site();
1228 let cursor: Cursor<'_> = tokens.begin();
1229 let unexpected: Rc> = Rc::new(Cell::new(Unexpected::None));
1230 new_parse_buffer(scope, cursor, unexpected)
1231}
1232
1233impl<F, T> Parser for F
1234where
1235 F: FnOnce(ParseStream) -> Result<T>,
1236{
1237 type Output = T;
1238
1239 fn parse2(self, tokens: TokenStream) -> Result<T> {
1240 let buf = TokenBuffer::new2(tokens);
1241 let state = tokens_to_parse_buffer(&buf);
1242 let node = self(&state)?;
1243 state.check_unexpected()?;
1244 if let Some(unexpected_span) = span_of_unexpected_ignoring_nones(state.cursor()) {
1245 Err(Error::new(unexpected_span, "unexpected token"))
1246 } else {
1247 Ok(node)
1248 }
1249 }
1250
1251 #[cfg(any(feature = "full", feature = "derive"))]
1252 fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> {
1253 let buf = TokenBuffer::new2(tokens);
1254 let cursor = buf.begin();
1255 let unexpected = Rc::new(Cell::new(Unexpected::None));
1256 let state = new_parse_buffer(scope, cursor, unexpected);
1257 let node = self(&state)?;
1258 state.check_unexpected()?;
1259 if let Some(unexpected_span) = span_of_unexpected_ignoring_nones(state.cursor()) {
1260 Err(Error::new(unexpected_span, "unexpected token"))
1261 } else {
1262 Ok(node)
1263 }
1264 }
1265}
1266
1267#[cfg(any(feature = "full", feature = "derive"))]
1268pub(crate) fn parse_scoped<F: Parser>(f: F, scope: Span, tokens: TokenStream) -> Result<F::Output> {
1269 f.__parse_scoped(scope, tokens)
1270}
1271
1272/// An empty syntax tree node that consumes no tokens when parsed.
1273///
1274/// This is useful for attribute macros that want to ensure they are not
1275/// provided any attribute args.
1276///
1277/// ```
1278/// # extern crate proc_macro;
1279/// #
1280/// use proc_macro::TokenStream;
1281/// use syn::parse_macro_input;
1282/// use syn::parse::Nothing;
1283///
1284/// # const IGNORE: &str = stringify! {
1285/// #[proc_macro_attribute]
1286/// # };
1287/// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream {
1288/// parse_macro_input!(args as Nothing);
1289///
1290/// /* ... */
1291/// # TokenStream::new()
1292/// }
1293/// ```
1294///
1295/// ```text
1296/// error: unexpected token
1297/// --> src/main.rs:3:19
1298/// |
1299/// 3 | #[my_attr(asdf)]
1300/// | ^^^^
1301/// ```
1302pub struct Nothing;
1303
1304impl Parse for Nothing {
1305 fn parse(_input: ParseStream) -> Result<Self> {
1306 Ok(Nothing)
1307 }
1308}
1309
1310#[cfg(feature = "extra-traits")]
1311#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
1312impl Debug for Nothing {
1313 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1314 f.write_str(data:"Nothing")
1315 }
1316}
1317
1318#[cfg(feature = "extra-traits")]
1319#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
1320impl Eq for Nothing {}
1321
1322#[cfg(feature = "extra-traits")]
1323#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
1324impl PartialEq for Nothing {
1325 fn eq(&self, _other: &Self) -> bool {
1326 true
1327 }
1328}
1329
1330#[cfg(feature = "extra-traits")]
1331#[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))]
1332impl Hash for Nothing {
1333 fn hash<H: Hasher>(&self, _state: &mut H) {}
1334}
1335