1 | //! Tokens representing Rust punctuation, keywords, and delimiters. |
2 | //! |
3 | //! The type names in this module can be difficult to keep straight, so we |
4 | //! prefer to use the [`Token!`] macro instead. This is a type-macro that |
5 | //! expands to the token type of the given token. |
6 | //! |
7 | //! [`Token!`]: crate::Token |
8 | //! |
9 | //! # Example |
10 | //! |
11 | //! The [`ItemStatic`] syntax tree node is defined like this. |
12 | //! |
13 | //! [`ItemStatic`]: crate::ItemStatic |
14 | //! |
15 | //! ``` |
16 | //! # use syn::{Attribute, Expr, Ident, Token, Type, Visibility}; |
17 | //! # |
18 | //! pub struct ItemStatic { |
19 | //! pub attrs: Vec<Attribute>, |
20 | //! pub vis: Visibility, |
21 | //! pub static_token: Token![static], |
22 | //! pub mutability: Option<Token![mut]>, |
23 | //! pub ident: Ident, |
24 | //! pub colon_token: Token![:], |
25 | //! pub ty: Box<Type>, |
26 | //! pub eq_token: Token![=], |
27 | //! pub expr: Box<Expr>, |
28 | //! pub semi_token: Token![;], |
29 | //! } |
30 | //! ``` |
31 | //! |
32 | //! # Parsing |
33 | //! |
34 | //! Keywords and punctuation can be parsed through the [`ParseStream::parse`] |
35 | //! method. Delimiter tokens are parsed using the [`parenthesized!`], |
36 | //! [`bracketed!`] and [`braced!`] macros. |
37 | //! |
38 | //! [`ParseStream::parse`]: crate::parse::ParseBuffer::parse() |
39 | //! [`parenthesized!`]: crate::parenthesized! |
40 | //! [`bracketed!`]: crate::bracketed! |
41 | //! [`braced!`]: crate::braced! |
42 | //! |
43 | //! ``` |
44 | //! use syn::{Attribute, Result}; |
45 | //! use syn::parse::{Parse, ParseStream}; |
46 | //! # |
47 | //! # enum ItemStatic {} |
48 | //! |
49 | //! // Parse the ItemStatic struct shown above. |
50 | //! impl Parse for ItemStatic { |
51 | //! fn parse(input: ParseStream) -> Result<Self> { |
52 | //! # use syn::ItemStatic; |
53 | //! # fn parse(input: ParseStream) -> Result<ItemStatic> { |
54 | //! Ok(ItemStatic { |
55 | //! attrs: input.call(Attribute::parse_outer)?, |
56 | //! vis: input.parse()?, |
57 | //! static_token: input.parse()?, |
58 | //! mutability: input.parse()?, |
59 | //! ident: input.parse()?, |
60 | //! colon_token: input.parse()?, |
61 | //! ty: input.parse()?, |
62 | //! eq_token: input.parse()?, |
63 | //! expr: input.parse()?, |
64 | //! semi_token: input.parse()?, |
65 | //! }) |
66 | //! # } |
67 | //! # unimplemented!() |
68 | //! } |
69 | //! } |
70 | //! ``` |
71 | //! |
72 | //! # Other operations |
73 | //! |
74 | //! Every keyword and punctuation token supports the following operations. |
75 | //! |
76 | //! - [Peeking] — `input.peek(Token![...])` |
77 | //! |
78 | //! - [Parsing] — `input.parse::<Token![...]>()?` |
79 | //! |
80 | //! - [Printing] — `quote!( ... #the_token ... )` |
81 | //! |
82 | //! - Construction from a [`Span`] — `let the_token = Token![...](sp)` |
83 | //! |
84 | //! - Field access to its span — `let sp = the_token.span` |
85 | //! |
86 | //! [Peeking]: crate::parse::ParseBuffer::peek() |
87 | //! [Parsing]: crate::parse::ParseBuffer::parse() |
88 | //! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html |
89 | //! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html |
90 | |
91 | use self::private::WithSpan; |
92 | #[cfg (feature = "parsing" )] |
93 | use crate::buffer::Cursor; |
94 | #[cfg (feature = "parsing" )] |
95 | use crate::error::Result; |
96 | #[cfg (feature = "parsing" )] |
97 | use crate::lifetime::Lifetime; |
98 | #[cfg (feature = "parsing" )] |
99 | use crate::lit::{Lit, LitBool, LitByte, LitByteStr, LitChar, LitFloat, LitInt, LitStr}; |
100 | #[cfg (feature = "parsing" )] |
101 | use crate::lookahead; |
102 | #[cfg (feature = "parsing" )] |
103 | use crate::parse::{Parse, ParseStream}; |
104 | use crate::span::IntoSpans; |
105 | #[cfg (any(feature = "parsing" , feature = "printing" ))] |
106 | use proc_macro2::Ident; |
107 | use proc_macro2::Span; |
108 | #[cfg (feature = "printing" )] |
109 | use proc_macro2::TokenStream; |
110 | #[cfg (feature = "parsing" )] |
111 | use proc_macro2::{Delimiter, Literal, Punct, TokenTree}; |
112 | #[cfg (feature = "printing" )] |
113 | use quote::{ToTokens, TokenStreamExt}; |
114 | #[cfg (feature = "extra-traits" )] |
115 | use std::cmp; |
116 | #[cfg (feature = "extra-traits" )] |
117 | use std::fmt::{self, Debug}; |
118 | #[cfg (feature = "extra-traits" )] |
119 | use std::hash::{Hash, Hasher}; |
120 | use std::ops::{Deref, DerefMut}; |
121 | |
122 | /// Marker trait for types that represent single tokens. |
123 | /// |
124 | /// This trait is sealed and cannot be implemented for types outside of Syn. |
125 | #[cfg (feature = "parsing" )] |
126 | pub trait Token: private::Sealed { |
127 | // Not public API. |
128 | #[doc (hidden)] |
129 | fn peek(cursor: Cursor) -> bool; |
130 | |
131 | // Not public API. |
132 | #[doc (hidden)] |
133 | fn display() -> &'static str; |
134 | } |
135 | |
136 | mod private { |
137 | use proc_macro2::Span; |
138 | |
139 | #[cfg (feature = "parsing" )] |
140 | pub trait Sealed {} |
141 | |
142 | /// Support writing `token.span` rather than `token.spans[0]` on tokens that |
143 | /// hold a single span. |
144 | #[repr (C)] |
145 | pub struct WithSpan { |
146 | pub span: Span, |
147 | } |
148 | } |
149 | |
150 | #[cfg (feature = "parsing" )] |
151 | impl private::Sealed for Ident {} |
152 | |
153 | #[cfg (feature = "parsing" )] |
154 | fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool { |
155 | use crate::parse::Unexpected; |
156 | use std::cell::Cell; |
157 | use std::rc::Rc; |
158 | |
159 | let scope: Span = Span::call_site(); |
160 | let unexpected: Rc> | = Rc::new(Cell::new(Unexpected::None)); |
161 | let buffer: ParseBuffer<'_> = crate::parse::new_parse_buffer(scope, cursor, unexpected); |
162 | peek(&buffer) |
163 | } |
164 | |
165 | macro_rules! impl_token { |
166 | ($display:tt $name:ty) => { |
167 | #[cfg(feature = "parsing" )] |
168 | impl Token for $name { |
169 | fn peek(cursor: Cursor) -> bool { |
170 | fn peek(input: ParseStream) -> bool { |
171 | <$name as Parse>::parse(input).is_ok() |
172 | } |
173 | peek_impl(cursor, peek) |
174 | } |
175 | |
176 | fn display() -> &'static str { |
177 | $display |
178 | } |
179 | } |
180 | |
181 | #[cfg(feature = "parsing" )] |
182 | impl private::Sealed for $name {} |
183 | }; |
184 | } |
185 | |
186 | impl_token!("lifetime" Lifetime); |
187 | impl_token!("literal" Lit); |
188 | impl_token!("string literal" LitStr); |
189 | impl_token!("byte string literal" LitByteStr); |
190 | impl_token!("byte literal" LitByte); |
191 | impl_token!("character literal" LitChar); |
192 | impl_token!("integer literal" LitInt); |
193 | impl_token!("floating point literal" LitFloat); |
194 | impl_token!("boolean literal" LitBool); |
195 | impl_token!("group token" proc_macro2::Group); |
196 | |
197 | macro_rules! impl_low_level_token { |
198 | ($display:tt $ty:ident $get:ident) => { |
199 | #[cfg(feature = "parsing" )] |
200 | impl Token for $ty { |
201 | fn peek(cursor: Cursor) -> bool { |
202 | cursor.$get().is_some() |
203 | } |
204 | |
205 | fn display() -> &'static str { |
206 | $display |
207 | } |
208 | } |
209 | |
210 | #[cfg(feature = "parsing" )] |
211 | impl private::Sealed for $ty {} |
212 | }; |
213 | } |
214 | |
215 | impl_low_level_token!("punctuation token" Punct punct); |
216 | impl_low_level_token!("literal" Literal literal); |
217 | impl_low_level_token!("token" TokenTree token_tree); |
218 | |
219 | // Not public API. |
220 | #[doc (hidden)] |
221 | #[cfg (feature = "parsing" )] |
222 | pub trait CustomToken { |
223 | fn peek(cursor: Cursor) -> bool; |
224 | fn display() -> &'static str; |
225 | } |
226 | |
227 | #[cfg (feature = "parsing" )] |
228 | impl<T: CustomToken> private::Sealed for T {} |
229 | |
230 | #[cfg (feature = "parsing" )] |
231 | impl<T: CustomToken> Token for T { |
232 | fn peek(cursor: Cursor) -> bool { |
233 | <Self as CustomToken>::peek(cursor) |
234 | } |
235 | |
236 | fn display() -> &'static str { |
237 | <Self as CustomToken>::display() |
238 | } |
239 | } |
240 | |
241 | macro_rules! define_keywords { |
242 | ($($token:tt pub struct $name:ident #[$doc:meta])*) => { |
243 | $( |
244 | #[$doc] |
245 | /// |
246 | /// Don't try to remember the name of this type — use the |
247 | /// [`Token!`] macro instead. |
248 | /// |
249 | /// [`Token!`]: crate::token |
250 | pub struct $name { |
251 | pub span: Span, |
252 | } |
253 | |
254 | #[doc(hidden)] |
255 | #[allow(non_snake_case)] |
256 | pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name { |
257 | $name { |
258 | span: span.into_spans()[0], |
259 | } |
260 | } |
261 | |
262 | impl std::default::Default for $name { |
263 | fn default() -> Self { |
264 | $name { |
265 | span: Span::call_site(), |
266 | } |
267 | } |
268 | } |
269 | |
270 | #[cfg(feature = "clone-impls" )] |
271 | #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls" )))] |
272 | impl Copy for $name {} |
273 | |
274 | #[cfg(feature = "clone-impls" )] |
275 | #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls" )))] |
276 | impl Clone for $name { |
277 | fn clone(&self) -> Self { |
278 | *self |
279 | } |
280 | } |
281 | |
282 | #[cfg(feature = "extra-traits" )] |
283 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
284 | impl Debug for $name { |
285 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
286 | f.write_str(stringify!($name)) |
287 | } |
288 | } |
289 | |
290 | #[cfg(feature = "extra-traits" )] |
291 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
292 | impl cmp::Eq for $name {} |
293 | |
294 | #[cfg(feature = "extra-traits" )] |
295 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
296 | impl PartialEq for $name { |
297 | fn eq(&self, _other: &$name) -> bool { |
298 | true |
299 | } |
300 | } |
301 | |
302 | #[cfg(feature = "extra-traits" )] |
303 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
304 | impl Hash for $name { |
305 | fn hash<H: Hasher>(&self, _state: &mut H) {} |
306 | } |
307 | |
308 | #[cfg(feature = "printing" )] |
309 | #[cfg_attr(doc_cfg, doc(cfg(feature = "printing" )))] |
310 | impl ToTokens for $name { |
311 | fn to_tokens(&self, tokens: &mut TokenStream) { |
312 | printing::keyword($token, self.span, tokens); |
313 | } |
314 | } |
315 | |
316 | #[cfg(feature = "parsing" )] |
317 | #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing" )))] |
318 | impl Parse for $name { |
319 | fn parse(input: ParseStream) -> Result<Self> { |
320 | Ok($name { |
321 | span: parsing::keyword(input, $token)?, |
322 | }) |
323 | } |
324 | } |
325 | |
326 | #[cfg(feature = "parsing" )] |
327 | impl Token for $name { |
328 | fn peek(cursor: Cursor) -> bool { |
329 | parsing::peek_keyword(cursor, $token) |
330 | } |
331 | |
332 | fn display() -> &'static str { |
333 | concat!("`" , $token, "`" ) |
334 | } |
335 | } |
336 | |
337 | #[cfg(feature = "parsing" )] |
338 | impl private::Sealed for $name {} |
339 | )* |
340 | }; |
341 | } |
342 | |
343 | macro_rules! impl_deref_if_len_is_1 { |
344 | ($name:ident/1) => { |
345 | impl Deref for $name { |
346 | type Target = WithSpan; |
347 | |
348 | fn deref(&self) -> &Self::Target { |
349 | unsafe { &*(self as *const Self as *const WithSpan) } |
350 | } |
351 | } |
352 | |
353 | impl DerefMut for $name { |
354 | fn deref_mut(&mut self) -> &mut Self::Target { |
355 | unsafe { &mut *(self as *mut Self as *mut WithSpan) } |
356 | } |
357 | } |
358 | }; |
359 | |
360 | ($name:ident/$len:tt) => {}; |
361 | } |
362 | |
363 | macro_rules! define_punctuation_structs { |
364 | ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => { |
365 | $( |
366 | #[repr(C)] |
367 | #[$doc] |
368 | /// |
369 | /// Don't try to remember the name of this type — use the |
370 | /// [`Token!`] macro instead. |
371 | /// |
372 | /// [`Token!`]: crate::token |
373 | pub struct $name { |
374 | pub spans: [Span; $len], |
375 | } |
376 | |
377 | #[doc(hidden)] |
378 | #[allow(non_snake_case)] |
379 | pub fn $name<S: IntoSpans<[Span; $len]>>(spans: S) -> $name { |
380 | $name { |
381 | spans: spans.into_spans(), |
382 | } |
383 | } |
384 | |
385 | impl std::default::Default for $name { |
386 | fn default() -> Self { |
387 | $name { |
388 | spans: [Span::call_site(); $len], |
389 | } |
390 | } |
391 | } |
392 | |
393 | #[cfg(feature = "clone-impls" )] |
394 | #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls" )))] |
395 | impl Copy for $name {} |
396 | |
397 | #[cfg(feature = "clone-impls" )] |
398 | #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls" )))] |
399 | impl Clone for $name { |
400 | fn clone(&self) -> Self { |
401 | *self |
402 | } |
403 | } |
404 | |
405 | #[cfg(feature = "extra-traits" )] |
406 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
407 | impl Debug for $name { |
408 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
409 | f.write_str(stringify!($name)) |
410 | } |
411 | } |
412 | |
413 | #[cfg(feature = "extra-traits" )] |
414 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
415 | impl cmp::Eq for $name {} |
416 | |
417 | #[cfg(feature = "extra-traits" )] |
418 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
419 | impl PartialEq for $name { |
420 | fn eq(&self, _other: &$name) -> bool { |
421 | true |
422 | } |
423 | } |
424 | |
425 | #[cfg(feature = "extra-traits" )] |
426 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
427 | impl Hash for $name { |
428 | fn hash<H: Hasher>(&self, _state: &mut H) {} |
429 | } |
430 | |
431 | impl_deref_if_len_is_1!($name/$len); |
432 | )* |
433 | }; |
434 | } |
435 | |
436 | macro_rules! define_punctuation { |
437 | ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => { |
438 | $( |
439 | define_punctuation_structs! { |
440 | $token pub struct $name/$len #[$doc] |
441 | } |
442 | |
443 | #[cfg(feature = "printing" )] |
444 | #[cfg_attr(doc_cfg, doc(cfg(feature = "printing" )))] |
445 | impl ToTokens for $name { |
446 | fn to_tokens(&self, tokens: &mut TokenStream) { |
447 | printing::punct($token, &self.spans, tokens); |
448 | } |
449 | } |
450 | |
451 | #[cfg(feature = "parsing" )] |
452 | #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing" )))] |
453 | impl Parse for $name { |
454 | fn parse(input: ParseStream) -> Result<Self> { |
455 | Ok($name { |
456 | spans: parsing::punct(input, $token)?, |
457 | }) |
458 | } |
459 | } |
460 | |
461 | #[cfg(feature = "parsing" )] |
462 | impl Token for $name { |
463 | fn peek(cursor: Cursor) -> bool { |
464 | parsing::peek_punct(cursor, $token) |
465 | } |
466 | |
467 | fn display() -> &'static str { |
468 | concat!("`" , $token, "`" ) |
469 | } |
470 | } |
471 | |
472 | #[cfg(feature = "parsing" )] |
473 | impl private::Sealed for $name {} |
474 | )* |
475 | }; |
476 | } |
477 | |
478 | macro_rules! define_delimiters { |
479 | ($($token:tt pub struct $name:ident #[$doc:meta])*) => { |
480 | $( |
481 | #[$doc] |
482 | pub struct $name { |
483 | pub span: Span, |
484 | } |
485 | |
486 | #[doc(hidden)] |
487 | #[allow(non_snake_case)] |
488 | pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name { |
489 | $name { |
490 | span: span.into_spans()[0], |
491 | } |
492 | } |
493 | |
494 | impl std::default::Default for $name { |
495 | fn default() -> Self { |
496 | $name { |
497 | span: Span::call_site(), |
498 | } |
499 | } |
500 | } |
501 | |
502 | #[cfg(feature = "clone-impls" )] |
503 | #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls" )))] |
504 | impl Copy for $name {} |
505 | |
506 | #[cfg(feature = "clone-impls" )] |
507 | #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls" )))] |
508 | impl Clone for $name { |
509 | fn clone(&self) -> Self { |
510 | *self |
511 | } |
512 | } |
513 | |
514 | #[cfg(feature = "extra-traits" )] |
515 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
516 | impl Debug for $name { |
517 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
518 | f.write_str(stringify!($name)) |
519 | } |
520 | } |
521 | |
522 | #[cfg(feature = "extra-traits" )] |
523 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
524 | impl cmp::Eq for $name {} |
525 | |
526 | #[cfg(feature = "extra-traits" )] |
527 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
528 | impl PartialEq for $name { |
529 | fn eq(&self, _other: &$name) -> bool { |
530 | true |
531 | } |
532 | } |
533 | |
534 | #[cfg(feature = "extra-traits" )] |
535 | #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits" )))] |
536 | impl Hash for $name { |
537 | fn hash<H: Hasher>(&self, _state: &mut H) {} |
538 | } |
539 | |
540 | impl $name { |
541 | #[cfg(feature = "printing" )] |
542 | pub fn surround<F>(&self, tokens: &mut TokenStream, f: F) |
543 | where |
544 | F: FnOnce(&mut TokenStream), |
545 | { |
546 | printing::delim($token, self.span, tokens, f); |
547 | } |
548 | } |
549 | |
550 | #[cfg(feature = "parsing" )] |
551 | impl private::Sealed for $name {} |
552 | )* |
553 | }; |
554 | } |
555 | |
556 | define_punctuation_structs! { |
557 | "_" pub struct Underscore/1 /// `_` |
558 | } |
559 | |
560 | #[cfg (feature = "printing" )] |
561 | #[cfg_attr (doc_cfg, doc(cfg(feature = "printing" )))] |
562 | impl ToTokens for Underscore { |
563 | fn to_tokens(&self, tokens: &mut TokenStream) { |
564 | tokens.append(token:Ident::new(string:"_" , self.span)); |
565 | } |
566 | } |
567 | |
568 | #[cfg (feature = "parsing" )] |
569 | #[cfg_attr (doc_cfg, doc(cfg(feature = "parsing" )))] |
570 | impl Parse for Underscore { |
571 | fn parse(input: ParseStream) -> Result<Self> { |
572 | input.step(|cursor: StepCursor<'_, '_>| { |
573 | if let Some((ident: Ident, rest: Cursor<'_>)) = cursor.ident() { |
574 | if ident == "_" { |
575 | return Ok((Underscore(spans:ident.span()), rest)); |
576 | } |
577 | } |
578 | if let Some((punct: Punct, rest: Cursor<'_>)) = cursor.punct() { |
579 | if punct.as_char() == '_' { |
580 | return Ok((Underscore(spans:punct.span()), rest)); |
581 | } |
582 | } |
583 | Err(cursor.error(message:"expected `_`" )) |
584 | }) |
585 | } |
586 | } |
587 | |
588 | #[cfg (feature = "parsing" )] |
589 | impl Token for Underscore { |
590 | fn peek(cursor: Cursor) -> bool { |
591 | if let Some((ident: Ident, _rest: Cursor<'_>)) = cursor.ident() { |
592 | return ident == "_" ; |
593 | } |
594 | if let Some((punct: Punct, _rest: Cursor<'_>)) = cursor.punct() { |
595 | return punct.as_char() == '_' ; |
596 | } |
597 | false |
598 | } |
599 | |
600 | fn display() -> &'static str { |
601 | "`_`" |
602 | } |
603 | } |
604 | |
605 | #[cfg (feature = "parsing" )] |
606 | impl private::Sealed for Underscore {} |
607 | |
608 | #[cfg (feature = "parsing" )] |
609 | impl Token for Paren { |
610 | fn peek(cursor: Cursor) -> bool { |
611 | lookahead::is_delimiter(cursor, Delimiter::Parenthesis) |
612 | } |
613 | |
614 | fn display() -> &'static str { |
615 | "parentheses" |
616 | } |
617 | } |
618 | |
619 | #[cfg (feature = "parsing" )] |
620 | impl Token for Brace { |
621 | fn peek(cursor: Cursor) -> bool { |
622 | lookahead::is_delimiter(cursor, Delimiter::Brace) |
623 | } |
624 | |
625 | fn display() -> &'static str { |
626 | "curly braces" |
627 | } |
628 | } |
629 | |
630 | #[cfg (feature = "parsing" )] |
631 | impl Token for Bracket { |
632 | fn peek(cursor: Cursor) -> bool { |
633 | lookahead::is_delimiter(cursor, Delimiter::Bracket) |
634 | } |
635 | |
636 | fn display() -> &'static str { |
637 | "square brackets" |
638 | } |
639 | } |
640 | |
641 | #[cfg (feature = "parsing" )] |
642 | impl Token for Group { |
643 | fn peek(cursor: Cursor) -> bool { |
644 | lookahead::is_delimiter(cursor, Delimiter::None) |
645 | } |
646 | |
647 | fn display() -> &'static str { |
648 | "invisible group" |
649 | } |
650 | } |
651 | |
652 | define_keywords! { |
653 | "abstract" pub struct Abstract /// `abstract` |
654 | "as" pub struct As /// `as` |
655 | "async" pub struct Async /// `async` |
656 | "auto" pub struct Auto /// `auto` |
657 | "await" pub struct Await /// `await` |
658 | "become" pub struct Become /// `become` |
659 | "box" pub struct Box /// `box` |
660 | "break" pub struct Break /// `break` |
661 | "const" pub struct Const /// `const` |
662 | "continue" pub struct Continue /// `continue` |
663 | "crate" pub struct Crate /// `crate` |
664 | "default" pub struct Default /// `default` |
665 | "do" pub struct Do /// `do` |
666 | "dyn" pub struct Dyn /// `dyn` |
667 | "else" pub struct Else /// `else` |
668 | "enum" pub struct Enum /// `enum` |
669 | "extern" pub struct Extern /// `extern` |
670 | "final" pub struct Final /// `final` |
671 | "fn" pub struct Fn /// `fn` |
672 | "for" pub struct For /// `for` |
673 | "if" pub struct If /// `if` |
674 | "impl" pub struct Impl /// `impl` |
675 | "in" pub struct In /// `in` |
676 | "let" pub struct Let /// `let` |
677 | "loop" pub struct Loop /// `loop` |
678 | "macro" pub struct Macro /// `macro` |
679 | "match" pub struct Match /// `match` |
680 | "mod" pub struct Mod /// `mod` |
681 | "move" pub struct Move /// `move` |
682 | "mut" pub struct Mut /// `mut` |
683 | "override" pub struct Override /// `override` |
684 | "priv" pub struct Priv /// `priv` |
685 | "pub" pub struct Pub /// `pub` |
686 | "ref" pub struct Ref /// `ref` |
687 | "return" pub struct Return /// `return` |
688 | "Self" pub struct SelfType /// `Self` |
689 | "self" pub struct SelfValue /// `self` |
690 | "static" pub struct Static /// `static` |
691 | "struct" pub struct Struct /// `struct` |
692 | "super" pub struct Super /// `super` |
693 | "trait" pub struct Trait /// `trait` |
694 | "try" pub struct Try /// `try` |
695 | "type" pub struct Type /// `type` |
696 | "typeof" pub struct Typeof /// `typeof` |
697 | "union" pub struct Union /// `union` |
698 | "unsafe" pub struct Unsafe /// `unsafe` |
699 | "unsized" pub struct Unsized /// `unsized` |
700 | "use" pub struct Use /// `use` |
701 | "virtual" pub struct Virtual /// `virtual` |
702 | "where" pub struct Where /// `where` |
703 | "while" pub struct While /// `while` |
704 | "yield" pub struct Yield /// `yield` |
705 | } |
706 | |
707 | define_punctuation! { |
708 | "+" pub struct Add/1 /// `+` |
709 | "+=" pub struct AddEq/2 /// `+=` |
710 | "&" pub struct And/1 /// `&` |
711 | "&&" pub struct AndAnd/2 /// `&&` |
712 | "&=" pub struct AndEq/2 /// `&=` |
713 | "@" pub struct At/1 /// `@` |
714 | "!" pub struct Bang/1 /// `!` |
715 | "^" pub struct Caret/1 /// `^` |
716 | "^=" pub struct CaretEq/2 /// `^=` |
717 | ":" pub struct Colon/1 /// `:` |
718 | "::" pub struct Colon2/2 /// `::` |
719 | "," pub struct Comma/1 /// `,` |
720 | "/" pub struct Div/1 /// `/` |
721 | "/=" pub struct DivEq/2 /// `/=` |
722 | "$" pub struct Dollar/1 /// `$` |
723 | "." pub struct Dot/1 /// `.` |
724 | ".." pub struct Dot2/2 /// `..` |
725 | "..." pub struct Dot3/3 /// `...` |
726 | "..=" pub struct DotDotEq/3 /// `..=` |
727 | "=" pub struct Eq/1 /// `=` |
728 | "==" pub struct EqEq/2 /// `==` |
729 | ">=" pub struct Ge/2 /// `>=` |
730 | ">" pub struct Gt/1 /// `>` |
731 | "<=" pub struct Le/2 /// `<=` |
732 | "<" pub struct Lt/1 /// `<` |
733 | "*=" pub struct MulEq/2 /// `*=` |
734 | "!=" pub struct Ne/2 /// `!=` |
735 | "|" pub struct Or/1 /// `|` |
736 | "|=" pub struct OrEq/2 /// `|=` |
737 | "||" pub struct OrOr/2 /// `||` |
738 | "#" pub struct Pound/1 /// `#` |
739 | "?" pub struct Question/1 /// `?` |
740 | "->" pub struct RArrow/2 /// `->` |
741 | "<-" pub struct LArrow/2 /// `<-` |
742 | "%" pub struct Rem/1 /// `%` |
743 | "%=" pub struct RemEq/2 /// `%=` |
744 | "=>" pub struct FatArrow/2 /// `=>` |
745 | ";" pub struct Semi/1 /// `;` |
746 | "<<" pub struct Shl/2 /// `<<` |
747 | "<<=" pub struct ShlEq/3 /// `<<=` |
748 | ">>" pub struct Shr/2 /// `>>` |
749 | ">>=" pub struct ShrEq/3 /// `>>=` |
750 | "*" pub struct Star/1 /// `*` |
751 | "-" pub struct Sub/1 /// `-` |
752 | "-=" pub struct SubEq/2 /// `-=` |
753 | "~" pub struct Tilde/1 /// `~` |
754 | } |
755 | |
756 | define_delimiters! { |
757 | "{" pub struct Brace /// `{...}` |
758 | "[" pub struct Bracket /// `[...]` |
759 | "(" pub struct Paren /// `(...)` |
760 | " " pub struct Group /// None-delimited group |
761 | } |
762 | |
763 | macro_rules! export_token_macro { |
764 | ($($await_rule:tt)*) => { |
765 | /// A type-macro that expands to the name of the Rust type representation of a |
766 | /// given token. |
767 | /// |
768 | /// See the [token module] documentation for details and examples. |
769 | /// |
770 | /// [token module]: crate::token |
771 | // Unfortunate duplication due to a rustdoc bug. |
772 | // https://github.com/rust-lang/rust/issues/45939 |
773 | #[macro_export] |
774 | macro_rules! Token { |
775 | [abstract] => { $crate::token::Abstract }; |
776 | [as] => { $crate::token::As }; |
777 | [async] => { $crate::token::Async }; |
778 | [auto] => { $crate::token::Auto }; |
779 | $($await_rule => { $crate::token::Await };)* |
780 | [become] => { $crate::token::Become }; |
781 | [box] => { $crate::token::Box }; |
782 | [break] => { $crate::token::Break }; |
783 | [const] => { $crate::token::Const }; |
784 | [continue] => { $crate::token::Continue }; |
785 | [crate] => { $crate::token::Crate }; |
786 | [default] => { $crate::token::Default }; |
787 | [do] => { $crate::token::Do }; |
788 | [dyn] => { $crate::token::Dyn }; |
789 | [else] => { $crate::token::Else }; |
790 | [enum] => { $crate::token::Enum }; |
791 | [extern] => { $crate::token::Extern }; |
792 | [final] => { $crate::token::Final }; |
793 | [fn] => { $crate::token::Fn }; |
794 | [for] => { $crate::token::For }; |
795 | [if] => { $crate::token::If }; |
796 | [impl] => { $crate::token::Impl }; |
797 | [in] => { $crate::token::In }; |
798 | [let] => { $crate::token::Let }; |
799 | [loop] => { $crate::token::Loop }; |
800 | [macro] => { $crate::token::Macro }; |
801 | [match] => { $crate::token::Match }; |
802 | [mod] => { $crate::token::Mod }; |
803 | [move] => { $crate::token::Move }; |
804 | [mut] => { $crate::token::Mut }; |
805 | [override] => { $crate::token::Override }; |
806 | [priv] => { $crate::token::Priv }; |
807 | [pub] => { $crate::token::Pub }; |
808 | [ref] => { $crate::token::Ref }; |
809 | [return] => { $crate::token::Return }; |
810 | [Self] => { $crate::token::SelfType }; |
811 | [self] => { $crate::token::SelfValue }; |
812 | [static] => { $crate::token::Static }; |
813 | [struct] => { $crate::token::Struct }; |
814 | [super] => { $crate::token::Super }; |
815 | [trait] => { $crate::token::Trait }; |
816 | [try] => { $crate::token::Try }; |
817 | [type] => { $crate::token::Type }; |
818 | [typeof] => { $crate::token::Typeof }; |
819 | [union] => { $crate::token::Union }; |
820 | [unsafe] => { $crate::token::Unsafe }; |
821 | [unsized] => { $crate::token::Unsized }; |
822 | [use] => { $crate::token::Use }; |
823 | [virtual] => { $crate::token::Virtual }; |
824 | [where] => { $crate::token::Where }; |
825 | [while] => { $crate::token::While }; |
826 | [yield] => { $crate::token::Yield }; |
827 | [+] => { $crate::token::Add }; |
828 | [+=] => { $crate::token::AddEq }; |
829 | [&] => { $crate::token::And }; |
830 | [&&] => { $crate::token::AndAnd }; |
831 | [&=] => { $crate::token::AndEq }; |
832 | [@] => { $crate::token::At }; |
833 | [!] => { $crate::token::Bang }; |
834 | [^] => { $crate::token::Caret }; |
835 | [^=] => { $crate::token::CaretEq }; |
836 | [:] => { $crate::token::Colon }; |
837 | [::] => { $crate::token::Colon2 }; |
838 | [,] => { $crate::token::Comma }; |
839 | [/] => { $crate::token::Div }; |
840 | [/=] => { $crate::token::DivEq }; |
841 | [$] => { $crate::token::Dollar }; |
842 | [.] => { $crate::token::Dot }; |
843 | [..] => { $crate::token::Dot2 }; |
844 | [...] => { $crate::token::Dot3 }; |
845 | [..=] => { $crate::token::DotDotEq }; |
846 | [=] => { $crate::token::Eq }; |
847 | [==] => { $crate::token::EqEq }; |
848 | [>=] => { $crate::token::Ge }; |
849 | [>] => { $crate::token::Gt }; |
850 | [<=] => { $crate::token::Le }; |
851 | [<] => { $crate::token::Lt }; |
852 | [*=] => { $crate::token::MulEq }; |
853 | [!=] => { $crate::token::Ne }; |
854 | [|] => { $crate::token::Or }; |
855 | [|=] => { $crate::token::OrEq }; |
856 | [||] => { $crate::token::OrOr }; |
857 | [#] => { $crate::token::Pound }; |
858 | [?] => { $crate::token::Question }; |
859 | [->] => { $crate::token::RArrow }; |
860 | [<-] => { $crate::token::LArrow }; |
861 | [%] => { $crate::token::Rem }; |
862 | [%=] => { $crate::token::RemEq }; |
863 | [=>] => { $crate::token::FatArrow }; |
864 | [;] => { $crate::token::Semi }; |
865 | [<<] => { $crate::token::Shl }; |
866 | [<<=] => { $crate::token::ShlEq }; |
867 | [>>] => { $crate::token::Shr }; |
868 | [>>=] => { $crate::token::ShrEq }; |
869 | [*] => { $crate::token::Star }; |
870 | [-] => { $crate::token::Sub }; |
871 | [-=] => { $crate::token::SubEq }; |
872 | [~] => { $crate::token::Tilde }; |
873 | [_] => { $crate::token::Underscore }; |
874 | } |
875 | }; |
876 | } |
877 | |
878 | // Old rustc does not permit `await` appearing anywhere in the source file. |
879 | // https://github.com/rust-lang/rust/issues/57919 |
880 | // We put the Token![await] rule in a place that is not lexed by old rustc. |
881 | #[cfg (not(syn_omit_await_from_token_macro))] |
882 | include!("await.rs" ); // export_token_macro! {[await]} |
883 | #[cfg (syn_omit_await_from_token_macro)] |
884 | export_token_macro! {} |
885 | |
886 | // Not public API. |
887 | #[doc (hidden)] |
888 | #[cfg (feature = "parsing" )] |
889 | pub mod parsing { |
890 | use crate::buffer::Cursor; |
891 | use crate::error::{Error, Result}; |
892 | use crate::parse::ParseStream; |
893 | use crate::span::FromSpans; |
894 | use proc_macro2::{Spacing, Span}; |
895 | |
896 | pub fn keyword(input: ParseStream, token: &str) -> Result<Span> { |
897 | input.step(|cursor| { |
898 | if let Some((ident, rest)) = cursor.ident() { |
899 | if ident == token { |
900 | return Ok((ident.span(), rest)); |
901 | } |
902 | } |
903 | Err(cursor.error(format!("expected ` {}`" , token))) |
904 | }) |
905 | } |
906 | |
907 | pub fn peek_keyword(cursor: Cursor, token: &str) -> bool { |
908 | if let Some((ident, _rest)) = cursor.ident() { |
909 | ident == token |
910 | } else { |
911 | false |
912 | } |
913 | } |
914 | |
915 | pub fn punct<S: FromSpans>(input: ParseStream, token: &str) -> Result<S> { |
916 | let mut spans = [input.span(); 3]; |
917 | punct_helper(input, token, &mut spans)?; |
918 | Ok(S::from_spans(&spans)) |
919 | } |
920 | |
921 | fn punct_helper(input: ParseStream, token: &str, spans: &mut [Span; 3]) -> Result<()> { |
922 | input.step(|cursor| { |
923 | let mut cursor = *cursor; |
924 | assert!(token.len() <= spans.len()); |
925 | |
926 | for (i, ch) in token.chars().enumerate() { |
927 | match cursor.punct() { |
928 | Some((punct, rest)) => { |
929 | spans[i] = punct.span(); |
930 | if punct.as_char() != ch { |
931 | break; |
932 | } else if i == token.len() - 1 { |
933 | return Ok(((), rest)); |
934 | } else if punct.spacing() != Spacing::Joint { |
935 | break; |
936 | } |
937 | cursor = rest; |
938 | } |
939 | None => break, |
940 | } |
941 | } |
942 | |
943 | Err(Error::new(spans[0], format!("expected ` {}`" , token))) |
944 | }) |
945 | } |
946 | |
947 | pub fn peek_punct(mut cursor: Cursor, token: &str) -> bool { |
948 | for (i, ch) in token.chars().enumerate() { |
949 | match cursor.punct() { |
950 | Some((punct, rest)) => { |
951 | if punct.as_char() != ch { |
952 | break; |
953 | } else if i == token.len() - 1 { |
954 | return true; |
955 | } else if punct.spacing() != Spacing::Joint { |
956 | break; |
957 | } |
958 | cursor = rest; |
959 | } |
960 | None => break, |
961 | } |
962 | } |
963 | false |
964 | } |
965 | } |
966 | |
967 | // Not public API. |
968 | #[doc (hidden)] |
969 | #[cfg (feature = "printing" )] |
970 | pub mod printing { |
971 | use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream}; |
972 | use quote::TokenStreamExt; |
973 | |
974 | pub fn punct(s: &str, spans: &[Span], tokens: &mut TokenStream) { |
975 | assert_eq!(s.len(), spans.len()); |
976 | |
977 | let mut chars = s.chars(); |
978 | let mut spans = spans.iter(); |
979 | let ch = chars.next_back().unwrap(); |
980 | let span = spans.next_back().unwrap(); |
981 | for (ch, span) in chars.zip(spans) { |
982 | let mut op = Punct::new(ch, Spacing::Joint); |
983 | op.set_span(*span); |
984 | tokens.append(op); |
985 | } |
986 | |
987 | let mut op = Punct::new(ch, Spacing::Alone); |
988 | op.set_span(*span); |
989 | tokens.append(op); |
990 | } |
991 | |
992 | pub fn keyword(s: &str, span: Span, tokens: &mut TokenStream) { |
993 | tokens.append(Ident::new(s, span)); |
994 | } |
995 | |
996 | pub fn delim<F>(s: &str, span: Span, tokens: &mut TokenStream, f: F) |
997 | where |
998 | F: FnOnce(&mut TokenStream), |
999 | { |
1000 | let delim = match s { |
1001 | "(" => Delimiter::Parenthesis, |
1002 | "[" => Delimiter::Bracket, |
1003 | "{" => Delimiter::Brace, |
1004 | " " => Delimiter::None, |
1005 | _ => panic!("unknown delimiter: {}" , s), |
1006 | }; |
1007 | let mut inner = TokenStream::new(); |
1008 | f(&mut inner); |
1009 | let mut g = Group::new(delim, inner); |
1010 | g.set_span(span); |
1011 | tokens.append(g); |
1012 | } |
1013 | } |
1014 | |