1#[cfg(wrap_proc_macro)]
2use crate::imp;
3#[cfg(span_locations)]
4use crate::location::LineColumn;
5use crate::parse::{self, Cursor};
6use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
7use crate::{Delimiter, Spacing, TokenTree};
8#[cfg(all(span_locations, not(fuzzing)))]
9use alloc::collections::BTreeMap;
10#[cfg(all(span_locations, not(fuzzing)))]
11use core::cell::RefCell;
12#[cfg(span_locations)]
13use core::cmp;
14use core::fmt::{self, Debug, Display, Write};
15use core::mem::ManuallyDrop;
16#[cfg(span_locations)]
17use core::ops::Range;
18use core::ops::RangeBounds;
19use core::ptr;
20use core::str;
21#[cfg(feature = "proc-macro")]
22use core::str::FromStr;
23use std::ffi::CStr;
24#[cfg(wrap_proc_macro)]
25use std::panic;
26#[cfg(procmacro2_semver_exempt)]
27use std::path::PathBuf;
28
29/// Force use of proc-macro2's fallback implementation of the API for now, even
30/// if the compiler's implementation is available.
31pub fn force() {
32 #[cfg(wrap_proc_macro)]
33 crate::detection::force_fallback();
34}
35
36/// Resume using the compiler's implementation of the proc macro API if it is
37/// available.
38pub fn unforce() {
39 #[cfg(wrap_proc_macro)]
40 crate::detection::unforce_fallback();
41}
42
43#[derive(Clone)]
44pub(crate) struct TokenStream {
45 inner: RcVec<TokenTree>,
46}
47
48#[derive(Debug)]
49pub(crate) struct LexError {
50 pub(crate) span: Span,
51}
52
53impl LexError {
54 pub(crate) fn span(&self) -> Span {
55 self.span
56 }
57
58 pub(crate) fn call_site() -> Self {
59 LexError {
60 span: Span::call_site(),
61 }
62 }
63}
64
65impl TokenStream {
66 pub(crate) fn new() -> Self {
67 TokenStream {
68 inner: RcVecBuilder::new().build(),
69 }
70 }
71
72 pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
73 // Create a dummy file & add it to the source map
74 let mut cursor = get_cursor(src);
75
76 // Strip a byte order mark if present
77 const BYTE_ORDER_MARK: &str = "\u{feff}";
78 if cursor.starts_with(BYTE_ORDER_MARK) {
79 cursor = cursor.advance(BYTE_ORDER_MARK.len());
80 }
81
82 parse::token_stream(cursor)
83 }
84
85 #[cfg(feature = "proc-macro")]
86 pub(crate) fn from_str_unchecked(src: &str) -> Self {
87 Self::from_str_checked(src).unwrap()
88 }
89
90 pub(crate) fn is_empty(&self) -> bool {
91 self.inner.len() == 0
92 }
93
94 fn take_inner(self) -> RcVecBuilder<TokenTree> {
95 let nodrop = ManuallyDrop::new(self);
96 unsafe { ptr::read(&nodrop.inner) }.make_owned()
97 }
98}
99
100fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
101 // https://github.com/dtolnay/proc-macro2/issues/235
102 match token {
103 TokenTree::Literal(crate::Literal {
104 #[cfg(wrap_proc_macro)]
105 inner: crate::imp::Literal::Fallback(literal: Literal),
106 #[cfg(not(wrap_proc_macro))]
107 inner: literal,
108 ..
109 }) if literal.repr.starts_with('-') => {
110 push_negative_literal(vec, literal);
111 }
112 _ => vec.push(element:token),
113 }
114
115 #[cold]
116 fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) {
117 literal.repr.remove(idx:0);
118 let mut punct: Punct = crate::Punct::new(ch:'-', Spacing::Alone);
119 punct.set_span(crate::Span::_new_fallback(inner:literal.span));
120 vec.push(element:TokenTree::Punct(punct));
121 vec.push(element:TokenTree::Literal(crate::Literal::_new_fallback(inner:literal)));
122 }
123}
124
125// Nonrecursive to prevent stack overflow.
126impl Drop for TokenStream {
127 fn drop(&mut self) {
128 let mut stack = Vec::new();
129 let mut current = match self.inner.get_mut() {
130 Some(inner) => inner.take().into_iter(),
131 None => return,
132 };
133 loop {
134 while let Some(token) = current.next() {
135 let group = match token {
136 TokenTree::Group(group) => group.inner,
137 _ => continue,
138 };
139 #[cfg(wrap_proc_macro)]
140 let group = match group {
141 crate::imp::Group::Fallback(group) => group,
142 crate::imp::Group::Compiler(_) => continue,
143 };
144 let mut group = group;
145 if let Some(inner) = group.stream.inner.get_mut() {
146 stack.push(current);
147 current = inner.take().into_iter();
148 }
149 }
150 match stack.pop() {
151 Some(next) => current = next,
152 None => return,
153 }
154 }
155 }
156}
157
158pub(crate) struct TokenStreamBuilder {
159 inner: RcVecBuilder<TokenTree>,
160}
161
162impl TokenStreamBuilder {
163 pub(crate) fn new() -> Self {
164 TokenStreamBuilder {
165 inner: RcVecBuilder::new(),
166 }
167 }
168
169 pub(crate) fn with_capacity(cap: usize) -> Self {
170 TokenStreamBuilder {
171 inner: RcVecBuilder::with_capacity(cap),
172 }
173 }
174
175 pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) {
176 self.inner.push(element:tt);
177 }
178
179 pub(crate) fn build(self) -> TokenStream {
180 TokenStream {
181 inner: self.inner.build(),
182 }
183 }
184}
185
186#[cfg(span_locations)]
187fn get_cursor(src: &str) -> Cursor {
188 #[cfg(fuzzing)]
189 return Cursor { rest: src, off: 1 };
190
191 // Create a dummy file & add it to the source map
192 #[cfg(not(fuzzing))]
193 SOURCE_MAP.with(|sm| {
194 let mut sm = sm.borrow_mut();
195 let span = sm.add_file(src);
196 Cursor {
197 rest: src,
198 off: span.lo,
199 }
200 })
201}
202
203#[cfg(not(span_locations))]
204fn get_cursor(src: &str) -> Cursor {
205 Cursor { rest: src }
206}
207
208impl Display for LexError {
209 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
210 f.write_str(data:"cannot parse string into token stream")
211 }
212}
213
214impl Display for TokenStream {
215 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
216 let mut joint: bool = false;
217 for (i: usize, tt: &TokenTree) in self.inner.iter().enumerate() {
218 if i != 0 && !joint {
219 write!(f, " ")?;
220 }
221 joint = false;
222 match tt {
223 TokenTree::Group(tt: &Group) => Display::fmt(self:tt, f),
224 TokenTree::Ident(tt: &Ident) => Display::fmt(self:tt, f),
225 TokenTree::Punct(tt: &Punct) => {
226 joint = tt.spacing() == Spacing::Joint;
227 Display::fmt(self:tt, f)
228 }
229 TokenTree::Literal(tt: &Literal) => Display::fmt(self:tt, f),
230 }?;
231 }
232
233 Ok(())
234 }
235}
236
237impl Debug for TokenStream {
238 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
239 f.write_str(data:"TokenStream ")?;
240 f.debug_list().entries(self.clone()).finish()
241 }
242}
243
244#[cfg(feature = "proc-macro")]
245impl From<proc_macro::TokenStream> for TokenStream {
246 fn from(inner: proc_macro::TokenStream) -> Self {
247 TokenStream::from_str_unchecked(&inner.to_string())
248 }
249}
250
251#[cfg(feature = "proc-macro")]
252impl From<TokenStream> for proc_macro::TokenStream {
253 fn from(inner: TokenStream) -> Self {
254 proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
255 }
256}
257
258impl From<TokenTree> for TokenStream {
259 fn from(tree: TokenTree) -> Self {
260 let mut stream: RcVecBuilder = RcVecBuilder::new();
261 push_token_from_proc_macro(vec:stream.as_mut(), token:tree);
262 TokenStream {
263 inner: stream.build(),
264 }
265 }
266}
267
268impl FromIterator<TokenTree> for TokenStream {
269 fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
270 let mut stream: TokenStream = TokenStream::new();
271 stream.extend(iter:tokens);
272 stream
273 }
274}
275
276impl FromIterator<TokenStream> for TokenStream {
277 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
278 let mut v: RcVecBuilder = RcVecBuilder::new();
279
280 for stream: TokenStream in streams {
281 v.extend(iter:stream.take_inner());
282 }
283
284 TokenStream { inner: v.build() }
285 }
286}
287
288impl Extend<TokenTree> for TokenStream {
289 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
290 let mut vec: RcVecMut<'_, TokenTree> = self.inner.make_mut();
291 tokens::IntoIter
292 .into_iter()
293 .for_each(|token: TokenTree| push_token_from_proc_macro(vec.as_mut(), token));
294 }
295}
296
297impl Extend<TokenStream> for TokenStream {
298 fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
299 self.inner.make_mut().extend(iter:streams.into_iter().flatten());
300 }
301}
302
303pub(crate) type TokenTreeIter = RcVecIntoIter<TokenTree>;
304
305impl IntoIterator for TokenStream {
306 type Item = TokenTree;
307 type IntoIter = TokenTreeIter;
308
309 fn into_iter(self) -> TokenTreeIter {
310 self.take_inner().into_iter()
311 }
312}
313
314#[cfg(all(span_locations, not(fuzzing)))]
315thread_local! {
316 static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
317 // Start with a single dummy file which all call_site() and def_site()
318 // spans reference.
319 files: vec![FileInfo {
320 source_text: String::new(),
321 span: Span { lo: 0, hi: 0 },
322 lines: vec![0],
323 char_index_to_byte_offset: BTreeMap::new(),
324 }],
325 });
326}
327
328#[cfg(span_locations)]
329pub(crate) fn invalidate_current_thread_spans() {
330 #[cfg(not(fuzzing))]
331 SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1));
332}
333
334#[cfg(all(span_locations, not(fuzzing)))]
335struct FileInfo {
336 source_text: String,
337 span: Span,
338 lines: Vec<usize>,
339 char_index_to_byte_offset: BTreeMap<usize, usize>,
340}
341
342#[cfg(all(span_locations, not(fuzzing)))]
343impl FileInfo {
344 fn offset_line_column(&self, offset: usize) -> LineColumn {
345 assert!(self.span_within(Span {
346 lo: offset as u32,
347 hi: offset as u32,
348 }));
349 let offset = offset - self.span.lo as usize;
350 match self.lines.binary_search(&offset) {
351 Ok(found) => LineColumn {
352 line: found + 1,
353 column: 0,
354 },
355 Err(idx) => LineColumn {
356 line: idx,
357 column: offset - self.lines[idx - 1],
358 },
359 }
360 }
361
362 fn span_within(&self, span: Span) -> bool {
363 span.lo >= self.span.lo && span.hi <= self.span.hi
364 }
365
366 fn byte_range(&mut self, span: Span) -> Range<usize> {
367 let lo_char = (span.lo - self.span.lo) as usize;
368
369 // Look up offset of the largest already-computed char index that is
370 // less than or equal to the current requested one. We resume counting
371 // chars from that point.
372 let (&last_char_index, &last_byte_offset) = self
373 .char_index_to_byte_offset
374 .range(..=lo_char)
375 .next_back()
376 .unwrap_or((&0, &0));
377
378 let lo_byte = if last_char_index == lo_char {
379 last_byte_offset
380 } else {
381 let total_byte_offset = match self.source_text[last_byte_offset..]
382 .char_indices()
383 .nth(lo_char - last_char_index)
384 {
385 Some((additional_offset, _ch)) => last_byte_offset + additional_offset,
386 None => self.source_text.len(),
387 };
388 self.char_index_to_byte_offset
389 .insert(lo_char, total_byte_offset);
390 total_byte_offset
391 };
392
393 let trunc_lo = &self.source_text[lo_byte..];
394 let char_len = (span.hi - span.lo) as usize;
395 lo_byte..match trunc_lo.char_indices().nth(char_len) {
396 Some((offset, _ch)) => lo_byte + offset,
397 None => self.source_text.len(),
398 }
399 }
400
401 fn source_text(&mut self, span: Span) -> String {
402 let byte_range = self.byte_range(span);
403 self.source_text[byte_range].to_owned()
404 }
405}
406
407/// Computes the offsets of each line in the given source string
408/// and the total number of characters
409#[cfg(all(span_locations, not(fuzzing)))]
410fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
411 let mut lines = vec![0];
412 let mut total = 0;
413
414 for ch in s.chars() {
415 total += 1;
416 if ch == '\n' {
417 lines.push(total);
418 }
419 }
420
421 (total, lines)
422}
423
424#[cfg(all(span_locations, not(fuzzing)))]
425struct SourceMap {
426 files: Vec<FileInfo>,
427}
428
429#[cfg(all(span_locations, not(fuzzing)))]
430impl SourceMap {
431 fn next_start_pos(&self) -> u32 {
432 // Add 1 so there's always space between files.
433 //
434 // We'll always have at least 1 file, as we initialize our files list
435 // with a dummy file.
436 self.files.last().unwrap().span.hi + 1
437 }
438
439 fn add_file(&mut self, src: &str) -> Span {
440 let (len, lines) = lines_offsets(src);
441 let lo = self.next_start_pos();
442 let span = Span {
443 lo,
444 hi: lo + (len as u32),
445 };
446
447 self.files.push(FileInfo {
448 source_text: src.to_owned(),
449 span,
450 lines,
451 // Populated lazily by source_text().
452 char_index_to_byte_offset: BTreeMap::new(),
453 });
454
455 span
456 }
457
458 #[cfg(procmacro2_semver_exempt)]
459 fn filepath(&self, span: Span) -> String {
460 for (i, file) in self.files.iter().enumerate() {
461 if file.span_within(span) {
462 return if i == 0 {
463 "<unspecified>".to_owned()
464 } else {
465 format!("<parsed string {}>", i)
466 };
467 }
468 }
469 unreachable!("Invalid span with no related FileInfo!");
470 }
471
472 fn fileinfo(&self, span: Span) -> &FileInfo {
473 for file in &self.files {
474 if file.span_within(span) {
475 return file;
476 }
477 }
478 unreachable!("Invalid span with no related FileInfo!");
479 }
480
481 fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
482 for file in &mut self.files {
483 if file.span_within(span) {
484 return file;
485 }
486 }
487 unreachable!("Invalid span with no related FileInfo!");
488 }
489}
490
491#[derive(Clone, Copy, PartialEq, Eq)]
492pub(crate) struct Span {
493 #[cfg(span_locations)]
494 pub(crate) lo: u32,
495 #[cfg(span_locations)]
496 pub(crate) hi: u32,
497}
498
499impl Span {
500 #[cfg(not(span_locations))]
501 pub(crate) fn call_site() -> Self {
502 Span {}
503 }
504
505 #[cfg(span_locations)]
506 pub(crate) fn call_site() -> Self {
507 Span { lo: 0, hi: 0 }
508 }
509
510 pub(crate) fn mixed_site() -> Self {
511 Span::call_site()
512 }
513
514 #[cfg(procmacro2_semver_exempt)]
515 pub(crate) fn def_site() -> Self {
516 Span::call_site()
517 }
518
519 pub(crate) fn resolved_at(&self, _other: Span) -> Span {
520 // Stable spans consist only of line/column information, so
521 // `resolved_at` and `located_at` only select which span the
522 // caller wants line/column information from.
523 *self
524 }
525
526 pub(crate) fn located_at(&self, other: Span) -> Span {
527 other
528 }
529
530 #[cfg(span_locations)]
531 pub(crate) fn byte_range(&self) -> Range<usize> {
532 #[cfg(fuzzing)]
533 return 0..0;
534
535 #[cfg(not(fuzzing))]
536 {
537 if self.is_call_site() {
538 0..0
539 } else {
540 SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self))
541 }
542 }
543 }
544
545 #[cfg(span_locations)]
546 pub(crate) fn start(&self) -> LineColumn {
547 #[cfg(fuzzing)]
548 return LineColumn { line: 0, column: 0 };
549
550 #[cfg(not(fuzzing))]
551 SOURCE_MAP.with(|sm| {
552 let sm = sm.borrow();
553 let fi = sm.fileinfo(*self);
554 fi.offset_line_column(self.lo as usize)
555 })
556 }
557
558 #[cfg(span_locations)]
559 pub(crate) fn end(&self) -> LineColumn {
560 #[cfg(fuzzing)]
561 return LineColumn { line: 0, column: 0 };
562
563 #[cfg(not(fuzzing))]
564 SOURCE_MAP.with(|sm| {
565 let sm = sm.borrow();
566 let fi = sm.fileinfo(*self);
567 fi.offset_line_column(self.hi as usize)
568 })
569 }
570
571 #[cfg(procmacro2_semver_exempt)]
572 pub(crate) fn file(&self) -> String {
573 #[cfg(fuzzing)]
574 return "<unspecified>".to_owned();
575
576 #[cfg(not(fuzzing))]
577 SOURCE_MAP.with(|sm| {
578 let sm = sm.borrow();
579 sm.filepath(*self)
580 })
581 }
582
583 #[cfg(procmacro2_semver_exempt)]
584 pub(crate) fn local_file(&self) -> Option<PathBuf> {
585 None
586 }
587
588 #[cfg(not(span_locations))]
589 pub(crate) fn join(&self, _other: Span) -> Option<Span> {
590 Some(Span {})
591 }
592
593 #[cfg(span_locations)]
594 pub(crate) fn join(&self, other: Span) -> Option<Span> {
595 #[cfg(fuzzing)]
596 return {
597 let _ = other;
598 None
599 };
600
601 #[cfg(not(fuzzing))]
602 SOURCE_MAP.with(|sm| {
603 let sm = sm.borrow();
604 // If `other` is not within the same FileInfo as us, return None.
605 if !sm.fileinfo(*self).span_within(other) {
606 return None;
607 }
608 Some(Span {
609 lo: cmp::min(self.lo, other.lo),
610 hi: cmp::max(self.hi, other.hi),
611 })
612 })
613 }
614
615 #[cfg(not(span_locations))]
616 pub(crate) fn source_text(&self) -> Option<String> {
617 None
618 }
619
620 #[cfg(span_locations)]
621 pub(crate) fn source_text(&self) -> Option<String> {
622 #[cfg(fuzzing)]
623 return None;
624
625 #[cfg(not(fuzzing))]
626 {
627 if self.is_call_site() {
628 None
629 } else {
630 Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
631 }
632 }
633 }
634
635 #[cfg(not(span_locations))]
636 pub(crate) fn first_byte(self) -> Self {
637 self
638 }
639
640 #[cfg(span_locations)]
641 pub(crate) fn first_byte(self) -> Self {
642 Span {
643 lo: self.lo,
644 hi: cmp::min(self.lo.saturating_add(1), self.hi),
645 }
646 }
647
648 #[cfg(not(span_locations))]
649 pub(crate) fn last_byte(self) -> Self {
650 self
651 }
652
653 #[cfg(span_locations)]
654 pub(crate) fn last_byte(self) -> Self {
655 Span {
656 lo: cmp::max(self.hi.saturating_sub(1), self.lo),
657 hi: self.hi,
658 }
659 }
660
661 #[cfg(span_locations)]
662 fn is_call_site(&self) -> bool {
663 self.lo == 0 && self.hi == 0
664 }
665}
666
667impl Debug for Span {
668 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
669 #[cfg(span_locations)]
670 return write!(f, "bytes({}..{})", self.lo, self.hi);
671
672 #[cfg(not(span_locations))]
673 write!(f, "Span")
674 }
675}
676
677pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
678 #[cfg(span_locations)]
679 {
680 if span.is_call_site() {
681 return;
682 }
683 }
684
685 if cfg!(span_locations) {
686 debug.field(name:"span", &span);
687 }
688}
689
690#[derive(Clone)]
691pub(crate) struct Group {
692 delimiter: Delimiter,
693 stream: TokenStream,
694 span: Span,
695}
696
697impl Group {
698 pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
699 Group {
700 delimiter,
701 stream,
702 span: Span::call_site(),
703 }
704 }
705
706 pub(crate) fn delimiter(&self) -> Delimiter {
707 self.delimiter
708 }
709
710 pub(crate) fn stream(&self) -> TokenStream {
711 self.stream.clone()
712 }
713
714 pub(crate) fn span(&self) -> Span {
715 self.span
716 }
717
718 pub(crate) fn span_open(&self) -> Span {
719 self.span.first_byte()
720 }
721
722 pub(crate) fn span_close(&self) -> Span {
723 self.span.last_byte()
724 }
725
726 pub(crate) fn set_span(&mut self, span: Span) {
727 self.span = span;
728 }
729}
730
731impl Display for Group {
732 // We attempt to match libproc_macro's formatting.
733 // Empty parens: ()
734 // Nonempty parens: (...)
735 // Empty brackets: []
736 // Nonempty brackets: [...]
737 // Empty braces: { }
738 // Nonempty braces: { ... }
739 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
740 let (open, close) = match self.delimiter {
741 Delimiter::Parenthesis => ("(", ")"),
742 Delimiter::Brace => ("{ ", "}"),
743 Delimiter::Bracket => ("[", "]"),
744 Delimiter::None => ("", ""),
745 };
746
747 f.write_str(open)?;
748 Display::fmt(&self.stream, f)?;
749 if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() {
750 f.write_str(" ")?;
751 }
752 f.write_str(close)?;
753
754 Ok(())
755 }
756}
757
758impl Debug for Group {
759 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
760 let mut debug: DebugStruct<'_, '_> = fmt.debug_struct(name:"Group");
761 debug.field(name:"delimiter", &self.delimiter);
762 debug.field(name:"stream", &self.stream);
763 debug_span_field_if_nontrivial(&mut debug, self.span);
764 debug.finish()
765 }
766}
767
768#[derive(Clone)]
769pub(crate) struct Ident {
770 sym: Box<str>,
771 span: Span,
772 raw: bool,
773}
774
775impl Ident {
776 #[track_caller]
777 pub(crate) fn new_checked(string: &str, span: Span) -> Self {
778 validate_ident(string);
779 Ident::new_unchecked(string, span)
780 }
781
782 pub(crate) fn new_unchecked(string: &str, span: Span) -> Self {
783 Ident {
784 sym: Box::from(string),
785 span,
786 raw: false,
787 }
788 }
789
790 #[track_caller]
791 pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
792 validate_ident_raw(string);
793 Ident::new_raw_unchecked(string, span)
794 }
795
796 pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self {
797 Ident {
798 sym: Box::from(string),
799 span,
800 raw: true,
801 }
802 }
803
804 pub(crate) fn span(&self) -> Span {
805 self.span
806 }
807
808 pub(crate) fn set_span(&mut self, span: Span) {
809 self.span = span;
810 }
811}
812
813pub(crate) fn is_ident_start(c: char) -> bool {
814 c == '_' || unicode_ident::is_xid_start(ch:c)
815}
816
817pub(crate) fn is_ident_continue(c: char) -> bool {
818 unicode_ident::is_xid_continue(ch:c)
819}
820
821#[track_caller]
822fn validate_ident(string: &str) {
823 if string.is_empty() {
824 panic!("Ident is not allowed to be empty; use Option<Ident>");
825 }
826
827 if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') {
828 panic!("Ident cannot be a number; use Literal instead");
829 }
830
831 fn ident_ok(string: &str) -> bool {
832 let mut chars = string.chars();
833 let first = chars.next().unwrap();
834 if !is_ident_start(first) {
835 return false;
836 }
837 for ch in chars {
838 if !is_ident_continue(ch) {
839 return false;
840 }
841 }
842 true
843 }
844
845 if !ident_ok(string) {
846 panic!("{:?} is not a valid Ident", string);
847 }
848}
849
850#[track_caller]
851fn validate_ident_raw(string: &str) {
852 validate_ident(string);
853
854 match string {
855 "_" | "super" | "self" | "Self" | "crate" => {
856 panic!("`r#{}` cannot be a raw identifier", string);
857 }
858 _ => {}
859 }
860}
861
862impl PartialEq for Ident {
863 fn eq(&self, other: &Ident) -> bool {
864 self.sym == other.sym && self.raw == other.raw
865 }
866}
867
868impl<T> PartialEq<T> for Ident
869where
870 T: ?Sized + AsRef<str>,
871{
872 fn eq(&self, other: &T) -> bool {
873 let other: &str = other.as_ref();
874 if self.raw {
875 other.starts_with("r#") && *self.sym == other[2..]
876 } else {
877 *self.sym == *other
878 }
879 }
880}
881
882impl Display for Ident {
883 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
884 if self.raw {
885 f.write_str(data:"r#")?;
886 }
887 Display::fmt(&self.sym, f)
888 }
889}
890
891#[allow(clippy::missing_fields_in_debug)]
892impl Debug for Ident {
893 // Ident(proc_macro), Ident(r#union)
894 #[cfg(not(span_locations))]
895 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
896 let mut debug: DebugTuple<'_, '_> = f.debug_tuple(name:"Ident");
897 debug.field(&format_args!("{}", self));
898 debug.finish()
899 }
900
901 // Ident {
902 // sym: proc_macro,
903 // span: bytes(128..138)
904 // }
905 #[cfg(span_locations)]
906 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
907 let mut debug = f.debug_struct("Ident");
908 debug.field("sym", &format_args!("{}", self));
909 debug_span_field_if_nontrivial(&mut debug, self.span);
910 debug.finish()
911 }
912}
913
914#[derive(Clone)]
915pub(crate) struct Literal {
916 pub(crate) repr: String,
917 span: Span,
918}
919
920macro_rules! suffixed_numbers {
921 ($($name:ident => $kind:ident,)*) => ($(
922 pub(crate) fn $name(n: $kind) -> Literal {
923 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
924 }
925 )*)
926}
927
928macro_rules! unsuffixed_numbers {
929 ($($name:ident => $kind:ident,)*) => ($(
930 pub(crate) fn $name(n: $kind) -> Literal {
931 Literal::_new(n.to_string())
932 }
933 )*)
934}
935
936impl Literal {
937 pub(crate) fn _new(repr: String) -> Self {
938 Literal {
939 repr,
940 span: Span::call_site(),
941 }
942 }
943
944 pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
945 let mut cursor = get_cursor(repr);
946 #[cfg(span_locations)]
947 let lo = cursor.off;
948
949 let negative = cursor.starts_with_char('-');
950 if negative {
951 cursor = cursor.advance(1);
952 if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
953 return Err(LexError::call_site());
954 }
955 }
956
957 if let Ok((rest, mut literal)) = parse::literal(cursor) {
958 if rest.is_empty() {
959 if negative {
960 literal.repr.insert(0, '-');
961 }
962 literal.span = Span {
963 #[cfg(span_locations)]
964 lo,
965 #[cfg(span_locations)]
966 hi: rest.off,
967 };
968 return Ok(literal);
969 }
970 }
971 Err(LexError::call_site())
972 }
973
974 pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self {
975 Literal::_new(repr.to_owned())
976 }
977
978 suffixed_numbers! {
979 u8_suffixed => u8,
980 u16_suffixed => u16,
981 u32_suffixed => u32,
982 u64_suffixed => u64,
983 u128_suffixed => u128,
984 usize_suffixed => usize,
985 i8_suffixed => i8,
986 i16_suffixed => i16,
987 i32_suffixed => i32,
988 i64_suffixed => i64,
989 i128_suffixed => i128,
990 isize_suffixed => isize,
991
992 f32_suffixed => f32,
993 f64_suffixed => f64,
994 }
995
996 unsuffixed_numbers! {
997 u8_unsuffixed => u8,
998 u16_unsuffixed => u16,
999 u32_unsuffixed => u32,
1000 u64_unsuffixed => u64,
1001 u128_unsuffixed => u128,
1002 usize_unsuffixed => usize,
1003 i8_unsuffixed => i8,
1004 i16_unsuffixed => i16,
1005 i32_unsuffixed => i32,
1006 i64_unsuffixed => i64,
1007 i128_unsuffixed => i128,
1008 isize_unsuffixed => isize,
1009 }
1010
1011 pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
1012 let mut s = f.to_string();
1013 if !s.contains('.') {
1014 s.push_str(".0");
1015 }
1016 Literal::_new(s)
1017 }
1018
1019 pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
1020 let mut s = f.to_string();
1021 if !s.contains('.') {
1022 s.push_str(".0");
1023 }
1024 Literal::_new(s)
1025 }
1026
1027 pub(crate) fn string(string: &str) -> Literal {
1028 let mut repr = String::with_capacity(string.len() + 2);
1029 repr.push('"');
1030 escape_utf8(string, &mut repr);
1031 repr.push('"');
1032 Literal::_new(repr)
1033 }
1034
1035 pub(crate) fn character(ch: char) -> Literal {
1036 let mut repr = String::new();
1037 repr.push('\'');
1038 if ch == '"' {
1039 // escape_debug turns this into '\"' which is unnecessary.
1040 repr.push(ch);
1041 } else {
1042 repr.extend(ch.escape_debug());
1043 }
1044 repr.push('\'');
1045 Literal::_new(repr)
1046 }
1047
1048 pub(crate) fn byte_character(byte: u8) -> Literal {
1049 let mut repr = "b'".to_string();
1050 #[allow(clippy::match_overlapping_arm)]
1051 match byte {
1052 b'\0' => repr.push_str(r"\0"),
1053 b'\t' => repr.push_str(r"\t"),
1054 b'\n' => repr.push_str(r"\n"),
1055 b'\r' => repr.push_str(r"\r"),
1056 b'\'' => repr.push_str(r"\'"),
1057 b'\\' => repr.push_str(r"\\"),
1058 b'\x20'..=b'\x7E' => repr.push(byte as char),
1059 _ => {
1060 let _ = write!(repr, r"\x{:02X}", byte);
1061 }
1062 }
1063 repr.push('\'');
1064 Literal::_new(repr)
1065 }
1066
1067 pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
1068 let mut repr = "b\"".to_string();
1069 let mut bytes = bytes.iter();
1070 while let Some(&b) = bytes.next() {
1071 #[allow(clippy::match_overlapping_arm)]
1072 match b {
1073 b'\0' => repr.push_str(match bytes.as_slice().first() {
1074 // circumvent clippy::octal_escapes lint
1075 Some(b'0'..=b'7') => r"\x00",
1076 _ => r"\0",
1077 }),
1078 b'\t' => repr.push_str(r"\t"),
1079 b'\n' => repr.push_str(r"\n"),
1080 b'\r' => repr.push_str(r"\r"),
1081 b'"' => repr.push_str("\\\""),
1082 b'\\' => repr.push_str(r"\\"),
1083 b'\x20'..=b'\x7E' => repr.push(b as char),
1084 _ => {
1085 let _ = write!(repr, r"\x{:02X}", b);
1086 }
1087 }
1088 }
1089 repr.push('"');
1090 Literal::_new(repr)
1091 }
1092
1093 pub(crate) fn c_string(string: &CStr) -> Literal {
1094 let mut repr = "c\"".to_string();
1095 let mut bytes = string.to_bytes();
1096 while !bytes.is_empty() {
1097 let (valid, invalid) = match str::from_utf8(bytes) {
1098 Ok(all_valid) => {
1099 bytes = b"";
1100 (all_valid, bytes)
1101 }
1102 Err(utf8_error) => {
1103 let (valid, rest) = bytes.split_at(utf8_error.valid_up_to());
1104 let valid = str::from_utf8(valid).unwrap();
1105 let invalid = utf8_error
1106 .error_len()
1107 .map_or(rest, |error_len| &rest[..error_len]);
1108 bytes = &bytes[valid.len() + invalid.len()..];
1109 (valid, invalid)
1110 }
1111 };
1112 escape_utf8(valid, &mut repr);
1113 for &byte in invalid {
1114 let _ = write!(repr, r"\x{:02X}", byte);
1115 }
1116 }
1117 repr.push('"');
1118 Literal::_new(repr)
1119 }
1120
1121 pub(crate) fn span(&self) -> Span {
1122 self.span
1123 }
1124
1125 pub(crate) fn set_span(&mut self, span: Span) {
1126 self.span = span;
1127 }
1128
1129 pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
1130 #[cfg(not(span_locations))]
1131 {
1132 let _ = range;
1133 None
1134 }
1135
1136 #[cfg(span_locations)]
1137 {
1138 use core::ops::Bound;
1139
1140 let lo = match range.start_bound() {
1141 Bound::Included(start) => {
1142 let start = u32::try_from(*start).ok()?;
1143 self.span.lo.checked_add(start)?
1144 }
1145 Bound::Excluded(start) => {
1146 let start = u32::try_from(*start).ok()?;
1147 self.span.lo.checked_add(start)?.checked_add(1)?
1148 }
1149 Bound::Unbounded => self.span.lo,
1150 };
1151 let hi = match range.end_bound() {
1152 Bound::Included(end) => {
1153 let end = u32::try_from(*end).ok()?;
1154 self.span.lo.checked_add(end)?.checked_add(1)?
1155 }
1156 Bound::Excluded(end) => {
1157 let end = u32::try_from(*end).ok()?;
1158 self.span.lo.checked_add(end)?
1159 }
1160 Bound::Unbounded => self.span.hi,
1161 };
1162 if lo <= hi && hi <= self.span.hi {
1163 Some(Span { lo, hi })
1164 } else {
1165 None
1166 }
1167 }
1168 }
1169}
1170
1171impl Display for Literal {
1172 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1173 Display::fmt(&self.repr, f)
1174 }
1175}
1176
1177impl Debug for Literal {
1178 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1179 let mut debug: DebugStruct<'_, '_> = fmt.debug_struct(name:"Literal");
1180 debug.field(name:"lit", &format_args!("{}", self.repr));
1181 debug_span_field_if_nontrivial(&mut debug, self.span);
1182 debug.finish()
1183 }
1184}
1185
1186fn escape_utf8(string: &str, repr: &mut String) {
1187 let mut chars: Chars<'_> = string.chars();
1188 while let Some(ch: char) = chars.next() {
1189 if ch == '\0' {
1190 repr.push_str(
1191 string:if chars&str
1192 .as_str()
1193 .starts_with(|next: char| '0' <= next && next <= '7')
1194 {
1195 // circumvent clippy::octal_escapes lint
1196 r"\x00"
1197 } else {
1198 r"\0"
1199 },
1200 );
1201 } else if ch == '\'' {
1202 // escape_debug turns this into "\'" which is unnecessary.
1203 repr.push(ch);
1204 } else {
1205 repr.extend(iter:ch.escape_debug());
1206 }
1207 }
1208}
1209
1210#[cfg(feature = "proc-macro")]
1211pub(crate) trait FromStr2: FromStr<Err = proc_macro::LexError> {
1212 #[cfg(wrap_proc_macro)]
1213 fn valid(src: &str) -> bool;
1214
1215 #[cfg(wrap_proc_macro)]
1216 fn from_str_checked(src: &str) -> Result<Self, imp::LexError> {
1217 // Validate using fallback parser, because rustc is incapable of
1218 // returning a recoverable Err for certain invalid token streams, and
1219 // will instead permanently poison the compilation.
1220 if !Self::valid(src) {
1221 return Err(imp::LexError::CompilerPanic);
1222 }
1223
1224 // Catch panic to work around https://github.com/rust-lang/rust/issues/58736.
1225 match panic::catch_unwind(|| Self::from_str(src)) {
1226 Ok(Ok(ok)) => Ok(ok),
1227 Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)),
1228 Err(_panic) => Err(imp::LexError::CompilerPanic),
1229 }
1230 }
1231
1232 fn from_str_unchecked(src: &str) -> Self {
1233 Self::from_str(src).unwrap()
1234 }
1235}
1236
1237#[cfg(feature = "proc-macro")]
1238impl FromStr2 for proc_macro::TokenStream {
1239 #[cfg(wrap_proc_macro)]
1240 fn valid(src: &str) -> bool {
1241 TokenStream::from_str_checked(src).is_ok()
1242 }
1243}
1244
1245#[cfg(feature = "proc-macro")]
1246impl FromStr2 for proc_macro::Literal {
1247 #[cfg(wrap_proc_macro)]
1248 fn valid(src: &str) -> bool {
1249 Literal::from_str_checked(repr:src).is_ok()
1250 }
1251}
1252