1#[cfg(wrap_proc_macro)]
2use crate::imp;
3#[cfg(span_locations)]
4use crate::location::LineColumn;
5use crate::parse::{self, Cursor};
6use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut};
7use crate::{Delimiter, Spacing, TokenTree};
8#[cfg(all(span_locations, not(fuzzing)))]
9use alloc::collections::BTreeMap;
10#[cfg(all(span_locations, not(fuzzing)))]
11use core::cell::RefCell;
12#[cfg(span_locations)]
13use core::cmp;
14use core::fmt::{self, Debug, Display, Write};
15use core::mem::ManuallyDrop;
16#[cfg(span_locations)]
17use core::ops::Range;
18use core::ops::RangeBounds;
19use core::ptr;
20use core::str;
21#[cfg(feature = "proc-macro")]
22use core::str::FromStr;
23use std::ffi::CStr;
24#[cfg(wrap_proc_macro)]
25use std::panic;
26#[cfg(procmacro2_semver_exempt)]
27use std::path::PathBuf;
28
29/// Force use of proc-macro2's fallback implementation of the API for now, even
30/// if the compiler's implementation is available.
31pub fn force() {
32 #[cfg(wrap_proc_macro)]
33 crate::detection::force_fallback();
34}
35
36/// Resume using the compiler's implementation of the proc macro API if it is
37/// available.
38pub fn unforce() {
39 #[cfg(wrap_proc_macro)]
40 crate::detection::unforce_fallback();
41}
42
43#[derive(Clone)]
44pub(crate) struct TokenStream {
45 inner: RcVec<TokenTree>,
46}
47
48#[derive(Debug)]
49pub(crate) struct LexError {
50 pub(crate) span: Span,
51}
52
53impl LexError {
54 pub(crate) fn span(&self) -> Span {
55 self.span
56 }
57
58 pub(crate) fn call_site() -> Self {
59 LexError {
60 span: Span::call_site(),
61 }
62 }
63}
64
65impl TokenStream {
66 pub(crate) fn new() -> Self {
67 TokenStream {
68 inner: RcVecBuilder::new().build(),
69 }
70 }
71
72 pub(crate) fn from_str_checked(src: &str) -> Result<Self, LexError> {
73 // Create a dummy file & add it to the source map
74 let mut cursor = get_cursor(src);
75
76 // Strip a byte order mark if present
77 const BYTE_ORDER_MARK: &str = "\u{feff}";
78 if cursor.starts_with(BYTE_ORDER_MARK) {
79 cursor = cursor.advance(BYTE_ORDER_MARK.len());
80 }
81
82 parse::token_stream(cursor)
83 }
84
85 #[cfg(feature = "proc-macro")]
86 pub(crate) fn from_str_unchecked(src: &str) -> Self {
87 Self::from_str_checked(src).unwrap()
88 }
89
90 pub(crate) fn is_empty(&self) -> bool {
91 self.inner.len() == 0
92 }
93
94 fn take_inner(self) -> RcVecBuilder<TokenTree> {
95 let nodrop = ManuallyDrop::new(self);
96 unsafe { ptr::read(&nodrop.inner) }.make_owned()
97 }
98}
99
100fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) {
101 // https://github.com/dtolnay/proc-macro2/issues/235
102 match token {
103 TokenTree::Literal(crate::Literal {
104 #[cfg(wrap_proc_macro)]
105 inner: crate::imp::Literal::Fallback(literal: Literal),
106 #[cfg(not(wrap_proc_macro))]
107 inner: literal,
108 ..
109 }) if literal.repr.starts_with('-') => {
110 push_negative_literal(vec, literal);
111 }
112 _ => vec.push(element:token),
113 }
114
115 #[cold]
116 fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) {
117 literal.repr.remove(idx:0);
118 let mut punct: Punct = crate::Punct::new(ch:'-', Spacing::Alone);
119 punct.set_span(crate::Span::_new_fallback(inner:literal.span));
120 vec.push(element:TokenTree::Punct(punct));
121 vec.push(element:TokenTree::Literal(crate::Literal::_new_fallback(inner:literal)));
122 }
123}
124
125// Nonrecursive to prevent stack overflow.
126impl Drop for TokenStream {
127 fn drop(&mut self) {
128 let mut stack = Vec::new();
129 let mut current = match self.inner.get_mut() {
130 Some(inner) => inner.take().into_iter(),
131 None => return,
132 };
133 loop {
134 while let Some(token) = current.next() {
135 let group = match token {
136 TokenTree::Group(group) => group.inner,
137 _ => continue,
138 };
139 #[cfg(wrap_proc_macro)]
140 let group = match group {
141 crate::imp::Group::Fallback(group) => group,
142 crate::imp::Group::Compiler(_) => continue,
143 };
144 let mut group = group;
145 if let Some(inner) = group.stream.inner.get_mut() {
146 stack.push(current);
147 current = inner.take().into_iter();
148 }
149 }
150 match stack.pop() {
151 Some(next) => current = next,
152 None => return,
153 }
154 }
155 }
156}
157
158pub(crate) struct TokenStreamBuilder {
159 inner: RcVecBuilder<TokenTree>,
160}
161
162impl TokenStreamBuilder {
163 pub(crate) fn new() -> Self {
164 TokenStreamBuilder {
165 inner: RcVecBuilder::new(),
166 }
167 }
168
169 pub(crate) fn with_capacity(cap: usize) -> Self {
170 TokenStreamBuilder {
171 inner: RcVecBuilder::with_capacity(cap),
172 }
173 }
174
175 pub(crate) fn push_token_from_parser(&mut self, tt: TokenTree) {
176 self.inner.push(element:tt);
177 }
178
179 pub(crate) fn build(self) -> TokenStream {
180 TokenStream {
181 inner: self.inner.build(),
182 }
183 }
184}
185
186#[cfg(span_locations)]
187fn get_cursor(src: &str) -> Cursor {
188 #[cfg(fuzzing)]
189 return Cursor { rest: src, off: 1 };
190
191 // Create a dummy file & add it to the source map
192 #[cfg(not(fuzzing))]
193 SOURCE_MAP.with(|sm| {
194 let mut sm = sm.borrow_mut();
195 let span = sm.add_file(src);
196 Cursor {
197 rest: src,
198 off: span.lo,
199 }
200 })
201}
202
203#[cfg(not(span_locations))]
204fn get_cursor(src: &str) -> Cursor {
205 Cursor { rest: src }
206}
207
208impl Display for LexError {
209 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
210 f.write_str(data:"cannot parse string into token stream")
211 }
212}
213
214impl Display for TokenStream {
215 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
216 let mut joint: bool = false;
217 for (i: usize, tt: &TokenTree) in self.inner.iter().enumerate() {
218 if i != 0 && !joint {
219 write!(f, " ")?;
220 }
221 joint = false;
222 match tt {
223 TokenTree::Group(tt: &Group) => Display::fmt(self:tt, f),
224 TokenTree::Ident(tt: &Ident) => Display::fmt(self:tt, f),
225 TokenTree::Punct(tt: &Punct) => {
226 joint = tt.spacing() == Spacing::Joint;
227 Display::fmt(self:tt, f)
228 }
229 TokenTree::Literal(tt: &Literal) => Display::fmt(self:tt, f),
230 }?;
231 }
232
233 Ok(())
234 }
235}
236
237impl Debug for TokenStream {
238 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
239 f.write_str(data:"TokenStream ")?;
240 f.debug_list().entries(self.clone()).finish()
241 }
242}
243
244#[cfg(feature = "proc-macro")]
245impl From<proc_macro::TokenStream> for TokenStream {
246 fn from(inner: proc_macro::TokenStream) -> Self {
247 TokenStream::from_str_unchecked(&inner.to_string())
248 }
249}
250
251#[cfg(feature = "proc-macro")]
252impl From<TokenStream> for proc_macro::TokenStream {
253 fn from(inner: TokenStream) -> Self {
254 proc_macro::TokenStream::from_str_unchecked(&inner.to_string())
255 }
256}
257
258impl From<TokenTree> for TokenStream {
259 fn from(tree: TokenTree) -> Self {
260 let mut stream: RcVecBuilder = RcVecBuilder::new();
261 push_token_from_proc_macro(vec:stream.as_mut(), token:tree);
262 TokenStream {
263 inner: stream.build(),
264 }
265 }
266}
267
268impl FromIterator<TokenTree> for TokenStream {
269 fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self {
270 let mut stream: TokenStream = TokenStream::new();
271 stream.extend(iter:tokens);
272 stream
273 }
274}
275
276impl FromIterator<TokenStream> for TokenStream {
277 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
278 let mut v: RcVecBuilder = RcVecBuilder::new();
279
280 for stream: TokenStream in streams {
281 v.extend(iter:stream.take_inner());
282 }
283
284 TokenStream { inner: v.build() }
285 }
286}
287
288impl Extend<TokenTree> for TokenStream {
289 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) {
290 let mut vec: RcVecMut<'_, TokenTree> = self.inner.make_mut();
291 tokens::IntoIter
292 .into_iter()
293 .for_each(|token: TokenTree| push_token_from_proc_macro(vec.as_mut(), token));
294 }
295}
296
297impl Extend<TokenStream> for TokenStream {
298 fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
299 self.inner.make_mut().extend(iter:streams.into_iter().flatten());
300 }
301}
302
303pub(crate) type TokenTreeIter = RcVecIntoIter<TokenTree>;
304
305impl IntoIterator for TokenStream {
306 type Item = TokenTree;
307 type IntoIter = TokenTreeIter;
308
309 fn into_iter(self) -> TokenTreeIter {
310 self.take_inner().into_iter()
311 }
312}
313
314#[cfg(procmacro2_semver_exempt)]
315#[derive(Clone, PartialEq, Eq)]
316pub(crate) struct SourceFile {
317 path: PathBuf,
318}
319
320#[cfg(procmacro2_semver_exempt)]
321impl SourceFile {
322 /// Get the path to this source file as a string.
323 pub(crate) fn path(&self) -> PathBuf {
324 self.path.clone()
325 }
326
327 pub(crate) fn is_real(&self) -> bool {
328 false
329 }
330}
331
332#[cfg(procmacro2_semver_exempt)]
333impl Debug for SourceFile {
334 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
335 f.debug_struct("SourceFile")
336 .field("path", &self.path())
337 .field("is_real", &self.is_real())
338 .finish()
339 }
340}
341
342#[cfg(all(span_locations, not(fuzzing)))]
343thread_local! {
344 static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
345 // Start with a single dummy file which all call_site() and def_site()
346 // spans reference.
347 files: vec![FileInfo {
348 source_text: String::new(),
349 span: Span { lo: 0, hi: 0 },
350 lines: vec![0],
351 char_index_to_byte_offset: BTreeMap::new(),
352 }],
353 });
354}
355
356#[cfg(span_locations)]
357pub(crate) fn invalidate_current_thread_spans() {
358 #[cfg(not(fuzzing))]
359 SOURCE_MAP.with(|sm| sm.borrow_mut().files.truncate(1));
360}
361
362#[cfg(all(span_locations, not(fuzzing)))]
363struct FileInfo {
364 source_text: String,
365 span: Span,
366 lines: Vec<usize>,
367 char_index_to_byte_offset: BTreeMap<usize, usize>,
368}
369
370#[cfg(all(span_locations, not(fuzzing)))]
371impl FileInfo {
372 fn offset_line_column(&self, offset: usize) -> LineColumn {
373 assert!(self.span_within(Span {
374 lo: offset as u32,
375 hi: offset as u32,
376 }));
377 let offset = offset - self.span.lo as usize;
378 match self.lines.binary_search(&offset) {
379 Ok(found) => LineColumn {
380 line: found + 1,
381 column: 0,
382 },
383 Err(idx) => LineColumn {
384 line: idx,
385 column: offset - self.lines[idx - 1],
386 },
387 }
388 }
389
390 fn span_within(&self, span: Span) -> bool {
391 span.lo >= self.span.lo && span.hi <= self.span.hi
392 }
393
394 fn byte_range(&mut self, span: Span) -> Range<usize> {
395 let lo_char = (span.lo - self.span.lo) as usize;
396
397 // Look up offset of the largest already-computed char index that is
398 // less than or equal to the current requested one. We resume counting
399 // chars from that point.
400 let (&last_char_index, &last_byte_offset) = self
401 .char_index_to_byte_offset
402 .range(..=lo_char)
403 .next_back()
404 .unwrap_or((&0, &0));
405
406 let lo_byte = if last_char_index == lo_char {
407 last_byte_offset
408 } else {
409 let total_byte_offset = match self.source_text[last_byte_offset..]
410 .char_indices()
411 .nth(lo_char - last_char_index)
412 {
413 Some((additional_offset, _ch)) => last_byte_offset + additional_offset,
414 None => self.source_text.len(),
415 };
416 self.char_index_to_byte_offset
417 .insert(lo_char, total_byte_offset);
418 total_byte_offset
419 };
420
421 let trunc_lo = &self.source_text[lo_byte..];
422 let char_len = (span.hi - span.lo) as usize;
423 lo_byte..match trunc_lo.char_indices().nth(char_len) {
424 Some((offset, _ch)) => lo_byte + offset,
425 None => self.source_text.len(),
426 }
427 }
428
429 fn source_text(&mut self, span: Span) -> String {
430 let byte_range = self.byte_range(span);
431 self.source_text[byte_range].to_owned()
432 }
433}
434
435/// Computes the offsets of each line in the given source string
436/// and the total number of characters
437#[cfg(all(span_locations, not(fuzzing)))]
438fn lines_offsets(s: &str) -> (usize, Vec<usize>) {
439 let mut lines = vec![0];
440 let mut total = 0;
441
442 for ch in s.chars() {
443 total += 1;
444 if ch == '\n' {
445 lines.push(total);
446 }
447 }
448
449 (total, lines)
450}
451
452#[cfg(all(span_locations, not(fuzzing)))]
453struct SourceMap {
454 files: Vec<FileInfo>,
455}
456
457#[cfg(all(span_locations, not(fuzzing)))]
458impl SourceMap {
459 fn next_start_pos(&self) -> u32 {
460 // Add 1 so there's always space between files.
461 //
462 // We'll always have at least 1 file, as we initialize our files list
463 // with a dummy file.
464 self.files.last().unwrap().span.hi + 1
465 }
466
467 fn add_file(&mut self, src: &str) -> Span {
468 let (len, lines) = lines_offsets(src);
469 let lo = self.next_start_pos();
470 let span = Span {
471 lo,
472 hi: lo + (len as u32),
473 };
474
475 self.files.push(FileInfo {
476 source_text: src.to_owned(),
477 span,
478 lines,
479 // Populated lazily by source_text().
480 char_index_to_byte_offset: BTreeMap::new(),
481 });
482
483 span
484 }
485
486 #[cfg(procmacro2_semver_exempt)]
487 fn filepath(&self, span: Span) -> PathBuf {
488 for (i, file) in self.files.iter().enumerate() {
489 if file.span_within(span) {
490 return PathBuf::from(if i == 0 {
491 "<unspecified>".to_owned()
492 } else {
493 format!("<parsed string {}>", i)
494 });
495 }
496 }
497 unreachable!("Invalid span with no related FileInfo!");
498 }
499
500 fn fileinfo(&self, span: Span) -> &FileInfo {
501 for file in &self.files {
502 if file.span_within(span) {
503 return file;
504 }
505 }
506 unreachable!("Invalid span with no related FileInfo!");
507 }
508
509 fn fileinfo_mut(&mut self, span: Span) -> &mut FileInfo {
510 for file in &mut self.files {
511 if file.span_within(span) {
512 return file;
513 }
514 }
515 unreachable!("Invalid span with no related FileInfo!");
516 }
517}
518
519#[derive(Clone, Copy, PartialEq, Eq)]
520pub(crate) struct Span {
521 #[cfg(span_locations)]
522 pub(crate) lo: u32,
523 #[cfg(span_locations)]
524 pub(crate) hi: u32,
525}
526
527impl Span {
528 #[cfg(not(span_locations))]
529 pub(crate) fn call_site() -> Self {
530 Span {}
531 }
532
533 #[cfg(span_locations)]
534 pub(crate) fn call_site() -> Self {
535 Span { lo: 0, hi: 0 }
536 }
537
538 pub(crate) fn mixed_site() -> Self {
539 Span::call_site()
540 }
541
542 #[cfg(procmacro2_semver_exempt)]
543 pub(crate) fn def_site() -> Self {
544 Span::call_site()
545 }
546
547 pub(crate) fn resolved_at(&self, _other: Span) -> Span {
548 // Stable spans consist only of line/column information, so
549 // `resolved_at` and `located_at` only select which span the
550 // caller wants line/column information from.
551 *self
552 }
553
554 pub(crate) fn located_at(&self, other: Span) -> Span {
555 other
556 }
557
558 #[cfg(procmacro2_semver_exempt)]
559 pub(crate) fn source_file(&self) -> SourceFile {
560 #[cfg(fuzzing)]
561 return SourceFile {
562 path: PathBuf::from("<unspecified>"),
563 };
564
565 #[cfg(not(fuzzing))]
566 SOURCE_MAP.with(|sm| {
567 let sm = sm.borrow();
568 let path = sm.filepath(*self);
569 SourceFile { path }
570 })
571 }
572
573 #[cfg(span_locations)]
574 pub(crate) fn byte_range(&self) -> Range<usize> {
575 #[cfg(fuzzing)]
576 return 0..0;
577
578 #[cfg(not(fuzzing))]
579 {
580 if self.is_call_site() {
581 0..0
582 } else {
583 SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).byte_range(*self))
584 }
585 }
586 }
587
588 #[cfg(span_locations)]
589 pub(crate) fn start(&self) -> LineColumn {
590 #[cfg(fuzzing)]
591 return LineColumn { line: 0, column: 0 };
592
593 #[cfg(not(fuzzing))]
594 SOURCE_MAP.with(|sm| {
595 let sm = sm.borrow();
596 let fi = sm.fileinfo(*self);
597 fi.offset_line_column(self.lo as usize)
598 })
599 }
600
601 #[cfg(span_locations)]
602 pub(crate) fn end(&self) -> LineColumn {
603 #[cfg(fuzzing)]
604 return LineColumn { line: 0, column: 0 };
605
606 #[cfg(not(fuzzing))]
607 SOURCE_MAP.with(|sm| {
608 let sm = sm.borrow();
609 let fi = sm.fileinfo(*self);
610 fi.offset_line_column(self.hi as usize)
611 })
612 }
613
614 #[cfg(not(span_locations))]
615 pub(crate) fn join(&self, _other: Span) -> Option<Span> {
616 Some(Span {})
617 }
618
619 #[cfg(span_locations)]
620 pub(crate) fn join(&self, other: Span) -> Option<Span> {
621 #[cfg(fuzzing)]
622 return {
623 let _ = other;
624 None
625 };
626
627 #[cfg(not(fuzzing))]
628 SOURCE_MAP.with(|sm| {
629 let sm = sm.borrow();
630 // If `other` is not within the same FileInfo as us, return None.
631 if !sm.fileinfo(*self).span_within(other) {
632 return None;
633 }
634 Some(Span {
635 lo: cmp::min(self.lo, other.lo),
636 hi: cmp::max(self.hi, other.hi),
637 })
638 })
639 }
640
641 #[cfg(not(span_locations))]
642 pub(crate) fn source_text(&self) -> Option<String> {
643 None
644 }
645
646 #[cfg(span_locations)]
647 pub(crate) fn source_text(&self) -> Option<String> {
648 #[cfg(fuzzing)]
649 return None;
650
651 #[cfg(not(fuzzing))]
652 {
653 if self.is_call_site() {
654 None
655 } else {
656 Some(SOURCE_MAP.with(|sm| sm.borrow_mut().fileinfo_mut(*self).source_text(*self)))
657 }
658 }
659 }
660
661 #[cfg(not(span_locations))]
662 pub(crate) fn first_byte(self) -> Self {
663 self
664 }
665
666 #[cfg(span_locations)]
667 pub(crate) fn first_byte(self) -> Self {
668 Span {
669 lo: self.lo,
670 hi: cmp::min(self.lo.saturating_add(1), self.hi),
671 }
672 }
673
674 #[cfg(not(span_locations))]
675 pub(crate) fn last_byte(self) -> Self {
676 self
677 }
678
679 #[cfg(span_locations)]
680 pub(crate) fn last_byte(self) -> Self {
681 Span {
682 lo: cmp::max(self.hi.saturating_sub(1), self.lo),
683 hi: self.hi,
684 }
685 }
686
687 #[cfg(span_locations)]
688 fn is_call_site(&self) -> bool {
689 self.lo == 0 && self.hi == 0
690 }
691}
692
693impl Debug for Span {
694 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
695 #[cfg(span_locations)]
696 return write!(f, "bytes({}..{})", self.lo, self.hi);
697
698 #[cfg(not(span_locations))]
699 write!(f, "Span")
700 }
701}
702
703pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
704 #[cfg(span_locations)]
705 {
706 if span.is_call_site() {
707 return;
708 }
709 }
710
711 if cfg!(span_locations) {
712 debug.field(name:"span", &span);
713 }
714}
715
716#[derive(Clone)]
717pub(crate) struct Group {
718 delimiter: Delimiter,
719 stream: TokenStream,
720 span: Span,
721}
722
723impl Group {
724 pub(crate) fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
725 Group {
726 delimiter,
727 stream,
728 span: Span::call_site(),
729 }
730 }
731
732 pub(crate) fn delimiter(&self) -> Delimiter {
733 self.delimiter
734 }
735
736 pub(crate) fn stream(&self) -> TokenStream {
737 self.stream.clone()
738 }
739
740 pub(crate) fn span(&self) -> Span {
741 self.span
742 }
743
744 pub(crate) fn span_open(&self) -> Span {
745 self.span.first_byte()
746 }
747
748 pub(crate) fn span_close(&self) -> Span {
749 self.span.last_byte()
750 }
751
752 pub(crate) fn set_span(&mut self, span: Span) {
753 self.span = span;
754 }
755}
756
757impl Display for Group {
758 // We attempt to match libproc_macro's formatting.
759 // Empty parens: ()
760 // Nonempty parens: (...)
761 // Empty brackets: []
762 // Nonempty brackets: [...]
763 // Empty braces: { }
764 // Nonempty braces: { ... }
765 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
766 let (open, close) = match self.delimiter {
767 Delimiter::Parenthesis => ("(", ")"),
768 Delimiter::Brace => ("{ ", "}"),
769 Delimiter::Bracket => ("[", "]"),
770 Delimiter::None => ("", ""),
771 };
772
773 f.write_str(open)?;
774 Display::fmt(&self.stream, f)?;
775 if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() {
776 f.write_str(" ")?;
777 }
778 f.write_str(close)?;
779
780 Ok(())
781 }
782}
783
784impl Debug for Group {
785 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
786 let mut debug: DebugStruct<'_, '_> = fmt.debug_struct(name:"Group");
787 debug.field(name:"delimiter", &self.delimiter);
788 debug.field(name:"stream", &self.stream);
789 debug_span_field_if_nontrivial(&mut debug, self.span);
790 debug.finish()
791 }
792}
793
794#[derive(Clone)]
795pub(crate) struct Ident {
796 sym: Box<str>,
797 span: Span,
798 raw: bool,
799}
800
801impl Ident {
802 #[track_caller]
803 pub(crate) fn new_checked(string: &str, span: Span) -> Self {
804 validate_ident(string);
805 Ident::new_unchecked(string, span)
806 }
807
808 pub(crate) fn new_unchecked(string: &str, span: Span) -> Self {
809 Ident {
810 sym: Box::from(string),
811 span,
812 raw: false,
813 }
814 }
815
816 #[track_caller]
817 pub(crate) fn new_raw_checked(string: &str, span: Span) -> Self {
818 validate_ident_raw(string);
819 Ident::new_raw_unchecked(string, span)
820 }
821
822 pub(crate) fn new_raw_unchecked(string: &str, span: Span) -> Self {
823 Ident {
824 sym: Box::from(string),
825 span,
826 raw: true,
827 }
828 }
829
830 pub(crate) fn span(&self) -> Span {
831 self.span
832 }
833
834 pub(crate) fn set_span(&mut self, span: Span) {
835 self.span = span;
836 }
837}
838
839pub(crate) fn is_ident_start(c: char) -> bool {
840 c == '_' || unicode_ident::is_xid_start(ch:c)
841}
842
843pub(crate) fn is_ident_continue(c: char) -> bool {
844 unicode_ident::is_xid_continue(ch:c)
845}
846
847#[track_caller]
848fn validate_ident(string: &str) {
849 if string.is_empty() {
850 panic!("Ident is not allowed to be empty; use Option<Ident>");
851 }
852
853 if string.bytes().all(|digit| b'0' <= digit && digit <= b'9') {
854 panic!("Ident cannot be a number; use Literal instead");
855 }
856
857 fn ident_ok(string: &str) -> bool {
858 let mut chars = string.chars();
859 let first = chars.next().unwrap();
860 if !is_ident_start(first) {
861 return false;
862 }
863 for ch in chars {
864 if !is_ident_continue(ch) {
865 return false;
866 }
867 }
868 true
869 }
870
871 if !ident_ok(string) {
872 panic!("{:?} is not a valid Ident", string);
873 }
874}
875
876#[track_caller]
877fn validate_ident_raw(string: &str) {
878 validate_ident(string);
879
880 match string {
881 "_" | "super" | "self" | "Self" | "crate" => {
882 panic!("`r#{}` cannot be a raw identifier", string);
883 }
884 _ => {}
885 }
886}
887
888impl PartialEq for Ident {
889 fn eq(&self, other: &Ident) -> bool {
890 self.sym == other.sym && self.raw == other.raw
891 }
892}
893
894impl<T> PartialEq<T> for Ident
895where
896 T: ?Sized + AsRef<str>,
897{
898 fn eq(&self, other: &T) -> bool {
899 let other: &str = other.as_ref();
900 if self.raw {
901 other.starts_with("r#") && *self.sym == other[2..]
902 } else {
903 *self.sym == *other
904 }
905 }
906}
907
908impl Display for Ident {
909 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
910 if self.raw {
911 f.write_str(data:"r#")?;
912 }
913 Display::fmt(&self.sym, f)
914 }
915}
916
917#[allow(clippy::missing_fields_in_debug)]
918impl Debug for Ident {
919 // Ident(proc_macro), Ident(r#union)
920 #[cfg(not(span_locations))]
921 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
922 let mut debug: DebugTuple<'_, '_> = f.debug_tuple(name:"Ident");
923 debug.field(&format_args!("{}", self));
924 debug.finish()
925 }
926
927 // Ident {
928 // sym: proc_macro,
929 // span: bytes(128..138)
930 // }
931 #[cfg(span_locations)]
932 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
933 let mut debug = f.debug_struct("Ident");
934 debug.field("sym", &format_args!("{}", self));
935 debug_span_field_if_nontrivial(&mut debug, self.span);
936 debug.finish()
937 }
938}
939
940#[derive(Clone)]
941pub(crate) struct Literal {
942 pub(crate) repr: String,
943 span: Span,
944}
945
946macro_rules! suffixed_numbers {
947 ($($name:ident => $kind:ident,)*) => ($(
948 pub(crate) fn $name(n: $kind) -> Literal {
949 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
950 }
951 )*)
952}
953
954macro_rules! unsuffixed_numbers {
955 ($($name:ident => $kind:ident,)*) => ($(
956 pub(crate) fn $name(n: $kind) -> Literal {
957 Literal::_new(n.to_string())
958 }
959 )*)
960}
961
962impl Literal {
963 pub(crate) fn _new(repr: String) -> Self {
964 Literal {
965 repr,
966 span: Span::call_site(),
967 }
968 }
969
970 pub(crate) fn from_str_checked(repr: &str) -> Result<Self, LexError> {
971 let mut cursor = get_cursor(repr);
972 #[cfg(span_locations)]
973 let lo = cursor.off;
974
975 let negative = cursor.starts_with_char('-');
976 if negative {
977 cursor = cursor.advance(1);
978 if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) {
979 return Err(LexError::call_site());
980 }
981 }
982
983 if let Ok((rest, mut literal)) = parse::literal(cursor) {
984 if rest.is_empty() {
985 if negative {
986 literal.repr.insert(0, '-');
987 }
988 literal.span = Span {
989 #[cfg(span_locations)]
990 lo,
991 #[cfg(span_locations)]
992 hi: rest.off,
993 };
994 return Ok(literal);
995 }
996 }
997 Err(LexError::call_site())
998 }
999
1000 pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self {
1001 Literal::_new(repr.to_owned())
1002 }
1003
1004 suffixed_numbers! {
1005 u8_suffixed => u8,
1006 u16_suffixed => u16,
1007 u32_suffixed => u32,
1008 u64_suffixed => u64,
1009 u128_suffixed => u128,
1010 usize_suffixed => usize,
1011 i8_suffixed => i8,
1012 i16_suffixed => i16,
1013 i32_suffixed => i32,
1014 i64_suffixed => i64,
1015 i128_suffixed => i128,
1016 isize_suffixed => isize,
1017
1018 f32_suffixed => f32,
1019 f64_suffixed => f64,
1020 }
1021
1022 unsuffixed_numbers! {
1023 u8_unsuffixed => u8,
1024 u16_unsuffixed => u16,
1025 u32_unsuffixed => u32,
1026 u64_unsuffixed => u64,
1027 u128_unsuffixed => u128,
1028 usize_unsuffixed => usize,
1029 i8_unsuffixed => i8,
1030 i16_unsuffixed => i16,
1031 i32_unsuffixed => i32,
1032 i64_unsuffixed => i64,
1033 i128_unsuffixed => i128,
1034 isize_unsuffixed => isize,
1035 }
1036
1037 pub(crate) fn f32_unsuffixed(f: f32) -> Literal {
1038 let mut s = f.to_string();
1039 if !s.contains('.') {
1040 s.push_str(".0");
1041 }
1042 Literal::_new(s)
1043 }
1044
1045 pub(crate) fn f64_unsuffixed(f: f64) -> Literal {
1046 let mut s = f.to_string();
1047 if !s.contains('.') {
1048 s.push_str(".0");
1049 }
1050 Literal::_new(s)
1051 }
1052
1053 pub(crate) fn string(string: &str) -> Literal {
1054 let mut repr = String::with_capacity(string.len() + 2);
1055 repr.push('"');
1056 escape_utf8(string, &mut repr);
1057 repr.push('"');
1058 Literal::_new(repr)
1059 }
1060
1061 pub(crate) fn character(ch: char) -> Literal {
1062 let mut repr = String::new();
1063 repr.push('\'');
1064 if ch == '"' {
1065 // escape_debug turns this into '\"' which is unnecessary.
1066 repr.push(ch);
1067 } else {
1068 repr.extend(ch.escape_debug());
1069 }
1070 repr.push('\'');
1071 Literal::_new(repr)
1072 }
1073
1074 pub(crate) fn byte_character(byte: u8) -> Literal {
1075 let mut repr = "b'".to_string();
1076 #[allow(clippy::match_overlapping_arm)]
1077 match byte {
1078 b'\0' => repr.push_str(r"\0"),
1079 b'\t' => repr.push_str(r"\t"),
1080 b'\n' => repr.push_str(r"\n"),
1081 b'\r' => repr.push_str(r"\r"),
1082 b'\'' => repr.push_str(r"\'"),
1083 b'\\' => repr.push_str(r"\\"),
1084 b'\x20'..=b'\x7E' => repr.push(byte as char),
1085 _ => {
1086 let _ = write!(repr, r"\x{:02X}", byte);
1087 }
1088 }
1089 repr.push('\'');
1090 Literal::_new(repr)
1091 }
1092
1093 pub(crate) fn byte_string(bytes: &[u8]) -> Literal {
1094 let mut repr = "b\"".to_string();
1095 let mut bytes = bytes.iter();
1096 while let Some(&b) = bytes.next() {
1097 #[allow(clippy::match_overlapping_arm)]
1098 match b {
1099 b'\0' => repr.push_str(match bytes.as_slice().first() {
1100 // circumvent clippy::octal_escapes lint
1101 Some(b'0'..=b'7') => r"\x00",
1102 _ => r"\0",
1103 }),
1104 b'\t' => repr.push_str(r"\t"),
1105 b'\n' => repr.push_str(r"\n"),
1106 b'\r' => repr.push_str(r"\r"),
1107 b'"' => repr.push_str("\\\""),
1108 b'\\' => repr.push_str(r"\\"),
1109 b'\x20'..=b'\x7E' => repr.push(b as char),
1110 _ => {
1111 let _ = write!(repr, r"\x{:02X}", b);
1112 }
1113 }
1114 }
1115 repr.push('"');
1116 Literal::_new(repr)
1117 }
1118
1119 pub(crate) fn c_string(string: &CStr) -> Literal {
1120 let mut repr = "c\"".to_string();
1121 let mut bytes = string.to_bytes();
1122 while !bytes.is_empty() {
1123 let (valid, invalid) = match str::from_utf8(bytes) {
1124 Ok(all_valid) => {
1125 bytes = b"";
1126 (all_valid, bytes)
1127 }
1128 Err(utf8_error) => {
1129 let (valid, rest) = bytes.split_at(utf8_error.valid_up_to());
1130 let valid = str::from_utf8(valid).unwrap();
1131 let invalid = utf8_error
1132 .error_len()
1133 .map_or(rest, |error_len| &rest[..error_len]);
1134 bytes = &bytes[valid.len() + invalid.len()..];
1135 (valid, invalid)
1136 }
1137 };
1138 escape_utf8(valid, &mut repr);
1139 for &byte in invalid {
1140 let _ = write!(repr, r"\x{:02X}", byte);
1141 }
1142 }
1143 repr.push('"');
1144 Literal::_new(repr)
1145 }
1146
1147 pub(crate) fn span(&self) -> Span {
1148 self.span
1149 }
1150
1151 pub(crate) fn set_span(&mut self, span: Span) {
1152 self.span = span;
1153 }
1154
1155 pub(crate) fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> {
1156 #[cfg(not(span_locations))]
1157 {
1158 let _ = range;
1159 None
1160 }
1161
1162 #[cfg(span_locations)]
1163 {
1164 use core::ops::Bound;
1165
1166 let lo = match range.start_bound() {
1167 Bound::Included(start) => {
1168 let start = u32::try_from(*start).ok()?;
1169 self.span.lo.checked_add(start)?
1170 }
1171 Bound::Excluded(start) => {
1172 let start = u32::try_from(*start).ok()?;
1173 self.span.lo.checked_add(start)?.checked_add(1)?
1174 }
1175 Bound::Unbounded => self.span.lo,
1176 };
1177 let hi = match range.end_bound() {
1178 Bound::Included(end) => {
1179 let end = u32::try_from(*end).ok()?;
1180 self.span.lo.checked_add(end)?.checked_add(1)?
1181 }
1182 Bound::Excluded(end) => {
1183 let end = u32::try_from(*end).ok()?;
1184 self.span.lo.checked_add(end)?
1185 }
1186 Bound::Unbounded => self.span.hi,
1187 };
1188 if lo <= hi && hi <= self.span.hi {
1189 Some(Span { lo, hi })
1190 } else {
1191 None
1192 }
1193 }
1194 }
1195}
1196
1197impl Display for Literal {
1198 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1199 Display::fmt(&self.repr, f)
1200 }
1201}
1202
1203impl Debug for Literal {
1204 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1205 let mut debug: DebugStruct<'_, '_> = fmt.debug_struct(name:"Literal");
1206 debug.field(name:"lit", &format_args!("{}", self.repr));
1207 debug_span_field_if_nontrivial(&mut debug, self.span);
1208 debug.finish()
1209 }
1210}
1211
1212fn escape_utf8(string: &str, repr: &mut String) {
1213 let mut chars: Chars<'_> = string.chars();
1214 while let Some(ch: char) = chars.next() {
1215 if ch == '\0' {
1216 repr.push_str(
1217 string:if chars&str
1218 .as_str()
1219 .starts_with(|next: char| '0' <= next && next <= '7')
1220 {
1221 // circumvent clippy::octal_escapes lint
1222 r"\x00"
1223 } else {
1224 r"\0"
1225 },
1226 );
1227 } else if ch == '\'' {
1228 // escape_debug turns this into "\'" which is unnecessary.
1229 repr.push(ch);
1230 } else {
1231 repr.extend(iter:ch.escape_debug());
1232 }
1233 }
1234}
1235
1236#[cfg(feature = "proc-macro")]
1237pub(crate) trait FromStr2: FromStr<Err = proc_macro::LexError> {
1238 #[cfg(wrap_proc_macro)]
1239 fn valid(src: &str) -> bool;
1240
1241 #[cfg(wrap_proc_macro)]
1242 fn from_str_checked(src: &str) -> Result<Self, imp::LexError> {
1243 // Validate using fallback parser, because rustc is incapable of
1244 // returning a recoverable Err for certain invalid token streams, and
1245 // will instead permanently poison the compilation.
1246 if !Self::valid(src) {
1247 return Err(imp::LexError::CompilerPanic);
1248 }
1249
1250 // Catch panic to work around https://github.com/rust-lang/rust/issues/58736.
1251 match panic::catch_unwind(|| Self::from_str(src)) {
1252 Ok(Ok(ok)) => Ok(ok),
1253 Ok(Err(lex)) => Err(imp::LexError::Compiler(lex)),
1254 Err(_panic) => Err(imp::LexError::CompilerPanic),
1255 }
1256 }
1257
1258 fn from_str_unchecked(src: &str) -> Self {
1259 Self::from_str(src).unwrap()
1260 }
1261}
1262
1263#[cfg(feature = "proc-macro")]
1264impl FromStr2 for proc_macro::TokenStream {
1265 #[cfg(wrap_proc_macro)]
1266 fn valid(src: &str) -> bool {
1267 TokenStream::from_str_checked(src).is_ok()
1268 }
1269}
1270
1271#[cfg(feature = "proc-macro")]
1272impl FromStr2 for proc_macro::Literal {
1273 #[cfg(wrap_proc_macro)]
1274 fn valid(src: &str) -> bool {
1275 Literal::from_str_checked(repr:src).is_ok()
1276 }
1277}
1278