1 | #[cfg (span_locations)] |
2 | use crate::location::LineColumn; |
3 | use crate::parse::{self, Cursor}; |
4 | use crate::rcvec::{RcVec, RcVecBuilder, RcVecIntoIter, RcVecMut}; |
5 | use crate::{Delimiter, Spacing, TokenTree}; |
6 | #[cfg (all(span_locations, not(fuzzing)))] |
7 | use core::cell::RefCell; |
8 | #[cfg (span_locations)] |
9 | use core::cmp; |
10 | use core::fmt::{self, Debug, Display, Write}; |
11 | use core::iter::FromIterator; |
12 | use core::mem::ManuallyDrop; |
13 | use core::ops::RangeBounds; |
14 | use core::ptr; |
15 | use core::str::FromStr; |
16 | use std::path::PathBuf; |
17 | |
18 | /// Force use of proc-macro2's fallback implementation of the API for now, even |
19 | /// if the compiler's implementation is available. |
20 | pub fn force() { |
21 | #[cfg (wrap_proc_macro)] |
22 | crate::detection::force_fallback(); |
23 | } |
24 | |
25 | /// Resume using the compiler's implementation of the proc macro API if it is |
26 | /// available. |
27 | pub fn unforce() { |
28 | #[cfg (wrap_proc_macro)] |
29 | crate::detection::unforce_fallback(); |
30 | } |
31 | |
32 | #[derive (Clone)] |
33 | pub(crate) struct TokenStream { |
34 | inner: RcVec<TokenTree>, |
35 | } |
36 | |
37 | #[derive (Debug)] |
38 | pub(crate) struct LexError { |
39 | pub(crate) span: Span, |
40 | } |
41 | |
42 | impl LexError { |
43 | pub(crate) fn span(&self) -> Span { |
44 | self.span |
45 | } |
46 | |
47 | fn call_site() -> Self { |
48 | LexError { |
49 | span: Span::call_site(), |
50 | } |
51 | } |
52 | } |
53 | |
54 | impl TokenStream { |
55 | pub fn new() -> Self { |
56 | TokenStream { |
57 | inner: RcVecBuilder::new().build(), |
58 | } |
59 | } |
60 | |
61 | pub fn is_empty(&self) -> bool { |
62 | self.inner.len() == 0 |
63 | } |
64 | |
65 | fn take_inner(self) -> RcVecBuilder<TokenTree> { |
66 | let nodrop: ManuallyDrop = ManuallyDrop::new(self); |
67 | unsafe { ptr::read(&nodrop.inner) }.make_owned() |
68 | } |
69 | } |
70 | |
71 | fn push_token_from_proc_macro(mut vec: RcVecMut<TokenTree>, token: TokenTree) { |
72 | // https://github.com/dtolnay/proc-macro2/issues/235 |
73 | match token { |
74 | #[cfg (not(no_bind_by_move_pattern_guard))] |
75 | TokenTree::Literal(crate::Literal { |
76 | #[cfg (wrap_proc_macro)] |
77 | inner: crate::imp::Literal::Fallback(literal), |
78 | #[cfg (not(wrap_proc_macro))] |
79 | inner: literal, |
80 | .. |
81 | }) if literal.repr.starts_with('-' ) => { |
82 | push_negative_literal(vec, literal); |
83 | } |
84 | #[cfg (no_bind_by_move_pattern_guard)] |
85 | TokenTree::Literal(crate::Literal { |
86 | #[cfg (wrap_proc_macro)] |
87 | inner: crate::imp::Literal::Fallback(literal), |
88 | #[cfg (not(wrap_proc_macro))] |
89 | inner: literal, |
90 | .. |
91 | }) => { |
92 | if literal.repr.starts_with('-' ) { |
93 | push_negative_literal(vec, literal); |
94 | } else { |
95 | vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal))); |
96 | } |
97 | } |
98 | _ => vec.push(token), |
99 | } |
100 | |
101 | #[cold ] |
102 | fn push_negative_literal(mut vec: RcVecMut<TokenTree>, mut literal: Literal) { |
103 | literal.repr.remove(0); |
104 | let mut punct = crate::Punct::new('-' , Spacing::Alone); |
105 | punct.set_span(crate::Span::_new_fallback(literal.span)); |
106 | vec.push(TokenTree::Punct(punct)); |
107 | vec.push(TokenTree::Literal(crate::Literal::_new_fallback(literal))); |
108 | } |
109 | } |
110 | |
111 | // Nonrecursive to prevent stack overflow. |
112 | impl Drop for TokenStream { |
113 | fn drop(&mut self) { |
114 | let mut inner: RcVecMut<'_, TokenTree> = match self.inner.get_mut() { |
115 | Some(inner: RcVecMut<'_, TokenTree>) => inner, |
116 | None => return, |
117 | }; |
118 | while let Some(token: TokenTree) = inner.pop() { |
119 | let group: Group = match token { |
120 | TokenTree::Group(group: Group) => group.inner, |
121 | _ => continue, |
122 | }; |
123 | #[cfg (wrap_proc_macro)] |
124 | let group: Group = match group { |
125 | crate::imp::Group::Fallback(group: Group) => group, |
126 | crate::imp::Group::Compiler(_) => continue, |
127 | }; |
128 | inner.extend(iter:group.stream.take_inner()); |
129 | } |
130 | } |
131 | } |
132 | |
133 | pub(crate) struct TokenStreamBuilder { |
134 | inner: RcVecBuilder<TokenTree>, |
135 | } |
136 | |
137 | impl TokenStreamBuilder { |
138 | pub fn new() -> Self { |
139 | TokenStreamBuilder { |
140 | inner: RcVecBuilder::new(), |
141 | } |
142 | } |
143 | |
144 | pub fn with_capacity(cap: usize) -> Self { |
145 | TokenStreamBuilder { |
146 | inner: RcVecBuilder::with_capacity(cap), |
147 | } |
148 | } |
149 | |
150 | pub fn push_token_from_parser(&mut self, tt: TokenTree) { |
151 | self.inner.push(element:tt); |
152 | } |
153 | |
154 | pub fn build(self) -> TokenStream { |
155 | TokenStream { |
156 | inner: self.inner.build(), |
157 | } |
158 | } |
159 | } |
160 | |
161 | #[cfg (span_locations)] |
162 | fn get_cursor(src: &str) -> Cursor { |
163 | #[cfg (fuzzing)] |
164 | return Cursor { rest: src, off: 1 }; |
165 | |
166 | // Create a dummy file & add it to the source map |
167 | #[cfg (not(fuzzing))] |
168 | SOURCE_MAP.with(|cm: &RefCell| { |
169 | let mut cm: RefMut<'_, SourceMap> = cm.borrow_mut(); |
170 | let span: Span = cm.add_file(src); |
171 | Cursor { |
172 | rest: src, |
173 | off: span.lo, |
174 | } |
175 | }) |
176 | } |
177 | |
178 | #[cfg (not(span_locations))] |
179 | fn get_cursor(src: &str) -> Cursor { |
180 | Cursor { rest: src } |
181 | } |
182 | |
183 | impl FromStr for TokenStream { |
184 | type Err = LexError; |
185 | |
186 | fn from_str(src: &str) -> Result<TokenStream, LexError> { |
187 | // Create a dummy file & add it to the source map |
188 | let mut cursor: Cursor<'_> = get_cursor(src); |
189 | |
190 | // Strip a byte order mark if present |
191 | const BYTE_ORDER_MARK: &str = " \u{feff}" ; |
192 | if cursor.starts_with(BYTE_ORDER_MARK) { |
193 | cursor = cursor.advance(BYTE_ORDER_MARK.len()); |
194 | } |
195 | |
196 | parse::token_stream(input:cursor) |
197 | } |
198 | } |
199 | |
200 | impl Display for LexError { |
201 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
202 | f.write_str(data:"cannot parse string into token stream" ) |
203 | } |
204 | } |
205 | |
206 | impl Display for TokenStream { |
207 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
208 | let mut joint: bool = false; |
209 | for (i: usize, tt: &TokenTree) in self.inner.iter().enumerate() { |
210 | if i != 0 && !joint { |
211 | write!(f, " " )?; |
212 | } |
213 | joint = false; |
214 | match tt { |
215 | TokenTree::Group(tt: &Group) => Display::fmt(self:tt, f), |
216 | TokenTree::Ident(tt: &Ident) => Display::fmt(self:tt, f), |
217 | TokenTree::Punct(tt: &Punct) => { |
218 | joint = tt.spacing() == Spacing::Joint; |
219 | Display::fmt(self:tt, f) |
220 | } |
221 | TokenTree::Literal(tt: &Literal) => Display::fmt(self:tt, f), |
222 | }?; |
223 | } |
224 | |
225 | Ok(()) |
226 | } |
227 | } |
228 | |
229 | impl Debug for TokenStream { |
230 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
231 | f.write_str(data:"TokenStream " )?; |
232 | f.debug_list().entries(self.clone()).finish() |
233 | } |
234 | } |
235 | |
236 | #[cfg (feature = "proc-macro" )] |
237 | impl From<proc_macro::TokenStream> for TokenStream { |
238 | fn from(inner: proc_macro::TokenStream) -> Self { |
239 | inner |
240 | .to_string() |
241 | .parse() |
242 | .expect(msg:"compiler token stream parse failed" ) |
243 | } |
244 | } |
245 | |
246 | #[cfg (feature = "proc-macro" )] |
247 | impl From<TokenStream> for proc_macro::TokenStream { |
248 | fn from(inner: TokenStream) -> Self { |
249 | inner |
250 | .to_string() |
251 | .parse() |
252 | .expect(msg:"failed to parse to compiler tokens" ) |
253 | } |
254 | } |
255 | |
256 | impl From<TokenTree> for TokenStream { |
257 | fn from(tree: TokenTree) -> Self { |
258 | let mut stream: RcVecBuilder = RcVecBuilder::new(); |
259 | push_token_from_proc_macro(vec:stream.as_mut(), token:tree); |
260 | TokenStream { |
261 | inner: stream.build(), |
262 | } |
263 | } |
264 | } |
265 | |
266 | impl FromIterator<TokenTree> for TokenStream { |
267 | fn from_iter<I: IntoIterator<Item = TokenTree>>(tokens: I) -> Self { |
268 | let mut stream: TokenStream = TokenStream::new(); |
269 | stream.extend(iter:tokens); |
270 | stream |
271 | } |
272 | } |
273 | |
274 | impl FromIterator<TokenStream> for TokenStream { |
275 | fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self { |
276 | let mut v: RcVecBuilder = RcVecBuilder::new(); |
277 | |
278 | for stream: TokenStream in streams { |
279 | v.extend(iter:stream.take_inner()); |
280 | } |
281 | |
282 | TokenStream { inner: v.build() } |
283 | } |
284 | } |
285 | |
286 | impl Extend<TokenTree> for TokenStream { |
287 | fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, tokens: I) { |
288 | let mut vec: RcVecMut<'_, TokenTree> = self.inner.make_mut(); |
289 | tokens::IntoIter |
290 | .into_iter() |
291 | .for_each(|token: TokenTree| push_token_from_proc_macro(vec:vec.as_mut(), token)); |
292 | } |
293 | } |
294 | |
295 | impl Extend<TokenStream> for TokenStream { |
296 | fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) { |
297 | self.inner.make_mut().extend(iter:streams.into_iter().flatten()); |
298 | } |
299 | } |
300 | |
301 | pub(crate) type TokenTreeIter = RcVecIntoIter<TokenTree>; |
302 | |
303 | impl IntoIterator for TokenStream { |
304 | type Item = TokenTree; |
305 | type IntoIter = TokenTreeIter; |
306 | |
307 | fn into_iter(self) -> TokenTreeIter { |
308 | self.take_inner().into_iter() |
309 | } |
310 | } |
311 | |
312 | #[derive (Clone, PartialEq, Eq)] |
313 | pub(crate) struct SourceFile { |
314 | path: PathBuf, |
315 | } |
316 | |
317 | impl SourceFile { |
318 | /// Get the path to this source file as a string. |
319 | pub fn path(&self) -> PathBuf { |
320 | self.path.clone() |
321 | } |
322 | |
323 | pub fn is_real(&self) -> bool { |
324 | // XXX(nika): Support real files in the future? |
325 | false |
326 | } |
327 | } |
328 | |
329 | impl Debug for SourceFile { |
330 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
331 | f&mut DebugStruct<'_, '_>.debug_struct("SourceFile" ) |
332 | .field("path" , &self.path()) |
333 | .field(name:"is_real" , &self.is_real()) |
334 | .finish() |
335 | } |
336 | } |
337 | |
338 | #[cfg (all(span_locations, not(fuzzing)))] |
339 | thread_local! { |
340 | static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap { |
341 | // NOTE: We start with a single dummy file which all call_site() and |
342 | // def_site() spans reference. |
343 | files: vec![FileInfo { |
344 | source_text: String::new(), |
345 | span: Span { lo: 0, hi: 0 }, |
346 | lines: vec![0], |
347 | }], |
348 | }); |
349 | } |
350 | |
351 | #[cfg (all(span_locations, not(fuzzing)))] |
352 | struct FileInfo { |
353 | source_text: String, |
354 | span: Span, |
355 | lines: Vec<usize>, |
356 | } |
357 | |
358 | #[cfg (all(span_locations, not(fuzzing)))] |
359 | impl FileInfo { |
360 | fn offset_line_column(&self, offset: usize) -> LineColumn { |
361 | assert!(self.span_within(Span { |
362 | lo: offset as u32, |
363 | hi: offset as u32 |
364 | })); |
365 | let offset = offset - self.span.lo as usize; |
366 | match self.lines.binary_search(&offset) { |
367 | Ok(found) => LineColumn { |
368 | line: found + 1, |
369 | column: 0, |
370 | }, |
371 | Err(idx) => LineColumn { |
372 | line: idx, |
373 | column: offset - self.lines[idx - 1], |
374 | }, |
375 | } |
376 | } |
377 | |
378 | fn span_within(&self, span: Span) -> bool { |
379 | span.lo >= self.span.lo && span.hi <= self.span.hi |
380 | } |
381 | |
382 | fn source_text(&self, span: Span) -> String { |
383 | let lo = (span.lo - self.span.lo) as usize; |
384 | let hi = (span.hi - self.span.lo) as usize; |
385 | self.source_text[lo..hi].to_owned() |
386 | } |
387 | } |
388 | |
389 | /// Computes the offsets of each line in the given source string |
390 | /// and the total number of characters |
391 | #[cfg (all(span_locations, not(fuzzing)))] |
392 | fn lines_offsets(s: &str) -> (usize, Vec<usize>) { |
393 | let mut lines: Vec = vec![0]; |
394 | let mut total: usize = 0; |
395 | |
396 | for ch: char in s.chars() { |
397 | total += 1; |
398 | if ch == ' \n' { |
399 | lines.push(total); |
400 | } |
401 | } |
402 | |
403 | (total, lines) |
404 | } |
405 | |
406 | #[cfg (all(span_locations, not(fuzzing)))] |
407 | struct SourceMap { |
408 | files: Vec<FileInfo>, |
409 | } |
410 | |
411 | #[cfg (all(span_locations, not(fuzzing)))] |
412 | impl SourceMap { |
413 | fn next_start_pos(&self) -> u32 { |
414 | // Add 1 so there's always space between files. |
415 | // |
416 | // We'll always have at least 1 file, as we initialize our files list |
417 | // with a dummy file. |
418 | self.files.last().unwrap().span.hi + 1 |
419 | } |
420 | |
421 | fn add_file(&mut self, src: &str) -> Span { |
422 | let (len, lines) = lines_offsets(src); |
423 | let lo = self.next_start_pos(); |
424 | // XXX(nika): Should we bother doing a checked cast or checked add here? |
425 | let span = Span { |
426 | lo, |
427 | hi: lo + (len as u32), |
428 | }; |
429 | |
430 | self.files.push(FileInfo { |
431 | source_text: src.to_owned(), |
432 | span, |
433 | lines, |
434 | }); |
435 | |
436 | span |
437 | } |
438 | |
439 | #[cfg (procmacro2_semver_exempt)] |
440 | fn filepath(&self, span: Span) -> PathBuf { |
441 | for (i, file) in self.files.iter().enumerate() { |
442 | if file.span_within(span) { |
443 | return PathBuf::from(if i == 0 { |
444 | "<unspecified>" .to_owned() |
445 | } else { |
446 | format!("<parsed string {}>" , i) |
447 | }); |
448 | } |
449 | } |
450 | unreachable!("Invalid span with no related FileInfo!" ); |
451 | } |
452 | |
453 | fn fileinfo(&self, span: Span) -> &FileInfo { |
454 | for file in &self.files { |
455 | if file.span_within(span) { |
456 | return file; |
457 | } |
458 | } |
459 | unreachable!("Invalid span with no related FileInfo!" ); |
460 | } |
461 | } |
462 | |
463 | #[derive (Clone, Copy, PartialEq, Eq)] |
464 | pub(crate) struct Span { |
465 | #[cfg (span_locations)] |
466 | pub(crate) lo: u32, |
467 | #[cfg (span_locations)] |
468 | pub(crate) hi: u32, |
469 | } |
470 | |
471 | impl Span { |
472 | #[cfg (not(span_locations))] |
473 | pub fn call_site() -> Self { |
474 | Span {} |
475 | } |
476 | |
477 | #[cfg (span_locations)] |
478 | pub fn call_site() -> Self { |
479 | Span { lo: 0, hi: 0 } |
480 | } |
481 | |
482 | #[cfg (not(no_hygiene))] |
483 | pub fn mixed_site() -> Self { |
484 | Span::call_site() |
485 | } |
486 | |
487 | #[cfg (procmacro2_semver_exempt)] |
488 | pub fn def_site() -> Self { |
489 | Span::call_site() |
490 | } |
491 | |
492 | pub fn resolved_at(&self, _other: Span) -> Span { |
493 | // Stable spans consist only of line/column information, so |
494 | // `resolved_at` and `located_at` only select which span the |
495 | // caller wants line/column information from. |
496 | *self |
497 | } |
498 | |
499 | pub fn located_at(&self, other: Span) -> Span { |
500 | other |
501 | } |
502 | |
503 | #[cfg (procmacro2_semver_exempt)] |
504 | pub fn source_file(&self) -> SourceFile { |
505 | #[cfg (fuzzing)] |
506 | return SourceFile { |
507 | path: PathBuf::from("<unspecified>" ), |
508 | }; |
509 | |
510 | #[cfg (not(fuzzing))] |
511 | SOURCE_MAP.with(|cm| { |
512 | let cm = cm.borrow(); |
513 | let path = cm.filepath(*self); |
514 | SourceFile { path } |
515 | }) |
516 | } |
517 | |
518 | #[cfg (span_locations)] |
519 | pub fn start(&self) -> LineColumn { |
520 | #[cfg (fuzzing)] |
521 | return LineColumn { line: 0, column: 0 }; |
522 | |
523 | #[cfg (not(fuzzing))] |
524 | SOURCE_MAP.with(|cm| { |
525 | let cm = cm.borrow(); |
526 | let fi = cm.fileinfo(*self); |
527 | fi.offset_line_column(self.lo as usize) |
528 | }) |
529 | } |
530 | |
531 | #[cfg (span_locations)] |
532 | pub fn end(&self) -> LineColumn { |
533 | #[cfg (fuzzing)] |
534 | return LineColumn { line: 0, column: 0 }; |
535 | |
536 | #[cfg (not(fuzzing))] |
537 | SOURCE_MAP.with(|cm| { |
538 | let cm = cm.borrow(); |
539 | let fi = cm.fileinfo(*self); |
540 | fi.offset_line_column(self.hi as usize) |
541 | }) |
542 | } |
543 | |
544 | #[cfg (not(span_locations))] |
545 | pub fn join(&self, _other: Span) -> Option<Span> { |
546 | Some(Span {}) |
547 | } |
548 | |
549 | #[cfg (span_locations)] |
550 | pub fn join(&self, other: Span) -> Option<Span> { |
551 | #[cfg (fuzzing)] |
552 | return { |
553 | let _ = other; |
554 | None |
555 | }; |
556 | |
557 | #[cfg (not(fuzzing))] |
558 | SOURCE_MAP.with(|cm| { |
559 | let cm = cm.borrow(); |
560 | // If `other` is not within the same FileInfo as us, return None. |
561 | if !cm.fileinfo(*self).span_within(other) { |
562 | return None; |
563 | } |
564 | Some(Span { |
565 | lo: cmp::min(self.lo, other.lo), |
566 | hi: cmp::max(self.hi, other.hi), |
567 | }) |
568 | }) |
569 | } |
570 | |
571 | #[cfg (not(span_locations))] |
572 | pub fn source_text(&self) -> Option<String> { |
573 | None |
574 | } |
575 | |
576 | #[cfg (span_locations)] |
577 | pub fn source_text(&self) -> Option<String> { |
578 | #[cfg (fuzzing)] |
579 | return None; |
580 | |
581 | #[cfg (not(fuzzing))] |
582 | { |
583 | if self.is_call_site() { |
584 | None |
585 | } else { |
586 | Some(SOURCE_MAP.with(|cm| cm.borrow().fileinfo(*self).source_text(*self))) |
587 | } |
588 | } |
589 | } |
590 | |
591 | #[cfg (not(span_locations))] |
592 | pub(crate) fn first_byte(self) -> Self { |
593 | self |
594 | } |
595 | |
596 | #[cfg (span_locations)] |
597 | pub(crate) fn first_byte(self) -> Self { |
598 | Span { |
599 | lo: self.lo, |
600 | hi: cmp::min(self.lo.saturating_add(1), self.hi), |
601 | } |
602 | } |
603 | |
604 | #[cfg (not(span_locations))] |
605 | pub(crate) fn last_byte(self) -> Self { |
606 | self |
607 | } |
608 | |
609 | #[cfg (span_locations)] |
610 | pub(crate) fn last_byte(self) -> Self { |
611 | Span { |
612 | lo: cmp::max(self.hi.saturating_sub(1), self.lo), |
613 | hi: self.hi, |
614 | } |
615 | } |
616 | |
617 | #[cfg (span_locations)] |
618 | fn is_call_site(&self) -> bool { |
619 | self.lo == 0 && self.hi == 0 |
620 | } |
621 | } |
622 | |
623 | impl Debug for Span { |
624 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
625 | #[cfg (span_locations)] |
626 | return write!(f, "bytes( {}.. {})" , self.lo, self.hi); |
627 | |
628 | #[cfg (not(span_locations))] |
629 | write!(f, "Span" ) |
630 | } |
631 | } |
632 | |
633 | pub(crate) fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) { |
634 | #[cfg (span_locations)] |
635 | { |
636 | if span.is_call_site() { |
637 | return; |
638 | } |
639 | } |
640 | |
641 | if cfg!(span_locations) { |
642 | debug.field(name:"span" , &span); |
643 | } |
644 | } |
645 | |
646 | #[derive (Clone)] |
647 | pub(crate) struct Group { |
648 | delimiter: Delimiter, |
649 | stream: TokenStream, |
650 | span: Span, |
651 | } |
652 | |
653 | impl Group { |
654 | pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self { |
655 | Group { |
656 | delimiter, |
657 | stream, |
658 | span: Span::call_site(), |
659 | } |
660 | } |
661 | |
662 | pub fn delimiter(&self) -> Delimiter { |
663 | self.delimiter |
664 | } |
665 | |
666 | pub fn stream(&self) -> TokenStream { |
667 | self.stream.clone() |
668 | } |
669 | |
670 | pub fn span(&self) -> Span { |
671 | self.span |
672 | } |
673 | |
674 | pub fn span_open(&self) -> Span { |
675 | self.span.first_byte() |
676 | } |
677 | |
678 | pub fn span_close(&self) -> Span { |
679 | self.span.last_byte() |
680 | } |
681 | |
682 | pub fn set_span(&mut self, span: Span) { |
683 | self.span = span; |
684 | } |
685 | } |
686 | |
687 | impl Display for Group { |
688 | // We attempt to match libproc_macro's formatting. |
689 | // Empty parens: () |
690 | // Nonempty parens: (...) |
691 | // Empty brackets: [] |
692 | // Nonempty brackets: [...] |
693 | // Empty braces: { } |
694 | // Nonempty braces: { ... } |
695 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
696 | let (open, close) = match self.delimiter { |
697 | Delimiter::Parenthesis => ("(" , ")" ), |
698 | Delimiter::Brace => ("{ " , "}" ), |
699 | Delimiter::Bracket => ("[" , "]" ), |
700 | Delimiter::None => ("" , "" ), |
701 | }; |
702 | |
703 | f.write_str(open)?; |
704 | Display::fmt(&self.stream, f)?; |
705 | if self.delimiter == Delimiter::Brace && !self.stream.inner.is_empty() { |
706 | f.write_str(" " )?; |
707 | } |
708 | f.write_str(close)?; |
709 | |
710 | Ok(()) |
711 | } |
712 | } |
713 | |
714 | impl Debug for Group { |
715 | fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { |
716 | let mut debug: DebugStruct<'_, '_> = fmt.debug_struct(name:"Group" ); |
717 | debug.field(name:"delimiter" , &self.delimiter); |
718 | debug.field(name:"stream" , &self.stream); |
719 | debug_span_field_if_nontrivial(&mut debug, self.span); |
720 | debug.finish() |
721 | } |
722 | } |
723 | |
724 | #[derive (Clone)] |
725 | pub(crate) struct Ident { |
726 | sym: String, |
727 | span: Span, |
728 | raw: bool, |
729 | } |
730 | |
731 | impl Ident { |
732 | fn _new(string: &str, raw: bool, span: Span) -> Self { |
733 | validate_ident(string, raw); |
734 | |
735 | Ident { |
736 | sym: string.to_owned(), |
737 | span, |
738 | raw, |
739 | } |
740 | } |
741 | |
742 | pub fn new(string: &str, span: Span) -> Self { |
743 | Ident::_new(string, false, span) |
744 | } |
745 | |
746 | pub fn new_raw(string: &str, span: Span) -> Self { |
747 | Ident::_new(string, true, span) |
748 | } |
749 | |
750 | pub fn span(&self) -> Span { |
751 | self.span |
752 | } |
753 | |
754 | pub fn set_span(&mut self, span: Span) { |
755 | self.span = span; |
756 | } |
757 | } |
758 | |
759 | pub(crate) fn is_ident_start(c: char) -> bool { |
760 | c == '_' || unicode_ident::is_xid_start(ch:c) |
761 | } |
762 | |
763 | pub(crate) fn is_ident_continue(c: char) -> bool { |
764 | unicode_ident::is_xid_continue(ch:c) |
765 | } |
766 | |
767 | fn validate_ident(string: &str, raw: bool) { |
768 | if string.is_empty() { |
769 | panic!("Ident is not allowed to be empty; use Option<Ident>" ); |
770 | } |
771 | |
772 | if string.bytes().all(|digit| digit >= b'0' && digit <= b'9' ) { |
773 | panic!("Ident cannot be a number; use Literal instead" ); |
774 | } |
775 | |
776 | fn ident_ok(string: &str) -> bool { |
777 | let mut chars = string.chars(); |
778 | let first = chars.next().unwrap(); |
779 | if !is_ident_start(first) { |
780 | return false; |
781 | } |
782 | for ch in chars { |
783 | if !is_ident_continue(ch) { |
784 | return false; |
785 | } |
786 | } |
787 | true |
788 | } |
789 | |
790 | if !ident_ok(string) { |
791 | panic!(" {:?} is not a valid Ident" , string); |
792 | } |
793 | |
794 | if raw { |
795 | match string { |
796 | "_" | "super" | "self" | "Self" | "crate" => { |
797 | panic!("`r# {}` cannot be a raw identifier" , string); |
798 | } |
799 | _ => {} |
800 | } |
801 | } |
802 | } |
803 | |
804 | impl PartialEq for Ident { |
805 | fn eq(&self, other: &Ident) -> bool { |
806 | self.sym == other.sym && self.raw == other.raw |
807 | } |
808 | } |
809 | |
810 | impl<T> PartialEq<T> for Ident |
811 | where |
812 | T: ?Sized + AsRef<str>, |
813 | { |
814 | fn eq(&self, other: &T) -> bool { |
815 | let other: &str = other.as_ref(); |
816 | if self.raw { |
817 | other.starts_with("r#" ) && self.sym == other[2..] |
818 | } else { |
819 | self.sym == other |
820 | } |
821 | } |
822 | } |
823 | |
824 | impl Display for Ident { |
825 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
826 | if self.raw { |
827 | f.write_str(data:"r#" )?; |
828 | } |
829 | Display::fmt(&self.sym, f) |
830 | } |
831 | } |
832 | |
833 | #[allow (clippy::missing_fields_in_debug)] |
834 | impl Debug for Ident { |
835 | // Ident(proc_macro), Ident(r#union) |
836 | #[cfg (not(span_locations))] |
837 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
838 | let mut debug = f.debug_tuple("Ident" ); |
839 | debug.field(&format_args!(" {}" , self)); |
840 | debug.finish() |
841 | } |
842 | |
843 | // Ident { |
844 | // sym: proc_macro, |
845 | // span: bytes(128..138) |
846 | // } |
847 | #[cfg (span_locations)] |
848 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
849 | let mut debug: DebugStruct<'_, '_> = f.debug_struct(name:"Ident" ); |
850 | debug.field(name:"sym" , &format_args!(" {}" , self)); |
851 | debug_span_field_if_nontrivial(&mut debug, self.span); |
852 | debug.finish() |
853 | } |
854 | } |
855 | |
856 | #[derive (Clone)] |
857 | pub(crate) struct Literal { |
858 | repr: String, |
859 | span: Span, |
860 | } |
861 | |
862 | macro_rules! suffixed_numbers { |
863 | ($($name:ident => $kind:ident,)*) => ($( |
864 | pub fn $name(n: $kind) -> Literal { |
865 | Literal::_new(format!(concat!("{}" , stringify!($kind)), n)) |
866 | } |
867 | )*) |
868 | } |
869 | |
870 | macro_rules! unsuffixed_numbers { |
871 | ($($name:ident => $kind:ident,)*) => ($( |
872 | pub fn $name(n: $kind) -> Literal { |
873 | Literal::_new(n.to_string()) |
874 | } |
875 | )*) |
876 | } |
877 | |
878 | impl Literal { |
879 | pub(crate) fn _new(repr: String) -> Self { |
880 | Literal { |
881 | repr, |
882 | span: Span::call_site(), |
883 | } |
884 | } |
885 | |
886 | pub(crate) unsafe fn from_str_unchecked(repr: &str) -> Self { |
887 | Literal::_new(repr.to_owned()) |
888 | } |
889 | |
890 | suffixed_numbers! { |
891 | u8_suffixed => u8, |
892 | u16_suffixed => u16, |
893 | u32_suffixed => u32, |
894 | u64_suffixed => u64, |
895 | u128_suffixed => u128, |
896 | usize_suffixed => usize, |
897 | i8_suffixed => i8, |
898 | i16_suffixed => i16, |
899 | i32_suffixed => i32, |
900 | i64_suffixed => i64, |
901 | i128_suffixed => i128, |
902 | isize_suffixed => isize, |
903 | |
904 | f32_suffixed => f32, |
905 | f64_suffixed => f64, |
906 | } |
907 | |
908 | unsuffixed_numbers! { |
909 | u8_unsuffixed => u8, |
910 | u16_unsuffixed => u16, |
911 | u32_unsuffixed => u32, |
912 | u64_unsuffixed => u64, |
913 | u128_unsuffixed => u128, |
914 | usize_unsuffixed => usize, |
915 | i8_unsuffixed => i8, |
916 | i16_unsuffixed => i16, |
917 | i32_unsuffixed => i32, |
918 | i64_unsuffixed => i64, |
919 | i128_unsuffixed => i128, |
920 | isize_unsuffixed => isize, |
921 | } |
922 | |
923 | pub fn f32_unsuffixed(f: f32) -> Literal { |
924 | let mut s = f.to_string(); |
925 | if !s.contains('.' ) { |
926 | s.push_str(".0" ); |
927 | } |
928 | Literal::_new(s) |
929 | } |
930 | |
931 | pub fn f64_unsuffixed(f: f64) -> Literal { |
932 | let mut s = f.to_string(); |
933 | if !s.contains('.' ) { |
934 | s.push_str(".0" ); |
935 | } |
936 | Literal::_new(s) |
937 | } |
938 | |
939 | pub fn string(t: &str) -> Literal { |
940 | let mut repr = String::with_capacity(t.len() + 2); |
941 | repr.push('"' ); |
942 | let mut chars = t.chars(); |
943 | while let Some(ch) = chars.next() { |
944 | if ch == ' \0' { |
945 | repr.push_str( |
946 | if chars |
947 | .as_str() |
948 | .starts_with(|next| '0' <= next && next <= '7' ) |
949 | { |
950 | // circumvent clippy::octal_escapes lint |
951 | " \\x00" |
952 | } else { |
953 | " \\0" |
954 | }, |
955 | ); |
956 | } else if ch == ' \'' { |
957 | // escape_debug turns this into "\'" which is unnecessary. |
958 | repr.push(ch); |
959 | } else { |
960 | repr.extend(ch.escape_debug()); |
961 | } |
962 | } |
963 | repr.push('"' ); |
964 | Literal::_new(repr) |
965 | } |
966 | |
967 | pub fn character(t: char) -> Literal { |
968 | let mut repr = String::new(); |
969 | repr.push(' \'' ); |
970 | if t == '"' { |
971 | // escape_debug turns this into '\"' which is unnecessary. |
972 | repr.push(t); |
973 | } else { |
974 | repr.extend(t.escape_debug()); |
975 | } |
976 | repr.push(' \'' ); |
977 | Literal::_new(repr) |
978 | } |
979 | |
980 | pub fn byte_string(bytes: &[u8]) -> Literal { |
981 | let mut escaped = "b \"" .to_string(); |
982 | let mut bytes = bytes.iter(); |
983 | while let Some(&b) = bytes.next() { |
984 | #[allow (clippy::match_overlapping_arm)] |
985 | match b { |
986 | b' \0' => escaped.push_str(match bytes.as_slice().first() { |
987 | // circumvent clippy::octal_escapes lint |
988 | Some(b'0' ..=b'7' ) => r"\x00" , |
989 | _ => r"\0" , |
990 | }), |
991 | b' \t' => escaped.push_str(r"\t" ), |
992 | b' \n' => escaped.push_str(r"\n" ), |
993 | b' \r' => escaped.push_str(r"\r" ), |
994 | b'"' => escaped.push_str(" \\\"" ), |
995 | b' \\' => escaped.push_str(" \\\\" ), |
996 | b' \x20' ..=b' \x7E' => escaped.push(b as char), |
997 | _ => { |
998 | let _ = write!(escaped, " \\x {:02X}" , b); |
999 | } |
1000 | } |
1001 | } |
1002 | escaped.push('"' ); |
1003 | Literal::_new(escaped) |
1004 | } |
1005 | |
1006 | pub fn span(&self) -> Span { |
1007 | self.span |
1008 | } |
1009 | |
1010 | pub fn set_span(&mut self, span: Span) { |
1011 | self.span = span; |
1012 | } |
1013 | |
1014 | pub fn subspan<R: RangeBounds<usize>>(&self, range: R) -> Option<Span> { |
1015 | #[cfg (not(span_locations))] |
1016 | { |
1017 | let _ = range; |
1018 | None |
1019 | } |
1020 | |
1021 | #[cfg (span_locations)] |
1022 | { |
1023 | use crate::convert::usize_to_u32; |
1024 | use core::ops::Bound; |
1025 | |
1026 | let lo = match range.start_bound() { |
1027 | Bound::Included(start) => { |
1028 | let start = usize_to_u32(*start)?; |
1029 | self.span.lo.checked_add(start)? |
1030 | } |
1031 | Bound::Excluded(start) => { |
1032 | let start = usize_to_u32(*start)?; |
1033 | self.span.lo.checked_add(start)?.checked_add(1)? |
1034 | } |
1035 | Bound::Unbounded => self.span.lo, |
1036 | }; |
1037 | let hi = match range.end_bound() { |
1038 | Bound::Included(end) => { |
1039 | let end = usize_to_u32(*end)?; |
1040 | self.span.lo.checked_add(end)?.checked_add(1)? |
1041 | } |
1042 | Bound::Excluded(end) => { |
1043 | let end = usize_to_u32(*end)?; |
1044 | self.span.lo.checked_add(end)? |
1045 | } |
1046 | Bound::Unbounded => self.span.hi, |
1047 | }; |
1048 | if lo <= hi && hi <= self.span.hi { |
1049 | Some(Span { lo, hi }) |
1050 | } else { |
1051 | None |
1052 | } |
1053 | } |
1054 | } |
1055 | } |
1056 | |
1057 | impl FromStr for Literal { |
1058 | type Err = LexError; |
1059 | |
1060 | fn from_str(repr: &str) -> Result<Self, Self::Err> { |
1061 | let mut cursor = get_cursor(repr); |
1062 | #[cfg (span_locations)] |
1063 | let lo = cursor.off; |
1064 | |
1065 | let negative = cursor.starts_with_char('-' ); |
1066 | if negative { |
1067 | cursor = cursor.advance(1); |
1068 | if !cursor.starts_with_fn(|ch| ch.is_ascii_digit()) { |
1069 | return Err(LexError::call_site()); |
1070 | } |
1071 | } |
1072 | |
1073 | if let Ok((rest, mut literal)) = parse::literal(cursor) { |
1074 | if rest.is_empty() { |
1075 | if negative { |
1076 | literal.repr.insert(0, '-' ); |
1077 | } |
1078 | literal.span = Span { |
1079 | #[cfg (span_locations)] |
1080 | lo, |
1081 | #[cfg (span_locations)] |
1082 | hi: rest.off, |
1083 | }; |
1084 | return Ok(literal); |
1085 | } |
1086 | } |
1087 | Err(LexError::call_site()) |
1088 | } |
1089 | } |
1090 | |
1091 | impl Display for Literal { |
1092 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { |
1093 | Display::fmt(&self.repr, f) |
1094 | } |
1095 | } |
1096 | |
1097 | impl Debug for Literal { |
1098 | fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { |
1099 | let mut debug: DebugStruct<'_, '_> = fmt.debug_struct(name:"Literal" ); |
1100 | debug.field(name:"lit" , &format_args!(" {}" , self.repr)); |
1101 | debug_span_field_if_nontrivial(&mut debug, self.span); |
1102 | debug.finish() |
1103 | } |
1104 | } |
1105 | |