1 #[cfg(span_locations)]
2 use std::cell::RefCell;
3 #[cfg(span_locations)]
4 use std::cmp;
5 use std::fmt;
6 use std::iter;
7 use std::ops::RangeBounds;
8 #[cfg(procmacro2_semver_exempt)]
9 use std::path::Path;
10 use std::path::PathBuf;
11 use std::str::FromStr;
12 use std::vec;
13
14 use crate::strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
15 use crate::{Delimiter, Punct, Spacing, TokenTree};
16 use unicode_xid::UnicodeXID;
17
18 #[derive(Clone)]
19 pub struct TokenStream {
20 inner: Vec<TokenTree>,
21 }
22
23 #[derive(Debug)]
24 pub struct LexError;
25
26 impl TokenStream {
new() -> TokenStream27 pub fn new() -> TokenStream {
28 TokenStream { inner: Vec::new() }
29 }
30
is_empty(&self) -> bool31 pub fn is_empty(&self) -> bool {
32 self.inner.len() == 0
33 }
34 }
35
36 #[cfg(span_locations)]
get_cursor(src: &str) -> Cursor37 fn get_cursor(src: &str) -> Cursor {
38 // Create a dummy file & add it to the source map
39 SOURCE_MAP.with(|cm| {
40 let mut cm = cm.borrow_mut();
41 let name = format!("<parsed string {}>", cm.files.len());
42 let span = cm.add_file(&name, src);
43 Cursor {
44 rest: src,
45 off: span.lo,
46 }
47 })
48 }
49
50 #[cfg(not(span_locations))]
get_cursor(src: &str) -> Cursor51 fn get_cursor(src: &str) -> Cursor {
52 Cursor { rest: src }
53 }
54
55 impl FromStr for TokenStream {
56 type Err = LexError;
57
from_str(src: &str) -> Result<TokenStream, LexError>58 fn from_str(src: &str) -> Result<TokenStream, LexError> {
59 // Create a dummy file & add it to the source map
60 let cursor = get_cursor(src);
61
62 match token_stream(cursor) {
63 Ok((input, output)) => {
64 if skip_whitespace(input).len() != 0 {
65 Err(LexError)
66 } else {
67 Ok(output)
68 }
69 }
70 Err(LexError) => Err(LexError),
71 }
72 }
73 }
74
75 impl fmt::Display for TokenStream {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result76 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
77 let mut joint = false;
78 for (i, tt) in self.inner.iter().enumerate() {
79 if i != 0 && !joint {
80 write!(f, " ")?;
81 }
82 joint = false;
83 match *tt {
84 TokenTree::Group(ref tt) => {
85 let (start, end) = match tt.delimiter() {
86 Delimiter::Parenthesis => ("(", ")"),
87 Delimiter::Brace => ("{", "}"),
88 Delimiter::Bracket => ("[", "]"),
89 Delimiter::None => ("", ""),
90 };
91 if tt.stream().into_iter().next().is_none() {
92 write!(f, "{} {}", start, end)?
93 } else {
94 write!(f, "{} {} {}", start, tt.stream(), end)?
95 }
96 }
97 TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
98 TokenTree::Punct(ref tt) => {
99 write!(f, "{}", tt.as_char())?;
100 match tt.spacing() {
101 Spacing::Alone => {}
102 Spacing::Joint => joint = true,
103 }
104 }
105 TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
106 }
107 }
108
109 Ok(())
110 }
111 }
112
113 impl fmt::Debug for TokenStream {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result114 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
115 f.write_str("TokenStream ")?;
116 f.debug_list().entries(self.clone()).finish()
117 }
118 }
119
120 #[cfg(use_proc_macro)]
121 impl From<proc_macro::TokenStream> for TokenStream {
from(inner: proc_macro::TokenStream) -> TokenStream122 fn from(inner: proc_macro::TokenStream) -> TokenStream {
123 inner
124 .to_string()
125 .parse()
126 .expect("compiler token stream parse failed")
127 }
128 }
129
130 #[cfg(use_proc_macro)]
131 impl From<TokenStream> for proc_macro::TokenStream {
from(inner: TokenStream) -> proc_macro::TokenStream132 fn from(inner: TokenStream) -> proc_macro::TokenStream {
133 inner
134 .to_string()
135 .parse()
136 .expect("failed to parse to compiler tokens")
137 }
138 }
139
140 impl From<TokenTree> for TokenStream {
from(tree: TokenTree) -> TokenStream141 fn from(tree: TokenTree) -> TokenStream {
142 TokenStream { inner: vec![tree] }
143 }
144 }
145
146 impl iter::FromIterator<TokenTree> for TokenStream {
from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self147 fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
148 let mut v = Vec::new();
149
150 for token in streams.into_iter() {
151 v.push(token);
152 }
153
154 TokenStream { inner: v }
155 }
156 }
157
158 impl iter::FromIterator<TokenStream> for TokenStream {
from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self159 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
160 let mut v = Vec::new();
161
162 for stream in streams.into_iter() {
163 v.extend(stream.inner);
164 }
165
166 TokenStream { inner: v }
167 }
168 }
169
170 impl Extend<TokenTree> for TokenStream {
extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I)171 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
172 self.inner.extend(streams);
173 }
174 }
175
176 impl Extend<TokenStream> for TokenStream {
extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I)177 fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
178 self.inner
179 .extend(streams.into_iter().flat_map(|stream| stream));
180 }
181 }
182
183 pub type TokenTreeIter = vec::IntoIter<TokenTree>;
184
185 impl IntoIterator for TokenStream {
186 type Item = TokenTree;
187 type IntoIter = TokenTreeIter;
188
into_iter(self) -> TokenTreeIter189 fn into_iter(self) -> TokenTreeIter {
190 self.inner.into_iter()
191 }
192 }
193
194 #[derive(Clone, PartialEq, Eq)]
195 pub struct SourceFile {
196 path: PathBuf,
197 }
198
199 impl SourceFile {
200 /// Get the path to this source file as a string.
path(&self) -> PathBuf201 pub fn path(&self) -> PathBuf {
202 self.path.clone()
203 }
204
is_real(&self) -> bool205 pub fn is_real(&self) -> bool {
206 // XXX(nika): Support real files in the future?
207 false
208 }
209 }
210
211 impl fmt::Debug for SourceFile {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result212 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
213 f.debug_struct("SourceFile")
214 .field("path", &self.path())
215 .field("is_real", &self.is_real())
216 .finish()
217 }
218 }
219
220 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
221 pub struct LineColumn {
222 pub line: usize,
223 pub column: usize,
224 }
225
226 #[cfg(span_locations)]
227 thread_local! {
228 static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
229 // NOTE: We start with a single dummy file which all call_site() and
230 // def_site() spans reference.
231 files: vec![{
232 #[cfg(procmacro2_semver_exempt)]
233 {
234 FileInfo {
235 name: "<unspecified>".to_owned(),
236 span: Span { lo: 0, hi: 0 },
237 lines: vec![0],
238 }
239 }
240
241 #[cfg(not(procmacro2_semver_exempt))]
242 {
243 FileInfo {
244 span: Span { lo: 0, hi: 0 },
245 lines: vec![0],
246 }
247 }
248 }],
249 });
250 }
251
252 #[cfg(span_locations)]
253 struct FileInfo {
254 #[cfg(procmacro2_semver_exempt)]
255 name: String,
256 span: Span,
257 lines: Vec<usize>,
258 }
259
260 #[cfg(span_locations)]
261 impl FileInfo {
offset_line_column(&self, offset: usize) -> LineColumn262 fn offset_line_column(&self, offset: usize) -> LineColumn {
263 assert!(self.span_within(Span {
264 lo: offset as u32,
265 hi: offset as u32
266 }));
267 let offset = offset - self.span.lo as usize;
268 match self.lines.binary_search(&offset) {
269 Ok(found) => LineColumn {
270 line: found + 1,
271 column: 0,
272 },
273 Err(idx) => LineColumn {
274 line: idx,
275 column: offset - self.lines[idx - 1],
276 },
277 }
278 }
279
span_within(&self, span: Span) -> bool280 fn span_within(&self, span: Span) -> bool {
281 span.lo >= self.span.lo && span.hi <= self.span.hi
282 }
283 }
284
285 /// Computesthe offsets of each line in the given source string.
286 #[cfg(span_locations)]
lines_offsets(s: &str) -> Vec<usize>287 fn lines_offsets(s: &str) -> Vec<usize> {
288 let mut lines = vec![0];
289 let mut prev = 0;
290 while let Some(len) = s[prev..].find('\n') {
291 prev += len + 1;
292 lines.push(prev);
293 }
294 lines
295 }
296
297 #[cfg(span_locations)]
298 struct SourceMap {
299 files: Vec<FileInfo>,
300 }
301
302 #[cfg(span_locations)]
303 impl SourceMap {
next_start_pos(&self) -> u32304 fn next_start_pos(&self) -> u32 {
305 // Add 1 so there's always space between files.
306 //
307 // We'll always have at least 1 file, as we initialize our files list
308 // with a dummy file.
309 self.files.last().unwrap().span.hi + 1
310 }
311
add_file(&mut self, name: &str, src: &str) -> Span312 fn add_file(&mut self, name: &str, src: &str) -> Span {
313 let lines = lines_offsets(src);
314 let lo = self.next_start_pos();
315 // XXX(nika): Shouild we bother doing a checked cast or checked add here?
316 let span = Span {
317 lo,
318 hi: lo + (src.len() as u32),
319 };
320
321 #[cfg(procmacro2_semver_exempt)]
322 self.files.push(FileInfo {
323 name: name.to_owned(),
324 span,
325 lines,
326 });
327
328 #[cfg(not(procmacro2_semver_exempt))]
329 self.files.push(FileInfo { span, lines });
330 let _ = name;
331
332 span
333 }
334
fileinfo(&self, span: Span) -> &FileInfo335 fn fileinfo(&self, span: Span) -> &FileInfo {
336 for file in &self.files {
337 if file.span_within(span) {
338 return file;
339 }
340 }
341 panic!("Invalid span with no related FileInfo!");
342 }
343 }
344
345 #[derive(Clone, Copy, PartialEq, Eq)]
346 pub struct Span {
347 #[cfg(span_locations)]
348 lo: u32,
349 #[cfg(span_locations)]
350 hi: u32,
351 }
352
353 impl Span {
354 #[cfg(not(span_locations))]
call_site() -> Span355 pub fn call_site() -> Span {
356 Span {}
357 }
358
359 #[cfg(span_locations)]
call_site() -> Span360 pub fn call_site() -> Span {
361 Span { lo: 0, hi: 0 }
362 }
363
364 #[cfg(procmacro2_semver_exempt)]
def_site() -> Span365 pub fn def_site() -> Span {
366 Span::call_site()
367 }
368
369 #[cfg(procmacro2_semver_exempt)]
resolved_at(&self, _other: Span) -> Span370 pub fn resolved_at(&self, _other: Span) -> Span {
371 // Stable spans consist only of line/column information, so
372 // `resolved_at` and `located_at` only select which span the
373 // caller wants line/column information from.
374 *self
375 }
376
377 #[cfg(procmacro2_semver_exempt)]
located_at(&self, other: Span) -> Span378 pub fn located_at(&self, other: Span) -> Span {
379 other
380 }
381
382 #[cfg(procmacro2_semver_exempt)]
source_file(&self) -> SourceFile383 pub fn source_file(&self) -> SourceFile {
384 SOURCE_MAP.with(|cm| {
385 let cm = cm.borrow();
386 let fi = cm.fileinfo(*self);
387 SourceFile {
388 path: Path::new(&fi.name).to_owned(),
389 }
390 })
391 }
392
393 #[cfg(span_locations)]
start(&self) -> LineColumn394 pub fn start(&self) -> LineColumn {
395 SOURCE_MAP.with(|cm| {
396 let cm = cm.borrow();
397 let fi = cm.fileinfo(*self);
398 fi.offset_line_column(self.lo as usize)
399 })
400 }
401
402 #[cfg(span_locations)]
end(&self) -> LineColumn403 pub fn end(&self) -> LineColumn {
404 SOURCE_MAP.with(|cm| {
405 let cm = cm.borrow();
406 let fi = cm.fileinfo(*self);
407 fi.offset_line_column(self.hi as usize)
408 })
409 }
410
411 #[cfg(not(span_locations))]
join(&self, _other: Span) -> Option<Span>412 pub fn join(&self, _other: Span) -> Option<Span> {
413 Some(Span {})
414 }
415
416 #[cfg(span_locations)]
join(&self, other: Span) -> Option<Span>417 pub fn join(&self, other: Span) -> Option<Span> {
418 SOURCE_MAP.with(|cm| {
419 let cm = cm.borrow();
420 // If `other` is not within the same FileInfo as us, return None.
421 if !cm.fileinfo(*self).span_within(other) {
422 return None;
423 }
424 Some(Span {
425 lo: cmp::min(self.lo, other.lo),
426 hi: cmp::max(self.hi, other.hi),
427 })
428 })
429 }
430
431 #[cfg(not(span_locations))]
first_byte(self) -> Self432 fn first_byte(self) -> Self {
433 self
434 }
435
436 #[cfg(span_locations)]
first_byte(self) -> Self437 fn first_byte(self) -> Self {
438 Span {
439 lo: self.lo,
440 hi: cmp::min(self.lo.saturating_add(1), self.hi),
441 }
442 }
443
444 #[cfg(not(span_locations))]
last_byte(self) -> Self445 fn last_byte(self) -> Self {
446 self
447 }
448
449 #[cfg(span_locations)]
last_byte(self) -> Self450 fn last_byte(self) -> Self {
451 Span {
452 lo: cmp::max(self.hi.saturating_sub(1), self.lo),
453 hi: self.hi,
454 }
455 }
456 }
457
458 impl fmt::Debug for Span {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result459 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
460 #[cfg(procmacro2_semver_exempt)]
461 return write!(f, "bytes({}..{})", self.lo, self.hi);
462
463 #[cfg(not(procmacro2_semver_exempt))]
464 write!(f, "Span")
465 }
466 }
467
debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span)468 pub fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
469 if cfg!(procmacro2_semver_exempt) {
470 debug.field("span", &span);
471 }
472 }
473
474 #[derive(Clone)]
475 pub struct Group {
476 delimiter: Delimiter,
477 stream: TokenStream,
478 span: Span,
479 }
480
481 impl Group {
new(delimiter: Delimiter, stream: TokenStream) -> Group482 pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
483 Group {
484 delimiter,
485 stream,
486 span: Span::call_site(),
487 }
488 }
489
delimiter(&self) -> Delimiter490 pub fn delimiter(&self) -> Delimiter {
491 self.delimiter
492 }
493
stream(&self) -> TokenStream494 pub fn stream(&self) -> TokenStream {
495 self.stream.clone()
496 }
497
span(&self) -> Span498 pub fn span(&self) -> Span {
499 self.span
500 }
501
span_open(&self) -> Span502 pub fn span_open(&self) -> Span {
503 self.span.first_byte()
504 }
505
span_close(&self) -> Span506 pub fn span_close(&self) -> Span {
507 self.span.last_byte()
508 }
509
set_span(&mut self, span: Span)510 pub fn set_span(&mut self, span: Span) {
511 self.span = span;
512 }
513 }
514
515 impl fmt::Display for Group {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result516 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
517 let (left, right) = match self.delimiter {
518 Delimiter::Parenthesis => ("(", ")"),
519 Delimiter::Brace => ("{", "}"),
520 Delimiter::Bracket => ("[", "]"),
521 Delimiter::None => ("", ""),
522 };
523
524 f.write_str(left)?;
525 self.stream.fmt(f)?;
526 f.write_str(right)?;
527
528 Ok(())
529 }
530 }
531
532 impl fmt::Debug for Group {
fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result533 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
534 let mut debug = fmt.debug_struct("Group");
535 debug.field("delimiter", &self.delimiter);
536 debug.field("stream", &self.stream);
537 #[cfg(procmacro2_semver_exempt)]
538 debug.field("span", &self.span);
539 debug.finish()
540 }
541 }
542
543 #[derive(Clone)]
544 pub struct Ident {
545 sym: String,
546 span: Span,
547 raw: bool,
548 }
549
550 impl Ident {
_new(string: &str, raw: bool, span: Span) -> Ident551 fn _new(string: &str, raw: bool, span: Span) -> Ident {
552 validate_ident(string);
553
554 Ident {
555 sym: string.to_owned(),
556 span,
557 raw,
558 }
559 }
560
new(string: &str, span: Span) -> Ident561 pub fn new(string: &str, span: Span) -> Ident {
562 Ident::_new(string, false, span)
563 }
564
new_raw(string: &str, span: Span) -> Ident565 pub fn new_raw(string: &str, span: Span) -> Ident {
566 Ident::_new(string, true, span)
567 }
568
span(&self) -> Span569 pub fn span(&self) -> Span {
570 self.span
571 }
572
set_span(&mut self, span: Span)573 pub fn set_span(&mut self, span: Span) {
574 self.span = span;
575 }
576 }
577
is_ident_start(c: char) -> bool578 fn is_ident_start(c: char) -> bool {
579 ('a' <= c && c <= 'z')
580 || ('A' <= c && c <= 'Z')
581 || c == '_'
582 || (c > '\x7f' && UnicodeXID::is_xid_start(c))
583 }
584
is_ident_continue(c: char) -> bool585 fn is_ident_continue(c: char) -> bool {
586 ('a' <= c && c <= 'z')
587 || ('A' <= c && c <= 'Z')
588 || c == '_'
589 || ('0' <= c && c <= '9')
590 || (c > '\x7f' && UnicodeXID::is_xid_continue(c))
591 }
592
validate_ident(string: &str)593 fn validate_ident(string: &str) {
594 let validate = string;
595 if validate.is_empty() {
596 panic!("Ident is not allowed to be empty; use Option<Ident>");
597 }
598
599 if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
600 panic!("Ident cannot be a number; use Literal instead");
601 }
602
603 fn ident_ok(string: &str) -> bool {
604 let mut chars = string.chars();
605 let first = chars.next().unwrap();
606 if !is_ident_start(first) {
607 return false;
608 }
609 for ch in chars {
610 if !is_ident_continue(ch) {
611 return false;
612 }
613 }
614 true
615 }
616
617 if !ident_ok(validate) {
618 panic!("{:?} is not a valid Ident", string);
619 }
620 }
621
622 impl PartialEq for Ident {
eq(&self, other: &Ident) -> bool623 fn eq(&self, other: &Ident) -> bool {
624 self.sym == other.sym && self.raw == other.raw
625 }
626 }
627
628 impl<T> PartialEq<T> for Ident
629 where
630 T: ?Sized + AsRef<str>,
631 {
eq(&self, other: &T) -> bool632 fn eq(&self, other: &T) -> bool {
633 let other = other.as_ref();
634 if self.raw {
635 other.starts_with("r#") && self.sym == other[2..]
636 } else {
637 self.sym == other
638 }
639 }
640 }
641
642 impl fmt::Display for Ident {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result643 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
644 if self.raw {
645 "r#".fmt(f)?;
646 }
647 self.sym.fmt(f)
648 }
649 }
650
651 impl fmt::Debug for Ident {
652 // Ident(proc_macro), Ident(r#union)
653 #[cfg(not(procmacro2_semver_exempt))]
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result654 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
655 let mut debug = f.debug_tuple("Ident");
656 debug.field(&format_args!("{}", self));
657 debug.finish()
658 }
659
660 // Ident {
661 // sym: proc_macro,
662 // span: bytes(128..138)
663 // }
664 #[cfg(procmacro2_semver_exempt)]
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result665 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
666 let mut debug = f.debug_struct("Ident");
667 debug.field("sym", &format_args!("{}", self));
668 debug.field("span", &self.span);
669 debug.finish()
670 }
671 }
672
673 #[derive(Clone)]
674 pub struct Literal {
675 text: String,
676 span: Span,
677 }
678
679 macro_rules! suffixed_numbers {
680 ($($name:ident => $kind:ident,)*) => ($(
681 pub fn $name(n: $kind) -> Literal {
682 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
683 }
684 )*)
685 }
686
687 macro_rules! unsuffixed_numbers {
688 ($($name:ident => $kind:ident,)*) => ($(
689 pub fn $name(n: $kind) -> Literal {
690 Literal::_new(n.to_string())
691 }
692 )*)
693 }
694
695 impl Literal {
_new(text: String) -> Literal696 fn _new(text: String) -> Literal {
697 Literal {
698 text,
699 span: Span::call_site(),
700 }
701 }
702
703 suffixed_numbers! {
704 u8_suffixed => u8,
705 u16_suffixed => u16,
706 u32_suffixed => u32,
707 u64_suffixed => u64,
708 u128_suffixed => u128,
709 usize_suffixed => usize,
710 i8_suffixed => i8,
711 i16_suffixed => i16,
712 i32_suffixed => i32,
713 i64_suffixed => i64,
714 i128_suffixed => i128,
715 isize_suffixed => isize,
716
717 f32_suffixed => f32,
718 f64_suffixed => f64,
719 }
720
721 unsuffixed_numbers! {
722 u8_unsuffixed => u8,
723 u16_unsuffixed => u16,
724 u32_unsuffixed => u32,
725 u64_unsuffixed => u64,
726 u128_unsuffixed => u128,
727 usize_unsuffixed => usize,
728 i8_unsuffixed => i8,
729 i16_unsuffixed => i16,
730 i32_unsuffixed => i32,
731 i64_unsuffixed => i64,
732 i128_unsuffixed => i128,
733 isize_unsuffixed => isize,
734 }
735
f32_unsuffixed(f: f32) -> Literal736 pub fn f32_unsuffixed(f: f32) -> Literal {
737 let mut s = f.to_string();
738 if !s.contains(".") {
739 s.push_str(".0");
740 }
741 Literal::_new(s)
742 }
743
f64_unsuffixed(f: f64) -> Literal744 pub fn f64_unsuffixed(f: f64) -> Literal {
745 let mut s = f.to_string();
746 if !s.contains(".") {
747 s.push_str(".0");
748 }
749 Literal::_new(s)
750 }
751
string(t: &str) -> Literal752 pub fn string(t: &str) -> Literal {
753 let mut text = String::with_capacity(t.len() + 2);
754 text.push('"');
755 for c in t.chars() {
756 if c == '\'' {
757 // escape_default turns this into "\'" which is unnecessary.
758 text.push(c);
759 } else {
760 text.extend(c.escape_default());
761 }
762 }
763 text.push('"');
764 Literal::_new(text)
765 }
766
character(t: char) -> Literal767 pub fn character(t: char) -> Literal {
768 let mut text = String::new();
769 text.push('\'');
770 if t == '"' {
771 // escape_default turns this into '\"' which is unnecessary.
772 text.push(t);
773 } else {
774 text.extend(t.escape_default());
775 }
776 text.push('\'');
777 Literal::_new(text)
778 }
779
byte_string(bytes: &[u8]) -> Literal780 pub fn byte_string(bytes: &[u8]) -> Literal {
781 let mut escaped = "b\"".to_string();
782 for b in bytes {
783 match *b {
784 b'\0' => escaped.push_str(r"\0"),
785 b'\t' => escaped.push_str(r"\t"),
786 b'\n' => escaped.push_str(r"\n"),
787 b'\r' => escaped.push_str(r"\r"),
788 b'"' => escaped.push_str("\\\""),
789 b'\\' => escaped.push_str("\\\\"),
790 b'\x20'..=b'\x7E' => escaped.push(*b as char),
791 _ => escaped.push_str(&format!("\\x{:02X}", b)),
792 }
793 }
794 escaped.push('"');
795 Literal::_new(escaped)
796 }
797
span(&self) -> Span798 pub fn span(&self) -> Span {
799 self.span
800 }
801
set_span(&mut self, span: Span)802 pub fn set_span(&mut self, span: Span) {
803 self.span = span;
804 }
805
subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span>806 pub fn subspan<R: RangeBounds<usize>>(&self, _range: R) -> Option<Span> {
807 None
808 }
809 }
810
811 impl fmt::Display for Literal {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result812 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
813 self.text.fmt(f)
814 }
815 }
816
817 impl fmt::Debug for Literal {
fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result818 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
819 let mut debug = fmt.debug_struct("Literal");
820 debug.field("lit", &format_args!("{}", self.text));
821 #[cfg(procmacro2_semver_exempt)]
822 debug.field("span", &self.span);
823 debug.finish()
824 }
825 }
826
token_stream(mut input: Cursor) -> PResult<TokenStream>827 fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
828 let mut trees = Vec::new();
829 loop {
830 let input_no_ws = skip_whitespace(input);
831 if input_no_ws.rest.len() == 0 {
832 break;
833 }
834 if let Ok((a, tokens)) = doc_comment(input_no_ws) {
835 input = a;
836 trees.extend(tokens);
837 continue;
838 }
839
840 let (a, tt) = match token_tree(input_no_ws) {
841 Ok(p) => p,
842 Err(_) => break,
843 };
844 trees.push(tt);
845 input = a;
846 }
847 Ok((input, TokenStream { inner: trees }))
848 }
849
850 #[cfg(not(span_locations))]
spanned<'a, T>( input: Cursor<'a>, f: fn(Cursor<'a>) -> PResult<'a, T>, ) -> PResult<'a, (T, crate::Span)>851 fn spanned<'a, T>(
852 input: Cursor<'a>,
853 f: fn(Cursor<'a>) -> PResult<'a, T>,
854 ) -> PResult<'a, (T, crate::Span)> {
855 let (a, b) = f(skip_whitespace(input))?;
856 Ok((a, ((b, crate::Span::_new_stable(Span::call_site())))))
857 }
858
859 #[cfg(span_locations)]
spanned<'a, T>( input: Cursor<'a>, f: fn(Cursor<'a>) -> PResult<'a, T>, ) -> PResult<'a, (T, crate::Span)>860 fn spanned<'a, T>(
861 input: Cursor<'a>,
862 f: fn(Cursor<'a>) -> PResult<'a, T>,
863 ) -> PResult<'a, (T, crate::Span)> {
864 let input = skip_whitespace(input);
865 let lo = input.off;
866 let (a, b) = f(input)?;
867 let hi = a.off;
868 let span = crate::Span::_new_stable(Span { lo, hi });
869 Ok((a, (b, span)))
870 }
871
token_tree(input: Cursor) -> PResult<TokenTree>872 fn token_tree(input: Cursor) -> PResult<TokenTree> {
873 let (rest, (mut tt, span)) = spanned(input, token_kind)?;
874 tt.set_span(span);
875 Ok((rest, tt))
876 }
877
878 named!(token_kind -> TokenTree, alt!(
879 map!(group, |g| TokenTree::Group(crate::Group::_new_stable(g)))
880 |
881 map!(literal, |l| TokenTree::Literal(crate::Literal::_new_stable(l))) // must be before symbol
882 |
883 map!(op, TokenTree::Punct)
884 |
885 symbol_leading_ws
886 ));
887
888 named!(group -> Group, alt!(
889 delimited!(
890 punct!("("),
891 token_stream,
892 punct!(")")
893 ) => { |ts| Group::new(Delimiter::Parenthesis, ts) }
894 |
895 delimited!(
896 punct!("["),
897 token_stream,
898 punct!("]")
899 ) => { |ts| Group::new(Delimiter::Bracket, ts) }
900 |
901 delimited!(
902 punct!("{"),
903 token_stream,
904 punct!("}")
905 ) => { |ts| Group::new(Delimiter::Brace, ts) }
906 ));
907
symbol_leading_ws(input: Cursor) -> PResult<TokenTree>908 fn symbol_leading_ws(input: Cursor) -> PResult<TokenTree> {
909 symbol(skip_whitespace(input))
910 }
911
symbol(input: Cursor) -> PResult<TokenTree>912 fn symbol(input: Cursor) -> PResult<TokenTree> {
913 let raw = input.starts_with("r#");
914 let rest = input.advance((raw as usize) << 1);
915
916 let (rest, sym) = symbol_not_raw(rest)?;
917
918 if !raw {
919 let ident = crate::Ident::new(sym, crate::Span::call_site());
920 return Ok((rest, ident.into()));
921 }
922
923 if sym == "_" {
924 return Err(LexError);
925 }
926
927 let ident = crate::Ident::_new_raw(sym, crate::Span::call_site());
928 Ok((rest, ident.into()))
929 }
930
symbol_not_raw(input: Cursor) -> PResult<&str>931 fn symbol_not_raw(input: Cursor) -> PResult<&str> {
932 let mut chars = input.char_indices();
933
934 match chars.next() {
935 Some((_, ch)) if is_ident_start(ch) => {}
936 _ => return Err(LexError),
937 }
938
939 let mut end = input.len();
940 for (i, ch) in chars {
941 if !is_ident_continue(ch) {
942 end = i;
943 break;
944 }
945 }
946
947 Ok((input.advance(end), &input.rest[..end]))
948 }
949
literal(input: Cursor) -> PResult<Literal>950 fn literal(input: Cursor) -> PResult<Literal> {
951 let input_no_ws = skip_whitespace(input);
952
953 match literal_nocapture(input_no_ws) {
954 Ok((a, ())) => {
955 let start = input.len() - input_no_ws.len();
956 let len = input_no_ws.len() - a.len();
957 let end = start + len;
958 Ok((a, Literal::_new(input.rest[start..end].to_string())))
959 }
960 Err(LexError) => Err(LexError),
961 }
962 }
963
964 named!(literal_nocapture -> (), alt!(
965 string
966 |
967 byte_string
968 |
969 byte
970 |
971 character
972 |
973 float
974 |
975 int
976 ));
977
978 named!(string -> (), alt!(
979 quoted_string
980 |
981 preceded!(
982 punct!("r"),
983 raw_string
984 ) => { |_| () }
985 ));
986
987 named!(quoted_string -> (), do_parse!(
988 punct!("\"") >>
989 cooked_string >>
990 tag!("\"") >>
991 option!(symbol_not_raw) >>
992 (())
993 ));
994
cooked_string(input: Cursor) -> PResult<()>995 fn cooked_string(input: Cursor) -> PResult<()> {
996 let mut chars = input.char_indices().peekable();
997 while let Some((byte_offset, ch)) = chars.next() {
998 match ch {
999 '"' => {
1000 return Ok((input.advance(byte_offset), ()));
1001 }
1002 '\r' => {
1003 if let Some((_, '\n')) = chars.next() {
1004 // ...
1005 } else {
1006 break;
1007 }
1008 }
1009 '\\' => match chars.next() {
1010 Some((_, 'x')) => {
1011 if !backslash_x_char(&mut chars) {
1012 break;
1013 }
1014 }
1015 Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
1016 | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
1017 Some((_, 'u')) => {
1018 if !backslash_u(&mut chars) {
1019 break;
1020 }
1021 }
1022 Some((_, '\n')) | Some((_, '\r')) => {
1023 while let Some(&(_, ch)) = chars.peek() {
1024 if ch.is_whitespace() {
1025 chars.next();
1026 } else {
1027 break;
1028 }
1029 }
1030 }
1031 _ => break,
1032 },
1033 _ch => {}
1034 }
1035 }
1036 Err(LexError)
1037 }
1038
1039 named!(byte_string -> (), alt!(
1040 delimited!(
1041 punct!("b\""),
1042 cooked_byte_string,
1043 tag!("\"")
1044 ) => { |_| () }
1045 |
1046 preceded!(
1047 punct!("br"),
1048 raw_string
1049 ) => { |_| () }
1050 ));
1051
cooked_byte_string(mut input: Cursor) -> PResult<()>1052 fn cooked_byte_string(mut input: Cursor) -> PResult<()> {
1053 let mut bytes = input.bytes().enumerate();
1054 'outer: while let Some((offset, b)) = bytes.next() {
1055 match b {
1056 b'"' => {
1057 return Ok((input.advance(offset), ()));
1058 }
1059 b'\r' => {
1060 if let Some((_, b'\n')) = bytes.next() {
1061 // ...
1062 } else {
1063 break;
1064 }
1065 }
1066 b'\\' => match bytes.next() {
1067 Some((_, b'x')) => {
1068 if !backslash_x_byte(&mut bytes) {
1069 break;
1070 }
1071 }
1072 Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
1073 | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
1074 Some((newline, b'\n')) | Some((newline, b'\r')) => {
1075 let rest = input.advance(newline + 1);
1076 for (offset, ch) in rest.char_indices() {
1077 if !ch.is_whitespace() {
1078 input = rest.advance(offset);
1079 bytes = input.bytes().enumerate();
1080 continue 'outer;
1081 }
1082 }
1083 break;
1084 }
1085 _ => break,
1086 },
1087 b if b < 0x80 => {}
1088 _ => break,
1089 }
1090 }
1091 Err(LexError)
1092 }
1093
raw_string(input: Cursor) -> PResult<()>1094 fn raw_string(input: Cursor) -> PResult<()> {
1095 let mut chars = input.char_indices();
1096 let mut n = 0;
1097 while let Some((byte_offset, ch)) = chars.next() {
1098 match ch {
1099 '"' => {
1100 n = byte_offset;
1101 break;
1102 }
1103 '#' => {}
1104 _ => return Err(LexError),
1105 }
1106 }
1107 for (byte_offset, ch) in chars {
1108 match ch {
1109 '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => {
1110 let rest = input.advance(byte_offset + 1 + n);
1111 return Ok((rest, ()));
1112 }
1113 '\r' => {}
1114 _ => {}
1115 }
1116 }
1117 Err(LexError)
1118 }
1119
1120 named!(byte -> (), do_parse!(
1121 punct!("b") >>
1122 tag!("'") >>
1123 cooked_byte >>
1124 tag!("'") >>
1125 (())
1126 ));
1127
cooked_byte(input: Cursor) -> PResult<()>1128 fn cooked_byte(input: Cursor) -> PResult<()> {
1129 let mut bytes = input.bytes().enumerate();
1130 let ok = match bytes.next().map(|(_, b)| b) {
1131 Some(b'\\') => match bytes.next().map(|(_, b)| b) {
1132 Some(b'x') => backslash_x_byte(&mut bytes),
1133 Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
1134 | Some(b'"') => true,
1135 _ => false,
1136 },
1137 b => b.is_some(),
1138 };
1139 if ok {
1140 match bytes.next() {
1141 Some((offset, _)) => {
1142 if input.chars().as_str().is_char_boundary(offset) {
1143 Ok((input.advance(offset), ()))
1144 } else {
1145 Err(LexError)
1146 }
1147 }
1148 None => Ok((input.advance(input.len()), ())),
1149 }
1150 } else {
1151 Err(LexError)
1152 }
1153 }
1154
1155 named!(character -> (), do_parse!(
1156 punct!("'") >>
1157 cooked_char >>
1158 tag!("'") >>
1159 (())
1160 ));
1161
cooked_char(input: Cursor) -> PResult<()>1162 fn cooked_char(input: Cursor) -> PResult<()> {
1163 let mut chars = input.char_indices();
1164 let ok = match chars.next().map(|(_, ch)| ch) {
1165 Some('\\') => match chars.next().map(|(_, ch)| ch) {
1166 Some('x') => backslash_x_char(&mut chars),
1167 Some('u') => backslash_u(&mut chars),
1168 Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
1169 true
1170 }
1171 _ => false,
1172 },
1173 ch => ch.is_some(),
1174 };
1175 if ok {
1176 match chars.next() {
1177 Some((idx, _)) => Ok((input.advance(idx), ())),
1178 None => Ok((input.advance(input.len()), ())),
1179 }
1180 } else {
1181 Err(LexError)
1182 }
1183 }
1184
1185 macro_rules! next_ch {
1186 ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
1187 match $chars.next() {
1188 Some((_, ch)) => match ch {
1189 $pat $(| $rest)* => ch,
1190 _ => return false,
1191 },
1192 None => return false
1193 }
1194 };
1195 }
1196
backslash_x_char<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, char)>,1197 fn backslash_x_char<I>(chars: &mut I) -> bool
1198 where
1199 I: Iterator<Item = (usize, char)>,
1200 {
1201 next_ch!(chars @ '0'..='7');
1202 next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
1203 true
1204 }
1205
backslash_x_byte<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, u8)>,1206 fn backslash_x_byte<I>(chars: &mut I) -> bool
1207 where
1208 I: Iterator<Item = (usize, u8)>,
1209 {
1210 next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
1211 next_ch!(chars @ b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F');
1212 true
1213 }
1214
backslash_u<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, char)>,1215 fn backslash_u<I>(chars: &mut I) -> bool
1216 where
1217 I: Iterator<Item = (usize, char)>,
1218 {
1219 next_ch!(chars @ '{');
1220 next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F');
1221 loop {
1222 let c = next_ch!(chars @ '0'..='9' | 'a'..='f' | 'A'..='F' | '_' | '}');
1223 if c == '}' {
1224 return true;
1225 }
1226 }
1227 }
1228
float(input: Cursor) -> PResult<()>1229 fn float(input: Cursor) -> PResult<()> {
1230 let (mut rest, ()) = float_digits(input)?;
1231 if let Some(ch) = rest.chars().next() {
1232 if is_ident_start(ch) {
1233 rest = symbol_not_raw(rest)?.0;
1234 }
1235 }
1236 word_break(rest)
1237 }
1238
float_digits(input: Cursor) -> PResult<()>1239 fn float_digits(input: Cursor) -> PResult<()> {
1240 let mut chars = input.chars().peekable();
1241 match chars.next() {
1242 Some(ch) if ch >= '0' && ch <= '9' => {}
1243 _ => return Err(LexError),
1244 }
1245
1246 let mut len = 1;
1247 let mut has_dot = false;
1248 let mut has_exp = false;
1249 while let Some(&ch) = chars.peek() {
1250 match ch {
1251 '0'..='9' | '_' => {
1252 chars.next();
1253 len += 1;
1254 }
1255 '.' => {
1256 if has_dot {
1257 break;
1258 }
1259 chars.next();
1260 if chars
1261 .peek()
1262 .map(|&ch| ch == '.' || is_ident_start(ch))
1263 .unwrap_or(false)
1264 {
1265 return Err(LexError);
1266 }
1267 len += 1;
1268 has_dot = true;
1269 }
1270 'e' | 'E' => {
1271 chars.next();
1272 len += 1;
1273 has_exp = true;
1274 break;
1275 }
1276 _ => break,
1277 }
1278 }
1279
1280 let rest = input.advance(len);
1281 if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
1282 return Err(LexError);
1283 }
1284
1285 if has_exp {
1286 let mut has_exp_value = false;
1287 while let Some(&ch) = chars.peek() {
1288 match ch {
1289 '+' | '-' => {
1290 if has_exp_value {
1291 break;
1292 }
1293 chars.next();
1294 len += 1;
1295 }
1296 '0'..='9' => {
1297 chars.next();
1298 len += 1;
1299 has_exp_value = true;
1300 }
1301 '_' => {
1302 chars.next();
1303 len += 1;
1304 }
1305 _ => break,
1306 }
1307 }
1308 if !has_exp_value {
1309 return Err(LexError);
1310 }
1311 }
1312
1313 Ok((input.advance(len), ()))
1314 }
1315
int(input: Cursor) -> PResult<()>1316 fn int(input: Cursor) -> PResult<()> {
1317 let (mut rest, ()) = digits(input)?;
1318 if let Some(ch) = rest.chars().next() {
1319 if is_ident_start(ch) {
1320 rest = symbol_not_raw(rest)?.0;
1321 }
1322 }
1323 word_break(rest)
1324 }
1325
digits(mut input: Cursor) -> PResult<()>1326 fn digits(mut input: Cursor) -> PResult<()> {
1327 let base = if input.starts_with("0x") {
1328 input = input.advance(2);
1329 16
1330 } else if input.starts_with("0o") {
1331 input = input.advance(2);
1332 8
1333 } else if input.starts_with("0b") {
1334 input = input.advance(2);
1335 2
1336 } else {
1337 10
1338 };
1339
1340 let mut len = 0;
1341 let mut empty = true;
1342 for b in input.bytes() {
1343 let digit = match b {
1344 b'0'..=b'9' => (b - b'0') as u64,
1345 b'a'..=b'f' => 10 + (b - b'a') as u64,
1346 b'A'..=b'F' => 10 + (b - b'A') as u64,
1347 b'_' => {
1348 if empty && base == 10 {
1349 return Err(LexError);
1350 }
1351 len += 1;
1352 continue;
1353 }
1354 _ => break,
1355 };
1356 if digit >= base {
1357 return Err(LexError);
1358 }
1359 len += 1;
1360 empty = false;
1361 }
1362 if empty {
1363 Err(LexError)
1364 } else {
1365 Ok((input.advance(len), ()))
1366 }
1367 }
1368
op(input: Cursor) -> PResult<Punct>1369 fn op(input: Cursor) -> PResult<Punct> {
1370 let input = skip_whitespace(input);
1371 match op_char(input) {
1372 Ok((rest, '\'')) => {
1373 symbol(rest)?;
1374 Ok((rest, Punct::new('\'', Spacing::Joint)))
1375 }
1376 Ok((rest, ch)) => {
1377 let kind = match op_char(rest) {
1378 Ok(_) => Spacing::Joint,
1379 Err(LexError) => Spacing::Alone,
1380 };
1381 Ok((rest, Punct::new(ch, kind)))
1382 }
1383 Err(LexError) => Err(LexError),
1384 }
1385 }
1386
op_char(input: Cursor) -> PResult<char>1387 fn op_char(input: Cursor) -> PResult<char> {
1388 if input.starts_with("//") || input.starts_with("/*") {
1389 // Do not accept `/` of a comment as an op.
1390 return Err(LexError);
1391 }
1392
1393 let mut chars = input.chars();
1394 let first = match chars.next() {
1395 Some(ch) => ch,
1396 None => {
1397 return Err(LexError);
1398 }
1399 };
1400 let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
1401 if recognized.contains(first) {
1402 Ok((input.advance(first.len_utf8()), first))
1403 } else {
1404 Err(LexError)
1405 }
1406 }
1407
doc_comment(input: Cursor) -> PResult<Vec<TokenTree>>1408 fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
1409 let mut trees = Vec::new();
1410 let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
1411 trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone)));
1412 if inner {
1413 trees.push(Punct::new('!', Spacing::Alone).into());
1414 }
1415 let mut stream = vec![
1416 TokenTree::Ident(crate::Ident::new("doc", span)),
1417 TokenTree::Punct(Punct::new('=', Spacing::Alone)),
1418 TokenTree::Literal(crate::Literal::string(comment)),
1419 ];
1420 for tt in stream.iter_mut() {
1421 tt.set_span(span);
1422 }
1423 let group = Group::new(Delimiter::Bracket, stream.into_iter().collect());
1424 trees.push(crate::Group::_new_stable(group).into());
1425 for tt in trees.iter_mut() {
1426 tt.set_span(span);
1427 }
1428 Ok((rest, trees))
1429 }
1430
1431 named!(doc_comment_contents -> (&str, bool), alt!(
1432 do_parse!(
1433 punct!("//!") >>
1434 s: take_until_newline_or_eof!() >>
1435 ((s, true))
1436 )
1437 |
1438 do_parse!(
1439 option!(whitespace) >>
1440 peek!(tag!("/*!")) >>
1441 s: block_comment >>
1442 ((s, true))
1443 )
1444 |
1445 do_parse!(
1446 punct!("///") >>
1447 not!(tag!("/")) >>
1448 s: take_until_newline_or_eof!() >>
1449 ((s, false))
1450 )
1451 |
1452 do_parse!(
1453 option!(whitespace) >>
1454 peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
1455 s: block_comment >>
1456 ((s, false))
1457 )
1458 ));
1459