1 #[cfg(span_locations)]
2 use std::cell::RefCell;
3 #[cfg(procmacro2_semver_exempt)]
4 use std::cmp;
5 use std::fmt;
6 use std::iter;
7 #[cfg(procmacro2_semver_exempt)]
8 use std::path::Path;
9 use std::path::PathBuf;
10 use std::str::FromStr;
11 use std::vec;
12
13 use strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
14 use unicode_xid::UnicodeXID;
15
16 use {Delimiter, Punct, Spacing, TokenTree};
17
18 #[derive(Clone)]
19 pub struct TokenStream {
20 inner: Vec<TokenTree>,
21 }
22
23 #[derive(Debug)]
24 pub struct LexError;
25
26 impl TokenStream {
new() -> TokenStream27 pub fn new() -> TokenStream {
28 TokenStream { inner: Vec::new() }
29 }
30
is_empty(&self) -> bool31 pub fn is_empty(&self) -> bool {
32 self.inner.len() == 0
33 }
34 }
35
36 #[cfg(span_locations)]
get_cursor(src: &str) -> Cursor37 fn get_cursor(src: &str) -> Cursor {
38 // Create a dummy file & add it to the source map
39 SOURCE_MAP.with(|cm| {
40 let mut cm = cm.borrow_mut();
41 let name = format!("<parsed string {}>", cm.files.len());
42 let span = cm.add_file(&name, src);
43 Cursor {
44 rest: src,
45 off: span.lo,
46 }
47 })
48 }
49
50 #[cfg(not(span_locations))]
get_cursor(src: &str) -> Cursor51 fn get_cursor(src: &str) -> Cursor {
52 Cursor { rest: src }
53 }
54
55 impl FromStr for TokenStream {
56 type Err = LexError;
57
from_str(src: &str) -> Result<TokenStream, LexError>58 fn from_str(src: &str) -> Result<TokenStream, LexError> {
59 // Create a dummy file & add it to the source map
60 let cursor = get_cursor(src);
61
62 match token_stream(cursor) {
63 Ok((input, output)) => {
64 if skip_whitespace(input).len() != 0 {
65 Err(LexError)
66 } else {
67 Ok(output)
68 }
69 }
70 Err(LexError) => Err(LexError),
71 }
72 }
73 }
74
75 impl fmt::Display for TokenStream {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result76 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
77 let mut joint = false;
78 for (i, tt) in self.inner.iter().enumerate() {
79 if i != 0 && !joint {
80 write!(f, " ")?;
81 }
82 joint = false;
83 match *tt {
84 TokenTree::Group(ref tt) => {
85 let (start, end) = match tt.delimiter() {
86 Delimiter::Parenthesis => ("(", ")"),
87 Delimiter::Brace => ("{", "}"),
88 Delimiter::Bracket => ("[", "]"),
89 Delimiter::None => ("", ""),
90 };
91 if tt.stream().into_iter().next().is_none() {
92 write!(f, "{} {}", start, end)?
93 } else {
94 write!(f, "{} {} {}", start, tt.stream(), end)?
95 }
96 }
97 TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
98 TokenTree::Punct(ref tt) => {
99 write!(f, "{}", tt.as_char())?;
100 match tt.spacing() {
101 Spacing::Alone => {}
102 Spacing::Joint => joint = true,
103 }
104 }
105 TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
106 }
107 }
108
109 Ok(())
110 }
111 }
112
113 impl fmt::Debug for TokenStream {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result114 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
115 f.write_str("TokenStream ")?;
116 f.debug_list().entries(self.clone()).finish()
117 }
118 }
119
120 #[cfg(use_proc_macro)]
121 impl From<::proc_macro::TokenStream> for TokenStream {
from(inner: ::proc_macro::TokenStream) -> TokenStream122 fn from(inner: ::proc_macro::TokenStream) -> TokenStream {
123 inner
124 .to_string()
125 .parse()
126 .expect("compiler token stream parse failed")
127 }
128 }
129
130 #[cfg(use_proc_macro)]
131 impl From<TokenStream> for ::proc_macro::TokenStream {
from(inner: TokenStream) -> ::proc_macro::TokenStream132 fn from(inner: TokenStream) -> ::proc_macro::TokenStream {
133 inner
134 .to_string()
135 .parse()
136 .expect("failed to parse to compiler tokens")
137 }
138 }
139
140 impl From<TokenTree> for TokenStream {
from(tree: TokenTree) -> TokenStream141 fn from(tree: TokenTree) -> TokenStream {
142 TokenStream { inner: vec![tree] }
143 }
144 }
145
146 impl iter::FromIterator<TokenTree> for TokenStream {
from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self147 fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
148 let mut v = Vec::new();
149
150 for token in streams.into_iter() {
151 v.push(token);
152 }
153
154 TokenStream { inner: v }
155 }
156 }
157
158 impl iter::FromIterator<TokenStream> for TokenStream {
from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self159 fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
160 let mut v = Vec::new();
161
162 for stream in streams.into_iter() {
163 v.extend(stream.inner);
164 }
165
166 TokenStream { inner: v }
167 }
168 }
169
170 impl Extend<TokenTree> for TokenStream {
extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I)171 fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
172 self.inner.extend(streams);
173 }
174 }
175
176 impl Extend<TokenStream> for TokenStream {
extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I)177 fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
178 self.inner
179 .extend(streams.into_iter().flat_map(|stream| stream));
180 }
181 }
182
183 pub type TokenTreeIter = vec::IntoIter<TokenTree>;
184
185 impl IntoIterator for TokenStream {
186 type Item = TokenTree;
187 type IntoIter = TokenTreeIter;
188
into_iter(self) -> TokenTreeIter189 fn into_iter(self) -> TokenTreeIter {
190 self.inner.into_iter()
191 }
192 }
193
194 #[derive(Clone, PartialEq, Eq)]
195 pub struct SourceFile {
196 path: PathBuf,
197 }
198
199 impl SourceFile {
200 /// Get the path to this source file as a string.
path(&self) -> PathBuf201 pub fn path(&self) -> PathBuf {
202 self.path.clone()
203 }
204
is_real(&self) -> bool205 pub fn is_real(&self) -> bool {
206 // XXX(nika): Support real files in the future?
207 false
208 }
209 }
210
211 impl fmt::Debug for SourceFile {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result212 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
213 f.debug_struct("SourceFile")
214 .field("path", &self.path())
215 .field("is_real", &self.is_real())
216 .finish()
217 }
218 }
219
220 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
221 pub struct LineColumn {
222 pub line: usize,
223 pub column: usize,
224 }
225
226 #[cfg(span_locations)]
227 thread_local! {
228 static SOURCE_MAP: RefCell<SourceMap> = RefCell::new(SourceMap {
229 // NOTE: We start with a single dummy file which all call_site() and
230 // def_site() spans reference.
231 files: vec![{
232 #[cfg(procmacro2_semver_exempt)]
233 {
234 FileInfo {
235 name: "<unspecified>".to_owned(),
236 span: Span { lo: 0, hi: 0 },
237 lines: vec![0],
238 }
239 }
240
241 #[cfg(not(procmacro2_semver_exempt))]
242 {
243 FileInfo {
244 span: Span { lo: 0, hi: 0 },
245 lines: vec![0],
246 }
247 }
248 }],
249 });
250 }
251
252 #[cfg(span_locations)]
253 struct FileInfo {
254 #[cfg(procmacro2_semver_exempt)]
255 name: String,
256 span: Span,
257 lines: Vec<usize>,
258 }
259
260 #[cfg(span_locations)]
261 impl FileInfo {
offset_line_column(&self, offset: usize) -> LineColumn262 fn offset_line_column(&self, offset: usize) -> LineColumn {
263 assert!(self.span_within(Span {
264 lo: offset as u32,
265 hi: offset as u32
266 }));
267 let offset = offset - self.span.lo as usize;
268 match self.lines.binary_search(&offset) {
269 Ok(found) => LineColumn {
270 line: found + 1,
271 column: 0,
272 },
273 Err(idx) => LineColumn {
274 line: idx,
275 column: offset - self.lines[idx - 1],
276 },
277 }
278 }
279
span_within(&self, span: Span) -> bool280 fn span_within(&self, span: Span) -> bool {
281 span.lo >= self.span.lo && span.hi <= self.span.hi
282 }
283 }
284
285 /// Computesthe offsets of each line in the given source string.
286 #[cfg(span_locations)]
lines_offsets(s: &str) -> Vec<usize>287 fn lines_offsets(s: &str) -> Vec<usize> {
288 let mut lines = vec![0];
289 let mut prev = 0;
290 while let Some(len) = s[prev..].find('\n') {
291 prev += len + 1;
292 lines.push(prev);
293 }
294 lines
295 }
296
297 #[cfg(span_locations)]
298 struct SourceMap {
299 files: Vec<FileInfo>,
300 }
301
302 #[cfg(span_locations)]
303 impl SourceMap {
next_start_pos(&self) -> u32304 fn next_start_pos(&self) -> u32 {
305 // Add 1 so there's always space between files.
306 //
307 // We'll always have at least 1 file, as we initialize our files list
308 // with a dummy file.
309 self.files.last().unwrap().span.hi + 1
310 }
311
add_file(&mut self, name: &str, src: &str) -> Span312 fn add_file(&mut self, name: &str, src: &str) -> Span {
313 let lines = lines_offsets(src);
314 let lo = self.next_start_pos();
315 // XXX(nika): Shouild we bother doing a checked cast or checked add here?
316 let span = Span {
317 lo: lo,
318 hi: lo + (src.len() as u32),
319 };
320
321 #[cfg(procmacro2_semver_exempt)]
322 self.files.push(FileInfo {
323 name: name.to_owned(),
324 span: span,
325 lines: lines,
326 });
327
328 #[cfg(not(procmacro2_semver_exempt))]
329 self.files.push(FileInfo {
330 span: span,
331 lines: lines,
332 });
333 let _ = name;
334
335 span
336 }
337
fileinfo(&self, span: Span) -> &FileInfo338 fn fileinfo(&self, span: Span) -> &FileInfo {
339 for file in &self.files {
340 if file.span_within(span) {
341 return file;
342 }
343 }
344 panic!("Invalid span with no related FileInfo!");
345 }
346 }
347
348 #[derive(Clone, Copy, PartialEq, Eq)]
349 pub struct Span {
350 #[cfg(span_locations)]
351 lo: u32,
352 #[cfg(span_locations)]
353 hi: u32,
354 }
355
356 impl Span {
357 #[cfg(not(span_locations))]
call_site() -> Span358 pub fn call_site() -> Span {
359 Span {}
360 }
361
362 #[cfg(span_locations)]
call_site() -> Span363 pub fn call_site() -> Span {
364 Span { lo: 0, hi: 0 }
365 }
366
367 #[cfg(procmacro2_semver_exempt)]
def_site() -> Span368 pub fn def_site() -> Span {
369 Span::call_site()
370 }
371
372 #[cfg(procmacro2_semver_exempt)]
resolved_at(&self, _other: Span) -> Span373 pub fn resolved_at(&self, _other: Span) -> Span {
374 // Stable spans consist only of line/column information, so
375 // `resolved_at` and `located_at` only select which span the
376 // caller wants line/column information from.
377 *self
378 }
379
380 #[cfg(procmacro2_semver_exempt)]
located_at(&self, other: Span) -> Span381 pub fn located_at(&self, other: Span) -> Span {
382 other
383 }
384
385 #[cfg(procmacro2_semver_exempt)]
source_file(&self) -> SourceFile386 pub fn source_file(&self) -> SourceFile {
387 SOURCE_MAP.with(|cm| {
388 let cm = cm.borrow();
389 let fi = cm.fileinfo(*self);
390 SourceFile {
391 path: Path::new(&fi.name).to_owned(),
392 }
393 })
394 }
395
396 #[cfg(span_locations)]
start(&self) -> LineColumn397 pub fn start(&self) -> LineColumn {
398 SOURCE_MAP.with(|cm| {
399 let cm = cm.borrow();
400 let fi = cm.fileinfo(*self);
401 fi.offset_line_column(self.lo as usize)
402 })
403 }
404
405 #[cfg(span_locations)]
end(&self) -> LineColumn406 pub fn end(&self) -> LineColumn {
407 SOURCE_MAP.with(|cm| {
408 let cm = cm.borrow();
409 let fi = cm.fileinfo(*self);
410 fi.offset_line_column(self.hi as usize)
411 })
412 }
413
414 #[cfg(procmacro2_semver_exempt)]
join(&self, other: Span) -> Option<Span>415 pub fn join(&self, other: Span) -> Option<Span> {
416 SOURCE_MAP.with(|cm| {
417 let cm = cm.borrow();
418 // If `other` is not within the same FileInfo as us, return None.
419 if !cm.fileinfo(*self).span_within(other) {
420 return None;
421 }
422 Some(Span {
423 lo: cmp::min(self.lo, other.lo),
424 hi: cmp::max(self.hi, other.hi),
425 })
426 })
427 }
428 }
429
430 impl fmt::Debug for Span {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result431 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
432 #[cfg(procmacro2_semver_exempt)]
433 return write!(f, "bytes({}..{})", self.lo, self.hi);
434
435 #[cfg(not(procmacro2_semver_exempt))]
436 write!(f, "Span")
437 }
438 }
439
debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span)440 pub fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
441 if cfg!(procmacro2_semver_exempt) {
442 debug.field("span", &span);
443 }
444 }
445
446 #[derive(Clone)]
447 pub struct Group {
448 delimiter: Delimiter,
449 stream: TokenStream,
450 span: Span,
451 }
452
453 impl Group {
new(delimiter: Delimiter, stream: TokenStream) -> Group454 pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
455 Group {
456 delimiter: delimiter,
457 stream: stream,
458 span: Span::call_site(),
459 }
460 }
461
delimiter(&self) -> Delimiter462 pub fn delimiter(&self) -> Delimiter {
463 self.delimiter
464 }
465
stream(&self) -> TokenStream466 pub fn stream(&self) -> TokenStream {
467 self.stream.clone()
468 }
469
span(&self) -> Span470 pub fn span(&self) -> Span {
471 self.span
472 }
473
474 #[cfg(procmacro2_semver_exempt)]
span_open(&self) -> Span475 pub fn span_open(&self) -> Span {
476 self.span
477 }
478
479 #[cfg(procmacro2_semver_exempt)]
span_close(&self) -> Span480 pub fn span_close(&self) -> Span {
481 self.span
482 }
483
set_span(&mut self, span: Span)484 pub fn set_span(&mut self, span: Span) {
485 self.span = span;
486 }
487 }
488
489 impl fmt::Display for Group {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result490 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
491 let (left, right) = match self.delimiter {
492 Delimiter::Parenthesis => ("(", ")"),
493 Delimiter::Brace => ("{", "}"),
494 Delimiter::Bracket => ("[", "]"),
495 Delimiter::None => ("", ""),
496 };
497
498 f.write_str(left)?;
499 self.stream.fmt(f)?;
500 f.write_str(right)?;
501
502 Ok(())
503 }
504 }
505
506 impl fmt::Debug for Group {
fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result507 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
508 let mut debug = fmt.debug_struct("Group");
509 debug.field("delimiter", &self.delimiter);
510 debug.field("stream", &self.stream);
511 #[cfg(procmacro2_semver_exempt)]
512 debug.field("span", &self.span);
513 debug.finish()
514 }
515 }
516
517 #[derive(Clone)]
518 pub struct Ident {
519 sym: String,
520 span: Span,
521 raw: bool,
522 }
523
524 impl Ident {
_new(string: &str, raw: bool, span: Span) -> Ident525 fn _new(string: &str, raw: bool, span: Span) -> Ident {
526 validate_ident(string);
527
528 Ident {
529 sym: string.to_owned(),
530 span: span,
531 raw: raw,
532 }
533 }
534
new(string: &str, span: Span) -> Ident535 pub fn new(string: &str, span: Span) -> Ident {
536 Ident::_new(string, false, span)
537 }
538
new_raw(string: &str, span: Span) -> Ident539 pub fn new_raw(string: &str, span: Span) -> Ident {
540 Ident::_new(string, true, span)
541 }
542
span(&self) -> Span543 pub fn span(&self) -> Span {
544 self.span
545 }
546
set_span(&mut self, span: Span)547 pub fn set_span(&mut self, span: Span) {
548 self.span = span;
549 }
550 }
551
552 #[inline]
is_ident_start(c: char) -> bool553 fn is_ident_start(c: char) -> bool {
554 ('a' <= c && c <= 'z')
555 || ('A' <= c && c <= 'Z')
556 || c == '_'
557 || (c > '\x7f' && UnicodeXID::is_xid_start(c))
558 }
559
560 #[inline]
is_ident_continue(c: char) -> bool561 fn is_ident_continue(c: char) -> bool {
562 ('a' <= c && c <= 'z')
563 || ('A' <= c && c <= 'Z')
564 || c == '_'
565 || ('0' <= c && c <= '9')
566 || (c > '\x7f' && UnicodeXID::is_xid_continue(c))
567 }
568
validate_ident(string: &str)569 fn validate_ident(string: &str) {
570 let validate = string;
571 if validate.is_empty() {
572 panic!("Ident is not allowed to be empty; use Option<Ident>");
573 }
574
575 if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
576 panic!("Ident cannot be a number; use Literal instead");
577 }
578
579 fn ident_ok(string: &str) -> bool {
580 let mut chars = string.chars();
581 let first = chars.next().unwrap();
582 if !is_ident_start(first) {
583 return false;
584 }
585 for ch in chars {
586 if !is_ident_continue(ch) {
587 return false;
588 }
589 }
590 true
591 }
592
593 if !ident_ok(validate) {
594 panic!("{:?} is not a valid Ident", string);
595 }
596 }
597
598 impl PartialEq for Ident {
eq(&self, other: &Ident) -> bool599 fn eq(&self, other: &Ident) -> bool {
600 self.sym == other.sym && self.raw == other.raw
601 }
602 }
603
604 impl<T> PartialEq<T> for Ident
605 where
606 T: ?Sized + AsRef<str>,
607 {
eq(&self, other: &T) -> bool608 fn eq(&self, other: &T) -> bool {
609 let other = other.as_ref();
610 if self.raw {
611 other.starts_with("r#") && self.sym == other[2..]
612 } else {
613 self.sym == other
614 }
615 }
616 }
617
618 impl fmt::Display for Ident {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result619 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
620 if self.raw {
621 "r#".fmt(f)?;
622 }
623 self.sym.fmt(f)
624 }
625 }
626
627 impl fmt::Debug for Ident {
628 // Ident(proc_macro), Ident(r#union)
629 #[cfg(not(procmacro2_semver_exempt))]
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result630 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
631 let mut debug = f.debug_tuple("Ident");
632 debug.field(&format_args!("{}", self));
633 debug.finish()
634 }
635
636 // Ident {
637 // sym: proc_macro,
638 // span: bytes(128..138)
639 // }
640 #[cfg(procmacro2_semver_exempt)]
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result641 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
642 let mut debug = f.debug_struct("Ident");
643 debug.field("sym", &format_args!("{}", self));
644 debug.field("span", &self.span);
645 debug.finish()
646 }
647 }
648
649 #[derive(Clone)]
650 pub struct Literal {
651 text: String,
652 span: Span,
653 }
654
655 macro_rules! suffixed_numbers {
656 ($($name:ident => $kind:ident,)*) => ($(
657 pub fn $name(n: $kind) -> Literal {
658 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
659 }
660 )*)
661 }
662
663 macro_rules! unsuffixed_numbers {
664 ($($name:ident => $kind:ident,)*) => ($(
665 pub fn $name(n: $kind) -> Literal {
666 Literal::_new(n.to_string())
667 }
668 )*)
669 }
670
671 impl Literal {
_new(text: String) -> Literal672 fn _new(text: String) -> Literal {
673 Literal {
674 text: text,
675 span: Span::call_site(),
676 }
677 }
678
679 suffixed_numbers! {
680 u8_suffixed => u8,
681 u16_suffixed => u16,
682 u32_suffixed => u32,
683 u64_suffixed => u64,
684 usize_suffixed => usize,
685 i8_suffixed => i8,
686 i16_suffixed => i16,
687 i32_suffixed => i32,
688 i64_suffixed => i64,
689 isize_suffixed => isize,
690
691 f32_suffixed => f32,
692 f64_suffixed => f64,
693 }
694
695 #[cfg(u128)]
696 suffixed_numbers! {
697 u128_suffixed => u128,
698 i128_suffixed => i128,
699 }
700
701 unsuffixed_numbers! {
702 u8_unsuffixed => u8,
703 u16_unsuffixed => u16,
704 u32_unsuffixed => u32,
705 u64_unsuffixed => u64,
706 usize_unsuffixed => usize,
707 i8_unsuffixed => i8,
708 i16_unsuffixed => i16,
709 i32_unsuffixed => i32,
710 i64_unsuffixed => i64,
711 isize_unsuffixed => isize,
712 }
713
714 #[cfg(u128)]
715 unsuffixed_numbers! {
716 u128_unsuffixed => u128,
717 i128_unsuffixed => i128,
718 }
719
f32_unsuffixed(f: f32) -> Literal720 pub fn f32_unsuffixed(f: f32) -> Literal {
721 let mut s = f.to_string();
722 if !s.contains(".") {
723 s.push_str(".0");
724 }
725 Literal::_new(s)
726 }
727
f64_unsuffixed(f: f64) -> Literal728 pub fn f64_unsuffixed(f: f64) -> Literal {
729 let mut s = f.to_string();
730 if !s.contains(".") {
731 s.push_str(".0");
732 }
733 Literal::_new(s)
734 }
735
string(t: &str) -> Literal736 pub fn string(t: &str) -> Literal {
737 let mut text = String::with_capacity(t.len() + 2);
738 text.push('"');
739 for c in t.chars() {
740 if c == '\'' {
741 // escape_default turns this into "\'" which is unnecessary.
742 text.push(c);
743 } else {
744 text.extend(c.escape_default());
745 }
746 }
747 text.push('"');
748 Literal::_new(text)
749 }
750
character(t: char) -> Literal751 pub fn character(t: char) -> Literal {
752 let mut text = String::new();
753 text.push('\'');
754 if t == '"' {
755 // escape_default turns this into '\"' which is unnecessary.
756 text.push(t);
757 } else {
758 text.extend(t.escape_default());
759 }
760 text.push('\'');
761 Literal::_new(text)
762 }
763
byte_string(bytes: &[u8]) -> Literal764 pub fn byte_string(bytes: &[u8]) -> Literal {
765 let mut escaped = "b\"".to_string();
766 for b in bytes {
767 match *b {
768 b'\0' => escaped.push_str(r"\0"),
769 b'\t' => escaped.push_str(r"\t"),
770 b'\n' => escaped.push_str(r"\n"),
771 b'\r' => escaped.push_str(r"\r"),
772 b'"' => escaped.push_str("\\\""),
773 b'\\' => escaped.push_str("\\\\"),
774 b'\x20'...b'\x7E' => escaped.push(*b as char),
775 _ => escaped.push_str(&format!("\\x{:02X}", b)),
776 }
777 }
778 escaped.push('"');
779 Literal::_new(escaped)
780 }
781
span(&self) -> Span782 pub fn span(&self) -> Span {
783 self.span
784 }
785
set_span(&mut self, span: Span)786 pub fn set_span(&mut self, span: Span) {
787 self.span = span;
788 }
789 }
790
791 impl fmt::Display for Literal {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result792 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
793 self.text.fmt(f)
794 }
795 }
796
797 impl fmt::Debug for Literal {
fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result798 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
799 let mut debug = fmt.debug_struct("Literal");
800 debug.field("lit", &format_args!("{}", self.text));
801 #[cfg(procmacro2_semver_exempt)]
802 debug.field("span", &self.span);
803 debug.finish()
804 }
805 }
806
token_stream(mut input: Cursor) -> PResult<TokenStream>807 fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
808 let mut trees = Vec::new();
809 loop {
810 let input_no_ws = skip_whitespace(input);
811 if input_no_ws.rest.len() == 0 {
812 break;
813 }
814 if let Ok((a, tokens)) = doc_comment(input_no_ws) {
815 input = a;
816 trees.extend(tokens);
817 continue;
818 }
819
820 let (a, tt) = match token_tree(input_no_ws) {
821 Ok(p) => p,
822 Err(_) => break,
823 };
824 trees.push(tt);
825 input = a;
826 }
827 Ok((input, TokenStream { inner: trees }))
828 }
829
830 #[cfg(not(span_locations))]
spanned<'a, T>( input: Cursor<'a>, f: fn(Cursor<'a>) -> PResult<'a, T>, ) -> PResult<'a, (T, ::Span)>831 fn spanned<'a, T>(
832 input: Cursor<'a>,
833 f: fn(Cursor<'a>) -> PResult<'a, T>,
834 ) -> PResult<'a, (T, ::Span)> {
835 let (a, b) = f(skip_whitespace(input))?;
836 Ok((a, ((b, ::Span::_new_stable(Span::call_site())))))
837 }
838
839 #[cfg(span_locations)]
spanned<'a, T>( input: Cursor<'a>, f: fn(Cursor<'a>) -> PResult<'a, T>, ) -> PResult<'a, (T, ::Span)>840 fn spanned<'a, T>(
841 input: Cursor<'a>,
842 f: fn(Cursor<'a>) -> PResult<'a, T>,
843 ) -> PResult<'a, (T, ::Span)> {
844 let input = skip_whitespace(input);
845 let lo = input.off;
846 let (a, b) = f(input)?;
847 let hi = a.off;
848 let span = ::Span::_new_stable(Span { lo: lo, hi: hi });
849 Ok((a, (b, span)))
850 }
851
token_tree(input: Cursor) -> PResult<TokenTree>852 fn token_tree(input: Cursor) -> PResult<TokenTree> {
853 let (rest, (mut tt, span)) = spanned(input, token_kind)?;
854 tt.set_span(span);
855 Ok((rest, tt))
856 }
857
858 named!(token_kind -> TokenTree, alt!(
859 map!(group, |g| TokenTree::Group(::Group::_new_stable(g)))
860 |
861 map!(literal, |l| TokenTree::Literal(::Literal::_new_stable(l))) // must be before symbol
862 |
863 map!(op, TokenTree::Punct)
864 |
865 symbol_leading_ws
866 ));
867
868 named!(group -> Group, alt!(
869 delimited!(
870 punct!("("),
871 token_stream,
872 punct!(")")
873 ) => { |ts| Group::new(Delimiter::Parenthesis, ts) }
874 |
875 delimited!(
876 punct!("["),
877 token_stream,
878 punct!("]")
879 ) => { |ts| Group::new(Delimiter::Bracket, ts) }
880 |
881 delimited!(
882 punct!("{"),
883 token_stream,
884 punct!("}")
885 ) => { |ts| Group::new(Delimiter::Brace, ts) }
886 ));
887
symbol_leading_ws(input: Cursor) -> PResult<TokenTree>888 fn symbol_leading_ws(input: Cursor) -> PResult<TokenTree> {
889 symbol(skip_whitespace(input))
890 }
891
symbol(input: Cursor) -> PResult<TokenTree>892 fn symbol(input: Cursor) -> PResult<TokenTree> {
893 let mut chars = input.char_indices();
894
895 let raw = input.starts_with("r#");
896 if raw {
897 chars.next();
898 chars.next();
899 }
900
901 match chars.next() {
902 Some((_, ch)) if is_ident_start(ch) => {}
903 _ => return Err(LexError),
904 }
905
906 let mut end = input.len();
907 for (i, ch) in chars {
908 if !is_ident_continue(ch) {
909 end = i;
910 break;
911 }
912 }
913
914 let a = &input.rest[..end];
915 if a == "r#_" {
916 Err(LexError)
917 } else {
918 let ident = if raw {
919 ::Ident::_new_raw(&a[2..], ::Span::call_site())
920 } else {
921 ::Ident::new(a, ::Span::call_site())
922 };
923 Ok((input.advance(end), ident.into()))
924 }
925 }
926
literal(input: Cursor) -> PResult<Literal>927 fn literal(input: Cursor) -> PResult<Literal> {
928 let input_no_ws = skip_whitespace(input);
929
930 match literal_nocapture(input_no_ws) {
931 Ok((a, ())) => {
932 let start = input.len() - input_no_ws.len();
933 let len = input_no_ws.len() - a.len();
934 let end = start + len;
935 Ok((a, Literal::_new(input.rest[start..end].to_string())))
936 }
937 Err(LexError) => Err(LexError),
938 }
939 }
940
941 named!(literal_nocapture -> (), alt!(
942 string
943 |
944 byte_string
945 |
946 byte
947 |
948 character
949 |
950 float
951 |
952 int
953 ));
954
955 named!(string -> (), alt!(
956 quoted_string
957 |
958 preceded!(
959 punct!("r"),
960 raw_string
961 ) => { |_| () }
962 ));
963
964 named!(quoted_string -> (), delimited!(
965 punct!("\""),
966 cooked_string,
967 tag!("\"")
968 ));
969
cooked_string(input: Cursor) -> PResult<()>970 fn cooked_string(input: Cursor) -> PResult<()> {
971 let mut chars = input.char_indices().peekable();
972 while let Some((byte_offset, ch)) = chars.next() {
973 match ch {
974 '"' => {
975 return Ok((input.advance(byte_offset), ()));
976 }
977 '\r' => {
978 if let Some((_, '\n')) = chars.next() {
979 // ...
980 } else {
981 break;
982 }
983 }
984 '\\' => match chars.next() {
985 Some((_, 'x')) => {
986 if !backslash_x_char(&mut chars) {
987 break;
988 }
989 }
990 Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
991 | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
992 Some((_, 'u')) => {
993 if !backslash_u(&mut chars) {
994 break;
995 }
996 }
997 Some((_, '\n')) | Some((_, '\r')) => {
998 while let Some(&(_, ch)) = chars.peek() {
999 if ch.is_whitespace() {
1000 chars.next();
1001 } else {
1002 break;
1003 }
1004 }
1005 }
1006 _ => break,
1007 },
1008 _ch => {}
1009 }
1010 }
1011 Err(LexError)
1012 }
1013
1014 named!(byte_string -> (), alt!(
1015 delimited!(
1016 punct!("b\""),
1017 cooked_byte_string,
1018 tag!("\"")
1019 ) => { |_| () }
1020 |
1021 preceded!(
1022 punct!("br"),
1023 raw_string
1024 ) => { |_| () }
1025 ));
1026
cooked_byte_string(mut input: Cursor) -> PResult<()>1027 fn cooked_byte_string(mut input: Cursor) -> PResult<()> {
1028 let mut bytes = input.bytes().enumerate();
1029 'outer: while let Some((offset, b)) = bytes.next() {
1030 match b {
1031 b'"' => {
1032 return Ok((input.advance(offset), ()));
1033 }
1034 b'\r' => {
1035 if let Some((_, b'\n')) = bytes.next() {
1036 // ...
1037 } else {
1038 break;
1039 }
1040 }
1041 b'\\' => match bytes.next() {
1042 Some((_, b'x')) => {
1043 if !backslash_x_byte(&mut bytes) {
1044 break;
1045 }
1046 }
1047 Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
1048 | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
1049 Some((newline, b'\n')) | Some((newline, b'\r')) => {
1050 let rest = input.advance(newline + 1);
1051 for (offset, ch) in rest.char_indices() {
1052 if !ch.is_whitespace() {
1053 input = rest.advance(offset);
1054 bytes = input.bytes().enumerate();
1055 continue 'outer;
1056 }
1057 }
1058 break;
1059 }
1060 _ => break,
1061 },
1062 b if b < 0x80 => {}
1063 _ => break,
1064 }
1065 }
1066 Err(LexError)
1067 }
1068
raw_string(input: Cursor) -> PResult<()>1069 fn raw_string(input: Cursor) -> PResult<()> {
1070 let mut chars = input.char_indices();
1071 let mut n = 0;
1072 while let Some((byte_offset, ch)) = chars.next() {
1073 match ch {
1074 '"' => {
1075 n = byte_offset;
1076 break;
1077 }
1078 '#' => {}
1079 _ => return Err(LexError),
1080 }
1081 }
1082 for (byte_offset, ch) in chars {
1083 match ch {
1084 '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => {
1085 let rest = input.advance(byte_offset + 1 + n);
1086 return Ok((rest, ()));
1087 }
1088 '\r' => {}
1089 _ => {}
1090 }
1091 }
1092 Err(LexError)
1093 }
1094
1095 named!(byte -> (), do_parse!(
1096 punct!("b") >>
1097 tag!("'") >>
1098 cooked_byte >>
1099 tag!("'") >>
1100 (())
1101 ));
1102
cooked_byte(input: Cursor) -> PResult<()>1103 fn cooked_byte(input: Cursor) -> PResult<()> {
1104 let mut bytes = input.bytes().enumerate();
1105 let ok = match bytes.next().map(|(_, b)| b) {
1106 Some(b'\\') => match bytes.next().map(|(_, b)| b) {
1107 Some(b'x') => backslash_x_byte(&mut bytes),
1108 Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
1109 | Some(b'"') => true,
1110 _ => false,
1111 },
1112 b => b.is_some(),
1113 };
1114 if ok {
1115 match bytes.next() {
1116 Some((offset, _)) => {
1117 if input.chars().as_str().is_char_boundary(offset) {
1118 Ok((input.advance(offset), ()))
1119 } else {
1120 Err(LexError)
1121 }
1122 }
1123 None => Ok((input.advance(input.len()), ())),
1124 }
1125 } else {
1126 Err(LexError)
1127 }
1128 }
1129
1130 named!(character -> (), do_parse!(
1131 punct!("'") >>
1132 cooked_char >>
1133 tag!("'") >>
1134 (())
1135 ));
1136
cooked_char(input: Cursor) -> PResult<()>1137 fn cooked_char(input: Cursor) -> PResult<()> {
1138 let mut chars = input.char_indices();
1139 let ok = match chars.next().map(|(_, ch)| ch) {
1140 Some('\\') => match chars.next().map(|(_, ch)| ch) {
1141 Some('x') => backslash_x_char(&mut chars),
1142 Some('u') => backslash_u(&mut chars),
1143 Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
1144 true
1145 }
1146 _ => false,
1147 },
1148 ch => ch.is_some(),
1149 };
1150 if ok {
1151 match chars.next() {
1152 Some((idx, _)) => Ok((input.advance(idx), ())),
1153 None => Ok((input.advance(input.len()), ())),
1154 }
1155 } else {
1156 Err(LexError)
1157 }
1158 }
1159
1160 macro_rules! next_ch {
1161 ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
1162 match $chars.next() {
1163 Some((_, ch)) => match ch {
1164 $pat $(| $rest)* => ch,
1165 _ => return false,
1166 },
1167 None => return false
1168 }
1169 };
1170 }
1171
backslash_x_char<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, char)>,1172 fn backslash_x_char<I>(chars: &mut I) -> bool
1173 where
1174 I: Iterator<Item = (usize, char)>,
1175 {
1176 next_ch!(chars @ '0'...'7');
1177 next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
1178 true
1179 }
1180
backslash_x_byte<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, u8)>,1181 fn backslash_x_byte<I>(chars: &mut I) -> bool
1182 where
1183 I: Iterator<Item = (usize, u8)>,
1184 {
1185 next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
1186 next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
1187 true
1188 }
1189
backslash_u<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, char)>,1190 fn backslash_u<I>(chars: &mut I) -> bool
1191 where
1192 I: Iterator<Item = (usize, char)>,
1193 {
1194 next_ch!(chars @ '{');
1195 next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
1196 loop {
1197 let c = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '_' | '}');
1198 if c == '}' {
1199 return true;
1200 }
1201 }
1202 }
1203
float(input: Cursor) -> PResult<()>1204 fn float(input: Cursor) -> PResult<()> {
1205 let (rest, ()) = float_digits(input)?;
1206 for suffix in &["f32", "f64"] {
1207 if rest.starts_with(suffix) {
1208 return word_break(rest.advance(suffix.len()));
1209 }
1210 }
1211 word_break(rest)
1212 }
1213
float_digits(input: Cursor) -> PResult<()>1214 fn float_digits(input: Cursor) -> PResult<()> {
1215 let mut chars = input.chars().peekable();
1216 match chars.next() {
1217 Some(ch) if ch >= '0' && ch <= '9' => {}
1218 _ => return Err(LexError),
1219 }
1220
1221 let mut len = 1;
1222 let mut has_dot = false;
1223 let mut has_exp = false;
1224 while let Some(&ch) = chars.peek() {
1225 match ch {
1226 '0'...'9' | '_' => {
1227 chars.next();
1228 len += 1;
1229 }
1230 '.' => {
1231 if has_dot {
1232 break;
1233 }
1234 chars.next();
1235 if chars
1236 .peek()
1237 .map(|&ch| ch == '.' || UnicodeXID::is_xid_start(ch))
1238 .unwrap_or(false)
1239 {
1240 return Err(LexError);
1241 }
1242 len += 1;
1243 has_dot = true;
1244 }
1245 'e' | 'E' => {
1246 chars.next();
1247 len += 1;
1248 has_exp = true;
1249 break;
1250 }
1251 _ => break,
1252 }
1253 }
1254
1255 let rest = input.advance(len);
1256 if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
1257 return Err(LexError);
1258 }
1259
1260 if has_exp {
1261 let mut has_exp_value = false;
1262 while let Some(&ch) = chars.peek() {
1263 match ch {
1264 '+' | '-' => {
1265 if has_exp_value {
1266 break;
1267 }
1268 chars.next();
1269 len += 1;
1270 }
1271 '0'...'9' => {
1272 chars.next();
1273 len += 1;
1274 has_exp_value = true;
1275 }
1276 '_' => {
1277 chars.next();
1278 len += 1;
1279 }
1280 _ => break,
1281 }
1282 }
1283 if !has_exp_value {
1284 return Err(LexError);
1285 }
1286 }
1287
1288 Ok((input.advance(len), ()))
1289 }
1290
int(input: Cursor) -> PResult<()>1291 fn int(input: Cursor) -> PResult<()> {
1292 let (rest, ()) = digits(input)?;
1293 for suffix in &[
1294 "isize", "i8", "i16", "i32", "i64", "i128", "usize", "u8", "u16", "u32", "u64", "u128",
1295 ] {
1296 if rest.starts_with(suffix) {
1297 return word_break(rest.advance(suffix.len()));
1298 }
1299 }
1300 word_break(rest)
1301 }
1302
digits(mut input: Cursor) -> PResult<()>1303 fn digits(mut input: Cursor) -> PResult<()> {
1304 let base = if input.starts_with("0x") {
1305 input = input.advance(2);
1306 16
1307 } else if input.starts_with("0o") {
1308 input = input.advance(2);
1309 8
1310 } else if input.starts_with("0b") {
1311 input = input.advance(2);
1312 2
1313 } else {
1314 10
1315 };
1316
1317 let mut len = 0;
1318 let mut empty = true;
1319 for b in input.bytes() {
1320 let digit = match b {
1321 b'0'...b'9' => (b - b'0') as u64,
1322 b'a'...b'f' => 10 + (b - b'a') as u64,
1323 b'A'...b'F' => 10 + (b - b'A') as u64,
1324 b'_' => {
1325 if empty && base == 10 {
1326 return Err(LexError);
1327 }
1328 len += 1;
1329 continue;
1330 }
1331 _ => break,
1332 };
1333 if digit >= base {
1334 return Err(LexError);
1335 }
1336 len += 1;
1337 empty = false;
1338 }
1339 if empty {
1340 Err(LexError)
1341 } else {
1342 Ok((input.advance(len), ()))
1343 }
1344 }
1345
op(input: Cursor) -> PResult<Punct>1346 fn op(input: Cursor) -> PResult<Punct> {
1347 let input = skip_whitespace(input);
1348 match op_char(input) {
1349 Ok((rest, '\'')) => {
1350 symbol(rest)?;
1351 Ok((rest, Punct::new('\'', Spacing::Joint)))
1352 }
1353 Ok((rest, ch)) => {
1354 let kind = match op_char(rest) {
1355 Ok(_) => Spacing::Joint,
1356 Err(LexError) => Spacing::Alone,
1357 };
1358 Ok((rest, Punct::new(ch, kind)))
1359 }
1360 Err(LexError) => Err(LexError),
1361 }
1362 }
1363
op_char(input: Cursor) -> PResult<char>1364 fn op_char(input: Cursor) -> PResult<char> {
1365 if input.starts_with("//") || input.starts_with("/*") {
1366 // Do not accept `/` of a comment as an op.
1367 return Err(LexError);
1368 }
1369
1370 let mut chars = input.chars();
1371 let first = match chars.next() {
1372 Some(ch) => ch,
1373 None => {
1374 return Err(LexError);
1375 }
1376 };
1377 let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
1378 if recognized.contains(first) {
1379 Ok((input.advance(first.len_utf8()), first))
1380 } else {
1381 Err(LexError)
1382 }
1383 }
1384
doc_comment(input: Cursor) -> PResult<Vec<TokenTree>>1385 fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
1386 let mut trees = Vec::new();
1387 let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
1388 trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone)));
1389 if inner {
1390 trees.push(Punct::new('!', Spacing::Alone).into());
1391 }
1392 let mut stream = vec![
1393 TokenTree::Ident(::Ident::new("doc", span)),
1394 TokenTree::Punct(Punct::new('=', Spacing::Alone)),
1395 TokenTree::Literal(::Literal::string(comment)),
1396 ];
1397 for tt in stream.iter_mut() {
1398 tt.set_span(span);
1399 }
1400 let group = Group::new(Delimiter::Bracket, stream.into_iter().collect());
1401 trees.push(::Group::_new_stable(group).into());
1402 for tt in trees.iter_mut() {
1403 tt.set_span(span);
1404 }
1405 Ok((rest, trees))
1406 }
1407
1408 named!(doc_comment_contents -> (&str, bool), alt!(
1409 do_parse!(
1410 punct!("//!") >>
1411 s: take_until_newline_or_eof!() >>
1412 ((s, true))
1413 )
1414 |
1415 do_parse!(
1416 option!(whitespace) >>
1417 peek!(tag!("/*!")) >>
1418 s: block_comment >>
1419 ((s, true))
1420 )
1421 |
1422 do_parse!(
1423 punct!("///") >>
1424 not!(tag!("/")) >>
1425 s: take_until_newline_or_eof!() >>
1426 ((s, false))
1427 )
1428 |
1429 do_parse!(
1430 option!(whitespace) >>
1431 peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
1432 s: block_comment >>
1433 ((s, false))
1434 )
1435 ));
1436