1 #![cfg_attr(not(procmacro2_semver_exempt), allow(dead_code))]
2
3 use std::borrow::Borrow;
4 use std::cell::RefCell;
5 #[cfg(procmacro2_semver_exempt)]
6 use std::cmp;
7 use std::collections::HashMap;
8 use std::fmt;
9 use std::iter;
10 use std::rc::Rc;
11 use std::str::FromStr;
12 use std::vec;
13
14 use strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
15 use unicode_xid::UnicodeXID;
16
17 use {Delimiter, Group, Op, Spacing, TokenTree};
18
19 #[derive(Clone, Debug)]
20 pub struct TokenStream {
21 inner: Vec<TokenTree>,
22 }
23
24 #[derive(Debug)]
25 pub struct LexError;
26
27 impl TokenStream {
empty() -> TokenStream28 pub fn empty() -> TokenStream {
29 TokenStream { inner: Vec::new() }
30 }
31
is_empty(&self) -> bool32 pub fn is_empty(&self) -> bool {
33 self.inner.len() == 0
34 }
35 }
36
37 #[cfg(procmacro2_semver_exempt)]
get_cursor(src: &str) -> Cursor38 fn get_cursor(src: &str) -> Cursor {
39 // Create a dummy file & add it to the codemap
40 CODEMAP.with(|cm| {
41 let mut cm = cm.borrow_mut();
42 let name = format!("<parsed string {}>", cm.files.len());
43 let span = cm.add_file(&name, src);
44 Cursor {
45 rest: src,
46 off: span.lo,
47 }
48 })
49 }
50
51 #[cfg(not(procmacro2_semver_exempt))]
get_cursor(src: &str) -> Cursor52 fn get_cursor(src: &str) -> Cursor {
53 Cursor { rest: src }
54 }
55
56 impl FromStr for TokenStream {
57 type Err = LexError;
58
from_str(src: &str) -> Result<TokenStream, LexError>59 fn from_str(src: &str) -> Result<TokenStream, LexError> {
60 // Create a dummy file & add it to the codemap
61 let cursor = get_cursor(src);
62
63 match token_stream(cursor) {
64 Ok((input, output)) => {
65 if skip_whitespace(input).len() != 0 {
66 Err(LexError)
67 } else {
68 Ok(output.inner)
69 }
70 }
71 Err(LexError) => Err(LexError),
72 }
73 }
74 }
75
76 impl fmt::Display for TokenStream {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result77 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
78 let mut joint = false;
79 for (i, tt) in self.inner.iter().enumerate() {
80 if i != 0 && !joint {
81 write!(f, " ")?;
82 }
83 joint = false;
84 match *tt {
85 TokenTree::Group(ref tt) => {
86 let (start, end) = match tt.delimiter() {
87 Delimiter::Parenthesis => ("(", ")"),
88 Delimiter::Brace => ("{", "}"),
89 Delimiter::Bracket => ("[", "]"),
90 Delimiter::None => ("", ""),
91 };
92 if tt.stream().inner.inner.len() == 0 {
93 write!(f, "{} {}", start, end)?
94 } else {
95 write!(f, "{} {} {}", start, tt.stream(), end)?
96 }
97 }
98 TokenTree::Term(ref tt) => write!(f, "{}", tt.as_str())?,
99 TokenTree::Op(ref tt) => {
100 write!(f, "{}", tt.op())?;
101 match tt.spacing() {
102 Spacing::Alone => {}
103 Spacing::Joint => joint = true,
104 }
105 }
106 TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
107 }
108 }
109
110 Ok(())
111 }
112 }
113
114 #[cfg(feature = "proc-macro")]
115 impl From<::proc_macro::TokenStream> for TokenStream {
from(inner: ::proc_macro::TokenStream) -> TokenStream116 fn from(inner: ::proc_macro::TokenStream) -> TokenStream {
117 inner
118 .to_string()
119 .parse()
120 .expect("compiler token stream parse failed")
121 }
122 }
123
124 #[cfg(feature = "proc-macro")]
125 impl From<TokenStream> for ::proc_macro::TokenStream {
from(inner: TokenStream) -> ::proc_macro::TokenStream126 fn from(inner: TokenStream) -> ::proc_macro::TokenStream {
127 inner
128 .to_string()
129 .parse()
130 .expect("failed to parse to compiler tokens")
131 }
132 }
133
134 impl From<TokenTree> for TokenStream {
from(tree: TokenTree) -> TokenStream135 fn from(tree: TokenTree) -> TokenStream {
136 TokenStream { inner: vec![tree] }
137 }
138 }
139
140 impl iter::FromIterator<TokenTree> for TokenStream {
from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self141 fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
142 let mut v = Vec::new();
143
144 for token in streams.into_iter() {
145 v.push(token);
146 }
147
148 TokenStream { inner: v }
149 }
150 }
151
152 pub type TokenTreeIter = vec::IntoIter<TokenTree>;
153
154 impl IntoIterator for TokenStream {
155 type Item = TokenTree;
156 type IntoIter = TokenTreeIter;
157
into_iter(self) -> TokenTreeIter158 fn into_iter(self) -> TokenTreeIter {
159 self.inner.into_iter()
160 }
161 }
162
163 #[cfg(procmacro2_semver_exempt)]
164 #[derive(Clone, PartialEq, Eq, Debug)]
165 pub struct FileName(String);
166
167 #[cfg(procmacro2_semver_exempt)]
168 impl fmt::Display for FileName {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result169 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
170 self.0.fmt(f)
171 }
172 }
173
174 #[cfg(procmacro2_semver_exempt)]
175 #[derive(Clone, PartialEq, Eq)]
176 pub struct SourceFile {
177 name: FileName,
178 }
179
180 #[cfg(procmacro2_semver_exempt)]
181 impl SourceFile {
182 /// Get the path to this source file as a string.
path(&self) -> &FileName183 pub fn path(&self) -> &FileName {
184 &self.name
185 }
186
is_real(&self) -> bool187 pub fn is_real(&self) -> bool {
188 // XXX(nika): Support real files in the future?
189 false
190 }
191 }
192
193 #[cfg(procmacro2_semver_exempt)]
194 impl AsRef<FileName> for SourceFile {
as_ref(&self) -> &FileName195 fn as_ref(&self) -> &FileName {
196 self.path()
197 }
198 }
199
200 #[cfg(procmacro2_semver_exempt)]
201 impl fmt::Debug for SourceFile {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result202 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
203 f.debug_struct("SourceFile")
204 .field("path", &self.path())
205 .field("is_real", &self.is_real())
206 .finish()
207 }
208 }
209
210 #[cfg(procmacro2_semver_exempt)]
211 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
212 pub struct LineColumn {
213 pub line: usize,
214 pub column: usize,
215 }
216
217 #[cfg(procmacro2_semver_exempt)]
218 thread_local! {
219 static CODEMAP: RefCell<Codemap> = RefCell::new(Codemap {
220 // NOTE: We start with a single dummy file which all call_site() and
221 // def_site() spans reference.
222 files: vec![FileInfo {
223 name: "<unspecified>".to_owned(),
224 span: Span { lo: 0, hi: 0 },
225 lines: vec![0],
226 }],
227 });
228 }
229
230 #[cfg(procmacro2_semver_exempt)]
231 struct FileInfo {
232 name: String,
233 span: Span,
234 lines: Vec<usize>,
235 }
236
237 #[cfg(procmacro2_semver_exempt)]
238 impl FileInfo {
offset_line_column(&self, offset: usize) -> LineColumn239 fn offset_line_column(&self, offset: usize) -> LineColumn {
240 assert!(self.span_within(Span {
241 lo: offset as u32,
242 hi: offset as u32
243 }));
244 let offset = offset - self.span.lo as usize;
245 match self.lines.binary_search(&offset) {
246 Ok(found) => LineColumn {
247 line: found + 1,
248 column: 0,
249 },
250 Err(idx) => LineColumn {
251 line: idx,
252 column: offset - self.lines[idx - 1],
253 },
254 }
255 }
256
span_within(&self, span: Span) -> bool257 fn span_within(&self, span: Span) -> bool {
258 span.lo >= self.span.lo && span.hi <= self.span.hi
259 }
260 }
261
262 /// Computesthe offsets of each line in the given source string.
263 #[cfg(procmacro2_semver_exempt)]
lines_offsets(s: &str) -> Vec<usize>264 fn lines_offsets(s: &str) -> Vec<usize> {
265 let mut lines = vec![0];
266 let mut prev = 0;
267 while let Some(len) = s[prev..].find('\n') {
268 prev += len + 1;
269 lines.push(prev);
270 }
271 lines
272 }
273
274 #[cfg(procmacro2_semver_exempt)]
275 struct Codemap {
276 files: Vec<FileInfo>,
277 }
278
279 #[cfg(procmacro2_semver_exempt)]
280 impl Codemap {
next_start_pos(&self) -> u32281 fn next_start_pos(&self) -> u32 {
282 // Add 1 so there's always space between files.
283 //
284 // We'll always have at least 1 file, as we initialize our files list
285 // with a dummy file.
286 self.files.last().unwrap().span.hi + 1
287 }
288
add_file(&mut self, name: &str, src: &str) -> Span289 fn add_file(&mut self, name: &str, src: &str) -> Span {
290 let lines = lines_offsets(src);
291 let lo = self.next_start_pos();
292 // XXX(nika): Shouild we bother doing a checked cast or checked add here?
293 let span = Span {
294 lo: lo,
295 hi: lo + (src.len() as u32),
296 };
297
298 self.files.push(FileInfo {
299 name: name.to_owned(),
300 span: span,
301 lines: lines,
302 });
303
304 span
305 }
306
fileinfo(&self, span: Span) -> &FileInfo307 fn fileinfo(&self, span: Span) -> &FileInfo {
308 for file in &self.files {
309 if file.span_within(span) {
310 return file;
311 }
312 }
313 panic!("Invalid span with no related FileInfo!");
314 }
315 }
316
317 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
318 pub struct Span {
319 #[cfg(procmacro2_semver_exempt)]
320 lo: u32,
321 #[cfg(procmacro2_semver_exempt)]
322 hi: u32,
323 }
324
325 impl Span {
326 #[cfg(not(procmacro2_semver_exempt))]
call_site() -> Span327 pub fn call_site() -> Span {
328 Span {}
329 }
330
331 #[cfg(procmacro2_semver_exempt)]
call_site() -> Span332 pub fn call_site() -> Span {
333 Span { lo: 0, hi: 0 }
334 }
335
def_site() -> Span336 pub fn def_site() -> Span {
337 Span::call_site()
338 }
339
resolved_at(&self, _other: Span) -> Span340 pub fn resolved_at(&self, _other: Span) -> Span {
341 // Stable spans consist only of line/column information, so
342 // `resolved_at` and `located_at` only select which span the
343 // caller wants line/column information from.
344 *self
345 }
346
located_at(&self, other: Span) -> Span347 pub fn located_at(&self, other: Span) -> Span {
348 other
349 }
350
351 #[cfg(procmacro2_semver_exempt)]
source_file(&self) -> SourceFile352 pub fn source_file(&self) -> SourceFile {
353 CODEMAP.with(|cm| {
354 let cm = cm.borrow();
355 let fi = cm.fileinfo(*self);
356 SourceFile {
357 name: FileName(fi.name.clone()),
358 }
359 })
360 }
361
362 #[cfg(procmacro2_semver_exempt)]
start(&self) -> LineColumn363 pub fn start(&self) -> LineColumn {
364 CODEMAP.with(|cm| {
365 let cm = cm.borrow();
366 let fi = cm.fileinfo(*self);
367 fi.offset_line_column(self.lo as usize)
368 })
369 }
370
371 #[cfg(procmacro2_semver_exempt)]
end(&self) -> LineColumn372 pub fn end(&self) -> LineColumn {
373 CODEMAP.with(|cm| {
374 let cm = cm.borrow();
375 let fi = cm.fileinfo(*self);
376 fi.offset_line_column(self.hi as usize)
377 })
378 }
379
380 #[cfg(procmacro2_semver_exempt)]
join(&self, other: Span) -> Option<Span>381 pub fn join(&self, other: Span) -> Option<Span> {
382 CODEMAP.with(|cm| {
383 let cm = cm.borrow();
384 // If `other` is not within the same FileInfo as us, return None.
385 if !cm.fileinfo(*self).span_within(other) {
386 return None;
387 }
388 Some(Span {
389 lo: cmp::min(self.lo, other.lo),
390 hi: cmp::max(self.hi, other.hi),
391 })
392 })
393 }
394 }
395
396 #[derive(Copy, Clone)]
397 pub struct Term {
398 intern: usize,
399 span: Span,
400 }
401
402 thread_local!(static SYMBOLS: RefCell<Interner> = RefCell::new(Interner::new()));
403
404 impl Term {
new(string: &str, span: Span) -> Term405 pub fn new(string: &str, span: Span) -> Term {
406 Term {
407 intern: SYMBOLS.with(|s| s.borrow_mut().intern(string)),
408 span: span,
409 }
410 }
411
as_str(&self) -> &str412 pub fn as_str(&self) -> &str {
413 SYMBOLS.with(|interner| {
414 let interner = interner.borrow();
415 let s = interner.get(self.intern);
416 unsafe { &*(s as *const str) }
417 })
418 }
419
span(&self) -> Span420 pub fn span(&self) -> Span {
421 self.span
422 }
423
set_span(&mut self, span: Span)424 pub fn set_span(&mut self, span: Span) {
425 self.span = span;
426 }
427 }
428
429 impl fmt::Debug for Term {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result430 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
431 f.debug_tuple("Term").field(&self.as_str()).finish()
432 }
433 }
434
435 struct Interner {
436 string_to_index: HashMap<MyRc, usize>,
437 index_to_string: Vec<Rc<String>>,
438 }
439
440 #[derive(Hash, Eq, PartialEq)]
441 struct MyRc(Rc<String>);
442
443 impl Borrow<str> for MyRc {
borrow(&self) -> &str444 fn borrow(&self) -> &str {
445 &self.0
446 }
447 }
448
449 impl Interner {
new() -> Interner450 fn new() -> Interner {
451 Interner {
452 string_to_index: HashMap::new(),
453 index_to_string: Vec::new(),
454 }
455 }
456
intern(&mut self, s: &str) -> usize457 fn intern(&mut self, s: &str) -> usize {
458 if let Some(&idx) = self.string_to_index.get(s) {
459 return idx;
460 }
461 let s = Rc::new(s.to_string());
462 self.index_to_string.push(s.clone());
463 self.string_to_index
464 .insert(MyRc(s), self.index_to_string.len() - 1);
465 self.index_to_string.len() - 1
466 }
467
get(&self, idx: usize) -> &str468 fn get(&self, idx: usize) -> &str {
469 &self.index_to_string[idx]
470 }
471 }
472
473 #[derive(Clone, Debug)]
474 pub struct Literal {
475 text: String,
476 span: Span,
477 }
478
479 macro_rules! suffixed_numbers {
480 ($($name:ident => $kind:ident,)*) => ($(
481 pub fn $name(n: $kind) -> Literal {
482 Literal::_new(format!(concat!("{}", stringify!($kind)), n))
483 }
484 )*)
485 }
486
487 macro_rules! unsuffixed_numbers {
488 ($($name:ident => $kind:ident,)*) => ($(
489 pub fn $name(n: $kind) -> Literal {
490 Literal::_new(n.to_string())
491 }
492 )*)
493 }
494
495 impl Literal {
_new(text: String) -> Literal496 fn _new(text: String) -> Literal {
497 Literal {
498 text: text,
499 span: Span::call_site(),
500 }
501 }
502
503 suffixed_numbers! {
504 u8_suffixed => u8,
505 u16_suffixed => u16,
506 u32_suffixed => u32,
507 u64_suffixed => u64,
508 usize_suffixed => usize,
509 i8_suffixed => i8,
510 i16_suffixed => i16,
511 i32_suffixed => i32,
512 i64_suffixed => i64,
513 isize_suffixed => isize,
514
515 f32_suffixed => f32,
516 f64_suffixed => f64,
517 }
518
519 unsuffixed_numbers! {
520 u8_unsuffixed => u8,
521 u16_unsuffixed => u16,
522 u32_unsuffixed => u32,
523 u64_unsuffixed => u64,
524 usize_unsuffixed => usize,
525 i8_unsuffixed => i8,
526 i16_unsuffixed => i16,
527 i32_unsuffixed => i32,
528 i64_unsuffixed => i64,
529 isize_unsuffixed => isize,
530 }
531
f32_unsuffixed(f: f32) -> Literal532 pub fn f32_unsuffixed(f: f32) -> Literal {
533 let mut s = f.to_string();
534 if !s.contains(".") {
535 s.push_str(".0");
536 }
537 Literal::_new(s)
538 }
539
f64_unsuffixed(f: f64) -> Literal540 pub fn f64_unsuffixed(f: f64) -> Literal {
541 let mut s = f.to_string();
542 if !s.contains(".") {
543 s.push_str(".0");
544 }
545 Literal::_new(s)
546 }
547
string(t: &str) -> Literal548 pub fn string(t: &str) -> Literal {
549 let mut s = t.chars()
550 .flat_map(|c| c.escape_default())
551 .collect::<String>();
552 s.push('"');
553 s.insert(0, '"');
554 Literal::_new(s)
555 }
556
character(t: char) -> Literal557 pub fn character(t: char) -> Literal {
558 Literal::_new(format!("'{}'", t.escape_default().collect::<String>()))
559 }
560
byte_string(bytes: &[u8]) -> Literal561 pub fn byte_string(bytes: &[u8]) -> Literal {
562 let mut escaped = "b\"".to_string();
563 for b in bytes {
564 match *b {
565 b'\0' => escaped.push_str(r"\0"),
566 b'\t' => escaped.push_str(r"\t"),
567 b'\n' => escaped.push_str(r"\n"),
568 b'\r' => escaped.push_str(r"\r"),
569 b'"' => escaped.push_str("\\\""),
570 b'\\' => escaped.push_str("\\\\"),
571 b'\x20'...b'\x7E' => escaped.push(*b as char),
572 _ => escaped.push_str(&format!("\\x{:02X}", b)),
573 }
574 }
575 escaped.push('"');
576 Literal::_new(escaped)
577 }
578
span(&self) -> Span579 pub fn span(&self) -> Span {
580 self.span
581 }
582
set_span(&mut self, span: Span)583 pub fn set_span(&mut self, span: Span) {
584 self.span = span;
585 }
586 }
587
588 impl fmt::Display for Literal {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result589 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
590 self.text.fmt(f)
591 }
592 }
593
token_stream(mut input: Cursor) -> PResult<::TokenStream>594 fn token_stream(mut input: Cursor) -> PResult<::TokenStream> {
595 let mut trees = Vec::new();
596 loop {
597 let input_no_ws = skip_whitespace(input);
598 if input_no_ws.rest.len() == 0 {
599 break
600 }
601 if let Ok((a, tokens)) = doc_comment(input_no_ws) {
602 input = a;
603 trees.extend(tokens);
604 continue
605 }
606
607 let (a, tt) = match token_tree(input_no_ws) {
608 Ok(p) => p,
609 Err(_) => break,
610 };
611 trees.push(tt);
612 input = a;
613 }
614 Ok((input, ::TokenStream::_new(TokenStream { inner: trees })))
615 }
616
617 #[cfg(not(procmacro2_semver_exempt))]
spanned<'a, T>( input: Cursor<'a>, f: fn(Cursor<'a>) -> PResult<'a, T>, ) -> PResult<'a, (T, ::Span)>618 fn spanned<'a, T>(
619 input: Cursor<'a>,
620 f: fn(Cursor<'a>) -> PResult<'a, T>,
621 ) -> PResult<'a, (T, ::Span)> {
622 let (a, b) = f(skip_whitespace(input))?;
623 Ok((a, ((b, ::Span::_new(Span { })))))
624 }
625
626 #[cfg(procmacro2_semver_exempt)]
spanned<'a, T>( input: Cursor<'a>, f: fn(Cursor<'a>) -> PResult<'a, T>, ) -> PResult<'a, (T, ::Span)>627 fn spanned<'a, T>(
628 input: Cursor<'a>,
629 f: fn(Cursor<'a>) -> PResult<'a, T>,
630 ) -> PResult<'a, (T, ::Span)> {
631 let input = skip_whitespace(input);
632 let lo = input.off;
633 let (a, b) = f(input)?;
634 let hi = a.off;
635 let span = ::Span::_new(Span { lo: lo, hi: hi });
636 Ok((a, (b, span)))
637 }
638
token_tree(input: Cursor) -> PResult<TokenTree>639 fn token_tree(input: Cursor) -> PResult<TokenTree> {
640 let (rest, (mut tt, span)) = spanned(input, token_kind)?;
641 tt.set_span(span);
642 Ok((rest, tt))
643 }
644
645 named!(token_kind -> TokenTree, alt!(
646 map!(group, TokenTree::Group)
647 |
648 map!(literal, TokenTree::Literal) // must be before symbol
649 |
650 symbol
651 |
652 map!(op, TokenTree::Op)
653 ));
654
655 named!(group -> Group, alt!(
656 delimited!(
657 punct!("("),
658 token_stream,
659 punct!(")")
660 ) => { |ts| Group::new(Delimiter::Parenthesis, ts) }
661 |
662 delimited!(
663 punct!("["),
664 token_stream,
665 punct!("]")
666 ) => { |ts| Group::new(Delimiter::Bracket, ts) }
667 |
668 delimited!(
669 punct!("{"),
670 token_stream,
671 punct!("}")
672 ) => { |ts| Group::new(Delimiter::Brace, ts) }
673 ));
674
symbol(mut input: Cursor) -> PResult<TokenTree>675 fn symbol(mut input: Cursor) -> PResult<TokenTree> {
676 input = skip_whitespace(input);
677
678 let mut chars = input.char_indices();
679
680 let lifetime = input.starts_with("'");
681 if lifetime {
682 chars.next();
683 }
684
685 let raw = !lifetime && input.starts_with("r#");
686 if raw {
687 chars.next();
688 chars.next();
689 }
690
691 match chars.next() {
692 Some((_, ch)) if UnicodeXID::is_xid_start(ch) || ch == '_' => {}
693 _ => return Err(LexError),
694 }
695
696 let mut end = input.len();
697 for (i, ch) in chars {
698 if !UnicodeXID::is_xid_continue(ch) {
699 end = i;
700 break;
701 }
702 }
703
704 let a = &input.rest[..end];
705 if a == "r#_" || lifetime && a != "'static" && KEYWORDS.contains(&&a[1..]) {
706 Err(LexError)
707 } else if a == "_" {
708 Ok((input.advance(end), Op::new('_', Spacing::Alone).into()))
709 } else {
710 Ok((
711 input.advance(end),
712 ::Term::new(a, ::Span::call_site()).into(),
713 ))
714 }
715 }
716
717 // From https://github.com/rust-lang/rust/blob/master/src/libsyntax_pos/symbol.rs
718 static KEYWORDS: &'static [&'static str] = &[
719 "abstract", "alignof", "as", "become", "box", "break", "const", "continue", "crate", "do",
720 "else", "enum", "extern", "false", "final", "fn", "for", "if", "impl", "in", "let", "loop",
721 "macro", "match", "mod", "move", "mut", "offsetof", "override", "priv", "proc", "pub", "pure",
722 "ref", "return", "self", "Self", "sizeof", "static", "struct", "super", "trait", "true",
723 "type", "typeof", "unsafe", "unsized", "use", "virtual", "where", "while", "yield",
724 ];
725
literal(input: Cursor) -> PResult<::Literal>726 fn literal(input: Cursor) -> PResult<::Literal> {
727 let input_no_ws = skip_whitespace(input);
728
729 match literal_nocapture(input_no_ws) {
730 Ok((a, ())) => {
731 let start = input.len() - input_no_ws.len();
732 let len = input_no_ws.len() - a.len();
733 let end = start + len;
734 Ok((
735 a,
736 ::Literal::_new(Literal::_new(input.rest[start..end].to_string())),
737 ))
738 }
739 Err(LexError) => Err(LexError),
740 }
741 }
742
743 named!(literal_nocapture -> (), alt!(
744 string
745 |
746 byte_string
747 |
748 byte
749 |
750 character
751 |
752 float
753 |
754 int
755 ));
756
757 named!(string -> (), alt!(
758 quoted_string
759 |
760 preceded!(
761 punct!("r"),
762 raw_string
763 ) => { |_| () }
764 ));
765
766 named!(quoted_string -> (), delimited!(
767 punct!("\""),
768 cooked_string,
769 tag!("\"")
770 ));
771
cooked_string(input: Cursor) -> PResult<()>772 fn cooked_string(input: Cursor) -> PResult<()> {
773 let mut chars = input.char_indices().peekable();
774 while let Some((byte_offset, ch)) = chars.next() {
775 match ch {
776 '"' => {
777 return Ok((input.advance(byte_offset), ()));
778 }
779 '\r' => {
780 if let Some((_, '\n')) = chars.next() {
781 // ...
782 } else {
783 break;
784 }
785 }
786 '\\' => match chars.next() {
787 Some((_, 'x')) => {
788 if !backslash_x_char(&mut chars) {
789 break;
790 }
791 }
792 Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
793 | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
794 Some((_, 'u')) => {
795 if !backslash_u(&mut chars) {
796 break;
797 }
798 }
799 Some((_, '\n')) | Some((_, '\r')) => {
800 while let Some(&(_, ch)) = chars.peek() {
801 if ch.is_whitespace() {
802 chars.next();
803 } else {
804 break;
805 }
806 }
807 }
808 _ => break,
809 },
810 _ch => {}
811 }
812 }
813 Err(LexError)
814 }
815
816 named!(byte_string -> (), alt!(
817 delimited!(
818 punct!("b\""),
819 cooked_byte_string,
820 tag!("\"")
821 ) => { |_| () }
822 |
823 preceded!(
824 punct!("br"),
825 raw_string
826 ) => { |_| () }
827 ));
828
cooked_byte_string(mut input: Cursor) -> PResult<()>829 fn cooked_byte_string(mut input: Cursor) -> PResult<()> {
830 let mut bytes = input.bytes().enumerate();
831 'outer: while let Some((offset, b)) = bytes.next() {
832 match b {
833 b'"' => {
834 return Ok((input.advance(offset), ()));
835 }
836 b'\r' => {
837 if let Some((_, b'\n')) = bytes.next() {
838 // ...
839 } else {
840 break;
841 }
842 }
843 b'\\' => match bytes.next() {
844 Some((_, b'x')) => {
845 if !backslash_x_byte(&mut bytes) {
846 break;
847 }
848 }
849 Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
850 | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
851 Some((newline, b'\n')) | Some((newline, b'\r')) => {
852 let rest = input.advance(newline + 1);
853 for (offset, ch) in rest.char_indices() {
854 if !ch.is_whitespace() {
855 input = rest.advance(offset);
856 bytes = input.bytes().enumerate();
857 continue 'outer;
858 }
859 }
860 break;
861 }
862 _ => break,
863 },
864 b if b < 0x80 => {}
865 _ => break,
866 }
867 }
868 Err(LexError)
869 }
870
raw_string(input: Cursor) -> PResult<()>871 fn raw_string(input: Cursor) -> PResult<()> {
872 let mut chars = input.char_indices();
873 let mut n = 0;
874 while let Some((byte_offset, ch)) = chars.next() {
875 match ch {
876 '"' => {
877 n = byte_offset;
878 break;
879 }
880 '#' => {}
881 _ => return Err(LexError),
882 }
883 }
884 for (byte_offset, ch) in chars {
885 match ch {
886 '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => {
887 let rest = input.advance(byte_offset + 1 + n);
888 return Ok((rest, ()));
889 }
890 '\r' => {}
891 _ => {}
892 }
893 }
894 Err(LexError)
895 }
896
897 named!(byte -> (), do_parse!(
898 punct!("b") >>
899 tag!("'") >>
900 cooked_byte >>
901 tag!("'") >>
902 (())
903 ));
904
cooked_byte(input: Cursor) -> PResult<()>905 fn cooked_byte(input: Cursor) -> PResult<()> {
906 let mut bytes = input.bytes().enumerate();
907 let ok = match bytes.next().map(|(_, b)| b) {
908 Some(b'\\') => match bytes.next().map(|(_, b)| b) {
909 Some(b'x') => backslash_x_byte(&mut bytes),
910 Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
911 | Some(b'"') => true,
912 _ => false,
913 },
914 b => b.is_some(),
915 };
916 if ok {
917 match bytes.next() {
918 Some((offset, _)) => {
919 if input.chars().as_str().is_char_boundary(offset) {
920 Ok((input.advance(offset), ()))
921 } else {
922 Err(LexError)
923 }
924 }
925 None => Ok((input.advance(input.len()), ())),
926 }
927 } else {
928 Err(LexError)
929 }
930 }
931
932 named!(character -> (), do_parse!(
933 punct!("'") >>
934 cooked_char >>
935 tag!("'") >>
936 (())
937 ));
938
cooked_char(input: Cursor) -> PResult<()>939 fn cooked_char(input: Cursor) -> PResult<()> {
940 let mut chars = input.char_indices();
941 let ok = match chars.next().map(|(_, ch)| ch) {
942 Some('\\') => match chars.next().map(|(_, ch)| ch) {
943 Some('x') => backslash_x_char(&mut chars),
944 Some('u') => backslash_u(&mut chars),
945 Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
946 true
947 }
948 _ => false,
949 },
950 ch => ch.is_some(),
951 };
952 if ok {
953 match chars.next() {
954 Some((idx, _)) => Ok((input.advance(idx), ())),
955 None => Ok((input.advance(input.len()), ())),
956 }
957 } else {
958 Err(LexError)
959 }
960 }
961
962 macro_rules! next_ch {
963 ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
964 match $chars.next() {
965 Some((_, ch)) => match ch {
966 $pat $(| $rest)* => ch,
967 _ => return false,
968 },
969 None => return false
970 }
971 };
972 }
973
backslash_x_char<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, char)>,974 fn backslash_x_char<I>(chars: &mut I) -> bool
975 where
976 I: Iterator<Item = (usize, char)>,
977 {
978 next_ch!(chars @ '0'...'7');
979 next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
980 true
981 }
982
backslash_x_byte<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, u8)>,983 fn backslash_x_byte<I>(chars: &mut I) -> bool
984 where
985 I: Iterator<Item = (usize, u8)>,
986 {
987 next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
988 next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
989 true
990 }
991
backslash_u<I>(chars: &mut I) -> bool where I: Iterator<Item = (usize, char)>,992 fn backslash_u<I>(chars: &mut I) -> bool
993 where
994 I: Iterator<Item = (usize, char)>,
995 {
996 next_ch!(chars @ '{');
997 next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
998 loop {
999 let c = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '_' | '}');
1000 if c == '}' {
1001 return true;
1002 }
1003 }
1004 }
1005
float(input: Cursor) -> PResult<()>1006 fn float(input: Cursor) -> PResult<()> {
1007 let (rest, ()) = float_digits(input)?;
1008 for suffix in &["f32", "f64"] {
1009 if rest.starts_with(suffix) {
1010 return word_break(rest.advance(suffix.len()));
1011 }
1012 }
1013 word_break(rest)
1014 }
1015
float_digits(input: Cursor) -> PResult<()>1016 fn float_digits(input: Cursor) -> PResult<()> {
1017 let mut chars = input.chars().peekable();
1018 match chars.next() {
1019 Some(ch) if ch >= '0' && ch <= '9' => {}
1020 _ => return Err(LexError),
1021 }
1022
1023 let mut len = 1;
1024 let mut has_dot = false;
1025 let mut has_exp = false;
1026 while let Some(&ch) = chars.peek() {
1027 match ch {
1028 '0'...'9' | '_' => {
1029 chars.next();
1030 len += 1;
1031 }
1032 '.' => {
1033 if has_dot {
1034 break;
1035 }
1036 chars.next();
1037 if chars
1038 .peek()
1039 .map(|&ch| ch == '.' || UnicodeXID::is_xid_start(ch))
1040 .unwrap_or(false)
1041 {
1042 return Err(LexError);
1043 }
1044 len += 1;
1045 has_dot = true;
1046 }
1047 'e' | 'E' => {
1048 chars.next();
1049 len += 1;
1050 has_exp = true;
1051 break;
1052 }
1053 _ => break,
1054 }
1055 }
1056
1057 let rest = input.advance(len);
1058 if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
1059 return Err(LexError);
1060 }
1061
1062 if has_exp {
1063 let mut has_exp_value = false;
1064 while let Some(&ch) = chars.peek() {
1065 match ch {
1066 '+' | '-' => {
1067 if has_exp_value {
1068 break;
1069 }
1070 chars.next();
1071 len += 1;
1072 }
1073 '0'...'9' => {
1074 chars.next();
1075 len += 1;
1076 has_exp_value = true;
1077 }
1078 '_' => {
1079 chars.next();
1080 len += 1;
1081 }
1082 _ => break,
1083 }
1084 }
1085 if !has_exp_value {
1086 return Err(LexError);
1087 }
1088 }
1089
1090 Ok((input.advance(len), ()))
1091 }
1092
int(input: Cursor) -> PResult<()>1093 fn int(input: Cursor) -> PResult<()> {
1094 let (rest, ()) = digits(input)?;
1095 for suffix in &[
1096 "isize", "i8", "i16", "i32", "i64", "i128", "usize", "u8", "u16", "u32", "u64", "u128"
1097 ] {
1098 if rest.starts_with(suffix) {
1099 return word_break(rest.advance(suffix.len()));
1100 }
1101 }
1102 word_break(rest)
1103 }
1104
digits(mut input: Cursor) -> PResult<()>1105 fn digits(mut input: Cursor) -> PResult<()> {
1106 let base = if input.starts_with("0x") {
1107 input = input.advance(2);
1108 16
1109 } else if input.starts_with("0o") {
1110 input = input.advance(2);
1111 8
1112 } else if input.starts_with("0b") {
1113 input = input.advance(2);
1114 2
1115 } else {
1116 10
1117 };
1118
1119 let mut len = 0;
1120 let mut empty = true;
1121 for b in input.bytes() {
1122 let digit = match b {
1123 b'0'...b'9' => (b - b'0') as u64,
1124 b'a'...b'f' => 10 + (b - b'a') as u64,
1125 b'A'...b'F' => 10 + (b - b'A') as u64,
1126 b'_' => {
1127 if empty && base == 10 {
1128 return Err(LexError);
1129 }
1130 len += 1;
1131 continue;
1132 }
1133 _ => break,
1134 };
1135 if digit >= base {
1136 return Err(LexError);
1137 }
1138 len += 1;
1139 empty = false;
1140 }
1141 if empty {
1142 Err(LexError)
1143 } else {
1144 Ok((input.advance(len), ()))
1145 }
1146 }
1147
op(input: Cursor) -> PResult<Op>1148 fn op(input: Cursor) -> PResult<Op> {
1149 let input = skip_whitespace(input);
1150 match op_char(input) {
1151 Ok((rest, ch)) => {
1152 let kind = match op_char(rest) {
1153 Ok(_) => Spacing::Joint,
1154 Err(LexError) => Spacing::Alone,
1155 };
1156 Ok((rest, Op::new(ch, kind)))
1157 }
1158 Err(LexError) => Err(LexError),
1159 }
1160 }
1161
op_char(input: Cursor) -> PResult<char>1162 fn op_char(input: Cursor) -> PResult<char> {
1163 let mut chars = input.chars();
1164 let first = match chars.next() {
1165 Some(ch) => ch,
1166 None => {
1167 return Err(LexError);
1168 }
1169 };
1170 let recognized = "~!@#$%^&*-=+|;:,<.>/?";
1171 if recognized.contains(first) {
1172 Ok((input.advance(first.len_utf8()), first))
1173 } else {
1174 Err(LexError)
1175 }
1176 }
1177
doc_comment(input: Cursor) -> PResult<Vec<TokenTree>>1178 fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
1179 let mut trees = Vec::new();
1180 let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
1181 trees.push(TokenTree::Op(Op::new('#', Spacing::Alone)));
1182 if inner {
1183 trees.push(Op::new('!', Spacing::Alone).into());
1184 }
1185 let mut stream = vec![
1186 TokenTree::Term(::Term::new("doc", span)),
1187 TokenTree::Op(Op::new('=', Spacing::Alone)),
1188 TokenTree::Literal(::Literal::string(comment)),
1189 ];
1190 for tt in stream.iter_mut() {
1191 tt.set_span(span);
1192 }
1193 trees.push(Group::new(Delimiter::Bracket, stream.into_iter().collect()).into());
1194 for tt in trees.iter_mut() {
1195 tt.set_span(span);
1196 }
1197 Ok((rest, trees))
1198 }
1199
1200 named!(doc_comment_contents -> (&str, bool), alt!(
1201 do_parse!(
1202 punct!("//!") >>
1203 s: take_until_newline_or_eof!() >>
1204 ((s, true))
1205 )
1206 |
1207 do_parse!(
1208 option!(whitespace) >>
1209 peek!(tag!("/*!")) >>
1210 s: block_comment >>
1211 ((s, true))
1212 )
1213 |
1214 do_parse!(
1215 punct!("///") >>
1216 not!(tag!("/")) >>
1217 s: take_until_newline_or_eof!() >>
1218 ((s, false))
1219 )
1220 |
1221 do_parse!(
1222 option!(whitespace) >>
1223 peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
1224 s: block_comment >>
1225 ((s, false))
1226 )
1227 ));
1228