1 use proc_macro2::{Literal, Span};
2 use std::str;
3
4 #[cfg(feature = "printing")]
5 use proc_macro2::Ident;
6
7 #[cfg(feature = "parsing")]
8 use proc_macro2::TokenStream;
9
10 use proc_macro2::TokenTree;
11
12 #[cfg(feature = "extra-traits")]
13 use std::hash::{Hash, Hasher};
14
15 #[cfg(feature = "parsing")]
16 use lookahead;
17 #[cfg(feature = "parsing")]
18 use parse::{Parse, Parser, Result};
19
20 ast_enum_of_structs! {
21 /// A Rust literal such as a string or integer or boolean.
22 ///
23 /// *This type is available if Syn is built with the `"derive"` or `"full"`
24 /// feature.*
25 ///
26 /// # Syntax tree enum
27 ///
28 /// This type is a [syntax tree enum].
29 ///
30 /// [syntax tree enum]: enum.Expr.html#syntax-tree-enums
31 //
32 // TODO: change syntax-tree-enum link to an intra rustdoc link, currently
33 // blocked on https://github.com/rust-lang/rust/issues/62833
34 pub enum Lit {
35 /// A UTF-8 string literal: `"foo"`.
36 ///
37 /// *This type is available if Syn is built with the `"derive"` or
38 /// `"full"` feature.*
39 pub Str(LitStr #manual_extra_traits {
40 token: Literal,
41 }),
42
43 /// A byte string literal: `b"foo"`.
44 ///
45 /// *This type is available if Syn is built with the `"derive"` or
46 /// `"full"` feature.*
47 pub ByteStr(LitByteStr #manual_extra_traits {
48 token: Literal,
49 }),
50
51 /// A byte literal: `b'f'`.
52 ///
53 /// *This type is available if Syn is built with the `"derive"` or
54 /// `"full"` feature.*
55 pub Byte(LitByte #manual_extra_traits {
56 token: Literal,
57 }),
58
59 /// A character literal: `'a'`.
60 ///
61 /// *This type is available if Syn is built with the `"derive"` or
62 /// `"full"` feature.*
63 pub Char(LitChar #manual_extra_traits {
64 token: Literal,
65 }),
66
67 /// An integer literal: `1` or `1u16`.
68 ///
69 /// Holds up to 64 bits of data. Use `LitVerbatim` for any larger
70 /// integer literal.
71 ///
72 /// *This type is available if Syn is built with the `"derive"` or
73 /// `"full"` feature.*
74 pub Int(LitInt #manual_extra_traits {
75 token: Literal,
76 }),
77
78 /// A floating point literal: `1f64` or `1.0e10f64`.
79 ///
80 /// Must be finite. May not be infinte or NaN.
81 ///
82 /// *This type is available if Syn is built with the `"derive"` or
83 /// `"full"` feature.*
84 pub Float(LitFloat #manual_extra_traits {
85 token: Literal,
86 }),
87
88 /// A boolean literal: `true` or `false`.
89 ///
90 /// *This type is available if Syn is built with the `"derive"` or
91 /// `"full"` feature.*
92 pub Bool(LitBool #manual_extra_traits {
93 pub value: bool,
94 pub span: Span,
95 }),
96
97 /// A raw token literal not interpreted by Syn, possibly because it
98 /// represents an integer larger than 64 bits.
99 ///
100 /// *This type is available if Syn is built with the `"derive"` or
101 /// `"full"` feature.*
102 pub Verbatim(LitVerbatim #manual_extra_traits {
103 pub token: Literal,
104 }),
105 }
106 }
107
108 impl LitStr {
new(value: &str, span: Span) -> Self109 pub fn new(value: &str, span: Span) -> Self {
110 let mut lit = Literal::string(value);
111 lit.set_span(span);
112 LitStr { token: lit }
113 }
114
value(&self) -> String115 pub fn value(&self) -> String {
116 value::parse_lit_str(&self.token.to_string())
117 }
118
119 /// Parse a syntax tree node from the content of this string literal.
120 ///
121 /// All spans in the syntax tree will point to the span of this `LitStr`.
122 ///
123 /// # Example
124 ///
125 /// ```edition2018
126 /// use proc_macro2::Span;
127 /// use syn::{Attribute, Error, Ident, Lit, Meta, MetaNameValue, Path, Result};
128 ///
129 /// // Parses the path from an attribute that looks like:
130 /// //
131 /// // #[path = "a::b::c"]
132 /// //
133 /// // or returns `None` if the input is some other attribute.
134 /// fn get_path(attr: &Attribute) -> Result<Option<Path>> {
135 /// if !attr.path.is_ident("path") {
136 /// return Ok(None);
137 /// }
138 ///
139 /// match attr.parse_meta()? {
140 /// Meta::NameValue(MetaNameValue { lit: Lit::Str(lit_str), .. }) => {
141 /// lit_str.parse().map(Some)
142 /// }
143 /// _ => {
144 /// let message = "expected #[path = \"...\"]";
145 /// Err(Error::new_spanned(attr, message))
146 /// }
147 /// }
148 /// }
149 /// ```
150 #[cfg(feature = "parsing")]
parse<T: Parse>(&self) -> Result<T>151 pub fn parse<T: Parse>(&self) -> Result<T> {
152 self.parse_with(T::parse)
153 }
154
155 /// Invoke parser on the content of this string literal.
156 ///
157 /// All spans in the syntax tree will point to the span of this `LitStr`.
158 ///
159 /// # Example
160 ///
161 /// ```edition2018
162 /// # use proc_macro2::Span;
163 /// # use syn::{LitStr, Result};
164 /// #
165 /// # fn main() -> Result<()> {
166 /// # let lit_str = LitStr::new("a::b::c", Span::call_site());
167 /// #
168 /// # const IGNORE: &str = stringify! {
169 /// let lit_str: LitStr = /* ... */;
170 /// # };
171 ///
172 /// // Parse a string literal like "a::b::c" into a Path, not allowing
173 /// // generic arguments on any of the path segments.
174 /// let basic_path = lit_str.parse_with(syn::Path::parse_mod_style)?;
175 /// #
176 /// # Ok(())
177 /// # }
178 /// ```
179 #[cfg(feature = "parsing")]
parse_with<F: Parser>(&self, parser: F) -> Result<F::Output>180 pub fn parse_with<F: Parser>(&self, parser: F) -> Result<F::Output> {
181 use proc_macro2::Group;
182
183 // Token stream with every span replaced by the given one.
184 fn respan_token_stream(stream: TokenStream, span: Span) -> TokenStream {
185 stream
186 .into_iter()
187 .map(|token| respan_token_tree(token, span))
188 .collect()
189 }
190
191 // Token tree with every span replaced by the given one.
192 fn respan_token_tree(mut token: TokenTree, span: Span) -> TokenTree {
193 match token {
194 TokenTree::Group(ref mut g) => {
195 let stream = respan_token_stream(g.stream().clone(), span);
196 *g = Group::new(g.delimiter(), stream);
197 g.set_span(span);
198 }
199 ref mut other => other.set_span(span),
200 }
201 token
202 }
203
204 // Parse string literal into a token stream with every span equal to the
205 // original literal's span.
206 let mut tokens = ::parse_str(&self.value())?;
207 tokens = respan_token_stream(tokens, self.span());
208
209 parser.parse2(tokens)
210 }
211
span(&self) -> Span212 pub fn span(&self) -> Span {
213 self.token.span()
214 }
215
set_span(&mut self, span: Span)216 pub fn set_span(&mut self, span: Span) {
217 self.token.set_span(span)
218 }
219 }
220
221 impl LitByteStr {
new(value: &[u8], span: Span) -> Self222 pub fn new(value: &[u8], span: Span) -> Self {
223 let mut token = Literal::byte_string(value);
224 token.set_span(span);
225 LitByteStr { token: token }
226 }
227
value(&self) -> Vec<u8>228 pub fn value(&self) -> Vec<u8> {
229 value::parse_lit_byte_str(&self.token.to_string())
230 }
231
span(&self) -> Span232 pub fn span(&self) -> Span {
233 self.token.span()
234 }
235
set_span(&mut self, span: Span)236 pub fn set_span(&mut self, span: Span) {
237 self.token.set_span(span)
238 }
239 }
240
241 impl LitByte {
new(value: u8, span: Span) -> Self242 pub fn new(value: u8, span: Span) -> Self {
243 let mut token = Literal::u8_suffixed(value);
244 token.set_span(span);
245 LitByte { token: token }
246 }
247
value(&self) -> u8248 pub fn value(&self) -> u8 {
249 value::parse_lit_byte(&self.token.to_string())
250 }
251
span(&self) -> Span252 pub fn span(&self) -> Span {
253 self.token.span()
254 }
255
set_span(&mut self, span: Span)256 pub fn set_span(&mut self, span: Span) {
257 self.token.set_span(span)
258 }
259 }
260
261 impl LitChar {
new(value: char, span: Span) -> Self262 pub fn new(value: char, span: Span) -> Self {
263 let mut token = Literal::character(value);
264 token.set_span(span);
265 LitChar { token: token }
266 }
267
value(&self) -> char268 pub fn value(&self) -> char {
269 value::parse_lit_char(&self.token.to_string())
270 }
271
span(&self) -> Span272 pub fn span(&self) -> Span {
273 self.token.span()
274 }
275
set_span(&mut self, span: Span)276 pub fn set_span(&mut self, span: Span) {
277 self.token.set_span(span)
278 }
279 }
280
281 impl LitInt {
new(value: u64, suffix: IntSuffix, span: Span) -> Self282 pub fn new(value: u64, suffix: IntSuffix, span: Span) -> Self {
283 let mut token = match suffix {
284 IntSuffix::Isize => Literal::isize_suffixed(value as isize),
285 IntSuffix::I8 => Literal::i8_suffixed(value as i8),
286 IntSuffix::I16 => Literal::i16_suffixed(value as i16),
287 IntSuffix::I32 => Literal::i32_suffixed(value as i32),
288 IntSuffix::I64 => Literal::i64_suffixed(value as i64),
289 IntSuffix::I128 => value::to_literal(&format!("{}i128", value)),
290 IntSuffix::Usize => Literal::usize_suffixed(value as usize),
291 IntSuffix::U8 => Literal::u8_suffixed(value as u8),
292 IntSuffix::U16 => Literal::u16_suffixed(value as u16),
293 IntSuffix::U32 => Literal::u32_suffixed(value as u32),
294 IntSuffix::U64 => Literal::u64_suffixed(value),
295 IntSuffix::U128 => value::to_literal(&format!("{}u128", value)),
296 IntSuffix::None => Literal::u64_unsuffixed(value),
297 };
298 token.set_span(span);
299 LitInt { token: token }
300 }
301
value(&self) -> u64302 pub fn value(&self) -> u64 {
303 value::parse_lit_int(&self.token.to_string()).unwrap()
304 }
305
suffix(&self) -> IntSuffix306 pub fn suffix(&self) -> IntSuffix {
307 let value = self.token.to_string();
308 for (s, suffix) in vec![
309 ("i8", IntSuffix::I8),
310 ("i16", IntSuffix::I16),
311 ("i32", IntSuffix::I32),
312 ("i64", IntSuffix::I64),
313 ("i128", IntSuffix::I128),
314 ("isize", IntSuffix::Isize),
315 ("u8", IntSuffix::U8),
316 ("u16", IntSuffix::U16),
317 ("u32", IntSuffix::U32),
318 ("u64", IntSuffix::U64),
319 ("u128", IntSuffix::U128),
320 ("usize", IntSuffix::Usize),
321 ] {
322 if value.ends_with(s) {
323 return suffix;
324 }
325 }
326 IntSuffix::None
327 }
328
span(&self) -> Span329 pub fn span(&self) -> Span {
330 self.token.span()
331 }
332
set_span(&mut self, span: Span)333 pub fn set_span(&mut self, span: Span) {
334 self.token.set_span(span)
335 }
336 }
337
338 impl LitFloat {
new(value: f64, suffix: FloatSuffix, span: Span) -> Self339 pub fn new(value: f64, suffix: FloatSuffix, span: Span) -> Self {
340 let mut token = match suffix {
341 FloatSuffix::F32 => Literal::f32_suffixed(value as f32),
342 FloatSuffix::F64 => Literal::f64_suffixed(value),
343 FloatSuffix::None => Literal::f64_unsuffixed(value),
344 };
345 token.set_span(span);
346 LitFloat { token: token }
347 }
348
value(&self) -> f64349 pub fn value(&self) -> f64 {
350 value::parse_lit_float(&self.token.to_string())
351 }
352
suffix(&self) -> FloatSuffix353 pub fn suffix(&self) -> FloatSuffix {
354 let value = self.token.to_string();
355 for (s, suffix) in vec![("f32", FloatSuffix::F32), ("f64", FloatSuffix::F64)] {
356 if value.ends_with(s) {
357 return suffix;
358 }
359 }
360 FloatSuffix::None
361 }
362
span(&self) -> Span363 pub fn span(&self) -> Span {
364 self.token.span()
365 }
366
set_span(&mut self, span: Span)367 pub fn set_span(&mut self, span: Span) {
368 self.token.set_span(span)
369 }
370 }
371
372 macro_rules! lit_extra_traits {
373 ($ty:ident, $field:ident) => {
374 #[cfg(feature = "extra-traits")]
375 impl Eq for $ty {}
376
377 #[cfg(feature = "extra-traits")]
378 impl PartialEq for $ty {
379 fn eq(&self, other: &Self) -> bool {
380 self.$field.to_string() == other.$field.to_string()
381 }
382 }
383
384 #[cfg(feature = "extra-traits")]
385 impl Hash for $ty {
386 fn hash<H>(&self, state: &mut H)
387 where
388 H: Hasher,
389 {
390 self.$field.to_string().hash(state);
391 }
392 }
393
394 #[cfg(feature = "parsing")]
395 #[doc(hidden)]
396 #[allow(non_snake_case)]
397 pub fn $ty(marker: lookahead::TokenMarker) -> $ty {
398 match marker {}
399 }
400 };
401 }
402
403 impl LitVerbatim {
span(&self) -> Span404 pub fn span(&self) -> Span {
405 self.token.span()
406 }
407
set_span(&mut self, span: Span)408 pub fn set_span(&mut self, span: Span) {
409 self.token.set_span(span)
410 }
411 }
412
413 lit_extra_traits!(LitStr, token);
414 lit_extra_traits!(LitByteStr, token);
415 lit_extra_traits!(LitByte, token);
416 lit_extra_traits!(LitChar, token);
417 lit_extra_traits!(LitInt, token);
418 lit_extra_traits!(LitFloat, token);
419 lit_extra_traits!(LitBool, value);
420 lit_extra_traits!(LitVerbatim, token);
421
422 ast_enum! {
423 /// The style of a string literal, either plain quoted or a raw string like
424 /// `r##"data"##`.
425 ///
426 /// *This type is available if Syn is built with the `"derive"` or `"full"`
427 /// feature.*
428 pub enum StrStyle #no_visit {
429 /// An ordinary string like `"data"`.
430 Cooked,
431 /// A raw string like `r##"data"##`.
432 ///
433 /// The unsigned integer is the number of `#` symbols used.
434 Raw(usize),
435 }
436 }
437
438 ast_enum! {
439 /// The suffix on an integer literal if any, like the `u8` in `127u8`.
440 ///
441 /// *This type is available if Syn is built with the `"derive"` or `"full"`
442 /// feature.*
443 pub enum IntSuffix #no_visit {
444 I8,
445 I16,
446 I32,
447 I64,
448 I128,
449 Isize,
450 U8,
451 U16,
452 U32,
453 U64,
454 U128,
455 Usize,
456 None,
457 }
458 }
459
460 ast_enum! {
461 /// The suffix on a floating point literal if any, like the `f32` in
462 /// `1.0f32`.
463 ///
464 /// *This type is available if Syn is built with the `"derive"` or `"full"`
465 /// feature.*
466 pub enum FloatSuffix #no_visit {
467 F32,
468 F64,
469 None,
470 }
471 }
472
473 #[cfg(feature = "parsing")]
474 #[doc(hidden)]
475 #[allow(non_snake_case)]
Lit(marker: lookahead::TokenMarker) -> Lit476 pub fn Lit(marker: lookahead::TokenMarker) -> Lit {
477 match marker {}
478 }
479
480 #[cfg(feature = "parsing")]
481 pub mod parsing {
482 use super::*;
483 use parse::{Parse, ParseStream, Result};
484
485 impl Parse for Lit {
parse(input: ParseStream) -> Result<Self>486 fn parse(input: ParseStream) -> Result<Self> {
487 input.step(|cursor| {
488 if let Some((lit, rest)) = cursor.literal() {
489 return Ok((Lit::new(lit), rest));
490 }
491 while let Some((ident, rest)) = cursor.ident() {
492 let value = if ident == "true" {
493 true
494 } else if ident == "false" {
495 false
496 } else {
497 break;
498 };
499 let lit_bool = LitBool {
500 value: value,
501 span: ident.span(),
502 };
503 return Ok((Lit::Bool(lit_bool), rest));
504 }
505 Err(cursor.error("expected literal"))
506 })
507 }
508 }
509
510 impl Parse for LitStr {
parse(input: ParseStream) -> Result<Self>511 fn parse(input: ParseStream) -> Result<Self> {
512 let head = input.fork();
513 match input.parse()? {
514 Lit::Str(lit) => Ok(lit),
515 _ => Err(head.error("expected string literal")),
516 }
517 }
518 }
519
520 impl Parse for LitByteStr {
parse(input: ParseStream) -> Result<Self>521 fn parse(input: ParseStream) -> Result<Self> {
522 let head = input.fork();
523 match input.parse()? {
524 Lit::ByteStr(lit) => Ok(lit),
525 _ => Err(head.error("expected byte string literal")),
526 }
527 }
528 }
529
530 impl Parse for LitByte {
parse(input: ParseStream) -> Result<Self>531 fn parse(input: ParseStream) -> Result<Self> {
532 let head = input.fork();
533 match input.parse()? {
534 Lit::Byte(lit) => Ok(lit),
535 _ => Err(head.error("expected byte literal")),
536 }
537 }
538 }
539
540 impl Parse for LitChar {
parse(input: ParseStream) -> Result<Self>541 fn parse(input: ParseStream) -> Result<Self> {
542 let head = input.fork();
543 match input.parse()? {
544 Lit::Char(lit) => Ok(lit),
545 _ => Err(head.error("expected character literal")),
546 }
547 }
548 }
549
550 impl Parse for LitInt {
parse(input: ParseStream) -> Result<Self>551 fn parse(input: ParseStream) -> Result<Self> {
552 let head = input.fork();
553 match input.parse()? {
554 Lit::Int(lit) => Ok(lit),
555 _ => Err(head.error("expected integer literal")),
556 }
557 }
558 }
559
560 impl Parse for LitFloat {
parse(input: ParseStream) -> Result<Self>561 fn parse(input: ParseStream) -> Result<Self> {
562 let head = input.fork();
563 match input.parse()? {
564 Lit::Float(lit) => Ok(lit),
565 _ => Err(head.error("expected floating point literal")),
566 }
567 }
568 }
569
570 impl Parse for LitBool {
parse(input: ParseStream) -> Result<Self>571 fn parse(input: ParseStream) -> Result<Self> {
572 let head = input.fork();
573 match input.parse()? {
574 Lit::Bool(lit) => Ok(lit),
575 _ => Err(head.error("expected boolean literal")),
576 }
577 }
578 }
579 }
580
581 #[cfg(feature = "printing")]
582 mod printing {
583 use super::*;
584 use proc_macro2::TokenStream;
585 use quote::{ToTokens, TokenStreamExt};
586
587 impl ToTokens for LitStr {
to_tokens(&self, tokens: &mut TokenStream)588 fn to_tokens(&self, tokens: &mut TokenStream) {
589 self.token.to_tokens(tokens);
590 }
591 }
592
593 impl ToTokens for LitByteStr {
to_tokens(&self, tokens: &mut TokenStream)594 fn to_tokens(&self, tokens: &mut TokenStream) {
595 self.token.to_tokens(tokens);
596 }
597 }
598
599 impl ToTokens for LitByte {
to_tokens(&self, tokens: &mut TokenStream)600 fn to_tokens(&self, tokens: &mut TokenStream) {
601 self.token.to_tokens(tokens);
602 }
603 }
604
605 impl ToTokens for LitChar {
to_tokens(&self, tokens: &mut TokenStream)606 fn to_tokens(&self, tokens: &mut TokenStream) {
607 self.token.to_tokens(tokens);
608 }
609 }
610
611 impl ToTokens for LitInt {
to_tokens(&self, tokens: &mut TokenStream)612 fn to_tokens(&self, tokens: &mut TokenStream) {
613 self.token.to_tokens(tokens);
614 }
615 }
616
617 impl ToTokens for LitFloat {
to_tokens(&self, tokens: &mut TokenStream)618 fn to_tokens(&self, tokens: &mut TokenStream) {
619 self.token.to_tokens(tokens);
620 }
621 }
622
623 impl ToTokens for LitBool {
to_tokens(&self, tokens: &mut TokenStream)624 fn to_tokens(&self, tokens: &mut TokenStream) {
625 let s = if self.value { "true" } else { "false" };
626 tokens.append(Ident::new(s, self.span));
627 }
628 }
629
630 impl ToTokens for LitVerbatim {
to_tokens(&self, tokens: &mut TokenStream)631 fn to_tokens(&self, tokens: &mut TokenStream) {
632 self.token.to_tokens(tokens);
633 }
634 }
635 }
636
637 mod value {
638 use super::*;
639 use proc_macro2::TokenStream;
640 use std::char;
641 use std::ops::{Index, RangeFrom};
642
643 impl Lit {
644 /// Interpret a Syn literal from a proc-macro2 literal.
645 ///
646 /// Not all proc-macro2 literals are valid Syn literals. In particular,
647 /// doc comments are considered by proc-macro2 to be literals but in Syn
648 /// they are [`Attribute`].
649 ///
650 /// [`Attribute`]: crate::Attribute
651 ///
652 /// # Panics
653 ///
654 /// Panics if the input is a doc comment literal.
new(token: Literal) -> Self655 pub fn new(token: Literal) -> Self {
656 let value = token.to_string();
657
658 match value::byte(&value, 0) {
659 b'"' | b'r' => return Lit::Str(LitStr { token: token }),
660 b'b' => match value::byte(&value, 1) {
661 b'"' | b'r' => return Lit::ByteStr(LitByteStr { token: token }),
662 b'\'' => return Lit::Byte(LitByte { token: token }),
663 _ => {}
664 },
665 b'\'' => return Lit::Char(LitChar { token: token }),
666 b'0'...b'9' => {
667 if number_is_int(&value) {
668 return Lit::Int(LitInt { token: token });
669 } else if number_is_float(&value) {
670 return Lit::Float(LitFloat { token: token });
671 } else {
672 // number overflow
673 return Lit::Verbatim(LitVerbatim { token: token });
674 }
675 }
676 _ => {
677 if value == "true" || value == "false" {
678 return Lit::Bool(LitBool {
679 value: value == "true",
680 span: token.span(),
681 });
682 }
683 }
684 }
685
686 panic!("Unrecognized literal: {}", value);
687 }
688 }
689
number_is_int(value: &str) -> bool690 fn number_is_int(value: &str) -> bool {
691 if number_is_float(value) {
692 false
693 } else {
694 value::parse_lit_int(value).is_some()
695 }
696 }
697
number_is_float(value: &str) -> bool698 fn number_is_float(value: &str) -> bool {
699 if value.contains('.') {
700 true
701 } else if value.starts_with("0x") || value.ends_with("size") {
702 false
703 } else {
704 value.contains('e') || value.contains('E')
705 }
706 }
707
708 /// Get the byte at offset idx, or a default of `b'\0'` if we're looking
709 /// past the end of the input buffer.
byte<S: AsRef<[u8]> + ?Sized>(s: &S, idx: usize) -> u8710 pub fn byte<S: AsRef<[u8]> + ?Sized>(s: &S, idx: usize) -> u8 {
711 let s = s.as_ref();
712 if idx < s.len() {
713 s[idx]
714 } else {
715 0
716 }
717 }
718
next_chr(s: &str) -> char719 fn next_chr(s: &str) -> char {
720 s.chars().next().unwrap_or('\0')
721 }
722
parse_lit_str(s: &str) -> String723 pub fn parse_lit_str(s: &str) -> String {
724 match byte(s, 0) {
725 b'"' => parse_lit_str_cooked(s),
726 b'r' => parse_lit_str_raw(s),
727 _ => unreachable!(),
728 }
729 }
730
731 // Clippy false positive
732 // https://github.com/rust-lang-nursery/rust-clippy/issues/2329
733 #[cfg_attr(feature = "cargo-clippy", allow(needless_continue))]
parse_lit_str_cooked(mut s: &str) -> String734 fn parse_lit_str_cooked(mut s: &str) -> String {
735 assert_eq!(byte(s, 0), b'"');
736 s = &s[1..];
737
738 let mut out = String::new();
739 'outer: loop {
740 let ch = match byte(s, 0) {
741 b'"' => break,
742 b'\\' => {
743 let b = byte(s, 1);
744 s = &s[2..];
745 match b {
746 b'x' => {
747 let (byte, rest) = backslash_x(s);
748 s = rest;
749 assert!(byte <= 0x80, "Invalid \\x byte in string literal");
750 char::from_u32(u32::from(byte)).unwrap()
751 }
752 b'u' => {
753 let (chr, rest) = backslash_u(s);
754 s = rest;
755 chr
756 }
757 b'n' => '\n',
758 b'r' => '\r',
759 b't' => '\t',
760 b'\\' => '\\',
761 b'0' => '\0',
762 b'\'' => '\'',
763 b'"' => '"',
764 b'\r' | b'\n' => loop {
765 let ch = next_chr(s);
766 if ch.is_whitespace() {
767 s = &s[ch.len_utf8()..];
768 } else {
769 continue 'outer;
770 }
771 },
772 b => panic!("unexpected byte {:?} after \\ character in byte literal", b),
773 }
774 }
775 b'\r' => {
776 assert_eq!(byte(s, 1), b'\n', "Bare CR not allowed in string");
777 s = &s[2..];
778 '\n'
779 }
780 _ => {
781 let ch = next_chr(s);
782 s = &s[ch.len_utf8()..];
783 ch
784 }
785 };
786 out.push(ch);
787 }
788
789 assert_eq!(s, "\"");
790 out
791 }
792
parse_lit_str_raw(mut s: &str) -> String793 fn parse_lit_str_raw(mut s: &str) -> String {
794 assert_eq!(byte(s, 0), b'r');
795 s = &s[1..];
796
797 let mut pounds = 0;
798 while byte(s, pounds) == b'#' {
799 pounds += 1;
800 }
801 assert_eq!(byte(s, pounds), b'"');
802 assert_eq!(byte(s, s.len() - pounds - 1), b'"');
803 for end in s[s.len() - pounds..].bytes() {
804 assert_eq!(end, b'#');
805 }
806
807 s[pounds + 1..s.len() - pounds - 1].to_owned()
808 }
809
parse_lit_byte_str(s: &str) -> Vec<u8>810 pub fn parse_lit_byte_str(s: &str) -> Vec<u8> {
811 assert_eq!(byte(s, 0), b'b');
812 match byte(s, 1) {
813 b'"' => parse_lit_byte_str_cooked(s),
814 b'r' => parse_lit_byte_str_raw(s),
815 _ => unreachable!(),
816 }
817 }
818
819 // Clippy false positive
820 // https://github.com/rust-lang-nursery/rust-clippy/issues/2329
821 #[cfg_attr(feature = "cargo-clippy", allow(needless_continue))]
parse_lit_byte_str_cooked(mut s: &str) -> Vec<u8>822 fn parse_lit_byte_str_cooked(mut s: &str) -> Vec<u8> {
823 assert_eq!(byte(s, 0), b'b');
824 assert_eq!(byte(s, 1), b'"');
825 s = &s[2..];
826
827 // We're going to want to have slices which don't respect codepoint boundaries.
828 let mut s = s.as_bytes();
829
830 let mut out = Vec::new();
831 'outer: loop {
832 let byte = match byte(s, 0) {
833 b'"' => break,
834 b'\\' => {
835 let b = byte(s, 1);
836 s = &s[2..];
837 match b {
838 b'x' => {
839 let (b, rest) = backslash_x(s);
840 s = rest;
841 b
842 }
843 b'n' => b'\n',
844 b'r' => b'\r',
845 b't' => b'\t',
846 b'\\' => b'\\',
847 b'0' => b'\0',
848 b'\'' => b'\'',
849 b'"' => b'"',
850 b'\r' | b'\n' => loop {
851 let byte = byte(s, 0);
852 let ch = char::from_u32(u32::from(byte)).unwrap();
853 if ch.is_whitespace() {
854 s = &s[1..];
855 } else {
856 continue 'outer;
857 }
858 },
859 b => panic!("unexpected byte {:?} after \\ character in byte literal", b),
860 }
861 }
862 b'\r' => {
863 assert_eq!(byte(s, 1), b'\n', "Bare CR not allowed in string");
864 s = &s[2..];
865 b'\n'
866 }
867 b => {
868 s = &s[1..];
869 b
870 }
871 };
872 out.push(byte);
873 }
874
875 assert_eq!(s, b"\"");
876 out
877 }
878
parse_lit_byte_str_raw(s: &str) -> Vec<u8>879 fn parse_lit_byte_str_raw(s: &str) -> Vec<u8> {
880 assert_eq!(byte(s, 0), b'b');
881 parse_lit_str_raw(&s[1..]).into_bytes()
882 }
883
parse_lit_byte(s: &str) -> u8884 pub fn parse_lit_byte(s: &str) -> u8 {
885 assert_eq!(byte(s, 0), b'b');
886 assert_eq!(byte(s, 1), b'\'');
887
888 // We're going to want to have slices which don't respect codepoint boundaries.
889 let mut s = s[2..].as_bytes();
890
891 let b = match byte(s, 0) {
892 b'\\' => {
893 let b = byte(s, 1);
894 s = &s[2..];
895 match b {
896 b'x' => {
897 let (b, rest) = backslash_x(s);
898 s = rest;
899 b
900 }
901 b'n' => b'\n',
902 b'r' => b'\r',
903 b't' => b'\t',
904 b'\\' => b'\\',
905 b'0' => b'\0',
906 b'\'' => b'\'',
907 b'"' => b'"',
908 b => panic!("unexpected byte {:?} after \\ character in byte literal", b),
909 }
910 }
911 b => {
912 s = &s[1..];
913 b
914 }
915 };
916
917 assert_eq!(byte(s, 0), b'\'');
918 b
919 }
920
parse_lit_char(mut s: &str) -> char921 pub fn parse_lit_char(mut s: &str) -> char {
922 assert_eq!(byte(s, 0), b'\'');
923 s = &s[1..];
924
925 let ch = match byte(s, 0) {
926 b'\\' => {
927 let b = byte(s, 1);
928 s = &s[2..];
929 match b {
930 b'x' => {
931 let (byte, rest) = backslash_x(s);
932 s = rest;
933 assert!(byte <= 0x80, "Invalid \\x byte in string literal");
934 char::from_u32(u32::from(byte)).unwrap()
935 }
936 b'u' => {
937 let (chr, rest) = backslash_u(s);
938 s = rest;
939 chr
940 }
941 b'n' => '\n',
942 b'r' => '\r',
943 b't' => '\t',
944 b'\\' => '\\',
945 b'0' => '\0',
946 b'\'' => '\'',
947 b'"' => '"',
948 b => panic!("unexpected byte {:?} after \\ character in byte literal", b),
949 }
950 }
951 _ => {
952 let ch = next_chr(s);
953 s = &s[ch.len_utf8()..];
954 ch
955 }
956 };
957 assert_eq!(s, "\'", "Expected end of char literal");
958 ch
959 }
960
backslash_x<S>(s: &S) -> (u8, &S) where S: Index<RangeFrom<usize>, Output = S> + AsRef<[u8]> + ?Sized,961 fn backslash_x<S>(s: &S) -> (u8, &S)
962 where
963 S: Index<RangeFrom<usize>, Output = S> + AsRef<[u8]> + ?Sized,
964 {
965 let mut ch = 0;
966 let b0 = byte(s, 0);
967 let b1 = byte(s, 1);
968 ch += 0x10
969 * match b0 {
970 b'0'...b'9' => b0 - b'0',
971 b'a'...b'f' => 10 + (b0 - b'a'),
972 b'A'...b'F' => 10 + (b0 - b'A'),
973 _ => panic!("unexpected non-hex character after \\x"),
974 };
975 ch += match b1 {
976 b'0'...b'9' => b1 - b'0',
977 b'a'...b'f' => 10 + (b1 - b'a'),
978 b'A'...b'F' => 10 + (b1 - b'A'),
979 _ => panic!("unexpected non-hex character after \\x"),
980 };
981 (ch, &s[2..])
982 }
983
backslash_u(mut s: &str) -> (char, &str)984 fn backslash_u(mut s: &str) -> (char, &str) {
985 if byte(s, 0) != b'{' {
986 panic!("expected {{ after \\u");
987 }
988 s = &s[1..];
989
990 let mut ch = 0;
991 for _ in 0..6 {
992 let b = byte(s, 0);
993 match b {
994 b'0'...b'9' => {
995 ch *= 0x10;
996 ch += u32::from(b - b'0');
997 s = &s[1..];
998 }
999 b'a'...b'f' => {
1000 ch *= 0x10;
1001 ch += u32::from(10 + b - b'a');
1002 s = &s[1..];
1003 }
1004 b'A'...b'F' => {
1005 ch *= 0x10;
1006 ch += u32::from(10 + b - b'A');
1007 s = &s[1..];
1008 }
1009 b'}' => break,
1010 _ => panic!("unexpected non-hex character after \\u"),
1011 }
1012 }
1013 assert!(byte(s, 0) == b'}');
1014 s = &s[1..];
1015
1016 if let Some(ch) = char::from_u32(ch) {
1017 (ch, s)
1018 } else {
1019 panic!("character code {:x} is not a valid unicode character", ch);
1020 }
1021 }
1022
parse_lit_int(mut s: &str) -> Option<u64>1023 pub fn parse_lit_int(mut s: &str) -> Option<u64> {
1024 let base = match (byte(s, 0), byte(s, 1)) {
1025 (b'0', b'x') => {
1026 s = &s[2..];
1027 16
1028 }
1029 (b'0', b'o') => {
1030 s = &s[2..];
1031 8
1032 }
1033 (b'0', b'b') => {
1034 s = &s[2..];
1035 2
1036 }
1037 (b'0'...b'9', _) => 10,
1038 _ => unreachable!(),
1039 };
1040
1041 let mut value = 0u64;
1042 loop {
1043 let b = byte(s, 0);
1044 let digit = match b {
1045 b'0'...b'9' => u64::from(b - b'0'),
1046 b'a'...b'f' if base > 10 => 10 + u64::from(b - b'a'),
1047 b'A'...b'F' if base > 10 => 10 + u64::from(b - b'A'),
1048 b'_' => {
1049 s = &s[1..];
1050 continue;
1051 }
1052 // NOTE: Looking at a floating point literal, we don't want to
1053 // consider these integers.
1054 b'.' if base == 10 => return None,
1055 b'e' | b'E' if base == 10 => return None,
1056 _ => break,
1057 };
1058
1059 if digit >= base {
1060 panic!("Unexpected digit {:x} out of base range", digit);
1061 }
1062
1063 value = match value.checked_mul(base) {
1064 Some(value) => value,
1065 None => return None,
1066 };
1067 value = match value.checked_add(digit) {
1068 Some(value) => value,
1069 None => return None,
1070 };
1071 s = &s[1..];
1072 }
1073
1074 Some(value)
1075 }
1076
parse_lit_float(input: &str) -> f641077 pub fn parse_lit_float(input: &str) -> f64 {
1078 // Rust's floating point literals are very similar to the ones parsed by
1079 // the standard library, except that rust's literals can contain
1080 // ignorable underscores. Let's remove those underscores.
1081 let mut bytes = input.to_owned().into_bytes();
1082 let mut write = 0;
1083 for read in 0..bytes.len() {
1084 if bytes[read] == b'_' {
1085 continue; // Don't increase write
1086 }
1087 if write != read {
1088 let x = bytes[read];
1089 bytes[write] = x;
1090 }
1091 write += 1;
1092 }
1093 bytes.truncate(write);
1094 let input = String::from_utf8(bytes).unwrap();
1095 let end = input.find('f').unwrap_or_else(|| input.len());
1096 input[..end].parse().unwrap()
1097 }
1098
to_literal(s: &str) -> Literal1099 pub fn to_literal(s: &str) -> Literal {
1100 let stream = s.parse::<TokenStream>().unwrap();
1101 match stream.into_iter().next().unwrap() {
1102 TokenTree::Literal(l) => l,
1103 _ => unreachable!(),
1104 }
1105 }
1106 }
1107