1 //! Tokens representing Rust punctuation, keywords, and delimiters.
2 //!
3 //! The type names in this module can be difficult to keep straight, so we
4 //! prefer to use the [`Token!`] macro instead. This is a type-macro that
5 //! expands to the token type of the given token.
6 //!
7 //! [`Token!`]: ../macro.Token.html
8 //!
9 //! # Example
10 //!
11 //! The [`ItemStatic`] syntax tree node is defined like this.
12 //!
13 //! [`ItemStatic`]: ../struct.ItemStatic.html
14 //!
15 //! ```
16 //! # use syn::{Attribute, Expr, Ident, Token, Type, Visibility};
17 //! #
18 //! pub struct ItemStatic {
19 //! pub attrs: Vec<Attribute>,
20 //! pub vis: Visibility,
21 //! pub static_token: Token![static],
22 //! pub mutability: Option<Token![mut]>,
23 //! pub ident: Ident,
24 //! pub colon_token: Token![:],
25 //! pub ty: Box<Type>,
26 //! pub eq_token: Token![=],
27 //! pub expr: Box<Expr>,
28 //! pub semi_token: Token![;],
29 //! }
30 //! ```
31 //!
32 //! # Parsing
33 //!
34 //! Keywords and punctuation can be parsed through the [`ParseStream::parse`]
35 //! method. Delimiter tokens are parsed using the [`parenthesized!`],
36 //! [`bracketed!`] and [`braced!`] macros.
37 //!
38 //! [`ParseStream::parse`]: ../parse/struct.ParseBuffer.html#method.parse
39 //! [`parenthesized!`]: ../macro.parenthesized.html
40 //! [`bracketed!`]: ../macro.bracketed.html
41 //! [`braced!`]: ../macro.braced.html
42 //!
43 //! ```
44 //! use syn::{Attribute, Result};
45 //! use syn::parse::{Parse, ParseStream};
46 //! #
47 //! # enum ItemStatic {}
48 //!
49 //! // Parse the ItemStatic struct shown above.
50 //! impl Parse for ItemStatic {
51 //! fn parse(input: ParseStream) -> Result<Self> {
52 //! # use syn::ItemStatic;
53 //! # fn parse(input: ParseStream) -> Result<ItemStatic> {
54 //! Ok(ItemStatic {
55 //! attrs: input.call(Attribute::parse_outer)?,
56 //! vis: input.parse()?,
57 //! static_token: input.parse()?,
58 //! mutability: input.parse()?,
59 //! ident: input.parse()?,
60 //! colon_token: input.parse()?,
61 //! ty: input.parse()?,
62 //! eq_token: input.parse()?,
63 //! expr: input.parse()?,
64 //! semi_token: input.parse()?,
65 //! })
66 //! # }
67 //! # unimplemented!()
68 //! }
69 //! }
70 //! ```
71 //!
72 //! # Other operations
73 //!
74 //! Every keyword and punctuation token supports the following operations.
75 //!
76 //! - [Peeking] — `input.peek(Token![...])`
77 //!
78 //! - [Parsing] — `input.parse::<Token![...]>()?`
79 //!
80 //! - [Printing] — `quote!( ... #the_token ... )`
81 //!
82 //! - Construction from a [`Span`] — `let the_token = Token![...](sp)`
83 //!
84 //! - Field access to its span — `let sp = the_token.span`
85 //!
86 //! [Peeking]: ../parse/struct.ParseBuffer.html#method.peek
87 //! [Parsing]: ../parse/struct.ParseBuffer.html#method.parse
88 //! [Printing]: https://docs.rs/quote/1.0/quote/trait.ToTokens.html
89 //! [`Span`]: https://docs.rs/proc-macro2/1.0/proc_macro2/struct.Span.html
90
91 #[cfg(feature = "extra-traits")]
92 use std::cmp;
93 #[cfg(feature = "extra-traits")]
94 use std::fmt::{self, Debug};
95 #[cfg(feature = "extra-traits")]
96 use std::hash::{Hash, Hasher};
97 use std::ops::{Deref, DerefMut};
98
99 #[cfg(any(feature = "parsing", feature = "printing"))]
100 use proc_macro2::Ident;
101 use proc_macro2::Span;
102 #[cfg(feature = "printing")]
103 use proc_macro2::TokenStream;
104 #[cfg(feature = "parsing")]
105 use proc_macro2::{Delimiter, Literal, Punct, TokenTree};
106 #[cfg(feature = "printing")]
107 use quote::{ToTokens, TokenStreamExt};
108
109 use self::private::WithSpan;
110 #[cfg(feature = "parsing")]
111 use crate::buffer::Cursor;
112 #[cfg(feature = "parsing")]
113 use crate::error::Result;
114 #[cfg(feature = "parsing")]
115 use crate::lifetime::Lifetime;
116 #[cfg(feature = "parsing")]
117 use crate::lit::{Lit, LitBool, LitByte, LitByteStr, LitChar, LitFloat, LitInt, LitStr};
118 #[cfg(feature = "parsing")]
119 use crate::lookahead;
120 #[cfg(feature = "parsing")]
121 use crate::parse::{Parse, ParseStream};
122 use crate::span::IntoSpans;
123
124 /// Marker trait for types that represent single tokens.
125 ///
126 /// This trait is sealed and cannot be implemented for types outside of Syn.
127 #[cfg(feature = "parsing")]
128 pub trait Token: private::Sealed {
129 // Not public API.
130 #[doc(hidden)]
peek(cursor: Cursor) -> bool131 fn peek(cursor: Cursor) -> bool;
132
133 // Not public API.
134 #[doc(hidden)]
display() -> &'static str135 fn display() -> &'static str;
136 }
137
138 mod private {
139 use proc_macro2::Span;
140
141 #[cfg(feature = "parsing")]
142 pub trait Sealed {}
143
144 /// Support writing `token.span` rather than `token.spans[0]` on tokens that
145 /// hold a single span.
146 #[repr(C)]
147 pub struct WithSpan {
148 pub span: Span,
149 }
150 }
151
152 #[cfg(feature = "parsing")]
153 impl private::Sealed for Ident {}
154
155 #[cfg(feature = "parsing")]
peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool156 fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool {
157 use crate::parse::Unexpected;
158 use std::cell::Cell;
159 use std::rc::Rc;
160
161 let scope = Span::call_site();
162 let unexpected = Rc::new(Cell::new(Unexpected::None));
163 let buffer = crate::parse::new_parse_buffer(scope, cursor, unexpected);
164 peek(&buffer)
165 }
166
167 macro_rules! impl_token {
168 ($display:tt $name:ty) => {
169 #[cfg(feature = "parsing")]
170 impl Token for $name {
171 fn peek(cursor: Cursor) -> bool {
172 fn peek(input: ParseStream) -> bool {
173 <$name as Parse>::parse(input).is_ok()
174 }
175 peek_impl(cursor, peek)
176 }
177
178 fn display() -> &'static str {
179 $display
180 }
181 }
182
183 #[cfg(feature = "parsing")]
184 impl private::Sealed for $name {}
185 };
186 }
187
188 impl_token!("lifetime" Lifetime);
189 impl_token!("literal" Lit);
190 impl_token!("string literal" LitStr);
191 impl_token!("byte string literal" LitByteStr);
192 impl_token!("byte literal" LitByte);
193 impl_token!("character literal" LitChar);
194 impl_token!("integer literal" LitInt);
195 impl_token!("floating point literal" LitFloat);
196 impl_token!("boolean literal" LitBool);
197 impl_token!("group token" proc_macro2::Group);
198
199 macro_rules! impl_low_level_token {
200 ($display:tt $ty:ident $get:ident) => {
201 #[cfg(feature = "parsing")]
202 impl Token for $ty {
203 fn peek(cursor: Cursor) -> bool {
204 cursor.$get().is_some()
205 }
206
207 fn display() -> &'static str {
208 $display
209 }
210 }
211
212 #[cfg(feature = "parsing")]
213 impl private::Sealed for $ty {}
214 };
215 }
216
217 impl_low_level_token!("punctuation token" Punct punct);
218 impl_low_level_token!("literal" Literal literal);
219 impl_low_level_token!("token" TokenTree token_tree);
220
221 // Not public API.
222 #[doc(hidden)]
223 #[cfg(feature = "parsing")]
224 pub trait CustomToken {
peek(cursor: Cursor) -> bool225 fn peek(cursor: Cursor) -> bool;
display() -> &'static str226 fn display() -> &'static str;
227 }
228
229 #[cfg(feature = "parsing")]
230 impl<T: CustomToken> private::Sealed for T {}
231
232 #[cfg(feature = "parsing")]
233 impl<T: CustomToken> Token for T {
peek(cursor: Cursor) -> bool234 fn peek(cursor: Cursor) -> bool {
235 <Self as CustomToken>::peek(cursor)
236 }
237
display() -> &'static str238 fn display() -> &'static str {
239 <Self as CustomToken>::display()
240 }
241 }
242
243 macro_rules! define_keywords {
244 ($($token:tt pub struct $name:ident #[$doc:meta])*) => {
245 $(
246 #[cfg_attr(feature = "clone-impls", derive(Copy, Clone))]
247 #[$doc]
248 ///
249 /// Don't try to remember the name of this type — use the
250 /// [`Token!`] macro instead.
251 ///
252 /// [`Token!`]: crate::token
253 pub struct $name {
254 pub span: Span,
255 }
256
257 #[doc(hidden)]
258 #[allow(non_snake_case)]
259 pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name {
260 $name {
261 span: span.into_spans()[0],
262 }
263 }
264
265 impl std::default::Default for $name {
266 fn default() -> Self {
267 $name {
268 span: Span::call_site(),
269 }
270 }
271 }
272
273 #[cfg(feature = "extra-traits")]
274 impl Debug for $name {
275 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
276 f.write_str(stringify!($name))
277 }
278 }
279
280 #[cfg(feature = "extra-traits")]
281 impl cmp::Eq for $name {}
282
283 #[cfg(feature = "extra-traits")]
284 impl PartialEq for $name {
285 fn eq(&self, _other: &$name) -> bool {
286 true
287 }
288 }
289
290 #[cfg(feature = "extra-traits")]
291 impl Hash for $name {
292 fn hash<H: Hasher>(&self, _state: &mut H) {}
293 }
294
295 #[cfg(feature = "printing")]
296 impl ToTokens for $name {
297 fn to_tokens(&self, tokens: &mut TokenStream) {
298 printing::keyword($token, self.span, tokens);
299 }
300 }
301
302 #[cfg(feature = "parsing")]
303 impl Parse for $name {
304 fn parse(input: ParseStream) -> Result<Self> {
305 Ok($name {
306 span: parsing::keyword(input, $token)?,
307 })
308 }
309 }
310
311 #[cfg(feature = "parsing")]
312 impl Token for $name {
313 fn peek(cursor: Cursor) -> bool {
314 parsing::peek_keyword(cursor, $token)
315 }
316
317 fn display() -> &'static str {
318 concat!("`", $token, "`")
319 }
320 }
321
322 #[cfg(feature = "parsing")]
323 impl private::Sealed for $name {}
324 )*
325 };
326 }
327
328 macro_rules! impl_deref_if_len_is_1 {
329 ($name:ident/1) => {
330 impl Deref for $name {
331 type Target = WithSpan;
332
333 fn deref(&self) -> &Self::Target {
334 unsafe { &*(self as *const Self as *const WithSpan) }
335 }
336 }
337
338 impl DerefMut for $name {
339 fn deref_mut(&mut self) -> &mut Self::Target {
340 unsafe { &mut *(self as *mut Self as *mut WithSpan) }
341 }
342 }
343 };
344
345 ($name:ident/$len:tt) => {};
346 }
347
348 macro_rules! define_punctuation_structs {
349 ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => {
350 $(
351 #[cfg_attr(feature = "clone-impls", derive(Copy, Clone))]
352 #[repr(C)]
353 #[$doc]
354 ///
355 /// Don't try to remember the name of this type — use the
356 /// [`Token!`] macro instead.
357 ///
358 /// [`Token!`]: crate::token
359 pub struct $name {
360 pub spans: [Span; $len],
361 }
362
363 #[doc(hidden)]
364 #[allow(non_snake_case)]
365 pub fn $name<S: IntoSpans<[Span; $len]>>(spans: S) -> $name {
366 $name {
367 spans: spans.into_spans(),
368 }
369 }
370
371 impl std::default::Default for $name {
372 fn default() -> Self {
373 $name {
374 spans: [Span::call_site(); $len],
375 }
376 }
377 }
378
379 #[cfg(feature = "extra-traits")]
380 impl Debug for $name {
381 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
382 f.write_str(stringify!($name))
383 }
384 }
385
386 #[cfg(feature = "extra-traits")]
387 impl cmp::Eq for $name {}
388
389 #[cfg(feature = "extra-traits")]
390 impl PartialEq for $name {
391 fn eq(&self, _other: &$name) -> bool {
392 true
393 }
394 }
395
396 #[cfg(feature = "extra-traits")]
397 impl Hash for $name {
398 fn hash<H: Hasher>(&self, _state: &mut H) {}
399 }
400
401 impl_deref_if_len_is_1!($name/$len);
402 )*
403 };
404 }
405
406 macro_rules! define_punctuation {
407 ($($token:tt pub struct $name:ident/$len:tt #[$doc:meta])*) => {
408 $(
409 define_punctuation_structs! {
410 $token pub struct $name/$len #[$doc]
411 }
412
413 #[cfg(feature = "printing")]
414 impl ToTokens for $name {
415 fn to_tokens(&self, tokens: &mut TokenStream) {
416 printing::punct($token, &self.spans, tokens);
417 }
418 }
419
420 #[cfg(feature = "parsing")]
421 impl Parse for $name {
422 fn parse(input: ParseStream) -> Result<Self> {
423 Ok($name {
424 spans: parsing::punct(input, $token)?,
425 })
426 }
427 }
428
429 #[cfg(feature = "parsing")]
430 impl Token for $name {
431 fn peek(cursor: Cursor) -> bool {
432 parsing::peek_punct(cursor, $token)
433 }
434
435 fn display() -> &'static str {
436 concat!("`", $token, "`")
437 }
438 }
439
440 #[cfg(feature = "parsing")]
441 impl private::Sealed for $name {}
442 )*
443 };
444 }
445
446 macro_rules! define_delimiters {
447 ($($token:tt pub struct $name:ident #[$doc:meta])*) => {
448 $(
449 #[cfg_attr(feature = "clone-impls", derive(Copy, Clone))]
450 #[$doc]
451 pub struct $name {
452 pub span: Span,
453 }
454
455 #[doc(hidden)]
456 #[allow(non_snake_case)]
457 pub fn $name<S: IntoSpans<[Span; 1]>>(span: S) -> $name {
458 $name {
459 span: span.into_spans()[0],
460 }
461 }
462
463 impl std::default::Default for $name {
464 fn default() -> Self {
465 $name {
466 span: Span::call_site(),
467 }
468 }
469 }
470
471 #[cfg(feature = "extra-traits")]
472 impl Debug for $name {
473 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
474 f.write_str(stringify!($name))
475 }
476 }
477
478 #[cfg(feature = "extra-traits")]
479 impl cmp::Eq for $name {}
480
481 #[cfg(feature = "extra-traits")]
482 impl PartialEq for $name {
483 fn eq(&self, _other: &$name) -> bool {
484 true
485 }
486 }
487
488 #[cfg(feature = "extra-traits")]
489 impl Hash for $name {
490 fn hash<H: Hasher>(&self, _state: &mut H) {}
491 }
492
493 impl $name {
494 #[cfg(feature = "printing")]
495 pub fn surround<F>(&self, tokens: &mut TokenStream, f: F)
496 where
497 F: FnOnce(&mut TokenStream),
498 {
499 printing::delim($token, self.span, tokens, f);
500 }
501 }
502
503 #[cfg(feature = "parsing")]
504 impl private::Sealed for $name {}
505 )*
506 };
507 }
508
509 define_punctuation_structs! {
510 "_" pub struct Underscore/1 /// `_`
511 }
512
513 #[cfg(feature = "printing")]
514 impl ToTokens for Underscore {
to_tokens(&self, tokens: &mut TokenStream)515 fn to_tokens(&self, tokens: &mut TokenStream) {
516 tokens.append(Ident::new("_", self.span));
517 }
518 }
519
520 #[cfg(feature = "parsing")]
521 impl Parse for Underscore {
parse(input: ParseStream) -> Result<Self>522 fn parse(input: ParseStream) -> Result<Self> {
523 input.step(|cursor| {
524 if let Some((ident, rest)) = cursor.ident() {
525 if ident == "_" {
526 return Ok((Underscore(ident.span()), rest));
527 }
528 }
529 if let Some((punct, rest)) = cursor.punct() {
530 if punct.as_char() == '_' {
531 return Ok((Underscore(punct.span()), rest));
532 }
533 }
534 Err(cursor.error("expected `_`"))
535 })
536 }
537 }
538
539 #[cfg(feature = "parsing")]
540 impl Token for Underscore {
peek(cursor: Cursor) -> bool541 fn peek(cursor: Cursor) -> bool {
542 if let Some((ident, _rest)) = cursor.ident() {
543 return ident == "_";
544 }
545 if let Some((punct, _rest)) = cursor.punct() {
546 return punct.as_char() == '_';
547 }
548 false
549 }
550
display() -> &'static str551 fn display() -> &'static str {
552 "`_`"
553 }
554 }
555
556 #[cfg(feature = "parsing")]
557 impl private::Sealed for Underscore {}
558
559 #[cfg(feature = "parsing")]
560 impl Token for Paren {
peek(cursor: Cursor) -> bool561 fn peek(cursor: Cursor) -> bool {
562 lookahead::is_delimiter(cursor, Delimiter::Parenthesis)
563 }
564
display() -> &'static str565 fn display() -> &'static str {
566 "parentheses"
567 }
568 }
569
570 #[cfg(feature = "parsing")]
571 impl Token for Brace {
peek(cursor: Cursor) -> bool572 fn peek(cursor: Cursor) -> bool {
573 lookahead::is_delimiter(cursor, Delimiter::Brace)
574 }
575
display() -> &'static str576 fn display() -> &'static str {
577 "curly braces"
578 }
579 }
580
581 #[cfg(feature = "parsing")]
582 impl Token for Bracket {
peek(cursor: Cursor) -> bool583 fn peek(cursor: Cursor) -> bool {
584 lookahead::is_delimiter(cursor, Delimiter::Bracket)
585 }
586
display() -> &'static str587 fn display() -> &'static str {
588 "square brackets"
589 }
590 }
591
592 #[cfg(feature = "parsing")]
593 impl Token for Group {
peek(cursor: Cursor) -> bool594 fn peek(cursor: Cursor) -> bool {
595 lookahead::is_delimiter(cursor, Delimiter::None)
596 }
597
display() -> &'static str598 fn display() -> &'static str {
599 "invisible group"
600 }
601 }
602
603 define_keywords! {
604 "abstract" pub struct Abstract /// `abstract`
605 "as" pub struct As /// `as`
606 "async" pub struct Async /// `async`
607 "auto" pub struct Auto /// `auto`
608 "await" pub struct Await /// `await`
609 "become" pub struct Become /// `become`
610 "box" pub struct Box /// `box`
611 "break" pub struct Break /// `break`
612 "const" pub struct Const /// `const`
613 "continue" pub struct Continue /// `continue`
614 "crate" pub struct Crate /// `crate`
615 "default" pub struct Default /// `default`
616 "do" pub struct Do /// `do`
617 "dyn" pub struct Dyn /// `dyn`
618 "else" pub struct Else /// `else`
619 "enum" pub struct Enum /// `enum`
620 "extern" pub struct Extern /// `extern`
621 "final" pub struct Final /// `final`
622 "fn" pub struct Fn /// `fn`
623 "for" pub struct For /// `for`
624 "if" pub struct If /// `if`
625 "impl" pub struct Impl /// `impl`
626 "in" pub struct In /// `in`
627 "let" pub struct Let /// `let`
628 "loop" pub struct Loop /// `loop`
629 "macro" pub struct Macro /// `macro`
630 "match" pub struct Match /// `match`
631 "mod" pub struct Mod /// `mod`
632 "move" pub struct Move /// `move`
633 "mut" pub struct Mut /// `mut`
634 "override" pub struct Override /// `override`
635 "priv" pub struct Priv /// `priv`
636 "pub" pub struct Pub /// `pub`
637 "ref" pub struct Ref /// `ref`
638 "return" pub struct Return /// `return`
639 "Self" pub struct SelfType /// `Self`
640 "self" pub struct SelfValue /// `self`
641 "static" pub struct Static /// `static`
642 "struct" pub struct Struct /// `struct`
643 "super" pub struct Super /// `super`
644 "trait" pub struct Trait /// `trait`
645 "try" pub struct Try /// `try`
646 "type" pub struct Type /// `type`
647 "typeof" pub struct Typeof /// `typeof`
648 "union" pub struct Union /// `union`
649 "unsafe" pub struct Unsafe /// `unsafe`
650 "unsized" pub struct Unsized /// `unsized`
651 "use" pub struct Use /// `use`
652 "virtual" pub struct Virtual /// `virtual`
653 "where" pub struct Where /// `where`
654 "while" pub struct While /// `while`
655 "yield" pub struct Yield /// `yield`
656 }
657
658 define_punctuation! {
659 "+" pub struct Add/1 /// `+`
660 "+=" pub struct AddEq/2 /// `+=`
661 "&" pub struct And/1 /// `&`
662 "&&" pub struct AndAnd/2 /// `&&`
663 "&=" pub struct AndEq/2 /// `&=`
664 "@" pub struct At/1 /// `@`
665 "!" pub struct Bang/1 /// `!`
666 "^" pub struct Caret/1 /// `^`
667 "^=" pub struct CaretEq/2 /// `^=`
668 ":" pub struct Colon/1 /// `:`
669 "::" pub struct Colon2/2 /// `::`
670 "," pub struct Comma/1 /// `,`
671 "/" pub struct Div/1 /// `/`
672 "/=" pub struct DivEq/2 /// `/=`
673 "$" pub struct Dollar/1 /// `$`
674 "." pub struct Dot/1 /// `.`
675 ".." pub struct Dot2/2 /// `..`
676 "..." pub struct Dot3/3 /// `...`
677 "..=" pub struct DotDotEq/3 /// `..=`
678 "=" pub struct Eq/1 /// `=`
679 "==" pub struct EqEq/2 /// `==`
680 ">=" pub struct Ge/2 /// `>=`
681 ">" pub struct Gt/1 /// `>`
682 "<=" pub struct Le/2 /// `<=`
683 "<" pub struct Lt/1 /// `<`
684 "*=" pub struct MulEq/2 /// `*=`
685 "!=" pub struct Ne/2 /// `!=`
686 "|" pub struct Or/1 /// `|`
687 "|=" pub struct OrEq/2 /// `|=`
688 "||" pub struct OrOr/2 /// `||`
689 "#" pub struct Pound/1 /// `#`
690 "?" pub struct Question/1 /// `?`
691 "->" pub struct RArrow/2 /// `->`
692 "<-" pub struct LArrow/2 /// `<-`
693 "%" pub struct Rem/1 /// `%`
694 "%=" pub struct RemEq/2 /// `%=`
695 "=>" pub struct FatArrow/2 /// `=>`
696 ";" pub struct Semi/1 /// `;`
697 "<<" pub struct Shl/2 /// `<<`
698 "<<=" pub struct ShlEq/3 /// `<<=`
699 ">>" pub struct Shr/2 /// `>>`
700 ">>=" pub struct ShrEq/3 /// `>>=`
701 "*" pub struct Star/1 /// `*`
702 "-" pub struct Sub/1 /// `-`
703 "-=" pub struct SubEq/2 /// `-=`
704 "~" pub struct Tilde/1 /// `~`
705 }
706
707 define_delimiters! {
708 "{" pub struct Brace /// `{...}`
709 "[" pub struct Bracket /// `[...]`
710 "(" pub struct Paren /// `(...)`
711 " " pub struct Group /// None-delimited group
712 }
713
714 macro_rules! export_token_macro {
715 ($($await_rule:tt)*) => {
716 /// A type-macro that expands to the name of the Rust type representation of a
717 /// given token.
718 ///
719 /// See the [token module] documentation for details and examples.
720 ///
721 /// [token module]: crate::token
722 // Unfortunate duplication due to a rustdoc bug.
723 // https://github.com/rust-lang/rust/issues/45939
724 #[macro_export]
725 macro_rules! Token {
726 (abstract) => { $crate::token::Abstract };
727 (as) => { $crate::token::As };
728 (async) => { $crate::token::Async };
729 (auto) => { $crate::token::Auto };
730 $($await_rule => { $crate::token::Await };)*
731 (become) => { $crate::token::Become };
732 (box) => { $crate::token::Box };
733 (break) => { $crate::token::Break };
734 (const) => { $crate::token::Const };
735 (continue) => { $crate::token::Continue };
736 (crate) => { $crate::token::Crate };
737 (default) => { $crate::token::Default };
738 (do) => { $crate::token::Do };
739 (dyn) => { $crate::token::Dyn };
740 (else) => { $crate::token::Else };
741 (enum) => { $crate::token::Enum };
742 (extern) => { $crate::token::Extern };
743 (final) => { $crate::token::Final };
744 (fn) => { $crate::token::Fn };
745 (for) => { $crate::token::For };
746 (if) => { $crate::token::If };
747 (impl) => { $crate::token::Impl };
748 (in) => { $crate::token::In };
749 (let) => { $crate::token::Let };
750 (loop) => { $crate::token::Loop };
751 (macro) => { $crate::token::Macro };
752 (match) => { $crate::token::Match };
753 (mod) => { $crate::token::Mod };
754 (move) => { $crate::token::Move };
755 (mut) => { $crate::token::Mut };
756 (override) => { $crate::token::Override };
757 (priv) => { $crate::token::Priv };
758 (pub) => { $crate::token::Pub };
759 (ref) => { $crate::token::Ref };
760 (return) => { $crate::token::Return };
761 (Self) => { $crate::token::SelfType };
762 (self) => { $crate::token::SelfValue };
763 (static) => { $crate::token::Static };
764 (struct) => { $crate::token::Struct };
765 (super) => { $crate::token::Super };
766 (trait) => { $crate::token::Trait };
767 (try) => { $crate::token::Try };
768 (type) => { $crate::token::Type };
769 (typeof) => { $crate::token::Typeof };
770 (union) => { $crate::token::Union };
771 (unsafe) => { $crate::token::Unsafe };
772 (unsized) => { $crate::token::Unsized };
773 (use) => { $crate::token::Use };
774 (virtual) => { $crate::token::Virtual };
775 (where) => { $crate::token::Where };
776 (while) => { $crate::token::While };
777 (yield) => { $crate::token::Yield };
778 (+) => { $crate::token::Add };
779 (+=) => { $crate::token::AddEq };
780 (&) => { $crate::token::And };
781 (&&) => { $crate::token::AndAnd };
782 (&=) => { $crate::token::AndEq };
783 (@) => { $crate::token::At };
784 (!) => { $crate::token::Bang };
785 (^) => { $crate::token::Caret };
786 (^=) => { $crate::token::CaretEq };
787 (:) => { $crate::token::Colon };
788 (::) => { $crate::token::Colon2 };
789 (,) => { $crate::token::Comma };
790 (/) => { $crate::token::Div };
791 (/=) => { $crate::token::DivEq };
792 ($) => { $crate::token::Dollar };
793 (.) => { $crate::token::Dot };
794 (..) => { $crate::token::Dot2 };
795 (...) => { $crate::token::Dot3 };
796 (..=) => { $crate::token::DotDotEq };
797 (=) => { $crate::token::Eq };
798 (==) => { $crate::token::EqEq };
799 (>=) => { $crate::token::Ge };
800 (>) => { $crate::token::Gt };
801 (<=) => { $crate::token::Le };
802 (<) => { $crate::token::Lt };
803 (*=) => { $crate::token::MulEq };
804 (!=) => { $crate::token::Ne };
805 (|) => { $crate::token::Or };
806 (|=) => { $crate::token::OrEq };
807 (||) => { $crate::token::OrOr };
808 (#) => { $crate::token::Pound };
809 (?) => { $crate::token::Question };
810 (->) => { $crate::token::RArrow };
811 (<-) => { $crate::token::LArrow };
812 (%) => { $crate::token::Rem };
813 (%=) => { $crate::token::RemEq };
814 (=>) => { $crate::token::FatArrow };
815 (;) => { $crate::token::Semi };
816 (<<) => { $crate::token::Shl };
817 (<<=) => { $crate::token::ShlEq };
818 (>>) => { $crate::token::Shr };
819 (>>=) => { $crate::token::ShrEq };
820 (*) => { $crate::token::Star };
821 (-) => { $crate::token::Sub };
822 (-=) => { $crate::token::SubEq };
823 (~) => { $crate::token::Tilde };
824 (_) => { $crate::token::Underscore };
825 }
826 };
827 }
828
829 // Old rustc does not permit `await` appearing anywhere in the source file.
830 // https://github.com/rust-lang/rust/issues/57919
831 // We put the Token![await] rule in a place that is not lexed by old rustc.
832 #[cfg(not(syn_omit_await_from_token_macro))]
833 include!("await.rs"); // export_token_macro![(await)];
834 #[cfg(syn_omit_await_from_token_macro)]
835 export_token_macro![];
836
837 // Not public API.
838 #[doc(hidden)]
839 #[cfg(feature = "parsing")]
840 pub mod parsing {
841 use proc_macro2::{Spacing, Span};
842
843 use crate::buffer::Cursor;
844 use crate::error::{Error, Result};
845 use crate::parse::ParseStream;
846 use crate::span::FromSpans;
847
keyword(input: ParseStream, token: &str) -> Result<Span>848 pub fn keyword(input: ParseStream, token: &str) -> Result<Span> {
849 input.step(|cursor| {
850 if let Some((ident, rest)) = cursor.ident() {
851 if ident == token {
852 return Ok((ident.span(), rest));
853 }
854 }
855 Err(cursor.error(format!("expected `{}`", token)))
856 })
857 }
858
peek_keyword(cursor: Cursor, token: &str) -> bool859 pub fn peek_keyword(cursor: Cursor, token: &str) -> bool {
860 if let Some((ident, _rest)) = cursor.ident() {
861 ident == token
862 } else {
863 false
864 }
865 }
866
punct<S: FromSpans>(input: ParseStream, token: &str) -> Result<S>867 pub fn punct<S: FromSpans>(input: ParseStream, token: &str) -> Result<S> {
868 let mut spans = [input.span(); 3];
869 punct_helper(input, token, &mut spans)?;
870 Ok(S::from_spans(&spans))
871 }
872
punct_helper(input: ParseStream, token: &str, spans: &mut [Span; 3]) -> Result<()>873 fn punct_helper(input: ParseStream, token: &str, spans: &mut [Span; 3]) -> Result<()> {
874 input.step(|cursor| {
875 let mut cursor = *cursor;
876 assert!(token.len() <= spans.len());
877
878 for (i, ch) in token.chars().enumerate() {
879 match cursor.punct() {
880 Some((punct, rest)) => {
881 spans[i] = punct.span();
882 if punct.as_char() != ch {
883 break;
884 } else if i == token.len() - 1 {
885 return Ok(((), rest));
886 } else if punct.spacing() != Spacing::Joint {
887 break;
888 }
889 cursor = rest;
890 }
891 None => break,
892 }
893 }
894
895 Err(Error::new(spans[0], format!("expected `{}`", token)))
896 })
897 }
898
peek_punct(mut cursor: Cursor, token: &str) -> bool899 pub fn peek_punct(mut cursor: Cursor, token: &str) -> bool {
900 for (i, ch) in token.chars().enumerate() {
901 match cursor.punct() {
902 Some((punct, rest)) => {
903 if punct.as_char() != ch {
904 break;
905 } else if i == token.len() - 1 {
906 return true;
907 } else if punct.spacing() != Spacing::Joint {
908 break;
909 }
910 cursor = rest;
911 }
912 None => break,
913 }
914 }
915 false
916 }
917 }
918
919 // Not public API.
920 #[doc(hidden)]
921 #[cfg(feature = "printing")]
922 pub mod printing {
923 use proc_macro2::{Delimiter, Group, Ident, Punct, Spacing, Span, TokenStream};
924 use quote::TokenStreamExt;
925
punct(s: &str, spans: &[Span], tokens: &mut TokenStream)926 pub fn punct(s: &str, spans: &[Span], tokens: &mut TokenStream) {
927 assert_eq!(s.len(), spans.len());
928
929 let mut chars = s.chars();
930 let mut spans = spans.iter();
931 let ch = chars.next_back().unwrap();
932 let span = spans.next_back().unwrap();
933 for (ch, span) in chars.zip(spans) {
934 let mut op = Punct::new(ch, Spacing::Joint);
935 op.set_span(*span);
936 tokens.append(op);
937 }
938
939 let mut op = Punct::new(ch, Spacing::Alone);
940 op.set_span(*span);
941 tokens.append(op);
942 }
943
keyword(s: &str, span: Span, tokens: &mut TokenStream)944 pub fn keyword(s: &str, span: Span, tokens: &mut TokenStream) {
945 tokens.append(Ident::new(s, span));
946 }
947
delim<F>(s: &str, span: Span, tokens: &mut TokenStream, f: F) where F: FnOnce(&mut TokenStream),948 pub fn delim<F>(s: &str, span: Span, tokens: &mut TokenStream, f: F)
949 where
950 F: FnOnce(&mut TokenStream),
951 {
952 let delim = match s {
953 "(" => Delimiter::Parenthesis,
954 "[" => Delimiter::Bracket,
955 "{" => Delimiter::Brace,
956 " " => Delimiter::None,
957 _ => panic!("unknown delimiter: {}", s),
958 };
959 let mut inner = TokenStream::new();
960 f(&mut inner);
961 let mut g = Group::new(delim, inner);
962 g.set_span(span);
963 tokens.append(g);
964 }
965 }
966