1 //! Parsing interface for parsing a token stream into a syntax tree node.
2 //!
3 //! Parsing in Syn is built on parser functions that take in a [`ParseStream`]
4 //! and produce a [`Result<T>`] where `T` is some syntax tree node. Underlying
5 //! these parser functions is a lower level mechanism built around the
6 //! [`Cursor`] type. `Cursor` is a cheaply copyable cursor over a range of
7 //! tokens in a token stream.
8 //!
9 //! [`ParseStream`]: type.ParseStream.html
10 //! [`Result<T>`]: type.Result.html
11 //! [`Cursor`]: ../buffer/index.html
12 //!
13 //! # Example
14 //!
15 //! Here is a snippet of parsing code to get a feel for the style of the
16 //! library. We define data structures for a subset of Rust syntax including
17 //! enums (not shown) and structs, then provide implementations of the [`Parse`]
18 //! trait to parse these syntax tree data structures from a token stream.
19 //!
20 //! Once `Parse` impls have been defined, they can be called conveniently from a
21 //! procedural macro through [`parse_macro_input!`] as shown at the bottom of
22 //! the snippet. If the caller provides syntactically invalid input to the
23 //! procedural macro, they will receive a helpful compiler error message
24 //! pointing out the exact token that triggered the failure to parse.
25 //!
26 //! [`parse_macro_input!`]: ../macro.parse_macro_input.html
27 //!
28 //! ```
29 //! # extern crate proc_macro;
30 //! #
31 //! use proc_macro::TokenStream;
32 //! use syn::{braced, parse_macro_input, token, Field, Ident, Result, Token};
33 //! use syn::parse::{Parse, ParseStream};
34 //! use syn::punctuated::Punctuated;
35 //!
36 //! enum Item {
37 //! Struct(ItemStruct),
38 //! Enum(ItemEnum),
39 //! }
40 //!
41 //! struct ItemStruct {
42 //! struct_token: Token![struct],
43 //! ident: Ident,
44 //! brace_token: token::Brace,
45 //! fields: Punctuated<Field, Token![,]>,
46 //! }
47 //! #
48 //! # enum ItemEnum {}
49 //!
50 //! impl Parse for Item {
51 //! fn parse(input: ParseStream) -> Result<Self> {
52 //! let lookahead = input.lookahead1();
53 //! if lookahead.peek(Token![struct]) {
54 //! input.parse().map(Item::Struct)
55 //! } else if lookahead.peek(Token![enum]) {
56 //! input.parse().map(Item::Enum)
57 //! } else {
58 //! Err(lookahead.error())
59 //! }
60 //! }
61 //! }
62 //!
63 //! impl Parse for ItemStruct {
64 //! fn parse(input: ParseStream) -> Result<Self> {
65 //! let content;
66 //! Ok(ItemStruct {
67 //! struct_token: input.parse()?,
68 //! ident: input.parse()?,
69 //! brace_token: braced!(content in input),
70 //! fields: content.parse_terminated(Field::parse_named)?,
71 //! })
72 //! }
73 //! }
74 //! #
75 //! # impl Parse for ItemEnum {
76 //! # fn parse(input: ParseStream) -> Result<Self> {
77 //! # unimplemented!()
78 //! # }
79 //! # }
80 //!
81 //! # const IGNORE: &str = stringify! {
82 //! #[proc_macro]
83 //! # };
84 //! pub fn my_macro(tokens: TokenStream) -> TokenStream {
85 //! let input = parse_macro_input!(tokens as Item);
86 //!
87 //! /* ... */
88 //! # "".parse().unwrap()
89 //! }
90 //! ```
91 //!
92 //! # The `syn::parse*` functions
93 //!
94 //! The [`syn::parse`], [`syn::parse2`], and [`syn::parse_str`] functions serve
95 //! as an entry point for parsing syntax tree nodes that can be parsed in an
96 //! obvious default way. These functions can return any syntax tree node that
97 //! implements the [`Parse`] trait, which includes most types in Syn.
98 //!
99 //! [`syn::parse`]: ../fn.parse.html
100 //! [`syn::parse2`]: ../fn.parse2.html
101 //! [`syn::parse_str`]: ../fn.parse_str.html
102 //! [`Parse`]: trait.Parse.html
103 //!
104 //! ```
105 //! use syn::Type;
106 //!
107 //! # fn run_parser() -> syn::Result<()> {
108 //! let t: Type = syn::parse_str("std::collections::HashMap<String, Value>")?;
109 //! # Ok(())
110 //! # }
111 //! #
112 //! # run_parser().unwrap();
113 //! ```
114 //!
115 //! The [`parse_quote!`] macro also uses this approach.
116 //!
117 //! [`parse_quote!`]: ../macro.parse_quote.html
118 //!
119 //! # The `Parser` trait
120 //!
121 //! Some types can be parsed in several ways depending on context. For example
122 //! an [`Attribute`] can be either "outer" like `#[...]` or "inner" like
123 //! `#![...]` and parsing the wrong one would be a bug. Similarly [`Punctuated`]
124 //! may or may not allow trailing punctuation, and parsing it the wrong way
125 //! would either reject valid input or accept invalid input.
126 //!
127 //! [`Attribute`]: ../struct.Attribute.html
128 //! [`Punctuated`]: ../punctuated/index.html
129 //!
130 //! The `Parse` trait is not implemented in these cases because there is no good
131 //! behavior to consider the default.
132 //!
133 //! ```compile_fail
134 //! # extern crate proc_macro;
135 //! #
136 //! # use syn::punctuated::Punctuated;
137 //! # use syn::{PathSegment, Result, Token};
138 //! #
139 //! # fn f(tokens: proc_macro::TokenStream) -> Result<()> {
140 //! #
141 //! // Can't parse `Punctuated` without knowing whether trailing punctuation
142 //! // should be allowed in this context.
143 //! let path: Punctuated<PathSegment, Token![::]> = syn::parse(tokens)?;
144 //! #
145 //! # Ok(())
146 //! # }
147 //! ```
148 //!
149 //! In these cases the types provide a choice of parser functions rather than a
150 //! single `Parse` implementation, and those parser functions can be invoked
151 //! through the [`Parser`] trait.
152 //!
153 //! [`Parser`]: trait.Parser.html
154 //!
155 //! ```
156 //! # extern crate proc_macro;
157 //! #
158 //! use proc_macro::TokenStream;
159 //! use syn::parse::Parser;
160 //! use syn::punctuated::Punctuated;
161 //! use syn::{Attribute, Expr, PathSegment, Result, Token};
162 //!
163 //! fn call_some_parser_methods(input: TokenStream) -> Result<()> {
164 //! // Parse a nonempty sequence of path segments separated by `::` punctuation
165 //! // with no trailing punctuation.
166 //! let tokens = input.clone();
167 //! let parser = Punctuated::<PathSegment, Token![::]>::parse_separated_nonempty;
168 //! let _path = parser.parse(tokens)?;
169 //!
170 //! // Parse a possibly empty sequence of expressions terminated by commas with
171 //! // an optional trailing punctuation.
172 //! let tokens = input.clone();
173 //! let parser = Punctuated::<Expr, Token![,]>::parse_terminated;
174 //! let _args = parser.parse(tokens)?;
175 //!
176 //! // Parse zero or more outer attributes but not inner attributes.
177 //! let tokens = input.clone();
178 //! let parser = Attribute::parse_outer;
179 //! let _attrs = parser.parse(tokens)?;
180 //!
181 //! Ok(())
182 //! }
183 //! ```
184 //!
185 //! ---
186 //!
187 //! *This module is available only if Syn is built with the `"parsing"` feature.*
188
189 #[path = "discouraged.rs"]
190 pub mod discouraged;
191
192 use std::cell::Cell;
193 use std::fmt::{self, Debug, Display};
194 use std::marker::PhantomData;
195 use std::mem;
196 use std::ops::Deref;
197 use std::rc::Rc;
198 use std::str::FromStr;
199
200 #[cfg(all(
201 not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
202 feature = "proc-macro"
203 ))]
204 use crate::proc_macro;
205 use proc_macro2::{self, Delimiter, Group, Literal, Punct, Span, TokenStream, TokenTree};
206
207 use crate::buffer::{Cursor, TokenBuffer};
208 use crate::error;
209 use crate::lookahead;
210 use crate::punctuated::Punctuated;
211 use crate::token::Token;
212
213 pub use crate::error::{Error, Result};
214 pub use crate::lookahead::{Lookahead1, Peek};
215
216 /// Parsing interface implemented by all types that can be parsed in a default
217 /// way from a token stream.
218 ///
219 /// Refer to the [module documentation] for details about implementing and using
220 /// the `Parse` trait.
221 ///
222 /// [module documentation]: self
223 pub trait Parse: Sized {
parse(input: ParseStream) -> Result<Self>224 fn parse(input: ParseStream) -> Result<Self>;
225 }
226
227 /// Input to a Syn parser function.
228 ///
229 /// See the methods of this type under the documentation of [`ParseBuffer`]. For
230 /// an overview of parsing in Syn, refer to the [module documentation].
231 ///
232 /// [module documentation]: self
233 pub type ParseStream<'a> = &'a ParseBuffer<'a>;
234
235 /// Cursor position within a buffered token stream.
236 ///
237 /// This type is more commonly used through the type alias [`ParseStream`] which
238 /// is an alias for `&ParseBuffer`.
239 ///
240 /// `ParseStream` is the input type for all parser functions in Syn. They have
241 /// the signature `fn(ParseStream) -> Result<T>`.
242 ///
243 /// ## Calling a parser function
244 ///
245 /// There is no public way to construct a `ParseBuffer`. Instead, if you are
246 /// looking to invoke a parser function that requires `ParseStream` as input,
247 /// you will need to go through one of the public parsing entry points.
248 ///
249 /// - The [`parse_macro_input!`] macro if parsing input of a procedural macro;
250 /// - One of [the `syn::parse*` functions][syn-parse]; or
251 /// - A method of the [`Parser`] trait.
252 ///
253 /// [syn-parse]: index.html#the-synparse-functions
254 pub struct ParseBuffer<'a> {
255 scope: Span,
256 // Instead of Cell<Cursor<'a>> so that ParseBuffer<'a> is covariant in 'a.
257 // The rest of the code in this module needs to be careful that only a
258 // cursor derived from this `cell` is ever assigned to this `cell`.
259 //
260 // Cell<Cursor<'a>> cannot be covariant in 'a because then we could take a
261 // ParseBuffer<'a>, upcast to ParseBuffer<'short> for some lifetime shorter
262 // than 'a, and then assign a Cursor<'short> into the Cell.
263 //
264 // By extension, it would not be safe to expose an API that accepts a
265 // Cursor<'a> and trusts that it lives as long as the cursor currently in
266 // the cell.
267 cell: Cell<Cursor<'static>>,
268 marker: PhantomData<Cursor<'a>>,
269 unexpected: Cell<Option<Rc<Cell<Unexpected>>>>,
270 }
271
272 impl<'a> Drop for ParseBuffer<'a> {
drop(&mut self)273 fn drop(&mut self) {
274 if let Some(unexpected_span) = span_of_unexpected_ignoring_nones(self.cursor()) {
275 let (inner, old_span) = inner_unexpected(self);
276 if old_span.is_none() {
277 inner.set(Unexpected::Some(unexpected_span));
278 }
279 }
280 }
281 }
282
283 impl<'a> Display for ParseBuffer<'a> {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result284 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
285 Display::fmt(&self.cursor().token_stream(), f)
286 }
287 }
288
289 impl<'a> Debug for ParseBuffer<'a> {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result290 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
291 Debug::fmt(&self.cursor().token_stream(), f)
292 }
293 }
294
295 /// Cursor state associated with speculative parsing.
296 ///
297 /// This type is the input of the closure provided to [`ParseStream::step`].
298 ///
299 /// [`ParseStream::step`]: ParseBuffer::step
300 ///
301 /// # Example
302 ///
303 /// ```
304 /// use proc_macro2::TokenTree;
305 /// use syn::Result;
306 /// use syn::parse::ParseStream;
307 ///
308 /// // This function advances the stream past the next occurrence of `@`. If
309 /// // no `@` is present in the stream, the stream position is unchanged and
310 /// // an error is returned.
311 /// fn skip_past_next_at(input: ParseStream) -> Result<()> {
312 /// input.step(|cursor| {
313 /// let mut rest = *cursor;
314 /// while let Some((tt, next)) = rest.token_tree() {
315 /// match &tt {
316 /// TokenTree::Punct(punct) if punct.as_char() == '@' => {
317 /// return Ok(((), next));
318 /// }
319 /// _ => rest = next,
320 /// }
321 /// }
322 /// Err(cursor.error("no `@` was found after this point"))
323 /// })
324 /// }
325 /// #
326 /// # fn remainder_after_skipping_past_next_at(
327 /// # input: ParseStream,
328 /// # ) -> Result<proc_macro2::TokenStream> {
329 /// # skip_past_next_at(input)?;
330 /// # input.parse()
331 /// # }
332 /// #
333 /// # use syn::parse::Parser;
334 /// # let remainder = remainder_after_skipping_past_next_at
335 /// # .parse_str("a @ b c")
336 /// # .unwrap();
337 /// # assert_eq!(remainder.to_string(), "b c");
338 /// ```
339 #[derive(Copy, Clone)]
340 pub struct StepCursor<'c, 'a> {
341 scope: Span,
342 // This field is covariant in 'c.
343 cursor: Cursor<'c>,
344 // This field is contravariant in 'c. Together these make StepCursor
345 // invariant in 'c. Also covariant in 'a. The user cannot cast 'c to a
346 // different lifetime but can upcast into a StepCursor with a shorter
347 // lifetime 'a.
348 //
349 // As long as we only ever construct a StepCursor for which 'c outlives 'a,
350 // this means if ever a StepCursor<'c, 'a> exists we are guaranteed that 'c
351 // outlives 'a.
352 marker: PhantomData<fn(Cursor<'c>) -> Cursor<'a>>,
353 }
354
355 impl<'c, 'a> Deref for StepCursor<'c, 'a> {
356 type Target = Cursor<'c>;
357
deref(&self) -> &Self::Target358 fn deref(&self) -> &Self::Target {
359 &self.cursor
360 }
361 }
362
363 impl<'c, 'a> StepCursor<'c, 'a> {
364 /// Triggers an error at the current position of the parse stream.
365 ///
366 /// The `ParseStream::step` invocation will return this same error without
367 /// advancing the stream state.
error<T: Display>(self, message: T) -> Error368 pub fn error<T: Display>(self, message: T) -> Error {
369 error::new_at(self.scope, self.cursor, message)
370 }
371 }
372
advance_step_cursor<'c, 'a>(proof: StepCursor<'c, 'a>, to: Cursor<'c>) -> Cursor<'a>373 pub(crate) fn advance_step_cursor<'c, 'a>(proof: StepCursor<'c, 'a>, to: Cursor<'c>) -> Cursor<'a> {
374 // Refer to the comments within the StepCursor definition. We use the
375 // fact that a StepCursor<'c, 'a> exists as proof that 'c outlives 'a.
376 // Cursor is covariant in its lifetime parameter so we can cast a
377 // Cursor<'c> to one with the shorter lifetime Cursor<'a>.
378 let _ = proof;
379 unsafe { mem::transmute::<Cursor<'c>, Cursor<'a>>(to) }
380 }
381
new_parse_buffer( scope: Span, cursor: Cursor, unexpected: Rc<Cell<Unexpected>>, ) -> ParseBuffer382 pub(crate) fn new_parse_buffer(
383 scope: Span,
384 cursor: Cursor,
385 unexpected: Rc<Cell<Unexpected>>,
386 ) -> ParseBuffer {
387 ParseBuffer {
388 scope,
389 // See comment on `cell` in the struct definition.
390 cell: Cell::new(unsafe { mem::transmute::<Cursor, Cursor<'static>>(cursor) }),
391 marker: PhantomData,
392 unexpected: Cell::new(Some(unexpected)),
393 }
394 }
395
396 #[derive(Clone)]
397 pub(crate) enum Unexpected {
398 None,
399 Some(Span),
400 Chain(Rc<Cell<Unexpected>>),
401 }
402
403 impl Default for Unexpected {
default() -> Self404 fn default() -> Self {
405 Unexpected::None
406 }
407 }
408
409 // We call this on Cell<Unexpected> and Cell<Option<T>> where temporarily
410 // swapping in a None is cheap.
cell_clone<T: Default + Clone>(cell: &Cell<T>) -> T411 fn cell_clone<T: Default + Clone>(cell: &Cell<T>) -> T {
412 let prev = cell.take();
413 let ret = prev.clone();
414 cell.set(prev);
415 ret
416 }
417
inner_unexpected(buffer: &ParseBuffer) -> (Rc<Cell<Unexpected>>, Option<Span>)418 fn inner_unexpected(buffer: &ParseBuffer) -> (Rc<Cell<Unexpected>>, Option<Span>) {
419 let mut unexpected = get_unexpected(buffer);
420 loop {
421 match cell_clone(&unexpected) {
422 Unexpected::None => return (unexpected, None),
423 Unexpected::Some(span) => return (unexpected, Some(span)),
424 Unexpected::Chain(next) => unexpected = next,
425 }
426 }
427 }
428
get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Unexpected>>429 pub(crate) fn get_unexpected(buffer: &ParseBuffer) -> Rc<Cell<Unexpected>> {
430 cell_clone(&buffer.unexpected).unwrap()
431 }
432
span_of_unexpected_ignoring_nones(mut cursor: Cursor) -> Option<Span>433 fn span_of_unexpected_ignoring_nones(mut cursor: Cursor) -> Option<Span> {
434 if cursor.eof() {
435 return None;
436 }
437 while let Some((inner, _span, rest)) = cursor.group(Delimiter::None) {
438 if let Some(unexpected) = span_of_unexpected_ignoring_nones(inner) {
439 return Some(unexpected);
440 }
441 cursor = rest;
442 }
443 if cursor.eof() {
444 None
445 } else {
446 Some(cursor.span())
447 }
448 }
449
450 impl<'a> ParseBuffer<'a> {
451 /// Parses a syntax tree node of type `T`, advancing the position of our
452 /// parse stream past it.
parse<T: Parse>(&self) -> Result<T>453 pub fn parse<T: Parse>(&self) -> Result<T> {
454 T::parse(self)
455 }
456
457 /// Calls the given parser function to parse a syntax tree node of type `T`
458 /// from this stream.
459 ///
460 /// # Example
461 ///
462 /// The parser below invokes [`Attribute::parse_outer`] to parse a vector of
463 /// zero or more outer attributes.
464 ///
465 /// [`Attribute::parse_outer`]: crate::Attribute::parse_outer
466 ///
467 /// ```
468 /// use syn::{Attribute, Ident, Result, Token};
469 /// use syn::parse::{Parse, ParseStream};
470 ///
471 /// // Parses a unit struct with attributes.
472 /// //
473 /// // #[path = "s.tmpl"]
474 /// // struct S;
475 /// struct UnitStruct {
476 /// attrs: Vec<Attribute>,
477 /// struct_token: Token![struct],
478 /// name: Ident,
479 /// semi_token: Token![;],
480 /// }
481 ///
482 /// impl Parse for UnitStruct {
483 /// fn parse(input: ParseStream) -> Result<Self> {
484 /// Ok(UnitStruct {
485 /// attrs: input.call(Attribute::parse_outer)?,
486 /// struct_token: input.parse()?,
487 /// name: input.parse()?,
488 /// semi_token: input.parse()?,
489 /// })
490 /// }
491 /// }
492 /// ```
call<T>(&self, function: fn(ParseStream) -> Result<T>) -> Result<T>493 pub fn call<T>(&self, function: fn(ParseStream) -> Result<T>) -> Result<T> {
494 function(self)
495 }
496
497 /// Looks at the next token in the parse stream to determine whether it
498 /// matches the requested type of token.
499 ///
500 /// Does not advance the position of the parse stream.
501 ///
502 /// # Syntax
503 ///
504 /// Note that this method does not use turbofish syntax. Pass the peek type
505 /// inside of parentheses.
506 ///
507 /// - `input.peek(Token![struct])`
508 /// - `input.peek(Token![==])`
509 /// - `input.peek(Ident)` *(does not accept keywords)*
510 /// - `input.peek(Ident::peek_any)`
511 /// - `input.peek(Lifetime)`
512 /// - `input.peek(token::Brace)`
513 ///
514 /// # Example
515 ///
516 /// In this example we finish parsing the list of supertraits when the next
517 /// token in the input is either `where` or an opening curly brace.
518 ///
519 /// ```
520 /// use syn::{braced, token, Generics, Ident, Result, Token, TypeParamBound};
521 /// use syn::parse::{Parse, ParseStream};
522 /// use syn::punctuated::Punctuated;
523 ///
524 /// // Parses a trait definition containing no associated items.
525 /// //
526 /// // trait Marker<'de, T>: A + B<'de> where Box<T>: Clone {}
527 /// struct MarkerTrait {
528 /// trait_token: Token![trait],
529 /// ident: Ident,
530 /// generics: Generics,
531 /// colon_token: Option<Token![:]>,
532 /// supertraits: Punctuated<TypeParamBound, Token![+]>,
533 /// brace_token: token::Brace,
534 /// }
535 ///
536 /// impl Parse for MarkerTrait {
537 /// fn parse(input: ParseStream) -> Result<Self> {
538 /// let trait_token: Token![trait] = input.parse()?;
539 /// let ident: Ident = input.parse()?;
540 /// let mut generics: Generics = input.parse()?;
541 /// let colon_token: Option<Token![:]> = input.parse()?;
542 ///
543 /// let mut supertraits = Punctuated::new();
544 /// if colon_token.is_some() {
545 /// loop {
546 /// supertraits.push_value(input.parse()?);
547 /// if input.peek(Token![where]) || input.peek(token::Brace) {
548 /// break;
549 /// }
550 /// supertraits.push_punct(input.parse()?);
551 /// }
552 /// }
553 ///
554 /// generics.where_clause = input.parse()?;
555 /// let content;
556 /// let empty_brace_token = braced!(content in input);
557 ///
558 /// Ok(MarkerTrait {
559 /// trait_token,
560 /// ident,
561 /// generics,
562 /// colon_token,
563 /// supertraits,
564 /// brace_token: empty_brace_token,
565 /// })
566 /// }
567 /// }
568 /// ```
peek<T: Peek>(&self, token: T) -> bool569 pub fn peek<T: Peek>(&self, token: T) -> bool {
570 let _ = token;
571 T::Token::peek(self.cursor())
572 }
573
574 /// Looks at the second-next token in the parse stream.
575 ///
576 /// This is commonly useful as a way to implement contextual keywords.
577 ///
578 /// # Example
579 ///
580 /// This example needs to use `peek2` because the symbol `union` is not a
581 /// keyword in Rust. We can't use just `peek` and decide to parse a union if
582 /// the very next token is `union`, because someone is free to write a `mod
583 /// union` and a macro invocation that looks like `union::some_macro! { ...
584 /// }`. In other words `union` is a contextual keyword.
585 ///
586 /// ```
587 /// use syn::{Ident, ItemUnion, Macro, Result, Token};
588 /// use syn::parse::{Parse, ParseStream};
589 ///
590 /// // Parses either a union or a macro invocation.
591 /// enum UnionOrMacro {
592 /// // union MaybeUninit<T> { uninit: (), value: T }
593 /// Union(ItemUnion),
594 /// // lazy_static! { ... }
595 /// Macro(Macro),
596 /// }
597 ///
598 /// impl Parse for UnionOrMacro {
599 /// fn parse(input: ParseStream) -> Result<Self> {
600 /// if input.peek(Token![union]) && input.peek2(Ident) {
601 /// input.parse().map(UnionOrMacro::Union)
602 /// } else {
603 /// input.parse().map(UnionOrMacro::Macro)
604 /// }
605 /// }
606 /// }
607 /// ```
peek2<T: Peek>(&self, token: T) -> bool608 pub fn peek2<T: Peek>(&self, token: T) -> bool {
609 let _ = token;
610 self.cursor().skip().map_or(false, T::Token::peek)
611 }
612
613 /// Looks at the third-next token in the parse stream.
peek3<T: Peek>(&self, token: T) -> bool614 pub fn peek3<T: Peek>(&self, token: T) -> bool {
615 let _ = token;
616 self.cursor()
617 .skip()
618 .and_then(Cursor::skip)
619 .map_or(false, T::Token::peek)
620 }
621
622 /// Parses zero or more occurrences of `T` separated by punctuation of type
623 /// `P`, with optional trailing punctuation.
624 ///
625 /// Parsing continues until the end of this parse stream. The entire content
626 /// of this parse stream must consist of `T` and `P`.
627 ///
628 /// # Example
629 ///
630 /// ```
631 /// # use quote::quote;
632 /// #
633 /// use syn::{parenthesized, token, Ident, Result, Token, Type};
634 /// use syn::parse::{Parse, ParseStream};
635 /// use syn::punctuated::Punctuated;
636 ///
637 /// // Parse a simplified tuple struct syntax like:
638 /// //
639 /// // struct S(A, B);
640 /// struct TupleStruct {
641 /// struct_token: Token![struct],
642 /// ident: Ident,
643 /// paren_token: token::Paren,
644 /// fields: Punctuated<Type, Token![,]>,
645 /// semi_token: Token![;],
646 /// }
647 ///
648 /// impl Parse for TupleStruct {
649 /// fn parse(input: ParseStream) -> Result<Self> {
650 /// let content;
651 /// Ok(TupleStruct {
652 /// struct_token: input.parse()?,
653 /// ident: input.parse()?,
654 /// paren_token: parenthesized!(content in input),
655 /// fields: content.parse_terminated(Type::parse)?,
656 /// semi_token: input.parse()?,
657 /// })
658 /// }
659 /// }
660 /// #
661 /// # let input = quote! {
662 /// # struct S(A, B);
663 /// # };
664 /// # syn::parse2::<TupleStruct>(input).unwrap();
665 /// ```
parse_terminated<T, P: Parse>( &self, parser: fn(ParseStream) -> Result<T>, ) -> Result<Punctuated<T, P>>666 pub fn parse_terminated<T, P: Parse>(
667 &self,
668 parser: fn(ParseStream) -> Result<T>,
669 ) -> Result<Punctuated<T, P>> {
670 Punctuated::parse_terminated_with(self, parser)
671 }
672
673 /// Returns whether there are tokens remaining in this stream.
674 ///
675 /// This method returns true at the end of the content of a set of
676 /// delimiters, as well as at the very end of the complete macro input.
677 ///
678 /// # Example
679 ///
680 /// ```
681 /// use syn::{braced, token, Ident, Item, Result, Token};
682 /// use syn::parse::{Parse, ParseStream};
683 ///
684 /// // Parses a Rust `mod m { ... }` containing zero or more items.
685 /// struct Mod {
686 /// mod_token: Token![mod],
687 /// name: Ident,
688 /// brace_token: token::Brace,
689 /// items: Vec<Item>,
690 /// }
691 ///
692 /// impl Parse for Mod {
693 /// fn parse(input: ParseStream) -> Result<Self> {
694 /// let content;
695 /// Ok(Mod {
696 /// mod_token: input.parse()?,
697 /// name: input.parse()?,
698 /// brace_token: braced!(content in input),
699 /// items: {
700 /// let mut items = Vec::new();
701 /// while !content.is_empty() {
702 /// items.push(content.parse()?);
703 /// }
704 /// items
705 /// },
706 /// })
707 /// }
708 /// }
709 /// ```
is_empty(&self) -> bool710 pub fn is_empty(&self) -> bool {
711 self.cursor().eof()
712 }
713
714 /// Constructs a helper for peeking at the next token in this stream and
715 /// building an error message if it is not one of a set of expected tokens.
716 ///
717 /// # Example
718 ///
719 /// ```
720 /// use syn::{ConstParam, Ident, Lifetime, LifetimeDef, Result, Token, TypeParam};
721 /// use syn::parse::{Parse, ParseStream};
722 ///
723 /// // A generic parameter, a single one of the comma-separated elements inside
724 /// // angle brackets in:
725 /// //
726 /// // fn f<T: Clone, 'a, 'b: 'a, const N: usize>() { ... }
727 /// //
728 /// // On invalid input, lookahead gives us a reasonable error message.
729 /// //
730 /// // error: expected one of: identifier, lifetime, `const`
731 /// // |
732 /// // 5 | fn f<!Sized>() {}
733 /// // | ^
734 /// enum GenericParam {
735 /// Type(TypeParam),
736 /// Lifetime(LifetimeDef),
737 /// Const(ConstParam),
738 /// }
739 ///
740 /// impl Parse for GenericParam {
741 /// fn parse(input: ParseStream) -> Result<Self> {
742 /// let lookahead = input.lookahead1();
743 /// if lookahead.peek(Ident) {
744 /// input.parse().map(GenericParam::Type)
745 /// } else if lookahead.peek(Lifetime) {
746 /// input.parse().map(GenericParam::Lifetime)
747 /// } else if lookahead.peek(Token![const]) {
748 /// input.parse().map(GenericParam::Const)
749 /// } else {
750 /// Err(lookahead.error())
751 /// }
752 /// }
753 /// }
754 /// ```
lookahead1(&self) -> Lookahead1<'a>755 pub fn lookahead1(&self) -> Lookahead1<'a> {
756 lookahead::new(self.scope, self.cursor())
757 }
758
759 /// Forks a parse stream so that parsing tokens out of either the original
760 /// or the fork does not advance the position of the other.
761 ///
762 /// # Performance
763 ///
764 /// Forking a parse stream is a cheap fixed amount of work and does not
765 /// involve copying token buffers. Where you might hit performance problems
766 /// is if your macro ends up parsing a large amount of content more than
767 /// once.
768 ///
769 /// ```
770 /// # use syn::{Expr, Result};
771 /// # use syn::parse::ParseStream;
772 /// #
773 /// # fn bad(input: ParseStream) -> Result<Expr> {
774 /// // Do not do this.
775 /// if input.fork().parse::<Expr>().is_ok() {
776 /// return input.parse::<Expr>();
777 /// }
778 /// # unimplemented!()
779 /// # }
780 /// ```
781 ///
782 /// As a rule, avoid parsing an unbounded amount of tokens out of a forked
783 /// parse stream. Only use a fork when the amount of work performed against
784 /// the fork is small and bounded.
785 ///
786 /// When complex speculative parsing against the forked stream is
787 /// unavoidable, use [`parse::discouraged::Speculative`] to advance the
788 /// original stream once the fork's parse is determined to have been
789 /// successful.
790 ///
791 /// For a lower level way to perform speculative parsing at the token level,
792 /// consider using [`ParseStream::step`] instead.
793 ///
794 /// [`parse::discouraged::Speculative`]: discouraged::Speculative
795 /// [`ParseStream::step`]: ParseBuffer::step
796 ///
797 /// # Example
798 ///
799 /// The parse implementation shown here parses possibly restricted `pub`
800 /// visibilities.
801 ///
802 /// - `pub`
803 /// - `pub(crate)`
804 /// - `pub(self)`
805 /// - `pub(super)`
806 /// - `pub(in some::path)`
807 ///
808 /// To handle the case of visibilities inside of tuple structs, the parser
809 /// needs to distinguish parentheses that specify visibility restrictions
810 /// from parentheses that form part of a tuple type.
811 ///
812 /// ```
813 /// # struct A;
814 /// # struct B;
815 /// # struct C;
816 /// #
817 /// struct S(pub(crate) A, pub (B, C));
818 /// ```
819 ///
820 /// In this example input the first tuple struct element of `S` has
821 /// `pub(crate)` visibility while the second tuple struct element has `pub`
822 /// visibility; the parentheses around `(B, C)` are part of the type rather
823 /// than part of a visibility restriction.
824 ///
825 /// The parser uses a forked parse stream to check the first token inside of
826 /// parentheses after the `pub` keyword. This is a small bounded amount of
827 /// work performed against the forked parse stream.
828 ///
829 /// ```
830 /// use syn::{parenthesized, token, Ident, Path, Result, Token};
831 /// use syn::ext::IdentExt;
832 /// use syn::parse::{Parse, ParseStream};
833 ///
834 /// struct PubVisibility {
835 /// pub_token: Token![pub],
836 /// restricted: Option<Restricted>,
837 /// }
838 ///
839 /// struct Restricted {
840 /// paren_token: token::Paren,
841 /// in_token: Option<Token![in]>,
842 /// path: Path,
843 /// }
844 ///
845 /// impl Parse for PubVisibility {
846 /// fn parse(input: ParseStream) -> Result<Self> {
847 /// let pub_token: Token![pub] = input.parse()?;
848 ///
849 /// if input.peek(token::Paren) {
850 /// let ahead = input.fork();
851 /// let mut content;
852 /// parenthesized!(content in ahead);
853 ///
854 /// if content.peek(Token![crate])
855 /// || content.peek(Token![self])
856 /// || content.peek(Token![super])
857 /// {
858 /// return Ok(PubVisibility {
859 /// pub_token,
860 /// restricted: Some(Restricted {
861 /// paren_token: parenthesized!(content in input),
862 /// in_token: None,
863 /// path: Path::from(content.call(Ident::parse_any)?),
864 /// }),
865 /// });
866 /// } else if content.peek(Token![in]) {
867 /// return Ok(PubVisibility {
868 /// pub_token,
869 /// restricted: Some(Restricted {
870 /// paren_token: parenthesized!(content in input),
871 /// in_token: Some(content.parse()?),
872 /// path: content.call(Path::parse_mod_style)?,
873 /// }),
874 /// });
875 /// }
876 /// }
877 ///
878 /// Ok(PubVisibility {
879 /// pub_token,
880 /// restricted: None,
881 /// })
882 /// }
883 /// }
884 /// ```
fork(&self) -> Self885 pub fn fork(&self) -> Self {
886 ParseBuffer {
887 scope: self.scope,
888 cell: self.cell.clone(),
889 marker: PhantomData,
890 // Not the parent's unexpected. Nothing cares whether the clone
891 // parses all the way unless we `advance_to`.
892 unexpected: Cell::new(Some(Rc::new(Cell::new(Unexpected::None)))),
893 }
894 }
895
896 /// Triggers an error at the current position of the parse stream.
897 ///
898 /// # Example
899 ///
900 /// ```
901 /// use syn::{Expr, Result, Token};
902 /// use syn::parse::{Parse, ParseStream};
903 ///
904 /// // Some kind of loop: `while` or `for` or `loop`.
905 /// struct Loop {
906 /// expr: Expr,
907 /// }
908 ///
909 /// impl Parse for Loop {
910 /// fn parse(input: ParseStream) -> Result<Self> {
911 /// if input.peek(Token![while])
912 /// || input.peek(Token![for])
913 /// || input.peek(Token![loop])
914 /// {
915 /// Ok(Loop {
916 /// expr: input.parse()?,
917 /// })
918 /// } else {
919 /// Err(input.error("expected some kind of loop"))
920 /// }
921 /// }
922 /// }
923 /// ```
error<T: Display>(&self, message: T) -> Error924 pub fn error<T: Display>(&self, message: T) -> Error {
925 error::new_at(self.scope, self.cursor(), message)
926 }
927
928 /// Speculatively parses tokens from this parse stream, advancing the
929 /// position of this stream only if parsing succeeds.
930 ///
931 /// This is a powerful low-level API used for defining the `Parse` impls of
932 /// the basic built-in token types. It is not something that will be used
933 /// widely outside of the Syn codebase.
934 ///
935 /// # Example
936 ///
937 /// ```
938 /// use proc_macro2::TokenTree;
939 /// use syn::Result;
940 /// use syn::parse::ParseStream;
941 ///
942 /// // This function advances the stream past the next occurrence of `@`. If
943 /// // no `@` is present in the stream, the stream position is unchanged and
944 /// // an error is returned.
945 /// fn skip_past_next_at(input: ParseStream) -> Result<()> {
946 /// input.step(|cursor| {
947 /// let mut rest = *cursor;
948 /// while let Some((tt, next)) = rest.token_tree() {
949 /// match &tt {
950 /// TokenTree::Punct(punct) if punct.as_char() == '@' => {
951 /// return Ok(((), next));
952 /// }
953 /// _ => rest = next,
954 /// }
955 /// }
956 /// Err(cursor.error("no `@` was found after this point"))
957 /// })
958 /// }
959 /// #
960 /// # fn remainder_after_skipping_past_next_at(
961 /// # input: ParseStream,
962 /// # ) -> Result<proc_macro2::TokenStream> {
963 /// # skip_past_next_at(input)?;
964 /// # input.parse()
965 /// # }
966 /// #
967 /// # use syn::parse::Parser;
968 /// # let remainder = remainder_after_skipping_past_next_at
969 /// # .parse_str("a @ b c")
970 /// # .unwrap();
971 /// # assert_eq!(remainder.to_string(), "b c");
972 /// ```
step<F, R>(&self, function: F) -> Result<R> where F: for<'c> FnOnce(StepCursor<'c, 'a>) -> Result<(R, Cursor<'c>)>,973 pub fn step<F, R>(&self, function: F) -> Result<R>
974 where
975 F: for<'c> FnOnce(StepCursor<'c, 'a>) -> Result<(R, Cursor<'c>)>,
976 {
977 // Since the user's function is required to work for any 'c, we know
978 // that the Cursor<'c> they return is either derived from the input
979 // StepCursor<'c, 'a> or from a Cursor<'static>.
980 //
981 // It would not be legal to write this function without the invariant
982 // lifetime 'c in StepCursor<'c, 'a>. If this function were written only
983 // in terms of 'a, the user could take our ParseBuffer<'a>, upcast it to
984 // a ParseBuffer<'short> which some shorter lifetime than 'a, invoke
985 // `step` on their ParseBuffer<'short> with a closure that returns
986 // Cursor<'short>, and we would wrongly write that Cursor<'short> into
987 // the Cell intended to hold Cursor<'a>.
988 //
989 // In some cases it may be necessary for R to contain a Cursor<'a>.
990 // Within Syn we solve this using `advance_step_cursor` which uses the
991 // existence of a StepCursor<'c, 'a> as proof that it is safe to cast
992 // from Cursor<'c> to Cursor<'a>. If needed outside of Syn, it would be
993 // safe to expose that API as a method on StepCursor.
994 let (node, rest) = function(StepCursor {
995 scope: self.scope,
996 cursor: self.cell.get(),
997 marker: PhantomData,
998 })?;
999 self.cell.set(rest);
1000 Ok(node)
1001 }
1002
1003 /// Returns the `Span` of the next token in the parse stream, or
1004 /// `Span::call_site()` if this parse stream has completely exhausted its
1005 /// input `TokenStream`.
span(&self) -> Span1006 pub fn span(&self) -> Span {
1007 let cursor = self.cursor();
1008 if cursor.eof() {
1009 self.scope
1010 } else {
1011 crate::buffer::open_span_of_group(cursor)
1012 }
1013 }
1014
1015 /// Provides low-level access to the token representation underlying this
1016 /// parse stream.
1017 ///
1018 /// Cursors are immutable so no operations you perform against the cursor
1019 /// will affect the state of this parse stream.
cursor(&self) -> Cursor<'a>1020 pub fn cursor(&self) -> Cursor<'a> {
1021 self.cell.get()
1022 }
1023
check_unexpected(&self) -> Result<()>1024 fn check_unexpected(&self) -> Result<()> {
1025 match inner_unexpected(self).1 {
1026 Some(span) => Err(Error::new(span, "unexpected token")),
1027 None => Ok(()),
1028 }
1029 }
1030 }
1031
1032 impl<T: Parse> Parse for Box<T> {
parse(input: ParseStream) -> Result<Self>1033 fn parse(input: ParseStream) -> Result<Self> {
1034 input.parse().map(Box::new)
1035 }
1036 }
1037
1038 impl<T: Parse + Token> Parse for Option<T> {
parse(input: ParseStream) -> Result<Self>1039 fn parse(input: ParseStream) -> Result<Self> {
1040 if T::peek(input.cursor()) {
1041 Ok(Some(input.parse()?))
1042 } else {
1043 Ok(None)
1044 }
1045 }
1046 }
1047
1048 impl Parse for TokenStream {
parse(input: ParseStream) -> Result<Self>1049 fn parse(input: ParseStream) -> Result<Self> {
1050 input.step(|cursor| Ok((cursor.token_stream(), Cursor::empty())))
1051 }
1052 }
1053
1054 impl Parse for TokenTree {
parse(input: ParseStream) -> Result<Self>1055 fn parse(input: ParseStream) -> Result<Self> {
1056 input.step(|cursor| match cursor.token_tree() {
1057 Some((tt, rest)) => Ok((tt, rest)),
1058 None => Err(cursor.error("expected token tree")),
1059 })
1060 }
1061 }
1062
1063 impl Parse for Group {
parse(input: ParseStream) -> Result<Self>1064 fn parse(input: ParseStream) -> Result<Self> {
1065 input.step(|cursor| {
1066 for delim in &[Delimiter::Parenthesis, Delimiter::Brace, Delimiter::Bracket] {
1067 if let Some((inside, span, rest)) = cursor.group(*delim) {
1068 let mut group = Group::new(*delim, inside.token_stream());
1069 group.set_span(span);
1070 return Ok((group, rest));
1071 }
1072 }
1073 Err(cursor.error("expected group token"))
1074 })
1075 }
1076 }
1077
1078 impl Parse for Punct {
parse(input: ParseStream) -> Result<Self>1079 fn parse(input: ParseStream) -> Result<Self> {
1080 input.step(|cursor| match cursor.punct() {
1081 Some((punct, rest)) => Ok((punct, rest)),
1082 None => Err(cursor.error("expected punctuation token")),
1083 })
1084 }
1085 }
1086
1087 impl Parse for Literal {
parse(input: ParseStream) -> Result<Self>1088 fn parse(input: ParseStream) -> Result<Self> {
1089 input.step(|cursor| match cursor.literal() {
1090 Some((literal, rest)) => Ok((literal, rest)),
1091 None => Err(cursor.error("expected literal token")),
1092 })
1093 }
1094 }
1095
1096 /// Parser that can parse Rust tokens into a particular syntax tree node.
1097 ///
1098 /// Refer to the [module documentation] for details about parsing in Syn.
1099 ///
1100 /// [module documentation]: self
1101 ///
1102 /// *This trait is available only if Syn is built with the `"parsing"` feature.*
1103 pub trait Parser: Sized {
1104 type Output;
1105
1106 /// Parse a proc-macro2 token stream into the chosen syntax tree node.
1107 ///
1108 /// This function will check that the input is fully parsed. If there are
1109 /// any unparsed tokens at the end of the stream, an error is returned.
parse2(self, tokens: TokenStream) -> Result<Self::Output>1110 fn parse2(self, tokens: TokenStream) -> Result<Self::Output>;
1111
1112 /// Parse tokens of source code into the chosen syntax tree node.
1113 ///
1114 /// This function will check that the input is fully parsed. If there are
1115 /// any unparsed tokens at the end of the stream, an error is returned.
1116 ///
1117 /// *This method is available only if Syn is built with both the `"parsing"` and
1118 /// `"proc-macro"` features.*
1119 #[cfg(all(
1120 not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))),
1121 feature = "proc-macro"
1122 ))]
parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output>1123 fn parse(self, tokens: proc_macro::TokenStream) -> Result<Self::Output> {
1124 self.parse2(proc_macro2::TokenStream::from(tokens))
1125 }
1126
1127 /// Parse a string of Rust code into the chosen syntax tree node.
1128 ///
1129 /// This function will check that the input is fully parsed. If there are
1130 /// any unparsed tokens at the end of the string, an error is returned.
1131 ///
1132 /// # Hygiene
1133 ///
1134 /// Every span in the resulting syntax tree will be set to resolve at the
1135 /// macro call site.
parse_str(self, s: &str) -> Result<Self::Output>1136 fn parse_str(self, s: &str) -> Result<Self::Output> {
1137 self.parse2(proc_macro2::TokenStream::from_str(s)?)
1138 }
1139
1140 // Not public API.
1141 #[doc(hidden)]
1142 #[cfg(any(feature = "full", feature = "derive"))]
__parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output>1143 fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> {
1144 let _ = scope;
1145 self.parse2(tokens)
1146 }
1147
1148 // Not public API.
1149 #[doc(hidden)]
1150 #[cfg(any(feature = "full", feature = "derive"))]
__parse_stream(self, input: ParseStream) -> Result<Self::Output>1151 fn __parse_stream(self, input: ParseStream) -> Result<Self::Output> {
1152 input.parse().and_then(|tokens| self.parse2(tokens))
1153 }
1154 }
1155
tokens_to_parse_buffer(tokens: &TokenBuffer) -> ParseBuffer1156 fn tokens_to_parse_buffer(tokens: &TokenBuffer) -> ParseBuffer {
1157 let scope = Span::call_site();
1158 let cursor = tokens.begin();
1159 let unexpected = Rc::new(Cell::new(Unexpected::None));
1160 new_parse_buffer(scope, cursor, unexpected)
1161 }
1162
1163 impl<F, T> Parser for F
1164 where
1165 F: FnOnce(ParseStream) -> Result<T>,
1166 {
1167 type Output = T;
1168
parse2(self, tokens: TokenStream) -> Result<T>1169 fn parse2(self, tokens: TokenStream) -> Result<T> {
1170 let buf = TokenBuffer::new2(tokens);
1171 let state = tokens_to_parse_buffer(&buf);
1172 let node = self(&state)?;
1173 state.check_unexpected()?;
1174 if let Some(unexpected_span) = span_of_unexpected_ignoring_nones(state.cursor()) {
1175 Err(Error::new(unexpected_span, "unexpected token"))
1176 } else {
1177 Ok(node)
1178 }
1179 }
1180
1181 #[doc(hidden)]
1182 #[cfg(any(feature = "full", feature = "derive"))]
__parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output>1183 fn __parse_scoped(self, scope: Span, tokens: TokenStream) -> Result<Self::Output> {
1184 let buf = TokenBuffer::new2(tokens);
1185 let cursor = buf.begin();
1186 let unexpected = Rc::new(Cell::new(Unexpected::None));
1187 let state = new_parse_buffer(scope, cursor, unexpected);
1188 let node = self(&state)?;
1189 state.check_unexpected()?;
1190 if let Some(unexpected_span) = span_of_unexpected_ignoring_nones(state.cursor()) {
1191 Err(Error::new(unexpected_span, "unexpected token"))
1192 } else {
1193 Ok(node)
1194 }
1195 }
1196
1197 #[doc(hidden)]
1198 #[cfg(any(feature = "full", feature = "derive"))]
__parse_stream(self, input: ParseStream) -> Result<Self::Output>1199 fn __parse_stream(self, input: ParseStream) -> Result<Self::Output> {
1200 self(input)
1201 }
1202 }
1203
1204 #[cfg(any(feature = "full", feature = "derive"))]
parse_scoped<F: Parser>(f: F, scope: Span, tokens: TokenStream) -> Result<F::Output>1205 pub(crate) fn parse_scoped<F: Parser>(f: F, scope: Span, tokens: TokenStream) -> Result<F::Output> {
1206 f.__parse_scoped(scope, tokens)
1207 }
1208
1209 #[cfg(any(feature = "full", feature = "derive"))]
parse_stream<F: Parser>(f: F, input: ParseStream) -> Result<F::Output>1210 pub(crate) fn parse_stream<F: Parser>(f: F, input: ParseStream) -> Result<F::Output> {
1211 f.__parse_stream(input)
1212 }
1213
1214 /// An empty syntax tree node that consumes no tokens when parsed.
1215 ///
1216 /// This is useful for attribute macros that want to ensure they are not
1217 /// provided any attribute args.
1218 ///
1219 /// ```
1220 /// # extern crate proc_macro;
1221 /// #
1222 /// use proc_macro::TokenStream;
1223 /// use syn::parse_macro_input;
1224 /// use syn::parse::Nothing;
1225 ///
1226 /// # const IGNORE: &str = stringify! {
1227 /// #[proc_macro_attribute]
1228 /// # };
1229 /// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream {
1230 /// parse_macro_input!(args as Nothing);
1231 ///
1232 /// /* ... */
1233 /// # "".parse().unwrap()
1234 /// }
1235 /// ```
1236 ///
1237 /// ```text
1238 /// error: unexpected token
1239 /// --> src/main.rs:3:19
1240 /// |
1241 /// 3 | #[my_attr(asdf)]
1242 /// | ^^^^
1243 /// ```
1244 pub struct Nothing;
1245
1246 impl Parse for Nothing {
parse(_input: ParseStream) -> Result<Self>1247 fn parse(_input: ParseStream) -> Result<Self> {
1248 Ok(Nothing)
1249 }
1250 }
1251