1 pub use BinOpToken::*;
2 pub use DelimToken::*;
3 pub use LitKind::*;
4 pub use Nonterminal::*;
5 pub use TokenKind::*;
6
7 use crate::ast;
8 use crate::ptr::P;
9 use crate::tokenstream::TokenTree;
10
11 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
12 use rustc_data_structures::sync::Lrc;
13 use rustc_macros::HashStable_Generic;
14 use rustc_span::symbol::{kw, sym};
15 use rustc_span::symbol::{Ident, Symbol};
16 use rustc_span::{self, edition::Edition, Span, DUMMY_SP};
17 use std::borrow::Cow;
18 use std::{fmt, mem};
19
20 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
21 pub enum CommentKind {
22 Line,
23 Block,
24 }
25
26 #[derive(Clone, PartialEq, Encodable, Decodable, Hash, Debug, Copy)]
27 #[derive(HashStable_Generic)]
28 pub enum BinOpToken {
29 Plus,
30 Minus,
31 Star,
32 Slash,
33 Percent,
34 Caret,
35 And,
36 Or,
37 Shl,
38 Shr,
39 }
40
41 /// A delimiter token.
42 #[derive(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Debug, Copy)]
43 #[derive(HashStable_Generic)]
44 pub enum DelimToken {
45 /// A round parenthesis (i.e., `(` or `)`).
46 Paren,
47 /// A square bracket (i.e., `[` or `]`).
48 Bracket,
49 /// A curly brace (i.e., `{` or `}`).
50 Brace,
51 /// An empty delimiter.
52 NoDelim,
53 }
54
55 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
56 pub enum LitKind {
57 Bool, // AST only, must never appear in a `Token`
58 Byte,
59 Char,
60 Integer,
61 Float,
62 Str,
63 StrRaw(u16), // raw string delimited by `n` hash symbols
64 ByteStr,
65 ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols
66 Err,
67 }
68
69 /// A literal token.
70 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
71 pub struct Lit {
72 pub kind: LitKind,
73 pub symbol: Symbol,
74 pub suffix: Option<Symbol>,
75 }
76
77 impl fmt::Display for Lit {
fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result78 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
79 let Lit { kind, symbol, suffix } = *self;
80 match kind {
81 Byte => write!(f, "b'{}'", symbol)?,
82 Char => write!(f, "'{}'", symbol)?,
83 Str => write!(f, "\"{}\"", symbol)?,
84 StrRaw(n) => write!(
85 f,
86 "r{delim}\"{string}\"{delim}",
87 delim = "#".repeat(n as usize),
88 string = symbol
89 )?,
90 ByteStr => write!(f, "b\"{}\"", symbol)?,
91 ByteStrRaw(n) => write!(
92 f,
93 "br{delim}\"{string}\"{delim}",
94 delim = "#".repeat(n as usize),
95 string = symbol
96 )?,
97 Integer | Float | Bool | Err => write!(f, "{}", symbol)?,
98 }
99
100 if let Some(suffix) = suffix {
101 write!(f, "{}", suffix)?;
102 }
103
104 Ok(())
105 }
106 }
107
108 impl LitKind {
109 /// An English article for the literal token kind.
article(self) -> &'static str110 pub fn article(self) -> &'static str {
111 match self {
112 Integer | Err => "an",
113 _ => "a",
114 }
115 }
116
descr(self) -> &'static str117 pub fn descr(self) -> &'static str {
118 match self {
119 Bool => panic!("literal token contains `Lit::Bool`"),
120 Byte => "byte",
121 Char => "char",
122 Integer => "integer",
123 Float => "float",
124 Str | StrRaw(..) => "string",
125 ByteStr | ByteStrRaw(..) => "byte string",
126 Err => "error",
127 }
128 }
129
may_have_suffix(self) -> bool130 crate fn may_have_suffix(self) -> bool {
131 matches!(self, Integer | Float | Err)
132 }
133 }
134
135 impl Lit {
new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit136 pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
137 Lit { kind, symbol, suffix }
138 }
139 }
140
ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool141 pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
142 let ident_token = Token::new(Ident(name, is_raw), span);
143
144 !ident_token.is_reserved_ident()
145 || ident_token.is_path_segment_keyword()
146 || [
147 kw::Async,
148 kw::Do,
149 kw::Box,
150 kw::Break,
151 kw::Const,
152 kw::Continue,
153 kw::False,
154 kw::For,
155 kw::If,
156 kw::Let,
157 kw::Loop,
158 kw::Match,
159 kw::Move,
160 kw::Return,
161 kw::True,
162 kw::Try,
163 kw::Unsafe,
164 kw::While,
165 kw::Yield,
166 kw::Static,
167 ]
168 .contains(&name)
169 }
170
ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool171 fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool {
172 let ident_token = Token::new(Ident(name, is_raw), span);
173
174 !ident_token.is_reserved_ident()
175 || ident_token.is_path_segment_keyword()
176 || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn]
177 .contains(&name)
178 }
179
180 #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
181 pub enum TokenKind {
182 /* Expression-operator symbols. */
183 Eq,
184 Lt,
185 Le,
186 EqEq,
187 Ne,
188 Ge,
189 Gt,
190 AndAnd,
191 OrOr,
192 Not,
193 Tilde,
194 BinOp(BinOpToken),
195 BinOpEq(BinOpToken),
196
197 /* Structural symbols */
198 At,
199 Dot,
200 DotDot,
201 DotDotDot,
202 DotDotEq,
203 Comma,
204 Semi,
205 Colon,
206 ModSep,
207 RArrow,
208 LArrow,
209 FatArrow,
210 Pound,
211 Dollar,
212 Question,
213 /// Used by proc macros for representing lifetimes, not generated by lexer right now.
214 SingleQuote,
215 /// An opening delimiter (e.g., `{`).
216 OpenDelim(DelimToken),
217 /// A closing delimiter (e.g., `}`).
218 CloseDelim(DelimToken),
219
220 /* Literals */
221 Literal(Lit),
222
223 /// Identifier token.
224 /// Do not forget about `NtIdent` when you want to match on identifiers.
225 /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
226 /// treat regular and interpolated identifiers in the same way.
227 Ident(Symbol, /* is_raw */ bool),
228 /// Lifetime identifier token.
229 /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
230 /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
231 /// treat regular and interpolated lifetime identifiers in the same way.
232 Lifetime(Symbol),
233
234 Interpolated(Lrc<Nonterminal>),
235
236 /// A doc comment token.
237 /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
238 /// similarly to symbols in string literal tokens.
239 DocComment(CommentKind, ast::AttrStyle, Symbol),
240
241 Eof,
242 }
243
244 // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger.
245 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
246 rustc_data_structures::static_assert_size!(TokenKind, 16);
247
248 #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
249 pub struct Token {
250 pub kind: TokenKind,
251 pub span: Span,
252 }
253
254 impl TokenKind {
lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind255 pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
256 Literal(Lit::new(kind, symbol, suffix))
257 }
258
259 // An approximation to proc-macro-style single-character operators used by rustc parser.
260 // If the operator token can be broken into two tokens, the first of which is single-character,
261 // then this function performs that operation, otherwise it returns `None`.
break_two_token_op(&self) -> Option<(TokenKind, TokenKind)>262 pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> {
263 Some(match *self {
264 Le => (Lt, Eq),
265 EqEq => (Eq, Eq),
266 Ne => (Not, Eq),
267 Ge => (Gt, Eq),
268 AndAnd => (BinOp(And), BinOp(And)),
269 OrOr => (BinOp(Or), BinOp(Or)),
270 BinOp(Shl) => (Lt, Lt),
271 BinOp(Shr) => (Gt, Gt),
272 BinOpEq(Plus) => (BinOp(Plus), Eq),
273 BinOpEq(Minus) => (BinOp(Minus), Eq),
274 BinOpEq(Star) => (BinOp(Star), Eq),
275 BinOpEq(Slash) => (BinOp(Slash), Eq),
276 BinOpEq(Percent) => (BinOp(Percent), Eq),
277 BinOpEq(Caret) => (BinOp(Caret), Eq),
278 BinOpEq(And) => (BinOp(And), Eq),
279 BinOpEq(Or) => (BinOp(Or), Eq),
280 BinOpEq(Shl) => (Lt, Le),
281 BinOpEq(Shr) => (Gt, Ge),
282 DotDot => (Dot, Dot),
283 DotDotDot => (Dot, DotDot),
284 ModSep => (Colon, Colon),
285 RArrow => (BinOp(Minus), Gt),
286 LArrow => (Lt, BinOp(Minus)),
287 FatArrow => (Eq, Gt),
288 _ => return None,
289 })
290 }
291
292 /// Returns tokens that are likely to be typed accidentally instead of the current token.
293 /// Enables better error recovery when the wrong token is found.
similar_tokens(&self) -> Option<Vec<TokenKind>>294 pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
295 match *self {
296 Comma => Some(vec![Dot, Lt, Semi]),
297 Semi => Some(vec![Colon, Comma]),
298 _ => None,
299 }
300 }
301
should_end_const_arg(&self) -> bool302 pub fn should_end_const_arg(&self) -> bool {
303 matches!(self, Gt | Ge | BinOp(Shr) | BinOpEq(Shr))
304 }
305 }
306
307 impl Token {
new(kind: TokenKind, span: Span) -> Self308 pub fn new(kind: TokenKind, span: Span) -> Self {
309 Token { kind, span }
310 }
311
312 /// Some token that will be thrown away later.
dummy() -> Self313 pub fn dummy() -> Self {
314 Token::new(TokenKind::Question, DUMMY_SP)
315 }
316
317 /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary.
from_ast_ident(ident: Ident) -> Self318 pub fn from_ast_ident(ident: Ident) -> Self {
319 Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
320 }
321
322 /// Return this token by value and leave a dummy token in its place.
take(&mut self) -> Self323 pub fn take(&mut self) -> Self {
324 mem::replace(self, Token::dummy())
325 }
326
327 /// For interpolated tokens, returns a span of the fragment to which the interpolated
328 /// token refers. For all other tokens this is just a regular span.
329 /// It is particularly important to use this for identifiers and lifetimes
330 /// for which spans affect name resolution and edition checks.
331 /// Note that keywords are also identifiers, so they should use this
332 /// if they keep spans or perform edition checks.
uninterpolated_span(&self) -> Span333 pub fn uninterpolated_span(&self) -> Span {
334 match &self.kind {
335 Interpolated(nt) => nt.span(),
336 _ => self.span,
337 }
338 }
339
is_op(&self) -> bool340 pub fn is_op(&self) -> bool {
341 !matches!(
342 self.kind,
343 OpenDelim(..)
344 | CloseDelim(..)
345 | Literal(..)
346 | DocComment(..)
347 | Ident(..)
348 | Lifetime(..)
349 | Interpolated(..)
350 | Eof
351 )
352 }
353
is_like_plus(&self) -> bool354 pub fn is_like_plus(&self) -> bool {
355 matches!(self.kind, BinOp(Plus) | BinOpEq(Plus))
356 }
357
358 /// Returns `true` if the token can appear at the start of an expression.
can_begin_expr(&self) -> bool359 pub fn can_begin_expr(&self) -> bool {
360 match self.uninterpolate().kind {
361 Ident(name, is_raw) =>
362 ident_can_begin_expr(name, self.span, is_raw), // value name or keyword
363 OpenDelim(..) | // tuple, array or block
364 Literal(..) | // literal
365 Not | // operator not
366 BinOp(Minus) | // unary minus
367 BinOp(Star) | // dereference
368 BinOp(Or) | OrOr | // closure
369 BinOp(And) | // reference
370 AndAnd | // double reference
371 // DotDotDot is no longer supported, but we need some way to display the error
372 DotDot | DotDotDot | DotDotEq | // range notation
373 Lt | BinOp(Shl) | // associated path
374 ModSep | // global path
375 Lifetime(..) | // labeled loop
376 Pound => true, // expression attributes
377 Interpolated(ref nt) => matches!(**nt, NtLiteral(..) |
378 NtExpr(..) |
379 NtBlock(..) |
380 NtPath(..)),
381 _ => false,
382 }
383 }
384
385 /// Returns `true` if the token can appear at the start of a type.
can_begin_type(&self) -> bool386 pub fn can_begin_type(&self) -> bool {
387 match self.uninterpolate().kind {
388 Ident(name, is_raw) =>
389 ident_can_begin_type(name, self.span, is_raw), // type name or keyword
390 OpenDelim(Paren) | // tuple
391 OpenDelim(Bracket) | // array
392 Not | // never
393 BinOp(Star) | // raw pointer
394 BinOp(And) | // reference
395 AndAnd | // double reference
396 Question | // maybe bound in trait object
397 Lifetime(..) | // lifetime bound in trait object
398 Lt | BinOp(Shl) | // associated path
399 ModSep => true, // global path
400 Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)),
401 _ => false,
402 }
403 }
404
405 /// Returns `true` if the token can appear at the start of a const param.
can_begin_const_arg(&self) -> bool406 pub fn can_begin_const_arg(&self) -> bool {
407 match self.kind {
408 OpenDelim(Brace) => true,
409 Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
410 _ => self.can_begin_literal_maybe_minus(),
411 }
412 }
413
414 /// Returns `true` if the token can appear at the start of a generic bound.
can_begin_bound(&self) -> bool415 pub fn can_begin_bound(&self) -> bool {
416 self.is_path_start()
417 || self.is_lifetime()
418 || self.is_keyword(kw::For)
419 || self == &Question
420 || self == &OpenDelim(Paren)
421 }
422
423 /// Returns `true` if the token is any literal.
is_lit(&self) -> bool424 pub fn is_lit(&self) -> bool {
425 matches!(self.kind, Literal(..))
426 }
427
428 /// Returns `true` if the token is any literal, a minus (which can prefix a literal,
429 /// for example a '-42', or one of the boolean idents).
430 ///
431 /// In other words, would this token be a valid start of `parse_literal_maybe_minus`?
432 ///
433 /// Keep this in sync with and `Lit::from_token`, excluding unary negation.
can_begin_literal_maybe_minus(&self) -> bool434 pub fn can_begin_literal_maybe_minus(&self) -> bool {
435 match self.uninterpolate().kind {
436 Literal(..) | BinOp(Minus) => true,
437 Ident(name, false) if name.is_bool_lit() => true,
438 Interpolated(ref nt) => match &**nt {
439 NtLiteral(_) => true,
440 NtExpr(e) => match &e.kind {
441 ast::ExprKind::Lit(_) => true,
442 ast::ExprKind::Unary(ast::UnOp::Neg, e) => {
443 matches!(&e.kind, ast::ExprKind::Lit(_))
444 }
445 _ => false,
446 },
447 _ => false,
448 },
449 _ => false,
450 }
451 }
452
453 // A convenience function for matching on identifiers during parsing.
454 // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
455 // into the regular identifier or lifetime token it refers to,
456 // otherwise returns the original token.
uninterpolate(&self) -> Cow<'_, Token>457 pub fn uninterpolate(&self) -> Cow<'_, Token> {
458 match &self.kind {
459 Interpolated(nt) => match **nt {
460 NtIdent(ident, is_raw) => {
461 Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
462 }
463 NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
464 _ => Cow::Borrowed(self),
465 },
466 _ => Cow::Borrowed(self),
467 }
468 }
469
470 /// Returns an identifier if this token is an identifier.
ident(&self) -> Option<(Ident, bool)>471 pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> {
472 let token = self.uninterpolate();
473 match token.kind {
474 Ident(name, is_raw) => Some((Ident::new(name, token.span), is_raw)),
475 _ => None,
476 }
477 }
478
479 /// Returns a lifetime identifier if this token is a lifetime.
lifetime(&self) -> Option<Ident>480 pub fn lifetime(&self) -> Option<Ident> {
481 let token = self.uninterpolate();
482 match token.kind {
483 Lifetime(name) => Some(Ident::new(name, token.span)),
484 _ => None,
485 }
486 }
487
488 /// Returns `true` if the token is an identifier.
is_ident(&self) -> bool489 pub fn is_ident(&self) -> bool {
490 self.ident().is_some()
491 }
492
493 /// Returns `true` if the token is a lifetime.
is_lifetime(&self) -> bool494 pub fn is_lifetime(&self) -> bool {
495 self.lifetime().is_some()
496 }
497
498 /// Returns `true` if the token is a identifier whose name is the given
499 /// string slice.
is_ident_named(&self, name: Symbol) -> bool500 pub fn is_ident_named(&self, name: Symbol) -> bool {
501 self.ident().map_or(false, |(ident, _)| ident.name == name)
502 }
503
504 /// Returns `true` if the token is an interpolated path.
is_path(&self) -> bool505 fn is_path(&self) -> bool {
506 if let Interpolated(ref nt) = self.kind {
507 if let NtPath(..) = **nt {
508 return true;
509 }
510 }
511 false
512 }
513
514 /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`?
515 /// That is, is this a pre-parsed expression dropped into the token stream
516 /// (which happens while parsing the result of macro expansion)?
is_whole_expr(&self) -> bool517 pub fn is_whole_expr(&self) -> bool {
518 if let Interpolated(ref nt) = self.kind {
519 if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt {
520 return true;
521 }
522 }
523
524 false
525 }
526
527 // Is the token an interpolated block (`$b:block`)?
is_whole_block(&self) -> bool528 pub fn is_whole_block(&self) -> bool {
529 if let Interpolated(ref nt) = self.kind {
530 if let NtBlock(..) = **nt {
531 return true;
532 }
533 }
534 false
535 }
536
537 /// Returns `true` if the token is either the `mut` or `const` keyword.
is_mutability(&self) -> bool538 pub fn is_mutability(&self) -> bool {
539 self.is_keyword(kw::Mut) || self.is_keyword(kw::Const)
540 }
541
is_qpath_start(&self) -> bool542 pub fn is_qpath_start(&self) -> bool {
543 self == &Lt || self == &BinOp(Shl)
544 }
545
is_path_start(&self) -> bool546 pub fn is_path_start(&self) -> bool {
547 self == &ModSep
548 || self.is_qpath_start()
549 || self.is_path()
550 || self.is_path_segment_keyword()
551 || self.is_ident() && !self.is_reserved_ident()
552 }
553
554 /// Returns `true` if the token is a given keyword, `kw`.
is_keyword(&self, kw: Symbol) -> bool555 pub fn is_keyword(&self, kw: Symbol) -> bool {
556 self.is_non_raw_ident_where(|id| id.name == kw)
557 }
558
is_path_segment_keyword(&self) -> bool559 pub fn is_path_segment_keyword(&self) -> bool {
560 self.is_non_raw_ident_where(Ident::is_path_segment_keyword)
561 }
562
563 // Returns true for reserved identifiers used internally for elided lifetimes,
564 // unnamed method parameters, crate root module, error recovery etc.
is_special_ident(&self) -> bool565 pub fn is_special_ident(&self) -> bool {
566 self.is_non_raw_ident_where(Ident::is_special)
567 }
568
569 /// Returns `true` if the token is a keyword used in the language.
is_used_keyword(&self) -> bool570 pub fn is_used_keyword(&self) -> bool {
571 self.is_non_raw_ident_where(Ident::is_used_keyword)
572 }
573
574 /// Returns `true` if the token is a keyword reserved for possible future use.
is_unused_keyword(&self) -> bool575 pub fn is_unused_keyword(&self) -> bool {
576 self.is_non_raw_ident_where(Ident::is_unused_keyword)
577 }
578
579 /// Returns `true` if the token is either a special identifier or a keyword.
is_reserved_ident(&self) -> bool580 pub fn is_reserved_ident(&self) -> bool {
581 self.is_non_raw_ident_where(Ident::is_reserved)
582 }
583
584 /// Returns `true` if the token is the identifier `true` or `false`.
is_bool_lit(&self) -> bool585 pub fn is_bool_lit(&self) -> bool {
586 self.is_non_raw_ident_where(|id| id.name.is_bool_lit())
587 }
588
589 /// Returns `true` if the token is a non-raw identifier for which `pred` holds.
is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool590 pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
591 match self.ident() {
592 Some((id, false)) => pred(id),
593 _ => false,
594 }
595 }
596
glue(&self, joint: &Token) -> Option<Token>597 pub fn glue(&self, joint: &Token) -> Option<Token> {
598 let kind = match self.kind {
599 Eq => match joint.kind {
600 Eq => EqEq,
601 Gt => FatArrow,
602 _ => return None,
603 },
604 Lt => match joint.kind {
605 Eq => Le,
606 Lt => BinOp(Shl),
607 Le => BinOpEq(Shl),
608 BinOp(Minus) => LArrow,
609 _ => return None,
610 },
611 Gt => match joint.kind {
612 Eq => Ge,
613 Gt => BinOp(Shr),
614 Ge => BinOpEq(Shr),
615 _ => return None,
616 },
617 Not => match joint.kind {
618 Eq => Ne,
619 _ => return None,
620 },
621 BinOp(op) => match joint.kind {
622 Eq => BinOpEq(op),
623 BinOp(And) if op == And => AndAnd,
624 BinOp(Or) if op == Or => OrOr,
625 Gt if op == Minus => RArrow,
626 _ => return None,
627 },
628 Dot => match joint.kind {
629 Dot => DotDot,
630 DotDot => DotDotDot,
631 _ => return None,
632 },
633 DotDot => match joint.kind {
634 Dot => DotDotDot,
635 Eq => DotDotEq,
636 _ => return None,
637 },
638 Colon => match joint.kind {
639 Colon => ModSep,
640 _ => return None,
641 },
642 SingleQuote => match joint.kind {
643 Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))),
644 _ => return None,
645 },
646
647 Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
648 | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
649 | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
650 | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
651 };
652
653 Some(Token::new(kind, self.span.to(joint.span)))
654 }
655 }
656
657 impl PartialEq<TokenKind> for Token {
eq(&self, rhs: &TokenKind) -> bool658 fn eq(&self, rhs: &TokenKind) -> bool {
659 self.kind == *rhs
660 }
661 }
662
663 #[derive(Clone, Encodable, Decodable)]
664 /// For interpolation during macro expansion.
665 pub enum Nonterminal {
666 NtItem(P<ast::Item>),
667 NtBlock(P<ast::Block>),
668 NtStmt(ast::Stmt),
669 NtPat(P<ast::Pat>),
670 NtExpr(P<ast::Expr>),
671 NtTy(P<ast::Ty>),
672 NtIdent(Ident, /* is_raw */ bool),
673 NtLifetime(Ident),
674 NtLiteral(P<ast::Expr>),
675 /// Stuff inside brackets for attributes
676 NtMeta(P<ast::AttrItem>),
677 NtPath(ast::Path),
678 NtVis(ast::Visibility),
679 NtTT(TokenTree),
680 }
681
682 // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
683 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
684 rustc_data_structures::static_assert_size!(Nonterminal, 48);
685
686 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
687 pub enum NonterminalKind {
688 Item,
689 Block,
690 Stmt,
691 PatParam {
692 /// Keep track of whether the user used `:pat_param` or `:pat` and we inferred it from the
693 /// edition of the span. This is used for diagnostics.
694 inferred: bool,
695 },
696 PatWithOr,
697 Expr,
698 Ty,
699 Ident,
700 Lifetime,
701 Literal,
702 Meta,
703 Path,
704 Vis,
705 TT,
706 }
707
708 impl NonterminalKind {
709 /// The `edition` closure is used to get the edition for the given symbol. Doing
710 /// `span.edition()` is expensive, so we do it lazily.
from_symbol( symbol: Symbol, edition: impl FnOnce() -> Edition, ) -> Option<NonterminalKind>711 pub fn from_symbol(
712 symbol: Symbol,
713 edition: impl FnOnce() -> Edition,
714 ) -> Option<NonterminalKind> {
715 Some(match symbol {
716 sym::item => NonterminalKind::Item,
717 sym::block => NonterminalKind::Block,
718 sym::stmt => NonterminalKind::Stmt,
719 sym::pat => match edition() {
720 Edition::Edition2015 | Edition::Edition2018 => {
721 NonterminalKind::PatParam { inferred: true }
722 }
723 Edition::Edition2021 => NonterminalKind::PatWithOr,
724 },
725 sym::pat_param => NonterminalKind::PatParam { inferred: false },
726 sym::expr => NonterminalKind::Expr,
727 sym::ty => NonterminalKind::Ty,
728 sym::ident => NonterminalKind::Ident,
729 sym::lifetime => NonterminalKind::Lifetime,
730 sym::literal => NonterminalKind::Literal,
731 sym::meta => NonterminalKind::Meta,
732 sym::path => NonterminalKind::Path,
733 sym::vis => NonterminalKind::Vis,
734 sym::tt => NonterminalKind::TT,
735 _ => return None,
736 })
737 }
symbol(self) -> Symbol738 fn symbol(self) -> Symbol {
739 match self {
740 NonterminalKind::Item => sym::item,
741 NonterminalKind::Block => sym::block,
742 NonterminalKind::Stmt => sym::stmt,
743 NonterminalKind::PatParam { inferred: false } => sym::pat_param,
744 NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat,
745 NonterminalKind::Expr => sym::expr,
746 NonterminalKind::Ty => sym::ty,
747 NonterminalKind::Ident => sym::ident,
748 NonterminalKind::Lifetime => sym::lifetime,
749 NonterminalKind::Literal => sym::literal,
750 NonterminalKind::Meta => sym::meta,
751 NonterminalKind::Path => sym::path,
752 NonterminalKind::Vis => sym::vis,
753 NonterminalKind::TT => sym::tt,
754 }
755 }
756 }
757
758 impl fmt::Display for NonterminalKind {
fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result759 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
760 write!(f, "{}", self.symbol())
761 }
762 }
763
764 impl Nonterminal {
span(&self) -> Span765 pub fn span(&self) -> Span {
766 match self {
767 NtItem(item) => item.span,
768 NtBlock(block) => block.span,
769 NtStmt(stmt) => stmt.span,
770 NtPat(pat) => pat.span,
771 NtExpr(expr) | NtLiteral(expr) => expr.span,
772 NtTy(ty) => ty.span,
773 NtIdent(ident, _) | NtLifetime(ident) => ident.span,
774 NtMeta(attr_item) => attr_item.span(),
775 NtPath(path) => path.span,
776 NtVis(vis) => vis.span,
777 NtTT(tt) => tt.span(),
778 }
779 }
780 }
781
782 impl PartialEq for Nonterminal {
eq(&self, rhs: &Self) -> bool783 fn eq(&self, rhs: &Self) -> bool {
784 match (self, rhs) {
785 (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
786 ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
787 }
788 (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
789 (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs,
790 // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
791 // correctly based on data from AST. This will prevent them from matching each other
792 // in macros. The comparison will become possible only when each nonterminal has an
793 // attached token stream from which it was parsed.
794 _ => false,
795 }
796 }
797 }
798
799 impl fmt::Debug for Nonterminal {
fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result800 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
801 match *self {
802 NtItem(..) => f.pad("NtItem(..)"),
803 NtBlock(..) => f.pad("NtBlock(..)"),
804 NtStmt(..) => f.pad("NtStmt(..)"),
805 NtPat(..) => f.pad("NtPat(..)"),
806 NtExpr(..) => f.pad("NtExpr(..)"),
807 NtTy(..) => f.pad("NtTy(..)"),
808 NtIdent(..) => f.pad("NtIdent(..)"),
809 NtLiteral(..) => f.pad("NtLiteral(..)"),
810 NtMeta(..) => f.pad("NtMeta(..)"),
811 NtPath(..) => f.pad("NtPath(..)"),
812 NtTT(..) => f.pad("NtTT(..)"),
813 NtVis(..) => f.pad("NtVis(..)"),
814 NtLifetime(..) => f.pad("NtLifetime(..)"),
815 }
816 }
817 }
818
819 impl<CTX> HashStable<CTX> for Nonterminal
820 where
821 CTX: crate::HashStableContext,
822 {
hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher)823 fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
824 panic!("interpolated tokens should not be present in the HIR")
825 }
826 }
827