From 470b0e9c3ca0d08678a9acd38a7308d5291cd1ae Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Sun, 23 Jun 2024 08:07:55 +1000 Subject: [PATCH 1/3] Import `NonterminalKind` in `compiler/rustc_expand/src/mbe/quoted.rs`. So we can omit the `token::` qualifier, which gives more space to some cramped code. --- compiler/rustc_expand/src/mbe/quoted.rs | 55 +++++++++++-------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index fdf187438d3d7..e60e4d70fbb73 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -2,7 +2,7 @@ use crate::errors; use crate::mbe::macro_parser::count_metavar_decls; use crate::mbe::{Delimited, KleeneOp, KleeneToken, MetaVarExpr, SequenceRepetition, TokenTree}; -use rustc_ast::token::{self, Delimiter, IdentIsRaw, Token}; +use rustc_ast::token::{self, Delimiter, IdentIsRaw, NonterminalKind, Token}; use rustc_ast::{tokenstream, NodeId}; use rustc_ast_pretty::pprust; use rustc_feature::Features; @@ -85,36 +85,31 @@ pub(super) fn parse( span.edition() } }; - let kind = - token::NonterminalKind::from_symbol(fragment.name, edition) - .unwrap_or_else(|| { - let help = match fragment.name { - sym::expr_2021 => { - format!( - "fragment specifier `expr_2021` \ - requires Rust 2021 or later\n\ - {VALID_FRAGMENT_NAMES_MSG}" - ) - } - _ if edition().at_least_rust_2021() - && features - .expr_fragment_specifier_2024 => - { - VALID_FRAGMENT_NAMES_MSG_2021.into() - } - _ => VALID_FRAGMENT_NAMES_MSG.into(), - }; - sess.dcx().emit_err( - errors::InvalidFragmentSpecifier { - span, - fragment, - help, - }, - ); - token::NonterminalKind::Ident + let kind = NonterminalKind::from_symbol(fragment.name, edition) + .unwrap_or_else(|| { + let help = match fragment.name { + sym::expr_2021 => { + format!( + "fragment specifier `expr_2021` \ + requires Rust 2021 or later\n\ + {VALID_FRAGMENT_NAMES_MSG}" + ) + } + _ if edition().at_least_rust_2021() + && features.expr_fragment_specifier_2024 => + { + VALID_FRAGMENT_NAMES_MSG_2021.into() + } + _ => VALID_FRAGMENT_NAMES_MSG.into(), + }; + sess.dcx().emit_err(errors::InvalidFragmentSpecifier { + span, + fragment, + help, }); - if kind - == (token::NonterminalKind::Expr2021 { inferred: false }) + NonterminalKind::Ident + }); + if kind == (NonterminalKind::Expr2021 { inferred: false }) && !features.expr_fragment_specifier_2024 { rustc_session::parse::feature_err( From 70fa67c0b2551b68b3d54bdebbb6565c95f25ab7 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Sun, 23 Jun 2024 08:13:41 +1000 Subject: [PATCH 2/3] Tweak some ugly formatting. --- compiler/rustc_expand/src/mbe/macro_rules.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 0050ff10539a8..0045baf7c569b 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -1146,7 +1146,10 @@ fn check_matcher_core<'tt>( // whereas macros from an external crate have a dummy id. if def.id != DUMMY_NODE_ID && matches!(kind, NonterminalKind::PatParam { inferred: true }) - && matches!(next_token, TokenTree::Token(token) if token.kind == BinOp(token::BinOpToken::Or)) + && matches!( + next_token, + TokenTree::Token(token) if token.kind == BinOp(token::BinOpToken::Or) + ) { // It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param. let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl( From e2aa38e6abf9c2ddd06fb2469628ee488dc49e30 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Sun, 23 Jun 2024 08:13:56 +1000 Subject: [PATCH 3/3] Rework pattern and expression nonterminal kinds. Merge `PatParam`/`PatWithOr`, and `Expr`/`Expr2021`, for a few reasons. - It's conceptually nice, because the two pattern kinds and the two expression kinds are very similar. - With expressions in particular, there are several places where both expression kinds get the same treatment. - It removes one unreachable match arm. - Most importantly, for #124141 I will need to introduce a new type `MetaVarKind` that is very similar to `NonterminalKind`, but records a couple of extra fields for expression metavars. It's nicer to have a single `MetaVarKind::Expr` expression variant to hold those extra fields instead of duplicating them across two variants `MetaVarKind::{Expr,Expr2021}`. And then it makes sense for patterns to be treated the same way, and for `NonterminalKind` to also be treated the same way. I also clarified the comments, because I have long found them a little hard to understand. --- compiler/rustc_ast/src/token.rs | 73 +++++++++++-------- compiler/rustc_expand/src/mbe/macro_rules.rs | 20 ++--- compiler/rustc_expand/src/mbe/quoted.rs | 4 +- .../rustc_parse/src/parser/nonterminal.rs | 28 +++---- src/tools/rustfmt/src/parse/macros/mod.rs | 6 +- 5 files changed, 71 insertions(+), 60 deletions(-) diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index cc66cc87652d0..efe1956615216 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -1,6 +1,8 @@ pub use BinOpToken::*; pub use LitKind::*; pub use Nonterminal::*; +pub use NtExprKind::*; +pub use NtPatKind::*; pub use TokenKind::*; use crate::ast; @@ -871,6 +873,27 @@ impl PartialEq for Token { } } +#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)] +pub enum NtPatKind { + // Matches or-patterns. Was written using `pat` in edition 2021 or later. + PatWithOr, + // Doesn't match or-patterns. + // - `inferred`: was written using `pat` in edition 2015 or 2018. + // - `!inferred`: was written using `pat_param`. + PatParam { inferred: bool }, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable)] +pub enum NtExprKind { + // Matches expressions using the post-edition 2024. Was written using + // `expr` in edition 2024 or later. + Expr, + // Matches expressions using the pre-edition 2024 rules. + // - `inferred`: was written using `expr` in edition 2021 or earlier. + // - `!inferred`: was written using `expr_2021`. + Expr2021 { inferred: bool }, +} + #[derive(Clone, Encodable, Decodable)] /// For interpolation during macro expansion. pub enum Nonterminal { @@ -892,19 +915,8 @@ pub enum NonterminalKind { Item, Block, Stmt, - PatParam { - /// Keep track of whether the user used `:pat_param` or `:pat` and we inferred it from the - /// edition of the span. This is used for diagnostics. - inferred: bool, - }, - PatWithOr, - Expr, - /// Matches an expression using the rules from edition 2021 and earlier. - Expr2021 { - /// Keep track of whether the user used `:expr` or `:expr_2021` and we inferred it from the - /// edition of the span. This is used for diagnostics AND feature gating. - inferred: bool, - }, + Pat(NtPatKind), + Expr(NtExprKind), Ty, Ident, Lifetime, @@ -926,20 +938,22 @@ impl NonterminalKind { sym::item => NonterminalKind::Item, sym::block => NonterminalKind::Block, sym::stmt => NonterminalKind::Stmt, - sym::pat => match edition() { - Edition::Edition2015 | Edition::Edition2018 => { - NonterminalKind::PatParam { inferred: true } + sym::pat => { + if edition().at_least_rust_2021() { + NonterminalKind::Pat(PatWithOr) + } else { + NonterminalKind::Pat(PatParam { inferred: true }) } - Edition::Edition2021 | Edition::Edition2024 => NonterminalKind::PatWithOr, - }, - sym::pat_param => NonterminalKind::PatParam { inferred: false }, - sym::expr => match edition() { - Edition::Edition2015 | Edition::Edition2018 | Edition::Edition2021 => { - NonterminalKind::Expr2021 { inferred: true } + } + sym::pat_param => NonterminalKind::Pat(PatParam { inferred: false }), + sym::expr => { + if edition().at_least_rust_2024() { + NonterminalKind::Expr(Expr) + } else { + NonterminalKind::Expr(Expr2021 { inferred: true }) } - Edition::Edition2024 => NonterminalKind::Expr, - }, - sym::expr_2021 => NonterminalKind::Expr2021 { inferred: false }, + } + sym::expr_2021 => NonterminalKind::Expr(Expr2021 { inferred: false }), sym::ty => NonterminalKind::Ty, sym::ident => NonterminalKind::Ident, sym::lifetime => NonterminalKind::Lifetime, @@ -951,15 +965,16 @@ impl NonterminalKind { _ => return None, }) } + fn symbol(self) -> Symbol { match self { NonterminalKind::Item => sym::item, NonterminalKind::Block => sym::block, NonterminalKind::Stmt => sym::stmt, - NonterminalKind::PatParam { inferred: false } => sym::pat_param, - NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat, - NonterminalKind::Expr | NonterminalKind::Expr2021 { inferred: true } => sym::expr, - NonterminalKind::Expr2021 { inferred: false } => sym::expr_2021, + NonterminalKind::Pat(PatParam { inferred: true } | PatWithOr) => sym::pat, + NonterminalKind::Pat(PatParam { inferred: false }) => sym::pat_param, + NonterminalKind::Expr(Expr2021 { inferred: true } | Expr) => sym::expr, + NonterminalKind::Expr(Expr2021 { inferred: false }) => sym::expr_2021, NonterminalKind::Ty => sym::ty, NonterminalKind::Ident => sym::ident, NonterminalKind::Lifetime => sym::lifetime, diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 0045baf7c569b..e43ba7c3a5a8f 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -10,7 +10,9 @@ use crate::mbe::transcribe::transcribe; use ast::token::IdentIsRaw; use rustc_ast as ast; -use rustc_ast::token::{self, Delimiter, NonterminalKind, Token, TokenKind, TokenKind::*}; +use rustc_ast::token::{ + self, Delimiter, NonterminalKind, NtPatKind::*, Token, TokenKind, TokenKind::*, +}; use rustc_ast::tokenstream::{DelimSpan, TokenStream}; use rustc_ast::{NodeId, DUMMY_NODE_ID}; use rustc_ast_pretty::pprust; @@ -1145,7 +1147,7 @@ fn check_matcher_core<'tt>( // Macros defined in the current crate have a real node id, // whereas macros from an external crate have a dummy id. if def.id != DUMMY_NODE_ID - && matches!(kind, NonterminalKind::PatParam { inferred: true }) + && matches!(kind, NonterminalKind::Pat(PatParam { inferred: true })) && matches!( next_token, TokenTree::Token(token) if token.kind == BinOp(token::BinOpToken::Or) @@ -1155,7 +1157,7 @@ fn check_matcher_core<'tt>( let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl( span, name, - Some(NonterminalKind::PatParam { inferred: false }), + Some(NonterminalKind::Pat(PatParam { inferred: false })), )); sess.psess.buffer_lint( RUST_2021_INCOMPATIBLE_OR_PATTERNS, @@ -1188,14 +1190,14 @@ fn check_matcher_core<'tt>( ); err.span_label(sp, format!("not allowed after `{kind}` fragments")); - if kind == NonterminalKind::PatWithOr + if kind == NonterminalKind::Pat(PatWithOr) && sess.psess.edition.at_least_rust_2021() && next_token.is_token(&BinOp(token::BinOpToken::Or)) { let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl( span, name, - Some(NonterminalKind::PatParam { inferred: false }), + Some(NonterminalKind::Pat(PatParam { inferred: false })), )); err.span_suggestion( span, @@ -1295,9 +1297,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { // maintain IsInFollow::Yes } - NonterminalKind::Stmt - | NonterminalKind::Expr - | NonterminalKind::Expr2021 { inferred: _ } => { + NonterminalKind::Stmt | NonterminalKind::Expr(_) => { const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"]; match tok { TokenTree::Token(token) => match token.kind { @@ -1307,7 +1307,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { _ => IsInFollow::No(TOKENS), } } - NonterminalKind::PatParam { .. } => { + NonterminalKind::Pat(PatParam { .. }) => { const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`|`", "`if`", "`in`"]; match tok { TokenTree::Token(token) => match token.kind { @@ -1320,7 +1320,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow { _ => IsInFollow::No(TOKENS), } } - NonterminalKind::PatWithOr => { + NonterminalKind::Pat(PatWithOr) => { const TOKENS: &[&str] = &["`=>`", "`,`", "`=`", "`if`", "`in`"]; match tok { TokenTree::Token(token) => match token.kind { diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs index e60e4d70fbb73..9c480f17b4215 100644 --- a/compiler/rustc_expand/src/mbe/quoted.rs +++ b/compiler/rustc_expand/src/mbe/quoted.rs @@ -2,7 +2,7 @@ use crate::errors; use crate::mbe::macro_parser::count_metavar_decls; use crate::mbe::{Delimited, KleeneOp, KleeneToken, MetaVarExpr, SequenceRepetition, TokenTree}; -use rustc_ast::token::{self, Delimiter, IdentIsRaw, NonterminalKind, Token}; +use rustc_ast::token::{self, Delimiter, IdentIsRaw, NonterminalKind, NtExprKind::*, Token}; use rustc_ast::{tokenstream, NodeId}; use rustc_ast_pretty::pprust; use rustc_feature::Features; @@ -109,7 +109,7 @@ pub(super) fn parse( }); NonterminalKind::Ident }); - if kind == (NonterminalKind::Expr2021 { inferred: false }) + if kind == NonterminalKind::Expr(Expr2021 { inferred: false }) && !features.expr_fragment_specifier_2024 { rustc_session::parse::feature_err( diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs index 59f6eff07b320..4a78b427832c5 100644 --- a/compiler/rustc_parse/src/parser/nonterminal.rs +++ b/compiler/rustc_parse/src/parser/nonterminal.rs @@ -1,5 +1,7 @@ use rustc_ast::ptr::P; -use rustc_ast::token::{self, Delimiter, Nonterminal::*, NonterminalKind, Token}; +use rustc_ast::token::{ + self, Delimiter, Nonterminal::*, NonterminalKind, NtExprKind::*, NtPatKind::*, Token, +}; use rustc_ast::HasTokens; use rustc_ast_pretty::pprust; use rustc_data_structures::sync::Lrc; @@ -36,14 +38,14 @@ impl<'a> Parser<'a> { } match kind { - NonterminalKind::Expr2021 { inferred: _ } => { + NonterminalKind::Expr(Expr2021 { .. }) => { token.can_begin_expr() // This exception is here for backwards compatibility. && !token.is_keyword(kw::Let) // This exception is here for backwards compatibility. && !token.is_keyword(kw::Const) } - NonterminalKind::Expr => { + NonterminalKind::Expr(Expr) => { token.can_begin_expr() // This exception is here for backwards compatibility. && !token.is_keyword(kw::Let) @@ -74,7 +76,7 @@ impl<'a> Parser<'a> { token::Interpolated(nt) => may_be_ident(nt), _ => false, }, - NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind { + NonterminalKind::Pat(pat_kind) => match &token.kind { // box, ref, mut, and other identifiers (can stricten) token::Ident(..) | token::NtIdent(..) | token::OpenDelim(Delimiter::Parenthesis) | // tuple pattern @@ -89,7 +91,7 @@ impl<'a> Parser<'a> { token::Lt | // path (UFCS constant) token::BinOp(token::Shl) => true, // path (double UFCS) // leading vert `|` or-pattern - token::BinOp(token::Or) => matches!(kind, NonterminalKind::PatWithOr), + token::BinOp(token::Or) => matches!(pat_kind, PatWithOr), token::Interpolated(nt) => may_be_ident(nt), _ => false, }, @@ -135,31 +137,25 @@ impl<'a> Parser<'a> { .create_err(UnexpectedNonterminal::Statement(self.token.span))); } }, - NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => { - NtPat(self.collect_tokens_no_attrs(|this| match kind { - NonterminalKind::PatParam { .. } => this.parse_pat_no_top_alt(None, None), - NonterminalKind::PatWithOr => this.parse_pat_allow_top_alt( + NonterminalKind::Pat(pat_kind) => { + NtPat(self.collect_tokens_no_attrs(|this| match pat_kind { + PatParam { .. } => this.parse_pat_no_top_alt(None, None), + PatWithOr => this.parse_pat_allow_top_alt( None, RecoverComma::No, RecoverColon::No, CommaRecoveryMode::EitherTupleOrPipe, ), - _ => unreachable!(), })?) } - - NonterminalKind::Expr | NonterminalKind::Expr2021 { inferred: _ } => { - NtExpr(self.parse_expr_force_collect()?) - } + NonterminalKind::Expr(_) => NtExpr(self.parse_expr_force_collect()?), NonterminalKind::Literal => { // The `:literal` matcher does not support attributes NtLiteral(self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?) } - NonterminalKind::Ty => { NtTy(self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?) } - // this could be handled like a token, since it is one NonterminalKind::Ident => { return if let Some((ident, is_raw)) = get_macro_ident(&self.token) { diff --git a/src/tools/rustfmt/src/parse/macros/mod.rs b/src/tools/rustfmt/src/parse/macros/mod.rs index 89169e10715b2..60c827fd03bbb 100644 --- a/src/tools/rustfmt/src/parse/macros/mod.rs +++ b/src/tools/rustfmt/src/parse/macros/mod.rs @@ -1,4 +1,4 @@ -use rustc_ast::token::{Delimiter, NonterminalKind, TokenKind}; +use rustc_ast::token::{Delimiter, NonterminalKind, NtExprKind::*, NtPatKind::*, TokenKind}; use rustc_ast::tokenstream::TokenStream; use rustc_ast::{ast, ptr}; use rustc_parse::parser::{ForceCollect, Parser, Recovery}; @@ -48,7 +48,7 @@ fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { parse_macro_arg!( Expr, - NonterminalKind::Expr, + NonterminalKind::Expr(Expr), |parser: &mut Parser<'b>| parser.parse_expr(), |x: ptr::P| Some(x) ); @@ -60,7 +60,7 @@ fn parse_macro_arg<'a, 'b: 'a>(parser: &'a mut Parser<'b>) -> Option { ); parse_macro_arg!( Pat, - NonterminalKind::PatParam { inferred: false }, + NonterminalKind::Pat(PatParam { inferred: false }), |parser: &mut Parser<'b>| parser.parse_pat_no_top_alt(None, None), |x: ptr::P| Some(x) );