Rollup merge of rust-lang#94368 - c410-f3r:metaaaaaaaaaaaaaaaaaaaaaaaaaaa, r=petrochenkov

[1/2] Implement macro meta-variable expressions

See rust-lang#93545 (comment)

The logic behind `length`, `index` and `count` was removed but the parsing code is still present, i.e., everything is simply ignored like `ignored`.

r? ``@petrochenkov``

Author: Dylan-DPC

compiler/rustc_ast/src/util/literal.rs

@@ -23,7 +23,7 @@ pub enum LitError {
 
 impl LitKind {
     /// Converts literal token into a semantic literal.
-    fn from_lit_token(lit: token::Lit) -> Result<LitKind, LitError> {
+    pub fn from_lit_token(lit: token::Lit) -> Result<LitKind, LitError> {
         let token::Lit { kind, symbol, suffix } = lit;
         if suffix.is_some() && !kind.may_have_suffix() {
             return Err(LitError::InvalidSuffix);

compiler/rustc_expand/src/lib.rs

@@ -3,6 +3,7 @@
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]
 #![feature(if_let_guard)]
+#![feature(let_chains)]
 #![feature(let_else)]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_internals)]

compiler/rustc_expand/src/mbe.rs

@@ -6,17 +6,17 @@
 crate mod macro_check;
 crate mod macro_parser;
 crate mod macro_rules;
+crate mod metavar_expr;
 crate mod quoted;
 crate mod transcribe;
 
+use metavar_expr::MetaVarExpr;
 use rustc_ast::token::{self, NonterminalKind, Token, TokenKind};
 use rustc_ast::tokenstream::DelimSpan;
-
+use rustc_data_structures::sync::Lrc;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
-use rustc_data_structures::sync::Lrc;
-
 /// Contains the sub-token-trees of a "delimited" token tree, such as the contents of `(`. Note
 /// that the delimiter itself might be `NoDelim`.
 #[derive(Clone, PartialEq, Encodable, Decodable, Debug)]
@@ -73,8 +73,8 @@ enum KleeneOp {
     ZeroOrOne,
 }
 
-/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, and `$(...)`
-/// are "first-class" token trees. Useful for parsing macros.
+/// Similar to `tokenstream::TokenTree`, except that `$i`, `$i:ident`, `$(...)`,
+/// and `${...}` are "first-class" token trees. Useful for parsing macros.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
 enum TokenTree {
     Token(Token),
@@ -85,6 +85,8 @@ enum TokenTree {
     MetaVar(Span, Ident),
     /// e.g., `$var:expr`. This is only used in the left hand side of MBE macros.
     MetaVarDecl(Span, Ident /* name to bind */, Option<NonterminalKind>),
+    /// A meta-variable expression inside `${...}`
+    MetaVarExpr(DelimSpan, MetaVarExpr),
 }
 
 impl TokenTree {
@@ -139,7 +141,9 @@ impl TokenTree {
             TokenTree::Token(Token { span, .. })
             | TokenTree::MetaVar(span, _)
             | TokenTree::MetaVarDecl(span, _, _) => span,
-            TokenTree::Delimited(span, _) | TokenTree::Sequence(span, _) => span.entire(),
+            TokenTree::Delimited(span, _)
+            | TokenTree::MetaVarExpr(span, _)
+            | TokenTree::Sequence(span, _) => span.entire(),
         }
     }
 

compiler/rustc_expand/src/mbe/macro_check.rs

@@ -278,6 +278,8 @@ fn check_binders(
                 binders.insert(name, BinderInfo { span, ops: ops.into() });
             }
         }
+        // `MetaVarExpr` can not appear in the LHS of a macro arm
+        TokenTree::MetaVarExpr(..) => {}
         TokenTree::Delimited(_, ref del) => {
             for tt in &del.tts {
                 check_binders(sess, node_id, tt, macros, binders, ops, valid);
@@ -335,6 +337,8 @@ fn check_occurrences(
             let name = MacroRulesNormalizedIdent::new(name);
             check_ops_is_prefix(sess, node_id, macros, binders, ops, span, name);
         }
+        // FIXME(c410-f3r) Check token (https://github.com/rust-lang/rust/issues/93902)
+        TokenTree::MetaVarExpr(..) => {}
         TokenTree::Delimited(_, ref del) => {
             check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid);
         }

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -200,7 +200,7 @@ struct MatcherPos<'root, 'tt> {
 
 // This type is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(MatcherPos<'_, '_>, 192);
+rustc_data_structures::static_assert_size!(MatcherPos<'_, '_>, 240);
 
 impl<'root, 'tt> MatcherPos<'root, 'tt> {
     /// Generates the top-level matcher position in which the "dot" is before the first token of
@@ -321,10 +321,13 @@ pub(super) fn count_names(ms: &[TokenTree]) -> usize {
     ms.iter().fold(0, |count, elt| {
         count
             + match *elt {
-                TokenTree::Sequence(_, ref seq) => seq.num_captures,
                 TokenTree::Delimited(_, ref delim) => count_names(&delim.tts),
                 TokenTree::MetaVar(..) => 0,
                 TokenTree::MetaVarDecl(..) => 1,
+                // FIXME(c410-f3r) MetaVarExpr should be handled instead of being ignored
+                // https://github.com/rust-lang/rust/issues/9390
+                TokenTree::MetaVarExpr(..) => 0,
+                TokenTree::Sequence(_, ref seq) => seq.num_captures,
                 TokenTree::Token(..) => 0,
             }
     })
@@ -436,7 +439,9 @@ fn nameize<I: Iterator<Item = NamedMatch>>(
                 }
                 Occupied(..) => return Err((sp, format!("duplicated bind name: {}", bind_name))),
             },
-            TokenTree::MetaVar(..) | TokenTree::Token(..) => (),
+            // FIXME(c410-f3r) MetaVar and MetaVarExpr should be handled instead of being ignored
+            // https://github.com/rust-lang/rust/issues/9390
+            TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) | TokenTree::Token(..) => {}
         }
 
         Ok(())
@@ -650,7 +655,7 @@ fn inner_parse_loop<'root, 'tt>(
                 // rules. NOTE that this is not necessarily an error unless _all_ items in
                 // `cur_items` end up doing this. There may still be some other matchers that do
                 // end up working out.
-                TokenTree::Token(..) | TokenTree::MetaVar(..) => {}
+                TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => {}
             }
         }
     }

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -580,7 +580,10 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool {
     use mbe::TokenTree;
     for tt in tts {
         match *tt {
-            TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => (),
+            TokenTree::Token(..)
+            | TokenTree::MetaVar(..)
+            | TokenTree::MetaVarDecl(..)
+            | TokenTree::MetaVarExpr(..) => (),
             TokenTree::Delimited(_, ref del) => {
                 if !check_lhs_no_empty_seq(sess, &del.tts) {
                     return false;
@@ -669,7 +672,10 @@ impl FirstSets {
             let mut first = TokenSet::empty();
             for tt in tts.iter().rev() {
                 match *tt {
-                    TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
+                    TokenTree::Token(..)
+                    | TokenTree::MetaVar(..)
+                    | TokenTree::MetaVarDecl(..)
+                    | TokenTree::MetaVarExpr(..) => {
                         first.replace_with(tt.clone());
                     }
                     TokenTree::Delimited(span, ref delimited) => {
@@ -731,7 +737,10 @@ impl FirstSets {
         for tt in tts.iter() {
             assert!(first.maybe_empty);
             match *tt {
-                TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
+                TokenTree::Token(..)
+                | TokenTree::MetaVar(..)
+                | TokenTree::MetaVarDecl(..)
+                | TokenTree::MetaVarExpr(..) => {
                     first.add_one(tt.clone());
                     return first;
                 }
@@ -907,7 +916,10 @@ fn check_matcher_core(
         // First, update `last` so that it corresponds to the set
         // of NT tokens that might end the sequence `... token`.
         match *token {
-            TokenTree::Token(..) | TokenTree::MetaVar(..) | TokenTree::MetaVarDecl(..) => {
+            TokenTree::Token(..)
+            | TokenTree::MetaVar(..)
+            | TokenTree::MetaVarDecl(..)
+            | TokenTree::MetaVarExpr(..) => {
                 if token_can_be_followed_by_any(token) {
                     // don't need to track tokens that work with any,
                     last.replace_with_irrelevant();

compiler/rustc_expand/src/mbe/metavar_expr.rs

@@ -0,0 +1,157 @@
+use rustc_ast::token;
+use rustc_ast::tokenstream::{Cursor, TokenStream, TokenTree};
+use rustc_ast::{LitIntType, LitKind};
+use rustc_ast_pretty::pprust;
+use rustc_errors::{Applicability, PResult};
+use rustc_session::parse::ParseSess;
+use rustc_span::symbol::Ident;
+use rustc_span::Span;
+
+/// A meta-variable expression, for expansions based on properties of meta-variables.
+#[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
+crate enum MetaVarExpr {
+    /// The number of repetitions of an identifier, optionally limited to a number
+    /// of outer-most repetition depths. If the depth limit is `None` then the depth is unlimited.
+    Count(Ident, Option<usize>),
+
+    /// Ignore a meta-variable for repetition without expansion.
+    Ignore(Ident),
+
+    /// The index of the repetition at a particular depth, where 0 is the inner-most
+    /// repetition. The `usize` is the depth.
+    Index(usize),
+
+    /// The length of the repetition at a particular depth, where 0 is the inner-most
+    /// repetition. The `usize` is the depth.
+    Length(usize),
+}
+
+impl MetaVarExpr {
+    /// Attempt to parse a meta-variable expression from a token stream.
+    crate fn parse<'sess>(
+        input: &TokenStream,
+        outer_span: Span,
+        sess: &'sess ParseSess,
+    ) -> PResult<'sess, MetaVarExpr> {
+        let mut tts = input.trees();
+        let ident = parse_ident(&mut tts, sess, outer_span)?;
+        let Some(TokenTree::Delimited(_, token::Paren, args)) = tts.next() else {
+            let msg = "meta-variable expression parameter must be wrapped in parentheses";
+            return Err(sess.span_diagnostic.struct_span_err(ident.span, msg));
+        };
+        check_trailing_token(&mut tts, sess)?;
+        let mut iter = args.trees();
+        let rslt = match &*ident.as_str() {
+            "count" => parse_count(&mut iter, sess, ident.span)?,
+            "ignore" => MetaVarExpr::Ignore(parse_ident(&mut iter, sess, ident.span)?),
+            "index" => MetaVarExpr::Index(parse_depth(&mut iter, sess, ident.span)?),
+            "length" => MetaVarExpr::Length(parse_depth(&mut iter, sess, ident.span)?),
+            _ => {
+                let err_msg = "unrecognized meta-variable expression";
+                let mut err = sess.span_diagnostic.struct_span_err(ident.span, err_msg);
+                err.span_suggestion(
+                    ident.span,
+                    "supported expressions are count, ignore, index and length",
+                    String::new(),
+                    Applicability::MachineApplicable,
+                );
+                return Err(err);
+            }
+        };
+        check_trailing_token(&mut iter, sess)?;
+        Ok(rslt)
+    }
+
+    crate fn ident(&self) -> Option<&Ident> {
+        match self {
+            MetaVarExpr::Count(ident, _) | MetaVarExpr::Ignore(ident) => Some(&ident),
+            MetaVarExpr::Index(..) | MetaVarExpr::Length(..) => None,
+        }
+    }
+}
+
+// Checks if there are any remaining tokens. For example, `${ignore(ident ... a b c ...)}`
+fn check_trailing_token<'sess>(iter: &mut Cursor, sess: &'sess ParseSess) -> PResult<'sess, ()> {
+    if let Some(tt) = iter.next() {
+        let mut diag = sess.span_diagnostic.struct_span_err(
+            tt.span(),
+            &format!("unexpected token: {}", pprust::tt_to_string(&tt)),
+        );
+        diag.span_note(tt.span(), "meta-variable expression must not have trailing tokens");
+        Err(diag)
+    } else {
+        Ok(())
+    }
+}
+
+/// Parse a meta-variable `count` expression: `count(ident[, depth])`
+fn parse_count<'sess>(
+    iter: &mut Cursor,
+    sess: &'sess ParseSess,
+    span: Span,
+) -> PResult<'sess, MetaVarExpr> {
+    let ident = parse_ident(iter, sess, span)?;
+    let depth = if try_eat_comma(iter) { Some(parse_depth(iter, sess, span)?) } else { None };
+    Ok(MetaVarExpr::Count(ident, depth))
+}
+
+/// Parses the depth used by index(depth) and length(depth).
+fn parse_depth<'sess>(
+    iter: &mut Cursor,
+    sess: &'sess ParseSess,
+    span: Span,
+) -> PResult<'sess, usize> {
+    let Some(tt) = iter.next() else { return Ok(0) };
+    let TokenTree::Token(token::Token {
+        kind: token::TokenKind::Literal(lit), ..
+    }) = tt else {
+        return Err(sess.span_diagnostic.struct_span_err(
+            span,
+            "meta-variable expression depth must be a literal"
+        ));
+    };
+    if let Ok(lit_kind) = LitKind::from_lit_token(lit)
+        && let LitKind::Int(n_u128, LitIntType::Unsuffixed) = lit_kind
+        && let Ok(n_usize) = usize::try_from(n_u128)
+    {
+        Ok(n_usize)
+    }
+    else {
+        let msg = "only unsuffixes integer literals are supported in meta-variable expressions";
+        Err(sess.span_diagnostic.struct_span_err(span, msg))
+    }
+}
+
+/// Parses an generic ident
+fn parse_ident<'sess>(
+    iter: &mut Cursor,
+    sess: &'sess ParseSess,
+    span: Span,
+) -> PResult<'sess, Ident> {
+    let err_fn = |msg| sess.span_diagnostic.struct_span_err(span, msg);
+    if let Some(tt) = iter.next() && let TokenTree::Token(token) = tt {
+        if let Some((elem, false)) = token.ident() {
+            return Ok(elem);
+        }
+        let token_str = pprust::token_to_string(&token);
+        let mut err = err_fn(&format!("expected identifier, found `{}`", &token_str));
+        err.span_suggestion(
+            token.span,
+            &format!("try removing `{}`", &token_str),
+            String::new(),
+            Applicability::MaybeIncorrect,
+        );
+        return Err(err);
+    }
+    Err(err_fn("expected identifier"))
+}
+
+/// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
+/// iterator is not modified and the result is `false`.
+fn try_eat_comma(iter: &mut Cursor) -> bool {
+    if let Some(TokenTree::Token(token::Token { kind: token::Comma, .. })) = iter.look_ahead(0) {
+        let _ = iter.next();
+        return true;
+    }
+    false
+}