Speed up Parser::expected_token_types.

The parser pushes a `TokenType` to `Parser::expected_token_types` on
every call to the various `check`/`eat` methods, and clears it on every
call to `bump`. Some of those `TokenType` values are full tokens that
require cloning and dropping. This is a *lot* of work for something
that is only used in error messages and it accounts for a significant
fraction of parsing execution time.

This commit overhauls `TokenType` so that `Parser::expected_token_types`
can be implemented as a bitset. This requires changing `TokenType` to a
C-style parameterless enum, and adding `TokenTypeSet` which uses a
`u128` for the bits. (The new `TokenType` has 105 variants.)

The new types `ExpTokenPair` and `ExpKeywordPair` are now arguments to
the `check`/`eat` methods. This is for maximum speed. The elements in
the pairs are always statically known; e.g. a
`token::BinOp(token::Star)` is always paired with a `TokenType::Star`.
So we now compute `TokenType`s in advance and pass them in to
`check`/`eat` rather than the current approach of constructing them on
insertion into `expected_token_types`.

Values of these pair types can be produced by the new `exp!` macro,
which is used at every `check`/`eat` call site. The macro is for
convenience, allowing any pair to be generated from a single identifier.

The ident/keyword filtering in `expected_one_of_not_found` is no longer
necessary. It was there to account for some sloppiness in
`TokenKind`/`TokenType` comparisons.

The existing `TokenType` is moved to a new file `token_type.rs`, and all
its new infrastructure is added to that file. There is more boilerplate
code than I would like, but I can't see how to make it shorter.

Author: nnethercote

src/parse/macros/cfg_if.rs

@@ -2,6 +2,7 @@ use std::panic::{AssertUnwindSafe, catch_unwind};
 
 use rustc_ast::ast;
 use rustc_ast::token::{Delimiter, TokenKind};
+use rustc_parse::exp;
 use rustc_parse::parser::ForceCollect;
 use rustc_span::symbol::kw;
 
@@ -31,7 +32,7 @@ fn parse_cfg_if_inner<'a>(
 
     while parser.token.kind != TokenKind::Eof {
         if process_if_cfg {
-            if !parser.eat_keyword(kw::If) {
+            if !parser.eat_keyword(exp!(If)) {
                 return Err("Expected `if`");
             }
 
@@ -55,7 +56,7 @@ fn parse_cfg_if_inner<'a>(
                 })?;
         }
 
-        if !parser.eat(&TokenKind::OpenDelim(Delimiter::Brace)) {
+        if !parser.eat(exp!(OpenBrace)) {
             return Err("Expected an opening brace");
         }
 
@@ -78,15 +79,15 @@ fn parse_cfg_if_inner<'a>(
             }
         }
 
-        if !parser.eat(&TokenKind::CloseDelim(Delimiter::Brace)) {
+        if !parser.eat(exp!(CloseBrace)) {
             return Err("Expected a closing brace");
         }
 
-        if parser.eat(&TokenKind::Eof) {
+        if parser.eat(exp!(Eof)) {
             break;
         }
 
-        if !parser.eat_keyword(kw::Else) {
+        if !parser.eat_keyword(exp!(Else)) {
             return Err("Expected `else`");
         }
 

src/parse/macros/lazy_static.rs

@@ -1,8 +1,9 @@
 use rustc_ast::ast;
 use rustc_ast::ptr::P;
-use rustc_ast::token::TokenKind;
+use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
-use rustc_span::symbol::{self, kw};
+use rustc_parse::exp;
+use rustc_span::symbol;
 
 use crate::rewrite::RewriteContext;
 
@@ -31,19 +32,19 @@ pub(crate) fn parse_lazy_static(
             }
         }
     }
-    while parser.token.kind != TokenKind::Eof {
+    while parser.token.kind != token::Eof {
         // Parse a `lazy_static!` item.
         // FIXME: These `eat_*` calls should be converted to `parse_or` to avoid
         // silently formatting malformed lazy-statics.
         let vis = parse_or!(parse_visibility, rustc_parse::parser::FollowedByType::No);
-        let _ = parser.eat_keyword(kw::Static);
-        let _ = parser.eat_keyword(kw::Ref);
+        let _ = parser.eat_keyword(exp!(Static));
+        let _ = parser.eat_keyword(exp!(Ref));
         let id = parse_or!(parse_ident);
-        let _ = parser.eat(&TokenKind::Colon);
+        let _ = parser.eat(exp!(Colon));
         let ty = parse_or!(parse_ty);
-        let _ = parser.eat(&TokenKind::Eq);
+        let _ = parser.eat(exp!(Eq));
         let expr = parse_or!(parse_expr);
-        let _ = parser.eat(&TokenKind::Semi);
+        let _ = parser.eat(exp!(Semi));
         result.push((vis, id, ty, expr));
     }
 

src/parse/parser.rs

@@ -1,11 +1,10 @@
 use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::path::{Path, PathBuf};
 
-use rustc_ast::token::TokenKind;
 use rustc_ast::{ast, attr, ptr};
 use rustc_errors::Diag;
 use rustc_parse::parser::Parser as RawParser;
-use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
+use rustc_parse::{exp, new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_span::{Span, sym};
 use thin_vec::ThinVec;
 
@@ -107,7 +106,7 @@ impl<'a> Parser<'a> {
         let result = catch_unwind(AssertUnwindSafe(|| {
             let mut parser =
                 unwrap_or_emit_fatal(new_parser_from_file(psess.inner(), path, Some(span)));
-            match parser.parse_mod(&TokenKind::Eof) {
+            match parser.parse_mod(exp!(Eof)) {
                 Ok((a, i, spans)) => Some((a, i, spans.inner_span)),
                 Err(e) => {
                     e.emit();