Clarify spacing computation.

The spacing computation is done in two parts. In the first part
`next_token` and `bump` use `Spacing::Alone` to mean "preceded by
whitespace" and `Spacing::Joint` to mean the opposite. In the second
part `parse_token_tree_other` then adjusts the `spacing` value to mean
the usual thing (i.e. "is the following token joinable punctuation?").
This shift in meaning is very confusing and it took me some time to
understand what was going on.

This commit changes the first part to use a bool, and adds some
comments, which makes things much clearer.

Author: nnethercote

compiler/rustc_parse/src/lexer/mod.rs

@@ -1,7 +1,7 @@
 use crate::lexer::unicode_chars::UNICODE_ARRAY;
 use rustc_ast::ast::{self, AttrStyle};
 use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind};
-use rustc_ast::tokenstream::{Spacing, TokenStream};
+use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::util::unicode::contains_text_flow_control_chars;
 use rustc_errors::{error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, PResult};
 use rustc_lexer::unescape::{self, Mode};
@@ -67,9 +67,10 @@ impl<'a> StringReader<'a> {
         self.override_span.unwrap_or_else(|| Span::with_root_ctxt(lo, hi))
     }
 
-    /// Returns the next token, and info about preceding whitespace, if any.
-    fn next_token(&mut self) -> (Spacing, Token) {
-        let mut spacing = Spacing::Joint;
+    /// Returns the next token, paired with a bool indicating if the token was
+    /// preceded by whitespace.
+    fn next_token(&mut self) -> (Token, bool) {
+        let mut preceded_by_whitespace = false;
 
         // Skip trivial (whitespace & comments) tokens
         loop {
@@ -78,7 +79,7 @@ impl<'a> StringReader<'a> {
 
             if text.is_empty() {
                 let span = self.mk_sp(self.pos, self.pos);
-                return (spacing, Token::new(token::Eof, span));
+                return (Token::new(token::Eof, span), preceded_by_whitespace);
             }
 
             let token = rustc_lexer::first_token(text);
@@ -91,9 +92,9 @@ impl<'a> StringReader<'a> {
             match self.cook_lexer_token(token.kind, start) {
                 Some(kind) => {
                     let span = self.mk_sp(start, self.pos);
-                    return (spacing, Token::new(kind, span));
+                    return (Token::new(kind, span), preceded_by_whitespace);
                 }
-                None => spacing = Spacing::Alone,
+                None => preceded_by_whitespace = true,
             }
         }
     }

compiler/rustc_parse/src/lexer/tokentrees.rs

@@ -246,16 +246,22 @@ impl<'a> TokenTreesReader<'a> {
 
     #[inline]
     fn parse_token_tree_other(&mut self) -> TokenTree {
+        // `spacing` for the returned token is determined by the next token:
+        // its kind and its `preceded_by_whitespace` status.
         let tok = self.token.take();
-        let mut spacing = self.bump();
-        if !self.token.is_op() {
-            spacing = Spacing::Alone;
-        }
+        let is_next_tok_preceded_by_whitespace = self.bump();
+        let spacing = if is_next_tok_preceded_by_whitespace || !self.token.is_op() {
+            Spacing::Alone
+        } else {
+            Spacing::Joint
+        };
         TokenTree::Token(tok, spacing)
     }
 
-    fn bump(&mut self) -> Spacing {
-        let (spacing, token) = self.string_reader.next_token();
+    // Set `self.token` to the next token. Returns a bool indicating if that
+    // token was preceded by whitespace.
+    fn bump(&mut self) -> bool {
+        let (token, spacing) = self.string_reader.next_token();
         self.token = token;
         spacing
     }