Skip to content
This repository was archived by the owner on May 28, 2025. It is now read-only.

Commit 4cfdbd3

Browse files
nnethercotennethercote
committed
Add spacing information to delimiters.
This is an extension of the previous commit. It means the output of something like this: ``` stringify!(let a: Vec<u32> = vec![];) ``` goes from this: ``` let a: Vec<u32> = vec![] ; ``` With this PR, it now produces this string: ``` let a: Vec<u32> = vec![]; ```
1 parent 925f7fa commit 4cfdbd3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+307
-231
lines changed

‎compiler/rustc_ast/src/attr/mod.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ impl MetaItemKind {
387387
tokens: &mut impl Iterator<Item = &'a TokenTree>,
388388
) -> Option<MetaItemKind> {
389389
match tokens.next() {
390-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
390+
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
391391
MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
392392
}
393393
Some(TokenTree::Token(token, _)) => {
@@ -401,7 +401,7 @@ impl MetaItemKind {
401401
tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
402402
) -> Option<MetaItemKind> {
403403
match tokens.peek() {
404-
Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
404+
Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
405405
let inner_tokens = inner_tokens.clone();
406406
tokens.next();
407407
MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
@@ -524,7 +524,7 @@ impl NestedMetaItem {
524524
tokens.next();
525525
return Some(NestedMetaItem::Lit(lit));
526526
}
527-
Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
527+
Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
528528
tokens.next();
529529
return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
530530
}

‎compiler/rustc_ast/src/mut_visit.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -682,7 +682,7 @@ pub fn visit_attr_tt<T: MutVisitor>(tt: &mut AttrTokenTree, vis: &mut T) {
682682
AttrTokenTree::Token(token, _) => {
683683
visit_token(token, vis);
684684
}
685-
AttrTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
685+
AttrTokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
686686
vis.visit_span(open);
687687
vis.visit_span(close);
688688
visit_attr_tts(tts, vis);
@@ -709,7 +709,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
709709
TokenTree::Token(token, _) => {
710710
visit_token(token, vis);
711711
}
712-
TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
712+
TokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
713713
vis.visit_span(open);
714714
vis.visit_span(close);
715715
visit_tts(tts, vis);

‎compiler/rustc_ast/src/tokenstream.rs‎

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ pub enum TokenTree {
4646
/// delimiters are implicitly represented by `Delimited`.
4747
Token(Token, Spacing),
4848
/// A delimited sequence of token trees.
49-
Delimited(DelimSpan, Delimiter, TokenStream),
49+
Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream),
5050
}
5151

5252
// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
@@ -62,11 +62,11 @@ where
6262
}
6363

6464
impl TokenTree {
65-
/// Checks if this `TokenTree` is equal to the other, regardless of span information.
65+
/// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information.
6666
pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
6767
match (self, other) {
6868
(TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
69-
(TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
69+
(TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
7070
delim == delim2 && tts.eq_unspanned(tts2)
7171
}
7272
_ => false,
@@ -188,7 +188,7 @@ pub struct AttrTokenStream(pub Lrc<Vec<AttrTokenTree>>);
188188
#[derive(Clone, Debug, Encodable, Decodable)]
189189
pub enum AttrTokenTree {
190190
Token(Token, Spacing),
191-
Delimited(DelimSpan, Delimiter, AttrTokenStream),
191+
Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream),
192192
/// Stores the attributes for an attribute target,
193193
/// along with the tokens for that attribute target.
194194
/// See `AttributesData` for more information
@@ -213,9 +213,14 @@ impl AttrTokenStream {
213213
AttrTokenTree::Token(inner, spacing) => {
214214
smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
215215
}
216-
AttrTokenTree::Delimited(span, delim, stream) => {
217-
smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
218-
.into_iter()
216+
AttrTokenTree::Delimited(span, spacing, delim, stream) => {
217+
smallvec![TokenTree::Delimited(
218+
*span,
219+
*spacing,
220+
*delim,
221+
stream.to_tokenstream()
222+
),]
223+
.into_iter()
219224
}
220225
AttrTokenTree::Attributes(data) => {
221226
let idx = data
@@ -235,7 +240,7 @@ impl AttrTokenStream {
235240
let mut found = false;
236241
// Check the last two trees (to account for a trailing semi)
237242
for tree in target_tokens.iter_mut().rev().take(2) {
238-
if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
243+
if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
239244
// Inner attributes are only supported on extern blocks, functions,
240245
// impls, and modules. All of these have their inner attributes
241246
// placed at the beginning of the rightmost outermost braced group:
@@ -255,7 +260,7 @@ impl AttrTokenStream {
255260
stream.push_stream(inner_attr.tokens());
256261
}
257262
stream.push_stream(delim_tokens.clone());
258-
*tree = TokenTree::Delimited(*span, *delim, stream);
263+
*tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
259264
found = true;
260265
break;
261266
}
@@ -477,11 +482,6 @@ impl TokenStream {
477482
TokenStream::new(vec![TokenTree::token_alone(kind, span)])
478483
}
479484

480-
/// Create a token stream containing a single `Delimited`.
481-
pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
482-
TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
483-
}
484-
485485
pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
486486
let Some(tokens) = node.tokens() else {
487487
panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
@@ -528,6 +528,7 @@ impl TokenStream {
528528
}
529529
token::Interpolated(nt) => TokenTree::Delimited(
530530
DelimSpan::from_single(token.span),
531+
DelimSpacing::new(Spacing::JointHidden, spacing),
531532
Delimiter::Invisible,
532533
TokenStream::from_nonterminal_ast(&nt.0).flattened(),
533534
),
@@ -538,8 +539,8 @@ impl TokenStream {
538539
fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
539540
match tree {
540541
TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
541-
TokenTree::Delimited(span, delim, tts) => {
542-
TokenTree::Delimited(*span, *delim, tts.flattened())
542+
TokenTree::Delimited(span, spacing, delim, tts) => {
543+
TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
543544
}
544545
}
545546
}
@@ -549,7 +550,7 @@ impl TokenStream {
549550
fn can_skip(stream: &TokenStream) -> bool {
550551
stream.trees().all(|tree| match tree {
551552
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
552-
TokenTree::Delimited(_, _, inner) => can_skip(inner),
553+
TokenTree::Delimited(.., inner) => can_skip(inner),
553554
})
554555
}
555556

@@ -638,9 +639,10 @@ impl TokenStream {
638639

639640
&TokenTree::Token(..) => i += 1,
640641

641-
&TokenTree::Delimited(sp, delim, ref delim_stream) => {
642+
&TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => {
642643
if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
643-
let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
644+
let new_tt =
645+
TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream);
644646
Lrc::make_mut(&mut stream.0)[i] = new_tt;
645647
modified = true;
646648
}
@@ -668,10 +670,11 @@ impl TokenStream {
668670
num_of_hashes = cmp::max(num_of_hashes, count);
669671
}
670672

671-
// `/// foo` becomes `doc = r"foo"`.
673+
// `/// foo` becomes `[doc = r"foo"]`.
672674
let delim_span = DelimSpan::from_single(span);
673675
let body = TokenTree::Delimited(
674676
delim_span,
677+
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
675678
Delimiter::Bracket,
676679
[
677680
TokenTree::token_alone(token::Ident(sym::doc, false), span),
@@ -784,6 +787,18 @@ impl DelimSpan {
784787
}
785788
}
786789

790+
#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
791+
pub struct DelimSpacing {
792+
pub open: Spacing,
793+
pub close: Spacing,
794+
}
795+
796+
impl DelimSpacing {
797+
pub fn new(open: Spacing, close: Spacing) -> DelimSpacing {
798+
DelimSpacing { open, close }
799+
}
800+
}
801+
787802
// Some types are used a lot. Make sure they don't unintentionally get bigger.
788803
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
789804
mod size_asserts {

‎compiler/rustc_ast_pretty/src/pprust/state.rs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -183,10 +183,10 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
183183
//
184184
// FIXME: Incorrect cases:
185185
// - Let: `let(a, b) = (1, 2)`
186-
(Tok(Token { kind: Ident(..), .. }, _), Del(_, Parenthesis, _)) => false,
186+
(Tok(Token { kind: Ident(..), .. }, _), Del(_, _, Parenthesis, _)) => false,
187187

188188
// `#` + `[`: `#[attr]`
189-
(Tok(Token { kind: Pound, .. }, _), Del(_, Bracket, _)) => false,
189+
(Tok(Token { kind: Pound, .. }, _), Del(_, _, Bracket, _)) => false,
190190

191191
_ => true,
192192
}
@@ -519,7 +519,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
519519
}
520520
*spacing
521521
}
522-
TokenTree::Delimited(dspan, delim, tts) => {
522+
TokenTree::Delimited(dspan, spacing, delim, tts) => {
523523
self.print_mac_common(
524524
None,
525525
false,
@@ -529,9 +529,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
529529
convert_dollar_crate,
530530
dspan.entire(),
531531
);
532-
// FIXME: add two `Spacing` fields to `TokenTree::Delimited`
533-
// and use the close delim one here.
534-
Spacing::Alone
532+
spacing.close
535533
}
536534
}
537535
}

‎compiler/rustc_expand/src/config.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ use crate::errors::{
66
};
77
use rustc_ast::ptr::P;
88
use rustc_ast::token::{Delimiter, Token, TokenKind};
9-
use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree};
10-
use rustc_ast::tokenstream::{DelimSpan, Spacing};
9+
use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, Spacing};
1110
use rustc_ast::tokenstream::{LazyAttrTokenStream, TokenTree};
1211
use rustc_ast::NodeId;
1312
use rustc_ast::{self as ast, AttrStyle, Attribute, HasAttrs, HasTokens, MetaItem};
@@ -242,7 +241,7 @@ impl<'a> StripUnconfigured<'a> {
242241
stream.0.iter().all(|tree| match tree {
243242
AttrTokenTree::Attributes(_) => false,
244243
AttrTokenTree::Token(..) => true,
245-
AttrTokenTree::Delimited(_, _, inner) => can_skip(inner),
244+
AttrTokenTree::Delimited(.., inner) => can_skip(inner),
246245
})
247246
}
248247

@@ -266,9 +265,9 @@ impl<'a> StripUnconfigured<'a> {
266265
None.into_iter()
267266
}
268267
}
269-
AttrTokenTree::Delimited(sp, delim, mut inner) => {
268+
AttrTokenTree::Delimited(sp, spacing, delim, mut inner) => {
270269
inner = self.configure_tokens(&inner);
271-
Some(AttrTokenTree::Delimited(sp, delim, inner)).into_iter()
270+
Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
272271
}
273272
AttrTokenTree::Token(ref token, _)
274273
if let TokenKind::Interpolated(nt) = &token.kind =>
@@ -376,6 +375,7 @@ impl<'a> StripUnconfigured<'a> {
376375
// in `#[attr]`, so just use the span of the `#` token.
377376
let bracket_group = AttrTokenTree::Delimited(
378377
DelimSpan::from_single(pound_span),
378+
DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
379379
Delimiter::Bracket,
380380
item.tokens
381381
.as_ref()

‎compiler/rustc_expand/src/mbe.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ pub(crate) mod transcribe;
1313

1414
use metavar_expr::MetaVarExpr;
1515
use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind};
16-
use rustc_ast::tokenstream::DelimSpan;
16+
use rustc_ast::tokenstream::{DelimSpacing, DelimSpan};
1717
use rustc_span::symbol::Ident;
1818
use rustc_span::Span;
1919

@@ -68,7 +68,7 @@ pub(crate) enum KleeneOp {
6868
enum TokenTree {
6969
Token(Token),
7070
/// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS).
71-
Delimited(DelimSpan, Delimited),
71+
Delimited(DelimSpan, DelimSpacing, Delimited),
7272
/// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
7373
Sequence(DelimSpan, SequenceRepetition),
7474
/// e.g., `$var`.
@@ -99,7 +99,7 @@ impl TokenTree {
9999
TokenTree::Token(Token { span, .. })
100100
| TokenTree::MetaVar(span, _)
101101
| TokenTree::MetaVarDecl(span, _, _) => span,
102-
TokenTree::Delimited(span, _)
102+
TokenTree::Delimited(span, ..)
103103
| TokenTree::MetaVarExpr(span, _)
104104
| TokenTree::Sequence(span, _) => span.entire(),
105105
}

‎compiler/rustc_expand/src/mbe/macro_check.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -290,7 +290,7 @@ fn check_binders(
290290
}
291291
// `MetaVarExpr` can not appear in the LHS of a macro arm
292292
TokenTree::MetaVarExpr(..) => {}
293-
TokenTree::Delimited(_, ref del) => {
293+
TokenTree::Delimited(.., ref del) => {
294294
for tt in &del.tts {
295295
check_binders(sess, node_id, tt, macros, binders, ops, valid);
296296
}
@@ -353,7 +353,7 @@ fn check_occurrences(
353353
};
354354
check_ops_is_prefix(sess, node_id, macros, binders, ops, dl.entire(), name);
355355
}
356-
TokenTree::Delimited(_, ref del) => {
356+
TokenTree::Delimited(.., ref del) => {
357357
check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid);
358358
}
359359
TokenTree::Sequence(_, ref seq) => {
@@ -435,8 +435,8 @@ fn check_nested_occurrences(
435435
// We check that the meta-variable is correctly used.
436436
check_occurrences(sess, node_id, tt, macros, binders, ops, valid);
437437
}
438-
(NestedMacroState::MacroRulesNotName, TokenTree::Delimited(_, del))
439-
| (NestedMacroState::MacroName, TokenTree::Delimited(_, del))
438+
(NestedMacroState::MacroRulesNotName, TokenTree::Delimited(.., del))
439+
| (NestedMacroState::MacroName, TokenTree::Delimited(.., del))
440440
if del.delim == Delimiter::Brace =>
441441
{
442442
let macro_rules = state == NestedMacroState::MacroRulesNotName;
@@ -466,7 +466,7 @@ fn check_nested_occurrences(
466466
// We check that the meta-variable is correctly used.
467467
check_occurrences(sess, node_id, tt, macros, binders, ops, valid);
468468
}
469-
(NestedMacroState::MacroName, TokenTree::Delimited(_, del))
469+
(NestedMacroState::MacroName, TokenTree::Delimited(.., del))
470470
if del.delim == Delimiter::Parenthesis =>
471471
{
472472
state = NestedMacroState::MacroNameParen;
@@ -481,7 +481,7 @@ fn check_nested_occurrences(
481481
valid,
482482
);
483483
}
484-
(NestedMacroState::MacroNameParen, TokenTree::Delimited(_, del))
484+
(NestedMacroState::MacroNameParen, TokenTree::Delimited(.., del))
485485
if del.delim == Delimiter::Brace =>
486486
{
487487
state = NestedMacroState::Empty;

‎compiler/rustc_expand/src/mbe/macro_parser.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
184184
TokenTree::Token(token) => {
185185
locs.push(MatcherLoc::Token { token: token.clone() });
186186
}
187-
TokenTree::Delimited(span, delimited) => {
187+
TokenTree::Delimited(span, _, delimited) => {
188188
let open_token = Token::new(token::OpenDelim(delimited.delim), span.open);
189189
let close_token = Token::new(token::CloseDelim(delimited.delim), span.close);
190190

@@ -335,7 +335,7 @@ pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize {
335335
.map(|tt| match tt {
336336
TokenTree::MetaVarDecl(..) => 1,
337337
TokenTree::Sequence(_, seq) => seq.num_captures,
338-
TokenTree::Delimited(_, delim) => count_metavar_decls(&delim.tts),
338+
TokenTree::Delimited(.., delim) => count_metavar_decls(&delim.tts),
339339
TokenTree::Token(..) => 0,
340340
TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => unreachable!(),
341341
})

0 commit comments

Comments
 (0)