Skip to content

Commit 1fe2c03

Browse files
nnethercotennethercote
committed
Remove tokenstream::Delimited.
Because it's an extra type layer that doesn't really help; in a couple of places it actively gets in the way, and overall removing it makes the code nicer. It does, however, move `tokenstream::TokenTree` further away from the `TokenTree` in `quote.rs`. More importantly, this change reduces the size of `TokenStream` from 48 bytes to 40 bytes on x86-64, which is enough to slightly reduce instruction counts on numerous benchmarks, the best by 1.5%. Note that `open_tt` and `close_tt` have gone from being methods on `Delimited` to associated methods of `TokenTree`.
1 parent b755501 commit 1fe2c03

File tree

18 files changed

+182
-203
lines changed

18 files changed

+182
-203
lines changed

‎src/librustc/hir/lowering.rs

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ use syntax::ptr::P;
7272
use syntax::source_map::{self, respan, CompilerDesugaringKind, Spanned};
7373
use syntax::std_inject;
7474
use syntax::symbol::{keywords, Symbol};
75-
use syntax::tokenstream::{Delimited, TokenStream, TokenTree};
75+
use syntax::tokenstream::{TokenStream, TokenTree};
7676
use syntax::parse::token::Token;
7777
use syntax::visit::{self, Visitor};
7878
use syntax_pos::{Span, MultiSpan};
@@ -1088,12 +1088,10 @@ impl<'a> LoweringContext<'a> {
10881088
fn lower_token_tree(&mut self, tree: TokenTree) -> TokenStream {
10891089
match tree {
10901090
TokenTree::Token(span, token) => self.lower_token(token, span),
1091-
TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
1091+
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
10921092
span,
1093-
Delimited {
1094-
delim: delimited.delim,
1095-
tts: self.lower_token_stream(delimited.tts.into()).into(),
1096-
},
1093+
delim,
1094+
self.lower_token_stream(tts.into()).into(),
10971095
).into(),
10981096
}
10991097
}

‎src/librustc/ich/impls_syntax.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -265,10 +265,10 @@ for tokenstream::TokenTree {
265265
span.hash_stable(hcx, hasher);
266266
hash_token(token, hcx, hasher);
267267
}
268-
tokenstream::TokenTree::Delimited(span, ref delimited) => {
268+
tokenstream::TokenTree::Delimited(span, delim, ref tts) => {
269269
span.hash_stable(hcx, hasher);
270-
std_hash::Hash::hash(&delimited.delim, hasher);
271-
for sub_tt in delimited.stream().trees() {
270+
std_hash::Hash::hash(&delim, hasher);
271+
for sub_tt in tts.stream().trees() {
272272
sub_tt.hash_stable(hcx, hasher);
273273
}
274274
}

‎src/librustc_lint/builtin.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1540,8 +1540,8 @@ impl KeywordIdents {
15401540
}
15411541
_ => {},
15421542
}
1543-
TokenTree::Delimited(_, ref delim) => {
1544-
self.check_tokens(cx, delim.tts.clone().into())
1543+
TokenTree::Delimited(_, _, tts) => {
1544+
self.check_tokens(cx, tts.stream())
15451545
},
15461546
}
15471547
}

‎src/libsyntax/ast.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1235,7 +1235,7 @@ pub enum MacDelimiter {
12351235

12361236
impl Mac_ {
12371237
pub fn stream(&self) -> TokenStream {
1238-
self.tts.clone().into()
1238+
self.tts.stream()
12391239
}
12401240
}
12411241

‎src/libsyntax/attr/mod.rs

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ use parse::token::{self, Token};
3434
use ptr::P;
3535
use symbol::Symbol;
3636
use ThinVec;
37-
use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
37+
use tokenstream::{TokenStream, TokenTree, DelimSpan};
3838
use GLOBALS;
3939

4040
use std::iter;
@@ -549,10 +549,11 @@ impl MetaItemKind {
549549
}
550550
tokens.push(item.node.tokens());
551551
}
552-
TokenTree::Delimited(DelimSpan::from_single(span), Delimited {
553-
delim: token::Paren,
554-
tts: TokenStream::concat(tokens).into(),
555-
}).into()
552+
TokenTree::Delimited(
553+
DelimSpan::from_single(span),
554+
token::Paren,
555+
TokenStream::concat(tokens).into(),
556+
).into()
556557
}
557558
}
558559
}
@@ -570,9 +571,9 @@ impl MetaItemKind {
570571
None
571572
};
572573
}
573-
Some(TokenTree::Delimited(_, ref delimited)) if delimited.delim == token::Paren => {
574+
Some(TokenTree::Delimited(_, delim, ref tts)) if delim == token::Paren => {
574575
tokens.next();
575-
delimited.stream()
576+
tts.stream()
576577
}
577578
_ => return Some(MetaItemKind::Word),
578579
};

‎src/libsyntax/ext/expand.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -622,9 +622,9 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
622622
fn extract_proc_macro_attr_input(&self, tokens: TokenStream, span: Span) -> TokenStream {
623623
let mut trees = tokens.trees();
624624
match trees.next() {
625-
Some(TokenTree::Delimited(_, delim)) => {
625+
Some(TokenTree::Delimited(_, _, tts)) => {
626626
if trees.next().is_none() {
627-
return delim.tts.into()
627+
return tts.into()
628628
}
629629
}
630630
Some(TokenTree::Token(..)) => {}

‎src/libsyntax/ext/quote.rs

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ pub mod rt {
3636
use symbol::Symbol;
3737
use ThinVec;
3838

39-
use tokenstream::{self, DelimSpan, TokenTree, TokenStream};
39+
use tokenstream::{DelimSpan, TokenTree, TokenStream};
4040

4141
pub use parse::new_parser_from_tts;
4242
pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName};
@@ -246,9 +246,9 @@ pub mod rt {
246246
inner.push(self.tokens.clone());
247247

248248
let delim_span = DelimSpan::from_single(self.span);
249-
r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited {
250-
delim: token::Bracket, tts: TokenStream::concat(inner).into()
251-
}));
249+
r.push(TokenTree::Delimited(
250+
delim_span, token::Bracket, TokenStream::concat(inner).into()
251+
));
252252
r
253253
}
254254
}
@@ -262,10 +262,9 @@ pub mod rt {
262262

263263
impl ToTokens for () {
264264
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
265-
vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited {
266-
delim: token::Paren,
267-
tts: TokenStream::empty().into(),
268-
})]
265+
vec![
266+
TokenTree::Delimited(DelimSpan::dummy(), token::Paren, TokenStream::empty().into())
267+
]
269268
}
270269
}
271270

@@ -382,8 +381,6 @@ pub mod rt {
382381

383382
// Replaces `Token::OpenDelim .. Token::CloseDelim` with `TokenTree::Delimited(..)`.
384383
pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
385-
use tokenstream::Delimited;
386-
387384
let mut results = Vec::new();
388385
let mut result = Vec::new();
389386
let mut open_span = DUMMY_SP;
@@ -395,10 +392,11 @@ pub fn unflatten(tts: Vec<TokenTree>) -> Vec<TokenTree> {
395392
}
396393
TokenTree::Token(span, token::CloseDelim(delim)) => {
397394
let delim_span = DelimSpan::from_pair(open_span, span);
398-
let tree = TokenTree::Delimited(delim_span, Delimited {
395+
let tree = TokenTree::Delimited(
396+
delim_span,
399397
delim,
400-
tts: result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
401-
});
398+
result.into_iter().map(TokenStream::from).collect::<TokenStream>().into(),
399+
);
402400
result = results.pop().unwrap();
403401
result.push(tree);
404402
}
@@ -758,10 +756,10 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec<ast::Stmt
758756
vec![e_tok]);
759757
vec![cx.stmt_expr(e_push)]
760758
},
761-
TokenTree::Delimited(span, ref delimed) => {
762-
let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false);
763-
stmts.extend(statements_mk_tts(cx, delimed.stream()));
764-
stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false));
759+
TokenTree::Delimited(span, delim, ref tts) => {
760+
let mut stmts = statements_mk_tt(cx, &TokenTree::open_tt(span.open, delim), false);
761+
stmts.extend(statements_mk_tts(cx, tts.stream()));
762+
stmts.extend(statements_mk_tt(cx, &TokenTree::close_tt(span.close, delim), false));
765763
stmts
766764
}
767765
}

‎src/libsyntax/ext/tt/quoted.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -280,17 +280,17 @@ where
280280
// `tree` is a `$` token. Look at the next token in `trees`
281281
tokenstream::TokenTree::Token(span, token::Dollar) => match trees.next() {
282282
// `tree` is followed by a delimited set of token trees. This indicates the beginning
283-
// of a repetition sequence in the macro (e.g., `$(pat)*`).
284-
Some(tokenstream::TokenTree::Delimited(span, delimited)) => {
283+
// of a repetition sequence in the macro (e.g. `$(pat)*`).
284+
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
285285
// Must have `(` not `{` or `[`
286-
if delimited.delim != token::Paren {
287-
let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim));
286+
if delim != token::Paren {
287+
let tok = pprust::token_to_string(&token::OpenDelim(delim));
288288
let msg = format!("expected `(`, found `{}`", tok);
289289
sess.span_diagnostic.span_err(span.entire(), &msg);
290290
}
291291
// Parse the contents of the sequence itself
292292
let sequence = parse(
293-
delimited.tts.into(),
293+
tts.into(),
294294
expect_matchers,
295295
sess,
296296
features,
@@ -354,12 +354,12 @@ where
354354

355355
// `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
356356
// descend into the delimited set and further parse it.
357-
tokenstream::TokenTree::Delimited(span, delimited) => TokenTree::Delimited(
357+
tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
358358
span,
359359
Lrc::new(Delimited {
360-
delim: delimited.delim,
360+
delim: delim,
361361
tts: parse(
362-
delimited.tts.into(),
362+
tts.into(),
363363
expect_matchers,
364364
sess,
365365
features,

‎src/libsyntax/ext/tt/transcribe.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ use fold::noop_fold_tt;
1717
use parse::token::{self, Token, NtTT};
1818
use smallvec::SmallVec;
1919
use syntax_pos::DUMMY_SP;
20-
use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan};
20+
use tokenstream::{TokenStream, TokenTree, DelimSpan};
2121

2222
use rustc_data_structures::fx::FxHashMap;
2323
use rustc_data_structures::sync::Lrc;
@@ -105,10 +105,11 @@ pub fn transcribe(cx: &ExtCtxt,
105105
if result_stack.is_empty() {
106106
return TokenStream::concat(result);
107107
}
108-
let tree = TokenTree::Delimited(span, Delimited {
109-
delim: forest.delim,
110-
tts: TokenStream::concat(result).into(),
111-
});
108+
let tree = TokenTree::Delimited(
109+
span,
110+
forest.delim,
111+
TokenStream::concat(result).into(),
112+
);
112113
result = result_stack.pop().unwrap();
113114
result.push(tree.into());
114115
}

‎src/libsyntax/fold.rs

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -605,12 +605,10 @@ pub fn noop_fold_tt<T: Folder>(tt: TokenTree, fld: &mut T) -> TokenTree {
605605
match tt {
606606
TokenTree::Token(span, tok) =>
607607
TokenTree::Token(fld.new_span(span), fld.fold_token(tok)),
608-
TokenTree::Delimited(span, delimed) => TokenTree::Delimited(
608+
TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited(
609609
DelimSpan::from_pair(fld.new_span(span.open), fld.new_span(span.close)),
610-
Delimited {
611-
tts: fld.fold_tts(delimed.stream()).into(),
612-
delim: delimed.delim,
613-
}
610+
delim,
611+
fld.fold_tts(tts.stream()).into(),
614612
),
615613
}
616614
}

0 commit comments

Comments
 (0)