diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index b7da276fc7e1d..4233f7806248b 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -1542,10 +1542,10 @@ pub enum MacArgs { } impl MacArgs { - pub fn delim(&self) -> DelimToken { + pub fn delim(&self) -> Option { match self { - MacArgs::Delimited(_, delim, _) => delim.to_token(), - MacArgs::Empty | MacArgs::Eq(..) => token::NoDelim, + MacArgs::Delimited(_, delim, _) => Some(delim.to_token()), + MacArgs::Empty | MacArgs::Eq(..) => None, } } diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs index 3c9bb81bedb1c..39824095e8663 100644 --- a/compiler/rustc_ast_pretty/src/pprust/state.rs +++ b/compiler/rustc_ast_pretty/src/pprust/state.rs @@ -464,7 +464,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere Some(MacHeader::Path(&item.path)), false, None, - delim.to_token(), + Some(delim.to_token()), tokens, true, span, @@ -530,7 +530,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere None, false, None, - *delim, + Some(*delim), tts, convert_dollar_crate, dspan.entire(), @@ -556,12 +556,12 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere header: Option>, has_bang: bool, ident: Option, - delim: DelimToken, + delim: Option, tts: &TokenStream, convert_dollar_crate: bool, span: Span, ) { - if delim == DelimToken::Brace { + if delim == Some(DelimToken::Brace) { self.cbox(INDENT_UNIT); } match header { @@ -577,7 +577,7 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere self.print_ident(ident); } match delim { - DelimToken::Brace => { + Some(DelimToken::Brace) => { if header.is_some() || has_bang || ident.is_some() { self.nbsp(); } @@ -585,23 +585,25 @@ pub trait PrintState<'a>: std::ops::Deref + std::ops::Dere if !tts.is_empty() { self.space(); } - } - _ => { - let token_str = self.token_kind_to_string(&token::OpenDelim(delim)); - self.word(token_str) - } - } - self.ibox(0); - self.print_tts(tts, convert_dollar_crate); - self.end(); - match delim { - DelimToken::Brace => { + self.ibox(0); + self.print_tts(tts, convert_dollar_crate); + self.end(); let empty = tts.is_empty(); self.bclose(span, empty); } - _ => { + Some(delim) => { + let token_str = self.token_kind_to_string(&token::OpenDelim(delim)); + self.word(token_str); + self.ibox(0); + self.print_tts(tts, convert_dollar_crate); + self.end(); let token_str = self.token_kind_to_string(&token::CloseDelim(delim)); - self.word(token_str) + self.word(token_str); + } + None => { + self.ibox(0); + self.print_tts(tts, convert_dollar_crate); + self.end(); } } } diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs index 2cfd6968accf7..f3dcdbf93c5f1 100644 --- a/compiler/rustc_expand/src/mbe/macro_rules.rs +++ b/compiler/rustc_expand/src/mbe/macro_rules.rs @@ -260,16 +260,15 @@ fn generic_extension<'cx, 'tt>( // Merge the gated spans from parsing the matcher with the pre-existing ones. sess.gated_spans.merge(gated_spans_snapshot); - // Ignore the delimiters on the RHS. - let rhs = match &rhses[i] { - mbe::TokenTree::Delimited(_, delimited) => &delimited.tts, + let (rhs, rhs_span): (&mbe::Delimited, DelimSpan) = match &rhses[i] { + mbe::TokenTree::Delimited(span, delimited) => (&delimited, *span), _ => cx.span_bug(sp, "malformed macro rhs"), }; let arm_span = rhses[i].span(); - let rhs_spans = rhs.iter().map(|t| t.span()).collect::>(); + let rhs_spans = rhs.tts.iter().map(|t| t.span()).collect::>(); // rhs has holes ( `$id` and `$(...)` that need filled) - let mut tts = match transcribe(cx, &named_matches, &rhs, transparency) { + let mut tts = match transcribe(cx, &named_matches, &rhs, rhs_span, transparency) { Ok(tts) => tts, Err(mut err) => { err.emit(); diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs index d25f044234cf4..d5b1913e1440c 100644 --- a/compiler/rustc_expand/src/mbe/transcribe.rs +++ b/compiler/rustc_expand/src/mbe/transcribe.rs @@ -29,8 +29,8 @@ impl MutVisitor for Marker { enum Frame<'a> { Delimited { tts: &'a [mbe::TokenTree], - delim_token: token::DelimToken, idx: usize, + delim_token: token::DelimToken, span: DelimSpan, }, Sequence { @@ -42,8 +42,8 @@ enum Frame<'a> { impl<'a> Frame<'a> { /// Construct a new frame around the delimited set of tokens. - fn new(tts: &'a [mbe::TokenTree]) -> Frame<'a> { - Frame::Delimited { tts, delim_token: token::NoDelim, idx: 0, span: DelimSpan::dummy() } + fn new(src: &'a mbe::Delimited, span: DelimSpan) -> Frame<'a> { + Frame::Delimited { tts: &src.tts, idx: 0, delim_token: src.delim, span } } } @@ -85,17 +85,18 @@ impl<'a> Iterator for Frame<'a> { pub(super) fn transcribe<'a>( cx: &ExtCtxt<'a>, interp: &FxHashMap, - src: &[mbe::TokenTree], + src: &mbe::Delimited, + src_span: DelimSpan, transparency: Transparency, ) -> PResult<'a, TokenStream> { // Nothing for us to transcribe... - if src.is_empty() { + if src.tts.is_empty() { return Ok(TokenStream::default()); } // We descend into the RHS (`src`), expanding things as we go. This stack contains the things // we have yet to expand/are still expanding. We start the stack off with the whole RHS. - let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src)]; + let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src, src_span)]; // As we descend in the RHS, we will need to be able to match nested sequences of matchers. // `repeats` keeps track of where we are in matching at each level, with the last element being diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs index 02749088c3139..575b01180df77 100644 --- a/compiler/rustc_parse/src/parser/attr_wrapper.rs +++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs @@ -5,7 +5,7 @@ use rustc_ast::tokenstream::{AttrAnnotatedTokenTree, DelimSpan, LazyTokenStream, use rustc_ast::{self as ast}; use rustc_ast::{AstLike, AttrVec, Attribute}; use rustc_errors::PResult; -use rustc_span::{sym, Span, DUMMY_SP}; +use rustc_span::{sym, Span}; use std::convert::TryInto; use std::ops::Range; @@ -400,24 +400,26 @@ fn make_token_stream( ) -> AttrAnnotatedTokenStream { #[derive(Debug)] struct FrameData { - open: Span, - open_delim: DelimToken, + // This is `None` for the first frame, `Some` for all others. + open_delim_sp: Option<(DelimToken, Span)>, inner: Vec<(AttrAnnotatedTokenTree, Spacing)>, } - let mut stack = - vec![FrameData { open: DUMMY_SP, open_delim: DelimToken::NoDelim, inner: vec![] }]; + let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }]; let mut token_and_spacing = iter.next(); while let Some((token, spacing)) = token_and_spacing { match token { FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => { - stack.push(FrameData { open: span, open_delim: delim, inner: vec![] }); + stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] }); } FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => { // HACK: If we encounter a mismatched `None` delimiter at the top // level, just ignore it. if matches!(delim, DelimToken::NoDelim) && (stack.len() == 1 - || !matches!(stack.last_mut().unwrap().open_delim, DelimToken::NoDelim)) + || !matches!( + stack.last_mut().unwrap().open_delim_sp.unwrap().0, + DelimToken::NoDelim + )) { token_and_spacing = iter.next(); continue; @@ -430,7 +432,7 @@ fn make_token_stream( // merge our current frame with the one above it. That is, transform // `[ { < first second } third ]` into `[ { first second } third ]` if !matches!(delim, DelimToken::NoDelim) - && matches!(frame_data.open_delim, DelimToken::NoDelim) + && matches!(frame_data.open_delim_sp.unwrap().0, DelimToken::NoDelim) { stack.last_mut().unwrap().inner.extend(frame_data.inner); // Process our closing delimiter again, this time at the previous @@ -439,12 +441,13 @@ fn make_token_stream( continue; } + let (open_delim, open_sp) = frame_data.open_delim_sp.unwrap(); assert_eq!( - frame_data.open_delim, delim, + open_delim, delim, "Mismatched open/close delims: open={:?} close={:?}", - frame_data.open, span + open_delim, span ); - let dspan = DelimSpan::from_pair(frame_data.open, span); + let dspan = DelimSpan::from_pair(open_sp, span); let stream = AttrAnnotatedTokenStream::new(frame_data.inner); let delimited = AttrAnnotatedTokenTree::Delimited(dspan, delim, stream); stack @@ -472,7 +475,7 @@ fn make_token_stream( // HACK: If we don't have a closing `None` delimiter for our last // frame, merge the frame with the top-level frame. That is, // turn `< first second` into `first second` - if stack.len() == 2 && stack[1].open_delim == DelimToken::NoDelim { + if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == DelimToken::NoDelim { let temp_buf = stack.pop().unwrap(); stack.last_mut().unwrap().inner.extend(temp_buf.inner); } diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 7efc0ca2da234..e4370809ebc24 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -2043,7 +2043,8 @@ impl<'a> Parser<'a> { self.sess.gated_spans.gate(sym::async_closure, span); } - if self.token.kind == TokenKind::Semi && self.token_cursor.frame.delim == DelimToken::Paren + if self.token.kind == TokenKind::Semi + && matches!(self.token_cursor.frame.delim_sp, Some((DelimToken::Paren, _))) { // It is likely that the closure body is a block but where the // braces have been removed. We will recover and eat the next diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 1686c5873e183..dfe758d0cdf01 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -244,14 +244,13 @@ struct TokenCursor { #[derive(Clone)] struct TokenCursorFrame { - delim: token::DelimToken, - span: DelimSpan, + delim_sp: Option<(DelimToken, DelimSpan)>, tree_cursor: tokenstream::Cursor, } impl TokenCursorFrame { - fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self { - TokenCursorFrame { delim, span, tree_cursor: tts.into_trees() } + fn new(delim_sp: Option<(DelimToken, DelimSpan)>, tts: TokenStream) -> Self { + TokenCursorFrame { delim_sp, tree_cursor: tts.into_trees() } } } @@ -266,7 +265,7 @@ impl TokenCursor { loop { // FIXME: we currently don't return `NoDelim` open/close delims. To fix #67062 we will // need to, whereupon the `delim != DelimToken::NoDelim` conditions below can be - // removed, as well as the loop. + // removed. if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing_ref() { match tree { &TokenTree::Token(ref token) => match (desugar_doc_comments, token) { @@ -277,7 +276,7 @@ impl TokenCursor { }, &TokenTree::Delimited(sp, delim, ref tts) => { // Set `open_delim` to true here because we deal with it immediately. - let frame = TokenCursorFrame::new(sp, delim, tts.clone()); + let frame = TokenCursorFrame::new(Some((delim, sp)), tts.clone()); self.stack.push(mem::replace(&mut self.frame, frame)); if delim != DelimToken::NoDelim { return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone); @@ -286,12 +285,11 @@ impl TokenCursor { } }; } else if let Some(frame) = self.stack.pop() { - let delim = self.frame.delim; - let span = self.frame.span; - self.frame = frame; - if delim != DelimToken::NoDelim { + if let Some((delim, span)) = self.frame.delim_sp && delim != DelimToken::NoDelim { + self.frame = frame; return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone); } + self.frame = frame; // No close delimiter to return; continue on to the next iteration. } else { return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone); @@ -330,8 +328,7 @@ impl TokenCursor { self.stack.push(mem::replace( &mut self.frame, TokenCursorFrame::new( - delim_span, - token::NoDelim, + None, if attr_style == AttrStyle::Inner { [TokenTree::token(token::Pound, span), TokenTree::token(token::Not, span), body] .iter() @@ -431,10 +428,6 @@ impl<'a> Parser<'a> { desugar_doc_comments: bool, subparser_name: Option<&'static str>, ) -> Self { - // Note: because of the way `TokenCursor::inlined_next` is structured, the `span` and - // `delim` arguments here are never used. - let start_frame = TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens); - let mut parser = Parser { sess, token: Token::dummy(), @@ -444,7 +437,7 @@ impl<'a> Parser<'a> { restrictions: Restrictions::empty(), expected_tokens: Vec::new(), token_cursor: TokenCursor { - frame: start_frame, + frame: TokenCursorFrame::new(None, tokens), stack: Vec::new(), num_next_calls: 0, desugar_doc_comments, @@ -1025,7 +1018,7 @@ impl<'a> Parser<'a> { } let frame = &self.token_cursor.frame; - if frame.delim != DelimToken::NoDelim { + if let Some((delim, span)) = frame.delim_sp && delim != DelimToken::NoDelim { let all_normal = (0..dist).all(|i| { let token = frame.tree_cursor.look_ahead(i); !matches!(token, Some(TokenTree::Delimited(_, DelimToken::NoDelim, _))) @@ -1038,7 +1031,7 @@ impl<'a> Parser<'a> { looker(&Token::new(token::OpenDelim(*delim), dspan.open)) } }, - None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)), + None => looker(&Token::new(token::CloseDelim(delim), span.close)), }; } } @@ -1198,8 +1191,7 @@ impl<'a> Parser<'a> { // Grab the tokens from this frame. let frame = &self.token_cursor.frame; let stream = frame.tree_cursor.stream.clone(); - let span = frame.span; - let delim = frame.delim; + let (delim, span) = frame.delim_sp.unwrap(); // Advance the token cursor through the entire delimited // sequence. After getting the `OpenDelim` we are *within* the diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs index 5b7ae5f7a7b82..14f1208b71f51 100644 --- a/compiler/rustc_parse/src/parser/stmt.rs +++ b/compiler/rustc_parse/src/parser/stmt.rs @@ -164,25 +164,29 @@ impl<'a> Parser<'a> { let delim = args.delim(); let hi = self.prev_token.span; - let style = - if delim == token::Brace { MacStmtStyle::Braces } else { MacStmtStyle::NoBraces }; + let style = match delim { + Some(token::Brace) => MacStmtStyle::Braces, + Some(_) => MacStmtStyle::NoBraces, + None => unreachable!(), + }; let mac = MacCall { path, args, prior_type_ascription: self.last_type_ascription }; - let kind = - if (delim == token::Brace && self.token != token::Dot && self.token != token::Question) - || self.token == token::Semi - || self.token == token::Eof - { - StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None })) - } else { - // Since none of the above applied, this is an expression statement macro. - let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new()); - let e = self.maybe_recover_from_bad_qpath(e, true)?; - let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?; - let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?; - StmtKind::Expr(e) - }; + let kind = if (style == MacStmtStyle::Braces + && self.token != token::Dot + && self.token != token::Question) + || self.token == token::Semi + || self.token == token::Eof + { + StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None })) + } else { + // Since none of the above applied, this is an expression statement macro. + let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new()); + let e = self.maybe_recover_from_bad_qpath(e, true)?; + let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?; + let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?; + StmtKind::Expr(e) + }; Ok(self.mk_stmt(lo.to(hi), kind)) } diff --git a/src/tools/rustfmt/src/expr.rs b/src/tools/rustfmt/src/expr.rs index 4f333cd27cefe..cd724373f4d16 100644 --- a/src/tools/rustfmt/src/expr.rs +++ b/src/tools/rustfmt/src/expr.rs @@ -1325,7 +1325,7 @@ pub(crate) fn can_be_overflowed_expr( } ast::ExprKind::MacCall(ref mac) => { match ( - rustc_ast::ast::MacDelimiter::from_token(mac.args.delim()), + rustc_ast::ast::MacDelimiter::from_token(mac.args.delim().unwrap()), context.config.overflow_delimited_expr(), ) { (Some(ast::MacDelimiter::Bracket), true) diff --git a/src/tools/rustfmt/src/macros.rs b/src/tools/rustfmt/src/macros.rs index 664f152e8be1d..92606902c5789 100644 --- a/src/tools/rustfmt/src/macros.rs +++ b/src/tools/rustfmt/src/macros.rs @@ -562,7 +562,7 @@ fn delim_token_to_str( ("{ ", " }") } } - DelimToken::NoDelim => ("", ""), + DelimToken::NoDelim => unreachable!(), }; if use_multiple_lines { let indent_str = shape.indent.to_string_with_newline(context.config);