From d0eaf60d5e4bd0d5be63a41ed9ff861dca95c932 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 16 Oct 2019 09:30:28 +1100 Subject: [PATCH 1/4] Remove two no-op `into()` calls. --- src/libsyntax/parse/parser.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 9cb410a8ae318..6bbd8be0cb982 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -285,10 +285,10 @@ impl TokenCursor { token::NoDelim, &if doc_comment_style(&name.as_str()) == AttrStyle::Inner { [TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body] - .iter().cloned().collect::().into() + .iter().cloned().collect::() } else { [TokenTree::token(token::Pound, sp), body] - .iter().cloned().collect::().into() + .iter().cloned().collect::() }, ))); From a6eef299d3b0ca24f8ffc0c3dc03283c09ec7945 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 14 Oct 2019 10:37:21 +1100 Subject: [PATCH 2/4] Make `TokenStream::from_iter` less general and more efficient. The current code has this impl: ``` impl> iter::FromIterator for TokenStream ``` If given an `IntoIterator`, it will convert each individual `TokenTree` to a `TokenStream` (at the cost of two allocations: a `Vec` and an `Lrc`). It will then merge those `TokenStream`s into a single `TokenStream`. This is inefficient. This commit changes the impl to this less general one: ``` impl iter::FromIterator for TokenStream ``` It collects the `TokenTree`s into a single `Vec` first and then converts that to a `TokenStream` by wrapping it in a single `Lrc`. The previous generality was unnecessary; no other code needs changing. This change speeds up several benchmarks by up to 4%. --- src/libsyntax/tokenstream.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index db6832d642322..3d89e73d729f3 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -202,9 +202,9 @@ impl From for TreeAndJoint { } } -impl> iter::FromIterator for TokenStream { - fn from_iter>(iter: I) -> Self { - TokenStream::from_streams(iter.into_iter().map(Into::into).collect::>()) +impl iter::FromIterator for TokenStream { + fn from_iter>(iter: I) -> Self { + TokenStream::new(iter.into_iter().map(Into::into).collect::>()) } } From 212ae58f36e4c9f2e0f46153f1c50a657e6c25db Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 14 Oct 2019 11:24:46 +1100 Subject: [PATCH 3/4] Change `Lit::tokens()` to `Lit::token_tree()`. Because most of the call sites have an easier time working with a `TokenTree` instead of a `TokenStream`. --- src/libsyntax/attr/mod.rs | 9 +++++---- src/libsyntax/parse/literal.rs | 8 ++++---- src/libsyntax/parse/parser/attr.rs | 9 ++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 402c2cad72f99..8786c850bfdd2 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -541,9 +541,10 @@ impl MetaItemKind { match *self { MetaItemKind::Word => TokenStream::default(), MetaItemKind::NameValue(ref lit) => { - let mut vec = vec![TokenTree::token(token::Eq, span).into()]; - lit.tokens().append_to_tree_and_joint_vec(&mut vec); - TokenStream::new(vec) + TokenStream::new(vec![ + TokenTree::token(token::Eq, span).into(), + lit.token_tree().into(), + ]) } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -606,7 +607,7 @@ impl NestedMetaItem { fn tokens(&self) -> TokenStream { match *self { NestedMetaItem::MetaItem(ref item) => item.tokens(), - NestedMetaItem::Literal(ref lit) => lit.tokens(), + NestedMetaItem::Literal(ref lit) => lit.token_tree().into(), } } diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs index 14e1696610a4d..7952e293a532d 100644 --- a/src/libsyntax/parse/literal.rs +++ b/src/libsyntax/parse/literal.rs @@ -3,7 +3,7 @@ use crate::ast::{self, Lit, LitKind}; use crate::parse::token::{self, Token}; use crate::symbol::{kw, sym, Symbol}; -use crate::tokenstream::{TokenStream, TokenTree}; +use crate::tokenstream::TokenTree; use log::debug; use rustc_data_structures::sync::Lrc; @@ -216,13 +216,13 @@ impl Lit { Lit { token: kind.to_lit_token(), kind, span } } - /// Losslessly convert an AST literal into a token stream. - crate fn tokens(&self) -> TokenStream { + /// Losslessly convert an AST literal into a token tree. + crate fn token_tree(&self) -> TokenTree { let token = match self.token.kind { token::Bool => token::Ident(self.token.symbol, false), _ => token::Literal(self.token), }; - TokenTree::token(token, self.span).into() + TokenTree::token(token, self.span) } } diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs index 6f7d1ead4c17c..188a144cac9de 100644 --- a/src/libsyntax/parse/parser/attr.rs +++ b/src/libsyntax/parse/parser/attr.rs @@ -6,7 +6,6 @@ use crate::tokenstream::{TokenStream, TokenTree}; use crate::source_map::Span; use log::debug; -use smallvec::smallvec; #[derive(Debug)] enum InnerAttributeParsePolicy<'a> { @@ -193,15 +192,15 @@ impl<'a> Parser<'a> { is_interpolated_expr = true; } } - let tokens = if is_interpolated_expr { + let token_tree = if is_interpolated_expr { // We need to accept arbitrary interpolated expressions to continue // supporting things like `doc = $expr` that work on stable. // Non-literal interpolated expressions are rejected after expansion. - self.parse_token_tree().into() + self.parse_token_tree() } else { - self.parse_unsuffixed_lit()?.tokens() + self.parse_unsuffixed_lit()?.token_tree() }; - TokenStream::from_streams(smallvec![eq.into(), tokens]) + TokenStream::new(vec![eq.into(), token_tree.into()]) } else { TokenStream::default() }; From e4ec4a6da350ae2564971ae826a1bc3ec9a41988 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Mon, 14 Oct 2019 14:06:00 +1100 Subject: [PATCH 4/4] Change `MetaItem::tokens()` to `MetaItem::token_trees_and_joints()`. Likewise for `NestedMetaItem::tokens()`. Also, add `MetaItemKind::token_trees_and_joints()`, which `MetaItemKind::tokens()` now calls. This avoids some unnecessary `TokenTree` to `TokenStream` conversions, and removes the need for the clumsy `TokenStream::append_to_tree_and_joint_vec()`. --- src/libsyntax/attr/mod.rs | 43 ++++++++++++++++++++++-------------- src/libsyntax/tokenstream.rs | 4 ---- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 8786c850bfdd2..4aec50408812f 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -22,7 +22,7 @@ use crate::ptr::P; use crate::sess::ParseSess; use crate::symbol::{sym, Symbol}; use crate::ThinVec; -use crate::tokenstream::{TokenStream, TokenTree, DelimSpan}; +use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use crate::GLOBALS; use log::debug; @@ -463,7 +463,7 @@ pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: Symbol) -> Option } impl MetaItem { - fn tokens(&self) -> TokenStream { + fn token_trees_and_joints(&self) -> Vec { let mut idents = vec![]; let mut last_pos = BytePos(0 as u32); for (i, segment) in self.path.segments.iter().enumerate() { @@ -477,8 +477,8 @@ impl MetaItem { idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into()); last_pos = segment.ident.span.hi(); } - self.kind.tokens(self.span).append_to_tree_and_joint_vec(&mut idents); - TokenStream::new(idents) + idents.extend(self.kind.token_trees_and_joints(self.span)); + idents } fn from_tokens(tokens: &mut iter::Peekable) -> Option @@ -537,14 +537,14 @@ impl MetaItem { } impl MetaItemKind { - pub fn tokens(&self, span: Span) -> TokenStream { + pub fn token_trees_and_joints(&self, span: Span) -> Vec { match *self { - MetaItemKind::Word => TokenStream::default(), + MetaItemKind::Word => vec![], MetaItemKind::NameValue(ref lit) => { - TokenStream::new(vec![ + vec![ TokenTree::token(token::Eq, span).into(), lit.token_tree().into(), - ]) + ] } MetaItemKind::List(ref list) => { let mut tokens = Vec::new(); @@ -552,17 +552,26 @@ impl MetaItemKind { if i > 0 { tokens.push(TokenTree::token(token::Comma, span).into()); } - item.tokens().append_to_tree_and_joint_vec(&mut tokens); + tokens.extend(item.token_trees_and_joints()) } - TokenTree::Delimited( - DelimSpan::from_single(span), - token::Paren, - TokenStream::new(tokens).into(), - ).into() + vec![ + TokenTree::Delimited( + DelimSpan::from_single(span), + token::Paren, + TokenStream::new(tokens).into(), + ).into() + ] } } } + // Premature conversions of `TokenTree`s to `TokenStream`s can hurt + // performance. Do not use this function if `token_trees_and_joints()` can + // be used instead. + pub fn tokens(&self, span: Span) -> TokenStream { + TokenStream::new(self.token_trees_and_joints(span)) + } + fn from_tokens(tokens: &mut iter::Peekable) -> Option where I: Iterator, { @@ -604,10 +613,10 @@ impl NestedMetaItem { } } - fn tokens(&self) -> TokenStream { + fn token_trees_and_joints(&self) -> Vec { match *self { - NestedMetaItem::MetaItem(ref item) => item.tokens(), - NestedMetaItem::Literal(ref lit) => lit.token_tree().into(), + NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(), + NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()], } } diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 3d89e73d729f3..ac155556cdae2 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -271,10 +271,6 @@ impl TokenStream { } } - pub fn append_to_tree_and_joint_vec(self, vec: &mut Vec) { - vec.extend(self.0.iter().cloned()); - } - pub fn trees(&self) -> Cursor { self.clone().into_trees() }