From a23dd0d1e6ddfe6624f1c59e9aefcb59e419610d Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 13 May 2020 02:43:23 +0100 Subject: [PATCH 1/8] Replace fcntl-based file lock with flock WSL1 does not support `fcntl`-based lock and will always report success, therefore creating a race condition when multiple rustc processes are modifying shared data such as `search-index.js`. WSL1 does however support `flock`. `flock` is supported by all unix-like platforms. The only caveat is that Linux 2.6.11 or earlier does not support `flock` on NFS mounts, but as the minimum supported Linux version is 2.6.18, it is not an issue. Fixes #72157 --- src/librustc_data_structures/flock.rs | 67 ++++++++------------------- 1 file changed, 19 insertions(+), 48 deletions(-) diff --git a/src/librustc_data_structures/flock.rs b/src/librustc_data_structures/flock.rs index 2a0139fa90d5a..d6fc552e66149 100644 --- a/src/librustc_data_structures/flock.rs +++ b/src/librustc_data_structures/flock.rs @@ -12,13 +12,12 @@ use std::path::Path; cfg_if! { if #[cfg(unix)] { - use std::ffi::{CString, OsStr}; - use std::mem; use std::os::unix::prelude::*; + use std::fs::{File, OpenOptions}; #[derive(Debug)] pub struct Lock { - fd: libc::c_int, + _file: File, } impl Lock { @@ -27,63 +26,35 @@ cfg_if! { create: bool, exclusive: bool) -> io::Result { - let os: &OsStr = p.as_ref(); - let buf = CString::new(os.as_bytes()).unwrap(); - let open_flags = if create { - libc::O_RDWR | libc::O_CREAT + let file = OpenOptions::new() + .read(true) + .write(true) + .create(create) + .mode(libc::S_IRWXU as u32) + .open(p)?; + + let mut operation = if exclusive { + libc::LOCK_EX } else { - libc::O_RDWR - }; - - let fd = unsafe { - libc::open(buf.as_ptr(), open_flags, - libc::S_IRWXU as libc::c_int) + libc::LOCK_SH }; - - if fd < 0 { - return Err(io::Error::last_os_error()); + if !wait { + operation |= libc::LOCK_NB } - let lock_type = if exclusive { - libc::F_WRLCK - } else { - libc::F_RDLCK - }; - - let mut flock: libc::flock = unsafe { mem::zeroed() }; - flock.l_type = lock_type as libc::c_short; - flock.l_whence = libc::SEEK_SET as libc::c_short; - flock.l_start = 0; - flock.l_len = 0; - - let cmd = if wait { libc::F_SETLKW } else { libc::F_SETLK }; - let ret = unsafe { - libc::fcntl(fd, cmd, &flock) - }; + let ret = unsafe { libc::flock(file.as_raw_fd(), operation) }; if ret == -1 { let err = io::Error::last_os_error(); - unsafe { libc::close(fd); } Err(err) } else { - Ok(Lock { fd }) + Ok(Lock { _file: file }) } } } - impl Drop for Lock { - fn drop(&mut self) { - let mut flock: libc::flock = unsafe { mem::zeroed() }; - flock.l_type = libc::F_UNLCK as libc::c_short; - flock.l_whence = libc::SEEK_SET as libc::c_short; - flock.l_start = 0; - flock.l_len = 0; - - unsafe { - libc::fcntl(self.fd, libc::F_SETLK, &flock); - libc::close(self.fd); - } - } - } + // Note that we don't need a Drop impl to execute `flock(fd, LOCK_UN)`. Lock acquired by + // `flock` is associated with the file descriptor and closing the file release it + // automatically. } else if #[cfg(windows)] { use std::mem; use std::os::windows::prelude::*; From 0148a7f26ce636ac22ac7797dcd7d292c59e8576 Mon Sep 17 00:00:00 2001 From: David Cook Date: Thu, 14 May 2020 07:46:43 -0500 Subject: [PATCH 2/8] InvalidUninitBytes: Track more info about access --- .../mir/interpret/allocation.rs | 37 ++++++++++++------ src/librustc_middle/mir/interpret/error.rs | 39 ++++++++++++++++--- src/librustc_middle/mir/interpret/mod.rs | 2 +- src/librustc_mir/interpret/validity.rs | 14 ++++--- 4 files changed, 67 insertions(+), 25 deletions(-) diff --git a/src/librustc_middle/mir/interpret/allocation.rs b/src/librustc_middle/mir/interpret/allocation.rs index 2b6cf224e2c1b..96195db0bacd2 100644 --- a/src/librustc_middle/mir/interpret/allocation.rs +++ b/src/librustc_middle/mir/interpret/allocation.rs @@ -11,6 +11,7 @@ use rustc_target::abi::{Align, HasDataLayout, Size}; use super::{ read_target_uint, write_target_uint, AllocId, InterpResult, Pointer, Scalar, ScalarMaybeUninit, + UninitBytesAccess, }; #[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash, RustcEncodable, RustcDecodable)] @@ -545,17 +546,23 @@ impl<'tcx, Tag: Copy, Extra> Allocation { impl<'tcx, Tag: Copy, Extra> Allocation { /// Checks whether the given range is entirely defined. /// - /// Returns `Ok(())` if it's defined. Otherwise returns the index of the byte - /// at which the first undefined access begins. - fn is_defined(&self, ptr: Pointer, size: Size) -> Result<(), Size> { + /// Returns `Ok(())` if it's defined. Otherwise returns the range of byte + /// indexes of the first contiguous undefined access. + fn is_defined(&self, ptr: Pointer, size: Size) -> Result<(), Range> { self.init_mask.is_range_initialized(ptr.offset, ptr.offset + size) // `Size` addition } - /// Checks that a range of bytes is defined. If not, returns the `ReadUndefBytes` - /// error which will report the first byte which is undefined. + /// Checks that a range of bytes is defined. If not, returns the `InvalidUndefBytes` + /// error which will report the first range of bytes which is undefined. fn check_defined(&self, ptr: Pointer, size: Size) -> InterpResult<'tcx> { - self.is_defined(ptr, size) - .or_else(|idx| throw_ub!(InvalidUninitBytes(Some(Pointer::new(ptr.alloc_id, idx))))) + self.is_defined(ptr, size).or_else(|idx_range| { + throw_ub!(InvalidUninitBytes(Some(Box::new(UninitBytesAccess { + access_ptr: ptr.erase_tag(), + access_size: size, + uninit_ptr: Pointer::new(ptr.alloc_id, idx_range.start), + uninit_size: idx_range.end - idx_range.start, // `Size` subtraction + })))) + }) } pub fn mark_definedness(&mut self, ptr: Pointer, size: Size, new_state: bool) { @@ -758,19 +765,25 @@ impl InitMask { /// Checks whether the range `start..end` (end-exclusive) is entirely initialized. /// - /// Returns `Ok(())` if it's initialized. Otherwise returns the index of the byte - /// at which the first uninitialized access begins. + /// Returns `Ok(())` if it's initialized. Otherwise returns a range of byte + /// indexes for the first contiguous span of the uninitialized access. #[inline] - pub fn is_range_initialized(&self, start: Size, end: Size) -> Result<(), Size> { + pub fn is_range_initialized(&self, start: Size, end: Size) -> Result<(), Range> { if end > self.len { - return Err(self.len); + return Err(self.len..end); } // FIXME(oli-obk): optimize this for allocations larger than a block. let idx = (start.bytes()..end.bytes()).map(Size::from_bytes).find(|&i| !self.get(i)); match idx { - Some(idx) => Err(idx), + Some(idx) => { + let undef_end = (idx.bytes()..end.bytes()) + .map(Size::from_bytes) + .find(|&i| self.get(i)) + .unwrap_or(end); + Err(idx..undef_end) + } None => Ok(()), } } diff --git a/src/librustc_middle/mir/interpret/error.rs b/src/librustc_middle/mir/interpret/error.rs index 06fe3793b2383..d32a147344992 100644 --- a/src/librustc_middle/mir/interpret/error.rs +++ b/src/librustc_middle/mir/interpret/error.rs @@ -6,7 +6,7 @@ use crate::ty::query::TyCtxtAt; use crate::ty::{self, layout, tls, FnSig, Ty}; use rustc_data_structures::sync::Lock; -use rustc_errors::{struct_span_err, DiagnosticBuilder, ErrorReported}; +use rustc_errors::{pluralize, struct_span_err, DiagnosticBuilder, ErrorReported}; use rustc_hir as hir; use rustc_hir::definitions::DefPathData; use rustc_macros::HashStable; @@ -327,6 +327,19 @@ impl fmt::Display for CheckInAllocMsg { } } +/// Details of an access to uninitialized bytes where it is not allowed. +#[derive(Debug)] +pub struct UninitBytesAccess { + /// Location of the original memory access. + pub access_ptr: Pointer, + /// Size of the original memory access. + pub access_size: Size, + /// Location of the first uninitialized byte that was accessed. + pub uninit_ptr: Pointer, + /// Number of consecutive uninitialized bytes that were accessed. + pub uninit_size: Size, +} + /// Error information for when the program caused Undefined Behavior. pub enum UndefinedBehaviorInfo<'tcx> { /// Free-form case. Only for errors that are never caught! @@ -384,7 +397,7 @@ pub enum UndefinedBehaviorInfo<'tcx> { /// Using a string that is not valid UTF-8, InvalidStr(std::str::Utf8Error), /// Using uninitialized data where it is not allowed. - InvalidUninitBytes(Option), + InvalidUninitBytes(Option>), /// Working with a local that is not currently live. DeadLocal, /// Data size is not equal to target size. @@ -455,10 +468,18 @@ impl fmt::Display for UndefinedBehaviorInfo<'_> { write!(f, "using {} as function pointer but it does not point to a function", p) } InvalidStr(err) => write!(f, "this string is not valid UTF-8: {}", err), - InvalidUninitBytes(Some(p)) => write!( + InvalidUninitBytes(Some(access)) => write!( f, - "reading uninitialized memory at {}, but this operation requires initialized memory", - p + "reading {} byte{} of memory starting at {}, \ + but {} byte{} {} uninitialized starting at {}, \ + and this operation requires initialized memory", + access.access_size.bytes(), + pluralize!(access.access_size.bytes()), + access.access_ptr, + access.uninit_size.bytes(), + pluralize!(access.uninit_size.bytes()), + if access.uninit_size.bytes() != 1 { "are" } else { "is" }, + access.uninit_ptr, ), InvalidUninitBytes(None) => write!( f, @@ -556,6 +577,9 @@ impl dyn MachineStopType { } } +#[cfg(target_arch = "x86_64")] +static_assert_size!(InterpError<'_>, 40); + pub enum InterpError<'tcx> { /// The program caused undefined behavior. UndefinedBehavior(UndefinedBehaviorInfo<'tcx>), @@ -604,7 +628,10 @@ impl InterpError<'_> { InterpError::MachineStop(b) => mem::size_of_val::(&**b) > 0, InterpError::Unsupported(UnsupportedOpInfo::Unsupported(_)) | InterpError::UndefinedBehavior(UndefinedBehaviorInfo::ValidationFailure(_)) - | InterpError::UndefinedBehavior(UndefinedBehaviorInfo::Ub(_)) => true, + | InterpError::UndefinedBehavior(UndefinedBehaviorInfo::Ub(_)) + | InterpError::UndefinedBehavior(UndefinedBehaviorInfo::InvalidUninitBytes(Some(_))) => { + true + } _ => false, } } diff --git a/src/librustc_middle/mir/interpret/mod.rs b/src/librustc_middle/mir/interpret/mod.rs index 71adb2fa477ad..d9e52af89007c 100644 --- a/src/librustc_middle/mir/interpret/mod.rs +++ b/src/librustc_middle/mir/interpret/mod.rs @@ -119,7 +119,7 @@ use crate::ty::{self, Instance, Ty, TyCtxt}; pub use self::error::{ struct_error, CheckInAllocMsg, ConstEvalErr, ConstEvalRawResult, ConstEvalResult, ErrorHandled, FrameInfo, InterpError, InterpErrorInfo, InterpResult, InvalidProgramInfo, MachineStopType, - ResourceExhaustionInfo, UndefinedBehaviorInfo, UnsupportedOpInfo, + ResourceExhaustionInfo, UndefinedBehaviorInfo, UninitBytesAccess, UnsupportedOpInfo, }; pub use self::value::{get_slice_bytes, ConstValue, RawConst, Scalar, ScalarMaybeUninit}; diff --git a/src/librustc_mir/interpret/validity.rs b/src/librustc_mir/interpret/validity.rs index 7eb05ba2d2c54..6ce93f21021ca 100644 --- a/src/librustc_mir/interpret/validity.rs +++ b/src/librustc_mir/interpret/validity.rs @@ -366,7 +366,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let place = try_validation!( self.ecx.ref_to_mplace(value), self.path, - err_ub!(InvalidUninitBytes(..)) => { "uninitialized {}", kind }, + err_ub!(InvalidUninitBytes { .. }) => { "uninitialized {}", kind }, ); if place.layout.is_unsized() { self.check_wide_ptr_meta(place.meta, place.layout)?; @@ -514,7 +514,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let place = try_validation!( self.ecx.ref_to_mplace(self.ecx.read_immediate(value)?), self.path, - err_ub!(InvalidUninitBytes(..)) => { "uninitialized raw pointer" }, + err_ub!(InvalidUninitBytes { .. } ) => { "uninitialized raw pointer" }, ); if place.layout.is_unsized() { self.check_wide_ptr_meta(place.meta, place.layout)?; @@ -593,7 +593,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, ' let value = try_validation!( value.not_undef(), self.path, - err_ub!(InvalidUninitBytes(..)) => { "{}", value } + err_ub!(InvalidUninitBytes { .. }) => { "{}", value } expected { "something {}", wrapping_range_format(valid_range, max_hi) }, ); let bits = match value.to_bits_or_ptr(op.layout.size, self.ecx) { @@ -804,12 +804,14 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M> // For some errors we might be able to provide extra information. // (This custom logic does not fit the `try_validation!` macro.) match err.kind { - err_ub!(InvalidUninitBytes(Some(ptr))) => { + err_ub!(InvalidUninitBytes(Some(access))) => { // Some byte was uninitialized, determine which // element that byte belongs to so we can // provide an index. - let i = usize::try_from(ptr.offset.bytes() / layout.size.bytes()) - .unwrap(); + let i = usize::try_from( + access.uninit_ptr.offset.bytes() / layout.size.bytes(), + ) + .unwrap(); self.path.push(PathElem::ArrayElem(i)); throw_validation_failure!(self.path, { "uninitialized bytes" }) From cc91041f26fa90920cc96c13c431610153a61586 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 18 May 2020 08:00:44 -0700 Subject: [PATCH 3/8] Always generated object code for `#![no_builtins]` This commit updates the code generation for `#![no_builtins]` to always produce object files instead of conditionally respecting `-Clinker-plugin-lto` and sometimes producing bitcode. This is intended to address rust-lang/cargo#8239. The issue at hand here is that Cargo has tried to get "smarter" about codegen in whole crate graph scenarios. When LTO is enabled it attempts to avoid codegen on as many crates as possible, opting to pass `-Clinker-plugin-lto` where it can to only generate bitcode. When this is combined with `-Zbuild-std`, however, it means that `compiler-builtins` only generates LLVM bitcode instead of object files. Rustc's own LTO passes then explicitly skip `compiler-builtins` (because it wouldn't work anyway) which means that LLVM bitcode gets sent to the linker, which chokes most of the time. The fix in this PR is to not actually respect `-Clinker-plugin-lto` for `#![no_builtins]` crates. These crates, even if slurped up by the linker rather than rustc, will not work with LTO. They define symbols which are only referenced as part of codegen, so LTO's aggressive internalization would trivially remove the symbols only to have the linker realize later that the symbol is undefined. Since pure-bitcode never makes sense for these libraries, the `-Clinker-plugin-lto` flag is silently ignored. --- src/librustc_codegen_ssa/back/write.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/librustc_codegen_ssa/back/write.rs b/src/librustc_codegen_ssa/back/write.rs index 53dfe7cb74998..f9ee7d8c5de71 100644 --- a/src/librustc_codegen_ssa/back/write.rs +++ b/src/librustc_codegen_ssa/back/write.rs @@ -142,8 +142,22 @@ impl ModuleConfig { let emit_obj = if !should_emit_obj { EmitObj::None } else if sess.target.target.options.obj_is_bitcode - || sess.opts.cg.linker_plugin_lto.enabled() + || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins) { + // This case is selected if the target uses objects as bitcode, or + // if linker plugin LTO is enabled. In the linker plugin LTO case + // the assumption is that the final link-step will read the bitcode + // and convert it to object code. This may be done by either the + // native linker or rustc itself. + // + // Note, however, that the linker-plugin-lto requested here is + // explicitly ignored for `#![no_builtins]` crates. These crates are + // specifically ignored by rustc's LTO passes and wouldn't work if + // loaded into the linker. These crates define symbols that LLVM + // lowers intrinsics to, and these symbol dependencies aren't known + // until after codegen. As a result any crate marked + // `#![no_builtins]` is assumed to not participate in LTO and + // instead goes on to generate object code. EmitObj::Bitcode } else if need_bitcode_in_object(sess) { EmitObj::ObjectCode(BitcodeSection::Full) From 9b2b8a5afa833795b66267684615497c9547878d Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Sun, 17 May 2020 15:51:01 -0400 Subject: [PATCH 4/8] Break tokens before checking if they are 'probably equal' Fixes #68489 When checking two `TokenStreams` to see if they are 'probably equal', we ignore the `IsJoint` information associated with each `TokenTree`. However, the `IsJoint` information determines whether adjacent tokens will be 'glued' (if possible) when construction the `TokenStream` - e.g. `[Gt Gt]` can be 'glued' to `BinOp(Shr)`. Since we are ignoring the `IsJoint` information, 'glued' and 'unglued' tokens are equivalent for determining if two `TokenStreams` are 'probably equal'. Therefore, we need to 'unglue' all tokens in the stream to avoid false negatives (which cause us to throw out the cached tokens, losing span information). --- src/librustc_ast/tokenstream.rs | 34 +++++++++++++++++-- src/test/ui/proc-macro/turbo-proc-macro.rs | 9 +++++ .../ui/proc-macro/turbo-proc-macro.stderr | 9 +++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 src/test/ui/proc-macro/turbo-proc-macro.rs create mode 100644 src/test/ui/proc-macro/turbo-proc-macro.stderr diff --git a/src/librustc_ast/tokenstream.rs b/src/librustc_ast/tokenstream.rs index 916a5ff6f46f4..38483360c0664 100644 --- a/src/librustc_ast/tokenstream.rs +++ b/src/librustc_ast/tokenstream.rs @@ -338,8 +338,38 @@ impl TokenStream { true } - let mut t1 = self.trees().filter(semantic_tree); - let mut t2 = other.trees().filter(semantic_tree); + // When comparing two `TokenStream`s, we ignore the `IsJoint` information. + // + // However, `rustc_parse::lexer::tokentrees::TokenStreamBuilder` will + // use `Token.glue` on adjacent tokens with the proper `IsJoint`. + // Since we are ignoreing `IsJoint`, a 'glued' token (e.g. `BinOp(Shr)`) + // and its 'split'/'unglued' compoenents (e.g. `Gt, Gt`) are equivalent + // when determining if two `TokenStream`s are 'probably equal'. + // + // Therefore, we use `break_two_token_op` to convert all tokens + // to the 'unglued' form (if it exists). This ensures that two + // `TokenStream`s which differ only in how their tokens are glued + // will be considered 'probably equal', which allows us to keep spans. + // + // This is important when the original `TokenStream` contained + // extra spaces (e.g. `f :: < Vec < _ > > ( ) ;'). These extra spaces + // will be omitted when we pretty-print, which can cause the original + // and reparsed `TokenStream`s to differ in the assignment of `IsJoint`, + // leading to some tokens being 'glued' together in one stream but not + // the other. See #68489 for more details. + fn break_tokens(tree: TokenTree) -> impl Iterator { + if let TokenTree::Token(token) = &tree { + if let Some((first, second)) = token.kind.break_two_token_op() { + return SmallVec::from_buf([TokenTree::Token(Token::new(first, DUMMY_SP)), TokenTree::Token(Token::new(second, DUMMY_SP))]).into_iter() + } + } + let mut vec = SmallVec::<[_; 2]>::new(); + vec.push(tree); + vec.into_iter() + } + + let mut t1 = self.trees().filter(semantic_tree).flat_map(break_tokens); + let mut t2 = other.trees().filter(semantic_tree).flat_map(break_tokens); for (t1, t2) in t1.by_ref().zip(t2.by_ref()) { if !t1.probably_equal_for_proc_macro(&t2) { return false; diff --git a/src/test/ui/proc-macro/turbo-proc-macro.rs b/src/test/ui/proc-macro/turbo-proc-macro.rs new file mode 100644 index 0000000000000..a255955f38da0 --- /dev/null +++ b/src/test/ui/proc-macro/turbo-proc-macro.rs @@ -0,0 +1,9 @@ +// aux-build:test-macros.rs + +extern crate test_macros; + +#[test_macros::recollect_attr] +fn repro() { + f :: < Vec < _ > > ( ) ; //~ ERROR cannot find +} +fn main() {} diff --git a/src/test/ui/proc-macro/turbo-proc-macro.stderr b/src/test/ui/proc-macro/turbo-proc-macro.stderr new file mode 100644 index 0000000000000..85c93b9345c37 --- /dev/null +++ b/src/test/ui/proc-macro/turbo-proc-macro.stderr @@ -0,0 +1,9 @@ +error[E0425]: cannot find function `f` in this scope + --> $DIR/turbo-proc-macro.rs:7:5 + | +LL | f :: < Vec < _ > > ( ) ; + | ^ not found in this scope + +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0425`. From 4a8ccdcc0b518e3c1878ce0be888fd85521b2026 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Tue, 19 May 2020 19:45:58 -0400 Subject: [PATCH 5/8] Use a fixed-point iteration when breaking tokens Some tokens need to be broken in a loop until we reach 'unbreakable' tokens. --- src/librustc_ast/tokenstream.rs | 45 ++++++++++++++++--- src/test/ui/proc-macro/break-token-spans.rs | 16 +++++++ .../ui/proc-macro/break-token-spans.stderr | 21 +++++++++ src/test/ui/proc-macro/turbo-proc-macro.rs | 9 ---- .../ui/proc-macro/turbo-proc-macro.stderr | 9 ---- 5 files changed, 77 insertions(+), 23 deletions(-) create mode 100644 src/test/ui/proc-macro/break-token-spans.rs create mode 100644 src/test/ui/proc-macro/break-token-spans.stderr delete mode 100644 src/test/ui/proc-macro/turbo-proc-macro.rs delete mode 100644 src/test/ui/proc-macro/turbo-proc-macro.stderr diff --git a/src/librustc_ast/tokenstream.rs b/src/librustc_ast/tokenstream.rs index 38483360c0664..075aaa7e5bc01 100644 --- a/src/librustc_ast/tokenstream.rs +++ b/src/librustc_ast/tokenstream.rs @@ -21,6 +21,8 @@ use rustc_macros::HashStable_Generic; use rustc_span::{Span, DUMMY_SP}; use smallvec::{smallvec, SmallVec}; +use log::debug; + use std::{iter, mem}; /// When the main rust parser encounters a syntax-extension invocation, it @@ -358,14 +360,47 @@ impl TokenStream { // leading to some tokens being 'glued' together in one stream but not // the other. See #68489 for more details. fn break_tokens(tree: TokenTree) -> impl Iterator { + // In almost all cases, we should have either zero or one levels + // of 'unglueing'. However, in some unusual cases, we may need + // to iterate breaking tokens mutliple times. For example: + // '[BinOpEq(Shr)] => [Gt, Ge] -> [Gt, Gt, Eq]' + let mut token_trees: SmallVec<[_; 2]>; if let TokenTree::Token(token) = &tree { - if let Some((first, second)) = token.kind.break_two_token_op() { - return SmallVec::from_buf([TokenTree::Token(Token::new(first, DUMMY_SP)), TokenTree::Token(Token::new(second, DUMMY_SP))]).into_iter() + let mut out = SmallVec::<[_; 2]>::new(); + out.push(token.clone()); + // Iterate to fixpoint: + // * We start off with 'out' containing our initial token, and `temp` empty + // * If we are able to break any tokens in `out`, then `out` will have + // at least one more element than 'temp', so we will try to break tokens + // again. + // * If we cannot break any tokens in 'out', we are done + loop { + let mut temp = SmallVec::<[_; 2]>::new(); + let mut changed = false; + + for token in out.into_iter() { + if let Some((first, second)) = token.kind.break_two_token_op() { + temp.push(Token::new(first, DUMMY_SP)); + temp.push(Token::new(second, DUMMY_SP)); + changed = true; + } else { + temp.push(token); + } + } + out = temp; + if !changed { + break; + } + } + token_trees = out.into_iter().map(|t| TokenTree::Token(t)).collect(); + if token_trees.len() != 1 { + debug!("break_tokens: broke {:?} to {:?}", tree, token_trees); } + } else { + token_trees = SmallVec::new(); + token_trees.push(tree); } - let mut vec = SmallVec::<[_; 2]>::new(); - vec.push(tree); - vec.into_iter() + token_trees.into_iter() } let mut t1 = self.trees().filter(semantic_tree).flat_map(break_tokens); diff --git a/src/test/ui/proc-macro/break-token-spans.rs b/src/test/ui/proc-macro/break-token-spans.rs new file mode 100644 index 0000000000000..ce8b9ebb4f9d2 --- /dev/null +++ b/src/test/ui/proc-macro/break-token-spans.rs @@ -0,0 +1,16 @@ +// aux-build:test-macros.rs +// Regression test for issues #68489 and #70987 +// Tests that we properly break tokens in `probably_equal_for_proc_macro` +// See #72306 +// +// Note that the weird spacing in this example is critical +// for testing the issue. + +extern crate test_macros; + +#[test_macros::recollect_attr] +fn repro() { + f :: < Vec < _ > > ( ) ; //~ ERROR cannot find + let a: Option>= true; //~ ERROR mismatched +} +fn main() {} diff --git a/src/test/ui/proc-macro/break-token-spans.stderr b/src/test/ui/proc-macro/break-token-spans.stderr new file mode 100644 index 0000000000000..caca973f252f7 --- /dev/null +++ b/src/test/ui/proc-macro/break-token-spans.stderr @@ -0,0 +1,21 @@ +error[E0425]: cannot find function `f` in this scope + --> $DIR/break-token-spans.rs:13:5 + | +LL | f :: < Vec < _ > > ( ) ; + | ^ not found in this scope + +error[E0308]: mismatched types + --> $DIR/break-token-spans.rs:14:32 + | +LL | let a: Option>= true; + | ------------------ ^^^^ expected enum `std::option::Option`, found `bool` + | | + | expected due to this + | + = note: expected enum `std::option::Option>` + found type `bool` + +error: aborting due to 2 previous errors + +Some errors have detailed explanations: E0308, E0425. +For more information about an error, try `rustc --explain E0308`. diff --git a/src/test/ui/proc-macro/turbo-proc-macro.rs b/src/test/ui/proc-macro/turbo-proc-macro.rs deleted file mode 100644 index a255955f38da0..0000000000000 --- a/src/test/ui/proc-macro/turbo-proc-macro.rs +++ /dev/null @@ -1,9 +0,0 @@ -// aux-build:test-macros.rs - -extern crate test_macros; - -#[test_macros::recollect_attr] -fn repro() { - f :: < Vec < _ > > ( ) ; //~ ERROR cannot find -} -fn main() {} diff --git a/src/test/ui/proc-macro/turbo-proc-macro.stderr b/src/test/ui/proc-macro/turbo-proc-macro.stderr deleted file mode 100644 index 85c93b9345c37..0000000000000 --- a/src/test/ui/proc-macro/turbo-proc-macro.stderr +++ /dev/null @@ -1,9 +0,0 @@ -error[E0425]: cannot find function `f` in this scope - --> $DIR/turbo-proc-macro.rs:7:5 - | -LL | f :: < Vec < _ > > ( ) ; - | ^ not found in this scope - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0425`. From 564ebbb0d19283894e87cd09333375aa0c84f8d9 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Wed, 20 May 2020 02:27:03 +0100 Subject: [PATCH 6/8] Use fcntl-based file lock for non-Linux unix --- src/librustc_data_structures/flock.rs | 77 +++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 10 deletions(-) diff --git a/src/librustc_data_structures/flock.rs b/src/librustc_data_structures/flock.rs index d6fc552e66149..655248e0f5221 100644 --- a/src/librustc_data_structures/flock.rs +++ b/src/librustc_data_structures/flock.rs @@ -7,13 +7,13 @@ #![allow(non_camel_case_types)] #![allow(nonstandard_style)] +use std::fs::{File, OpenOptions}; use std::io; use std::path::Path; cfg_if! { - if #[cfg(unix)] { + if #[cfg(target_os = "linux")] { use std::os::unix::prelude::*; - use std::fs::{File, OpenOptions}; #[derive(Debug)] pub struct Lock { @@ -27,11 +27,11 @@ cfg_if! { exclusive: bool) -> io::Result { let file = OpenOptions::new() - .read(true) - .write(true) - .create(create) - .mode(libc::S_IRWXU as u32) - .open(p)?; + .read(true) + .write(true) + .create(create) + .mode(libc::S_IRWXU as u32) + .open(p)?; let mut operation = if exclusive { libc::LOCK_EX @@ -44,8 +44,7 @@ cfg_if! { let ret = unsafe { libc::flock(file.as_raw_fd(), operation) }; if ret == -1 { - let err = io::Error::last_os_error(); - Err(err) + Err(io::Error::last_os_error()) } else { Ok(Lock { _file: file }) } @@ -55,10 +54,68 @@ cfg_if! { // Note that we don't need a Drop impl to execute `flock(fd, LOCK_UN)`. Lock acquired by // `flock` is associated with the file descriptor and closing the file release it // automatically. + } else if #[cfg(unix)] { + use std::mem; + use std::os::unix::prelude::*; + + #[derive(Debug)] + pub struct Lock { + file: File, + } + + impl Lock { + pub fn new(p: &Path, + wait: bool, + create: bool, + exclusive: bool) + -> io::Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(create) + .mode(libc::S_IRWXU as u32) + .open(p)?; + + let lock_type = if exclusive { + libc::F_WRLCK + } else { + libc::F_RDLCK + }; + + let mut flock: libc::flock = unsafe { mem::zeroed() }; + flock.l_type = lock_type as libc::c_short; + flock.l_whence = libc::SEEK_SET as libc::c_short; + flock.l_start = 0; + flock.l_len = 0; + + let cmd = if wait { libc::F_SETLKW } else { libc::F_SETLK }; + let ret = unsafe { + libc::fcntl(file.as_raw_fd(), cmd, &flock) + }; + if ret == -1 { + Err(io::Error::last_os_error()) + } else { + Ok(Lock { file }) + } + } + } + + impl Drop for Lock { + fn drop(&mut self) { + let mut flock: libc::flock = unsafe { mem::zeroed() }; + flock.l_type = libc::F_UNLCK as libc::c_short; + flock.l_whence = libc::SEEK_SET as libc::c_short; + flock.l_start = 0; + flock.l_len = 0; + + unsafe { + libc::fcntl(self.file.as_raw_fd(), libc::F_SETLK, &flock); + } + } + } } else if #[cfg(windows)] { use std::mem; use std::os::windows::prelude::*; - use std::fs::{File, OpenOptions}; use winapi::um::minwinbase::{OVERLAPPED, LOCKFILE_FAIL_IMMEDIATELY, LOCKFILE_EXCLUSIVE_LOCK}; use winapi::um::fileapi::LockFileEx; From 633293fc3a54247f4308507632040424356a9e19 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Wed, 20 May 2020 15:33:58 -0400 Subject: [PATCH 7/8] Fix tests --- src/test/ui/proc-macro/break-token-spans.rs | 2 +- src/test/ui/suggestions/issue-61963.rs | 1 + src/test/ui/suggestions/issue-61963.stderr | 10 ++++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/test/ui/proc-macro/break-token-spans.rs b/src/test/ui/proc-macro/break-token-spans.rs index ce8b9ebb4f9d2..59dc3b5043cd7 100644 --- a/src/test/ui/proc-macro/break-token-spans.rs +++ b/src/test/ui/proc-macro/break-token-spans.rs @@ -2,7 +2,7 @@ // Regression test for issues #68489 and #70987 // Tests that we properly break tokens in `probably_equal_for_proc_macro` // See #72306 -// +// // Note that the weird spacing in this example is critical // for testing the issue. diff --git a/src/test/ui/suggestions/issue-61963.rs b/src/test/ui/suggestions/issue-61963.rs index c9d738f5a283e..666fc965f02f5 100644 --- a/src/test/ui/suggestions/issue-61963.rs +++ b/src/test/ui/suggestions/issue-61963.rs @@ -16,6 +16,7 @@ pub struct Qux(T); #[dom_struct] pub struct Foo { + //~^ ERROR trait objects without an explicit `dyn` are deprecated [bare_trait_objects] qux: Qux>, bar: Box, //~^ ERROR trait objects without an explicit `dyn` are deprecated [bare_trait_objects] diff --git a/src/test/ui/suggestions/issue-61963.stderr b/src/test/ui/suggestions/issue-61963.stderr index 0e2eb7616cf9d..62ae5fa3fe54f 100644 --- a/src/test/ui/suggestions/issue-61963.stderr +++ b/src/test/ui/suggestions/issue-61963.stderr @@ -1,5 +1,5 @@ error: trait objects without an explicit `dyn` are deprecated - --> $DIR/issue-61963.rs:20:14 + --> $DIR/issue-61963.rs:21:14 | LL | bar: Box, | ^^^ help: use `dyn`: `dyn Bar` @@ -10,5 +10,11 @@ note: the lint level is defined here LL | #![deny(bare_trait_objects)] | ^^^^^^^^^^^^^^^^^^ -error: aborting due to previous error +error: trait objects without an explicit `dyn` are deprecated + --> $DIR/issue-61963.rs:18:1 + | +LL | pub struct Foo { + | ^^^ help: use `dyn`: `dyn pub` + +error: aborting due to 2 previous errors From a05acbf36f5a1f49919253281fc9bf9465606070 Mon Sep 17 00:00:00 2001 From: Gary Guo Date: Thu, 21 May 2020 01:10:52 +0100 Subject: [PATCH 8/8] Comment flock usage on Linux --- src/librustc_data_structures/flock.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/librustc_data_structures/flock.rs b/src/librustc_data_structures/flock.rs index 655248e0f5221..9383be474fd5a 100644 --- a/src/librustc_data_structures/flock.rs +++ b/src/librustc_data_structures/flock.rs @@ -12,6 +12,11 @@ use std::io; use std::path::Path; cfg_if! { + // We use `flock` rather than `fcntl` on Linux, because WSL1 does not support + // `fcntl`-style advisory locks properly (rust-lang/rust#72157). + // + // For other Unix targets we still use `fcntl` because it's more portable than + // `flock`. if #[cfg(target_os = "linux")] { use std::os::unix::prelude::*;