From 23d3ff1b9756c768c4412dcd1d80cff0617fd5c5 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Sun, 6 Oct 2019 23:45:25 +0200 Subject: [PATCH 01/43] Fix zero-size uninitialized boxes Requesting a zero-size allocation is not allowed, return a dangling pointer instead. CC https://github.com/rust-lang/rust/issues/63291#issuecomment-538692745 --- src/liballoc/boxed.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/liballoc/boxed.rs b/src/liballoc/boxed.rs index 9b5d9431ae204..2693a64e13ba1 100644 --- a/src/liballoc/boxed.rs +++ b/src/liballoc/boxed.rs @@ -141,6 +141,9 @@ impl Box { /// ``` #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit() -> Box> { + if mem::size_of::() == 0 { + return Box(NonNull::dangling().into()) + } let layout = alloc::Layout::new::>(); let ptr = unsafe { Global.alloc(layout) @@ -181,10 +184,17 @@ impl Box<[T]> { /// ``` #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit]> { - let layout = alloc::Layout::array::>(len).unwrap(); - let ptr = unsafe { alloc::alloc(layout) }; - let unique = Unique::new(ptr).unwrap_or_else(|| alloc::handle_alloc_error(layout)); - let slice = unsafe { slice::from_raw_parts_mut(unique.cast().as_ptr(), len) }; + let ptr = if mem::size_of::() == 0 || len == 0 { + NonNull::dangling() + } else { + let layout = alloc::Layout::array::>(len).unwrap(); + unsafe { + Global.alloc(layout) + .unwrap_or_else(|_| alloc::handle_alloc_error(layout)) + .cast() + } + }; + let slice = unsafe { slice::from_raw_parts_mut(ptr.as_ptr(), len) }; Box(Unique::from(slice)) } } From 9d5208a0d2b54232c4d8ddc0927b8417d62dbbd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= Date: Sun, 6 Oct 2019 00:00:00 +0000 Subject: [PATCH 02/43] Remove sanitizer runtime crates --- Cargo.lock | 48 -------------- src/bootstrap/dist.rs | 4 -- src/build_helper/lib.rs | 119 ----------------------------------- src/librustc_asan/Cargo.toml | 20 ------ src/librustc_asan/build.rs | 30 --------- src/librustc_asan/lib.rs | 8 --- src/librustc_lsan/Cargo.toml | 20 ------ src/librustc_lsan/build.rs | 29 --------- src/librustc_lsan/lib.rs | 8 --- src/librustc_msan/Cargo.toml | 20 ------ src/librustc_msan/build.rs | 29 --------- src/librustc_msan/lib.rs | 8 --- src/librustc_tsan/Cargo.toml | 20 ------ src/librustc_tsan/build.rs | 30 --------- src/librustc_tsan/lib.rs | 8 --- src/libstd/Cargo.toml | 10 --- 16 files changed, 411 deletions(-) delete mode 100644 src/librustc_asan/Cargo.toml delete mode 100644 src/librustc_asan/build.rs delete mode 100644 src/librustc_asan/lib.rs delete mode 100644 src/librustc_lsan/Cargo.toml delete mode 100644 src/librustc_lsan/build.rs delete mode 100644 src/librustc_lsan/lib.rs delete mode 100644 src/librustc_msan/Cargo.toml delete mode 100644 src/librustc_msan/build.rs delete mode 100644 src/librustc_msan/lib.rs delete mode 100644 src/librustc_tsan/Cargo.toml delete mode 100644 src/librustc_tsan/build.rs delete mode 100644 src/librustc_tsan/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 844320fff3f13..eb3cefd98aaec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3387,17 +3387,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "rustc_asan" -version = "0.0.0" -dependencies = [ - "alloc", - "build_helper", - "cmake", - "compiler_builtins", - "core", -] - [[package]] name = "rustc_codegen_llvm" version = "0.0.0" @@ -3592,17 +3581,6 @@ dependencies = [ "cc", ] -[[package]] -name = "rustc_lsan" -version = "0.0.0" -dependencies = [ - "alloc", - "build_helper", - "cmake", - "compiler_builtins", - "core", -] - [[package]] name = "rustc_macros" version = "0.1.0" @@ -3656,17 +3634,6 @@ dependencies = [ "syntax_pos", ] -[[package]] -name = "rustc_msan" -version = "0.0.0" -dependencies = [ - "alloc", - "build_helper", - "cmake", - "compiler_builtins", - "core", -] - [[package]] name = "rustc_passes" version = "0.0.0" @@ -3778,17 +3745,6 @@ dependencies = [ "syntax_pos", ] -[[package]] -name = "rustc_tsan" -version = "0.0.0" -dependencies = [ - "alloc", - "build_helper", - "cmake", - "compiler_builtins", - "core", -] - [[package]] name = "rustc_typeck" version = "0.0.0" @@ -4143,10 +4099,6 @@ dependencies = [ "panic_unwind", "profiler_builtins", "rand 0.7.0", - "rustc_asan", - "rustc_lsan", - "rustc_msan", - "rustc_tsan", "unwind", "wasi", ] diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index d9dff77a30e6b..02384f103fd05 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -901,10 +901,6 @@ impl Step for Src { "src/libcore", "src/libpanic_abort", "src/libpanic_unwind", - "src/librustc_asan", - "src/librustc_lsan", - "src/librustc_msan", - "src/librustc_tsan", "src/libstd", "src/libunwind", "src/libtest", diff --git a/src/build_helper/lib.rs b/src/build_helper/lib.rs index f035a7119188a..54991df825a40 100644 --- a/src/build_helper/lib.rs +++ b/src/build_helper/lib.rs @@ -1,9 +1,7 @@ -use std::fs::File; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::{SystemTime, UNIX_EPOCH}; use std::{env, fs}; -use std::thread; /// A helper macro to `unwrap` a result except also print out details like: /// @@ -182,123 +180,6 @@ pub fn up_to_date(src: &Path, dst: &Path) -> bool { } } -#[must_use] -pub struct NativeLibBoilerplate { - pub src_dir: PathBuf, - pub out_dir: PathBuf, -} - -impl NativeLibBoilerplate { - /// On macOS we don't want to ship the exact filename that compiler-rt builds. - /// This conflicts with the system and ours is likely a wildly different - /// version, so they can't be substituted. - /// - /// As a result, we rename it here but we need to also use - /// `install_name_tool` on macOS to rename the commands listed inside of it to - /// ensure it's linked against correctly. - pub fn fixup_sanitizer_lib_name(&self, sanitizer_name: &str) { - if env::var("TARGET").unwrap() != "x86_64-apple-darwin" { - return - } - - let dir = self.out_dir.join("build/lib/darwin"); - let name = format!("clang_rt.{}_osx_dynamic", sanitizer_name); - let src = dir.join(&format!("lib{}.dylib", name)); - let new_name = format!("lib__rustc__{}.dylib", name); - let dst = dir.join(&new_name); - - println!("{} => {}", src.display(), dst.display()); - fs::rename(&src, &dst).unwrap(); - let status = Command::new("install_name_tool") - .arg("-id") - .arg(format!("@rpath/{}", new_name)) - .arg(&dst) - .status() - .expect("failed to execute `install_name_tool`"); - assert!(status.success()); - } -} - -impl Drop for NativeLibBoilerplate { - fn drop(&mut self) { - if !thread::panicking() { - t!(File::create(self.out_dir.join("rustbuild.timestamp"))); - } - } -} - -// Perform standard preparations for native libraries that are build only once for all stages. -// Emit rerun-if-changed and linking attributes for Cargo, check if any source files are -// updated, calculate paths used later in actual build with CMake/make or C/C++ compiler. -// If Err is returned, then everything is up-to-date and further build actions can be skipped. -// Timestamps are created automatically when the result of `native_lib_boilerplate` goes out -// of scope, so all the build actions should be completed until then. -pub fn native_lib_boilerplate( - src_dir: &Path, - out_name: &str, - link_name: &str, - search_subdir: &str, -) -> Result { - rerun_if_changed_anything_in_dir(src_dir); - - let out_dir = env::var_os("RUSTBUILD_NATIVE_DIR").unwrap_or_else(|| - env::var_os("OUT_DIR").unwrap()); - let out_dir = PathBuf::from(out_dir).join(out_name); - t!(fs::create_dir_all(&out_dir)); - if link_name.contains('=') { - println!("cargo:rustc-link-lib={}", link_name); - } else { - println!("cargo:rustc-link-lib=static={}", link_name); - } - println!( - "cargo:rustc-link-search=native={}", - out_dir.join(search_subdir).display() - ); - - let timestamp = out_dir.join("rustbuild.timestamp"); - if !up_to_date(Path::new("build.rs"), ×tamp) || !up_to_date(src_dir, ×tamp) { - Ok(NativeLibBoilerplate { - src_dir: src_dir.to_path_buf(), - out_dir, - }) - } else { - Err(()) - } -} - -pub fn sanitizer_lib_boilerplate(sanitizer_name: &str) - -> Result<(NativeLibBoilerplate, String), ()> -{ - let (link_name, search_path, apple) = match &*env::var("TARGET").unwrap() { - "x86_64-unknown-linux-gnu" => ( - format!("clang_rt.{}-x86_64", sanitizer_name), - "build/lib/linux", - false, - ), - "x86_64-apple-darwin" => ( - format!("clang_rt.{}_osx_dynamic", sanitizer_name), - "build/lib/darwin", - true, - ), - _ => return Err(()), - }; - let to_link = if apple { - format!("dylib=__rustc__{}", link_name) - } else { - format!("static={}", link_name) - }; - // This env var is provided by rustbuild to tell us where `compiler-rt` - // lives. - let dir = env::var_os("RUST_COMPILER_RT_ROOT").unwrap(); - let lib = native_lib_boilerplate( - dir.as_ref(), - sanitizer_name, - &to_link, - search_path, - )?; - Ok((lib, link_name)) -} - fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool { t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| { let meta = t!(e.metadata()); diff --git a/src/librustc_asan/Cargo.toml b/src/librustc_asan/Cargo.toml deleted file mode 100644 index df117de8720e0..0000000000000 --- a/src/librustc_asan/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -build = "build.rs" -name = "rustc_asan" -version = "0.0.0" -edition = "2018" - -[lib] -name = "rustc_asan" -path = "lib.rs" -test = false - -[build-dependencies] -build_helper = { path = "../build_helper" } -cmake = "0.1.38" - -[dependencies] -alloc = { path = "../liballoc" } -core = { path = "../libcore" } -compiler_builtins = "0.1.0" diff --git a/src/librustc_asan/build.rs b/src/librustc_asan/build.rs deleted file mode 100644 index 645707ccc0338..0000000000000 --- a/src/librustc_asan/build.rs +++ /dev/null @@ -1,30 +0,0 @@ -use std::env; -use build_helper::sanitizer_lib_boilerplate; - -use cmake::Config; - -fn main() { - println!("cargo:rerun-if-env-changed=RUSTC_BUILD_SANITIZERS"); - if env::var("RUSTC_BUILD_SANITIZERS") != Ok("1".to_string()) { - return; - } - if let Some(llvm_config) = env::var_os("LLVM_CONFIG") { - build_helper::restore_library_path(); - - let (native, target) = match sanitizer_lib_boilerplate("asan") { - Ok(native) => native, - _ => return, - }; - - Config::new(&native.src_dir) - .define("COMPILER_RT_BUILD_SANITIZERS", "ON") - .define("COMPILER_RT_BUILD_BUILTINS", "OFF") - .define("COMPILER_RT_BUILD_XRAY", "OFF") - .define("LLVM_CONFIG_PATH", llvm_config) - .out_dir(&native.out_dir) - .build_target(&target) - .build(); - native.fixup_sanitizer_lib_name("asan"); - } - println!("cargo:rerun-if-env-changed=LLVM_CONFIG"); -} diff --git a/src/librustc_asan/lib.rs b/src/librustc_asan/lib.rs deleted file mode 100644 index d6c8e54c18db7..0000000000000 --- a/src/librustc_asan/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![sanitizer_runtime] -#![feature(nll)] -#![feature(sanitizer_runtime)] -#![feature(staged_api)] -#![no_std] -#![unstable(feature = "sanitizer_runtime_lib", - reason = "internal implementation detail of sanitizers", - issue = "0")] diff --git a/src/librustc_lsan/Cargo.toml b/src/librustc_lsan/Cargo.toml deleted file mode 100644 index 9a24361f44e64..0000000000000 --- a/src/librustc_lsan/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -build = "build.rs" -name = "rustc_lsan" -version = "0.0.0" -edition = "2018" - -[lib] -name = "rustc_lsan" -path = "lib.rs" -test = false - -[build-dependencies] -build_helper = { path = "../build_helper" } -cmake = "0.1.38" - -[dependencies] -alloc = { path = "../liballoc" } -core = { path = "../libcore" } -compiler_builtins = "0.1.0" diff --git a/src/librustc_lsan/build.rs b/src/librustc_lsan/build.rs deleted file mode 100644 index 73720d8c2d64e..0000000000000 --- a/src/librustc_lsan/build.rs +++ /dev/null @@ -1,29 +0,0 @@ -use std::env; -use build_helper::sanitizer_lib_boilerplate; - -use cmake::Config; - -fn main() { - println!("cargo:rerun-if-env-changed=RUSTC_BUILD_SANITIZERS"); - if env::var("RUSTC_BUILD_SANITIZERS") != Ok("1".to_string()) { - return; - } - if let Some(llvm_config) = env::var_os("LLVM_CONFIG") { - build_helper::restore_library_path(); - - let (native, target) = match sanitizer_lib_boilerplate("lsan") { - Ok(native) => native, - _ => return, - }; - - Config::new(&native.src_dir) - .define("COMPILER_RT_BUILD_SANITIZERS", "ON") - .define("COMPILER_RT_BUILD_BUILTINS", "OFF") - .define("COMPILER_RT_BUILD_XRAY", "OFF") - .define("LLVM_CONFIG_PATH", llvm_config) - .out_dir(&native.out_dir) - .build_target(&target) - .build(); - } - println!("cargo:rerun-if-env-changed=LLVM_CONFIG"); -} diff --git a/src/librustc_lsan/lib.rs b/src/librustc_lsan/lib.rs deleted file mode 100644 index d6c8e54c18db7..0000000000000 --- a/src/librustc_lsan/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![sanitizer_runtime] -#![feature(nll)] -#![feature(sanitizer_runtime)] -#![feature(staged_api)] -#![no_std] -#![unstable(feature = "sanitizer_runtime_lib", - reason = "internal implementation detail of sanitizers", - issue = "0")] diff --git a/src/librustc_msan/Cargo.toml b/src/librustc_msan/Cargo.toml deleted file mode 100644 index bda4078572501..0000000000000 --- a/src/librustc_msan/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -build = "build.rs" -name = "rustc_msan" -version = "0.0.0" -edition = "2018" - -[lib] -name = "rustc_msan" -path = "lib.rs" -test = false - -[build-dependencies] -build_helper = { path = "../build_helper" } -cmake = "0.1.38" - -[dependencies] -alloc = { path = "../liballoc" } -core = { path = "../libcore" } -compiler_builtins = "0.1.0" diff --git a/src/librustc_msan/build.rs b/src/librustc_msan/build.rs deleted file mode 100644 index a81786ee36d04..0000000000000 --- a/src/librustc_msan/build.rs +++ /dev/null @@ -1,29 +0,0 @@ -use std::env; -use build_helper::sanitizer_lib_boilerplate; - -use cmake::Config; - -fn main() { - println!("cargo:rerun-if-env-changed=RUSTC_BUILD_SANITIZERS"); - if env::var("RUSTC_BUILD_SANITIZERS") != Ok("1".to_string()) { - return; - } - if let Some(llvm_config) = env::var_os("LLVM_CONFIG") { - build_helper::restore_library_path(); - - let (native, target) = match sanitizer_lib_boilerplate("msan") { - Ok(native) => native, - _ => return, - }; - - Config::new(&native.src_dir) - .define("COMPILER_RT_BUILD_SANITIZERS", "ON") - .define("COMPILER_RT_BUILD_BUILTINS", "OFF") - .define("COMPILER_RT_BUILD_XRAY", "OFF") - .define("LLVM_CONFIG_PATH", llvm_config) - .out_dir(&native.out_dir) - .build_target(&target) - .build(); - } - println!("cargo:rerun-if-env-changed=LLVM_CONFIG"); -} diff --git a/src/librustc_msan/lib.rs b/src/librustc_msan/lib.rs deleted file mode 100644 index d6c8e54c18db7..0000000000000 --- a/src/librustc_msan/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![sanitizer_runtime] -#![feature(nll)] -#![feature(sanitizer_runtime)] -#![feature(staged_api)] -#![no_std] -#![unstable(feature = "sanitizer_runtime_lib", - reason = "internal implementation detail of sanitizers", - issue = "0")] diff --git a/src/librustc_tsan/Cargo.toml b/src/librustc_tsan/Cargo.toml deleted file mode 100644 index 82045dd0cddc7..0000000000000 --- a/src/librustc_tsan/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -authors = ["The Rust Project Developers"] -build = "build.rs" -name = "rustc_tsan" -version = "0.0.0" -edition = "2018" - -[lib] -name = "rustc_tsan" -path = "lib.rs" -test = false - -[build-dependencies] -build_helper = { path = "../build_helper" } -cmake = "0.1.38" - -[dependencies] -alloc = { path = "../liballoc" } -core = { path = "../libcore" } -compiler_builtins = "0.1.0" diff --git a/src/librustc_tsan/build.rs b/src/librustc_tsan/build.rs deleted file mode 100644 index f9333e1502327..0000000000000 --- a/src/librustc_tsan/build.rs +++ /dev/null @@ -1,30 +0,0 @@ -use std::env; -use build_helper::sanitizer_lib_boilerplate; - -use cmake::Config; - -fn main() { - println!("cargo:rerun-if-env-changed=RUSTC_BUILD_SANITIZERS"); - if env::var("RUSTC_BUILD_SANITIZERS") != Ok("1".to_string()) { - return; - } - if let Some(llvm_config) = env::var_os("LLVM_CONFIG") { - build_helper::restore_library_path(); - - let (native, target) = match sanitizer_lib_boilerplate("tsan") { - Ok(native) => native, - _ => return, - }; - - Config::new(&native.src_dir) - .define("COMPILER_RT_BUILD_SANITIZERS", "ON") - .define("COMPILER_RT_BUILD_BUILTINS", "OFF") - .define("COMPILER_RT_BUILD_XRAY", "OFF") - .define("LLVM_CONFIG_PATH", llvm_config) - .out_dir(&native.out_dir) - .build_target(&target) - .build(); - native.fixup_sanitizer_lib_name("tsan"); - } - println!("cargo:rerun-if-env-changed=LLVM_CONFIG"); -} diff --git a/src/librustc_tsan/lib.rs b/src/librustc_tsan/lib.rs deleted file mode 100644 index d6c8e54c18db7..0000000000000 --- a/src/librustc_tsan/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -#![sanitizer_runtime] -#![feature(nll)] -#![feature(sanitizer_runtime)] -#![feature(staged_api)] -#![no_std] -#![unstable(feature = "sanitizer_runtime_lib", - reason = "internal implementation detail of sanitizers", - issue = "0")] diff --git a/src/libstd/Cargo.toml b/src/libstd/Cargo.toml index 5309af6f4c342..8806a93c65765 100644 --- a/src/libstd/Cargo.toml +++ b/src/libstd/Cargo.toml @@ -34,16 +34,6 @@ features = [ "rustc-dep-of-std" ] # enable build support for integrating into li [dev-dependencies] rand = "0.7" -[target.x86_64-apple-darwin.dependencies] -rustc_asan = { path = "../librustc_asan" } -rustc_tsan = { path = "../librustc_tsan" } - -[target.x86_64-unknown-linux-gnu.dependencies] -rustc_asan = { path = "../librustc_asan" } -rustc_lsan = { path = "../librustc_lsan" } -rustc_msan = { path = "../librustc_msan" } -rustc_tsan = { path = "../librustc_tsan" } - [target.'cfg(any(all(target_arch = "wasm32", not(target_os = "emscripten")), all(target_vendor = "fortanix", target_env = "sgx")))'.dependencies] dlmalloc = { version = "0.1", features = ['rustc-dep-of-std'] } From c1b5249dc72705c940db253dcf25258a45ce1dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= Date: Mon, 7 Oct 2019 00:00:00 +0000 Subject: [PATCH 03/43] Remove sanitizer_runtime attribute --- .../src/library-features/sanitizer-runtime-lib.md | 5 ----- src/librustc/query/mod.rs | 4 ---- src/librustc_codegen_ssa/base.rs | 4 ---- src/librustc_codegen_ssa/lib.rs | 1 - src/librustc_metadata/cstore_impl.rs | 1 - src/librustc_metadata/encoder.rs | 1 - src/librustc_metadata/schema.rs | 1 - src/libsyntax/feature_gate/active.rs | 3 --- src/libsyntax/feature_gate/builtin_attrs.rs | 5 ----- src/libsyntax/feature_gate/removed.rs | 2 ++ .../feature-gates/feature-gate-sanitizer-runtime.rs | 3 --- .../feature-gate-sanitizer-runtime.stderr | 11 ----------- 12 files changed, 2 insertions(+), 39 deletions(-) delete mode 100644 src/doc/unstable-book/src/library-features/sanitizer-runtime-lib.md delete mode 100644 src/test/ui/feature-gates/feature-gate-sanitizer-runtime.rs delete mode 100644 src/test/ui/feature-gates/feature-gate-sanitizer-runtime.stderr diff --git a/src/doc/unstable-book/src/library-features/sanitizer-runtime-lib.md b/src/doc/unstable-book/src/library-features/sanitizer-runtime-lib.md deleted file mode 100644 index 82ae67fc05ac3..0000000000000 --- a/src/doc/unstable-book/src/library-features/sanitizer-runtime-lib.md +++ /dev/null @@ -1,5 +0,0 @@ -# `sanitizer_runtime_lib` - -This feature is internal to the Rust compiler and is not intended for general use. - ------------------------- diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index c95652f274e36..b9fd4a7e702d9 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -639,10 +639,6 @@ rustc_queries! { fatal_cycle desc { "checking if the crate has_panic_handler" } } - query is_sanitizer_runtime(_: CrateNum) -> bool { - fatal_cycle - desc { "query a crate is `#![sanitizer_runtime]`" } - } query is_profiler_runtime(_: CrateNum) -> bool { fatal_cycle desc { "query a crate is `#![profiler_runtime]`" } diff --git a/src/librustc_codegen_ssa/base.rs b/src/librustc_codegen_ssa/base.rs index 1c441ca7cbf2e..315e25976be56 100644 --- a/src/librustc_codegen_ssa/base.rs +++ b/src/librustc_codegen_ssa/base.rs @@ -707,7 +707,6 @@ impl CrateInfo { panic_runtime: None, compiler_builtins: None, profiler_runtime: None, - sanitizer_runtime: None, is_no_builtins: Default::default(), native_libraries: Default::default(), used_libraries: tcx.native_libraries(LOCAL_CRATE), @@ -743,9 +742,6 @@ impl CrateInfo { if tcx.is_profiler_runtime(cnum) { info.profiler_runtime = Some(cnum); } - if tcx.is_sanitizer_runtime(cnum) { - info.sanitizer_runtime = Some(cnum); - } if tcx.is_no_builtins(cnum) { info.is_no_builtins.insert(cnum); } diff --git a/src/librustc_codegen_ssa/lib.rs b/src/librustc_codegen_ssa/lib.rs index 0221a04b04518..a53612474b4f2 100644 --- a/src/librustc_codegen_ssa/lib.rs +++ b/src/librustc_codegen_ssa/lib.rs @@ -130,7 +130,6 @@ pub struct CrateInfo { pub panic_runtime: Option, pub compiler_builtins: Option, pub profiler_runtime: Option, - pub sanitizer_runtime: Option, pub is_no_builtins: FxHashSet, pub native_libraries: FxHashMap>>, pub crate_name: FxHashMap, diff --git a/src/librustc_metadata/cstore_impl.rs b/src/librustc_metadata/cstore_impl.rs index fd011265da7cb..1a4e79b57d2cf 100644 --- a/src/librustc_metadata/cstore_impl.rs +++ b/src/librustc_metadata/cstore_impl.rs @@ -160,7 +160,6 @@ provide! { <'tcx> tcx, def_id, other, cdata, is_compiler_builtins => { cdata.root.compiler_builtins } has_global_allocator => { cdata.root.has_global_allocator } has_panic_handler => { cdata.root.has_panic_handler } - is_sanitizer_runtime => { cdata.root.sanitizer_runtime } is_profiler_runtime => { cdata.root.profiler_runtime } panic_strategy => { cdata.root.panic_strategy } extern_crate => { diff --git a/src/librustc_metadata/encoder.rs b/src/librustc_metadata/encoder.rs index 03a14f886455a..91c740a695948 100644 --- a/src/librustc_metadata/encoder.rs +++ b/src/librustc_metadata/encoder.rs @@ -519,7 +519,6 @@ impl<'tcx> EncodeContext<'tcx> { no_builtins: attr::contains_name(&attrs, sym::no_builtins), panic_runtime: attr::contains_name(&attrs, sym::panic_runtime), profiler_runtime: attr::contains_name(&attrs, sym::profiler_runtime), - sanitizer_runtime: attr::contains_name(&attrs, sym::sanitizer_runtime), symbol_mangling_version: tcx.sess.opts.debugging_opts.symbol_mangling_version, crate_deps, diff --git a/src/librustc_metadata/schema.rs b/src/librustc_metadata/schema.rs index d3539e7140121..c8b18ba5d76d7 100644 --- a/src/librustc_metadata/schema.rs +++ b/src/librustc_metadata/schema.rs @@ -197,7 +197,6 @@ crate struct CrateRoot<'tcx> { pub no_builtins: bool, pub panic_runtime: bool, pub profiler_runtime: bool, - pub sanitizer_runtime: bool, pub symbol_mangling_version: SymbolManglingVersion, } diff --git a/src/libsyntax/feature_gate/active.rs b/src/libsyntax/feature_gate/active.rs index 94f0995566f52..fe5584465e412 100644 --- a/src/libsyntax/feature_gate/active.rs +++ b/src/libsyntax/feature_gate/active.rs @@ -198,9 +198,6 @@ declare_features! ( /// Allows using the `unadjusted` ABI; perma-unstable. (active, abi_unadjusted, "1.16.0", None, None), - /// Allows identifying crates that contain sanitizer runtimes. - (active, sanitizer_runtime, "1.17.0", None, None), - /// Used to identify crates that contain the profiler runtime. (active, profiler_runtime, "1.18.0", None, None), diff --git a/src/libsyntax/feature_gate/builtin_attrs.rs b/src/libsyntax/feature_gate/builtin_attrs.rs index ab1620b952491..922c40e4db0ae 100644 --- a/src/libsyntax/feature_gate/builtin_attrs.rs +++ b/src/libsyntax/feature_gate/builtin_attrs.rs @@ -396,11 +396,6 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ "the `#[compiler_builtins]` attribute is used to identify the `compiler_builtins` crate \ which contains compiler-rt intrinsics and will never be stable", ), - gated!( - sanitizer_runtime, Whitelisted, template!(Word), - "the `#[sanitizer_runtime]` attribute is used to identify crates that contain the runtime \ - of a sanitizer and will never be stable", - ), gated!( profiler_runtime, Whitelisted, template!(Word), "the `#[profiler_runtime]` attribute is used to identify the `profiler_builtins` crate \ diff --git a/src/libsyntax/feature_gate/removed.rs b/src/libsyntax/feature_gate/removed.rs index 2c29e1ebf1493..445715d3ecaaf 100644 --- a/src/libsyntax/feature_gate/removed.rs +++ b/src/libsyntax/feature_gate/removed.rs @@ -70,6 +70,8 @@ declare_features! ( (removed, pushpop_unsafe, "1.2.0", None, None, None), (removed, needs_allocator, "1.4.0", Some(27389), None, Some("subsumed by `#![feature(allocator_internals)]`")), + /// Allows identifying crates that contain sanitizer runtimes. + (removed, sanitizer_runtime, "1.17.0", None, None, None), (removed, proc_macro_mod, "1.27.0", Some(54727), None, Some("subsumed by `#![feature(proc_macro_hygiene)]`")), (removed, proc_macro_expr, "1.27.0", Some(54727), None, diff --git a/src/test/ui/feature-gates/feature-gate-sanitizer-runtime.rs b/src/test/ui/feature-gates/feature-gate-sanitizer-runtime.rs deleted file mode 100644 index 3b972c117a6ff..0000000000000 --- a/src/test/ui/feature-gates/feature-gate-sanitizer-runtime.rs +++ /dev/null @@ -1,3 +0,0 @@ -#![sanitizer_runtime] //~ ERROR the `#[sanitizer_runtime]` attribute is - -fn main() {} diff --git a/src/test/ui/feature-gates/feature-gate-sanitizer-runtime.stderr b/src/test/ui/feature-gates/feature-gate-sanitizer-runtime.stderr deleted file mode 100644 index b13ec215f8c08..0000000000000 --- a/src/test/ui/feature-gates/feature-gate-sanitizer-runtime.stderr +++ /dev/null @@ -1,11 +0,0 @@ -error[E0658]: the `#[sanitizer_runtime]` attribute is used to identify crates that contain the runtime of a sanitizer and will never be stable - --> $DIR/feature-gate-sanitizer-runtime.rs:1:1 - | -LL | #![sanitizer_runtime] - | ^^^^^^^^^^^^^^^^^^^^^ - | - = help: add `#![feature(sanitizer_runtime)]` to the crate attributes to enable - -error: aborting due to previous error - -For more information about this error, try `rustc --explain E0658`. From 87c3eedffba64830b67e54e75dd479f9fd83cc7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= Date: Sun, 6 Oct 2019 00:00:00 +0000 Subject: [PATCH 04/43] Link sanitizer runtimes instead of injecting crate dependencies --- src/bootstrap/check.rs | 2 +- src/bootstrap/compile.rs | 122 ++++++++++++------ src/bootstrap/doc.rs | 2 +- src/bootstrap/native.rs | 7 + src/bootstrap/test.rs | 2 +- src/librustc/session/mod.rs | 11 ++ src/librustc_codegen_ssa/back/link.rs | 88 ++++++------- src/librustc_metadata/creader.rs | 101 +-------------- .../sanitizer-address/Makefile | 2 +- .../sanitizer-invalid-target/Makefile | 2 +- .../run-make-fulldeps/sanitizer-leak/Makefile | 2 +- .../sanitizer-memory/Makefile | 2 +- 12 files changed, 155 insertions(+), 188 deletions(-) diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index cadb9a7e441f2..c79c046a4f6e6 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -48,7 +48,7 @@ impl Step for Std { let compiler = builder.compiler(0, builder.config.build); let mut cargo = builder.cargo(compiler, Mode::Std, target, cargo_subcommand(builder.kind)); - std_cargo(builder, &compiler, target, &mut cargo); + std_cargo(builder, target, &mut cargo); builder.info(&format!("Checking std artifacts ({} -> {})", &compiler.host, target)); run_cargo(builder, diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 6ea32edfb208b..c50818cba6db7 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -94,7 +94,7 @@ impl Step for Std { copy_third_party_objects(builder, &compiler, target); let mut cargo = builder.cargo(compiler, Mode::Std, target, "build"); - std_cargo(builder, &compiler, target, &mut cargo); + std_cargo(builder, target, &mut cargo); builder.info(&format!("Building stage{} std artifacts ({} -> {})", compiler.stage, &compiler.host, target)); @@ -155,7 +155,6 @@ fn copy_third_party_objects(builder: &Builder<'_>, compiler: &Compiler, target: /// Configure cargo to compile the standard library, adding appropriate env vars /// and such. pub fn std_cargo(builder: &Builder<'_>, - compiler: &Compiler, target: Interned, cargo: &mut Cargo) { if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { @@ -200,22 +199,6 @@ pub fn std_cargo(builder: &Builder<'_>, let mut features = builder.std_features(); features.push_str(&compiler_builtins_c_feature); - if compiler.stage != 0 && builder.config.sanitizers { - // This variable is used by the sanitizer runtime crates, e.g. - // rustc_lsan, to build the sanitizer runtime from C code - // When this variable is missing, those crates won't compile the C code, - // so we don't set this variable during stage0 where llvm-config is - // missing - // We also only build the runtimes when --enable-sanitizers (or its - // config.toml equivalent) is used - let llvm_config = builder.ensure(native::Llvm { - target: builder.config.build, - emscripten: false, - }); - cargo.env("LLVM_CONFIG", llvm_config); - cargo.env("RUSTC_BUILD_SANITIZERS", "1"); - } - cargo.arg("--features").arg(features) .arg("--manifest-path") .arg(builder.src.join("src/libtest/Cargo.toml")); @@ -274,30 +257,95 @@ impl Step for StdLink { let hostdir = builder.sysroot_libdir(target_compiler, compiler.host); add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); - if builder.config.sanitizers && compiler.stage != 0 && target == "x86_64-apple-darwin" { - // The sanitizers are only built in stage1 or above, so the dylibs will - // be missing in stage0 and causes panic. See the `std()` function above - // for reason why the sanitizers are not built in stage0. - copy_apple_sanitizer_dylibs(builder, &builder.native_dir(target), "osx", &libdir); + if builder.config.sanitizers && target_compiler.stage != 0 { + // The sanitizers are only copied in stage1 or above, + // to avoid creating dependency on LLVM. + copy_sanitizers(builder, &target_compiler, target); } } } -fn copy_apple_sanitizer_dylibs( - builder: &Builder<'_>, - native_dir: &Path, - platform: &str, - into: &Path, -) { - for &sanitizer in &["asan", "tsan"] { - let filename = format!("lib__rustc__clang_rt.{}_{}_dynamic.dylib", sanitizer, platform); - let mut src_path = native_dir.join(sanitizer); - src_path.push("build"); - src_path.push("lib"); - src_path.push("darwin"); - src_path.push(&filename); - builder.copy(&src_path, &into.join(filename)); +/// Copies sanitizer runtime libraries into target libdir. +fn copy_sanitizers(builder: &Builder<'_>, compiler: &Compiler, target: Interned) { + let llvm_config = builder.ensure(native::Llvm { + target, + emscripten: false, + }); + + let llvm_version = output(Command::new(&llvm_config).arg("--version")); + let llvm_version = llvm_version.trim(); + + // The compiler-rt uses CLANG_VERSION as a part of COMPILER_RT_INSTALL_PATH. + // The CLANG_VERSION is based on LLVM_VERSION but it does not not include + // LLVM_VERSION_SUFFIX. On the other hand value returned from llvm-config + // --version does include it, so lets strip it to obtain CLANG_VERSION. + let mut non_digits = 0; + let mut third_non_digit = llvm_version.len(); + for (i, c) in llvm_version.char_indices() { + if !c.is_digit(10) { + non_digits += 1; + if non_digits == 3 { + third_non_digit = i; + break; + } + } + } + let llvm_version = &llvm_version[..third_non_digit]; + + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + let llvm_libdir = PathBuf::from(llvm_libdir.trim()); + + let clang_resourcedir = llvm_libdir.join("clang").join(&llvm_version); + let sanitizers = supported_sanitizers(&clang_resourcedir, target); + let libdir = builder.sysroot_libdir(*compiler, target); + + for (src, name) in &sanitizers { + let dst = libdir.join(name); + if !src.exists() { + println!("Ignoring missing runtime: {}", src.display()); + continue; + } + builder.copy(&src, &dst); + + if target == "x86_64-apple-darwin" { + // Update the library install name reflect the fact it has been renamed. + let status = Command::new("install_name_tool") + .arg("-id") + .arg(format!("@rpath/{}", name)) + .arg(&dst) + .status() + .expect("failed to execute `install_name_tool`"); + assert!(status.success()); + } + } +} + +/// Returns a list of paths to sanitizer libraries supported on given target, +/// and corresponding names we plan to give them when placed in target libdir. +fn supported_sanitizers(resourcedir: &Path, target: Interned) -> Vec<(PathBuf, String)> { + let sanitizers = &["asan", "lsan", "msan", "tsan"]; + let mut result = Vec::new(); + match &*target { + "x86_64-apple-darwin" => { + let srcdir = resourcedir.join("lib/darwin"); + for s in sanitizers { + let src = format!("libclang_rt.{}_osx_dynamic.dylib", s); + let dst = format!("librustc_rt.{}.dylib", s); + result.push((srcdir.join(src), dst)); + } + + } + "x86_64-unknown-linux-gnu" => { + let srcdir = resourcedir.join("lib/linux"); + for s in sanitizers { + let src = format!("libclang_rt.{}-x86_64.a", s); + let dst = format!("librustc_rt.{}.a", s); + result.push((srcdir.join(src), dst)); + } + } + _ => {} } + result } #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 4ee8cd2485c02..5e68b9d27c2d8 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -458,7 +458,7 @@ impl Step for Std { let run_cargo_rustdoc_for = |package: &str| { let mut cargo = builder.cargo(compiler, Mode::Std, target, "rustdoc"); - compile::std_cargo(builder, &compiler, target, &mut cargo); + compile::std_cargo(builder, target, &mut cargo); // Keep a whitelist so we do not build internal stdlib crates, these will be // build by the rustc step later if enabled. diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 7bf9ea2688f4c..839ee8a9742f9 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -217,6 +217,13 @@ impl Step for Llvm { enabled_llvm_projects.push("compiler-rt"); } + if builder.config.sanitizers { + enabled_llvm_projects.push("compiler-rt"); + cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON"); + // Avoids building instrumented version of libcxx. + cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); + } + if want_lldb { enabled_llvm_projects.push("clang"); enabled_llvm_projects.push("lldb"); diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index b7ce9c7b39709..a39f2ce5407c8 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1759,7 +1759,7 @@ impl Step for Crate { let mut cargo = builder.cargo(compiler, mode, target, test_kind.subcommand()); match mode { Mode::Std => { - compile::std_cargo(builder, &compiler, target, &mut cargo); + compile::std_cargo(builder, target, &mut cargo); } Mode::Rustc => { builder.ensure(compile::Rustc { compiler, target }); diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index fa2902e4f0ede..55452d9982d79 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -1296,6 +1296,17 @@ fn validate_commandline_args_with_session_available(sess: &Session) { with `-Cpanic=unwind` on Windows when targeting MSVC. \ See https://github.com/rust-lang/rust/issues/61002 for details."); } + + // Sanitizers can only be used on some tested platforms. + if let Some(ref sanitizer) = sess.opts.debugging_opts.sanitizer { + const SUPPORTED_TARGETS: &[&str] = &["x86_64-unknown-linux-gnu", "x86_64-apple-darwin"]; + if !SUPPORTED_TARGETS.contains(&&*sess.opts.target_triple.triple()) { + sess.err(&format!("{:?}Sanitizer only works with the `{}` target", + sanitizer, + SUPPORTED_TARGETS.join("` or `") + )); + } + } } /// Hash value constructed out of all the `-C metadata` arguments passed to the diff --git a/src/librustc_codegen_ssa/back/link.rs b/src/librustc_codegen_ssa/back/link.rs index 1c5d3b1a890ee..774beb631f872 100644 --- a/src/librustc_codegen_ssa/back/link.rs +++ b/src/librustc_codegen_ssa/back/link.rs @@ -522,6 +522,7 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>(sess: &'a Session, { let mut linker = codegen_results.linker_info.to_linker(cmd, &sess, flavor, target_cpu); + link_sanitizer_runtime(sess, crate_type, &mut *linker); link_args::(&mut *linker, flavor, sess, crate_type, tmpdir, out_filename, codegen_results); cmd = linker.finalize(); @@ -725,6 +726,49 @@ fn link_natively<'a, B: ArchiveBuilder<'a>>(sess: &'a Session, } } +fn link_sanitizer_runtime(sess: &Session, + crate_type: config::CrateType, + linker: &mut dyn Linker) { + let sanitizer = match &sess.opts.debugging_opts.sanitizer { + Some(s) => s, + None => return, + }; + + if crate_type != config::CrateType::Executable { + return; + } + + let name = match sanitizer { + Sanitizer::Address => "asan", + Sanitizer::Leak => "lsan", + Sanitizer::Memory => "msan", + Sanitizer::Thread => "tsan", + }; + + let default_sysroot = filesearch::get_or_default_sysroot(); + let default_tlib = filesearch::make_target_lib_path( + &default_sysroot, sess.opts.target_triple.triple()); + + match sess.opts.target_triple.triple() { + "x86_64-apple-darwin" => { + // On Apple platforms, the sanitizer is always built as a dylib, and + // LLVM will link to `@rpath/*.dylib`, so we need to specify an + // rpath to the library as well (the rpath should be absolute, see + // PR #41352 for details). + let filename = format!("librustc_rt.{}.dylib", name); + let rpath = default_tlib.to_str().expect("non-utf8 component in path"); + linker.args(&["-Wl,-rpath".into(), "-Xlinker".into(), rpath.into()]); + linker.link_dylib(Symbol::intern(&filename)); + } + "x86_64-unknown-linux-gnu" => { + let filename = format!("librustc_rt.{}.a", name); + let path = default_tlib.join(&filename); + linker.link_whole_rlib(&path); + } + _ => {} + } +} + /// Returns a boolean indicating whether the specified crate should be ignored /// during LTO. /// @@ -1391,11 +1435,6 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( _ if codegen_results.crate_info.profiler_runtime == Some(cnum) => { add_static_crate::(cmd, sess, codegen_results, tmpdir, crate_type, cnum); } - _ if codegen_results.crate_info.sanitizer_runtime == Some(cnum) && - crate_type == config::CrateType::Executable => { - // Link the sanitizer runtimes only if we are actually producing an executable - link_sanitizer_runtime::(cmd, sess, codegen_results, tmpdir, cnum); - } // compiler-builtins are always placed last to ensure that they're // linked correctly. _ if codegen_results.crate_info.compiler_builtins == Some(cnum) => { @@ -1435,45 +1474,6 @@ fn add_upstream_rust_crates<'a, B: ArchiveBuilder<'a>>( } } - // We must link the sanitizer runtime using -Wl,--whole-archive but since - // it's packed in a .rlib, it contains stuff that are not objects that will - // make the linker error. So we must remove those bits from the .rlib before - // linking it. - fn link_sanitizer_runtime<'a, B: ArchiveBuilder<'a>>(cmd: &mut dyn Linker, - sess: &'a Session, - codegen_results: &CodegenResults, - tmpdir: &Path, - cnum: CrateNum) { - let src = &codegen_results.crate_info.used_crate_source[&cnum]; - let cratepath = &src.rlib.as_ref().unwrap().0; - - if sess.target.target.options.is_like_osx { - // On Apple platforms, the sanitizer is always built as a dylib, and - // LLVM will link to `@rpath/*.dylib`, so we need to specify an - // rpath to the library as well (the rpath should be absolute, see - // PR #41352 for details). - // - // FIXME: Remove this logic into librustc_*san once Cargo supports it - let rpath = cratepath.parent().unwrap(); - let rpath = rpath.to_str().expect("non-utf8 component in path"); - cmd.args(&["-Wl,-rpath".into(), "-Xlinker".into(), rpath.into()]); - } - - let dst = tmpdir.join(cratepath.file_name().unwrap()); - let mut archive = ::new(sess, &dst, Some(cratepath)); - archive.update_symbols(); - - for f in archive.src_files() { - if f.ends_with(RLIB_BYTECODE_EXTENSION) || f == METADATA_FILENAME { - archive.remove_file(&f); - } - } - - archive.build(); - - cmd.link_whole_rlib(&dst); - } - // Adds the static "rlib" versions of all crates to the command line. // There's a bit of magic which happens here specifically related to LTO and // dynamic libraries. Specifically: diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index 9850121d2ce5d..07c5ee5ef161d 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -11,7 +11,7 @@ use rustc::dep_graph::DepNodeIndex; use rustc::middle::cstore::DepKind; use rustc::mir::interpret::AllocDecodingState; use rustc::session::{Session, CrateDisambiguator}; -use rustc::session::config::{Sanitizer, self}; +use rustc::session::config; use rustc_target::spec::{PanicStrategy, TargetTriple}; use rustc::session::search_paths::PathKind; use rustc::middle::cstore::{CrateSource, ExternCrate, ExternCrateSource}; @@ -706,104 +706,6 @@ impl<'a> CrateLoader<'a> { &|data| data.root.needs_panic_runtime); } - fn inject_sanitizer_runtime(&self) { - if let Some(ref sanitizer) = self.sess.opts.debugging_opts.sanitizer { - // Sanitizers can only be used on some tested platforms with - // executables linked to `std` - const ASAN_SUPPORTED_TARGETS: &[&str] = &["x86_64-unknown-linux-gnu", - "x86_64-apple-darwin"]; - const TSAN_SUPPORTED_TARGETS: &[&str] = &["x86_64-unknown-linux-gnu", - "x86_64-apple-darwin"]; - const LSAN_SUPPORTED_TARGETS: &[&str] = &["x86_64-unknown-linux-gnu"]; - const MSAN_SUPPORTED_TARGETS: &[&str] = &["x86_64-unknown-linux-gnu"]; - - let supported_targets = match *sanitizer { - Sanitizer::Address => ASAN_SUPPORTED_TARGETS, - Sanitizer::Thread => TSAN_SUPPORTED_TARGETS, - Sanitizer::Leak => LSAN_SUPPORTED_TARGETS, - Sanitizer::Memory => MSAN_SUPPORTED_TARGETS, - }; - if !supported_targets.contains(&&*self.sess.opts.target_triple.triple()) { - self.sess.err(&format!("{:?}Sanitizer only works with the `{}` target", - sanitizer, - supported_targets.join("` or `") - )); - return - } - - // firstyear 2017 - during testing I was unable to access an OSX machine - // to make this work on different crate types. As a result, today I have - // only been able to test and support linux as a target. - if self.sess.opts.target_triple.triple() == "x86_64-unknown-linux-gnu" { - if !self.sess.crate_types.borrow().iter().all(|ct| { - match *ct { - // Link the runtime - config::CrateType::Staticlib | - config::CrateType::Executable => true, - // This crate will be compiled with the required - // instrumentation pass - config::CrateType::Rlib | - config::CrateType::Dylib | - config::CrateType::Cdylib => - false, - _ => { - self.sess.err(&format!("Only executables, staticlibs, \ - cdylibs, dylibs and rlibs can be compiled with \ - `-Z sanitizer`")); - false - } - } - }) { - return - } - } else { - if !self.sess.crate_types.borrow().iter().all(|ct| { - match *ct { - // Link the runtime - config::CrateType::Executable => true, - // This crate will be compiled with the required - // instrumentation pass - config::CrateType::Rlib => false, - _ => { - self.sess.err(&format!("Only executables and rlibs can be \ - compiled with `-Z sanitizer`")); - false - } - } - }) { - return - } - } - - let mut uses_std = false; - self.cstore.iter_crate_data(|_, data| { - if data.root.name == sym::std { - uses_std = true; - } - }); - - if uses_std { - let name = Symbol::intern(match sanitizer { - Sanitizer::Address => "rustc_asan", - Sanitizer::Leak => "rustc_lsan", - Sanitizer::Memory => "rustc_msan", - Sanitizer::Thread => "rustc_tsan", - }); - info!("loading sanitizer: {}", name); - - let data = self.resolve_crate(name, DUMMY_SP, DepKind::Explicit, None).1; - - // Sanity check the loaded crate to ensure it is indeed a sanitizer runtime - if !data.root.sanitizer_runtime { - self.sess.err(&format!("the crate `{}` is not a sanitizer runtime", - name)); - } - } else { - self.sess.err("Must link std to be compiled with `-Z sanitizer`"); - } - } - } - fn inject_profiler_runtime(&self) { if self.sess.opts.debugging_opts.profile || self.sess.opts.cg.profile_generate.enabled() @@ -961,7 +863,6 @@ impl<'a> CrateLoader<'a> { impl<'a> CrateLoader<'a> { pub fn postprocess(&self, krate: &ast::Crate) { - self.inject_sanitizer_runtime(); self.inject_profiler_runtime(); self.inject_allocator_crate(krate); self.inject_panic_runtime(krate); diff --git a/src/test/run-make-fulldeps/sanitizer-address/Makefile b/src/test/run-make-fulldeps/sanitizer-address/Makefile index 3a377c32993d5..6715ef16eb620 100644 --- a/src/test/run-make-fulldeps/sanitizer-address/Makefile +++ b/src/test/run-make-fulldeps/sanitizer-address/Makefile @@ -23,7 +23,7 @@ endif endif all: - $(RUSTC) -g -Z sanitizer=address -Z print-link-args $(EXTRA_RUSTFLAG) overflow.rs | $(CGREP) librustc_asan + $(RUSTC) -g -Z sanitizer=address -Z print-link-args $(EXTRA_RUSTFLAG) overflow.rs | $(CGREP) librustc_rt.asan # Verify that stack buffer overflow is detected: $(TMPDIR)/overflow 2>&1 | $(CGREP) stack-buffer-overflow # Verify that variable name is included in address sanitizer report: diff --git a/src/test/run-make-fulldeps/sanitizer-invalid-target/Makefile b/src/test/run-make-fulldeps/sanitizer-invalid-target/Makefile index df8afee15ce07..2a23f0fe3d4ef 100644 --- a/src/test/run-make-fulldeps/sanitizer-invalid-target/Makefile +++ b/src/test/run-make-fulldeps/sanitizer-invalid-target/Makefile @@ -2,4 +2,4 @@ all: $(RUSTC) -Z sanitizer=leak --target i686-unknown-linux-gnu hello.rs 2>&1 | \ - $(CGREP) 'LeakSanitizer only works with the `x86_64-unknown-linux-gnu` target' + $(CGREP) 'LeakSanitizer only works with the `x86_64-unknown-linux-gnu` or `x86_64-apple-darwin` target' diff --git a/src/test/run-make-fulldeps/sanitizer-leak/Makefile b/src/test/run-make-fulldeps/sanitizer-leak/Makefile index 101e8272ab91e..070a4948e9c1c 100644 --- a/src/test/run-make-fulldeps/sanitizer-leak/Makefile +++ b/src/test/run-make-fulldeps/sanitizer-leak/Makefile @@ -7,5 +7,5 @@ # FIXME(#46126) ThinLTO for libstd broke this test all: - $(RUSTC) -C opt-level=1 -g -Z sanitizer=leak -Z print-link-args leak.rs | $(CGREP) librustc_lsan + $(RUSTC) -C opt-level=1 -g -Z sanitizer=leak -Z print-link-args leak.rs | $(CGREP) librustc_rt.lsan $(TMPDIR)/leak 2>&1 | $(CGREP) 'detected memory leaks' diff --git a/src/test/run-make-fulldeps/sanitizer-memory/Makefile b/src/test/run-make-fulldeps/sanitizer-memory/Makefile index b3376f8a72358..cf41390d9f7f1 100644 --- a/src/test/run-make-fulldeps/sanitizer-memory/Makefile +++ b/src/test/run-make-fulldeps/sanitizer-memory/Makefile @@ -5,5 +5,5 @@ # only-x86_64 all: - $(RUSTC) -g -Z sanitizer=memory -Z print-link-args uninit.rs | $(CGREP) librustc_msan + $(RUSTC) -g -Z sanitizer=memory -Z print-link-args uninit.rs | $(CGREP) librustc_rt.msan $(TMPDIR)/uninit 2>&1 | $(CGREP) use-of-uninitialized-value From ac3cf6b13c34fbf3ac61d81c3d0649c2615f5558 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= Date: Mon, 7 Oct 2019 00:00:00 +0000 Subject: [PATCH 05/43] Compile main executable with rustc in staticlib test The static libraries produced by rustc no longer contain sanitizer runtime libraries, so the final linking step needs to be performed with rustc and sanitizer flag. --- .../sanitizer-staticlib-link/Makefile | 4 ++-- .../sanitizer-staticlib-link/program.c | 7 ------- .../sanitizer-staticlib-link/program.rs | 10 ++++++++++ 3 files changed, 12 insertions(+), 9 deletions(-) delete mode 100644 src/test/run-make-fulldeps/sanitizer-staticlib-link/program.c create mode 100644 src/test/run-make-fulldeps/sanitizer-staticlib-link/program.rs diff --git a/src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile b/src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile index 200dc1be4dee6..5e62fcc7a5688 100644 --- a/src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile +++ b/src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile @@ -10,6 +10,6 @@ all: $(RUSTC) -g -Z sanitizer=address --crate-type staticlib --target $(TARGET) library.rs - $(CC) program.c $(call STATICLIB,library) $(call OUT_EXE,program) $(EXTRACFLAGS) $(EXTRACXXFLAGS) - LD_LIBRARY_PATH=$(TMPDIR) $(TMPDIR)/program 2>&1 | $(CGREP) stack-buffer-overflow + $(RUSTC) -g -Z sanitizer=address --crate-type bin --target $(TARGET) -L . program.rs + $(TMPDIR)/program 2>&1 | $(CGREP) stack-buffer-overflow diff --git a/src/test/run-make-fulldeps/sanitizer-staticlib-link/program.c b/src/test/run-make-fulldeps/sanitizer-staticlib-link/program.c deleted file mode 100644 index a6d3bcdc5ac9d..0000000000000 --- a/src/test/run-make-fulldeps/sanitizer-staticlib-link/program.c +++ /dev/null @@ -1,7 +0,0 @@ -// ignore-license -void overflow(); - -int main() { - overflow(); - return 0; -} diff --git a/src/test/run-make-fulldeps/sanitizer-staticlib-link/program.rs b/src/test/run-make-fulldeps/sanitizer-staticlib-link/program.rs new file mode 100644 index 0000000000000..21e1ade2cd52c --- /dev/null +++ b/src/test/run-make-fulldeps/sanitizer-staticlib-link/program.rs @@ -0,0 +1,10 @@ +#[link(name = "library")] +extern { + fn overflow(); +} + +fn main() { + unsafe { + overflow(); + } +} From 7864cb4da78a18b1c5754eac105a4dbabfe030e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= Date: Tue, 8 Oct 2019 00:00:00 +0000 Subject: [PATCH 06/43] Enable sanitizer-leak test case --- src/test/run-make-fulldeps/sanitizer-leak/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/run-make-fulldeps/sanitizer-leak/Makefile b/src/test/run-make-fulldeps/sanitizer-leak/Makefile index 070a4948e9c1c..2f485ae4fa4c3 100644 --- a/src/test/run-make-fulldeps/sanitizer-leak/Makefile +++ b/src/test/run-make-fulldeps/sanitizer-leak/Makefile @@ -3,8 +3,6 @@ # needs-sanitizer-support # only-linux # only-x86_64 -# ignore-test -# FIXME(#46126) ThinLTO for libstd broke this test all: $(RUSTC) -C opt-level=1 -g -Z sanitizer=leak -Z print-link-args leak.rs | $(CGREP) librustc_rt.lsan From f721547bf852c5d3ce220e13c62871515d1d4e95 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 4 Oct 2019 18:24:10 +0200 Subject: [PATCH 07/43] ci: move install-awscli.sh into scripts/ --- src/ci/azure-pipelines/steps/run.yml | 2 +- src/ci/{ => scripts}/install-awscli.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/ci/{ => scripts}/install-awscli.sh (100%) diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 15a2499e4609e..be4e7e02ab08f 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -127,7 +127,7 @@ steps: # Ensure the `aws` CLI is installed so we can deploy later on, cache docker # images, etc. -- bash: src/ci/install-awscli.sh +- bash: src/ci/scripts/install-awscli.sh env: AGENT_OS: $(Agent.OS) condition: and(succeeded(), not(variables.SKIP_JOB)) diff --git a/src/ci/install-awscli.sh b/src/ci/scripts/install-awscli.sh similarity index 100% rename from src/ci/install-awscli.sh rename to src/ci/scripts/install-awscli.sh From e9974f9b362da125545bb97e4707db1fda4e5e13 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 4 Oct 2019 18:31:31 +0200 Subject: [PATCH 08/43] ci: extract dumping the environment into a script --- src/ci/azure-pipelines/steps/run.yml | 12 ++---------- src/ci/scripts/dump-environment.sh | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) create mode 100755 src/ci/scripts/dump-environment.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index be4e7e02ab08f..86399e5828f46 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -48,16 +48,8 @@ steps: - bash: python src/ci/cpu-usage-over-time.py &> cpu-usage.csv & displayName: "Collect CPU-usage statistics in the background" -- bash: printenv | sort - displayName: Show environment variables - -- bash: | - set -e - df -h - du . | sort -nr | head -n100 - displayName: Show disk usage - # FIXME: this hasn't been tested, but maybe it works on Windows? Should test! - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) +- bash: src/ci/scripts/dump-environment.sh + displayName: Show the current environment - template: install-sccache.yml - template: install-clang.yml diff --git a/src/ci/scripts/dump-environment.sh b/src/ci/scripts/dump-environment.sh new file mode 100755 index 0000000000000..c6774b52ab92d --- /dev/null +++ b/src/ci/scripts/dump-environment.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# This script dumps information about the build environment to stdout. + +set -euo pipefail +IFS=$'\n\t' + +echo "environment variables:" +printenv | sort +echo + +echo "disk usage:" +df -h +echo + +echo "biggest files in the working dir:" +set +o pipefail +du . | sort -nr | head -n100 +set -o pipefail +echo From 6b89d5904ef09684a5013303dd1ecc858d8478b7 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 4 Oct 2019 18:42:53 +0200 Subject: [PATCH 09/43] ci: extract installing sccache into a script --- .../azure-pipelines/steps/install-sccache.yml | 21 ------------------- src/ci/azure-pipelines/steps/run.yml | 7 ++++++- src/ci/scripts/install-sccache.sh | 20 ++++++++++++++++++ src/ci/shared.sh | 20 ++++++++++++++++++ 4 files changed, 46 insertions(+), 22 deletions(-) delete mode 100644 src/ci/azure-pipelines/steps/install-sccache.yml create mode 100755 src/ci/scripts/install-sccache.sh diff --git a/src/ci/azure-pipelines/steps/install-sccache.yml b/src/ci/azure-pipelines/steps/install-sccache.yml deleted file mode 100644 index d4679c1c6733e..0000000000000 --- a/src/ci/azure-pipelines/steps/install-sccache.yml +++ /dev/null @@ -1,21 +0,0 @@ -steps: - -- bash: | - set -e - curl -fo /usr/local/bin/sccache https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-apple-darwin - chmod +x /usr/local/bin/sccache - displayName: Install sccache (OSX) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - -- script: | - md sccache - powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf sccache\sccache.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-26-sccache-x86_64-pc-windows-msvc" - echo ##vso[task.prependpath]%CD%\sccache - displayName: Install sccache (Windows) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -# Note that we don't install sccache on Linux since it's installed elsewhere -# through all the containers. -# -# FIXME: we should probably install sccache outside the containers and then -# mount it inside the containers so we can centralize all installation here. diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 86399e5828f46..66405f2d2bfde 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -51,7 +51,12 @@ steps: - bash: src/ci/scripts/dump-environment.sh displayName: Show the current environment -- template: install-sccache.yml +- bash: src/ci/scripts/install-sccache.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install sccache + condition: and(succeeded(), not(variables.SKIP_JOB)) + - template: install-clang.yml # Switch to XCode 9.3 on OSX since it seems to be the last version that supports diff --git a/src/ci/scripts/install-sccache.sh b/src/ci/scripts/install-sccache.sh new file mode 100755 index 0000000000000..d3c298992254e --- /dev/null +++ b/src/ci/scripts/install-sccache.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# This script installs sccache on the local machine. Note that we don't install +# sccache on Linux since it's installed elsewhere through all the containers. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isMacOS; then + curl -fo /usr/local/bin/sccache "${MIRRORS_BASE}/2018-04-02-sccache-x86_64-apple-darwin" + chmod +x /usr/local/bin/sccache +elif isWindows; then + mkdir -p sccache + curl -fo sccache/sccache.exe "${MIRRORS_BASE}/2018-04-26-sccache-x86_64-pc-windows-msvc" + ciCommandAddPath "$(pwd)/sccache" +fi + +# FIXME: we should probably install sccache outside the containers and then +# mount it inside the containers so we can centralize all installation here. diff --git a/src/ci/shared.sh b/src/ci/shared.sh index b093a07ec5c5a..49fe3841ceb3b 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -4,6 +4,8 @@ # `source shared.sh`, hence the invalid shebang and not being # marked as an executable file in git. +export MIRRORS_BASE="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" + # See http://unix.stackexchange.com/questions/82598 # Duplicated in docker/dist-various-2/shared.sh function retry { @@ -32,6 +34,24 @@ function isOSX { [ "$AGENT_OS" = "Darwin" ] } +function isMacOS { + isOSX +} + +function isWindows { + [ "$AGENT_OS" = "Windows_NT" ] +} + function getCIBranch { echo "$BUILD_SOURCEBRANCHNAME" } + +function ciCommandAddPath { + if [[ $# -ne 1 ]]; then + echo "usage: $0 " + exit 1 + fi + path="$1" + + echo "##vso[task.prependpath]${path}" +} From aa760064c8aee2b0b47fcde39104de4743d63894 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 4 Oct 2019 18:53:52 +0200 Subject: [PATCH 10/43] ci: extract installing clang into a script --- .../azure-pipelines/steps/install-clang.yml | 46 ------------------- src/ci/azure-pipelines/steps/run.yml | 6 ++- src/ci/scripts/install-clang.sh | 42 +++++++++++++++++ src/ci/shared.sh | 11 +++++ 4 files changed, 58 insertions(+), 47 deletions(-) delete mode 100644 src/ci/azure-pipelines/steps/install-clang.yml create mode 100755 src/ci/scripts/install-clang.sh diff --git a/src/ci/azure-pipelines/steps/install-clang.yml b/src/ci/azure-pipelines/steps/install-clang.yml deleted file mode 100644 index 14daf81b43075..0000000000000 --- a/src/ci/azure-pipelines/steps/install-clang.yml +++ /dev/null @@ -1,46 +0,0 @@ -steps: - -- bash: | - set -e - curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - - - export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang - echo "##vso[task.setvariable variable=CC]$CC" - - export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ - echo "##vso[task.setvariable variable=CXX]$CXX" - - # Configure `AR` specifically so rustbuild doesn't try to infer it as - # `clang-ar` by accident. - echo "##vso[task.setvariable variable=AR]ar" - displayName: Install clang (OSX) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - -# If we're compiling for MSVC then we, like most other distribution builders, -# switch to clang as the compiler. This'll allow us eventually to enable LTO -# amongst LLVM and rustc. Note that we only do this on MSVC as I don't think -# clang has an output mode compatible with MinGW that we need. If it does we -# should switch to clang for MinGW as well! -# -# Note that the LLVM installer is an NSIS installer -# -# Original downloaded here came from -# http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe -# That installer was run through `wine` on Linux and then the resulting -# installation directory (found in `$HOME/.wine/drive_c/Program Files/LLVM`) was -# packaged up into a tarball. We've had issues otherwise that the installer will -# randomly hang, provide not a lot of useful information, pollute global state, -# etc. In general the tarball is just more confined and easier to deal with when -# working with various CI environments. -- bash: | - set -e - mkdir -p citools - cd citools - curl -f https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/LLVM-7.0.0-win64.tar.gz | tar xzf - - echo "##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]$RUST_CONFIGURE_ARGS --set llvm.clang-cl=`pwd`/clang-rust/bin/clang-cl.exe" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) - displayName: Install clang (Windows) - -# Note that we don't install clang on Linux since its compiler story is just so -# different. Each container has its own toolchain configured appropriately -# already. diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 66405f2d2bfde..abf6164b8b797 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -57,7 +57,11 @@ steps: displayName: Install sccache condition: and(succeeded(), not(variables.SKIP_JOB)) -- template: install-clang.yml +- bash: src/ci/scripts/install-clang.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install clang + condition: and(succeeded(), not(variables.SKIP_JOB)) # Switch to XCode 9.3 on OSX since it seems to be the last version that supports # i686-apple-darwin. We'll eventually want to upgrade this and it will probably diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh new file mode 100755 index 0000000000000..79cbc73a61a71 --- /dev/null +++ b/src/ci/scripts/install-clang.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# This script installs clang on the local machine. Note that we don't install +# clang on Linux since its compiler story is just so different. Each container +# has its own toolchain configured appropriately already. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isMacOS; then + curl -f https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/clang%2Bllvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - + + ciCommandSetEnv CC "$(pwd)/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang" + ciCommandSetEnv CXX "$(pwd)/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++" + + # Configure `AR` specifically so rustbuild doesn't try to infer it as + # `clang-ar` by accident. + ciCommandSetEnv AR "ar" +elif isWindows; then + # If we're compiling for MSVC then we, like most other distribution builders, + # switch to clang as the compiler. This'll allow us eventually to enable LTO + # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think + # clang has an output mode compatible with MinGW that we need. If it does we + # should switch to clang for MinGW as well! + # + # Note that the LLVM installer is an NSIS installer + # + # Original downloaded here came from + # http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe + # That installer was run through `wine` on Linux and then the resulting + # installation directory (found in `$HOME/.wine/drive_c/Program Files/LLVM`) was + # packaged up into a tarball. We've had issues otherwise that the installer will + # randomly hang, provide not a lot of useful information, pollute global state, + # etc. In general the tarball is just more confined and easier to deal with when + # working with various CI environments. + + mkdir -p citools + cd citools + curl -f "${MIRRORS_BASE}/LLVM-7.0.0-win64.tar.gz" | tar xzf - + ciCommandSetEnv RUST_CONFIGURE_ARGS "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe" +fi diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 49fe3841ceb3b..3d54c94de38f5 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -55,3 +55,14 @@ function ciCommandAddPath { echo "##vso[task.prependpath]${path}" } + +function ciCommandSetEnv { + if [[ $# -ne 2 ]]; then + echo "usage: $0 " + exit 1 + fi + name="$1" + value="$2" + + echo "##vso[task.setvariable variable=${name}]${value}" +} From 7c23f5826aeee189f19a894ee8b5246a2013a97c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 7 Oct 2019 13:44:37 +0200 Subject: [PATCH 11/43] ci: extract switching xcode into a script --- src/ci/azure-pipelines/steps/run.yml | 13 +++++-------- src/ci/scripts/switch-xcode.sh | 13 +++++++++++++ 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100755 src/ci/scripts/switch-xcode.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index abf6164b8b797..cfade9a6f5240 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -63,14 +63,11 @@ steps: displayName: Install clang condition: and(succeeded(), not(variables.SKIP_JOB)) -# Switch to XCode 9.3 on OSX since it seems to be the last version that supports -# i686-apple-darwin. We'll eventually want to upgrade this and it will probably -# force us to drop i686-apple-darwin, but let's keep the wheels turning for now. -- bash: | - set -e - sudo xcode-select --switch /Applications/Xcode_9.3.app - displayName: Switch to Xcode 9.3 (OSX) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) +- bash: src/ci/scripts/switch-xcode.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Switch to Xcode 9.3 + condition: and(succeeded(), not(variables.SKIP_JOB)) - template: install-windows-build-deps.yml diff --git a/src/ci/scripts/switch-xcode.sh b/src/ci/scripts/switch-xcode.sh new file mode 100755 index 0000000000000..2cbb2ddbc7046 --- /dev/null +++ b/src/ci/scripts/switch-xcode.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Switch to XCode 9.3 on OSX since it seems to be the last version that supports +# i686-apple-darwin. We'll eventually want to upgrade this and it will probably +# force us to drop i686-apple-darwin, but let's keep the wheels turning for now. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isMacOS; then + sudo xcode-select --switch /Applications/Xcode_9.3.app +fi From 6cfe5d9045461e4faf010eaa1f5b32b7ab90c0c8 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 7 Oct 2019 14:01:18 +0200 Subject: [PATCH 12/43] ci: extract parts of windows-build-deps into scripts --- .../steps/install-windows-build-deps.yml | 35 ------------------- src/ci/azure-pipelines/steps/run.yml | 18 ++++++++++ src/ci/scripts/install-innosetup.sh | 18 ++++++++++ src/ci/scripts/install-wix.sh | 17 +++++++++ src/ci/scripts/windows-symlink-build-dir.sh | 15 ++++++++ 5 files changed, 68 insertions(+), 35 deletions(-) create mode 100755 src/ci/scripts/install-innosetup.sh create mode 100755 src/ci/scripts/install-wix.sh create mode 100755 src/ci/scripts/windows-symlink-build-dir.sh diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml index bd4f1ed0cea43..ef87551d637f6 100644 --- a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml +++ b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml @@ -1,39 +1,4 @@ steps: -# We use the WIX toolset to create combined installers for Windows, and these -# binaries are downloaded from -# https://github.com/wixtoolset/wix3 originally -- bash: | - set -e - curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/wix311-binaries.zip - echo "##vso[task.setvariable variable=WIX]`pwd`/wix" - mkdir -p wix/bin - cd wix/bin - 7z x ../../wix311-binaries.zip - displayName: Install wix - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -# We use InnoSetup and its `iscc` program to also create combined installers. -# Honestly at this point WIX above and `iscc` are just holdovers from -# oh-so-long-ago and are required for creating installers on Windows. I think -# one is MSI installers and one is EXE, but they're not used so frequently at -# this point anyway so perhaps it's a wash! -- script: | - echo ##vso[task.prependpath]C:\Program Files (x86)\Inno Setup 5 - curl.exe -o is-install.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe - is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- - displayName: Install InnoSetup - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -# We've had issues with the default drive in use running out of space during a -# build, and it looks like the `C:` drive has more space than the default `D:` -# drive. We should probably confirm this with the azure pipelines team at some -# point, but this seems to fix our "disk space full" problems. -- script: | - mkdir c:\MORE_SPACE - mklink /J build c:\MORE_SPACE - displayName: "Ensure build happens on C:/ instead of D:/" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - - bash: git config --replace-all --global core.autocrlf false displayName: "Disable git automatic line ending conversion (on C:/)" diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index cfade9a6f5240..19745f3f8c964 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -69,6 +69,24 @@ steps: displayName: Switch to Xcode 9.3 condition: and(succeeded(), not(variables.SKIP_JOB)) +- bash: src/ci/scripts/install-wix.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install wix + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/install-innosetup.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install InnoSetup + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/windows-symlink-build-dir.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Ensure the build happens on C:\ instead of D:\ + condition: and(succeeded(), not(variables.SKIP_JOB)) + - template: install-windows-build-deps.yml # Looks like docker containers have IPv6 disabled by default, so let's turn it diff --git a/src/ci/scripts/install-innosetup.sh b/src/ci/scripts/install-innosetup.sh new file mode 100755 index 0000000000000..c410bb9b4ff64 --- /dev/null +++ b/src/ci/scripts/install-innosetup.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# We use InnoSetup and its `iscc` program to also create combined installers. +# Honestly at this point WIX above and `iscc` are just holdovers from +# oh-so-long-ago and are required for creating installers on Windows. I think +# one is MSI installers and one is EXE, but they're not used so frequently at +# this point anyway so perhaps it's a wash! + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + curl.exe -o is-install.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe + is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- + + ciCommandAddPath "C:\\Program Files (x86)\\Inno Setup 5" +fi diff --git a/src/ci/scripts/install-wix.sh b/src/ci/scripts/install-wix.sh new file mode 100755 index 0000000000000..b0b22b347e0d5 --- /dev/null +++ b/src/ci/scripts/install-wix.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# We use the WIX toolset to create combined installers for Windows, and these +# binaries are downloaded from https://github.com/wixtoolset/wix3 originally + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/wix311-binaries.zip + mkdir -p wix/bin + cd wix/bin + 7z x ../../wix311-binaries.zip + + ciCommandSetEnv WIX "$(pwd)/wix" +fi diff --git a/src/ci/scripts/windows-symlink-build-dir.sh b/src/ci/scripts/windows-symlink-build-dir.sh new file mode 100755 index 0000000000000..e57128c70f5f1 --- /dev/null +++ b/src/ci/scripts/windows-symlink-build-dir.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# We've had issues with the default drive in use running out of space during a +# build, and it looks like the `C:` drive has more space than the default `D:` +# drive. We should probably confirm this with the azure pipelines team at some +# point, but this seems to fix our "disk space full" problems. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + cmd //c "mkdir c:\\MORE_SPACE" + cmd //c "mklink /J build c:\\MORE_SPACE" +fi From 1ac2ad353d56469c8255227e25d2c4feed1c401a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 7 Oct 2019 14:57:09 +0200 Subject: [PATCH 13/43] ci: extract disabling git crlf handling into a script --- .../steps/install-windows-build-deps.yml | 3 --- src/ci/azure-pipelines/steps/run.yml | 9 +++++++-- src/ci/scripts/disable-git-crlf-conversion.sh | 13 +++++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) create mode 100755 src/ci/scripts/disable-git-crlf-conversion.sh diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml index ef87551d637f6..90e8f3e9df93c 100644 --- a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml +++ b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml @@ -1,7 +1,4 @@ steps: -- bash: git config --replace-all --global core.autocrlf false - displayName: "Disable git automatic line ending conversion (on C:/)" - # Download and install MSYS2, needed primarily for the test suite (run-make) but # also used by the MinGW toolchain for assembling things. # diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 19745f3f8c964..df8e8e59d7260 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -87,6 +87,10 @@ steps: displayName: Ensure the build happens on C:\ instead of D:\ condition: and(succeeded(), not(variables.SKIP_JOB)) +- bash: src/ci/scripts/disable-git-crlf-conversion.sh + displayName: "Disable git automatic line ending conversion (on C:/)" + condition: and(succeeded(), not(variables.SKIP_JOB)) + - template: install-windows-build-deps.yml # Looks like docker containers have IPv6 disabled by default, so let's turn it @@ -104,8 +108,9 @@ steps: # re-enables autocrlf. We've not tracked down the exact cause -- and there may # be multiple -- but this should ensure submodules are checked out with the # appropriate line endings. -- bash: git config --replace-all --global core.autocrlf false - displayName: "Disable git automatic line ending conversion" +- bash: src/ci/scripts/disable-git-crlf-conversion.sh + displayName: Disable git automatic line ending conversion + condition: and(succeeded(), not(variables.SKIP_JOB)) # Check out all our submodules, but more quickly than using git by using one of # our custom scripts diff --git a/src/ci/scripts/disable-git-crlf-conversion.sh b/src/ci/scripts/disable-git-crlf-conversion.sh new file mode 100755 index 0000000000000..836145fbb8e60 --- /dev/null +++ b/src/ci/scripts/disable-git-crlf-conversion.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Disable automatic line ending conversion, which is enabled by default on +# Azure's Windows image. Having the conversion enabled caused regressions both +# in our test suite (it broke miri tests) and in the ecosystem, since we +# started shipping install scripts with CRLF endings instead of the old LF. +# +# Note that we do this a couple times during the build as the PATH and current +# user/directory change, e.g. when mingw is enabled. + +set -euo pipefail +IFS=$'\n\t' + +git config --replace-all --global core.autocrlf false From f828a41639549797d5ebe32c17e11da1f124fce3 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 7 Oct 2019 18:48:50 +0200 Subject: [PATCH 14/43] ci: extract installing msys2 into a script --- .../steps/install-windows-build-deps.yml | 29 ------------------- src/ci/azure-pipelines/steps/run.yml | 7 +++++ src/ci/scripts/install-msys2.sh | 29 +++++++++++++++++++ 3 files changed, 36 insertions(+), 29 deletions(-) create mode 100755 src/ci/scripts/install-msys2.sh diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml index 90e8f3e9df93c..1a5b14d66aa0f 100644 --- a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml +++ b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml @@ -1,22 +1,4 @@ steps: -# Download and install MSYS2, needed primarily for the test suite (run-make) but -# also used by the MinGW toolchain for assembling things. -# -# FIXME: we should probe the default azure image and see if we can use the MSYS2 -# toolchain there. (if there's even one there). For now though this gets the job -# done. -- bash: | - set -e - choco install msys2 --params="/InstallDir:$(System.Workfolder)/msys2 /NoPath" -y --no-progress - echo "##vso[task.prependpath]$(System.Workfolder)/msys2/usr/bin" - mkdir -p "$(System.Workfolder)/msys2/home/$USERNAME" - displayName: Install msys2 - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -- bash: pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar - displayName: Install msys2 base deps - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - # If we need to download a custom MinGW, do so here and set the path # appropriately. # @@ -58,17 +40,6 @@ steps: condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) displayName: Download standard MinGW -# Make sure we use the native python interpreter instead of some msys equivalent -# one way or another. The msys interpreters seem to have weird path conversions -# baked in which break LLVM's build system one way or another, so let's use the -# native version which keeps everything as native as possible. -- bash: | - set -e - cp C:/Python27amd64/python.exe C:/Python27amd64/python2.7.exe - echo "##vso[task.prependpath]C:/Python27amd64" - displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - # Note that this is originally from the github releases patch of Ninja - bash: | set -e diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index df8e8e59d7260..4700af0148e55 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -91,6 +91,13 @@ steps: displayName: "Disable git automatic line ending conversion (on C:/)" condition: and(succeeded(), not(variables.SKIP_JOB)) +- bash: src/ci/scripts/install-msys2.sh + env: + AGENT_OS: $(Agent.OS) + SYSTEM_WORKFOLDER: $(System.Workfolder) + displayName: Install msys2 + condition: and(succeeded(), not(variables.SKIP_JOB)) + - template: install-windows-build-deps.yml # Looks like docker containers have IPv6 disabled by default, so let's turn it diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh new file mode 100755 index 0000000000000..c5ad8a79cea8b --- /dev/null +++ b/src/ci/scripts/install-msys2.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Download and install MSYS2, needed primarily for the test suite (run-make) but +# also used by the MinGW toolchain for assembling things. +# +# FIXME: we should probe the default azure image and see if we can use the MSYS2 +# toolchain there. (if there's even one there). For now though this gets the job +# done. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + choco install msys2 --params="/InstallDir:${SYSTEM_WORKFOLDER}/msys2 /NoPath" -y --no-progress + mkdir -p "${SYSTEM_WORKFOLDER}/msys2/home/${USERNAME}" + + ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/usr/bin" + export PATH="${SYSTEM_WORKFOLDER}/msys2/usr/bin" + + pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar + + # Make sure we use the native python interpreter instead of some msys equivalent + # one way or another. The msys interpreters seem to have weird path conversions + # baked in which break LLVM's build system one way or another, so let's use the + # native version which keeps everything as native as possible. + cp C:/Python27amd64/python.exe C:/Python27amd64/python2.7.exe + ciCommandAddPath "C:\\Python27amd64" +fi From a5ea6ce6eae2039ffe3c92430735a0e1d857b252 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 8 Oct 2019 10:57:07 +0200 Subject: [PATCH 15/43] ci: extract installing mingw into a script --- .../steps/install-windows-build-deps.yml | 40 ----------------- src/ci/azure-pipelines/steps/run.yml | 7 +++ src/ci/scripts/install-mingw.sh | 44 +++++++++++++++++++ 3 files changed, 51 insertions(+), 40 deletions(-) create mode 100755 src/ci/scripts/install-mingw.sh diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml index 1a5b14d66aa0f..7aab5658dc05a 100644 --- a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml +++ b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml @@ -1,44 +1,4 @@ steps: -# If we need to download a custom MinGW, do so here and set the path -# appropriately. -# -# Here we also do a pretty heinous thing which is to mangle the MinGW -# installation we just downloaded. Currently, as of this writing, we're using -# MinGW-w64 builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it -# appears to be the first version which contains a fix for #40546, builds -# randomly failing during LLVM due to ar.exe/ranlib.exe failures. -# -# Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds -# to contain a regression in gdb (#40184). As a result if we were to use the -# gdb provided (7.11.1) then we would fail all debuginfo tests. -# -# In order to fix spurious failures (pretty high priority) we use 6.3.0. To -# avoid disabling gdb tests we download an *old* version of gdb, specifically -# that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb -# with the 6.2.0 gdb to get tests passing. -# -# Note that we don't literally overwrite the gdb.exe binary because it appears -# to just use gdborig.exe, so that's the binary we deal with instead. -- bash: | - set -e - curl -o mingw.7z $MINGW_URL/$MINGW_ARCHIVE - 7z x -y mingw.7z > /dev/null - curl -o $MINGW_DIR/bin/gdborig.exe $MINGW_URL/2017-04-20-${MSYS_BITS}bit-gdborig.exe - echo "##vso[task.prependpath]`pwd`/$MINGW_DIR/bin" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],'')) - displayName: Download custom MinGW - -# Otherwise install MinGW through `pacman` -- bash: | - set -e - arch=i686 - if [ "$MSYS_BITS" = "64" ]; then - arch=x86_64 - fi - pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2 - echo "##vso[task.prependpath]$(System.Workfolder)/msys2/mingw$MSYS_BITS/bin" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) - displayName: Download standard MinGW # Note that this is originally from the github releases patch of Ninja - bash: | diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 4700af0148e55..7478121306edc 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -98,6 +98,13 @@ steps: displayName: Install msys2 condition: and(succeeded(), not(variables.SKIP_JOB)) +- bash: src/ci/scripts/install-mingw.sh + env: + AGENT_OS: $(Agent.OS) + SYSTEM_WORKFOLDER: $(System.Workfolder) + displayName: Install MinGW + condition: and(succeeded(), not(variables.SKIP_JOB)) + - template: install-windows-build-deps.yml # Looks like docker containers have IPv6 disabled by default, so let's turn it diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh new file mode 100755 index 0000000000000..cf3e3e7ec258c --- /dev/null +++ b/src/ci/scripts/install-mingw.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# If we need to download a custom MinGW, do so here and set the path +# appropriately. +# +# Here we also do a pretty heinous thing which is to mangle the MinGW +# installation we just downloaded. Currently, as of this writing, we're using +# MinGW-w64 builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it +# appears to be the first version which contains a fix for #40546, builds +# randomly failing during LLVM due to ar.exe/ranlib.exe failures. +# +# Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds +# to contain a regression in gdb (#40184). As a result if we were to use the +# gdb provided (7.11.1) then we would fail all debuginfo tests. +# +# In order to fix spurious failures (pretty high priority) we use 6.3.0. To +# avoid disabling gdb tests we download an *old* version of gdb, specifically +# that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb +# with the 6.2.0 gdb to get tests passing. +# +# Note that we don't literally overwrite the gdb.exe binary because it appears +# to just use gdborig.exe, so that's the binary we deal with instead. +# +# Otherwise install MinGW through `pacman` + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + if [[ -z "${MINGW_URL+x}" ]]; then + curl -o mingw.7z "${MINGW_URL}/${MINGW_ARCHIVE}" + 7z x -y mingw.7z > /dev/null + curl -o "${MINGW_DIR}/bin/gdborig.exe" "${MINGW_URL}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" + ciCommandAddPath "$(pwd)/${MINGW_DIR}/bin" + else + arch=i686 + if [ "$MSYS_BITS" = "64" ]; then + arch=x86_64 + fi + pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2 + ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" + fi +fi From 69a946adad37ec748e086d5a630e6f3d3e85f6b9 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 8 Oct 2019 11:01:47 +0200 Subject: [PATCH 16/43] ci: extract installing ninja into a script --- .../steps/install-windows-build-deps.yml | 13 ------------- src/ci/azure-pipelines/steps/run.yml | 6 +++++- src/ci/scripts/install-ninja.sh | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 14 deletions(-) delete mode 100644 src/ci/azure-pipelines/steps/install-windows-build-deps.yml create mode 100755 src/ci/scripts/install-ninja.sh diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml deleted file mode 100644 index 7aab5658dc05a..0000000000000 --- a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml +++ /dev/null @@ -1,13 +0,0 @@ -steps: - -# Note that this is originally from the github releases patch of Ninja -- bash: | - set -e - mkdir ninja - curl -o ninja.zip https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip - 7z x -oninja ninja.zip - rm ninja.zip - echo "##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]$RUST_CONFIGURE_ARGS --enable-ninja" - echo "##vso[task.prependpath]`pwd`/ninja" - displayName: Download and install ninja - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 7478121306edc..04281e54cf4c1 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -105,7 +105,11 @@ steps: displayName: Install MinGW condition: and(succeeded(), not(variables.SKIP_JOB)) -- template: install-windows-build-deps.yml +- bash: src/ci/scripts/install-ninja.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install ninja + condition: and(succeeded(), not(variables.SKIP_JOB)) # Looks like docker containers have IPv6 disabled by default, so let's turn it # on since libstd tests require it diff --git a/src/ci/scripts/install-ninja.sh b/src/ci/scripts/install-ninja.sh new file mode 100755 index 0000000000000..c3d0217706ad9 --- /dev/null +++ b/src/ci/scripts/install-ninja.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Note that this is originally from the github releases patch of Ninja + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + mkdir ninja + curl -o ninja.zip https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip + 7z x -oninja ninja.zip + rm ninja.zip + ciCommandSetEnv "RUST_CONFIGURE_ARGS" "${RUST_CONFIGURE_ARGS} --enable-ninja" + ciCommandAddPath "$(pwd)/ninja" +fi From d778f95b24abda4315e71eff63f26a0ff267a1ea Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 8 Oct 2019 11:46:55 +0200 Subject: [PATCH 17/43] ci: extract enabling ipv6 on docker into a script --- src/ci/azure-pipelines/steps/run.yml | 14 +++++--------- src/ci/scripts/enable-docker-ipv6.sh | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) create mode 100755 src/ci/scripts/enable-docker-ipv6.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 04281e54cf4c1..0562b28ca9cb7 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -111,15 +111,11 @@ steps: displayName: Install ninja condition: and(succeeded(), not(variables.SKIP_JOB)) -# Looks like docker containers have IPv6 disabled by default, so let's turn it -# on since libstd tests require it -- bash: | - set -e - sudo mkdir -p /etc/docker - echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json - sudo service docker restart - displayName: Enable IPv6 - condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Linux')) +- bash: src/ci/scripts/enable-docker-ipv6.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Enable IPv6 on Docker + condition: and(succeeded(), not(variables.SKIP_JOB)) # Disable automatic line ending conversion (again). On Windows, when we're # installing dependencies, something switches the git configuration directory or diff --git a/src/ci/scripts/enable-docker-ipv6.sh b/src/ci/scripts/enable-docker-ipv6.sh new file mode 100755 index 0000000000000..35bb0ad7d29a4 --- /dev/null +++ b/src/ci/scripts/enable-docker-ipv6.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Looks like docker containers have IPv6 disabled by default, so let's turn it +# on since libstd tests require it + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isLinux; then + sudo mkdir -p /etc/docker + echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json + sudo service docker restart +fi From 09bf3cf75a96d64715cc54009ae3a209e9f7332d Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 8 Oct 2019 11:57:06 +0200 Subject: [PATCH 18/43] ci: extract checking out submodules into a script --- src/ci/azure-pipelines/steps/run.yml | 16 +++++----------- src/ci/scripts/checkout-submodules.sh | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 11 deletions(-) create mode 100755 src/ci/scripts/checkout-submodules.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 0562b28ca9cb7..0b15ae36e0c4b 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -128,17 +128,11 @@ steps: # Check out all our submodules, but more quickly than using git by using one of # our custom scripts -- bash: | - set -e - mkdir -p $HOME/rustsrc - $BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc - condition: and(succeeded(), not(variables.SKIP_JOB), ne(variables['Agent.OS'], 'Windows_NT')) - displayName: Check out submodules (Unix) -- script: | - if not exist C:\cache\rustsrc\NUL mkdir C:\cache\rustsrc - sh src/ci/init_repo.sh . /c/cache/rustsrc - condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Windows_NT')) - displayName: Check out submodules (Windows) +- bash: src/ci/scripts/checkout-submodules.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Checkout submodules + condition: and(succeeded(), not(variables.SKIP_JOB)) # See also the disable for autocrlf above, this just checks that it worked # diff --git a/src/ci/scripts/checkout-submodules.sh b/src/ci/scripts/checkout-submodules.sh new file mode 100755 index 0000000000000..0b44ea3c90bc9 --- /dev/null +++ b/src/ci/scripts/checkout-submodules.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Check out all our submodules, but more quickly than using git by using one of +# our custom scripts + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + path="/c/cache/rustsrc" +else + path="${HOME}/rustsrc" +fi + +mkdir -p "${path}" +"$(cd "$(dirname "$0")" && pwd)/../init_repo.sh" . "${path}" From 646da3a993aec0d26ebd71aa1af365345aab188b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 8 Oct 2019 12:02:41 +0200 Subject: [PATCH 19/43] ci: extract verifying line endings into a script --- src/ci/azure-pipelines/steps/run.yml | 26 +++++--------------------- src/ci/scripts/verify-line-endings.sh | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 21 deletions(-) create mode 100755 src/ci/scripts/verify-line-endings.sh diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 0b15ae36e0c4b..1f8ef40ec21ab 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -126,33 +126,17 @@ steps: displayName: Disable git automatic line ending conversion condition: and(succeeded(), not(variables.SKIP_JOB)) -# Check out all our submodules, but more quickly than using git by using one of -# our custom scripts - bash: src/ci/scripts/checkout-submodules.sh env: AGENT_OS: $(Agent.OS) displayName: Checkout submodules condition: and(succeeded(), not(variables.SKIP_JOB)) -# See also the disable for autocrlf above, this just checks that it worked -# -# We check both in rust-lang/rust and in a submodule to make sure both are -# accurate. Submodules are checked out significantly later than the main -# repository in this script, so settings can (and do!) change between then. -# -# Linux (and maybe macOS) builders don't currently have dos2unix so just only -# run this step on Windows. -- bash: | - set -x - # print out the git configuration so we can better investigate failures in - # the following - git config --list --show-origin - dos2unix -ih Cargo.lock src/tools/rust-installer/install-template.sh - endings=$(dos2unix -ic Cargo.lock src/tools/rust-installer/install-template.sh) - # if endings has non-zero length, error out - if [ -n "$endings" ]; then exit 1 ; fi - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - displayName: Verify line endings are LF +- bash: src/ci/scripts/verify-line-endings.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Verify line endings + condition: and(succeeded(), not(variables.SKIP_JOB)) # Ensure the `aws` CLI is installed so we can deploy later on, cache docker # images, etc. diff --git a/src/ci/scripts/verify-line-endings.sh b/src/ci/scripts/verify-line-endings.sh new file mode 100755 index 0000000000000..f3cac13ea4802 --- /dev/null +++ b/src/ci/scripts/verify-line-endings.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# See also the disable for autocrlf, this just checks that it worked. +# +# We check both in rust-lang/rust and in a submodule to make sure both are +# accurate. Submodules are checked out significantly later than the main +# repository in this script, so settings can (and do!) change between then. +# +# Linux (and maybe macOS) builders don't currently have dos2unix so just only +# run this step on Windows. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + # print out the git configuration so we can better investigate failures in + # the following + git config --list --show-origin + dos2unix -ih Cargo.lock src/tools/rust-installer/install-template.sh + endings=$(dos2unix -ic Cargo.lock src/tools/rust-installer/install-template.sh) + # if endings has non-zero length, error out + if [ -n "$endings" ]; then exit 1 ; fi +fi From c139db70872d878ca80c8389ddddf6b427a55a3a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 8 Oct 2019 12:04:28 +0200 Subject: [PATCH 20/43] ci: use shared.sh in scripts/install-awscli.sh --- src/ci/scripts/install-awscli.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ci/scripts/install-awscli.sh b/src/ci/scripts/install-awscli.sh index 69c8d2e3099ab..f104310430290 100755 --- a/src/ci/scripts/install-awscli.sh +++ b/src/ci/scripts/install-awscli.sh @@ -16,12 +16,14 @@ set -euo pipefail IFS=$'\n\t' +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + MIRROR="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-07-27-awscli.tar" DEPS_DIR="/tmp/awscli-deps" pip="pip" pipflags="" -if [[ "${AGENT_OS}" == "Linux" ]]; then +if isLinux; then pip="pip3" pipflags="--user" From 9458c9bcb4ca60dac6b1da04efc1f0c7ce209e11 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 8 Oct 2019 12:05:45 +0200 Subject: [PATCH 21/43] ci: cleanup platform detection --- src/ci/run.sh | 2 +- src/ci/shared.sh | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ci/run.sh b/src/ci/run.sh index 0d5ea371245e4..bce35670c8d46 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -117,7 +117,7 @@ make check-bootstrap # Display the CPU and memory information. This helps us know why the CI timing # is fluctuating. -if isOSX; then +if isMacOS; then system_profiler SPHardwareDataType || true sysctl hw || true ncpus=$(sysctl -n hw.ncpu) diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 3d54c94de38f5..37e45b5639dc9 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -30,16 +30,16 @@ function isCI { [ "$CI" = "true" ] || [ "$TF_BUILD" = "True" ] } -function isOSX { +function isMacOS { [ "$AGENT_OS" = "Darwin" ] } -function isMacOS { - isOSX +function isWindows { + [ "$AGENT_OS" = "Windows_NT" ] } -function isWindows { - [ "$AGENT_OS" = "Windows_NT" ] +function isLinux { + [ "$AGENT_OS" = "Linux" ] } function getCIBranch { From 11173f8664516bf174ab16fbb929c27be87dbb6b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 8 Oct 2019 16:10:25 +0200 Subject: [PATCH 22/43] ci: fix tidy --- src/ci/scripts/enable-docker-ipv6.sh | 3 ++- src/ci/scripts/install-clang.sh | 3 ++- src/ci/scripts/install-innosetup.sh | 3 ++- src/ci/scripts/install-mingw.sh | 3 ++- src/ci/scripts/install-ninja.sh | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ci/scripts/enable-docker-ipv6.sh b/src/ci/scripts/enable-docker-ipv6.sh index 35bb0ad7d29a4..03d5a75e24e27 100755 --- a/src/ci/scripts/enable-docker-ipv6.sh +++ b/src/ci/scripts/enable-docker-ipv6.sh @@ -9,6 +9,7 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isLinux; then sudo mkdir -p /etc/docker - echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json + echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' \ + | sudo tee /etc/docker/daemon.json sudo service docker restart fi diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index 79cbc73a61a71..13740e7ba5a70 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -38,5 +38,6 @@ elif isWindows; then mkdir -p citools cd citools curl -f "${MIRRORS_BASE}/LLVM-7.0.0-win64.tar.gz" | tar xzf - - ciCommandSetEnv RUST_CONFIGURE_ARGS "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe" + ciCommandSetEnv RUST_CONFIGURE_ARGS \ + "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe" fi diff --git a/src/ci/scripts/install-innosetup.sh b/src/ci/scripts/install-innosetup.sh index c410bb9b4ff64..4ad5f24240288 100755 --- a/src/ci/scripts/install-innosetup.sh +++ b/src/ci/scripts/install-innosetup.sh @@ -11,7 +11,8 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then - curl.exe -o is-install.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe + curl.exe -o is-install.exe \ + https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- ciCommandAddPath "C:\\Program Files (x86)\\Inno Setup 5" diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh index cf3e3e7ec258c..5a1c19b8981ec 100755 --- a/src/ci/scripts/install-mingw.sh +++ b/src/ci/scripts/install-mingw.sh @@ -38,7 +38,8 @@ if isWindows; then if [ "$MSYS_BITS" = "64" ]; then arch=x86_64 fi - pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2 + pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \ + mingw-w64-$arch-gcc mingw-w64-$arch-python2 ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" fi fi diff --git a/src/ci/scripts/install-ninja.sh b/src/ci/scripts/install-ninja.sh index c3d0217706ad9..f5ab17edb71bb 100755 --- a/src/ci/scripts/install-ninja.sh +++ b/src/ci/scripts/install-ninja.sh @@ -8,7 +8,8 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then mkdir ninja - curl -o ninja.zip https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip + curl -o ninja.zip \ + https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip 7z x -oninja ninja.zip rm ninja.zip ciCommandSetEnv "RUST_CONFIGURE_ARGS" "${RUST_CONFIGURE_ARGS} --enable-ninja" From 11c59e3a8def171a8d3ae28b69d6bf3c014dcf4f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 9 Oct 2019 15:43:30 +0200 Subject: [PATCH 23/43] ci: reuse the mirrors base url from shared.sh in scripts --- src/ci/scripts/install-awscli.sh | 2 +- src/ci/scripts/install-clang.sh | 2 +- src/ci/scripts/install-innosetup.sh | 3 +-- src/ci/scripts/install-ninja.sh | 3 +-- src/ci/scripts/install-wix.sh | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/ci/scripts/install-awscli.sh b/src/ci/scripts/install-awscli.sh index f104310430290..e21187938504c 100755 --- a/src/ci/scripts/install-awscli.sh +++ b/src/ci/scripts/install-awscli.sh @@ -18,7 +18,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" -MIRROR="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-07-27-awscli.tar" +MIRROR="${MIRRORS_BASE}/2019-07-27-awscli.tar" DEPS_DIR="/tmp/awscli-deps" pip="pip" diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index 13740e7ba5a70..ccb6a0bd9dc16 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -9,7 +9,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isMacOS; then - curl -f https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/clang%2Bllvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - + curl -f "${MIRRORS_BASE}/clang%2Bllvm-7.0.0-x86_64-apple-darwin.tar.xz" | tar xJf - ciCommandSetEnv CC "$(pwd)/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang" ciCommandSetEnv CXX "$(pwd)/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++" diff --git a/src/ci/scripts/install-innosetup.sh b/src/ci/scripts/install-innosetup.sh index 4ad5f24240288..ff1a9954ee6e3 100755 --- a/src/ci/scripts/install-innosetup.sh +++ b/src/ci/scripts/install-innosetup.sh @@ -11,8 +11,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then - curl.exe -o is-install.exe \ - https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe + curl.exe -o is-install.exe "${MIRRORS_BASE}/2017-08-22-is.exe" is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- ciCommandAddPath "C:\\Program Files (x86)\\Inno Setup 5" diff --git a/src/ci/scripts/install-ninja.sh b/src/ci/scripts/install-ninja.sh index f5ab17edb71bb..b8261d8a6f284 100755 --- a/src/ci/scripts/install-ninja.sh +++ b/src/ci/scripts/install-ninja.sh @@ -8,8 +8,7 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then mkdir ninja - curl -o ninja.zip \ - https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip + curl -o ninja.zip "${MIRRORS_BASE}/2017-03-15-ninja-win.zip" 7z x -oninja ninja.zip rm ninja.zip ciCommandSetEnv "RUST_CONFIGURE_ARGS" "${RUST_CONFIGURE_ARGS} --enable-ninja" diff --git a/src/ci/scripts/install-wix.sh b/src/ci/scripts/install-wix.sh index b0b22b347e0d5..950e19ff6407c 100755 --- a/src/ci/scripts/install-wix.sh +++ b/src/ci/scripts/install-wix.sh @@ -8,7 +8,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then - curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/wix311-binaries.zip + curl -O "${MIRRORS_BASE}/wix311-binaries.zip" mkdir -p wix/bin cd wix/bin 7z x ../../wix311-binaries.zip From 6351efbd2a6370bcd83d02bde619f09f9c7be46d Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 15 Oct 2019 10:21:11 +0200 Subject: [PATCH 24/43] ci: fix innosetup installation --- src/ci/scripts/install-innosetup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/scripts/install-innosetup.sh b/src/ci/scripts/install-innosetup.sh index ff1a9954ee6e3..04ca249777a11 100755 --- a/src/ci/scripts/install-innosetup.sh +++ b/src/ci/scripts/install-innosetup.sh @@ -12,7 +12,7 @@ source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if isWindows; then curl.exe -o is-install.exe "${MIRRORS_BASE}/2017-08-22-is.exe" - is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- + cmd.exe //c "is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP-" ciCommandAddPath "C:\\Program Files (x86)\\Inno Setup 5" fi From 01008e4cd68a92e585d7f9e718393568c800589e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Miku=C5=82a?= Date: Wed, 16 Oct 2019 12:50:59 +0200 Subject: [PATCH 25/43] Update libc to 0.2.64 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d40b19892790..01637a212cba0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1713,9 +1713,9 @@ checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" [[package]] name = "libc" -version = "0.2.62" +version = "0.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" +checksum = "74dfca3d9957906e8d1e6a0b641dc9a59848e793f1da2165889fd4f62d10d79c" dependencies = [ "rustc-std-workspace-core", ] From c2c290eaf48357bc16e315ecd2ae3ab555a14c45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= Date: Wed, 16 Oct 2019 00:00:00 +0000 Subject: [PATCH 26/43] Document sanitizers in unstable-book --- .../src/compiler-flags/sanitizer.md | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 src/doc/unstable-book/src/compiler-flags/sanitizer.md diff --git a/src/doc/unstable-book/src/compiler-flags/sanitizer.md b/src/doc/unstable-book/src/compiler-flags/sanitizer.md new file mode 100644 index 0000000000000..cbb90bd3bb331 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/sanitizer.md @@ -0,0 +1,163 @@ +# `sanitizer` + +The tracking issue for this feature is: [#39699](https://github.com/rust-lang/rust/issues/39699). + +------------------------ + +This feature allows for use of one of following sanitizers: + +* [AddressSanitizer][clang-asan] a faster memory error detector. Can + detect out-of-bounds access to heap, stack, and globals, use after free, use + after return, double free, invalid free, memory leaks. +* [LeakSanitizer][clang-lsan] a run-time memory leak detector. +* [MemorySanitizer][clang-msan] a detector of uninitialized reads. +* [ThreadSanitizer][clang-tsan] a fast data race detector. + +To enable a sanitizer compile with `-Zsanitizer=...` option, where value is one +of `address`, `leak`, `memory` or `thread`. + +# Examples + +This sections show various issues that can be detected with sanitizers. For +simplicity, the examples are prepared under assumption that optimization level +used is zero. + +## AddressSanitizer + +Stack buffer overflow: + +```shell +$ cat a.rs +fn main() { + let xs = [0, 1, 2, 3]; + let _y = unsafe { *xs.as_ptr().offset(4) }; +} +$ rustc -Zsanitizer=address a.rs +$ ./a +================================================================= +==10029==ERROR: AddressSanitizer: stack-buffer-overflow on address 0x7ffcc15f43d0 at pc 0x55f77dc015c5 bp 0x7ffcc15f4390 sp 0x7ffcc15f4388 +READ of size 4 at 0x7ffcc15f43d0 thread T0 + #0 0x55f77dc015c4 in a::main::hab3bd2a745c2d0ac (/tmp/a+0xa5c4) + #1 0x55f77dc01cdb in std::rt::lang_start::_$u7b$$u7b$closure$u7d$$u7d$::haa8c76d1faa7b7ca (/tmp/a+0xacdb) + #2 0x55f77dc90f02 in std::rt::lang_start_internal::_$u7b$$u7b$closure$u7d$$u7d$::hfeb9a1aef9ac820d /rustc/c27f7568bc74c418996892028a629eed5a7f5f00/src/libstd/rt.rs:48:12 + #3 0x55f77dc90f02 in std::panicking::try::do_call::h12f0919717b8e0a6 /rustc/c27f7568bc74c418996892028a629eed5a7f5f00/src/libstd/panicking.rs:288:39 + #4 0x55f77dc926c9 in __rust_maybe_catch_panic /rustc/c27f7568bc74c418996892028a629eed5a7f5f00/src/libpanic_unwind/lib.rs:80:7 + #5 0x55f77dc9197c in std::panicking::try::h413b21cdcd6cfd86 /rustc/c27f7568bc74c418996892028a629eed5a7f5f00/src/libstd/panicking.rs:267:12 + #6 0x55f77dc9197c in std::panic::catch_unwind::hc5cc8ef2fd73424d /rustc/c27f7568bc74c418996892028a629eed5a7f5f00/src/libstd/panic.rs:396:8 + #7 0x55f77dc9197c in std::rt::lang_start_internal::h2039f418ab92218f /rustc/c27f7568bc74c418996892028a629eed5a7f5f00/src/libstd/rt.rs:47:24 + #8 0x55f77dc01c61 in std::rt::lang_start::ha905d28f6b61d691 (/tmp/a+0xac61) + #9 0x55f77dc0163a in main (/tmp/a+0xa63a) + #10 0x7f9b3cf5bbba in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x26bba) + #11 0x55f77dc01289 in _start (/tmp/a+0xa289) + +Address 0x7ffcc15f43d0 is located in stack of thread T0 at offset 48 in frame + #0 0x55f77dc0135f in a::main::hab3bd2a745c2d0ac (/tmp/a+0xa35f) + + This frame has 1 object(s): + [32, 48) 'xs' <== Memory access at offset 48 overflows this variable +HINT: this may be a false positive if your program uses some custom stack unwind mechanism, swapcontext or vfork + (longjmp and C++ exceptions *are* supported) +SUMMARY: AddressSanitizer: stack-buffer-overflow (/tmp/a+0xa5c4) in a::main::hab3bd2a745c2d0ac +Shadow bytes around the buggy address: + 0x1000182b6820: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 0x1000182b6830: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 0x1000182b6840: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 0x1000182b6850: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 0x1000182b6860: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +=>0x1000182b6870: 00 00 00 00 f1 f1 f1 f1 00 00[f3]f3 00 00 00 00 + 0x1000182b6880: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 0x1000182b6890: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 0x1000182b68a0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 0x1000182b68b0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 + 0x1000182b68c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +Shadow byte legend (one shadow byte represents 8 application bytes): + Addressable: 00 + Partially addressable: 01 02 03 04 05 06 07 + Heap left redzone: fa + Freed heap region: fd + Stack left redzone: f1 + Stack mid redzone: f2 + Stack right redzone: f3 + Stack after return: f5 + Stack use after scope: f8 + Global redzone: f9 + Global init order: f6 + Poisoned by user: f7 + Container overflow: fc + Array cookie: ac + Intra object redzone: bb + ASan internal: fe + Left alloca redzone: ca + Right alloca redzone: cb + Shadow gap: cc +==10029==ABORTING +``` + +## MemorySanitizer + +Use of uninitialized memory. Note that we are using `-Zbuild-std` to instrument +standard library, and passing `-msan-track-origins=2` to the LLVM to track +origins of uninitialized memory: + +```shell +$ cat src/main.rs +use std::mem::MaybeUninit; + +fn main() { + unsafe { + let a = MaybeUninit::<[usize; 4]>::uninit(); + let a = a.assume_init(); + println!("{}", a[2]); + } +} + +$ env RUSTFLAGS="-Zsanitizer=memory -Cllvm-args=-msan-track-origins=2" cargo -Zbuild-std run --target x86_64-unknown-linux-gnu +==9416==WARNING: MemorySanitizer: use-of-uninitialized-value + #0 0x560c04f7488a in core::fmt::num::imp::fmt_u64::haa293b0b098501ca $RUST/build/x86_64-unknown-linux-gnu/stage1/lib/rustlib/src/rust/src/libcore/fmt/num.rs:202:16 +... + Uninitialized value was stored to memory at + #0 0x560c04ae898a in __msan_memcpy.part.0 $RUST/src/llvm-project/compiler-rt/lib/msan/msan_interceptors.cc:1558:3 + #1 0x560c04b2bf88 in memory::main::hd2333c1899d997f5 $CWD/src/main.rs:6:16 + + Uninitialized value was created by an allocation of 'a' in the stack frame of function '_ZN6memory4main17hd2333c1899d997f5E' + #0 0x560c04b2bc50 in memory::main::hd2333c1899d997f5 $CWD/src/main.rs:3 +``` + + +# Instrumentation of external dependencies and std + +The sanitizers to varying degrees work correctly with partially instrumented +code. On the one extreme is LeakSanitizer that doesn't use any compile time +instrumentation, on the other is MemorySanitizer that requires that all program +code to be instrumented (failing to achieve that will inevitably result in +false positives). + +It is strongly recommended to combine sanitizers with recompiled and +instrumented standard library, for example using [cargo `-Zbuild-std` +functionality][build-std]. + +[build-std]: https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#build-std + +# Build scripts and procedural macros + +Use of sanitizers together with build scripts and procedural macros is +technically possible, but in almost all cases it would be best avoided. This +is especially true for procedural macros which would require an instrumented +version of rustc. + +In more practical terms when using cargo always remember to pass `--target` +flag, so that rustflags will not be applied to build scripts and procedural +macros. + +# Additional Information + +* [Sanitizers project page](https://github.com/google/sanitizers/wiki/) +* [AddressSanitizer in Clang][clang-asan] +* [LeakSanitizer in Clang][clang-lsan] +* [MemorySanitizer in Clang][clang-msan] +* [ThreadSanitizer in Clang][clang-tsan] + +[clang-asan]: https://clang.llvm.org/docs/AddressSanitizer.html +[clang-lsan]: https://clang.llvm.org/docs/LeakSanitizer.html +[clang-msan]: https://clang.llvm.org/docs/MemorySanitizer.html +[clang-tsan]: https://clang.llvm.org/docs/ThreadSanitizer.html From ca1cfdab78d1966efd492cd550a3684f3b95527c Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Wed, 16 Oct 2019 20:32:58 +0200 Subject: [PATCH 27/43] Uninitialized boxes: check for zero-size allocation based on Layout::size --- src/liballoc/boxed.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/liballoc/boxed.rs b/src/liballoc/boxed.rs index 2693a64e13ba1..567b8ea722491 100644 --- a/src/liballoc/boxed.rs +++ b/src/liballoc/boxed.rs @@ -141,10 +141,10 @@ impl Box { /// ``` #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit() -> Box> { - if mem::size_of::() == 0 { + let layout = alloc::Layout::new::>(); + if layout.size() == 0 { return Box(NonNull::dangling().into()) } - let layout = alloc::Layout::new::>(); let ptr = unsafe { Global.alloc(layout) .unwrap_or_else(|_| alloc::handle_alloc_error(layout)) @@ -184,10 +184,10 @@ impl Box<[T]> { /// ``` #[unstable(feature = "new_uninit", issue = "63291")] pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit]> { - let ptr = if mem::size_of::() == 0 || len == 0 { + let layout = alloc::Layout::array::>(len).unwrap(); + let ptr = if layout.size() == 0 { NonNull::dangling() } else { - let layout = alloc::Layout::array::>(len).unwrap(); unsafe { Global.alloc(layout) .unwrap_or_else(|_| alloc::handle_alloc_error(layout)) From dd4d6a989278004547e504f4c7e755eb39be604d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= Date: Thu, 17 Oct 2019 00:00:00 +0000 Subject: [PATCH 28/43] Verify that sanitizer runtime is not part of staticlib --- src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile b/src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile index 5e62fcc7a5688..d12969f16bc35 100644 --- a/src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile +++ b/src/test/run-make-fulldeps/sanitizer-staticlib-link/Makefile @@ -10,6 +10,9 @@ all: $(RUSTC) -g -Z sanitizer=address --crate-type staticlib --target $(TARGET) library.rs + # Verify that linking without sanitizer fails, i.e., runtime is not part of staticlib: + $(RUSTC) -g --crate-type bin --target $(TARGET) -L . program.rs && exit 1 || true + # Verify that linking with sanitizer succeeds and detects overflow in library: $(RUSTC) -g -Z sanitizer=address --crate-type bin --target $(TARGET) -L . program.rs $(TMPDIR)/program 2>&1 | $(CGREP) stack-buffer-overflow From 75f4dac5f38cfe46cf9ccd3b2c602a5e1336453c Mon Sep 17 00:00:00 2001 From: Dylan MacKenzie Date: Wed, 16 Oct 2019 18:00:44 -0700 Subject: [PATCH 29/43] Add regression test for #65394 --- src/test/ui/consts/const-eval/issue-65394.rs | 13 +++++++++++++ src/test/ui/consts/const-eval/issue-65394.stderr | 11 +++++++++++ 2 files changed, 24 insertions(+) create mode 100644 src/test/ui/consts/const-eval/issue-65394.rs create mode 100644 src/test/ui/consts/const-eval/issue-65394.stderr diff --git a/src/test/ui/consts/const-eval/issue-65394.rs b/src/test/ui/consts/const-eval/issue-65394.rs new file mode 100644 index 0000000000000..978e227bcc817 --- /dev/null +++ b/src/test/ui/consts/const-eval/issue-65394.rs @@ -0,0 +1,13 @@ +// Test for absence of validation mismatch ICE in #65394 + +#![feature(rustc_attrs)] + +#[rustc_mir(borrowck_graphviz_postflow="hello.dot")] +const _: Vec = { + let mut x = Vec::::new(); + let r = &mut x; //~ ERROR references in constants may only refer to immutable values + let y = x; + y +}; + +fn main() {} diff --git a/src/test/ui/consts/const-eval/issue-65394.stderr b/src/test/ui/consts/const-eval/issue-65394.stderr new file mode 100644 index 0000000000000..f48c551cb50f5 --- /dev/null +++ b/src/test/ui/consts/const-eval/issue-65394.stderr @@ -0,0 +1,11 @@ +error[E0017]: references in constants may only refer to immutable values + --> $DIR/issue-65394.rs:8:13 + | +LL | let r = &mut x; + | ^^^^^^ constants require immutable values + +[ERROR rustc_mir::transform::qualify_consts] old validator: [($DIR/issue-65394.rs:8:13: 8:19, "MutBorrow(Mut { allow_two_phase_borrow: false })")] +[ERROR rustc_mir::transform::qualify_consts] new validator: [($DIR/issue-65394.rs:8:13: 8:19, "MutBorrow(Mut { allow_two_phase_borrow: false })"), ($DIR/issue-65394.rs:7:9: 7:14, "LiveDrop")] +error: aborting due to previous error + +For more information about this error, try `rustc --explain E0017`. From 22a0856641dec573ab3b83b09c57aa597ca87b19 Mon Sep 17 00:00:00 2001 From: Dylan MacKenzie Date: Wed, 16 Oct 2019 18:01:01 -0700 Subject: [PATCH 30/43] Enable `drain_filter` --- src/librustc_mir/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/librustc_mir/lib.rs b/src/librustc_mir/lib.rs index a837c34e8d474..98d5487870a4d 100644 --- a/src/librustc_mir/lib.rs +++ b/src/librustc_mir/lib.rs @@ -14,6 +14,7 @@ Rust MIR: a lowered representation of Rust. Also: an experiment! #![feature(core_intrinsics)] #![feature(const_fn)] #![feature(decl_macro)] +#![feature(drain_filter)] #![feature(exhaustive_patterns)] #![feature(never_type)] #![feature(specialization)] From af691de9c1d9923e8a6b7965d1eebf21e1d87ad2 Mon Sep 17 00:00:00 2001 From: Dylan MacKenzie Date: Wed, 16 Oct 2019 18:01:26 -0700 Subject: [PATCH 31/43] Suppress validation mismatch ICE in the presence of mut borrows --- src/librustc_mir/transform/qualify_consts.rs | 75 +++++++++++++++----- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/src/librustc_mir/transform/qualify_consts.rs b/src/librustc_mir/transform/qualify_consts.rs index fbcf9c8cb5eba..da1abb9747c1a 100644 --- a/src/librustc_mir/transform/qualify_consts.rs +++ b/src/librustc_mir/transform/qualify_consts.rs @@ -1024,23 +1024,12 @@ impl<'a, 'tcx> Checker<'a, 'tcx> { new_errors.dedup(); if self.errors != new_errors { - error!("old validator: {:?}", self.errors); - error!("new validator: {:?}", new_errors); - - // ICE on nightly if the validators do not emit exactly the same errors. - // Users can supress this panic with an unstable compiler flag (hopefully after - // filing an issue). - let opts = &self.tcx.sess.opts; - let trigger_ice = opts.unstable_features.is_nightly_build() - && !opts.debugging_opts.suppress_const_validation_back_compat_ice; - - if trigger_ice { - span_bug!( - body.span, - "{}", - VALIDATOR_MISMATCH_ERR, - ); - } + validator_mismatch( + self.tcx, + body, + std::mem::replace(&mut self.errors, vec![]), + new_errors, + ); } } @@ -1870,6 +1859,58 @@ fn args_required_const(tcx: TyCtxt<'_>, def_id: DefId) -> Option, + body: &Body<'tcx>, + mut old_errors: Vec<(Span, String)>, + mut new_errors: Vec<(Span, String)>, +) { + error!("old validator: {:?}", old_errors); + error!("new validator: {:?}", new_errors); + + // ICE on nightly if the validators do not emit exactly the same errors. + // Users can supress this panic with an unstable compiler flag (hopefully after + // filing an issue). + let opts = &tcx.sess.opts; + let strict_validation_enabled = opts.unstable_features.is_nightly_build() + && !opts.debugging_opts.suppress_const_validation_back_compat_ice; + + if !strict_validation_enabled { + return; + } + + // If this difference would cause a regression from the old to the new or vice versa, trigger + // the ICE. + if old_errors.is_empty() || new_errors.is_empty() { + span_bug!(body.span, "{}", VALIDATOR_MISMATCH_ERR); + } + + // HACK: Borrows that would allow mutation are forbidden in const contexts, but they cause the + // new validator to be more conservative about when a dropped local has been moved out of. + // + // Supress the mismatch ICE in cases where the validators disagree only on the number of + // `LiveDrop` errors and both observe the same sequence of `MutBorrow`s. + + let is_live_drop = |(_, s): &mut (_, String)| s.starts_with("LiveDrop"); + let is_mut_borrow = |(_, s): &&(_, String)| s.starts_with("MutBorrow"); + + let old_live_drops: Vec<_> = old_errors.drain_filter(is_live_drop).collect(); + let new_live_drops: Vec<_> = new_errors.drain_filter(is_live_drop).collect(); + + let only_live_drops_differ = old_live_drops != new_live_drops && old_errors == new_errors; + + let old_mut_borrows = old_errors.iter().filter(is_mut_borrow); + let new_mut_borrows = new_errors.iter().filter(is_mut_borrow); + + let at_least_one_mut_borrow = old_mut_borrows.clone().next().is_some(); + + if only_live_drops_differ && at_least_one_mut_borrow && old_mut_borrows.eq(new_mut_borrows) { + return; + } + + span_bug!(body.span, "{}", VALIDATOR_MISMATCH_ERR); +} + const VALIDATOR_MISMATCH_ERR: &str = r"Disagreement between legacy and dataflow-based const validators. After filing an issue, use `-Zsuppress-const-validation-back-compat-ice` to compile your code."; From 6de4924b6c1ff5a99397ca1a3894c51f085f3e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Miku=C5=82a?= Date: Thu, 17 Oct 2019 12:09:11 +0200 Subject: [PATCH 32/43] Update emscripten functions declarations --- src/libstd/sys/unix/fd.rs | 34 ++-------------------------------- src/libstd/sys/unix/fs.rs | 2 -- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/src/libstd/sys/unix/fd.rs b/src/libstd/sys/unix/fd.rs index ac43526b50f2c..ba611a6b7e7b9 100644 --- a/src/libstd/sys/unix/fd.rs +++ b/src/libstd/sys/unix/fd.rs @@ -71,22 +71,7 @@ impl FileDesc { #[cfg(target_os = "android")] use super::android::cvt_pread64; - #[cfg(target_os = "emscripten")] - unsafe fn cvt_pread64(fd: c_int, buf: *mut c_void, count: usize, offset: i64) - -> io::Result - { - use crate::convert::TryInto; - use libc::pread64; - // pread64 on emscripten actually takes a 32 bit offset - if let Ok(o) = offset.try_into() { - cvt(pread64(fd, buf, count, o)) - } else { - Err(io::Error::new(io::ErrorKind::InvalidInput, - "cannot pread >2GB")) - } - } - - #[cfg(not(any(target_os = "android", target_os = "emscripten")))] + #[cfg(not(target_os = "android"))] unsafe fn cvt_pread64(fd: c_int, buf: *mut c_void, count: usize, offset: i64) -> io::Result { @@ -128,22 +113,7 @@ impl FileDesc { #[cfg(target_os = "android")] use super::android::cvt_pwrite64; - #[cfg(target_os = "emscripten")] - unsafe fn cvt_pwrite64(fd: c_int, buf: *const c_void, count: usize, offset: i64) - -> io::Result - { - use crate::convert::TryInto; - use libc::pwrite64; - // pwrite64 on emscripten actually takes a 32 bit offset - if let Ok(o) = offset.try_into() { - cvt(pwrite64(fd, buf, count, o)) - } else { - Err(io::Error::new(io::ErrorKind::InvalidInput, - "cannot pwrite >2GB")) - } - } - - #[cfg(not(any(target_os = "android", target_os = "emscripten")))] + #[cfg(not(target_os = "android"))] unsafe fn cvt_pwrite64(fd: c_int, buf: *const c_void, count: usize, offset: i64) -> io::Result { diff --git a/src/libstd/sys/unix/fs.rs b/src/libstd/sys/unix/fs.rs index 3b1eb86b84fe1..fdfc350995d08 100644 --- a/src/libstd/sys/unix/fs.rs +++ b/src/libstd/sys/unix/fs.rs @@ -602,8 +602,6 @@ impl File { SeekFrom::End(off) => (libc::SEEK_END, off), SeekFrom::Current(off) => (libc::SEEK_CUR, off), }; - #[cfg(target_os = "emscripten")] - let pos = pos as i32; let n = cvt(unsafe { lseek64(self.0.raw(), pos, whence) })?; Ok(n as u64) } From 11011013f2b609b6cf58c96555b9e31dfd19d7ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20K=C3=BCber?= Date: Thu, 17 Oct 2019 19:00:19 -0700 Subject: [PATCH 33/43] Refer to "associated functions" instead of "static methods" --- src/librustc_resolve/error_codes.rs | 22 ++++++++++++++-------- src/librustc_resolve/late/diagnostics.rs | 22 ++++++++++------------ src/libsyntax_ext/deriving/clone.rs | 22 +++++++++------------- src/libsyntax_ext/deriving/default.rs | 2 +- src/libsyntax_ext/deriving/generic/mod.rs | 4 +--- src/test/ui/error-codes/E0424.stderr | 2 +- src/test/ui/resolve/issue-2356.stderr | 4 ++-- 7 files changed, 38 insertions(+), 40 deletions(-) diff --git a/src/librustc_resolve/error_codes.rs b/src/librustc_resolve/error_codes.rs index ab3d95dd8edfe..29ca797fc065c 100644 --- a/src/librustc_resolve/error_codes.rs +++ b/src/librustc_resolve/error_codes.rs @@ -1013,7 +1013,12 @@ fn h1() -> i32 { "##, E0424: r##" -The `self` keyword was used in a static method. +The `self` keyword was used inside of an associated function instead of inside +of a method. Associated functions have no "`self` receiver" argument, and are +equivalent to regular functions which exist in the namespace of a trait. +Methods, on the other hand, have a `self` reciver argument, like `self`, +`&self`, `&mut self` or `self: &mut Pin` (this last one is an example of +an ["abitrary `self` type"](https://github.com/rust-lang/rust/issues/44874)). Erroneous code example: @@ -1021,25 +1026,26 @@ Erroneous code example: struct Foo; impl Foo { - fn bar(self) {} + // `bar` is a method, because it has a receiver argument. + fn bar(&self) {} + // `foo` is an associated function, because it has no receiver argument. fn foo() { - self.bar(); // error: `self` is not available in a static method. + self.bar(); // error: `self` is not available in an associated function } } ``` -Please check if the method's argument list should have contained `self`, -`&self`, or `&mut self` (in case you didn't want to create a static -method), and add it if so. Example: +Check if the associated function's argument list should have contained a `self` +receiver for it to be a method, and add it if so. Example: ``` struct Foo; impl Foo { - fn bar(self) {} + fn bar(&self) {} - fn foo(self) { + fn foo(self) { // `foo` is now a method. self.bar(); // ok! } } diff --git a/src/librustc_resolve/late/diagnostics.rs b/src/librustc_resolve/late/diagnostics.rs index 412734eabe05b..dba4226e98363 100644 --- a/src/librustc_resolve/late/diagnostics.rs +++ b/src/librustc_resolve/late/diagnostics.rs @@ -115,8 +115,10 @@ impl<'a> LateResolutionVisitor<'a, '_> { if is_self_type(path, ns) { syntax::diagnostic_used!(E0411); err.code(DiagnosticId::Error("E0411".into())); - err.span_label(span, format!("`Self` is only available in impls, traits, \ - and type definitions")); + err.span_label( + span, + format!("`Self` is only available in impls, traits, and type definitions"), + ); return (err, Vec::new()); } if is_self_value(path, ns) { @@ -125,16 +127,12 @@ impl<'a> LateResolutionVisitor<'a, '_> { syntax::diagnostic_used!(E0424); err.code(DiagnosticId::Error("E0424".into())); err.span_label(span, match source { - PathSource::Pat => { - format!("`self` value is a keyword \ - and may not be bound to \ - variables or shadowed") - } - _ => { - format!("`self` value is a keyword \ - only available in methods \ - with `self` parameter") - } + PathSource::Pat => format!( + "`self` value is a keyword and may not be bound to variables or shadowed", + ), + _ => format!( + "`self` value is a keyword only available in methods with a `self` parameter", + ), }); return (err, Vec::new()); } diff --git a/src/libsyntax_ext/deriving/clone.rs b/src/libsyntax_ext/deriving/clone.rs index eb7d480aa9831..67ef69babdc0a 100644 --- a/src/libsyntax_ext/deriving/clone.rs +++ b/src/libsyntax_ext/deriving/clone.rs @@ -174,14 +174,12 @@ fn cs_clone(name: &str, all_fields = af; vdata = &variant.data; } - EnumNonMatchingCollapsed(..) => { - cx.span_bug(trait_span, - &format!("non-matching enum variants in \ - `derive({})`", - name)) - } + EnumNonMatchingCollapsed(..) => cx.span_bug(trait_span, &format!( + "non-matching enum variants in `derive({})`", + name, + )), StaticEnum(..) | StaticStruct(..) => { - cx.span_bug(trait_span, &format!("static method in `derive({})`", name)) + cx.span_bug(trait_span, &format!("associated function in `derive({})`", name)) } } @@ -191,12 +189,10 @@ fn cs_clone(name: &str, .map(|field| { let ident = match field.name { Some(i) => i, - None => { - cx.span_bug(trait_span, - &format!("unnamed field in normal struct in \ - `derive({})`", - name)) - } + None => cx.span_bug(trait_span, &format!( + "unnamed field in normal struct in `derive({})`", + name, + )), }; let call = subcall(cx, field); cx.field_imm(field.span, ident, call) diff --git a/src/libsyntax_ext/deriving/default.rs b/src/libsyntax_ext/deriving/default.rs index 6176791c31b18..cfc0f3cd6cbf9 100644 --- a/src/libsyntax_ext/deriving/default.rs +++ b/src/libsyntax_ext/deriving/default.rs @@ -75,6 +75,6 @@ fn default_substructure(cx: &mut ExtCtxt<'_>, // let compilation continue DummyResult::raw_expr(trait_span, true) } - _ => cx.span_bug(trait_span, "Non-static method in `derive(Default)`"), + _ => cx.span_bug(trait_span, "method in `derive(Default)`"), }; } diff --git a/src/libsyntax_ext/deriving/generic/mod.rs b/src/libsyntax_ext/deriving/generic/mod.rs index 1886a5154b7b6..fd2c1552d32a8 100644 --- a/src/libsyntax_ext/deriving/generic/mod.rs +++ b/src/libsyntax_ext/deriving/generic/mod.rs @@ -1055,9 +1055,7 @@ impl<'a> MethodDef<'a> { }) .collect() } else { - cx.span_bug(trait_.span, - "no self arguments to non-static method in generic \ - `derive`") + cx.span_bug(trait_.span, "no self arguments for method in generic `derive`") }; // body of the inner most destructuring match diff --git a/src/test/ui/error-codes/E0424.stderr b/src/test/ui/error-codes/E0424.stderr index d67a2660dac38..99b5e01abb192 100644 --- a/src/test/ui/error-codes/E0424.stderr +++ b/src/test/ui/error-codes/E0424.stderr @@ -2,7 +2,7 @@ error[E0424]: expected value, found module `self` --> $DIR/E0424.rs:7:9 | LL | self.bar(); - | ^^^^ `self` value is a keyword only available in methods with `self` parameter + | ^^^^ `self` value is a keyword only available in methods with a `self` parameter error[E0424]: expected unit struct/variant or constant, found module `self` --> $DIR/E0424.rs:12:9 diff --git a/src/test/ui/resolve/issue-2356.stderr b/src/test/ui/resolve/issue-2356.stderr index 7790383843e17..e0a2088ab8b98 100644 --- a/src/test/ui/resolve/issue-2356.stderr +++ b/src/test/ui/resolve/issue-2356.stderr @@ -62,7 +62,7 @@ error[E0424]: expected value, found module `self` --> $DIR/issue-2356.rs:65:8 | LL | if self.whiskers > 3 { - | ^^^^ `self` value is a keyword only available in methods with `self` parameter + | ^^^^ `self` value is a keyword only available in methods with a `self` parameter error[E0425]: cannot find function `grow_older` in this scope --> $DIR/issue-2356.rs:72:5 @@ -98,7 +98,7 @@ error[E0424]: expected value, found module `self` --> $DIR/issue-2356.rs:92:5 | LL | self += 1; - | ^^^^ `self` value is a keyword only available in methods with `self` parameter + | ^^^^ `self` value is a keyword only available in methods with a `self` parameter error: aborting due to 17 previous errors From d8fca9ee4ea9399dc8244f744f0198821d233ecf Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 18 Oct 2019 09:24:56 +1100 Subject: [PATCH 34/43] Use `with` in `Symbol` trait methods. Instead of `as_str()`, which unnecessarily involves `LocalInternedString`. --- src/libsyntax_pos/symbol.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libsyntax_pos/symbol.rs b/src/libsyntax_pos/symbol.rs index 8845b66a7cefd..e4f309259e7c1 100644 --- a/src/libsyntax_pos/symbol.rs +++ b/src/libsyntax_pos/symbol.rs @@ -934,19 +934,19 @@ impl Symbol { impl fmt::Debug for Symbol { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, f) + self.with(|str| fmt::Display::fmt(&str, f)) } } impl fmt::Display for Symbol { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.as_str(), f) + self.with(|str| fmt::Display::fmt(&str, f)) } } impl Encodable for Symbol { fn encode(&self, s: &mut S) -> Result<(), S::Error> { - s.emit_str(&self.as_str()) + self.with(|string| s.emit_str(string)) } } From 3532863a96499df049589aa5120db149498ab544 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 18 Oct 2019 10:06:23 +1100 Subject: [PATCH 35/43] Change how `Symbol::Debug` works. Currently, `Symbol::Debug` and `Symbol::Display` produce the same output; neither wraps the symbol in double quotes. This commit changes `Symbol::Debug` so it wraps the symbol in quotes. This change brings `Symbol`'s behaviour in line with `String` and `InternedString`. The change requires a couple of trivial test output adjustments. --- src/libsyntax_pos/symbol.rs | 2 +- src/test/ui/hygiene/unpretty-debug.stdout | 2 +- .../lint/redundant-semicolon/redundant-semi-proc-macro.stderr | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libsyntax_pos/symbol.rs b/src/libsyntax_pos/symbol.rs index e4f309259e7c1..efb01ff84dcda 100644 --- a/src/libsyntax_pos/symbol.rs +++ b/src/libsyntax_pos/symbol.rs @@ -934,7 +934,7 @@ impl Symbol { impl fmt::Debug for Symbol { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.with(|str| fmt::Display::fmt(&str, f)) + self.with(|str| fmt::Debug::fmt(&str, f)) } } diff --git a/src/test/ui/hygiene/unpretty-debug.stdout b/src/test/ui/hygiene/unpretty-debug.stdout index 6971873ba601e..acd852103cae3 100644 --- a/src/test/ui/hygiene/unpretty-debug.stdout +++ b/src/test/ui/hygiene/unpretty-debug.stdout @@ -17,7 +17,7 @@ fn y /* 0#0 */() { } /* Expansions: 0: parent: ExpnId(0), call_site_ctxt: #0, kind: Root -1: parent: ExpnId(0), call_site_ctxt: #0, kind: Macro(Bang, foo) +1: parent: ExpnId(0), call_site_ctxt: #0, kind: Macro(Bang, "foo") SyntaxContexts: #0: parent: #0, outer_mark: (ExpnId(0), Opaque) diff --git a/src/test/ui/lint/redundant-semicolon/redundant-semi-proc-macro.stderr b/src/test/ui/lint/redundant-semicolon/redundant-semi-proc-macro.stderr index 5f289c0914d6e..2160df51a8375 100644 --- a/src/test/ui/lint/redundant-semicolon/redundant-semi-proc-macro.stderr +++ b/src/test/ui/lint/redundant-semicolon/redundant-semi-proc-macro.stderr @@ -1,4 +1,4 @@ -TokenStream [Ident { ident: "fn", span: #0 bytes(197..199) }, Ident { ident: "span_preservation", span: #0 bytes(200..217) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(217..219) }, Group { delimiter: Brace, stream: TokenStream [Ident { ident: "let", span: #0 bytes(227..230) }, Ident { ident: "tst", span: #0 bytes(231..234) }, Punct { ch: '=', spacing: Alone, span: #0 bytes(235..236) }, Literal { lit: Lit { kind: Integer, symbol: 123, suffix: None }, span: Span { lo: BytePos(237), hi: BytePos(240), ctxt: #0 } }, Punct { ch: ';', spacing: Joint, span: #0 bytes(240..241) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(241..242) }, Ident { ident: "match", span: #0 bytes(288..293) }, Ident { ident: "tst", span: #0 bytes(294..297) }, Group { delimiter: Brace, stream: TokenStream [Literal { lit: Lit { kind: Integer, symbol: 123, suffix: None }, span: Span { lo: BytePos(482), hi: BytePos(485), ctxt: #0 } }, Punct { ch: '=', spacing: Joint, span: #0 bytes(486..488) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(486..488) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(489..491) }, Punct { ch: ',', spacing: Alone, span: #0 bytes(491..492) }, Ident { ident: "_", span: #0 bytes(501..502) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(503..505) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(503..505) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(506..508) }], span: #0 bytes(298..514) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(514..515) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(515..516) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(516..517) }], span: #0 bytes(221..561) }] +TokenStream [Ident { ident: "fn", span: #0 bytes(197..199) }, Ident { ident: "span_preservation", span: #0 bytes(200..217) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(217..219) }, Group { delimiter: Brace, stream: TokenStream [Ident { ident: "let", span: #0 bytes(227..230) }, Ident { ident: "tst", span: #0 bytes(231..234) }, Punct { ch: '=', spacing: Alone, span: #0 bytes(235..236) }, Literal { lit: Lit { kind: Integer, symbol: "123", suffix: None }, span: Span { lo: BytePos(237), hi: BytePos(240), ctxt: #0 } }, Punct { ch: ';', spacing: Joint, span: #0 bytes(240..241) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(241..242) }, Ident { ident: "match", span: #0 bytes(288..293) }, Ident { ident: "tst", span: #0 bytes(294..297) }, Group { delimiter: Brace, stream: TokenStream [Literal { lit: Lit { kind: Integer, symbol: "123", suffix: None }, span: Span { lo: BytePos(482), hi: BytePos(485), ctxt: #0 } }, Punct { ch: '=', spacing: Joint, span: #0 bytes(486..488) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(486..488) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(489..491) }, Punct { ch: ',', spacing: Alone, span: #0 bytes(491..492) }, Ident { ident: "_", span: #0 bytes(501..502) }, Punct { ch: '=', spacing: Joint, span: #0 bytes(503..505) }, Punct { ch: '>', spacing: Alone, span: #0 bytes(503..505) }, Group { delimiter: Parenthesis, stream: TokenStream [], span: #0 bytes(506..508) }], span: #0 bytes(298..514) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(514..515) }, Punct { ch: ';', spacing: Joint, span: #0 bytes(515..516) }, Punct { ch: ';', spacing: Alone, span: #0 bytes(516..517) }], span: #0 bytes(221..561) }] error: unnecessary trailing semicolon --> $DIR/redundant-semi-proc-macro.rs:9:19 | From f65a492afcd9cd892a1a5591f938efd41b5e0d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20K=C3=BCber?= Date: Thu, 17 Oct 2019 20:26:21 -0700 Subject: [PATCH 36/43] Point at enclosing function without `self` receiver --- src/librustc_resolve/late.rs | 8 +++++++- src/librustc_resolve/late/diagnostics.rs | 3 +++ src/test/ui/error-codes/E0424.stderr | 14 ++++++++++---- src/test/ui/resolve/issue-2356.stderr | 18 ++++++++++++++---- 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/librustc_resolve/late.rs b/src/librustc_resolve/late.rs index bb9f895c5f39b..73a282b1a0ec1 100644 --- a/src/librustc_resolve/late.rs +++ b/src/librustc_resolve/late.rs @@ -345,6 +345,9 @@ struct LateResolutionVisitor<'a, 'b> { /// The current self item if inside an ADT (used for better errors). current_self_item: Option, + /// The current enclosing funciton (used for better errors). + current_function: Option, + /// A list of labels as of yet unused. Labels will be removed from this map when /// they are used (in a `break` or `continue` statement) unused_labels: FxHashMap, @@ -415,7 +418,8 @@ impl<'a, 'tcx> Visitor<'tcx> for LateResolutionVisitor<'a, '_> { } } } - fn visit_fn(&mut self, fn_kind: FnKind<'tcx>, declaration: &'tcx FnDecl, _: Span, _: NodeId) { + fn visit_fn(&mut self, fn_kind: FnKind<'tcx>, declaration: &'tcx FnDecl, sp: Span, _: NodeId) { + let previous_value = replace(&mut self.current_function, Some(sp)); debug!("(resolving function) entering function"); let rib_kind = match fn_kind { FnKind::ItemFn(..) => FnItemRibKind, @@ -441,6 +445,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LateResolutionVisitor<'a, '_> { debug!("(resolving function) leaving function"); }) }); + self.current_function = previous_value; } fn visit_generics(&mut self, generics: &'tcx Generics) { @@ -546,6 +551,7 @@ impl<'a, 'b> LateResolutionVisitor<'a, '_> { current_trait_assoc_types: Vec::new(), current_self_type: None, current_self_item: None, + current_function: None, unused_labels: Default::default(), current_type_ascription: Vec::new(), } diff --git a/src/librustc_resolve/late/diagnostics.rs b/src/librustc_resolve/late/diagnostics.rs index dba4226e98363..2721df4c68763 100644 --- a/src/librustc_resolve/late/diagnostics.rs +++ b/src/librustc_resolve/late/diagnostics.rs @@ -134,6 +134,9 @@ impl<'a> LateResolutionVisitor<'a, '_> { "`self` value is a keyword only available in methods with a `self` parameter", ), }); + if let Some(span) = &self.current_function { + err.span_label(*span, "this function doesn't have a `self` parameter"); + } return (err, Vec::new()); } diff --git a/src/test/ui/error-codes/E0424.stderr b/src/test/ui/error-codes/E0424.stderr index 99b5e01abb192..567d1b3cc75f4 100644 --- a/src/test/ui/error-codes/E0424.stderr +++ b/src/test/ui/error-codes/E0424.stderr @@ -1,14 +1,20 @@ error[E0424]: expected value, found module `self` --> $DIR/E0424.rs:7:9 | -LL | self.bar(); - | ^^^^ `self` value is a keyword only available in methods with a `self` parameter +LL | / fn foo() { +LL | | self.bar(); + | | ^^^^ `self` value is a keyword only available in methods with a `self` parameter +LL | | } + | |_____- this function doesn't have a `self` parameter error[E0424]: expected unit struct/variant or constant, found module `self` --> $DIR/E0424.rs:12:9 | -LL | let self = "self"; - | ^^^^ `self` value is a keyword and may not be bound to variables or shadowed +LL | / fn main () { +LL | | let self = "self"; + | | ^^^^ `self` value is a keyword and may not be bound to variables or shadowed +LL | | } + | |_- this function doesn't have a `self` parameter error: aborting due to 2 previous errors diff --git a/src/test/ui/resolve/issue-2356.stderr b/src/test/ui/resolve/issue-2356.stderr index e0a2088ab8b98..329543114a610 100644 --- a/src/test/ui/resolve/issue-2356.stderr +++ b/src/test/ui/resolve/issue-2356.stderr @@ -61,8 +61,14 @@ LL | purr(); error[E0424]: expected value, found module `self` --> $DIR/issue-2356.rs:65:8 | -LL | if self.whiskers > 3 { - | ^^^^ `self` value is a keyword only available in methods with a `self` parameter +LL | / fn meow() { +LL | | if self.whiskers > 3 { + | | ^^^^ `self` value is a keyword only available in methods with a `self` parameter +LL | | +LL | | println!("MEOW"); +LL | | } +LL | | } + | |___- this function doesn't have a `self` parameter error[E0425]: cannot find function `grow_older` in this scope --> $DIR/issue-2356.rs:72:5 @@ -97,8 +103,12 @@ LL | purr_louder(); error[E0424]: expected value, found module `self` --> $DIR/issue-2356.rs:92:5 | -LL | self += 1; - | ^^^^ `self` value is a keyword only available in methods with a `self` parameter +LL | / fn main() { +LL | | self += 1; + | | ^^^^ `self` value is a keyword only available in methods with a `self` parameter +LL | | +LL | | } + | |_- this function doesn't have a `self` parameter error: aborting due to 17 previous errors From 0879f630741550f0d4989e908071de0bb05a50f1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 18 Oct 2019 16:05:14 +1100 Subject: [PATCH 37/43] Remove `Copy` and `Clone` impls for `LocalInternedString`. They aren't used. --- src/libsyntax_pos/symbol.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libsyntax_pos/symbol.rs b/src/libsyntax_pos/symbol.rs index efb01ff84dcda..fa9567fb62c0f 100644 --- a/src/libsyntax_pos/symbol.rs +++ b/src/libsyntax_pos/symbol.rs @@ -1106,8 +1106,8 @@ fn with_interner T>(f: F) -> T { } /// An alternative to `Symbol` and `InternedString`, useful when the chars -/// within the symbol need to be accessed. It is best used for temporary -/// values. +/// within the symbol need to be accessed. It deliberately has limited +/// functionality and should only be used for temporary values. /// /// Because the interner outlives any thread which uses this type, we can /// safely treat `string` which points to interner data, as an immortal string, @@ -1116,7 +1116,7 @@ fn with_interner T>(f: F) -> T { // FIXME: ensure that the interner outlives any thread which uses // `LocalInternedString`, by creating a new thread right after constructing the // interner. -#[derive(Clone, Copy, Eq, PartialOrd, Ord)] +#[derive(Eq, PartialOrd, Ord)] pub struct LocalInternedString { string: &'static str, } From d343ee839b6fe26c992d0609281443776839039c Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Fri, 18 Oct 2019 16:36:17 +1100 Subject: [PATCH 38/43] Remove `Hash` impls for `DefPath`, `DisambiguatedDefPathData`, and `DefKey`. They aren't used. --- src/librustc/hir/map/definitions.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/librustc/hir/map/definitions.rs b/src/librustc/hir/map/definitions.rs index cc099fcc40f6d..d2732c92d2688 100644 --- a/src/librustc/hir/map/definitions.rs +++ b/src/librustc/hir/map/definitions.rs @@ -111,7 +111,7 @@ pub struct Definitions { /// A unique identifier that we can use to lookup a definition /// precisely. It combines the index of the definition's parent (if /// any) with a `DisambiguatedDefPathData`. -#[derive(Clone, PartialEq, Debug, Hash, RustcEncodable, RustcDecodable)] +#[derive(Clone, PartialEq, Debug, RustcEncodable, RustcDecodable)] pub struct DefKey { /// The parent path. pub parent: Option, @@ -162,13 +162,13 @@ impl DefKey { /// between them. This introduces some artificial ordering dependency /// but means that if you have, e.g., two impls for the same type in /// the same module, they do get distinct `DefId`s. -#[derive(Clone, PartialEq, Debug, Hash, RustcEncodable, RustcDecodable)] +#[derive(Clone, PartialEq, Debug, RustcEncodable, RustcDecodable)] pub struct DisambiguatedDefPathData { pub data: DefPathData, pub disambiguator: u32 } -#[derive(Clone, Debug, Hash, RustcEncodable, RustcDecodable)] +#[derive(Clone, Debug, RustcEncodable, RustcDecodable)] pub struct DefPath { /// The path leading from the crate root to the item. pub data: Vec, From 375e34d9db0f83b42242c9aa78124de1fe713ef6 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 18 Oct 2019 16:32:58 +0200 Subject: [PATCH 39/43] ci: fix wrong path being set in install-msys2.sh --- src/ci/scripts/install-msys2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh index c5ad8a79cea8b..8b9a55fb17972 100755 --- a/src/ci/scripts/install-msys2.sh +++ b/src/ci/scripts/install-msys2.sh @@ -16,7 +16,7 @@ if isWindows; then mkdir -p "${SYSTEM_WORKFOLDER}/msys2/home/${USERNAME}" ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/usr/bin" - export PATH="${SYSTEM_WORKFOLDER}/msys2/usr/bin" + export PATH="${SYSTEM_WORKFOLDER}/msys2/usr/bin:${PATH}" pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar From 865c4bcff6b52f69b4122e3a0f02f647cf588553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20K=C3=BCber?= Date: Fri, 18 Oct 2019 08:36:46 -0700 Subject: [PATCH 40/43] review comments: help wording --- src/librustc_resolve/error_codes.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/librustc_resolve/error_codes.rs b/src/librustc_resolve/error_codes.rs index 29ca797fc065c..ee207025e0c32 100644 --- a/src/librustc_resolve/error_codes.rs +++ b/src/librustc_resolve/error_codes.rs @@ -1013,12 +1013,12 @@ fn h1() -> i32 { "##, E0424: r##" -The `self` keyword was used inside of an associated function instead of inside -of a method. Associated functions have no "`self` receiver" argument, and are -equivalent to regular functions which exist in the namespace of a trait. -Methods, on the other hand, have a `self` reciver argument, like `self`, -`&self`, `&mut self` or `self: &mut Pin` (this last one is an example of -an ["abitrary `self` type"](https://github.com/rust-lang/rust/issues/44874)). +The `self` keyword was used inside of an associated function without a "`self` +receiver" parameter. The `self` keyword can only be used inside methods, which +are associated functions (functions defined inside of a `trait` or `impl` block) +that have a `self` receiver as its first parameter, like `self`, `&self`, +`&mut self` or `self: &mut Pin` (this last one is an example of an +["abitrary `self` type"](https://github.com/rust-lang/rust/issues/44874)). Erroneous code example: @@ -1026,17 +1026,18 @@ Erroneous code example: struct Foo; impl Foo { - // `bar` is a method, because it has a receiver argument. + // `bar` is a method, because it has a receiver parameter. fn bar(&self) {} - // `foo` is an associated function, because it has no receiver argument. + // `foo` is not a method, because it has no receiver parameter. fn foo() { - self.bar(); // error: `self` is not available in an associated function + self.bar(); // error: `self` value is a keyword only available in + // methods with a `self` parameter } } ``` -Check if the associated function's argument list should have contained a `self` +Check if the associated function's parameter list should have contained a `self` receiver for it to be a method, and add it if so. Example: ``` From bd813bf1acbef7643a06278af48603397c7eb5c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20K=C3=BCber?= Date: Fri, 18 Oct 2019 08:38:08 -0700 Subject: [PATCH 41/43] review comment: span bug label --- src/libsyntax_ext/deriving/generic/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libsyntax_ext/deriving/generic/mod.rs b/src/libsyntax_ext/deriving/generic/mod.rs index fd2c1552d32a8..216338c1a8861 100644 --- a/src/libsyntax_ext/deriving/generic/mod.rs +++ b/src/libsyntax_ext/deriving/generic/mod.rs @@ -1055,7 +1055,7 @@ impl<'a> MethodDef<'a> { }) .collect() } else { - cx.span_bug(trait_.span, "no self arguments for method in generic `derive`") + cx.span_bug(trait_.span, "no `self` parameter for method in generic `derive`") }; // body of the inner most destructuring match From 2b76c8b95ff866d8806fdc82fe270c4df02efc0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20K=C3=BCber?= Date: Fri, 18 Oct 2019 13:00:14 -0700 Subject: [PATCH 42/43] review comments --- src/librustc_resolve/error_codes.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/librustc_resolve/error_codes.rs b/src/librustc_resolve/error_codes.rs index ee207025e0c32..2288aab3c27a6 100644 --- a/src/librustc_resolve/error_codes.rs +++ b/src/librustc_resolve/error_codes.rs @@ -1014,11 +1014,7 @@ fn h1() -> i32 { E0424: r##" The `self` keyword was used inside of an associated function without a "`self` -receiver" parameter. The `self` keyword can only be used inside methods, which -are associated functions (functions defined inside of a `trait` or `impl` block) -that have a `self` receiver as its first parameter, like `self`, `&self`, -`&mut self` or `self: &mut Pin` (this last one is an example of an -["abitrary `self` type"](https://github.com/rust-lang/rust/issues/44874)). +receiver" parameter. Erroneous code example: @@ -1037,6 +1033,12 @@ impl Foo { } ``` +The `self` keyword can only be used inside methods, which are associated +functions (functions defined inside of a `trait` or `impl` block) that have a +`self` receiver as its first parameter, like `self`, `&self`, `&mut self` or +`self: &mut Pin` (this last one is an example of an ["abitrary `self` +type"](https://github.com/rust-lang/rust/issues/44874)). + Check if the associated function's parameter list should have contained a `self` receiver for it to be a method, and add it if so. Example: From 227db40a98e5bd903aa3658c16f19a3d6f694deb Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Wed, 16 Oct 2019 20:33:55 +0200 Subject: [PATCH 43/43] Uninitialized boxes: add test for zero-size allocations --- src/liballoc/tests/boxed.rs | 18 ++++++++++++++++++ src/liballoc/tests/lib.rs | 2 ++ 2 files changed, 20 insertions(+) create mode 100644 src/liballoc/tests/boxed.rs diff --git a/src/liballoc/tests/boxed.rs b/src/liballoc/tests/boxed.rs new file mode 100644 index 0000000000000..bc3d53bf30da3 --- /dev/null +++ b/src/liballoc/tests/boxed.rs @@ -0,0 +1,18 @@ +use std::ptr::NonNull; +use std::mem::MaybeUninit; + +#[test] +fn unitialized_zero_size_box() { + assert_eq!( + &*Box::<()>::new_uninit() as *const _, + NonNull::>::dangling().as_ptr(), + ); + assert_eq!( + Box::<[()]>::new_uninit_slice(4).as_ptr(), + NonNull::>::dangling().as_ptr(), + ); + assert_eq!( + Box::<[String]>::new_uninit_slice(0).as_ptr(), + NonNull::>::dangling().as_ptr(), + ); +} diff --git a/src/liballoc/tests/lib.rs b/src/liballoc/tests/lib.rs index 5723a30c0f34f..59f5c8dfb8a87 100644 --- a/src/liballoc/tests/lib.rs +++ b/src/liballoc/tests/lib.rs @@ -2,6 +2,7 @@ #![feature(box_syntax)] #![feature(drain_filter)] #![feature(exact_size_is_empty)] +#![feature(new_uninit)] #![feature(option_flattening)] #![feature(pattern)] #![feature(repeat_generic_slice)] @@ -15,6 +16,7 @@ use std::collections::hash_map::DefaultHasher; mod arc; mod binary_heap; +mod boxed; mod btree; mod cow_str; mod fmt;