Retry in all curl calls.

cross-rs · Jun 30, 2020 · 8a4ad26 · 8a4ad26
1 parent 6126ef0
commit 8a4ad26
Show file tree

Hide file tree

Showing 13 changed files with 51 additions and 43 deletions.
diff --git a/ci/azure-install-rust.yml b/ci/azure-install-rust.yml
@@ -13,7 +13,9 @@ steps:
           rustup self update
         fi
       else
-        curl -sSL https://sh.rustup.rs | sh -s -- -y --default-toolchain="$TOOLCHAIN" --profile=minimal
+        curl --retry 3 -sSfL https://sh.rustup.rs -o rustup-init.sh
+        sh rustup-init.sh -y --default-toolchain="$TOOLCHAIN" --profile=minimal
+        rm rustup-init.sh
         echo "##vso[task.prependpath]$HOME/.cargo/bin"
       fi
     displayName: Install rustup

diff --git a/docker/Dockerfile.arm-unknown-linux-gnueabihf b/docker/Dockerfile.arm-unknown-linux-gnueabihf
@@ -9,10 +9,12 @@ RUN /cmake.sh
 COPY xargo.sh /
 RUN /xargo.sh
 
-RUN mkdir /usr/arm-linux-gnueabihf && \
-    cd /usr/arm-linux-gnueabihf && \
-    curl -L https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-arm-linux-gnueabihf.tar.xz | \
-    tar --strip-components 1 -xJ
+RUN mkdir /usr/arm-linux-gnueabihf \
+ && cd /usr/arm-linux-gnueabihf \
+ && export TOOLCHAIN_VERSION=8.3-2019.03 \
+ && curl --retry 3 -sSfL https://developer.arm.com/-/media/Files/downloads/gnu-a/${TOOLCHAIN_VERSION}/binrel/gcc-arm-${TOOLCHAIN_VERSION}-x86_64-arm-linux-gnueabihf.tar.xz -O \
+ && tar --strip-components 1 -xJf gcc-arm-${TOOLCHAIN_VERSION}-x86_64-arm-linux-gnueabihf.tar.xz \
+ && rm gcc-arm-${TOOLCHAIN_VERSION}-x86_64-arm-linux-gnueabihf.tar.xz
 
 ENV PATH /usr/arm-linux-gnueabihf/bin:$PATH
 

diff --git a/docker/android-ndk.sh b/docker/android-ndk.sh
@@ -28,7 +28,7 @@ main() {
     td="$(mktemp -d)"
 
     pushd "${td}"
-    curl -O "${NDK_URL}"
+    curl --retry 3 -sSfL "${NDK_URL}" -O
     unzip -q android-ndk-*.zip
     pushd android-ndk-*
     ./build/tools/make_standalone_toolchain.py \

diff --git a/docker/android-system.sh b/docker/android-system.sh
@@ -50,7 +50,7 @@ EOF
         fi
     done
 
-    curl -O https://storage.googleapis.com/git-repo-downloads/repo
+    curl --retry 3 -sSfL https://storage.googleapis.com/git-repo-downloads/repo -O
     chmod +x repo
 
     # this is the minimum set of modules that are need to build bionic

diff --git a/docker/cmake.sh b/docker/cmake.sh
@@ -22,7 +22,7 @@ main() {
 
     pushd "${td}"
 
-    curl -sSfL "https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-Linux-x86_64.sh" -o cmake.sh
+    curl --retry 3 -sSfL "https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-Linux-x86_64.sh" -o cmake.sh
     sh cmake.sh --skip-license --prefix=/usr/local
 
     popd

diff --git a/docker/disabled/dragonfly.sh b/docker/disabled/dragonfly.sh
@@ -35,11 +35,11 @@ main() {
 
     mkdir "${td}"/{binutils,gcc}{,-build} "${td}/dragonfly"
 
-    curl "https://ftp.gnu.org/gnu/binutils/binutils-${binutils}.tar.bz2" | \
-        tar -C "${td}/binutils" --strip-components=1 -xj
+    curl --retry 3 -sSfL "https://ftp.gnu.org/gnu/binutils/binutils-${binutils}.tar.bz2" -O
+    tar -C "${td}/binutils" --strip-components=1 -xjf "binutils-${binutils}.tar.bz2"
 
-    curl "https://ftp.gnu.org/gnu/gcc/gcc-${gcc}/gcc-${gcc}.tar.bz2" | \
-        tar -C "${td}/gcc" --strip-components=1 -xj
+    curl --retry 3 -sSfL "https://ftp.gnu.org/gnu/gcc/gcc-${gcc}/gcc-${gcc}.tar.bz2" -O
+    tar -C "${td}/gcc" --strip-components=1 -xjf "gcc-${gcc}.tar.bz2"
 
     pushd "${td}"
 
@@ -90,9 +90,8 @@ EOF
 EOF
     cd ..
 
-    curl "https://mirror-master.dragonflybsd.org/iso-images/dfly-x86_64-${dragonfly}.iso.bz2" | \
-        bzcat | \
-        bsdtar xf - -C "${td}/dragonfly" ./usr/include ./usr/lib ./lib
+    curl --retry 3 -sSfL "https://mirror-master.dragonflybsd.org/iso-images/dfly-x86_64-${dragonfly}.iso.bz2" -O
+    bzcat "dfly-x86_64-${dragonfly}.iso.bz2" | bsdtar xf - -C "${td}/dragonfly" ./usr/include ./usr/lib ./lib
 
     cd binutils-build
     ../binutils/configure \

diff --git a/docker/dropbear.sh b/docker/dropbear.sh
@@ -29,8 +29,8 @@ main() {
 
     pushd "${td}"
 
-    curl -L "https://matt.ucc.asn.au/dropbear/dropbear-${version}.tar.bz2" | \
-        tar --strip-components=1 -xj
+    curl --retry 3 -sSfL "https://matt.ucc.asn.au/dropbear/dropbear-${version}.tar.bz2" -O
+    tar --strip-components=1 -xjf "dropbear-${version}.tar.bz2"
 
     # Remove some unwanted message
     sed -i '/skipping hostkey/d' cli-kex.c

diff --git a/docker/freebsd.sh b/docker/freebsd.sh
@@ -34,11 +34,11 @@ main() {
 
     mkdir "${td}"/{binutils,gcc}{,-build} "${td}/freebsd"
 
-    curl "https://ftp.gnu.org/gnu/binutils/binutils-${binutils}.tar.bz2" | \
-        tar -C "${td}/binutils" --strip-components=1 -xj
+    curl --retry 3 -sSfL "https://ftp.gnu.org/gnu/binutils/binutils-${binutils}.tar.bz2" -O
+    tar -C "${td}/binutils" --strip-components=1 -xjf "binutils-${binutils}.tar.bz2"
 
-    curl "https://ftp.gnu.org/gnu/gcc/gcc-${gcc}/gcc-${gcc}.tar.bz2" | \
-        tar -C "${td}/gcc" --strip-components=1 -xj
+    curl --retry 3 -sSfL "https://ftp.gnu.org/gnu/gcc/gcc-${gcc}/gcc-${gcc}.tar.bz2" -O
+    tar -C "${td}/gcc" --strip-components=1 -xjf "gcc-${gcc}.tar.bz2"
 
     pushd "${td}"
 
@@ -57,8 +57,8 @@ main() {
             ;;
     esac
 
-    curl "http://ftp.freebsd.org/pub/FreeBSD/releases/${bsd_arch}/10.2-RELEASE/base.txz" | \
-        tar -C "${td}/freebsd" -xJ ./usr/include ./usr/lib ./lib
+    curl --retry 3 -sSfL "http://ftp.freebsd.org/pub/FreeBSD/releases/${bsd_arch}/10.2-RELEASE/base.txz" -O
+    tar -C "${td}/freebsd" -xJf base.txz ./usr/include ./usr/lib ./lib
 
     cd binutils-build
     ../binutils/configure \

diff --git a/docker/musl.sh b/docker/musl.sh
@@ -19,6 +19,8 @@ hide_output() {
 }
 
 main() {
+    local version=0.9.8
+
     local dependencies=(
         ca-certificates
         curl
@@ -38,13 +40,13 @@ main() {
     td="$(mktemp -d)"
 
     pushd "${td}"
-    curl -L https://github.com/richfelker/musl-cross-make/archive/v0.9.8.tar.gz | \
-        tar --strip-components=1 -xz
+    curl --retry 3 -sSfL "https://github.com/richfelker/musl-cross-make/archive/v${version}.tar.gz" -O
+    tar --strip-components=1 -xzf "v${version}.tar.gz"
 
     hide_output make install "-j$(nproc)" \
         GCC_VER=6.4.0 \
         MUSL_VER=1.1.22 \
-        DL_CMD='curl -C - -L -o' \
+        DL_CMD='curl --retry 3 -sSfL -C - -o' \
         OUTPUT=/usr/local/ \
         "${@}"
 

diff --git a/docker/netbsd.sh b/docker/netbsd.sh
@@ -33,11 +33,11 @@ main() {
 
     mkdir "${td}"/{binutils,gcc}{,-build} "${td}/netbsd"
 
-    curl "https://ftp.gnu.org/gnu/binutils/binutils-${binutils}.tar.bz2" | \
-        tar -C "${td}/binutils" --strip-components=1 -xj
+    curl --retry 3 -sSfL "https://ftp.gnu.org/gnu/binutils/binutils-${binutils}.tar.bz2" -O
+    tar -C "${td}/binutils" --strip-components=1 -xjf "binutils-${binutils}.tar.bz2"
 
-    curl "https://ftp.gnu.org/gnu/gcc/gcc-${gcc}/gcc-${gcc}.tar.bz2" | \
-        tar -C "${td}/gcc" --strip-components=1 -xj
+    curl --retry 3 -sSfL "https://ftp.gnu.org/gnu/gcc/gcc-${gcc}/gcc-${gcc}.tar.bz2" -O
+    tar -C "${td}/gcc" --strip-components=1 -xjf "gcc-${gcc}.tar.bz2"
 
     pushd "${td}"
 
@@ -51,16 +51,19 @@ main() {
 
     local patch
     for patch in "${patches[@]}"; do
-        patch="$(curl "${patch}")"
-        echo "${patch}" | patch -Np0
+        local patch_file
+        patch_file="$(mktemp)"
+        curl --retry 3 -sSfL "${patch}" -o "${patch_file}"
+        patch -Np0 < "${patch_file}"
+        rm "${patch_file}"
     done
     cd ..
 
-    curl ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-7.0/amd64/binary/sets/base.tgz | \
-        tar -C "${td}/netbsd" -xz ./usr/include ./usr/lib ./lib
+    curl --retry 3 -sSfL ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-7.0/amd64/binary/sets/base.tgz -O
+    tar -C "${td}/netbsd" -xzf base.tgz ./usr/include ./usr/lib ./lib
 
-    curl ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-7.0/amd64/binary/sets/comp.tgz | \
-        tar -C "${td}/netbsd" -xz ./usr/include ./usr/lib
+    curl --retry 3 -sSfL ftp://ftp.netbsd.org/pub/NetBSD/NetBSD-7.0/amd64/binary/sets/comp.tgz -O
+    tar -C "${td}/netbsd" -xzf comp.tgz ./usr/include ./usr/lib
 
     cd binutils-build
     ../binutils/configure \

diff --git a/docker/qemu.sh b/docker/qemu.sh
@@ -53,8 +53,8 @@ main() {
 
     pushd "${td}"
 
-    curl -L "https://download.qemu.org/qemu-${version}.tar.xz" | \
-        tar --strip-components=1 -xJ
+    curl --retry 3 -sSfL "https://download.qemu.org/qemu-${version}.tar.xz" -O
+    tar --strip-components=1 -xJf "qemu-${version}.tar.xz"
 
    local targets="${arch}-linux-user"
    local virtfs=""

diff --git a/docker/solaris.sh b/docker/solaris.sh
@@ -36,11 +36,11 @@ main() {
 
     mkdir "${td}"/{binutils,gcc}{,-build} "${td}/solaris"
 
-    curl "https://ftp.gnu.org/gnu/binutils/binutils-${binutils}.tar.xz" | \
-        tar -C "${td}/binutils" --strip-components=1 -xJ
+    curl --retry 3 -sSfL "https://ftp.gnu.org/gnu/binutils/binutils-${binutils}.tar.xz" -O
+    tar -C "${td}/binutils" --strip-components=1 -xJf "binutils-${binutils}.tar.xz"
 
-    curl "https://ftp.gnu.org/gnu/gcc/gcc-${gcc}/gcc-${gcc}.tar.xz" | \
-        tar -C "${td}/gcc" --strip-components=1 -xJ
+    curl --retry 3 -sSfL "https://ftp.gnu.org/gnu/gcc/gcc-${gcc}/gcc-${gcc}.tar.xz" -O
+    tar -C "${td}/gcc" --strip-components=1 -xJf "gcc-${gcc}.tar.xz"
 
     pushd "${td}"
 

diff --git a/docker/xargo.sh b/docker/xargo.sh
@@ -21,7 +21,7 @@ main() {
     export RUSTUP_HOME=/tmp/rustup
     export CARGO_HOME=/tmp/cargo
 
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o rustup-init.sh
+    curl --retry 3 -sSfL https://sh.rustup.rs -o rustup-init.sh
     sh rustup-init.sh -y --no-modify-path --profile minimal
     rm rustup-init.sh