diff --git a/src/usr/local/buildpack/tools/v2/node.sh b/src/usr/local/buildpack/tools/v2/node.sh index 8495e0ee3..5bcb3a329 100644 --- a/src/usr/local/buildpack/tools/v2/node.sh +++ b/src/usr/local/buildpack/tools/v2/node.sh @@ -39,9 +39,19 @@ function check_tool_requirements () { function install_tool () { local versioned_tool_path local npm # temp npm executable + local arch=linux-x64 + + checksums=$(get_from_url "https://nodejs.org/dist/v${TOOL_VERSION}/SHASUMS256.txt") + + # get checksum from file + original_checksum=$(grep "node-v${TOOL_VERSION}-${arch}.tar.xz" "${checksums}" | cut -d' ' -f1) # download file - file=$(get_from_url "https://nodejs.org/dist/v${TOOL_VERSION}/node-v${TOOL_VERSION}-linux-x64.tar.xz") + file=$(get_from_url \ + "https://nodejs.org/dist/v${TOOL_VERSION}/node-v${TOOL_VERSION}-${arch}.tar.xz" \ + "node-v${TOOL_VERSION}-${arch}.tar.xz" \ + "${original_checksum}" \ + "sha256sum" ) versioned_tool_path=$(create_versioned_tool_path) diff --git a/src/usr/local/buildpack/utils/cache.sh b/src/usr/local/buildpack/utils/cache.sh index fc791e3c7..84012c85b 100644 --- a/src/usr/local/buildpack/utils/cache.sh +++ b/src/usr/local/buildpack/utils/cache.sh @@ -30,7 +30,10 @@ # will attempt to download the file at the given url and stores it in the cache. # If this file already exists, will return the cached version # The cache will only used if BUILDPACK_CACHE_DIR is set -# First argument is the url, second one is the filename (optional) +# First argument is the url +# Second argument is the filename (optional) +# Third argument is the checksum that the file should have (optional) +# Fourth argument is the checksum algorithm (optional) function get_from_url () { local url=${1} check url true @@ -41,21 +44,35 @@ function get_from_url () { local name name=${2:-$(basename "${url}")} + local expected_checksum + expected_checksum=${3} + + local checksum_algo + checksum_algo=${4} + local filename="${checksum}/${name}" if [ -n "${BUILDPACK_CACHE_DIR}" ] && [ -e "${BUILDPACK_CACHE_DIR}/${filename}" ]; then - # file in cache - # echo "Found file in cache: ${BUILDPACK_CACHE_DIR}/${filename}" >&2 - echo "${BUILDPACK_CACHE_DIR}/${filename}" + # file in cache, verify checksum first + if [ -n "${expected_checksum}" ] && ! verify_checksum "${BUILDPACK_CACHE_DIR}/${filename}" "${expected_checksum}" "${checksum_algo}" ; then + # file in cache but checksum doesn't match, so remove file and download again + echo "Cached file is corrupt, redownloading: ${BUILDPACK_CACHE_DIR}/${filename}" >&2 + rm -rf "${BUILDPACK_CACHE_DIR:?}/${filename}" + download_file "${url}" "${filename}" "${expected_checksum}" "${checksum_algo}" + else + echo "${BUILDPACK_CACHE_DIR}/${filename}" + fi else # cache disabled or not in cache - download_file "${url}" "${filename}" + download_file "${url}" "${filename}" "${expected_checksum}" "${checksum_algo}" fi } # Will download the file into the cache folder and returns the path # If the cache is not enabled it will download it to a temp folder -# The second argument will be the filename if given +# The second argument will be the filename if given (optional) +# The third argument is the exepcted checksum of the file if given (optional) +# The fourth argument is the checksum algorithm (optional) function download_file () { local url=${1} check url true @@ -63,14 +80,91 @@ function download_file () { local name name=${2:-$(basename "${url}")} + local expected_checksum + expected_checksum=${3} + + local checksum_algo + checksum_algo=${4} + + local retry=3 local temp_folder=${BUILDPACK_CACHE_DIR:-${TEMP_DIR}} - if ! curl --retry 3 --create-dirs -sSfLo "${temp_folder}/${name}" "${url}" ; then - echo "Download failed: ${url}" >&2 - exit 1 - fi; + while [ "${retry}" -gt 0 ]; do + retry=$((retry-1)) + if ! curl --retry 3 --create-dirs -sSfLo "${temp_folder}/${name}" "${url}" ; then + echo "Download failed: ${url}" >&2 + exit 1 + fi; + + # verify checksum if given + if [ -n "${expected_checksum}" ]; then + if ! verify_checksum "${temp_folder}/${name}" "${expected_checksum}" "${checksum_algo}" ; then + echo "Retries left: ${retry}" >&2 + # clean up what we downloaded so far + rm "${temp_folder}/${name}" + if [ "${retry}" -le 0 ]; then + echo "Checksum verification failed: ${url}" >&2 + exit 1 + fi + echo "Checksum verification failed, retrying" >&2 + continue + fi + fi + retry=0 + done + echo "${temp_folder}/${name}" } +# Will verify if the given checksum matches the given file +# First argument is the path to the file +# Second argument is the checksum +# Third argument is the type, currently supported: +# * sha1 +# * sha224sum +# * sha256sum +# * sha384sum +# * sha512sum +function verify_checksum () { + local file=${1} + check file true + + local expected_checksum=${2} + check expected_checksum true + + local algorithm=${3} + check algorithm true + + + # prevent executing the algorithm blindly + # so use this switch case statement + case $algorithm in + sha1sum) + real_checksum=$(sha1sum "${file}" | cut -d' ' -f1) + ;; + sha224sum) + real_checksum=$(sha224sum "${file}" | cut -d' ' -f1) + ;; + sha256sum) + real_checksum=$(sha256sum "${file}" | cut -d' ' -f1) + ;; + sha384sum) + real_checksum=$(sha384sum "${file}" | cut -d' ' -f1) + ;; + sha512sum) + real_checksum=$(sha512sum "${file}" | cut -d' ' -f1) + ;; + *) + echo "Non supported checksum algorithm: ${algorithm}" >&2 + return 1 + ;; + esac + if [ "$real_checksum" != "$expected_checksum" ]; then + echo "Checksum does not match for file ${file}. Expected: ${expected_checksum} - Got: ${real_checksum}" >&2 + return 1 + fi + return 0 +} + # will try to clean up the oldest file in the cache until the cache is empty # or unless the threshold is reached # When given true as first argument, will only delete a single file diff --git a/test/bash/cache.bats b/test/bash/cache.bats index d61e07697..7d4e64c73 100644 --- a/test/bash/cache.bats +++ b/test/bash/cache.bats @@ -198,6 +198,71 @@ teardown() { assert_output --regexp "${BUILDPACK_CACHE_DIR}/[0-9a-f]{64}/test" } +@test "get_from_url_with_checksum" { + # create cache dir + BUILDPACK_CACHE_DIR="${TEST_ROOT_DIR}/cache" + mkdir -p "${BUILDPACK_CACHE_DIR}" + + # sha256sum of file + local checksum="72e5ba348fdddc06d7b0561403377581c927d62e8f22a911531ad07f616b8c21" + local file="https://github.com/containerbase/base/releases/download/1.0.0/buildpack.tar.xz" + + run get_from_url "${file}" $(basename "${file}") "${checksum}" "sha256sum" + assert_success + assert_output --regexp "^${BUILDPACK_CACHE_DIR}/[0-9a-f]{64}/buildpack\.tar\.xz" + + rm -rf "${BUILDPACK_CACHE_DIR}" + + run get_from_url "${file}" test "${checksum}" "sha256sum" + assert_success + assert_output --regexp "${BUILDPACK_CACHE_DIR}/[0-9a-f]{64}/test" + + rm -rf "${BUILDPACK_CACHE_DIR}" + + # wrong checksum + run get_from_url "${file}" $(basename "${file}") "123" "sha256sum" + assert_failure + assert_output --partial "Retries left: 2" + assert_output --partial "Retries left: 1" + assert_output --partial "Retries left: 0" + + rm -rf "${BUILDPACK_CACHE_DIR}" + + run get_from_url "${file}" test "123" "sha256sum" + assert_failure + assert_output --partial "Retries left: 2" + assert_output --partial "Retries left: 1" + assert_output --partial "Retries left: 0" +} + +@test "get_from_url_with_cache_and_checksum" { + # create cache dir + BUILDPACK_CACHE_DIR="${TEST_ROOT_DIR}/cache" + mkdir -p "${BUILDPACK_CACHE_DIR}" + + # sha256sum of file + local checksum="72e5ba348fdddc06d7b0561403377581c927d62e8f22a911531ad07f616b8c21" + local file="https://github.com/containerbase/base/releases/download/1.0.0/buildpack.tar.xz" + + run get_from_url "${file}" $(basename "${file}") "${checksum}" "sha256sum" + assert_success + assert_output --regexp "^${BUILDPACK_CACHE_DIR}/[0-9a-f]{64}/buildpack\.tar\.xz" + + file_path="${output}" + + run get_from_url "${file}" test "${checksum}" "sha256sum" + assert_success + assert_output --regexp "${BUILDPACK_CACHE_DIR}/[0-9a-f]{64}/test" + + # change checksum of cached file + echo "a" >> "${file_path}" + + # corrupt file in cache + run get_from_url "${file}" $(basename "${file}") "${checksum}" "sha256sum" + assert_success + assert_output --partial "Cached file is corrupt" +} + @test "cache_folder" { # set up the cache load "$TEST_DIR/cache.sh"