Support zstd decompression for external dependencies

WORKSPACE

@@ -183,6 +183,8 @@ distdir_tar(
         "aecba11114cf1fac5497aeb844b6966106de3eb6.tar.gz",
         # abseil/abseil-cpp
         "df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz",
+        # com_github_luben_zstd_jni
+        "v1.4.5-11.tar.gz",
     ],
     dirname = "derived/distdir",
     sha256 = {
@@ -224,6 +226,8 @@ distdir_tar(
         "aecba11114cf1fac5497aeb844b6966106de3eb6.tar.gz": "9f385e146410a8150b6f4cb1a57eab7ec806ced48d427554b1e754877ff26c3e",
         # abseil/abseil-cpp
         "df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz": "f368a8476f4e2e0eccf8a7318b98dafbe30b2600f4e3cf52636e5eb145aba06a",
+        # com_github_luben_zstd_jni
+        "v1.4.5-11.tar.gz": "9a33de540e4a10a080a22ff918c1bc2f585647f73ea30694eb095a1844b3f052",
     },
     urls = {
         "e0b0291b2c51fbe5a7cfa14473a1ae850f94f021.zip": [
@@ -322,6 +326,11 @@ distdir_tar(
             "https://mirror.bazel.build/github.com/abseil/abseil-cpp/archive/df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz",
             "https://github.com/abseil/abseil-cpp/archive/df3ea785d8c30a9503321a3d35ee7d35808f190d.tar.gz",
         ],
+        # com_github_luben_zstd_jni
+        "v1.4.5-11.tar.gz": [
+            # "https://mirror.bazel.build/github.com/luben/zstd-jni/archive/v1.4.5-11.tar.gz",
+            "https://github.com/luben/zstd-jni/archive/v1.4.5-11.tar.gz",
+        ],
     },
 )
 
@@ -543,6 +552,20 @@ http_archive(
     ],
 )
 
+http_archive(
+    name = "com_github_luben_zstd_jni",
+    build_file = "//third_party/zstd-jni:BUILD.bazel",
+    patch_args = ["-p1"],
+    patches = ["//third_party/zstd-jni:zstd-jni_1.4.5-11.patch"],
+    sha256 = "9a33de540e4a10a080a22ff918c1bc2f585647f73ea30694eb095a1844b3f052",
+    strip_prefix = "zstd-jni-1.4.5-11",
+    urls = [
+        # TODO: Uncomment after publishing library to the mirror.
+        # "https://mirror.bazel.build/github.com/luben/zstd-jni/archive/v1.4.5-11.tar.gz",
+        "https://github.com/luben/zstd-jni/archive/v1.4.5-11.tar.gz",
+    ],
+)
+
 # For testing, have an distdir_tar with all the archives implicit in every
 # WORKSPACE, to that they don't have to be refetched for every test
 # calling `bazel sync`.

src/BUILD

@@ -341,6 +341,7 @@ filegroup(
         "//src/main/tools:process-wrapper",
         "//src/main/tools:linux-sandbox",
         "//tools/osx:xcode-locator",
+        "@com_github_luben_zstd_jni//:zstd-native-lib",
     ] + select({
         "//src/conditions:windows": [],
         "//conditions:default": [

src/main/java/com/google/devtools/build/lib/bazel/repository/DecompressorValue.java

@@ -106,13 +106,15 @@ static Decompressor getDecompressor(Path archivePath)
       return TarGzFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.xz") || baseName.endsWith(".txz")) {
       return TarXzFunction.INSTANCE;
+    } else if (baseName.endsWith(".tar.zst") || baseName.endsWith(".tzst")) {
+      return TarZstdFunction.INSTANCE;
     } else if (baseName.endsWith(".tar.bz2")) {
       return TarBz2Function.INSTANCE;
     } else {
       throw new RepositoryFunctionException(
           Starlark.errorf(
-              "Expected a file with a .zip, .jar, .war, .tar, .tar.gz, .tgz, .tar.xz, .txz, or "
-                  + ".tar.bz2 suffix (got %s)",
+              "Expected a file with a .zip, .jar, .war, .tar, .tar.gz, .tgz, .tar.xz, .txz, "
+                  + ".tar.zst, .tzst, or .tar.bz2 suffix (got %s)",
               archivePath),
           Transience.PERSISTENT);
     }

src/main/java/com/google/devtools/build/lib/bazel/repository/TarZstdFunction.java

@@ -0,0 +1,41 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import com.google.devtools.build.lib.bazel.repository.DecompressorValue.Decompressor;
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import org.apache.commons.compress.compressors.zstandard.ZstdCompressorInputStream;
+
+/**
+ * Creates a repository by unarchiving a .tar.zst file.
+ */
+public class TarZstdFunction extends CompressedTarFunction {
+  public static final Decompressor INSTANCE = new TarZstdFunction();
+  private static final int BUFFER_SIZE = 32 * 1024;
+
+  private TarZstdFunction() {
+  }
+
+  @Override
+  protected InputStream getDecompressorStream(DecompressorDescriptor descriptor)
+      throws IOException {
+    return new ZstdCompressorInputStream(
+        new BufferedInputStream(
+            new FileInputStream(descriptor.archivePath().getPathFile()), BUFFER_SIZE));
+  }
+}

src/test/java/com/google/devtools/build/lib/bazel/repository/BUILD

@@ -26,6 +26,7 @@ java_library(
     ),
     data = [
         "test_decompress_archive.tar.gz",
+        "test_decompress_archive.tar.zst",
         "test_decompress_archive.zip",
     ],
     deps = [

src/test/java/com/google/devtools/build/lib/bazel/repository/DecompressorValueTest.java

@@ -50,6 +50,10 @@ public void testKnownFileExtensionsDoNotThrow() throws Exception {
     DecompressorDescriptor.builder().setArchivePath(path).build();
     path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.txz");
     DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.zst");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
+    path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tzst");
+    DecompressorDescriptor.builder().setArchivePath(path).build();
     path = fs.getPath("/foo/.external-repositories/some-repo/bar.baz.tar.bz2");
     DecompressorDescriptor.builder().setArchivePath(path).build();
   }

src/test/java/com/google/devtools/build/lib/bazel/repository/TarZstdFunctionTest.java

@@ -0,0 +1,47 @@
+// Copyright 2020 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.devtools.build.lib.bazel.repository;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.devtools.build.lib.bazel.repository.TestArchiveDescriptor.INNER_FOLDER_NAME;
+import static com.google.devtools.build.lib.bazel.repository.TestArchiveDescriptor.ROOT_FOLDER_NAME;
+
+import com.google.devtools.build.lib.vfs.Path;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Tests Zstandard (zstd) decompression. */
+@RunWith(JUnit4.class)
+public class TarZstdFunctionTest {
+  private static final String ARCHIVE_NAME = "test_decompress_archive.tar.zst";
+
+  /**
+   * Test decompressing a zstd file.
+   */
+  @Test
+  public void testZstdDecompress() throws Exception {
+    TestArchiveDescriptor archiveDescriptor = new TestArchiveDescriptor(ARCHIVE_NAME, "out", false);
+
+    Path outputDir = decompress(archiveDescriptor.createDescriptorBuilder());
+
+    archiveDescriptor.assertOutputFiles(outputDir, INNER_FOLDER_NAME);
+  }
+
+  private Path decompress(DecompressorDescriptor.Builder descriptorBuilder) throws Exception {
+    descriptorBuilder.setDecompressor(TarZstdFunction.INSTANCE);
+    return TarZstdFunction.INSTANCE.decompress(descriptorBuilder.build());
+  }
+}

src/test/shell/bazel/BUILD

@@ -689,7 +689,10 @@ sh_test(
     name = "external_integration_test",
     size = "large",
     srcs = ["external_integration_test.sh"],
-    data = [":test-deps"],
+    data = [
+        ":test-deps",
+        "//src/test/shell/bazel/testdata:zstd_test_archive",
+    ],
     shard_count = 15,
     tags = [
         "no_windows",

src/test/shell/bazel/external_integration_test.sh

@@ -205,6 +205,29 @@ function test_http_archive_tar_xz() {
   http_archive_helper tar_xz_up
 }
 
+# Test zstd support using pre-created archive with a single file.
+# Not using http_archive_helper because zstd may not be available on the
+# testing platform.
+function test_http_archive_tar_zstd() {
+  serve_file "$(rlocation \
+      io_bazel/src/test/shell/bazel/testdata/zstd_test_archive.tar.zst)"
+
+  cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+    name = 'test_zstd_repo',
+    url = 'http://127.0.0.1:$nc_port/archive.tar.zst',
+    sha256 = '12b0116f2a3c804859438e102a8a1d5f494c108d1b026da9f6ca55fb5107c7e9',
+    build_file_content = 'filegroup(name="x", srcs=glob(["*"]))',
+)
+EOF
+  bazel build @test_zstd_repo//...
+  kill_nc
+
+  base_external_path=bazel-out/../external/test_zstd_repo
+  assert_contains "test content" "${base_external_path}/test_dir/test_file"
+}
+
 function test_http_archive_no_server() {
   cat >> $(create_workspace_with_default_repos WORKSPACE) <<EOF
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

src/test/shell/bazel/testdata/BUILD

@@ -32,3 +32,12 @@ filegroup(
     ],
     visibility = ["//src/test/shell/bazel:__pkg__"],
 )
+
+filegroup(
+    name = "zstd_test_archive",
+    testonly = 1,
+    srcs = [
+        "zstd_test_archive.tar.zst",
+    ],
+    visibility = ["//src/test/shell/bazel:__pkg__"],
+)

third_party/BUILD

@@ -47,6 +47,7 @@ filegroup(
         "//third_party/py/six:srcs",
         "//third_party/rules_python:srcs",
         "//third_party/zlib:srcs",
+        "//third_party/zstd-jni:srcs",
         "@googleapis//:srcs",
         "@remoteapis//:srcs",
     ],
@@ -142,6 +143,9 @@ distrib_java_import(
     name = "apache_commons_compress",
     enable_distributions = ["debian"],
     jars = ["apache_commons_compress/apache-commons-compress-1.19.jar"],
+    deps = [
+        "@com_github_luben_zstd_jni//:zstd-jni",
+    ],
 )
 
 distrib_java_import(

third_party/zstd-jni/BUILD.bazel

@@ -0,0 +1,129 @@
+load("@rules_java//java:defs.bzl", "java_library")
+
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])  # BSD License
+
+filegroup(
+    name = "srcs",
+    srcs = glob(["**"]),
+    visibility = ["//third_party:__pkg__"],
+)
+
+# Below two rules are copied from src/main/native/BUILD.
+# Move to some common place?
+genrule(
+    name = "copy_link_jni_md_header",
+    srcs = select({
+        "@io_bazel//src/conditions:darwin": ["@io_bazel//tools/jdk:jni_md_header-darwin"],
+        "@io_bazel//src/conditions:darwin_x86_64": ["@io_bazel//tools/jdk:jni_md_header-darwin"],
+        "@io_bazel//src/conditions:freebsd": ["@io_bazel//tools/jdk:jni_md_header-freebsd"],
+        "@io_bazel//src/conditions:openbsd": ["@io_bazel//tools/jdk:jni_md_header-openbsd"],
+        "@io_bazel//src/conditions:windows": ["@io_bazel//tools/jdk:jni_md_header-windows"],
+        "@io_bazel//conditions:default": ["@io_bazel//tools/jdk:jni_md_header-linux"],
+    }),
+    outs = ["jni_md.h"],
+    cmd = "cp -f $< $@",
+)
+
+genrule(
+    name = "copy_link_jni_header",
+    srcs = ["@io_bazel//tools/jdk:jni_header"],
+    outs = ["jni.h"],
+    cmd = "cp -f $< $@",
+)
+
+cc_library(
+    name = "jni_headers",
+    hdrs = [
+        ":jni.h",
+        ":jni_md.h",
+    ],
+    includes = ["."],
+)
+
+filegroup(
+    name = "zstd_sources",
+    srcs = glob([
+        "src/main/native/**/*.c",
+        "src/main/native/**/*.h",
+    ]),
+)
+
+cc_binary(
+    name = "libzstd-jni.so",
+    srcs = [":zstd_sources"],
+    copts = [
+        "-std=c99",
+        "-Wundef",
+        "-Wshadow",
+        "-Wcast-align",
+        "-Wstrict-prototypes",
+        "-Wno-unused-variable",
+        "-Wpointer-arith",
+        "-Wno-maybe-uninitialized",
+    ],
+    includes = [
+        "src/main/native",
+        "src/main/native/common",
+        "src/main/native/legacy",
+    ],
+    linkshared = 1,
+    local_defines = [
+        "ZSTD_MULTITHREAD=1",
+        "ZSTD_LEGACY_SUPPORT=4",
+    ],
+    deps = [":jni_headers"],
+)
+
+cc_binary(
+    name = "zstd-jni.dll",
+    srcs = [":zstd_sources"],
+    includes = [
+        "src/main/native",
+        "src/main/native/common",
+        "src/main/native/legacy",
+    ],
+    linkshared = 1,
+    local_defines = [
+        "ZSTD_MULTITHREAD=1",
+        "ZSTD_LEGACY_SUPPORT=4",
+        "_JNI_IMPLEMENTATION_",
+    ],
+    deps = [":jni_headers"],
+)
+
+# TODO(bazel-team): Come up with a way to support platform-specific dynamic
+# library extensions.  This is issue #914.
+genrule(
+    name = "mac-compat",
+    srcs = [":libzstd-jni.so"],
+    outs = ["libzstd-jni.dylib"],
+    cmd = "cp $< $@",
+    output_to_bindir = 1,
+)
+
+filegroup(
+    name = "zstd-native-lib",
+    srcs = select({
+        "@io_bazel//src/conditions:darwin": ["libzstd-jni.dylib"],
+        "@io_bazel//src/conditions:darwin_x86_64": ["libzstd-jni.dylib"],
+        "@io_bazel//src/conditions:windows": ["zstd-jni.dll"],
+        "@io_bazel//conditions:default": ["libzstd-jni.so"],
+    }),
+    visibility = [
+        "@io_bazel//src:__pkg__",
+    ],
+)
+
+java_library(
+    name = "zstd-jni",
+    srcs = glob(
+        [
+            "src/main/java/**/*.java",
+        ],
+    ),
+    data = [":zstd-native-lib"],
+    visibility = ["//visibility:public"],
+    deps = ["@bazel_tools//tools/java/runfiles"],
+)