diff --git a/BUILD b/BUILD index 07eb0bc076c0a1..cbb83d0f688f82 100644 --- a/BUILD +++ b/BUILD @@ -70,7 +70,7 @@ filegroup( srcs = [ ":WORKSPACE", ":distdir.bzl", - ":distdir_deps.bzl", + ":workspace_deps.bzl", ], visibility = [ "//src/test/shell/bazel:__subpackages__", diff --git a/MODULE.bazel b/MODULE.bazel index d2b6d192b10868..afcbb50ac4625d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -52,7 +52,7 @@ local_path_override( ) # The following Bazel modules are not direct dependencies for building Bazel, -# but are required for visibility from DIST_ARCHIVE_REPOS in distdir_deps.bzl +# but are required for visibility from DIST_ARCHIVE_REPOS in repositories.bzl bazel_dep(name = "apple_support", version = "1.5.0") bazel_dep(name = "abseil-cpp", version = "20220623.1") bazel_dep(name = "c-ares", version = "1.15.0") @@ -211,7 +211,7 @@ java_toolchains = use_extension("@rules_java//java:extensions.bzl", "toolchains" use_repo( java_toolchains, "local_jdk", - # The following are required for visibility from TEST_REPOS in distdir_deps.bzl + # The following are required for visibility in //src:test_repos "remote_java_tools", "remote_java_tools_darwin_arm64", "remote_java_tools_darwin_x86_64", @@ -273,6 +273,7 @@ use_repo( "openjdk_macos_x86_64_vanilla", "openjdk_win_arm64_vanilla", "openjdk_win_vanilla", + "workspace_repo_cache", ) # Required only by `--extra_toolchains=@local_config_cc//:cc-toolchain-arm64_windows` from .bazelrc diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index c04564ef39116d..df4d1a33a9035f 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 3, - "moduleFileHash": "413e7533bebfc4aaf273d5c909589a297de330ac07bd6970c59aad7034fae1cd", + "moduleFileHash": "e04ec0b3c6964d9e4b1c7eadd0fa8ca5531b5cac26e31f5b861ba3d61e6532d1", "flags": { "cmdRegistries": [ "https://bcr.bazel.build/" @@ -330,7 +330,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 321, + "line": 322, "column": 22 } } @@ -461,7 +461,8 @@ "openjdk_macos_aarch64_vanilla": "openjdk_macos_aarch64_vanilla", "openjdk_macos_x86_64_vanilla": "openjdk_macos_x86_64_vanilla", "openjdk_win_arm64_vanilla": "openjdk_win_arm64_vanilla", - "openjdk_win_vanilla": "openjdk_win_vanilla" + "openjdk_win_vanilla": "openjdk_win_vanilla", + "workspace_repo_cache": "workspace_repo_cache" }, "devImports": [], "tags": [], @@ -474,7 +475,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 279, + "line": 280, "column": 29 }, "imports": { @@ -491,7 +492,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 286, + "line": 287, "column": 32 }, "imports": { @@ -510,7 +511,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 294, + "line": 295, "column": 31 }, "imports": { @@ -527,7 +528,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 297, + "line": 298, "column": 48 }, "imports": { @@ -544,7 +545,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 343, + "line": 344, "column": 35 }, "imports": { @@ -561,7 +562,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 346, + "line": 347, "column": 42 }, "imports": { @@ -2106,7 +2107,7 @@ "moduleExtensions": { "//:extensions.bzl%bazel_android_deps": { "general": { - "bzlTransitiveDigest": "YE1RlA6YS4ztUyZvffGDzF2vAIQTseoX8IwjeYyKcFc=", + "bzlTransitiveDigest": "K69taXcfsnBnbeRoTfCfYi0j4PJru+aKdY+KJoNB/dA=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -2116,14 +2117,8 @@ "attributes": { "name": "_main~bazel_android_deps~desugar_jdk_libs", "sha256": "ef71be474fbb3b3b7bd70cda139f01232c63b9e1bbd08c058b00a8d538d4db17", - "urls": [ - "https://github.com/google/desugar_jdk_libs/archive/24dcd1dead0b64aae3d7c89ca9646b5dc4068009.zip" - ], - "patch_args": [ - "-p0" - ], - "patches": [], - "strip_prefix": "desugar_jdk_libs-24dcd1dead0b64aae3d7c89ca9646b5dc4068009" + "strip_prefix": "desugar_jdk_libs-24dcd1dead0b64aae3d7c89ca9646b5dc4068009", + "url": "https://github.com/google/desugar_jdk_libs/archive/24dcd1dead0b64aae3d7c89ca9646b5dc4068009.zip" } } } @@ -2131,7 +2126,7 @@ }, "//:extensions.bzl%bazel_build_deps": { "general": { - "bzlTransitiveDigest": "YE1RlA6YS4ztUyZvffGDzF2vAIQTseoX8IwjeYyKcFc=", + "bzlTransitiveDigest": "K69taXcfsnBnbeRoTfCfYi0j4PJru+aKdY+KJoNB/dA=", "accumulatedFileDigests": { "@@//src/test/tools/bzlmod:MODULE.bazel.lock": "23e00a4ebe85282fdd1c8206adeec448eba0618b7083739bbd78704651e32d6b" }, @@ -2143,11 +2138,8 @@ "attributes": { "name": "_main~bazel_build_deps~openjdk_macos_aarch64_vanilla", "sha256": "2a7a99a3ea263dbd8d32a67d1e6e363ba8b25c645c826f5e167a02bbafaff1fa", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_aarch64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_aarch64.tar.gz" - ], - "downloaded_file_path": "zulu-macos-aarch64-vanilla.tar.gz" + "downloaded_file_path": "zulu-macos-aarch64-vanilla.tar.gz", + "url": "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_aarch64.tar.gz" } }, "bazel_tools_repo_cache": { @@ -2176,11 +2168,8 @@ "attributes": { "name": "_main~bazel_build_deps~openjdk_linux_vanilla", "sha256": "0c0eadfbdc47a7ca64aeab51b9c061f71b6e4d25d2d87674512e9b6387e9e3a6", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_x64.tar.gz" - ], - "downloaded_file_path": "zulu-linux-vanilla.tar.gz" + "downloaded_file_path": "zulu-linux-vanilla.tar.gz", + "url": "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_x64.tar.gz" } }, "debian_cc_deps": { @@ -2209,11 +2198,8 @@ "attributes": { "name": "_main~bazel_build_deps~openjdk_linux_s390x_vanilla", "sha256": "f2512f9a8e9847dd5d3557c39b485a8e7a1ef37b601dcbcb748d22e49f44815c", - "urls": [ - "https://mirror.bazel.build/github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_s390x_linux_hotspot_19.0.2_7.tar.gz", - "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_s390x_linux_hotspot_19.0.2_7.tar.gz" - ], - "downloaded_file_path": "adoptopenjdk-s390x-vanilla.tar.gz" + "downloaded_file_path": "adoptopenjdk-s390x-vanilla.tar.gz", + "url": "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_s390x_linux_hotspot_19.0.2_7.tar.gz" } }, "bootstrap_repo_cache": { @@ -2272,11 +2258,8 @@ "attributes": { "name": "_main~bazel_build_deps~openjdk_win_arm64_vanilla", "sha256": "975603e684f2ec5a525b3b5336d6aa0b09b5b7d2d0d9e271bd6a9892ad550181", - "urls": [ - "https://mirror.bazel.build/aka.ms/download-jdk/microsoft-jdk-21.0.0-windows-aarch64.zip", - "https://aka.ms/download-jdk/microsoft-jdk-21.0.0-windows-aarch64.zip" - ], - "downloaded_file_path": "zulu-win-arm64.zip" + "downloaded_file_path": "zulu-win-arm64.zip", + "url": "https://aka.ms/download-jdk/microsoft-jdk-21.0.0-windows-aarch64.zip" } }, "openjdk_linux_ppc64le_vanilla": { @@ -2285,11 +2268,8 @@ "attributes": { "name": "_main~bazel_build_deps~openjdk_linux_ppc64le_vanilla", "sha256": "45dde71faf8cbb78fab3c976894259655c8d3de827347f23e0ebe5710921dded", - "urls": [ - "https://mirror.bazel.build/github.com/adoptium/temurin20-binaries/releases/download/jdk-20%2B36/OpenJDK20U-jdk_ppc64le_linux_hotspot_20_36.tar.gz", - "https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20%2B36/OpenJDK20U-jdk_ppc64le_linux_hotspot_20_36.tar.gz" - ], - "downloaded_file_path": "adoptopenjdk-ppc64le-vanilla.tar.gz" + "downloaded_file_path": "adoptopenjdk-ppc64le-vanilla.tar.gz", + "url": "https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20%2B36/OpenJDK20U-jdk_ppc64le_linux_hotspot_20_36.tar.gz" } }, "openjdk_macos_x86_64_vanilla": { @@ -2298,11 +2278,66 @@ "attributes": { "name": "_main~bazel_build_deps~openjdk_macos_x86_64_vanilla", "sha256": "9639b87db586d0c89f7a9892ae47f421e442c64b97baebdff31788fbe23265bd", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_x64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_x64.tar.gz" - ], - "downloaded_file_path": "zulu-macos-vanilla.tar.gz" + "downloaded_file_path": "zulu-macos-vanilla.tar.gz", + "url": "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_x64.tar.gz" + } + }, + "workspace_repo_cache": { + "bzlFile": "@@//:distdir.bzl", + "ruleClassName": "_distdir_tar", + "attributes": { + "name": "_main~bazel_build_deps~workspace_repo_cache", + "archives": [ + "rules_cc-0.0.9.tar.gz", + "rules_java-7.0.6.tar.gz", + "5.3.0-21.7.tar.gz", + "bazel-skylib-1.4.1.tar.gz", + "rules_license-0.0.7.tar.gz", + "rules_python-0.24.0.tar.gz", + "rules_pkg-0.9.1.tar.gz", + "rules_testing-v0.0.4.tar.gz", + "coverage_output_generator-v2.6.zip" + ], + "sha256": { + "rules_cc-0.0.9.tar.gz": "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf", + "rules_java-7.0.6.tar.gz": "e81e9deaae0d9d99ef3dd5f6c1b32338447fe16d5564155531ea4eb7ef38854b", + "5.3.0-21.7.tar.gz": "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", + "bazel-skylib-1.4.1.tar.gz": "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7", + "rules_license-0.0.7.tar.gz": "4531deccb913639c30e5c7512a054d5d875698daeb75d8cf90f284375fe7c360", + "rules_python-0.24.0.tar.gz": "0a8003b044294d7840ac7d9d73eef05d6ceb682d7516781a4ec62eeb34702578", + "rules_pkg-0.9.1.tar.gz": "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", + "rules_testing-v0.0.4.tar.gz": "4e21f9aa7996944ce91431f27bca374bff56e680acfe497276074d56bc5d9af2", + "coverage_output_generator-v2.6.zip": "7006375f6756819b7013ca875eab70a541cf7d89142d9c511ed78ea4fefa38af" + }, + "urls": { + "rules_cc-0.0.9.tar.gz": [ + "https://github.com/bazelbuild/rules_cc/releases/download/0.0.9/rules_cc-0.0.9.tar.gz" + ], + "rules_java-7.0.6.tar.gz": [ + "https://github.com/bazelbuild/rules_java/releases/download/7.0.6/rules_java-7.0.6.tar.gz" + ], + "5.3.0-21.7.tar.gz": [ + "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz" + ], + "bazel-skylib-1.4.1.tar.gz": [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz" + ], + "rules_license-0.0.7.tar.gz": [ + "https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz" + ], + "rules_python-0.24.0.tar.gz": [ + "https://github.com/bazelbuild/rules_python/releases/download/0.24.0/rules_python-0.24.0.tar.gz" + ], + "rules_pkg-0.9.1.tar.gz": [ + "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz" + ], + "rules_testing-v0.0.4.tar.gz": [ + "https://github.com/bazelbuild/rules_testing/releases/download/v0.0.4/rules_testing-v0.0.4.tar.gz" + ], + "coverage_output_generator-v2.6.zip": [ + "https://mirror.bazel.build/bazel_coverage_output_generator/releases/coverage_output_generator-v2.6.zip" + ] + } } }, "openjdk_win_vanilla": { @@ -2311,11 +2346,8 @@ "attributes": { "name": "_main~bazel_build_deps~openjdk_win_vanilla", "sha256": "e9959d500a0d9a7694ac243baf657761479da132f0f94720cbffd092150bd802", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-win_x64.zip", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-win_x64.zip" - ], - "downloaded_file_path": "zulu-win-vanilla.zip" + "downloaded_file_path": "zulu-win-vanilla.zip", + "url": "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-win_x64.zip" } }, "openjdk_linux_aarch64_vanilla": { @@ -2324,11 +2356,8 @@ "attributes": { "name": "_main~bazel_build_deps~openjdk_linux_aarch64_vanilla", "sha256": "1fb64b8036c5d463d8ab59af06bf5b6b006811e6012e3b0eb6bccf57f1c55835", - "urls": [ - "https://mirror.bazel.build/cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_aarch64.tar.gz", - "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_aarch64.tar.gz" - ], - "downloaded_file_path": "zulu-linux-aarch64-vanilla.tar.gz" + "downloaded_file_path": "zulu-linux-aarch64-vanilla.tar.gz", + "url": "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_aarch64.tar.gz" } }, "debian_proto_deps": { @@ -2347,7 +2376,7 @@ }, "//:extensions.bzl%bazel_test_deps": { "general": { - "bzlTransitiveDigest": "YE1RlA6YS4ztUyZvffGDzF2vAIQTseoX8IwjeYyKcFc=", + "bzlTransitiveDigest": "K69taXcfsnBnbeRoTfCfYi0j4PJru+aKdY+KJoNB/dA=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -2371,15 +2400,8 @@ "attributes": { "name": "_main~bazel_test_deps~bazelci_rules", "sha256": "eca21884e6f66a88c358e580fd67a6b148d30ab57b1680f62a96c00f9bc6a07e", - "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/continuous-integration/releases/download/rules-1.0.0/bazelci_rules-1.0.0.tar.gz", - "https://github.com/bazelbuild/continuous-integration/releases/download/rules-1.0.0/bazelci_rules-1.0.0.tar.gz" - ], - "patch_args": [ - "-p0" - ], - "patches": [], - "strip_prefix": "bazelci_rules-1.0.0" + "strip_prefix": "bazelci_rules-1.0.0", + "url": "https://github.com/bazelbuild/continuous-integration/releases/download/rules-1.0.0/bazelci_rules-1.0.0.tar.gz" } } } @@ -7921,7 +7943,7 @@ }, "@rules_python~0.24.0//python/extensions:pip.bzl%pip": { "general": { - "bzlTransitiveDigest": "TEllPVzarB4s4TpELB3gWHqo8xpekIiVpXI5FLmg+CQ=", + "bzlTransitiveDigest": "Ft3hTgShxX/7p6AciATaTUQJgVShv/1SJ64filkse6c=", "accumulatedFileDigests": { "@@//:requirements.txt": "ff12967a755bb8e9b4c92524f6471a99e14c30474a3d428547c55745ec8f23a0" }, @@ -7956,7 +7978,7 @@ "repo": "bazel_pip_dev_deps_38", "repo_prefix": "bazel_pip_dev_deps_38_", "python_interpreter": "", - "python_interpreter_target": "@@rules_python~0.24.0~python~python_3_8_aarch64-apple-darwin//:bin/python3", + "python_interpreter_target": "@@rules_python~0.24.0~python~python_3_8_x86_64-unknown-linux-gnu//:bin/python3", "quiet": true, "timeout": 600, "isolated": true, diff --git a/distdir.bzl b/distdir.bzl index c1dcfd9f14b7fd..e872dc4d253b1e 100644 --- a/distdir.bzl +++ b/distdir.bzl @@ -13,9 +13,7 @@ # limitations under the License. """Defines a repository rule that generates an archive consisting of the specified files to fetch""" -load("//:distdir_deps.bzl", "DEPS_BY_NAME") load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts") -load("//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar") _BUILD = """ load("@rules_pkg//pkg:tar.bzl", "pkg_tar") @@ -57,29 +55,26 @@ _distdir_tar = repository_rule( attrs = _distdir_tar_attrs, ) -def distdir_tar(name, archives, sha256, urls, dirname, dist_deps = None): +def distdir_tar(name, dist_deps): """Creates a repository whose content is a set of tar files. Args: name: repo name. - archives: list of tar file names. - sha256: map of tar file names to SHAs. - urls: map of tar file names to URL lists. - dirname: output directory in repo. dist_deps: map of repo names to dict of archive, sha256, and urls. """ - if dist_deps: - for dep, info in dist_deps.items(): - archive_file = info["archive"] - archives.append(archive_file) - sha256[archive_file] = info["sha256"] - urls[archive_file] = info["urls"] + archives = [] + sha256 = {} + urls = {} + for _, info in dist_deps.items(): + archive_file = info["archive"] + archives.append(archive_file) + sha256[archive_file] = info["sha256"] + urls[archive_file] = info["urls"] _distdir_tar( name = name, archives = archives, sha256 = sha256, urls = urls, - dirname = dirname, ) def _repo_cache_tar_impl(ctx): @@ -133,63 +128,3 @@ repo_cache_tar = repository_rule( implementation = _repo_cache_tar_impl, attrs = _repo_cache_tar_attrs, ) - -def dist_http_archive(name, **kwargs): - """Wraps http_archive, providing attributes like sha and urls from the central list. - - dist_http_archive wraps an http_archive invocation, but looks up relevant attributes - from distdir_deps.bzl so the user does not have to specify them. - - Args: - name: repo name - **kwargs: see http_archive for allowed args. - """ - info = DEPS_BY_NAME[name] - if "patch_args" not in kwargs: - kwargs["patch_args"] = info.get("patch_args") - if "patches" not in kwargs: - kwargs["patches"] = info.get("patches") - if "strip_prefix" not in kwargs: - kwargs["strip_prefix"] = info.get("strip_prefix") - http_archive( - name = name, - sha256 = info["sha256"], - urls = info["urls"], - **kwargs - ) - -def dist_http_file(name, **kwargs): - """Wraps http_file, providing attributes like sha and urls from the central list. - - dist_http_file wraps an http_file invocation, but looks up relevant attributes - from distdir_deps.bzl so the user does not have to specify them. - - Args: - name: repo name - **kwargs: see http_file for allowed args. - """ - info = DEPS_BY_NAME[name] - http_file( - name = name, - sha256 = info["sha256"], - urls = info["urls"], - **kwargs - ) - -def dist_http_jar(name, **kwargs): - """Wraps http_jar, providing attributes like sha and urls from the central list. - - dist_http_jar wraps an http_jar invocation, but looks up relevant attributes - from distdir_deps.bzl so the user does not have to specify them. - - Args: - name: repo name - **kwargs: see http_jar for allowed args. - """ - info = DEPS_BY_NAME[name] - http_jar( - name = name, - sha256 = info["sha256"], - urls = info["urls"], - **kwargs - ) diff --git a/extensions.bzl b/extensions.bzl index f933cfff843475..002d172734d695 100644 --- a/extensions.bzl +++ b/extensions.bzl @@ -16,9 +16,9 @@ """ -load("//:distdir.bzl", "dist_http_archive", "repo_cache_tar") -load("//:distdir_deps.bzl", "DIST_ARCHIVE_REPOS") -load("//:repositories.bzl", "embedded_jdk_repositories") +load("//:distdir.bzl", "distdir_tar", "repo_cache_tar") +load("//:repositories.bzl", "DIST_ARCHIVE_REPOS", "android_deps_repos", "bazelci_rules_repo", "embedded_jdk_repositories") +load("//:workspace_deps.bzl", "WORKSPACE_REPOS") load("//src/main/res:winsdk_configure.bzl", "winsdk_configure") load("//src/test/shell/bazel:list_source_repository.bzl", "list_source_repository") load("//src/tools/bzlmod:utils.bzl", "parse_bazel_module_repos") @@ -31,19 +31,20 @@ def _bazel_build_deps(_ctx): repo_cache_tar(name = "bootstrap_repo_cache", repos = DIST_ARCHIVE_REPOS, dirname = "derived/repository_cache") BAZEL_TOOLS_DEPS_REPOS = parse_bazel_module_repos(_ctx, _ctx.path(Label("//src/test/tools/bzlmod:MODULE.bazel.lock"))) repo_cache_tar(name = "bazel_tools_repo_cache", repos = BAZEL_TOOLS_DEPS_REPOS, lockfile = "//src/test/tools/bzlmod:MODULE.bazel.lock") + distdir_tar(name = "workspace_repo_cache", dist_deps = WORKSPACE_REPOS) bazel_build_deps = module_extension(implementation = _bazel_build_deps) ### Dependencies for testing Bazel def _bazel_test_deps(_ctx): + bazelci_rules_repo() list_source_repository(name = "local_bazel_source_list") - dist_http_archive(name = "bazelci_rules") winsdk_configure(name = "local_config_winsdk") bazel_test_deps = module_extension(implementation = _bazel_test_deps) ### Dependencies for Bazel Android tools def _bazel_android_deps(_ctx): - dist_http_archive(name = "desugar_jdk_libs") + android_deps_repos() bazel_android_deps = module_extension(implementation = _bazel_android_deps) diff --git a/repositories.bzl b/repositories.bzl index 2c1ab847d6898b..03e47487537f9b 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -15,46 +15,123 @@ """ -load("//:distdir.bzl", "dist_http_file") +load("//src/tools/bzlmod:utils.bzl", "get_canonical_repo_name") +load("//tools/build_defs/repo:http.bzl", "http_archive", "http_file") + +################################################################################## +# +# The list of repositories required while bootstrapping Bazel offline +# +################################################################################## +DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [ + "abseil-cpp", + "apple_support", + "bazel_skylib", + "blake3", + "c-ares", + "com_github_grpc_grpc", + "com_google_protobuf", + "io_bazel_skydoc", + "platforms", + "rules_cc", + "rules_go", + "rules_java", + "rules_jvm_external", + "rules_license", + "rules_pkg", + "rules_proto", + "rules_python", + "upb", + "zlib", + "zstd-jni", +]] + [(get_canonical_repo_name("com_github_grpc_grpc") + suffix) for suffix in [ + # Extra grpc dependencies introduced via its module extension + "~grpc_repo_deps_ext~bazel_gazelle", # TODO: Should be a bazel_dep + "~grpc_repo_deps_ext~bazel_skylib", # TODO: Should be removed + "~grpc_repo_deps_ext~com_envoyproxy_protoc_gen_validate", + "~grpc_repo_deps_ext~com_github_cncf_udpa", + "~grpc_repo_deps_ext~com_google_googleapis", + "~grpc_repo_deps_ext~envoy_api", + "~grpc_repo_deps_ext~rules_cc", # TODO: Should be removed +]] + +################################################################################## +# +# Make sure all URLs below are mirrored to https://mirror.bazel.build +# +################################################################################## def embedded_jdk_repositories(): """OpenJDK distributions used to create a version of Bazel bundled with the OpenJDK.""" - dist_http_file( + http_file( name = "openjdk_linux_vanilla", + sha256 = "0c0eadfbdc47a7ca64aeab51b9c061f71b6e4d25d2d87674512e9b6387e9e3a6", downloaded_file_path = "zulu-linux-vanilla.tar.gz", + url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_x64.tar.gz", ) - - dist_http_file( + http_file( name = "openjdk_linux_aarch64_vanilla", + sha256 = "1fb64b8036c5d463d8ab59af06bf5b6b006811e6012e3b0eb6bccf57f1c55835", downloaded_file_path = "zulu-linux-aarch64-vanilla.tar.gz", + url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_aarch64.tar.gz", ) - dist_http_file( - name = "openjdk_linux_ppc64le_vanilla", - downloaded_file_path = "adoptopenjdk-ppc64le-vanilla.tar.gz", - ) - - dist_http_file( + # JDK21 unavailable so use JDK19 instead for linux s390x. + http_file( name = "openjdk_linux_s390x_vanilla", + sha256 = "f2512f9a8e9847dd5d3557c39b485a8e7a1ef37b601dcbcb748d22e49f44815c", downloaded_file_path = "adoptopenjdk-s390x-vanilla.tar.gz", + url = "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_s390x_linux_hotspot_19.0.2_7.tar.gz", ) - dist_http_file( + # JDK21 unavailable so use JDK19 instead for linux ppc64le. + http_file( + name = "openjdk_linux_ppc64le_vanilla", + sha256 = "45dde71faf8cbb78fab3c976894259655c8d3de827347f23e0ebe5710921dded", + downloaded_file_path = "adoptopenjdk-ppc64le-vanilla.tar.gz", + url = "https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20%2B36/OpenJDK20U-jdk_ppc64le_linux_hotspot_20_36.tar.gz", + ) + http_file( name = "openjdk_macos_x86_64_vanilla", + sha256 = "9639b87db586d0c89f7a9892ae47f421e442c64b97baebdff31788fbe23265bd", downloaded_file_path = "zulu-macos-vanilla.tar.gz", + url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_x64.tar.gz", ) - - dist_http_file( + http_file( name = "openjdk_macos_aarch64_vanilla", + sha256 = "2a7a99a3ea263dbd8d32a67d1e6e363ba8b25c645c826f5e167a02bbafaff1fa", downloaded_file_path = "zulu-macos-aarch64-vanilla.tar.gz", + url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_aarch64.tar.gz", ) - - dist_http_file( + http_file( name = "openjdk_win_vanilla", + sha256 = "e9959d500a0d9a7694ac243baf657761479da132f0f94720cbffd092150bd802", downloaded_file_path = "zulu-win-vanilla.zip", + url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-win_x64.zip", ) - dist_http_file( + # JDK21 unavailable from zulu, we'll use Microsoft's OpenJDK build instead. + http_file( name = "openjdk_win_arm64_vanilla", + sha256 = "975603e684f2ec5a525b3b5336d6aa0b09b5b7d2d0d9e271bd6a9892ad550181", downloaded_file_path = "zulu-win-arm64.zip", + url = "https://aka.ms/download-jdk/microsoft-jdk-21.0.0-windows-aarch64.zip", + ) + +def bazelci_rules_repo(): + """Required by the Bazel CI jobs.""" + http_archive( + name = "bazelci_rules", + sha256 = "eca21884e6f66a88c358e580fd67a6b148d30ab57b1680f62a96c00f9bc6a07e", + strip_prefix = "bazelci_rules-1.0.0", + url = "https://github.com/bazelbuild/continuous-integration/releases/download/rules-1.0.0/bazelci_rules-1.0.0.tar.gz", + ) + +def android_deps_repos(): + """Required by building the android tools.""" + http_archive( + name = "desugar_jdk_libs", + sha256 = "ef71be474fbb3b3b7bd70cda139f01232c63b9e1bbd08c058b00a8d538d4db17", + strip_prefix = "desugar_jdk_libs-24dcd1dead0b64aae3d7c89ca9646b5dc4068009", + url = "https://github.com/google/desugar_jdk_libs/archive/24dcd1dead0b64aae3d7c89ca9646b5dc4068009.zip", ) diff --git a/src/BUILD b/src/BUILD index 8f70304152da0c..90c19f27cc0b33 100644 --- a/src/BUILD +++ b/src/BUILD @@ -587,5 +587,8 @@ filegroup( "@remotejdk%s_%s//:WORKSPACE" % (version, os) for version in ("17", "21") for os in ("macos", "macos_aarch64", "linux", "win") - ] + ["@bazel_tools_repo_cache//:files"], + ] + [ + "@bazel_tools_repo_cache//:files", + "@workspace_repo_cache//:files", + ], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD index 372ee20bf826ec..ddab0cd622ee6a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD @@ -1,4 +1,4 @@ -load("//:distdir_deps.bzl", "gen_workspace_stanza") +load("//:workspace_deps.bzl", "gen_workspace_stanza") load("@rules_java//java:defs.bzl", "java_library") package( @@ -125,7 +125,7 @@ genrule( # 1. bazel build tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator_zip # 2. Copy and rename the zip file with a new version locally. # 3. Upload the file under https://mirror.bazel.build/bazel_coverage_output_generator/releases. -# 4. Update //distdir_deps.bzl and //tools/test/extensions.bzl to point to the new release. +# 4. Update //workspace_deps.bzl and //tools/test/extensions.bzl to point to the new release. gen_workspace_stanza( name = "workspace_with_coverage_output_generator", out = "coverage.WORKSPACE", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD index b46e763878c654..67150729b29009 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD @@ -1,5 +1,5 @@ +load("//:workspace_deps.bzl", "gen_workspace_stanza") load("@rules_java//java:defs.bzl", "java_library") -load("//:distdir_deps.bzl", "gen_workspace_stanza") package( default_applicable_licenses = ["//:license"], diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD index 06bb2c27b1c69f..72e589410700ce 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD @@ -1,4 +1,4 @@ -load("//:distdir_deps.bzl", "gen_workspace_stanza") +load("//:workspace_deps.bzl", "gen_workspace_stanza") load("@rules_java//java:defs.bzl", "java_library") package( diff --git a/src/test/java/com/google/devtools/build/lib/blackbox/framework/BUILD b/src/test/java/com/google/devtools/build/lib/blackbox/framework/BUILD index 814d1137a6bf27..d860a82e67e39c 100644 --- a/src/test/java/com/google/devtools/build/lib/blackbox/framework/BUILD +++ b/src/test/java/com/google/devtools/build/lib/blackbox/framework/BUILD @@ -1,5 +1,5 @@ load("@rules_java//java:defs.bzl", "java_library", "java_test") -load("//:distdir_deps.bzl", "gen_workspace_stanza") +load("//:workspace_deps.bzl", "gen_workspace_stanza") java_library( name = "deps", diff --git a/src/test/py/bazel/BUILD b/src/test/py/bazel/BUILD index bc9e9361840712..6d089f11d59793 100644 --- a/src/test/py/bazel/BUILD +++ b/src/test/py/bazel/BUILD @@ -1,6 +1,6 @@ load("@bazel_pip_dev_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_library", "py_test") -load("//:distdir_deps.bzl", "gen_workspace_stanza") +load("//:workspace_deps.bzl", "gen_workspace_stanza") package(default_visibility = ["//visibility:private"]) diff --git a/src/test/shell/BUILD b/src/test/shell/BUILD index f820527754829d..14f941a68df89d 100644 --- a/src/test/shell/BUILD +++ b/src/test/shell/BUILD @@ -1,5 +1,5 @@ load("@rules_python//python:defs.bzl", "py_test") -load("//:distdir_deps.bzl", "gen_workspace_stanza") +load("//:workspace_deps.bzl", "gen_workspace_stanza") package(default_visibility = ["//visibility:private"]) diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index d278bd41eb411e..8dbd1750d8b2da 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -1,4 +1,4 @@ -load("//:distdir_deps.bzl", "gen_workspace_stanza") +load("//:workspace_deps.bzl", "gen_workspace_stanza") package(default_visibility = ["//visibility:private"]) diff --git a/src/test/shell/bazel/verify_workspace.sh b/src/test/shell/bazel/verify_workspace.sh index 26f3d136a6a29b..5c1a86abf14251 100755 --- a/src/test/shell/bazel/verify_workspace.sh +++ b/src/test/shell/bazel/verify_workspace.sh @@ -40,7 +40,7 @@ fi source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \ || { echo "integration_test_setup.sh not found!" >&2; exit 1; } -WORKSPACE_FILES=("$(rlocation io_bazel/WORKSPACE)" "$(rlocation io_bazel/distdir_deps.bzl)") +WORKSPACE_FILES=("$(rlocation io_bazel/WORKSPACE)" "$(rlocation io_bazel/workspace_deps.bzl)") # base maven repository URLs can return 404s. URL_ALLOWLIST=("https://dl.google.com/android/maven2" "https://repo1.maven.org/maven2") diff --git a/src/test/shell/integration/BUILD b/src/test/shell/integration/BUILD index 251a1e0e3a6ae1..acc995ebc6b66a 100644 --- a/src/test/shell/integration/BUILD +++ b/src/test/shell/integration/BUILD @@ -1,4 +1,4 @@ -load("//:distdir_deps.bzl", "gen_workspace_stanza") +load("//:workspace_deps.bzl", "gen_workspace_stanza") package(default_visibility = ["//visibility:private"]) diff --git a/workspace_deps.bzl b/workspace_deps.bzl new file mode 100644 index 00000000000000..82236da99987ed --- /dev/null +++ b/workspace_deps.bzl @@ -0,0 +1,152 @@ +# Copyright 2020 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""WORKSPACE default repository definitions.""" + +WORKSPACE_REPOS = { + # Used in src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/cc_configure.WORKSPACE. + # Used in src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE. + # Used in src/test/java/com/google/devtools/build/lib/blackbox/framework/blackbox.WORKSAPCE + "rules_cc": { + "archive": "rules_cc-0.0.9.tar.gz", + "sha256": "2037875b9a4456dce4a79d112a8ae885bbc4aad968e6587dca6e64f3a0900cdf", + "urls": ["https://github.com/bazelbuild/rules_cc/releases/download/0.0.9/rules_cc-0.0.9.tar.gz"], + "strip_prefix": "rules_cc-0.0.9", + }, + "rules_java": { + "archive": "rules_java-7.0.6.tar.gz", + "sha256": "e81e9deaae0d9d99ef3dd5f6c1b32338447fe16d5564155531ea4eb7ef38854b", + "urls": ["https://github.com/bazelbuild/rules_java/releases/download/7.0.6/rules_java-7.0.6.tar.gz"], + }, + # Used in src/test/java/com/google/devtools/build/lib/blackbox/framework/blackbox.WORKSAPCE + "rules_proto": { + "archive": "5.3.0-21.7.tar.gz", + "sha256": "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", + "strip_prefix": "rules_proto-5.3.0-21.7", + "urls": ["https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz"], + }, + "bazel_skylib": { + "archive": "bazel-skylib-1.4.1.tar.gz", + "sha256": "b8a1527901774180afc798aeb28c4634bdccf19c4d98e7bdd1ce79d1fe9aaad7", + "urls": ["https://github.com/bazelbuild/bazel-skylib/releases/download/1.4.1/bazel-skylib-1.4.1.tar.gz"], + }, + "rules_license": { + "archive": "rules_license-0.0.7.tar.gz", + "sha256": "4531deccb913639c30e5c7512a054d5d875698daeb75d8cf90f284375fe7c360", + "urls": ["https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz"], + }, + "rules_python": { + "archive": "rules_python-0.24.0.tar.gz", + "sha256": "0a8003b044294d7840ac7d9d73eef05d6ceb682d7516781a4ec62eeb34702578", + "strip_prefix": "rules_python-0.24.0", + "urls": ["https://github.com/bazelbuild/rules_python/releases/download/0.24.0/rules_python-0.24.0.tar.gz"], + }, + "rules_pkg": { + "archive": "rules_pkg-0.9.1.tar.gz", + "sha256": "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", + "urls": ["https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz"], + }, + "rules_testing": { + "archive": "rules_testing-v0.0.4.tar.gz", + "sha256": "4e21f9aa7996944ce91431f27bca374bff56e680acfe497276074d56bc5d9af2", + "strip_prefix": "rules_testing-0.0.4", + "urls": ["https://github.com/bazelbuild/rules_testing/releases/download/v0.0.4/rules_testing-v0.0.4.tar.gz"], + }, + "remote_coverage_tools": { + "archive": "coverage_output_generator-v2.6.zip", + "sha256": "7006375f6756819b7013ca875eab70a541cf7d89142d9c511ed78ea4fefa38af", + "urls": ["https://mirror.bazel.build/bazel_coverage_output_generator/releases/coverage_output_generator-v2.6.zip"], + }, +} + +def _gen_workspace_stanza_impl(ctx): + if ctx.attr.template and (ctx.attr.preamble or ctx.attr.postamble): + fail("Can not use template with either preamble or postamble") + if ctx.attr.use_maybe and ctx.attr.repo_clause: + fail("Can not use use_maybe with repo_clause") + + if ctx.attr.use_maybe: + repo_clause = """ +maybe( + http_archive, + name = "{repo}", + sha256 = "{sha256}", + strip_prefix = {strip_prefix}, + urls = {urls}, +) +""" + elif ctx.attr.repo_clause: + repo_clause = ctx.attr.repo_clause + else: + repo_clause = """ +http_archive( + name = "{repo}", + sha256 = "{sha256}", + strip_prefix = {strip_prefix}, + urls = {urls}, +) +""" + + repo_stanzas = {} + for repo in ctx.attr.repos: + info = WORKSPACE_REPOS[repo] + strip_prefix = info.get("strip_prefix") + if strip_prefix: + strip_prefix = "\"%s\"" % strip_prefix + else: + strip_prefix = "None" + + repo_stanzas["{%s}" % repo] = repo_clause.format( + repo = repo, + sha256 = str(info["sha256"]), + strip_prefix = strip_prefix, + urls = info["urls"], + ) + + if ctx.attr.template: + ctx.actions.expand_template( + output = ctx.outputs.out, + template = ctx.file.template, + substitutions = repo_stanzas, + ) + else: + content = "\n".join([p.strip() for p in ctx.attr.preamble.strip().split("\n")]) + content += "\n" + content += "".join(repo_stanzas.values()) + content += "\n" + content += "\n".join([p.strip() for p in ctx.attr.postamble.strip().split("\n")]) + content += "\n" + ctx.actions.write(ctx.outputs.out, content) + + return [DefaultInfo(files = depset([ctx.outputs.out]))] + +gen_workspace_stanza = rule( + attrs = { + "repos": attr.string_list(doc = "Set of repos to include."), + "out": attr.output(mandatory = True), + "preamble": attr.string(doc = "Preamble."), + "postamble": attr.string(doc = "Set of rules to follow repos."), + "template": attr.label( + doc = "Template WORKSPACE file. May not be used with preamble or postamble." + + "Repo stanzas can be included using the syntax '{repo name}'.", + allow_single_file = True, + mandatory = False, + ), + "use_maybe": attr.bool(doc = "Use maybe() invocation instead of http_archive."), + "repo_clause": attr.string(doc = "Use a custom clause for each repository."), + }, + doc = "Use specifications from WORKSPACE_REPOS to generate WORKSPACE http_archive stanzas or to" + + "drop them into a template.", + implementation = _gen_workspace_stanza_impl, +)