From 36a8d39a3de42eafe94e0f2d080ec34e5161eda7 Mon Sep 17 00:00:00 2001 From: Robert Burke Date: Tue, 1 Oct 2024 19:31:06 -0700 Subject: [PATCH] [prism][java] Update Prism locator to match Python SDK semantics. (#32619) * Update PrismLocator to match the python SDK semantics. * Update validate beam release with prism RC validation instructions. * rename resolveLocation to resolveSource --------- Co-authored-by: lostluck <13907733+lostluck@users.noreply.github.com> --- .../beam/runners/prism/PrismLocator.java | 69 ++++++++++----- .../runners/prism/PrismPipelineOptions.java | 7 +- .../beam/runners/prism/PrismLocatorTest.java | 85 ++++++++++++++----- .../content/en/blog/validate-beam-release.md | 31 +++++++ 4 files changed, 147 insertions(+), 45 deletions(-) diff --git a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java index f69260344d12..27aea3f64df0 100644 --- a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java +++ b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java @@ -54,9 +54,9 @@ class PrismLocator { private static final String PRISM_BIN_PATH = ".apache_beam/cache/prism/bin"; private static final Set PERMS = PosixFilePermissions.fromString("rwxr-xr-x"); - private static final String GITHUB_DOWNLOAD_PREFIX = - "https://github.com/apache/beam/releases/download"; - private static final String GITHUB_TAG_PREFIX = "https://github.com/apache/beam/releases/tag"; + private static final String GITHUB_COMMON_PREFIX = "https://github.com/apache/beam/releases/"; + private static final String GITHUB_DOWNLOAD_PREFIX = GITHUB_COMMON_PREFIX + "download"; + private static final String GITHUB_TAG_PREFIX = GITHUB_COMMON_PREFIX + "tag"; private final PrismPipelineOptions options; @@ -64,24 +64,51 @@ class PrismLocator { this.options = options; } + String resolveSource() { + String from = + String.format( + "%s/v%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, RELEASE_INFO.getSdkVersion(), buildFileName()); + + if (Strings.isNullOrEmpty(options.getPrismLocation())) { + return from; + } + from = options.getPrismLocation(); + + // Likely a local file, return it directly. + if (!from.startsWith("http")) { + return from; + } + + // Validate that it's from a safe location: A Beam Github Release + checkArgument( + options.getPrismLocation().startsWith(GITHUB_COMMON_PREFIX), + "Provided --prismLocation URL is not an Apache Beam Github " + + "Release page URL or download URL: ", + options.getPrismLocation()); + + from = options.getPrismLocation(); + + // If this is the tag prefix, then build the release download with the version + // from the given url. + if (options.getPrismLocation().startsWith(GITHUB_TAG_PREFIX)) { + Path tagPath = Paths.get(options.getPrismLocation()); + Path locVersion = tagPath.getName(tagPath.getNameCount() - 1); + // The "v" prefix is already included in the version name segment. + from = String.format("%s/%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, locVersion, buildFileName()); + } + checkArgument( + from.startsWith(GITHUB_DOWNLOAD_PREFIX), + "Provided --prismLocation URL could not be resolved to a download URL. ", + options.getPrismLocation()); + return from; + } + /** * Downloads and prepares a Prism executable for use with the {@link PrismRunner}. The returned * {@link String} is the absolute path to the Prism executable. */ String resolve() throws IOException { - - String from = - String.format("%s/v%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, getSDKVersion(), buildFileName()); - - if (!Strings.isNullOrEmpty(options.getPrismLocation())) { - checkArgument( - !options.getPrismLocation().startsWith(GITHUB_TAG_PREFIX), - "Provided --prismLocation URL is not an Apache Beam Github " - + "Release page URL or download URL: ", - from); - - from = options.getPrismLocation(); - } + String from = resolveSource(); String fromFileName = getNameWithoutExtension(from); Path to = Paths.get(userHome(), PRISM_BIN_PATH, fromFileName); @@ -135,11 +162,6 @@ private String resolve(Path from, Path to) throws IOException { return to.toString(); } - String buildFileName() { - String version = getSDKVersion(); - return String.format("apache_beam-v%s-prism-%s-%s", version, os(), arch()); - } - private static void unzip(URL from, Path to) { try { unzip(from.openStream(), to); @@ -181,6 +203,11 @@ private static String getNameWithoutExtension(String path) { .getNameWithoutExtension(path); } + private String buildFileName() { + String version = getSDKVersion(); + return String.format("apache_beam-v%s-prism-%s-%s", version, os(), arch()); + } + private String getSDKVersion() { if (Strings.isNullOrEmpty(options.getPrismVersionOverride())) { return RELEASE_INFO.getSdkVersion(); diff --git a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java index 6a6ca4e615d0..9b280d0a70d4 100644 --- a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java +++ b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java @@ -32,8 +32,8 @@ public interface PrismPipelineOptions extends PortablePipelineOptions { @Description( "Path or URL to a prism binary, or zipped binary for the current " + "platform (Operating System and Architecture). May also be an Apache " - + "Beam Github Release page URL, with a matching --prismVersionOverride " - + "set. This option overrides all others for finding a prism binary.") + + "Beam Github Release page URL, which be used to construct download URL for " + + " the current platform. ") String getPrismLocation(); void setPrismLocation(String prismLocation); @@ -41,7 +41,8 @@ public interface PrismPipelineOptions extends PortablePipelineOptions { @Description( "Override the SDK's version for deriving the Github Release URLs for " + "downloading a zipped prism binary, for the current platform. If " - + "set to a Github Release page URL, then it will use that release page as a base when constructing the download URL.") + + "the --prismLocation flag is set to a Github Release page URL, " + + "then it will use that release page as a base when constructing the download URL.") String getPrismVersionOverride(); void setPrismVersionOverride(String prismVersionOverride); diff --git a/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java b/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java index 9054e9c99a04..095d3c9bde61 100644 --- a/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java +++ b/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java @@ -30,6 +30,7 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import org.apache.beam.sdk.options.PipelineOptionsFactory; +import org.apache.beam.sdk.util.ReleaseInfo; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -40,6 +41,7 @@ @RunWith(JUnit4.class) public class PrismLocatorTest { + private static final ReleaseInfo RELEASE_INFO = ReleaseInfo.getReleaseInfo(); private static final Path DESTINATION_DIRECTORY = prismBinDirectory(); @Before @@ -61,29 +63,66 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) } @Test - public void givenVersionOverride_thenResolves() throws IOException { - assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse(); + public void givenVersionOverride_thenResolvesLocation() throws IOException { + PrismPipelineOptions options = options(); + options.setPrismVersionOverride("2.57.0-RC1"); + + PrismLocator underTest = new PrismLocator(options); + String got = underTest.resolveSource(); + + assertThat(got) + .contains( + "https://github.com/apache/beam/releases/download/v" + RELEASE_INFO.getSdkVersion()); + assertThat(got).contains("apache_beam-v2.57.0-RC1-prism"); + assertThat(got).contains(".zip"); + } + + // This testcase validates a user override to download a different release version specifically. + @Test + public void givenHttpPrismLocationOption_thenResolvesLocation() throws IOException { + PrismPipelineOptions options = options(); + String want = + "https://github.com/apache/beam/releases/download/v2.57.0/apache_beam-v2.57.0-prism-darwin-arm64.zip"; + options.setPrismLocation(want); + + PrismLocator underTest = new PrismLocator(options); + String got = underTest.resolveSource(); + + assertThat(got).isEqualTo(want); + } + + // This testcase is the Release Validation behavior, where we provide an RC option, but + // need to resolve the download for the non-RC version. + // Copy the URL directly, and set the location, override the file's RC version with the final + // version. + @Test + public void givenRCGithubTagPrismLocationOption_thenResolvesLocation() { PrismPipelineOptions options = options(); + options.setPrismLocation("https://github.com/apache/beam/releases/tag/v2.57.0-RC1/"); options.setPrismVersionOverride("2.57.0"); + PrismLocator underTest = new PrismLocator(options); - String got = underTest.resolve(); - assertThat(got).contains(DESTINATION_DIRECTORY.toString()); - assertThat(got).contains("2.57.0"); - Path gotPath = Paths.get(got); - assertThat(Files.exists(gotPath)).isTrue(); + String got = underTest.resolveSource(); + + assertThat(got) + .contains( + "https://github.com/apache/beam/releases/download/v2.57.0-RC1/apache_beam-v2.57.0-prism"); + assertThat(got).contains(".zip"); } @Test - public void givenHttpPrismLocationOption_thenResolves() throws IOException { - assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse(); + public void givenRCGithubTagPrismLocationOptionNoTrailingSlash_thenResolvesLocation() { PrismPipelineOptions options = options(); - options.setPrismLocation( - "https://github.com/apache/beam/releases/download/v2.57.0/apache_beam-v2.57.0-prism-darwin-arm64.zip"); + options.setPrismLocation("https://github.com/apache/beam/releases/tag/v2.57.0-RC2"); + options.setPrismVersionOverride("2.57.0"); + PrismLocator underTest = new PrismLocator(options); - String got = underTest.resolve(); - assertThat(got).contains(DESTINATION_DIRECTORY.toString()); - Path gotPath = Paths.get(got); - assertThat(Files.exists(gotPath)).isTrue(); + String got = underTest.resolveSource(); + + assertThat(got) + .contains( + "https://github.com/apache/beam/releases/download/v2.57.0-RC2/apache_beam-v2.57.0-prism"); + assertThat(got).contains(".zip"); } @Test @@ -91,24 +130,27 @@ public void givenFilePrismLocationOption_thenResolves() throws IOException { assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse(); PrismPipelineOptions options = options(); options.setPrismLocation(getLocalPrismBuildOrIgnoreTest()); + PrismLocator underTest = new PrismLocator(options); String got = underTest.resolve(); + assertThat(got).contains(DESTINATION_DIRECTORY.toString()); Path gotPath = Paths.get(got); assertThat(Files.exists(gotPath)).isTrue(); } @Test - public void givenGithubTagPrismLocationOption_thenThrows() { + public void givenIncorrectGithubPrismLocationOption_thenThrows() { PrismPipelineOptions options = options(); + // This is an incorrect github download path. Downloads are under /download/ not tag. options.setPrismLocation( "https://github.com/apache/beam/releases/tag/v2.57.0/apache_beam-v2.57.0-prism-darwin-amd64.zip"); + PrismLocator underTest = new PrismLocator(options); - IllegalArgumentException error = - assertThrows(IllegalArgumentException.class, underTest::resolve); - assertThat(error.getMessage()) - .contains( - "Provided --prismLocation URL is not an Apache Beam Github Release page URL or download URL"); + + RuntimeException error = assertThrows(RuntimeException.class, underTest::resolve); + // Message should contain the incorrectly constructed download link. + assertThat(error.getMessage()).contains(".zip/apache_beam"); } @Test @@ -116,6 +158,7 @@ public void givenGithubTagPrismLocationOption_thenThrows() { public void givenPrismLocation404_thenThrows() { PrismPipelineOptions options = options(); options.setPrismLocation("https://example.com/i/dont/exist.zip"); + PrismLocator underTest = new PrismLocator(options); RuntimeException error = assertThrows(RuntimeException.class, underTest::resolve); assertThat(error.getMessage()).contains("NotFoundException"); diff --git a/website/www/site/content/en/blog/validate-beam-release.md b/website/www/site/content/en/blog/validate-beam-release.md index e4335530cfe3..31d0d6eed1a0 100644 --- a/website/www/site/content/en/blog/validate-beam-release.md +++ b/website/www/site/content/en/blog/validate-beam-release.md @@ -107,6 +107,37 @@ With that, the Beam version in your environment will be the latest release candidate, and you can go ahead and run your tests to verify that everything works well. +### Validating Prism Runner RC against RC SDKs + +Replace v2.59.0-RC1 with the tag of the RC version being validated. + +#### Python + +To validate the prism runner with Python, +`--runner=PrismRunner --prism_location=https://github.com/apache/beam/releases/tag/v2.59.0-RC1 --prism_beam_version_override=v2.59.0` + +* The `runner` flag sets Beam to use Prism. +* The `prism_location` sets the source of Prism assets. +* The `prism_beam_version_override` flag sets what those artifacts are labeled as. +* The assets are packaged as the final release version, so the override is required. + +#### Java + +For Gradle, add the Prism, and the JAMM depdendencies to your `build.gradle`. + +``` + implementation "org.apache.beam:beam-runners-prism-java:2.59.0" + implementation "com.github.jbellis:jamm:0.4.0" +``` + +Then add the following flags, substituting the version accordingly. + +`--runner=PrismRunner --prismLocation="https://github.com/apache/beam/releases/tag/v2.59.0-RC1/" --prismVersionOverride=v2.59.0 + +* The `runner` flag sets Beam to use Prism. +* The `prismLocation` sets the source of Prism assets, specifically the zip file of the version in question. + + ### Configuring a Go build to validate a Beam release candidate For Go SDK releases, you can fetch the Go SDK RC using [`go get`](https://golang.org/ref/mod#go-get),