Skip to content

Commit

Permalink
[prism][java] Update Prism locator to match Python SDK semantics. (ap…
Browse files Browse the repository at this point in the history 
…ache#32619)

* Update PrismLocator to match the python SDK semantics.

* Update validate beam release with prism RC validation instructions.

* rename resolveLocation to resolveSource

---------

Co-authored-by: lostluck <[email protected]>
  • Loading branch information
@lostluck
lostluck and lostluck authored Oct 2, 2024
1 parent 415fdd3 commit 36a8d39
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 45 deletions.
69 changes: 48 additions & 21 deletions runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,34 +54,61 @@ class PrismLocator {
private static final String PRISM_BIN_PATH = ".apache_beam/cache/prism/bin";
private static final Set<PosixFilePermission> PERMS =
PosixFilePermissions.fromString("rwxr-xr-x");
private static final String GITHUB_DOWNLOAD_PREFIX =
"https://github.com/apache/beam/releases/download";
private static final String GITHUB_TAG_PREFIX = "https://github.com/apache/beam/releases/tag";
private static final String GITHUB_COMMON_PREFIX = "https://github.com/apache/beam/releases/";
private static final String GITHUB_DOWNLOAD_PREFIX = GITHUB_COMMON_PREFIX + "download";
private static final String GITHUB_TAG_PREFIX = GITHUB_COMMON_PREFIX + "tag";

private final PrismPipelineOptions options;

PrismLocator(PrismPipelineOptions options) {
this.options = options;
}

String resolveSource() {
String from =
String.format(
"%s/v%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, RELEASE_INFO.getSdkVersion(), buildFileName());

if (Strings.isNullOrEmpty(options.getPrismLocation())) {
return from;
}
from = options.getPrismLocation();

// Likely a local file, return it directly.
if (!from.startsWith("http")) {
return from;
}

// Validate that it's from a safe location: A Beam Github Release
checkArgument(
options.getPrismLocation().startsWith(GITHUB_COMMON_PREFIX),
"Provided --prismLocation URL is not an Apache Beam Github "
+ "Release page URL or download URL: ",
options.getPrismLocation());

from = options.getPrismLocation();

// If this is the tag prefix, then build the release download with the version
// from the given url.
if (options.getPrismLocation().startsWith(GITHUB_TAG_PREFIX)) {
Path tagPath = Paths.get(options.getPrismLocation());
Path locVersion = tagPath.getName(tagPath.getNameCount() - 1);
// The "v" prefix is already included in the version name segment.
from = String.format("%s/%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, locVersion, buildFileName());
}
checkArgument(
from.startsWith(GITHUB_DOWNLOAD_PREFIX),
"Provided --prismLocation URL could not be resolved to a download URL. ",
options.getPrismLocation());
return from;
}

/**
* Downloads and prepares a Prism executable for use with the {@link PrismRunner}. The returned
* {@link String} is the absolute path to the Prism executable.
*/
String resolve() throws IOException {

String from =
String.format("%s/v%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, getSDKVersion(), buildFileName());

if (!Strings.isNullOrEmpty(options.getPrismLocation())) {
checkArgument(
!options.getPrismLocation().startsWith(GITHUB_TAG_PREFIX),
"Provided --prismLocation URL is not an Apache Beam Github "
+ "Release page URL or download URL: ",
from);

from = options.getPrismLocation();
}
String from = resolveSource();

String fromFileName = getNameWithoutExtension(from);
Path to = Paths.get(userHome(), PRISM_BIN_PATH, fromFileName);
Expand Down Expand Up @@ -135,11 +162,6 @@ private String resolve(Path from, Path to) throws IOException {
return to.toString();
}

String buildFileName() {
String version = getSDKVersion();
return String.format("apache_beam-v%s-prism-%s-%s", version, os(), arch());
}

private static void unzip(URL from, Path to) {
try {
unzip(from.openStream(), to);
Expand Down Expand Up @@ -181,6 +203,11 @@ private static String getNameWithoutExtension(String path) {
.getNameWithoutExtension(path);
}

private String buildFileName() {
String version = getSDKVersion();
return String.format("apache_beam-v%s-prism-%s-%s", version, os(), arch());
}

private String getSDKVersion() {
if (Strings.isNullOrEmpty(options.getPrismVersionOverride())) {
return RELEASE_INFO.getSdkVersion();
Expand Down
7 changes: 4 additions & 3 deletions runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,17 @@ public interface PrismPipelineOptions extends PortablePipelineOptions {
@Description(
"Path or URL to a prism binary, or zipped binary for the current "
+ "platform (Operating System and Architecture). May also be an Apache "
+ "Beam Github Release page URL, with a matching --prismVersionOverride "
+ "set. This option overrides all others for finding a prism binary.")
+ "Beam Github Release page URL, which be used to construct download URL for "
+ " the current platform. ")
String getPrismLocation();

void setPrismLocation(String prismLocation);

@Description(
"Override the SDK's version for deriving the Github Release URLs for "
+ "downloading a zipped prism binary, for the current platform. If "
+ "set to a Github Release page URL, then it will use that release page as a base when constructing the download URL.")
+ "the --prismLocation flag is set to a Github Release page URL, "
+ "then it will use that release page as a base when constructing the download URL.")
String getPrismVersionOverride();

void setPrismVersionOverride(String prismVersionOverride);
Expand Down
85 changes: 64 additions & 21 deletions runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.util.ReleaseInfo;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
Expand All @@ -40,6 +41,7 @@
@RunWith(JUnit4.class)
public class PrismLocatorTest {

private static final ReleaseInfo RELEASE_INFO = ReleaseInfo.getReleaseInfo();
private static final Path DESTINATION_DIRECTORY = prismBinDirectory();

@Before
Expand All @@ -61,61 +63,102 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
}

@Test
public void givenVersionOverride_thenResolves() throws IOException {
assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
public void givenVersionOverride_thenResolvesLocation() throws IOException {
PrismPipelineOptions options = options();
options.setPrismVersionOverride("2.57.0-RC1");

PrismLocator underTest = new PrismLocator(options);
String got = underTest.resolveSource();

assertThat(got)
.contains(
"https://github.com/apache/beam/releases/download/v" + RELEASE_INFO.getSdkVersion());
assertThat(got).contains("apache_beam-v2.57.0-RC1-prism");
assertThat(got).contains(".zip");
}

// This testcase validates a user override to download a different release version specifically.
@Test
public void givenHttpPrismLocationOption_thenResolvesLocation() throws IOException {
PrismPipelineOptions options = options();
String want =
"https://github.com/apache/beam/releases/download/v2.57.0/apache_beam-v2.57.0-prism-darwin-arm64.zip";
options.setPrismLocation(want);

PrismLocator underTest = new PrismLocator(options);
String got = underTest.resolveSource();

assertThat(got).isEqualTo(want);
}

// This testcase is the Release Validation behavior, where we provide an RC option, but
// need to resolve the download for the non-RC version.
// Copy the URL directly, and set the location, override the file's RC version with the final
// version.
@Test
public void givenRCGithubTagPrismLocationOption_thenResolvesLocation() {
PrismPipelineOptions options = options();
options.setPrismLocation("https://github.com/apache/beam/releases/tag/v2.57.0-RC1/");
options.setPrismVersionOverride("2.57.0");

PrismLocator underTest = new PrismLocator(options);
String got = underTest.resolve();
assertThat(got).contains(DESTINATION_DIRECTORY.toString());
assertThat(got).contains("2.57.0");
Path gotPath = Paths.get(got);
assertThat(Files.exists(gotPath)).isTrue();
String got = underTest.resolveSource();

assertThat(got)
.contains(
"https://github.com/apache/beam/releases/download/v2.57.0-RC1/apache_beam-v2.57.0-prism");
assertThat(got).contains(".zip");
}

@Test
public void givenHttpPrismLocationOption_thenResolves() throws IOException {
assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
public void givenRCGithubTagPrismLocationOptionNoTrailingSlash_thenResolvesLocation() {
PrismPipelineOptions options = options();
options.setPrismLocation(
"https://github.com/apache/beam/releases/download/v2.57.0/apache_beam-v2.57.0-prism-darwin-arm64.zip");
options.setPrismLocation("https://github.com/apache/beam/releases/tag/v2.57.0-RC2");
options.setPrismVersionOverride("2.57.0");

PrismLocator underTest = new PrismLocator(options);
String got = underTest.resolve();
assertThat(got).contains(DESTINATION_DIRECTORY.toString());
Path gotPath = Paths.get(got);
assertThat(Files.exists(gotPath)).isTrue();
String got = underTest.resolveSource();

assertThat(got)
.contains(
"https://github.com/apache/beam/releases/download/v2.57.0-RC2/apache_beam-v2.57.0-prism");
assertThat(got).contains(".zip");
}

@Test
public void givenFilePrismLocationOption_thenResolves() throws IOException {
assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
PrismPipelineOptions options = options();
options.setPrismLocation(getLocalPrismBuildOrIgnoreTest());

PrismLocator underTest = new PrismLocator(options);
String got = underTest.resolve();

assertThat(got).contains(DESTINATION_DIRECTORY.toString());
Path gotPath = Paths.get(got);
assertThat(Files.exists(gotPath)).isTrue();
}

@Test
public void givenGithubTagPrismLocationOption_thenThrows() {
public void givenIncorrectGithubPrismLocationOption_thenThrows() {
PrismPipelineOptions options = options();
// This is an incorrect github download path. Downloads are under /download/ not tag.
options.setPrismLocation(
"https://github.com/apache/beam/releases/tag/v2.57.0/apache_beam-v2.57.0-prism-darwin-amd64.zip");

PrismLocator underTest = new PrismLocator(options);
IllegalArgumentException error =
assertThrows(IllegalArgumentException.class, underTest::resolve);
assertThat(error.getMessage())
.contains(
"Provided --prismLocation URL is not an Apache Beam Github Release page URL or download URL");

RuntimeException error = assertThrows(RuntimeException.class, underTest::resolve);
// Message should contain the incorrectly constructed download link.
assertThat(error.getMessage()).contains(".zip/apache_beam");
}

@Test
@Ignore // TODO: use mock site. Currently failing with response code 500 instead of 404
public void givenPrismLocation404_thenThrows() {
PrismPipelineOptions options = options();
options.setPrismLocation("https://example.com/i/dont/exist.zip");

PrismLocator underTest = new PrismLocator(options);
RuntimeException error = assertThrows(RuntimeException.class, underTest::resolve);
assertThat(error.getMessage()).contains("NotFoundException");
Expand Down
31 changes: 31 additions & 0 deletions website/www/site/content/en/blog/validate-beam-release.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,37 @@ With that, the Beam version in your environment will be the latest release
candidate, and you can go ahead and run your tests to verify that everything
works well.

### Validating Prism Runner RC against RC SDKs

Replace v2.59.0-RC1 with the tag of the RC version being validated.

#### Python

To validate the prism runner with Python,
`--runner=PrismRunner --prism_location=https://github.com/apache/beam/releases/tag/v2.59.0-RC1 --prism_beam_version_override=v2.59.0`

* The `runner` flag sets Beam to use Prism.
* The `prism_location` sets the source of Prism assets.
* The `prism_beam_version_override` flag sets what those artifacts are labeled as.
* The assets are packaged as the final release version, so the override is required.

#### Java

For Gradle, add the Prism, and the JAMM depdendencies to your `build.gradle`.

```
implementation "org.apache.beam:beam-runners-prism-java:2.59.0"
implementation "com.github.jbellis:jamm:0.4.0"
```

Then add the following flags, substituting the version accordingly.

`--runner=PrismRunner --prismLocation="https://github.com/apache/beam/releases/tag/v2.59.0-RC1/" --prismVersionOverride=v2.59.0

* The `runner` flag sets Beam to use Prism.
* The `prismLocation` sets the source of Prism assets, specifically the zip file of the version in question.


### Configuring a Go build to validate a Beam release candidate

For Go SDK releases, you can fetch the Go SDK RC using [`go get`](https://golang.org/ref/mod#go-get),
Expand Down

0 comments on commit 36a8d39

Please sign in to comment.