[prism][java] Update Prism locator to match Python SDK semantics. (#32619)

lostluck · web-flow · commit 36a8d39a3de4 · 2024-10-01T19:31:06.000-07:00
* Update PrismLocator to match the python SDK semantics.

* Update validate beam release with prism RC validation instructions.

* rename resolveLocation to resolveSource

---------

Co-authored-by: lostluck &lt;13907733+lostluck@users.noreply.github.com&gt;
diff --git a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java
@@ -54,34 +54,61 @@ class PrismLocator {
   private static final String PRISM_BIN_PATH = ".apache_beam/cache/prism/bin";
   private static final Set<PosixFilePermission> PERMS =
       PosixFilePermissions.fromString("rwxr-xr-x");
-  private static final String GITHUB_DOWNLOAD_PREFIX =
-      "https://github.com/apache/beam/releases/download";
-  private static final String GITHUB_TAG_PREFIX = "https://github.com/apache/beam/releases/tag";
+  private static final String GITHUB_COMMON_PREFIX = "https://github.com/apache/beam/releases/";
+  private static final String GITHUB_DOWNLOAD_PREFIX = GITHUB_COMMON_PREFIX + "download";
+  private static final String GITHUB_TAG_PREFIX = GITHUB_COMMON_PREFIX + "tag";
 
   private final PrismPipelineOptions options;
 
   PrismLocator(PrismPipelineOptions options) {
     this.options = options;
   }
 
+  String resolveSource() {
+    String from =
+        String.format(
+            "%s/v%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, RELEASE_INFO.getSdkVersion(), buildFileName());
+
+    if (Strings.isNullOrEmpty(options.getPrismLocation())) {
+      return from;
+    }
+    from = options.getPrismLocation();
+
+    // Likely a local file, return it directly.
+    if (!from.startsWith("http")) {
+      return from;
+    }
+
+    // Validate that it's from a safe location: A Beam Github Release
+    checkArgument(
+        options.getPrismLocation().startsWith(GITHUB_COMMON_PREFIX),
+        "Provided --prismLocation URL is not an Apache Beam Github "
+            + "Release page URL or download URL: ",
+        options.getPrismLocation());
+
+    from = options.getPrismLocation();
+
+    // If this is the tag prefix, then build the release download with the version
+    // from the given url.
+    if (options.getPrismLocation().startsWith(GITHUB_TAG_PREFIX)) {
+      Path tagPath = Paths.get(options.getPrismLocation());
+      Path locVersion = tagPath.getName(tagPath.getNameCount() - 1);
+      // The "v" prefix is already included in the version name segment.
+      from = String.format("%s/%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, locVersion, buildFileName());
+    }
+    checkArgument(
+        from.startsWith(GITHUB_DOWNLOAD_PREFIX),
+        "Provided --prismLocation URL could not be resolved to a download URL. ",
+        options.getPrismLocation());
+    return from;
+  }
+
   /**
    * Downloads and prepares a Prism executable for use with the {@link PrismRunner}. The returned
    * {@link String} is the absolute path to the Prism executable.
    */
   String resolve() throws IOException {
-
-    String from =
-        String.format("%s/v%s/%s.zip", GITHUB_DOWNLOAD_PREFIX, getSDKVersion(), buildFileName());
-
-    if (!Strings.isNullOrEmpty(options.getPrismLocation())) {
-      checkArgument(
-          !options.getPrismLocation().startsWith(GITHUB_TAG_PREFIX),
-          "Provided --prismLocation URL is not an Apache Beam Github "
-              + "Release page URL or download URL: ",
-          from);
-
-      from = options.getPrismLocation();
-    }
+    String from = resolveSource();
 
     String fromFileName = getNameWithoutExtension(from);
     Path to = Paths.get(userHome(), PRISM_BIN_PATH, fromFileName);
@@ -135,11 +162,6 @@ private String resolve(Path from, Path to) throws IOException {
     return to.toString();
   }
 
-  String buildFileName() {
-    String version = getSDKVersion();
-    return String.format("apache_beam-v%s-prism-%s-%s", version, os(), arch());
-  }
-
   private static void unzip(URL from, Path to) {
     try {
       unzip(from.openStream(), to);
@@ -181,6 +203,11 @@ private static String getNameWithoutExtension(String path) {
         .getNameWithoutExtension(path);
   }
 
+  private String buildFileName() {
+    String version = getSDKVersion();
+    return String.format("apache_beam-v%s-prism-%s-%s", version, os(), arch());
+  }
+
   private String getSDKVersion() {
     if (Strings.isNullOrEmpty(options.getPrismVersionOverride())) {
       return RELEASE_INFO.getSdkVersion();
diff --git a/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java b/runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismPipelineOptions.java
@@ -32,16 +32,17 @@ public interface PrismPipelineOptions extends PortablePipelineOptions {
   @Description(
       "Path or URL to a prism binary, or zipped binary for the current "
           + "platform (Operating System and Architecture). May also be an Apache "
-          + "Beam Github Release page URL, with a matching --prismVersionOverride "
-          + "set. This option overrides all others for finding a prism binary.")
+          + "Beam Github Release page URL, which be used to construct  download URL for "
+          + " the current platform. ")
   String getPrismLocation();
 
   void setPrismLocation(String prismLocation);
 
   @Description(
       "Override the SDK's version for deriving the Github Release URLs for "
           + "downloading a zipped prism binary, for the current platform. If "
-          + "set to a Github Release page URL, then it will use that release page as a base when constructing the download URL.")
+          + "the --prismLocation flag is set to a Github Release page URL, "
+          + "then it will use that release page as a base when constructing the download URL.")
   String getPrismVersionOverride();
 
   void setPrismVersionOverride(String prismVersionOverride);
diff --git a/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java b/runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java
@@ -30,6 +30,7 @@
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
+import org.apache.beam.sdk.util.ReleaseInfo;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -40,6 +41,7 @@
 @RunWith(JUnit4.class)
 public class PrismLocatorTest {
 
+  private static final ReleaseInfo RELEASE_INFO = ReleaseInfo.getReleaseInfo();
   private static final Path DESTINATION_DIRECTORY = prismBinDirectory();
 
   @Before
@@ -61,61 +63,102 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
   }
 
   @Test
-  public void givenVersionOverride_thenResolves() throws IOException {
-    assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
+  public void givenVersionOverride_thenResolvesLocation() throws IOException {
+    PrismPipelineOptions options = options();
+    options.setPrismVersionOverride("2.57.0-RC1");
+
+    PrismLocator underTest = new PrismLocator(options);
+    String got = underTest.resolveSource();
+
+    assertThat(got)
+        .contains(
+            "https://github.com/apache/beam/releases/download/v" + RELEASE_INFO.getSdkVersion());
+    assertThat(got).contains("apache_beam-v2.57.0-RC1-prism");
+    assertThat(got).contains(".zip");
+  }
+
+  // This testcase validates a user override to download a different release version specifically.
+  @Test
+  public void givenHttpPrismLocationOption_thenResolvesLocation() throws IOException {
+    PrismPipelineOptions options = options();
+    String want =
+        "https://github.com/apache/beam/releases/download/v2.57.0/apache_beam-v2.57.0-prism-darwin-arm64.zip";
+    options.setPrismLocation(want);
+
+    PrismLocator underTest = new PrismLocator(options);
+    String got = underTest.resolveSource();
+
+    assertThat(got).isEqualTo(want);
+  }
+
+  // This testcase is the Release Validation behavior, where we provide an RC option, but
+  // need to resolve the download for the non-RC version.
+  // Copy the URL directly, and set the location, override the file's RC version with the final
+  // version.
+  @Test
+  public void givenRCGithubTagPrismLocationOption_thenResolvesLocation() {
     PrismPipelineOptions options = options();
+    options.setPrismLocation("https://github.com/apache/beam/releases/tag/v2.57.0-RC1/");
     options.setPrismVersionOverride("2.57.0");
+
     PrismLocator underTest = new PrismLocator(options);
-    String got = underTest.resolve();
-    assertThat(got).contains(DESTINATION_DIRECTORY.toString());
-    assertThat(got).contains("2.57.0");
-    Path gotPath = Paths.get(got);
-    assertThat(Files.exists(gotPath)).isTrue();
+    String got = underTest.resolveSource();
+
+    assertThat(got)
+        .contains(
+            "https://github.com/apache/beam/releases/download/v2.57.0-RC1/apache_beam-v2.57.0-prism");
+    assertThat(got).contains(".zip");
   }
 
   @Test
-  public void givenHttpPrismLocationOption_thenResolves() throws IOException {
-    assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
+  public void givenRCGithubTagPrismLocationOptionNoTrailingSlash_thenResolvesLocation() {
     PrismPipelineOptions options = options();
-    options.setPrismLocation(
-        "https://github.com/apache/beam/releases/download/v2.57.0/apache_beam-v2.57.0-prism-darwin-arm64.zip");
+    options.setPrismLocation("https://github.com/apache/beam/releases/tag/v2.57.0-RC2");
+    options.setPrismVersionOverride("2.57.0");
+
     PrismLocator underTest = new PrismLocator(options);
-    String got = underTest.resolve();
-    assertThat(got).contains(DESTINATION_DIRECTORY.toString());
-    Path gotPath = Paths.get(got);
-    assertThat(Files.exists(gotPath)).isTrue();
+    String got = underTest.resolveSource();
+
+    assertThat(got)
+        .contains(
+            "https://github.com/apache/beam/releases/download/v2.57.0-RC2/apache_beam-v2.57.0-prism");
+    assertThat(got).contains(".zip");
   }
 
   @Test
   public void givenFilePrismLocationOption_thenResolves() throws IOException {
     assertThat(Files.exists(DESTINATION_DIRECTORY)).isFalse();
     PrismPipelineOptions options = options();
     options.setPrismLocation(getLocalPrismBuildOrIgnoreTest());
+
     PrismLocator underTest = new PrismLocator(options);
     String got = underTest.resolve();
+
     assertThat(got).contains(DESTINATION_DIRECTORY.toString());
     Path gotPath = Paths.get(got);
     assertThat(Files.exists(gotPath)).isTrue();
   }
 
   @Test
-  public void givenGithubTagPrismLocationOption_thenThrows() {
+  public void givenIncorrectGithubPrismLocationOption_thenThrows() {
     PrismPipelineOptions options = options();
+    // This is an incorrect github download path. Downloads are under /download/ not tag.
     options.setPrismLocation(
         "https://github.com/apache/beam/releases/tag/v2.57.0/apache_beam-v2.57.0-prism-darwin-amd64.zip");
+
     PrismLocator underTest = new PrismLocator(options);
-    IllegalArgumentException error =
-        assertThrows(IllegalArgumentException.class, underTest::resolve);
-    assertThat(error.getMessage())
-        .contains(
-            "Provided --prismLocation URL is not an Apache Beam Github Release page URL or download URL");
+
+    RuntimeException error = assertThrows(RuntimeException.class, underTest::resolve);
+    // Message should contain the incorrectly constructed download link.
+    assertThat(error.getMessage()).contains(".zip/apache_beam");
   }
 
   @Test
   @Ignore // TODO: use mock site. Currently failing with response code 500 instead of 404
   public void givenPrismLocation404_thenThrows() {
     PrismPipelineOptions options = options();
     options.setPrismLocation("https://example.com/i/dont/exist.zip");
+
     PrismLocator underTest = new PrismLocator(options);
     RuntimeException error = assertThrows(RuntimeException.class, underTest::resolve);
     assertThat(error.getMessage()).contains("NotFoundException");
diff --git a/website/www/site/content/en/blog/validate-beam-release.md b/website/www/site/content/en/blog/validate-beam-release.md
@@ -107,6 +107,37 @@ With that, the Beam version in your environment will be the latest release
 candidate, and you can go ahead and run your tests to verify that everything
 works well.
 
+### Validating Prism Runner RC against RC SDKs
+
+Replace v2.59.0-RC1 with the tag of the RC version being validated.
+
+#### Python
+
+To validate the prism runner with Python,
+`--runner=PrismRunner --prism_location=https://github.com/apache/beam/releases/tag/v2.59.0-RC1 --prism_beam_version_override=v2.59.0`
+
+* The `runner` flag sets Beam to use Prism.
+* The `prism_location` sets the source of Prism assets.
+* The `prism_beam_version_override` flag sets what those artifacts are labeled as.
+* The assets are packaged as the final release version, so the override is required.
+
+#### Java
+
+For Gradle, add the Prism, and the JAMM depdendencies to your `build.gradle`.
+
+```
+    implementation "org.apache.beam:beam-runners-prism-java:2.59.0"
+    implementation "com.github.jbellis:jamm:0.4.0"
+```
+
+Then add the following flags, substituting the version accordingly.
+
+`--runner=PrismRunner --prismLocation="https://github.com/apache/beam/releases/tag/v2.59.0-RC1/" --prismVersionOverride=v2.59.0
+
+* The `runner` flag sets Beam to use Prism.
+* The `prismLocation` sets the source of Prism assets, specifically the zip file of the version in question.
+
+
 ### Configuring a Go build to validate a Beam release candidate
 
 For Go SDK releases, you can fetch the Go SDK RC using [`go get`](https://golang.org/ref/mod#go-get),
