feat: bake gapic-generator-java into the hermetic build docker image

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -12,11 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# install gapic-generator-java in a separate layer so we don't overload the image
+# with the transferred source code and jars
+FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build
+
+WORKDIR /sdk-platform-java
+COPY . .
+# {x-version-update-start:gapic-generator-java:current}
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.42.1-SNAPSHOT" 
+
+RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip
+RUN ls "/root/.m2/repository/com/google/api/gapic-generator-java/" 
+RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" .
+
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
 SHELL [ "/bin/bash", "-c" ]
 
+
 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ARG PROTOC_VERSION=25.3
 ARG GRPC_VERSION=1.65.1
@@ -47,7 +61,16 @@ RUN source /src/utils/utilities.sh \
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
 ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 
-# use python 3.11 (the base image has several python versions; here we define the default one)
+
+# we transfer gapic-generator-java from the previous stage.
+# here we redeclare this env var since they are not preserved between stages
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.42.1-SNAPSHOT" 
+# {x-version-update-end:gapic-generator-java:current}
+ENV DOCKER_GAPIC_GENERATOR_LOCATION="/gapic-generator-java/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar"
+COPY --from=ggj-build "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" "${DOCKER_GAPIC_GENERATOR_LOCATION}"
+RUN chmod 755 "${DOCKER_GAPIC_GENERATOR_LOCATION}"
+
+#  use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3

.github/scripts/hermetic_library_generation.sh

@@ -90,9 +90,6 @@ fi
 git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}"
 config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true)
 
-generator_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java)
-echo "Local generator version: ${generator_version}"
-
 # install generator locally since we're using a SNAPSHOT version.
 mvn -V -B -ntp clean install -DskipTests
 

library_generation/generate_library.sh

@@ -16,8 +16,6 @@ case $key in
     ;;
   --gapic_generator_version)
     gapic_generator_version="$2"
-    # export this variable so that it can be used in gapic-generator-java-wrapper.sh
-    export gapic_generator_version
     shift
     ;;
   --protoc_version)
@@ -77,7 +75,7 @@ script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}"/utils/utilities.sh
 output_folder="$(get_output_folder)"
 
-if [ -z "${gapic_generator_version}" ]; then
+if [[ -z "${gapic_generator_version}" ]] ; then
   echo 'missing required argument --gapic_generator_version'
   exit 1
 fi
@@ -126,6 +124,9 @@ if [ -z "${os_architecture}" ]; then
   os_architecture=$(detect_os_architecture)
 fi
 
+# export this variable so that it can be used in gapic-generator-java-wrapper.sh
+export gapic_generator_version
+
 temp_destination_path="${output_folder}/temp_preprocessed"
 mkdir -p "${output_folder}/${destination_path}"
 if [ -d "${temp_destination_path}" ]; then

library_generation/model/generation_config.py

@@ -25,7 +25,7 @@
 COMMON_PROTOS_LIBRARY_NAME = "common-protos"
 GAPIC_GENERATOR_VERSION = "gapic_generator_version"
 LIBRARIES_BOM_VERSION = "libraries_bom_version"
-GENERATOR_VERSION_ENV_KEY = "GENERATOR_VERSION"
+GENERATOR_VERSION_ENV_KEY = "DOCKER_GAPIC_GENERATOR_VERSION"
 
 
 class GenerationConfig:
@@ -90,9 +90,11 @@ def __set_generator_version(gapic_generator_version: Optional[str]) -> str:
         gapic_generator_version = os.getenv(GENERATOR_VERSION_ENV_KEY)
         if not gapic_generator_version:
             raise ValueError(
-                f"Environment variable {GENERATOR_VERSION_ENV_KEY}"
-                f" is not set when the generator version is not"
-                f" specified in the generation config."
+                f"gapic_generator_version was not specified in the "
+                f"configuration yaml and the fall-back env var "
+                f"{GENERATOR_VERSION_ENV_KEY} was not found. At "
+                f"least one of them must be set to proceed with the "
+                f"generation."
             )
         return gapic_generator_version
 

library_generation/test/generate_library_unit_tests.sh

@@ -212,6 +212,50 @@ download_tools_succeed_with_baked_grpc() {
   unset grpc_path
 }
 
+download_tools_succeed_with_baked_generator() {
+  # This test has the same structure as
+  # download_tools_succeed_with_baked_protoc, but meant for
+  # gapic-generator-java.
+  local test_dir=$(mktemp -d)
+  pushd "${test_dir}"
+  generator_folder=$(mktemp -d)
+  touch "${generator_folder}/gapic-generator-java.fakejar"
+  export DOCKER_GAPIC_GENERATOR_LOCATION="${generator_folder}/gapic-generator-java.fakejar"
+  export DOCKER_GAPIC_GENERATOR_VERSION="99.99"
+  export output_folder=$(get_output_folder)
+  mkdir "${output_folder}"
+
+  local test_protoc_version="1.64.0"
+  local test_grpc_version="1.64.0"
+  # we expect download_tools to decide to use DOCKER_GAPIC_GENERATOR_LOCATION because
+  # the protoc version we want to download is the same as DOCKER_GRPC_VERSION
+  log=$(download_tools "99.99" "${test_protoc_version}" "${test_grpc_version}" "linux-x86_64" 2>&1 1>/dev/null)
+
+  # the assertion functions are designed to be called only once per function.
+  # Here we do two manual tests that are then coverged into a single
+  # pseudo-boolean
+  has_expected_log="false"
+  has_expected_file="false"
+  is_output_correct="false"
+  if grep -q "Using gapic-generator-java version baked into the container: 99.99" <<< "${log}"; then
+    has_expected_log="true"
+  fi
+  if [[ -f "${output_folder}/gapic-generator-java.fakejar" ]]; then
+    has_expected_file="true"
+  fi
+  if [[ "${has_expected_log}" == "true" ]] && [[ "${has_expected_file}" == "true" ]]; then
+    is_output_correct="true"
+  fi
+
+  assertEquals "true" "${is_output_correct}"
+
+  rm -rdf "${output_folder}"
+  unset DOCKER_GRPC_LOCATION
+  unset DOCKER_GRPC_VERSION
+  unset output_folder
+  unset grpc_path
+}
+
 download_grpc_plugin_succeed_with_valid_version_linux_test() {
   download_grpc_plugin "1.55.1" "linux-x86_64"
   assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe"
@@ -350,6 +394,7 @@ test_list=(
   download_protoc_failed_with_invalid_arch_test
   download_tools_succeed_with_baked_protoc
   download_tools_succeed_with_baked_grpc
+  download_tools_succeed_with_baked_generator
   download_grpc_plugin_succeed_with_valid_version_linux_test
   download_grpc_plugin_succeed_with_valid_version_macos_test
   download_grpc_plugin_failed_with_invalid_version_linux_test

library_generation/test/model/generation_config_unit_tests.py

@@ -56,20 +56,20 @@ def test_generation_config_default_value(self):
     def test_generation_config_with_generator_version_env_raise_exception(self):
         self.assertRaisesRegex(
             ValueError,
-            "Environment variable GENERATOR_VERSION is not set",
+            "the fall-back env var DOCKER_GAPIC_GENERATOR_VERSION was not found",
             GenerationConfig,
             googleapis_commitish="",
             libraries=[],
         )
 
     def test_generation_config_set_generator_version_from_env(self):
-        os.environ["GENERATOR_VERSION"] = "1.2.3"
+        os.environ["DOCKER_GAPIC_GENERATOR_VERSION"] = "1.2.3"
         config = GenerationConfig(
             googleapis_commitish="",
             libraries=[],
         )
         self.assertEqual("1.2.3", config.gapic_generator_version)
-        os.environ.pop("GENERATOR_VERSION")
+        os.environ.pop("DOCKER_GAPIC_GENERATOR_VERSION")
 
     def test_from_yaml_succeeds(self):
         config = from_yaml(f"{test_config_dir}/generation_config.yaml")

library_generation/test/test_utilities.sh

@@ -27,7 +27,7 @@ __test_failed() {
 }
 
 
-############# Functions used in test execution #############
+############# Functions used in test execution. They can only be called once per test #############
 
 assertEquals() {
   local expected=$1

library_generation/utils/utilities.sh

@@ -158,7 +158,6 @@ download_tools() {
   local grpc_version=$3
   local os_architecture=$4
   pushd "${output_folder}"
-  download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-${gapic_generator_version}.jar"
 
   # the variable protoc_path is used in generate_library.sh. It is explicitly
   # exported to make clear that it is used outside this utilities file.
@@ -179,6 +178,19 @@ download_tools() {
     export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}")
   fi
 
+  # similar case with gapic-generator-java
+  if [[ "${DOCKER_GAPIC_GENERATOR_VERSION}" == "${gapic_generator_version}" ]]; then
+    # if the specified gapic_generator_version matches the one baked in the docker
+    # container, we copy the generator jar into the output folder so it can be
+    # picked up by gapic-generator-java-wrapper
+    >&2 echo "Using gapic-generator-java version baked into the container: ${DOCKER_GAPIC_GENERATOR_VERSION}"
+    cp "${DOCKER_GAPIC_GENERATOR_LOCATION}" "${output_folder}"
+  else
+    download_generator_artifact \
+      "${gapic_generator_version}" \
+      "gapic-generator-java-${gapic_generator_version}.jar"
+  fi
+
   popd
 }