Upgrade GCP BOM to 26.32.0 (#30335)

* Upgrade GCP BOM to 26.32.0

* trigger postcommits

* rerun script fix rebase

* Remove no longer exist method

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -605,12 +605,12 @@ class BeamModulePlugin implements Plugin<Project> {
     def dbcp2_version = "2.9.0"
     def errorprone_version = "2.10.0"
     // [bomupgrader] determined by: com.google.api:gax, consistent with: google_cloud_platform_libraries_bom
-    def gax_version = "2.41.0"
+    def gax_version = "2.42.0"
     def google_ads_version = "26.0.0"
     def google_clients_version = "2.0.0"
     def google_cloud_bigdataoss_version = "2.2.16"
     // [bomupgrader] determined by: com.google.cloud:google-cloud-spanner, consistent with: google_cloud_platform_libraries_bom
-    def google_cloud_spanner_version = "6.57.0"
+    def google_cloud_spanner_version = "6.58.0"
     def google_code_gson_version = "2.10.1"
     def google_oauth_clients_version = "1.34.1"
     // [bomupgrader] determined by: io.grpc:grpc-netty, consistent with: google_cloud_platform_libraries_bom
@@ -734,15 +734,12 @@ class BeamModulePlugin implements Plugin<Project> {
         google_api_client_jackson2                  : "com.google.api-client:google-api-client-jackson2:$google_clients_version",
         google_api_client_java6                     : "com.google.api-client:google-api-client-java6:$google_clients_version",
         google_api_common                           : "com.google.api:api-common", // google_cloud_platform_libraries_bom sets version
-        // Keep version consistent with the version in google_cloud_bigquery, managed by google_cloud_platform_libraries_bom
-        google_api_services_bigquery                : "com.google.apis:google-api-services-bigquery:v2-rev20230812-$google_clients_version",
-        // Keep version consistent with the version in google_cloud_resourcemanager, managed by google_cloud_platform_libraries_bom
-        google_api_services_cloudresourcemanager    : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20230806-$google_clients_version",
+        google_api_services_bigquery                : "com.google.apis:google-api-services-bigquery:v2-rev20240124-2.0.0",  // [bomupgrader] sets version
+        google_api_services_cloudresourcemanager    : "com.google.apis:google-api-services-cloudresourcemanager:v1-rev20240128-2.0.0",  // [bomupgrader] sets version
         google_api_services_dataflow                : "com.google.apis:google-api-services-dataflow:v1b3-rev20240113-$google_clients_version",
         google_api_services_healthcare              : "com.google.apis:google-api-services-healthcare:v1-rev20240130-$google_clients_version",
         google_api_services_pubsub                  : "com.google.apis:google-api-services-pubsub:v1-rev20220904-$google_clients_version",
-        // Keep version consistent with the version in google_cloud_nio, managed by google_cloud_platform_libraries_bom
-        google_api_services_storage                 : "com.google.apis:google-api-services-storage:v1-rev20231202-$google_clients_version",
+        google_api_services_storage                 : "com.google.apis:google-api-services-storage:v1-rev20240205-2.0.0",  // [bomupgrader] sets version
         google_auth_library_credentials             : "com.google.auth:google-auth-library-credentials", // google_cloud_platform_libraries_bom sets version
         google_auth_library_oauth2_http             : "com.google.auth:google-auth-library-oauth2-http", // google_cloud_platform_libraries_bom sets version
         google_cloud_bigquery                       : "com.google.cloud:google-cloud-bigquery", // google_cloud_platform_libraries_bom sets version
@@ -754,14 +751,13 @@ class BeamModulePlugin implements Plugin<Project> {
         google_cloud_core_grpc                      : "com.google.cloud:google-cloud-core-grpc", // google_cloud_platform_libraries_bom sets version
         google_cloud_datacatalog_v1beta1            : "com.google.cloud:google-cloud-datacatalog", // google_cloud_platform_libraries_bom sets version
         google_cloud_dataflow_java_proto_library_all: "com.google.cloud.dataflow:google-cloud-dataflow-java-proto-library-all:0.5.160304",
-        // Keep version consistent with the version in google_cloud_datastore, managed by google_cloud_platform_libraries_bom
-        google_cloud_datastore_v1_proto_client      : "com.google.cloud.datastore:datastore-v1-proto-client:2.17.1",
+        google_cloud_datastore_v1_proto_client      : "com.google.cloud.datastore:datastore-v1-proto-client:2.18.3",   // [bomupgrader] sets version
         google_cloud_firestore                      : "com.google.cloud:google-cloud-firestore", // google_cloud_platform_libraries_bom sets version
         google_cloud_pubsub                         : "com.google.cloud:google-cloud-pubsub", // google_cloud_platform_libraries_bom sets version
         google_cloud_pubsublite                     : "com.google.cloud:google-cloud-pubsublite",  // google_cloud_platform_libraries_bom sets version
         // [bomupgrader] the BOM version is set by scripts/tools/bomupgrader.py. If update manually, also update
         // libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml
-        google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.31.0",
+        google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:26.32.0",
         google_cloud_spanner                        : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version
         google_cloud_spanner_test                   : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests",
         google_code_gson                            : "com.google.code.gson:gson:$google_code_gson_version",

scripts/tools/bomupgrader.py

@@ -57,7 +57,10 @@ class BeamModulePluginProcessor:
   # dependencies managed by GCP-BOM that used the dependencies in KNOWN_DEPS
   # So we need to add it to the example project to get the version to sync
   OTHER_CONSTRANTS = [
-      "com.google.cloud:google-cloud-bigquery"  # uses arrow
+      "com.google.cloud:google-cloud-bigquery",  # for arrow
+      'com.google.cloud:google-cloud-nio',  # for google-api-services-storage
+      'com.google.cloud:google-cloud-resourcemanager',  # for google-api-services-cloudresourcemanager
+      'com.google.cloud:google-cloud-datastore',  # for google-cloud-dataflow-java-proto-library-all
   ]
 
   # TODO: the logic can be generalized to support multiple BOM
@@ -66,6 +69,9 @@ class BeamModulePluginProcessor:
   )
   # e.g.  def grpc_version = "1.61.0"
   VERSION_STRING = re.compile(r'^\s*def (\w+)_version\s*=\s*[\'"](\S+)[\'"]')
+  SINGLE_VERSION_STRING = re.compile(
+      r'.+:\s*[\'"]([\w\-.]+:[\w\-.]+):(.+)[\'"],\s*//\s*\[bomupgrader\] sets version'
+  )
   BOM_VERSION_STRING = re.compile(
       r'\s*google_cloud_platform_libraries_bom\s*:\s*[\'"]com\.google\.cloud:libraries-bom:([0-9\.]+)[\'"],?'
   )
@@ -92,7 +98,24 @@ def __init__(self, bom_version, project_root='.', runnable=None):
     # e.g. {"io.grpc:grpc-netty", "1.61.0"}
     self.dep_versions = {}
     self.dep_versions_current = {}
-    self.known_deps = {}
+    # dependencies managed by bomupgrader. They are declared inline in BeamModulePlugin,
+    # different from KNOWN_DEPS which first define a version
+    self.set_deps = {}
+    self.set_deps_current = {}
+
+  @staticmethod
+  def resolve_actual_dep(line, id):
+    """Resolve actual dependency from dependencyTree line"""
+    idx = line.find(id + ':')
+    if idx == -1: return ""  # not found
+    dep_and_other = line[idx + len(id) + 1:].split()
+    try:
+      jdx = dep_and_other.index('->')
+      ver = dep_and_other[jdx + 1]
+    except ValueError:
+      # there might be multiple ':', e.g. come.group.id:some-package:test:1.2.3
+      ver = dep_and_other[0].split(':')[-1]
+    return ver
 
   def check_dependencies(self):
     """Check dependencies in KNOWN_DEPS are found in BeamModulePlugin, and vice versa."""
@@ -107,6 +130,11 @@ def check_dependencies(self):
               "Version definition not found after anchor comment. Try standardize it."
           )
         found_deps[n.group(1)] = n.group(2)
+        continue
+      m = self.SINGLE_VERSION_STRING.match(line)
+      if m:
+        self.set_deps_current[m.group(1)] = m.group(2)
+
     assert sorted(self.KNOWN_DEPS.keys()) == sorted(found_deps.keys()), f"expect {self.KNOWN_DEPS.keys()} == {found_deps.keys()}"
     self.dep_versions_current = {
         self.KNOWN_DEPS[k]: v
@@ -122,7 +150,8 @@ def prepare_gradle(self, bom_version):
         raise
 
     deps = []
-    for dep in list(self.KNOWN_DEPS.values()) + self.OTHER_CONSTRANTS:
+    for dep in list(self.KNOWN_DEPS.values()) + self.OTHER_CONSTRANTS + list(
+        self.set_deps_current.keys()):
       deps.append(f"implementation '{dep}'")
     gradle_file = self.GRADLE_TEMPLATE % (bom_version, "\n".join(deps))
     with open(os.path.join(self.BUILD_DIR, 'build.gradle'), 'w') as fout:
@@ -145,26 +174,34 @@ def resolve(self):
     result = subp.stdout.decode('utf-8')
     # example line: |    +--- com.google.guava:guava:32.1.3-android -> 32.1.3-jre (*)
     logging.debug(result)
+    get_dep_line = re.compile('\s+([\w\-.]+:[\w\-.]+):(.+)')
+
     for line in result.splitlines():
+      # search self.set_deps version
+      m = get_dep_line.search(line)
+      if m and m.group(1) in self.set_deps_current:
+        ver = self.resolve_actual_dep(line, m.group(1))
+        self.set_deps[m.group(1)] = ver
+        continue
+
+      # search KNOWN_DEPS version
       for id in self.KNOWN_DEPS.values():
-        idx = line.find(id + ':')
-        if idx == -1:
-          continue
-        dep_and_other = line[idx + len(id) + 1:].split()
-        try:
-          jdx = dep_and_other.index('->')
-          ver = dep_and_other[jdx + 1]
-        except ValueError:
-          # there might be multiple ':', e.g. come.group.id:some-package:test:1.2.3
-          ver = dep_and_other[0].split(':')[-1]
-        self.dep_versions[id] = ver
-        break
+        if id in self.dep_versions: continue
+        ver = self.resolve_actual_dep(line, id)
+        if ver:
+          self.dep_versions[id] = ver
+          break
 
     if len(self.dep_versions) < len(self.KNOWN_DEPS):
       logging.warning(
           "Warning: not all dependencies are resolved: %s", self.dep_versions)
       logging.info(result)
 
+    if len(self.set_deps) < len(self.set_deps_current):
+      logging.warning(
+          "Warning: not all dependencies are resolved: %s", self.set_deps)
+      logging.info(result)
+
   def write_back(self):
     logging.info("-----Update BeamModulePlugin-----")
     # make a shallow copy
@@ -188,13 +225,26 @@ def write_back(self):
           logging.info('Changed %s: %s -> %s', id, old_v, new_v)
         else:
           logging.info('Unchanged: %s:%s', id, new_v)
-      else:
-        # replace GCP BOM version
-        n = self.BOM_VERSION_STRING.match(line)
-        if n:
+        continue
+
+      # single_ver replace
+      m = self.SINGLE_VERSION_STRING.match(line)
+      if m:
+        id = m.group(1)
+        old_v = m.group(2)
+        new_v = self.set_deps[id]
+        if new_v != old_v:
           self.target_lines[idx] = self.original_lines[idx].replace(
-              n.group(1), self.bom_version)
-          found_bom = True
+              old_v, new_v)
+          logging.info('Changed %s: %s -> %s', id, old_v, new_v)
+        else:
+          logging.info('Unchanged: %s:%s', id, new_v)
+      # replace GCP BOM version
+      n = self.BOM_VERSION_STRING.match(line)
+      if n:
+        self.target_lines[idx] = self.original_lines[idx].replace(
+            n.group(1), self.bom_version)
+        found_bom = True
 
     if not found_bom:
       logging.warning(
@@ -224,6 +274,7 @@ def run(self):
     self.write_back()
     self.write_license_script()
 
+
 if __name__ == '__main__':
   logging.getLogger().setLevel(logging.INFO)
   if len(sys.argv) < 2:

sdks/java/container/license_scripts/dep_urls_java.yaml

@@ -46,7 +46,7 @@ jaxen:
   '1.1.6':
     type: "3-Clause BSD"
 libraries-bom:
-  '26.31.0':
+  '26.32.0':
     license: "https://raw.githubusercontent.com/GoogleCloudPlatform/cloud-opensource-java/master/LICENSE"
     type: "Apache License 2.0"
 paranamer:

sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/datastore/DatastoreV1.java

@@ -2041,10 +2041,7 @@ public Datastore getDatastore(
       }
 
       DatastoreOptions.Builder builder =
-          new DatastoreOptions.Builder()
-              .projectId(projectId)
-              .databaseId(databaseId)
-              .initializer(initializer);
+          new DatastoreOptions.Builder().projectId(projectId).initializer(initializer);
 
       if (localhost != null) {
         builder.localHost(localhost);

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/datastore/V1TestUtil.java

@@ -151,10 +151,7 @@ static Datastore getDatastore(
     }
 
     DatastoreOptions.Builder builder =
-        new DatastoreOptions.Builder()
-            .projectId(projectId)
-            .databaseId(databaseId)
-            .initializer(initializer);
+        new DatastoreOptions.Builder().projectId(projectId).initializer(initializer);
 
     return DatastoreFactory.get().create(builder.build());
   }