Merge branch 'master' into python-blunderbuss

docker/owlbot/java/src/requirements.txt

@@ -318,9 +318,9 @@ typing==3.7.4.3 \
     --hash=sha256:1187fb9c82fd670d10aa07bbb6cfcfe4bdda42d6fab8d5134f04e8c4d0b71cc9 \
     --hash=sha256:283d868f5071ab9ad873e5e52268d611e851c870a2ba354193026f2dfb29d8b5
     # via -r requirements.in
-urllib3==2.0.6 \
-    --hash=sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2 \
-    --hash=sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564
+urllib3==2.0.7 \
+    --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \
+    --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e
     # via requests
 watchdog==3.0.0 \
     --hash=sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a \

synthtool/gcp/templates/java_library/.kokoro/presubmit/graalvm-native-17.cfg

@@ -3,7 +3,7 @@
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
   key: "TRAMPOLINE_IMAGE"
-  value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17:22.3.2"
+  value: "gcr.io/cloud-devrel-kokoro-resources/graalvm17:22.3.3"
 }
 
 env_vars: {

synthtool/gcp/templates/java_library/.kokoro/presubmit/graalvm-native.cfg

@@ -3,7 +3,7 @@
 # Configure the docker image for kokoro-trampoline.
 env_vars: {
   key: "TRAMPOLINE_IMAGE"
-  value: "gcr.io/cloud-devrel-kokoro-resources/graalvm:22.3.2"
+  value: "gcr.io/cloud-devrel-kokoro-resources/graalvm:22.3.3"
 }
 
 env_vars: {

synthtool/gcp/templates/java_library/.kokoro/requirements.txt

@@ -470,9 +470,9 @@ typing-extensions==4.7.1 \
     --hash=sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36 \
     --hash=sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2
     # via -r requirements.in
-urllib3==1.26.17 \
-    --hash=sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21 \
-    --hash=sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b
+urllib3==1.26.18 \
+    --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \
+    --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0
     # via
     #   google-auth
     #   requests

synthtool/gcp/templates/python_mono_repo_library/noxfile.py.j2

@@ -28,7 +28,11 @@ import nox
 
 BLACK_VERSION = "black[jupyter]==23.7.0"
 ISORT_VERSION = "isort==5.11.0"
+{% if metadata['repo']['distribution_name'].startswith('google') %}
 LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"]
+{% else %}
+LINT_PATHS = ["docs", "{{ metadata['repo']['distribution_name'] }}", "tests", "noxfile.py", "setup.py"]
+{% endif %}
 
 DEFAULT_PYTHON_VERSION = "{{ default_python_version }}"
 
@@ -156,7 +160,11 @@ def lint(session):
         "--check",
         *LINT_PATHS,
     )
+    {% if metadata['repo']['distribution_name'].startswith('google') %}
     session.run("flake8", "google", "tests")
+    {% else %}
+    session.run("flake8", "{{ metadata['repo']['distribution_name'] }}", "tests")
+    {% endif %}
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
@@ -237,7 +245,11 @@ def default(session):
         "py.test",
         "--quiet",
         f"--junitxml=unit_{session.python}_sponge_log.xml",
+    {% if metadata['repo']['distribution_name'].startswith('google') %}
         "--cov=google",
+    {% else %}
+        "--cov={{ metadata['repo']['distribution_name'] }}",
+    {% endif %}
         "--cov=tests/unit",
         "--cov-append",
         "--cov-config=.coveragerc",

synthtool/languages/python_mono_repo.py

@@ -16,8 +16,9 @@
 import os
 from pathlib import Path
 import shutil
-import synthtool as s
+import synthtool
 import synthtool.gcp as gcp
+import yaml
 
 
 def create_symlink_in_docs_dir(package_dir: str, filename: str):
@@ -98,6 +99,72 @@ def update_url_in_setup_py(package_dir: str):
         f.writelines(new_setup_py)
 
 
+def apply_client_specific_post_processing(
+    post_processing_dir: str, package_name: str
+) -> None:
+    """Applies client-specific post processing which exists in the Path `post_processing_dir`.
+    This function is only called from `owlbot_main` when there is an `owl-bot-staging` folder
+    which contains generated client library code. Re-running the script more than once is
+    expected to be idempotent. The client-specific post processing YAML is in the following format:
+    ```
+        description: Verbose description about the need for the workaround.
+        url: URL of the issue in gapic-generator-python tracking eventual removal of the workaround
+        replacements:
+          - replacement:
+            paths: [<List of files where the replacement should occur relative to the monorepo root directory>]
+            before: "The string to search for in the specified paths"
+            after:  "The string to replace in the the specified paths",
+            count: <integer indicating number of replacements that should have occurred across all files after the script is run>
+    ```
+
+    Note: The `paths` key above must only include paths for the same package so that the number of replacements
+    made in a given package can be verified.
+
+    Args:
+        post_processing_dir (str): Path to the directory which contains YAML files which will
+            be used to apply client-specific post processing, e.g. 'packages/<package_name>/scripts/client-post-processing'
+            relative to the monorepo root directory.
+        package_name (str): The name of the package where client specific post processing will be applied.
+    """
+
+    if Path(post_processing_dir).exists():
+        for post_processing_path in Path(post_processing_dir).iterdir():
+            with open(post_processing_path, "r") as post_processing_path_file:
+                post_processing_json = yaml.safe_load(post_processing_path_file)
+                all_replacements = post_processing_json["replacements"]
+                # For each workaround related to the specified issue
+                for replacement in all_replacements:
+                    replacement_count = 0
+                    number_of_paths_with_replacements = 0
+                    # For each file that needs the workaround applied
+                    for client_library_path in replacement["paths"]:
+                        if package_name in client_library_path:
+                            number_of_paths_with_replacements += 1
+                            replacement_count += synthtool.replace(
+                                client_library_path,
+                                replacement["before"],
+                                replacement["after"],
+                            )
+                            # Ensure idempotency by checking that subsequent calls won't
+                            # trigger additional replacements within the same path
+                            assert (
+                                synthtool.replace(
+                                    client_library_path,
+                                    replacement["before"],
+                                    replacement["after"],
+                                )
+                                == 0
+                            )
+                    if number_of_paths_with_replacements:
+                        # Ensure that the numner of paths where a replacement occurred matches the number of paths.
+                        assert number_of_paths_with_replacements == len(
+                            replacement["paths"]
+                        )
+                        # Ensure that the total number of replacements matches the value specified in `count`
+                        # for all paths in `replacement["paths"]`
+                        assert replacement_count == replacement["count"]
+
+
 def walk_through_owlbot_dirs(dir: Path):
     """
     Walks through all API packages in google-cloud-python/packages
@@ -142,31 +209,35 @@ def owlbot_main(package_dir: str) -> None:
     except FileNotFoundError:
         raise Exception("Could not find the default version")
 
-    if Path(f"owl-bot-staging/{Path(package_dir).name}").exists():
-        for library in s.get_staging_dirs(
-            default_version, f"owl-bot-staging/{Path(package_dir).name}"
+    package_name = Path(package_dir).name
+
+    if Path(f"owl-bot-staging/{package_name}").exists():
+        for library in synthtool.get_staging_dirs(
+            default_version, f"owl-bot-staging/{package_name}"
         ):
             if clean_up_generated_samples:
                 shutil.rmtree(
                     f"{package_dir}/samples/generated_samples", ignore_errors=True
                 )
                 clean_up_generated_samples = False
-            s.move([library], package_dir, excludes=[])
+            synthtool.move([library], package_dir, excludes=[])
 
         templated_files = gcp.CommonTemplates().py_mono_repo_library(
-            relative_dir=f"packages/{Path(package_dir).name}",
+            relative_dir=f"packages/{package_name}",
             microgenerator=True,
             default_python_version="3.9",
             unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11"],
             system_test_python_versions=["3.8", "3.9", "3.10", "3.11"],
             cov_level=100,
             versions=gcp.common.detect_versions(
-                path=f"{package_dir}/google",
+                path=f"{package_dir}/google"
+                if package_name.startswith("google")
+                else f"{package_dir}/{package_name}",
                 default_version=default_version,
                 default_first=True,
             ),
         )
-        s.move([templated_files], package_dir)
+        synthtool.move([templated_files], package_dir)
 
         # create symlink docs/README.rst if it doesn't exist
         create_symlink_docs_readme(package_dir)
@@ -178,15 +249,19 @@ def owlbot_main(package_dir: str) -> None:
         update_url_in_setup_py(package_dir)
 
         # run format nox session for all directories which have a noxfile
-        for noxfile in Path(".").glob(
-            f"packages/{Path(package_dir).name}/**/noxfile.py"
-        ):
-            s.shell.run(["nox", "-s", "format"], cwd=noxfile.parent, hide_output=False)
+        for noxfile in Path(".").glob(f"packages/{package_name}/**/noxfile.py"):
+            synthtool.shell.run(
+                ["nox", "-s", "format"], cwd=noxfile.parent, hide_output=False
+            )
+
+        apply_client_specific_post_processing(
+            f"packages/{package_name}/scripts/client-post-processing", package_name
+        )
 
 
 if __name__ == "__main__":
     owlbot_dirs = walk_through_owlbot_dirs(Path.cwd())
     for package_dir in owlbot_dirs:
         owlbot_main(package_dir)
 
-    s.remove_staging_dirs()
+    synthtool.remove_staging_dirs()

tests/fixtures/python_mono_repo/packages/google-cloud-asset/another_path/another_file.rst

@@ -0,0 +1,3 @@
+First replacement
+First replacement
+Second replacement

tests/fixtures/python_mono_repo/packages/google-cloud-asset/docs/index.rst

@@ -0,0 +1,3 @@
+First replacement
+First replacement
+Second replacement

tests/fixtures/python_mono_repo/packages/google-cloud-asset/scripts/client-post-processing/paths-must-be-for-the-same-package.yaml

@@ -0,0 +1,20 @@
+description: Update docs/index.rst to show that a bot was here.
+url: N/A
+replacements:
+  - paths: [
+      "packages/google-cloud-asset/docs/index.rst",
+      "packages/google-cloud-asset/another_path/another_file.rst",
+      # adding a path for a different package should cause an assertion error
+      "packages/another-path/another_path/another_file.rst",
+    ]
+    before: "First replacement"
+    after:  "Completed replacement 1."
+    count: 4
+  - paths: ["packages/google-cloud-asset/docs/index.rst"]
+    before: "Second replacement"
+    after: "Completed replacement 2."
+    count: 1
+  - paths: ["packages/some-other-asset/docs/index.rst"]
+    before: "Does not exist"
+    after: "Replacement does not exist"
+    count: 1

tests/fixtures/python_mono_repo/packages/google-cloud-workflows/another_path/another_file.rst

@@ -0,0 +1,3 @@
+First replacement
+First replacement
+Second replacement

tests/fixtures/python_mono_repo/packages/google-cloud-workflows/docs/index.rst

@@ -0,0 +1,3 @@
+First replacement
+First replacement
+Second replacement

tests/fixtures/python_mono_repo/packages/google-cloud-workflows/scripts/client-post-processing/update-docs-index-rst.yaml

@@ -0,0 +1,18 @@
+description: Update docs/index.rst to show that a bot was here.
+url: N/A
+replacements:
+  - paths: [
+      "packages/google-cloud-workflows/docs/index.rst",
+      "packages/google-cloud-workflows/another_path/another_file.rst",
+    ]
+    before: "First replacement"
+    after:  "Completed replacement 1."
+    count: 4
+  - paths: ["packages/google-cloud-workflows/docs/index.rst"]
+    before: "Second replacement"
+    after: "Completed replacement 2."
+    count: 1
+  - paths: ["packages/some-other-package/docs/index.rst"]
+    before: "Does not exist"
+    after: "Replacement does not exist"
+    count: 1

tests/fixtures/python_mono_repo/scripts/client-post-processing/paths-must-be-for-the-same-package.yaml

@@ -0,0 +1,20 @@
+description: Update docs/index.rst to show that a bot was here.
+url: N/A
+replacements:
+  - paths: [
+      "packages/google-cloud-asset/docs/index.rst",
+      "packages/google-cloud-asset/another_path/another_file.rst",
+      # adding a path for a different package should cause an assertion error
+      "packages/another-path/another_path/another_file.rst",
+    ]
+    before: "First replacement"
+    after:  "Completed replacement 1."
+    count: 4
+  - paths: ["packages/google-cloud-asset/docs/index.rst"]
+    before: "Second replacement"
+    after: "Completed replacement 2."
+    count: 1
+  - paths: ["packages/some-other-asset/docs/index.rst"]
+    before: "Does not exist"
+    after: "Replacement does not exist"
+    count: 1

tests/fixtures/python_mono_repo/scripts/client-post-processing/update-docs-index-rst.yaml

@@ -0,0 +1,18 @@
+description: Update docs/index.rst to show that a bot was here.
+url: N/A
+replacements:
+  - paths: [
+      "packages/google-cloud-workflows/docs/index.rst",
+      "packages/google-cloud-workflows/another_path/another_file.rst",
+    ]
+    before: "First replacement"
+    after:  "Completed replacement 1."
+    count: 4
+  - paths: ["packages/google-cloud-workflows/docs/index.rst"]
+    before: "Second replacement"
+    after: "Completed replacement 2."
+    count: 1
+  - paths: ["packages/some-other-package/docs/index.rst"]
+    before: "Does not exist"
+    after: "Replacement does not exist"
+    count: 1

tests/test_python_mono_repo.py

@@ -0,0 +1,60 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from pathlib import Path
+import pytest
+
+from synthtool.languages import python_mono_repo
+from . import util
+
+
+TEST_FIXTURE_DIR = Path(__file__).parent / "fixtures" / "python_mono_repo"
+
+
+def test_apply_workaround():
+    with util.copied_fixtures_dir(TEST_FIXTURE_DIR):
+        docs_index_rst = Path(
+            "packages/google-cloud-workflows/docs/index.rst"
+        ).read_text()
+        assert (
+            docs_index_rst
+            == "First replacement\nFirst replacement\nSecond replacement\n"
+        )
+        python_mono_repo.apply_client_specific_post_processing(
+            "packages/google-cloud-workflows/scripts/client-post-processing",
+            "google-cloud-workflows",
+        )
+        docs_index_rst = Path(
+            "packages/google-cloud-workflows/docs/index.rst"
+        ).read_text()
+        assert (
+            docs_index_rst
+            == "Completed replacement 1.\nCompleted replacement 1.\nCompleted replacement 2.\n"
+        )
+
+        # Confirm that `AssertionError`` is raised if you run the post processing more than once
+        # because the replacement should only occur once.
+        with pytest.raises(AssertionError):
+            python_mono_repo.apply_client_specific_post_processing(
+                "packages/google-cloud-workflows/scripts/client-post-processing",
+                "google-cloud-workflows",
+            )
+
+        # Confirm that `AssertionError`` is raised if
+        # there is a replacement is defined for multiple packages.
+        # Each path in replacement["paths"] should only be for a single package.
+        with pytest.raises(AssertionError):
+            python_mono_repo.apply_client_specific_post_processing(
+                "packages/google-cloud-asset/scripts/client-post-processing",
+                "google-cloud-asset",
+            )