feat: get PR description from googleapis commits

library_generation/generate_pr_description.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#  Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import click
+
+from library_generation.model.generation_config import from_yaml
+from library_generation.utilities import get_commit_messages
+from library_generation.utilities import get_file_paths
+
+
+@click.group(invoke_without_command=False)
+@click.pass_context
+@click.version_option(message="%(version)s")
+def main(ctx):
+    pass
+
+
+@main.command()
+@click.option(
+    "--generation-config-yaml",
+    required=True,
+    type=str,
+    help="""
+    Path to generation_config.yaml that contains the metadata about
+    library generation.
+    """,
+)
+@click.option(
+    "--baseline-commit",
+    required=True,
+    type=str,
+    help="""
+    The baseline commit from which the commit message is considered.
+    This commit should be an ancestor of googleapis commit in configuration.
+    """,
+)
+@click.option(
+    "--repo-url",
+    type=str,
+    default="https://github.com/googleapis/googleapis.git",
+    show_default=True,
+    help="""
+    GitHub repository URL.
+    """,
+)
+def generate(
+    generation_config_yaml: str,
+    repo_url: str,
+    baseline_commit: str,
+) -> str:
+    return generate_pr_descriptions(
+        generation_config_yaml=generation_config_yaml,
+        repo_url=repo_url,
+        baseline_commit=baseline_commit,
+    )
+
+
+def generate_pr_descriptions(
+    generation_config_yaml: str,
+    repo_url: str,
+    baseline_commit: str,
+) -> str:
+    config = from_yaml(generation_config_yaml)
+    paths = get_file_paths(config)
+    return get_commit_messages(
+        repo_url=repo_url,
+        latest_commit=config.googleapis_commitish,
+        baseline_commit=baseline_commit,
+        paths=paths,
+    )
+
+
+if __name__ == "__main__":
+    main()

library_generation/test/integration_tests.py

@@ -17,12 +17,14 @@
 import unittest
 from distutils.dir_util import copy_tree
 from distutils.file_util import copy_file
+from filecmp import cmp
 from filecmp import dircmp
 
 from git import Repo
 from pathlib import Path
 from typing import List
-from typing import Dict
+
+from library_generation.generate_pr_description import generate_pr_descriptions
 from library_generation.generate_repo import generate_from_yaml
 from library_generation.model.generation_config import from_yaml, GenerationConfig
 from library_generation.test.compare_poms import compare_xml
@@ -49,6 +51,28 @@
 
 
 class IntegrationTest(unittest.TestCase):
+    def test_get_commit_message_success(self):
+        repo_url = "https://github.com/googleapis/googleapis.git"
+        baseline_commit = "a17d4caf184b050d50cacf2b0d579ce72c31ce74"
+        description = generate_pr_descriptions(
+            f"{config_dir}/google-cloud-java/{config_name}",
+            repo_url=repo_url,
+            baseline_commit=baseline_commit,
+        )
+        description_file = f"{config_dir}/pr-description.txt"
+        if os.path.isfile(f"{description_file}"):
+            os.remove(f"{description_file}")
+        with open(f"{description_file}", "w+") as f:
+            f.write(description)
+        # There are five commits from googleapis commitish to baseline commit,
+        # inclusively. Only two of the commits contain changes in proto_path
+        # that are in configuration. Therefore, only two commit messages should
+        # be included in the PR description.
+        self.assertTrue(
+            cmp(f"{config_dir}/pr-description-golden.txt", f"{description_file}")
+        )
+        os.remove(f"{description_file}")
+
     def test_generate_repo(self):
         shutil.rmtree(f"{golden_dir}", ignore_errors=True)
         os.makedirs(f"{golden_dir}", exist_ok=True)
@@ -150,7 +174,7 @@ def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str:
             repo = Repo(dest)
         else:
             dest = default_dest
-            repo_dest = f"{golden_dir}/{repo}"
+            shutil.rmtree(dest, ignore_errors=True)
             repo_url = f"{repo_prefix}/{repo}"
             print(f"Cloning repository {repo_url}")
             repo = Repo.clone_from(repo_url, dest)
@@ -169,6 +193,8 @@ def __get_library_names_from_config(cls, config: GenerationConfig) -> List[str]:
     def __get_config_files(cls, path: str) -> List[tuple[str, str]]:
         config_files = []
         for sub_dir in Path(path).resolve().iterdir():
+            if sub_dir.is_file():
+                continue
             repo = sub_dir.name
             if repo == "golden":
                 continue

library_generation/test/resources/integration/google-cloud-java/generation_config.yaml

@@ -48,3 +48,17 @@ libraries:
     - proto_path: google/cloud/alloydb/connectors/v1
     - proto_path: google/cloud/alloydb/connectors/v1alpha
     - proto_path: google/cloud/alloydb/connectors/v1beta
+
+  - api_shortname: documentai
+    name_pretty: Document AI
+    product_documentation: https://cloud.google.com/compute/docs/documentai/
+    api_description: allows developers to unlock insights from your documents with machine
+        learning.
+    library_name: document-ai
+    release_level: stable
+    issue_tracker: https://issuetracker.google.com/savedsearches/559755
+    GAPICs:
+    - proto_path: google/cloud/documentai/v1
+    - proto_path: google/cloud/documentai/v1beta1
+    - proto_path: google/cloud/documentai/v1beta2
+    - proto_path: google/cloud/documentai/v1beta3

library_generation/test/resources/integration/pr-description-golden.txt

@@ -0,0 +1,10 @@
+7a9a855287b5042410c93e5a510f40efd4ce6cb1
+feat: expose model_type in v1 processor, so that user can see the model_type after get or list processor version
+
+PiperOrigin-RevId: 603727585
+
+
+c7fd8bd652ac690ca84f485014f70b52eef7cb9e
+feat: add model_type in v1beta3 processor proto
+
+PiperOrigin-RevId: 603726122

library_generation/test/unit_tests.py

@@ -30,6 +30,7 @@
 from library_generation.model.gapic_inputs import parse as parse_build_file
 from library_generation.model.generation_config import from_yaml
 from library_generation.model.library_config import LibraryConfig
+from library_generation.utilities import get_file_paths
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 resources_dir = os.path.join(script_dir, "resources")
@@ -214,6 +215,24 @@ def test_from_yaml_succeeds(self):
         self.assertEqual("google/cloud/asset/v1p5beta1", gapics[3].proto_path)
         self.assertEqual("google/cloud/asset/v1p7beta1", gapics[4].proto_path)
 
+    def test_get_file_paths_from_yaml_success(self):
+        paths = get_file_paths(from_yaml(f"{test_config_dir}/generation_config.yaml"))
+        self.assertEqual(
+            {
+                "google/cloud/asset/v1",
+                "google/cloud/asset/v1p1beta1",
+                "google/cloud/asset/v1p2beta1",
+                "google/cloud/asset/v1p5beta1",
+                "google/cloud/asset/v1p7beta1",
+                "google/api",
+                "google/longrunning",
+                "google/rpc",
+                "google/shopping/type",
+                "google/type",
+            },
+            paths,
+        )
+
     @parameterized.expand(
         [
             ("BUILD_no_additional_protos.bazel", " "),

library_generation/utilities.py

@@ -17,6 +17,7 @@
 import os
 import shutil
 import re
+from git import Commit, Repo
 from pathlib import Path
 from lxml import etree
 from library_generation.model.bom_config import BomConfig
@@ -33,6 +34,13 @@
 group_id_tag = "groupId"
 artifact_tag = "artifactId"
 version_tag = "version"
+common_protos = {
+    "google/api",
+    "google/longrunning",
+    "google/rpc",
+    "google/shopping/type",
+    "google/type",
+}
 
 
 def __render(template_name: str, output_name: str, **kwargs):
@@ -134,6 +142,14 @@ def __handle_special_bom(
     ]
 
 
+def __is_qualified_commit(paths: set[str], commit: Commit) -> bool:
+    for file in commit.stats.files.keys():
+        idx = file.rfind("/")
+        if file[:idx] in paths:
+            return True
+    return False
+
+
 def create_argument(arg_key: str, arg_container: object) -> List[str]:
     """
     Generates a list of two elements [argument, value], or returns
@@ -483,3 +499,52 @@ def get_version_from(
         for line in f.readlines():
             if artifact_id in line:
                 return line.split(":")[index].strip()
+
+
+def get_file_paths(config: GenerationConfig) -> set[str]:
+    """
+    Get versioned proto_path from configuration file, plus known paths of
+    common protos.
+
+    :param config:
+    :return: versioned proto_path plus paths of common protos
+    """
+    paths = set()
+    for library in config.libraries:
+        for gapic_config in library.gapic_configs:
+            paths.add(gapic_config.proto_path)
+    return paths.union(common_protos)
+
+
+def get_commit_messages(
+    repo_url: str, latest_commit: str, baseline_commit: str, paths: set[str]
+) -> str:
+    """
+    Combine commit messages of a repository from latest_commit to
+    baseline_commit. Only commits which change files in a pre-defined
+    file paths will be considered.
+    Note that baseline_commit should be an ancestor of latest_commit.
+
+    :param repo_url: the url of the repository.
+    :param latest_commit: the newest commit to be considered in
+    selecting commit message.
+    :param baseline_commit: the oldest commit to be considered in
+    selecting commit message. This commit should be an ancestor of
+    :param paths: a set of file paths
+    :return: commit messages.
+    """
+    tmp_dir = "/tmp/repo"
+    shutil.rmtree(tmp_dir, ignore_errors=True)
+    os.mkdir(tmp_dir)
+    messages = []
+    repo = Repo.clone_from(repo_url, tmp_dir)
+    commit = repo.commit(latest_commit)
+    while str(commit.hexsha) != baseline_commit:
+        if __is_qualified_commit(paths, commit):
+            messages.append(f"{commit.hexsha}\n{commit.message}")
+        commit_parents = commit.parents
+        if len(commit_parents) == 0:
+            break
+        commit = commit_parents[0]
+    shutil.rmtree(tmp_dir, ignore_errors=True)
+    return "\n\n".join(messages)