feat: track customization in templates

requirements.txt

@@ -305,3 +305,15 @@ setuptools==65.5.1 \
     --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \
     --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f
     # via -r requirements.in
+
+setuptools~=65.5.1
+pytest~=7.4.2
+PyYAML~=6.0.1
+lxml~=4.9.3
+Jinja2~=3.1.2
+click~=8.1.7
+requests~=2.31.0
+colorlog~=6.7.0
+watchdog~=3.0.0
+packaging~=23.1
+nox~=2023.4.22

synthtool/gcp/common.py

@@ -21,8 +21,8 @@
 from pathlib import Path
 from typing import Dict, List, Optional
 import jinja2
+import yaml
 from datetime import date
-
 from synthtool import shell, _tracked_paths
 from synthtool.gcp import partials
 from synthtool.languages import node, node_mono_repo
@@ -49,10 +49,31 @@ def __init__(self, template_path: Optional[Path] = None):
         self._templates = templates.Templates(self._template_root)
         self.excludes = []  # type: List[str]
 
-    def _generic_library(self, directory: str, relative_dir=None, **kwargs) -> Path:
-        # load common repo meta information (metadata that's not language specific).
+    def _generic_library(
+        self,
+        directory: str,
+        relative_dir=None,
+        partial_files: List[str] = None,
+        **kwargs
+    ) -> Path:
+        defaults_path = self._template_root / directory / "defaults"
+        kwargs["metadata"]["defaults"] = {}
+        for default_file in defaults_path.glob("*-default.yaml"):
+            self.excludes.append(
+                "defaults/{}".format(os.path.basename(default_file))
+            )
+            with open(default_file) as f:
+                kwargs["metadata"]["defaults"].update(
+                    yaml.load(f, Loader=yaml.SafeLoader)
+                )
+        # load common repo meta information (metadata that's not language
+        # specific).
         if "metadata" in kwargs:
-            self._load_generic_metadata(kwargs["metadata"], relative_dir=relative_dir)
+            self._load_generic_metadata(
+                kwargs["metadata"],
+                relative_dir=relative_dir,
+                partial_files=partial_files
+            )
             # if no samples were found, don't attempt to render a
             # samples/README.md.
             if "samples" not in kwargs["metadata"] or not kwargs["metadata"]["samples"]:
@@ -321,11 +342,15 @@ def py_library(self, **kwargs) -> Path:
                     f.write(content)
         return ret
 
-    def java_library(self, **kwargs) -> Path:
+    def java_library(self, partial_files: List[str] = None, **kwargs) -> Path:
         # kwargs["metadata"] is required to load values from .repo-metadata.json
         if "metadata" not in kwargs:
             kwargs["metadata"] = {}
-        return self._generic_library("java_library", **kwargs)
+        return self._generic_library(
+            "java_library",
+            partial_files=partial_files,
+            **kwargs
+        )
 
     def node_library(self, **kwargs) -> Path:
         # TODO: once we've migrated all Node.js repos to either having
@@ -397,11 +422,16 @@ def render(self, template_name: str, **kwargs) -> Path:
         _tracked_paths.add(template)
         return template
 
-    def _load_generic_metadata(self, metadata: Dict, relative_dir=None):
+    def _load_generic_metadata(
+        self,
+        metadata: Dict,
+        relative_dir=None,
+        partial_files: List[str] = None
+    ):
         """
         loads additional meta information from .repo-metadata.json.
         """
-        metadata["partials"] = partials.load_partials()
+        metadata["partials"] = partials.load_partials(partial_files)
 
         # Loads repo metadata information from the default location if it
         # hasn't already been set. Some callers may have already loaded repo

synthtool/gcp/partials.py

@@ -18,12 +18,15 @@
 from typing import Dict, List
 
 # these are the default locations to look up
-_DEFAULT_PARTIAL_FILES = [".readme-partials.yml", ".readme-partials.yaml"]
+_DEFAULT_PARTIAL_FILES = [
+    ".readme-partials.yml",
+    ".readme-partials.yaml",
+]
 
 
-def load_partials(files: List[str] = _DEFAULT_PARTIAL_FILES) -> Dict:
+def load_partials(files: List[str] = None) -> Dict:
     """
-    hand-crafted artisinal markdown can be provided in a .readme-partials.yml.
+    hand-crafted artisanal markdown can be provided in a .readme-partials.yml.
     The following fields are currently supported:
 
     body: custom body to include in the usage section of the document.
@@ -34,13 +37,15 @@ def load_partials(files: List[str] = _DEFAULT_PARTIAL_FILES) -> Dict:
     deprecation_warning: a warning to indicate that the library has been
         deprecated and a pointer to an alternate option
     """
+    if files is None:
+        files = _DEFAULT_PARTIAL_FILES
+    else:
+        files.extend(_DEFAULT_PARTIAL_FILES)
+    res = {}
     cwd_path = Path(os.getcwd())
-    partials_file = None
     for file in files:
         if os.path.exists(cwd_path / file):
             partials_file = cwd_path / file
-            break
-    if not partials_file:
-        return {}
-    with open(partials_file) as f:
-        return yaml.load(f, Loader=yaml.SafeLoader)
+            with open(partials_file) as f:
+                res.update(yaml.load(f, Loader=yaml.SafeLoader))
+    return res

synthtool/gcp/templates/java_library/.kokoro/nightly/integration.cfg

@@ -1,37 +1,30 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-# Configure the docker image for kokoro-trampoline.
+{%- if 'defaults' in metadata and 'env_vars' in metadata['defaults'] -%}
+{%- for default_key, default_value in metadata['defaults']['env_vars'].items() -%}
+{%- if 'partials' in metadata
+  and 'kokoro_nightly_env_vars' in metadata['partials']
+  and default_key not in metadata['partials']['kokoro_nightly_env_vars'] -%}
+{%- if default_key == "ENABLE_FLAKYBOT" %}
 env_vars: {
-  key: "TRAMPOLINE_IMAGE"
-  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
-}
-
-env_vars: {
-  key: "JOB_TYPE"
-  value: "integration"
-}
-# TODO: remove this after we've migrated all tests and scripts
-env_vars: {
-  key: "GCLOUD_PROJECT"
-  value: "java-docs-samples-testing"
-}
-
-env_vars: {
-  key: "GOOGLE_CLOUD_PROJECT"
-  value: "java-docs-samples-testing"
-}
-
-env_vars: {
-  key: "ENABLE_FLAKYBOT"
+  key: "{{ default_key }}"
   value: {% if migrated_split_repo %}"false"{% else %}"true"{% endif %}
 }
-
+{% else %}
 env_vars: {
-  key: "GOOGLE_APPLICATION_CREDENTIALS"
-  value: "secret_manager/java-it-service-account"
+  key: "{{ default_key }}"
+  value: "{{ default_value }}"
 }
+{% endif %}
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
 
+{%- if 'partials' in metadata and 'kokoro_nightly_env_vars' in metadata['partials'] -%}
+{% for partial_key, partial_value in metadata['partials']['kokoro_nightly_env_vars'].items() %}
 env_vars: {
-  key: "SECRET_MANAGER_KEYS"
-  value: "java-it-service-account"
+  key: "{{ partial_key }}"
+  value: "{{ partial_value }}"
 }
+{% endfor -%}
+{%- endif -%}

synthtool/gcp/templates/java_library/defaults/integration-default.yaml

@@ -0,0 +1,8 @@
+env_vars:
+  TRAMPOLINE_IMAGE: gcr.io/cloud-devrel-kokoro-resources/java8
+  JOB_TYPE: integration
+  GCLOUD_PROJECT: java-docs-samples-testing
+  GOOGLE_CLOUD_PROJECT: java-docs-samples-testing
+  ENABLE_FLAKYBOT: true
+  GOOGLE_APPLICATION_CREDENTIALS: secret_manager/java-it-service-account
+  SECRET_MANAGER_KEYS: java-it-service-account

synthtool/languages/java.py

@@ -472,7 +472,10 @@ def _common_template_metadata() -> Dict[str, Any]:
 
 
 def common_templates(
-    excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs
+    excludes: List[str] = [],
+    template_path: Optional[Path] = None,
+    partial_files: List[str] = None,
+    **kwargs
 ) -> None:
     """Generate common templates for a Java Library
 
@@ -481,8 +484,11 @@ def common_templates(
     their expected location.
 
     Args:
-        excludes (List[str], optional): List of template paths to ignore
-        **kwargs: Additional options for CommonTemplates.java_library()
+        :param excludes: List of template paths to ignore
+        :param template_path:
+        :param partial_files:
+        :param kwargs: Additional options for CommonTemplates.java_library()
+
     """
     metadata = _common_template_metadata()
     kwargs["metadata"] = metadata
@@ -516,11 +522,14 @@ def common_templates(
         )
     )
 
-    templates = gcp.CommonTemplates(template_path=template_path).java_library(**kwargs)
+    templates = gcp\
+        .CommonTemplates(template_path=template_path)\
+        .java_library(partial_files, **kwargs)
 
     # skip README generation on Kokoro (autosynth)
     if os.environ.get("KOKORO_ROOT") is not None:
-        # README.md is now synthesized separately. This prevents synthtool from deleting the
+        # README.md is now synthesized separately.
+        # This prevents synthtool from deleting the
         # README as it's no longer generated here.
         excludes.append("README.md")
 

tests/fixtures/java_templates/partials/.repo-metadata.json

@@ -0,0 +1,18 @@
+{
+    "api_shortname": "cloudasset",
+    "name_pretty": "Cloud Asset Inventory",
+    "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.",
+    "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html",
+    "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0",
+    "release_level": "stable",
+    "transport": "grpc",
+    "requires_billing": true,
+    "language": "java",
+    "repo": "googleapis/java-asset",
+    "repo_short": "java-asset",
+    "distribution_name": "com.google.cloud:google-cloud-asset",
+    "library_type": "GAPIC_AUTO",
+    "api_id": "cloudasset.googleapis.com"
+}

tests/fixtures/java_templates/partials/integration-golden.cfg

@@ -0,0 +1,40 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Configure the docker image for kokoro-trampoline.
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "integration"
+}
+# TODO: remove this after we've migrated all tests and scripts
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: {% if migrated_split_repo %}"false"{% else %}"true"{% endif %}
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}
+{% if 'partials' in metadata and metadata['partials']['env_vars_list'] %}
+{{ metadata['partials']['env_vars_list'] }}
+{% endif %}