diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index e5055d8030..5ca6d93219 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -13,7 +13,7 @@ # limitations under the License. import os import sys - +from typing import Optional import click as click from library_generation.generate_pr_description import generate_pr_descriptions from library_generation.generate_repo import generate_from_yaml @@ -51,6 +51,17 @@ def main(ctx): metadata about library generation. """, ) +@click.option( + "--library-names", + type=str, + default=None, + show_default=True, + help=""" + A list of library names that will be generated, separated by comma. + The library name of a library is the value of library_name or api_shortname, + if library_name is not specified, in the generation configuration. + """, +) @click.option( "--repository-path", type=str, @@ -77,6 +88,7 @@ def main(ctx): def generate( baseline_generation_config_path: str, current_generation_config_path: str, + library_names: Optional[str], repository_path: str, api_definitions_path: str, ): @@ -86,8 +98,8 @@ def generate( history. If baseline generation config is not specified but current generation - config is specified, generate all libraries based on current generation - config without commit history. + config is specified, generate all libraries if `library_names` is not + specified, based on current generation config without commit history. If current generation config is not specified but baseline generation config is specified, raise FileNotFoundError because current generation @@ -95,8 +107,15 @@ def generate( If both baseline generation config and current generation config are not specified, generate all libraries based on the default generation config, - which is generation_config.yaml in the current working directory. Raise - FileNotFoundError if the default config does not exist. + which is generation_config.yaml in the current working directory. + + If `library_names` is specified, only libraries whose name can be found in + the current generation config or default generation config, if current + generation config is not specified, will be generated. Changed libraries + will be ignored even if baseline and current generation config are + specified. + + Raise FileNotFoundError if the default config does not exist. The commit history, if generated, will be available in repository_path/pr_description.txt. @@ -104,6 +123,7 @@ def generate( __generate_repo_and_pr_description_impl( baseline_generation_config_path=baseline_generation_config_path, current_generation_config_path=current_generation_config_path, + library_names=library_names, repository_path=repository_path, api_definitions_path=api_definitions_path, ) @@ -112,6 +132,7 @@ def generate( def __generate_repo_and_pr_description_impl( baseline_generation_config_path: str, current_generation_config_path: str, + library_names: Optional[str], repository_path: str, api_definitions_path: str, ): @@ -146,30 +167,39 @@ def __generate_repo_and_pr_description_impl( current_generation_config_path = os.path.abspath(current_generation_config_path) repository_path = os.path.abspath(repository_path) api_definitions_path = os.path.abspath(api_definitions_path) + include_library_names = _parse_library_name_from(library_names) + if not baseline_generation_config_path: - # Execute full generation based on current_generation_config if + # Execute selective generation based on current_generation_config if # baseline_generation_config is not specified. # Do not generate pull request description. generate_from_yaml( config=from_yaml(current_generation_config_path), repository_path=repository_path, api_definitions_path=api_definitions_path, + target_library_names=include_library_names, ) return - # Compare two generation configs and only generate changed libraries. + # Compare two generation configs to get changed libraries. # Generate pull request description. baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) config_change = compare_config( baseline_config=from_yaml(baseline_generation_config_path), current_config=from_yaml(current_generation_config_path), ) - # pass None if we want to fully generate the repository. - target_library_names = ( + # Pass None if we want to fully generate the repository. + changed_library_names = ( config_change.get_changed_libraries() if not _needs_full_repo_generation(config_change=config_change) else None ) + # Include library names takes preference if specified. + target_library_names = ( + include_library_names + if include_library_names is not None + else changed_library_names + ) generate_from_yaml( config=config_change.current_config, repository_path=repository_path, @@ -191,6 +221,12 @@ def _needs_full_repo_generation(config_change: ConfigChange) -> bool: return not current_config.is_monorepo() or current_config.contains_common_protos() +def _parse_library_name_from(includes: str) -> Optional[list[str]]: + if includes is None: + return None + return [library_name.strip() for library_name in includes.split(",")] + + @main.command() @click.option( "--generation-config-path", diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index cb44fadcbe..fa3062f091 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -12,9 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import shutil - +from typing import Optional import library_generation.utils.utilities as util from library_generation.generate_composed_library import generate_composed_library from library_generation.model.generation_config import GenerationConfig @@ -26,7 +25,7 @@ def generate_from_yaml( config: GenerationConfig, repository_path: str, api_definitions_path: str, - target_library_names: list[str] = None, + target_library_names: Optional[list[str]], ) -> None: """ Based on the generation config, generates libraries via diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index 171be31236..ca70bdd647 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -104,6 +104,7 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation( generate_impl( baseline_generation_config_path=config_path, current_generation_config_path=config_path, + library_names=None, repository_path=".", api_definitions_path=".", ) @@ -114,6 +115,37 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation( target_library_names=None, ) + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of non monorepos + (HW libraries). + generate() should call generate_from_yaml() with + target_library_names equals includes. + """ + config_path = f"{test_resource_dir}/generation_config.yaml" + self.assertFalse(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names="cloudasset,non-existent-library", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["cloudasset", "non-existent-library"], + ) + @patch("library_generation.cli.entry_point.generate_from_yaml") @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_non_monorepo_with_changes_triggers_full_generation( @@ -137,6 +169,7 @@ def test_generate_non_monorepo_with_changes_triggers_full_generation( generate_impl( baseline_generation_config_path=baseline_config_path, current_generation_config_path=current_config_path, + library_names=None, repository_path=".", api_definitions_path=".", ) @@ -147,6 +180,41 @@ def test_generate_non_monorepo_with_changes_triggers_full_generation( target_library_names=None, ) + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_non_monorepo_with_changes_with_includes_triggers_selective_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of non monorepos + (HW libraries). + generate() should call generate_from_yaml() with + target_library_names equals includes + """ + baseline_config_path = f"{test_resource_dir}/generation_config.yaml" + current_config_path = ( + f"{test_resource_dir}/generation_config_library_modified.yaml" + ) + self.assertFalse(from_yaml(current_config_path).is_monorepo()) + self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=baseline_config_path, + current_generation_config_path=current_config_path, + library_names="cloudasset,non-existent-library", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["cloudasset", "non-existent-library"], + ) + @patch("library_generation.cli.entry_point.generate_from_yaml") @patch("library_generation.cli.entry_point.generate_pr_descriptions") def test_generate_monorepo_with_common_protos_triggers_full_generation( @@ -167,6 +235,7 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation( generate_impl( baseline_generation_config_path=config_path, current_generation_config_path=config_path, + library_names=None, repository_path=".", api_definitions_path=".", ) @@ -179,7 +248,69 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation( @patch("library_generation.cli.entry_point.generate_from_yaml") @patch("library_generation.cli.entry_point.generate_pr_descriptions") - def test_generate_monorepo_without_common_protos_does_not_trigger_full_generation( + def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo with + common protos. + target_library_names is the same as includes. + """ + config_path = f"{test_resource_dir}/monorepo_with_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names="iam,non-existent-library", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["iam", "non-existent-library"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_monorepo_without_change_does_not_trigger_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + generate() should call generate_from_yaml() with + target_library_names=changed libraries which does not trigger the full + generation. + """ + config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names=None, + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=[], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_monorepo_without_change_with_includes_trigger_selective_generation( self, generate_pr_descriptions, generate_from_yaml, @@ -198,6 +329,104 @@ def test_generate_monorepo_without_common_protos_does_not_trigger_full_generatio generate_impl( baseline_generation_config_path=config_path, current_generation_config_path=config_path, + library_names="asset", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["asset"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_monorepo_with_changed_config_without_includes_trigger_changed_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + target_library_names should be the changed libraries if includes + is not specified. + """ + current_config_path = f"{test_resource_dir}/monorepo_current.yaml" + baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" + self.assertTrue(from_yaml(current_config_path).is_monorepo()) + self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=baseline_config_path, + current_generation_config_path=current_config_path, + library_names=None, + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["asset"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_monorepo_with_changed_config_and_includes_trigger_selective_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + target_library_names should be the same as include libraries, regardless + the library exists or not. + """ + current_config_path = f"{test_resource_dir}/monorepo_current.yaml" + baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" + self.assertTrue(from_yaml(current_config_path).is_monorepo()) + self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=baseline_config_path, + current_generation_config_path=current_config_path, + library_names="cloudbuild,non-existent-library", + repository_path=".", + api_definitions_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, + repository_path=ANY, + api_definitions_path=ANY, + target_library_names=["cloudbuild", "non-existent-library"], + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_monorepo_without_changed_config_without_includes_does_not_trigger_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + target_library_names should be the changed libraries if includes + is not specified. + """ + config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + library_names=None, repository_path=".", api_definitions_path=".", ) diff --git a/library_generation/test/generate_repo_unit_tests.py b/library_generation/test/generate_repo_unit_tests.py index 6085c237a6..470f0a4b18 100644 --- a/library_generation/test/generate_repo_unit_tests.py +++ b/library_generation/test/generate_repo_unit_tests.py @@ -40,6 +40,20 @@ def test_get_target_library_given_null_returns_all_libraries(self): target_libraries = get_target_libraries(config) self.assertEqual([one_library, another_library], target_libraries) + def test_get_target_library_given_an_non_existent_library_returns_only_existing_libraries( + self, + ): + one_library = GenerateRepoTest.__get_an_empty_library_config() + one_library.api_shortname = "one_library" + another_library = GenerateRepoTest.__get_an_empty_library_config() + another_library.api_shortname = "another_library" + config = GenerateRepoTest.__get_an_empty_generation_config() + config.libraries.extend([one_library, another_library]) + target_libraries = get_target_libraries( + config, ["one_library", "another_library", "non_existent_library"] + ) + self.assertEqual([one_library, another_library], target_libraries) + @staticmethod def __get_an_empty_generation_config() -> GenerationConfig: return GenerationConfig( diff --git a/library_generation/test/resources/test-config/monorepo_baseline.yaml b/library_generation/test/resources/test-config/monorepo_baseline.yaml new file mode 100644 index 0000000000..c2c4fd4a3b --- /dev/null +++ b/library_generation/test/resources/test-config/monorepo_baseline.yaml @@ -0,0 +1,30 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: preview + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2 diff --git a/library_generation/test/resources/test-config/monorepo_current.yaml b/library_generation/test/resources/test-config/monorepo_current.yaml new file mode 100644 index 0000000000..3ee2c8be2c --- /dev/null +++ b/library_generation/test/resources/test-config/monorepo_current.yaml @@ -0,0 +1,30 @@ +gapic_generator_version: 2.34.0 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: stable + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2