From c8dc4c5b11c08fd6f21c4bb5e7c18e6f5f1d3a25 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Thu, 10 Nov 2022 16:24:16 -0500 Subject: [PATCH 1/5] chore(java): pom generation to look at root versions.txt --- docker/owlbot/java/src/fix-poms.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docker/owlbot/java/src/fix-poms.py b/docker/owlbot/java/src/fix-poms.py index 670a7096e..479f0a3c7 100644 --- a/docker/owlbot/java/src/fix-poms.py +++ b/docker/owlbot/java/src/fix-poms.py @@ -281,6 +281,14 @@ def main(): group_id, artifact_id = repo_metadata["distribution_name"].split(":") name = repo_metadata["name_pretty"] existing_modules = load_versions("versions.txt", group_id) + monorepo = False + if not existing_modules: + # For single-component Release Please setup, the root versions.txt + # manages the versions of all submodules. + existing_modules = load_versions("../versions.txt", group_id) + if existing_modules: + monorepo = True + print(f"monoreop? {monorepo}") # extra modules that need to be manages in versions.txt if "extra_versioned_modules" in repo_metadata: @@ -463,10 +471,12 @@ def main(): name=name, ) - if os.path.isfile("versions.txt"): - print("updating modules in versions.txt") + # For monorepo, we use the versions.txt at the root + versions_txt_file = "../versions.txt" if monorepo else "versions.txt" + if os.path.isfile(versions_txt_file): + print(f"updating modules in {versions_txt_file}") else: - print("creating missing versions.txt") + print(f"creating missing {versions_txt_file}") existing_modules.pop(parent_artifact_id) # add extra modules to versions.txt @@ -479,7 +489,7 @@ def main(): release_version=main_module.release_version, ) templates.render( - template_name="versions.txt.j2", output_name="./versions.txt", modules=existing_modules.values(), + template_name="versions.txt.j2", output_name=versions_txt_file, modules=existing_modules.values(), ) if __name__ == "__main__": From 082e1e2c9320ac4360b10024f0ded533000dfe28 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 14 Nov 2022 13:20:23 -0500 Subject: [PATCH 2/5] not to include irrelevant modules in monorepo --- docker/owlbot/java/bin/entrypoint.sh | 4 ++-- docker/owlbot/java/src/fix-poms.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docker/owlbot/java/bin/entrypoint.sh b/docker/owlbot/java/bin/entrypoint.sh index a3721cc75..b54e127da 100755 --- a/docker/owlbot/java/bin/entrypoint.sh +++ b/docker/owlbot/java/bin/entrypoint.sh @@ -17,8 +17,8 @@ set -e # Runs template and etc in current working directory function processModule() { - # templates - echo "Generating templates..." + # templates as well as retrieving files from owl-bot-staging + echo "Generating templates and retrieving files from owl-bot-staging directory..." /owlbot/bin/write_templates.sh echo "...done" diff --git a/docker/owlbot/java/src/fix-poms.py b/docker/owlbot/java/src/fix-poms.py index 479f0a3c7..f25c67713 100644 --- a/docker/owlbot/java/src/fix-poms.py +++ b/docker/owlbot/java/src/fix-poms.py @@ -308,6 +308,7 @@ def main(): else: excluded_poms_list = "" + # Missing Case 1: When this library ('java-XXX' module) is new. if artifact_id not in existing_modules: existing_modules[artifact_id] = module.Module( group_id=group_id, @@ -330,9 +331,18 @@ def main(): required_dependencies = {} for dependency_module in existing_modules: - if dependency_module not in excluded_dependencies_list: - required_dependencies[dependency_module] = existing_modules[dependency_module] + if dependency_module in excluded_dependencies_list: + continue + artifact_id = existing_modules[dependency_module].artifact_id + if monorepo and not os.path.isdir(artifact_id): + # In monorepo, existing_modules are loaded form the root + # versions.txt and thus includes irrelevant artifacts + continue + required_dependencies[dependency_module] = existing_modules[dependency_module] + # Missing Case 2: There's a new proto-XXX and grpc-XXX directory. It's a new + # version in the proto file to a library. Both a new library and existing + # library. for path in glob.glob("proto-google-*"): if not path in existing_modules: existing_modules[path] = module.Module( From 97fed3c8ad7d9c4257529ee954220e44f5aa25eb Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 15 Nov 2022 17:21:45 -0500 Subject: [PATCH 3/5] not to override global artifact_id --- docker/owlbot/java/src/fix-poms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/owlbot/java/src/fix-poms.py b/docker/owlbot/java/src/fix-poms.py index f25c67713..1f7eaaa02 100644 --- a/docker/owlbot/java/src/fix-poms.py +++ b/docker/owlbot/java/src/fix-poms.py @@ -333,8 +333,8 @@ def main(): for dependency_module in existing_modules: if dependency_module in excluded_dependencies_list: continue - artifact_id = existing_modules[dependency_module].artifact_id - if monorepo and not os.path.isdir(artifact_id): + dep_artifact_id = existing_modules[dependency_module].artifact_id + if monorepo and not os.path.isdir(dep_artifact_id): # In monorepo, existing_modules are loaded form the root # versions.txt and thus includes irrelevant artifacts continue From 547eb55bd861704fe229002d524e8eefb6e0ae1e Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 15 Nov 2022 17:50:08 -0500 Subject: [PATCH 4/5] ./versions.txt --- docker/owlbot/java/src/fix-poms.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/docker/owlbot/java/src/fix-poms.py b/docker/owlbot/java/src/fix-poms.py index 1f7eaaa02..390d36058 100644 --- a/docker/owlbot/java/src/fix-poms.py +++ b/docker/owlbot/java/src/fix-poms.py @@ -275,9 +275,9 @@ def update_bom_pom(filename: str, modules: List[module.Module]): tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") def main(): + print(f"working directory: {os.getcwd()}") with open(".repo-metadata.json", "r") as fp: repo_metadata = json.load(fp) - group_id, artifact_id = repo_metadata["distribution_name"].split(":") name = repo_metadata["name_pretty"] existing_modules = load_versions("versions.txt", group_id) @@ -288,7 +288,7 @@ def main(): existing_modules = load_versions("../versions.txt", group_id) if existing_modules: monorepo = True - print(f"monoreop? {monorepo}") + print(f"monorepo? {monorepo}") # extra modules that need to be manages in versions.txt if "extra_versioned_modules" in repo_metadata: @@ -481,12 +481,10 @@ def main(): name=name, ) - # For monorepo, we use the versions.txt at the root - versions_txt_file = "../versions.txt" if monorepo else "versions.txt" - if os.path.isfile(versions_txt_file): - print(f"updating modules in {versions_txt_file}") - else: - print(f"creating missing {versions_txt_file}") + # For monorepo, we use the versions.txt at the root. The "./" is needed + # for the templates.render(), which tries to create a directory. + versions_txt_file = "../versions.txt" if monorepo else "./versions.txt" + print(f"updating modules in {versions_txt_file}") existing_modules.pop(parent_artifact_id) # add extra modules to versions.txt From 275a22555bc5e3e4d59d449711bfd9a41c821176 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Wed, 16 Nov 2022 09:18:35 -0500 Subject: [PATCH 5/5] Update docker/owlbot/java/src/fix-poms.py Co-authored-by: Burke Davison <40617934+burkedavison@users.noreply.github.com> --- docker/owlbot/java/src/fix-poms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/owlbot/java/src/fix-poms.py b/docker/owlbot/java/src/fix-poms.py index 390d36058..b14877ef9 100644 --- a/docker/owlbot/java/src/fix-poms.py +++ b/docker/owlbot/java/src/fix-poms.py @@ -335,7 +335,7 @@ def main(): continue dep_artifact_id = existing_modules[dependency_module].artifact_id if monorepo and not os.path.isdir(dep_artifact_id): - # In monorepo, existing_modules are loaded form the root + # In monorepo, existing_modules are loaded from the root # versions.txt and thus includes irrelevant artifacts continue required_dependencies[dependency_module] = existing_modules[dependency_module]