From eb912d5e5971959e00419b2283ecdf5dc5fb8d0e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 23 Jul 2024 01:25:08 +0000 Subject: [PATCH 1/8] fix: make grpc deps `common-protos` and `iam-v1` test-scoped --- library_generation/owlbot/src/fix_poms.py | 42 ++++++++++++++++++- .../owlbot/templates/poms/cloud_pom.xml.j2 | 16 +++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/library_generation/owlbot/src/fix_poms.py b/library_generation/owlbot/src/fix_poms.py index b88cd3d6bb..a91fedbed5 100644 --- a/library_generation/owlbot/src/fix_poms.py +++ b/library_generation/owlbot/src/fix_poms.py @@ -92,7 +92,10 @@ def _is_cloud_client(existing_modules: List[module.Module]) -> bool: def update_cloud_pom( - filename: str, proto_modules: List[module.Module], grpc_modules: List[module.Module] + filename: str, + proto_modules: List[module.Module], + grpc_modules: List[module.Module], + repo_metadata: dict, ): tree = etree.parse(filename) root = tree.getroot() @@ -104,6 +107,39 @@ def update_cloud_pom( if m.find("{http://maven.apache.org/POM/4.0.0}artifactId") is not None ] + # as of July 2024, we have two dependencies that should be declared as + # test-scoped: grpc-google-common-protos and grpc-google-iam-v1. Only in + # java-storage and java-spanner we keep them as they are + TEST_SCOPED_DEPENDENCIES = ["grpc-google-common-protos", "grpc-google-iam-v1"] + print( + 'converting old dependencies "grpc-google-common-protos" and "grpc-google-iam-v1" to test-scoped' + ) + for d in dependencies: + if repo_metadata["repo_short"] in ["java-spanner", "java-storage"]: + print( + f"skipping test-scoped-dependency fix for special case repo: {repo_metadata['repo_short']}" + ) + continue + artifact_query = "{http://maven.apache.org/POM/4.0.0}artifactId" + scope_query = "{http://maven.apache.org/POM/4.0.0}scope" + current_scope = d.find(scope_query) + artifact_id_elem = d.find(artifact_query) + if artifact_id_elem is None: + continue + artifact_id = artifact_id_elem.text + is_test_scoped = ( + current_scope.text == "test" if current_scope is not None else False + ) + if artifact_id in TEST_SCOPED_DEPENDENCIES and not is_test_scoped: + new_scope = etree.Element(scope_query) + new_scope.text = "test" + if current_scope is not None: + d.replace(current_scope, new_scope) + else: + d.append(new_scope) + new_scope.tail = "\n " + new_scope.getprevious().tail = "\n " + try: grpc_index = _find_dependency_index( dependencies, "com.google.api.grpc", "grpc-" @@ -492,7 +528,9 @@ def main(versions_file, monorepo): if os.path.isfile(f"{artifact_id}/pom.xml"): print("updating modules in cloud pom.xml") if artifact_id not in excluded_poms_list: - update_cloud_pom(f"{artifact_id}/pom.xml", proto_modules, grpc_modules) + update_cloud_pom( + f"{artifact_id}/pom.xml", proto_modules, grpc_modules, repo_metadata + ) elif artifact_id not in excluded_poms_list: print("creating missing cloud pom.xml") templates.render( diff --git a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 index ed1f6fbfea..e60cd96e4a 100644 --- a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 @@ -39,10 +39,12 @@ com.google.protobuf protobuf-java +{%- if repo in ['java-storage', 'java-spanner'] %} com.google.api.grpc proto-google-common-protos +{%- endif %} {% for module in proto_modules %} {{module.group_id}} @@ -72,16 +74,30 @@ com.google.api.grpc proto-google-iam-v1 +{%- if repo not in ['java-storage', 'java-spanner'] %} com.google.api.grpc grpc-google-iam-v1 +{%- endif %} org.threeten threetenbp +{%- if repo not in ['java-storage', 'java-spanner'] %} + + com.google.api.grpc + proto-google-common-protos + test + + + com.google.api.grpc + grpc-google-iam-v1 + test + +{%- endif %} junit junit From daf93fb100be8ed641902a5e2d0038cf3e95f683 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 25 Jul 2024 17:08:46 +0000 Subject: [PATCH 2/8] add java-dataproc to ignored modules --- library_generation/owlbot/src/fix_poms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library_generation/owlbot/src/fix_poms.py b/library_generation/owlbot/src/fix_poms.py index a91fedbed5..9f07bcbb00 100644 --- a/library_generation/owlbot/src/fix_poms.py +++ b/library_generation/owlbot/src/fix_poms.py @@ -109,13 +109,13 @@ def update_cloud_pom( # as of July 2024, we have two dependencies that should be declared as # test-scoped: grpc-google-common-protos and grpc-google-iam-v1. Only in - # java-storage and java-spanner we keep them as they are + # java-storage, java-spanner and java-dataproc we keep them as they are TEST_SCOPED_DEPENDENCIES = ["grpc-google-common-protos", "grpc-google-iam-v1"] print( 'converting old dependencies "grpc-google-common-protos" and "grpc-google-iam-v1" to test-scoped' ) for d in dependencies: - if repo_metadata["repo_short"] in ["java-spanner", "java-storage"]: + if repo_metadata["repo_short"] in ["java-spanner", "java-storage", "java-dataproc"]: print( f"skipping test-scoped-dependency fix for special case repo: {repo_metadata['repo_short']}" ) From 0c4be7941f62a005523f0ce5dbe642606b94912d Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 26 Jul 2024 12:56:58 +0000 Subject: [PATCH 3/8] fix new library template --- library_generation/owlbot/templates/poms/cloud_pom.xml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 index e60cd96e4a..ec33600bf8 100644 --- a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 @@ -39,7 +39,7 @@ com.google.protobuf protobuf-java -{%- if repo in ['java-storage', 'java-spanner'] %} +{%- if repo in ['java-storage', 'java-spanner', 'java-dataproc'] %} com.google.api.grpc proto-google-common-protos @@ -74,7 +74,7 @@ com.google.api.grpc proto-google-iam-v1 -{%- if repo not in ['java-storage', 'java-spanner'] %} +{%- if repo in ['java-storage', 'java-spanner', 'java-dataproc'] %} com.google.api.grpc grpc-google-iam-v1 @@ -86,7 +86,7 @@ -{%- if repo not in ['java-storage', 'java-spanner'] %} +{%- if repo not in ['java-storage', 'java-spanner','java-dataproc'] %} com.google.api.grpc proto-google-common-protos From 1fabb63c42898eaa6795cee50497a815a9d4c837 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 26 Jul 2024 13:40:09 +0000 Subject: [PATCH 4/8] modify golden file --- .../test-owlbot/java-admanager/ad-manager/pom-golden.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library_generation/test/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml b/library_generation/test/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml index 106b55c8c5..4c8a8a8343 100644 --- a/library_generation/test/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml +++ b/library_generation/test/resources/test-owlbot/java-admanager/ad-manager/pom-golden.xml @@ -64,6 +64,7 @@ com.google.api.grpc grpc-google-common-protos + test com.google.api.grpc @@ -72,6 +73,7 @@ com.google.api.grpc grpc-google-iam-v1 + test org.threeten From fdd9d27cdd8c42ddada5b41df7bd06e2e474ccf0 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 26 Jul 2024 13:49:07 +0000 Subject: [PATCH 5/8] lint --- library_generation/owlbot/src/fix_poms.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/library_generation/owlbot/src/fix_poms.py b/library_generation/owlbot/src/fix_poms.py index 9f07bcbb00..c886f494b0 100644 --- a/library_generation/owlbot/src/fix_poms.py +++ b/library_generation/owlbot/src/fix_poms.py @@ -115,7 +115,11 @@ def update_cloud_pom( 'converting old dependencies "grpc-google-common-protos" and "grpc-google-iam-v1" to test-scoped' ) for d in dependencies: - if repo_metadata["repo_short"] in ["java-spanner", "java-storage", "java-dataproc"]: + if repo_metadata["repo_short"] in [ + "java-spanner", + "java-storage", + "java-dataproc", + ]: print( f"skipping test-scoped-dependency fix for special case repo: {repo_metadata['repo_short']}" ) From c9156146c2bf075f57772622450a14ee59d383dc Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 30 Jul 2024 17:32:29 +0000 Subject: [PATCH 6/8] correct pom template --- .../owlbot/templates/poms/cloud_pom.xml.j2 | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 index ec33600bf8..95ad72c497 100644 --- a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 @@ -39,12 +39,10 @@ com.google.protobuf protobuf-java -{%- if repo in ['java-storage', 'java-spanner', 'java-dataproc'] %} com.google.api.grpc proto-google-common-protos -{%- endif %} {% for module in proto_modules %} {{module.group_id}} @@ -68,13 +66,13 @@ com.google.api.grpc - grpc-google-common-protos + proto-google-iam-v1 +{%- if repo in ['java-storage', 'java-spanner', 'java-dataproc'] %} com.google.api.grpc - proto-google-iam-v1 + grpc-google-common-protos -{%- if repo in ['java-storage', 'java-spanner', 'java-dataproc'] %} com.google.api.grpc grpc-google-iam-v1 @@ -89,7 +87,7 @@ {%- if repo not in ['java-storage', 'java-spanner','java-dataproc'] %} com.google.api.grpc - proto-google-common-protos + grpc-google-common-protos test From 18591bd320825f762c949aa865a2ed425fd72864 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 23 Aug 2024 14:07:29 +0000 Subject: [PATCH 7/8] apply this change only on monorepos --- library_generation/owlbot/src/fix_poms.py | 77 ++++++++++++----------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/library_generation/owlbot/src/fix_poms.py b/library_generation/owlbot/src/fix_poms.py index c886f494b0..9ee7514a4d 100644 --- a/library_generation/owlbot/src/fix_poms.py +++ b/library_generation/owlbot/src/fix_poms.py @@ -15,6 +15,8 @@ import sys import glob import json +from xml.etree.ElementTree import ElementTree + from lxml import etree import os import re @@ -95,7 +97,7 @@ def update_cloud_pom( filename: str, proto_modules: List[module.Module], grpc_modules: List[module.Module], - repo_metadata: dict, + is_monorepo: bool, ): tree = etree.parse(filename) root = tree.getroot() @@ -107,42 +109,8 @@ def update_cloud_pom( if m.find("{http://maven.apache.org/POM/4.0.0}artifactId") is not None ] - # as of July 2024, we have two dependencies that should be declared as - # test-scoped: grpc-google-common-protos and grpc-google-iam-v1. Only in - # java-storage, java-spanner and java-dataproc we keep them as they are - TEST_SCOPED_DEPENDENCIES = ["grpc-google-common-protos", "grpc-google-iam-v1"] - print( - 'converting old dependencies "grpc-google-common-protos" and "grpc-google-iam-v1" to test-scoped' - ) - for d in dependencies: - if repo_metadata["repo_short"] in [ - "java-spanner", - "java-storage", - "java-dataproc", - ]: - print( - f"skipping test-scoped-dependency fix for special case repo: {repo_metadata['repo_short']}" - ) - continue - artifact_query = "{http://maven.apache.org/POM/4.0.0}artifactId" - scope_query = "{http://maven.apache.org/POM/4.0.0}scope" - current_scope = d.find(scope_query) - artifact_id_elem = d.find(artifact_query) - if artifact_id_elem is None: - continue - artifact_id = artifact_id_elem.text - is_test_scoped = ( - current_scope.text == "test" if current_scope is not None else False - ) - if artifact_id in TEST_SCOPED_DEPENDENCIES and not is_test_scoped: - new_scope = etree.Element(scope_query) - new_scope.text = "test" - if current_scope is not None: - d.replace(current_scope, new_scope) - else: - d.append(new_scope) - new_scope.tail = "\n " - new_scope.getprevious().tail = "\n " + if is_monorepo: + _set_test_scoped_deps(dependencies) try: grpc_index = _find_dependency_index( @@ -209,6 +177,39 @@ def update_cloud_pom( tree.write(filename, pretty_print=True, xml_declaration=True, encoding="utf-8") +def _set_test_scoped_deps(dependencies: list[ElementTree]) -> None: + """ + As of July 2024, we have two dependencies that should be declared as + test-scoped in a monorepo: grpc-google-common-protos and grpc-google-iam-v1. + HW libraries are treated as usual + :param dependencies: List of XML Objects representing a + """ + TEST_SCOPED_DEPENDENCIES = ["grpc-google-common-protos", "grpc-google-iam-v1"] + print( + 'converting dependencies "grpc-google-common-protos" and "grpc-google-iam-v1" to test-scoped' + ) + for d in dependencies: + artifact_query = "{http://maven.apache.org/POM/4.0.0}artifactId" + scope_query = "{http://maven.apache.org/POM/4.0.0}scope" + current_scope = d.find(scope_query) + artifact_id_elem = d.find(artifact_query) + if artifact_id_elem is None: + continue + artifact_id = artifact_id_elem.text + is_test_scoped = ( + current_scope.text == "test" if current_scope is not None else False + ) + if artifact_id in TEST_SCOPED_DEPENDENCIES and not is_test_scoped: + new_scope = etree.Element(scope_query) + new_scope.text = "test" + if current_scope is not None: + d.replace(current_scope, new_scope) + else: + d.append(new_scope) + new_scope.tail = "\n " + new_scope.getprevious().tail = "\n " + + def update_parent_pom(filename: str, modules: List[module.Module]): tree = etree.parse(filename) root = tree.getroot() @@ -533,7 +534,7 @@ def main(versions_file, monorepo): print("updating modules in cloud pom.xml") if artifact_id not in excluded_poms_list: update_cloud_pom( - f"{artifact_id}/pom.xml", proto_modules, grpc_modules, repo_metadata + f"{artifact_id}/pom.xml", proto_modules, grpc_modules, monorepo ) elif artifact_id not in excluded_poms_list: print("creating missing cloud pom.xml") From 06e2f564089f64de629d865500117eb6ae41a1a3 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Fri, 23 Aug 2024 14:13:37 +0000 Subject: [PATCH 8/8] update templates to decide via monorepo variable --- library_generation/owlbot/templates/poms/cloud_pom.xml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 index 95ad72c497..d0ae8cd7c5 100644 --- a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 @@ -68,7 +68,7 @@ com.google.api.grpc proto-google-iam-v1 -{%- if repo in ['java-storage', 'java-spanner', 'java-dataproc'] %} +{%- if not monorepo %} com.google.api.grpc grpc-google-common-protos @@ -84,7 +84,7 @@ -{%- if repo not in ['java-storage', 'java-spanner','java-dataproc'] %} +{%- if monorepo %} com.google.api.grpc grpc-google-common-protos