From ced8c08ffe902a01a631dd4ac343c8b4505af075 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 30 May 2024 17:58:44 +0200 Subject: [PATCH 1/7] WIP: methods to run gh create issue Signed-off-by: Jose Luis Rivero --- release.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/release.py b/release.py index 265796308..27023033a 100755 --- a/release.py +++ b/release.py @@ -2,6 +2,7 @@ from __future__ import print_function from argparse import RawTextHelpFormatter +from tempfile import NamedTemporaryFile import subprocess import sys import tempfile @@ -515,9 +516,51 @@ def display_help_job_chain_for_source_calls(args): f'{releasepy_check_url}') +def get_vendor_repo_name(args): + canonical_name = get_canonical_package_name(args.package) + return canonical_name.replace('-', '_') + "_vendor" + + +def create_issue_in_repo(github_repo, title, body): + _out = "" + with NamedTemporaryFile("w", delete=True) as f: + f.write(body) + f.flush() + cmd = ['gh', 'issue', 'create', + '--repo', github_repo, + '--title', title, + '--body-file', f.name] + _out, _err = check_call(cmd) + if _err: + print(f"An error happened running gh issue cmd: {_err}") + sys.exit(1) + return _out + + + +def create_issue_in_gz_vendor_repo(args, ros_distro): + gz_vendor_repo = 'gazebo-release/' + get_vendor_repo_name(args) + print(gz_vendor_repo) + gz_vendor_repo = 'j-rivero/test' + title = f'Update version for {ros_distro} to the latest tag of {args.package}: {args.version}' + body = f'The {get_canonical_package_name(args.package)} repository tagged a new: {args.version} '\ + 'This repository needs to be updated accordingly for the branch ${ros_distro}:\n'\ + ' * Sync to the new version in CMakelists.txt \n'\ + ' * Bump the patch version in package.xml \n'\ + ' * Run the release process for this ROS package' + + return create_issue_in_repo(gz_vendor_repo, title, body) + + + def go(argv): args = parse_args(argv) + ros_distro = 'rolling' + out = create_issue_in_gz_vendor_repo(args, ros_distro) + print(out) + sys.exit(2) + # Default to release 1 if not present if not args.release_version: args.release_version = 1 From b68bd353397740b26d3dba319aafff9ca6c4237b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 31 May 2024 13:24:50 +0200 Subject: [PATCH 2/7] Fully working version Signed-off-by: Jose Luis Rivero --- release.py | 49 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/release.py b/release.py index 27023033a..784736b4b 100755 --- a/release.py +++ b/release.py @@ -23,6 +23,7 @@ LINUX_DISTROS = ['ubuntu', 'debian'] SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' +ROS_VENDOR = {'harmonic': 'rolling'} OSRF_REPOS_SUPPORTED = "stable prerelease nightly testing none" @@ -516,9 +517,25 @@ def display_help_job_chain_for_source_calls(args): f'{releasepy_check_url}') -def get_vendor_repo_name(args): - canonical_name = get_canonical_package_name(args.package) - return canonical_name.replace('-', '_') + "_vendor" +def get_collections_for_package(package_name, version): + script_directory = os.path.dirname(os.path.abspath(sys.argv[0])) + helper_script = f'{script_directory}/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py' + collection_yaml = f'{script_directory}/jenkins-scripts/dsl/gz-collections.yaml' + cmd = [helper_script, + get_canonical_package_name(package_name), + version, + collection_yaml] + _out, _err = check_call(cmd, IGNORE_DRY_RUN) + if _err: + print(f"An error happened running get_collections_from_package_and_version: {_err}") + sys.exit(1) + collection_list = _out.decode().strip().split(' ') + return collection_list + + +def get_vendor_github_repo(package_name): + canonical_name = get_canonical_package_name(package_name) + return f"gazebo-release/{canonical_name.replace('-', '_')}_vendor" def create_issue_in_repo(github_repo, title, body): @@ -534,33 +551,26 @@ def create_issue_in_repo(github_repo, title, body): if _err: print(f"An error happened running gh issue cmd: {_err}") sys.exit(1) - return _out - + if DRY_RUN: + return f"http://github.com/{github_repo}/issues/#dry-run-no-number" + return _out.decode().replace('\n', '') def create_issue_in_gz_vendor_repo(args, ros_distro): - gz_vendor_repo = 'gazebo-release/' + get_vendor_repo_name(args) - print(gz_vendor_repo) + gz_vendor_repo = get_vendor_github_repo(args.package) gz_vendor_repo = 'j-rivero/test' title = f'Update version for {ros_distro} to the latest tag of {args.package}: {args.version}' body = f'The {get_canonical_package_name(args.package)} repository tagged a new: {args.version} '\ - 'This repository needs to be updated accordingly for the branch ${ros_distro}:\n'\ + 'This repository needs to be updated accordingly for the branch {ros_distro}:\n'\ ' * Sync to the new version in CMakelists.txt \n'\ ' * Bump the patch version in package.xml \n'\ ' * Run the release process for this ROS package' - return create_issue_in_repo(gz_vendor_repo, title, body) - def go(argv): args = parse_args(argv) - ros_distro = 'rolling' - out = create_issue_in_gz_vendor_repo(args, ros_distro) - print(out) - sys.exit(2) - # Default to release 1 if not present if not args.release_version: args.release_version = 1 @@ -684,6 +694,15 @@ def go(argv): args.version) display_help_job_chain_for_source_calls(args) + print("ROS vendor packages that can be updated:") + for collection in get_collections_for_package(args.package, args.version): + if collection in ROS_VENDOR: + ros_distro = ROS_VENDOR[collection] + print(f" * Github {get_vendor_github_repo(args.package)} " + f"part of {collection} in ROS 2 {ros_distro}") + issue_url = create_issue_in_gz_vendor_repo(args, ros_distro) + print(f" + Issue created: {issue_url}") + if __name__ == '__main__': go(sys.argv) From db115270e12bc2e18c1151aa2024eb70794a977b Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 31 May 2024 16:12:00 +0200 Subject: [PATCH 3/7] Missing helper file Signed-off-by: Jose Luis Rivero --- ...et_collections_from_package_and_version.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py diff --git a/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py b/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py new file mode 100755 index 000000000..f7840304a --- /dev/null +++ b/jenkins-scripts/dsl/tools/get_collections_from_package_and_version.py @@ -0,0 +1,35 @@ +#!/usr/bin/python3 + +import sys +import yaml + + +# Function to find the collection name based on lib name and major version +def find_collection(data, lib_name, major_version): + instances = [] + + for collection in data['collections']: + for lib in collection['libs']: + if lib['name'] == lib_name and lib['major_version'] == major_version: + instances.append(collection['name']) + return instances + + +def get_major_version(version): + elements = version.split('.') + return int(elements[0]) + + +if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + +lib_name = sys.argv[1] +version = sys.argv[2] +yaml_file = sys.argv[3] + +with open(yaml_file, 'r') as file: + data = yaml.safe_load(file) + +collection_names = find_collection(data, lib_name, get_major_version(version)) +print(f"{' '.join(collection_names)}") From 67f4a2b904a8fb62957d9f8f0f4df9a3ca355c93 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 31 May 2024 18:09:49 +0200 Subject: [PATCH 4/7] Improve the error reporting on gh Signed-off-by: Jose Luis Rivero --- release.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/release.py b/release.py index 784736b4b..2e5e3f5d6 100755 --- a/release.py +++ b/release.py @@ -539,21 +539,21 @@ def get_vendor_github_repo(package_name): def create_issue_in_repo(github_repo, title, body): - _out = "" + _out = b"" with NamedTemporaryFile("w", delete=True) as f: f.write(body) f.flush() - cmd = ['gh', 'issue', 'create', + cmd = ['ghxx', 'issue', 'create', '--repo', github_repo, '--title', title, '--body-file', f.name] - _out, _err = check_call(cmd) - if _err: - print(f"An error happened running gh issue cmd: {_err}") - sys.exit(1) + try: + _out, _err = check_call(cmd) + except Exception as e: + _err = str(e) if DRY_RUN: - return f"http://github.com/{github_repo}/issues/#dry-run-no-number" - return _out.decode().replace('\n', '') + return f"http://github.com/{github_repo}/issues/#dry-run-no-number", "" + return _out.decode().replace('\n', ''), _err def create_issue_in_gz_vendor_repo(args, ros_distro): @@ -565,7 +565,13 @@ def create_issue_in_gz_vendor_repo(args, ros_distro): ' * Sync to the new version in CMakelists.txt \n'\ ' * Bump the patch version in package.xml \n'\ ' * Run the release process for this ROS package' - return create_issue_in_repo(gz_vendor_repo, title, body) + _out, _err = create_issue_in_repo(gz_vendor_repo, title, body) + if _err: + print(' !! An error happened running the "gh issue" cmd. Do not run this script again.\n' + ' Please create the issue manually. The error reported was:\n' + f' {_err}') + sys.exit(1) + return _out def go(argv): From 3236d73a33aabbf22c9163ac716e5eb46fa5c72f Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 31 May 2024 18:39:32 +0200 Subject: [PATCH 5/7] Implement a basic testing Signed-off-by: Jose Luis Rivero --- check_releasepy.bash | 46 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/check_releasepy.bash b/check_releasepy.bash index 921153017..c623d9e38 100755 --- a/check_releasepy.bash +++ b/check_releasepy.bash @@ -12,7 +12,7 @@ exec_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - gz-foo 1.2.3 token ${test_params}"" + gz-foo 1.2.3 token ${test_params} } exec_ignition_releasepy_test() @@ -22,7 +22,7 @@ exec_ignition_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-foo 1.2.3 token ${test_params}"" + ign-foo 1.2.3 token ${test_params} } exec_ignition_gazebo_releasepy_test() @@ -32,7 +32,18 @@ exec_ignition_gazebo_releasepy_test() ./release.py \ --dry-run \ --no-sanity-checks \ - ign-gazebo 1.2.3 token ${test_params}"" + ign-gazebo 1.2.3 token ${test_params} +} + +exec_releasepy_with_real_gz() +{ + gz_pkg=${1} major_version=${2} + ./release.py \ + --dry-run \ + --no-sanity-checks \ + --source-repo-uri http://github.com/gazebosim/gz-common \ + --source-repo-existing-ref http://github.com/gazebosim/gz-common/foo-tag \ + "${gz_pkg}" "${major_version}.x.y" token } expect_job_run() @@ -73,7 +84,26 @@ expect_param() echo "${param} not found in test output" exit 1 fi +} +expect_vendor_repo() +{ + output="${1}" repo="${2}" + + if ! grep "Github ${repo}" <<< "${output}"; then + echo "${repo} not found in test output" + exit 1 + fi +} + +expect_no_vendor() +{ + output="${1}" + + if grep -q 'in ROS 2' <<< "${output}"; then + echo "ROS 2 string found in output" + exit 1 + fi } source_repo_uri_test=$(exec_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") @@ -81,6 +111,7 @@ expect_job_run "${source_repo_uri_test}" "gz-foo-source" expect_job_not_run "${source_repo_uri_test}" "gz-foo-debbuilder" expect_number_of_jobs "${source_repo_uri_test}" "1" expect_param "${source_repo_uri_test}" "SOURCE_REPO_URI=https%3A%2F%2Fgithub.com%2Fgazebosim%2Fgz-foo.git" +expect_no_vendor "${source_repo_uri_test}" # non existing package source_tarball_uri_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz") expect_job_run "${source_tarball_uri_test}" "gz-foo-debbuilder" @@ -88,6 +119,7 @@ expect_job_run "${source_tarball_uri_test}" "generic-release-homebrew_pull_reque expect_job_not_run "${source_tarball_uri_test}" "gz-foo-source" expect_number_of_jobs "${source_tarball_uri_test}" "7" expect_param "${source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fgz-foo-1.2.3.tar.gz" +expect_no_vendor "${source_tarball_uri_test}" nightly_test=$(exec_releasepy_test "--nightly-src-branch my-nightly-branch3 --upload-to-repo nightly") expect_job_run "${nightly_test}" "gz-foo-debbuilder" @@ -95,6 +127,7 @@ expect_job_not_run "${nightly_test}" "generic-release-homebrew_pull_request_upda expect_job_not_run "${nightly_test}" "gz-foo-source" expect_number_of_jobs "${nightly_test}" "2" expect_param "${nightly_test}" "SOURCE_TARBALL_URI=my-nightly-branch3" +expect_no_vendor "${nightly_test}" bump_linux_test=$(exec_releasepy_test "--source-tarball-uri https://gazebosim/gz-foo-1.2.3.tar.gz --only-bump-revision-linux -r 2") expect_job_run "${bump_linux_test}" "gz-foo-debbuilder" @@ -102,6 +135,7 @@ expect_job_not_run "${bump_linux_test}" "generic-release-homebrew_pull_request_u expect_job_not_run "${bump_linux_test}" "gz-foo-source" expect_number_of_jobs "${bump_linux_test}" "6" expect_param "${bump_linux_test}" "RELEASE_VERSION=2" +expect_no_vendor "${bump_linux_test}" ignition_test=$(exec_ignition_releasepy_test "--source-repo-uri https://github.com/gazebosim/gz-foo.git") expect_job_run "${ignition_test}" "gz-foo-source" @@ -128,3 +162,9 @@ expect_number_of_jobs "${ign_gazebo_source_tarball_uri_test}" "7" expect_param "${ign_gazebo_source_tarball_uri_test}" "SOURCE_TARBALL_URI=https%3A%2F%2Fgazebosim%2Fign-gazebo-1.2.3.tar.gz" expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE=ign-gazebo" expect_param "${ign_gazebo_source_tarball_uri_test}" "PACKAGE_ALIAS=ignition-gazebo" + +ros_vendor_test=$(exec_releasepy_with_real_gz gz-fuel-tools 9) +expect_vendor_repo "${ros_vendor_test}" gazebo-release/gz_fuel_tools_vendor + +ros_vendor_test=$(exec_releasepy_with_real_gz gz-cmake 2) +expect_no_vendor "${bump_linux_test}" From 8ed28301c45bcc1016946d77de85f3df7988b790 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 3 Jun 2024 12:48:42 +0200 Subject: [PATCH 6/7] Cleanup Signed-off-by: Jose Luis Rivero --- release.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release.py b/release.py index 2e5e3f5d6..0fc79b5b0 100755 --- a/release.py +++ b/release.py @@ -540,6 +540,8 @@ def get_vendor_github_repo(package_name): def create_issue_in_repo(github_repo, title, body): _out = b"" + # For parsing the \n correctly we need to store the content in a temp + # file and pass it as --body-file with NamedTemporaryFile("w", delete=True) as f: f.write(body) f.flush() @@ -558,7 +560,6 @@ def create_issue_in_repo(github_repo, title, body): def create_issue_in_gz_vendor_repo(args, ros_distro): gz_vendor_repo = get_vendor_github_repo(args.package) - gz_vendor_repo = 'j-rivero/test' title = f'Update version for {ros_distro} to the latest tag of {args.package}: {args.version}' body = f'The {get_canonical_package_name(args.package)} repository tagged a new: {args.version} '\ 'This repository needs to be updated accordingly for the branch {ros_distro}:\n'\ From 0087404ca01c13117f9e20105881af8044e04812 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 7 Jun 2024 19:45:16 +0200 Subject: [PATCH 7/7] WIP: create PR for vendor_repositories Signed-off-by: Jose Luis Rivero --- release.py | 84 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/release.py b/release.py index 0fc79b5b0..a21d47e1b 100755 --- a/release.py +++ b/release.py @@ -3,6 +3,7 @@ from __future__ import print_function from argparse import RawTextHelpFormatter from tempfile import NamedTemporaryFile +from tempfile import TemporaryDirectory import subprocess import sys import tempfile @@ -538,46 +539,61 @@ def get_vendor_github_repo(package_name): return f"gazebo-release/{canonical_name.replace('-', '_')}_vendor" -def create_issue_in_repo(github_repo, title, body): - _out = b"" - # For parsing the \n correctly we need to store the content in a temp - # file and pass it as --body-file - with NamedTemporaryFile("w", delete=True) as f: - f.write(body) - f.flush() - cmd = ['ghxx', 'issue', 'create', - '--repo', github_repo, - '--title', title, - '--body-file', f.name] - try: - _out, _err = check_call(cmd) - except Exception as e: - _err = str(e) - if DRY_RUN: - return f"http://github.com/{github_repo}/issues/#dry-run-no-number", "" - return _out.decode().replace('\n', ''), _err - - -def create_issue_in_gz_vendor_repo(args, ros_distro): - gz_vendor_repo = get_vendor_github_repo(args.package) - title = f'Update version for {ros_distro} to the latest tag of {args.package}: {args.version}' - body = f'The {get_canonical_package_name(args.package)} repository tagged a new: {args.version} '\ - 'This repository needs to be updated accordingly for the branch {ros_distro}:\n'\ - ' * Sync to the new version in CMakelists.txt \n'\ - ' * Bump the patch version in package.xml \n'\ - ' * Run the release process for this ROS package' - _out, _err = create_issue_in_repo(gz_vendor_repo, title, body) - if _err: - print(' !! An error happened running the "gh issue" cmd. Do not run this script again.\n' - ' Please create the issue manually. The error reported was:\n' - f' {_err}') +def get_vendor_repo_url(package_name): + return f"https://github.com/{get_vendor_github_repo(package_name)}" + + +def prepare_vendor_pr_temp_workspace(package_name): + pr_ws_dir = tempfile.mkdtemp() + gz_vendor_tool = os.path.join(pr_ws_dir, "gz_vendor") + cmd = ['git', 'clone', '-q', + 'https://github.com/gazebo-tooling/gz_vendor/', + gz_vendor_tool] + _, _err_tool = check_call(cmd) + gz_vendor_repo = os.path.join(pr_ws_dir, 'gz_vendor_repo') + cmd = ['git', 'clone', '-q', + get_vendor_repo_url(package_name), + gz_vendor_repo] + _, _err_repo = check_call(cmd) + if _err_tool or _err_repo: + print("Problems with cloning vendor and tool repos:") + print(f"{_err_tool} {_err_repo}") + sys.exit(1) + + return gz_vendor_tool, gz_vendor_repo + + +def execute_update_vendor_package_tool(vendor_tool_path, vendor_repo_path): + run_cmd = ['python3', + f"{vendor_tool_path}/create_gz_vendor_pkg/create_vendor_package.py", + 'package.xml', + '--output_dir', vendor_repo_path] + _, _err_run = check_call(run_cmd) + if _err_run: + print("Problems running the create_vendor_package.py script:") sys.exit(1) - return _out + + +def create_pr_in_gz_vendor_repo(args): + vendor_tool_path, vendor_repo_path = \ + prepare_vendor_pr_temp_workspace(args.package) + execute_update_vendor_package_tool( + vendor_tool_path, vendor_repo_path) + + cmd_diff = ['git', "-C", vendor_repo_path, 'diff'] + _out, _ = check_call(cmd_diff) + print(vendor_repo_path) + print(_out.decode()) + + def go(argv): args = parse_args(argv) + create_pr_in_gz_vendor_repo(args) + sys.exit(0) + # Default to release 1 if not present if not args.release_version: args.release_version = 1