Skip to content

Commit

Permalink
update release process
Browse files Browse the repository at this point in the history
  • Loading branch information
nickfyson committed Dec 5, 2023
1 parent bc50092 commit 7b040eb
Show file tree
Hide file tree
Showing 14 changed files with 407 additions and 77 deletions.
2 changes: 1 addition & 1 deletion .github/actions/prepare-test/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ runs:
echo "tools-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/codeql-bundle-$version-manual/$artifact_name" >> $GITHUB_OUTPUT
elif [[ ${{ inputs.version }} == *"stable"* ]]; then
version=`echo ${{ inputs.version }} | sed -e 's/^.*\-//'`
echo "tools-url=https://github.com/github/codeql-action/releases/download/codeql-bundle-$version/$artifact_name" >> $GITHUB_OUTPUT
echo "tools-url=https://github.com/nickfyson-org/codeql-action/releases/download/codeql-bundle-$version/$artifact_name" >> $GITHUB_OUTPUT
elif [[ ${{ inputs.version }} == "latest" ]]; then
echo "tools-url=latest" >> $GITHUB_OUTPUT
elif [[ ${{ inputs.version }} == "default" ]]; then
Expand Down
25 changes: 25 additions & 0 deletions .github/actions/release-branches/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: 'Release branches'
description: 'Determine branches for release & backport'
inputs:
major_version:
description: 'The version as extracted from the package.json file'
required: true
latest_tag:
description: 'The most recent tag published to the repository'
required: true
outputs:
backport_source_branch:
description: "The release branch for the given tag"
value: ${{ steps.branches.outputs.backport_source_branch }}
backport_target_branches:
description: "JSON encoded list of branches to target with backports"
value: ${{ steps.branches.outputs.backport_target_branches }}
runs:
using: "composite"
steps:
- id: branches
run: |
python ${{ github.action_path }}/release-branches.py \
--major-version ${{ inputs.major_version }} \
--latest-tag ${{ inputs.latest_tag }}
shell: bash
57 changes: 57 additions & 0 deletions .github/actions/release-branches/release-branches.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import argparse
import os, json
import subprocess

# Name of the remote
ORIGIN = 'origin'

OLDEST_SUPPORTED_MAJOR_VERSION = 2

# Runs git with the given args and returns the stdout.
# Raises an error if git does not exit successfully (unless passed
# allow_non_zero_exit_code=True).
def run_git(*args, allow_non_zero_exit_code=False):
cmd = ['git', *args]
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if not allow_non_zero_exit_code and p.returncode != 0:
raise Exception(f'Call to {" ".join(cmd)} exited with code {p.returncode} stderr: {p.stderr.decode("ascii")}.')
return p.stdout.decode('ascii')

def main():

parser = argparse.ArgumentParser()
parser.add_argument("--major-version", required=True, type=str, help="The major version of the release")
parser.add_argument("--latest-tag", required=True, type=str, help="The most recent tag published to the repository")
args = parser.parse_args()

major_version = args.major_version
latest_tag = args.latest_tag

print("major_version: " + major_version)
print("latest_tag: " + latest_tag)

# If this is a primary release, we backport to all supported branches,
# so we check whether the major_version taken from the package.json
# is greater than or equal to the latest tag pulled from the repo.
# For example...
# 'v1' >= 'v2' is False # we're operating from an older release branch and should not backport
# 'v2' >= 'v2' is True # the normal case where we're updating the current version
# 'v3' >= 'v2' is True # in this case we are making the first release of a new major version
consider_backports = ( major_version >= latest_tag.split(".")[0] )

with open(os.environ["GITHUB_OUTPUT"], "a") as f:

f.write(f"backport_source_branch=releases/{major_version}\n")

backport_target_branches = []

if consider_backports:
for i in range(int(major_version.strip("v"))-1, 0, -1):
branch_name = f"releases/v{i}"
if i >= OLDEST_SUPPORTED_MAJOR_VERSION:
backport_target_branches.append(branch_name)

f.write("backport_target_branches="+json.dumps(backport_target_branches)+"\n")

if __name__ == "__main__":
main()
33 changes: 33 additions & 0 deletions .github/actions/release-initialise/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: 'Prepare release job'
description: 'Executed preparatory steps before update a release branch'

runs:
using: "composite"
steps:

- name: Dump environment
run: env
shell: bash

- name: Dump GitHub context
env:
GITHUB_CONTEXT: '${{ toJson(github) }}'
run: echo "$GITHUB_CONTEXT"
shell: bash

- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.8

- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install PyGithub==1.55 requests
shell: bash

- name: Update git config
run: |
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --global user.name "github-actions[bot]"
shell: bash
4 changes: 2 additions & 2 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
### Merge / deployment checklist

- [ ] Confirm this change is backwards compatible with existing workflows.
- [ ] Confirm the [readme](https://github.com/github/codeql-action/blob/main/README.md) has been updated if necessary.
- [ ] Confirm the [changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) has been updated if necessary.
- [ ] Confirm the [readme](https://github.com/nickfyson-org/codeql-action/blob/main/README.md) has been updated if necessary.
- [ ] Confirm the [changelog](https://github.com/nickfyson-org/codeql-action/blob/main/CHANGELOG.md) has been updated if necessary.
169 changes: 142 additions & 27 deletions .github/update-release-branch.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@
"""

SOURCE_BRANCH = 'main'
TARGET_BRANCH = 'releases/v2'

# Name of the remote
ORIGIN = 'origin'
Expand All @@ -34,7 +32,9 @@ def branch_exists_on_remote(branch_name):
return run_git('ls-remote', '--heads', ORIGIN, branch_name).strip() != ''

# Opens a PR from the given branch to the target branch
def open_pr(repo, all_commits, source_branch_short_sha, new_branch_name, conductor):
def open_pr(
repo, all_commits, source_branch_short_sha, new_branch_name, source_branch, target_branch,
conductor, is_primary_release, conflicted_files):
# Sort the commits into the pull requests that introduced them,
# and any commits that don't have a pull request
pull_requests = []
Expand All @@ -56,7 +56,7 @@ def open_pr(repo, all_commits, source_branch_short_sha, new_branch_name, conduct

# Start constructing the body text
body = []
body.append(f'Merging {source_branch_short_sha} into {TARGET_BRANCH}.')
body.append(f'Merging {source_branch_short_sha} into {target_branch}.')

body.append('')
body.append(f'Conductor for this PR is @{conductor}.')
Expand All @@ -79,20 +79,38 @@ def open_pr(repo, all_commits, source_branch_short_sha, new_branch_name, conduct

body.append('')
body.append('Please do the following:')
if len(conflicted_files) > 0:
body.append(' - [ ] Ensure `package.json` file contains the correct version.')
body.append(' - [ ] Add commits to this branch to resolve the merge conflicts ' +
'in the following files:')
body.extend([f' - [ ] `{file}`' for file in conflicted_files])
body.append(' - [ ] Ensure another maintainer has reviewed the additional commits you added to this ' +
'branch to resolve the merge conflicts.')
body.append(' - [ ] Ensure the CHANGELOG displays the correct version and date.')
body.append(' - [ ] Ensure the CHANGELOG includes all relevant, user-facing changes since the last release.')
body.append(f' - [ ] Check that there are not any unexpected commits being merged into the {TARGET_BRANCH} branch.')
body.append(f' - [ ] Check that there are not any unexpected commits being merged into the {target_branch} branch.')
body.append(' - [ ] Ensure the docs team is aware of any documentation changes that need to be released.')

if not is_primary_release:
body.append(' - [ ] Remove and re-add the "Update dependencies" label to the PR to trigger just this workflow.')
body.append(' - [ ] Wait for the "Update dependencies" workflow to push a commit updating the dependencies.')
body.append(' - [ ] Mark the PR as ready for review to trigger the full set of PR checks.')

body.append(' - [ ] Approve and merge this PR. Make sure `Create a merge commit` is selected rather than `Squash and merge` or `Rebase and merge`.')
body.append(' - [ ] Merge the mergeback PR that will automatically be created once this PR is merged.')

title = f'Merge {SOURCE_BRANCH} into {TARGET_BRANCH}'
if is_primary_release:
body.append(' - [ ] Merge the mergeback PR that will automatically be created once this PR is merged.')
body.append(' - [ ] Merge the v1 release PR that will automatically be created once this PR is merged.')

title = f'Merge {source_branch} into {target_branch}'
labels = ['Update dependencies'] if not is_primary_release else []

# Create the pull request
# PR checks won't be triggered on PRs created by Actions. Therefore mark the PR as draft so that
# a maintainer can take the PR out of draft, thereby triggering the PR checks.
pr = repo.create_pull(title=title, body='\n'.join(body), head=new_branch_name, base=TARGET_BRANCH, draft=True)
print(f'Created PR #{pr.number}')
pr = repo.create_pull(title=title, body='\n'.join(body), head=new_branch_name, base=target_branch, draft=True)
pr.add_to_labels(*labels)
print(f'Created PR #{str(pr.number)}')

# Assign the conductor
pr.add_to_assignees(conductor)
Expand All @@ -102,10 +120,10 @@ def open_pr(repo, all_commits, source_branch_short_sha, new_branch_name, conduct
# since the last release to the target branch.
# This will not include any commits that exist on the target branch
# that aren't on the source branch.
def get_commit_difference(repo):
def get_commit_difference(repo, source_branch, target_branch):
# Passing split nothing means that the empty string splits to nothing: compare `''.split() == []`
# to `''.split('\n') == ['']`.
commits = run_git('log', '--pretty=format:%H', f'{ORIGIN}/{TARGET_BRANCH}..{ORIGIN}/{SOURCE_BRANCH}').strip().split()
commits = run_git('log', '--pretty=format:%H', f'{ORIGIN}/{target_branch}..{ORIGIN}/{source_branch}').strip().split()

# Convert to full-fledged commit objects
commits = [repo.get_commit(c) for c in commits]
Expand Down Expand Up @@ -180,7 +198,25 @@ def main():
'--repository-nwo',
type=str,
required=True,
help='The nwo of the repository, for example github/codeql-action.'
help='The nwo of the repository, for example nickfyson-org/codeql-action.'
)
parser.add_argument(
'--source-branch',
type=str,
required=True,
help='Source branch for release branch update.'
)
parser.add_argument(
'--target-branch',
type=str,
required=True,
help='Target branch for release branch update.'
)
parser.add_argument(
'--is-primary-release',
action='store_true',
default=False,
help='Whether this update is the primary release for the current major version.'
)
parser.add_argument(
'--conductor',
Expand All @@ -191,18 +227,29 @@ def main():

args = parser.parse_args()

source_branch = args.source_branch
target_branch = args.target_branch
is_primary_release = args.is_primary_release

repo = Github(args.github_token).get_repo(args.repository_nwo)
version = get_current_version()

# the target branch will be of the form releases/vN, where N is the major version number
target_branch_major_version = target_branch.strip('releases/v')

# split version into major, minor, patch
_, v_minor, v_patch = get_current_version().split('.')

version = f"{target_branch_major_version}.{v_minor}.{v_patch}"

# Print what we intend to go
print(f'Considering difference between {SOURCE_BRANCH} and {TARGET_BRANCH}...')
source_branch_short_sha = run_git('rev-parse', '--short', f'{ORIGIN}/{SOURCE_BRANCH}').strip()
print(f'Current head of {SOURCE_BRANCH} is {source_branch_short_sha}.')
print(f'Considering difference between {source_branch} and {target_branch}...')
source_branch_short_sha = run_git('rev-parse', '--short', f'{ORIGIN}/{source_branch}').strip()
print(f'Current head of {source_branch} is {source_branch_short_sha}.')

# See if there are any commits to merge in
commits = get_commit_difference(repo=repo)
commits = get_commit_difference(repo=repo, source_branch=source_branch, target_branch=target_branch)
if len(commits) == 0:
print(f'No commits to merge from {SOURCE_BRANCH} to {TARGET_BRANCH}.')
print(f'No commits to merge from {source_branch} to {target_branch}.')
return

# The branch name is based off of the name of branch being merged into
Expand All @@ -220,17 +267,81 @@ def main():
# Create the new branch and push it to the remote
print(f'Creating branch {new_branch_name}.')

# If we're performing a standard release, there won't be any new commits on the target branch,
# as these will have already been merged back into the source branch. Therefore we can just
# start from the source branch.
run_git('checkout', '-b', new_branch_name, f'{ORIGIN}/{SOURCE_BRANCH}')
# The process of creating the v{Older} release can run into merge conflicts. We commit the unresolved
# conflicts so a maintainer can easily resolve them (vs erroring and requiring maintainers to
# reconstruct the release manually)
conflicted_files = []

if not is_primary_release:

# the source branch will be of the form releases/vN, where N is the major version number
source_branch_major_version = source_branch.strip('releases/v')

# If we're performing a backport, start from the target branch
print(f'Creating {new_branch_name} from the {ORIGIN}/{target_branch} branch')
run_git('checkout', '-b', new_branch_name, f'{ORIGIN}/{target_branch}')

# Revert the commit that we made as part of the last release that updated the version number and
# changelog to refer to {older}.x.x variants. This avoids merge conflicts in the changelog and
# package.json files when we merge in the v{latest} branch.
# This commit will not exist the first time we release the v{N-1} branch from the v{N} branch, so we
# use `git log --grep` to conditionally revert the commit.
print('Reverting the version number and changelog updates from the last release to avoid conflicts')
vOlder_update_commits = run_git('log', '--grep', '^Update version and changelog for v', '--format=%H').split()

if len(vOlder_update_commits) > 0:
print(f' Reverting {vOlder_update_commits[0]}')
# Only revert the newest commit as older ones will already have been reverted in previous
# releases.
run_git('revert', vOlder_update_commits[0], '--no-edit')

# Also revert the "Update checked-in dependencies" commit created by Actions.
update_dependencies_commit = run_git('log', '--grep', '^Update checked-in dependencies', '--format=%H').split()[0]
# TODO: why is this failing for the v2 branch currently...?
print(f' Reverting {update_dependencies_commit}')
run_git('revert', update_dependencies_commit, '--no-edit')

else:
print(' Nothing to revert.')

print(f'Merging {ORIGIN}/{source_branch} into the release prep branch')
# Commit any conflicts (see the comment for `conflicted_files`)
run_git('merge', f'{ORIGIN}/{source_branch}', allow_non_zero_exit_code=True)
conflicted_files = run_git('diff', '--name-only', '--diff-filter', 'U').splitlines()
if len(conflicted_files) > 0:
run_git('add', '.')
run_git('commit', '--no-edit')

# Migrate the package version number from a vLatest version number to a vOlder version number
print(f'Setting version number to {version}')
subprocess.check_output(['npm', 'version', version, '--no-git-tag-version'])
run_git('add', 'package.json', 'package-lock.json')

# Migrate the changelog notes from v2 version numbers to v1 version numbers
print(f'Migrating changelog notes from v{source_branch_major_version} to v{target_branch_major_version}')
subprocess.check_output(['sed', '-i', f's/^## {source_branch_major_version}\./## {target_branch_major_version}./g', 'CHANGELOG.md'])

# Remove changelog notes from all versions that do not apply to the vOlder branch
print(f'Removing changelog notes that do not apply to v{target_branch_major_version}')
for v in range(int(target_branch_major_version)+1, int(source_branch_major_version)+1):
print(f'Removing changelog notes that are tagged [v{v}+ only\]')
subprocess.check_output(['sed', '-i', f'/^- \[v{v}+ only\]/d', 'CHANGELOG.md'])

# Amend the commit generated by `npm version` to update the CHANGELOG
run_git('add', 'CHANGELOG.md')
run_git('commit', '-m', f'Update version and changelog for v{version}')
else:
# If we're performing a standard release, there won't be any new commits on the target branch,
# as these will have already been merged back into the source branch. Therefore we can just
# start from the source branch.
run_git('checkout', '-b', new_branch_name, f'{ORIGIN}/{source_branch}')

print('Updating changelog')
update_changelog(version)
print('Updating changelog')
update_changelog(version)

# Create a commit that updates the CHANGELOG
run_git('add', 'CHANGELOG.md')
run_git('commit', '-m', f'Update changelog for v{version}')
# Create a commit that updates the CHANGELOG
run_git('add', 'CHANGELOG.md')
run_git('commit', '-m', f'Update changelog for v{version}')

run_git('push', ORIGIN, new_branch_name)

Expand All @@ -240,7 +351,11 @@ def main():
commits,
source_branch_short_sha,
new_branch_name,
source_branch=source_branch,
target_branch=target_branch,
conductor=args.conductor,
is_primary_release=is_primary_release,
conflicted_files=conflicted_files
)

if __name__ == '__main__':
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/check-expected-release-files.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@ jobs:
bundle_version="$(cat "./src/defaults.json" | jq -r ".bundleVersion")"
set -x
for expected_file in "codeql-bundle.tar.gz" "codeql-bundle-linux64.tar.gz" "codeql-bundle-osx64.tar.gz" "codeql-bundle-win64.tar.gz"; do
curl --location --fail --head --request GET "https://github.com/github/codeql-action/releases/download/$bundle_version/$expected_file" > /dev/null
curl --location --fail --head --request GET "https://github.com/nickfyson-org/codeql-action/releases/download/$bundle_version/$expected_file" > /dev/null
done
Loading

0 comments on commit 7b040eb

Please sign in to comment.