From 0c758b8f87e73b6df6627615fe0d1eef0f8977e9 Mon Sep 17 00:00:00 2001 From: Leona Maroni Date: Mon, 25 Nov 2024 13:12:59 +0100 Subject: [PATCH] ci: Add update-nixpkgs tooling PL-133100 Co-Authored-By: Max Bosch --- .../workflows/update-nixpkgs-on-merge.yaml | 38 +++ .github/workflows/update-nixpkgs.yaml | 41 +++ .../20241125_131649_nixpkgs-updates_scriv.md | 18 ++ ci/gh_get_app_token.py | 40 +++ ci/update-nixpkgs-on-merge.py | 149 +++++++++ ci/update-nixpkgs.py | 300 ++++++++++++++++++ flake.nix | 17 +- 7 files changed, 601 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/update-nixpkgs-on-merge.yaml create mode 100644 .github/workflows/update-nixpkgs.yaml create mode 100644 changelog.d/20241125_131649_nixpkgs-updates_scriv.md create mode 100644 ci/gh_get_app_token.py create mode 100644 ci/update-nixpkgs-on-merge.py create mode 100755 ci/update-nixpkgs.py diff --git a/.github/workflows/update-nixpkgs-on-merge.yaml b/.github/workflows/update-nixpkgs-on-merge.yaml new file mode 100644 index 000000000..8c0329cc3 --- /dev/null +++ b/.github/workflows/update-nixpkgs-on-merge.yaml @@ -0,0 +1,38 @@ +name: update-nixpkgs-on-merge + +on: + pull_request: + types: + - closed + +jobs: + update-nixpkgs-on-merge: + if: github.event.pull_request.merged == true && startsWith(github.head_ref, 'nixpkgs-auto-update/') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.NIXPKGS_UPDATE_APP_ID }} + private-key: ${{ secrets.NIXPKGS_UPDATE_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + - run: | + echo "::add-mask::${{steps.app-token.outputs.token}}" + - name: Get GitHub App User ID + id: get-user-id + run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + - run: | + git config --global user.name '${{ steps.app-token.outputs.app-slug }}[bot]' + git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com>' + - run: | + pip install pygithub gitpython + - run: | + python ci/update-nixpkgs-on-merge.py \ + --merged-pr-id ${{ github.event.number }} \ + --nixpkgs-dir ../nixpkgs \ + --nixpkgs-origin-url https://x-access-token:${{steps.app-token.outputs.token}}@github.com/flyingcircusio/nixpkgs.git + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/.github/workflows/update-nixpkgs.yaml b/.github/workflows/update-nixpkgs.yaml new file mode 100644 index 000000000..a06a6723b --- /dev/null +++ b/.github/workflows/update-nixpkgs.yaml @@ -0,0 +1,41 @@ +name: update-nixpkgs + +on: + workflow_dispatch: {} + schedule: + - cron: "5 8 * * *" + +jobs: + run-nixpkgs-update: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: cachix/install-nix-action@v21 + with: + install_url: https://releases.nixos.org/nix/nix-2.18.9/install + - uses: actions/create-github-app-token@v1 + id: app-token + with: + app-id: ${{ vars.NIXPKGS_UPDATE_APP_ID }} + private-key: ${{ secrets.NIXPKGS_UPDATE_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + - run: | + echo "::add-mask::${{steps.app-token.outputs.token}}" + - name: Get GitHub App User ID + id: get-user-id + run: echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + - run: | + git config --global user.name '${{ steps.app-token.outputs.app-slug }}[bot]' + git config --global user.email '${{ steps.get-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com>' + - run: | + pip install pygithub gitpython + - run: | + python ci/update-nixpkgs.py \ + --nixpkgs-dir ../nixpkgs \ + --nixpkgs-upstream-url https://github.com/NixOS/nixpkgs \ + --nixpkgs-origin-url https://x-access-token:${{steps.app-token.outputs.token}}@github.com/flyingcircusio/nixpkgs.git \ + --platform-versions 24.05 + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} diff --git a/changelog.d/20241125_131649_nixpkgs-updates_scriv.md b/changelog.d/20241125_131649_nixpkgs-updates_scriv.md new file mode 100644 index 000000000..7dcbc79c6 --- /dev/null +++ b/changelog.d/20241125_131649_nixpkgs-updates_scriv.md @@ -0,0 +1,18 @@ + + +### Impact + +### NixOS XX.XX platform + +- internal: Automate nixpkgs updates with GitHub Actions (PL-133100) diff --git a/ci/gh_get_app_token.py b/ci/gh_get_app_token.py new file mode 100644 index 000000000..dd8f41f1e --- /dev/null +++ b/ci/gh_get_app_token.py @@ -0,0 +1,40 @@ +from argparse import ArgumentParser +from logging import INFO, basicConfig + +from github import Auth, GithubIntegration + + +def main(): + basicConfig(level=INFO) + argparser = ArgumentParser("GitHub get App Token") + argparser.add_argument("--app-id", help="App ID", required=True) + argparser.add_argument( + "--private-key-path", help="Path to the private key", required=True + ) + argparser.add_argument( + "--installation_id", + help="GitHub App installation ID. If not given the first one is picked", + required=False, + ) + args = argparser.parse_args() + + # This script very easily just return + with open(args.private_key_path, "r") as pk_file: + private_key = pk_file.read() + auth = Auth.AppAuth(args.app_id, private_key) + + gh_int = GithubIntegration(auth=auth) + installation_id = args.installation_id + if not installation_id: + installation_id = gh_int.get_installations()[0].id + access_token = gh_int.get_access_token(installation_id) + print( + "access token:", + access_token.token, + "expires at:", + access_token.expires_at.isoformat(), + ) + + +if __name__ == "__main__": + main() diff --git a/ci/update-nixpkgs-on-merge.py b/ci/update-nixpkgs-on-merge.py new file mode 100644 index 000000000..3ca3edd42 --- /dev/null +++ b/ci/update-nixpkgs-on-merge.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +""" +This script should be run when an automatic update-nixpkgs PR has been merged. +It will merge the corresponding flyingcircus/nixpkgs PR and cleanup +all old fc-nixos and nixpkgs PRs/branches that haven't been merged. +""" +import datetime +import os +from argparse import ArgumentParser +from dataclasses import dataclass +from logging import INFO, basicConfig, debug, info, warning + +from git import GitCommandError, Repo +from github import Auth, Github + +INTEGRATION_BRANCH_SCHEME = "nixpkgs-auto-update/{target_branch}/{now}" +FC_NIXOS_REPO = "flyingcircusio/fc-nixos-testing" +NIXPKGS_REPO = "flyingcircusio/nixpkgs-testing" + + +@dataclass +class Remote: + url: str + branches: list[str] + + +def nixpkgs_repository(directory: str, remotes: dict[str, Remote]) -> Repo: + info("Updating nixpkgs repository.") + if os.path.exists(directory): + repo = Repo(directory) + else: + repo = Repo.init(directory, mkdir=True) + + for name, remote in remotes.items(): + info(f"Updating nixpkgs repository remote `{name}`.") + if name in repo.remotes and repo.remotes[name].url != remote.url: + repo.delete_remote(repo.remote(name)) + if name not in repo.remotes: + repo.create_remote(name, remote.url) + + for branch in remote.branches: + info( + f"Fetching nixpkgs repository remote `{name}` - branch `{branch}`." + ) + getattr(repo.remotes, name).fetch( + refspec=branch, filter="blob:none" + ) + + return repo + + +def rebase_nixpkgs( + gh: Github, nixpkgs_repo: Repo, target_branch: str, integration_branch: str +) -> bool: + """Rebase nixpkgs repo integration branch onto target branch + Returns: True when successful, False when unsuccessful. + """ + info(f"Rebase nixpkgs repo integration branch onto target branch.") + if nixpkgs_repo.is_dirty(): + raise Exception("Repository is dirty!") + + nixpkgs_repo.git.checkout(target_branch) + + try: + nixpkgs_repo.git.rebase(f"origin/{integration_branch}") + except GitCommandError as e: + warning(f"Rebase failed:\n{e.stderr}") + nixpkgs_repo.git.rebase(abort=True) + warning("Aborted rebase.") + return False + + nixpkgs_repo.git.push(force_with_lease=True) + gh.get_repo(NIXPKGS_REPO).get_git_ref( + f"heads/{integration_branch}" + ).delete() + return True + + +def cleanup_old_prs_and_branches(gh: Github, merged_integration_branch: str): + info("Cleaning up old PRs and branches.") + fc_nixos_repo = gh.get_repo(FC_NIXOS_REPO) + nixpkgs_repo = gh.get_repo(NIXPKGS_REPO) + merged_integration_branch_date = datetime.date.fromisoformat( + merged_integration_branch.split("/")[2] + ) + # branches will be closed automatically by GitHub, when the branch is deleted + for repo in [fc_nixos_repo, nixpkgs_repo]: + for branch in repo.get_branches(): + if not branch.name.startswith("nixpkgs-auto-update/"): + continue + branch_datestr = branch.name.split("/")[2] + if ( + datetime.date.fromisoformat(branch_datestr) + < merged_integration_branch_date + ): + repo.get_git_ref(f"heads/{branch.name}").delete() + + +def main(): + basicConfig(level=INFO) + argparser = ArgumentParser("nixpkgs updater for fc-nixos") + argparser.add_argument( + "--merged-pr-id", help="merged fc-nixos PR ID", required=True + ) + argparser.add_argument( + "--nixpkgs-dir", + help="Directory where the nixpkgs git checkout is in", + required=True, + ) + argparser.add_argument( + "--nixpkgs-origin-url", + help="URL to push the nixpkgs updates to", + required=True, + ) + args = argparser.parse_args() + + try: + github_access_token = os.environ["GH_TOKEN"] + except KeyError: + raise Exception("Missing `GH_TOKEN` environment variable.") + + gh = Github(auth=Auth.Token(github_access_token)) + fc_nixos_pr = gh.get_repo(FC_NIXOS_REPO).get_pull(int(args.merged_pr_id)) + pr_platform_version = fc_nixos_pr.base.ref.split("-")[1] + integration_branch = fc_nixos_pr.head.ref + nixpkgs_target_branch = f"nixos-{pr_platform_version}" + + remotes = { + "origin": Remote( + args.nixpkgs_origin_url, + [integration_branch, nixpkgs_target_branch], + ) + } + nixpkgs_repo = nixpkgs_repository(args.nixpkgs_dir, remotes) + if rebase_nixpkgs( + gh, + nixpkgs_repo, + nixpkgs_target_branch, + integration_branch, + ): + fc_nixos_pr.create_issue_comment( + f"Rebased nixpkgs `{nixpkgs_target_branch}` branch successfully." + ) + cleanup_old_prs_and_branches(gh, integration_branch) + + +if __name__ == "__main__": + main() diff --git a/ci/update-nixpkgs.py b/ci/update-nixpkgs.py new file mode 100755 index 000000000..35d8a149d --- /dev/null +++ b/ci/update-nixpkgs.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 + +""" +Workflow: +* get executed in a given interval (e.g. daily - channel bumps are happening approximately every other day) +* pull all given releases into fc fork (rebase strategy) + * post error on merge conflict +* if new: + * push to integration branch (nixpkgs-auto-update/fc-XX.XX-dev/YYYY-MM-DD) + * update fc-nixos & create PR + * comment diff/changelog since last commit into PR +On Merge (fc-nixos): + * merge updated nixpkgs into nixos-XX.XX branch + * delete old integration branches in nixpkgs and fc-nixos + +Manual merges work by pushing manually in the nixpkgs integration branch and running the GHA manually. + +""" +import datetime +import os +from argparse import ArgumentParser +from dataclasses import dataclass +from logging import INFO, basicConfig, debug, info +from os import path +from pathlib import Path +from subprocess import check_output + +from git import Commit, Repo +from git.exc import GitCommandError +from github import Auth, Github + +NIXOS_VERSION_PATH = "release/nixos-version" +PACKAGE_VERSIONS_PATH = "release/package-versions.json" +VERSIONS_PATH = "release/versions.json" +CHANGELOG_DIR = "changelog.d" +FC_NIXOS_REPO = "flyingcircusio/fc-nixos" +NIXPKGS_REPO = "flyingcircusio/nixpkgs" + + +@dataclass +class NixpkgsRebaseResult: + upstream_commit: Commit + + # This is the latest commit on the release branch in our fork. + # If we have multiple consecutive updates, it is not the same as + # fork_before_rebase since this is the state of the tracking branch before + # the last rebase. This commit is important to generate the full + # changelog. + fork_commit: Commit + fork_before_rebase: Commit + fork_after_rebase: Commit + + +@dataclass +class Remote: + url: str + branches: list[str] + + +def nixpkgs_repository(directory: str, remotes: dict[str, Remote]) -> Repo: + info("Updating nixpkgs repository.") + if path.exists(directory): + repo = Repo(directory) + else: + repo = Repo.init(directory, mkdir=True) + + for name, remote in remotes.items(): + info(f"Updating nixpkgs repository remote `{name}`.") + if name in repo.remotes and repo.remotes[name].url != remote.url: + repo.delete_remote(repo.remote(name)) + if name not in repo.remotes: + repo.create_remote(name, remote.url) + + for branch in remote.branches: + info( + f"Fetching nixpkgs repository remote `{name}` - branch `{branch}`." + ) + # Ignore errors. This is intended as the last day integration branch may not exist + try: + getattr(repo.remotes, name).fetch( + refspec=branch, filter="blob:none" + ) + except GitCommandError as e: + debug("Error while fetching branch ", e) + pass + + return repo + + +def rebase_nixpkgs( + nixpkgs_repo: Repo, + branch_to_rebase: str, + integration_branch: str, + last_day_integration_branch: str, +) -> NixpkgsRebaseResult | None: + info(f"Trying to rebase nixpkgs repository.") + if nixpkgs_repo.is_dirty(): + raise Exception("Repository is dirty!") + + if not any(integration_branch == head.name for head in nixpkgs_repo.heads): + tracking_branch = nixpkgs_repo.create_head( + integration_branch, f"origin/{branch_to_rebase}" + ) + tracking_branch.checkout() + else: + nixpkgs_repo.git.checkout(integration_branch) + + latest_upstream = nixpkgs_repo.refs[f"upstream/{branch_to_rebase}"].commit + common_grounds = nixpkgs_repo.merge_base( + f"upstream/{branch_to_rebase}", "HEAD" + ) + + if all( + latest_upstream.hexsha != commit.hexsha for commit in common_grounds + ): + info( + f"Latest commit of {branch_to_rebase} is '{latest_upstream.hexsha}' which is not part of our fork, rebasing." + ) + current_state = nixpkgs_repo.head.commit + try: + nixpkgs_repo.git.rebase(f"upstream/{branch_to_rebase}") + except GitCommandError as e: + return None + + # Check if there are new commits compared to the last day's integration branch. + if f"origin/{last_day_integration_branch}" in nixpkgs_repo.refs: + diff_index = nixpkgs_repo.git.diff_index( + f"origin/{last_day_integration_branch}" + ) + + if diff_index == "": + info( + "No changes compared to the last day's integration branch. Not creating a new PR." + ) + return None + + nixpkgs_repo.git.push("origin", integration_branch, force=True) + + return NixpkgsRebaseResult( + upstream_commit=latest_upstream, + fork_commit=nixpkgs_repo.refs[f"origin/{branch_to_rebase}"].commit, + fork_before_rebase=current_state, + fork_after_rebase=nixpkgs_repo.head.commit, + ) + + info("Nothing to do.") + + +def update_fc_nixos( + target_branch: str, + integration_branch: str, + previous_hex_sha: str, + new_hex_sha: str, +): + info(f"Update fc-nixos.") + repo = Repo(Path.cwd()) + if not any(integration_branch == head.name for head in repo.heads): + tracking_branch = repo.create_head( + integration_branch, f"origin/{target_branch}" + ) + tracking_branch.checkout() + else: + repo.git.checkout(integration_branch) + + check_output( + [ + "nix", + "flake", + "lock", + "--override-input", + "nixpkgs", + f"github:{NIXPKGS_REPO}/{new_hex_sha}", + ] + ) + check_output(["nix", "run", ".#buildVersionsJson"]).decode("utf-8") + check_output(["nix", "run", ".#buildPackageVersionsJson"]).decode("utf-8") + + changelog_path = ( + Path(CHANGELOG_DIR) + / f"{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}_nixpkgs-auto-update-{target_branch}.md" + ) + changelog_path.write_text( + f""" +### NixOS XX.XX platform + +- Update nixpkgs from {previous_hex_sha} to {new_hex_sha} +""" + ) + + repo.git.add( + [ + "flake.lock", + VERSIONS_PATH, + PACKAGE_VERSIONS_PATH, + str(changelog_path), + ] + ) + repo.git.commit(message=f"Auto update nixpkgs to {new_hex_sha}") + repo.git.push("origin", integration_branch, force=True) + + +def create_fc_nixos_pr( + target_branch: str, + integration_branch: str, + github_access_token: str, + now: str, +): + info(f"Create PR in fc-nixos.") + gh = Github(auth=Auth.Token(github_access_token)) + fc_nixos_repo = gh.get_repo(FC_NIXOS_REPO) + fc_nixos_repo.create_pull( + base=target_branch, + head=integration_branch, + title=f"Auto update nixpkgs {now}", + body=f"""\ +View nixpkgs update branch: [{integration_branch}](https://github.com/{NIXPKGS_REPO}/tree/{integration_branch}) +""", + ) + + +def main(): + basicConfig(level=INFO) + argparser = ArgumentParser("nixpkgs updater for fc-nixos") + argparser.add_argument( + "--nixpkgs-dir", + help="Directory where the nixpkgs git checkout is in", + required=True, + ) + argparser.add_argument( + "--nixpkgs-upstream-url", + help="URL to the upstream nixpkgs repository", + required=True, + ) + argparser.add_argument( + "--nixpkgs-origin-url", + help="URL to push the nixpkgs updates to", + required=True, + ) + argparser.add_argument( + "--platform-versions", + help="Platform versions", + required=True, + nargs="+", + ) + args = argparser.parse_args() + + try: + github_access_token = os.environ["GH_TOKEN"] + except KeyError: + raise Exception("Missing `GH_TOKEN` environment variable.") + + today = datetime.date.today().isoformat() + yesterday = ( + datetime.date.today() - datetime.timedelta(days=1) + ).isoformat() + + for platform_version in args.platform_versions: + info(f"Updating platform {platform_version}") + nixpkgs_target_branch = f"nixos-{platform_version}" + fc_nixos_target_branch = f"fc-{platform_version}-dev" + integration_branch = ( + f"nixpkgs-auto-update/{fc_nixos_target_branch}/{today}" + ) + last_day_integration_branch = ( + f"nixpkgs-auto-update/{fc_nixos_target_branch}/{yesterday}" + ) + + remotes = { + "upstream": Remote( + args.nixpkgs_upstream_url, [nixpkgs_target_branch] + ), + "origin": Remote( + args.nixpkgs_origin_url, + [nixpkgs_target_branch, last_day_integration_branch], + ), + } + nixpkgs_repo = nixpkgs_repository(args.nixpkgs_dir, remotes) + if result := rebase_nixpkgs( + nixpkgs_repo, + nixpkgs_target_branch, + integration_branch, + last_day_integration_branch, + ): + info(f"Updated 'nixpkgs' to '{result.fork_after_rebase.hexsha}'") + update_fc_nixos( + fc_nixos_target_branch, + integration_branch, + result.fork_commit.hexsha, + result.fork_after_rebase.hexsha, + ) + create_fc_nixos_pr( + fc_nixos_target_branch, + integration_branch, + github_access_token, + today, + ) + + +if __name__ == "__main__": + main() diff --git a/flake.nix b/flake.nix index f96f05c27..f5ca3a28a 100644 --- a/flake.nix +++ b/flake.nix @@ -67,6 +67,19 @@ }; }; + apps.buildVersionsJson = { + type = "app"; + program = "${pkgs.writeShellScript "update-versions-json" '' + jq < $(nix build .#versionsJson --print-out-paths) > release/versions.json + ''}"; + }; + apps.buildPackageVersionsJson = { + type = "app"; + program = "${pkgs.writeShellScript "update-package-versions-json" '' + jq < $(nix build .#packageVersions --print-out-paths) > release/package-versions.json + ''}"; + }; + packages = { # These are packages that work on all systems. # Also see release/flake-part-linux-only-packages.nix @@ -153,11 +166,11 @@ # only build this script on Linux. It just produces an error # message on Non-Linux because packageVersions is missing. build_package_versions_json.exec = '' - jq < $(nix build .#packageVersions --print-out-paths) > release/package-versions.json + nix run .#buildPackageVersionsJson ''; build_versions_json.exec = '' - jq < $(nix build .#versionsJson --print-out-paths) > release/versions.json + nix run .#buildVersionsJson ''; build_channels_dir.exec = ''