From 86d43e5523e0f03b4022ca19863f7557009aec1e Mon Sep 17 00:00:00 2001 From: Khosrow Moossavi Date: Sat, 14 Dec 2024 17:33:11 -0500 Subject: [PATCH] feat: add new action to rebuild package sources --- .github/scripts/rebuild-sources.sh | 206 ++++++++++++++++++++++++++ .github/workflows/rebuild-sources.yml | 152 +++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100755 .github/scripts/rebuild-sources.sh create mode 100644 .github/workflows/rebuild-sources.yml diff --git a/.github/scripts/rebuild-sources.sh b/.github/scripts/rebuild-sources.sh new file mode 100755 index 00000000..b1b01461 --- /dev/null +++ b/.github/scripts/rebuild-sources.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# +# Rebuild the source files. +# +# The following files will be rebuilt out of exisiting .orig.tar.gz file which +# is previously repacked without /debian folder in it. +# +# - .dsc +# - .debian.tar.xz +# +# This will ensure one single .orig.tar.gz file can be used for all the +# packages of the same version and same component of different codenames. + +rebuild_packages() { + local distro=$1 + local codename=$2 + local component=$3 + + echo "Distro : $distro" + echo "Codename : $codename" + echo "Component: $component" + echo "==============================" + + pushd "$PKG_BUILD_PATH/$distro/$codename/$component" >/dev/null + + for f in $(find . -maxdepth 1 -type f -name "*.orig.tar.gz" | sort); do + base_name=$(basename $f) + + # not the provided --only-package + if [ -n "$ONLY_PACKAGE" ] && [ "$ONLY_PACKAGE" != "$base_name" ]; then + continue + fi + + tmp=$(mktemp -d) + if [ -z "$tmp" ]; then + continue + fi + if [ ! -d "$tmp" ]; then + continue + fi + + pkg_full_name=$(echo $base_name | sed 's/.orig.tar.gz//g') + pkg_name=$(echo $base_name | cut -d"_" -f1) + + if [ ! -f "$pkg_full_name.orig.tar.gz" ]; then + echo "$pkg_full_name.orig.tar.gz is missing" + continue + fi + if [ ! -f "$pkg_full_name-$codename.debian.tar.xz" ]; then + echo "$pkg_full_name-$codename.debian.tar.xz is missing" + continue + fi + + cp "$pkg_full_name.orig.tar.gz" "$tmp" + cp "$pkg_full_name-$codename.debian.tar.xz" "$tmp" + + # entering /tmp/tmp.XXXXXXXXXX + pushd $tmp >/dev/null + + tar -xzf "$pkg_full_name.orig.tar.gz" + tar -xf "$pkg_full_name-$codename.debian.tar.xz" + mv "debian/" "$pkg_name" + + if [ -d "$pkg_name" ]; then + pushd $pkg_name >/dev/null + + sudo apt update + sudo apt build-dep -y . + debuild -S -sa + + popd >/dev/null + fi + + # existing /tmp/tmp.XXXXXXXXXX + popd >/dev/null + + # copy newly generated .dsc and .debian.tar.xz file back to the repo + cp $tmp/$pkg_full_name-$codename.dsc . + cp $tmp/$pkg_full_name-$codename.debian.tar.xz . + + rm -rf $tmp >/dev/null + echo "==============================" + done + + popd >/dev/null +} + +find_packages() { + local distro="$1" + + pushd "$PKG_BUILD_PATH" >/dev/null + + for dir in $(find "$distro/" -mindepth 2 -maxdepth 2 -type d | sort); do + codename=$(echo "$dir" | cut -d"/" -f2) + component=$(echo "$dir" | cut -d"/" -f3) + + # not the provided --only-codename or --only-component + if [ -n "$ONLY_CODENAME" ] && [ "$ONLY_CODENAME" != "$codename" ]; then + continue + fi + if [ -n "$ONLY_COMPONENT" ] && [ "$ONLY_COMPONENT" != "$component" ]; then + continue + fi + + # skip named version folder (e.g. 3.x), the contents are symlinks + if [[ $component == *"."* ]]; then + continue + fi + + rebuild_packages "$distro" "$codename" "$component" + done + + popd >/dev/null +} + +is_supported() { + local distro="$1" + + if [ "$distro" == "ubuntu" ]; then + return 0 + fi + if [ "$distro" == "debian" ]; then + return 0 + fi + + return 1 +} + +main() { + for dir in $(find "$PKG_BUILD_PATH" -mindepth 1 -maxdepth 1 -type d); do + distro="${dir##*/}" + + if ! is_supported "$distro"; then + echo "Skipping $distro, it's not a supported distro." + continue + fi + if [ -n "$ONLY_DISTRO" ] && [ "$ONLY_DISTRO" != "$distro" ]; then + continue + fi + + find_packages "$distro" + done +} + +usage() { +cat << EOF +Rebuild debian package sources + +Usage: $0 --pkg-build-path [options...] + +Options: + --pkg-build-path Path to build package folder + + --only-distro Only rebuild sources of this distro + --only-codename Only rebuild sources of this codename + --only-component Only rebuild sources of this component + --only-package Only rebuild sources of this package + + --help Show this message +EOF +} + +parse_flag() { + declare -n argument=$3 + + if [ -n "$2" ] && [ ${2:0:1} != "-" ]; then + argument=$2 + return + fi + + echo "Error: argument for $1 is missing" >&2 + exit 1 +} + +PKG_BUILD_PATH="" +ONLY_DISTRO="" +ONLY_CODENAME="" +ONLY_COMPONENT="" +ONLY_PACKAGE="" + +while [[ $# -gt 0 ]]; do + case $1 in + --pkg-build-path) parse_flag "$1" "$2" PKG_BUILD_PATH; shift 2 ;; + + --only-distro) parse_flag "$1" "$2" ONLY_DISTRO; shift 2 ;; + --only-codename) parse_flag "$1" "$2" ONLY_CODENAME; shift 2 ;; + --only-component) parse_flag "$1" "$2" ONLY_COMPONENT; shift 2 ;; + --only-package) parse_flag "$1" "$2" ONLY_PACKAGE; shift 2 ;; + + -h|--help) usage; exit 0; ;; + -*|--*) echo "Unknown option $1"; exit 1; ;; + *) echo "Unknown command $1"; exit 1; ;; +esac +done + +if [ -z "$PKG_BUILD_PATH" ]; then + echo "Error: required value for --pkg-build-path is missing" + exit 1 +fi +if [ ! -d "$PKG_BUILD_PATH" ]; then + echo "Error: $PKG_BUILD_PATH not found" + exit 1 +fi +PKG_BUILD_PATH=$(realpath $PKG_BUILD_PATH) + +main diff --git a/.github/workflows/rebuild-sources.yml b/.github/workflows/rebuild-sources.yml new file mode 100644 index 00000000..695bcaea --- /dev/null +++ b/.github/workflows/rebuild-sources.yml @@ -0,0 +1,152 @@ +name: Rebuild Sources + +on: + workflow_dispatch: + inputs: + pull-from: + description: "Path on disk to pull from" + required: true + default: "/opt/archives/workspace/" + push-to: + description: "Path on disk to push to" + required: true + default: "/opt/archives/packages/" + only-distro: + description: "Only rebuild sources of this distro" + required: false + default: "" + only-codename: + description: "Only rebuild sources of this codename" + required: false + default: "" + only-component: + description: "Only rebuild sources of this component" + required: false + default: "" + only-package: + description: "Only rebuild sources of this package" + required: false + default: "" + workflow_call: + inputs: + pull-from: + description: "Path on disk to pull from" + type: string + required: true + default: "/opt/archives/workspace/" + push-to: + description: "Path on disk to push to" + type: string + required: true + default: "/opt/archives/packages/" + only-distro: + description: "Only rebuild sources of this distro" + type: string + required: false + default: "" + only-codename: + description: "Only rebuild sources of this codename" + type: string + required: false + default: "" + only-component: + description: "Only rebuild sources of this component" + type: string + required: false + default: "" + only-package: + description: "Only rebuild sources of this package" + type: string + required: false + default: "" + +jobs: + rebuild: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set Job Parameters + id: init + run: | + echo "package-build-path=${{ github.workspace }}/packages" >> $GITHUB_OUTPUT + + - name: Environment Setup + run: | + set -e + + rm -Rf "${{ steps.init.outputs.package-build-path }}" + + sudo apt update + DEBIAN_FRONTEND=noninteractive sudo apt install -y --no-install-recommends git devscripts wget dput + + export DEBEMAIL="regolith.linux@gmail.com" + export DEBFULLNAME="Regolith Linux" + export DEBIAN_FRONTEND=noninteractive + + mkdir -p ~/.gnupg/ + printf "${{ secrets.PACKAGE_PRIVATE_KEY2 }}" | base64 --decode > ~/.gnupg/private.key + gpg --batch --import ~/.gnupg/private.key + + - name: Setup SSH agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.KAMATERA_SSH_KEY }} + + - name: Pull Packages + run: | + set -e + set -x + + ssh-keyscan -H ${{ secrets.KAMATERA_HOSTNAME2 }} >> ~/.ssh/known_hosts + mkdir -p "${{ steps.init.outputs.package-build-path }}" + + ssh root@${{ secrets.KAMATERA_HOSTNAME2 }} "get-published-sources.sh --workspace-path \"${{ inputs.pull-from }}\"" + + rsync \ + -avzh \ + --exclude="*.*/" \ + --include="*/" \ + --include="*.orig.tar.gz" \ + --include="*.debian.tar.xz" \ + --include="*.diff.gz" \ + --exclude="*" \ + root@${{ secrets.KAMATERA_HOSTNAME2 }}:${{ inputs.pull-from }} \ + "${{ steps.init.outputs.package-build-path }}" + + - name: Rebuild Sources + run: | + set -e + + command_arguments=(--pkg-build-path "${{ steps.init.outputs.package-build-path }}") + + if [ -n "${{ inputs.only-distro }}" ]; then + command_arguments+=(--only-distro "${{ inputs.only-distro }}") + fi + if [ -n "${{ inputs.only-codename }}" ]; then + command_arguments+=(--only-codename "${{ inputs.only-codename }}") + fi + if [ -n "${{ inputs.only-component }}" ]; then + command_arguments+=(--only-component "${{ inputs.only-component }}") + fi + if [ -n "${{ inputs.only-package }}" ]; then + command_arguments+=(--only-package "${{ inputs.only-package }}") + fi + + ./.github/scripts/rebuild-sources.sh ${command_arguments[@]} + + - name: Deploy via rsync + run: | + set -e + set -x + + for i in 1 2 3 4 5; do + echo "Attempt $i" + rsync \ + -avzhH \ + ${{ steps.init.outputs.package-build-path }}/* \ + root@${{ secrets.KAMATERA_HOSTNAME2 }}:${{ inputs.push-to }} && break || sleep 5 + done + + ssh root@${{ secrets.KAMATERA_HOSTNAME2 }} "cleanup-workspace.sh --workspace-path \"${{ inputs.pull-from }}\""