From cdf3c6abf589fdc0cf9e24eaadc75c67e576b3c5 Mon Sep 17 00:00:00 2001 From: Ahmet Ibrahim Aksoy Date: Fri, 27 Sep 2024 15:31:39 +0200 Subject: [PATCH] Add Alpine packaging and testing github action workflow (#4554) (#4571) --- .github/workflows/package-alpine-linux.yml | 90 ++++++++++++++++++++++ docs/Release.md | 21 +++++ scripts/alpine-configure-packaging-key.sh | 10 +++ scripts/alpine-generate-hash.sh | 8 ++ scripts/docker-script.sh | 31 +++++--- scripts/generate-alpine-packaging-file.ps1 | 54 +++++++++++++ scripts/package-build.sh | 32 ++++++++ scripts/templates/APKBUILD.template | 54 +++++++++++++ 8 files changed, 290 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/package-alpine-linux.yml create mode 100755 scripts/alpine-configure-packaging-key.sh create mode 100755 scripts/alpine-generate-hash.sh create mode 100644 scripts/generate-alpine-packaging-file.ps1 create mode 100755 scripts/package-build.sh create mode 100644 scripts/templates/APKBUILD.template diff --git a/.github/workflows/package-alpine-linux.yml b/.github/workflows/package-alpine-linux.yml new file mode 100644 index 0000000000..e0728e278a --- /dev/null +++ b/.github/workflows/package-alpine-linux.yml @@ -0,0 +1,90 @@ +name: Alpine Linux Packages + +on: + workflow_dispatch: + push: + branches: + - main + - release/* + pull_request: + branches: + - main + - release/* + +concurrency: + # Cancel any workflow currently in progress for the same PR. + # Allow running concurrently with any other commits. + group: package-alpine-linux-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +permissions: read-all + +jobs: + build-alpine-packages: + name: Generate Alpine Packages + needs: [] + strategy: + fail-fast: false + matrix: + vec: [ + { friendlyName: "Alpine-3.20-x64", config: "Release", arch: "x64", tls: "openssl3", image: "mcr.microsoft.com/dotnet/sdk:8.0-alpine3.20-amd64" }, + { friendlyName: "Alpine-3.20-ARM64", config: "Release", arch: "arm64", tls: "openssl3", image: "mcr.microsoft.com/dotnet/sdk:8.0-alpine3.20-arm64v8" }, + # .NET is not working properly for ARM32 Alpine with QEMU, so keep it disabled for now. + # { friendlyName: "Alpine-3.20-ARM32", config: "Release", arch: "arm", tls: "openssl3", image: "mcr.microsoft.com/dotnet/sdk:8.0-alpine3.20-arm32v7" }, + ] + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + - name: Set up QEMU + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf + - name: Generate APKBUILD + shell: pwsh + run: | + ./scripts/generate-alpine-packaging-file.ps1 -ArchiveUri https://github.com/${{ github.repository }}/archive/${{ github.sha }}.tar.gz -SHA ${{ github.sha }} + mkdir -p packages + - name: Docker Run and Build Package + run: | + docker run \ + -v $(pwd)/packages:/artifacts \ + -v $(pwd):/msquic \ + ${{ matrix.vec.image }} /msquic/scripts/package-build.sh + - name: Upload Package + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a + with: + name: ${{ matrix.vec.friendlyName }}-package + path: packages/*.apk + + test-packages-on-docker: + name: Test Linux Packages + needs: [build-alpine-packages] + strategy: + fail-fast: false + matrix: + vec: [ + { friendlyName: "Alpine-3.20-x64", config: "Release", arch: "x64", tls: "openssl3", image: "mcr.microsoft.com/dotnet/runtime:9.0-alpine3.20-amd64", dotnetVersion: "9.0" }, + { friendlyName: "Alpine-3.20-ARM64", config: "Release", arch: "arm64", tls: "openssl3", image: "mcr.microsoft.com/dotnet/runtime:9.0-alpine3.20-arm64v8", dotnetVersion: "9.0" }, + # .NET is not working properly for ARM32 Alpine with QEMU, so keep it disabled for now. + # { friendlyName: "Alpine-3.20-ARM32", config: "Release", arch: "arm", tls: "openssl3", image: "mcr.microsoft.com/dotnet/runtime:9.0-alpine3.20-arm32v7", dotnetVersion: "9.0" }, + ] + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + - name: Download Package + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 + with: + name: ${{ matrix.vec.friendlyName }}-package + path: artifacts + - name: Set up QEMU + uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf + - name: Set up .NET 9.0 + uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee + with: + dotnet-version: ${{ matrix.vec.dotnetVersion }} + - name: Build .NET QUIC Test Project + run: | + pushd src/cs/QuicSimpleTest && dotnet build QuicHello.net${{ matrix.vec.dotnetVersion }}.csproj -a ${{ matrix.vec.arch }} -c ${{ matrix.vec.config }} -o artifacts/net${{ matrix.vec.dotnetVersion }} -f net${{ matrix.vec.dotnetVersion }} && popd + - name: Docker Run + run: | + docker run -v $(pwd):/main ${{ matrix.vec.image }} /main/scripts/docker-script.sh ${{ matrix.vec.arch }} ${{ matrix.vec.config }} ${{ matrix.vec.tls }} ${{ matrix.vec.dotnetVersion }} diff --git a/docs/Release.md b/docs/Release.md index 152735ea13..a09e31165b 100644 --- a/docs/Release.md +++ b/docs/Release.md @@ -99,6 +99,27 @@ This table describes all officially supported MsQuic releases. - Run `cargo publish` from the `release/X.Y` branch. 9. Update (via PR) `main` branch's `test-down-level.yml` to point the newly uploaded `*_test.zip` release binaries. +## Publishing Linux packages to packages.microsoft.com (PMC) +1. The publishing [pipeline](https://mscodehub.visualstudio.com/msquic/_build?definitionId=2068) automatically uploads packages into PMC when a tag is created. + +Sometimes the pipeline fails due to PMC infra issues (e.g. the PMC HTTP endpoint returning errors). The publishing pipeline can be run manually to retry. When running manually, please ensure that the right tag is chosen and the right resources (under "Advanced options") are chosen. By default, the pipeline picks up the latest resources from the official build pipeline which are not always the right ones. + +When testing the pipeline, please make sure to comment out the PMC cli commands in [upload-linux-packages.sh](https://github.com/microsoft/msquic/blob/main/scripts/upload-linux-packages.sh) to avoid accidentally publishing packages into prod. + +## Publishing MsQuic for Alpine + +1. Run `generate-alpine-packaging-file.ps1` script to create `APKBUILD` file for the release. (This script can run on any Linux distro.) +2. If you don't have account for [AlpineLinux GitLab](https://gitlab.alpinelinux.org). Create an account and [configure your SSH](https://docs.gitlab.com/ee/user/ssh.html). +3. If you didn't fork `aports` repository yet, Fork `https://gitlab.alpinelinux.org/alpine/aports`. +4. Clone `https://gitlab.alpinelinux.org//aports` repository. +5. Navigate to `aports/testing/libmsquic` folder. +6. Replace the `APKBUILD` file with newly created `APKBUILD` file. +7. Create a commit using `testing/libmsquic: upgrade to ` (version_number e.g. 2.5.0 or 2.4.4). +8. Create a merge request using `testing/libmsquic: upgrade to ` (version_number e.g. 2.5.0 or 2.4.4). +9. Owners of the `aports` repository will respond to the PR or merge it in couple of days/hours. + +For future reference: [Official documentation](https://wiki.alpinelinux.org/wiki/Creating_an_Alpine_package) + ## Synchronizing with Windows 1. Once the release branch is created, set the pipeline [here](https://mscodehub.visualstudio.com/msquic/_build?definitionId=1868) to ingest the release branch into Windows, and run it. diff --git a/scripts/alpine-configure-packaging-key.sh b/scripts/alpine-configure-packaging-key.sh new file mode 100755 index 0000000000..07eca2863b --- /dev/null +++ b/scripts/alpine-configure-packaging-key.sh @@ -0,0 +1,10 @@ +#! /bin/sh + +# The argument is private key path + +sudo cp $1.pub /etc/apk/keys/ +sudo cp $1 /home/packaging/.abuild/ + +sudo echo PACKAGER_PRIVKEY="$1" > /home/packaging/.abuild/abuild.conf + +sudo chown -R packaging:abuild /home/packaging/.abuild diff --git a/scripts/alpine-generate-hash.sh b/scripts/alpine-generate-hash.sh new file mode 100755 index 0000000000..c456a5e945 --- /dev/null +++ b/scripts/alpine-generate-hash.sh @@ -0,0 +1,8 @@ +#! /bin/sh +set -e +apk add --upgrade --no-cache alpine-sdk +adduser -D packaging -G abuild +cp /msquic/APKBUILD /home/packaging/ +cd /home/packaging +su packaging -c "abuild checksum" +cp /home/packaging/APKBUILD /msquic/APKBUILD diff --git a/scripts/docker-script.sh b/scripts/docker-script.sh index 5a10c48048..bad0289967 100755 --- a/scripts/docker-script.sh +++ b/scripts/docker-script.sh @@ -1,8 +1,7 @@ -#! /bin/bash +#! /bin/sh -if [[ $(id -u) -ne 0 ]]; -then - #Beware of how you compose the command +if [ $(id -u) -ne 0 ]; then + # Beware of how you compose the command echo "This script must be run as root. Running the script with sudo..." printf -v cmd_str '%q ' "$0" "$@" exec sudo su -c "$cmd_str" @@ -54,22 +53,34 @@ install_libmsquic_azure_linux() find -name "libmsquic*.rpm" -exec tdnf install -y {} \; } -if [[ "$OS" == "ubuntu" ]] || [[ "$OS" == "debian" ]]; then +install_libmsquic_alpine() +{ + if ! [ -f /usr/bin/dotnet ]; then + apk add --upgrade --no-cache wget gzip tar + fi + find -name "libmsquic*.apk" -exec apk add --allow-untrusted {} \; +} + +if [ "$OS" = "ubuntu" ] || [ "$OS" = "debian" ]; then install_dependencies_apt -elif [[ "$OS" == "centos" ]] || [[ "$OS" == "almalinux" ]] || [[ "$OS" == "rhel" ]] || [[ "$OS" == "fedora" ]]; then +elif [ "$OS" = "centos" ] || [ "$OS" = "almalinux" ] || [ "$OS" = "rhel" ] || [ "$OS" = "fedora" ]; then install_dependencies_rpm -elif [[ "$OS" == 'opensuse-leap' ]]; then +elif [ "$OS" = 'opensuse-leap' ]; then install_dependencies_opensuse -elif [[ "$OS" == 'azurelinux' ]] || [[ "$OS" == 'mariner' ]]; then +elif [ "$OS" = 'azurelinux' ] || [ "$OS" = 'mariner' ]; then install_libmsquic_azure_linux +elif [ "$OS" = 'alpine' ]; then + install_libmsquic_alpine else echo "Unsupported OS: ${OS}" exit 1 fi set -e -chmod +x artifacts/bin/linux/${1}_${2}_${3}/msquictest -artifacts/bin/linux/${1}_${2}_${3}/msquictest --gtest_filter=ParameterValidation.ValidateApi +if ! [ "$OS" = 'alpine' ]; then + chmod +x artifacts/bin/linux/${1}_${2}_${3}/msquictest + artifacts/bin/linux/${1}_${2}_${3}/msquictest --gtest_filter=ParameterValidation.ValidateApi +fi # Install .NET if it is not installed if ! [ -f /usr/bin/dotnet ]; then diff --git a/scripts/generate-alpine-packaging-file.ps1 b/scripts/generate-alpine-packaging-file.ps1 new file mode 100644 index 0000000000..b5979cd1b4 --- /dev/null +++ b/scripts/generate-alpine-packaging-file.ps1 @@ -0,0 +1,54 @@ +param ( + [Parameter(Mandatory = $false)] + [string]$ArchiveUri = 'https://github.com/microsoft/msquic/archive/refs/tags/v$pkgver.tar.gz', + + [Parameter(Mandatory = $false)] + [string]$SHA = "" +) + +class Version +{ + [string]$Major + [string]$Minor + [string]$Patch +} + +$submodules = git submodule + +$processedSubmodules = @("clog", "openssl3", "googletest") +$placeholderVariables = @{ + "clog" = "CLOG_COMMIT_HASH" + "openssl3" = "OPENSSL3_COMMIT_HASH" + "googletest" = "GOOGLETEST_COMMIT_HASH" +} +$versionPlaceholder = "VERSION_PLACEHOLDER" +$alpinePackagingFile = ((Get-Content "$PSScriptRoot/templates/APKBUILD.template") -join "`n") + "`n" +$alpinePackagingFile = $alpinePackagingFile -replace "ARCHIVE_URI_PLACEHOLDER", $ArchiveUri + +if ($SHA -ne "") +{ + $alpinePackagingFile = $alpinePackagingFile -replace "SHA_PLACEHOLDER", $SHA +} +else +{ + $alpinePackagingFile = $alpinePackagingFile -replace "SHA_PLACEHOLDER", '$pkgver' +} + +foreach ($submodule in $submodules) +{ + $submoduleInfo = $submodule.Trim().Trim('-').Split(" ") + $submoduleName = $submoduleInfo[1].Replace("submodules/", "") + if ($processedSubmodules -contains $submoduleName) + { + $alpinePackagingFile = $alpinePackagingFile -replace $placeholderVariables[$submoduleName], $submoduleInfo[0] + } +} + +$version = [Version](Get-Content "$PSScriptRoot/../version.json" | Out-String | ConvertFrom-Json) +$alpinePackagingFile = $alpinePackagingFile -replace $versionPlaceholder, "$($version.Major).$($version.Minor).$($version.Patch)" +Write-Output $alpinePackagingFile | Out-File APKBUILD -NoNewline +Write-Output "APKBUILD file for msquic v$($version.Major).$($version.Minor).$($version.Patch) has been generated successfully." +Write-Output "Starting to add file hashes into APKBUILD file..." + +docker run -v .:/msquic -w /msquic alpine:latest /msquic/scripts/alpine-generate-hash.sh +Write-Output "File hashes have been added successfully." diff --git a/scripts/package-build.sh b/scripts/package-build.sh new file mode 100755 index 0000000000..2b027ad277 --- /dev/null +++ b/scripts/package-build.sh @@ -0,0 +1,32 @@ +#! /bin/sh + +set -e +apk add --upgrade sudo alpine-sdk + +git config --global user.name "Microsoft QUIC Team" +git config --global user.email "quicdev@microsoft.com" + +# Add the packaging user to the abuild group +adduser -D packaging -G abuild + +# Give the packaging user sudo access +echo "packaging ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/packaging + +mkdir -p /var/cache/distfiles +chmod a+w /var/cache/distfiles + +mkdir -p /home/packaging/github-actions/packages/ +chown -R packaging:abuild /home/packaging/github-actions/packages/ + +mkdir -p /home/packaging/tools +cp /msquic/APKBUILD /home/packaging/tools +chown -R packaging:abuild /home/packaging/tools + +su packaging -c "abuild-keygen -n" +find /home/packaging/.abuild -name '*.rsa' -exec /msquic/scripts/alpine-configure-packaging-key.sh {} \; + +# msquic is using submodules and we need to get them inside +cd /home/packaging/tools +su packaging -c "abuild -r" + +cp /home/packaging/packages/packaging/**/*.apk /artifacts diff --git a/scripts/templates/APKBUILD.template b/scripts/templates/APKBUILD.template new file mode 100644 index 0000000000..58969e7090 --- /dev/null +++ b/scripts/templates/APKBUILD.template @@ -0,0 +1,54 @@ +# Contributor: Ahmet Ibrahim AKSOY +# Maintainer: Microsoft QUIC Team +pkgname=libmsquic +pkgver=VERSION_PLACEHOLDER +pkgrel=0 +_clog=CLOG_COMMIT_HASH +_gtest=GOOGLETEST_COMMIT_HASH +_openssl3=OPENSSL3_COMMIT_HASH +pkgdesc="Cross-platform, C implementation of the IETF QUIC protocol, exposed to C, C++, C# and Rust." +url="https://github.com/microsoft/msquic" +arch="x86_64 armv7 aarch64" +license="MIT" +makedepends="cmake numactl-dev linux-headers lttng-ust-dev openssl-dev perl xz" +checkdepends="perf" +subpackages="$pkgname-dev $pkgname-doc" +source="msquic-$pkgver.tar.gz::ARCHIVE_URI_PLACEHOLDER + clog-$_clog.tar.gz::https://github.com/microsoft/CLOG/archive/$_clog.tar.gz + gtest-$_gtest.tar.gz::https://github.com/google/googletest/archive/$_gtest.tar.gz + openssl3-$_openssl3.tar.gz::https://github.com/quictls/openssl/archive/$_openssl3.tar.gz + " +builddir="$srcdir/msquic-SHA_PLACEHOLDER" + +prepare() { + default_prepare + + cd "$builddir/submodules" + rm -rf clog googletest openssl openssl3 xdp-for-windows + mv ../../CLOG-*/ clog/ + mv ../../googletest-*/ googletest/ + mv ../../openssl-*/ openssl3/ +} + +build() { + cmake -B build \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -DQUIC_TLS=openssl3 \ + -DQUIC_ENABLE_LOGGING=true \ + -DQUIC_USE_SYSTEM_LIBCRYPTO=true \ + -DQUIC_BUILD_TOOLS=off \ + -DQUIC_BUILD_TEST=on \ + -DQUIC_BUILD_PERF=off + cmake --build build +} + +check() { + build/bin/Release/msquictest --gtest_filter=ParameterValidation.ValidateApi +} + +package() { + DESTDIR="$pkgdir" cmake --install build + rm -rf "$pkgdir"/usr/share/msquic/ + install -Dm644 LICENSE "$pkgdir"/usr/share/licenses/$pkgname/LICENSE +}