Skip to content

Commit

Permalink
Add Alpine packaging and testing github action workflow (#4554) (#4571)
Browse files Browse the repository at this point in the history
  • Loading branch information
liveans committed Sep 27, 2024
1 parent 66ddc27 commit cdf3c6a
Show file tree
Hide file tree
Showing 8 changed files with 290 additions and 10 deletions.
90 changes: 90 additions & 0 deletions .github/workflows/package-alpine-linux.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
name: Alpine Linux Packages

on:
workflow_dispatch:
push:
branches:
- main
- release/*
pull_request:
branches:
- main
- release/*

concurrency:
# Cancel any workflow currently in progress for the same PR.
# Allow running concurrently with any other commits.
group: package-alpine-linux-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true

permissions: read-all

jobs:
build-alpine-packages:
name: Generate Alpine Packages
needs: []
strategy:
fail-fast: false
matrix:
vec: [
{ friendlyName: "Alpine-3.20-x64", config: "Release", arch: "x64", tls: "openssl3", image: "mcr.microsoft.com/dotnet/sdk:8.0-alpine3.20-amd64" },
{ friendlyName: "Alpine-3.20-ARM64", config: "Release", arch: "arm64", tls: "openssl3", image: "mcr.microsoft.com/dotnet/sdk:8.0-alpine3.20-arm64v8" },
# .NET is not working properly for ARM32 Alpine with QEMU, so keep it disabled for now.
# { friendlyName: "Alpine-3.20-ARM32", config: "Release", arch: "arm", tls: "openssl3", image: "mcr.microsoft.com/dotnet/sdk:8.0-alpine3.20-arm32v7" },
]
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
- name: Set up QEMU
uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf
- name: Generate APKBUILD
shell: pwsh
run: |
./scripts/generate-alpine-packaging-file.ps1 -ArchiveUri https://github.com/${{ github.repository }}/archive/${{ github.sha }}.tar.gz -SHA ${{ github.sha }}
mkdir -p packages
- name: Docker Run and Build Package
run: |
docker run \
-v $(pwd)/packages:/artifacts \
-v $(pwd):/msquic \
${{ matrix.vec.image }} /msquic/scripts/package-build.sh
- name: Upload Package
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
with:
name: ${{ matrix.vec.friendlyName }}-package
path: packages/*.apk

test-packages-on-docker:
name: Test Linux Packages
needs: [build-alpine-packages]
strategy:
fail-fast: false
matrix:
vec: [
{ friendlyName: "Alpine-3.20-x64", config: "Release", arch: "x64", tls: "openssl3", image: "mcr.microsoft.com/dotnet/runtime:9.0-alpine3.20-amd64", dotnetVersion: "9.0" },
{ friendlyName: "Alpine-3.20-ARM64", config: "Release", arch: "arm64", tls: "openssl3", image: "mcr.microsoft.com/dotnet/runtime:9.0-alpine3.20-arm64v8", dotnetVersion: "9.0" },
# .NET is not working properly for ARM32 Alpine with QEMU, so keep it disabled for now.
# { friendlyName: "Alpine-3.20-ARM32", config: "Release", arch: "arm", tls: "openssl3", image: "mcr.microsoft.com/dotnet/runtime:9.0-alpine3.20-arm32v7", dotnetVersion: "9.0" },
]
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
- name: Download Package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
with:
name: ${{ matrix.vec.friendlyName }}-package
path: artifacts
- name: Set up QEMU
uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf
- name: Set up .NET 9.0
uses: actions/setup-dotnet@6bd8b7f7774af54e05809fcc5431931b3eb1ddee
with:
dotnet-version: ${{ matrix.vec.dotnetVersion }}
- name: Build .NET QUIC Test Project
run: |
pushd src/cs/QuicSimpleTest && dotnet build QuicHello.net${{ matrix.vec.dotnetVersion }}.csproj -a ${{ matrix.vec.arch }} -c ${{ matrix.vec.config }} -o artifacts/net${{ matrix.vec.dotnetVersion }} -f net${{ matrix.vec.dotnetVersion }} && popd
- name: Docker Run
run: |
docker run -v $(pwd):/main ${{ matrix.vec.image }} /main/scripts/docker-script.sh ${{ matrix.vec.arch }} ${{ matrix.vec.config }} ${{ matrix.vec.tls }} ${{ matrix.vec.dotnetVersion }}
21 changes: 21 additions & 0 deletions docs/Release.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,27 @@ This table describes all officially supported MsQuic releases.
- Run `cargo publish` from the `release/X.Y` branch.
9. Update (via PR) `main` branch's `test-down-level.yml` to point the newly uploaded `*_test.zip` release binaries.

## Publishing Linux packages to packages.microsoft.com (PMC)
1. The publishing [pipeline](https://mscodehub.visualstudio.com/msquic/_build?definitionId=2068) automatically uploads packages into PMC when a tag is created.

Sometimes the pipeline fails due to PMC infra issues (e.g. the PMC HTTP endpoint returning errors). The publishing pipeline can be run manually to retry. When running manually, please ensure that the right tag is chosen and the right resources (under "Advanced options") are chosen. By default, the pipeline picks up the latest resources from the official build pipeline which are not always the right ones.

When testing the pipeline, please make sure to comment out the PMC cli commands in [upload-linux-packages.sh](https://github.com/microsoft/msquic/blob/main/scripts/upload-linux-packages.sh) to avoid accidentally publishing packages into prod.

## Publishing MsQuic for Alpine

1. Run `generate-alpine-packaging-file.ps1` script to create `APKBUILD` file for the release. (This script can run on any Linux distro.)
2. If you don't have account for [AlpineLinux GitLab](https://gitlab.alpinelinux.org). Create an account and [configure your SSH](https://docs.gitlab.com/ee/user/ssh.html).
3. If you didn't fork `aports` repository yet, Fork `https://gitlab.alpinelinux.org/alpine/aports`.
4. Clone `https://gitlab.alpinelinux.org/<your_username>/aports` repository.
5. Navigate to `aports/testing/libmsquic` folder.
6. Replace the `APKBUILD` file with newly created `APKBUILD` file.
7. Create a commit using `testing/libmsquic: upgrade to <version_number>` (version_number e.g. 2.5.0 or 2.4.4).
8. Create a merge request using `testing/libmsquic: upgrade to <version_number>` (version_number e.g. 2.5.0 or 2.4.4).
9. Owners of the `aports` repository will respond to the PR or merge it in couple of days/hours.

For future reference: [Official documentation](https://wiki.alpinelinux.org/wiki/Creating_an_Alpine_package)

## Synchronizing with Windows

1. Once the release branch is created, set the pipeline [here](https://mscodehub.visualstudio.com/msquic/_build?definitionId=1868) to ingest the release branch into Windows, and run it.
Expand Down
10 changes: 10 additions & 0 deletions scripts/alpine-configure-packaging-key.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#! /bin/sh

# The argument is private key path

sudo cp $1.pub /etc/apk/keys/
sudo cp $1 /home/packaging/.abuild/

sudo echo PACKAGER_PRIVKEY="$1" > /home/packaging/.abuild/abuild.conf

sudo chown -R packaging:abuild /home/packaging/.abuild
8 changes: 8 additions & 0 deletions scripts/alpine-generate-hash.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#! /bin/sh
set -e
apk add --upgrade --no-cache alpine-sdk
adduser -D packaging -G abuild
cp /msquic/APKBUILD /home/packaging/
cd /home/packaging
su packaging -c "abuild checksum"
cp /home/packaging/APKBUILD /msquic/APKBUILD
31 changes: 21 additions & 10 deletions scripts/docker-script.sh
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
#! /bin/bash
#! /bin/sh

if [[ $(id -u) -ne 0 ]];
then
#Beware of how you compose the command
if [ $(id -u) -ne 0 ]; then
# Beware of how you compose the command
echo "This script must be run as root. Running the script with sudo..."
printf -v cmd_str '%q ' "$0" "$@"
exec sudo su -c "$cmd_str"
Expand Down Expand Up @@ -54,22 +53,34 @@ install_libmsquic_azure_linux()
find -name "libmsquic*.rpm" -exec tdnf install -y {} \;
}

if [[ "$OS" == "ubuntu" ]] || [[ "$OS" == "debian" ]]; then
install_libmsquic_alpine()
{
if ! [ -f /usr/bin/dotnet ]; then
apk add --upgrade --no-cache wget gzip tar
fi
find -name "libmsquic*.apk" -exec apk add --allow-untrusted {} \;
}

if [ "$OS" = "ubuntu" ] || [ "$OS" = "debian" ]; then
install_dependencies_apt
elif [[ "$OS" == "centos" ]] || [[ "$OS" == "almalinux" ]] || [[ "$OS" == "rhel" ]] || [[ "$OS" == "fedora" ]]; then
elif [ "$OS" = "centos" ] || [ "$OS" = "almalinux" ] || [ "$OS" = "rhel" ] || [ "$OS" = "fedora" ]; then
install_dependencies_rpm
elif [[ "$OS" == 'opensuse-leap' ]]; then
elif [ "$OS" = 'opensuse-leap' ]; then
install_dependencies_opensuse
elif [[ "$OS" == 'azurelinux' ]] || [[ "$OS" == 'mariner' ]]; then
elif [ "$OS" = 'azurelinux' ] || [ "$OS" = 'mariner' ]; then
install_libmsquic_azure_linux
elif [ "$OS" = 'alpine' ]; then
install_libmsquic_alpine
else
echo "Unsupported OS: ${OS}"
exit 1
fi

set -e
chmod +x artifacts/bin/linux/${1}_${2}_${3}/msquictest
artifacts/bin/linux/${1}_${2}_${3}/msquictest --gtest_filter=ParameterValidation.ValidateApi
if ! [ "$OS" = 'alpine' ]; then
chmod +x artifacts/bin/linux/${1}_${2}_${3}/msquictest
artifacts/bin/linux/${1}_${2}_${3}/msquictest --gtest_filter=ParameterValidation.ValidateApi
fi

# Install .NET if it is not installed
if ! [ -f /usr/bin/dotnet ]; then
Expand Down
54 changes: 54 additions & 0 deletions scripts/generate-alpine-packaging-file.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
param (
[Parameter(Mandatory = $false)]
[string]$ArchiveUri = 'https://github.com/microsoft/msquic/archive/refs/tags/v$pkgver.tar.gz',

[Parameter(Mandatory = $false)]
[string]$SHA = ""
)

class Version
{
[string]$Major
[string]$Minor
[string]$Patch
}

$submodules = git submodule

$processedSubmodules = @("clog", "openssl3", "googletest")
$placeholderVariables = @{
"clog" = "CLOG_COMMIT_HASH"
"openssl3" = "OPENSSL3_COMMIT_HASH"
"googletest" = "GOOGLETEST_COMMIT_HASH"
}
$versionPlaceholder = "VERSION_PLACEHOLDER"
$alpinePackagingFile = ((Get-Content "$PSScriptRoot/templates/APKBUILD.template") -join "`n") + "`n"
$alpinePackagingFile = $alpinePackagingFile -replace "ARCHIVE_URI_PLACEHOLDER", $ArchiveUri

if ($SHA -ne "")
{
$alpinePackagingFile = $alpinePackagingFile -replace "SHA_PLACEHOLDER", $SHA
}
else
{
$alpinePackagingFile = $alpinePackagingFile -replace "SHA_PLACEHOLDER", '$pkgver'
}

foreach ($submodule in $submodules)
{
$submoduleInfo = $submodule.Trim().Trim('-').Split(" ")
$submoduleName = $submoduleInfo[1].Replace("submodules/", "")
if ($processedSubmodules -contains $submoduleName)
{
$alpinePackagingFile = $alpinePackagingFile -replace $placeholderVariables[$submoduleName], $submoduleInfo[0]
}
}

$version = [Version](Get-Content "$PSScriptRoot/../version.json" | Out-String | ConvertFrom-Json)
$alpinePackagingFile = $alpinePackagingFile -replace $versionPlaceholder, "$($version.Major).$($version.Minor).$($version.Patch)"
Write-Output $alpinePackagingFile | Out-File APKBUILD -NoNewline
Write-Output "APKBUILD file for msquic v$($version.Major).$($version.Minor).$($version.Patch) has been generated successfully."
Write-Output "Starting to add file hashes into APKBUILD file..."

docker run -v .:/msquic -w /msquic alpine:latest /msquic/scripts/alpine-generate-hash.sh
Write-Output "File hashes have been added successfully."
32 changes: 32 additions & 0 deletions scripts/package-build.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#! /bin/sh

set -e
apk add --upgrade sudo alpine-sdk

git config --global user.name "Microsoft QUIC Team"
git config --global user.email "quicdev@microsoft.com"

# Add the packaging user to the abuild group
adduser -D packaging -G abuild

# Give the packaging user sudo access
echo "packaging ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/packaging

mkdir -p /var/cache/distfiles
chmod a+w /var/cache/distfiles

mkdir -p /home/packaging/github-actions/packages/
chown -R packaging:abuild /home/packaging/github-actions/packages/

mkdir -p /home/packaging/tools
cp /msquic/APKBUILD /home/packaging/tools
chown -R packaging:abuild /home/packaging/tools

su packaging -c "abuild-keygen -n"
find /home/packaging/.abuild -name '*.rsa' -exec /msquic/scripts/alpine-configure-packaging-key.sh {} \;

# msquic is using submodules and we need to get them inside
cd /home/packaging/tools
su packaging -c "abuild -r"

cp /home/packaging/packages/packaging/**/*.apk /artifacts
54 changes: 54 additions & 0 deletions scripts/templates/APKBUILD.template
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Contributor: Ahmet Ibrahim AKSOY <aaksoy@microsoft.com>
# Maintainer: Microsoft QUIC Team <quicdev@microsoft.com>
pkgname=libmsquic
pkgver=VERSION_PLACEHOLDER
pkgrel=0
_clog=CLOG_COMMIT_HASH
_gtest=GOOGLETEST_COMMIT_HASH
_openssl3=OPENSSL3_COMMIT_HASH
pkgdesc="Cross-platform, C implementation of the IETF QUIC protocol, exposed to C, C++, C# and Rust."
url="https://github.com/microsoft/msquic"
arch="x86_64 armv7 aarch64"
license="MIT"
makedepends="cmake numactl-dev linux-headers lttng-ust-dev openssl-dev perl xz"
checkdepends="perf"
subpackages="$pkgname-dev $pkgname-doc"
source="msquic-$pkgver.tar.gz::ARCHIVE_URI_PLACEHOLDER
clog-$_clog.tar.gz::https://github.com/microsoft/CLOG/archive/$_clog.tar.gz
gtest-$_gtest.tar.gz::https://github.com/google/googletest/archive/$_gtest.tar.gz
openssl3-$_openssl3.tar.gz::https://github.com/quictls/openssl/archive/$_openssl3.tar.gz
"
builddir="$srcdir/msquic-SHA_PLACEHOLDER"

prepare() {
default_prepare

cd "$builddir/submodules"
rm -rf clog googletest openssl openssl3 xdp-for-windows
mv ../../CLOG-*/ clog/
mv ../../googletest-*/ googletest/
mv ../../openssl-*/ openssl3/
}

build() {
cmake -B build \
-DCMAKE_INSTALL_PREFIX=/usr \
-DCMAKE_BUILD_TYPE=Release \
-DQUIC_TLS=openssl3 \
-DQUIC_ENABLE_LOGGING=true \
-DQUIC_USE_SYSTEM_LIBCRYPTO=true \
-DQUIC_BUILD_TOOLS=off \
-DQUIC_BUILD_TEST=on \
-DQUIC_BUILD_PERF=off
cmake --build build
}

check() {
build/bin/Release/msquictest --gtest_filter=ParameterValidation.ValidateApi
}

package() {
DESTDIR="$pkgdir" cmake --install build
rm -rf "$pkgdir"/usr/share/msquic/
install -Dm644 LICENSE "$pkgdir"/usr/share/licenses/$pkgname/LICENSE
}

0 comments on commit cdf3c6a

Please sign in to comment.