Skip to content

Fix the k8ssandra-operator ITs #2571

Fix the k8ssandra-operator ITs

Fix the k8ssandra-operator ITs #2571

Workflow file for this run

name: Build, test and Release
on:
push:
branches:
- master
- '*.*'
pull_request:
release:
types:
- published
jobs:
build:
strategy:
matrix:
python-version: [3.9, "3.10", "3.11"]
include:
- python-version: 3.9
tox-py: py39
- python-version: "3.10"
tox-py: py310
- python-version: "3.11"
tox-py: py311
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Set up Poetry
uses: abatilo/actions-poetry@v2
with:
poetry-version: "1.8.0"
- name: Install dependencies
run: |
poetry install
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=W503
- name: Run tox
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
poetry run tox -e ${{ matrix.tox-py }}
- uses: codecov/codecov-action@v1
name: Report code coverage
debian-build:
# Build debian packages
strategy:
matrix:
suite: [focal]
include:
- suite: focal
os-version: ubuntu-22.04
fail-fast: false
runs-on: ${{ matrix.os-version }}
steps:
- uses: actions/checkout@v3
- name: Build Debian Package
run: |
version=$(cat VERSION)
version_exists=$(cat debian/changelog | grep "($VERSION)" || echo "")
if [ -z "$version_exists" ]
then
echo "Updating debian changelog..."
cd packaging/docker-build
docker compose build release && docker compose run release
cd ../..
fi
cd packaging/docker-build
docker compose build "cassandra-medusa-builder-${{ matrix.suite }}" \
&& docker compose run "cassandra-medusa-builder-${{ matrix.suite }}"
cd ../../packages
if [ -f "cassandra-medusa_${version}-0~${{ matrix.suite }}0_amd64.deb" ]; then
echo "${suite} debian package was successfully built"
else
echo "Error: no packages found for ${{ matrix.suite }}"
exit 1
fi
- name: Install Debian Package and run Medusa
run: |
version=$(cat VERSION)
cd packages
sudo dpkg -i cassandra-medusa_${version}-0~${{ matrix.suite }}0_amd64.deb
medusa
exit $?
integration-tests:
continue-on-error: ${{ matrix.experimental }}
needs: build
strategy:
fail-fast: false
matrix:
python-version: [3.9, "3.10", "3.11"]
it-backend: [local, s3, gcs, minio, azure, azure-hierarchical]
# IBM not included by default due to lite plan quota being easily exceeded
#it-backend: [local, s3, gcs, minio, ibm, azure]
cassandra-version: [2.2.19, 3.11.17, 4.0.15, 'github:apache/trunk']
java-version: [8.0.252, 11.0.25]
include:
# tweak the experimental flag for cassandra versions
- cassandra-version: 2.2.19
experimental: false
- cassandra-version: 3.11.17
experimental: false
- cassandra-version: 4.0.15
experimental: false
# explicitly include tests against python 3.11 and one version of cassandra
- python-version: "3.11"
cassandra-version: 4.0.15
it-backend: gcs
experimental: false
java-version: 11.0.25
# explicitly include tests against python 3.10 and one version of cassandra
- python-version: "3.10"
cassandra-version: 4.0.15
it-backend: gcs
experimental: false
java-version: 11.0.25
exclude:
# no tests against trunk
- cassandra-version: 'github:apache/trunk'
# no tests for C* 2.2 with java 11
- cassandra-version: 2.2.19
java-version: 11.0.25
# no tests for C* 3.11 with java 11
- cassandra-version: 3.11.17
java-version: 11.0.25
# no tests for C* 4.0 with java 8
- cassandra-version: 4.0.15
java-version: 8.0.252
# fewer tests against cassandra 3.11.17 (exclude all but s3 storage backends)
# we are not doing the local because it would run a scenario with mgmt-api which no longer supports 3.11
# but we still want some tests against 3.11.17, so we use s3 for at least some coverage
- it-backend: local
cassandra-version: "3.11.17"
- it-backend: gcs
cassandra-version: "3.11.17"
- it-backend: minio
cassandra-version: "3.11.17"
- it-backend: azure
cassandra-version: "3.11.17"
- it-backend: azure-hierarchical
cassandra-version: "3.11.17"
# no tests against non-python3.9, except the explicitly allowed combinations
- python-version: "3.10"
- python-version: "3.11"
runs-on: ubuntu-22.04
services:
minio:
image: lazybit/minio
ports:
- 9000:9000
env:
MINIO_ACCESS_KEY: minio_key
MINIO_SECRET_KEY: minio_secret
volumes:
- ${{ github.workspace }}/../data:/data
options: --name=minio --health-cmd "curl http://localhost:9000/minio/health/live"
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: Setup Java Action
uses: actions/setup-java@v1
with:
java-version: ${{ matrix.java-version}}
architecture: x64
- name: Setup Poetry
uses: abatilo/actions-poetry@v2
with:
poetry-version: "1.8.0"
- name: Install dependencies
run: |
poetry install
sudo apt-get remove azure-cli || true
pip install ccm
- name: Check if integration tests can run
env:
AWS_ACCESS_KEY_ID: ${{ secrets.BUCKET_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.BUCKET_SECRET }}
run: |
if [[ ( -n "${AWS_ACCESS_KEY_ID}" && "${{ matrix.it-backend }}" == "s3" ) \
|| ( "${{ matrix.it-backend }}" == "local" ) \
|| ( "${{ matrix.it-backend }}" == "minio" ) \
|| ( -n '${{ secrets.MEDUSA_GCS_CREDENTIALS }}' && "${{ matrix.it-backend }}" == "gcs" ) \
|| ( -n '${{ secrets.MEDUSA_IBM_CREDENTIALS }}' && "${{ matrix.it-backend }}" == "ibm" ) \
|| ( -n '${{ secrets.MEDUSA_AZURE_CREDENTIALS }}' && "${{ matrix.it-backend }}" == "azure" ) \
|| ( -n '${{ secrets.MEDUSA_AZURE_HIERARCHICAL_CREDENTIALS }}' && "${{ matrix.it-backend }}" == "azure-hierarchical" ) ]];
then
echo "IT_CAN_RUN=yes" >> $GITHUB_ENV
else
echo "IT_CAN_RUN=no" >> $GITHUB_ENV
fi
- name: Run integration tests
if: ${{ env.IT_CAN_RUN == 'yes' }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.BUCKET_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.BUCKET_SECRET }}
run: |
set -e
if [[ "${{ matrix.it-backend }}" == "ibm" ]];
then
# Prevent awscli from using AWS secrets in case we're running against IBM cloud
unset AWS_ACCESS_KEY_ID
unset AWS_SECRET_ACCESS_KEY
fi
# Write GCS service account credentials to a file
mkdir ~/.aws
# This fake cluster needs to be created first so that the integration tests pass in GH actions. Don't ask me why...
ccm create test_cluster -v binary:${{ matrix.cassandra-version }} -n 1 --vnodes
ccm node1 updateconf 'storage_port: 7011'
ccm node1 updateconf 'concurrent_reads: 4'
ccm node1 updateconf 'concurrent_writes: 4'
ccm node1 updateconf 'concurrent_counter_writes: 4'
ccm node1 updateconf 'num_tokens: 4'
sed -i 's/#MAX_HEAP_SIZE="4G"/MAX_HEAP_SIZE="256m"/' ~/.ccm/test_cluster/node1/conf/cassandra-env.sh
sed -i 's/#HEAP_NEWSIZE="800M"/HEAP_NEWSIZE="200M"/' ~/.ccm/test_cluster/node1/conf/cassandra-env.sh
# remove the ThreadPriorityPolicy option for cases where we run with java 11
sed -i 's/-XX:ThreadPriorityPolicy=42//' ~/.ccm/test_cluster/node1/conf/jvm.options || true
sed -i 's/-XX:ThreadPriorityPolicy=42//' ~/.ccm/test_cluster/node1/conf/jvm8-server.options || true
ccm start -v
ccm showlastlog|tail -100
ccm stop
# if we are dealing with C* 4.0+, we need to fix the java-driver dependency.
# it has a bug that breaks sstableloader tests with client encryption enabled
# if cassandra version starts with 4.0 or 4.1
if [[ "${{ matrix.cassandra-version }}" =~ ^4\.[01]\. ]];
then
ccm create driver-fix-cluster -v ${{ matrix.cassandra-version }} -n 1
# enclosed in () so we return to current pwd once we are done fixing the driver
(
cd ~/.ccm/repository/${{ matrix.cassandra-version }}/lib
rm cassandra-driver-core-3.11.0-shaded.jar
wget https://repo1.maven.org/maven2/com/datastax/cassandra/cassandra-driver-core/3.11.5/cassandra-driver-core-3.11.5-shaded.jar
)
fi
if [ "${{ matrix.it-backend }}" == "s3" ]
then
# AWS S3 Storage tests
./run_integration_tests.sh -v --s3 --no-local --cassandra-version=${{ matrix.cassandra-version }}
elif [ "${{ matrix.it-backend }}" == "gcs" ]
then
# Google Cloud Storage tests
echo '${{ secrets.MEDUSA_GCS_CREDENTIALS }}' > ~/medusa_credentials.json
./run_integration_tests.sh -v --gcs --no-local --cassandra-version=${{ matrix.cassandra-version }}
elif [ "${{ matrix.it-backend }}" == "ibm" ]
then
# IBM Cloud Object Storage tests
printf "%s" '${{ secrets.MEDUSA_IBM_CREDENTIALS }}' > ~/.aws/ibm_credentials
./run_integration_tests.sh -v --ibm --no-local --cassandra-version=${{ matrix.cassandra-version }}
elif [ "${{ matrix.it-backend }}" == "minio" ]
then
# MinIO Object Storage tests
unset AWS_ACCESS_KEY_ID
unset AWS_SECRET_ACCESS_KEY
wget https://dl.min.io/client/mc/release/linux-amd64/mc
chmod +x mc
./mc alias set minio http://127.0.0.1:9000 minio_key minio_secret
./mc mb minio/medusa-dev
cp ./tests/resources/minio/minio_credentials ~/.aws/minio_credentials
./run_integration_tests.sh -v --minio --no-local --cassandra-version=${{ matrix.cassandra-version }}
elif [ "${{ matrix.it-backend }}" == "azure" ]
then
# Azure Blob Storage tests
printf "%s" '${{ secrets.MEDUSA_AZURE_CREDENTIALS }}' > ~/medusa_azure_credentials.json
./run_integration_tests.sh -v --azure --no-local --cassandra-version=${{ matrix.cassandra-version }}
elif [ "${{ matrix.it-backend }}" == "azure-hierarchical" ]
then
# Azure Blob Storage with hierarchical namespace tests
printf "%s" '${{ secrets.MEDUSA_AZURE_HIERARCHICAL_CREDENTIALS }}' > ~/medusa_azure_credentials.json
./run_integration_tests.sh -v --azure --no-local --cassandra-version=${{ matrix.cassandra-version }}
else
# Local storage tests
./run_integration_tests.sh -v --cassandra-version=${{ matrix.cassandra-version }}
fi
# Move and convert the coverage analysis file to XML
mv tests/integration/.coverage .
poetry run coverage xml
- uses: codecov/codecov-action@v1
name: Report code coverage
k8ssandra-e2e-tests:
needs: [build]
runs-on: ubuntu-latest
strategy:
matrix:
e2e_test:
- CreateSingleMedusaJob
- CreateSingleDseSearchDatacenterCluster
fail-fast: false
name: k8ssandra-${{ matrix.e2e_test }}
env:
CGO_ENABLED: 0
KUBECONFIG_FILE: "./build/kind-kubeconfig"
CONTROL_PLANE: kind-k8ssandra-0
DATA_PLANES: kind-k8ssandra-0
steps:
- name: Raise open files limit
run: |
echo fs.file-max=1000000 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
echo fs.inotify.max_user_instances=1280 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
echo fs.inotify.max_user_watches=655360 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
ulimit -a
- name: Free diskspace by removing unused packages
run: |
sudo rm -rf /usr/local/lib/android
sudo rm -rf /usr/share/dotnet
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.K8SSANDRA_DOCKER_HUB_USERNAME }}
password: ${{ secrets.K8SSANDRA_DOCKER_HUB_PASSWORD }}
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Set up Poetry
uses: abatilo/actions-poetry@v2
with:
poetry-version: "1.8.0"
- uses: actions/checkout@v3
with:
path: cassandra-medusa
- uses: actions/checkout@v3
with:
repository: k8ssandra/k8ssandra-operator
ref: radovan/ec2-dse-medusa
path: k8ssandra-operator
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.20'
cache: true
cache-dependency-path: "**/*.sum"
- name: Install Kind
working-directory: k8ssandra-operator
run: go get sigs.k8s.io/kind
- name: Install kustomize
uses: imranismail/setup-kustomize@v2
with:
kustomize-version: 4.x
- name: Build Medusa
working-directory: cassandra-medusa
run: |
echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
poetry install && poetry build
- name: Build Medusa image
id: docker_build
uses: docker/build-push-action@v4
with:
file: cassandra-medusa/k8s/Dockerfile
context: ./cassandra-medusa
tags: k8ssandra/medusa:${{ env.short_sha }}-tmp
platforms: linux/amd64
push: true
- name: Setup kind cluster
working-directory: k8ssandra-operator
run: |
make single-up
- name: Run e2e test ( ${{ matrix.e2e_test }} )
working-directory: k8ssandra-operator
run: |
e2e_test="TestOperator/${{ matrix.e2e_test }}"
args="-kubeconfigFile '${{ env.KUBECONFIG_FILE }}'"
args="$args -controlPlane '${{ env.CONTROL_PLANE }}'"
args="$args -dataPlanes '${{ env.DATA_PLANES }}'"
args="$args -medusaImageTag ${{ env.short_sha }}-tmp"
make E2E_TEST="$e2e_test" TEST_ARGS="$args" e2e-test
- name: Setup tmate session
if: ${{ failure() }}
uses: mxschmitt/action-tmate@v3
timeout-minutes: 15
- name: Get artefact upload directory
working-directory: k8ssandra-operator
if: ${{ failure() }}
run: |
uploaddir_name=$(echo ${{ matrix.e2e_test }}| sed 's/\//__/g')
echo 'setting uploaddir_name to' $uploaddir_name
echo "uploaddir_name=$uploaddir_name" >> $GITHUB_ENV
- name: Archive k8s logs
if: ${{ failure() }}
uses: actions/upload-artifact@v3
with:
name: k8s-logs-${{ env.uploaddir_name }}
path: ./k8ssandra-operator/build/test
publish-docker-master:
needs: [debian-build, build, integration-tests, k8ssandra-e2e-tests]
if: github.event_name == 'push' && github.ref == 'refs/heads/master'
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: 'arm64,arm'
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.K8SSANDRA_DOCKER_HUB_USERNAME }}
password: ${{ secrets.K8SSANDRA_DOCKER_HUB_PASSWORD }}
- name: Set git parsed values
id: vars
shell: bash
run: |
echo "sha_short=$(git rev-parse --short=8 ${{ github.sha }})" >> $GITHUB_OUTPUT
- name: Set up Poetry
uses: abatilo/actions-poetry@v2
with:
poetry-version: "1.8.0"
- name: Build Medusa
run: |
echo "Publishing release $(git rev-parse --short HEAD) in Docker Hub"
poetry install && poetry build
- name: Build image and push
id: docker_build
uses: docker/build-push-action@v4
with:
file: k8s/Dockerfile
context: .
push: true
tags: k8ssandra/medusa:${{ steps.vars.outputs.sha_short}},k8ssandra/medusa:master
platforms: linux/amd64,linux/arm64
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
publish-debian:
# We can only release if the build above succeeded first
needs: [debian-build, build, integration-tests, k8ssandra-e2e-tests]
if: github.event_name == 'release' && github.event.action == 'published'
strategy:
matrix:
suite: [focal]
include:
- suite: focal
os-version: ubuntu-22.04
fail-fast: false
runs-on: ${{ matrix.os-version }}
steps:
- uses: actions/checkout@v3
- name: Build Debian
run: |
version=$(cat VERSION)
echo "VERSION=$version" >> $GITHUB_ENV
cd packaging/docker-build
docker compose build "cassandra-medusa-builder-${{ matrix.suite }}" \
&& docker compose run "cassandra-medusa-builder-${{ matrix.suite }}"
- name: Push Debian to Cloudsmith
id: push-deb
uses: cloudsmith-io/action@master
with:
api-key: ${{ secrets.CLOUDSMITH_API_KEY }}
command: 'push'
format: 'deb'
owner: 'thelastpickle'
repo: 'medusa'
distro: 'ubuntu'
release: ${{ matrix.suite }}
republish: 'true'
file: "packages/cassandra-medusa_${{ env.VERSION }}-0~${{ matrix.suite }}0_amd64.deb"
publish-docker:
needs: [publish-debian]
if: github.event_name == 'release' && github.event.action == 'published'
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
with:
platforms: 'arm64,arm'
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.K8SSANDRA_DOCKER_HUB_USERNAME }}
password: ${{ secrets.K8SSANDRA_DOCKER_HUB_PASSWORD }}
- name: Set git parsed values
id: vars
shell: bash
run: |
echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT
- name: Set up Poetry
uses: abatilo/actions-poetry@v2
with:
poetry-version: "1.8.0"
- name: Build Medusa
run: |
echo "Publishing release ${{ steps.vars.outputs.version}} in Docker Hub"
poetry install && poetry build
- name: Build image and push
id: docker_build
uses: docker/build-push-action@v4
with:
file: k8s/Dockerfile
context: .
push: true
tags: k8ssandra/medusa:${{ steps.vars.outputs.version}}
platforms: linux/amd64,linux/arm64
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
publish-pypi:
# We can only release if the build above succeeded first
needs: [publish-docker]
if: github.event_name == 'release' && github.event.action == 'published'
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.10
uses: actions/setup-python@v1
with:
python-version: 3.10
- name: Set up Poetry
uses: abatilo/actions-poetry@v2
with:
poetry-version: "1.8.0"
- name: Install dependencies
run: |
python -m venv venv
. venv/bin/activate
python -m pip install --upgrade pip
- name: Build and publish to pypi
env:
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
. venv/bin/activate
pip install setuptools wheel twine
poetry install && poetry build
twine upload dist/*