Skip to content
OPERANDI CI/CD

Transfer the project from the SCC to the NHR partition #337

Sign in to view logs

Transfer the project from the SCC to the NHR partition

Transfer the project from the SCC to the NHR partition #337

Workflow file for this run

.github/workflows/ci_cd.yml at ca824f5
# For more information see:
# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: OPERANDI CI/CD
on:
push:
# branches: [ main ]
paths:
- .github/**
- src/**
- tests/**
- .*
- docker*
- Makefile
pull_request:
# branches: [ main ]
paths:
- .github/**
- src/**
- tests/**
- .*
- docker*
- Makefile
env:
REGISTRY: ghcr.io
REPO_NAME: ${{ github.repository }}
jobs:
build-native:
name: Native build of Operandi modules
strategy:
fail-fast: true
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install operandi dependencies
run: |
sudo apt-get update && sudo apt-get -y install make
python3 -m pip install --upgrade pip setuptools
pip3 install -U pip wheel
pip install -r tests/requirements.txt
- name: Lint with flake8
run: |
python3 -m pip install flake8
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics
- name: Install operandi modules
run: make install
run-operandi-tests:
name: Run operandi tests
needs: build-native
env:
ENV_FILE: ./.github/workflows/tests/.env
strategy:
fail-fast: false
matrix:
python-version: [ "3.8" ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install operandi dependencies
run: |
sudo apt-get update && sudo apt-get -y install make
python3 -m pip install --upgrade pip setuptools
pip3 install -U pip wheel
pip install -r tests/requirements.txt
- name: Install operandi modules
run: make install
- name: Import variables from env file
shell: bash
run: |
while read line; do
echo "$line" >> $GITHUB_ENV
done < ${{ env.ENV_FILE }}
- name: copy hpc ssh key
env:
SSH_KEY_HPC: ${{ secrets.OPERANDI_SSH_KEY_HPC }}
run: |
mkdir -p /home/runner/.ssh/
echo "$SSH_KEY_HPC" > /home/runner/.ssh/key_hpc
chmod 600 /home/runner/.ssh/key_hpc
- name: start RabbitMQ Server
run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-rabbitmq
- name: start MongoDB
run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-mongodb
- name: wait starting of RabbitMQ and MongoDB
run: sleep 45
- name: run utils tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/tests_utils/test_*.py -v
- name: run broker tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/tests_broker/test_*.py -v
- name: run server tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/tests_server/test_*.py -v
- name: run harvester tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/tests_harvester/test_*.py -v
- name: run integration tests
run: |
export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
pytest -p no:cacheprovider tests/integration_tests/test_*.py -s -v
build-and-push-docker-images:
name: Push ${{matrix.services.module}} to image registry
environment:
name: development
needs: run-operandi-tests
strategy:
fail-fast: true
matrix:
services: [
{ module: "broker", dockerfile: "Dockerfile_broker" },
{ module: "server", dockerfile: "Dockerfile_server" }
]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels) for Docker
uses: docker/metadata-action@v4
id: meta
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}-${{matrix.services.module}}
- name: Build and push Docker image
uses: docker/build-push-action@v4
with:
context: ./src
file: ./src/${{matrix.services.dockerfile}}
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
deploy-development:
name: Deploy to development VM
environment:
name: development
needs: build-and-push-docker-images
runs-on: ubuntu-latest
steps:
- name: copy ssh key
env:
SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }}
run: |
mkdir -p ~/.ssh/
echo "$SSH_KEY" > ~/.ssh/key
chmod 600 ~/.ssh/key
- name: deploy
env:
SSH_USER: ${{ secrets.OPERANDI_SSH_USER }}
SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_DEV }}
run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/operandi/start-operandi-docker.sh
deploy-production:
name: Deploy to live VM
environment:
name: production
needs: build-and-push-docker-images
runs-on: ubuntu-latest
steps:
- name: copy ssh key
env:
SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }}
run: |
mkdir -p ~/.ssh/
echo "$SSH_KEY" > ~/.ssh/key
chmod 600 ~/.ssh/key
- name: deploy
env:
SSH_USER: ${{ secrets.OPERANDI_SSH_USER }}
SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_PROD }}
run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/start-operandi-docker.sh