always create new sftp connection #226

Workflow file for this run

	# For more information see:
	# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

	name: OPERANDI CI/CD

	on:
	push:
	# branches: [ main ]
	paths:
	- .github/**
	- src/**
	- tests/**
	- .*
	- docker*
	- Makefile
	pull_request:
	# branches: [ main ]
	paths:
	- .github/**
	- src/**
	- tests/**
	- .*
	- docker*
	- Makefile

	env:
	REGISTRY: ghcr.io
	REPO_NAME: ${{ github.repository }}

	jobs:
	build-native:
	name: Native build of Operandi modules
	strategy:
	fail-fast: true
	matrix:
	python-version: ["3.8", "3.9", "3.10", "3.11"]
	os: [ubuntu-latest]
	runs-on: ${{ matrix.os }}

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3
	with:
	ref: main

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install operandi dependencies
	run: \|
	sudo apt-get update && sudo apt-get -y install make
	python3 -m pip install --upgrade pip setuptools
	pip3 install -U pip wheel
	pip install -r tests/requirements.txt

	- name: Lint with flake8
	run: \|
	python3 -m pip install flake8
	# stop the build if there are Python syntax errors or undefined names
	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
	# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
	flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics

	- name: Install operandi modules
	run: make install

	run-operandi-tests:
	name: Run operandi tests
	needs: build-native
	env:
	ENV_FILE: ./.github/workflows/tests/.env
	strategy:
	fail-fast: false
	matrix:
	python-version: [ "3.8" ]
	os: [ ubuntu-latest ]
	runs-on: ${{ matrix.os }}

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3
	with:
	ref: main

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install operandi dependencies
	run: \|
	sudo apt-get update && sudo apt-get -y install make
	python3 -m pip install --upgrade pip setuptools
	pip3 install -U pip wheel
	pip install -r tests/requirements.txt

	- name: Install operandi modules
	run: make install

	- name: Import variables from env file
	shell: bash
	run: \|
	while read line; do
	echo "$line" >> $GITHUB_ENV
	done < ${{ env.ENV_FILE }}

	- name: copy hpc ssh key
	env:
	SSH_KEY_HPC: ${{ secrets.OPERANDI_SSH_KEY_HPC }}
	run: \|
	mkdir -p /home/runner/.ssh/
	echo "$SSH_KEY_HPC" > /home/runner/.ssh/key_hpc
	chmod 600 /home/runner/.ssh/key_hpc

	- name: start RabbitMQ Server
	run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-rabbitmq

	- name: start MongoDB
	run: docker compose -f ./docker-compose.yml --env-file ${{ env.ENV_FILE }} up -d operandi-mongodb

	- name: wait starting of RabbitMQ and MongoDB
	run: sleep 25

	- name: run utils tests
	run: \|
	export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
	pytest tests/tests_utils/test_*.py

	- name: run broker tests
	run: \|
	export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
	pytest tests/tests_broker/test_*.py

	- name: run server tests
	run: \|
	export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
	pytest tests/tests_server/test_*.py

	- name: run harvester tests
	run: \|
	export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
	pytest tests/tests_harvester/test_*.py

	- name: run integration tests
	run: \|
	export $(shell sed 's/=.*//' ${{ env.ENV_FILE }})
	pytest tests/integration_tests/test_*.py -s -v

	build-and-push-docker-images:
	name: Push ${{matrix.services.module}} to image registry
	environment:
	name: development
	needs: run-operandi-tests
	strategy:
	fail-fast: true
	matrix:
	services: [
	{ module: "broker", dockerfile: "Dockerfile_broker" },
	{ module: "server", dockerfile: "Dockerfile_server" }
	]
	os: [ ubuntu-latest ]
	runs-on: ${{ matrix.os }}

	permissions:
	contents: read
	packages: write

	steps:
	- name: Checkout repository
	uses: actions/checkout@v3
	with:
	ref: main

	- name: Log in to the Container registry
	uses: docker/login-action@v2
	with:
	registry: ${{ env.REGISTRY }}
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Extract metadata (tags, labels) for Docker
	uses: docker/metadata-action@v4
	id: meta
	with:
	github-token: ${{ secrets.GITHUB_TOKEN }}
	images: ${{ env.REGISTRY }}/${{ env.REPO_NAME }}-${{matrix.services.module}}

	- name: Build and push Docker image
	uses: docker/build-push-action@v4
	with:
	context: ./src
	file: ./src/${{matrix.services.dockerfile}}
	push: true
	tags: ${{ steps.meta.outputs.tags }}
	labels: ${{ steps.meta.outputs.labels }}

	deploy-development:
	name: Deploy to development VM
	environment:
	name: development
	needs: build-and-push-docker-images
	runs-on: ubuntu-latest

	steps:
	- name: copy ssh key
	env:
	SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }}
	run: \|
	mkdir -p ~/.ssh/
	echo "$SSH_KEY" > ~/.ssh/key
	chmod 600 ~/.ssh/key

	- name: deploy
	env:
	SSH_USER: ${{ secrets.OPERANDI_SSH_USER }}
	SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_DEV }}
	run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/operandi/start-operandi-docker.sh

	deploy-production:
	name: Deploy to live VM
	environment:
	name: production
	needs: build-and-push-docker-images
	runs-on: ubuntu-latest

	steps:
	- name: copy ssh key
	env:
	SSH_KEY: ${{ secrets.OPERANDI_SSH_KEY }}
	run: \|
	mkdir -p ~/.ssh/
	echo "$SSH_KEY" > ~/.ssh/key
	chmod 600 ~/.ssh/key

	- name: deploy
	env:
	SSH_USER: ${{ secrets.OPERANDI_SSH_USER }}
	SSH_HOST: ${{ secrets.OPERANDI_SSH_HOST_PROD }}
	run: ssh $SSH_USER@$SSH_HOST -o StrictHostKeyChecking=no -i ~/.ssh/key /home/$SSH_USER/start-operandi-docker.sh

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

always create new sftp connection #226

Workflow file

always create new sftp connection #226

Jobs

Run details

Workflow file for this run