feat: Initial working pipeline with minimum tests

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 90

.gitignore

@@ -0,0 +1,4 @@
+credentials.json
+data_pipeline.egg-info
+.pytest_cache
+venv

.pylintrc

@@ -0,0 +1,7 @@
+[MESSAGES CONTROL]
+
+min-similarity-lines=5
+disable=
+    missing-docstring,
+    too-few-public-methods,
+    no-self-use

Dockerfile

@@ -0,0 +1,39 @@
+FROM puckel/docker-airflow:1.10.4
+ARG install_dev
+
+USER root
+
+RUN apt update && apt install sudo -yqq
+RUN usermod -aG sudo airflow
+RUN echo "airflow ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+
+RUN sed -i 's/LocalExecutor/SequentialExecutor/' /entrypoint.sh
+COPY requirements.txt ./
+RUN pip install --upgrade -r requirements.txt
+RUN if [ "${install_dev}" = "y" ]; then  pip install bokeh; fi
+
+USER airflow
+COPY --chown=airflow:airflow requirements.dev.txt ./
+RUN if [ "${install_dev}" = "y" ]; then pip install --user -r requirements.dev.txt; fi
+
+ENV PATH /usr/local/airflow/.local/bin:$PATH
+
+COPY --chown=airflow:airflow data_pipeline ./data_pipeline
+COPY --chown=airflow:airflow dags ./dags
+COPY --chown=airflow:airflow setup.py ./setup.py
+RUN pip install -e . --user --no-dependencies
+
+COPY .pylintrc ./.pylintrc
+COPY .flake8 ./.flake8
+COPY --chown=airflow:airflow tests ./tests
+COPY --chown=airflow:airflow run_test.sh ./
+RUN if [ "${install_dev}" = "y" ]; then chmod +x run_test.sh; fi
+
+COPY --chown=airflow:airflow worker.sh ./
+RUN chmod +x worker.sh
+
+RUN mkdir -p $AIRFLOW_HOME/serve
+RUN ln -s $AIRFLOW_HOME/logs $AIRFLOW_HOME/serve/log
+
+ENTRYPOINT []

Jenkinsfile

@@ -0,0 +1,46 @@
+elifePipeline {
+    node('containers-jenkins-plugin') {
+        def commit
+        def jenkins_image_building_ci_pipeline = 'process/process-data-hub-airflow-image-update-repo-list'
+        def git_url
+
+        stage 'Checkout', {
+            checkout scm
+            commit = elifeGitRevision()
+            git_url = getGitUrl()
+        }
+
+        stage 'Build and run tests', {
+            withDataPipelineGcpCredentials {
+                try {
+                    sh "make ci-end2end-test"
+                } finally {
+                    sh "make ci-clean"
+                }
+            }
+        }
+
+        elifeMainlineOnly {
+            stage 'Build data pipeline image with latest commit', {
+                triggerImageBuild(jenkins_image_building_ci_pipeline, git_url, commit)
+            }
+        }
+    }
+}
+
+def withDataPipelineGcpCredentials(doSomething) {
+    try {
+        sh 'vault.sh kv get -field credentials secret/containers/data-pipeline/gcp > credentials.json'
+        doSomething()
+    } finally {
+        sh 'echo > credentials.json'
+    }
+}
+
+def triggerImageBuild(jenkins_image_building_ci_pipeline, gitUrl, gitCommitRef){
+    build job: jenkins_image_building_ci_pipeline,  wait: false, parameters: [string(name: 'gitUrl', value: gitUrl), string(name: 'gitCommitRef', value: gitCommitRef)]
+}
+
+def getGitUrl() {
+    return sh(script: "git config --get remote.origin.url", returnStdout: true).trim()
+}

Makefile

@@ -0,0 +1,73 @@
+#!/usr/bin/make -f
+
+DOCKER_COMPOSE_CI = docker-compose
+DOCKER_COMPOSE_DEV = docker-compose -f docker-compose.yml -f docker-compose.dev.override.yml
+DOCKER_COMPOSE = $(DOCKER_COMPOSE_CI)
+
+
+VENV = venv
+PIP = $(VENV)/bin/pip
+PYTHON = PYTHONPATH=dags $(VENV)/bin/python
+
+
+venv-clean:
+	@if [ -d "$(VENV)" ]; then \
+		rm -rf "$(VENV)"; \
+	fi
+
+venv-create:
+	python3 -m venv $(VENV)
+
+venv-activate:
+	chmod +x venv/bin/activate
+	bash -c "venv/bin/activate"
+
+dev-install:
+	SLUGIFY_USES_TEXT_UNIDECODE=yes $(PIP) install -r requirements.txt
+	$(PIP) install -r requirements.dev.txt
+	$(PIP) install -e . --no-deps
+
+dev-venv: venv-create dev-install
+
+dev-flake8:
+	$(PYTHON) -m flake8 data_pipeline dags tests
+
+dev-pylint:
+	$(PYTHON) -m pylint data_pipeline dags tests
+
+dev-lint: dev-flake8 dev-pylint
+
+dev-unittest:
+	$(PYTHON) -m pytest -p no:cacheprovider $(ARGS) tests/unit_test
+
+dev-dagtest:
+	$(PYTHON) -m pytest -p no:cacheprovider $(ARGS) tests/dag_validation_test
+
+dev-integration-test: dev-install
+	(VENV)/bin/airflow upgradedb
+	$(PYTHON) -m pytest -p no:cacheprovider $(ARGS) tests/integration_test
+
+dev-test: dev-lint dev-unittest dev-dagtest
+
+build:
+	$(DOCKER_COMPOSE) build data-hub_image
+
+build-dev:
+	$(DOCKER_COMPOSE) build data-hub-dags-dev
+
+ci-test-exclude-e2e: build-dev
+	$(DOCKER_COMPOSE) run --rm data-hub-dags-dev ./run_test.sh
+
+ci-end2end-test: build-dev
+	$(DOCKER_COMPOSE) run --rm  test-client
+	make ci-clean
+
+dev-env: build-dev
+	$(DOCKER_COMPOSE_DEV) up  --scale dask-worker=1 scheduler
+
+dev-end2end-test: build-dev
+	$(DOCKER_COMPOSE_DEV) run --rm  test-client
+	make ci-clean
+
+ci-clean:
+	$(DOCKER_COMPOSE) down -v