Mise à jour de l'environnement python et amélioration du build (#2626)

## Linked issues

- Resolve #2625

.dockerignore

@@ -1,27 +1,39 @@
 # git repo
 .git
 
+# Frontend environment
+frontend/node_modules
+
 # Env variables
 datascience/.env
 
+#Docs
+/datascience/docs
+
 # Notebooks
 /datascience/notebooks
-/datascience/.local
-/datascience/.ipython
-/datascience/.cache
-.ipynb_checkpoints/
+datascience/dam-si-jupyter-config.py
+**/.ipython
+**/.ipynb_checkpoints/
 .jupyter/
 
 # Local data
 /datascience/data
 
 # Python
 *.pyc
-__pycache__
+datascience/monitorfish.egg-info
+/datascience/.local
+/datascience/.cache
+/datascience/.pytest_cache
+**/__pycache__
 
 # Coverage reports
-htmlcov
-.coverage
+**/htmlcov
+**/.coverage
+
+# Tools
+**/.pre-commit-config.yaml
 
 # IDEs
 .vscode

.github/workflows/cicd-data-processing.yml

@@ -46,9 +46,9 @@ jobs:
         uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache-pipeline
-          key: ${{ runner.os }}-single-buildx-${{ github.sha }}
+          key: ${{ runner.os }}-single-buildx-data-pipeline-${{ github.ref_name }}
           restore-keys: |
-            ${{ runner.os }}-single-buildx
+            ${{ runner.os }}-single-buildx-data-pipeline-
 
       - name: Build image
         uses: docker/build-push-action@v2

.github/workflows/cicd.yml

@@ -144,9 +144,9 @@ jobs:
         uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache-app
-          key: ${{ runner.os }}-single-buildx-${{ github.sha }}
+          key: ${{ runner.os }}-single-buildx-app-${{ github.ref_name }}
           restore-keys: |
-            ${{ runner.os }}-single-buildx
+            ${{ runner.os }}-single-buildx-app-
 
       - name: Build image
         uses: docker/build-push-action@v2

Makefile

@@ -1,4 +1,5 @@
 INFRA_FOLDER="$(shell pwd)/infra/configurations/"
+HOST_MIGRATIONS_FOLDER=$(shell pwd)/backend/src/main/resources/db/migration
 
 .PHONY: install init-sig run-front run-back docker-build docker-tag docker-push check-clean-archi test restart-app
 
@@ -62,7 +63,7 @@ docker-compose-up:
 docker-build-pipeline:
 	docker build -f "infra/docker/Dockerfile.DataPipeline" . -t monitorfish-pipeline:$(VERSION)
 docker-test-pipeline:
-	docker run --network host -v /var/run/docker.sock:/var/run/docker.sock -u monitorfish-pipeline:$(DOCKER_GROUP) --env-file datascience/.env.test monitorfish-pipeline:$(VERSION) coverage run -m pytest --pdb tests
+	docker run --network host -v /var/run/docker.sock:/var/run/docker.sock -u monitorfish-pipeline:$(DOCKER_GROUP) --env-file datascience/.env.test --env HOST_MIGRATIONS_FOLDER=$(HOST_MIGRATIONS_FOLDER) monitorfish-pipeline:$(VERSION) coverage run -m pytest --pdb tests
 docker-tag-pipeline:
 	docker tag monitorfish-pipeline:$(VERSION) docker.pkg.github.com/mtes-mct/monitorfish/monitorfish-pipeline:$(VERSION)
 docker-push-pipeline:
@@ -92,8 +93,6 @@ install-pipeline:
 	cd datascience && poetry install
 test-pipeline:
 	cd datascience && export TEST_LOCAL=True && poetry run coverage run -m pytest --pdb tests/ && poetry run coverage report && poetry run coverage html
-update-python-dependencies:
-	cd datascience && poetry export --without-hashes -o requirements.txt && poetry export --without-hashes --dev -o requirements-dev.txt
 
 # DOC commands
 push-docs-to-transifex:

datascience/.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
-  - repo: https://github.com/myint/autoflake
-    rev: v1.4
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.1
     hooks:
       - id: autoflake
         name: autoflake
@@ -10,17 +10,17 @@ repos:
         require_serial: true
         args: ['--in-place', '--remove-all-unused-imports', '--ignore-init-module-imports']
   - repo: https://github.com/pycqa/isort
-    rev: 5.6.3
+    rev: 5.12.0
     hooks:
       - id: isort
         name: isort (python)
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.10.0
     hooks:
       - id: black
         language_version: python3
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
     hooks:
       - id: flake8
         args: # arguments to configure flake8

datascience/.python-version

@@ -1 +1 @@
-3.8.13
+3.11.6

datascience/config.py

@@ -8,6 +8,13 @@
 LIBRARY_LOCATION = ROOT_DIRECTORY / Path("src")
 QUERIES_LOCATION = LIBRARY_LOCATION / Path("pipeline/queries")
 TEST_DATA_LOCATION = ROOT_DIRECTORY / Path("tests/test_data")
+LOCAL_MIGRATIONS_FOLDER = str(
+    (ROOT_DIRECTORY / Path("../backend/src/main/resources/db/migration")).resolve()
+)
+# HOST_MIGRATIONS_FOLDER envirionment variable is needed when running tests in CI to
+# mount migrations folder from the host to the database container
+HOST_MIGRATIONS_FOLDER = os.getenv("HOST_MIGRATIONS_FOLDER", LOCAL_MIGRATIONS_FOLDER)
+
 EMAIL_TEMPLATES_LOCATION = LIBRARY_LOCATION / Path("pipeline/emails/templates")
 EMAIL_IMAGES_LOCATION = LIBRARY_LOCATION / Path("pipeline/emails/images")
 EMAIL_FONTS_LOCATION = LIBRARY_LOCATION / Path("pipeline/emails/fonts")