build: update backend dependency management to pdm + dockerfile

.circleci/config.yml

@@ -167,7 +167,8 @@ jobs:
             yarn
             cd ${CIRCLE_WORKING_DIRECTORY}
             # Install Python dependencies
-            pip install --upgrade pip
+            pip install --upgrade pip pdm
+            pdm export --prod --without-hashes > requirements.txt
             pip install -r requirements.txt
       - run:
           name: Run backend code checks

.dockerignore

@@ -1,6 +1,7 @@
 .*
 *.yml
 **/*.pyc
+**/__pypackages__
 **/node_modules
 **/npm-debug.log
 logs/

.github/labeler.yml

@@ -5,7 +5,7 @@
   - tests/**/*
   - migrations/**/*
   - ./manage.py
-  - ./requirements.txt
+  - ./pyproject.toml
 "Component: Infrastructure":
   - .circleci/*
   - .github/**/*
@@ -15,10 +15,10 @@
   - docker-*.yml
 "dependencies":
   - frontend/yarn.lock
-  - ./requirements.txt
+  - ./pyproject.toml
 "javascript":
   - frontend/yarn.lock
 "python":
-  - ./requirements.txt
+  - ./pyproject.toml
 "Type: Translations":
   - frontend/src/locales/*

.gitignore

@@ -26,6 +26,8 @@ logs/
 #python
 venv/
 *pyc
+.pdm.toml
+__pypackages__/
 
 # Editor generated files #
 .idea/

Dockerfile

@@ -1,12 +1,34 @@
-FROM quay.io/hotosm/base-python-image as base
-LABEL version=0.1
-LABEL maintainer="HOT Sysadmin <sysadmin@hotosm.org>"
-LABEL description="Builds backend docker image"
+ARG ALPINE_IMG_TAG=3.17
+ARG PYTHON_IMG_TAG=3.9
 
-WORKDIR /usr/src/app
+FROM docker.io/python:${PYTHON_IMG_TAG}-alpine${ALPINE_IMG_TAG} as base
+ARG APP_VERSION=0.1.0
+ARG DOCKERFILE_VERSION=0.4.0
+ARG ALPINE_IMG_TAG
+ARG PYTHON_IMG_TAG
+ARG MAINTAINER=sysadmin@hotosm.org
+LABEL org.hotosm.tasks.app-version="${APP_VERSION}" \
+      org.hotosm.tasks.alpine-img-tag="${ALPINE_IMG_TAG}" \
+      org.hotosm.tasks.python-img-tag="${PYTHON_IMG_TAG}" \
+      org.hotosm.tasks.dockerfile-version="${DOCKERFILE_VERSION}" \
+      org.hotosm.tasks.maintainer="${MAINTAINER}" \
+      org.hotosm.tasks.api-port="5000"
+# Fix timezone (do not change - see issue #3638)
+ENV TZ UTC
+
+
+
+FROM base as extract-deps
+WORKDIR /opt/python
+COPY pyproject.toml pdm.lock README.md /opt/python/
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir pdm==2.5.3
+RUN pdm export --prod --without-hashes > requirements.txt
 
-FROM base as builder
 
+
+FROM base as build
+WORKDIR /opt/python
 # Setup backend build-time dependencies
 RUN apk update && \
     apk add \
@@ -20,29 +42,55 @@ RUN apk update && \
         proj-util \
         proj-dev \
         make
+# Setup backend Python dependencies
+COPY --from=extract-deps \
+    /opt/python/requirements.txt /opt/python/
+RUN pip install --user --no-warn-script-location \
+    --no-cache-dir -r /opt/python/requirements.txt
 
-COPY requirements.txt .
 
-RUN pip install \
-    --prefix=/install \
-    --no-cache-dir \
-    --no-warn-script-location \
-    -r requirements.txt
 
+FROM base as runtime
+ARG PYTHON_IMG_TAG
+WORKDIR /usr/src/app
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONFAULTHANDLER=1 \
+    PATH="/home/appuser/.local/bin:$PATH" \
+    PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_IMG_TAG/site-packages" \
+    SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
+    REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 # Setup backend runtime dependencies
-FROM base
-
 RUN apk update && \
-    apk add \
+    apk add --no-cache \
         postgresql-libs geos proj-util
-
-COPY --from=builder /install /usr/local
+COPY --from=build \
+    /root/.local \
+    /home/appuser/.local
 COPY backend backend/
 COPY migrations migrations/
 COPY scripts/world scripts/world/
 COPY scripts/database scripts/database/
 COPY manage.py .
+# Add non-root user, permissions, init log dir
+RUN adduser -D -u 900 -h /home/appuser -s /bin/false appuser \
+    && chown -R appuser:appuser /usr/src /home/appuser
+
+
+
+FROM runtime as debug
+RUN pip install --no-warn-script-location \
+    --no-cache-dir debugpy==1.6.7
+USER appuser
+CMD ["python", "-m", "debugpy", "--wait-for-client", "--listen", "0.0.0.0:5678", \
+    "-m", "gunicorn", "-c", "python:backend.gunicorn", "manage:application", \
+    "--reload", "--log-level", "error"]
+
+
 
-ENV TZ UTC # Fix timezone (do not change - see issue #3638)
-EXPOSE 5000
-CMD ["gunicorn", "-c", "python:backend.gunicorn", "manage:application"]
+FROM runtime as prod
+# Pre-compile packages to .pyc (init speed gains)
+RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
+USER appuser
+CMD ["gunicorn", "-c", "python:backend.gunicorn", "manage:application", \
+    "--workers", "1", "--log-level", "error"]

backend/config.py

@@ -83,7 +83,7 @@ class EnvironmentConfig:
 
     # Logging settings
     LOG_LEVEL = os.getenv("TM_LOG_LEVEL", logging.DEBUG)
-    LOG_DIR = os.getenv("TM_LOG_DIR", "logs")
+    LOG_DIR = os.getenv("TM_LOG_DIR", "/home/appuser/logs")
 
     # Mapper Level values represent number of OSM changesets
     MAPPER_LEVEL_INTERMEDIATE = int(os.getenv("TM_MAPPER_LEVEL_INTERMEDIATE", 250))

docs-old/contributing-translation.md

@@ -9,7 +9,7 @@ Everybody is welcome to support translations through the Transifex website.
 ## Developers
 
 For developers, Transifex offers a [CLI client](https://docs.transifex.com/client/introduction/) and the Tasking
-Manager offers commands to interact with it. The client is already included in `requirements.txt` so you should have the Transifex commands installed once you have set up your backend side code.
+Manager offers commands to interact with it. The client is already included in the Python requirements so you should have the Transifex commands installed once you have set up your backend side code.
 
 The Tasking Manager is using Angular Translate to display the translated strings. It works with key/value pairs in .json format, which is also the format used to store the translations in Transifex.
 

docs-old/review-pr.md

@@ -52,7 +52,18 @@ and ask the person to check on the error.
 
 1. Test behaviour and edge cases
   Install the PR on your local setup, make sure you run
-  a. backend dependency installation: `pip install -r requirements.txt`
+  a. backend dependency installation:
+    * Linux/Mac (Option 1: pep582):
+        * First ensure the Python version in `pyproject.toml:requires-python` is installed on your system.
+        * ```pip install --upgrade pdm```
+        * ```pdm config --global python.use_venv False```
+        * ```pdm --pep582 >> ~/.bash_profile```
+        * ```source ~/.bash_profile```
+        * ```pdm install```
+    * Linux/Mac (Option 2: pip (system/venv)):
+        * ```pip install --upgrade pdm```
+        * ```pdm export --without-hashes > requirements.txt```
+        * ```pip install requirements.txt```
   b. introduced database migrations: `python manage.py db upgrade`
   c. frontend dependency installation: `cd frontend && yarn && cd ..`
   d. rebuild the frontend: `cd frontend && yarn build && cd ..`
@@ -100,7 +111,11 @@ git pull https://github.com/author/tasking-manager.git branchname
 
 Make sure that python dependencies are up to date:
 
-`pip install -r requirements.txt`
+* Option 1:
+    * ```pdm install```
+* Option 2:
+    * ```pdm export --without-hashes > requirements.txt```
+    * ```pip install requirements.txt```
 
 ### Frontend
 
@@ -127,7 +142,7 @@ If you get an error, you may have an upgraded database from a prior PR. Try down
 
 I have found it better to run tests on a separate database from the live version, but the choice is up to you. Again, it is probably wise to back up your database first if you choose to run it on your main database.
 
-`venv/bin/  ./tests --with-xunit --xunit-file unitresults.xml --with-coverage --cover-erase --cover-package=./backend`
+`python3 -m unittest discover ./tests --with-xunit --xunit-file unitresults.xml --with-coverage --cover-erase --cover-package=./backend`
 
 ### Check changes
 

docs-old/setup-development.md

@@ -90,12 +90,18 @@ There are two ways to configure Tasking Manager. You can set some environment va
 
 #### Build
 
-* Create a Python Virtual Environment, using Python 3.6+:
-    * ```python3 -m venv ./venv```
-* Activate your virtual environment and install dependencies:
-    * Linux/Mac:
-        * ```. ./venv/bin/activate```
-        * ```pip install -r requirements.txt```
+* Install project dependencies:
+    * Linux/Mac (Option 1: pep582):
+        * First ensure the Python version in `pyproject.toml:requires-python` is installed on your system.
+        * ```pip install --upgrade pdm```
+        * ```pdm config --global python.use_venv False```
+        * ```pdm --pep582 >> ~/.bash_profile```
+        * ```source ~/.bash_profile```
+        * ```pdm install```
+    * Linux/Mac (Option 2: pip (system/venv)):
+        * ```pip install --upgrade pdm```
+        * ```pdm export --without-hashes > requirements.txt```
+        * ```pip install requirements.txt```
 
 #### Tests
 
@@ -149,7 +155,7 @@ In order to authenticate on the API, you need to have an Authorization Token.
 1. Run the command line `manage.py` with the `gen_token` option and `-u <OSM_User_ID_number>`. The command line can be run in any shell session as long as you are in the tasking-manager directory.
 
 ```
-venv/bin/python manage.py gen_token -u 99999999
+python manage.py gen_token -u 99999999
 ```
 
 This will generate a line that looks like this: