Merge pull request #47 from digital-land/update_docker_image

Updating the docker to get the image from slim-bookworm

request-processor/Dockerfile

@@ -1,16 +1,24 @@
-FROM python:3.12-alpine3.19
+FROM python:3.12.7-slim-bookworm
 
 ARG GIT_COMMIT
 ENV GIT_COMMIT=$GIT_COMMIT
 
-RUN apk add --no-cache musl-dev curl
 RUN pip3 install cython
-RUN apk add --no-cache libcurl curl-dev build-base git proj proj-dev proj-util geos-dev
 
-# Install GDAL dependencies
-RUN apk update && \
-    apk add --no-cache gdal-dev gdal gdal-tools aws-cli postgresql-client jq && \
-    rm -rf /var/cache/apk/*
+RUN apt update && \
+    apt install gcc -y && \
+    apt install g++ -y && \
+    apt install make && \
+    apt install git -y && \
+    apt install libc-dev curl proj-bin -y libproj-dev -y \
+    gdal-bin \
+    gdal-data \
+    awscli \
+    postgresql-client \
+    jq
+
+
+RUN apt clean
 
 COPY request-processor/requirements/requirements.txt requirements/requirements.txt
 COPY request-processor/makefile makefile

request-processor/requirements/requirements.txt

@@ -1,30 +1,30 @@
 #
-# This file is autogenerated by pip-compile with Python 3.9
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile requirements/requirements.in
 #
 -e git+https://github.com/digital-land/pipeline.git#egg=digital-land
     # via -r requirements/requirements.in
-aiofiles==23.2.1
+aiofiles==24.1.0
     # via datasette
-amqp==5.2.0
+amqp==5.3.1
     # via kombu
-annotated-types==0.6.0
+annotated-types==0.7.0
     # via pydantic
-anyio==4.3.0
+anyio==4.6.2.post1
     # via httpx
-asgi-csrf==0.9
+asgi-csrf==0.11
     # via datasette
 asgiref==3.8.1
     # via datasette
-billiard==4.2.0
+billiard==4.2.1
     # via celery
-boto3==1.34.89
+boto3==1.35.64
     # via
     #   celery
     #   kombu
-botocore==1.34.89
+botocore==1.35.64
     # via
     #   boto3
     #   s3transfer
@@ -38,16 +38,18 @@ celery[sqs]==5.3.6
     # via
     #   -r requirements/requirements.in
     #   sentry-sdk
-certifi==2024.2.2
+certifi==2024.8.30
     # via
     #   httpcore
     #   httpx
     #   pyproj
     #   requests
     #   sentry-sdk
+cffi==1.17.1
+    # via pygit2
 cfgv==3.4.0
     # via pre-commit
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via requests
 click==8.1.7
     # via
@@ -56,6 +58,7 @@ click==8.1.7
     #   click-didyoumean
     #   click-plugins
     #   click-repl
+    #   dask
     #   datasette
     #   digital-land
     #   uvicorn
@@ -67,97 +70,126 @@ click-plugins==1.1.1
     # via celery
 click-repl==0.3.0
     # via celery
-dataclasses-json==0.6.4
+cloudpickle==3.1.0
+    # via dask
+dask==2024.11.2
     # via digital-land
-datasette==0.64.6
+dataclasses-json==0.6.7
     # via digital-land
-distlib==0.3.8
+datasette==0.65
+    # via digital-land
+distlib==0.3.9
     # via virtualenv
-dnspython==2.6.1
+dnspython==2.7.0
     # via eventlet
+duckdb==1.1.3
+    # via digital-land
 esridump==1.13.0
     # via digital-land
-et-xmlfile==1.1.0
+et-xmlfile==2.0.0
     # via openpyxl
 eventlet==0.35.2
     # via -r requirements/requirements.in
-exceptiongroup==1.2.1
+exceptiongroup==1.2.2
     # via anyio
-filelock==3.13.4
+filelock==3.16.1
     # via virtualenv
+flexcache==0.3
+    # via datasette
+flexparser==0.4
+    # via datasette
+fsspec==2024.10.0
+    # via dask
 geojson==3.1.0
     # via digital-land
-greenlet==3.0.3
+greenlet==3.1.1
     # via
     #   eventlet
     #   sqlalchemy
 h11==0.14.0
     # via
     #   httpcore
     #   uvicorn
-httpcore==1.0.5
+httpcore==1.0.7
     # via httpx
-httpx==0.27.0
+httpx==0.27.2
     # via
     #   -r requirements/requirements.in
     #   datasette
 hupper==1.12.1
     # via datasette
-identify==2.5.36
+identify==2.6.2
     # via pre-commit
-idna==3.7
+idna==3.10
     # via
     #   anyio
     #   httpx
     #   requests
-isodate==0.6.1
+importlib-metadata==8.5.0
+    # via dask
+isodate==0.7.2
     # via rdflib
 itsdangerous==2.2.0
     # via
     #   asgi-csrf
     #   datasette
-janus==1.0.0
+janus==1.1.0
     # via datasette
-jinja2==3.1.3
+jinja2==3.1.4
     # via datasette
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-kombu[sqs]==5.3.7
+json-stream==2.3.2
+    # via digital-land
+json-stream-rs-tokenizer==0.4.26
+    # via json-stream
+kombu[sqs]==5.4.2
     # via celery
-markupsafe==2.1.5
+locket==1.0.0
+    # via partd
+markupsafe==3.0.2
     # via jinja2
-marshmallow==3.21.1
+marshmallow==3.23.1
     # via dataclasses-json
 mergedeep==1.3.4
     # via datasette
 mypy-extensions==1.0.0
     # via typing-inspect
-nodeenv==1.8.0
+nodeenv==1.9.1
     # via pre-commit
 numpy==1.26.4
     # via
+    #   digital-land
     #   pandas
     #   shapely
-openpyxl==3.1.2
+openpyxl==3.1.5
     # via digital-land
-packaging==24.0
-    # via marshmallow
-pandas==2.2.2
+packaging==24.2
+    # via
+    #   dask
+    #   marshmallow
+pandas==2.2.3
     # via digital-land
-pint==0.23
-    # via datasette
-platformdirs==4.2.0
-    # via virtualenv
+partd==1.4.2
+    # via dask
+platformdirs==4.3.6
+    # via
+    #   datasette
+    #   virtualenv
 pluggy==1.5.0
     # via datasette
-pre-commit==3.7.0
+pre-commit==4.0.1
     # via -r requirements/requirements.in
-prompt-toolkit==3.0.43
+prompt-toolkit==3.0.48
     # via click-repl
-psycopg2-binary==2.9.9
+psycopg2-binary==2.9.10
     # via -r requirements/requirements.in
+pyarrow==18.0.0
+    # via digital-land
+pycparser==2.22
+    # via cffi
 pycurl==7.45.3
     # via
     #   celery
@@ -168,31 +200,34 @@ pydantic==2.7.0
     #   digital-land
 pydantic-core==2.18.1
     # via pydantic
-pyparsing==3.1.2
+pygit2==1.16.0
+    # via digital-land
+pyparsing==3.2.0
     # via rdflib
-pyproj==3.6.1
+pyproj==3.7.0
     # via digital-land
 python-dateutil==2.9.0.post0
     # via
     #   botocore
     #   celery
     #   pandas
-python-multipart==0.0.9
+python-multipart==0.0.17
     # via asgi-csrf
-pytz==2024.1
+pytz==2024.2
     # via pandas
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via
+    #   dask
     #   datasette
     #   digital-land
     #   pre-commit
-rdflib==7.0.0
+rdflib==7.1.1
     # via sparqlwrapper
-requests==2.31.0
+requests==2.32.3
     # via
     #   digital-land
     #   esridump
-s3transfer==0.10.1
+s3transfer==0.10.3
     # via boto3
 sentry-sdk[celery]==2.2.1
     # via -r requirements/requirements.in
@@ -203,7 +238,6 @@ shortuuid==1.0.13
 six==1.16.0
     # via
     #   esridump
-    #   isodate
     #   python-dateutil
 sniffio==1.3.1
     # via
@@ -213,48 +247,55 @@ sparqlwrapper==2.0.0
     # via digital-land
 spatialite==0.0.3
     # via digital-land
-sqlalchemy==2.0.29
+sqlalchemy==2.0.36
     # via -r requirements/requirements.in
-typing-extensions==4.11.0
+toolz==1.0.0
+    # via
+    #   dask
+    #   partd
+typing-extensions==4.12.2
     # via
     #   anyio
     #   asgiref
-    #   janus
-    #   kombu
-    #   pint
+    #   datasette
+    #   flexcache
+    #   flexparser
     #   pydantic
     #   pydantic-core
     #   sqlalchemy
     #   typing-inspect
     #   uvicorn
 typing-inspect==0.9.0
     # via dataclasses-json
-tzdata==2024.1
+tzdata==2024.2
     # via
     #   celery
+    #   kombu
     #   pandas
-urllib3==1.26.18
+urllib3==2.2.3
     # via
     #   botocore
     #   celery
     #   kombu
     #   requests
     #   sentry-sdk
-uvicorn==0.29.0
+uvicorn==0.32.0
     # via datasette
-validators==0.28.1
+validators==0.34.0
     # via digital-land
 vine==5.1.0
     # via
     #   amqp
     #   celery
     #   kombu
-virtualenv==20.25.3
+virtualenv==20.27.1
     # via pre-commit
 wcwidth==0.2.13
     # via prompt-toolkit
 xlrd==1.2.0
     # via digital-land
+zipp==3.21.0
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

request-processor/src/tasks.py

@@ -68,7 +68,8 @@ def check_datafile(request: Dict, directories=None):
                 fileName = utils.save_content(content, tmp_dir)
             else:
                 save_response_to_db(request_schema.id, log)
-                raise CustomException(log)
+                logger.warning(f"URL check failed: {log}")
+                return
         response = workflow.run_workflow(
             fileName,
             request_schema.id,