Updating Docker image to use new AL2 image, Gunicorn, and multi-stage build

.github/workflows/integration_tests.yaml

@@ -0,0 +1,25 @@
+name: Integration Tests
+
+on: pull_request
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  integration_tests:
+    name: Integration tests
+    uses: uc-cdis/.github/.github/workflows/integration_tests.yaml@master
+    with:
+      SERVICE_TO_TEST: manifestservice
+    secrets:
+      CI_AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
+      CI_AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}
+      JENKINS_API_TOKEN: ${{ secrets.JENKINS_API_TOKEN }}
+      QA_DASHBOARD_S3_PATH: ${{ secrets.QA_DASHBOARD_S3_PATH }}
+      CI_TEST_ORCID_USERID: ${{ secrets.CI_TEST_ORCID_USERID }}
+      CI_TEST_ORCID_PASSWORD: ${{ secrets.CI_TEST_ORCID_PASSWORD }}
+      CI_TEST_RAS_USERID: ${{ secrets.CI_TEST_RAS_USERID }}
+      CI_TEST_RAS_PASSWORD: ${{ secrets.CI_TEST_RAS_PASSWORD }}
+      CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
+      CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }}

Dockerfile

@@ -1,52 +1,67 @@
-# To run: docker run -v /path/to/wsgi.py:/var/www/manifestservice/wsgi.py --name=manifestservice -p 81:80 manifestservice
-# To check running container: docker exec -it manifestservice /bin/bash
+ARG AZLINUX_BASE_VERSION=master
 
-FROM quay.io/cdis/python:python3.9-buster-2.0.0
+# Base stage with python-build-base
+FROM quay.io/cdis/python-build-base:${AZLINUX_BASE_VERSION} as base
+
+# Comment this in, and comment out the line above, if quay is down
+# FROM 707767160287.dkr.ecr.us-east-1.amazonaws.com/gen3/python-build-base:${AZLINUX_BASE_VERSION} as base
 
 ENV appname=manifestservice
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1
+
+WORKDIR /${appname}
+
+# create gen3 user
+# Create a group 'gen3' with GID 1000 and a user 'gen3' with UID 1000
+RUN groupadd -g 1000 gen3 && \
+    useradd -m -s /bin/bash -u 1000 -g gen3 gen3  && \
+    chown -R gen3:gen3 /$appname && \
+    chown -R gen3:gen3 /venv
+
+
+# Builder stage
+FROM base as builder
+
+USER gen3
+
+
+RUN python -m venv /venv
+
+COPY poetry.lock pyproject.toml /${appname}/
 
-RUN pip install --upgrade pip
+RUN pip install poetry && \
+    poetry install -vv --only main --no-interaction
 
-# install poetry
-RUN pip install --upgrade poetry
+COPY --chown=gen3:gen3 . /$appname
+COPY --chown=gen3:gen3 ./deployment/wsgi/wsgi.py /$appname/wsgi.py
 
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends\
-    libmcrypt4 libmhash2 mcrypt \
-    curl bash git vim \
-    && apt-get clean
+# Run poetry again so this app itself gets installed too
+RUN poetry install --without dev --no-interaction
 
-RUN mkdir -p /var/www/$appname \
-    && mkdir -p /var/www/.cache/Python-Eggs/ \
-    && mkdir /run/nginx/ \
-    && ln -sf /dev/stdout /var/log/nginx/access.log \
-    && ln -sf /dev/stderr /var/log/nginx/error.log \
-    && chown nginx -R /var/www/.cache/Python-Eggs/ \
-    && chown nginx /var/www/$appname
+RUN git config --global --add safe.directory /${appname} && COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" > /$appname/version_data.py \
+    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >> /$appname/version_data.py
 
-EXPOSE 80
+# Final stage
+FROM base
 
-WORKDIR /$appname
+COPY --from=builder /venv /venv
+COPY --from=builder /$appname /$appname
 
-# copy ONLY poetry artifact and install
-# this will make sure than the dependencies is cached
-COPY poetry.lock pyproject.toml /$appname/
-RUN poetry config virtualenvs.create false \
-    && poetry install -vv --no-root --no-dev --no-interaction \
-    && poetry show -v
 
-COPY . /$appname
-COPY ./deployment/uwsgi/uwsgi.ini /etc/uwsgi/uwsgi.ini
-COPY ./deployment/uwsgi/wsgi.py /$appname/wsgi.py
+# Switch to non-root user 'gen3' for the serving process
+USER gen3
 
-# install Indexd and dependencies via poetry
-RUN poetry config virtualenvs.create false \
-    && poetry install -vv --no-dev --no-interaction \
-    && poetry show -v
+RUN source /venv/bin/activate
 
-RUN COMMIT=`git rev-parse HEAD` && echo "COMMIT=\"${COMMIT}\"" >$appname/version_data.py \
-    && VERSION=`git describe --always --tags` && echo "VERSION=\"${VERSION}\"" >>$appname/version_data.py
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONIOENCODING=UTF-8
 
-WORKDIR /var/www/$appname
+CMD ["gunicorn", "-c", "deployment/wsgi/gunicorn.conf.py"]
 
-CMD /dockerrun.sh
+# RUN apt-get update \
+#     && apt-get install -y --no-install-recommends\
+#     libmcrypt4 libmhash2 mcrypt \
+#     curl bash git vim \
+#     && apt-get clean

README.md

@@ -1,4 +1,5 @@
 # Manifest Service
+
 ### Overview
 This service handles reading from and writing to a user's s3 folder containing their manifests. A manifest is a JSON file that lists records a researcher may be interested in analyzing. This service stores a manifest to a user folder in an s3 bucket and delivers it for later use, such as when the researcher wants to mount the manifest in their workspace. If the "prefix" config variable is set, user folders will be stored in a directory of that name within the s3 bucket.
 
@@ -48,10 +49,29 @@ Create a cohort GUID in the user's folder:
     Post body: { "guid": "5183a350-9d56-4084-8a03-6471cafeb7fe" }
     Returns: { "filename" : "5183a350-9d56-4084-8a03-6471cafeb7fe" }
 
+Lists a user's exported metadata objects:
+
+    GET /metadata
+    Returns: { "metadata" : [ { "filename" : "metadata-2024-06-13T17-14-46.026593.json", "last_modified" : "2024-06-13 17:14:47" }, ... ] }
+
+Create an exported metadata object in the user's folder:
+
+    POST /metadata
+    Post body: { "some_metadata_key": "some_metadata_value" }
+    Returns: { "filename" : "metadata-2024-06-13T17-14-46.026593.json" }
+
+Read the contents of an exported metadata object file in the user's folder:
+
+    GET /metadata/<filename.json>
+    Returns: { "body" : "the-body-of-the-exported-metadata-object-file-as-a-string" }
+
 On failure, the above endpoints all return JSON in the form
 
     { "error" : "error-message" }
 
+### OpenAPI spec
+
+The [OpenAPI](https://github.com/OAI/OpenAPI-Specification)/[Swagger 2.0](https://swagger.io/) specification of a service is stored in its `swagger.yaml` and can be visualized [here](http://petstore.swagger.io/?url=https://raw.githubusercontent.com/uc-cdis/manifestservice/master/openapi/swagger.yaml).
 
 ### Running the service locally
 If you want to run this service locally, fill out the config.json file with the correct values and then run:

deployment/wsgi/gunicorn.conf.py

@@ -0,0 +1,6 @@
+wsgi_app = "deployment.wsgi.wsgi:application"
+bind = "0.0.0.0:8000"
+workers = 1
+user = "gen3"
+group = "gen3"
+timeout = 300