feat(python): add 3.9 dockerfile and GH action for building 3.9/10 ba… (

#1861) * feat(python): add 3.9 dockerfile and GH action for building 3.9/10 base images * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml * Update image_build_push.yaml Co-authored-by: Alexander VT <alexander.m.vantol@gmail.com>
uc-cdis · Mar 14, 2022 · ab0b1d2 · ab0b1d2
1 parent 1a23751
commit ab0b1d2
Show file tree

Hide file tree

Showing 14 changed files with 538 additions and 1 deletion.
diff --git a/.github/workflows/image_build_push.yaml b/.github/workflows/image_build_push.yaml
@@ -0,0 +1,31 @@
+name: Build Python Base Images and Push to Quay and ECR
+
+on: push
+
+jobs:
+  python_3-9:
+    name: Python 3.9 Build and Push
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.9-buster/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.9-buster"
+      OVERRIDE_REPO_NAME: "python"
+      OVERRIDE_TAG_NAME: "python3.9-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  python_3-10:
+    name: Python 3.10 Build and Push
+    uses: uc-cdis/.github/.github/workflows/image_build_push.yaml@master
+    with:
+      DOCKERFILE_LOCATION: "./Docker/python-nginx/python3.10-buster/Dockerfile"
+      DOCKERFILE_BUILD_CONTEXT: "./Docker/python-nginx/python3.10-buster"
+      OVERRIDE_REPO_NAME: "python"
+      OVERRIDE_TAG_NAME: "python3.10-buster-$(echo ${GITHUB_REF#refs/*/} | tr / _)"
+    secrets:
+      ECR_AWS_ACCESS_KEY_ID: ${{ secrets.ECR_AWS_ACCESS_KEY_ID }}
+      ECR_AWS_SECRET_ACCESS_KEY: ${{ secrets.ECR_AWS_SECRET_ACCESS_KEY }}
+      QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }}
+      QUAY_ROBOT_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
diff --git a/Docker/python-nginx/README.md b/Docker/python-nginx/README.md
@@ -4,4 +4,6 @@ The `python3.6-alpine3.7` image is at https://quay.io/repository/cdis/python-ngi
 
 The `python3.6-buster/Dockerfile` triggers an image build here: https://quay.io/repository/cdis/python and builds image with naming convention: `python3.6-buster-{{branch}}`
 
+The `python3.9-buster/Dockerfile` triggers an image build here: https://quay.io/repository/cdis/python and builds image with naming convention: `python3.9-buster-{{branch}}`
+
 The `python3.10-buster/Dockerfile` triggers an image build here: https://quay.io/repository/cdis/python and builds image with naming convention: `python3.10-buster-{{branch}}`
diff --git a/Docker/python-nginx/python3.10-buster/app/main.py b/Docker/python-nginx/python3.10-buster/app/main.py
@@ -1,4 +1,4 @@
 def application(env, start_response):
     start_response('200 OK', [('Content-Type', 'text/html')])
-    return [b"Hello World from a default Nginx uWSGI Python 3.6 app in a\
+    return [b"Hello World from a default Nginx uWSGI Python 3.10 app in a\
             Docker container (default)"]
diff --git a/Docker/python-nginx/python3.9-buster/Dockerfile b/Docker/python-nginx/python3.9-buster/Dockerfile
@@ -0,0 +1,168 @@
+# https://github.com/tiangolo/uwsgi-nginx-docker/blob/master/docker-images/python3.9.dockerfile
+FROM quay.io/cdis/python:3.9-buster
+
+# https://github.com/nginxinc/docker-nginx/blob/f958fbacada447737319e979db45a1da49123142/mainline/debian/Dockerfile
+ENV NGINX_VERSION   1.21.1
+ENV NJS_VERSION     0.6.1
+ENV PKG_RELEASE     1~buster
+
+RUN set -x \
+# create nginx user/group first, to be consistent throughout docker variants
+    && addgroup --system --gid 102 nginx \
+    && adduser --system --disabled-login --ingroup nginx --no-create-home --home /nonexistent --gecos "nginx user" --shell /bin/false --uid 102 nginx \
+# also add nginx user to gid 101 (ssh) groups
+    && adduser nginx ssh \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y gnupg1 ca-certificates \
+    && \
+    NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \
+    found=''; \
+    for server in \
+        ha.pool.sks-keyservers.net \
+        hkp://keyserver.ubuntu.com:80 \
+        hkp://p80.pool.sks-keyservers.net:80 \
+        pgp.mit.edu \
+    ; do \
+        echo "Fetching GPG key $NGINX_GPGKEY from $server"; \
+        apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \
+    done; \
+    test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \
+    apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \
+    && dpkgArch="$(dpkg --print-architecture)" \
+    && nginxPackages=" \
+        nginx=${NGINX_VERSION}-${PKG_RELEASE} \
+        nginx-module-xslt=${NGINX_VERSION}-${PKG_RELEASE} \
+        nginx-module-geoip=${NGINX_VERSION}-${PKG_RELEASE} \
+        nginx-module-image-filter=${NGINX_VERSION}-${PKG_RELEASE} \
+        nginx-module-njs=${NGINX_VERSION}+${NJS_VERSION}-${PKG_RELEASE} \
+    " \
+    && case "$dpkgArch" in \
+        amd64|i386|arm64) \
+# arches officialy built by upstream
+            echo "deb https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \
+            && apt-get update \
+            ;; \
+        *) \
+# we're on an architecture upstream doesn't officially build for
+# let's build binaries from the published source packages
+            echo "deb-src https://nginx.org/packages/mainline/debian/ buster nginx" >> /etc/apt/sources.list.d/nginx.list \
+            \
+# new directory for storing sources and .deb files
+            && tempDir="$(mktemp -d)" \
+            && chmod 777 "$tempDir" \
+# (777 to ensure APT's "_apt" user can access it too)
+            \
+# save list of currently-installed packages so build dependencies can be cleanly removed later
+            && savedAptMark="$(apt-mark showmanual)" \
+            \
+# build .deb files from upstream's source packages (which are verified by apt-get)
+            && apt-get update \
+            && apt-get build-dep -y $nginxPackages \
+            && ( \
+                cd "$tempDir" \
+                && DEB_BUILD_OPTIONS="nocheck parallel=$(nproc)" \
+                    apt-get source --compile $nginxPackages \
+            ) \
+# we don't remove APT lists here because they get re-downloaded and removed later
+            \
+# reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies
+# (which is done after we install the built packages so we don't have to redownload any overlapping dependencies)
+            && apt-mark showmanual | xargs apt-mark auto > /dev/null \
+            && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; } \
+            \
+# create a temporary local APT repo to install from (so that dependency resolution can be handled by APT, as it should be)
+            && ls -lAFh "$tempDir" \
+            && ( cd "$tempDir" && dpkg-scanpackages . > Packages ) \
+            && grep '^Package: ' "$tempDir/Packages" \
+            && echo "deb [ trusted=yes ] file://$tempDir ./" > /etc/apt/sources.list.d/temp.list \
+# work around the following APT issue by using "Acquire::GzipIndexes=false" (overriding "/etc/apt/apt.conf.d/docker-gzip-indexes")
+#   Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
+#   ...
+#   E: Failed to fetch store:/var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages  Could not open file /var/lib/apt/lists/partial/_tmp_tmp.ODWljpQfkE_._Packages - open (13: Permission denied)
+            && apt-get -o Acquire::GzipIndexes=false update \
+            ;; \
+    esac \
+    \
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+                        $nginxPackages \
+                        gettext-base \
+                        curl \
+    && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx.list \
+    \
+# if we have leftovers from building, let's purge them (including extra, unnecessary build deps)
+    && if [ -n "$tempDir" ]; then \
+        apt-get purge -y --auto-remove \
+        && rm -rf "$tempDir" /etc/apt/sources.list.d/temp.list; \
+    fi \
+# forward request and error logs to docker log collector
+    && ln -sf /dev/stdout /var/log/nginx/access.log \
+    && ln -sf /dev/stderr /var/log/nginx/error.log \
+# create a docker-entrypoint.d directory
+    && mkdir /docker-entrypoint.d
+
+EXPOSE 80
+
+# # Expose 443, in case of LTS / HTTPS
+EXPOSE 443
+
+# install uwsgi
+# https://uwsgi-docs.readthedocs.io/en/latest/Install.html
+RUN python -m pip install --upgrade pip
+RUN pip install uwsgi
+
+# Remove default configuration from Nginx
+RUN rm /etc/nginx/conf.d/default.conf
+# Copy the base uWSGI ini file to enable default dynamic uwsgi process number
+COPY uwsgi.ini /etc/uwsgi/
+
+COPY uwsgi.conf /etc/nginx/sites-available/
+
+RUN ln -s /etc/nginx/sites-available/uwsgi.conf /etc/nginx/conf.d/uwsgi.conf
+
+# Install Supervisord
+RUN apt-get update && apt-get install -y supervisor \
+&& rm -rf /var/lib/apt/lists/*
+# Custom Supervisord config
+COPY supervisord.ini /etc/supervisor.d/supervisord.ini
+
+# Which uWSGI .ini file should be used, to make it customizable
+ENV UWSGI_INI /app/uwsgi.ini
+
+# By default, disable uwsgi cheaper mode and run 2 processes.
+# If UWSGI_CHEAPER=N and UWSGI_PROCESSES=M, N is the min and M is the max
+# number of processes. UWSGI_CHEAPER must be lower than UWSGI_PROCESSES.
+# We set them here instead of in uwsgi.ini so that they can be overwritten.
+ENV UWSGI_CHEAPER=
+ENV UWSGI_PROCESSES=2
+
+# By default, allow unlimited file sizes, modify it to limit the file sizes
+# To have a maximum of 1 MB (Nginx's default) change the line to:
+# ENV NGINX_MAX_UPLOAD 1m
+ENV NGINX_MAX_UPLOAD 0
+
+# By default, Nginx will run a single worker process, setting it to auto
+# will create a worker for each CPU core
+ENV NGINX_WORKER_PROCESSES 1
+
+# By default, Nginx listens on port 80.
+# To modify this, change LISTEN_PORT environment variable.
+# (in a Dockerfile or with an option for `docker run`)
+ENV LISTEN_PORT 80
+
+# Copy the entrypoint that will generate Nginx additional configs
+COPY entrypoint.sh /entrypoint.sh
+COPY logrotate-nginx.conf /etc/logrotate.d/nginx
+RUN chmod +x /entrypoint.sh
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+COPY dockerrun.sh /dockerrun.sh
+RUN mkdir -p /var/www/metrics/ && chmod +x /dockerrun.sh
+
+ENTRYPOINT ["sh", "/entrypoint.sh"]
+
+# Add demo app
+COPY ./app /app
+WORKDIR /app
+
+CMD ["/usr/bin/supervisord"]
diff --git a/Docker/python-nginx/python3.9-buster/app/main.py b/Docker/python-nginx/python3.9-buster/app/main.py
@@ -0,0 +1,4 @@
+def application(env, start_response):
+    start_response('200 OK', [('Content-Type', 'text/html')])
+    return [b"Hello World from a default Nginx uWSGI Python 3.9 app in a\
+            Docker container (default)"]
diff --git a/Docker/python-nginx/python3.9-buster/app/uwsgi.ini b/Docker/python-nginx/python3.9-buster/app/uwsgi.ini
@@ -0,0 +1,2 @@
+[uwsgi]
+wsgi-file=/app/main.py
diff --git a/Docker/python-nginx/python3.9-buster/dockerrun.sh b/Docker/python-nginx/python3.9-buster/dockerrun.sh
@@ -0,0 +1,102 @@
+#!/bin/sh
+#
+# Note: base alpine Linux image may not include bash shell,
+#    and we probably want to move to that for service images,
+#    so just use bourn shell ...
+
+#
+# Update certificate authority index -
+# environment may have mounted more authorities
+# - ex: /usr/local/share/ca-certificates/cdis-ca.crt into system bundle       
+#
+
+GEN3_DEBUG="${GEN3_DEBUG:-False}"
+GEN3_DRYRUN="${GEN3_DRYRUN:-False}"
+GEN3_UWSGI_TIMEOUT="${GEN3_UWSGI_TIMEOUT:-45s}"
+
+run() {
+  if [ "$GEN3_DRYRUN" = True ]; then
+    echo "DRY RUN - not running: $@"
+  else
+    echo "Running $@"
+    "$@"
+  fi
+}
+
+help() {
+    cat - <<EOM
+Gen3 base (generic) launch script
+Use: 
+  dockkerrun.bash [--help] [--debug=False] [--uwsgiTimeout=45s] [--dryrun=False]
+EOM
+}
+
+
+while [ $# -gt 0 ]; do
+  arg="$1"
+  shift
+  key=""
+  value=""
+  key="$(echo "$arg" | sed -e 's/^-*//' | sed -e 's/=.*$//')"
+  value="$(echo "$arg" | sed -e 's/^.*=//')"
+
+  if [ "$value" = "$arg" ]; then # =value not given, so use default
+    value=""
+  fi
+  case "$key" in
+  debug)
+    GEN3_DEBUG="${value:-True}"
+    ;;
+  uwsgiTimeout)
+    GEN3_UWSGI_TIMEOUT="${value:-45s}"
+    ;;
+  dryrun)
+    GEN3_DRYRUN="${value:-True}"
+    ;;
+  help)
+    help
+    exit 0
+    ;;
+  *)
+    echo "ERROR: unknown argument $arg - bailing out"
+    exit 1
+    ;;
+  esac
+done
+
+cat - <<EOM
+Got configuration:
+GEN3_DEBUG=$GEN3_DEBUG
+GEN3_UWSGI_TIMEOUT=$GEN3_UWSGI_TIMEOUT
+GEN3_DRYRUN=$GEN3_DRYRUN
+EOM
+
+run update-ca-certificates
+run mkdir -p /var/run/gen3
+
+# fill in timeout in the uwsgi.conf template
+if [ -f /etc/nginx/sites-available/uwsgi.conf ]; then
+  sed -i -e "s/GEN3_UWSGI_TIMEOUT/$GEN3_UWSGI_TIMEOUT/g" /etc/nginx/sites-available/uwsgi.conf
+fi
+
+#
+# Enable debug flag based on GEN3_DEBUG environment
+#
+if [ -f ./wsgi.py ] && [ "$GEN3_DEBUG" = "True" ]; then
+  printf "\napplication.debug=True\n\n" >> ./wsgi.py
+fi
+
+if [ -z $DD_ENABLED ]; then
+(
+  run uwsgi --ini /etc/uwsgi/uwsgi.ini
+) &
+else
+pip install ddtrace
+echo "import=ddtrace.bootstrap.sitecustomize" >> /etc/uwsgi/uwsgi.ini
+(
+  ddtrace-run uwsgi --enable-threads --ini /etc/uwsgi/uwsgi.ini
+) &
+fi
+
+run nginx -g 'daemon off;'
+wait
diff --git a/Docker/python-nginx/python3.9-buster/entrypoint.sh b/Docker/python-nginx/python3.9-buster/entrypoint.sh
@@ -0,0 +1,55 @@
+#!/usr/bin/env sh
+set -e
+
+rate_limit=""
+if [ ! -z $NGINX_RATE_LIMIT ]; then
+  echo "Found NGINX_RATE_LIMIT environment variable..."
+  if [ ! -z $OVERRIDE_NGINX_RATE_LIMIT ]; then
+    rate_limit=$OVERRIDE_NGINX_RATE_LIMIT
+    echo "Overriding Nginx rate limit with new value ${rate_limit}..."
+  else
+    rate_limit=$NGINX_RATE_LIMIT
+    echo "Applying Nginx rate limit from k8s deployment descriptor..."
+  fi
+
+  # Add rate_limit config
+  rate_limit_conf="\ \ \ \ limit_req_zone \$binary_remote_addr zone=one:10m rate=${rate_limit}r/s;"
+  sed -i "/http\ {/a ${rate_limit_conf}" /etc/nginx/nginx.conf
+  if [ -f /etc/nginx/sites-available/uwsgi.conf ]; then
+    limit_req_config="\ \ \ \ \ \ \ \ limit_req zone=one;"
+    sed -i "/location\ \/\ {/a ${limit_req_config}" /etc/nginx/sites-available/uwsgi.conf
+  fi
+fi
+
+# Get the maximum upload file size for Nginx, default to 0: unlimited
+USE_NGINX_MAX_UPLOAD=${NGINX_MAX_UPLOAD:-0}
+# Generate Nginx config for maximum upload file size
+echo "client_max_body_size $USE_NGINX_MAX_UPLOAD;" > /etc/nginx/conf.d/upload.conf
+
+# Explicitly add installed Python packages and uWSGI Python packages to PYTHONPATH
+# Otherwise uWSGI can't import Flask
+export PYTHONPATH=$PYTHONPATH:/usr/local/lib/python3.6/site-packages:/usr/lib/python3.6/site-packages
+
+# Get the number of workers for Nginx, default to 1
+USE_NGINX_WORKER_PROCESSES=${NGINX_WORKER_PROCESSES:-1}
+# Modify the number of worker processes in Nginx config
+sed -i "/worker_processes\s/c\worker_processes ${USE_NGINX_WORKER_PROCESSES};" /etc/nginx/nginx.conf
+
+# Set the max number of connections per worker for Nginx, if requested
+# Cannot exceed worker_rlimit_nofile, see NGINX_WORKER_OPEN_FILES below
+if [ -n "$NGINX_WORKER_CONNECTIONS" ] ; then
+    sed -i "/worker_connections\s/c\    worker_connections ${NGINX_WORKER_CONNECTIONS};" /etc/nginx/nginx.conf
+fi
+
+# Set the max number of open file descriptors for Nginx workers, if requested
+if [ -n "$NGINX_WORKER_OPEN_FILES" ] ; then
+    echo "worker_rlimit_nofile ${NGINX_WORKER_OPEN_FILES};" >> /etc/nginx/nginx.conf
+fi
+
+# Get the listen port for Nginx, default to 80
+USE_LISTEN_PORT=${LISTEN_PORT:-80}
+# Modify Nignx config for listen port
+if ! grep -q "listen ${USE_LISTEN_PORT};" /etc/nginx/nginx.conf ; then
+    sed -i -e "/server {/a\    listen ${USE_LISTEN_PORT};" /etc/nginx/nginx.conf
+fi
+exec "$@"
diff --git a/Docker/python-nginx/python3.9-buster/logrotate-nginx.conf b/Docker/python-nginx/python3.9-buster/logrotate-nginx.conf
@@ -0,0 +1,9 @@
+# nginx log rotation
+/var/log/nginx {
+    weekly
+    size 10M
+    postrotate
+        [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`
+    endscript
+    rotate 5
+}