[ENH] Updating resen-base and resen-core

resen-base/Dockerfile

@@ -1,4 +1,6 @@
-FROM ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
+# based on jupyter:dockerstacks/base-notebook
+# https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/Dockerfile
+FROM ubuntu:focal-20200703@sha256:d5a6519d9f048100123c568eb83f7ef5bfcad69b01424f420f17c932b00dea76
 
 LABEL maintainer="Ingeo Team <ingeo-team@ingeo.datatransport.org>"
 LABEL description="The base docker image from which resen-core images are built."
@@ -16,16 +18,16 @@ ARG NB_GID="100"
 USER root
 
 ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -yq dist-upgrade && \
-    apt-get install -yq --no-install-recommends \
+RUN apt-get update \
+ && apt-get install -yq --no-install-recommends \
     wget \
     bzip2 \
     ca-certificates \
     sudo \
     locales \
     fonts-liberation \
-    run-one && \
-    rm -rf /var/lib/apt/lists/*
+    run-one \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # install tini
 ARG TINI_VERSION="v0.18.0"
@@ -38,8 +40,8 @@ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
     locale-gen
 
 # Add a script that we will use to correct permissions after running certain commands
-ADD resources/fix-permissions /usr/local/bin/fix-permissions
-RUN chmod +x /usr/local/bin/fix-permissions
+COPY resources/fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
 
 # Enable prompt color in the skeleton .bashrc before creating the default NB_USER
 RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc
@@ -61,15 +63,21 @@ CMD ["start-notebook.sh"]
 COPY resources/02-resen.sh /etc/profile.d/
 
 # Add scripts for starting servers
+# these came from github jupyter:dockerstacks/base-notebook
 COPY resources/start.sh /usr/local/bin/
 COPY resources/start-notebook.sh /usr/local/bin/
 COPY resources/start-singleuser.sh /usr/local/bin/
 RUN chmod +x /usr/local/bin/start.sh && \
     chmod +x /usr/local/bin/start-notebook.sh && \
     chmod +x /usr/local/bin/start-singleuser.sh
 
+# might want to add this in?
+#COPY jupyter_notebook_config.py /etc/jupyter/
+#RUN fix-permissions /etc/jupyter/
 
-#stuff from jupyter/scipy-notebook
+
+#stuff from jupyter/scipy-notebook (we don't use conda)
+#https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/scipy-notebook/Dockerfile
 RUN apt-get update && apt-get install -yq --no-install-recommends \
     build-essential \
     git \
@@ -81,15 +89,15 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \
     netcat \
     pandoc \
     python-dev \
-    texlive-fonts-extra \
-    texlive-fonts-recommended \
-    texlive-generic-recommended \
-    texlive-latex-base \
-    texlive-latex-extra \
+    # ---- nbconvert dependencies ----
     texlive-xetex \
+    texlive-fonts-recommended \
+    texlive-plain-generic \
+    # ----
     tzdata \
     unzip && \
-    rm -rf /var/lib/apt/lists/*
+    apt-get install -y --no-install-recommends ffmpeg dvipng \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 USER $NB_USER
 WORKDIR /home/$NB_USER

resen-base/resources/02-resen.sh

@@ -1,2 +1 @@
-export SHELL=/bin/bash
 export LANG=C.UTF-8

resen-base/resources/start-notebook.sh

@@ -4,11 +4,16 @@
 
 set -e
 
+wrapper=""
+if [[ "${RESTARTABLE}" == "yes" ]]; then
+    wrapper="run-one-constantly"
+fi
+
 if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then
-  # launched by JupyterHub, use single-user entrypoint
-  exec /usr/local/bin/start-singleuser.sh "$@"
+    # launched by JupyterHub, use single-user entrypoint
+    exec /usr/local/bin/start-singleuser.sh "$@"
 elif [[ ! -z "${JUPYTER_ENABLE_LAB}" ]]; then
-  . /usr/local/bin/start.sh jupyter lab "$@"
+    . /usr/local/bin/start.sh $wrapper jupyter lab "$@"
 else
-  . /usr/local/bin/start.sh jupyter notebook "$@"
+    . /usr/local/bin/start.sh $wrapper jupyter notebook "$@"
 fi

resen-base/resources/start-singleuser.sh

@@ -4,42 +4,39 @@
 
 set -e
 
-source /home/jovyan/envs/py36/bin/activate
+# modified 9 Sept 2020 from https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/start-singleuser.sh
+source /home/jovyan/envs/py38/bin/activate
 
 # set default ip to 0.0.0.0
 if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then
-  NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS"
+    NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS"
 fi
 
 # handle some deprecated environment variables
 # from DockerSpawner < 0.8.
 # These won't be passed from DockerSpawner 0.9,
 # so avoid specifying --arg=empty-string
 if [ ! -z "$NOTEBOOK_DIR" ]; then
-  NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS"
+    NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS"
 fi
 if [ ! -z "$JPY_PORT" ]; then
-  NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS"
+    NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS"
 fi
 if [ ! -z "$JPY_USER" ]; then
-  NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS"
+    NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS"
 fi
 if [ ! -z "$JPY_COOKIE_NAME" ]; then
-  NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS"
+    NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS"
 fi
 if [ ! -z "$JPY_BASE_URL" ]; then
-  NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS"
+    NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS"
 fi
 if [ ! -z "$JPY_HUB_PREFIX" ]; then
-  NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS"
+    NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS"
 fi
 if [ ! -z "$JPY_HUB_API_URL" ]; then
-  NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS"
-fi
-if [ ! -z "$JUPYTER_ENABLE_LAB" ]; then
-  NOTEBOOK_BIN="jupyter labhub"
-else
-  NOTEBOOK_BIN="jupyterhub-singleuser"
+    NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS"
 fi
+NOTEBOOK_BIN="jupyterhub-singleuser"
 
 . /usr/local/bin/start.sh $NOTEBOOK_BIN $NOTEBOOK_ARGS "$@"

resen-base/resources/start.sh

@@ -66,7 +66,7 @@ if [ $(id -u) == 0 ] ; then
         # (it could be mounted, and we shouldn't create it if it already exists)
         if [[ ! -e "/home/$NB_USER" ]]; then
             echo "Relocating home dir to /home/$NB_USER"
-            mv /home/jovyan "/home/$NB_USER"
+            mv /home/jovyan "/home/$NB_USER" || ln -s /home/jovyan "/home/$NB_USER"
         fi
         # if workdir is in /home/jovyan, cd to /home/$NB_USER
         if [[ "$PWD/" == "/home/jovyan/"* ]]; then
@@ -76,37 +76,35 @@ if [ $(id -u) == 0 ] ; then
         fi
     fi
 
-    # Change UID of NB_USER to NB_UID if it does not match
-    if [ "$NB_UID" != $(id -u $NB_USER) ] ; then
-        echo "Set $NB_USER UID to: $NB_UID"
-        usermod -u $NB_UID $NB_USER
-    fi
-
-    # Set NB_USER primary gid to NB_GID (after making the group).  Set
-    # supplementary gids to NB_GID and 100.
-    if [ "$NB_GID" != $(id -g $NB_USER) ] ; then
-        echo "Add $NB_USER to group: $NB_GID"
-        groupadd -g $NB_GID -o ${NB_GROUP:-${NB_USER}}
-        usermod  -g $NB_GID -aG 100 $NB_USER
+    # Change UID:GID of NB_USER to NB_UID:NB_GID if it does not match
+    if [ "$NB_UID" != $(id -u $NB_USER) ] || [ "$NB_GID" != $(id -g $NB_USER) ]; then
+        echo "Set user $NB_USER UID:GID to: $NB_UID:$NB_GID"
+        if [ "$NB_GID" != $(id -g $NB_USER) ]; then
+            groupadd -f -g $NB_GID -o ${NB_GROUP:-${NB_USER}}
+        fi
+        userdel $NB_USER
+        useradd --home /home/$NB_USER -u $NB_UID -g $NB_GID -G 100 -l $NB_USER
     fi
 
     # Enable sudo if requested
     if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then
-        # echo "Granting $NB_USER sudo access and appending $CONDA_DIR/bin to sudo PATH"
         echo "Granting $NB_USER sudo access"
+        # and appending $CONDA_DIR/bin to sudo PATH"  # see comment below 
         echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook
     fi
 
-    # Add $CONDA_DIR/bin to sudo secure_path (not needed for resen-core)
-    #sed -r "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path
+    # Add $CONDA_DIR/bin to sudo secure_path
+    # removed 9 Sept 2020 from https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/start.sh
+    # we don't use conda in our docker image
+    # sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path
 
     # Exec the command as NB_USER with the PATH and the rest of
     # the environment preserved
     run-hooks /usr/local/bin/before-notebook.d
     echo "Executing the command: ${cmd[@]}"
     exec sudo -E -H -u $NB_USER PATH=$PATH XDG_CACHE_HOME=/home/$NB_USER/.cache PYTHONPATH=${PYTHONPATH:-} "${cmd[@]}"
 else
-    if [[ "$NB_UID" == "$(id -u jovyan)" && "$NB_GID" == "$(id -g jovyan)" ]]; then
+    if [[ "$NB_UID" == "$(id -u jovyan 2>/dev/null)" && "$NB_GID" == "$(id -g jovyan 2>/dev/null)" ]]; then
         # User is not attempting to override user/group via environment
         # variables, but they could still have overridden the uid/gid that
         # container runs as. Check that the user has an entry in the passwd
@@ -149,4 +147,4 @@ else
     run-hooks /usr/local/bin/before-notebook.d
     echo "Executing the command: ${cmd[@]}"
     exec "${cmd[@]}"
-fi
+fi

resen-core/Dockerfile

@@ -1,4 +1,5 @@
-FROM earthcubeingeo/resen-base:2020.1.0
+#FROM earthcubeingeo/resen-base:2020.2.0
+FROM resenbasetest:latest
 
 LABEL maintainer="Ingeo Team <ingeo-team@ingeo.datatransport.org>"
 LABEL description="The core docker image used by resen."
@@ -27,33 +28,28 @@ RUN apt-get update && \
 # Install python stuff
 RUN apt-get update && \
     apt-get install -yq --no-install-recommends \
-    python2.7 python3.6 \
-    python-virtualenv python3-virtualenv \
-    python-setuptools python3-setuptools \
-    python-dev python3-dev \
+    python3-virtualenv \
+    python3-setuptools \
+    python3-dev \
+    python3.8 \
     libproj-dev proj-data proj-bin \
     libgeos++-dev libgeos-dev \
     libdb-dev \
-    python-tk python3-tk && \
+    python3-tk && \
     rm -rf /var/lib/apt/lists/*
 
 # Install system libraries needed by python packages
 RUN apt-get update && \
     apt-get install -yq --no-install-recommends \
-    libncurses5-dev && \
+    libncurses5-dev \
+    cm-super \
+    npm nodejs && \
     rm -rf /var/lib/apt/lists/*
 
-# Install nodejs and npm needed for matplotlib widgets
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \
-    apt-get install -yq --no-install-recommends nodejs && \
-    rm -rf /var/lib/apt/lists/*
-
-# Create python 2.7 and python 3.6 virtual environments
+# Create python 3.8 virtual environment
 USER $NB_USER
 RUN /bin/bash -c 'mkdir -p /home/$NB_USER/envs && \
-        python3 -m virtualenv -p /usr/bin/python3.6 /home/$NB_USER/envs/py36 && \
-        python3 -m virtualenv -p /usr/bin/python2.7 /home/$NB_USER/envs/py27'
-
+        python3 -m virtualenv -p /usr/bin/python3.8 /home/$NB_USER/envs/py38'
 
 WORKDIR /home/$NB_USER
 
@@ -67,23 +63,17 @@ RUN /bin/bash -c 'bash install_CDF.sh && rm install_CDF.sh'
 # Now install all python packages
 USER $NB_USER
 
-COPY resources/helpers/setup_py36_env.sh .
-RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && bash setup_py36_env.sh && rm setup_py36_env.sh'
-
-COPY resources/helpers/setup_py27_env.sh .
-RUN /bin/bash -cl 'source /home/$NB_USER/envs/py27/bin/activate && bash setup_py27_env.sh && rm setup_py27_env.sh'
+COPY resources/helpers/setup_py38_env.sh .
+RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && bash setup_py38_env.sh && rm setup_py38_env.sh'
 
 # register python environments with ipykernel
-RUN /bin/bash -c 'source /home/$NB_USER/envs/py27/bin/activate && \
-                  pip install ipykernel==4.10.0 && \
-                  python -m ipykernel install --user --name py27 --display-name "py27"'
 
-RUN /bin/bash -c 'source /home/$NB_USER/envs/py36/bin/activate && \
+RUN /bin/bash -c 'source /home/$NB_USER/envs/py38/bin/activate && \
                   pip install ipykernel==5.1.1 && \
-                  python -m ipykernel install --user --name py36 --display-name "py36"'
+                  python -m ipykernel install --user --name py38 --display-name "py38"'
 
 # remove python3 kernel from showing up in jupyter
-RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \
+RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \
                    yes | jupyter kernelspec remove python3'
 
 # Install basemap
@@ -95,35 +85,39 @@ COPY resources/helpers/setup_pyglow.sh .
 RUN /bin/bash -cl 'bash setup_pyglow.sh && rm setup_pyglow.sh'
 
 # Install citationhelper
+USER root
 COPY resources/pkg_citations.json /home/$NB_USER/cache/
+RUN chown -R $NB_USER:users "/home/$NB_USER/cache/"
+USER $NB_USER
 
 # create work directory for user
 RUN mkdir /home/$NB_USER/work
 
-# set default python environment to py36
-RUN /bin/bash -c 'echo "source /home/$NB_USER/envs/py36/bin/activate" >> /home/$NB_USER/.bashrc'
+# set default python environment to py38
+RUN /bin/bash -c 'echo "source /home/$NB_USER/envs/py38/bin/activate" >> /home/$NB_USER/.bashrc'
 
 # post installation stuff
 # Download 110m scale cartopy data
 COPY resources/feature_download.py .
-RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \
+RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \
                    python feature_download.py physical cultural cultural-extra --do_scales 110m && \
                    rm feature_download.py'
 
-# Installing jupyter-widgets
-RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \
+# Installing jupyter-widgets with interaction with matplotlib and bokeh
+RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \
                    jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
-                   jupyter labextension install jupyter-matplotlib'
+                   jupyter labextension install jupyter-matplotlib && \
+                   jupyter labextension install @bokeh/jupyter_bokeh'
 
 # Finally set up some stuff to make user experience better
 RUN echo "TERM=xterm-256color" >> /home/$NB_USER/.bashrc && \
     echo "MPLBACKEND=Agg" >> /home/$NB_USER/.bashrc && \
     echo "export CITEHELP_REFFILE=/home/$NB_USER/cache/pkg_citations.json" >> /home/$NB_USER/.bashrc
 
 # Set default jupyter terminal shell
-RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \
+RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \
                    jupyter lab --generate-config'
-RUN sed -i -e 's/#c.NotebookApp.terminado_settings = {}/c.NotebookApp.terminado_settings = {\"shell_command\"\:[\"\/bin\/bash\"]}/' /home/jovyan/.jupyter/jupyter_notebook_config.py
+RUN sed -i -e 's/# c.NotebookApp.terminado_settings = {}/c.NotebookApp.terminado_settings = {\"shell_command\"\:[\"\/bin\/bash\"]}/' /home/jovyan/.jupyter/jupyter_notebook_config.py
 
 # Set the jupyterlab user-settings
 RUN /bin/bash -cl 'mkdir -p /home/jovyan/.jupyter/lab/user-settings/@jupyterlab/extensionmanager-extension && \

resen-core/resources/helpers/setup_basemap.sh

@@ -3,26 +3,21 @@
 #
 #    A helper script for installing basemap
 #
-#    Assumes you have py27 and py36 python virtual environments
+#    Assumes you have py38 python virtual environments
 #
 #######################################################################################
 
 # git clone --depth 0 https://github.com/matplotlib/basemap.git # VERY SLOW
-wget https://github.com/matplotlib/basemap/archive/v1.2.0rel.tar.gz
-tar -zxf v1.2.0rel.tar.gz
-cd basemap-1.2.0rel
+wget https://github.com/matplotlib/basemap/archive/v1.2.2rel.tar.gz
+tar -zxf v1.2.2rel.tar.gz
+cd basemap-1.2.2rel
 
-source ${HOME}/envs/py27/bin/activate
-pip install .
-
-deactivate
-
-source ${HOME}/envs/py36/bin/activate
+source ${HOME}/envs/py38/bin/activate
 pip install .
 
 deactivate
 
 # cleanup
 cd ..
-rm -rf basemap-1.2.0rel v1.2.0rel.tar.gz
+rm -rf basemap-1.2.2rel v1.2.2rel.tar.gz
 rm ${HOME}/.wget-hsts