From 5d5c363c68f1adc419346c72e0d87e15f2ec11e5 Mon Sep 17 00:00:00 2001 From: Ashton Reimer Date: Thu, 10 Sep 2020 22:30:42 -0700 Subject: [PATCH 1/9] Updated resen-base to ubuntu 20.04 and updated start scripts to track latest jupyter-stacks. --- resen-base/Dockerfile | 36 +++++++++++-------- resen-base/resources/02-resen.sh | 1 - resen-base/resources/start-notebook.sh | 13 ++++--- resen-base/resources/start-singleuser.sh | 23 ++++++------ resen-base/resources/start.sh | 34 +++++++++--------- .../{setup_py36_env.sh => setup_py38_env.sh} | 0 6 files changed, 57 insertions(+), 50 deletions(-) rename resen-core/resources/helpers/{setup_py36_env.sh => setup_py38_env.sh} (100%) diff --git a/resen-base/Dockerfile b/resen-base/Dockerfile index c963646..375c587 100644 --- a/resen-base/Dockerfile +++ b/resen-base/Dockerfile @@ -1,4 +1,6 @@ -FROM ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f +# based on jupyter:dockerstacks/base-notebook +# https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/Dockerfile +FROM ubuntu:focal-20200703@sha256:d5a6519d9f048100123c568eb83f7ef5bfcad69b01424f420f17c932b00dea76 LABEL maintainer="Ingeo Team " LABEL description="The base docker image from which resen-core images are built." @@ -16,16 +18,16 @@ ARG NB_GID="100" USER root ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get -yq dist-upgrade && \ - apt-get install -yq --no-install-recommends \ +RUN apt-get update \ + && apt-get install -yq --no-install-recommends \ wget \ bzip2 \ ca-certificates \ sudo \ locales \ fonts-liberation \ - run-one && \ - rm -rf /var/lib/apt/lists/* + run-one \ + && apt-get clean && rm -rf /var/lib/apt/lists/* # install tini ARG TINI_VERSION="v0.18.0" @@ -38,8 +40,8 @@ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen # Add a script that we will use to correct permissions after running certain commands -ADD resources/fix-permissions /usr/local/bin/fix-permissions -RUN chmod +x /usr/local/bin/fix-permissions +COPY resources/fix-permissions /usr/local/bin/fix-permissions +RUN chmod a+rx /usr/local/bin/fix-permissions # Enable prompt color in the skeleton .bashrc before creating the default NB_USER RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc @@ -61,6 +63,7 @@ CMD ["start-notebook.sh"] COPY resources/02-resen.sh /etc/profile.d/ # Add scripts for starting servers +# these came from github jupyter:dockerstacks/base-notebook COPY resources/start.sh /usr/local/bin/ COPY resources/start-notebook.sh /usr/local/bin/ COPY resources/start-singleuser.sh /usr/local/bin/ @@ -68,8 +71,13 @@ RUN chmod +x /usr/local/bin/start.sh && \ chmod +x /usr/local/bin/start-notebook.sh && \ chmod +x /usr/local/bin/start-singleuser.sh +# might want to add this in? +#COPY jupyter_notebook_config.py /etc/jupyter/ +#RUN fix-permissions /etc/jupyter/ -#stuff from jupyter/scipy-notebook + +#stuff from jupyter/scipy-notebook (we don't use conda) +#https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/scipy-notebook/Dockerfile RUN apt-get update && apt-get install -yq --no-install-recommends \ build-essential \ git \ @@ -81,15 +89,15 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ netcat \ pandoc \ python-dev \ - texlive-fonts-extra \ - texlive-fonts-recommended \ - texlive-generic-recommended \ - texlive-latex-base \ - texlive-latex-extra \ + # ---- nbconvert dependencies ---- texlive-xetex \ + texlive-fonts-recommended \ + texlive-plain-generic \ + # ---- tzdata \ unzip && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y --no-install-recommends ffmpeg dvipng \ + && apt-get clean && rm -rf /var/lib/apt/lists/* USER $NB_USER WORKDIR /home/$NB_USER diff --git a/resen-base/resources/02-resen.sh b/resen-base/resources/02-resen.sh index b92bd9c..de51944 100644 --- a/resen-base/resources/02-resen.sh +++ b/resen-base/resources/02-resen.sh @@ -1,2 +1 @@ -export SHELL=/bin/bash export LANG=C.UTF-8 diff --git a/resen-base/resources/start-notebook.sh b/resen-base/resources/start-notebook.sh index 4eb2e22..c40baa5 100644 --- a/resen-base/resources/start-notebook.sh +++ b/resen-base/resources/start-notebook.sh @@ -4,11 +4,16 @@ set -e +wrapper="" +if [[ "${RESTARTABLE}" == "yes" ]]; then + wrapper="run-one-constantly" +fi + if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then - # launched by JupyterHub, use single-user entrypoint - exec /usr/local/bin/start-singleuser.sh "$@" + # launched by JupyterHub, use single-user entrypoint + exec /usr/local/bin/start-singleuser.sh "$@" elif [[ ! -z "${JUPYTER_ENABLE_LAB}" ]]; then - . /usr/local/bin/start.sh jupyter lab "$@" + . /usr/local/bin/start.sh $wrapper jupyter lab "$@" else - . /usr/local/bin/start.sh jupyter notebook "$@" + . /usr/local/bin/start.sh $wrapper jupyter notebook "$@" fi \ No newline at end of file diff --git a/resen-base/resources/start-singleuser.sh b/resen-base/resources/start-singleuser.sh index 21125bc..02c7d1d 100644 --- a/resen-base/resources/start-singleuser.sh +++ b/resen-base/resources/start-singleuser.sh @@ -4,11 +4,12 @@ set -e +# modified 9 Sept 2020 from https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/start-singleuser.sh source /home/jovyan/envs/py36/bin/activate # set default ip to 0.0.0.0 if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then - NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" fi # handle some deprecated environment variables @@ -16,30 +17,26 @@ fi # These won't be passed from DockerSpawner 0.9, # so avoid specifying --arg=empty-string if [ ! -z "$NOTEBOOK_DIR" ]; then - NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_PORT" ]; then - NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_USER" ]; then - NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_COOKIE_NAME" ]; then - NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_BASE_URL" ]; then - NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_HUB_PREFIX" ]; then - NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_HUB_API_URL" ]; then - NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" -fi -if [ ! -z "$JUPYTER_ENABLE_LAB" ]; then - NOTEBOOK_BIN="jupyter labhub" -else - NOTEBOOK_BIN="jupyterhub-singleuser" + NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" fi +NOTEBOOK_BIN="jupyterhub-singleuser" . /usr/local/bin/start.sh $NOTEBOOK_BIN $NOTEBOOK_ARGS "$@" \ No newline at end of file diff --git a/resen-base/resources/start.sh b/resen-base/resources/start.sh index beb4f13..e9f7634 100644 --- a/resen-base/resources/start.sh +++ b/resen-base/resources/start.sh @@ -66,7 +66,7 @@ if [ $(id -u) == 0 ] ; then # (it could be mounted, and we shouldn't create it if it already exists) if [[ ! -e "/home/$NB_USER" ]]; then echo "Relocating home dir to /home/$NB_USER" - mv /home/jovyan "/home/$NB_USER" + mv /home/jovyan "/home/$NB_USER" || ln -s /home/jovyan "/home/$NB_USER" fi # if workdir is in /home/jovyan, cd to /home/$NB_USER if [[ "$PWD/" == "/home/jovyan/"* ]]; then @@ -76,29 +76,27 @@ if [ $(id -u) == 0 ] ; then fi fi - # Change UID of NB_USER to NB_UID if it does not match - if [ "$NB_UID" != $(id -u $NB_USER) ] ; then - echo "Set $NB_USER UID to: $NB_UID" - usermod -u $NB_UID $NB_USER - fi - - # Set NB_USER primary gid to NB_GID (after making the group). Set - # supplementary gids to NB_GID and 100. - if [ "$NB_GID" != $(id -g $NB_USER) ] ; then - echo "Add $NB_USER to group: $NB_GID" - groupadd -g $NB_GID -o ${NB_GROUP:-${NB_USER}} - usermod -g $NB_GID -aG 100 $NB_USER + # Change UID:GID of NB_USER to NB_UID:NB_GID if it does not match + if [ "$NB_UID" != $(id -u $NB_USER) ] || [ "$NB_GID" != $(id -g $NB_USER) ]; then + echo "Set user $NB_USER UID:GID to: $NB_UID:$NB_GID" + if [ "$NB_GID" != $(id -g $NB_USER) ]; then + groupadd -f -g $NB_GID -o ${NB_GROUP:-${NB_USER}} + fi + userdel $NB_USER + useradd --home /home/$NB_USER -u $NB_UID -g $NB_GID -G 100 -l $NB_USER fi # Enable sudo if requested if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then - # echo "Granting $NB_USER sudo access and appending $CONDA_DIR/bin to sudo PATH" echo "Granting $NB_USER sudo access" + # and appending $CONDA_DIR/bin to sudo PATH" # see comment below echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook fi - # Add $CONDA_DIR/bin to sudo secure_path (not needed for resen-core) - #sed -r "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path + # Add $CONDA_DIR/bin to sudo secure_path + # removed 9 Sept 2020 from https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/start.sh + # we don't use conda in our docker image + # sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path # Exec the command as NB_USER with the PATH and the rest of # the environment preserved @@ -106,7 +104,7 @@ if [ $(id -u) == 0 ] ; then echo "Executing the command: ${cmd[@]}" exec sudo -E -H -u $NB_USER PATH=$PATH XDG_CACHE_HOME=/home/$NB_USER/.cache PYTHONPATH=${PYTHONPATH:-} "${cmd[@]}" else - if [[ "$NB_UID" == "$(id -u jovyan)" && "$NB_GID" == "$(id -g jovyan)" ]]; then + if [[ "$NB_UID" == "$(id -u jovyan 2>/dev/null)" && "$NB_GID" == "$(id -g jovyan 2>/dev/null)" ]]; then # User is not attempting to override user/group via environment # variables, but they could still have overridden the uid/gid that # container runs as. Check that the user has an entry in the passwd @@ -149,4 +147,4 @@ else run-hooks /usr/local/bin/before-notebook.d echo "Executing the command: ${cmd[@]}" exec "${cmd[@]}" -fi +fi \ No newline at end of file diff --git a/resen-core/resources/helpers/setup_py36_env.sh b/resen-core/resources/helpers/setup_py38_env.sh similarity index 100% rename from resen-core/resources/helpers/setup_py36_env.sh rename to resen-core/resources/helpers/setup_py38_env.sh From 49de0afac6347b25cbb2d1a198df04807fd7a8ee Mon Sep 17 00:00:00 2001 From: Ashton Reimer Date: Thu, 10 Sep 2020 22:38:36 -0700 Subject: [PATCH 2/9] Updated resen-core: removed py27, install new packages --- resen-core/Dockerfile | 55 ++++++-------- resen-core/resources/helpers/setup_basemap.sh | 9 +-- .../resources/helpers/setup_py38_env.sh | 73 +++++++++++-------- resen-core/resources/helpers/setup_pyglow.sh | 13 +--- resen-core/testing/tests/test_py36_imports.py | 5 +- 5 files changed, 72 insertions(+), 83 deletions(-) diff --git a/resen-core/Dockerfile b/resen-core/Dockerfile index d93269a..8afa217 100644 --- a/resen-core/Dockerfile +++ b/resen-core/Dockerfile @@ -1,4 +1,5 @@ -FROM earthcubeingeo/resen-base:2020.1.0 +#FROM earthcubeingeo/resen-base:2020.1.0 +FROM resenbasetest:latest LABEL maintainer="Ingeo Team " LABEL description="The core docker image used by resen." @@ -27,33 +28,28 @@ RUN apt-get update && \ # Install python stuff RUN apt-get update && \ apt-get install -yq --no-install-recommends \ - python2.7 python3.6 \ - python-virtualenv python3-virtualenv \ - python-setuptools python3-setuptools \ - python-dev python3-dev \ + python3-virtualenv \ + python3-setuptools \ + python3-dev \ + python3.8 \ libproj-dev proj-data proj-bin \ libgeos++-dev libgeos-dev \ libdb-dev \ - python-tk python3-tk && \ + python3-tk && \ rm -rf /var/lib/apt/lists/* # Install system libraries needed by python packages RUN apt-get update && \ apt-get install -yq --no-install-recommends \ - libncurses5-dev && \ - rm -rf /var/lib/apt/lists/* - -# Install nodejs and npm needed for matplotlib widgets -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \ - apt-get install -yq --no-install-recommends nodejs && \ + libncurses5-dev \ + cm-super \ + npm nodejs && \ rm -rf /var/lib/apt/lists/* # Create python 2.7 and python 3.6 virtual environments USER $NB_USER RUN /bin/bash -c 'mkdir -p /home/$NB_USER/envs && \ - python3 -m virtualenv -p /usr/bin/python3.6 /home/$NB_USER/envs/py36 && \ - python3 -m virtualenv -p /usr/bin/python2.7 /home/$NB_USER/envs/py27' - + python3 -m virtualenv -p /usr/bin/python3.8 /home/$NB_USER/envs/py38' WORKDIR /home/$NB_USER @@ -67,23 +63,17 @@ RUN /bin/bash -c 'bash install_CDF.sh && rm install_CDF.sh' # Now install all python packages USER $NB_USER -COPY resources/helpers/setup_py36_env.sh . -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && bash setup_py36_env.sh && rm setup_py36_env.sh' - -COPY resources/helpers/setup_py27_env.sh . -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py27/bin/activate && bash setup_py27_env.sh && rm setup_py27_env.sh' +COPY resources/helpers/setup_py38_env.sh . +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && bash setup_py38_env.sh && rm setup_py38_env.sh' # register python environments with ipykernel -RUN /bin/bash -c 'source /home/$NB_USER/envs/py27/bin/activate && \ - pip install ipykernel==4.10.0 && \ - python -m ipykernel install --user --name py27 --display-name "py27"' -RUN /bin/bash -c 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -c 'source /home/$NB_USER/envs/py38/bin/activate && \ pip install ipykernel==5.1.1 && \ - python -m ipykernel install --user --name py36 --display-name "py36"' + python -m ipykernel install --user --name py38 --display-name "py38"' # remove python3 kernel from showing up in jupyter -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ yes | jupyter kernelspec remove python3' # Install basemap @@ -95,23 +85,26 @@ COPY resources/helpers/setup_pyglow.sh . RUN /bin/bash -cl 'bash setup_pyglow.sh && rm setup_pyglow.sh' # Install citationhelper +USER root COPY resources/pkg_citations.json /home/$NB_USER/cache/ +RUN chown -R $NB_USER:users "/home/$NB_USER/cache/" +USER $NB_USER # create work directory for user RUN mkdir /home/$NB_USER/work -# set default python environment to py36 -RUN /bin/bash -c 'echo "source /home/$NB_USER/envs/py36/bin/activate" >> /home/$NB_USER/.bashrc' +# set default python environment to py38 +RUN /bin/bash -c 'echo "source /home/$NB_USER/envs/py38/bin/activate" >> /home/$NB_USER/.bashrc' # post installation stuff # Download 110m scale cartopy data COPY resources/feature_download.py . -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ python feature_download.py physical cultural cultural-extra --do_scales 110m && \ rm feature_download.py' # Installing jupyter-widgets -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ jupyter labextension install @jupyter-widgets/jupyterlab-manager && \ jupyter labextension install jupyter-matplotlib' @@ -121,7 +114,7 @@ RUN echo "TERM=xterm-256color" >> /home/$NB_USER/.bashrc && \ echo "export CITEHELP_REFFILE=/home/$NB_USER/cache/pkg_citations.json" >> /home/$NB_USER/.bashrc # Set default jupyter terminal shell -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ jupyter lab --generate-config' RUN sed -i -e 's/#c.NotebookApp.terminado_settings = {}/c.NotebookApp.terminado_settings = {\"shell_command\"\:[\"\/bin\/bash\"]}/' /home/jovyan/.jupyter/jupyter_notebook_config.py diff --git a/resen-core/resources/helpers/setup_basemap.sh b/resen-core/resources/helpers/setup_basemap.sh index dbb7e2e..518b61f 100644 --- a/resen-core/resources/helpers/setup_basemap.sh +++ b/resen-core/resources/helpers/setup_basemap.sh @@ -3,7 +3,7 @@ # # A helper script for installing basemap # -# Assumes you have py27 and py36 python virtual environments +# Assumes you have py38 python virtual environments # ####################################################################################### @@ -12,12 +12,7 @@ wget https://github.com/matplotlib/basemap/archive/v1.2.0rel.tar.gz tar -zxf v1.2.0rel.tar.gz cd basemap-1.2.0rel -source ${HOME}/envs/py27/bin/activate -pip install . - -deactivate - -source ${HOME}/envs/py36/bin/activate +source ${HOME}/envs/py38/bin/activate pip install . deactivate diff --git a/resen-core/resources/helpers/setup_py38_env.sh b/resen-core/resources/helpers/setup_py38_env.sh index 5288a43..e367341 100644 --- a/resen-core/resources/helpers/setup_py38_env.sh +++ b/resen-core/resources/helpers/setup_py38_env.sh @@ -1,59 +1,68 @@ #!/bin/bash ####################################################################################### # -# A helper script for setting up a python 3.6 environment +# A helper script for setting up a python 3.8 environment # ####################################################################################### -echo "**** Installing python 3.6 packages ****" +echo "**** Installing python 3.8 packages ****" # upgrade pip -pip install pip==20.1.1 +pip install pip==20.2.3 # Now use pip to install everything we can -# Notes: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap -pip install -U jupyterhub==1.0.0 \ - jupyterlab==2.1.2 \ - notebook==6.0.3 \ - paramiko==2.7.1 \ - ipython==7.14.0 \ - pymongo==3.10.1 \ - mechanize==0.4.5 \ - cryptography==2.9.2 \ - pytest==5.4.2 \ - nbformat==5.0.6 \ - numpy==1.18.4 \ - matplotlib==3.2.1 \ - scipy==1.4.1 \ +# NOTES: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap +# jupyterhub, jupyterlab, notebook should all follow jupyterstacks versions +pip install -U jupyterhub==1.1.0 \ + jupyterlab==2.2.5 \ + notebook==6.1.3 \ + paramiko==2.7.2 \ + ipython==7.18.1 \ + pytest==5.4.3 \ + nbformat==5.0.7 \ + numpy==1.19.1 \ + matplotlib==3.3.1 \ + scipy==1.5.2 \ tables==3.6.1 \ - sgp4==2.8 \ - pandas==1.0.3 \ - sympy==1.5.1 \ - sqlalchemy==1.3.17 \ + sgp4==2.12 \ + pandas==1.1.2 \ + sympy==1.6.2 \ + sqlalchemy==1.3.19 \ h5py==2.10.0 \ pytz==2020.1 \ - netcdf4==1.5.3 \ - bokeh==2.0.2 \ + netcdf4==1.5.4 \ + bokeh==2.2.1 \ pyyaml==5.3.1 \ - cython==0.29.18 \ - pyproj==2.6.1.post1 \ + cython==0.29.21 \ + pyproj==1.9.6 \ madrigalweb==3.1.12 \ bsddb3==6.2.7 \ aacgmv2==2.6.0 \ - pymap3d==2.4.1 \ - astropy==3.2.1 \ - plasmapy==0.3.1 \ - pydarn==1.0.0.1 \ - viresclient==0.6.1 \ - ipympl==0.5.6 \ + astropy==4.0.1.post1 \ + plasmapy==0.4.0 \ + pydarn==1.1.0 \ + viresclient==0.6.2 \ + ipympl==0.5.7 \ hdfviewer==0.11.0 \ - watermark==2.0.2 + watermark==2.0.2 \ + seaborn==0.11.0 \ + pyephem==3.7.7.0 \ + skyfield==1.26 + +#build matplotlib font cache +MPLBACKEND=Agg python -uc "import matplotlib.pyplot" + +# install sunpy +pip install -U sunpy==2.0.1 beautifulsoup4==4.9.1 drms==0.5.7 zeep==3.4.0 tqdm==4.48.2 # Custom pip installation for any package that needs it pip install apexpy==1.0.3 # have to install after installing numpy pip install cartopy==0.18.0 # have to install after installing numpy +pip install pymap3d==2.4.1 # has side effects when importing first time +python -uc "import pymap3d" + pip install spacepy==0.2.1 source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB # only update omni and qin-denton since maia.usno.navy.mil/ser7/tai-utc.dat leapseconds website was down at the time... diff --git a/resen-core/resources/helpers/setup_pyglow.sh b/resen-core/resources/helpers/setup_pyglow.sh index 1605910..2af1560 100755 --- a/resen-core/resources/helpers/setup_pyglow.sh +++ b/resen-core/resources/helpers/setup_pyglow.sh @@ -3,8 +3,7 @@ # # A helper script for installing pyglow # -# Assumes you have conda installed with a py27 virtual environment -# and another environment with the name py36 +# Assumes you have a py38 python virtual environments # ####################################################################################### @@ -16,15 +15,7 @@ git clone https://github.com/timduly4/pyglow.git $PYGLOW_BUILD_DIR cd $PYGLOW_BUILD_DIR git checkout 054bf800263c01ecfa783741c9cfe8d3d6c259ca -b master_20190904 -source ${HOME}/envs/py27/bin/activate - -make -C src/pyglow/models source -pip install . -python -c "import pyglow" - -deactivate - -source ${HOME}/envs/py36/bin/activate +source ${HOME}/envs/py38/bin/activate make -C src/pyglow/models source pip install . diff --git a/resen-core/testing/tests/test_py36_imports.py b/resen-core/testing/tests/test_py36_imports.py index f29031d..15ee56d 100644 --- a/resen-core/testing/tests/test_py36_imports.py +++ b/resen-core/testing/tests/test_py36_imports.py @@ -15,11 +15,12 @@ class ImportTester(unittest.TestCase): # In the future, get this list from a Pipfile and # include a test for package version number - packages = ['paramiko','IPython','pymongo','mechanize','netCDF4','cryptography','pytest', + packages = ['paramiko','IPython','seaborn','netCDF4','cryptography','pytest', 'nbformat','numpy','matplotlib','scipy','tables','sgp4','pandas','sympy', 'sqlalchemy','h5py','pytz','bokeh','yaml','cython','apexpy','cartopy', 'spacepy','aacgmv2','pymap3d','mangopy','madrigalWeb','visuamisr', - 'citationhelper','viresclient','hdfviewer','watermark','pydarn','plasmapy'] + 'citationhelper','viresclient','hdfviewer','watermark','pydarn','plasmapy', + 'sunpy','ephem','skyfield'] def test_imports(self): # Try to import all the packages From 069d595c379b165314aa4b0ed55a022ef9ee7138 Mon Sep 17 00:00:00 2001 From: Ashton Reimer Date: Thu, 10 Sep 2020 22:40:12 -0700 Subject: [PATCH 3/9] remove py27 scripts --- .../resources/helpers/setup_py27_env.sh | 69 ------------------- resen-core/testing/tests/test_py27_imports.py | 54 --------------- ...t_py36_imports.py => test_py38_imports.py} | 0 3 files changed, 123 deletions(-) delete mode 100644 resen-core/resources/helpers/setup_py27_env.sh delete mode 100644 resen-core/testing/tests/test_py27_imports.py rename resen-core/testing/tests/{test_py36_imports.py => test_py38_imports.py} (100%) diff --git a/resen-core/resources/helpers/setup_py27_env.sh b/resen-core/resources/helpers/setup_py27_env.sh deleted file mode 100644 index 940c081..0000000 --- a/resen-core/resources/helpers/setup_py27_env.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -####################################################################################### -# -# A helper script for setting up a python 2.7 environment -# -####################################################################################### - -echo "**** Installing python 2.7 packages ****" - -# upgrade pip -pip install pip==20.1.1 - -# Now use pip to install everything we can -# Notes: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap -pip install -U paramiko==2.4.2 \ - ipython==5.8.0 \ - prompt_toolkit==1.0.16 \ - pymongo==3.7.2 \ - mechanize==0.4.1 \ - cryptography==2.6.1 \ - pytest==3.0.5 \ - nbformat==4.4.0 \ - numpy==1.16.2 \ - matplotlib==2.2.4 \ - scipy==1.2.1 \ - tables==3.5.1 \ - sgp4==1.4 \ - pandas==0.24.2 \ - sympy==1.1.0 \ - sqlalchemy==1.3.1 \ - h5py==2.9.0 \ - pytz==2018.9 \ - netcdf4==1.4.3 \ - bokeh==1.0.4 \ - pyyaml==5.1 \ - cython==0.29.6 \ - sciunit2==0.4.post3.dev95073526 \ - madrigalweb==3.1.10 \ - pyproj==1.9.6 \ - bsddb3==6.2.6 \ - aacgmv2==2.5.2 \ - pymap3d==1.8.1 \ - astropy==2.0.14 \ - ipympl==0.5.6 \ - # sunpy==0.9.10 - -# Custom pip installation for any package that needs it -pip install apexpy==1.0.3 # have to install after installing numpy - -pip install cartopy==0.17.0 - -pip install spacepy==0.2.1 -source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB -# only update omni and qin-denton since maia.usno.navy.mil/ser7/tai-utc.dat leapseconds website was down at the time... -python -uc "import spacepy.toolbox; spacepy.toolbox.update(QDomni=True)" -# spacepy 0.2.1 doesn't clean up this file. Submitted issue and PR to spacepy, see https://github.com/spacepy/spacepy/pull/219 -rm OMNI_OMNI2_merged_20120213-v1.cdf - -# Installing mangopy (14 June 2018) -pip install git+https://github.com/astib/MANGO.git@2dd4ca5380dca54cac8d2180c3ad63fc041a5c67 - -# Install visuamisr -pip install git+https://github.com/asreimer/visuamisr.git@v2.0.3 - -# Install citationhelper -pip install git+https://github.com/EarthCubeInGeo/citationhelper.git@v0.2 - -# cleanup -rm -rf ~/.cache/pip/* diff --git a/resen-core/testing/tests/test_py27_imports.py b/resen-core/testing/tests/test_py27_imports.py deleted file mode 100644 index f88c18c..0000000 --- a/resen-core/testing/tests/test_py27_imports.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - test_imports.py - - This is intended to be used to check that all expected python packages are avaiable. Works - well for identifying strangeness that may occur with conda when installing multiple python - packages with differing dependency requirements. - -""" - -import unittest -import importlib - -class ImportTester(unittest.TestCase): - # In the future, get this list from a Pipfile and - # include a test for package version number - packages = ['paramiko','IPython','pymongo','mechanize','netCDF4','cryptography','pytest', - 'nbformat','numpy','matplotlib','scipy','tables','sgp4','pandas','sympy', - 'sqlalchemy','h5py','pytz','bokeh','yaml','cython','sciunit2','bsddb3', - 'apexpy','cartopy','spacepy','aacgmv2','pymap3d','mangopy','astropy', - 'madrigalWeb','visuamisr','citationhelper'] - - def test_imports(self): - # Try to import all the packages - for package in self.packages: - print("Importing %s" % package) - importlib.import_module(package) - - # if we get here, everything imported fine! - self.assertTrue(True) - -if __name__ == '__main__': - unittest.main() - -# Still need to do: -# import tables -# import pyglow -# import sgp4 -# import pydarn -# import sympy -# import SQLAlchemy - -# import bokeh -# import ipyparallel - -# from Cython.Build import cythonize -# import sunpy -# import heliopy - - - - - diff --git a/resen-core/testing/tests/test_py36_imports.py b/resen-core/testing/tests/test_py38_imports.py similarity index 100% rename from resen-core/testing/tests/test_py36_imports.py rename to resen-core/testing/tests/test_py38_imports.py From 35417ef094243645de1d8e20da20aa707893ca86 Mon Sep 17 00:00:00 2001 From: Ashton Reimer Date: Thu, 10 Sep 2020 22:42:24 -0700 Subject: [PATCH 4/9] update python env to py38 --- resen-base/resources/start-singleuser.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resen-base/resources/start-singleuser.sh b/resen-base/resources/start-singleuser.sh index 02c7d1d..a70a115 100644 --- a/resen-base/resources/start-singleuser.sh +++ b/resen-base/resources/start-singleuser.sh @@ -5,7 +5,7 @@ set -e # modified 9 Sept 2020 from https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/start-singleuser.sh -source /home/jovyan/envs/py36/bin/activate +source /home/jovyan/envs/py38/bin/activate # set default ip to 0.0.0.0 if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then From b70e14fb961f09dfee954fc2cb28195064ee316f Mon Sep 17 00:00:00 2001 From: Ashton Reimer Date: Thu, 15 Oct 2020 22:26:28 -0700 Subject: [PATCH 5/9] Fixed bug with jupyterlab terminado not using bash. Fixed basemap version. --- resen-core/Dockerfile | 6 +++--- resen-core/resources/helpers/setup_basemap.sh | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/resen-core/Dockerfile b/resen-core/Dockerfile index 8afa217..a06e309 100644 --- a/resen-core/Dockerfile +++ b/resen-core/Dockerfile @@ -1,4 +1,4 @@ -#FROM earthcubeingeo/resen-base:2020.1.0 +#FROM earthcubeingeo/resen-base:2020.2.0 FROM resenbasetest:latest LABEL maintainer="Ingeo Team " @@ -46,7 +46,7 @@ RUN apt-get update && \ npm nodejs && \ rm -rf /var/lib/apt/lists/* -# Create python 2.7 and python 3.6 virtual environments +# Create python 3.8 virtual environment USER $NB_USER RUN /bin/bash -c 'mkdir -p /home/$NB_USER/envs && \ python3 -m virtualenv -p /usr/bin/python3.8 /home/$NB_USER/envs/py38' @@ -116,7 +116,7 @@ RUN echo "TERM=xterm-256color" >> /home/$NB_USER/.bashrc && \ # Set default jupyter terminal shell RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ jupyter lab --generate-config' -RUN sed -i -e 's/#c.NotebookApp.terminado_settings = {}/c.NotebookApp.terminado_settings = {\"shell_command\"\:[\"\/bin\/bash\"]}/' /home/jovyan/.jupyter/jupyter_notebook_config.py +RUN sed -i -e 's/# c.NotebookApp.terminado_settings = {}/c.NotebookApp.terminado_settings = {\"shell_command\"\:[\"\/bin\/bash\"]}/' /home/jovyan/.jupyter/jupyter_notebook_config.py # Set the jupyterlab user-settings RUN /bin/bash -cl 'mkdir -p /home/jovyan/.jupyter/lab/user-settings/@jupyterlab/extensionmanager-extension && \ diff --git a/resen-core/resources/helpers/setup_basemap.sh b/resen-core/resources/helpers/setup_basemap.sh index 518b61f..ee05f5b 100644 --- a/resen-core/resources/helpers/setup_basemap.sh +++ b/resen-core/resources/helpers/setup_basemap.sh @@ -8,9 +8,9 @@ ####################################################################################### # git clone --depth 0 https://github.com/matplotlib/basemap.git # VERY SLOW -wget https://github.com/matplotlib/basemap/archive/v1.2.0rel.tar.gz -tar -zxf v1.2.0rel.tar.gz -cd basemap-1.2.0rel +wget https://github.com/matplotlib/basemap/archive/v1.2.2rel.tar.gz +tar -zxf v1.2.2rel.tar.gz +cd basemap-1.2.2rel source ${HOME}/envs/py38/bin/activate pip install . @@ -19,5 +19,5 @@ deactivate # cleanup cd .. -rm -rf basemap-1.2.0rel v1.2.0rel.tar.gz +rm -rf basemap-1.2.2rel v1.2.2rel.tar.gz rm ${HOME}/.wget-hsts From a81f8248e6094e3d52c8f491ede7fb78c311ccab Mon Sep 17 00:00:00 2001 From: Pablo Reyes Date: Mon, 19 Oct 2020 11:53:55 -0700 Subject: [PATCH 6/9] Updating the latest commit(19 Oct 2020) of pyglow in setup_pyglow.sh that fixes an iri2016 issue for dates after year 2010. --- resen-core/resources/helpers/setup_davitpy.sh | 35 +++++++++ .../resources/helpers/setup_hdf5viewer.sh | 32 ++++++++ resen-core/resources/helpers/setup_mangopy.sh | 36 +++++++++ .../resources/helpers/setup_py27_env.sh | 69 +++++++++++++++++ .../resources/helpers/setup_py36_env.sh | 74 +++++++++++++++++++ resen-core/resources/helpers/setup_pyglow.sh | 2 +- resen-core/resources/helpers/setup_spacepy.sh | 41 ++++++++++ 7 files changed, 288 insertions(+), 1 deletion(-) create mode 100644 resen-core/resources/helpers/setup_davitpy.sh create mode 100644 resen-core/resources/helpers/setup_hdf5viewer.sh create mode 100644 resen-core/resources/helpers/setup_mangopy.sh create mode 100644 resen-core/resources/helpers/setup_py27_env.sh create mode 100644 resen-core/resources/helpers/setup_py36_env.sh create mode 100644 resen-core/resources/helpers/setup_spacepy.sh diff --git a/resen-core/resources/helpers/setup_davitpy.sh b/resen-core/resources/helpers/setup_davitpy.sh new file mode 100644 index 0000000..5c1d604 --- /dev/null +++ b/resen-core/resources/helpers/setup_davitpy.sh @@ -0,0 +1,35 @@ +#!/bin/bash +####################################################################################### +# +# A helper script for installing davitpy +# +# Assumes you have a py27 python virtual environment +# +####################################################################################### + +source ${HOME}/envs/py27/bin/activate +CURDIR=$(pwd) +DAVITPY_BUILD_DIR=${CURDIR}/davitpy_build + +git clone https://github.com/vtsuperdarn/davitpy.git $DAVITPY_BUILD_DIR +cd $DAVITPY_BUILD_DIR +git checkout tags/0.8-master -b 0.8-master + +pip install . + +# get hdw.dat and radar.dat - needed so davitpy can get radar information locally +mkdir -p ${HOME}/cache +cd ${HOME}/cache +git clone https://github.com/vtsuperdarn/hdw.dat.git +wget https://raw.githubusercontent.com/SuperDARN/rst/v4.2/tables/superdarn/radar.dat + +deactivate + +# cleanup +cd ${CURDIR} +rm -rf $DAVITPY_BUILD_DIR +rm ${HOME}/.wget-hsts + +# set nessisary environment variables +echo "SD_HDWPATH=${HOME}/cache/hdw.dat" >> ${HOME}/.bashrc +echo "SD_RADAR=${HOME}/cache/radar.dat" >> ${HOME}/.bashrc \ No newline at end of file diff --git a/resen-core/resources/helpers/setup_hdf5viewer.sh b/resen-core/resources/helpers/setup_hdf5viewer.sh new file mode 100644 index 0000000..6a4ab02 --- /dev/null +++ b/resen-core/resources/helpers/setup_hdf5viewer.sh @@ -0,0 +1,32 @@ +#!/bin/bash +####################################################################################### +# +# A helper script for installing pyglow +# +# Assumes you have conda installed with a py27 virtual environment +# and another environment with the name py36 +# +####################################################################################### + +echo "**** Installing hdf5 viewer ****" + +cd $BUILD +HDF5VIEWER_BUILD_DIR=hdf5viewer_build + +git clone https://github.com/pmreyes2/jupyterlab-hdf5.git $HDF5VIEWER_BUILD_DIR + +cd $HDF5VIEWER_BUILD_DIR + +git checkout a8a0e9bb1771ffa630cd8ead940ac7b258e1ad8b -b master_20190921 + +source /home/$NB_USER/envs/py36/bin/activate + +export PATH=/usr/local/lib/nodejs/node-v10.16.3-linux-x64/bin:$PATH +pip install . +jupyter labextension install jupyterlab-hdf + +deactivate + +# cleanup +cd $BUILD +rm -r $HDF5VIEWER_BUILD_DIR diff --git a/resen-core/resources/helpers/setup_mangopy.sh b/resen-core/resources/helpers/setup_mangopy.sh new file mode 100644 index 0000000..db90868 --- /dev/null +++ b/resen-core/resources/helpers/setup_mangopy.sh @@ -0,0 +1,36 @@ +#!/bin/bash +####################################################################################### +# +# A helper script for installing mangopy +# +# Assumes you have a py27 virtual environment +# and another environment with the name py36 +# +####################################################################################### + +echo "**** Installing mangopy ****" + +CWD=$(pwd) + +MANGOPY_BUILD_DIR=$CWD/mangopy_build +mkdir -p $MANGOPY_BUILD_DIR + +cd $MANGOPY_BUILD_DIR +git clone https://github.com/astib/MANGO.git +cd MANGO +#If we wanted a specific commit: +#git checkout b25cba78e58197394809cb8323656a1d636c3e3d -b master_06jun2019 + +source /home/$NB_USER/envs/py27/bin/activate +LDFLAGS="-shared" pip install . + +deactivate + +source /home/$NB_USER/envs/py36/bin/activate +LDFLAGS="-shared" pip install . + +deactivate + +# cleanup +cd $CWD +rm -r $MANGOPY_BUILD_DIR diff --git a/resen-core/resources/helpers/setup_py27_env.sh b/resen-core/resources/helpers/setup_py27_env.sh new file mode 100644 index 0000000..940c081 --- /dev/null +++ b/resen-core/resources/helpers/setup_py27_env.sh @@ -0,0 +1,69 @@ +#!/bin/bash +####################################################################################### +# +# A helper script for setting up a python 2.7 environment +# +####################################################################################### + +echo "**** Installing python 2.7 packages ****" + +# upgrade pip +pip install pip==20.1.1 + +# Now use pip to install everything we can +# Notes: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap +pip install -U paramiko==2.4.2 \ + ipython==5.8.0 \ + prompt_toolkit==1.0.16 \ + pymongo==3.7.2 \ + mechanize==0.4.1 \ + cryptography==2.6.1 \ + pytest==3.0.5 \ + nbformat==4.4.0 \ + numpy==1.16.2 \ + matplotlib==2.2.4 \ + scipy==1.2.1 \ + tables==3.5.1 \ + sgp4==1.4 \ + pandas==0.24.2 \ + sympy==1.1.0 \ + sqlalchemy==1.3.1 \ + h5py==2.9.0 \ + pytz==2018.9 \ + netcdf4==1.4.3 \ + bokeh==1.0.4 \ + pyyaml==5.1 \ + cython==0.29.6 \ + sciunit2==0.4.post3.dev95073526 \ + madrigalweb==3.1.10 \ + pyproj==1.9.6 \ + bsddb3==6.2.6 \ + aacgmv2==2.5.2 \ + pymap3d==1.8.1 \ + astropy==2.0.14 \ + ipympl==0.5.6 \ + # sunpy==0.9.10 + +# Custom pip installation for any package that needs it +pip install apexpy==1.0.3 # have to install after installing numpy + +pip install cartopy==0.17.0 + +pip install spacepy==0.2.1 +source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB +# only update omni and qin-denton since maia.usno.navy.mil/ser7/tai-utc.dat leapseconds website was down at the time... +python -uc "import spacepy.toolbox; spacepy.toolbox.update(QDomni=True)" +# spacepy 0.2.1 doesn't clean up this file. Submitted issue and PR to spacepy, see https://github.com/spacepy/spacepy/pull/219 +rm OMNI_OMNI2_merged_20120213-v1.cdf + +# Installing mangopy (14 June 2018) +pip install git+https://github.com/astib/MANGO.git@2dd4ca5380dca54cac8d2180c3ad63fc041a5c67 + +# Install visuamisr +pip install git+https://github.com/asreimer/visuamisr.git@v2.0.3 + +# Install citationhelper +pip install git+https://github.com/EarthCubeInGeo/citationhelper.git@v0.2 + +# cleanup +rm -rf ~/.cache/pip/* diff --git a/resen-core/resources/helpers/setup_py36_env.sh b/resen-core/resources/helpers/setup_py36_env.sh new file mode 100644 index 0000000..5288a43 --- /dev/null +++ b/resen-core/resources/helpers/setup_py36_env.sh @@ -0,0 +1,74 @@ +#!/bin/bash +####################################################################################### +# +# A helper script for setting up a python 3.6 environment +# +####################################################################################### + +echo "**** Installing python 3.6 packages ****" + +# upgrade pip +pip install pip==20.1.1 + +# Now use pip to install everything we can +# Notes: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap +pip install -U jupyterhub==1.0.0 \ + jupyterlab==2.1.2 \ + notebook==6.0.3 \ + paramiko==2.7.1 \ + ipython==7.14.0 \ + pymongo==3.10.1 \ + mechanize==0.4.5 \ + cryptography==2.9.2 \ + pytest==5.4.2 \ + nbformat==5.0.6 \ + numpy==1.18.4 \ + matplotlib==3.2.1 \ + scipy==1.4.1 \ + tables==3.6.1 \ + sgp4==2.8 \ + pandas==1.0.3 \ + sympy==1.5.1 \ + sqlalchemy==1.3.17 \ + h5py==2.10.0 \ + pytz==2020.1 \ + netcdf4==1.5.3 \ + bokeh==2.0.2 \ + pyyaml==5.3.1 \ + cython==0.29.18 \ + pyproj==2.6.1.post1 \ + madrigalweb==3.1.12 \ + bsddb3==6.2.7 \ + aacgmv2==2.6.0 \ + pymap3d==2.4.1 \ + astropy==3.2.1 \ + plasmapy==0.3.1 \ + pydarn==1.0.0.1 \ + viresclient==0.6.1 \ + ipympl==0.5.6 \ + hdfviewer==0.11.0 \ + watermark==2.0.2 + +# Custom pip installation for any package that needs it +pip install apexpy==1.0.3 # have to install after installing numpy + +pip install cartopy==0.18.0 # have to install after installing numpy + +pip install spacepy==0.2.1 +source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB +# only update omni and qin-denton since maia.usno.navy.mil/ser7/tai-utc.dat leapseconds website was down at the time... +python -uc "import spacepy.toolbox; spacepy.toolbox.update(QDomni=True)" +# spacepy 0.2.1 doesn't clean up this file. Submitted issue and PR to spacepy, see https://github.com/spacepy/spacepy/pull/219 +rm OMNI_OMNI2_merged_20120213-v1.cdf + +# Installing mangopy (14 June 2018) +pip install git+https://github.com/astib/MANGO.git@2dd4ca5380dca54cac8d2180c3ad63fc041a5c67 + +# Install visuamisr +pip install git+https://github.com/asreimer/visuamisr.git@v2.0.3 + +# Install citationhelper +pip install git+https://github.com/EarthCubeInGeo/citationhelper.git@v0.2 + +# cleanup +rm -rf ~/.cache/pip/* diff --git a/resen-core/resources/helpers/setup_pyglow.sh b/resen-core/resources/helpers/setup_pyglow.sh index 2af1560..6363b1f 100755 --- a/resen-core/resources/helpers/setup_pyglow.sh +++ b/resen-core/resources/helpers/setup_pyglow.sh @@ -13,7 +13,7 @@ PYGLOW_BUILD_DIR=pyglow_build git clone https://github.com/timduly4/pyglow.git $PYGLOW_BUILD_DIR cd $PYGLOW_BUILD_DIR -git checkout 054bf800263c01ecfa783741c9cfe8d3d6c259ca -b master_20190904 +git checkout 36e998524cf603cf61ca7d98ced61e2f945ab1df -b master_20201019 source ${HOME}/envs/py38/bin/activate diff --git a/resen-core/resources/helpers/setup_spacepy.sh b/resen-core/resources/helpers/setup_spacepy.sh new file mode 100644 index 0000000..589c9cc --- /dev/null +++ b/resen-core/resources/helpers/setup_spacepy.sh @@ -0,0 +1,41 @@ +#!/bin/bash +####################################################################################### +# +# A helper script for installing spacepy +# +# Assumes you have conda installed with a py27 virtual environment +# and another environment with the name py36 +# +####################################################################################### + +echo "**** Installing spacepy ****" + +cd $BUILD +SPACEPY_BUILD_DIR=spacepy_build + +git clone https://github.com/spacepy/spacepy.git $SPACEPY_BUILD_DIR +cd $SPACEPY_BUILD_DIR +git checkout 8b8c2d56ebdcb1d3a59fd868a1c92f34ce16ec64 -b 0.1.6_withfix + +source /home/$NB_USER/envs/py27/bin/activate +pip install . + +deactivate + +source /home/$NB_USER/envs/py36/bin/activate +pip install . + +# some post-pip initialization of packages +cd $BUILD +source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB +python -c "import spacepy.toolbox; spacepy.toolbox.update()" +# a file about 195 MB called OMNI_OMNI2_merged_20120213-v1.cdf +# has been created and will be removed later on. The data is kept +# inside ~/.spacepy +echo "support_notice: False" >> /home/$NB_USER/.spacepy/spacepy.rc + +deactivate + +# cleanup +cd $BUILD +rm -r $SPACEPY_BUILD_DIR From 86eb1bf366563b8423d075b41707e66732bfe88f Mon Sep 17 00:00:00 2001 From: Pablo Reyes Date: Mon, 19 Oct 2020 16:32:53 -0700 Subject: [PATCH 7/9] Adding sciunit2 to setup_py38_env.sh updating pip to 20.2.4 (2020-10-16) --- resen-core/resources/helpers/setup_py38_env.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/resen-core/resources/helpers/setup_py38_env.sh b/resen-core/resources/helpers/setup_py38_env.sh index e367341..376c9e8 100644 --- a/resen-core/resources/helpers/setup_py38_env.sh +++ b/resen-core/resources/helpers/setup_py38_env.sh @@ -8,7 +8,7 @@ echo "**** Installing python 3.8 packages ****" # upgrade pip -pip install pip==20.2.3 +pip install pip==20.2.4 # Now use pip to install everything we can # NOTES: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap @@ -47,7 +47,8 @@ pip install -U jupyterhub==1.1.0 \ watermark==2.0.2 \ seaborn==0.11.0 \ pyephem==3.7.7.0 \ - skyfield==1.26 + skyfield==1.26 \ + sciunit2==0.4.post58.dev224550607 #build matplotlib font cache MPLBACKEND=Agg python -uc "import matplotlib.pyplot" From 72f5a2d464ae4a0a1a85b9675c6d1840096eff48 Mon Sep 17 00:00:00 2001 From: Pablo Reyes Date: Mon, 19 Oct 2020 22:18:37 -0700 Subject: [PATCH 8/9] Removing files accidentally reintroduced in the two previous commits --- resen-core/resources/helpers/setup_davitpy.sh | 35 --------- .../resources/helpers/setup_hdf5viewer.sh | 32 -------- resen-core/resources/helpers/setup_mangopy.sh | 36 --------- .../resources/helpers/setup_py27_env.sh | 69 ----------------- .../resources/helpers/setup_py36_env.sh | 74 ------------------- resen-core/resources/helpers/setup_spacepy.sh | 41 ---------- 6 files changed, 287 deletions(-) delete mode 100644 resen-core/resources/helpers/setup_davitpy.sh delete mode 100644 resen-core/resources/helpers/setup_hdf5viewer.sh delete mode 100644 resen-core/resources/helpers/setup_mangopy.sh delete mode 100644 resen-core/resources/helpers/setup_py27_env.sh delete mode 100644 resen-core/resources/helpers/setup_py36_env.sh delete mode 100644 resen-core/resources/helpers/setup_spacepy.sh diff --git a/resen-core/resources/helpers/setup_davitpy.sh b/resen-core/resources/helpers/setup_davitpy.sh deleted file mode 100644 index 5c1d604..0000000 --- a/resen-core/resources/helpers/setup_davitpy.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -####################################################################################### -# -# A helper script for installing davitpy -# -# Assumes you have a py27 python virtual environment -# -####################################################################################### - -source ${HOME}/envs/py27/bin/activate -CURDIR=$(pwd) -DAVITPY_BUILD_DIR=${CURDIR}/davitpy_build - -git clone https://github.com/vtsuperdarn/davitpy.git $DAVITPY_BUILD_DIR -cd $DAVITPY_BUILD_DIR -git checkout tags/0.8-master -b 0.8-master - -pip install . - -# get hdw.dat and radar.dat - needed so davitpy can get radar information locally -mkdir -p ${HOME}/cache -cd ${HOME}/cache -git clone https://github.com/vtsuperdarn/hdw.dat.git -wget https://raw.githubusercontent.com/SuperDARN/rst/v4.2/tables/superdarn/radar.dat - -deactivate - -# cleanup -cd ${CURDIR} -rm -rf $DAVITPY_BUILD_DIR -rm ${HOME}/.wget-hsts - -# set nessisary environment variables -echo "SD_HDWPATH=${HOME}/cache/hdw.dat" >> ${HOME}/.bashrc -echo "SD_RADAR=${HOME}/cache/radar.dat" >> ${HOME}/.bashrc \ No newline at end of file diff --git a/resen-core/resources/helpers/setup_hdf5viewer.sh b/resen-core/resources/helpers/setup_hdf5viewer.sh deleted file mode 100644 index 6a4ab02..0000000 --- a/resen-core/resources/helpers/setup_hdf5viewer.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -####################################################################################### -# -# A helper script for installing pyglow -# -# Assumes you have conda installed with a py27 virtual environment -# and another environment with the name py36 -# -####################################################################################### - -echo "**** Installing hdf5 viewer ****" - -cd $BUILD -HDF5VIEWER_BUILD_DIR=hdf5viewer_build - -git clone https://github.com/pmreyes2/jupyterlab-hdf5.git $HDF5VIEWER_BUILD_DIR - -cd $HDF5VIEWER_BUILD_DIR - -git checkout a8a0e9bb1771ffa630cd8ead940ac7b258e1ad8b -b master_20190921 - -source /home/$NB_USER/envs/py36/bin/activate - -export PATH=/usr/local/lib/nodejs/node-v10.16.3-linux-x64/bin:$PATH -pip install . -jupyter labextension install jupyterlab-hdf - -deactivate - -# cleanup -cd $BUILD -rm -r $HDF5VIEWER_BUILD_DIR diff --git a/resen-core/resources/helpers/setup_mangopy.sh b/resen-core/resources/helpers/setup_mangopy.sh deleted file mode 100644 index db90868..0000000 --- a/resen-core/resources/helpers/setup_mangopy.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -####################################################################################### -# -# A helper script for installing mangopy -# -# Assumes you have a py27 virtual environment -# and another environment with the name py36 -# -####################################################################################### - -echo "**** Installing mangopy ****" - -CWD=$(pwd) - -MANGOPY_BUILD_DIR=$CWD/mangopy_build -mkdir -p $MANGOPY_BUILD_DIR - -cd $MANGOPY_BUILD_DIR -git clone https://github.com/astib/MANGO.git -cd MANGO -#If we wanted a specific commit: -#git checkout b25cba78e58197394809cb8323656a1d636c3e3d -b master_06jun2019 - -source /home/$NB_USER/envs/py27/bin/activate -LDFLAGS="-shared" pip install . - -deactivate - -source /home/$NB_USER/envs/py36/bin/activate -LDFLAGS="-shared" pip install . - -deactivate - -# cleanup -cd $CWD -rm -r $MANGOPY_BUILD_DIR diff --git a/resen-core/resources/helpers/setup_py27_env.sh b/resen-core/resources/helpers/setup_py27_env.sh deleted file mode 100644 index 940c081..0000000 --- a/resen-core/resources/helpers/setup_py27_env.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -####################################################################################### -# -# A helper script for setting up a python 2.7 environment -# -####################################################################################### - -echo "**** Installing python 2.7 packages ****" - -# upgrade pip -pip install pip==20.1.1 - -# Now use pip to install everything we can -# Notes: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap -pip install -U paramiko==2.4.2 \ - ipython==5.8.0 \ - prompt_toolkit==1.0.16 \ - pymongo==3.7.2 \ - mechanize==0.4.1 \ - cryptography==2.6.1 \ - pytest==3.0.5 \ - nbformat==4.4.0 \ - numpy==1.16.2 \ - matplotlib==2.2.4 \ - scipy==1.2.1 \ - tables==3.5.1 \ - sgp4==1.4 \ - pandas==0.24.2 \ - sympy==1.1.0 \ - sqlalchemy==1.3.1 \ - h5py==2.9.0 \ - pytz==2018.9 \ - netcdf4==1.4.3 \ - bokeh==1.0.4 \ - pyyaml==5.1 \ - cython==0.29.6 \ - sciunit2==0.4.post3.dev95073526 \ - madrigalweb==3.1.10 \ - pyproj==1.9.6 \ - bsddb3==6.2.6 \ - aacgmv2==2.5.2 \ - pymap3d==1.8.1 \ - astropy==2.0.14 \ - ipympl==0.5.6 \ - # sunpy==0.9.10 - -# Custom pip installation for any package that needs it -pip install apexpy==1.0.3 # have to install after installing numpy - -pip install cartopy==0.17.0 - -pip install spacepy==0.2.1 -source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB -# only update omni and qin-denton since maia.usno.navy.mil/ser7/tai-utc.dat leapseconds website was down at the time... -python -uc "import spacepy.toolbox; spacepy.toolbox.update(QDomni=True)" -# spacepy 0.2.1 doesn't clean up this file. Submitted issue and PR to spacepy, see https://github.com/spacepy/spacepy/pull/219 -rm OMNI_OMNI2_merged_20120213-v1.cdf - -# Installing mangopy (14 June 2018) -pip install git+https://github.com/astib/MANGO.git@2dd4ca5380dca54cac8d2180c3ad63fc041a5c67 - -# Install visuamisr -pip install git+https://github.com/asreimer/visuamisr.git@v2.0.3 - -# Install citationhelper -pip install git+https://github.com/EarthCubeInGeo/citationhelper.git@v0.2 - -# cleanup -rm -rf ~/.cache/pip/* diff --git a/resen-core/resources/helpers/setup_py36_env.sh b/resen-core/resources/helpers/setup_py36_env.sh deleted file mode 100644 index 5288a43..0000000 --- a/resen-core/resources/helpers/setup_py36_env.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -####################################################################################### -# -# A helper script for setting up a python 3.6 environment -# -####################################################################################### - -echo "**** Installing python 3.6 packages ****" - -# upgrade pip -pip install pip==20.1.1 - -# Now use pip to install everything we can -# Notes: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap -pip install -U jupyterhub==1.0.0 \ - jupyterlab==2.1.2 \ - notebook==6.0.3 \ - paramiko==2.7.1 \ - ipython==7.14.0 \ - pymongo==3.10.1 \ - mechanize==0.4.5 \ - cryptography==2.9.2 \ - pytest==5.4.2 \ - nbformat==5.0.6 \ - numpy==1.18.4 \ - matplotlib==3.2.1 \ - scipy==1.4.1 \ - tables==3.6.1 \ - sgp4==2.8 \ - pandas==1.0.3 \ - sympy==1.5.1 \ - sqlalchemy==1.3.17 \ - h5py==2.10.0 \ - pytz==2020.1 \ - netcdf4==1.5.3 \ - bokeh==2.0.2 \ - pyyaml==5.3.1 \ - cython==0.29.18 \ - pyproj==2.6.1.post1 \ - madrigalweb==3.1.12 \ - bsddb3==6.2.7 \ - aacgmv2==2.6.0 \ - pymap3d==2.4.1 \ - astropy==3.2.1 \ - plasmapy==0.3.1 \ - pydarn==1.0.0.1 \ - viresclient==0.6.1 \ - ipympl==0.5.6 \ - hdfviewer==0.11.0 \ - watermark==2.0.2 - -# Custom pip installation for any package that needs it -pip install apexpy==1.0.3 # have to install after installing numpy - -pip install cartopy==0.18.0 # have to install after installing numpy - -pip install spacepy==0.2.1 -source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB -# only update omni and qin-denton since maia.usno.navy.mil/ser7/tai-utc.dat leapseconds website was down at the time... -python -uc "import spacepy.toolbox; spacepy.toolbox.update(QDomni=True)" -# spacepy 0.2.1 doesn't clean up this file. Submitted issue and PR to spacepy, see https://github.com/spacepy/spacepy/pull/219 -rm OMNI_OMNI2_merged_20120213-v1.cdf - -# Installing mangopy (14 June 2018) -pip install git+https://github.com/astib/MANGO.git@2dd4ca5380dca54cac8d2180c3ad63fc041a5c67 - -# Install visuamisr -pip install git+https://github.com/asreimer/visuamisr.git@v2.0.3 - -# Install citationhelper -pip install git+https://github.com/EarthCubeInGeo/citationhelper.git@v0.2 - -# cleanup -rm -rf ~/.cache/pip/* diff --git a/resen-core/resources/helpers/setup_spacepy.sh b/resen-core/resources/helpers/setup_spacepy.sh deleted file mode 100644 index 589c9cc..0000000 --- a/resen-core/resources/helpers/setup_spacepy.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -####################################################################################### -# -# A helper script for installing spacepy -# -# Assumes you have conda installed with a py27 virtual environment -# and another environment with the name py36 -# -####################################################################################### - -echo "**** Installing spacepy ****" - -cd $BUILD -SPACEPY_BUILD_DIR=spacepy_build - -git clone https://github.com/spacepy/spacepy.git $SPACEPY_BUILD_DIR -cd $SPACEPY_BUILD_DIR -git checkout 8b8c2d56ebdcb1d3a59fd868a1c92f34ce16ec64 -b 0.1.6_withfix - -source /home/$NB_USER/envs/py27/bin/activate -pip install . - -deactivate - -source /home/$NB_USER/envs/py36/bin/activate -pip install . - -# some post-pip initialization of packages -cd $BUILD -source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB -python -c "import spacepy.toolbox; spacepy.toolbox.update()" -# a file about 195 MB called OMNI_OMNI2_merged_20120213-v1.cdf -# has been created and will be removed later on. The data is kept -# inside ~/.spacepy -echo "support_notice: False" >> /home/$NB_USER/.spacepy/spacepy.rc - -deactivate - -# cleanup -cd $BUILD -rm -r $SPACEPY_BUILD_DIR From 77bf1a0ee4cce50a06697790fe9f960d9deb6c63 Mon Sep 17 00:00:00 2001 From: Pablo Reyes Date: Tue, 20 Oct 2020 10:21:48 -0700 Subject: [PATCH 9/9] Adding jupyter-widget interaction with bokeh --- resen-core/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/resen-core/Dockerfile b/resen-core/Dockerfile index a06e309..ee83f7c 100644 --- a/resen-core/Dockerfile +++ b/resen-core/Dockerfile @@ -103,10 +103,11 @@ RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ python feature_download.py physical cultural cultural-extra --do_scales 110m && \ rm feature_download.py' -# Installing jupyter-widgets +# Installing jupyter-widgets with interaction with matplotlib and bokeh RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ jupyter labextension install @jupyter-widgets/jupyterlab-manager && \ - jupyter labextension install jupyter-matplotlib' + jupyter labextension install jupyter-matplotlib && \ + jupyter labextension install @bokeh/jupyter_bokeh' # Finally set up some stuff to make user experience better RUN echo "TERM=xterm-256color" >> /home/$NB_USER/.bashrc && \