diff --git a/resen-base/Dockerfile b/resen-base/Dockerfile index c963646..375c587 100644 --- a/resen-base/Dockerfile +++ b/resen-base/Dockerfile @@ -1,4 +1,6 @@ -FROM ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f +# based on jupyter:dockerstacks/base-notebook +# https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/Dockerfile +FROM ubuntu:focal-20200703@sha256:d5a6519d9f048100123c568eb83f7ef5bfcad69b01424f420f17c932b00dea76 LABEL maintainer="Ingeo Team " LABEL description="The base docker image from which resen-core images are built." @@ -16,16 +18,16 @@ ARG NB_GID="100" USER root ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get -yq dist-upgrade && \ - apt-get install -yq --no-install-recommends \ +RUN apt-get update \ + && apt-get install -yq --no-install-recommends \ wget \ bzip2 \ ca-certificates \ sudo \ locales \ fonts-liberation \ - run-one && \ - rm -rf /var/lib/apt/lists/* + run-one \ + && apt-get clean && rm -rf /var/lib/apt/lists/* # install tini ARG TINI_VERSION="v0.18.0" @@ -38,8 +40,8 @@ RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ locale-gen # Add a script that we will use to correct permissions after running certain commands -ADD resources/fix-permissions /usr/local/bin/fix-permissions -RUN chmod +x /usr/local/bin/fix-permissions +COPY resources/fix-permissions /usr/local/bin/fix-permissions +RUN chmod a+rx /usr/local/bin/fix-permissions # Enable prompt color in the skeleton .bashrc before creating the default NB_USER RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc @@ -61,6 +63,7 @@ CMD ["start-notebook.sh"] COPY resources/02-resen.sh /etc/profile.d/ # Add scripts for starting servers +# these came from github jupyter:dockerstacks/base-notebook COPY resources/start.sh /usr/local/bin/ COPY resources/start-notebook.sh /usr/local/bin/ COPY resources/start-singleuser.sh /usr/local/bin/ @@ -68,8 +71,13 @@ RUN chmod +x /usr/local/bin/start.sh && \ chmod +x /usr/local/bin/start-notebook.sh && \ chmod +x /usr/local/bin/start-singleuser.sh +# might want to add this in? +#COPY jupyter_notebook_config.py /etc/jupyter/ +#RUN fix-permissions /etc/jupyter/ -#stuff from jupyter/scipy-notebook + +#stuff from jupyter/scipy-notebook (we don't use conda) +#https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/scipy-notebook/Dockerfile RUN apt-get update && apt-get install -yq --no-install-recommends \ build-essential \ git \ @@ -81,15 +89,15 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ netcat \ pandoc \ python-dev \ - texlive-fonts-extra \ - texlive-fonts-recommended \ - texlive-generic-recommended \ - texlive-latex-base \ - texlive-latex-extra \ + # ---- nbconvert dependencies ---- texlive-xetex \ + texlive-fonts-recommended \ + texlive-plain-generic \ + # ---- tzdata \ unzip && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y --no-install-recommends ffmpeg dvipng \ + && apt-get clean && rm -rf /var/lib/apt/lists/* USER $NB_USER WORKDIR /home/$NB_USER diff --git a/resen-base/resources/02-resen.sh b/resen-base/resources/02-resen.sh index b92bd9c..de51944 100644 --- a/resen-base/resources/02-resen.sh +++ b/resen-base/resources/02-resen.sh @@ -1,2 +1 @@ -export SHELL=/bin/bash export LANG=C.UTF-8 diff --git a/resen-base/resources/start-notebook.sh b/resen-base/resources/start-notebook.sh index 4eb2e22..c40baa5 100644 --- a/resen-base/resources/start-notebook.sh +++ b/resen-base/resources/start-notebook.sh @@ -4,11 +4,16 @@ set -e +wrapper="" +if [[ "${RESTARTABLE}" == "yes" ]]; then + wrapper="run-one-constantly" +fi + if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then - # launched by JupyterHub, use single-user entrypoint - exec /usr/local/bin/start-singleuser.sh "$@" + # launched by JupyterHub, use single-user entrypoint + exec /usr/local/bin/start-singleuser.sh "$@" elif [[ ! -z "${JUPYTER_ENABLE_LAB}" ]]; then - . /usr/local/bin/start.sh jupyter lab "$@" + . /usr/local/bin/start.sh $wrapper jupyter lab "$@" else - . /usr/local/bin/start.sh jupyter notebook "$@" + . /usr/local/bin/start.sh $wrapper jupyter notebook "$@" fi \ No newline at end of file diff --git a/resen-base/resources/start-singleuser.sh b/resen-base/resources/start-singleuser.sh index 21125bc..a70a115 100644 --- a/resen-base/resources/start-singleuser.sh +++ b/resen-base/resources/start-singleuser.sh @@ -4,11 +4,12 @@ set -e -source /home/jovyan/envs/py36/bin/activate +# modified 9 Sept 2020 from https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/start-singleuser.sh +source /home/jovyan/envs/py38/bin/activate # set default ip to 0.0.0.0 if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then - NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" fi # handle some deprecated environment variables @@ -16,30 +17,26 @@ fi # These won't be passed from DockerSpawner 0.9, # so avoid specifying --arg=empty-string if [ ! -z "$NOTEBOOK_DIR" ]; then - NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_PORT" ]; then - NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_USER" ]; then - NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_COOKIE_NAME" ]; then - NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_BASE_URL" ]; then - NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_HUB_PREFIX" ]; then - NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" + NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" fi if [ ! -z "$JPY_HUB_API_URL" ]; then - NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" -fi -if [ ! -z "$JUPYTER_ENABLE_LAB" ]; then - NOTEBOOK_BIN="jupyter labhub" -else - NOTEBOOK_BIN="jupyterhub-singleuser" + NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" fi +NOTEBOOK_BIN="jupyterhub-singleuser" . /usr/local/bin/start.sh $NOTEBOOK_BIN $NOTEBOOK_ARGS "$@" \ No newline at end of file diff --git a/resen-base/resources/start.sh b/resen-base/resources/start.sh index beb4f13..e9f7634 100644 --- a/resen-base/resources/start.sh +++ b/resen-base/resources/start.sh @@ -66,7 +66,7 @@ if [ $(id -u) == 0 ] ; then # (it could be mounted, and we shouldn't create it if it already exists) if [[ ! -e "/home/$NB_USER" ]]; then echo "Relocating home dir to /home/$NB_USER" - mv /home/jovyan "/home/$NB_USER" + mv /home/jovyan "/home/$NB_USER" || ln -s /home/jovyan "/home/$NB_USER" fi # if workdir is in /home/jovyan, cd to /home/$NB_USER if [[ "$PWD/" == "/home/jovyan/"* ]]; then @@ -76,29 +76,27 @@ if [ $(id -u) == 0 ] ; then fi fi - # Change UID of NB_USER to NB_UID if it does not match - if [ "$NB_UID" != $(id -u $NB_USER) ] ; then - echo "Set $NB_USER UID to: $NB_UID" - usermod -u $NB_UID $NB_USER - fi - - # Set NB_USER primary gid to NB_GID (after making the group). Set - # supplementary gids to NB_GID and 100. - if [ "$NB_GID" != $(id -g $NB_USER) ] ; then - echo "Add $NB_USER to group: $NB_GID" - groupadd -g $NB_GID -o ${NB_GROUP:-${NB_USER}} - usermod -g $NB_GID -aG 100 $NB_USER + # Change UID:GID of NB_USER to NB_UID:NB_GID if it does not match + if [ "$NB_UID" != $(id -u $NB_USER) ] || [ "$NB_GID" != $(id -g $NB_USER) ]; then + echo "Set user $NB_USER UID:GID to: $NB_UID:$NB_GID" + if [ "$NB_GID" != $(id -g $NB_USER) ]; then + groupadd -f -g $NB_GID -o ${NB_GROUP:-${NB_USER}} + fi + userdel $NB_USER + useradd --home /home/$NB_USER -u $NB_UID -g $NB_GID -G 100 -l $NB_USER fi # Enable sudo if requested if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then - # echo "Granting $NB_USER sudo access and appending $CONDA_DIR/bin to sudo PATH" echo "Granting $NB_USER sudo access" + # and appending $CONDA_DIR/bin to sudo PATH" # see comment below echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook fi - # Add $CONDA_DIR/bin to sudo secure_path (not needed for resen-core) - #sed -r "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path + # Add $CONDA_DIR/bin to sudo secure_path + # removed 9 Sept 2020 from https://github.com/jupyter/docker-stacks/blob/d4a632faefb34ca9f502b84b8e5d6a1108dbda5b/base-notebook/start.sh + # we don't use conda in our docker image + # sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path # Exec the command as NB_USER with the PATH and the rest of # the environment preserved @@ -106,7 +104,7 @@ if [ $(id -u) == 0 ] ; then echo "Executing the command: ${cmd[@]}" exec sudo -E -H -u $NB_USER PATH=$PATH XDG_CACHE_HOME=/home/$NB_USER/.cache PYTHONPATH=${PYTHONPATH:-} "${cmd[@]}" else - if [[ "$NB_UID" == "$(id -u jovyan)" && "$NB_GID" == "$(id -g jovyan)" ]]; then + if [[ "$NB_UID" == "$(id -u jovyan 2>/dev/null)" && "$NB_GID" == "$(id -g jovyan 2>/dev/null)" ]]; then # User is not attempting to override user/group via environment # variables, but they could still have overridden the uid/gid that # container runs as. Check that the user has an entry in the passwd @@ -149,4 +147,4 @@ else run-hooks /usr/local/bin/before-notebook.d echo "Executing the command: ${cmd[@]}" exec "${cmd[@]}" -fi +fi \ No newline at end of file diff --git a/resen-core/Dockerfile b/resen-core/Dockerfile index d93269a..ee83f7c 100644 --- a/resen-core/Dockerfile +++ b/resen-core/Dockerfile @@ -1,4 +1,5 @@ -FROM earthcubeingeo/resen-base:2020.1.0 +#FROM earthcubeingeo/resen-base:2020.2.0 +FROM resenbasetest:latest LABEL maintainer="Ingeo Team " LABEL description="The core docker image used by resen." @@ -27,33 +28,28 @@ RUN apt-get update && \ # Install python stuff RUN apt-get update && \ apt-get install -yq --no-install-recommends \ - python2.7 python3.6 \ - python-virtualenv python3-virtualenv \ - python-setuptools python3-setuptools \ - python-dev python3-dev \ + python3-virtualenv \ + python3-setuptools \ + python3-dev \ + python3.8 \ libproj-dev proj-data proj-bin \ libgeos++-dev libgeos-dev \ libdb-dev \ - python-tk python3-tk && \ + python3-tk && \ rm -rf /var/lib/apt/lists/* # Install system libraries needed by python packages RUN apt-get update && \ apt-get install -yq --no-install-recommends \ - libncurses5-dev && \ + libncurses5-dev \ + cm-super \ + npm nodejs && \ rm -rf /var/lib/apt/lists/* -# Install nodejs and npm needed for matplotlib widgets -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - && \ - apt-get install -yq --no-install-recommends nodejs && \ - rm -rf /var/lib/apt/lists/* - -# Create python 2.7 and python 3.6 virtual environments +# Create python 3.8 virtual environment USER $NB_USER RUN /bin/bash -c 'mkdir -p /home/$NB_USER/envs && \ - python3 -m virtualenv -p /usr/bin/python3.6 /home/$NB_USER/envs/py36 && \ - python3 -m virtualenv -p /usr/bin/python2.7 /home/$NB_USER/envs/py27' - + python3 -m virtualenv -p /usr/bin/python3.8 /home/$NB_USER/envs/py38' WORKDIR /home/$NB_USER @@ -67,23 +63,17 @@ RUN /bin/bash -c 'bash install_CDF.sh && rm install_CDF.sh' # Now install all python packages USER $NB_USER -COPY resources/helpers/setup_py36_env.sh . -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && bash setup_py36_env.sh && rm setup_py36_env.sh' - -COPY resources/helpers/setup_py27_env.sh . -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py27/bin/activate && bash setup_py27_env.sh && rm setup_py27_env.sh' +COPY resources/helpers/setup_py38_env.sh . +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && bash setup_py38_env.sh && rm setup_py38_env.sh' # register python environments with ipykernel -RUN /bin/bash -c 'source /home/$NB_USER/envs/py27/bin/activate && \ - pip install ipykernel==4.10.0 && \ - python -m ipykernel install --user --name py27 --display-name "py27"' -RUN /bin/bash -c 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -c 'source /home/$NB_USER/envs/py38/bin/activate && \ pip install ipykernel==5.1.1 && \ - python -m ipykernel install --user --name py36 --display-name "py36"' + python -m ipykernel install --user --name py38 --display-name "py38"' # remove python3 kernel from showing up in jupyter -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ yes | jupyter kernelspec remove python3' # Install basemap @@ -95,25 +85,29 @@ COPY resources/helpers/setup_pyglow.sh . RUN /bin/bash -cl 'bash setup_pyglow.sh && rm setup_pyglow.sh' # Install citationhelper +USER root COPY resources/pkg_citations.json /home/$NB_USER/cache/ +RUN chown -R $NB_USER:users "/home/$NB_USER/cache/" +USER $NB_USER # create work directory for user RUN mkdir /home/$NB_USER/work -# set default python environment to py36 -RUN /bin/bash -c 'echo "source /home/$NB_USER/envs/py36/bin/activate" >> /home/$NB_USER/.bashrc' +# set default python environment to py38 +RUN /bin/bash -c 'echo "source /home/$NB_USER/envs/py38/bin/activate" >> /home/$NB_USER/.bashrc' # post installation stuff # Download 110m scale cartopy data COPY resources/feature_download.py . -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ python feature_download.py physical cultural cultural-extra --do_scales 110m && \ rm feature_download.py' -# Installing jupyter-widgets -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \ +# Installing jupyter-widgets with interaction with matplotlib and bokeh +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ jupyter labextension install @jupyter-widgets/jupyterlab-manager && \ - jupyter labextension install jupyter-matplotlib' + jupyter labextension install jupyter-matplotlib && \ + jupyter labextension install @bokeh/jupyter_bokeh' # Finally set up some stuff to make user experience better RUN echo "TERM=xterm-256color" >> /home/$NB_USER/.bashrc && \ @@ -121,9 +115,9 @@ RUN echo "TERM=xterm-256color" >> /home/$NB_USER/.bashrc && \ echo "export CITEHELP_REFFILE=/home/$NB_USER/cache/pkg_citations.json" >> /home/$NB_USER/.bashrc # Set default jupyter terminal shell -RUN /bin/bash -cl 'source /home/$NB_USER/envs/py36/bin/activate && \ +RUN /bin/bash -cl 'source /home/$NB_USER/envs/py38/bin/activate && \ jupyter lab --generate-config' -RUN sed -i -e 's/#c.NotebookApp.terminado_settings = {}/c.NotebookApp.terminado_settings = {\"shell_command\"\:[\"\/bin\/bash\"]}/' /home/jovyan/.jupyter/jupyter_notebook_config.py +RUN sed -i -e 's/# c.NotebookApp.terminado_settings = {}/c.NotebookApp.terminado_settings = {\"shell_command\"\:[\"\/bin\/bash\"]}/' /home/jovyan/.jupyter/jupyter_notebook_config.py # Set the jupyterlab user-settings RUN /bin/bash -cl 'mkdir -p /home/jovyan/.jupyter/lab/user-settings/@jupyterlab/extensionmanager-extension && \ diff --git a/resen-core/resources/helpers/setup_basemap.sh b/resen-core/resources/helpers/setup_basemap.sh index dbb7e2e..ee05f5b 100644 --- a/resen-core/resources/helpers/setup_basemap.sh +++ b/resen-core/resources/helpers/setup_basemap.sh @@ -3,26 +3,21 @@ # # A helper script for installing basemap # -# Assumes you have py27 and py36 python virtual environments +# Assumes you have py38 python virtual environments # ####################################################################################### # git clone --depth 0 https://github.com/matplotlib/basemap.git # VERY SLOW -wget https://github.com/matplotlib/basemap/archive/v1.2.0rel.tar.gz -tar -zxf v1.2.0rel.tar.gz -cd basemap-1.2.0rel +wget https://github.com/matplotlib/basemap/archive/v1.2.2rel.tar.gz +tar -zxf v1.2.2rel.tar.gz +cd basemap-1.2.2rel -source ${HOME}/envs/py27/bin/activate -pip install . - -deactivate - -source ${HOME}/envs/py36/bin/activate +source ${HOME}/envs/py38/bin/activate pip install . deactivate # cleanup cd .. -rm -rf basemap-1.2.0rel v1.2.0rel.tar.gz +rm -rf basemap-1.2.2rel v1.2.2rel.tar.gz rm ${HOME}/.wget-hsts diff --git a/resen-core/resources/helpers/setup_py27_env.sh b/resen-core/resources/helpers/setup_py27_env.sh deleted file mode 100644 index 940c081..0000000 --- a/resen-core/resources/helpers/setup_py27_env.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -####################################################################################### -# -# A helper script for setting up a python 2.7 environment -# -####################################################################################### - -echo "**** Installing python 2.7 packages ****" - -# upgrade pip -pip install pip==20.1.1 - -# Now use pip to install everything we can -# Notes: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap -pip install -U paramiko==2.4.2 \ - ipython==5.8.0 \ - prompt_toolkit==1.0.16 \ - pymongo==3.7.2 \ - mechanize==0.4.1 \ - cryptography==2.6.1 \ - pytest==3.0.5 \ - nbformat==4.4.0 \ - numpy==1.16.2 \ - matplotlib==2.2.4 \ - scipy==1.2.1 \ - tables==3.5.1 \ - sgp4==1.4 \ - pandas==0.24.2 \ - sympy==1.1.0 \ - sqlalchemy==1.3.1 \ - h5py==2.9.0 \ - pytz==2018.9 \ - netcdf4==1.4.3 \ - bokeh==1.0.4 \ - pyyaml==5.1 \ - cython==0.29.6 \ - sciunit2==0.4.post3.dev95073526 \ - madrigalweb==3.1.10 \ - pyproj==1.9.6 \ - bsddb3==6.2.6 \ - aacgmv2==2.5.2 \ - pymap3d==1.8.1 \ - astropy==2.0.14 \ - ipympl==0.5.6 \ - # sunpy==0.9.10 - -# Custom pip installation for any package that needs it -pip install apexpy==1.0.3 # have to install after installing numpy - -pip install cartopy==0.17.0 - -pip install spacepy==0.2.1 -source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB -# only update omni and qin-denton since maia.usno.navy.mil/ser7/tai-utc.dat leapseconds website was down at the time... -python -uc "import spacepy.toolbox; spacepy.toolbox.update(QDomni=True)" -# spacepy 0.2.1 doesn't clean up this file. Submitted issue and PR to spacepy, see https://github.com/spacepy/spacepy/pull/219 -rm OMNI_OMNI2_merged_20120213-v1.cdf - -# Installing mangopy (14 June 2018) -pip install git+https://github.com/astib/MANGO.git@2dd4ca5380dca54cac8d2180c3ad63fc041a5c67 - -# Install visuamisr -pip install git+https://github.com/asreimer/visuamisr.git@v2.0.3 - -# Install citationhelper -pip install git+https://github.com/EarthCubeInGeo/citationhelper.git@v0.2 - -# cleanup -rm -rf ~/.cache/pip/* diff --git a/resen-core/resources/helpers/setup_py36_env.sh b/resen-core/resources/helpers/setup_py38_env.sh similarity index 50% rename from resen-core/resources/helpers/setup_py36_env.sh rename to resen-core/resources/helpers/setup_py38_env.sh index 5288a43..376c9e8 100644 --- a/resen-core/resources/helpers/setup_py36_env.sh +++ b/resen-core/resources/helpers/setup_py38_env.sh @@ -1,59 +1,69 @@ #!/bin/bash ####################################################################################### # -# A helper script for setting up a python 3.6 environment +# A helper script for setting up a python 3.8 environment # ####################################################################################### -echo "**** Installing python 3.6 packages ****" +echo "**** Installing python 3.8 packages ****" # upgrade pip -pip install pip==20.1.1 +pip install pip==20.2.4 # Now use pip to install everything we can -# Notes: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap -pip install -U jupyterhub==1.0.0 \ - jupyterlab==2.1.2 \ - notebook==6.0.3 \ - paramiko==2.7.1 \ - ipython==7.14.0 \ - pymongo==3.10.1 \ - mechanize==0.4.5 \ - cryptography==2.9.2 \ - pytest==5.4.2 \ - nbformat==5.0.6 \ - numpy==1.18.4 \ - matplotlib==3.2.1 \ - scipy==1.4.1 \ +# NOTES: pyproj==1.9.6 required for basemap, 2.0.0 breaks basemap +# jupyterhub, jupyterlab, notebook should all follow jupyterstacks versions +pip install -U jupyterhub==1.1.0 \ + jupyterlab==2.2.5 \ + notebook==6.1.3 \ + paramiko==2.7.2 \ + ipython==7.18.1 \ + pytest==5.4.3 \ + nbformat==5.0.7 \ + numpy==1.19.1 \ + matplotlib==3.3.1 \ + scipy==1.5.2 \ tables==3.6.1 \ - sgp4==2.8 \ - pandas==1.0.3 \ - sympy==1.5.1 \ - sqlalchemy==1.3.17 \ + sgp4==2.12 \ + pandas==1.1.2 \ + sympy==1.6.2 \ + sqlalchemy==1.3.19 \ h5py==2.10.0 \ pytz==2020.1 \ - netcdf4==1.5.3 \ - bokeh==2.0.2 \ + netcdf4==1.5.4 \ + bokeh==2.2.1 \ pyyaml==5.3.1 \ - cython==0.29.18 \ - pyproj==2.6.1.post1 \ + cython==0.29.21 \ + pyproj==1.9.6 \ madrigalweb==3.1.12 \ bsddb3==6.2.7 \ aacgmv2==2.6.0 \ - pymap3d==2.4.1 \ - astropy==3.2.1 \ - plasmapy==0.3.1 \ - pydarn==1.0.0.1 \ - viresclient==0.6.1 \ - ipympl==0.5.6 \ + astropy==4.0.1.post1 \ + plasmapy==0.4.0 \ + pydarn==1.1.0 \ + viresclient==0.6.2 \ + ipympl==0.5.7 \ hdfviewer==0.11.0 \ - watermark==2.0.2 + watermark==2.0.2 \ + seaborn==0.11.0 \ + pyephem==3.7.7.0 \ + skyfield==1.26 \ + sciunit2==0.4.post58.dev224550607 + +#build matplotlib font cache +MPLBACKEND=Agg python -uc "import matplotlib.pyplot" + +# install sunpy +pip install -U sunpy==2.0.1 beautifulsoup4==4.9.1 drms==0.5.7 zeep==3.4.0 tqdm==4.48.2 # Custom pip installation for any package that needs it pip install apexpy==1.0.3 # have to install after installing numpy pip install cartopy==0.18.0 # have to install after installing numpy +pip install pymap3d==2.4.1 # has side effects when importing first time +python -uc "import pymap3d" + pip install spacepy==0.2.1 source /usr/local/bin/definitions.B # to set the CDF definitios including $CDF_LIB # only update omni and qin-denton since maia.usno.navy.mil/ser7/tai-utc.dat leapseconds website was down at the time... diff --git a/resen-core/resources/helpers/setup_pyglow.sh b/resen-core/resources/helpers/setup_pyglow.sh index 1605910..6363b1f 100755 --- a/resen-core/resources/helpers/setup_pyglow.sh +++ b/resen-core/resources/helpers/setup_pyglow.sh @@ -3,8 +3,7 @@ # # A helper script for installing pyglow # -# Assumes you have conda installed with a py27 virtual environment -# and another environment with the name py36 +# Assumes you have a py38 python virtual environments # ####################################################################################### @@ -14,17 +13,9 @@ PYGLOW_BUILD_DIR=pyglow_build git clone https://github.com/timduly4/pyglow.git $PYGLOW_BUILD_DIR cd $PYGLOW_BUILD_DIR -git checkout 054bf800263c01ecfa783741c9cfe8d3d6c259ca -b master_20190904 +git checkout 36e998524cf603cf61ca7d98ced61e2f945ab1df -b master_20201019 -source ${HOME}/envs/py27/bin/activate - -make -C src/pyglow/models source -pip install . -python -c "import pyglow" - -deactivate - -source ${HOME}/envs/py36/bin/activate +source ${HOME}/envs/py38/bin/activate make -C src/pyglow/models source pip install . diff --git a/resen-core/testing/tests/test_py27_imports.py b/resen-core/testing/tests/test_py27_imports.py deleted file mode 100644 index f88c18c..0000000 --- a/resen-core/testing/tests/test_py27_imports.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" - test_imports.py - - This is intended to be used to check that all expected python packages are avaiable. Works - well for identifying strangeness that may occur with conda when installing multiple python - packages with differing dependency requirements. - -""" - -import unittest -import importlib - -class ImportTester(unittest.TestCase): - # In the future, get this list from a Pipfile and - # include a test for package version number - packages = ['paramiko','IPython','pymongo','mechanize','netCDF4','cryptography','pytest', - 'nbformat','numpy','matplotlib','scipy','tables','sgp4','pandas','sympy', - 'sqlalchemy','h5py','pytz','bokeh','yaml','cython','sciunit2','bsddb3', - 'apexpy','cartopy','spacepy','aacgmv2','pymap3d','mangopy','astropy', - 'madrigalWeb','visuamisr','citationhelper'] - - def test_imports(self): - # Try to import all the packages - for package in self.packages: - print("Importing %s" % package) - importlib.import_module(package) - - # if we get here, everything imported fine! - self.assertTrue(True) - -if __name__ == '__main__': - unittest.main() - -# Still need to do: -# import tables -# import pyglow -# import sgp4 -# import pydarn -# import sympy -# import SQLAlchemy - -# import bokeh -# import ipyparallel - -# from Cython.Build import cythonize -# import sunpy -# import heliopy - - - - - diff --git a/resen-core/testing/tests/test_py36_imports.py b/resen-core/testing/tests/test_py38_imports.py similarity index 89% rename from resen-core/testing/tests/test_py36_imports.py rename to resen-core/testing/tests/test_py38_imports.py index f29031d..15ee56d 100644 --- a/resen-core/testing/tests/test_py36_imports.py +++ b/resen-core/testing/tests/test_py38_imports.py @@ -15,11 +15,12 @@ class ImportTester(unittest.TestCase): # In the future, get this list from a Pipfile and # include a test for package version number - packages = ['paramiko','IPython','pymongo','mechanize','netCDF4','cryptography','pytest', + packages = ['paramiko','IPython','seaborn','netCDF4','cryptography','pytest', 'nbformat','numpy','matplotlib','scipy','tables','sgp4','pandas','sympy', 'sqlalchemy','h5py','pytz','bokeh','yaml','cython','apexpy','cartopy', 'spacepy','aacgmv2','pymap3d','mangopy','madrigalWeb','visuamisr', - 'citationhelper','viresclient','hdfviewer','watermark','pydarn','plasmapy'] + 'citationhelper','viresclient','hdfviewer','watermark','pydarn','plasmapy', + 'sunpy','ephem','skyfield'] def test_imports(self): # Try to import all the packages