From 5ce5888b6f9a233c1921700e2028f207051ec9c1 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 15 Jun 2021 22:07:54 -0700 Subject: [PATCH 01/16] Dynamically select the right miniforge arch and unpin the ROOT_CONTAINER so that buildx can build arm64 and amd64 cleanup --- Makefile | 3 ++- base-notebook/Dockerfile | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 35722294a8..70ece996d1 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ # Use bash for inline if-statements in arch_patch target SHELL:=bash +ARCH:=$(shell uname -m) OWNER?=jupyter # Need to list the images in build dependency order @@ -29,7 +30,7 @@ help: build/%: DARGS?= build/%: ## build the latest image for a stack - docker build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) + docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --push --platform linux/arm64,linux/amd64 @echo -n "Built image size: " @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 88dc3eb73d..f24ebdfa17 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -3,8 +3,7 @@ # Ubuntu 20.04 (focal) # https://hub.docker.com/_/ubuntu/?tab=tags&name=focal -# OS/ARCH: linux/amd64 -ARG ROOT_CONTAINER=ubuntu:focal-20210609@sha256:376209074d481dca0a9cf4282710cd30a9e7ff402dea8261acdaaf57a18971dd +ARG ROOT_CONTAINER=ubuntu:focal-20210609 ARG BASE_CONTAINER=$ROOT_CONTAINER FROM $BASE_CONTAINER @@ -28,7 +27,7 @@ ARG conda_version="4.10.2" # Miniforge installer patch version ARG miniforge_patch_number="0" # Miniforge installer architecture -ARG miniforge_arch="x86_64" +#ARG miniforge_arch="x86_64" # Package Manager and Python implementation to use (https://github.com/conda-forge/miniforge) # - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy # - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy @@ -37,9 +36,11 @@ ARG miniforge_python="Mambaforge" # Miniforge archive to install ARG miniforge_version="${conda_version}-${miniforge_patch_number}" # Miniforge installer -ARG miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${miniforge_arch}.sh" -# Miniforge checksum -ARG miniforge_checksum="1e89ee86afa06e23b2478579be16a33fff6cff346314f6a6382fd20b1f83e669" +RUN export arch=$(uname -m) && \ + if [ "$arch" == "aarm64" ]; then \ + arch="arm64"; \ + fi; \ +RUN export miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${arch}.sh" # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) @@ -105,8 +106,7 @@ WORKDIR /tmp # Prerequisites installation: conda, mamba, pip, tini RUN wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}" && \ - echo "${miniforge_checksum} *${miniforge_installer}" | sha256sum --check && \ - /bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \ + /bin/bash "${miniforge_installer}" -f -b -p ${CONDA_DIR} && \ rm "${miniforge_installer}" && \ # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html echo "conda ${CONDA_VERSION}" >> "${CONDA_DIR}/conda-meta/pinned" && \ From 5a125bdbbb1bd65954adb4b76a1348fa39f6ec3e Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 15 Jun 2021 22:08:59 -0700 Subject: [PATCH 02/16] Remove the commented out hard set of arch --- base-notebook/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index f24ebdfa17..0e5212bbaf 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -25,9 +25,7 @@ USER root # Conda version ARG conda_version="4.10.2" # Miniforge installer patch version -ARG miniforge_patch_number="0" -# Miniforge installer architecture -#ARG miniforge_arch="x86_64" +ARG miniforge_patch_number="5" # Package Manager and Python implementation to use (https://github.com/conda-forge/miniforge) # - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy # - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy From e6a4667f6ac064ee46b4d6a180983fc9ba810c09 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 21 Jun 2021 20:41:12 -0700 Subject: [PATCH 03/16] Address code review comments from mathbunnyru --- Makefile | 1 - base-notebook/Dockerfile | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 70ece996d1..164d273a6d 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,6 @@ # Use bash for inline if-statements in arch_patch target SHELL:=bash -ARCH:=$(shell uname -m) OWNER?=jupyter # Need to list the images in build dependency order diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 0e5212bbaf..75033c906f 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -34,11 +34,11 @@ ARG miniforge_python="Mambaforge" # Miniforge archive to install ARG miniforge_version="${conda_version}-${miniforge_patch_number}" # Miniforge installer -RUN export arch=$(uname -m) && \ +RUN export miniforge_arch=$(uname -m) && \ if [ "$arch" == "aarm64" ]; then \ - arch="arm64"; \ + miniforge_arch="arm64"; \ fi; \ -RUN export miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${arch}.sh" +RUN export miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${miniforge_arch}.sh" # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) @@ -104,7 +104,7 @@ WORKDIR /tmp # Prerequisites installation: conda, mamba, pip, tini RUN wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}" && \ - /bin/bash "${miniforge_installer}" -f -b -p ${CONDA_DIR} && \ + /bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \ rm "${miniforge_installer}" && \ # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html echo "conda ${CONDA_VERSION}" >> "${CONDA_DIR}/conda-meta/pinned" && \ From 3b7d143af464b15ba2a5e61b7cf4d04debdeeb1a Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Mon, 21 Jun 2021 20:50:24 -0700 Subject: [PATCH 04/16] Add setting the owner to respective dockerhbub username --- docs/contributing/features.md | 1 + docs/contributing/packages.md | 1 + docs/contributing/tests.md | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/contributing/features.md b/docs/contributing/features.md index 54ecc15774..81ec860b70 100644 --- a/docs/contributing/features.md +++ b/docs/contributing/features.md @@ -42,6 +42,7 @@ If there's agreement that the feature belongs in one or more of the core stacks: If you use `make`, call: ```bash + OWNER={yourdockerhubusername} make build/somestack-notebook ``` diff --git a/docs/contributing/packages.md b/docs/contributing/packages.md index d9fc9c7ed7..2cc99c4d1f 100644 --- a/docs/contributing/packages.md +++ b/docs/contributing/packages.md @@ -18,6 +18,7 @@ Please follow the process below to update a package version: If you use `make`, call: ```bash + OWNER={yourdockerhubusername} make build/somestack-notebook ``` diff --git a/docs/contributing/tests.md b/docs/contributing/tests.md index 58bed9092d..a6175420db 100644 --- a/docs/contributing/tests.md +++ b/docs/contributing/tests.md @@ -26,6 +26,7 @@ Please follow the process below to add new tests: If you use `make`, call: ```bash + OWNER={yourdockerhubusername} make build/somestack-notebook make test/somestack-notebook ``` From ec579faeeef24c86fade37999dd0eb4ce89c44b4 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Jun 2021 14:02:07 -0700 Subject: [PATCH 05/16] Revert "Add setting the owner to respective dockerhbub username" This reverts commit a8d9c468a7d35dbda33ae902cc524db599519d8a. --- docs/contributing/features.md | 1 - docs/contributing/packages.md | 1 - docs/contributing/tests.md | 1 - 3 files changed, 3 deletions(-) diff --git a/docs/contributing/features.md b/docs/contributing/features.md index 81ec860b70..54ecc15774 100644 --- a/docs/contributing/features.md +++ b/docs/contributing/features.md @@ -42,7 +42,6 @@ If there's agreement that the feature belongs in one or more of the core stacks: If you use `make`, call: ```bash - OWNER={yourdockerhubusername} make build/somestack-notebook ``` diff --git a/docs/contributing/packages.md b/docs/contributing/packages.md index 2cc99c4d1f..d9fc9c7ed7 100644 --- a/docs/contributing/packages.md +++ b/docs/contributing/packages.md @@ -18,7 +18,6 @@ Please follow the process below to update a package version: If you use `make`, call: ```bash - OWNER={yourdockerhubusername} make build/somestack-notebook ``` diff --git a/docs/contributing/tests.md b/docs/contributing/tests.md index a6175420db..58bed9092d 100644 --- a/docs/contributing/tests.md +++ b/docs/contributing/tests.md @@ -26,7 +26,6 @@ Please follow the process below to add new tests: If you use `make`, call: ```bash - OWNER={yourdockerhubusername} make build/somestack-notebook make test/somestack-notebook ``` From bae7ce7859d39c1e49a59f539c632034718dcf0c Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 29 Jun 2021 14:18:07 -0700 Subject: [PATCH 06/16] Fix up the dockerfile, make the default buildx compatible with the CI and add a comment about how to do cross-platform builds. --- Makefile | 6 +++++- base-notebook/Dockerfile | 17 +++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 164d273a6d..1da4b073a3 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,10 @@ # Use bash for inline if-statements in arch_patch target SHELL:=bash OWNER?=jupyter +# By default we still target single image amd64 & docker output type. +# To cross build run make PLATFORM="linux/amd64,linux/arm64" BUILDX_OUTPUT_TYPE="registry" +PLATFORM ?= "linux/amd64" +BUILDX_OUTPUT_TYPE ?= "docker" # Need to list the images in build dependency order ALL_IMAGES:=base-notebook \ @@ -29,7 +33,7 @@ help: build/%: DARGS?= build/%: ## build the latest image for a stack - docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --push --platform linux/arm64,linux/amd64 + docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --output=type="${BUILDX_OUTPUT_TYPE}" --platform "${PLATFORM}" @echo -n "Built image size: " @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 75033c906f..8b4988cbf4 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -30,15 +30,9 @@ ARG miniforge_patch_number="5" # - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy # - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy ARG miniforge_python="Mambaforge" - # Miniforge archive to install ARG miniforge_version="${conda_version}-${miniforge_patch_number}" -# Miniforge installer -RUN export miniforge_arch=$(uname -m) && \ - if [ "$arch" == "aarm64" ]; then \ - miniforge_arch="arm64"; \ - fi; \ -RUN export miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${miniforge_arch}.sh" + # Install all OS dependencies for notebook server that starts but lacks all # features (e.g., download as all possible file formats) @@ -103,7 +97,14 @@ RUN mkdir "/home/${NB_USER}/work" && \ WORKDIR /tmp # Prerequisites installation: conda, mamba, pip, tini -RUN wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}" && \ +RUN set -x && \ + # Miniforge installer + export miniforge_arch=$(uname -m) && \ + if [ "$miniforge_arch" == "aarm64" ]; then \ + miniforge_arch="arm64"; \ + fi; \ + export miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${miniforge_arch}.sh" && \ + wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}" && \ /bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \ rm "${miniforge_installer}" && \ # Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html From c87bf141fc9214ce16ab0a87bbd97581dbec18dc Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 30 Jun 2021 16:45:00 -0700 Subject: [PATCH 07/16] Update the docs --- README.md | 4 ++++ docs/index.rst | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/README.md b/README.md index ac3b42efef..d55282777f 100644 --- a/README.md +++ b/README.md @@ -104,3 +104,7 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st - [Jupyter Discourse Forum](https://discourse.jupyter.org/) - [Jupyter Website](https://jupyter.org) - [Images on DockerHub](https://hub.docker.com/u/jupyter) + +## Architectures + +Currently published containers only support x86, however cross-platform image support for x86 & arm64 images is now present. diff --git a/docs/index.rst b/docs/index.rst index 65119716d1..0d97c0209d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,11 @@ Docker destroys the container after notebook server exit, but any files written docker run --rm -p 10000:8888 -e JUPYTER_ENABLE_LAB=yes -v "${PWD}":/home/jovyan/work jupyter/datascience-notebook:33add21fab64 +Architectures +----------- +Currently published containers only support x86, however cross-platform image support for x86 & arm64 images is now present. + + Table of Contents ----------------- From 55d5f31a28e851c854890e6c842cbdd0f97971de Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 6 Jul 2021 11:53:29 -0700 Subject: [PATCH 08/16] Refactor the Makefile to support cross-building images incrementally, add ARG OWNER to the Dockerfile's so people can more easily push to their own, add docker buildx & ARM64 support to the CI --- .github/workflows/docker.yml | 25 ++++++++++++++++++++++--- Makefile | 32 ++++++++++++++++++++------------ all-spark-notebook/Dockerfile | 3 ++- base-notebook/Dockerfile | 3 +-- binder/Dockerfile | 3 ++- datascience-notebook/Dockerfile | 3 ++- minimal-notebook/Dockerfile | 3 ++- pyspark-notebook/Dockerfile | 3 ++- r-notebook/Dockerfile | 3 ++- scipy-notebook/Dockerfile | 3 ++- tensorflow-notebook/Dockerfile | 3 ++- 11 files changed, 59 insertions(+), 25 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1a6d9d19db..2e878651a6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,6 +31,28 @@ jobs: !contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.pull_request.title, 'ci skip') steps: + - name: Should we push this image to a public registry? + run: | + if [ "${{ startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/main') }}" = "true" ]; then + # Empty => Docker Hub + echo "REGISTRY=" >> $GITHUB_ENV + else + echo "REGISTRY=localhost:5000/" >> $GITHUB_ENV + fi + - uses: actions/checkout@v2 + + # Setup docker to build for multiple platforms, see: + # https://github.com/docker/build-push-action/tree/v2.4.0#usage + # https://github.com/docker/build-push-action/blob/v2.4.0/docs/advanced/multi-platform.md + + - name: Set up QEMU (for docker buildx) + uses: docker/setup-qemu-action@25f0500ff22e406f7191a2a8ba8cda16901ca018 # associated tag: v1.0.2 + + - name: Set up Docker Buildx (for multi-arch builds) + uses: docker/setup-buildx-action@2a4b53665e15ce7d7049afb11ff1f70ff1610609 # associated tag: v1.1.2 + with: + # Allows pushing to registry on localhost:5000 + driver-opts: network=host - name: Clone Main Repo uses: actions/checkout@v2 with: @@ -67,6 +89,3 @@ jobs: with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} - - name: Push Images to DockerHub - if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' - run: make -C main push-all diff --git a/Makefile b/Makefile index 1da4b073a3..afbcfe49e3 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,19 @@ SHELL:=bash OWNER?=jupyter # By default we still target single image amd64 & docker output type. -# To cross build run make PLATFORM="linux/amd64,linux/arm64" BUILDX_OUTPUT_TYPE="registry" -PLATFORM ?= "linux/amd64" -BUILDX_OUTPUT_TYPE ?= "docker" # Need to list the images in build dependency order +# These are images we can cross-build +CROSS_IMAGES:= base-notebook \ + minimal-notebook +# These images that aren't currently supported for cross-building, your help is welcome. +X86_IMAGES:= r-notebook \ + scipy-notebook \ + tensorflow-notebook \ + datascience-notebook \ + pyspark-notebook \ + all-spark-notebook +# All of the images ALL_IMAGES:=base-notebook \ minimal-notebook \ r-notebook \ @@ -33,12 +41,18 @@ help: build/%: DARGS?= build/%: ## build the latest image for a stack - docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --output=type="${BUILDX_OUTPUT_TYPE}" --platform "${PLATFORM}" + docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --platform "linux/amd64" --build-arg OWNER="${OWNER}" --push @echo -n "Built image size: " @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" -build-all: $(foreach I,$(ALL_IMAGES), build/$(I) ) ## build all stacks -build-test-all: $(foreach I,$(ALL_IMAGES), build/$(I) test/$(I) ) ## build and test all stacks +build-cross/%: DARGS?= +build-cross/%: ## build the latest image for a stack on x86 and ARM + docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --platform "linux/amd64,linux/arm64" --build-arg OWNER="${OWNER}" --push + @echo -n "Built image size: " + @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" + +build-all: $(foreach I,$(CROSS_IMAGES), build-cross/$(I) ) $(foreach I,$(X86_IMAGES), build/$(I) ) ## build all stacks +build-test-all: build-all $(foreach I,$(ALL_IMAGES), test/$(I) ) ## build and test all stacks check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental) @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py @@ -100,12 +114,6 @@ pull/%: ## pull a jupyter image pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I) ) ## pull all images -push/%: DARGS?= -push/%: ## push all tags for a jupyter image - docker push --all-tags $(DARGS) $(OWNER)/$(notdir $@) - -push-all: $(foreach I,$(ALL_IMAGES),push/$(I) ) ## push all tagged images - run/%: DARGS?= run/%: ## run a bash in interactive mode in a stack docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL) diff --git a/all-spark-notebook/Dockerfile b/all-spark-notebook/Dockerfile index 21b54bf508..264668636e 100644 --- a/all-spark-notebook/Dockerfile +++ b/all-spark-notebook/Dockerfile @@ -1,6 +1,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG BASE_CONTAINER=jupyter/pyspark-notebook +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/pyspark-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 8b4988cbf4..0f283555b0 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -5,8 +5,7 @@ # https://hub.docker.com/_/ubuntu/?tab=tags&name=focal ARG ROOT_CONTAINER=ubuntu:focal-20210609 -ARG BASE_CONTAINER=$ROOT_CONTAINER -FROM $BASE_CONTAINER +FROM $ROOT_CONTAINER LABEL maintainer="Jupyter Project " ARG NB_USER="jovyan" diff --git a/binder/Dockerfile b/binder/Dockerfile index 75fb0c3428..d9f4a013e4 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -2,7 +2,8 @@ # Distributed under the terms of the Modified BSD License. # https://hub.docker.com/r/jupyter/base-notebook/tags -ARG BASE_CONTAINER=jupyter/base-notebook:33add21fab64 +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/base-notebook:33add21fab64 FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/datascience-notebook/Dockerfile b/datascience-notebook/Dockerfile index 61cb5711c7..c8d99e5649 100644 --- a/datascience-notebook/Dockerfile +++ b/datascience-notebook/Dockerfile @@ -1,6 +1,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG BASE_CONTAINER=jupyter/scipy-notebook +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/scipy-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/minimal-notebook/Dockerfile b/minimal-notebook/Dockerfile index 5f573bdc71..3450e7ca75 100644 --- a/minimal-notebook/Dockerfile +++ b/minimal-notebook/Dockerfile @@ -1,6 +1,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG BASE_CONTAINER=jupyter/base-notebook +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/base-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/pyspark-notebook/Dockerfile b/pyspark-notebook/Dockerfile index 8dc2dcc4f4..c400a882a1 100644 --- a/pyspark-notebook/Dockerfile +++ b/pyspark-notebook/Dockerfile @@ -1,6 +1,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG BASE_CONTAINER=jupyter/scipy-notebook +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/scipy-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/r-notebook/Dockerfile b/r-notebook/Dockerfile index 20053c470b..5a098a4eb9 100644 --- a/r-notebook/Dockerfile +++ b/r-notebook/Dockerfile @@ -1,6 +1,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG BASE_CONTAINER=jupyter/minimal-notebook +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/minimal-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/scipy-notebook/Dockerfile b/scipy-notebook/Dockerfile index c2b769eb38..4f82be5b05 100644 --- a/scipy-notebook/Dockerfile +++ b/scipy-notebook/Dockerfile @@ -1,6 +1,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG BASE_CONTAINER=jupyter/minimal-notebook +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/minimal-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/tensorflow-notebook/Dockerfile b/tensorflow-notebook/Dockerfile index 3271de7520..96b78ba14c 100644 --- a/tensorflow-notebook/Dockerfile +++ b/tensorflow-notebook/Dockerfile @@ -1,6 +1,7 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG BASE_CONTAINER=jupyter/scipy-notebook +ARG OWNER=jupyter +ARG BASE_CONTAINER=$OWNER/scipy-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " From 480a3295138e4e379edae1d358973c3dde2d7dba Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 6 Jul 2021 11:54:38 -0700 Subject: [PATCH 09/16] Simplify build-test-all rule --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index afbcfe49e3..85cd1c9b2c 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ build-cross/%: ## build the latest image for a stack on x86 and ARM @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" build-all: $(foreach I,$(CROSS_IMAGES), build-cross/$(I) ) $(foreach I,$(X86_IMAGES), build/$(I) ) ## build all stacks -build-test-all: build-all $(foreach I,$(ALL_IMAGES), test/$(I) ) ## build and test all stacks +build-test-all: build-all test-all ## build and test all stacks check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental) @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py From 65312e27f831201a12acb508e58b8eaec0a5b2ae Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 6 Jul 2021 12:03:29 -0700 Subject: [PATCH 10/16] Match patch version --- base-notebook/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 0f283555b0..1be16e8f60 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -24,7 +24,7 @@ USER root # Conda version ARG conda_version="4.10.2" # Miniforge installer patch version -ARG miniforge_patch_number="5" +ARG miniforge_patch_number="0" # Package Manager and Python implementation to use (https://github.com/conda-forge/miniforge) # - conda only: either Miniforge3 to use Python or Miniforge-pypy3 to use PyPy # - conda + mamba: either Mambaforge to use Python or Mambaforge-pypy3 to use PyPy From 22fef8680ffb8549889e46b03b0af81b003f9e35 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 6 Jul 2021 12:11:55 -0700 Subject: [PATCH 11/16] Run prettier on docker.yml --- .github/workflows/docker.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2e878651a6..e908ca5958 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -40,7 +40,6 @@ jobs: echo "REGISTRY=localhost:5000/" >> $GITHUB_ENV fi - uses: actions/checkout@v2 - # Setup docker to build for multiple platforms, see: # https://github.com/docker/build-push-action/tree/v2.4.0#usage # https://github.com/docker/build-push-action/blob/v2.4.0/docs/advanced/multi-platform.md From 8a16192e13c8fb8db917e6f5827b29707b9d03dd Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 6 Jul 2021 12:13:22 -0700 Subject: [PATCH 12/16] Declare and export seperately per docker lint --- base-notebook/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/base-notebook/Dockerfile b/base-notebook/Dockerfile index 1be16e8f60..3cd4379b11 100644 --- a/base-notebook/Dockerfile +++ b/base-notebook/Dockerfile @@ -98,11 +98,13 @@ WORKDIR /tmp # Prerequisites installation: conda, mamba, pip, tini RUN set -x && \ # Miniforge installer - export miniforge_arch=$(uname -m) && \ + miniforge_arch=$(uname -m) && \ + export miniforge_arch && \ if [ "$miniforge_arch" == "aarm64" ]; then \ miniforge_arch="arm64"; \ fi; \ - export miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${miniforge_arch}.sh" && \ + miniforge_installer="${miniforge_python}-${miniforge_version}-Linux-${miniforge_arch}.sh" && \ + export miniforge_installer && \ wget --quiet "https://github.com/conda-forge/miniforge/releases/download/${miniforge_version}/${miniforge_installer}" && \ /bin/bash "${miniforge_installer}" -f -b -p "${CONDA_DIR}" && \ rm "${miniforge_installer}" && \ From ad949afc22b923320c9203bb0859ebae8e88722b Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Tue, 6 Jul 2021 15:22:29 -0700 Subject: [PATCH 13/16] Skip CI changes --- .github/workflows/docker.yml | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e908ca5958..1a6d9d19db 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -31,27 +31,6 @@ jobs: !contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.pull_request.title, 'ci skip') steps: - - name: Should we push this image to a public registry? - run: | - if [ "${{ startsWith(github.ref, 'refs/tags/') || (github.ref == 'refs/heads/main') }}" = "true" ]; then - # Empty => Docker Hub - echo "REGISTRY=" >> $GITHUB_ENV - else - echo "REGISTRY=localhost:5000/" >> $GITHUB_ENV - fi - - uses: actions/checkout@v2 - # Setup docker to build for multiple platforms, see: - # https://github.com/docker/build-push-action/tree/v2.4.0#usage - # https://github.com/docker/build-push-action/blob/v2.4.0/docs/advanced/multi-platform.md - - - name: Set up QEMU (for docker buildx) - uses: docker/setup-qemu-action@25f0500ff22e406f7191a2a8ba8cda16901ca018 # associated tag: v1.0.2 - - - name: Set up Docker Buildx (for multi-arch builds) - uses: docker/setup-buildx-action@2a4b53665e15ce7d7049afb11ff1f70ff1610609 # associated tag: v1.1.2 - with: - # Allows pushing to registry on localhost:5000 - driver-opts: network=host - name: Clone Main Repo uses: actions/checkout@v2 with: @@ -88,3 +67,6 @@ jobs: with: username: ${{secrets.DOCKERHUB_USERNAME}} password: ${{secrets.DOCKERHUB_TOKEN}} + - name: Push Images to DockerHub + if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main' + run: make -C main push-all From 8f8b0d12fc9a96289ed565fa1d42bde6f57a0da1 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 7 Jul 2021 11:26:42 -0700 Subject: [PATCH 14/16] Revert the makefile changes --- Makefile | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/Makefile b/Makefile index 85cd1c9b2c..35722294a8 100644 --- a/Makefile +++ b/Makefile @@ -5,20 +5,8 @@ # Use bash for inline if-statements in arch_patch target SHELL:=bash OWNER?=jupyter -# By default we still target single image amd64 & docker output type. # Need to list the images in build dependency order -# These are images we can cross-build -CROSS_IMAGES:= base-notebook \ - minimal-notebook -# These images that aren't currently supported for cross-building, your help is welcome. -X86_IMAGES:= r-notebook \ - scipy-notebook \ - tensorflow-notebook \ - datascience-notebook \ - pyspark-notebook \ - all-spark-notebook -# All of the images ALL_IMAGES:=base-notebook \ minimal-notebook \ r-notebook \ @@ -41,18 +29,12 @@ help: build/%: DARGS?= build/%: ## build the latest image for a stack - docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --platform "linux/amd64" --build-arg OWNER="${OWNER}" --push + docker build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) @echo -n "Built image size: " @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" -build-cross/%: DARGS?= -build-cross/%: ## build the latest image for a stack on x86 and ARM - docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --platform "linux/amd64,linux/arm64" --build-arg OWNER="${OWNER}" --push - @echo -n "Built image size: " - @docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}" - -build-all: $(foreach I,$(CROSS_IMAGES), build-cross/$(I) ) $(foreach I,$(X86_IMAGES), build/$(I) ) ## build all stacks -build-test-all: build-all test-all ## build and test all stacks +build-all: $(foreach I,$(ALL_IMAGES), build/$(I) ) ## build all stacks +build-test-all: $(foreach I,$(ALL_IMAGES), build/$(I) test/$(I) ) ## build and test all stacks check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental) @TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py @@ -114,6 +96,12 @@ pull/%: ## pull a jupyter image pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I) ) ## pull all images +push/%: DARGS?= +push/%: ## push all tags for a jupyter image + docker push --all-tags $(DARGS) $(OWNER)/$(notdir $@) + +push-all: $(foreach I,$(ALL_IMAGES),push/$(I) ) ## push all tagged images + run/%: DARGS?= run/%: ## run a bash in interactive mode in a stack docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL) From 99a75a238f71f9e315103041111fb06e0e883a84 Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 7 Jul 2021 11:27:52 -0700 Subject: [PATCH 15/16] Update the Arch comment to match --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d55282777f..a023019a33 100644 --- a/README.md +++ b/README.md @@ -107,4 +107,4 @@ This change is tracked in the issue [#1217](https://github.com/jupyter/docker-st ## Architectures -Currently published containers only support x86, however cross-platform image support for x86 & arm64 images is now present. +Currently published containers only support x86, some containers may support cross-building with docker buildx. From cf7dc259644c2c8c0f0de84a32fbe4b7ce97c2ec Mon Sep 17 00:00:00 2001 From: Holden Karau Date: Wed, 7 Jul 2021 13:40:44 -0700 Subject: [PATCH 16/16] back out unrelated changes --- all-spark-notebook/Dockerfile | 3 +-- binder/Dockerfile | 3 +-- datascience-notebook/Dockerfile | 3 +-- docs/index.rst | 2 +- minimal-notebook/Dockerfile | 3 +-- pyspark-notebook/Dockerfile | 3 +-- r-notebook/Dockerfile | 3 +-- scipy-notebook/Dockerfile | 3 +-- tensorflow-notebook/Dockerfile | 3 +-- 9 files changed, 9 insertions(+), 17 deletions(-) diff --git a/all-spark-notebook/Dockerfile b/all-spark-notebook/Dockerfile index 264668636e..21b54bf508 100644 --- a/all-spark-notebook/Dockerfile +++ b/all-spark-notebook/Dockerfile @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/pyspark-notebook +ARG BASE_CONTAINER=jupyter/pyspark-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/binder/Dockerfile b/binder/Dockerfile index d9f4a013e4..75fb0c3428 100644 --- a/binder/Dockerfile +++ b/binder/Dockerfile @@ -2,8 +2,7 @@ # Distributed under the terms of the Modified BSD License. # https://hub.docker.com/r/jupyter/base-notebook/tags -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/base-notebook:33add21fab64 +ARG BASE_CONTAINER=jupyter/base-notebook:33add21fab64 FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/datascience-notebook/Dockerfile b/datascience-notebook/Dockerfile index c8d99e5649..61cb5711c7 100644 --- a/datascience-notebook/Dockerfile +++ b/datascience-notebook/Dockerfile @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/scipy-notebook +ARG BASE_CONTAINER=jupyter/scipy-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/docs/index.rst b/docs/index.rst index 0d97c0209d..9b42b9ac01 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -46,7 +46,7 @@ Docker destroys the container after notebook server exit, but any files written Architectures ----------- -Currently published containers only support x86, however cross-platform image support for x86 & arm64 images is now present. +Currently published containers only support x86, some containers may support cross-building with docker buildx. Table of Contents diff --git a/minimal-notebook/Dockerfile b/minimal-notebook/Dockerfile index 3450e7ca75..5f573bdc71 100644 --- a/minimal-notebook/Dockerfile +++ b/minimal-notebook/Dockerfile @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/base-notebook +ARG BASE_CONTAINER=jupyter/base-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/pyspark-notebook/Dockerfile b/pyspark-notebook/Dockerfile index c400a882a1..8dc2dcc4f4 100644 --- a/pyspark-notebook/Dockerfile +++ b/pyspark-notebook/Dockerfile @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/scipy-notebook +ARG BASE_CONTAINER=jupyter/scipy-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/r-notebook/Dockerfile b/r-notebook/Dockerfile index 5a098a4eb9..20053c470b 100644 --- a/r-notebook/Dockerfile +++ b/r-notebook/Dockerfile @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/minimal-notebook +ARG BASE_CONTAINER=jupyter/minimal-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/scipy-notebook/Dockerfile b/scipy-notebook/Dockerfile index 4f82be5b05..c2b769eb38 100644 --- a/scipy-notebook/Dockerfile +++ b/scipy-notebook/Dockerfile @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/minimal-notebook +ARG BASE_CONTAINER=jupyter/minimal-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project " diff --git a/tensorflow-notebook/Dockerfile b/tensorflow-notebook/Dockerfile index 96b78ba14c..3271de7520 100644 --- a/tensorflow-notebook/Dockerfile +++ b/tensorflow-notebook/Dockerfile @@ -1,7 +1,6 @@ # Copyright (c) Jupyter Development Team. # Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter -ARG BASE_CONTAINER=$OWNER/scipy-notebook +ARG BASE_CONTAINER=jupyter/scipy-notebook FROM $BASE_CONTAINER LABEL maintainer="Jupyter Project "