Add option to use digests for images referenced in registry

arbitrary-users-patch/build_images.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

build.sh

@@ -14,6 +14,7 @@ REGISTRY="quay.io"
 ORGANIZATION="eclipse"
 TAG="nightly"
 TARGET="registry"
+USE_DIGESTS=false
 DOCKERFILE="./build/dockerfiles/Dockerfile"
 
 USAGE="
@@ -27,6 +28,8 @@ Options:
         Docker registry to be used for image; default 'quay.io'
     --organization, -o [ORGANIZATION]
         Docker image organization to be used for image; default: 'eclipse'
+    --use-digests
+        Build registry to use images pinned by digest instead of tag
     --offline
         Build offline version of registry, with all sample projects
         cached in the registry; disabled by default.
@@ -54,6 +57,10 @@ function parse_arguments() {
             ORGANIZATION="$2"
             shift; shift;
             ;;
+            --use-digests)
+            USE_DIGESTS=true
+            shift
+            ;;
             --offline)
             TARGET="offline-registry"
             shift
@@ -76,10 +83,19 @@ VERSION=$(head -n 1 VERSION)
 case $VERSION in
   *SNAPSHOT)
     echo "Snapshot version (${VERSION}) specified in $(find . -name VERSION): building nightly plugin registry."
-    docker build -t "${IMAGE}" -f ${DOCKERFILE} --target ${TARGET} .
+    docker build \
+        -t "${IMAGE}" \
+        -f ${DOCKERFILE} \
+        --build-arg "USE_DIGESTS=${USE_DIGESTS}" \
+        --target ${TARGET} .
     ;;
   *)
     echo "Release version specified in $(find . -name VERSION): Building plugin registry for release ${VERSION}."
-    docker build -t "${IMAGE}" -f ${DOCKERFILE} --target ${TARGET} --build-arg "PATCHED_IMAGES_TAG=${VERSION}" .
+    docker build \
+        -t "${IMAGE}" \
+        -f ${DOCKERFILE} \
+        --build-arg "USE_DIGESTS=${USE_DIGESTS}" \
+        --build-arg "PATCHED_IMAGES_TAG=${VERSION}" \
+        --target ${TARGET} .
     ;;
 esac

build/dockerfiles/Dockerfile

@@ -7,13 +7,14 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 FROM alpine:3.10 AS builder
-RUN apk add --no-cache py-pip jq bash wget git && pip install yq
+RUN apk add --no-cache py-pip jq bash wget git skopeo && pip install yq
 
 # Registry, organization, and tag to use for base images in dockerfiles. Devfiles
 # will be rewritten during build to use these values for base images.
 ARG PATCHED_IMAGES_REG="quay.io"
 ARG PATCHED_IMAGES_ORG="eclipse"
 ARG PATCHED_IMAGES_TAG="nightly"
+ARG USE_DIGESTS=false
 
 COPY ./build/scripts ./arbitrary-users-patch/base_images /build/
 COPY ./devfiles /build/devfiles
@@ -23,6 +24,7 @@ RUN TAG=${PATCHED_IMAGES_TAG} \
     REGISTRY=${PATCHED_IMAGES_REG} \
     ./update_devfile_patched_image_tags.sh
 RUN ./check_mandatory_fields.sh devfiles
+RUN if [[ ${USE_DIGESTS} == "true" ]]; then ./write_image_digests.sh devfiles; fi
 RUN ./index.sh > /build/devfiles/index.json
 RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
 RUN chmod -R g+rwX /build/devfiles

build/dockerfiles/content_sets_centos8_appstream.repo

@@ -0,0 +1,5 @@
+[AppStream]
+name=CentOS-8 - AppStream
+baseurl=http://mirror.centos.org/centos-8/8/AppStream/x86_64/os/
+gpgcheck=0
+enabled=1

build/dockerfiles/entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
@@ -40,9 +40,10 @@ INDEX_JSON="${DEVFILES_DIR}/index.json"
 #   \2 - Registry portion of image, e.g. (quay.io)/eclipse/che-theia:tag
 #   \3 - Organization portion of image, e.g. quay.io/(eclipse)/che-theia:tag
 #   \4 - Image name portion of image, e.g. quay.io/eclipse/(che-theia):tag
-#   \5 - Tag of image, e.g. quay.io/eclipse/che-theia:(tag)
-#   \6 - Optional quotation following image reference
-IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*):([._a-zA-Z0-9-]*)("?)'
+#   \5 - Optional image digest identifier (empty for tags), e.g. quay.io/eclipse/che-theia(@sha256):digest
+#   \6 - Tag of image or digest, e.g. quay.io/eclipse/che-theia:(tag)
+#   \7 - Optional quotation following image reference
+IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)(@sha256)?:([._a-zA-Z0-9-]*)("?)'
 
 # We can't use the `-d` option for readarray because
 # registry.centos.org/centos/httpd-24-centos7 ships with Bash 4.2
@@ -55,15 +56,15 @@ for devfile in "${devfiles[@]}"; do
   # Defaults don't work because registry and tags may be different.
   if [ -n "$REGISTRY" ]; then
     echo "    Updating image registry to $REGISTRY"
-    sed -i -E "s|image:$IMAGE_REGEX|image:\1${REGISTRY}/\3/\4:\5\6|" "$devfile"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1${REGISTRY}/\3/\4\5:\6\7|" "$devfile"
   fi
   if [ -n "$ORGANIZATION" ]; then
     echo "    Updating image organization to $ORGANIZATION"
-    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/${ORGANIZATION}/\4:\5\6|" "$devfile"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/${ORGANIZATION}/\4\5:\6\7|" "$devfile"
   fi
   if [ -n "$TAG" ]; then
     echo "    Updating image tag to $TAG"
-    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/\3/\4:${TAG}\6|" "$devfile"
+    sed -i -E "s|image:$IMAGE_REGEX|image:\1\2/\3/\4:${TAG}\7|" "$devfile"
   fi
 done
 

build/dockerfiles/rhel.Dockerfile

@@ -21,8 +21,8 @@ USER 0
 
 ARG BOOTSTRAP=false
 ENV BOOTSTRAP=${BOOTSTRAP}
-ARG LATEST_ONLY=false
-ENV LATEST_ONLY=${LATEST_ONLY}
+ARG USE_DIGESTS=false
+ENV USE_DIGESTS=${USE_DIGESTS}
 
 # to get all the python deps pre-fetched so we can build in Brew:
 # 1. extract files in the container to your local filesystem
@@ -39,7 +39,7 @@ ENV LATEST_ONLY=${LATEST_ONLY}
 
 # NOTE: uncomment for local build. Must also set full registry path in FROM to registry.redhat.io or registry.access.redhat.com
 # enable rhel 7 or 8 content sets (from Brew) to resolve jq as rpm
-COPY ./build/dockerfiles/content_sets_epel7.repo /etc/yum.repos.d/
+COPY ./build/dockerfiles/content_sets_centos8_appstream.repo /etc/yum.repos.d/
 
 COPY ./build/dockerfiles/rhel.install.sh /tmp
 RUN /tmp/rhel.install.sh && rm -f /tmp/rhel.install.sh
@@ -58,6 +58,7 @@ RUN TAG=${PATCHED_IMAGES_TAG} \
     REGISTRY=${PATCHED_IMAGES_REG} \
     ./update_devfile_patched_image_tags.sh
 RUN ./check_mandatory_fields.sh devfiles
+RUN if [[ ${USE_DIGESTS} == "true" ]]; then ./write_image_digests.sh devfiles; fi
 RUN ./index.sh > /build/devfiles/index.json
 RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
 RUN chmod -R g+rwX /build/devfiles

build/dockerfiles/rhel.entrypoint.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

build/dockerfiles/rhel.install.sh

@@ -1,14 +1,14 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
 #
 # SPDX-License-Identifier: EPL-2.0
 #
 
-microdnf install -y findutils bash wget yum gzip git tar jq python3-six python3-pip && microdnf -y clean all && \
+microdnf install -y findutils bash wget yum gzip git tar jq python3-six python3-pip skopeo && microdnf -y clean all && \
 # install yq (depends on jq and pyyaml - if jq and pyyaml not already installed, this will try to compile it)
 if [[ -f /tmp/root-local.tgz ]] || [[ ${BOOTSTRAP} == "true" ]]; then \
     mkdir -p /root/.local; tar xf /tmp/root-local.tgz -C /root/.local/; rm -fr /tmp/root-local.tgz;  \

build/scripts/cache_images.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

build/scripts/cache_projects.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

build/scripts/index.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

build/scripts/write_image_digests.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Copyright (c) 2020 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+
+
+readarray -d '' devfiles < <(find "$1" -name 'devfile.yaml' -print0)
+for image in $(yq -r '.components[]?.image' "${devfiles[@]}" | grep -v "null" | sort | uniq); do
+  echo "Rewriting image $image"
+  digest=$(skopeo inspect "docker://${image}" | jq -r '.Digest')
+  echo "  to use digest $digest"
+  digest_image="${image%:*}@${digest}"
+
+  # Rewrite images to use sha-256 digests
+  sed -i -E 's|"?'"${image}"'"?|"'"${digest_image}"'" # tag: '"${image}"'|g' "${devfiles[@]}"
+done

cico_build_ci.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2019 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

cico_build_nightly.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2019 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

cico_build_release.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2019 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

cico_functions.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Copyright (c) 2012-2019 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

deploy/kubernetes/che-devfile-registry/Chart.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

deploy/kubernetes/che-devfile-registry/templates/configmap.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

deploy/kubernetes/che-devfile-registry/templates/deployment.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

deploy/kubernetes/che-devfile-registry/templates/ingress.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

deploy/kubernetes/che-devfile-registry/templates/service.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

deploy/kubernetes/che-devfile-registry/values.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/

deploy/openshift/che-devfile-registry.yaml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018 Red Hat, Inc.
+# Copyright (c) 2018-2020 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/