eclipse-che · ibuziuk · Oct 28, 2019 · Oct 17, 2019 · Oct 18, 2019 · Oct 22, 2019
@@ -30,7 +30,7 @@ The default docker build has multiple targets:
 - `--target registry` is used to build the default devfile registry, where projects in devfiles refer to publically hosted git repos
 - `--target offline-registry` is used to build a devfile registry which self-hosts projects as zip files.
 
-The offline registry build will, during the docker build, pull zips from all projects hosted on github and store them in the `/resources` path. This registry should be deployed with environment variable `CHE_DEVFILE_HTTPS_ENDPOINT` set to the URL of the route/endpoint that exposes the devfile registry, as devfiles need to be rewritten to point to internally hosted zip files.
+The offline registry build will, during the docker build, pull zips from all projects hosted on github and store them in the `/resources` path. This registry should be deployed with environment variable `CHE_DEVFILE_REGISTRY_URL` set to the URL of the route/endpoint that exposes the devfile registry, as devfiles need to be rewritten to point to internally hosted zip files.
 
 ## OpenShift
 You can deploy Che devfile registry on Openshift with command.

@@ -1 +1 @@
-7.3.0
+7.3.1
@@ -1,6 +1,5 @@
 che-cpp-rhel7           registry.access.redhat.com/devtools/llvm-toolset-rhel7
 che-dotnet-2.2          mcr.microsoft.com/dotnet/core/sdk:2.2-stretch
-che-golang-1.10         golang:1.10.7-stretch
 che-golang-1.12         golang:1.12-stretch
 che-java11-gradle       gradle:5.2.1-jdk11
 che-java11-maven        maven:3.6.0-jdk-11

@@ -1,28 +1,85 @@
-#!/bin/sh
+#!/bin/bash
 #
-# Copyright (c) 2012-2018 Red Hat, Inc.
+# Copyright (c) 2019 Red Hat, Inc.
 # This program and the accompanying materials are made
 # available under the terms of the Eclipse Public License 2.0
 # which is available at https://www.eclipse.org/legal/epl-2.0/
 #
 # SPDX-License-Identifier: EPL-2.0
 #
-# Build Che devfile registry image. Note that this script will read the version
-# in ./VERSION; if it is *-SNAPSHOT, devfiles in the registry will use nightly-tagged
-# images with the arbitrary user IDs patch. If ./VERSION contains otherwise,
-# the devfiles in the registry will instead use the value in ./VERSION.
-#
 
+set -e
+
+REGISTRY="quay.io"
+ORGANIZATION="eclipse"
+TAG="nightly"
+TARGET="registry"
+DOCKERFILE="./build/dockerfiles/Dockerfile"
+
+USAGE="
+Usage: ./build.sh [OPTIONS]
+Options:
+    --help
+        Print this message.
+    --tag, -t [TAG]
+        Docker image tag to be used for image; default: 'nightly'
+    --registry, -r [REGISTRY]
+        Docker registry to be used for image; default 'quay.io'
+    --organization, -o [ORGANIZATION]
+        Docker image organization to be used for image; default: 'eclipse'
+    --offline
+        Build offline version of registry, with all sample projects
+        cached in the registry; disabled by default.
+    --rhel
+        Build registry using UBI images instead of default
+"
+
+function print_usage() {
+    echo -e "$USAGE"
+}
+
+function parse_arguments() {
+    while [[ $# -gt 0 ]]; do
+        key="$1"
+        case $key in
+            -t|--tag)
+            TAG="$2"
+            shift; shift;
+            ;;
+            -r|--registry)
+            REGISTRY="$2"
+            shift; shift;
+            ;;
+            -o|--organization)
+            ORGANIZATION="$2"
+            shift; shift;
+            ;;
+            --offline)
+            TARGET="offline-registry"
+            shift
+            ;;
+            --rhel)
+            DOCKERFILE="./build/dockerfiles/rhel.Dockerfile"
+            shift
+            ;;
+            *)
+            print_usage
+            exit 0
+        esac
+    done
+}
+
+parse_arguments "$@"
+
+IMAGE="${REGISTRY}/${ORGANIZATION}/che-devfile-registry:${TAG}"
 VERSION=$(head -n 1 VERSION)
 case $VERSION in
   *SNAPSHOT)
     echo "Snapshot version (${VERSION}) specified in $(find . -name VERSION): building nightly plugin registry."
-    docker build -t "quay.io/eclipse/che-devfile-registry:nightly" -f ./build/dockerfiles/Dockerfile --target registry .
+    docker build -t "${IMAGE}" -f ${DOCKERFILE} --target ${TARGET} .
     ;;
   *)
     echo "Release version specified in $(find . -name VERSION): Building plugin registry for release ${VERSION}."
-    docker build -t "quay.io/eclipse/che-devfile-registry:${VERSION}" -f ./build/dockerfiles/Dockerfile . \
-        --build-arg "PATCHED_IMAGES_TAG=${VERSION}" \
-        --target registry
+    docker build -t "${IMAGE}" -f ${DOCKERFILE} --target ${TARGET} --build-arg "PATCHED_IMAGES_TAG=${VERSION}" .
     ;;
 esac
@@ -7,7 +7,7 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 FROM alpine:3.10 AS builder
-RUN apk add --no-cache py-pip jq bash && pip install yq
+RUN apk add --no-cache py-pip jq bash git && pip install yq
 
 # Registry, organization, and tag to use for base images in dockerfiles. Devfiles
 # will be rewritten during build to use these values for base images.
@@ -37,6 +37,7 @@ CMD ["/usr/bin/run-httpd"]
 
 # Offline registry: download project zips and place them in /build/resources
 FROM builder AS offline-builder
+RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
 RUN ./cache_projects.sh devfiles resources && chmod -R g+rwX /build
 
 # Offline registry: copy updated devfile.yamls and cached projects

@@ -19,7 +19,7 @@
 # In addition, this script will perform the necessary set up for the offline
 # devfile registry, replacing placeholders in all devfiles based off environment
 # variable
-#     CHE_DEVFILE_HTTPS_ENDPOINT
+#     CHE_DEVFILE_REGISTRY_URL
 # which should be set to the public endpoint for this registry.
 #
 # Will execute any arguments on completion (`exec $@`)
@@ -29,7 +29,7 @@ set -e
 REGISTRY=${CHE_DEVFILE_IMAGES_REGISTRY_URL}
 ORGANIZATION=${CHE_DEVFILE_IMAGES_REGISTRY_ORGANIZATION}
 TAG=${CHE_DEVFILE_IMAGES_REGISTRY_TAG}
-PUBLIC_URL=${CHE_DEVFILE_HTTPS_ENDPOINT}
+PUBLIC_URL=${CHE_DEVFILE_REGISTRY_URL}
 
 DEFAULT_DEVFILES_DIR="/var/www/html/devfiles"
 DEVFILES_DIR="${DEVFILES_DIR:-${DEFAULT_DEVFILES_DIR}}"
@@ -41,7 +41,7 @@ DEVFILES_DIR="${DEVFILES_DIR:-${DEFAULT_DEVFILES_DIR}}"
 #   \4 - Image name portion of image, e.g. quay.io/eclipse/(che-theia):tag
 #   \5 - Tag of image, e.g. quay.io/eclipse/che-theia:(tag)
 #   \6 - Optional quotation following image reference
-IMAGE_REGEX='([[:space:]]*"?)([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*):([._a-zA-Z0-9-]*)("?)'
+IMAGE_REGEX='([[:space:]]*"?)([._:a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*)/([._a-zA-Z0-9-]*):([._a-zA-Z0-9-]*)("?)'
 
 # We can't use the `-d` option for readarray because
 # registry.centos.org/centos/httpd-24-centos7 ships with Bash 4.2
@@ -71,7 +71,7 @@ if [ -n "$PUBLIC_URL" ]; then
   sed -i "s|{{ DEVFILE_REGISTRY_URL }}|${PUBLIC_URL}|" "${devfiles[@]}"
 else
   if grep -q '{{ DEVFILE_REGISTRY_URL }}' "${devfiles[@]}"; then
-    echo "WARNING: environment variable 'CHE_DEVFILE_HTTPS_ENDPOINT' not configured" \
+    echo "WARNING: environment variable 'CHE_DEVFILE_REGISTRY_URL' not configured" \
          "for an offline build of this registry. This may cause issues with importing" \
          "projects in a workspace."
     # Experimental workaround -- detect service IP for che-devfile-registry

@@ -68,7 +68,7 @@ RUN chmod -R g+rwX /build/devfiles
 # Build registry, copying meta.yamls and index.json from builder
 # UPSTREAM: use RHEL7/RHSCL/httpd image so we're not required to authenticate with registry.redhat.io
 # https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/rhscl/httpd-24-rhel7
-FROM registry.access.redhat.com/rhscl/httpd-24-rhel7:2.4-104 AS registry
+FROM registry.access.redhat.com/rhscl/httpd-24-rhel7:2.4-105 AS registry
 
 # DOWNSTREAM: use RHEL8/httpd
 # https://access.redhat.com/containers/?tab=tags#/registry.access.redhat.com/rhel8/httpd-24
@@ -97,6 +97,7 @@ CMD ["/usr/local/bin/rhel.entrypoint.sh"]
 
 # Offline devfile registry build
 FROM builder AS offline-builder
+RUN ./list_referenced_images.sh devfiles > /build/devfiles/external_images.txt
 RUN ./cache_projects.sh devfiles resources && chmod -R g+rwX /build
 
 FROM registry AS offline-registry

@@ -8,7 +8,7 @@
 # SPDX-License-Identifier: EPL-2.0
 #
 
-microdnf install -y findutils bash wget yum gzip tar jq python3-six python3-pip && microdnf -y clean all && \
+microdnf install -y findutils bash wget yum gzip git tar jq python3-six python3-pip && microdnf -y clean all && \
 # install yq (depends on jq and pyyaml - if jq and pyyaml not already installed, this will try to compile it)
 if [[ -f /tmp/root-local.tgz ]] || [[ ${BOOTSTRAP} == "true" ]]; then \
     mkdir -p /root/.local; tar xf /tmp/root-local.tgz -C /root/.local/; rm -fr /tmp/root-local.tgz;  \

@@ -15,46 +15,24 @@
 
 set -e
 
-DEVFILES_DIR="${1}"
-RESOURCES_DIR="${2}"
-TEMP_DIR="${2}/devfiles_temp/"
-TEMP_FILE="${TEMP_DIR}temp.yaml"
+DEVFILES_DIR="${1%/}"
+RESOURCES_DIR="${2%/}"
+TEMP_DIR="${RESOURCES_DIR}/devfiles_temp"
+TEMP_FILE="${TEMP_DIR}/temp.yaml"
+TEMP_REPO="${TEMP_DIR}/cloned"
 
-# Builds the URL for downloading a GitHub project as a .zip
+# Clone a git repository and create an archive zip at a specified location
 # Args:
-#   $1 - main repo URL; if it ends in '.git', this will be trimmed
-#   $2 - branch to download; if empty or 'null', 'master' is used
-function build_project_zip_url() {
-  location="$1"
-  branch="$2"
-
-  # Trim unwanted path portions
-  location="${location%/}"
-  location="${location%.git}"
-
-  # set branch to "master" if undefined
-  if [ -z "$branch" ] || [ "$branch" = "null" ]; then
-    branch="master"
-  fi
-
-  URL="${location}/archive/${branch}.zip"
-  echo "$URL"
-}
-
-# Download a project's zip to specified directory. If file already exists, nothing
-# is done.
-# Args:
-#   $1 - URL to download from
-#   $2 - path + name of file to save
-function download_project_zip() {
-  URL="$1"
-  destination="$2"
-  if [ -f "$destination" ]; then
-    echo "    Project already cached"
-  else
-    echo "    Downloading $URL to $destination"
-    wget -O "$destination" -nv "$URL" 2>&1 | sed "s/^/        /g"
-  fi
+#   $1 - URL of git repo
+#   $2 - (optional) branch to archive
+#   $3 - destination path for the archived project zip file
+function cache_project() {
+  local repo="$1"
+  local branch="$2"
+  local destination="$3"
+  git clone "$location" -b "$branch" --depth 1 "$TEMP_REPO" &>/dev/null
+  git archive "$branch" --remote="$TEMP_REPO" --format zip --output "$destination"
+  rm -rf "$TEMP_REPO"
 }
 
 # Update devfile to refer to a locally stored zip instead of a public git repo
@@ -63,13 +41,12 @@ function download_project_zip() {
 #   $2 - name of project to update within devfile
 #   $3 - path to downloaded project zip
 function update_devfile() {
-  devfile="$1"
-  project_name="$2"
-  destination="$3"
-  echo "    Updating devfile $devfile to point at cached project zip $destination"
+  local devfile="$1"
+  local project_name="$2"
+  local zip_path="$3"
   # The yq script below will rewrite the project with $project_name to be a zip-type
   # project with provided path. The location field contains a placeholder
-  # '{{ DEVFILE_REGISTRY_URL }}' which must be filled at runtime (see 
+  # '{{ DEVFILE_REGISTRY_URL }}' which must be filled at runtime (see
   # build/dockerfiles/entrypoint.sh script)
   # shellcheck disable=SC2016
   yq -y \
@@ -86,7 +63,7 @@ function update_devfile() {
       )
     }' "$devfile" \
     --arg "PROJECT_NAME" "${project_name}" \
-    --arg "PROJECT_PATH" "${destination}" \
+    --arg "PROJECT_PATH" "${zip_path}" \
     > "$TEMP_FILE"
   # As a workaround since jq does not support in-place updates, we need to copy
   # to a temp file and then overwrite the original.
@@ -95,10 +72,25 @@ function update_devfile() {
 
 }
 
+function get_devfile_name() {
+  devfile=$1
+  yq -r '.metadata |
+    if has("name") then
+      .name
+    elif has("generateName") then
+      .generateName
+    else
+      "unnamed-devfile"
+    end
+  ' $devfile
+}
+
 readarray -d '' devfiles < <(find "$DEVFILES_DIR" -name 'devfile.yaml' -print0)
 mkdir -p "$TEMP_DIR" "$RESOURCES_DIR"
 for devfile in "${devfiles[@]}"; do
   echo "Caching project files for devfile $devfile"
+  devfile_name=$(get_devfile_name "$devfile")
+  devfile_name=${devfile_name%-}
   for project in $(yq -c '.projects[]?' "$devfile"); do
     project_name=$(echo "$project" | jq -r '.name')
     echo "    Caching project $project_name"
@@ -111,20 +103,14 @@ for devfile in "${devfiles[@]}"; do
 
     location=$(echo "$project" | jq -r '.source.location')
     branch=$(echo "$project" | jq -r '.source.branch')
-    if ! echo "$location" | grep -q "github"; then
-      echo "    [WARN]: Project is not hosted on GitHub; skipping."
-      continue
+    if [ -n $branch ]; then
+      branch="master"
     fi
+    destination="${RESOURCES_DIR}/${devfile_name}-${project_name}-${branch}.zip"
+    echo "    Caching project to $(realpath $destination)"
+    cache_project "$location" "$branch" "$(realpath $destination)"
 
-    URL=$(build_project_zip_url "$location" "$branch")
-
-    filename=$(basename "$URL")
-    target=${URL#*//}
-    destination="${RESOURCES_DIR%/}/${target}"
-
-    mkdir -p "${destination%${filename}}"
-    download_project_zip "$URL" "$destination"
-
+    echo "    Updating devfile $devfile to point at cached project zip $destination"
     update_devfile "$devfile" "$project_name" "$destination"
   done
 done

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# Copyright (c) 2019 Red Hat, Inc.
+# This program and the accompanying materials are made
+# available under the terms of the Eclipse Public License 2.0
+# which is available at https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# List all images referenced in meta.yaml files
+#
+
+set -e
+
+readarray -d '' devfiles < <(find "$1" -name 'devfile.yaml' -print0)
+yq -r '.components[] | if has("image") then .image else empty end' "${devfiles[@]}" | sort | uniq
@@ -18,7 +18,7 @@ components:
 -
   type: dockerimage
   # this version is used in the plugin
-  image: quay.io/eclipse/che-golang-1.10:nightly
+  image: quay.io/eclipse/che-golang-1.12:nightly
   alias: go-cli
   env:
     - name: GOPATH

@@ -1,6 +1,6 @@
 ---
 displayName: Go
-description: Stack with Go 1.10.7
+description: Stack with Go 1.12.10
 tags: ["Debian", "Go"]
 icon: https://www.eclipse.org/che/images/logo-eclipseche.svg
 globalMemoryLimit: 1686Mi