Cherry-pick #20576 to 7.x: Build docker images based on Red Hat UBI

CHANGELOG-developer.next.asciidoc

@@ -92,3 +92,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Added SQL helper that can be used from any Metricbeat module {pull}18955[18955]
 - Update Go version to 1.14.4. {pull}19753[19753]
 - Update Go version to 1.14.7. {pull}20508[20508]
+- Add packaging for docker image based on UBI minimal 8. {pull}20576[20576]

dev-tools/mage/dockerbuilder.go

@@ -151,19 +151,14 @@ func isDockerFile(path string) bool {
 }
 
 func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]interface{}) error {
-	// has specific dockerfile
-	dockerfile := fmt.Sprintf("Dockerfile.%s.tmpl", b.imageName)
-	_, err := os.Stat(filepath.Join(templatesDir, dockerfile))
-	if err != nil {
-		// specific missing fallback to generic
-		dockerfile = "Dockerfile.tmpl"
+	dockerfile := "Dockerfile.tmpl"
+	if f, found := b.ExtraVars["dockerfile"]; found {
+		dockerfile = f
 	}
 
-	entrypoint := fmt.Sprintf("docker-entrypoint.%s.tmpl", b.imageName)
-	_, err = os.Stat(filepath.Join(templatesDir, entrypoint))
-	if err != nil {
-		// specific missing fallback to generic
-		entrypoint = "docker-entrypoint.tmpl"
+	entrypoint := "docker-entrypoint.tmpl"
+	if e, found := b.ExtraVars["docker_entrypoint"]; found {
+		entrypoint = e
 	}
 
 	type fileExpansion struct {
@@ -176,7 +171,7 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in
 			".tmpl",
 		)
 		path := filepath.Join(templatesDir, file.source)
-		err = b.ExpandFile(path, target, data)
+		err := b.ExpandFile(path, target, data)
 		if err != nil {
 			return errors.Wrapf(err, "expanding template '%s' to '%s'", path, target)
 		}

dev-tools/mage/settings.go

@@ -91,6 +91,7 @@ var (
 		"repo":              GetProjectRepoInfo,
 		"title":             strings.Title,
 		"tolower":           strings.ToLower,
+		"contains":          strings.Contains,
 	}
 )
 

dev-tools/packaging/package_test.go

@@ -48,13 +48,15 @@ const (
 )
 
 var (
-	configFilePattern      = regexp.MustCompile(`.*beat\.yml$|apm-server\.yml$`)
+	configFilePattern      = regexp.MustCompile(`.*beat\.yml$|apm-server\.yml|elastic-agent\.yml$`)
 	manifestFilePattern    = regexp.MustCompile(`manifest.yml`)
 	modulesDirPattern      = regexp.MustCompile(`module/.+`)
 	modulesDDirPattern     = regexp.MustCompile(`modules.d/$`)
 	modulesDFilePattern    = regexp.MustCompile(`modules.d/.+`)
 	monitorsDFilePattern   = regexp.MustCompile(`monitors.d/.+`)
 	systemdUnitFilePattern = regexp.MustCompile(`/lib/systemd/system/.*\.service`)
+
+	licenseFiles = []string{"LICENSE.txt", "NOTICE.txt"}
 )
 
 var (
@@ -122,6 +124,7 @@ func checkRPM(t *testing.T, file string) {
 	checkModulesPresent(t, "/usr/share", p)
 	checkModulesDPresent(t, "/etc/", p)
 	checkMonitorsDPresent(t, "/etc", p)
+	checkLicensesPresent(t, "/usr/share", p)
 	checkSystemdUnitPermissions(t, p)
 	ensureNoBuildIDLinks(t, p)
 }
@@ -141,6 +144,7 @@ func checkDeb(t *testing.T, file string, buf *bytes.Buffer) {
 	checkModulesPresent(t, "./usr/share", p)
 	checkModulesDPresent(t, "./etc/", p)
 	checkMonitorsDPresent(t, "./etc/", p)
+	checkLicensesPresent(t, "./usr/share", p)
 	checkModulesOwner(t, p, true)
 	checkModulesPermissions(t, p)
 	checkSystemdUnitPermissions(t, p)
@@ -160,6 +164,7 @@ func checkTar(t *testing.T, file string) {
 	checkModulesDPresent(t, "", p)
 	checkModulesPermissions(t, p)
 	checkModulesOwner(t, p, true)
+	checkLicensesPresent(t, "", p)
 }
 
 func checkZip(t *testing.T, file string) {
@@ -174,6 +179,7 @@ func checkZip(t *testing.T, file string) {
 	checkModulesPresent(t, "", p)
 	checkModulesDPresent(t, "", p)
 	checkModulesPermissions(t, p)
+	checkLicensesPresent(t, "", p)
 }
 
 func checkDocker(t *testing.T, file string) {
@@ -190,6 +196,7 @@ func checkDocker(t *testing.T, file string) {
 	checkManifestPermissionsWithMode(t, p, os.FileMode(0640))
 	checkModulesPresent(t, "", p)
 	checkModulesDPresent(t, "", p)
+	checkLicensesPresent(t, "licenses/", p)
 }
 
 // Verify that the main configuration file is installed with a 0600 file mode.
@@ -373,6 +380,22 @@ func checkMonitors(t *testing.T, name, prefix string, r *regexp.Regexp, p *packa
 	})
 }
 
+func checkLicensesPresent(t *testing.T, prefix string, p *packageFile) {
+	for _, licenseFile := range licenseFiles {
+		t.Run("License file "+licenseFile, func(t *testing.T) {
+			for _, entry := range p.Contents {
+				if strings.HasPrefix(entry.File, prefix) && strings.HasSuffix(entry.File, "/"+licenseFile) {
+					return
+				}
+			}
+			if prefix != "" {
+				t.Fatalf("not found under %s", prefix)
+			}
+			t.Fatal("not found")
+		})
+	}
+}
+
 func checkDockerEntryPoint(t *testing.T, p *packageFile, info *dockerInfo) {
 	expectedMode := os.FileMode(0755)
 
@@ -402,7 +425,8 @@ func checkDockerLabels(t *testing.T, p *packageFile, info *dockerInfo, file stri
 	if vendor != "Elastic" {
 		return
 	}
-	t.Run(fmt.Sprintf("%s labels", p.Name), func(t *testing.T) {
+
+	t.Run(fmt.Sprintf("%s license labels", p.Name), func(t *testing.T) {
 		expectedLicense := "Elastic License"
 		ossPrefix := strings.Join([]string{
 			info.Config.Labels["org.label-schema.name"],
@@ -412,8 +436,24 @@ func checkDockerLabels(t *testing.T, p *packageFile, info *dockerInfo, file stri
 		if strings.HasPrefix(filepath.Base(file), ossPrefix) {
 			expectedLicense = "ASL 2.0"
 		}
-		if license, present := info.Config.Labels["license"]; !present || license != expectedLicense {
-			t.Errorf("unexpected license label: %s", license)
+		licenseLabels := []string{
+			"license",
+			"org.label-schema.license",
+		}
+		for _, licenseLabel := range licenseLabels {
+			if license, present := info.Config.Labels[licenseLabel]; !present || license != expectedLicense {
+				t.Errorf("unexpected license label %s: %s", licenseLabel, license)
+			}
+		}
+	})
+
+	t.Run(fmt.Sprintf("%s required labels", p.Name), func(t *testing.T) {
+		// From https://redhat-connect.gitbook.io/partner-guide-for-red-hat-openshift-and-container/program-on-boarding/technical-prerequisites
+		requiredLabels := []string{"name", "vendor", "version", "release", "summary", "description"}
+		for _, label := range requiredLabels {
+			if value, present := info.Config.Labels[label]; !present || value == "" {
+				t.Errorf("missing required label %s", label)
+			}
 		}
 	})
 }
@@ -657,6 +697,12 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) {
 			if strings.HasPrefix("/"+name, workingDir) || "/"+name == entrypoint {
 				p.Contents[name] = entry
 			}
+			// Add also licenses
+			for _, licenseFile := range licenseFiles {
+				if strings.Contains(name, licenseFile) {
+					p.Contents[name] = entry
+				}
+			}
 		}
 	}
 

dev-tools/packaging/packages.yml

@@ -28,6 +28,9 @@ shared:
       /usr/share/{{.BeatName}}/LICENSE.txt:
         source: '{{ repo.RootDir }}/LICENSE.txt'
         mode: 0644
+      /usr/share/{{.BeatName}}/NOTICE.txt:
+        source: '{{ repo.RootDir }}/NOTICE.txt'
+        mode: 0644
       /usr/share/{{.BeatName}}/README.md:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
         mode: 0644
@@ -117,6 +120,9 @@ shared:
       /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/LICENSE.txt:
         source: '{{ repo.RootDir }}/LICENSE.txt'
         mode: 0644
+      /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/NOTICE.txt:
+        source: '{{ repo.RootDir }}/NOTICE.txt'
+        mode: 0644
       /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/README.md:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
         mode: 0644
@@ -186,6 +192,9 @@ shared:
     LICENSE.txt:
       source: '{{ repo.RootDir }}/LICENSE.txt'
       mode: 0644
+    NOTICE.txt:
+      source: '{{ repo.RootDir }}/NOTICE.txt'
+      mode: 0644
     README.md:
       template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/common/README.md.tmpl'
       mode: 0644
@@ -307,6 +316,9 @@ shared:
     <<: *agent_binary_spec
     extra_vars:
       from: 'centos:7'
+      buildFrom: 'centos:7'
+      dockerfile: 'Dockerfile.elastic-agent.tmpl'
+      docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl'
       user: 'root'
       linux_capabilities: ''
     files:
@@ -460,6 +472,7 @@ shared:
     <<: *binary_spec
     extra_vars:
       from: 'centos:7'
+      buildFrom: 'centos:7'
       user: '{{ .BeatName }}'
       linux_capabilities: ''
     files:
@@ -468,6 +481,11 @@ shared:
         mode: 0600
         config: true
 
+  - &docker_ubi_spec
+    extra_vars:
+      image_name: '{{.BeatName}}-ubi8'
+      from: 'registry.access.redhat.com/ubi8/ubi-minimal'
+
   - &elastic_docker_spec
     extra_vars:
       repository: 'docker.elastic.co/beats'
@@ -637,6 +655,14 @@ specs:
         <<: *elastic_docker_spec
         <<: *elastic_license_for_binaries
 
+    - os: linux
+      types: [docker]
+      spec:
+        <<: *docker_spec
+        <<: *docker_ubi_spec
+        <<: *elastic_docker_spec
+        <<: *elastic_license_for_binaries
+
   # Elastic Beat with Elastic License and binary taken the current directory.
   elastic_beat_xpack_reduced:
     ###
@@ -721,6 +747,17 @@ specs:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
+    - os: linux
+      types: [docker]
+      spec:
+        <<: *docker_spec
+        <<: *docker_ubi_spec
+        <<: *elastic_docker_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./{{.XPackDir}}/{{.BeatName}}/build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
   # Elastic Beat with Elastic License and binary taken from the x-pack dir.
   elastic_beat_agent_binaries:
     ###
@@ -782,6 +819,17 @@ specs:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
 
+    - os: linux
+      types: [docker]
+      spec:
+        <<: *agent_docker_spec
+        <<: *docker_ubi_spec
+        <<: *elastic_docker_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
 
   # Elastic Beat with Elastic License and binary taken from the x-pack dir.
   elastic_beat_agent_demo_binaries:

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -2,13 +2,38 @@
 {{- $beatBinary := printf "%s/%s" $beatHome .BeatName }}
 {{- $repoInfo := repo }}
 
+# Prepare home in a different stage to avoid creating additional layers on
+# the final image because of permission changes.
+FROM {{ .buildFrom }} AS home
+
+COPY beat {{ $beatHome }}
+
+RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
+    chown -R root:root {{ $beatHome }} && \
+    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
+    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
+    chmod 0750 {{ $beatBinary }} && \
+{{- if .linux_capabilities }}
+    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
+{{- end }}
+{{- range $i, $modulesd := .ModulesDirs }}
+    chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
+{{- end }}
+    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+
 FROM {{ .from }}
 
+{{- if contains .from "ubi-minimal" }}
+RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils &&  microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)
+RUN curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o /usr/local/bin/jq && \
+    chmod +x /usr/local/bin/jq
+{{- else }}
 # Installing jq needs to be installed after epel-release and cannot be in the same yum install command.
 RUN yum -y --setopt=tsflags=nodocs update && \
     yum install epel-release -y && \
     yum install jq -y && \
     yum clean all
+{{- end }}
 
 LABEL \
   org.label-schema.build-date="{{ date }}" \
@@ -20,33 +45,37 @@ LABEL \
   org.label-schema.url="{{ .BeatURL }}" \
   org.label-schema.vcs-url="{{ $repoInfo.RootImportPath }}" \
   org.label-schema.vcs-ref="{{ commit }}" \
+  io.k8s.description="{{ .BeatDescription }}" \
+  io.k8s.display-name="{{ .BeatName | title }} image" \
+  org.opencontainers.image.created="{{ date }}" \
+  org.opencontainers.image.licenses="{{ .License }}" \
+  org.opencontainers.image.title="{{ .BeatName | title }}" \
+  org.opencontainers.image.vendor="{{ .BeatVendor }}" \
+  name="{{ .BeatName }}" \
+  maintainer="infra@elastic.co" \
+  vendor="{{ .BeatVendor }}" \
+  version="{{ beat_version }}" \
+  release="1" \
+  url="{{ .BeatURL }}" \
+  summary="{{ .BeatName }}" \
   license="{{ .License }}" \
   description="{{ .BeatDescription }}"
 
 ENV ELASTIC_CONTAINER "true"
 ENV PATH={{ $beatHome }}:$PATH
 
-COPY beat {{ $beatHome }}
 COPY docker-entrypoint /usr/local/bin/docker-entrypoint
 RUN chmod 755 /usr/local/bin/docker-entrypoint
 
-RUN groupadd --gid 1000 {{ .BeatName }}
+COPY --from=home {{ $beatHome }} {{ $beatHome }}
 
-RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
-    chown -R root:{{ .BeatName }} {{ $beatHome }} && \
-    find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
-    find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
-    chmod 0750 {{ $beatBinary }} && \
-{{- if .linux_capabilities }}
-    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
-{{- end }}
-{{- range $i, $modulesd := .ModulesDirs }}
-    chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
-{{- end }}
-    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+RUN mkdir /licenses
+COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses
+COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
 
 {{- if ne .user "root" }}
-RUN useradd -M --uid 1000 --gid 1000 --home {{ $beatHome }} {{ .user }}
+RUN groupadd --gid 1000 {{ .BeatName }}
+RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }}
 {{- end }}
 USER {{ .user }}