Merge pull request #14482 from andrewhamilton-okta/fix-containerd-ins…

…ecure-registry Fixes containerd configuration issue with insecure registries
kubernetes · Aug 31, 2022 · f2df65f · f2df65f
2 parents 87cb961 + c26af98
commit f2df65f
Show file tree

Hide file tree

Showing 16 changed files with 59 additions and 50 deletions.
diff --git a/Makefile b/Makefile
@@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co
 KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2)
 
 # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions
-ISO_VERSION ?= v1.26.1-1661377864-14783
+ISO_VERSION ?= v1.26.1-1661795462-14482
 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta
 DEB_VERSION ?= $(subst -,~,$(RAW_VERSION))
 DEB_REVISION ?= 0

diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml
@@ -57,9 +57,8 @@ oom_score = 0
       conf_dir = "/etc/cni/net.mk"
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
-          endpoint = ["https://registry-1.docker.io"]
+      config_path = "/etc/containerd/certs.d"
+
   [plugins."io.containerd.service.v1.diff-service"]
     default = ["walking"]
   [plugins."io.containerd.gc.v1.scheduler"]

diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml.default b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml.default
@@ -100,9 +100,7 @@ oom_score = 0
       max_conf_num = 1
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
-          endpoint = ["https://registry-1.docker.io"]
+      config_path = "/etc/containerd/certs.d"
     [plugins."io.containerd.grpc.v1.cri".image_decryption]
       key_model = ""
     [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]

diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk
@@ -53,6 +53,9 @@ define CONTAINERD_BIN_AARCH64_INSTALL_TARGET_CMDS
 	$(INSTALL) -Dm644 \
 		$(CONTAINERD_BIN_AARCH64_PKGDIR)/config.toml \
 		$(TARGET_DIR)/etc/containerd/config.toml
+	$(INSTALL) -Dm644 \
+		$(CONTAINERD_BIN_AARCH64_PKGDIR)/containerd_docker_io_hosts.toml \
+		$(TARGET_DIR)/etc/containerd/certs.d/docker.io/hosts.toml
 endef
 
 define CONTAINERD_BIN_AARCH64_INSTALL_INIT_SYSTEMD

diff --git a/.../minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd_docker_io_hosts.toml b/.../minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd_docker_io_hosts.toml
@@ -0,0 +1 @@
+server = "https://registry-1.docker.io"
diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml
@@ -57,9 +57,8 @@ oom_score = 0
       conf_dir = "/etc/cni/net.mk"
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
-          endpoint = ["https://registry-1.docker.io"]
+      config_path = "/etc/containerd/certs.d"
+
   [plugins."io.containerd.service.v1.diff-service"]
     default = ["walking"]
   [plugins."io.containerd.gc.v1.scheduler"]

diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml.default b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml.default
@@ -100,9 +100,7 @@ oom_score = 0
       max_conf_num = 1
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
-          endpoint = ["https://registry-1.docker.io"]
+      config_path = "/etc/containerd/certs.d"
     [plugins."io.containerd.grpc.v1.cri".image_decryption]
       key_model = ""
     [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]

diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd-bin.mk b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd-bin.mk
@@ -54,6 +54,9 @@ define CONTAINERD_BIN_INSTALL_TARGET_CMDS
 	$(INSTALL) -Dm644 \
 		$(CONTAINERD_BIN_PKGDIR)/config.toml \
 		$(TARGET_DIR)/etc/containerd/config.toml
+	$(INSTALL) -Dm644 \
+		$(CONTAINERD_BIN_PKGDIR)/containerd_docker_io_hosts.toml \
+		$(TARGET_DIR)/etc/containerd/certs.d/docker.io/hosts.toml
 endef
 
 define CONTAINERD_BIN_INSTALL_INIT_SYSTEMD

diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd_docker_io_hosts.toml b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd_docker_io_hosts.toml
@@ -0,0 +1 @@
+server = "https://registry-1.docker.io"
diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile
@@ -50,6 +50,7 @@ COPY deploy/kicbase/10-network-security.conf /etc/sysctl.d/10-network-security.c
 COPY deploy/kicbase/11-tcp-mtu-probing.conf /etc/sysctl.d/11-tcp-mtu-probing.conf
 COPY deploy/kicbase/02-crio.conf /etc/crio/crio.conf.d/02-crio.conf
 COPY deploy/kicbase/containerd.toml /etc/containerd/config.toml
+COPY deploy/kicbase/containerd_docker_io_hosts.toml /etc/containerd/certs.d/docker.io/hosts.toml
 COPY deploy/kicbase/clean-install /usr/local/bin/clean-install
 COPY deploy/kicbase/entrypoint /usr/local/bin/entrypoint
 COPY --from=auto-pause /src/cmd/auto-pause/auto-pause-${TARGETARCH} /bin/auto-pause

diff --git a/deploy/kicbase/containerd.toml b/deploy/kicbase/containerd.toml
@@ -57,9 +57,8 @@ oom_score = 0
       conf_dir = "/etc/cni/net.mk"
       conf_template = ""
     [plugins."io.containerd.grpc.v1.cri".registry]
-      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
-        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
-          endpoint = ["https://registry-1.docker.io"]
+      config_path = "/etc/containerd/certs.d"
+
   [plugins."io.containerd.service.v1.diff-service"]
     default = ["walking"]
   [plugins."io.containerd.gc.v1.scheduler"]

diff --git a/deploy/kicbase/containerd_docker_io_hosts.toml b/deploy/kicbase/containerd_docker_io_hosts.toml
@@ -0,0 +1 @@
+server = "https://registry-1.docker.io"
diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go
@@ -24,9 +24,9 @@ import (
 
 const (
 	// Version is the current version of kic
-	Version = "v0.0.33-1659486857-14721"
+	Version = "v0.0.33-1661795577-14482"
 	// SHA of the kic base image
-	baseImageSHA = "98c8007234ca882b63abc707dc184c585fcb5372828b49a4b639961324d291b3"
+	baseImageSHA = "e92c29880a4b3b095ed3b61b1f4a696b57c5cd5212bc8256f9599a777020645d"
 	// The name of the GCR kicbase repository
 	gcrRepo = "gcr.io/k8s-minikube/kicbase-builds"
 	// The name of the Dockerhub kicbase repository

diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go
@@ -21,12 +21,12 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"html/template"
 	"net/url"
 	"os"
 	"os/exec"
 	"path"
 	"strings"
-	"text/template"
 	"time"
 
 	"github.com/blang/semver/v4"
@@ -45,13 +45,12 @@ import (
 const (
 	containerdNamespaceRoot = "/run/containerd/runc/k8s.io"
 	// ContainerdConfFile is the path to the containerd configuration
-	containerdConfigFile         = "/etc/containerd/config.toml"
-	containerdImportedConfigFile = "/etc/containerd/containerd.conf.d/02-containerd.conf"
-	containerdConfigTemplate     = `version = 2
-{{ range .InsecureRegistry -}}
-[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{. -}}"]
-  endpoint = ["http://{{. -}}"]
-{{ end -}}
+	containerdConfigFile               = "/etc/containerd/config.toml"
+	containerdMirrorsRoot              = "/etc/containerd/certs.d"
+	containerdInsecureRegistryTemplate = `server = "{{.InsecureRegistry -}}"
+
+[host."{{.InsecureRegistry -}}"]
+  skip_verify = true
 `
 )
 
@@ -142,28 +141,35 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve
 	if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*conf_dir = .*$|conf_dir = \"%s\"|' -i %s", cni.ConfDir, containerdConfigFile))); err != nil {
 		return errors.Wrap(err, "update conf_dir")
 	}
-	imports := `imports = ["/etc/containerd/containerd.conf.d/02-containerd.conf"]`
-	if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^# imports|%s|' -i %s", imports, containerdConfigFile))); err != nil {
-		return errors.Wrap(err, "update conf_dir")
-	}
 
-	cPath := containerdImportedConfigFile
-	t, err := template.New("02-containerd.conf").Parse(containerdConfigTemplate)
-	if err != nil {
-		return err
-	}
-	opts := struct {
-		InsecureRegistry []string
-	}{
-		InsecureRegistry: insecureRegistry,
-	}
-	var b bytes.Buffer
-	if err := t.Execute(&b, opts); err != nil {
-		return err
-	}
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath))
-	if _, err := cr.RunCmd(c); err != nil {
-		return errors.Wrap(err, "generate containerd cfg")
+	for _, registry := range insecureRegistry {
+		addr := registry
+		if strings.HasPrefix(strings.ToLower(registry), "http://") || strings.HasPrefix(strings.ToLower(registry), "https://") {
+			i := strings.Index(addr, "//")
+			addr = addr[i+2:]
+		} else {
+			registry = "http://" + registry
+		}
+
+		t, err := template.New("hosts.toml").Parse(containerdInsecureRegistryTemplate)
+		if err != nil {
+			return errors.Wrap(err, "unable to parse insecure registry template")
+		}
+		opts := struct {
+			InsecureRegistry string
+		}{
+			InsecureRegistry: registry,
+		}
+		var b bytes.Buffer
+		if err := t.Execute(&b, opts); err != nil {
+			return errors.Wrap(err, "unable to create insecure registry template")
+		}
+		regRootPath := path.Join(containerdMirrorsRoot, addr)
+
+		c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", regRootPath, base64.StdEncoding.EncodeToString(b.Bytes()), path.Join(regRootPath, "hosts.toml")))
+		if _, err := cr.RunCmd(c); err != nil {
+			return errors.Wrap(err, "unable to generate insecure registry cfg")
+		}
 	}
 	return nil
 }

diff --git a/pkg/minikube/download/iso.go b/pkg/minikube/download/iso.go
@@ -41,7 +41,7 @@ const fileScheme = "file"
 // DefaultISOURLs returns a list of ISO URL's to consult by default, in priority order
 func DefaultISOURLs() []string {
 	v := version.GetISOVersion()
-	isoBucket := "minikube-builds/iso/14783"
+	isoBucket := "minikube-builds/iso/14482"
 	return []string{
 		fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s-%s.iso", isoBucket, v, runtime.GOARCH),
 		fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/%s/minikube-%s-%s.iso", v, v, runtime.GOARCH),

diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
@@ -26,7 +26,7 @@ minikube start [flags]
       --apiserver-names strings           A set of apiserver names which are used in the generated certificate for kubernetes.  This can be used if you want to make the apiserver available from outside the machine
       --apiserver-port int                The apiserver listening port (default 8443)
       --auto-update-drivers               If set, automatically updates drivers to the latest version. Defaults to true. (default true)
-      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.33-1659486857-14721@sha256:98c8007234ca882b63abc707dc184c585fcb5372828b49a4b639961324d291b3")
+      --base-image string                 The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.33-1661795577-14482@sha256:e92c29880a4b3b095ed3b61b1f4a696b57c5cd5212bc8256f9599a777020645d")
       --binary-mirror string              Location to fetch kubectl, kubelet, & kubeadm binaries from.
       --cache-images                      If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true)
       --cert-expiration duration          Duration until minikube certificate expiration, defaults to three years (26280h). (default 26280h0m0s)
@@ -69,7 +69,7 @@ minikube start [flags]
       --insecure-registry strings         Insecure Docker registries to pass to the Docker daemon.  The default service CIDR range will automatically be added.
       --install-addons                    If set, install addons. Defaults to true. (default true)
       --interactive                       Allow user prompts for more information (default true)
-      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/14783/minikube-v1.26.1-1661377864-14783-amd64.iso,https://github.com/kubernetes/minikube/releases/download/v1.26.1-1661377864-14783/minikube-v1.26.1-1661377864-14783-amd64.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.26.1-1661377864-14783-amd64.iso])
+      --iso-url strings                   Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/14482/minikube-v1.26.1-1661795462-14482-amd64.iso,https://github.com/kubernetes/minikube/releases/download/v1.26.1-1661795462-14482/minikube-v1.26.1-1661795462-14482-amd64.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.26.1-1661795462-14482-amd64.iso])
       --keep-context                      This will keep the existing kubectl context and will create a minikube context.
       --kubernetes-version string         The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.24.4, 'latest' for v1.25.0-rc.1). Defaults to 'stable'.
       --kvm-gpu                           Enable experimental NVIDIA GPU support in minikube