From 20470cfc8be26e3f6528acebf74a8dc66fa2b824 Mon Sep 17 00:00:00 2001 From: Andrew Hamilton Date: Thu, 30 Jun 2022 14:34:52 -0700 Subject: [PATCH 1/4] Fixes containerd configuration issue with insecure registry - Updates containerd configuration to use the new format for specifying container registry mirrors. - Updates the start code to produce files in the correct location for registry mirrors specified with --insecure-registry --- .../containerd-bin-aarch64/config.toml | 5 +- .../config.toml.default | 4 +- .../containerd-bin-aarch64/containerd-bin.mk | 3 + .../containerd_docker_io_hosts.toml | 1 + .../x86_64/package/containerd-bin/config.toml | 5 +- .../containerd-bin/config.toml.default | 4 +- .../package/containerd-bin/containerd-bin.mk | 3 + .../containerd_docker_io_hosts.toml | 1 + deploy/kicbase/Dockerfile | 1 + deploy/kicbase/containerd.toml | 5 +- .../kicbase/containerd_docker_io_hosts.toml | 1 + pkg/minikube/cruntime/containerd.go | 64 ++++++++++--------- 12 files changed, 53 insertions(+), 44 deletions(-) create mode 100644 deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd_docker_io_hosts.toml create mode 100644 deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd_docker_io_hosts.toml create mode 100644 deploy/kicbase/containerd_docker_io_hosts.toml diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml index b060e08f19d4..d5de73eae405 100644 --- a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml +++ b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml @@ -57,9 +57,8 @@ oom_score = 0 conf_dir = "/etc/cni/net.mk" conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] + config_path = "/etc/containerd/certs.d" + [plugins."io.containerd.service.v1.diff-service"] default = ["walking"] [plugins."io.containerd.gc.v1.scheduler"] diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml.default b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml.default index c54c96c32022..54a396a43572 100644 --- a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml.default +++ b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/config.toml.default @@ -100,9 +100,7 @@ oom_score = 0 max_conf_num = 1 conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] + config_path = "/etc/containerd/certs.d" [plugins."io.containerd.grpc.v1.cri".image_decryption] key_model = "" [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk index c92faee4edc5..dbf7acd486af 100644 --- a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk +++ b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk @@ -53,6 +53,9 @@ define CONTAINERD_BIN_AARCH64_INSTALL_TARGET_CMDS $(INSTALL) -Dm644 \ $(CONTAINERD_BIN_AARCH64_PKGDIR)/config.toml \ $(TARGET_DIR)/etc/containerd/config.toml + $(INSTALL) -Dm644 \ + $(CONTAINERD_BIN_AARCH64_PKGDIR)/containerd_docker_io_hosts.toml \ + $(TARGET_DIR)/etc/containerd/docker.io/hosts.toml endef define CONTAINERD_BIN_AARCH64_INSTALL_INIT_SYSTEMD diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd_docker_io_hosts.toml b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd_docker_io_hosts.toml new file mode 100644 index 000000000000..00df747eba97 --- /dev/null +++ b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd_docker_io_hosts.toml @@ -0,0 +1 @@ +server = "https://registry-1.docker.io" \ No newline at end of file diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml index b060e08f19d4..e63ad23c34db 100644 --- a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml +++ b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml @@ -57,9 +57,8 @@ oom_score = 0 conf_dir = "/etc/cni/net.mk" conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] + config_path = "/etc/containerd/certs.d" + [plugins."io.containerd.service.v1.diff-service"] default = ["walking"] [plugins."io.containerd.gc.v1.scheduler"] diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml.default b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml.default index c54c96c32022..54a396a43572 100644 --- a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml.default +++ b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/config.toml.default @@ -100,9 +100,7 @@ oom_score = 0 max_conf_num = 1 conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] + config_path = "/etc/containerd/certs.d" [plugins."io.containerd.grpc.v1.cri".image_decryption] key_model = "" [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd-bin.mk b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd-bin.mk index f07d8d0009aa..3845fe6eb838 100644 --- a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd-bin.mk +++ b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd-bin.mk @@ -54,6 +54,9 @@ define CONTAINERD_BIN_INSTALL_TARGET_CMDS $(INSTALL) -Dm644 \ $(CONTAINERD_BIN_PKGDIR)/config.toml \ $(TARGET_DIR)/etc/containerd/config.toml + $(INSTALL) -Dm644 \ + $(CONTAINERD_BIN_PKGDIR)/containerd_docker_io_hosts.toml \ + $(TARGET_DIR)/etc/containerd/certs.d/docker.io/hosts.toml endef define CONTAINERD_BIN_INSTALL_INIT_SYSTEMD diff --git a/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd_docker_io_hosts.toml b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd_docker_io_hosts.toml new file mode 100644 index 000000000000..00df747eba97 --- /dev/null +++ b/deploy/iso/minikube-iso/arch/x86_64/package/containerd-bin/containerd_docker_io_hosts.toml @@ -0,0 +1 @@ +server = "https://registry-1.docker.io" \ No newline at end of file diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index e5e66fbe7de1..265e3ef0b455 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -50,6 +50,7 @@ COPY deploy/kicbase/10-network-security.conf /etc/sysctl.d/10-network-security.c COPY deploy/kicbase/11-tcp-mtu-probing.conf /etc/sysctl.d/11-tcp-mtu-probing.conf COPY deploy/kicbase/02-crio.conf /etc/crio/crio.conf.d/02-crio.conf COPY deploy/kicbase/containerd.toml /etc/containerd/config.toml +COPY deploy/kicbase/containerd_docker_io_hosts.toml /etc/containerd/certs.d/docker.io/hosts.toml COPY deploy/kicbase/clean-install /usr/local/bin/clean-install COPY deploy/kicbase/entrypoint /usr/local/bin/entrypoint COPY --from=auto-pause /src/cmd/auto-pause/auto-pause-${TARGETARCH} /bin/auto-pause diff --git a/deploy/kicbase/containerd.toml b/deploy/kicbase/containerd.toml index 6270ba5879f1..98b902d7a1c0 100644 --- a/deploy/kicbase/containerd.toml +++ b/deploy/kicbase/containerd.toml @@ -57,9 +57,8 @@ oom_score = 0 conf_dir = "/etc/cni/net.mk" conf_template = "" [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] + config_path = "/etc/containerd/certs.d" + [plugins."io.containerd.service.v1.diff-service"] default = ["walking"] [plugins."io.containerd.gc.v1.scheduler"] diff --git a/deploy/kicbase/containerd_docker_io_hosts.toml b/deploy/kicbase/containerd_docker_io_hosts.toml new file mode 100644 index 000000000000..00df747eba97 --- /dev/null +++ b/deploy/kicbase/containerd_docker_io_hosts.toml @@ -0,0 +1 @@ +server = "https://registry-1.docker.io" \ No newline at end of file diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index f5e4be75afeb..b9167b6147e3 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -21,12 +21,12 @@ import ( "encoding/base64" "encoding/json" "fmt" + "html/template" "net/url" "os" "os/exec" "path" "strings" - "text/template" "time" "github.com/blang/semver/v4" @@ -45,13 +45,12 @@ import ( const ( containerdNamespaceRoot = "/run/containerd/runc/k8s.io" // ContainerdConfFile is the path to the containerd configuration - containerdConfigFile = "/etc/containerd/config.toml" - containerdImportedConfigFile = "/etc/containerd/containerd.conf.d/02-containerd.conf" - containerdConfigTemplate = `version = 2 -{{ range .InsecureRegistry -}} -[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{. -}}"] - endpoint = ["http://{{. -}}"] -{{ end -}} + containerdConfigFile = "/etc/containerd/config.toml" + containerdMirrorsRoot = "/etc/containerd/certs.d" + containerdInsecureRegistryTemplate = `server = "{{.InsecureRegistry -}}" + +[host."{{.InsecureRegistry -}}"] + skip_verify = true ` ) @@ -142,28 +141,35 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*conf_dir = .*$|conf_dir = \"%s\"|' -i %s", cni.ConfDir, containerdConfigFile))); err != nil { return errors.Wrap(err, "update conf_dir") } - imports := `imports = ["/etc/containerd/containerd.conf.d/02-containerd.conf"]` - if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^# imports|%s|' -i %s", imports, containerdConfigFile))); err != nil { - return errors.Wrap(err, "update conf_dir") - } - cPath := containerdImportedConfigFile - t, err := template.New("02-containerd.conf").Parse(containerdConfigTemplate) - if err != nil { - return err - } - opts := struct { - InsecureRegistry []string - }{ - InsecureRegistry: insecureRegistry, - } - var b bytes.Buffer - if err := t.Execute(&b, opts); err != nil { - return err - } - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", path.Dir(cPath), base64.StdEncoding.EncodeToString(b.Bytes()), cPath)) - if _, err := cr.RunCmd(c); err != nil { - return errors.Wrap(err, "generate containerd cfg") + for _, registry := range insecureRegistry { + addr := registry + if strings.HasPrefix(strings.ToLower(registry), "http://") || strings.HasPrefix(strings.ToLower(registry), "https://") { + i := strings.Index(addr, "//") + addr = addr[i+2:] + } else { + registry = "http://" + registry + } + + t, err := template.New("hosts.toml").Parse(containerdInsecureRegistryTemplate) + if err != nil { + return errors.Wrap(err, "unable to parse insecure registry template") + } + opts := struct { + InsecureRegistry string + }{ + InsecureRegistry: registry, + } + var b bytes.Buffer + if err := t.Execute(&b, opts); err != nil { + return errors.Wrap(err, "unable to create insecure registry template") + } + regRootPath := path.Join(containerdMirrorsRoot, addr) + + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo mkdir -p %s && printf %%s \"%s\" | base64 -d | sudo tee %s", regRootPath, base64.StdEncoding.EncodeToString(b.Bytes()), path.Join(regRootPath, "hosts.toml"))) + if _, err := cr.RunCmd(c); err != nil { + return errors.Wrap(err, "unable to generate insecure registry cfg") + } } return nil } From c929340ea939a0403d2c3ba65c436e4c8d9e59d5 Mon Sep 17 00:00:00 2001 From: Andrew Hamilton Date: Mon, 29 Aug 2022 10:29:33 -0700 Subject: [PATCH 2/4] Update target dir for aarch64 Adds missing certs.d directory in the installation directory path. Co-authored-by: Steven Powell <44844360+spowelljr@users.noreply.github.com> --- .../aarch64/package/containerd-bin-aarch64/containerd-bin.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk index dbf7acd486af..1dd8ac38a8e1 100644 --- a/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk +++ b/deploy/iso/minikube-iso/arch/aarch64/package/containerd-bin-aarch64/containerd-bin.mk @@ -55,7 +55,7 @@ define CONTAINERD_BIN_AARCH64_INSTALL_TARGET_CMDS $(TARGET_DIR)/etc/containerd/config.toml $(INSTALL) -Dm644 \ $(CONTAINERD_BIN_AARCH64_PKGDIR)/containerd_docker_io_hosts.toml \ - $(TARGET_DIR)/etc/containerd/docker.io/hosts.toml + $(TARGET_DIR)/etc/containerd/certs.d/docker.io/hosts.toml endef define CONTAINERD_BIN_AARCH64_INSTALL_INIT_SYSTEMD From 1599f52e3fa68557bb7f1719d13076e299b48e9d Mon Sep 17 00:00:00 2001 From: minikube-bot Date: Mon, 29 Aug 2022 18:16:10 +0000 Subject: [PATCH 3/4] Updating kicbase image to v0.0.33-1661795577-14482 --- pkg/drivers/kic/types.go | 4 ++-- site/content/en/docs/commands/start.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/drivers/kic/types.go b/pkg/drivers/kic/types.go index 934090974b62..69e39df4aa64 100644 --- a/pkg/drivers/kic/types.go +++ b/pkg/drivers/kic/types.go @@ -24,9 +24,9 @@ import ( const ( // Version is the current version of kic - Version = "v0.0.33-1659486857-14721" + Version = "v0.0.33-1661795577-14482" // SHA of the kic base image - baseImageSHA = "98c8007234ca882b63abc707dc184c585fcb5372828b49a4b639961324d291b3" + baseImageSHA = "e92c29880a4b3b095ed3b61b1f4a696b57c5cd5212bc8256f9599a777020645d" // The name of the GCR kicbase repository gcrRepo = "gcr.io/k8s-minikube/kicbase-builds" // The name of the Dockerhub kicbase repository diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 9a0029cc2710..a0f38219bce7 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -26,7 +26,7 @@ minikube start [flags] --apiserver-names strings A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine --apiserver-port int The apiserver listening port (default 8443) --auto-update-drivers If set, automatically updates drivers to the latest version. Defaults to true. (default true) - --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.33-1659486857-14721@sha256:98c8007234ca882b63abc707dc184c585fcb5372828b49a4b639961324d291b3") + --base-image string The base image to use for docker/podman drivers. Intended for local development. (default "gcr.io/k8s-minikube/kicbase-builds:v0.0.33-1661795577-14482@sha256:e92c29880a4b3b095ed3b61b1f4a696b57c5cd5212bc8256f9599a777020645d") --binary-mirror string Location to fetch kubectl, kubelet, & kubeadm binaries from. --cache-images If true, cache docker images for the current bootstrapper and load them into the machine. Always false with --driver=none. (default true) --cert-expiration duration Duration until minikube certificate expiration, defaults to three years (26280h). (default 26280h0m0s) From c26af98e814d2bdea3ebd394fa9808cfbb35f553 Mon Sep 17 00:00:00 2001 From: minikube-bot Date: Mon, 29 Aug 2022 22:51:31 +0000 Subject: [PATCH 4/4] Updating ISO to v1.26.1-1661795462-14482 --- Makefile | 2 +- pkg/minikube/download/iso.go | 2 +- site/content/en/docs/commands/start.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 89902f80e6c8..225b80b21762 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ KUBERNETES_VERSION ?= $(shell egrep "DefaultKubernetesVersion =" pkg/minikube/co KIC_VERSION ?= $(shell egrep "Version =" pkg/drivers/kic/types.go | cut -d \" -f2) # Default to .0 for higher cache hit rates, as build increments typically don't require new ISO versions -ISO_VERSION ?= v1.26.1-1661377864-14783 +ISO_VERSION ?= v1.26.1-1661795462-14482 # Dashes are valid in semver, but not Linux packaging. Use ~ to delimit alpha/beta DEB_VERSION ?= $(subst -,~,$(RAW_VERSION)) DEB_REVISION ?= 0 diff --git a/pkg/minikube/download/iso.go b/pkg/minikube/download/iso.go index 189eeb1c5a96..fdab34c110e7 100644 --- a/pkg/minikube/download/iso.go +++ b/pkg/minikube/download/iso.go @@ -41,7 +41,7 @@ const fileScheme = "file" // DefaultISOURLs returns a list of ISO URL's to consult by default, in priority order func DefaultISOURLs() []string { v := version.GetISOVersion() - isoBucket := "minikube-builds/iso/14783" + isoBucket := "minikube-builds/iso/14482" return []string{ fmt.Sprintf("https://storage.googleapis.com/%s/minikube-%s-%s.iso", isoBucket, v, runtime.GOARCH), fmt.Sprintf("https://github.com/kubernetes/minikube/releases/download/%s/minikube-%s-%s.iso", v, v, runtime.GOARCH), diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index a0f38219bce7..c4946e026039 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -69,7 +69,7 @@ minikube start [flags] --insecure-registry strings Insecure Docker registries to pass to the Docker daemon. The default service CIDR range will automatically be added. --install-addons If set, install addons. Defaults to true. (default true) --interactive Allow user prompts for more information (default true) - --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/14783/minikube-v1.26.1-1661377864-14783-amd64.iso,https://github.com/kubernetes/minikube/releases/download/v1.26.1-1661377864-14783/minikube-v1.26.1-1661377864-14783-amd64.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.26.1-1661377864-14783-amd64.iso]) + --iso-url strings Locations to fetch the minikube ISO from. (default [https://storage.googleapis.com/minikube-builds/iso/14482/minikube-v1.26.1-1661795462-14482-amd64.iso,https://github.com/kubernetes/minikube/releases/download/v1.26.1-1661795462-14482/minikube-v1.26.1-1661795462-14482-amd64.iso,https://kubernetes.oss-cn-hangzhou.aliyuncs.com/minikube/iso/minikube-v1.26.1-1661795462-14482-amd64.iso]) --keep-context This will keep the existing kubectl context and will create a minikube context. --kubernetes-version string The Kubernetes version that the minikube VM will use (ex: v1.2.3, 'stable' for v1.24.4, 'latest' for v1.25.0-rc.1). Defaults to 'stable'. --kvm-gpu Enable experimental NVIDIA GPU support in minikube