Merge pull request #8501 from medyagh/sol_msg_check_running

docker/podman: add solution message when container exits after start/create

cmd/minikube/cmd/start.go

@@ -156,6 +156,7 @@ func runStart(cmd *cobra.Command, args []string) {
 	starter, err := provisionWithDriver(cmd, ds, existing)
 	if err != nil {
 		maybeExitWithAdvice(err)
+		machine.MaybeDisplayAdvice(err, viper.GetString("driver"))
 		if specified {
 			// If the user specified a driver, don't fallback to anything else
 			exit.WithError("error provisioning host", err)

pkg/drivers/kic/kic.go

@@ -305,6 +305,11 @@ func (d *Driver) Restart() error {
 // Start an already created kic container
 func (d *Driver) Start() error {
 	if err := oci.StartContainer(d.NodeConfig.OCIBinary, d.MachineName); err != nil {
+		oci.LogContainerDebug(d.OCIBinary, d.MachineName)
+		_, err := oci.DaemonInfo(d.OCIBinary)
+		if err != nil {
+			return errors.Wrapf(oci.ErrDaemonInfo, "debug daemon info %q", d.MachineName)
+		}
 		return errors.Wrap(err, "start")
 	}
 	checkRunning := func() error {
@@ -320,7 +325,13 @@ func (d *Driver) Start() error {
 	}
 
 	if err := retry.Expo(checkRunning, 500*time.Microsecond, time.Second*30); err != nil {
-		return err
+		oci.LogContainerDebug(d.OCIBinary, d.MachineName)
+		_, err := oci.DaemonInfo(d.OCIBinary)
+		if err != nil {
+			return errors.Wrapf(oci.ErrDaemonInfo, "container name %q", d.MachineName)
+		}
+
+		return errors.Wrapf(oci.ErrExitedUnexpectedly, "container name %q", d.MachineName)
 	}
 	return nil
 }

pkg/drivers/kic/oci/errors.go

@@ -16,7 +16,12 @@ limitations under the License.
 
 package oci
 
-import "errors"
+import (
+	"errors"
+	"os/exec"
+
+	"github.com/golang/glog"
+)
 
 // FailFastError type is an error that could not be solved by trying again
 type FailFastError error
@@ -26,3 +31,55 @@ var ErrWindowsContainers = FailFastError(errors.New("docker container type is wi
 
 // ErrCPUCountLimit is thrown when docker daemon doesn't have enough CPUs for the requested container
 var ErrCPUCountLimit = FailFastError(errors.New("not enough CPUs is available for container"))
+
+// ErrExitedUnexpectedly is thrown when container is created/started without error but later it exists and it's status is not running anymore.
+var ErrExitedUnexpectedly = errors.New("container exited unexpectedly")
+
+// ErrDaemonInfo is thrown when docker/podman info is failing or not responding
+var ErrDaemonInfo = errors.New("daemon info not responding")
+
+// LogContainerDebug will print relevant docker/podman infos after a container fails
+func LogContainerDebug(ociBin string, name string) {
+	rr, err := containerInspect(ociBin, name)
+	if err != nil {
+		glog.Warningf("Filed to get postmortem inspect. %s :%v", rr.Command(), err)
+	} else {
+		glog.Infof("Postmortem inspect (%q): %s", rr.Command(), rr.Output())
+	}
+
+	rr, err = containerLogs(ociBin, name)
+	if err != nil {
+		glog.Warningf("Filed to get postmortem logs. %s :%v", rr.Command(), err)
+	} else {
+		glog.Infof("Postmortem logs (%q): %s", rr.Command(), rr.Output())
+	}
+	if ociBin == Docker {
+		di, err := dockerSystemInfo()
+		if err != nil {
+			glog.Warningf("Failed to get postmortem docker info: %v", err)
+		} else {
+			glog.Infof("postmortem docker info: %+v", di)
+		}
+	} else {
+		pi, err := podmanSystemInfo()
+		if err != nil {
+			glog.Warningf("couldn't get postmortem info, failed to to run podman info: %v", err)
+		} else {
+			glog.Infof("postmortem podman info: %+v", pi)
+		}
+	}
+}
+
+// containerLogs will return out the logs for a container
+func containerLogs(ociBin string, name string) (*RunResult, error) {
+	if ociBin == Docker {
+		return runCmd(exec.Command(ociBin, "logs", "--timestamps", "--details", name))
+	}
+	// podman doesn't have --details
+	return runCmd(exec.Command(ociBin, "logs", "--timestamps", name))
+}
+
+// containerInspect will return the inspect for a container
+func containerInspect(ociBin string, name string) (*RunResult, error) {
+	return runCmd(exec.Command(ociBin, "inspect", name))
+}

pkg/drivers/kic/oci/oci.go

@@ -218,7 +218,12 @@ func CreateContainerNode(p CreateParams) error {
 
 	// retry up to up 13 seconds to make sure the created container status is running.
 	if err := retry.Expo(checkRunning, 13*time.Millisecond, time.Second*13); err != nil {
-		return errors.Wrapf(err, "check container %q running", p.Name)
+		LogContainerDebug(p.OCIBinary, p.Name)
+		_, err := DaemonInfo(p.OCIBinary)
+		if err != nil {
+			return errors.Wrapf(ErrDaemonInfo, "container name %q", p.Name)
+		}
+		return errors.Wrapf(ErrExitedUnexpectedly, "container name %q", p.Name)
 	}
 
 	return nil

pkg/minikube/machine/advice.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package machine
+
+import (
+	"runtime"
+
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/out"
+)
+
+// MaybeDisplayAdvice will provide advice without exiting, so minikube has a chance to try the failover
+func MaybeDisplayAdvice(err error, driver string) {
+	if errors.Is(err, oci.ErrDaemonInfo) {
+		out.ErrLn("")
+		out.ErrT(out.Conflict, "{{.driver_name}} couldn't proceed because {{.driver_name}} service is not healthy.", out.V{"driver_name": driver})
+	}
+
+	if errors.Is(err, oci.ErrExitedUnexpectedly) {
+		out.ErrLn("")
+		out.ErrT(out.Conflict, "The minikube {{.driver_name}} container exited unexpectedly.", out.V{"driver_name": driver})
+	}
+
+	if errors.Is(err, oci.ErrExitedUnexpectedly) || errors.Is(err, oci.ErrDaemonInfo) {
+		out.T(out.Tip, "If you are still interested to make {{.driver_name}} driver work. The following suggestions might help you get passed this issue:", out.V{"driver_name": driver})
+		out.T(out.Empty, `
+	- Prune unused {{.driver_name}} images, volumes and abandoned containers.`, out.V{"driver_name": driver})
+		out.T(out.Empty, `
+	- Restart your {{.driver_name}} service`, out.V{"driver_name": driver})
+		if runtime.GOOS != "linux" {
+			out.T(out.Empty, `
+	- Ensure your {{.driver_name}} daemon has access to enough CPU/memory resources. `, out.V{"driver_name": driver})
+			if runtime.GOOS == "darwin" && driver == oci.Docker {
+				out.T(out.Empty, `
+	- Docs https://docs.docker.com/docker-for-mac/#resources`, out.V{"driver_name": driver})
+			}
+			if runtime.GOOS == "windows" && driver == oci.Docker {
+				out.T(out.Empty, `
+	- Docs https://docs.docker.com/docker-for-windows/#resources`, out.V{"driver_name": driver})
+			}
+		}
+		out.T(out.Empty, `
+	- Delete and recreate minikube cluster
+		minikube delete
+		minikube start --driver={{.driver_name}}`, out.V{"driver_name": driver})
+		// TODO #8348: maybe advice user if to set the --force-systemd https://github.com/kubernetes/minikube/issues/8348
+	}
+
+}

pkg/minikube/machine/fix.go

@@ -142,6 +142,7 @@ func recreateIfNeeded(api libmachine.API, cc *config.ClusterConfig, n *config.No
 		out.T(out.Restarting, `Restarting existing {{.driver_name}} {{.machine_type}} for "{{.cluster}}" ...`, out.V{"driver_name": cc.Driver, "cluster": machineName, "machine_type": machineType})
 	}
 	if err := h.Driver.Start(); err != nil {
+		MaybeDisplayAdvice(err, h.DriverName)
 		return h, errors.Wrap(err, "driver start")
 	}
 	if err := saveHost(api, h, cc, n); err != nil {