WIP: create tunnel for qemu

cmd/minikube/cmd/docker-env.go

@@ -301,7 +301,7 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc
 			if err != nil {
 				exit.Message(reason.DrvPortForward, "Error getting port binding for '{{.driver_name}} driver: {{.error}}", out.V{"driver_name": driverName, "error": err})
 			}
-		} else if driver.NeedsPortForward(driverName) && driverName == driver.QEMU2 {
+		} else if driverName == driver.QEMU2 {
 			port = d.(*qemu.Driver).EnginePort
 		}
 

cmd/minikube/cmd/service.go

@@ -86,11 +86,6 @@ var serviceCmd = &cobra.Command{
 		cname := ClusterFlagValue()
 		co := mustload.Healthy(cname)
 
-		// Bail cleanly for qemu2 until implemented
-		if driver.IsQEMU(co.Config.Driver) {
-			exit.Message(reason.Unimplemented, "minikube service is not currently implemented with the qemu2 driver. See https://github.com/kubernetes/minikube/issues/14146 for details.")
-		}
-
 		var services service.URLs
 		services, err := service.GetServiceURLs(co.API, co.Config.Name, namespace, serviceURLTemplate)
 		if err != nil {
@@ -148,7 +143,7 @@ You may select another namespace by using 'minikube service {{.service}} -n <nam
 
 		if driver.NeedsPortForward(co.Config.Driver) && driver.IsKIC(co.Config.Driver) && services != nil {
 			startKicServiceTunnel(services, cname, co.Config.Driver)
-		} else if driver.NeedsPortForward(co.Config.Driver) && driver.IsQEMU(co.Config.Driver) && services != nil {
+		} else if driver.IsQEMU(co.Config.Driver) && services != nil {
 			startQemuServiceTunnel(services, cname, co.Config.Driver)
 		} else if !serviceURLMode {
 			openURLs(data)

cmd/minikube/cmd/start_flags.go

@@ -461,7 +461,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str
 		out.WarningT("With --network-plugin=cni, you will need to provide your own CNI. See --cni flag as a user-friendly alternative")
 	}
 
-	if !(driver.IsKIC(drvName) || driver.IsKVM(drvName)) && viper.GetString(network) != "" {
+	if !(driver.IsKIC(drvName) || driver.IsKVM(drvName) || driver.IsQEMU(drvName)) && viper.GetString(network) != "" {
 		out.WarningT("--network flag is only valid with the docker/podman and KVM drivers, it will be ignored")
 	}
 

cmd/minikube/cmd/tunnel.go

@@ -56,11 +56,6 @@ var tunnelCmd = &cobra.Command{
 		cname := ClusterFlagValue()
 		co := mustload.Healthy(cname)
 
-		// Bail cleanly for qemu2 until implemented
-		if driver.IsQEMU(co.Config.Driver) {
-			exit.Message(reason.Unimplemented, "minikube tunnel is not currently implemented with the qemu2 driver. See https://github.com/kubernetes/minikube/issues/14146 for details.")
-		}
-
 		if cleanup {
 			klog.Info("Checking for tunnels to cleanup...")
 			if err := manager.CleanupNotRunningTunnels(); err != nil {

pkg/drivers/qemu/qemu.go

@@ -105,7 +105,7 @@ func (d *Driver) GetSSHPort() (int, error) {
 
 func (d *Driver) GetSSHUsername() string {
 	if d.SSHUser == "" {
-		d.SSHUser = "docker"
+		d.SSHUser = defaultSSHUser
 	}
 	return d.SSHUser
 }
@@ -146,6 +146,9 @@ func (d *Driver) GetIP() (string, error) {
 	if d.Network == "user" {
 		return "127.0.0.1", nil
 	}
+	if d.Network == "socket" {
+		return "192.168.105.1", nil
+	}
 	return d.NetworkAddress, nil
 }
 
@@ -155,6 +158,9 @@ func (d *Driver) GetPort() int {
 		d.FirstQuery = false
 		port = 2376
 	}
+	if d.Network == "socket" {
+		port = 8443
+	}
 	return port
 }
 
@@ -366,12 +372,12 @@ func (d *Driver) Start() error {
 	} else {
 		if d.Nographic {
 			startCmd = append(startCmd,
-				"-nographic",
-			)
+				"-nographic")
 		} else {
 			startCmd = append(startCmd,
-				"-display", "none",
-			)
+				"-vga", "none")
+			startCmd = append(startCmd,
+				"-monitor", "stdio") // default: none
 		}
 	}
 
@@ -412,10 +418,21 @@ func (d *Driver) Start() error {
 		startCmd = append(startCmd,
 			"-nic", fmt.Sprintf("tap,model=virtio,ifname=%s,script=no,downscript=no", d.NetworkInterface),
 		)
-	case "vde":
+	case "socket":
 		startCmd = append(startCmd,
-			"-nic", fmt.Sprintf("vde,model=virtio,sock=%s", d.NetworkSocket),
-		)
+			"-nic", "socket,model=virtio-net-pci,listen=192.168.105.1:8443")
+		// startCmd = append(startCmd,
+		// VDE ref: -nic", fmt.Sprintf("vde,model=virtio,sock=%s", d.NetworkSocket)
+
+		// -M virt															# set.
+		// -device virtio-net-pci,netdev=net0		# enhanced.
+		// -netdev socket,id=net0,fd=3					# changed.
+		// -m 1024															# param.
+		// -accel hvf														# set.
+		// -cdrom ~/boot2docker.iso							# param.
+
+		// "-netdev", "socket,id=net0,fd=3",
+		// )
 	case "bridge":
 		startCmd = append(startCmd,
 			"-nic", fmt.Sprintf("bridge,model=virtio,br=%s", d.NetworkBridge),
@@ -424,16 +441,17 @@ func (d *Driver) Start() error {
 		log.Errorf("unknown network: %s", d.Network)
 	}
 
-	startCmd = append(startCmd,
-		"-daemonize")
+	// startCmd = append(startCmd,
+	// 	"-daemonize")
 
 	if d.CloudConfigRoot != "" {
 		startCmd = append(startCmd,
 			"-fsdev",
 			fmt.Sprintf("local,security_model=passthrough,readonly,id=fsdev0,path=%s", d.CloudConfigRoot))
 		startCmd = append(startCmd,
 			"-device",
-			"virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=config-2")
+			// "virtio-9p-pci,id=fs0,fsdev=fsdev0,mount_tag=config-2")
+			"virtio-net-pci,id=fs0,fsdev=fsdev0,mount_tag=config-2,netdev=net0")
 	}
 
 	if d.VirtioDrives {
@@ -445,7 +463,11 @@ func (d *Driver) Start() error {
 			d.diskPath())
 	}
 
-	if stdout, stderr, err := cmdOutErr(d.Program, startCmd...); err != nil {
+	socketCmd := append([]string{"/var/run/socket_vmnet", d.Program}, startCmd...)
+
+	fmt.Printf("\n\nABC QEMU Flags: %+v\n\n", socketCmd) // Debug.
+
+	if stdout, stderr, err := cmdOutErr("/opt/socket_vmnet/bin/socket_vmnet_client", socketCmd...); err != nil {
 		fmt.Printf("OUTPUT: %s\n", stdout)
 		fmt.Printf("ERROR: %s\n", stderr)
 		return err
@@ -457,16 +479,16 @@ func (d *Driver) Start() error {
 
 func cmdOutErr(cmdStr string, args ...string) (string, string, error) {
 	cmd := exec.Command(cmdStr, args...)
-	log.Debugf("executing: %v %v", cmdStr, strings.Join(args, " "))
+	log.Debugf("executing: %s %s", cmdStr, strings.Join(args, " "))
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
 	err := cmd.Run()
 	stdoutStr := stdout.String()
 	stderrStr := stderr.String()
-	log.Debugf("STDOUT: %v", stdoutStr)
-	log.Debugf("STDERR: %v", stderrStr)
+	log.Debugf("STDOUT: %s", stdoutStr)
+	log.Debugf("STDERR: %s", stderrStr)
 	if err != nil {
 		if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound {
 			err = fmt.Errorf("mystery error: %v", ee)
@@ -570,7 +592,7 @@ func (d *Driver) pidfilePath() string {
 func (d *Driver) generateDiskImage(size int) error {
 	log.Debugf("Creating %d MB hard disk image...", size)
 
-	magicString := "boot2docker, please format-me"
+	const magicString = "boot2docker, please format-me"
 
 	buf := new(bytes.Buffer)
 	tw := tar.NewWriter(buf)

pkg/minikube/cluster/ip.go

@@ -152,7 +152,7 @@ func DriverIP(api libmachine.API, machineName string) (net.IP, error) {
 		ipStr = oci.DefaultBindIPV4
 	}
 	if driver.IsQEMU(host.DriverName) {
-		ipStr = "127.0.0.1"
+		ipStr = "192.168.105.1"
 	}
 	ip := net.ParseIP(ipStr)
 	if ip == nil {

pkg/minikube/driver/driver.go

@@ -198,7 +198,7 @@ func AllowsPreload(driverName string) bool {
 // NeedsPortForward returns true if driver is unable provide direct IP connectivity
 func NeedsPortForward(name string) bool {
 	if IsQEMU(name) {
-		return true
+		return false
 	}
 	if !IsKIC(name) {
 		return false

pkg/minikube/driver/endpoint.go

@@ -45,8 +45,8 @@ func ControlPlaneEndpoint(cc *config.ClusterConfig, cp *config.Node, driverName
 			hostname = cc.KubernetesConfig.APIServerName
 		}
 		return hostname, ips[0], port, err
-	} else if NeedsPortForward(driverName) && IsQEMU(driverName) {
-		return "localhost", net.IPv4(127, 0, 0, 1), cc.APIServerPort, nil
+	} else if IsQEMU(driverName) {
+		return "localhost", net.IPv4(192, 168, 105, 1), cc.APIServerPort, nil
 	}
 
 	// https://github.com/kubernetes/minikube/issues/3878

pkg/minikube/machine/machine.go

@@ -122,9 +122,6 @@ func saveHost(api libmachine.API, h *host.Host, cfg *config.ClusterConfig, n *co
 	if err != nil {
 		return err
 	}
-	if ip == "127.0.0.1" && driver.IsQEMU(h.Driver.DriverName()) {
-		ip = "10.0.2.15"
-	}
 	n.IP = ip
 	return config.SaveNode(cfg, n)
 }

pkg/minikube/node/start.go

@@ -561,13 +561,13 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool)
 		return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network")
 	}
 
-	if driver.IsQEMU(host.Driver.DriverName()) {
-		apiServerPort, err := getPort()
-		if err != nil {
-			return runner, preExists, m, host, errors.Wrap(err, "Failed to find apiserver port")
-		}
-		cfg.APIServerPort = apiServerPort
-	}
+	// if driver.IsQEMU(host.Driver.DriverName()) {
+	// 	apiServerPort, err := getPort()
+	// 	if err != nil {
+	// 		return runner, preExists, m, host, errors.Wrap(err, "Failed to find apiserver port")
+	// 	}
+	// 	cfg.APIServerPort = apiServerPort
+	// }
 
 	// Bypass proxy for minikube's vm host ip
 	err = proxy.ExcludeIP(ip)
@@ -579,19 +579,19 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool)
 }
 
 // getPort asks the kernel for a free open port that is ready to use
-func getPort() (int, error) {
-	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
-	if err != nil {
-		panic(err)
-	}
-
-	l, err := net.ListenTCP("tcp", addr)
-	if err != nil {
-		return -1, errors.Errorf("Error accessing port %d", addr.Port)
-	}
-	defer l.Close()
-	return l.Addr().(*net.TCPAddr).Port, nil
-}
+// func getPort() (int, error) {
+// 	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+// 	if err != nil {
+// 		panic(err)
+// 	}
+
+// 	l, err := net.ListenTCP("tcp", addr)
+// 	if err != nil {
+// 		return -1, errors.Errorf("Error accessing port %d", addr.Port)
+// 	}
+// 	defer l.Close()
+// 	return l.Addr().(*net.TCPAddr).Port, nil
+// }
 
 // startHostInternal starts a new minikube host using a VM or None
 func startHostInternal(api libmachine.API, cc *config.ClusterConfig, n *config.Node, delOnFail bool) (*host.Host, bool, error) {
@@ -664,7 +664,7 @@ func validateNetwork(h *host.Host, r command.Runner, imageRepository string) (st
 		}
 	}
 
-	if !driver.BareMetal(h.Driver.DriverName()) && !driver.IsKIC(h.Driver.DriverName()) && !driver.IsQEMU(h.Driver.DriverName()) {
+	if !driver.BareMetal(h.Driver.DriverName()) && !driver.IsKIC(h.Driver.DriverName()) {
 		if err := trySSH(h, ip); err != nil {
 			return ip, err
 		}

pkg/minikube/registry/drvs/qemu2/qemu2.go

@@ -78,6 +78,7 @@ func qemuFirmwarePath(customPath string) (string, error) {
 			p = "/usr/local/Cellar/qemu"
 			fw = "share/qemu/edk2-x86_64-code.fd"
 		case "arm64":
+			// TODO() if installed via Macports, use "/opt/local/", see #14765.
 			p = "/opt/homebrew/Cellar/qemu"
 			fw = "share/qemu/edk2-aarch64-code.fd"
 		default:
@@ -177,7 +178,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) {
 		CPUType:        qemuCPU,
 		Firmware:       qemuFirmware,
 		VirtioDrives:   false,
-		Network:        "user",
+		Network:        "socket",
 		CacheMode:      "default",
 		IOMode:         "threads",
 	}, nil

pkg/minikube/tunnel/qemu/service_tunnel.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package qemu
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	typed_core "k8s.io/client-go/kubernetes/typed/core/v1"
+
+	"k8s.io/klog/v2"
+)
+
+// ServiceTunnel ...
+type ServiceTunnel struct {
+	sshPort        string
+	sshKey         string
+	v1Core         typed_core.CoreV1Interface
+	sshConn        *sshConn
+	suppressStdOut bool
+}
+
+// NewServiceTunnel ...
+func NewServiceTunnel(sshPort, sshKey string, v1Core typed_core.CoreV1Interface, suppressStdOut bool) *ServiceTunnel {
+	return &ServiceTunnel{
+		sshPort:        sshPort,
+		sshKey:         sshKey,
+		v1Core:         v1Core,
+		suppressStdOut: suppressStdOut,
+	}
+}
+
+// Start ...
+func (t *ServiceTunnel) Start(svcName, namespace string) ([]string, error) {
+	svc, err := t.v1Core.Services(namespace).Get(context.Background(), svcName, metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "Service %s was not found in %q namespace. You may select another namespace by using 'minikube service %s -n <namespace>", svcName, namespace, svcName)
+	}
+
+	t.sshConn, err = createSSHConnWithRandomPorts(svcName, t.sshPort, t.sshKey, svc)
+	if err != nil {
+		return nil, errors.Wrap(err, "creating ssh conn")
+	}
+
+	go func() {
+		t.sshConn.suppressStdOut = t.suppressStdOut
+		err = t.sshConn.startAndWait()
+		if err != nil {
+			klog.Errorf("error starting ssh tunnel: %v", err)
+		}
+	}()
+
+	urls := make([]string, 0, len(svc.Spec.Ports))
+	for _, port := range t.sshConn.ports {
+		urls = append(urls, fmt.Sprintf("http://127.0.0.1:%d", port))
+	}
+
+	return urls, nil
+}
+
+// Stop ...
+func (t *ServiceTunnel) Stop() {
+	err := t.sshConn.stop()
+	if err != nil {
+		klog.Warningf("Failed to stop ssh tunnel", err)
+	}
+}