Enhacements to support external networkers

gqt/create_test.go

@@ -337,6 +337,7 @@ var _ = Describe("Creating a Container", func() {
 	})
 
 	Context("when running with an external network plugin", func() {
+		var pluginOutput string
 		BeforeEach(func() {
 			tmpDir, err := ioutil.TempDir("", "netplugtest")
 			Expect(err).NotTo(HaveOccurred())
@@ -350,51 +351,57 @@ var _ = Describe("Creating a Container", func() {
 			}
 		})
 
-		It("does not run kawasaki", func() {
-			container, err := client.Create(garden.ContainerSpec{})
-			Expect(err).NotTo(HaveOccurred())
-
-			out := gbytes.NewBuffer()
-			process, err := container.Run(garden.ProcessSpec{
-				Path: "ip",
-				Args: []string{
-					"-o",
-					"link",
-					"show",
-				},
-			}, garden.ProcessIO{
-				Stdout: io.MultiWriter(GinkgoWriter, out),
+		Context("when the plugin returns a properties key", func() {
+			BeforeEach(func() {
+				pluginOutput = `{"properties": {"key":"value", "garden.network.container-ip":"10.10.24.3"}}`
+				args = append(args, "--network-plugin-extra-arg", pluginOutput)
 			})
-			Expect(err).NotTo(HaveOccurred())
 
-			exitCode, err := process.Wait()
-			Expect(err).NotTo(HaveOccurred())
-			Expect(exitCode).To(BeZero())
+			It("does not run kawasaki", func() {
+				container, err := client.Create(garden.ContainerSpec{})
+				Expect(err).NotTo(HaveOccurred())
+
+				out := gbytes.NewBuffer()
+				process, err := container.Run(garden.ProcessSpec{
+					Path: "ip",
+					Args: []string{
+						"-o",
+						"link",
+						"show",
+					},
+				}, garden.ProcessIO{
+					Stdout: io.MultiWriter(GinkgoWriter, out),
+				})
+				Expect(err).NotTo(HaveOccurred())
 
-			// ip link appends a new line on the end so let's trim that first
-			contents := strings.TrimRight(string(out.Contents()), "\n")
+				exitCode, err := process.Wait()
+				Expect(err).NotTo(HaveOccurred())
+				Expect(exitCode).To(BeZero())
 
-			// Check that we only have 1 interface, the loopback interface
-			Expect(strings.Split(contents, "\n")).To(HaveLen(1))
-			Expect(contents).To(ContainSubstring("LOOPBACK"))
+				// ip link appends a new line on the end so let's trim that first
+				contents := strings.TrimRight(string(out.Contents()), "\n")
+
+				// Check that we only have 1 interface, the loopback interface
+				Expect(strings.Split(contents, "\n")).To(HaveLen(1))
+				Expect(contents).To(ContainSubstring("LOOPBACK"))
+			})
 		})
 
 		Context("when the external network plugin returns invalid JSON", func() {
 			BeforeEach(func() {
-				pluginOutput := "invalid-json"
+				pluginOutput = "invalid-json"
 				args = append(args, "--network-plugin-extra-arg", pluginOutput)
-
 			})
 
 			It("returns a useful error message", func() {
 				_, err := client.Create(garden.ContainerSpec{})
-				Expect(err).To(MatchError(ContainSubstring("network plugin returned invalid JSON")))
+				Expect(err).To(MatchError(ContainSubstring("unmarshaling result from external networker: invalid character")))
 			})
 		})
 
 		Context("when the external network plugin returns JSON without the 'properties' key", func() {
 			BeforeEach(func() {
-				pluginOutput := `{"not-properties-key":{"foo":"bar"}}`
+				pluginOutput = `{"not-properties-key":{"foo":"bar"}}`
 				args = append(args, "--network-plugin-extra-arg", pluginOutput)
 
 			})

gqt/net_test.go

@@ -296,13 +296,15 @@ var _ = Describe("Networking", func() {
 	Context("when a network plugin path is provided at startup", func() {
 		var argsFile string
 		var stdinFile string
+		var pluginReturn string
 
 		BeforeEach(func() {
 			tmpDir, err := ioutil.TempDir("", "netplugtest")
 			Expect(err).NotTo(HaveOccurred())
 
 			argsFile = path.Join(tmpDir, "args.log")
 			stdinFile = path.Join(tmpDir, "stdin.log")
+			args = append(args, "--network-plugin-extra-arg", pluginReturn)
 
 			args = []string{
 				"--network-plugin", testNetPluginBin,
@@ -311,76 +313,72 @@ var _ = Describe("Networking", func() {
 			}
 		})
 
-		It("executes the network plugin during container creation", func() {
-			containerHandle := container.Handle()
-
-			Eventually(getContent(argsFile)).Should(
-				ContainSubstring(
-					fmt.Sprintf("%s %s --action up --handle %s --network %s", argsFile, stdinFile, containerHandle, containerNetwork),
-				),
-			)
-		})
+		Context("when the network plugin returns properties", func() {
+			BeforeEach(func() {
+				pluginReturn = `{"properties":{
+					"foo":"bar",
+					"kawasaki.mtu":"1499",
+					"garden.network.container-ip":"10.255.10.10",
+					"garden.network.host-ip":"255.255.255.255"
+				}}`
+				args = append(args, "--network-plugin-extra-arg", pluginReturn)
+				extraProperties = garden.Properties{
+					"some-property-on-the-spec": "some-value",
+					"network.some-key":          "some-value",
+					"network.some-other-key":    "some-other-value",
+					"some-other-key":            "do-not-propagate",
+					"garden.whatever":           "do-not-propagate",
+					"kawasaki.nope":             "do-not-propagate",
+				}
+			})
 
-		It("passes the container pid to plugin's stdin", func() {
-			Eventually(getContent(stdinFile)).Should(
-				MatchRegexp(`.*{"PID":[0-9]+}.*`),
-			)
-		})
+			Context("when the container spec has properties that start with 'network.'", func() {
+				var expectedJSON string
 
-		It("executes the network plugin during container destroy", func() {
-			containerHandle := container.Handle()
+				BeforeEach(func() {
+					expectedJSON = `"some-key":"some-value","some-other-key":"some-other-value"}`
+				})
 
-			Expect(client.Destroy(containerHandle)).To(Succeed())
-			Expect(argsFile).To(BeAnExistingFile())
+				It("propagates those properties as JSON to the network plugin up action", func() {
+					Eventually(getContent(stdinFile)).Should(ContainSubstring(expectedJSON))
+				})
+			})
 
-			Eventually(getContent(argsFile)).Should(
-				ContainSubstring(
-					fmt.Sprintf("%s %s --action down --handle %s", argsFile, stdinFile, containerHandle),
-				),
-			)
-		})
+			It("executes the network plugin during container destroy", func() {
+				containerHandle := container.Handle()
 
-		Context("when the container spec has properties that start with 'network.'", func() {
-			var expectedJSON string
+				Expect(client.Destroy(containerHandle)).To(Succeed())
+				Expect(argsFile).To(BeAnExistingFile())
 
-			BeforeEach(func() {
-				extraProperties = garden.Properties{
-					"network.some-key":       "some-value",
-					"network.some-other-key": "some-other-value",
-					"some-other-key":         "do-not-propagate",
-					"garden.whatever":        "do-not-propagate",
-					"kawasaki.nope":          "do-not-propagate",
-				}
-				expectedJSON = `{ "some-key": "some-value", "some-other-key": "some-other-value" }`
+				Eventually(getContent(argsFile)).Should(ContainSubstring(fmt.Sprintf("%s %s", argsFile, stdinFile)))
+				Eventually(getContent(argsFile)).Should(ContainSubstring(fmt.Sprintf("--action down --handle %s", containerHandle)))
 			})
 
-			It("propagates those properties as JSON to the network plugin up action", func() {
-				Eventually(getFlagValue(argsFile, "--properties")).Should(MatchJSON(expectedJSON))
+			It("passes the container pid to plugin's stdin", func() {
+				Eventually(getContent(stdinFile)).Should(
+					MatchRegexp(`.*{"Pid":[0-9]+.*}.*`),
+				)
 			})
-		})
 
-		Context("when the network plugin returns properties", func() {
-			BeforeEach(func() {
-				pluginReturn := `{"properties":{
-					"foo":"bar",
-					"kawasaki.mtu":"1499",
-					"garden.network.container-ip":"10.10.10.10",
-					"garden.network.host-ip":"11.11.11.11",
-					"garden.network.external-ip":"12.12.12.12"
-				}}`
-				args = append(args, "--network-plugin-extra-arg", pluginReturn)
-				extraProperties = garden.Properties{
-					"some-property-on-the-spec": "some-value",
-				}
+			It("executes the network plugin during container creation", func() {
+				containerHandle := container.Handle()
+
+				Eventually(getContent(argsFile)).Should(
+					ContainSubstring(
+						fmt.Sprintf("%s %s %s --action up --handle %s", argsFile, stdinFile, pluginReturn, containerHandle),
+					),
+				)
 			})
 
-			It("persits the returned properties to the container's properties", func() {
+			It("persists the returned properties to the container's properties", func() {
 				info, err := container.Info()
 				Expect(err).NotTo(HaveOccurred())
 
 				containerProperties := info.Properties
 
 				Expect(containerProperties["foo"]).To(Equal("bar"))
+				Expect(containerProperties["garden.network.container-ip"]).To(Equal("10.255.10.10"))
+				Expect(containerProperties["garden.network.host-ip"]).To(Equal("255.255.255.255"))
 			})
 
 			It("doesn't remove existing properties", func() {
@@ -389,6 +387,14 @@ var _ = Describe("Networking", func() {
 
 				Expect(info.Properties).To(HaveKey("some-property-on-the-spec"))
 			})
+
+			It("sets the ExternalIP and ContainerIP fields on the container.Info()", func() {
+				info, err := container.Info()
+				Expect(err).NotTo(HaveOccurred())
+
+				Expect(info.ExternalIP).NotTo(BeEmpty())
+				Expect(info.ContainerIP).To(Equal("10.255.10.10"))
+			})
 		})
 	})
 

guardiancmd/command.go

@@ -24,6 +24,7 @@ import (
 	"code.cloudfoundry.org/garden/server"
 	"code.cloudfoundry.org/guardian/gardener"
 	"code.cloudfoundry.org/guardian/kawasaki"
+	"code.cloudfoundry.org/guardian/kawasaki/dns"
 	"code.cloudfoundry.org/guardian/kawasaki/factory"
 	"code.cloudfoundry.org/guardian/kawasaki/iptables"
 	"code.cloudfoundry.org/guardian/kawasaki/ports"
@@ -359,14 +360,6 @@ func (cmd *GuardianCommand) wireRunDMCStarter(logger lager.Logger) gardener.Star
 }
 
 func (cmd *GuardianCommand) wireNetworker(log lager.Logger, propManager kawasaki.ConfigStore, portPool *ports.PortPool) (gardener.Networker, gardener.Starter, error) {
-	interfacePrefix := fmt.Sprintf("w%s", cmd.Server.Tag)
-	chainPrefix := fmt.Sprintf("w-%s-", cmd.Server.Tag)
-
-	var denyNetworksList []string
-	for _, network := range cmd.Network.DenyNetworks {
-		denyNetworksList = append(denyNetworksList, network.String())
-	}
-
 	externalIP, err := defaultExternalIP(cmd.Network.ExternalIP)
 	if err != nil {
 		return nil, nil, err
@@ -377,34 +370,49 @@ func (cmd *GuardianCommand) wireNetworker(log lager.Logger, propManager kawasaki
 		dnsServers[i] = ip.IP()
 	}
 
-	iptRunner := &logging.Runner{CommandRunner: linux_command_runner.New(), Logger: log.Session("iptables-runner")}
-	ipTables := iptables.New(cmd.Bin.IPTables.Path(), iptRunner, chainPrefix)
-	ipTablesStarter := iptables.NewStarter(ipTables, cmd.Network.AllowHostAccess, interfacePrefix, denyNetworksList)
-
-	idGenerator := kawasaki.NewSequentialIDGenerator(time.Now().UnixNano())
-
-	var networker kawasaki.Networker
 	if cmd.Network.Plugin.Path() != "" {
-		networker = netplugin.New(
+		resolvConfigurer := &kawasaki.ResolvConfigurer{
+			HostsFileCompiler:  &dns.HostsFileCompiler{},
+			ResolvFileCompiler: &dns.ResolvFileCompiler{},
+			FileWriter:         &dns.RootfsWriter{},
+			IDMapReader:        &kawasaki.RootIdMapReader{},
+		}
+		externalNetworker := netplugin.New(
 			linux_command_runner.New(),
 			propManager,
+			externalIP,
+			dnsServers,
+			resolvConfigurer,
 			cmd.Network.Plugin.Path(),
-			cmd.Network.PluginExtraArgs...,
-		)
-	} else {
-		networker = kawasaki.New(
-			cmd.Bin.IPTables.Path(),
-			kawasaki.SpecParserFunc(kawasaki.ParseSpec),
-			subnets.NewPool(cmd.Network.Pool.CIDR()),
-			kawasaki.NewConfigCreator(idGenerator, interfacePrefix, chainPrefix, externalIP, dnsServers, cmd.Network.Mtu),
-			propManager,
-			factory.NewDefaultConfigurer(ipTables),
-			portPool,
-			iptables.NewPortForwarder(ipTables),
-			iptables.NewFirewallOpener(ipTables),
+			cmd.Network.PluginExtraArgs,
 		)
+		return externalNetworker, externalNetworker, nil
+	}
+
+	var denyNetworksList []string
+	for _, network := range cmd.Network.DenyNetworks {
+		denyNetworksList = append(denyNetworksList, network.String())
 	}
 
+	interfacePrefix := fmt.Sprintf("w%s", cmd.Server.Tag)
+	chainPrefix := fmt.Sprintf("w-%s-", cmd.Server.Tag)
+	idGenerator := kawasaki.NewSequentialIDGenerator(time.Now().UnixNano())
+	iptRunner := &logging.Runner{CommandRunner: linux_command_runner.New(), Logger: log.Session("iptables-runner")}
+	ipTables := iptables.New(cmd.Bin.IPTables.Path(), iptRunner, chainPrefix)
+	ipTablesStarter := iptables.NewStarter(ipTables, cmd.Network.AllowHostAccess, interfacePrefix, denyNetworksList)
+
+	networker := kawasaki.New(
+		cmd.Bin.IPTables.Path(),
+		kawasaki.SpecParserFunc(kawasaki.ParseSpec),
+		subnets.NewPool(cmd.Network.Pool.CIDR()),
+		kawasaki.NewConfigCreator(idGenerator, interfacePrefix, chainPrefix, externalIP, dnsServers, cmd.Network.Mtu),
+		propManager,
+		factory.NewDefaultConfigurer(ipTables),
+		portPool,
+		iptables.NewPortForwarder(ipTables),
+		iptables.NewFirewallOpener(ipTables),
+	)
+
 	return networker, ipTablesStarter, nil
 }
 

kawasaki/networker.go

@@ -206,7 +206,7 @@ func (n *networker) NetIn(log lager.Logger, handle string, externalPort, contain
 		return 0, 0, err
 	}
 
-	if err := addPortMapping(log, n.configStore, handle, garden.PortMapping{
+	if err := AddPortMapping(log, n.configStore, handle, garden.PortMapping{
 		HostPort:      externalPort,
 		ContainerPort: containerPort,
 	}); err != nil {
@@ -289,7 +289,7 @@ func (n *networker) Restore(log lager.Logger, handle string) error {
 	return nil
 }
 
-func addPortMapping(logger lager.Logger, configStore ConfigStore, handle string, newMapping garden.PortMapping) error {
+func AddPortMapping(logger lager.Logger, configStore ConfigStore, handle string, newMapping garden.PortMapping) error {
 	var currentMappings portMappingList
 	if currentMappingsJson, ok := configStore.Get(handle, gardener.MappedPortsKey); ok {
 		var err error