Merge pull request moby#47871 from robmry/portmapper_fixes_and_nonat

Portmapper improvements, and options to disable NAT

daemon/daemon.go

@@ -584,6 +584,17 @@ func (daemon *Daemon) restore(cfg *configStore) error {
 		return fmt.Errorf("Error initializing network controller: %v", err)
 	}
 
+	// If port-mapping failed during live-restore of a container, perhaps because
+	// a host port that was previously mapped to a container is now in-use by some
+	// other process - ports will not be mapped for the restored container, but it
+	// will be running. Replace the restored mappings in NetworkSettings with the
+	// current state so that the problem is visible in 'inspect'.
+	for _, c := range containers {
+		if sb, err := daemon.netController.SandboxByID(c.NetworkSettings.SandboxID); err == nil {
+			c.NetworkSettings.Ports = getPortMapInfo(sb)
+		}
+	}
+
 	// Now that all the containers are registered, register the links
 	for _, c := range containers {
 		group.Add(1)

daemon/network.go

@@ -1026,11 +1026,16 @@ func getEndpointPortMapInfo(ep *libnetwork.Endpoint) (nat.PortMap, error) {
 
 	if portMapping, ok := mapData.([]networktypes.PortBinding); ok {
 		for _, pp := range portMapping {
+			// Use an empty string for the host port if there's no port assigned.
 			natPort, err := nat.NewPort(pp.Proto.String(), strconv.Itoa(int(pp.Port)))
 			if err != nil {
 				return pm, err
 			}
-			natBndg := nat.PortBinding{HostIP: pp.HostIP.String(), HostPort: strconv.Itoa(int(pp.HostPort))}
+			var hp string
+			if pp.HostPort > 0 {
+				hp = strconv.Itoa(int(pp.HostPort))
+			}
+			natBndg := nat.PortBinding{HostIP: pp.HostIP.String(), HostPort: hp}
 			pm[natPort] = append(pm[natPort], natBndg)
 		}
 	}

integration/internal/container/ops.go

@@ -2,6 +2,7 @@ package container
 
 import (
 	"maps"
+	"slices"
 	"strings"
 
 	"github.com/docker/docker/api/types/container"
@@ -71,6 +72,16 @@ func WithExposedPorts(ports ...string) func(*TestContainerConfig) {
 	}
 }
 
+// WithPortMap sets/replaces port mappings.
+func WithPortMap(pm nat.PortMap) func(*TestContainerConfig) {
+	return func(c *TestContainerConfig) {
+		c.HostConfig.PortBindings = nat.PortMap{}
+		for p, b := range pm {
+			c.HostConfig.PortBindings[p] = slices.Clone(b)
+		}
+	}
+}
+
 // WithTty sets the TTY mode of the container
 func WithTty(tty bool) func(*TestContainerConfig) {
 	return func(c *TestContainerConfig) {

integration/networking/bridge_test.go

@@ -13,9 +13,11 @@ import (
 	networktypes "github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/integration/internal/container"
 	"github.com/docker/docker/integration/internal/network"
+	"github.com/docker/docker/libnetwork/drivers/bridge"
 	"github.com/docker/docker/libnetwork/netlabel"
 	"github.com/docker/docker/testutil"
 	"github.com/docker/docker/testutil/daemon"
+	"github.com/docker/go-connections/nat"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
@@ -912,3 +914,84 @@ func TestSetEndpointSysctl(t *testing.T) {
 		}
 	}
 }
+
+func TestDisableNAT(t *testing.T) {
+	skip.If(t, testEnv.DaemonInfo.OSType == "windows", "bridge driver option doesn't apply to Windows")
+
+	ctx := setupTest(t)
+	d := daemon.New(t)
+	d.StartWithBusybox(ctx, t)
+	defer d.Stop(t)
+
+	c := d.NewClientT(t)
+	defer c.Close()
+
+	testcases := []struct {
+		name       string
+		gwMode4    string
+		gwMode6    string
+		expPortMap nat.PortMap
+	}{
+		{
+			name: "defaults",
+			expPortMap: nat.PortMap{
+				"80/tcp": []nat.PortBinding{
+					{HostIP: "0.0.0.0", HostPort: "8080"},
+					{HostIP: "::", HostPort: "8080"},
+				},
+			},
+		},
+		{
+			name:    "nat4 routed6",
+			gwMode4: "nat",
+			gwMode6: "routed",
+			expPortMap: nat.PortMap{
+				"80/tcp": []nat.PortBinding{
+					{HostIP: "0.0.0.0", HostPort: "8080"},
+					{HostIP: "::", HostPort: ""},
+				},
+			},
+		},
+		{
+			name:    "nat6 routed4",
+			gwMode4: "routed",
+			gwMode6: "nat",
+			expPortMap: nat.PortMap{
+				"80/tcp": []nat.PortBinding{
+					{HostIP: "0.0.0.0", HostPort: ""},
+					{HostIP: "::", HostPort: "8080"},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := testutil.StartSpan(ctx, t)
+
+			const netName = "testnet"
+			nwOpts := []func(options *networktypes.CreateOptions){
+				network.WithIPv6(),
+				network.WithIPAM("fd2a:a2c3:4448::/64", "fd2a:a2c3:4448::1"),
+			}
+			if tc.gwMode4 != "" {
+				nwOpts = append(nwOpts, network.WithOption(bridge.IPv4GatewayMode, tc.gwMode4))
+			}
+			if tc.gwMode6 != "" {
+				nwOpts = append(nwOpts, network.WithOption(bridge.IPv6GatewayMode, tc.gwMode6))
+			}
+			network.CreateNoError(ctx, t, c, netName, nwOpts...)
+			defer network.RemoveNoError(ctx, t, c, netName)
+
+			id := container.Run(ctx, t, c,
+				container.WithNetworkMode(netName),
+				container.WithExposedPorts("80/tcp"),
+				container.WithPortMap(nat.PortMap{"80/tcp": {{HostPort: "8080"}}}),
+			)
+			defer c.ContainerRemove(ctx, id, containertypes.RemoveOptions{Force: true})
+
+			inspect := container.Inspect(ctx, t, c, id)
+			assert.Check(t, is.DeepEqual(inspect.NetworkSettings.Ports, tc.expPortMap))
+		})
+	}
+}

libnetwork/drivers/bridge/bridge_linux.go

@@ -20,8 +20,6 @@ import (
 	"github.com/docker/docker/libnetwork/netutils"
 	"github.com/docker/docker/libnetwork/ns"
 	"github.com/docker/docker/libnetwork/options"
-	"github.com/docker/docker/libnetwork/portallocator"
-	"github.com/docker/docker/libnetwork/portmapper"
 	"github.com/docker/docker/libnetwork/scope"
 	"github.com/docker/docker/libnetwork/types"
 	"github.com/pkg/errors"
@@ -63,6 +61,8 @@ type networkConfiguration struct {
 	BridgeName           string
 	EnableIPv6           bool
 	EnableIPMasquerade   bool
+	GwModeIPv4           gwMode
+	GwModeIPv6           gwMode
 	EnableICC            bool
 	InhibitIPv4          bool
 	Mtu                  int
@@ -113,7 +113,7 @@ type bridgeEndpoint struct {
 	macAddress      net.HardwareAddr
 	containerConfig *containerConfiguration
 	extConnConfig   *connectivityConfiguration
-	portMapping     []types.PortBinding // Operation port bindings
+	portMapping     []portBinding // Operational port bindings
 	dbIndex         uint64
 	dbExists        bool
 }
@@ -123,9 +123,7 @@ type bridgeNetwork struct {
 	bridge        *bridgeInterface // The bridge's L3 interface
 	config        *networkConfiguration
 	endpoints     map[string]*bridgeEndpoint // key: endpoint id
-	portMapper    *portmapper.PortMapper
-	portMapperV6  *portmapper.PortMapper
-	driver        *driver // The network's driver
+	driver        *driver                    // The network's driver
 	iptCleanFuncs iptablesCleanFuncs
 	sync.Mutex
 }
@@ -144,15 +142,21 @@ type driver struct {
 	store             *datastore.Store
 	nlh               *netlink.Handle
 	configNetwork     sync.Mutex
-	portAllocator     *portallocator.PortAllocator // Overridable for tests.
 	sync.Mutex
 }
 
+type gwMode string
+
+const (
+	gwModeDefault gwMode = ""
+	gwModeNAT     gwMode = "nat"
+	gwModeRouted  gwMode = "routed"
+)
+
 // New constructs a new bridge driver
 func newDriver() *driver {
 	return &driver{
-		networks:      map[string]*bridgeNetwork{},
-		portAllocator: portallocator.Get(),
+		networks: map[string]*bridgeNetwork{},
 	}
 }
 
@@ -292,6 +296,14 @@ func (c *networkConfiguration) fromLabels(labels map[string]string) error {
 			if c.EnableIPMasquerade, err = strconv.ParseBool(value); err != nil {
 				return parseErr(label, value, err.Error())
 			}
+		case IPv4GatewayMode:
+			if c.GwModeIPv4, err = newGwMode(value); err != nil {
+				return parseErr(label, value, err.Error())
+			}
+		case IPv6GatewayMode:
+			if c.GwModeIPv6, err = newGwMode(value); err != nil {
+				return parseErr(label, value, err.Error())
+			}
 		case EnableICC:
 			if c.EnableICC, err = strconv.ParseBool(value); err != nil {
 				return parseErr(label, value, err.Error())
@@ -324,6 +336,20 @@ func (c *networkConfiguration) fromLabels(labels map[string]string) error {
 	return nil
 }
 
+func newGwMode(gwMode string) (gwMode, error) {
+	switch gwMode {
+	case "nat":
+		return gwModeNAT, nil
+	case "routed":
+		return gwModeRouted, nil
+	}
+	return gwModeDefault, fmt.Errorf("unknown gateway mode %s", gwMode)
+}
+
+func (m gwMode) natDisabled() bool {
+	return m == gwModeRouted
+}
+
 func parseErr(label, value, errString string) error {
 	return types.InvalidParameterErrorf("failed to parse %s value: %v (%s)", label, value, errString)
 }
@@ -332,6 +358,21 @@ func (n *bridgeNetwork) registerIptCleanFunc(clean iptableCleanFunc) {
 	n.iptCleanFuncs = append(n.iptCleanFuncs, clean)
 }
 
+func (n *bridgeNetwork) iptablesEnabled(version iptables.IPVersion) (bool, error) {
+	n.Lock()
+	defer n.Unlock()
+	if n.driver == nil {
+		return false, types.InvalidParameterErrorf("no driver found")
+	}
+
+	n.driver.Lock()
+	defer n.driver.Unlock()
+	if version == iptables.IPv6 {
+		return n.driver.config.EnableIP6Tables, nil
+	}
+	return n.driver.config.EnableIPTables, nil
+}
+
 func (n *bridgeNetwork) getDriverChains(version iptables.IPVersion) (*iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, *iptables.ChainInfo, error) {
 	n.Lock()
 	defer n.Unlock()
@@ -355,6 +396,21 @@ func (n *bridgeNetwork) getNetworkBridgeName() string {
 	return config.BridgeName
 }
 
+func (n *bridgeNetwork) getNATDisabled() (ipv4, ipv6 bool) {
+	n.Lock()
+	defer n.Unlock()
+	return n.config.GwModeIPv4.natDisabled(), n.config.GwModeIPv6.natDisabled()
+}
+
+func (n *bridgeNetwork) userlandProxyPath() string {
+	n.Lock()
+	defer n.Unlock()
+	if n.driver == nil {
+		return ""
+	}
+	return n.driver.userlandProxyPath()
+}
+
 func (n *bridgeNetwork) getEndpoint(eid string) (*bridgeEndpoint, error) {
 	if eid == "" {
 		return nil, InvalidEndpointIDError(eid)
@@ -508,6 +564,16 @@ func (d *driver) getNetwork(id string) (*bridgeNetwork, error) {
 	return nil, types.NotFoundErrorf("network not found: %s", id)
 }
 
+func (d *driver) userlandProxyPath() string {
+	d.Lock()
+	defer d.Unlock()
+
+	if d.config.EnableUserlandProxy {
+		return d.config.UserlandProxyPath
+	}
+	return ""
+}
+
 func parseNetworkGenericOptions(data interface{}) (*networkConfiguration, error) {
 	var (
 		err    error
@@ -714,13 +780,11 @@ func (d *driver) createNetwork(config *networkConfiguration) (err error) {
 
 	// Create and set network handler in driver
 	network := &bridgeNetwork{
-		id:           config.ID,
-		endpoints:    make(map[string]*bridgeEndpoint),
-		config:       config,
-		portMapper:   portmapper.NewWithPortAllocator(d.portAllocator, d.config.UserlandProxyPath),
-		portMapperV6: portmapper.NewWithPortAllocator(d.portAllocator, d.config.UserlandProxyPath),
-		bridge:       bridgeIface,
-		driver:       d,
+		id:        config.ID,
+		endpoints: make(map[string]*bridgeEndpoint),
+		config:    config,
+		bridge:    bridgeIface,
+		driver:    d,
 	}
 
 	d.Lock()
@@ -1221,7 +1285,7 @@ func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, erro
 		// Return a copy of the operational data
 		pmc := make([]types.PortBinding, 0, len(ep.portMapping))
 		for _, pm := range ep.portMapping {
-			pmc = append(pmc, pm.GetCopy())
+			pmc = append(pmc, pm.PortBinding.GetCopy())
 		}
 		m[netlabel.PortMap] = pmc
 	}
@@ -1321,9 +1385,16 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string
 	}
 
 	// Program any required port mapping and store them in the endpoint
-	endpoint.portMapping, err = network.allocatePorts(endpoint, network.config.DefaultBindingIP, d.config.EnableUserlandProxy)
-	if err != nil {
-		return err
+	if endpoint.extConnConfig != nil && endpoint.extConnConfig.PortBindings != nil {
+		endpoint.portMapping, err = network.addPortMappings(
+			endpoint.addr,
+			endpoint.addrv6,
+			endpoint.extConnConfig.PortBindings,
+			network.config.DefaultBindingIP,
+		)
+		if err != nil {
+			return err
+		}
 	}
 
 	defer func() {