Randomise outgoing port for connections in the SNAT iptables rule.

Gopkg.toml

@@ -92,3 +92,7 @@
   name = "k8s.io/client-go"
   # revision for tag "kubernetes-1.10.1"
   revision = "989be4278f353e42f26c416c53757d16fcff77db"
+
+[[constraint]]
+  name = "firepear.net/qsplit"
+  version = "2.2.2"

README.md

@@ -99,6 +99,13 @@ Default: `false`
 Specifies whether an external NAT gateway should be used to provide SNAT of secondary ENI IP addresses\. If set to `true`, the SNAT `iptables` rule and off\-VPC IP rule are not applied, and these rules are removed if they have already been applied\.  
 Disable SNAT if you need to allow inbound communication to your pods from external VPNs, direct connections, and external VPCs, and your pods do not need to access the Internet directly via an Internet Gateway\. However, your nodes must be running in a private subnet and connected to the internet through an AWS NAT Gateway or another external NAT device\.
 
+`AWS_VPC_K8S_CNI_RANDOMIZESNAT`  
+Type: String 
+Default: `hashrandom` 
+Valid Values: `hashrandom`, `prng`, `none`
+Specifies weather the SNAT `iptables` rule should randomize the outgoing ports for connections. When enabled (`hashrandom`) the `--random` flag will be added to the SNAT `iptables` rule. To use pseudo random number generation rather than hash based (i.e. `--random-fully`) use `prng` for the environment variable.
+Disable (`none`) this functionality if you rely on sequential port allocation for outgoing connections. For old versions if iptables that do not support `--random-fully` this option will fall back to `--random`.
+
 `WARM_ENI_TARGET`  
 Type: Integer  
 Default: `1`  

pkg/networkutils/network.go

@@ -56,6 +56,12 @@ const (
 	// be installed and will be removed if they are already installed.  Defaults to false.
 	envExternalSNAT = "AWS_VPC_K8S_CNI_EXTERNALSNAT"
 
+	// This environment is used to specify weather the SNAT rule added to iptables should randomize port
+	// allocation for outgoing connections. If set to "hashrandom" the SNAT iptables rule will have the "--random" flag
+	// added to it. Set it to "prng" if you want to use a pseudo random numbers, i.e. "--random-fully".
+	// Defaults to hashrandom.
+	envRandomizeSNAT = "AWS_VPC_K8S_CNI_RANDOMIZESNAT"
+
 	// envNodePortSupport is the name of environment variable that configures whether we implement support for
 	// NodePorts on the primary ENI.  This requires that we add additional iptables rules and loosen the kernel's
 	// RPF check as described below.  Defaults to true.
@@ -98,6 +104,7 @@ type NetworkAPIs interface {
 
 type linuxNetwork struct {
 	useExternalSNAT        bool
+	typeOfSNAT             snatType
 	nodePortSupportEnabled bool
 	connmark               uint32
 
@@ -110,15 +117,30 @@ type linuxNetwork struct {
 
 type iptablesIface interface {
 	Exists(table, chain string, rulespec ...string) (bool, error)
+	Insert(table, chain string, pos int, rulespec ...string) error
 	Append(table, chain string, rulespec ...string) error
 	Delete(table, chain string, rulespec ...string) error
+	List(table, chain string) ([]string, error)
 	NewChain(table, chain string) error
+	ClearChain(table, chain string) error
+	DeleteChain(table, chain string) error
+	ListChains(table string) ([]string, error)
+	HasRandomFully() bool
 }
 
+type snatType uint32
+
+const (
+	sequentialSNAT snatType = iota
+	randomHashSNAT
+	randomPRNGSNAT
+)
+
 // New creates a linuxNetwork object
 func New() NetworkAPIs {
 	return &linuxNetwork{
 		useExternalSNAT:        useExternalSNAT(),
+		typeOfSNAT:             typeOfSNAT(),
 		nodePortSupportEnabled: nodePortSupportEnabled(),
 		mainENIMark:            getConnmark(),
 
@@ -294,22 +316,34 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDR *net.IPNet, vpcCIDRs []*string,
 			}})
 	}
 
+	// Prepare the Desired Rule for SNAT Rule
 	curChain := fmt.Sprintf("AWS-SNAT-CHAIN-%d", len(vpcCIDRs))
+	snatRule := []string{"-m", "comment", "--comment", "AWS, SNAT",
+		"-m", "addrtype", "!", "--dst-type", "LOCAL",
+		"-j", "SNAT", "--to-source", primaryAddr.String()}
+	if n.typeOfSNAT == randomHashSNAT {
+		snatRule = append(snatRule, "--random")
+	}
+	if n.typeOfSNAT == randomPRNGSNAT {
+		if ipt.HasRandomFully() {
+			snatRule = append(snatRule, "--random-fully")
+		} else {
+			log.Warn("prng (--random-fully) requested, but iptables version does not support it. " +
+				"Falling back to hashrandom (--random)")
+			snatRule = append(snatRule, "--random")
+		}
+	}
 	iptableRules = append(iptableRules, iptablesRule{
 		name:        "last SNAT rule for non-VPC outbound traffic",
 		shouldExist: !n.useExternalSNAT,
 		table:       "nat",
 		chain:       curChain,
-		rule: []string{
-			"-m", "comment", "--comment", "AWS, SNAT",
-			"-m", "addrtype", "!", "--dst-type", "LOCAL",
-			"-j", "SNAT", "--to-source", primaryAddr.String()},
+		rule:        snatRule,
 	})
 
 	log.Debugf("iptableRules: %v", iptableRules)
 
 	iptableRules = append(iptableRules, iptablesRule{
-
 		name:        "connmark for primary ENI",
 		shouldExist: n.nodePortSupportEnabled,
 		table:       "mangle",
@@ -427,6 +461,33 @@ func useExternalSNAT() bool {
 	return getBoolEnvVar(envExternalSNAT, false)
 }
 
+func typeOfSNAT() snatType {
+	defaultValue := randomHashSNAT
+	defaultString := "hashrandom"
+	strValue := os.Getenv(envRandomizeSNAT)
+	switch strValue {
+	case "":
+		// empty means default
+		return defaultValue
+	case "prng":
+		// prng means to use --random-fully
+		// note: for old versions of iptables, this will fall back to --random
+		return randomPRNGSNAT
+	case "none":
+		// none means to disable randomisation (no flag)
+		return sequentialSNAT
+
+	case defaultString:
+		// hashrandom means to use --random
+		return randomHashSNAT
+	default:
+		// if we get to this point, the environment variable has an invalid value
+		log.Errorf("Failed to parse %s; using default: %s. Provided string was \"%s\"", envRandomizeSNAT, defaultString,
+			strValue)
+		return defaultValue
+	}
+}
+
 func nodePortSupportEnabled() bool {
 	return getBoolEnvVar(envNodePortSupport, true)
 }

pkg/networkutils/network_test.go

@@ -313,6 +313,10 @@ func (ipt *mockIptables) Exists(table, chainName string, rulespec ...string) (bo
 	return false, nil
 }
 
+func (ipt *mockIptables) Insert(table, chain string, pos int, rulespec ...string) error {
+	return nil
+}
+
 func (ipt *mockIptables) Append(table, chain string, rulespec ...string) error {
 	if ipt.dataplaneState[table] == nil {
 		ipt.dataplaneState[table] = map[string][][]string{}
@@ -339,10 +343,32 @@ func (ipt *mockIptables) Delete(table, chainName string, rulespec ...string) err
 	return nil
 }
 
+func (ipt *mockIptables) List(table, chain string) ([]string, error) {
+	return nil, nil
+
+}
+
 func (ipt *mockIptables) NewChain(table, chain string) error {
 	return nil
 }
 
+func (ipt *mockIptables) ClearChain(table, chain string) error {
+	return nil
+}
+
+func (ipt *mockIptables) DeleteChain(table, chain string) error {
+	return nil
+}
+
+func (ipt *mockIptables) ListChains(table string) ([]string, error) {
+	return nil, nil
+}
+
+func (ipt *mockIptables) HasFullyRandom() bool {
+	// TODO: Work out how to write a test case for this
+	return true
+}
+
 type mockFile struct {
 	closed bool
 	data   string