feat(transparent-proxy): allow to wait for xtables lock and retry whe…

…n installing tproxy fails (backport of #7870) (#7892)

Signed-off-by: Bart Smykla <bartek@smykla.com>

app/kumactl/cmd/completion/testdata/bash.golden

@@ -5378,6 +5378,10 @@ _kumactl_install_transparent-proxy()
     two_word_flags+=("--kuma-dp-user")
     local_nonpersistent_flags+=("--kuma-dp-user")
     local_nonpersistent_flags+=("--kuma-dp-user=")
+    flags+=("--max-retries=")
+    two_word_flags+=("--max-retries")
+    local_nonpersistent_flags+=("--max-retries")
+    local_nonpersistent_flags+=("--max-retries=")
     flags+=("--redirect-all-dns-traffic")
     local_nonpersistent_flags+=("--redirect-all-dns-traffic")
     flags+=("--redirect-dns")
@@ -5406,6 +5410,10 @@ _kumactl_install_transparent-proxy()
     local_nonpersistent_flags+=("--redirect-outbound-port=")
     flags+=("--skip-dns-conntrack-zone-split")
     local_nonpersistent_flags+=("--skip-dns-conntrack-zone-split")
+    flags+=("--sleep-between-retries=")
+    two_word_flags+=("--sleep-between-retries")
+    local_nonpersistent_flags+=("--sleep-between-retries")
+    local_nonpersistent_flags+=("--sleep-between-retries=")
     flags+=("--store-firewalld")
     local_nonpersistent_flags+=("--store-firewalld")
     flags+=("--use-transparent-proxy-engine-v1")
@@ -5416,6 +5424,14 @@ _kumactl_install_transparent-proxy()
     two_word_flags+=("--vnet")
     local_nonpersistent_flags+=("--vnet")
     local_nonpersistent_flags+=("--vnet=")
+    flags+=("--wait=")
+    two_word_flags+=("--wait")
+    local_nonpersistent_flags+=("--wait")
+    local_nonpersistent_flags+=("--wait=")
+    flags+=("--wait-interval=")
+    two_word_flags+=("--wait-interval")
+    local_nonpersistent_flags+=("--wait-interval")
+    local_nonpersistent_flags+=("--wait-interval=")
     flags+=("--api-timeout=")
     two_word_flags+=("--api-timeout")
     flags+=("--config-file=")

app/kumactl/cmd/install/install_transparent_proxy.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	os_user "os/user"
 	"runtime"
+	"time"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
@@ -43,6 +44,10 @@ type transparentProxyArgs struct {
 	EbpfCgroupPath                 string
 	EbpfTCAttachIface              string
 	VnetNetworks                   []string
+	Wait                           uint
+	WaitInterval                   uint
+	MaxRetries                     int
+	SleepBetweenRetries            time.Duration
 }
 
 func newInstallTransparentProxy() *cobra.Command {
@@ -72,6 +77,10 @@ func newInstallTransparentProxy() *cobra.Command {
 		EbpfCgroupPath:                 "/sys/fs/cgroup",
 		EbpfTCAttachIface:              "",
 		VnetNetworks:                   []string{},
+		Wait:                           5,
+		WaitInterval:                   0,
+		MaxRetries:                     5,
+		SleepBetweenRetries:            2 * time.Second,
 	}
 	cmd := &cobra.Command{
 		Use:   "transparent-proxy",
@@ -142,7 +151,7 @@ runuser -u kuma-dp -- \
 			}
 
 			if args.DNSUpstreamTargetChain != "RETURN" {
-				_, _ = cmd.ErrOrStderr().Write([]byte("# `--redirect-dns-upstream-target-chain` is deprecated, please avoid using it"))
+				_, _ = cmd.ErrOrStderr().Write([]byte("# [WARNING] `--redirect-dns-upstream-target-chain` is deprecated, please avoid using it"))
 			}
 
 			if args.EbpfEnabled {
@@ -155,19 +164,19 @@ runuser -u kuma-dp -- \
 				}
 
 				if args.StoreFirewalld {
-					_, _ = cmd.ErrOrStderr().Write([]byte("# --store-firewalld will be ignored when --ebpf-enabled is being used"))
+					_, _ = cmd.ErrOrStderr().Write([]byte("# [WARNING] --store-firewalld will be ignored when --ebpf-enabled is being used"))
 				}
 
 				if args.SkipDNSConntrackZoneSplit {
-					_, _ = cmd.ErrOrStderr().Write([]byte("# --skip-dns-conntrack-zone-split will be ignored when --ebpf-enabled is being used"))
+					_, _ = cmd.ErrOrStderr().Write([]byte("# [WARNING] --skip-dns-conntrack-zone-split will be ignored when --ebpf-enabled is being used"))
 				}
 			}
 
 			if err := configureTransparentProxy(cmd, &args); err != nil {
 				return err
 			}
 
-			_, _ = cmd.OutOrStdout().Write([]byte("Transparent proxy set up successfully, you can now run kuma-dp using transparent-proxy.\n"))
+			_, _ = cmd.OutOrStdout().Write([]byte("# transparent proxy set up successfully\n"))
 			return nil
 		},
 	}
@@ -207,6 +216,10 @@ runuser -u kuma-dp -- \
 	cmd.Flags().StringArrayVar(&args.ExcludeOutboundTCPPortsForUIDs, "exclude-outbound-tcp-ports-for-uids", []string{}, "tcp outbound ports to exclude for specific UIDs in a format of ports:uids where both ports and uids can be a single value, a list, a range or a combination of all, e.g. 3000-5000:103,104,106-108 would mean exclude ports from 3000 to 5000 for UIDs 103, 104, 106, 107, 108")
 	cmd.Flags().StringArrayVar(&args.ExcludeOutboundUDPPortsForUIDs, "exclude-outbound-udp-ports-for-uids", []string{}, "udp outbound ports to exclude for specific UIDs in a format of ports:uids where both ports and uids can be a single value, a list, a range or a combination of all, e.g. 3000-5000:103,104,106-108 would mean exclude ports from 3000 to 5000 for UIDs 103, 104, 106, 107, 108")
 	cmd.Flags().StringArrayVar(&args.VnetNetworks, "vnet", []string{}, "virtual networks in a format of interfaceNameRegex:CIDR split by ':' where interface name doesn't have to be exact name e.g. docker0:172.17.0.0/16, br+:172.18.0.0/16, iface:::1/64")
+	cmd.Flags().UintVar(&args.Wait, "wait", args.Wait, "specify the amount of time, in seconds, that the application should wait for the xtables exclusive lock before exiting. If the lock is not available within the specified time, the application will exit with an error")
+	cmd.Flags().UintVar(&args.WaitInterval, "wait-interval", args.WaitInterval, "flag can be used to specify the amount of time, in microseconds, that iptables should wait between each iteration of the lock acquisition loop. This can be useful if the xtables lock is being held by another application for a long time, and you want to reduce the amount of CPU that iptables uses while waiting for the lock")
+	cmd.Flags().IntVar(&args.MaxRetries, "max-retries", args.MaxRetries, "flag can be used to specify the maximum number of times to retry an installation before giving up")
+	cmd.Flags().DurationVar(&args.SleepBetweenRetries, "sleep-between-retries", args.SleepBetweenRetries, "flag can be used to specify the amount of time to sleep between retries")
 
 	return cmd
 }
@@ -266,6 +279,10 @@ func configureTransparentProxy(cmd *cobra.Command, args *transparentProxyArgs) e
 		VnetNetworks:                   args.VnetNetworks,
 		Stdout:                         cmd.OutOrStdout(),
 		Stderr:                         cmd.OutOrStderr(),
+		Wait:                           args.Wait,
+		WaitInterval:                   args.WaitInterval,
+		MaxRetries:                     args.MaxRetries,
+		SleepBetweenRetries:            args.SleepBetweenRetries,
 	}
 
 	if args.UseTransparentProxyEngineV1 {

app/kumactl/cmd/install/install_transparent_proxy_test.go

@@ -74,7 +74,7 @@ var _ = Describe("kumactl install transparent proxy", func() {
 				"--redirect-dns-upstream-target-chain", "DOCKER_OUTPUT",
 			},
 			goldenFile:   "install-transparent-proxy.dns.golden.txt",
-			errorMessage: "# `--redirect-dns-upstream-target-chain` is deprecated, please avoid using it",
+			errorMessage: "# [WARNING] `--redirect-dns-upstream-target-chain` is deprecated, please avoid using it",
 		}),
 		Entry("should generate defaults with user id and DNS redirected without conntrack zone splitting and log deprecate", testCase{
 			extraArgs: []string{
@@ -85,7 +85,7 @@ var _ = Describe("kumactl install transparent proxy", func() {
 				"--skip-dns-conntrack-zone-split",
 			},
 			goldenFile:   "install-transparent-proxy.dns.golden.txt",
-			errorMessage: "# `--redirect-dns-upstream-target-chain` is deprecated, please avoid using it",
+			errorMessage: "# [WARNING] `--redirect-dns-upstream-target-chain` is deprecated, please avoid using it",
 		}),
 		Entry("should generate defaults with overrides", testCase{
 			extraArgs: []string{

docs/generated/cmd/kumactl/kumactl_install_transparent-proxy.md

@@ -74,6 +74,7 @@ kumactl install transparent-proxy [flags]
   -h, --help                                                                            help for transparent-proxy
       --kuma-dp-uid string                                                              the UID of the user that will run kuma-dp
       --kuma-dp-user string                                                             the user that will run kuma-dp
+      --max-retries int                                                                 flag can be used to specify the maximum number of times to retry an installation before giving up (default 5)
       --redirect-all-dns-traffic                                                        redirect all DNS traffic to a specified port, unlike --redirect-dns this will not be limited to the dns servers identified in /etc/resolve.conf
       --redirect-dns                                                                    redirect only DNS requests targeted to the servers listed in /etc/resolv.conf to a specified port
       --redirect-dns-port string                                                        the port where the DNS agent is listening (default "15053")
@@ -83,10 +84,13 @@ kumactl install transparent-proxy [flags]
       --redirect-inbound-port-v6 networking.transparentProxying.redirectPortInboundV6   IPv6 inbound port redirected to Envoy, as specified in dataplane's networking.transparentProxying.redirectPortInboundV6 (default "15010")
       --redirect-outbound-port networking.transparentProxying.redirectPortOutbound      outbound port redirected to Envoy, as specified in dataplane's networking.transparentProxying.redirectPortOutbound (default "15001")
       --skip-dns-conntrack-zone-split                                                   skip applying conntrack zone splitting iptables rules
+      --sleep-between-retries duration                                                  flag can be used to specify the amount of time to sleep between retries (default 2s)
       --store-firewalld                                                                 store the iptables changes with firewalld
       --use-transparent-proxy-engine-v1                                                 use legacy transparent proxy engine v1
       --verbose                                                                         verbose
       --vnet stringArray                                                                virtual networks in a format of interfaceNameRegex:CIDR split by ':' where interface name doesn't have to be exact name e.g. docker0:172.17.0.0/16, br+:172.18.0.0/16, iface:::1/64
+      --wait uint                                                                       specify the amount of time, in seconds, that the application should wait for the xtables exclusive lock before exiting. If the lock is not available within the specified time, the application will exit with an error (default 5)
+      --wait-interval uint                                                              flag can be used to specify the amount of time, in microseconds, that iptables should wait between each iteration of the lock acquisition loop. This can be useful if the xtables lock is being held by another application for a long time, and you want to reduce the amount of CPU that iptables uses while waiting for the lock
 ```
 
 ### Options inherited from parent commands

pkg/transparentproxy/config/config.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"time"
 )
 
 type TransparentProxyConfig struct {
@@ -35,6 +36,11 @@ type TransparentProxyConfig struct {
 	VnetNetworks                   []string
 	Stdout                         io.Writer
 	Stderr                         io.Writer
+	RestoreLegacy                  bool
+	Wait                           uint
+	WaitInterval                   uint
+	MaxRetries                     int
+	SleepBetweenRetries            time.Duration
 }
 
 const DebugLogLevel uint16 = 7
@@ -116,6 +122,11 @@ type LogConfig struct {
 	Level   uint16
 }
 
+type RetryConfig struct {
+	MaxRetries         int
+	SleepBetweenReties time.Duration
+}
+
 type Config struct {
 	Owner    Owner
 	Redirect Redirect
@@ -140,6 +151,22 @@ type Config struct {
 	// Log is the place where configuration for logging iptables rules will
 	// be placed
 	Log LogConfig
+	// Wait is the amount of time, in seconds, that the application should wait
+	// for the xtables exclusive lock before exiting. If the lock is not
+	// available within the specified time, the application will exit with
+	// an error. Default value *(0) means wait forever. To disable this behavior
+	// and exit immediately if the xtables lock is not available, set this to
+	// nil
+	Wait uint
+	// WaitInterval is the amount of time, in microseconds, that iptables should
+	// wait between each iteration of the lock acquisition loop. This can be
+	// useful if the xtables lock is being held by another application for
+	// a long time, and you want to reduce the amount of CPU that iptables uses
+	// while waiting for the lock
+	WaitInterval uint
+	// Retry allows you to configure the number of times that the system should
+	// retry an installation if it fails
+	Retry RetryConfig
 }
 
 // ShouldDropInvalidPackets is just a convenience function which can be used in
@@ -184,7 +211,7 @@ func (c Config) ShouldConntrackZoneSplit() bool {
 	// skip conntrack related rules and move forward
 	if err := exec.Command("iptables", "-m", "conntrack", "--help").Run(); err != nil {
 		_, _ = fmt.Fprintf(c.RuntimeStdout,
-			"[WARNING] error occurred when validating if 'conntrack' iptables "+
+			"# [WARNING] error occurred when validating if 'conntrack' iptables "+
 				"module is present. Rules for DNS conntrack zone "+
 				"splitting won't be applied: %s\n", err,
 		)
@@ -243,6 +270,12 @@ func defaultConfig() Config {
 			Enabled: false,
 			Level:   DebugLogLevel,
 		},
+		Wait:         5,
+		WaitInterval: 0,
+		Retry: RetryConfig{
+			MaxRetries:         4,
+			SleepBetweenReties: 2 * time.Second,
+		},
 	}
 }
 
@@ -374,5 +407,20 @@ func MergeConfigWithDefaults(cfg Config) Config {
 		result.Log.Level = cfg.Log.Level
 	}
 
+	// .Wait
+	result.Wait = cfg.Wait
+
+	// .WaitInterval
+	result.WaitInterval = cfg.WaitInterval
+
+	// .Retry
+	if cfg.Retry.MaxRetries > 0 {
+		result.Retry.MaxRetries = cfg.Retry.MaxRetries
+	}
+
+	if cfg.Retry.SleepBetweenReties != 0 {
+		result.Retry.SleepBetweenReties = cfg.Retry.SleepBetweenReties
+	}
+
 	return result
 }