Add fqdnCacheMinTTL configuration parameter (#6808)

fqdnCacheMinTTL helps address the issue of applications caching DNS
response IPs beyond the TTL value for the DNS record. It is used to
enforce FQDN policy rules, ensuring that resolved IPs are included in
datapath rules for as long as the application caches them. Ideally, this
value should be set to the maximum caching duration across all
applications.

Signed-off-by: Hemant <hkbiet@gmail.com>

build/charts/antrea/README.md

@@ -97,6 +97,7 @@ Kubernetes: `>= 1.19.0-0`
 | flowExporter.flowCollectorAddr | string | `"flow-aggregator/flow-aggregator:4739:tls"` | IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>]. If the collector is running in-cluster as a Service, set <HOST> to <Service namespace>/<Service name>. |
 | flowExporter.flowPollInterval | string | `"5s"` | Determines how often the flow exporter polls for new connections. |
 | flowExporter.idleFlowExportTimeout | string | `"15s"` | timeout after which a flow record is sent to the collector for idle flows. |
+| fqdnCacheMinTTL | int | `0` | fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included in datapath rules for as long as the application caches them. Ideally, this value should be set to the maximum caching duration across all applications. |
 | hostGateway | string | `"antrea-gw0"` | Name of the interface antrea-agent will create and use for host <-> Pod communication. |
 | image | object | `{}` | Container image to use for Antrea components. DEPRECATED: use agentImage and controllerImage instead. |
 | ipsec.authenticationMode | string | `"psk"` | The authentication mode to use for IPsec. Must be one of "psk" or "cert". |

build/charts/antrea/conf/antrea-agent.conf

@@ -307,6 +307,12 @@ kubeAPIServerOverride: {{ .Values.kubeAPIServerOverride | quote }}
 # 10.96.0.10:53, [fd00:10:96::a]:53).
 dnsServerOverride: {{ .Values.dnsServerOverride | quote }}
 
+# fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value
+# for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included
+# in datapath rules for as long as the application caches them. Ideally, this value should be set to
+# the maximum caching duration across all applications.
+fqdnCacheMinTTL: {{ .Values.fqdnCacheMinTTL }}
+
 # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used.
 # https://golang.org/pkg/crypto/tls/#pkg-constants
 # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always

build/charts/antrea/values.yaml

@@ -187,6 +187,11 @@ kubeAPIServerOverride: ""
 # -- Address of DNS server, to override the kube-dns Service. It's used to
 # resolve hostnames in a FQDN policy.
 dnsServerOverride: ""
+# -- fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value
+# for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included
+# in datapath rules for as long as the application caches them. Ideally, this value should be set to
+# the maximum caching duration across all applications.
+fqdnCacheMinTTL: 0
 # -- IPv4 CIDR range used for Services. Required when AntreaProxy is disabled.
 serviceCIDR: ""
 # -- IPv6 CIDR range used for Services. Required when AntreaProxy is disabled.

build/yamls/antrea-aks.yml

@@ -4235,6 +4235,12 @@ data:
     # 10.96.0.10:53, [fd00:10:96::a]:53).
     dnsServerOverride: ""
 
+    # fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value
+    # for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included
+    # in datapath rules for as long as the application caches them. Ideally, this value should be set to
+    # the maximum caching duration across all applications.
+    fqdnCacheMinTTL: 0
+
     # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used.
     # https://golang.org/pkg/crypto/tls/#pkg-constants
     # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always
@@ -5394,7 +5400,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5dd823245aab41ce7ca74d05693aa96e1537615f6966b6b78879cde5d3a0b215
+        checksum/config: f7ac1903ae9edfd45361cb67b991cb23f708f15cb5cb862bffd70e95dcd776fb
       labels:
         app: antrea
         component: antrea-agent
@@ -5632,7 +5638,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5dd823245aab41ce7ca74d05693aa96e1537615f6966b6b78879cde5d3a0b215
+        checksum/config: f7ac1903ae9edfd45361cb67b991cb23f708f15cb5cb862bffd70e95dcd776fb
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-eks.yml

@@ -4235,6 +4235,12 @@ data:
     # 10.96.0.10:53, [fd00:10:96::a]:53).
     dnsServerOverride: ""
 
+    # fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value
+    # for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included
+    # in datapath rules for as long as the application caches them. Ideally, this value should be set to
+    # the maximum caching duration across all applications.
+    fqdnCacheMinTTL: 0
+
     # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used.
     # https://golang.org/pkg/crypto/tls/#pkg-constants
     # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always
@@ -5394,7 +5400,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5dd823245aab41ce7ca74d05693aa96e1537615f6966b6b78879cde5d3a0b215
+        checksum/config: f7ac1903ae9edfd45361cb67b991cb23f708f15cb5cb862bffd70e95dcd776fb
       labels:
         app: antrea
         component: antrea-agent
@@ -5633,7 +5639,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5dd823245aab41ce7ca74d05693aa96e1537615f6966b6b78879cde5d3a0b215
+        checksum/config: f7ac1903ae9edfd45361cb67b991cb23f708f15cb5cb862bffd70e95dcd776fb
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-gke.yml

@@ -4235,6 +4235,12 @@ data:
     # 10.96.0.10:53, [fd00:10:96::a]:53).
     dnsServerOverride: ""
 
+    # fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value
+    # for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included
+    # in datapath rules for as long as the application caches them. Ideally, this value should be set to
+    # the maximum caching duration across all applications.
+    fqdnCacheMinTTL: 0
+
     # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used.
     # https://golang.org/pkg/crypto/tls/#pkg-constants
     # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always
@@ -5394,7 +5400,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e9ea48fa57cb11513f69a1fb2b44dd3c6cb96aa739598c1db5091ea91f097f4b
+        checksum/config: 00ba3a60f132691721ba2e84c5c8f0a9eddc32593b38798de8f59d52fff54169
       labels:
         app: antrea
         component: antrea-agent
@@ -5630,7 +5636,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: e9ea48fa57cb11513f69a1fb2b44dd3c6cb96aa739598c1db5091ea91f097f4b
+        checksum/config: 00ba3a60f132691721ba2e84c5c8f0a9eddc32593b38798de8f59d52fff54169
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea-ipsec.yml

@@ -4248,6 +4248,12 @@ data:
     # 10.96.0.10:53, [fd00:10:96::a]:53).
     dnsServerOverride: ""
 
+    # fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value
+    # for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included
+    # in datapath rules for as long as the application caches them. Ideally, this value should be set to
+    # the maximum caching duration across all applications.
+    fqdnCacheMinTTL: 0
+
     # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used.
     # https://golang.org/pkg/crypto/tls/#pkg-constants
     # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always
@@ -5407,7 +5413,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 38e19ea8db3838e3f5cff4aaa2684db1586fb457d095ac3ea49e8bf405a04e41
+        checksum/config: 4b9bbfbbda1ab405ade14e797ea88fbd6f3795bb6aae9df0496409d542799145
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -5689,7 +5695,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 38e19ea8db3838e3f5cff4aaa2684db1586fb457d095ac3ea49e8bf405a04e41
+        checksum/config: 4b9bbfbbda1ab405ade14e797ea88fbd6f3795bb6aae9df0496409d542799145
       labels:
         app: antrea
         component: antrea-controller

build/yamls/antrea.yml

@@ -4235,6 +4235,12 @@ data:
     # 10.96.0.10:53, [fd00:10:96::a]:53).
     dnsServerOverride: ""
 
+    # fqdnCacheMinTTL helps address the issue of applications caching DNS response IPs beyond the TTL value
+    # for the DNS record. It is used to enforce FQDN policy rules, ensuring that resolved IPs are included
+    # in datapath rules for as long as the application caches them. Ideally, this value should be set to
+    # the maximum caching duration across all applications.
+    fqdnCacheMinTTL: 0
+
     # Comma-separated list of Cipher Suites. If omitted, the default Go Cipher Suites will be used.
     # https://golang.org/pkg/crypto/tls/#pkg-constants
     # Note that TLS1.3 Cipher Suites cannot be added to the list. But the apiserver will always
@@ -5394,7 +5400,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 59fb1ea496577015058d4e99e4f64136aa68d5340db13c00ced565da750a22fc
+        checksum/config: e4e94ba89524d8fdc7eb3ad6e0f6948767f3d92ef767f17c47da348f08b5c2e0
       labels:
         app: antrea
         component: antrea-agent
@@ -5630,7 +5636,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 59fb1ea496577015058d4e99e4f64136aa68d5340db13c00ced565da750a22fc
+        checksum/config: e4e94ba89524d8fdc7eb3ad6e0f6948767f3d92ef767f17c47da348f08b5c2e0
       labels:
         app: antrea
         component: antrea-controller

cmd/antrea-agent/agent.go

@@ -528,6 +528,7 @@ func run(o *Options) error {
 		nodeConfig,
 		podNetworkWait,
 		l7Reconciler,
+		uint32(o.config.FQDNCacheMinTTL),
 	)
 	if err != nil {
 		return fmt.Errorf("error creating new NetworkPolicy controller: %v", err)

cmd/antrea-agent/options.go

@@ -155,6 +155,10 @@ func (o *Options) validate(args []string) error {
 		return fmt.Errorf("nodeType %s requires feature gate ExternalNode to be enabled", o.config.NodeType)
 	}
 
+	if o.config.FQDNCacheMinTTL < 0 {
+		return fmt.Errorf("fqdnCacheMinTTL must be greater than or equal to 0")
+	}
+
 	if o.config.NodeType == config.ExternalNode.String() {
 		o.nodeType = config.ExternalNode
 		return o.validateExternalNodeOptions()

pkg/agent/controller/networkpolicy/fqdn.go

@@ -127,6 +127,7 @@ type fqdnController struct {
 	ofClient openflow.Client
 	// dnsServerAddr stores the coreDNS server address, or the user provided DNS server address.
 	dnsServerAddr string
+	minTTL        uint32
 
 	// dirtyRuleHandler is a callback that is run upon finding a rule out-of-sync.
 	dirtyRuleHandler func(string)
@@ -160,7 +161,7 @@ type fqdnController struct {
 	clock clock.Clock
 }
 
-func newFQDNController(client openflow.Client, allocator *idAllocator, dnsServerOverride string, dirtyRuleHandler func(string), v4Enabled, v6Enabled bool, gwPort uint32, clock clock.WithTicker) (*fqdnController, error) {
+func newFQDNController(client openflow.Client, allocator *idAllocator, dnsServerOverride string, dirtyRuleHandler func(string), v4Enabled, v6Enabled bool, gwPort uint32, clock clock.WithTicker, fqdnCacheMinTTL uint32) (*fqdnController, error) {
 	controller := &fqdnController{
 		ofClient:         client,
 		dirtyRuleHandler: dirtyRuleHandler,
@@ -182,6 +183,7 @@ func newFQDNController(client openflow.Client, allocator *idAllocator, dnsServer
 		ipv6Enabled:            v6Enabled,
 		gwPort:                 gwPort,
 		clock:                  clock,
+		minTTL:                 fqdnCacheMinTTL,
 	}
 	if controller.ofClient != nil {
 		if err := controller.ofClient.NewDNSPacketInConjunction(dnsInterceptRuleID); err != nil {
@@ -643,15 +645,15 @@ func (f *fqdnController) parseDNSResponse(msg *dns.Msg) (string, map[string]ipWi
 			if f.ipv4Enabled {
 				responseIPs[r.A.String()] = ipWithExpiration{
 					ip:             r.A,
-					expirationTime: currentTime.Add(time.Duration(r.Header().Ttl) * time.Second),
+					expirationTime: currentTime.Add(time.Duration(max(f.minTTL, r.Header().Ttl)) * time.Second),
 				}
 
 			}
 		case *dns.AAAA:
 			if f.ipv6Enabled {
 				responseIPs[r.AAAA.String()] = ipWithExpiration{
 					ip:             r.AAAA,
-					expirationTime: currentTime.Add(time.Duration(r.Header().Ttl) * time.Second),
+					expirationTime: currentTime.Add(time.Duration(max(f.minTTL, r.Header().Ttl)) * time.Second),
 				}
 			}
 		}