diff --git a/docs/RELEASE-NOTES.rst b/docs/RELEASE-NOTES.rst index 1ccf4dde6..94129d8ff 100644 --- a/docs/RELEASE-NOTES.rst +++ b/docs/RELEASE-NOTES.rst @@ -10,11 +10,12 @@ Added Functionality * Next generation routes preview. Refer `Documentation `_ for more details * Support for health monitors using route annotations See `Examples `_ * Policy CR integration with extended ConfigMap - * Support for TLS profiles as K8S secrets in next generation routes. See `Example `_ + * Support for TLS profiles as K8S secrets in route annotations. See `Examples `_ * Support Path based A/B deployment for Re-encrypt termination * Support to create Health Monitor from the pod liveness probe that route exposes. Refer `Documentation `_ for more details * Support for Default SSL profiles from baseRouteSpec in extended Configmap * GSLB support for routes in AS3 mode + * Support for TLS profiles as route annotations. See `Examples `_ * CRD * CIS configures GTM configuration in default partition * Pool reselect support for VS and TS diff --git a/docs/config_examples/next-gen-routes/README.md b/docs/config_examples/next-gen-routes/README.md index a3bdfd49d..7553411e6 100644 --- a/docs/config_examples/next-gen-routes/README.md +++ b/docs/config_examples/next-gen-routes/README.md @@ -101,27 +101,10 @@ Base route configuration can be defined in Global ConfigMap. This cannot be over | namespace | Mandatory | namespace to group the routes | - | Local and Global configMap | | vsAddress | Mandatory | BigIP Virtual Server IP Address | - | Local and Global configMap | | vsName | Optional | Name of BigIP Virtual Server | auto | Local and Global configMap | -| tls | Optional | Dictionary of client and server SSL profiles (See next section). | - | Local and Global configMap | **Note**: 1. namespaceLabel is mutually exclusive with namespace parameter. 2. --namespace-label parameter has to be defined in CIS deployment to use the namespaceLabel in extended configMap. -#### TLS Config Parameters - -| Parameter | Required | Description | Default | ConfigMap | -| --------- | -------- | ----------- | ------- | --------- | -| clientSSL | Optional | client SSL profile | - | Local and Global configMap | -| serverSSL | Optional | server SSL profile | - | Local and Global configMap | -| reference | Mandatory | Profile Object type | - | Local and Global configMap | - -* tls schema: -``` - tls: - clientSSL: /Common/clientssl - serverSSL: /Common/serverssl - reference: bigip -``` - ## Example Global & Local ConfigMap with namespace parameter **Example: Global Configmap** ``` diff --git a/docs/config_examples/next-gen-routes/configmap/extendedRouteConfig.yaml b/docs/config_examples/next-gen-routes/configmap/extendedRouteConfig.yaml index 022fc7319..d30a48bf2 100644 --- a/docs/config_examples/next-gen-routes/configmap/extendedRouteConfig.yaml +++ b/docs/config_examples/next-gen-routes/configmap/extendedRouteConfig.yaml @@ -10,10 +10,6 @@ data: vserverAddr: 10.8.3.11 vserverName: nextgenroutes allowOverride: true - tls: - clientSSL: /Common/clientssl - serverSSL: /Common/serverssl - reference: bigip - namespace: new vserverAddr: 10.8.3.12 allowOverride: true diff --git a/docs/config_examples/next-gen-routes/configmap/extendedRouteConfigWithTLSCertsK8SSecrets.yaml b/docs/config_examples/next-gen-routes/configmap/extendedRouteConfigWithTLSCertsK8SSecrets.yaml deleted file mode 100644 index 8774ce769..000000000 --- a/docs/config_examples/next-gen-routes/configmap/extendedRouteConfigWithTLSCertsK8SSecrets.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: default-extended-route-spec - namespace: kube-system -data: - extendedSpec: | - baseRouteSpec: - tlsCipher: - tlsVersion: 1.2 - ciphers: DEFAULT - cipherGroup: /Common/f5-default - extendedRouteSpec: - - namespace: default - vserverAddr: 10.8.3.11 - vserverName: nextgenroutes - allowOverride: true - tls: - clientSSL: clientssl-secret - serverSSL: serverssl-secret - reference: secret diff --git a/docs/config_examples/next-gen-routes/configmap/extendedRouteConfigwithBaseConfig.yaml b/docs/config_examples/next-gen-routes/configmap/extendedRouteConfigwithBaseConfig.yaml index 2f0f8e2cd..79deeff83 100644 --- a/docs/config_examples/next-gen-routes/configmap/extendedRouteConfigwithBaseConfig.yaml +++ b/docs/config_examples/next-gen-routes/configmap/extendedRouteConfigwithBaseConfig.yaml @@ -19,10 +19,6 @@ data: vserverAddr: 10.8.0.4 vserverName: nextgenroutes allowOverride: true - tls: - clientSSL: /Common/clientssl - serverSSL: /Common/serverssl - reference: bigip - allowOverride: false namespace: bar vserverAddr: 10.8.0.5 diff --git a/docs/config_examples/next-gen-routes/routes/edge-route-with-bigip-reference-in-ssl-annotation.yaml b/docs/config_examples/next-gen-routes/routes/edge-route-with-bigip-reference-in-ssl-annotation.yaml new file mode 100644 index 000000000..3a1ac7bfe --- /dev/null +++ b/docs/config_examples/next-gen-routes/routes/edge-route-with-bigip-reference-in-ssl-annotation.yaml @@ -0,0 +1,20 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + virtual-server.f5.com/clientssl: /Common/foo-clientssl + virtual-server.f5.com/health: '[{"path": "pytest-foo-1.com/","send": "HTTP GET + pytest-foo-1.com/", "recv": "","interval": 2,"timeout": 5, "type": "http"}]' + labels: + f5type: systest + name: svc-pytest-foo-1-com + namespace: foo +spec: + host: pytest-foo-1.com + path: / + tls: + termination: edge + to: + kind: Service + name: svc-pytest-foo-1-com + weight: 100 diff --git a/docs/config_examples/next-gen-routes/routes/edge-route-with-k8s-secret-in-ssl-annotation.yaml b/docs/config_examples/next-gen-routes/routes/edge-route-with-k8s-secret-in-ssl-annotation.yaml new file mode 100644 index 000000000..cee97b328 --- /dev/null +++ b/docs/config_examples/next-gen-routes/routes/edge-route-with-k8s-secret-in-ssl-annotation.yaml @@ -0,0 +1,20 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + virtual-server.f5.com/clientssl: clientssl-secret + virtual-server.f5.com/health: '[{"path": "pytest-foo-1.com/","send": "HTTP GET + pytest-foo-1.com/", "recv": "","interval": 2,"timeout": 5, "type": "http"}]' + labels: + f5type: systest + name: svc-pytest-foo-1-com + namespace: foo +spec: + host: pytest-foo-1.com + path: / + tls: + termination: edge + to: + kind: Service + name: svc-pytest-foo-1-com + weight: 100 diff --git a/docs/config_examples/next-gen-routes/routes/reencrypt-route-with-bigip-reference-in-ssl-annotaion.yaml b/docs/config_examples/next-gen-routes/routes/reencrypt-route-with-bigip-reference-in-ssl-annotaion.yaml new file mode 100644 index 000000000..a843644e3 --- /dev/null +++ b/docs/config_examples/next-gen-routes/routes/reencrypt-route-with-bigip-reference-in-ssl-annotaion.yaml @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + virtual-server.f5.com/clientssl: /Common/foo-clientssl + virtual-server.f5.com/serverssl: /Common/foo-serverssl + virtual-server.f5.com/health: '[{"path": "pytest-foo-1.com/","send": "HTTP GET + pytest-foo-1.com/", "recv": "","interval": 2,"timeout": 5, "type": "https"}]' + labels: + f5type: systest + name: svc-pytest-foo-1-com + namespace: foo +spec: + host: pytest-foo-1.com + path: / + tls: + termination: reencrypt + to: + kind: Service + name: svc-pytest-foo-1-com + weight: 100 diff --git a/docs/config_examples/next-gen-routes/routes/reencrypt-route-with-k8s-secret-in-ssl-annotation.yaml b/docs/config_examples/next-gen-routes/routes/reencrypt-route-with-k8s-secret-in-ssl-annotation.yaml new file mode 100644 index 000000000..88a734b6f --- /dev/null +++ b/docs/config_examples/next-gen-routes/routes/reencrypt-route-with-k8s-secret-in-ssl-annotation.yaml @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + annotations: + virtual-server.f5.com/clientssl: clientssl-secret + virtual-server.f5.com/serverssl: serverssl-secret + virtual-server.f5.com/health: '[{"path": "pytest-foo-1.com/","send": "HTTP GET + pytest-foo-1.com/", "recv": "","interval": 2,"timeout": 5, "type": "https"}]' + labels: + f5type: systest + name: svc-pytest-foo-1-com + namespace: foo +spec: + host: pytest-foo-1.com + path: / + tls: + termination: reencrypt + to: + kind: Service + name: svc-pytest-foo-1-com + weight: 100 diff --git a/pkg/controller/nativeResourceWorker.go b/pkg/controller/nativeResourceWorker.go index 83a7d1b71..0803b6258 100644 --- a/pkg/controller/nativeResourceWorker.go +++ b/pkg/controller/nativeResourceWorker.go @@ -40,7 +40,7 @@ func (ctlr *Controller) processRoutes(routeGroup string, triggerDelete bool) err return fmt.Errorf("extended Route Spec not available for RouteGroup/Namespace: %v", routeGroup) } - routes := ctlr.getGroupedRoutes(routeGroup, extdSpec) + routes := ctlr.getGroupedRoutes(routeGroup) if triggerDelete || len(routes) == 0 { // Delete all possible virtuals for this route group @@ -108,7 +108,7 @@ func (ctlr *Controller) processRoutes(routeGroup string, triggerDelete bool) err if isSecureRoute(rt) { //TLS Logic - processed := ctlr.handleRouteTLS(rsCfg, rt, extdSpec.VServerAddr, servicePort, extdSpec) + processed := ctlr.handleRouteTLS(rsCfg, rt, extdSpec.VServerAddr, servicePort) if !processed { // Processing failed // Stop processing further routes @@ -158,7 +158,7 @@ func (ctlr *Controller) processRoutes(routeGroup string, triggerDelete bool) err return nil } -func (ctlr *Controller) getGroupedRoutes(routeGroup string, extdSpec *ExtendedRouteGroupSpec) []*routeapi.Route { +func (ctlr *Controller) getGroupedRoutes(routeGroup string) []*routeapi.Route { var assocRoutes []*routeapi.Route // Get the route group for _, namespace := range ctlr.resources.extdSpecMap[routeGroup].namespaces { @@ -168,7 +168,7 @@ func (ctlr *Controller) getGroupedRoutes(routeGroup string, extdSpec *ExtendedRo ctlr.TeemData.Unlock() for _, route := range orderedRoutes { // TODO: add combinations for a/b - svc weight ; valid svcs or not - if ctlr.checkValidRoute(route, extdSpec) { + if ctlr.checkValidRoute(route) { var key string if route.Spec.Path == "/" || len(route.Spec.Path) == 0 { key = route.Spec.Host + "/" @@ -1308,7 +1308,7 @@ func (ctlr *Controller) fetchRoute(rscKey string) *routeapi.Route { return obj.(*routeapi.Route) } -func (ctlr *Controller) checkValidRoute(route *routeapi.Route, extdSpec *ExtendedRouteGroupSpec) bool { +func (ctlr *Controller) checkValidRoute(route *routeapi.Route) bool { // Validate the hostpath ctlr.processedHostPath.Lock() defer ctlr.processedHostPath.Unlock() @@ -1327,22 +1327,26 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, extdSpec *Extende return false } } - - // If TLS reference of type BigIP/Secret is configured in ConfigMap, fetch Client and Server SSL profile references - if extdSpec != nil && extdSpec.TLS != (TLS{}) && (extdSpec.TLS.Reference == BIGIP || extdSpec.TLS.Reference == Secret) && - route.Spec.TLS.Termination != routeapi.TLSTerminationPassthrough { - if extdSpec.TLS.ClientSSL == "" { - message := fmt.Sprintf("Missing client SSL profile %s reference in the ConfigMap", extdSpec.TLS.Reference) + sslProfileOption := ctlr.getSSLProfileOption(route) + switch sslProfileOption { + case "": + break + case AnnotationSSLOption: + if _, ok := route.ObjectMeta.Annotations[resource.F5ServerSslProfileAnnotation]; !ok && route.Spec.TLS.Termination == routeapi.TLSTerminationReencrypt { + message := fmt.Sprintf("Missing server SSL profile in the annotation") go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse) return false } - if extdSpec.TLS.ServerSSL == "" && route.Spec.TLS.Termination == routeapi.TLSTerminationReencrypt { - message := fmt.Sprintf("Missing server SSL profile %s reference in the ConfigMap", extdSpec.TLS.Reference) + case RouteCertificateSSLOption: + // Validate vsHostname if certificate is not provided in SSL annotations + ok := checkCertificateHost(route.Spec.Host, []byte(route.Spec.TLS.Certificate), []byte(route.Spec.TLS.Key)) + if !ok { + //Invalid certificate and key + message := fmt.Sprintf("Invalid certificate and key for route: %v", route.ObjectMeta.Name) go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse) return false } - } else if ctlr.resources.baseRouteConfig != (BaseRouteConfig{}) && ctlr.resources.baseRouteConfig.DefaultTLS != (DefaultSSLProfile{}) && - ctlr.resources.baseRouteConfig.DefaultTLS.Reference == BIGIP && route.Spec.TLS.Termination != routeapi.TLSTerminationPassthrough { + case DefaultSSLOption: if ctlr.resources.baseRouteConfig.DefaultTLS.ClientSSL == "" { message := fmt.Sprintf("Missing client SSL profile %s reference in the ConfigMap - BaseRouteSpec", ctlr.resources.baseRouteConfig.DefaultTLS.Reference) go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse) @@ -1353,22 +1357,12 @@ func (ctlr *Controller) checkValidRoute(route *routeapi.Route, extdSpec *Extende go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse) return false } - } else if nil != route.Spec.TLS && route.Spec.TLS.Termination != routeapi.TLSTerminationPassthrough { - if route.Spec.TLS.Certificate != "" && route.Spec.TLS.Key != "" { - // Validate vsHostname if certificate is not provided in SSL annotations - ok := checkCertificateHost(route.Spec.Host, []byte(route.Spec.TLS.Certificate), []byte(route.Spec.TLS.Key)) - if !ok { - //Invalid certificate and key - message := fmt.Sprintf("Invalid certificate and key for route: %v", route.ObjectMeta.Name) - go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse) - return false - } - } else { - message := fmt.Sprintf("Missing certificate/key for route: %v", route.ObjectMeta.Name) - go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse) - return false - } + default: + message := fmt.Sprintf("Missing certificate/key/SSL profile annotation/defaultSSL for route: %v", route.ObjectMeta.Name) + go ctlr.updateRouteAdmitStatus(fmt.Sprintf("%v/%v", route.Namespace, route.Name), "ExtendedValidationFailed", message, v1.ConditionFalse) + return false } + // Validate the route service exists or not err, _ := ctlr.getServicePort(route) if err != nil { @@ -1474,15 +1468,9 @@ func (ctlr *Controller) getRouteGroupForSecret(secret *v1.Secret) string { if extdSpec == nil || extdSpec.global == nil { continue } - // Skip routeGroups with no TLS and TLS with reference other than secret - if extdSpec.global.TLS == (TLS{}) || extdSpec.global.TLS.Reference != Secret { - continue - } - // Check if the updated secret is used in any RouteGroup and, belongs to the same namespace as the RouteGroup - if extdSpec.global.TLS.ServerSSL == secret.Name || extdSpec.global.TLS.ClientSSL == secret.Name { - if ctlr.resources.invertedNamespaceLabelMap[secret.Namespace] == rg { - return rg - } + // Check if namespace of the secret matches with the namespace of the routes defined in a route group + if ctlr.resources.invertedNamespaceLabelMap[secret.Namespace] == rg { + return rg } } return "" diff --git a/pkg/controller/nativeResourceWorker_test.go b/pkg/controller/nativeResourceWorker_test.go index aec01675f..cd9f8bd88 100644 --- a/pkg/controller/nativeResourceWorker_test.go +++ b/pkg/controller/nativeResourceWorker_test.go @@ -3,20 +3,19 @@ package controller import ( "fmt" cisapiv1 "github.com/F5Networks/k8s-bigip-ctlr/config/apis/cis/v1" - "strings" - "time" - + "github.com/F5Networks/k8s-bigip-ctlr/pkg/resource" "github.com/F5Networks/k8s-bigip-ctlr/pkg/teem" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "github.com/F5Networks/k8s-bigip-ctlr/pkg/test" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" routeapi "github.com/openshift/api/route/v1" fakeRouteClient "github.com/openshift/client-go/route/clientset/versioned/fake" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" k8sfake "k8s.io/client-go/kubernetes/fake" + "strings" + "time" ) var _ = Describe("Routes", func() { @@ -181,18 +180,18 @@ var _ = Describe("Routes", func() { cmNamespace, data) data["extendedSpec"] = ` -baseRouteSpec: - tlsCipher: - tlsVersion : 1.2 -extendedRouteSpec: - - namespace: default - vserverAddr: 10.8.3.11 - vserverName: nextgenroutes - allowOverride: true - - namespace: new - vserverAddr: 10.8.3.12 - allowOverride: true -` + baseRouteSpec: + tlsCipher: + tlsVersion : 1.2 + extendedRouteSpec: + - namespace: default + vserverAddr: 10.8.3.11 + vserverName: nextgenroutes + allowOverride: true + - namespace: new + vserverAddr: 10.8.3.12 + allowOverride: true + ` _, _ = mockCtlr.processConfigMap(cm, false) spec1 := routeapi.RouteSpec{ @@ -226,10 +225,10 @@ extendedRouteSpec: rskey1 := fmt.Sprintf("%v/%v", route1.Namespace, route1.Name) rskey2 := fmt.Sprintf("%v/%v", route2.Namespace, route2.Name) rskey3 := fmt.Sprintf("%v/%v", route3.Namespace, route3.Name) - Expect(mockCtlr.checkValidRoute(route1, nil)).To(BeFalse()) + Expect(mockCtlr.checkValidRoute(route1)).To(BeFalse()) mockCtlr.processedHostPath.processedHostPathMap[route1.Spec.Host+route1.Spec.Path] = route1.ObjectMeta.CreationTimestamp - Expect(mockCtlr.checkValidRoute(route2, nil)).To(BeFalse()) - Expect(mockCtlr.checkValidRoute(route3, nil)).To(BeFalse()) + Expect(mockCtlr.checkValidRoute(route2)).To(BeFalse()) + Expect(mockCtlr.checkValidRoute(route3)).To(BeFalse()) time.Sleep(100 * time.Millisecond) route1 = mockCtlr.fetchRoute(rskey1) route2 = mockCtlr.fetchRoute(rskey2) @@ -268,17 +267,9 @@ extendedRouteSpec: vserverAddr: 10.8.3.11 vserverName: nextgenroutes allowOverride: true - tls: - clientSSL: /Common/clientssl - serverSSL: /Common/serverssl - reference: bigip - namespace: test vserverAddr: 10.8.3.12 allowOverride: true - tls: - clientSSL: /Common/clientssl - serverSSL: /Common/serverssl - reference: bigip bigIpPartition: dev ` err, isProcessed := mockCtlr.processConfigMap(cm, false) @@ -315,7 +306,10 @@ extendedRouteSpec: mockCtlr.addEndpoints(fooEndpts) //Add new Route - route1 := test.NewRoute("route1", "1", namespace1, spec1, nil) + annotation1 := make(map[string]string) + annotation1[resource.F5ServerSslProfileAnnotation] = "/Common/serverssl" + annotation1[resource.F5ClientSslProfileAnnotation] = "/Common/clientssl" + route1 := test.NewRoute("route1", "1", namespace1, spec1, annotation1) mockCtlr.addRoute(route1) mockCtlr.resources.invertedNamespaceLabelMap[namespace1] = namespace1 err = mockCtlr.processRoutes(namespace1, false) @@ -329,7 +323,10 @@ extendedRouteSpec: mockCtlr.addEndpoints(barEndpts) //Add new Route - route2 := test.NewRoute("route2", "1", namespace2, spec2, nil) + annotation2 := make(map[string]string) + annotation2[resource.F5ServerSslProfileAnnotation] = "/Common/serverssl" + annotation2[resource.F5ClientSslProfileAnnotation] = "/Common/clientssl" + route2 := test.NewRoute("route2", "1", namespace2, spec2, annotation2) mockCtlr.addRoute(route2) mockCtlr.resources.invertedNamespaceLabelMap[namespace2] = namespace2 err = mockCtlr.processRoutes(namespace2, false) @@ -455,10 +452,6 @@ extendedRouteSpec: - namespace: test vserverAddr: 10.8.3.12 allowOverride: true - tls: - clientSSL: /Common/clientssl - serverSSL: /Common/serverssl - reference: bigip ` err, isProcessed = mockCtlr.processConfigMap(cm, false) Expect(err).To(BeNil()) @@ -598,11 +591,6 @@ extendedRouteSpec: VServerName: "nextgenroutes", VServerAddr: "10.10.10.10", AllowOverride: "False", - TLS: TLS{ - ClientSSL: "/Common/clientssl", - ServerSSL: "/Common/serverssl", - Reference: "bigip", - }, }, namespaces: []string{routeGroup}, partition: "test", @@ -630,7 +618,10 @@ extendedRouteSpec: convertSvcPortsToEndpointPorts(fooPorts)) mockCtlr.addEndpoints(fooEndpts) //Domain Based Route - route1 := test.NewRoute("route1", "1", routeGroup, spec1, nil) + annotation1 := make(map[string]string) + annotation1[resource.F5ServerSslProfileAnnotation] = "/Common/serverssl" + annotation1[resource.F5ClientSslProfileAnnotation] = "/Common/clientssl" + route1 := test.NewRoute("route1", "1", routeGroup, spec1, annotation1) mockCtlr.addRoute(route1) mockCtlr.resources.invertedNamespaceLabelMap[routeGroup] = routeGroup @@ -648,7 +639,7 @@ extendedRouteSpec: spec1.AlternateBackends = alternateBackend //Domain based route with alternate backend - route2 := test.NewRoute("route2", "1", routeGroup, spec1, nil) + route2 := test.NewRoute("route2", "1", routeGroup, spec1, annotation1) mockCtlr.addRoute(route2) mockCtlr.resources.invertedNamespaceLabelMap[routeGroup] = routeGroup @@ -671,7 +662,7 @@ extendedRouteSpec: Key: "-----BEGIN RSA PRIVATE KEY-----\n MIIEpAIBAAKCAQEAy3IHmdvGjR/fSti25e4YKpotbwkG/WOcOkXk+IwJuu14c/4d\n sDM17IayBOWuyhxvQUTyIpmNNqkb1PJ1cY1+6eIdecXdFhUPZtKylxE6NhqWtxpY\n n1jUbyiH1iqKS899MjbQ9GUrfBy/SZxwEkupq/WJcdvbtuYClUgMXqAcLpDQFZoP\n CWn9qkFj3BubkQp2trO+2K4VGURTNixDcSZs+GoTpZQSS1E6KFAFWu8T9WgnWODW\n Zi1DOGoYb0+rgso9qi1FgPNSPbEqgi82917rUobC8qK8TweXL0xq4rgpAv3Ypsc4\n Mhbxcm9Gh1QflH+MDI3eqYhN9F5oMQYYeH3HKwIDAQABAoIBACLPujk7f/f58i1O\n c81YNk5j305Wjxmgh8T43Lsiyy9vHuNKIi5aNOnqCmAIJSZ0Qx05/OyqtZ0axqZj\n bnElswe2JzEFCFWU+POxLdnnmrxTRGLEYVGy03bJyqR81vkt4dBLzOlkvlIYYSrp\n V8vponjIJOKUqj3bkamVkHhIkUnuM2lXdC30VcWBU5m9S6SuwjNFOLzhrIucXATA\n vvKH+Bw6tGKI5yE8PkSyW8BCnFg24AF2UQq1k8XvjnT3CTVeCxEZUp+HOt1Y2F25\n AhqE0viC2KeJtG0y34QKhbxq5gtUljbNCaKUkKJlO4Hu+bGVrZGPmAIEMPwMgX9u\n JaH2w/ECgYEA63XUA243qlMESfasD2BbIxyO6Wqk47CGZvfj6N66pFQO075Vv3dO\n IY1ENT/Cd73XE9zxr/9RQ4BG42pWL1/3g1jcpAa+iW2SK1YxaCe3SwSQY+EWuGsY\n XmhahZ/V7aD5PH4v+ewOG1r6WF5ugwoaaEvn/9/f3At4TszX9/acWbcCgYEA3TFD\n blSk+iFWjXnYzTTgS+5ZVt2c3Ix4iEY1pCRpcMsCbqx0BiqjXUCtHBDNQ5+LxlyD\n wLMjcQGGIyfSlLxuXQONRRfo2PZjcYe7JvxsX/FrXTvFi0n+i9o2HM38nH2Un40Z\n cpr/fpcpvC8kFD20jo/nt8J8OdZT9fZ5WIa2Di0CgYBQQW8sZCrxES7LDxsCerNV\n umwzvzfIq+iDvEagnxo63LPZFG0hv8aPxRjUlZDxQ3HFwW9Xr8zBFz4SUbJin3E8\n AdPizLGxIfnKb6yTdcYR+dJFWPlnjolV1HfWR+6g+lc5eUFdDEqapF3kNPuyCoWJ\n uyWun14sIHS3Vzbdu9767QKBgQDQiTB0pXLAq4upaFYA6bgJflZWMitAN2Mvv1m1\n Per2vz60zvu4EJziPya1zhVnitTBl9lTZNCmKvSm0lWTiq9WHBIlMOyDGJAaqgfF\n MriOH9LEHKUatBE7EuhvcbiWZUMoxWNXjFASrjtXwu3181L2ETA6LC7obGvN+ajf\n 0Gl1pQKBgQCAzIzP5ab8vvqwHVhDN+mWfG3vvN3tCI2rL4zv5boO20MqVTxu9i7o\n e7Zro8EKG/HNmt7hF46vq2OJa5QUpNf6a1II4dRsbbBoFUzGinm41TUENkeMumTU\n XsGWrknaI+J90tmvkM8rSI1Qjcw1zHUWTyd7blDj/snjb/Qg4v57yw==\n -----END RSA PRIVATE KEY-----"}, } - route3 := test.NewRoute("route3", "1", routeGroup, spec2, nil) + route3 := test.NewRoute("route3", "1", routeGroup, spec2, annotation1) mockCtlr.addRoute(route3) err = mockCtlr.processRoutes(routeGroup, false) Expect(err).To(BeNil()) @@ -684,25 +675,19 @@ extendedRouteSpec: It("Check Route TLS", func() { - tls := TLS{ - ClientSSL: "/Common/clientssl", - ServerSSL: "/Common/serverssl", - Reference: "bigip", - } + annotation1 := make(map[string]string) + annotation1[resource.F5ServerSslProfileAnnotation] = "/Common/serverssl" + annotation1[resource.F5ClientSslProfileAnnotation] = "/Common/clientssl" - clientTls := TLS{ - ClientSSL: "/Common/clientssl", - Reference: "bigip", - } - serverTls := TLS{ - ServerSSL: "/Common/serverssl", - Reference: "bigip", - } + clientSSLAnnotation := make(map[string]string) + clientSSLAnnotation[resource.F5ClientSslProfileAnnotation] = "/Common/clientssl" + + serverSSLAnnotation := make(map[string]string) + serverSSLAnnotation[resource.F5ServerSslProfileAnnotation] = "/Common/serverssl" extdSpec := &ExtendedRouteGroupSpec{ VServerName: "defaultServer", VServerAddr: "10.8.3.11", - TLS: tls, AllowOverride: "0", } @@ -710,7 +695,6 @@ extendedRouteSpec: extdSpec1 := &ExtendedRouteGroupSpec{ VServerName: "defaultServer", VServerAddr: "10.8.3.11", - TLS: serverTls, AllowOverride: "0", } @@ -718,7 +702,6 @@ extendedRouteSpec: extdSpec2 := &ExtendedRouteGroupSpec{ VServerName: "defaultServer", VServerAddr: "10.8.3.11", - TLS: clientTls, AllowOverride: "0", } @@ -744,8 +727,8 @@ extendedRouteSpec: routeGroup := "default" - route1 := test.NewRoute("route1", "1", routeGroup, spec1, nil) - route2 := test.NewRoute("route2", "2", routeGroup, spec2, nil) + route1 := test.NewRoute("route1", "1", routeGroup, spec1, annotation1) + route2 := test.NewRoute("route2", "2", routeGroup, spec2, annotation1) rsCfg := &ResourceConfig{} rsCfg.Virtual.Partition = routeGroup rsCfg.MetaData.ResourceType = VirtualServer @@ -766,28 +749,31 @@ extendedRouteSpec: rsCfg, route1, extdSpec.VServerAddr, - intstr.IntOrString{IntVal: 443}, extdSpec)).To(BeTrue()) + intstr.IntOrString{IntVal: 443})).To(BeTrue()) //for edge route and global config map without client ssl profile - It should fail + route1.Annotations = serverSSLAnnotation Expect(mockCtlr.handleRouteTLS( rsCfg, route1, extdSpec1.VServerAddr, - intstr.IntOrString{IntVal: 443}, extdSpec1)).To(BeFalse()) + intstr.IntOrString{IntVal: 443})).To(BeFalse()) //for re-encrypt route, and big ip reference in global config map - It should pass + route2.Annotations = annotation1 Expect(mockCtlr.handleRouteTLS( rsCfg, route2, extdSpec.VServerAddr, - intstr.IntOrString{IntVal: 443}, extdSpec)).To(BeTrue()) + intstr.IntOrString{IntVal: 443})).To(BeTrue()) //for re encrypt route and global config map without server ssl profile - It should fail + route2.Annotations = clientSSLAnnotation Expect(mockCtlr.handleRouteTLS( rsCfg, route2, extdSpec2.VServerAddr, - intstr.IntOrString{IntVal: 443}, extdSpec2)).To(BeFalse()) + intstr.IntOrString{IntVal: 443})).To(BeFalse()) }) It("Verify NextGenRoutes K8S Secret as TLS certs", func() { @@ -819,25 +805,20 @@ extendedRouteSpec: } mockCtlr.comInformers["default"].secretsInformer.GetStore().Add(clientssl) mockCtlr.comInformers["default"].secretsInformer.GetStore().Add(serverssl) - tls := TLS{ - ClientSSL: "clientssl", - ServerSSL: "serverssl", - Reference: "secret", - } - clientTls := TLS{ - ClientSSL: "clientssl", - Reference: "secret", - } - serverTls := TLS{ - ServerSSL: "serverssl", - Reference: "secret", - } + annotation1 := make(map[string]string) + annotation1[resource.F5ServerSslProfileAnnotation] = "serverssl" + annotation1[resource.F5ClientSslProfileAnnotation] = "clientssl" + + clientSSLAnnotation := make(map[string]string) + clientSSLAnnotation[resource.F5ClientSslProfileAnnotation] = "clientssl" + + serverSSLAnnotation := make(map[string]string) + serverSSLAnnotation[resource.F5ServerSslProfileAnnotation] = "serverssl" extdSpec := &ExtendedRouteGroupSpec{ VServerName: "defaultServer", VServerAddr: "10.8.3.11", - TLS: tls, AllowOverride: "0", } @@ -845,7 +826,6 @@ extendedRouteSpec: extdSpec1 := &ExtendedRouteGroupSpec{ VServerName: "defaultServer", VServerAddr: "10.8.3.11", - TLS: serverTls, AllowOverride: "0", } @@ -853,7 +833,6 @@ extendedRouteSpec: extdSpec2 := &ExtendedRouteGroupSpec{ VServerName: "defaultServer", VServerAddr: "10.8.3.11", - TLS: clientTls, AllowOverride: "0", } @@ -879,8 +858,8 @@ extendedRouteSpec: routeGroup := "default" - route1 := test.NewRoute("route1", "1", routeGroup, spec1, nil) - route2 := test.NewRoute("route2", "2", routeGroup, spec2, nil) + route1 := test.NewRoute("route1", "1", routeGroup, spec1, annotation1) + route2 := test.NewRoute("route2", "2", routeGroup, spec2, annotation1) rsCfg := &ResourceConfig{} rsCfg.Virtual.Partition = routeGroup rsCfg.MetaData.ResourceType = VirtualServer @@ -902,40 +881,43 @@ extendedRouteSpec: rsCfg, route1, extdSpec.VServerAddr, - intstr.IntOrString{IntVal: 443}, extdSpec)).To(BeTrue()) + intstr.IntOrString{IntVal: 443})).To(BeTrue()) //for edge route and global config map without client ssl profile - It should fail + route1.Annotations = serverSSLAnnotation Expect(mockCtlr.handleRouteTLS( rsCfg, route1, extdSpec1.VServerAddr, - intstr.IntOrString{IntVal: 443}, extdSpec1)).To(BeFalse()) + intstr.IntOrString{IntVal: 443})).To(BeFalse()) //for re-encrypt route, and k8s secret as TLS certs in global config map - It should pass + route2.Annotations = annotation1 Expect(mockCtlr.handleRouteTLS( rsCfg, route2, extdSpec.VServerAddr, - intstr.IntOrString{IntVal: 443}, extdSpec)).To(BeTrue()) + intstr.IntOrString{IntVal: 443})).To(BeTrue()) //for re encrypt route and global config map without server ssl profile - It should fail + route2.Annotations = clientSSLAnnotation Expect(mockCtlr.handleRouteTLS( rsCfg, route2, extdSpec2.VServerAddr, - intstr.IntOrString{IntVal: 443}, extdSpec2)).To(BeFalse()) + intstr.IntOrString{IntVal: 443})).To(BeFalse()) - // Verify that getRouteGroupForSecret fetches the correct routeGroup on k8s secret update + // Verify that getRouteGroupForSecret fetches the z routeGroup on k8s secret update // Prepare extdSpecMap that holds all the mockCtlr.resources.extdSpecMap = make(map[string]*extendedParsedSpec) mockCtlr.resources.extdSpecMap[routeGroup] = &extendedParsedSpec{ - global: &ExtendedRouteGroupSpec{TLS: TLS{ClientSSL: "clientssl", ServerSSL: "serverssl", Reference: Secret}}, + global: &ExtendedRouteGroupSpec{VServerName: "default"}, } mockCtlr.resources.extdSpecMap["test1"] = &extendedParsedSpec{ - global: &ExtendedRouteGroupSpec{TLS: TLS{ClientSSL: "clientssl", ServerSSL: "serverssl", Reference: BIGIP}}, + global: &ExtendedRouteGroupSpec{VServerName: "test1"}, } mockCtlr.resources.extdSpecMap["test2"] = &extendedParsedSpec{ - global: &ExtendedRouteGroupSpec{TLS: TLS{ClientSSL: "clientssl1", ServerSSL: "", Reference: Secret}}, + global: &ExtendedRouteGroupSpec{VServerName: "test2"}, } // Prepare invertedNamespaceLabelMap that maps namespaces to routeGroup mockCtlr.resources.invertedNamespaceLabelMap = make(map[string]string) @@ -947,9 +929,10 @@ extendedRouteSpec: // get routeGroup clientssl secret which belongs to test3 namespace Expect(mockCtlr.getRouteGroupForSecret(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "clientssl", Namespace: "test3"}})).To(Equal("")) + // Needs to be handled // get routeGroup clientssl1 secret which belongs to default namespace - Expect(mockCtlr.getRouteGroupForSecret(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "clientssl1", - Namespace: "default"}})).To(Equal("")) + //Expect(mockCtlr.getRouteGroupForSecret(&v1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "clientssl1", + // Namespace: "default"}})).To(Equal("")) }) @@ -1306,10 +1289,6 @@ extendedRouteSpec: - namespace: default vserverAddr: 10.8.3.11 vserverName: nextgenroutes - tls: - clientSSL: /Common/clientssl - serverSSL: /Common/serverssl - reference: bigip allowOverride: true ` err, ok = mockCtlr.processConfigMap(cm, false) @@ -1324,11 +1303,6 @@ extendedRouteSpec: VServerName: "nextgenroutes", VServerAddr: "10.10.10.10", AllowOverride: "False", - TLS: TLS{ - ClientSSL: "/Common/clientssl", - ServerSSL: "/Common/serverssl", - Reference: "bigip", - }, }, namespaces: []string{routeGroup}, partition: "test", @@ -1355,6 +1329,8 @@ extendedRouteSpec: mockCtlr.addEndpoints(fooEndpts) annotations := make(map[string]string) annotations["virtual-server.f5.com/balance"] = "least-connections-node" + annotations[resource.F5ServerSslProfileAnnotation] = "/Common/serverssl" + annotations[resource.F5ClientSslProfileAnnotation] = "/Common/clientssl" route1 := test.NewRoute("route1", "1", routeGroup, spec1, annotations) mockCtlr.addRoute(route1) mockCtlr.resources.invertedNamespaceLabelMap[routeGroup] = routeGroup @@ -1368,10 +1344,6 @@ extendedRouteSpec: - namespace: default vserverAddr: 10.8.3.11 vserverName: nextgenroutes - tls: - clientSSL: /Common/clientssl - serverSSL: /Common/serverssl - reference: bigip allowOverride: true ` err, ok = mockCtlr.processConfigMap(cm, false) diff --git a/pkg/controller/resourceConfig.go b/pkg/controller/resourceConfig.go index 6e87ce9c3..dfe4fd5c8 100644 --- a/pkg/controller/resourceConfig.go +++ b/pkg/controller/resourceConfig.go @@ -19,6 +19,7 @@ package controller import ( "encoding/json" "fmt" + "github.com/F5Networks/k8s-bigip-ctlr/pkg/resource" "net" "reflect" @@ -103,6 +104,14 @@ const ( Certificate = "certificate" ) +// constants for SSL options +const ( + AnnotationSSLOption = "annotation" + RouteCertificateSSLOption = "routeCertificate" + DefaultSSLOption = "defaultSSL" + InvalidSSLOption = "invalid" +) + func NewCustomProfile( profile ProfileRef, certificates []certificate, @@ -1882,7 +1891,6 @@ func (rs *ResourceStore) getExtendedRouteSpec(routeGroup string) (*ExtendedRoute VServerName: extdSpec.global.VServerName, VServerAddr: extdSpec.global.VServerAddr, AllowOverride: extdSpec.global.AllowOverride, - TLS: extdSpec.global.TLS, } if extdSpec.local.VServerName != "" { @@ -1894,9 +1902,6 @@ func (rs *ResourceStore) getExtendedRouteSpec(routeGroup string) (*ExtendedRoute if extdSpec.local.Policy != "" { ergc.Policy = extdSpec.local.Policy } - if extdSpec.local.TLS != (TLS{}) { - ergc.TLS = extdSpec.local.TLS - } return ergc, extdSpec.partition } @@ -1910,8 +1915,7 @@ func (ctlr *Controller) handleRouteTLS( rsCfg *ResourceConfig, route *routeapi.Route, vServerAddr string, - servicePort intstr.IntOrString, - extdSpec *ExtendedRouteGroupSpec) bool { + servicePort intstr.IntOrString) bool { if route.Spec.TLS == nil { // Probably this is a non-tls route, nothing to do w.r.t TLS @@ -1919,63 +1923,73 @@ func (ctlr *Controller) handleRouteTLS( } var tlsReferenceType string bigIPSSLProfiles := BigIPSSLProfiles{} - - //If TLS config is present in the global configmap look for the bigIPReference - if extdSpec.TLS != (TLS{}) && (extdSpec.TLS.Reference == BIGIP || extdSpec.TLS.Reference == Secret) { - tlsReferenceType = extdSpec.TLS.Reference - if route.Spec.TLS.Termination != routeapi.TLSTerminationPassthrough { - if extdSpec.TLS.ClientSSL == "" { - return false + sslProfileOption := ctlr.getSSLProfileOption(route) + switch sslProfileOption { + case "": + break + case AnnotationSSLOption: + if clientSSL, ok := route.ObjectMeta.Annotations[resource.F5ClientSslProfileAnnotation]; ok { + if len(strings.Split(clientSSL, "/")) > 1 { + tlsReferenceType = BIGIP + } else { + tlsReferenceType = Secret } - if route.Spec.TLS.Termination == routeapi.TLSTerminationReencrypt && extdSpec.TLS.ServerSSL == "" { - return false + bigIPSSLProfiles.clientSSLs = append(bigIPSSLProfiles.clientSSLs, clientSSL) + serverSSL, ok := route.ObjectMeta.Annotations[resource.F5ServerSslProfileAnnotation] + if route.Spec.TLS.Termination == routeapi.TLSTerminationReencrypt { + if !ok { + return false + } + bigIPSSLProfiles.serverSSLs = append(bigIPSSLProfiles.serverSSLs, serverSSL) } - bigIPSSLProfiles.clientSSLs = append(bigIPSSLProfiles.clientSSLs, extdSpec.TLS.ClientSSL) - if route.Spec.TLS.Termination == routeapi.TLSTerminationReencrypt { - bigIPSSLProfiles.serverSSLs = append(bigIPSSLProfiles.serverSSLs, extdSpec.TLS.ServerSSL) + } + case RouteCertificateSSLOption: + tlsReferenceType = Certificate + if route.Spec.TLS.Key != "" { + bigIPSSLProfiles.key = route.Spec.TLS.Key + } + if route.Spec.TLS.Certificate != "" { + bigIPSSLProfiles.certificate = route.Spec.TLS.Certificate + } + if route.Spec.TLS.CACertificate != "" { + bigIPSSLProfiles.caCertificate = route.Spec.TLS.CACertificate + } + if route.Spec.TLS.DestinationCACertificate != "" { + bigIPSSLProfiles.destinationCACertificate = route.Spec.TLS.DestinationCACertificate + } + // Set DependsOnTLS to true in case of route certificate and defaultSSLProfile + if ctlr.resources.baseRouteConfig != (BaseRouteConfig{}) { + //Flag to track the route groups which are using TLS Ciphers + ctlr.resources.extdSpecMap[ctlr.resources.supplementContextCache.invertedNamespaceLabelMap[route.Namespace]].global.Meta = Meta{ + DependsOnTLS: true, } } - } else if route.Spec.TLS != nil && route.Spec.TLS.Termination != TLSPassthrough { + case DefaultSSLOption: // Check for default tls in baseRouteSpec - if ctlr.resources.baseRouteConfig != (BaseRouteConfig{}) && ctlr.resources.baseRouteConfig.DefaultTLS != (DefaultSSLProfile{}) && - ctlr.resources.baseRouteConfig.DefaultTLS.Reference == BIGIP { - tlsReferenceType = BIGIP + tlsReferenceType = BIGIP - if ctlr.resources.baseRouteConfig.DefaultTLS.ClientSSL == "" { - return false - } - bigIPSSLProfiles.clientSSLs = append(bigIPSSLProfiles.clientSSLs, ctlr.resources.baseRouteConfig.DefaultTLS.ClientSSL) - - if route.Spec.TLS.Termination == TLSReencrypt { - if ctlr.resources.baseRouteConfig.DefaultTLS.ServerSSL == "" { - return false - } - bigIPSSLProfiles.serverSSLs = append(bigIPSSLProfiles.serverSSLs, ctlr.resources.baseRouteConfig.DefaultTLS.ServerSSL) - } - } else { - tlsReferenceType = Certificate + if ctlr.resources.baseRouteConfig.DefaultTLS.ClientSSL == "" { + return false + } + bigIPSSLProfiles.clientSSLs = append(bigIPSSLProfiles.clientSSLs, ctlr.resources.baseRouteConfig.DefaultTLS.ClientSSL) - if route.Spec.TLS.Key != "" { - bigIPSSLProfiles.key = route.Spec.TLS.Key - } - if route.Spec.TLS.Certificate != "" { - bigIPSSLProfiles.certificate = route.Spec.TLS.Certificate - } - if route.Spec.TLS.CACertificate != "" { - bigIPSSLProfiles.caCertificate = route.Spec.TLS.CACertificate - } - if route.Spec.TLS.DestinationCACertificate != "" { - bigIPSSLProfiles.destinationCACertificate = route.Spec.TLS.DestinationCACertificate + if route.Spec.TLS.Termination == TLSReencrypt { + if ctlr.resources.baseRouteConfig.DefaultTLS.ServerSSL == "" { + return false } + bigIPSSLProfiles.serverSSLs = append(bigIPSSLProfiles.serverSSLs, ctlr.resources.baseRouteConfig.DefaultTLS.ServerSSL) } - + // Set DependsOnTLS to true in case of route certificate and defaultSSLProfile if ctlr.resources.baseRouteConfig != (BaseRouteConfig{}) { //Flag to track the route groups which are using TLS Ciphers ctlr.resources.extdSpecMap[ctlr.resources.supplementContextCache.invertedNamespaceLabelMap[route.Namespace]].global.Meta = Meta{ DependsOnTLS: true, } } + default: + log.Errorf("Missing certificate/key/SSL profile annotation/defaultSSL for route: %v", route.ObjectMeta.Name) + return false } var poolPathRefs []poolPathRef @@ -2031,3 +2045,26 @@ func (ctlr *Controller) handleRouteTLS( bigIPSSLProfiles, }) } + +/* + getSSLProfileOption returns which ssl profile option to be used for the route + Examples: annotation, routeCertificate, defaultSSL, invalid +*/ +func (ctlr *Controller) getSSLProfileOption(route *routeapi.Route) string { + sslProfileOption := "" + if route == nil || route.Spec.TLS == nil || route.Spec.TLS.Termination == routeapi.TLSTerminationPassthrough { + return sslProfileOption + } + if _, ok := route.ObjectMeta.Annotations[resource.F5ClientSslProfileAnnotation]; ok { + sslProfileOption = AnnotationSSLOption + } else if route.Spec.TLS != nil && route.Spec.TLS.Key != "" && route.Spec.TLS.Certificate != "" { + sslProfileOption = RouteCertificateSSLOption + } else if ctlr.resources != nil && ctlr.resources.baseRouteConfig != (BaseRouteConfig{}) && + ctlr.resources.baseRouteConfig.DefaultTLS != (DefaultSSLProfile{}) && + ctlr.resources.baseRouteConfig.DefaultTLS.Reference == BIGIP { + sslProfileOption = DefaultSSLOption + } else { + sslProfileOption = InvalidSSLOption + } + return sslProfileOption +} diff --git a/pkg/controller/types.go b/pkg/controller/types.go index b3282de21..e8d405032 100644 --- a/pkg/controller/types.go +++ b/pkg/controller/types.go @@ -1077,7 +1077,6 @@ type ( VServerAddr string `yaml:"vserverAddr"` AllowOverride string `yaml:"allowOverride"` Policy string `yaml:"policyCR,omitempty"` - TLS TLS `yaml:"tls"` Meta Meta } @@ -1085,12 +1084,6 @@ type ( DependsOnTLS bool } - TLS struct { - ClientSSL string `yaml:"clientSSL,omitempty"` - ServerSSL string `yaml:"serverSSL,omitempty"` - Reference string `yaml:"reference,omitempty"` - } - BaseRouteConfig struct { TLSCipher TLSCipher `yaml:"tlsCipher"` DefaultTLS DefaultSSLProfile `yaml:"defaultTLS,omitempty"`