connect: introduce ExternalSNI field on service-defaults

agent/config/runtime_test.go

@@ -2783,6 +2783,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 							"kind": "service-defaults",
 							"name": "web",
 							"protocol": "http",
+							"external_sni": "abc-123",
 							"mesh_gateway": {
 								"mode": "remote"
 							}
@@ -2796,6 +2797,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 						kind = "service-defaults"
 						name = "web"
 						protocol = "http"
+						external_sni = "abc-123"
 						mesh_gateway {
 							mode = "remote"
 						}
@@ -2805,9 +2807,10 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.DataDir = dataDir
 				rt.ConfigEntryBootstrap = []structs.ConfigEntry{
 					&structs.ServiceConfigEntry{
-						Kind:     structs.ServiceDefaults,
-						Name:     "web",
-						Protocol: "http",
+						Kind:        structs.ServiceDefaults,
+						Name:        "web",
+						Protocol:    "http",
+						ExternalSNI: "abc-123",
 						MeshGateway: structs.MeshGatewayConfig{
 							Mode: structs.MeshGatewayModeRemote,
 						},
@@ -2825,6 +2828,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 							"Kind": "service-defaults",
 							"Name": "web",
 							"Protocol": "http",
+							"ExternalSNI": "abc-123",
 							"MeshGateway": {
 								"Mode": "remote"
 							}
@@ -2838,6 +2842,7 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 						Kind = "service-defaults"
 						Name = "web"
 						Protocol = "http"
+						ExternalSNI = "abc-123"
 						MeshGateway {
 							Mode = "remote"
 						}
@@ -2847,9 +2852,10 @@ func TestConfigFlagsAndEdgecases(t *testing.T) {
 				rt.DataDir = dataDir
 				rt.ConfigEntryBootstrap = []structs.ConfigEntry{
 					&structs.ServiceConfigEntry{
-						Kind:     structs.ServiceDefaults,
-						Name:     "web",
-						Protocol: "http",
+						Kind:        structs.ServiceDefaults,
+						Name:        "web",
+						Protocol:    "http",
+						ExternalSNI: "abc-123",
 						MeshGateway: structs.MeshGatewayConfig{
 							Mode: structs.MeshGatewayModeRemote,
 						},

agent/consul/discoverychain/compile.go

@@ -813,9 +813,48 @@ RESOLVE_AGAIN:
 
 	target.Subset = resolver.Subsets[target.ServiceSubset]
 
+	if serviceDefault := c.entries.GetService(target.Service); serviceDefault != nil && serviceDefault.ExternalSNI != "" {
+		// Explicitly set the SNI value.
+		target.SNI = serviceDefault.ExternalSNI
+		target.External = true
+	}
+
+	// If using external SNI the service is fundamentally external.
+	if target.External {
+		if resolver.Redirect != nil {
+			return nil, &structs.ConfigEntryGraphError{
+				Message: fmt.Sprintf(
+					"service %q has an external SNI set; cannot define redirects for external services",
+					target.Service,
+				),
+			}
+		}
+		if len(resolver.Subsets) > 0 {
+			return nil, &structs.ConfigEntryGraphError{
+				Message: fmt.Sprintf(
+					"service %q has an external SNI set; cannot define subsets for external services",
+					target.Service,
+				),
+			}
+		}
+		if len(resolver.Failover) > 0 {
+			return nil, &structs.ConfigEntryGraphError{
+				Message: fmt.Sprintf(
+					"service %q has an external SNI set; cannot define failover for external services",
+					target.Service,
+				),
+			}
+		}
+	}
+
 	// TODO (mesh-gateway)- maybe allow using a gateway within a datacenter at some point
 	if target.Datacenter == c.useInDatacenter {
 		target.MeshGateway.Mode = structs.MeshGatewayModeDefault
+
+	} else if target.External {
+		// Bypass mesh gateways if it is an external service.
+		target.MeshGateway.Mode = structs.MeshGatewayModeDefault
+
 	} else {
 		// Default mesh gateway settings
 		if serviceDefault := c.entries.GetService(target.Service); serviceDefault != nil {

agent/consul/discoverychain/compile_test.go

@@ -46,6 +46,7 @@ func TestCompile(t *testing.T) {
 		"datacenter failover with mesh gateways":           testcase_DatacenterFailover_WithMeshGateways(),
 		"noop split to resolver with default subset":       testcase_NoopSplit_WithDefaultSubset(),
 		"resolver with default subset":                     testcase_Resolve_WithDefaultSubset(),
+		"default resolver with external sni":               testcase_DefaultResolver_ExternalSNI(),
 		"resolver with no entries and inferring defaults":  testcase_DefaultResolver(),
 		"default resolver with proxy defaults":             testcase_DefaultResolver_WithProxyDefaults(),
 		"service redirect to service with default resolver is not a default chain": testcase_RedirectToDefaultResolverIsNotDefaultChain(),
@@ -54,13 +55,16 @@ func TestCompile(t *testing.T) {
 		"multi dc canary":            testcase_MultiDatacenterCanary(),
 
 		// various errors
-		"splitter requires valid protocol": testcase_SplitterRequiresValidProtocol(),
-		"router requires valid protocol":   testcase_RouterRequiresValidProtocol(),
-		"split to unsplittable protocol":   testcase_SplitToUnsplittableProtocol(),
-		"route to unroutable protocol":     testcase_RouteToUnroutableProtocol(),
-		"failover crosses protocols":       testcase_FailoverCrossesProtocols(),
-		"redirect crosses protocols":       testcase_RedirectCrossesProtocols(),
-		"redirect to missing subset":       testcase_RedirectToMissingSubset(),
+		"splitter requires valid protocol":        testcase_SplitterRequiresValidProtocol(),
+		"router requires valid protocol":          testcase_RouterRequiresValidProtocol(),
+		"split to unsplittable protocol":          testcase_SplitToUnsplittableProtocol(),
+		"route to unroutable protocol":            testcase_RouteToUnroutableProtocol(),
+		"failover crosses protocols":              testcase_FailoverCrossesProtocols(),
+		"redirect crosses protocols":              testcase_RedirectCrossesProtocols(),
+		"redirect to missing subset":              testcase_RedirectToMissingSubset(),
+		"resolver with failover and external sni": testcase_Resolver_ExternalSNI_FailoverNotAllowed(),
+		"resolver with subsets and external sni":  testcase_Resolver_ExternalSNI_SubsetsNotAllowed(),
+		"resolver with redirect and external sni": testcase_Resolver_ExternalSNI_RedirectNotAllowed(),
 
 		// overrides
 		"resolver with protocol from override":         testcase_ResolverProtocolOverride(),
@@ -1435,6 +1439,109 @@ func testcase_Resolve_WithDefaultSubset() compileTestCase {
 	return compileTestCase{entries: entries, expect: expect}
 }
 
+func testcase_DefaultResolver_ExternalSNI() compileTestCase {
+	entries := newEntries()
+	entries.AddServices(&structs.ServiceConfigEntry{
+		Kind:        structs.ServiceDefaults,
+		Name:        "main",
+		ExternalSNI: "main.some.other.service.mesh",
+	})
+
+	expect := &structs.CompiledDiscoveryChain{
+		Protocol:  "tcp",
+		StartNode: "resolver:main.default.dc1",
+		Nodes: map[string]*structs.DiscoveryGraphNode{
+			"resolver:main.default.dc1": &structs.DiscoveryGraphNode{
+				Type: structs.DiscoveryGraphNodeTypeResolver,
+				Name: "main.default.dc1",
+				Resolver: &structs.DiscoveryResolver{
+					Default:        true,
+					ConnectTimeout: 5 * time.Second,
+					Target:         "main.default.dc1",
+				},
+			},
+		},
+		Targets: map[string]*structs.DiscoveryTarget{
+			"main.default.dc1": newTarget("main", "", "default", "dc1", func(t *structs.DiscoveryTarget) {
+				t.SNI = "main.some.other.service.mesh"
+				t.External = true
+			}),
+		},
+	}
+	return compileTestCase{entries: entries, expect: expect, expectIsDefault: true}
+}
+
+func testcase_Resolver_ExternalSNI_FailoverNotAllowed() compileTestCase {
+	entries := newEntries()
+	entries.AddServices(&structs.ServiceConfigEntry{
+		Kind:        structs.ServiceDefaults,
+		Name:        "main",
+		ExternalSNI: "main.some.other.service.mesh",
+	})
+	entries.AddResolvers(&structs.ServiceResolverConfigEntry{
+		Kind:           "service-resolver",
+		Name:           "main",
+		ConnectTimeout: 33 * time.Second,
+		Failover: map[string]structs.ServiceResolverFailover{
+			"*": {Service: "backup"},
+		},
+	})
+
+	return compileTestCase{
+		entries:        entries,
+		expectErr:      `service "main" has an external SNI set; cannot define failover for external services`,
+		expectGraphErr: true,
+	}
+}
+
+func testcase_Resolver_ExternalSNI_SubsetsNotAllowed() compileTestCase {
+	entries := newEntries()
+	entries.AddServices(&structs.ServiceConfigEntry{
+		Kind:        structs.ServiceDefaults,
+		Name:        "main",
+		ExternalSNI: "main.some.other.service.mesh",
+	})
+	entries.AddResolvers(&structs.ServiceResolverConfigEntry{
+		Kind:           "service-resolver",
+		Name:           "main",
+		ConnectTimeout: 33 * time.Second,
+		Subsets: map[string]structs.ServiceResolverSubset{
+			"canary": {
+				Filter: "Service.Meta.version == canary",
+			},
+		},
+	})
+
+	return compileTestCase{
+		entries:        entries,
+		expectErr:      `service "main" has an external SNI set; cannot define subsets for external services`,
+		expectGraphErr: true,
+	}
+}
+
+func testcase_Resolver_ExternalSNI_RedirectNotAllowed() compileTestCase {
+	entries := newEntries()
+	entries.AddServices(&structs.ServiceConfigEntry{
+		Kind:        structs.ServiceDefaults,
+		Name:        "main",
+		ExternalSNI: "main.some.other.service.mesh",
+	})
+	entries.AddResolvers(&structs.ServiceResolverConfigEntry{
+		Kind:           "service-resolver",
+		Name:           "main",
+		ConnectTimeout: 33 * time.Second,
+		Redirect: &structs.ServiceResolverRedirect{
+			Datacenter: "dc2",
+		},
+	})
+
+	return compileTestCase{
+		entries:        entries,
+		expectErr:      `service "main" has an external SNI set; cannot define redirects for external services`,
+		expectGraphErr: true,
+	}
+}
+
 func testcase_MultiDatacenterCanary() compileTestCase {
 	entries := newEntries()
 	setServiceProtocol(entries, "main", "http")

agent/proxycfg/testing.go

@@ -592,6 +592,10 @@ func TestConfigSnapshotDiscoveryChain(t testing.T) *ConfigSnapshot {
 	return testConfigSnapshotDiscoveryChain(t, "simple")
 }
 
+func TestConfigSnapshotDiscoveryChainExternalSNI(t testing.T) *ConfigSnapshot {
+	return testConfigSnapshotDiscoveryChain(t, "external-sni")
+}
+
 func TestConfigSnapshotDiscoveryChainWithOverrides(t testing.T) *ConfigSnapshot {
 	return testConfigSnapshotDiscoveryChain(t, "simple-with-overrides")
 }
@@ -664,6 +668,19 @@ func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalE
 				ConnectTimeout: 33 * time.Second,
 			},
 		)
+	case "external-sni":
+		entries = append(entries,
+			&structs.ServiceConfigEntry{
+				Kind:        structs.ServiceDefaults,
+				Name:        "db",
+				ExternalSNI: "db.some.other.service.mesh",
+			},
+			&structs.ServiceResolverConfigEntry{
+				Kind:           structs.ServiceResolver,
+				Name:           "db",
+				ConnectTimeout: 33 * time.Second,
+			},
+		)
 	case "failover":
 		entries = append(entries,
 			&structs.ServiceResolverConfigEntry{
@@ -858,6 +875,7 @@ func testConfigSnapshotDiscoveryChain(t testing.T, variation string, additionalE
 	switch variation {
 	case "simple-with-overrides":
 	case "simple":
+	case "external-sni":
 	case "failover":
 		snap.ConnectProxy.WatchedUpstreamEndpoints["db"]["fail.default.dc1"] =
 			TestUpstreamNodesAlternate(t)

agent/structs/config_entry.go

@@ -52,6 +52,8 @@ type ServiceConfigEntry struct {
 	Protocol    string
 	MeshGateway MeshGatewayConfig `json:",omitempty"`
 
+	ExternalSNI string `json:",omitempty"`
+
 	// TODO(banks): enable this once we have upstreams supported too. Enabling
 	// sidecars actually makes no sense and adds complications when you don't
 	// allow upstreams to be specified centrally too.
@@ -306,6 +308,7 @@ func ConfigEntryDecodeRulesForKind(kind string) (skipWhenPatching []string, tran
 	case ServiceDefaults:
 		return nil, map[string]string{
 			"mesh_gateway": "meshgateway",
+			"external_sni": "externalsni",
 		}, nil
 	case ServiceRouter:
 		return []string{

agent/structs/config_entry_test.go

@@ -93,6 +93,7 @@ func TestDecodeConfigEntry(t *testing.T) {
 				kind = "service-defaults"
 				name = "main"
 				protocol = "http"
+				external_sni = "abc-123"
 				mesh_gateway {
 					mode = "remote"
 				}
@@ -101,14 +102,16 @@ func TestDecodeConfigEntry(t *testing.T) {
 				Kind = "service-defaults"
 				Name = "main"
 				Protocol = "http"
+				ExternalSNI = "abc-123"
 				MeshGateway {
 					Mode = "remote"
 				}
 			`,
 			expect: &ServiceConfigEntry{
-				Kind:     "service-defaults",
-				Name:     "main",
-				Protocol: "http",
+				Kind:        "service-defaults",
+				Name:        "main",
+				Protocol:    "http",
+				ExternalSNI: "abc-123",
 				MeshGateway: MeshGatewayConfig{
 					Mode: MeshGatewayModeRemote,
 				},

agent/structs/discovery_chain.go

@@ -160,6 +160,11 @@ type DiscoveryTarget struct {
 
 	MeshGateway MeshGatewayConfig     `json:",omitempty"`
 	Subset      ServiceResolverSubset `json:",omitempty"`
+
+	// SNI if set is the sni field to use when addressing this set of
+	// endpoints. If not configured then the Connect default should be used.
+	SNI      string `json:",omitempty"`
+	External bool   `json:",omitempty"`
 }
 
 func NewDiscoveryTarget(service, serviceSubset, namespace, datacenter string) *DiscoveryTarget {

agent/xds/clusters.go

@@ -321,10 +321,15 @@ func (s *Server) makeUpstreamClustersForDiscoveryChain(
 			c.Http2ProtocolOptions = &envoycore.Http2ProtocolOptions{}
 		}
 
+		finalSNI := sni
+		if target.SNI != "" {
+			finalSNI = target.SNI
+		}
+
 		// Enable TLS upstream with the configured client certificate.
 		c.TlsContext = &envoyauth.UpstreamTlsContext{
 			CommonTlsContext: makeCommonTLSContext(cfgSnap),
-			Sni:              sni,
+			Sni:              finalSNI,
 		}
 
 		out = append(out, c)

agent/xds/clusters_test.go

@@ -105,6 +105,11 @@ func TestClustersFromSnapshot(t *testing.T) {
 			create: proxycfg.TestConfigSnapshotDiscoveryChain,
 			setup:  nil,
 		},
+		{
+			name:   "connect-proxy-with-chain-external-sni",
+			create: proxycfg.TestConfigSnapshotDiscoveryChainExternalSNI,
+			setup:  nil,
+		},
 		{
 			name:   "connect-proxy-with-chain-and-overrides",
 			create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides,

agent/xds/endpoints_test.go

@@ -243,6 +243,11 @@ func Test_endpointsFromSnapshot(t *testing.T) {
 			create: proxycfg.TestConfigSnapshotDiscoveryChain,
 			setup:  nil,
 		},
+		{
+			name:   "connect-proxy-with-chain-external-sni",
+			create: proxycfg.TestConfigSnapshotDiscoveryChainExternalSNI,
+			setup:  nil,
+		},
 		{
 			name:   "connect-proxy-with-chain-and-overrides",
 			create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides,

agent/xds/listeners_test.go

@@ -184,6 +184,11 @@ func TestListenersFromSnapshot(t *testing.T) {
 			},
 			setup: nil,
 		},
+		{
+			name:   "connect-proxy-with-chain-external-sni",
+			create: proxycfg.TestConfigSnapshotDiscoveryChainExternalSNI,
+			setup:  nil,
+		},
 		{
 			name:   "connect-proxy-with-chain-and-overrides",
 			create: proxycfg.TestConfigSnapshotDiscoveryChainWithOverrides,